Alarm-Mail bei fehlgeschlagenen geplanten Aufgaben und Queue-Jobs

- OperationalAlert: Log (Kanal monitoring) immer, Mail an
  MONITORING_ALERT_EMAIL (Standard EXCEPTION_MAIL), optional ntfy;
  gleiche Meldung hoechstens einmal pro Stunde
- Kernel: onFailure fuer alle geplanten Aufgaben (auch kuenftige)
- AppServiceProvider: Queue::failing
- withoutOverlapping mit kurzen Sperrdauern (10-60 Min.) statt 24 h

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin 2026-10-08 09:49:18 +00:00
parent fdbb4f3853
commit c9cd17a41a
7 changed files with 300 additions and 4 deletions

View file

@ -9,8 +9,10 @@ use App\Console\Commands\CheckPaymentUptime;
use App\Console\Commands\DhlUpdateTracking;
use App\Console\Commands\UserCleanUp;
use App\Console\Commands\UserMakeAboOrder;
use App\Support\OperationalAlert;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
use Illuminate\Support\Str;
class Kernel extends ConsoleKernel
{
@ -42,7 +44,7 @@ class Kernel extends ConsoleKernel
// Uptime-Check: PAYONE-Erreichbarkeit alle 5 Minuten prüfen
$schedule->command('payment:check-uptime')
->everyFiveMinutes()
->withoutOverlapping()
->withoutOverlapping(10)
->runInBackground();
// Job 1: Überprüft täglich um 02:00 Uhr die Zahlungskonten.
@ -62,7 +64,7 @@ class Kernel extends ConsoleKernel
// verzögerter Stripe-Retry; Stripe CLI auf dem Test-Server retryt gar nicht).
$schedule->command('stripe:process-pending-events')
->everyTenMinutes()
->withoutOverlapping()
->withoutOverlapping(30)
->runInBackground();
// Stripe: offene Zahlungen aktiv gegen die API abgleichen. Fängt den Fall
@ -72,7 +74,7 @@ class Kernel extends ConsoleKernel
// Läuft über denselben, idempotenten Paid-Pfad wie ein echter Webhook.
$schedule->command('stripe:sync-payment-status --pending --days=3')
->hourly()
->withoutOverlapping()
->withoutOverlapping(60)
->runInBackground();
// Abo-Einmalprodukte: Reminder-Mail vor der Ausführung (Einmalprodukte-Fenster).
@ -108,8 +110,19 @@ class Kernel extends ConsoleKernel
// DHL Tracking Update: Stündlich mit status-basierten Intervallen und Batch-API
$schedule->command('dhl:update-tracking --days=30 --send-emails')
->hourly()
->withoutOverlapping()
->withoutOverlapping(60)
->runInBackground();
// Betriebs-Alert (Log, Mail, optional ntfy), wenn eine geplante Aufgabe fehlschlaegt
foreach ($schedule->events() as $event) {
$event->onFailure(function () use ($event): void {
$task = $event->description ?: trim(Str::after((string) $event->command, "'artisan'"));
OperationalAlert::send('Geplante Aufgabe fehlgeschlagen: '.$task, [
'exit_code' => $event->exitCode,
]);
});
}
}
/**

View file

@ -6,9 +6,12 @@ use App\Services\Backoffice\BackofficeDashboardService;
use App\Services\Payment\Payone\PayonePaymentService;
use App\Services\Payment\Stripe\StripeClientWrapper;
use App\Services\Util;
use App\Support\OperationalAlert;
use Barryvdh\LaravelIdeHelper\IdeHelperServiceProvider;
use Illuminate\Foundation\Console\AboutCommand;
use Illuminate\Pagination\Paginator;
use Illuminate\Queue\Events\JobFailed;
use Illuminate\Support\Facades\Queue;
use Illuminate\Support\Facades\Schema;
use Illuminate\Support\Facades\URL;
use Illuminate\Support\ServiceProvider;
@ -30,6 +33,14 @@ class AppServiceProvider extends ServiceProvider
URL::forceScheme('https');
}
Queue::failing(function (JobFailed $event): void {
OperationalAlert::send('Queue-Job fehlgeschlagen: '.$event->job->resolveName(), [
'connection' => $event->connectionName,
'queue' => $event->job->getQueue(),
'exception' => $event->exception->getMessage(),
]);
});
AboutCommand::add('mivita', fn (): array => [
'Version' => config('app.version') ?? 'unbekannt',
]);

View file

@ -0,0 +1,107 @@
<?php
namespace App\Support;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Mail;
use Illuminate\Support\Str;
use Throwable;
/**
* Zentrale Betriebs-Alerts: meldet fehlgeschlagene geplante Aufgaben
* (Console\Kernel) und Queue-Jobs (AppServiceProvider). Der Log-Eintrag
* (`critical`, Kanal monitoring) läuft immer; Mail und ntfy-Push sind
* Best-Effort, unabhängig voneinander, und dürfen den auslösenden Prozess nie
* stören. Gleiche Meldungen gehen höchstens einmal je Drosselzeit raus, damit
* ein Task im 5-Minuten-Takt kein Postfach flutet.
*/
class OperationalAlert
{
/**
* @param array<string, mixed> $context
*/
public static function send(string $summary, array $context = []): void
{
Log::channel('monitoring')->critical('[monitoring] '.$summary, $context);
if (! config('monitoring.alerts.enabled') || ! self::firstWithinThrottle($summary)) {
return;
}
self::mail($summary, $context);
self::push($summary, $context);
}
private static function firstWithinThrottle(string $summary): bool
{
try {
return Cache::add('monitoring:alert:'.sha1($summary), true, now()->addMinutes((int) config('monitoring.alerts.throttle_minutes')));
} catch (Throwable) {
return true;
}
}
/**
* @param array<string, mixed> $context
*/
private static function mail(string $summary, array $context): void
{
$recipient = config('monitoring.alerts.mail_to');
if (empty($recipient)) {
return;
}
try {
Mail::raw(
$summary."\n\n".json_encode($context, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
fn ($message) => $message->to($recipient)->subject('['.config('app.name').'] Betriebs-Alert: '.Str::limit($summary, 80)),
);
} catch (Throwable $e) {
Log::channel('monitoring')->error('[monitoring] Alert-Mail konnte nicht versendet werden.', [
'summary' => $summary,
'error' => $e->getMessage(),
]);
}
}
/**
* JSON-Publish an die ntfy-Basis-URL (UTF-8-sicher, anders als Header).
*
* @param array<string, mixed> $context
*/
private static function push(string $summary, array $context): void
{
$url = (string) config('monitoring.alerts.ntfy.url');
$topic = (string) config('monitoring.alerts.ntfy.topic');
if ($url === '' || $topic === '') {
return;
}
$details = collect($context)
->map(fn (mixed $value, string $key): string => $key.': '.(is_scalar($value) || $value === null ? (string) $value : json_encode($value)))
->implode("\n");
try {
Http::timeout(5)
->connectTimeout(3)
->when(filled(config('monitoring.alerts.ntfy.token')), fn ($request) => $request->withToken((string) config('monitoring.alerts.ntfy.token')))
->post(rtrim($url, '/'), [
'topic' => $topic,
'title' => config('app.name').' ('.app()->environment().'): Betriebs-Alert',
'message' => Str::limit(trim($summary."\n\n".$details), 3500),
'priority' => (int) config('monitoring.alerts.ntfy.priority'),
'tags' => ['rotating_light'],
])
->throw();
} catch (Throwable $e) {
Log::channel('monitoring')->error('[monitoring] ntfy-Push konnte nicht versendet werden.', [
'summary' => $summary,
'error' => $e->getMessage(),
]);
}
}
}