Alarm-Mail bei fehlgeschlagenen geplanten Aufgaben und Queue-Jobs
- OperationalAlert: Log (Kanal monitoring) immer, Mail an MONITORING_ALERT_EMAIL (Standard EXCEPTION_MAIL), optional ntfy; gleiche Meldung hoechstens einmal pro Stunde - Kernel: onFailure fuer alle geplanten Aufgaben (auch kuenftige) - AppServiceProvider: Queue::failing - withoutOverlapping mit kurzen Sperrdauern (10-60 Min.) statt 24 h Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
fdbb4f3853
commit
c9cd17a41a
7 changed files with 300 additions and 4 deletions
|
|
@ -9,8 +9,10 @@ use App\Console\Commands\CheckPaymentUptime;
|
|||
use App\Console\Commands\DhlUpdateTracking;
|
||||
use App\Console\Commands\UserCleanUp;
|
||||
use App\Console\Commands\UserMakeAboOrder;
|
||||
use App\Support\OperationalAlert;
|
||||
use Illuminate\Console\Scheduling\Schedule;
|
||||
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
class Kernel extends ConsoleKernel
|
||||
{
|
||||
|
|
@ -42,7 +44,7 @@ class Kernel extends ConsoleKernel
|
|||
// Uptime-Check: PAYONE-Erreichbarkeit alle 5 Minuten prüfen
|
||||
$schedule->command('payment:check-uptime')
|
||||
->everyFiveMinutes()
|
||||
->withoutOverlapping()
|
||||
->withoutOverlapping(10)
|
||||
->runInBackground();
|
||||
|
||||
// Job 1: Überprüft täglich um 02:00 Uhr die Zahlungskonten.
|
||||
|
|
@ -62,7 +64,7 @@ class Kernel extends ConsoleKernel
|
|||
// verzögerter Stripe-Retry; Stripe CLI auf dem Test-Server retryt gar nicht).
|
||||
$schedule->command('stripe:process-pending-events')
|
||||
->everyTenMinutes()
|
||||
->withoutOverlapping()
|
||||
->withoutOverlapping(30)
|
||||
->runInBackground();
|
||||
|
||||
// Stripe: offene Zahlungen aktiv gegen die API abgleichen. Fängt den Fall
|
||||
|
|
@ -72,7 +74,7 @@ class Kernel extends ConsoleKernel
|
|||
// Läuft über denselben, idempotenten Paid-Pfad wie ein echter Webhook.
|
||||
$schedule->command('stripe:sync-payment-status --pending --days=3')
|
||||
->hourly()
|
||||
->withoutOverlapping()
|
||||
->withoutOverlapping(60)
|
||||
->runInBackground();
|
||||
|
||||
// Abo-Einmalprodukte: Reminder-Mail vor der Ausführung (Einmalprodukte-Fenster).
|
||||
|
|
@ -108,8 +110,19 @@ class Kernel extends ConsoleKernel
|
|||
// DHL Tracking Update: Stündlich mit status-basierten Intervallen und Batch-API
|
||||
$schedule->command('dhl:update-tracking --days=30 --send-emails')
|
||||
->hourly()
|
||||
->withoutOverlapping()
|
||||
->withoutOverlapping(60)
|
||||
->runInBackground();
|
||||
|
||||
// Betriebs-Alert (Log, Mail, optional ntfy), wenn eine geplante Aufgabe fehlschlaegt
|
||||
foreach ($schedule->events() as $event) {
|
||||
$event->onFailure(function () use ($event): void {
|
||||
$task = $event->description ?: trim(Str::after((string) $event->command, "'artisan'"));
|
||||
|
||||
OperationalAlert::send('Geplante Aufgabe fehlgeschlagen: '.$task, [
|
||||
'exit_code' => $event->exitCode,
|
||||
]);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -6,9 +6,12 @@ use App\Services\Backoffice\BackofficeDashboardService;
|
|||
use App\Services\Payment\Payone\PayonePaymentService;
|
||||
use App\Services\Payment\Stripe\StripeClientWrapper;
|
||||
use App\Services\Util;
|
||||
use App\Support\OperationalAlert;
|
||||
use Barryvdh\LaravelIdeHelper\IdeHelperServiceProvider;
|
||||
use Illuminate\Foundation\Console\AboutCommand;
|
||||
use Illuminate\Pagination\Paginator;
|
||||
use Illuminate\Queue\Events\JobFailed;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
use Illuminate\Support\Facades\URL;
|
||||
use Illuminate\Support\ServiceProvider;
|
||||
|
|
@ -30,6 +33,14 @@ class AppServiceProvider extends ServiceProvider
|
|||
URL::forceScheme('https');
|
||||
}
|
||||
|
||||
Queue::failing(function (JobFailed $event): void {
|
||||
OperationalAlert::send('Queue-Job fehlgeschlagen: '.$event->job->resolveName(), [
|
||||
'connection' => $event->connectionName,
|
||||
'queue' => $event->job->getQueue(),
|
||||
'exception' => $event->exception->getMessage(),
|
||||
]);
|
||||
});
|
||||
|
||||
AboutCommand::add('mivita', fn (): array => [
|
||||
'Version' => config('app.version') ?? 'unbekannt',
|
||||
]);
|
||||
|
|
|
|||
107
app/Support/OperationalAlert.php
Normal file
107
app/Support/OperationalAlert.php
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
<?php
|
||||
|
||||
namespace App\Support;
|
||||
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Mail;
|
||||
use Illuminate\Support\Str;
|
||||
use Throwable;
|
||||
|
||||
/**
|
||||
* Zentrale Betriebs-Alerts: meldet fehlgeschlagene geplante Aufgaben
|
||||
* (Console\Kernel) und Queue-Jobs (AppServiceProvider). Der Log-Eintrag
|
||||
* (`critical`, Kanal monitoring) läuft immer; Mail und ntfy-Push sind
|
||||
* Best-Effort, unabhängig voneinander, und dürfen den auslösenden Prozess nie
|
||||
* stören. Gleiche Meldungen gehen höchstens einmal je Drosselzeit raus, damit
|
||||
* ein Task im 5-Minuten-Takt kein Postfach flutet.
|
||||
*/
|
||||
class OperationalAlert
|
||||
{
|
||||
/**
|
||||
* @param array<string, mixed> $context
|
||||
*/
|
||||
public static function send(string $summary, array $context = []): void
|
||||
{
|
||||
Log::channel('monitoring')->critical('[monitoring] '.$summary, $context);
|
||||
|
||||
if (! config('monitoring.alerts.enabled') || ! self::firstWithinThrottle($summary)) {
|
||||
return;
|
||||
}
|
||||
|
||||
self::mail($summary, $context);
|
||||
self::push($summary, $context);
|
||||
}
|
||||
|
||||
private static function firstWithinThrottle(string $summary): bool
|
||||
{
|
||||
try {
|
||||
return Cache::add('monitoring:alert:'.sha1($summary), true, now()->addMinutes((int) config('monitoring.alerts.throttle_minutes')));
|
||||
} catch (Throwable) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $context
|
||||
*/
|
||||
private static function mail(string $summary, array $context): void
|
||||
{
|
||||
$recipient = config('monitoring.alerts.mail_to');
|
||||
|
||||
if (empty($recipient)) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
Mail::raw(
|
||||
$summary."\n\n".json_encode($context, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
fn ($message) => $message->to($recipient)->subject('['.config('app.name').'] Betriebs-Alert: '.Str::limit($summary, 80)),
|
||||
);
|
||||
} catch (Throwable $e) {
|
||||
Log::channel('monitoring')->error('[monitoring] Alert-Mail konnte nicht versendet werden.', [
|
||||
'summary' => $summary,
|
||||
'error' => $e->getMessage(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON-Publish an die ntfy-Basis-URL (UTF-8-sicher, anders als Header).
|
||||
*
|
||||
* @param array<string, mixed> $context
|
||||
*/
|
||||
private static function push(string $summary, array $context): void
|
||||
{
|
||||
$url = (string) config('monitoring.alerts.ntfy.url');
|
||||
$topic = (string) config('monitoring.alerts.ntfy.topic');
|
||||
|
||||
if ($url === '' || $topic === '') {
|
||||
return;
|
||||
}
|
||||
|
||||
$details = collect($context)
|
||||
->map(fn (mixed $value, string $key): string => $key.': '.(is_scalar($value) || $value === null ? (string) $value : json_encode($value)))
|
||||
->implode("\n");
|
||||
|
||||
try {
|
||||
Http::timeout(5)
|
||||
->connectTimeout(3)
|
||||
->when(filled(config('monitoring.alerts.ntfy.token')), fn ($request) => $request->withToken((string) config('monitoring.alerts.ntfy.token')))
|
||||
->post(rtrim($url, '/'), [
|
||||
'topic' => $topic,
|
||||
'title' => config('app.name').' ('.app()->environment().'): Betriebs-Alert',
|
||||
'message' => Str::limit(trim($summary."\n\n".$details), 3500),
|
||||
'priority' => (int) config('monitoring.alerts.ntfy.priority'),
|
||||
'tags' => ['rotating_light'],
|
||||
])
|
||||
->throw();
|
||||
} catch (Throwable $e) {
|
||||
Log::channel('monitoring')->error('[monitoring] ntfy-Push konnte nicht versendet werden.', [
|
||||
'summary' => $summary,
|
||||
'error' => $e->getMessage(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue