Alarm-Mail bei fehlgeschlagenen geplanten Aufgaben und Queue-Jobs

- OperationalAlert: Log (Kanal monitoring) immer, Mail an
  MONITORING_ALERT_EMAIL (Standard EXCEPTION_MAIL), optional ntfy;
  gleiche Meldung hoechstens einmal pro Stunde
- Kernel: onFailure fuer alle geplanten Aufgaben (auch kuenftige)
- AppServiceProvider: Queue::failing
- withoutOverlapping mit kurzen Sperrdauern (10-60 Min.) statt 24 h

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin 2026-10-08 09:49:18 +00:00
parent fdbb4f3853
commit c9cd17a41a
7 changed files with 300 additions and 4 deletions

View file

@ -0,0 +1,123 @@
<?php
use App\Support\OperationalAlert;
use Illuminate\Console\Scheduling\Event as ScheduledEvent;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Contracts\Queue\Job;
use Illuminate\Http\Client\Request;
use Illuminate\Mail\Events\MessageSent;
use Illuminate\Queue\Events\JobFailed;
use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
use Psr\Log\LoggerInterface;
use Tests\TestCase;
uses(TestCase::class);
/**
* Betriebs-Alerts: Log immer, Mail und ntfy unabhängig voneinander und
* Best-Effort — ein ausgefallener Kanal darf den auslösenden Prozess nie stören.
*/
beforeEach(function () {
config()->set('cache.default', 'array');
config()->set('monitoring.alerts.enabled', true);
config()->set('monitoring.alerts.mail_to', null);
config()->set('monitoring.alerts.throttle_minutes', 60);
config()->set('monitoring.alerts.ntfy', ['url' => null, 'topic' => null, 'token' => null, 'priority' => 4]);
});
test('an alert always logs critical to the monitoring channel', function () {
$monitoringLog = Mockery::spy(LoggerInterface::class);
Log::shouldReceive('channel')->with('monitoring')->andReturn($monitoringLog);
OperationalAlert::send('Geplante Aufgabe fehlgeschlagen: demo', ['exit_code' => 1]);
$monitoringLog->shouldHaveReceived('critical')->once();
});
test('an alert mails the configured recipient', function () {
config()->set('monitoring.alerts.mail_to', 'ops@example.com');
Event::fake([MessageSent::class]);
OperationalAlert::send('Etwas ist kaputt', ['exit_code' => 1]);
Event::assertDispatched(MessageSent::class, function (MessageSent $event): bool {
$recipients = array_map(fn ($address): string => $address->getAddress(), $event->message->getTo());
return in_array('ops@example.com', $recipients, true)
&& str_contains((string) $event->message->getSubject(), 'Etwas ist kaputt');
});
});
test('the same alert is mailed only once within the throttle window', function () {
config()->set('monitoring.alerts.mail_to', 'ops@example.com');
Event::fake([MessageSent::class]);
OperationalAlert::send('PAYONE nicht erreichbar');
OperationalAlert::send('PAYONE nicht erreichbar');
OperationalAlert::send('Anderer Fehler');
Event::assertDispatchedTimes(MessageSent::class, 2);
$this->travel(61)->minutes();
OperationalAlert::send('PAYONE nicht erreichbar');
Event::assertDispatchedTimes(MessageSent::class, 3);
});
test('an alert is pushed to ntfy as json with token and priority', function () {
config()->set('monitoring.alerts.ntfy', ['url' => 'https://push.example.com/', 'topic' => 'mivita-ops', 'token' => 'tk_geheim', 'priority' => 4]);
Http::fake(['push.example.com*' => Http::response(['id' => 'x'])]);
OperationalAlert::send('Queue-Job fehlgeschlagen: Demo', ['exit_code' => 1]);
Http::assertSent(fn (Request $request): bool => $request->url() === 'https://push.example.com'
&& $request->hasHeader('Authorization', 'Bearer tk_geheim')
&& $request['topic'] === 'mivita-ops'
&& $request['priority'] === 4
&& str_contains($request['message'], 'exit_code: 1'));
});
test('a failing push never bubbles up and nothing is sent when alerts are off', function () {
config()->set('monitoring.alerts.ntfy', ['url' => 'https://push.example.com', 'topic' => 'mivita-ops', 'token' => null, 'priority' => 4]);
Http::fake(['push.example.com*' => Http::response('down', 502)]);
OperationalAlert::send('Etwas ist kaputt');
Http::assertSentCount(1);
config()->set('monitoring.alerts.enabled', false);
OperationalAlert::send('Etwas anderes ist kaputt');
Http::assertSentCount(1);
});
test('a failed queue job triggers an alert via the registered listener', function () {
config()->set('monitoring.alerts.mail_to', 'ops@example.com');
Event::fake([MessageSent::class]);
$job = Mockery::mock(Job::class)->shouldIgnoreMissing();
$job->shouldReceive('resolveName')->andReturn('App\\Jobs\\SyncStripeInvoiceReference');
$job->shouldReceive('getQueue')->andReturn('default');
event(new JobFailed('redis', $job, new RuntimeException('boom')));
Event::assertDispatched(MessageSent::class, fn (MessageSent $sent): bool => str_contains((string) $sent->message->getSubject(), 'SyncStripeInvoiceReference'));
});
test('every scheduled task alerts when it fails', function (string $command) {
config()->set('monitoring.alerts.mail_to', 'ops@example.com');
Event::fake([MessageSent::class]);
$event = collect(app(Schedule::class)->events())
->first(fn (ScheduledEvent $event): bool => str_contains((string) $event->command, $command));
expect($event)->not->toBeNull();
$event->finish(app(), 1);
Event::assertDispatched(MessageSent::class, fn (MessageSent $sent): bool => str_contains((string) $sent->message->getSubject(), $command));
})->with([
'Provisionen' => 'business:store-optimized',
'Abo-Bestellungen' => 'user:make_abo_order',
'Zahlungsprüfung' => 'payments:check-accounts',
]);