Alarm-Mail bei fehlgeschlagenen geplanten Aufgaben und Queue-Jobs
- OperationalAlert: Log (Kanal monitoring) immer, Mail an MONITORING_ALERT_EMAIL (Standard EXCEPTION_MAIL), optional ntfy; gleiche Meldung hoechstens einmal pro Stunde - Kernel: onFailure fuer alle geplanten Aufgaben (auch kuenftige) - AppServiceProvider: Queue::failing - withoutOverlapping mit kurzen Sperrdauern (10-60 Min.) statt 24 h Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
fdbb4f3853
commit
c9cd17a41a
7 changed files with 300 additions and 4 deletions
123
tests/Feature/Monitoring/OperationalAlertTest.php
Normal file
123
tests/Feature/Monitoring/OperationalAlertTest.php
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
<?php
|
||||
|
||||
use App\Support\OperationalAlert;
|
||||
use Illuminate\Console\Scheduling\Event as ScheduledEvent;
|
||||
use Illuminate\Console\Scheduling\Schedule;
|
||||
use Illuminate\Contracts\Queue\Job;
|
||||
use Illuminate\Http\Client\Request;
|
||||
use Illuminate\Mail\Events\MessageSent;
|
||||
use Illuminate\Queue\Events\JobFailed;
|
||||
use Illuminate\Support\Facades\Event;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Tests\TestCase;
|
||||
|
||||
uses(TestCase::class);
|
||||
|
||||
/**
|
||||
* Betriebs-Alerts: Log immer, Mail und ntfy unabhängig voneinander und
|
||||
* Best-Effort — ein ausgefallener Kanal darf den auslösenden Prozess nie stören.
|
||||
*/
|
||||
beforeEach(function () {
|
||||
config()->set('cache.default', 'array');
|
||||
config()->set('monitoring.alerts.enabled', true);
|
||||
config()->set('monitoring.alerts.mail_to', null);
|
||||
config()->set('monitoring.alerts.throttle_minutes', 60);
|
||||
config()->set('monitoring.alerts.ntfy', ['url' => null, 'topic' => null, 'token' => null, 'priority' => 4]);
|
||||
});
|
||||
|
||||
test('an alert always logs critical to the monitoring channel', function () {
|
||||
$monitoringLog = Mockery::spy(LoggerInterface::class);
|
||||
Log::shouldReceive('channel')->with('monitoring')->andReturn($monitoringLog);
|
||||
|
||||
OperationalAlert::send('Geplante Aufgabe fehlgeschlagen: demo', ['exit_code' => 1]);
|
||||
|
||||
$monitoringLog->shouldHaveReceived('critical')->once();
|
||||
});
|
||||
|
||||
test('an alert mails the configured recipient', function () {
|
||||
config()->set('monitoring.alerts.mail_to', 'ops@example.com');
|
||||
Event::fake([MessageSent::class]);
|
||||
|
||||
OperationalAlert::send('Etwas ist kaputt', ['exit_code' => 1]);
|
||||
|
||||
Event::assertDispatched(MessageSent::class, function (MessageSent $event): bool {
|
||||
$recipients = array_map(fn ($address): string => $address->getAddress(), $event->message->getTo());
|
||||
|
||||
return in_array('ops@example.com', $recipients, true)
|
||||
&& str_contains((string) $event->message->getSubject(), 'Etwas ist kaputt');
|
||||
});
|
||||
});
|
||||
|
||||
test('the same alert is mailed only once within the throttle window', function () {
|
||||
config()->set('monitoring.alerts.mail_to', 'ops@example.com');
|
||||
Event::fake([MessageSent::class]);
|
||||
|
||||
OperationalAlert::send('PAYONE nicht erreichbar');
|
||||
OperationalAlert::send('PAYONE nicht erreichbar');
|
||||
OperationalAlert::send('Anderer Fehler');
|
||||
|
||||
Event::assertDispatchedTimes(MessageSent::class, 2);
|
||||
|
||||
$this->travel(61)->minutes();
|
||||
OperationalAlert::send('PAYONE nicht erreichbar');
|
||||
|
||||
Event::assertDispatchedTimes(MessageSent::class, 3);
|
||||
});
|
||||
|
||||
test('an alert is pushed to ntfy as json with token and priority', function () {
|
||||
config()->set('monitoring.alerts.ntfy', ['url' => 'https://push.example.com/', 'topic' => 'mivita-ops', 'token' => 'tk_geheim', 'priority' => 4]);
|
||||
Http::fake(['push.example.com*' => Http::response(['id' => 'x'])]);
|
||||
|
||||
OperationalAlert::send('Queue-Job fehlgeschlagen: Demo', ['exit_code' => 1]);
|
||||
|
||||
Http::assertSent(fn (Request $request): bool => $request->url() === 'https://push.example.com'
|
||||
&& $request->hasHeader('Authorization', 'Bearer tk_geheim')
|
||||
&& $request['topic'] === 'mivita-ops'
|
||||
&& $request['priority'] === 4
|
||||
&& str_contains($request['message'], 'exit_code: 1'));
|
||||
});
|
||||
|
||||
test('a failing push never bubbles up and nothing is sent when alerts are off', function () {
|
||||
config()->set('monitoring.alerts.ntfy', ['url' => 'https://push.example.com', 'topic' => 'mivita-ops', 'token' => null, 'priority' => 4]);
|
||||
Http::fake(['push.example.com*' => Http::response('down', 502)]);
|
||||
|
||||
OperationalAlert::send('Etwas ist kaputt');
|
||||
Http::assertSentCount(1);
|
||||
|
||||
config()->set('monitoring.alerts.enabled', false);
|
||||
OperationalAlert::send('Etwas anderes ist kaputt');
|
||||
Http::assertSentCount(1);
|
||||
});
|
||||
|
||||
test('a failed queue job triggers an alert via the registered listener', function () {
|
||||
config()->set('monitoring.alerts.mail_to', 'ops@example.com');
|
||||
Event::fake([MessageSent::class]);
|
||||
|
||||
$job = Mockery::mock(Job::class)->shouldIgnoreMissing();
|
||||
$job->shouldReceive('resolveName')->andReturn('App\\Jobs\\SyncStripeInvoiceReference');
|
||||
$job->shouldReceive('getQueue')->andReturn('default');
|
||||
|
||||
event(new JobFailed('redis', $job, new RuntimeException('boom')));
|
||||
|
||||
Event::assertDispatched(MessageSent::class, fn (MessageSent $sent): bool => str_contains((string) $sent->message->getSubject(), 'SyncStripeInvoiceReference'));
|
||||
});
|
||||
|
||||
test('every scheduled task alerts when it fails', function (string $command) {
|
||||
config()->set('monitoring.alerts.mail_to', 'ops@example.com');
|
||||
Event::fake([MessageSent::class]);
|
||||
|
||||
$event = collect(app(Schedule::class)->events())
|
||||
->first(fn (ScheduledEvent $event): bool => str_contains((string) $event->command, $command));
|
||||
|
||||
expect($event)->not->toBeNull();
|
||||
|
||||
$event->finish(app(), 1);
|
||||
|
||||
Event::assertDispatched(MessageSent::class, fn (MessageSent $sent): bool => str_contains((string) $sent->message->getSubject(), $command));
|
||||
})->with([
|
||||
'Provisionen' => 'business:store-optimized',
|
||||
'Abo-Bestellungen' => 'user:make_abo_order',
|
||||
'Zahlungsprüfung' => 'payments:check-accounts',
|
||||
]);
|
||||
Loading…
Add table
Add a link
Reference in a new issue