- GlitchTip über die Sentry-SDKs; Datenschutzfilter (nur User-ID, keine
Request-Bodys, Tokens aus URL/Query/Referer, E-Mails aus Meldungen).
Browser-SDK erst beim ersten Fehler nachgeladen.
- ReportErrorSignals: tote interne Links (404 mit eigenem Referrer), 403/429
für Angemeldete, 419 als gedrosselte Warnung.
- /health (Bearer-Token): DB, Cache, Queue-Rückstau, Scheduler-Herzschlag,
Plattenplatz → 200/503; Scheduler und Plattenplatz auch im Systemstatus.
- OperationalAlert zusätzlich per ntfy (JSON-POST), unabhängig von der Mail.
- Verlauf: login_failed (Grund unbekannt/kein Passwort/falsch),
login_lockout, two_factor_failed; IP nur als gekürzter HMAC.
- Umami: Kundenbereich mit eigenem Schalter und identify('user-<id>'),
nie im Admin/bei Impersonation; Query-Filter auch auf den Webseiten
(signierte Links gingen bisher an Umami).
- queue:prune-failed nach 30 Tagen.
- Versionsnummer: config('app.version') aus APP_VERSION oder Datei RELEASE
(git describe --tags), in GlitchTip, /health, Systemstatus, artisan about.
Plan: dev/betrieb/monitoring-plan.md (Entscheidungen M1–M9).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
132 lines
5 KiB
PHP
132 lines
5 KiB
PHP
<?php
|
|
|
|
use App\Mail\OperationalAlertMail;
|
|
use App\Support\OperationalAlert;
|
|
use Illuminate\Contracts\Queue\Job;
|
|
use Illuminate\Http\Client\Request;
|
|
use Illuminate\Queue\Events\JobFailed;
|
|
use Illuminate\Support\Facades\Http;
|
|
use Illuminate\Support\Facades\Log;
|
|
use Illuminate\Support\Facades\Mail;
|
|
use Psr\Log\LoggerInterface;
|
|
|
|
test('an alert always logs critical and mails the configured recipient', function () {
|
|
config()->set('monitoring.alerts.enabled', true);
|
|
config()->set('monitoring.alerts.mail_to', 'ops@example.com');
|
|
Mail::fake();
|
|
$monitoringLog = Mockery::spy(LoggerInterface::class);
|
|
Log::shouldReceive('channel')->with('monitoring')->andReturn($monitoringLog);
|
|
|
|
OperationalAlert::send('Geplanter Task fehlgeschlagen: billing:generate-manual-invoices', ['exit_code' => 1]);
|
|
|
|
$monitoringLog->shouldHaveReceived('critical')->once();
|
|
Mail::assertSent(OperationalAlertMail::class, function (OperationalAlertMail $mail) {
|
|
return $mail->hasTo('ops@example.com')
|
|
&& str_contains($mail->summary, 'billing:generate-manual-invoices')
|
|
&& $mail->context === ['exit_code' => 1];
|
|
});
|
|
});
|
|
|
|
test('no mail is sent when alerts are disabled', function () {
|
|
config()->set('monitoring.alerts.enabled', false);
|
|
config()->set('monitoring.alerts.mail_to', 'ops@example.com');
|
|
Mail::fake();
|
|
|
|
OperationalAlert::send('Etwas ist kaputt');
|
|
|
|
Mail::assertNothingSent();
|
|
});
|
|
|
|
test('no mail is sent when no recipient is configured', function () {
|
|
config()->set('monitoring.alerts.enabled', true);
|
|
config()->set('monitoring.alerts.mail_to', null);
|
|
Mail::fake();
|
|
|
|
OperationalAlert::send('Etwas ist kaputt');
|
|
|
|
Mail::assertNothingSent();
|
|
});
|
|
|
|
test('a mail failure never bubbles up to the caller', function () {
|
|
config()->set('monitoring.alerts.enabled', true);
|
|
config()->set('monitoring.alerts.mail_to', 'ops@example.com');
|
|
|
|
// Mailserver down: der Versand wirft — der Alert muss das schlucken,
|
|
// damit der auslösende Scheduler-/Queue-Prozess weiterläuft.
|
|
Mail::shouldReceive('to')->andReturnSelf();
|
|
Mail::shouldReceive('send')->andThrow(new RuntimeException('mail down'));
|
|
|
|
OperationalAlert::send('Etwas ist kaputt');
|
|
|
|
expect(true)->toBeTrue();
|
|
});
|
|
|
|
test('a failing queue job triggers an operational alert via the registered listener', function () {
|
|
config()->set('monitoring.alerts.enabled', true);
|
|
config()->set('monitoring.alerts.mail_to', 'ops@example.com');
|
|
Mail::fake();
|
|
|
|
$job = Mockery::mock(Job::class);
|
|
$job->shouldReceive('resolveName')->andReturn('App\\Jobs\\ClassifyPressRelease');
|
|
$job->shouldReceive('getQueue')->andReturn('classification');
|
|
|
|
event(new JobFailed('database', $job, new RuntimeException('job boom')));
|
|
|
|
Mail::assertSent(OperationalAlertMail::class, function (OperationalAlertMail $mail) {
|
|
return str_contains($mail->summary, 'ClassifyPressRelease');
|
|
});
|
|
});
|
|
|
|
test('an alert is also pushed to ntfy as json with token and priority', function () {
|
|
config()->set('monitoring.alerts.enabled', true);
|
|
config()->set('monitoring.alerts.mail_to', null);
|
|
config()->set('monitoring.alerts.ntfy', [
|
|
'url' => 'https://ntfy.example.com/',
|
|
'topic' => 'pressekonto-ops',
|
|
'token' => 'tk_geheim',
|
|
'priority' => 4,
|
|
]);
|
|
Http::fake(['ntfy.example.com/*' => Http::response(['id' => 'abc'])]);
|
|
|
|
OperationalAlert::send('Geplanter Task fehlgeschlagen: archive:purge', ['exit_code' => 1, 'queue' => ['a' => 'b']]);
|
|
|
|
Http::assertSent(function (Request $request) {
|
|
return $request->url() === 'https://ntfy.example.com'
|
|
&& $request->hasHeader('Authorization', 'Bearer tk_geheim')
|
|
&& $request['topic'] === 'pressekonto-ops'
|
|
&& $request['priority'] === 4
|
|
&& str_contains($request['message'], 'archive:purge')
|
|
&& str_contains($request['message'], 'exit_code: 1')
|
|
&& str_contains($request['message'], 'queue: {"a":"b"}');
|
|
});
|
|
});
|
|
|
|
test('a failing ntfy push neither bubbles up nor stops the mail', function () {
|
|
config()->set('monitoring.alerts.enabled', true);
|
|
config()->set('monitoring.alerts.mail_to', 'ops@example.com');
|
|
config()->set('monitoring.alerts.ntfy.url', 'https://ntfy.example.com');
|
|
config()->set('monitoring.alerts.ntfy.topic', 'pressekonto-ops');
|
|
Mail::fake();
|
|
Http::fake(['ntfy.example.com/*' => Http::response('down', 502)]);
|
|
|
|
OperationalAlert::send('Etwas ist kaputt');
|
|
|
|
Mail::assertSent(OperationalAlertMail::class);
|
|
Http::assertSentCount(1);
|
|
});
|
|
|
|
test('no push is sent when alerts are disabled or ntfy is not configured', function () {
|
|
Http::fake();
|
|
config()->set('monitoring.alerts.ntfy.url', 'https://ntfy.example.com');
|
|
config()->set('monitoring.alerts.ntfy.topic', null);
|
|
config()->set('monitoring.alerts.enabled', true);
|
|
|
|
OperationalAlert::send('Etwas ist kaputt');
|
|
|
|
config()->set('monitoring.alerts.ntfy.topic', 'pressekonto-ops');
|
|
config()->set('monitoring.alerts.enabled', false);
|
|
|
|
OperationalAlert::send('Etwas ist kaputt');
|
|
|
|
Http::assertNothingSent();
|
|
});
|