monitoring: Betriebsüberwachung Phase 1 — GlitchTip (Server- und Browserfehler), Fehlersignale ohne Exception, /health mit Scheduler-Herzschlag, ntfy-Alarme, fehlgeschlagene Anmeldungen im Verlauf, Umami im Kundenbereich, Versionsnummer
- GlitchTip über die Sentry-SDKs; Datenschutzfilter (nur User-ID, keine
Request-Bodys, Tokens aus URL/Query/Referer, E-Mails aus Meldungen).
Browser-SDK erst beim ersten Fehler nachgeladen.
- ReportErrorSignals: tote interne Links (404 mit eigenem Referrer), 403/429
für Angemeldete, 419 als gedrosselte Warnung.
- /health (Bearer-Token): DB, Cache, Queue-Rückstau, Scheduler-Herzschlag,
Plattenplatz → 200/503; Scheduler und Plattenplatz auch im Systemstatus.
- OperationalAlert zusätzlich per ntfy (JSON-POST), unabhängig von der Mail.
- Verlauf: login_failed (Grund unbekannt/kein Passwort/falsch),
login_lockout, two_factor_failed; IP nur als gekürzter HMAC.
- Umami: Kundenbereich mit eigenem Schalter und identify('user-<id>'),
nie im Admin/bei Impersonation; Query-Filter auch auf den Webseiten
(signierte Links gingen bisher an Umami).
- queue:prune-failed nach 30 Tagen.
- Versionsnummer: config('app.version') aus APP_VERSION oder Datei RELEASE
(git describe --tags), in GlitchTip, /health, Systemstatus, artisan about.
Plan: dev/betrieb/monitoring-plan.md (Entscheidungen M1–M9).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f2ed268b51
commit
8769a71100
42 changed files with 1896 additions and 9 deletions
|
|
@ -8,6 +8,7 @@ use App\Models\PressReleaseAttachment;
|
|||
use App\Models\PressReleaseImage;
|
||||
use App\Services\Archive\ArchivePendingQueue;
|
||||
use App\Services\Archive\EvidenceArchive;
|
||||
use App\Services\Monitoring\HealthProbe;
|
||||
use App\Services\PressRelease\Images\PressReleaseImageRemover;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
|
@ -160,6 +161,12 @@ class AdminSystemHealth
|
|||
'detail' => $this->queueWorkerDetail($queue),
|
||||
];
|
||||
|
||||
// Scheduler-Herzschlag und Plattenplatz — dieselben Prüfungen, die
|
||||
// der externe Monitor über /health abfragt (Monitoring-Plan 1.4/1.5).
|
||||
$probe = app(HealthProbe::class);
|
||||
$checks[] = $probe->scheduler();
|
||||
$checks[] = $probe->disk();
|
||||
|
||||
// Neue Permissions ausgerollt?
|
||||
$expected = PermissionCatalog::adminOnly();
|
||||
$existing = Permission::query()->whereIn('name', $expected)->pluck('name')->all();
|
||||
|
|
|
|||
193
app/Services/Monitoring/HealthProbe.php
Normal file
193
app/Services/Monitoring/HealthProbe.php
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Monitoring;
|
||||
|
||||
use App\Models\AppSetting;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Str;
|
||||
use Throwable;
|
||||
|
||||
/**
|
||||
* Die Prüfungen, die ein externer Monitor jede Minute abfragt
|
||||
* (Monitoring-Plan 1.4, 1.5): Datenbank, Cache, Queue-Rückstau,
|
||||
* Scheduler-Herzschlag, Plattenplatz. Bewusst billig — keine Zählungen über
|
||||
* große Tabellen, damit `/health` selbst keine Last erzeugt.
|
||||
*
|
||||
* Der Systemstatus im Admin (AdminSystemHealth) übernimmt Scheduler und
|
||||
* Plattenplatz von hier.
|
||||
*/
|
||||
class HealthProbe
|
||||
{
|
||||
public const StatusOk = 'ok';
|
||||
|
||||
public const StatusWarn = 'warn';
|
||||
|
||||
public const StatusCritical = 'critical';
|
||||
|
||||
public const HeartbeatKey = 'monitoring.scheduler_heartbeat';
|
||||
|
||||
public function recordHeartbeat(): void
|
||||
{
|
||||
AppSetting::setValue(self::HeartbeatKey, now()->toIso8601String());
|
||||
}
|
||||
|
||||
public function lastHeartbeat(): ?Carbon
|
||||
{
|
||||
$value = AppSetting::getValueFresh(self::HeartbeatKey);
|
||||
|
||||
return is_string($value) ? Carbon::parse($value) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list<array{key: string, label: string, status: string, detail: string}>
|
||||
*/
|
||||
public function checks(): array
|
||||
{
|
||||
return [
|
||||
$this->database(),
|
||||
$this->cache(),
|
||||
$this->queue(),
|
||||
$this->scheduler(),
|
||||
$this->disk(),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<array{status: string}> $checks
|
||||
*/
|
||||
public static function worstStatus(array $checks): string
|
||||
{
|
||||
$statuses = array_column($checks, 'status');
|
||||
|
||||
return match (true) {
|
||||
in_array(self::StatusCritical, $statuses, true) => self::StatusCritical,
|
||||
in_array(self::StatusWarn, $statuses, true) => self::StatusWarn,
|
||||
default => self::StatusOk,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
public function scheduler(): array
|
||||
{
|
||||
$lastBeat = $this->lastHeartbeat();
|
||||
$warnMinutes = (int) config('monitoring.scheduler.heartbeat_warn_minutes');
|
||||
$criticalMinutes = (int) config('monitoring.scheduler.heartbeat_critical_minutes');
|
||||
$minutesAgo = $lastBeat !== null ? (int) $lastBeat->diffInMinutes(now(), absolute: true) : null;
|
||||
|
||||
return $this->check(
|
||||
'scheduler',
|
||||
'Scheduler',
|
||||
match (true) {
|
||||
$minutesAgo === null, $minutesAgo >= $criticalMinutes => self::StatusCritical,
|
||||
$minutesAgo >= $warnMinutes => self::StatusWarn,
|
||||
default => self::StatusOk,
|
||||
},
|
||||
$lastBeat === null
|
||||
? 'Noch kein Herzschlag — läuft `schedule:run` per Cron (bzw. `schedule:work`)?'
|
||||
: 'Letzter Herzschlag '.$lastBeat->format('d.m.Y H:i:s').' (vor '.$minutesAgo.' Min.).',
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
public function disk(): array
|
||||
{
|
||||
$path = storage_path();
|
||||
$free = @disk_free_space($path);
|
||||
$total = @disk_total_space($path);
|
||||
|
||||
if ($free === false || $total === false || $total <= 0) {
|
||||
return $this->check('disk', 'Plattenplatz', self::StatusWarn, 'Plattenplatz nicht ermittelbar.');
|
||||
}
|
||||
|
||||
$freePercent = round($free / $total * 100, 1);
|
||||
$minPercent = (float) config('monitoring.health.disk_min_free_percent');
|
||||
|
||||
return $this->check(
|
||||
'disk',
|
||||
'Plattenplatz',
|
||||
match (true) {
|
||||
$freePercent < $minPercent => self::StatusCritical,
|
||||
$freePercent < $minPercent * 2 => self::StatusWarn,
|
||||
default => self::StatusOk,
|
||||
},
|
||||
sprintf('%s %% frei (%s GB von %s GB).', $freePercent, round($free / 1024 ** 3, 1), round($total / 1024 ** 3, 1)),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
private function database(): array
|
||||
{
|
||||
try {
|
||||
DB::select('select 1');
|
||||
|
||||
return $this->check('database', 'Datenbank', self::StatusOk, 'Verbindung steht.');
|
||||
} catch (Throwable $exception) {
|
||||
return $this->check('database', 'Datenbank', self::StatusCritical, 'Keine Verbindung: '.Str::limit($exception->getMessage(), 120));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
private function cache(): array
|
||||
{
|
||||
try {
|
||||
$value = Str::random(8);
|
||||
Cache::put('monitoring:health-probe', $value, 60);
|
||||
|
||||
return Cache::get('monitoring:health-probe') === $value
|
||||
? $this->check('cache', 'Cache', self::StatusOk, 'Schreiben und Lesen funktionieren.')
|
||||
: $this->check('cache', 'Cache', self::StatusCritical, 'Geschriebener Wert kommt nicht zurück.');
|
||||
} catch (Throwable $exception) {
|
||||
return $this->check('cache', 'Cache', self::StatusCritical, 'Nicht erreichbar: '.Str::limit($exception->getMessage(), 120));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
private function queue(): array
|
||||
{
|
||||
if (config('queue.default') !== 'database') {
|
||||
return $this->check('queue', 'Queue', self::StatusOk, 'Treiber '.config('queue.default').' — Rückstau hier nicht prüfbar.');
|
||||
}
|
||||
|
||||
try {
|
||||
$now = now()->getTimestamp();
|
||||
$oldestAvailable = DB::table('jobs')
|
||||
->whereNull('reserved_at')
|
||||
->where('available_at', '<=', $now)
|
||||
->min('available_at');
|
||||
} catch (Throwable $exception) {
|
||||
return $this->check('queue', 'Queue', self::StatusCritical, 'Jobs-Tabelle nicht lesbar: '.Str::limit($exception->getMessage(), 120));
|
||||
}
|
||||
|
||||
$waitingSeconds = $oldestAvailable !== null ? max(0, $now - (int) $oldestAvailable) : 0;
|
||||
$criticalSeconds = (int) config('monitoring.health.queue_backlog_critical_seconds');
|
||||
|
||||
return $this->check(
|
||||
'queue',
|
||||
'Queue',
|
||||
$waitingSeconds >= $criticalSeconds ? self::StatusCritical : self::StatusOk,
|
||||
$oldestAvailable === null
|
||||
? 'Kein wartender Job.'
|
||||
: 'Ältester wartender Job seit '.$waitingSeconds.' s.'.($waitingSeconds >= $criticalSeconds ? ' Worker gestoppt?' : ''),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
private function check(string $key, string $label, string $status, string $detail): array
|
||||
{
|
||||
return ['key' => $key, 'label' => $label, 'status' => $status, 'detail' => $detail];
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue