Kevin
8ad779ea3e
- HealthProbe: Datenbank, Cache, Queue (sync / database / Horizon), fehlgeschlagene Jobs (24 h), Scheduler-Herzschlag, Plattenplatz - /health auf jeder Domain, ohne web-Gruppe, nur mit MONITORING_HEALTH_TOKEN - monitoring:heartbeat minuetlich im Scheduler - Sysadmin -> Systemstatus (my.mivita.care/sysadmin/status) mit Version, Umgebung und zuletzt fehlgeschlagenen Jobs - Anleitung Abschnitt G Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
265 lines
9.8 KiB
PHP
265 lines
9.8 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Monitoring;
|
|
|
|
use Illuminate\Support\Carbon;
|
|
use Illuminate\Support\Facades\Cache;
|
|
use Illuminate\Support\Facades\DB;
|
|
use Illuminate\Support\Facades\Redis;
|
|
use Illuminate\Support\Str;
|
|
use Laravel\Horizon\Contracts\MasterSupervisorRepository;
|
|
use Laravel\Horizon\WaitTimeCalculator;
|
|
use Throwable;
|
|
|
|
/**
|
|
* Die Prüfungen hinter `/health` (externer Monitor) und dem Systemstatus im
|
|
* Sysadmin-Bereich: Datenbank, Cache, Queue, fehlgeschlagene Jobs,
|
|
* Scheduler-Herzschlag, Plattenplatz. Bewusst billig — keine Zählungen über
|
|
* große Tabellen, damit `/health` selbst keine Last erzeugt.
|
|
*/
|
|
class HealthProbe
|
|
{
|
|
public const StatusOk = 'ok';
|
|
|
|
public const StatusWarn = 'warn';
|
|
|
|
public const StatusCritical = 'critical';
|
|
|
|
/**
|
|
* Datei statt Datenbank/Cache: übersteht `cache:clear`, braucht keine
|
|
* Migration. Bei mehreren App-Servern stattdessen eine DB-Zeile nehmen.
|
|
*/
|
|
public static function heartbeatPath(): string
|
|
{
|
|
return storage_path('framework/scheduler-heartbeat');
|
|
}
|
|
|
|
public function recordHeartbeat(): void
|
|
{
|
|
file_put_contents(self::heartbeatPath(), now()->toIso8601String(), LOCK_EX);
|
|
}
|
|
|
|
public function lastHeartbeat(): ?Carbon
|
|
{
|
|
$value = @file_get_contents(self::heartbeatPath());
|
|
|
|
return is_string($value) && trim($value) !== '' ? Carbon::parse(trim($value)) : null;
|
|
}
|
|
|
|
/**
|
|
* @return list<array{key: string, label: string, status: string, detail: string}>
|
|
*/
|
|
public function checks(): array
|
|
{
|
|
return [
|
|
$this->database(),
|
|
$this->cache(),
|
|
$this->queue(),
|
|
$this->failedJobs(),
|
|
$this->scheduler(),
|
|
$this->disk(),
|
|
];
|
|
}
|
|
|
|
/**
|
|
* @param list<array{status: string}> $checks
|
|
*/
|
|
public static function worstStatus(array $checks): string
|
|
{
|
|
$statuses = array_column($checks, 'status');
|
|
|
|
return match (true) {
|
|
in_array(self::StatusCritical, $statuses, true) => self::StatusCritical,
|
|
in_array(self::StatusWarn, $statuses, true) => self::StatusWarn,
|
|
default => self::StatusOk,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @return array{key: string, label: string, status: string, detail: string}
|
|
*/
|
|
public function scheduler(): array
|
|
{
|
|
$lastBeat = $this->lastHeartbeat();
|
|
$warnMinutes = (int) config('monitoring.scheduler.heartbeat_warn_minutes');
|
|
$criticalMinutes = (int) config('monitoring.scheduler.heartbeat_critical_minutes');
|
|
$minutesAgo = $lastBeat !== null ? (int) $lastBeat->diffInMinutes(now(), absolute: true) : null;
|
|
|
|
return $this->check(
|
|
'scheduler',
|
|
'Scheduler',
|
|
match (true) {
|
|
$minutesAgo === null, $minutesAgo >= $criticalMinutes => self::StatusCritical,
|
|
$minutesAgo >= $warnMinutes => self::StatusWarn,
|
|
default => self::StatusOk,
|
|
},
|
|
$lastBeat === null
|
|
? 'Noch kein Herzschlag — läuft `schedule:run` minütlich per Cron?'
|
|
: 'Letzter Herzschlag '.$lastBeat->format('d.m.Y H:i:s').' (vor '.$minutesAgo.' Min.).',
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @return array{key: string, label: string, status: string, detail: string}
|
|
*/
|
|
public function disk(): array
|
|
{
|
|
$path = storage_path();
|
|
$free = @disk_free_space($path);
|
|
$total = @disk_total_space($path);
|
|
|
|
if ($free === false || $total === false || $total <= 0) {
|
|
return $this->check('disk', 'Plattenplatz', self::StatusWarn, 'Plattenplatz nicht ermittelbar.');
|
|
}
|
|
|
|
$freePercent = round($free / $total * 100, 1);
|
|
$minPercent = (float) config('monitoring.health.disk_min_free_percent');
|
|
|
|
return $this->check(
|
|
'disk',
|
|
'Plattenplatz',
|
|
match (true) {
|
|
$freePercent < $minPercent => self::StatusCritical,
|
|
$freePercent < $minPercent * 2 => self::StatusWarn,
|
|
default => self::StatusOk,
|
|
},
|
|
sprintf('%s %% frei (%s GB von %s GB).', $freePercent, round($free / 1024 ** 3, 1), round($total / 1024 ** 3, 1)),
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @return array{key: string, label: string, status: string, detail: string}
|
|
*/
|
|
private function database(): array
|
|
{
|
|
try {
|
|
DB::select('select 1');
|
|
|
|
return $this->check('database', 'Datenbank', self::StatusOk, 'Verbindung steht.');
|
|
} catch (Throwable $exception) {
|
|
return $this->check('database', 'Datenbank', self::StatusCritical, 'Keine Verbindung: '.Str::limit($exception->getMessage(), 120));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @return array{key: string, label: string, status: string, detail: string}
|
|
*/
|
|
private function cache(): array
|
|
{
|
|
try {
|
|
$value = Str::random(8);
|
|
Cache::put('monitoring:health-probe', $value, 60);
|
|
|
|
return Cache::get('monitoring:health-probe') === $value
|
|
? $this->check('cache', 'Cache', self::StatusOk, 'Schreiben und Lesen funktionieren ('.config('cache.default').').')
|
|
: $this->check('cache', 'Cache', self::StatusCritical, 'Geschriebener Wert kommt nicht zurück.');
|
|
} catch (Throwable $exception) {
|
|
return $this->check('cache', 'Cache', self::StatusCritical, 'Nicht erreichbar: '.Str::limit($exception->getMessage(), 120));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Je nach QUEUE_DRIVER: `sync` braucht keinen Worker, `database` wird am
|
|
* ältesten wartenden Job gemessen, `redis` an Horizon (läuft es, wie lange
|
|
* warten die Jobs).
|
|
*
|
|
* @return array{key: string, label: string, status: string, detail: string}
|
|
*/
|
|
private function queue(): array
|
|
{
|
|
return match (config('queue.default')) {
|
|
'sync' => $this->check('queue', 'Queue', self::StatusOk, 'Treiber sync — Jobs laufen direkt im Request, kein Worker nötig.'),
|
|
'database' => $this->databaseQueue(),
|
|
'redis' => $this->horizonQueue(),
|
|
default => $this->check('queue', 'Queue', self::StatusWarn, 'Treiber '.config('queue.default').' — hier nicht prüfbar.'),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @return array{key: string, label: string, status: string, detail: string}
|
|
*/
|
|
private function databaseQueue(): array
|
|
{
|
|
try {
|
|
$now = now()->getTimestamp();
|
|
$oldestAvailable = DB::table('jobs')
|
|
->whereNull('reserved_at')
|
|
->where('available_at', '<=', $now)
|
|
->min('available_at');
|
|
} catch (Throwable $exception) {
|
|
return $this->check('queue', 'Queue', self::StatusCritical, 'Jobs-Tabelle nicht lesbar: '.Str::limit($exception->getMessage(), 120));
|
|
}
|
|
|
|
$waitingSeconds = $oldestAvailable !== null ? max(0, $now - (int) $oldestAvailable) : 0;
|
|
$criticalSeconds = (int) config('monitoring.health.queue_backlog_critical_seconds');
|
|
|
|
return $this->check(
|
|
'queue',
|
|
'Queue',
|
|
$waitingSeconds >= $criticalSeconds ? self::StatusCritical : self::StatusOk,
|
|
$oldestAvailable === null
|
|
? 'Treiber database — kein wartender Job.'
|
|
: 'Treiber database — ältester wartender Job seit '.$waitingSeconds.' s.'.($waitingSeconds >= $criticalSeconds ? ' Worker gestoppt?' : ''),
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @return array{key: string, label: string, status: string, detail: string}
|
|
*/
|
|
private function horizonQueue(): array
|
|
{
|
|
try {
|
|
Redis::connection()->ping();
|
|
$masters = app(MasterSupervisorRepository::class)->all();
|
|
} catch (Throwable $exception) {
|
|
return $this->check('queue', 'Queue', self::StatusCritical, 'Redis nicht erreichbar: '.Str::limit($exception->getMessage(), 120));
|
|
}
|
|
|
|
if ($masters === []) {
|
|
return $this->check('queue', 'Queue', self::StatusCritical, 'Treiber redis — Horizon läuft nicht. Daemon in Ploi prüfen.');
|
|
}
|
|
|
|
if (collect($masters)->every(fn (object $master): bool => $master->status === 'paused')) {
|
|
return $this->check('queue', 'Queue', self::StatusWarn, 'Treiber redis — Horizon ist pausiert.');
|
|
}
|
|
|
|
$longestWait = (int) collect(app(WaitTimeCalculator::class)->calculate())->max();
|
|
$criticalSeconds = (int) config('monitoring.health.queue_backlog_critical_seconds');
|
|
|
|
return $this->check(
|
|
'queue',
|
|
'Queue',
|
|
$longestWait >= $criticalSeconds ? self::StatusCritical : self::StatusOk,
|
|
'Treiber redis — Horizon läuft, geschätzte Wartezeit höchstens '.$longestWait.' s.',
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @return array{key: string, label: string, status: string, detail: string}
|
|
*/
|
|
private function failedJobs(): array
|
|
{
|
|
$hours = (int) config('monitoring.health.failed_jobs_window_hours');
|
|
|
|
try {
|
|
$count = DB::table('failed_jobs')->where('failed_at', '>=', now()->subHours($hours))->count();
|
|
} catch (Throwable $exception) {
|
|
return $this->check('failed_jobs', 'Fehlgeschlagene Jobs', self::StatusWarn, 'Tabelle failed_jobs nicht lesbar: '.Str::limit($exception->getMessage(), 120));
|
|
}
|
|
|
|
return $this->check(
|
|
'failed_jobs',
|
|
'Fehlgeschlagene Jobs',
|
|
$count > 0 ? self::StatusWarn : self::StatusOk,
|
|
$count > 0 ? $count.' in den letzten '.$hours.' Stunden.' : 'Keine in den letzten '.$hours.' Stunden.',
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @return array{key: string, label: string, status: string, detail: string}
|
|
*/
|
|
private function check(string $key, string $label, string $status, string $detail): array
|
|
{
|
|
return ['key' => $key, 'label' => $label, 'status' => $status, 'detail' => $detail];
|
|
}
|
|
}
|