From fec88ee2655c042622ea14f6e1bc82fe98a5c118 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 7 Oct 2026 11:49:14 +0000 Subject: [PATCH] Systemstatus im Sysadmin-Bereich und /health fuer externe Ueberwachung - HealthProbe: Datenbank, Cache, Queue (sync / database / Horizon), fehlgeschlagene Jobs (24 h), Scheduler-Herzschlag, Plattenplatz - /health auf jeder Domain, ohne web-Gruppe, nur mit MONITORING_HEALTH_TOKEN - monitoring:heartbeat minuetlich im Scheduler - Sysadmin -> Systemstatus (my.mivita.care/sysadmin/status) mit Version, Umgebung und zuletzt fehlgeschlagenen Jobs - Anleitung Abschnitt G Co-Authored-By: Claude Opus 5.5 --- app/Console/Commands/MonitoringHeartbeat.php | 25 ++ app/Console/Kernel.php | 3 + app/Http/Controllers/HealthController.php | 39 +++ .../SyS/SystemStatusController.php | 59 ++++ app/Providers/RouteServiceProvider.php | 13 +- app/Services/Monitoring/HealthProbe.php | 265 ++++++++++++++++++ config/monitoring.php | 30 ++ dev/betrieb/README.md | 23 ++ resources/lang/de/navigation.php | 1 + resources/lang/en/navigation.php | 1 + resources/lang/es/navigation.php | 1 + resources/lang/fr/navigation.php | 1 + .../layouts/includes/layout-sidenav.blade.php | 6 + resources/views/sys/status/index.blade.php | 109 +++++++ routes/domains/crm.php | 1 + .../Feature/Monitoring/HealthEndpointTest.php | 131 +++++++++ .../Monitoring/SystemStatusPageTest.php | 78 ++++++ 17 files changed, 783 insertions(+), 3 deletions(-) create mode 100644 app/Console/Commands/MonitoringHeartbeat.php create mode 100644 app/Http/Controllers/HealthController.php create mode 100644 app/Http/Controllers/SyS/SystemStatusController.php create mode 100644 app/Services/Monitoring/HealthProbe.php create mode 100644 config/monitoring.php create mode 100644 resources/views/sys/status/index.blade.php create mode 100644 tests/Feature/Monitoring/HealthEndpointTest.php create mode 100644 tests/Feature/Monitoring/SystemStatusPageTest.php diff --git a/app/Console/Commands/MonitoringHeartbeat.php b/app/Console/Commands/MonitoringHeartbeat.php new file mode 100644 index 00000000..c75e5fd8 --- /dev/null +++ b/app/Console/Commands/MonitoringHeartbeat.php @@ -0,0 +1,25 @@ +recordHeartbeat(); + + return self::SUCCESS; + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 91760d73..44d2f3d0 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -36,6 +36,9 @@ class Kernel extends ConsoleKernel */ protected function schedule(Schedule $schedule) { + // Herzschlag fuer /health und den Systemstatus: bleibt er aus, laeuft der Cron nicht + $schedule->command('monitoring:heartbeat')->everyMinute()->withoutOverlapping(2); + // Uptime-Check: PAYONE-Erreichbarkeit alle 5 Minuten prüfen $schedule->command('payment:check-uptime') ->everyFiveMinutes() diff --git a/app/Http/Controllers/HealthController.php b/app/Http/Controllers/HealthController.php new file mode 100644 index 00000000..fc2fb952 --- /dev/null +++ b/app/Http/Controllers/HealthController.php @@ -0,0 +1,39 @@ +bearerToken() ?? $request->query('token', '')); + + abort_if($token === '' || ! hash_equals($token, $given), 404); + + $checks = $probe->checks(); + $status = HealthProbe::worstStatus($checks); + + return response()->json([ + 'status' => $status, + 'version' => config('app.version'), + 'checked_at' => now()->toIso8601String(), + 'checks' => collect($checks)->mapWithKeys(fn (array $check): array => [ + $check['key'] => ['status' => $check['status'], 'detail' => $check['detail']], + ]), + ], $status === HealthProbe::StatusCritical ? 503 : 200) + ->header('Cache-Control', 'no-store'); + } +} diff --git a/app/Http/Controllers/SyS/SystemStatusController.php b/app/Http/Controllers/SyS/SystemStatusController.php new file mode 100644 index 00000000..de9164e4 --- /dev/null +++ b/app/Http/Controllers/SyS/SystemStatusController.php @@ -0,0 +1,59 @@ +checks(); + + return view('sys.status.index', [ + 'checks' => $checks, + 'overall' => HealthProbe::worstStatus($checks), + 'environment' => [ + 'Version' => config('app.version') ?? 'unbekannt', + 'Umgebung' => app()->environment(), + 'PHP' => PHP_VERSION, + 'Laravel' => app()->version(), + 'Queue-Treiber' => config('queue.default'), + 'Cache-Treiber' => config('cache.default'), + ], + 'healthTokenConfigured' => filled(config('monitoring.health.token')), + 'horizonAvailable' => config('queue.default') === 'redis', + 'failedJobs' => $this->recentFailedJobs(), + ]); + } + + /** + * @return Collection + */ + private function recentFailedJobs(): Collection + { + try { + $rows = DB::table('failed_jobs')->latest('failed_at')->limit(10)->get(); + } catch (Throwable) { + return collect(); + } + + return $rows->map(fn (object $row): array => [ + 'uuid' => $row->uuid ?? null, + 'job' => json_decode($row->payload, true)['displayName'] ?? 'unbekannt', + 'queue' => $row->connection.' / '.$row->queue, + 'failed_at' => (string) $row->failed_at, + 'error' => Str::limit(Str::before((string) $row->exception, "\n"), 200), + ]); + } +} diff --git a/app/Providers/RouteServiceProvider.php b/app/Providers/RouteServiceProvider.php index 3b11af66..cdb7445f 100644 --- a/app/Providers/RouteServiceProvider.php +++ b/app/Providers/RouteServiceProvider.php @@ -4,6 +4,7 @@ namespace App\Providers; // use App\Domain\DomainContext; // HOTFIX: Temporär deaktiviert use App\Domain\EarlyDomainParser; +use App\Http\Controllers\HealthController; use Illuminate\Cache\RateLimiting\Limit; use Illuminate\Foundation\Support\Providers\RouteServiceProvider as ServiceProvider; use Illuminate\Http\Request; @@ -36,8 +37,14 @@ class RouteServiceProvider extends ServiceProvider $this->configureRateLimiting(); $this->routes(function () { + // Tiefer Health-Check für den externen Monitor: auf jeder Domain, + // ohne web-Gruppe (keine Session, kein CSRF, kein SubdomainResolver) + Route::get('/health', HealthController::class) + ->middleware('throttle:30,1') + ->name('health.deep'); + // API-Routen werden global geladen - Route::domain('api.' . config('app.domain') . config('app.tld_care')) + Route::domain('api.'.config('app.domain').config('app.tld_care')) ->middleware('api') ->namespace($this->namespace) ->group(base_path('routes/api.php')); @@ -74,7 +81,7 @@ class RouteServiceProvider extends ServiceProvider default => ['main'], }; foreach ($routesToLoad as $routeType) { - $this->loadDomainRoutes($routeType, $routeType . '.php'); + $this->loadDomainRoutes($routeType, $routeType.'.php'); } if (config('app.debug')) { @@ -91,7 +98,7 @@ class RouteServiceProvider extends ServiceProvider */ protected function loadDomainRoutes(string $domainType, string $fileName): void { - $routePath = base_path('routes/domains/' . $fileName); + $routePath = base_path('routes/domains/'.$fileName); if (file_exists($routePath)) { Route::group([], $routePath); diff --git a/app/Services/Monitoring/HealthProbe.php b/app/Services/Monitoring/HealthProbe.php new file mode 100644 index 00000000..93429c32 --- /dev/null +++ b/app/Services/Monitoring/HealthProbe.php @@ -0,0 +1,265 @@ +toIso8601String(), LOCK_EX); + } + + public function lastHeartbeat(): ?Carbon + { + $value = @file_get_contents(self::heartbeatPath()); + + return is_string($value) && trim($value) !== '' ? Carbon::parse(trim($value)) : null; + } + + /** + * @return list + */ + public function checks(): array + { + return [ + $this->database(), + $this->cache(), + $this->queue(), + $this->failedJobs(), + $this->scheduler(), + $this->disk(), + ]; + } + + /** + * @param list $checks + */ + public static function worstStatus(array $checks): string + { + $statuses = array_column($checks, 'status'); + + return match (true) { + in_array(self::StatusCritical, $statuses, true) => self::StatusCritical, + in_array(self::StatusWarn, $statuses, true) => self::StatusWarn, + default => self::StatusOk, + }; + } + + /** + * @return array{key: string, label: string, status: string, detail: string} + */ + public function scheduler(): array + { + $lastBeat = $this->lastHeartbeat(); + $warnMinutes = (int) config('monitoring.scheduler.heartbeat_warn_minutes'); + $criticalMinutes = (int) config('monitoring.scheduler.heartbeat_critical_minutes'); + $minutesAgo = $lastBeat !== null ? (int) $lastBeat->diffInMinutes(now(), absolute: true) : null; + + return $this->check( + 'scheduler', + 'Scheduler', + match (true) { + $minutesAgo === null, $minutesAgo >= $criticalMinutes => self::StatusCritical, + $minutesAgo >= $warnMinutes => self::StatusWarn, + default => self::StatusOk, + }, + $lastBeat === null + ? 'Noch kein Herzschlag — läuft `schedule:run` minütlich per Cron?' + : 'Letzter Herzschlag '.$lastBeat->format('d.m.Y H:i:s').' (vor '.$minutesAgo.' Min.).', + ); + } + + /** + * @return array{key: string, label: string, status: string, detail: string} + */ + public function disk(): array + { + $path = storage_path(); + $free = @disk_free_space($path); + $total = @disk_total_space($path); + + if ($free === false || $total === false || $total <= 0) { + return $this->check('disk', 'Plattenplatz', self::StatusWarn, 'Plattenplatz nicht ermittelbar.'); + } + + $freePercent = round($free / $total * 100, 1); + $minPercent = (float) config('monitoring.health.disk_min_free_percent'); + + return $this->check( + 'disk', + 'Plattenplatz', + match (true) { + $freePercent < $minPercent => self::StatusCritical, + $freePercent < $minPercent * 2 => self::StatusWarn, + default => self::StatusOk, + }, + sprintf('%s %% frei (%s GB von %s GB).', $freePercent, round($free / 1024 ** 3, 1), round($total / 1024 ** 3, 1)), + ); + } + + /** + * @return array{key: string, label: string, status: string, detail: string} + */ + private function database(): array + { + try { + DB::select('select 1'); + + return $this->check('database', 'Datenbank', self::StatusOk, 'Verbindung steht.'); + } catch (Throwable $exception) { + return $this->check('database', 'Datenbank', self::StatusCritical, 'Keine Verbindung: '.Str::limit($exception->getMessage(), 120)); + } + } + + /** + * @return array{key: string, label: string, status: string, detail: string} + */ + private function cache(): array + { + try { + $value = Str::random(8); + Cache::put('monitoring:health-probe', $value, 60); + + return Cache::get('monitoring:health-probe') === $value + ? $this->check('cache', 'Cache', self::StatusOk, 'Schreiben und Lesen funktionieren ('.config('cache.default').').') + : $this->check('cache', 'Cache', self::StatusCritical, 'Geschriebener Wert kommt nicht zurück.'); + } catch (Throwable $exception) { + return $this->check('cache', 'Cache', self::StatusCritical, 'Nicht erreichbar: '.Str::limit($exception->getMessage(), 120)); + } + } + + /** + * Je nach QUEUE_DRIVER: `sync` braucht keinen Worker, `database` wird am + * ältesten wartenden Job gemessen, `redis` an Horizon (läuft es, wie lange + * warten die Jobs). + * + * @return array{key: string, label: string, status: string, detail: string} + */ + private function queue(): array + { + return match (config('queue.default')) { + 'sync' => $this->check('queue', 'Queue', self::StatusOk, 'Treiber sync — Jobs laufen direkt im Request, kein Worker nötig.'), + 'database' => $this->databaseQueue(), + 'redis' => $this->horizonQueue(), + default => $this->check('queue', 'Queue', self::StatusWarn, 'Treiber '.config('queue.default').' — hier nicht prüfbar.'), + }; + } + + /** + * @return array{key: string, label: string, status: string, detail: string} + */ + private function databaseQueue(): array + { + try { + $now = now()->getTimestamp(); + $oldestAvailable = DB::table('jobs') + ->whereNull('reserved_at') + ->where('available_at', '<=', $now) + ->min('available_at'); + } catch (Throwable $exception) { + return $this->check('queue', 'Queue', self::StatusCritical, 'Jobs-Tabelle nicht lesbar: '.Str::limit($exception->getMessage(), 120)); + } + + $waitingSeconds = $oldestAvailable !== null ? max(0, $now - (int) $oldestAvailable) : 0; + $criticalSeconds = (int) config('monitoring.health.queue_backlog_critical_seconds'); + + return $this->check( + 'queue', + 'Queue', + $waitingSeconds >= $criticalSeconds ? self::StatusCritical : self::StatusOk, + $oldestAvailable === null + ? 'Treiber database — kein wartender Job.' + : 'Treiber database — ältester wartender Job seit '.$waitingSeconds.' s.'.($waitingSeconds >= $criticalSeconds ? ' Worker gestoppt?' : ''), + ); + } + + /** + * @return array{key: string, label: string, status: string, detail: string} + */ + private function horizonQueue(): array + { + try { + Redis::connection()->ping(); + $masters = app(MasterSupervisorRepository::class)->all(); + } catch (Throwable $exception) { + return $this->check('queue', 'Queue', self::StatusCritical, 'Redis nicht erreichbar: '.Str::limit($exception->getMessage(), 120)); + } + + if ($masters === []) { + return $this->check('queue', 'Queue', self::StatusCritical, 'Treiber redis — Horizon läuft nicht. Daemon in Ploi prüfen.'); + } + + if (collect($masters)->every(fn (object $master): bool => $master->status === 'paused')) { + return $this->check('queue', 'Queue', self::StatusWarn, 'Treiber redis — Horizon ist pausiert.'); + } + + $longestWait = (int) collect(app(WaitTimeCalculator::class)->calculate())->max(); + $criticalSeconds = (int) config('monitoring.health.queue_backlog_critical_seconds'); + + return $this->check( + 'queue', + 'Queue', + $longestWait >= $criticalSeconds ? self::StatusCritical : self::StatusOk, + 'Treiber redis — Horizon läuft, geschätzte Wartezeit höchstens '.$longestWait.' s.', + ); + } + + /** + * @return array{key: string, label: string, status: string, detail: string} + */ + private function failedJobs(): array + { + $hours = (int) config('monitoring.health.failed_jobs_window_hours'); + + try { + $count = DB::table('failed_jobs')->where('failed_at', '>=', now()->subHours($hours))->count(); + } catch (Throwable $exception) { + return $this->check('failed_jobs', 'Fehlgeschlagene Jobs', self::StatusWarn, 'Tabelle failed_jobs nicht lesbar: '.Str::limit($exception->getMessage(), 120)); + } + + return $this->check( + 'failed_jobs', + 'Fehlgeschlagene Jobs', + $count > 0 ? self::StatusWarn : self::StatusOk, + $count > 0 ? $count.' in den letzten '.$hours.' Stunden.' : 'Keine in den letzten '.$hours.' Stunden.', + ); + } + + /** + * @return array{key: string, label: string, status: string, detail: string} + */ + private function check(string $key, string $label, string $status, string $detail): array + { + return ['key' => $key, 'label' => $label, 'status' => $status, 'detail' => $detail]; + } +} diff --git a/config/monitoring.php b/config/monitoring.php new file mode 100644 index 00000000..95fdfc26 --- /dev/null +++ b/config/monitoring.php @@ -0,0 +1,30 @@ + [ + 'token' => env('MONITORING_HEALTH_TOKEN'), + 'queue_backlog_critical_seconds' => (int) env('MONITORING_QUEUE_BACKLOG_SECONDS', 300), + 'disk_min_free_percent' => (float) env('MONITORING_DISK_MIN_FREE_PERCENT', 10), + 'failed_jobs_window_hours' => 24, + ], + + 'scheduler' => [ + 'heartbeat_warn_minutes' => 3, + 'heartbeat_critical_minutes' => 10, + ], + +]; diff --git a/dev/betrieb/README.md b/dev/betrieb/README.md index f9ad58d1..02d4470a 100644 --- a/dev/betrieb/README.md +++ b/dev/betrieb/README.md @@ -221,6 +221,29 @@ grep -E '^(APP_ENV|QUEUE_DRIVER|REDIS_HOST)=' .env # APP_ENV=production Zurück: `QUEUE_DRIVER=sync` in `.env`, in `deploy.sh` `QUEUE_WORKER="none"`. Liegen gebliebene Jobs vorher mit `php artisan queue:work redis --stop-when-empty` abarbeiten. +## G — Überwachung: Systemstatus und `/health` + +**Systemstatus** im CRM unter Sysadmin → „Systemstatus“ (`https://my.mivita.care/sysadmin/status`, nur admin ≥ 4): +Datenbank, Cache, Queue (sync / database / Horizon), fehlgeschlagene Jobs der letzten 24 h, +Scheduler-Herzschlag, Plattenplatz, Version, zuletzt fehlgeschlagene Jobs. + +**`/health`** liefert dieselben Prüfungen als JSON für einen externen Monitor, auf jeder Domain +(z. B. `https://mivita.care/health`). 200 = in Ordnung oder Warnung, 503 = kritisch. Ohne Token 404. + +Einrichtung auf Live (einmalig): + +```bash +cd /home/ploi/mivita.care +openssl rand -hex 24 # Wert als MONITORING_HEALTH_TOKEN in die .env +curl -s -H "Authorization: Bearer " https://mivita.care/health +``` + +Der **Scheduler-Herzschlag** (`monitoring:heartbeat`, minütlich) zeigt, ob der Cron `schedule:run` +wirklich läuft. Steht er nach dem Deploy auf „Kritisch“, ist in Ploi kein Scheduler-Eintrag +(`php artisan schedule:run` jede Minute) aktiv. + +Schwellen in `.env` (optional): `MONITORING_QUEUE_BACKLOG_SECONDS=300`, `MONITORING_DISK_MIN_FREE_PERCENT=10`. + ## Fallstricke - **FPM-Restart statt Reload** → 502 für Sekunden. diff --git a/resources/lang/de/navigation.php b/resources/lang/de/navigation.php index 0f438d44..ef50d4e6 100644 --- a/resources/lang/de/navigation.php +++ b/resources/lang/de/navigation.php @@ -85,6 +85,7 @@ return [ 'customer_orders' => 'Kundenbestellungen', 'external_orders' => 'Externe Bestellungen', 'tools' => 'Tools', + 'system_status' => 'Systemstatus', 'news_archive' => 'News Archiv', 'incentive' => 'Incentive', 'incentives' => 'Incentives', diff --git a/resources/lang/en/navigation.php b/resources/lang/en/navigation.php index f10bc7e2..9c2221d0 100644 --- a/resources/lang/en/navigation.php +++ b/resources/lang/en/navigation.php @@ -85,6 +85,7 @@ return [ 'customer_orders' => 'Customer Orders', 'external_orders' => 'External Orders', 'tools' => 'Tools', + 'system_status' => 'System status', 'news_archive' => 'News Archive', 'incentive' => 'Incentive', 'incentives' => 'Incentives', diff --git a/resources/lang/es/navigation.php b/resources/lang/es/navigation.php index 24f2b755..ccc6f19d 100644 --- a/resources/lang/es/navigation.php +++ b/resources/lang/es/navigation.php @@ -82,6 +82,7 @@ return [ 'customer_orders' => 'Pedidos de clientes', 'external_orders' => 'Pedidos externos', 'tools' => 'Herramientas', + 'system_status' => 'Estado del sistema', 'news_archive' => 'Archivo de Noticias', 'incentive' => 'Incentivo', 'incentives' => 'Incentivos', diff --git a/resources/lang/fr/navigation.php b/resources/lang/fr/navigation.php index 8d440e53..72eafb7c 100644 --- a/resources/lang/fr/navigation.php +++ b/resources/lang/fr/navigation.php @@ -82,6 +82,7 @@ return [ 'customer_orders' => 'Commandes clients', 'external_orders' => 'Commandes externes', 'tools' => 'Outils', + 'system_status' => 'État du système', 'news_archive' => 'Archive des actualités', 'incentive' => 'Incentive', 'incentives' => 'Incentives', diff --git a/resources/views/layouts/includes/layout-sidenav.blade.php b/resources/views/layouts/includes/layout-sidenav.blade.php index aacb786c..5c221656 100755 --- a/resources/views/layouts/includes/layout-sidenav.blade.php +++ b/resources/views/layouts/includes/layout-sidenav.blade.php @@ -713,6 +713,12 @@
{{ __('navigation.tools') }}
+
  • + +
    {{ __('navigation.system_status') }}
    +     +
  • diff --git a/resources/views/sys/status/index.blade.php b/resources/views/sys/status/index.blade.php new file mode 100644 index 00000000..de0e81a6 --- /dev/null +++ b/resources/views/sys/status/index.blade.php @@ -0,0 +1,109 @@ +@extends('layouts.layout-2') + +@php + $badges = ['ok' => 'badge-success', 'warn' => 'badge-warning', 'critical' => 'badge-danger']; + $labels = ['ok' => 'OK', 'warn' => 'Warnung', 'critical' => 'Kritisch']; +@endphp + +@section('content') +

    + Systemstatus + {{ $labels[$overall] }} + Stand {{ now()->format('d.m.Y H:i:s') }} +

    + +
    +
    Prüfungen
    +
    + + + + + + + + + + @foreach ($checks as $check) + + + + + + @endforeach + +
    PrüfungStatusDetail
    {{ $check['label'] }}{{ $labels[$check['status']] }}{{ $check['detail'] }}
    +
    +
    + +
    +
    +
    +
    Umgebung
    + + + @foreach ($environment as $label => $value) + + + + + @endforeach + +
    {{ $label }}{{ $value }}
    +
    +
    +
    +
    +
    Externe Überwachung
    +
    +

    + Endpunkt für den externen Monitor: + {{ url('/health') }} +

    + @if ($healthTokenConfigured) +

    Token gesetzt + Abruf mit Header Authorization: Bearer <MONITORING_HEALTH_TOKEN>. + Antwort 200 = in Ordnung, 503 = kritisch.

    + @else +

    Kein Token + MONITORING_HEALTH_TOKEN ist in der .env nicht gesetzt — der Endpunkt + antwortet mit 404.

    + @endif + @if ($horizonAvailable) +     Horizon öffnen + @endif +
    +
    +
    +
    + +
    +
    Zuletzt fehlgeschlagene Jobs
    + @if ($failedJobs->isEmpty()) +
    Keine fehlgeschlagenen Jobs gespeichert.
    + @else +
    + + + + + + + + + + + @foreach ($failedJobs as $job) + + + + + + + @endforeach + +
    ZeitpunktJobQueueFehler
    {{ $job['failed_at'] }}{{ $job['job'] }}{{ $job['queue'] }}{{ $job['error'] }}
    +
    + @endif +
    +@endsection diff --git a/routes/domains/crm.php b/routes/domains/crm.php index 11ef735d..39074b85 100644 --- a/routes/domains/crm.php +++ b/routes/domains/crm.php @@ -570,6 +570,7 @@ Route::domain(config('app.pre_url_crm').config('app.domain').config('app.tld_car Route::post('/sysadmin/tool/store/{tool}', 'SyS\SysController@store')->name('sysadmin_tool_store'); Route::get('/sysadmin/settings', 'SyS\SettingController@index')->name('sysadmin_settings'); + Route::get('/sysadmin/status', 'SyS\SystemStatusController@index')->name('sysadmin_status'); Route::post('/sysadmin/setting/store', 'SyS\SettingController@store')->name('sysadmin_setting_store'); }); }); diff --git a/tests/Feature/Monitoring/HealthEndpointTest.php b/tests/Feature/Monitoring/HealthEndpointTest.php new file mode 100644 index 00000000..9934385d --- /dev/null +++ b/tests/Feature/Monitoring/HealthEndpointTest.php @@ -0,0 +1,131 @@ +set('cache.default', 'array'); + config()->set('monitoring.health.token', 'geheimes-monitor-token'); + config()->set('monitoring.health.disk_min_free_percent', 0); + config()->set('queue.default', 'sync'); +}); + +afterEach(function () { + @unlink(HealthProbe::heartbeatPath()); +}); + +function healthHeaders(): array +{ + return ['Authorization' => 'Bearer geheimes-monitor-token']; +} + +test('the endpoint does not exist without a configured token or with a wrong one', function () { + app(HealthProbe::class)->recordHeartbeat(); + + $this->get('/health', healthHeaders())->assertOk(); + + $this->get('/health', ['Authorization' => 'Bearer falsch'])->assertNotFound(); + $this->get('/health')->assertNotFound(); + + config()->set('monitoring.health.token', null); + $this->get('/health', healthHeaders())->assertNotFound(); +}); + +test('a healthy system answers 200 with every check', function () { + config()->set('app.version', 'v1.1.0'); + $this->artisan('monitoring:heartbeat')->assertSuccessful(); + + $this->get('/health?token=geheimes-monitor-token') + ->assertOk() + ->assertHeader('Cache-Control', 'no-store, private') + ->assertJsonPath('status', 'ok') + ->assertJsonPath('version', 'v1.1.0') + ->assertJsonPath('checks.database.status', 'ok') + ->assertJsonPath('checks.cache.status', 'ok') + ->assertJsonPath('checks.queue.status', 'ok') + ->assertJsonPath('checks.failed_jobs.status', 'ok') + ->assertJsonPath('checks.scheduler.status', 'ok') + ->assertJsonStructure(['checks' => ['disk' => ['status', 'detail']]]); +}); + +test('a missing or stale heartbeat makes the endpoint fail', function () { + $this->get('/health', healthHeaders()) + ->assertStatus(503) + ->assertJsonPath('checks.scheduler.status', 'critical'); + + app(HealthProbe::class)->recordHeartbeat(); + + $this->travel(4)->minutes(); + $this->get('/health', healthHeaders())->assertOk()->assertJsonPath('checks.scheduler.status', 'warn'); + + $this->travel(7)->minutes(); + $this->get('/health', healthHeaders())->assertStatus(503)->assertJsonPath('checks.scheduler.status', 'critical'); +}); + +test('a database queue backlog older than five minutes makes the endpoint fail', function () { + config()->set('queue.default', 'database'); + app(HealthProbe::class)->recordHeartbeat(); + + DB::table('jobs')->insert([ + 'queue' => 'default', + 'payload' => '{}', + 'attempts' => 0, + 'reserved_at' => null, + 'available_at' => now()->subMinutes(6)->getTimestamp(), + 'created_at' => now()->subMinutes(6)->getTimestamp(), + ]); + + $this->get('/health', healthHeaders()) + ->assertStatus(503) + ->assertJsonPath('checks.queue.status', 'critical'); +}); + +test('with the redis driver a stopped horizon makes the endpoint fail', function () { + config()->set('queue.default', 'redis'); + app(HealthProbe::class)->recordHeartbeat(); + + Redis::shouldReceive('connection->ping')->andReturn(true); + $this->mock(MasterSupervisorRepository::class)->shouldReceive('all')->andReturn([]); + + $this->get('/health', healthHeaders()) + ->assertStatus(503) + ->assertJsonPath('checks.queue.status', 'critical') + ->assertJsonPath('checks.queue.detail', 'Treiber redis — Horizon läuft nicht. Daemon in Ploi prüfen.'); +}); + +test('recently failed jobs are a warning, not an outage', function () { + app(HealthProbe::class)->recordHeartbeat(); + + DB::table('failed_jobs')->insert([ + 'uuid' => 'a1b2c3', + 'connection' => 'redis', + 'queue' => 'default', + 'payload' => json_encode(['displayName' => 'App\\Jobs\\SyncStripeInvoiceReference']), + 'exception' => "RuntimeException: Stripe nicht erreichbar\n#0 ...", + 'failed_at' => now()->subHour(), + ]); + + $this->get('/health', healthHeaders()) + ->assertOk() + ->assertJsonPath('status', 'warn') + ->assertJsonPath('checks.failed_jobs.status', 'warn'); +}); + +test('the endpoint skips the session and the web middleware group', function () { + app(HealthProbe::class)->recordHeartbeat(); + + $this->get('/health', healthHeaders()) + ->assertOk() + ->assertCookieMissing(config('session.cookie')); +}); diff --git a/tests/Feature/Monitoring/SystemStatusPageTest.php b/tests/Feature/Monitoring/SystemStatusPageTest.php new file mode 100644 index 00000000..1db0ed1c --- /dev/null +++ b/tests/Feature/Monitoring/SystemStatusPageTest.php @@ -0,0 +1,78 @@ +previousHttpHost = $_SERVER['HTTP_HOST'] ?? null; + $_SERVER['HTTP_HOST'] = config('domains.domains.crm.host'); + $this->refreshApplication(); + $this->artisan('migrate'); + config()->set('cache.default', 'array'); + config()->set('queue.default', 'sync'); + @unlink(HealthProbe::heartbeatPath()); +}); + +afterEach(function () { + @unlink(HealthProbe::heartbeatPath()); + + if ($this->previousHttpHost === null) { + unset($_SERVER['HTTP_HOST']); + } else { + $_SERVER['HTTP_HOST'] = $this->previousHttpHost; + } +}); + +function statusPageUser(int $admin): User +{ + $account = UserAccount::create([ + 'first_name' => 'Sys', + 'last_name' => 'Admin', + 'm_first_name' => 'Sys', + 'm_last_name' => 'Admin', + 'm_account' => 'M'.random_int(1000, 9999), + ]); + + return User::forceCreate([ + 'email' => 'sys-'.uniqid('', true).'@example.com', + 'password' => bcrypt('secret'), + 'lang' => 'de', + 'active' => 1, + 'blocked' => 0, + 'wizard' => 100, + 'admin' => $admin, + 'payment_account' => now()->addYear(), + 'account_id' => $account->id, + ])->fresh(); +} + +test('system admins see the checks, the version and the menu entry', function () { + config()->set('app.version', 'v1.0.2'); + app(HealthProbe::class)->recordHeartbeat(); + + $this->actingAs(statusPageUser(4)) + ->get(route('sysadmin_status')) + ->assertOk() + ->assertSee('Systemstatus') + ->assertSee('Datenbank') + ->assertSee('Scheduler') + ->assertSee('Fehlgeschlagene Jobs') + ->assertSee('v1.0.2') + ->assertSee(route('sysadmin_status'), false) + ->assertSee('Kein Token'); +}); + +test('other admins are not allowed to open the page', function () { + $this->actingAs(statusPageUser(3)) + ->get(route('sysadmin_status')) + ->assertForbidden(); +});