Systemstatus im Sysadmin-Bereich und /health fuer externe Ueberwachung

- HealthProbe: Datenbank, Cache, Queue (sync / database / Horizon),
  fehlgeschlagene Jobs (24 h), Scheduler-Herzschlag, Plattenplatz
- /health auf jeder Domain, ohne web-Gruppe, nur mit MONITORING_HEALTH_TOKEN
- monitoring:heartbeat minuetlich im Scheduler
- Sysadmin -> Systemstatus (my.mivita.care/sysadmin/status) mit Version,
  Umgebung und zuletzt fehlgeschlagenen Jobs
- Anleitung Abschnitt G

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/Console/Commands/MonitoringHeartbeat.php

@@ -0,0 +1,25 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Services\Monitoring\HealthProbe;
+use Illuminate\Console\Command;
+
+/**
+ * Minütlicher Herzschlag des Schedulers. Bleibt er aus,
+ * melden `/health` und der Systemstatus den Scheduler als ausgefallen —
+ * `onFailure` der einzelnen Tasks greift in diesem Fall nicht mehr.
+ */
+class MonitoringHeartbeat extends Command
+{
+    protected $signature = 'monitoring:heartbeat';
+
+    protected $description = 'Schreibt den Herzschlag des Schedulers (Grundlage für /health und den Systemstatus).';
+
+    public function handle(HealthProbe $probe): int
+    {
+        $probe->recordHeartbeat();
+
+        return self::SUCCESS;
+    }
+}

app/Console/Kernel.php

@@ -36,6 +36,9 @@ class Kernel extends ConsoleKernel
      */
     protected function schedule(Schedule $schedule)
     {
+        // Herzschlag fuer /health und den Systemstatus: bleibt er aus, laeuft der Cron nicht
+        $schedule->command('monitoring:heartbeat')->everyMinute()->withoutOverlapping(2);
+
         // Uptime-Check: PAYONE-Erreichbarkeit alle 5 Minuten prüfen
         $schedule->command('payment:check-uptime')
             ->everyFiveMinutes()

app/Http/Controllers/HealthController.php

@@ -0,0 +1,39 @@
+<?php
+
+namespace App\Http\Controllers;
+
+use App\Services\Monitoring\HealthProbe;
+use Illuminate\Http\JsonResponse;
+use Illuminate\Http\Request;
+
+/**
+ * Tiefer Health-Check für den externen Monitor.
+ *
+ * `/up` zeigt nur, dass die App bootet. `/health` prüft zusätzlich DB,
+ * Cache, Queue, fehlgeschlagene Jobs, Scheduler-Herzschlag und Plattenplatz: 200 solange
+ * nichts kritisch ist, sonst 503. Zugang nur mit MONITORING_HEALTH_TOKEN
+ * (Bearer oder `?token=`); ohne konfiguriertes Token gibt es den Endpunkt nicht.
+ */
+class HealthController extends Controller
+{
+    public function __invoke(Request $request, HealthProbe $probe): JsonResponse
+    {
+        $token = (string) config('monitoring.health.token');
+        $given = (string) ($request->bearerToken() ?? $request->query('token', ''));
+
+        abort_if($token === '' || ! hash_equals($token, $given), 404);
+
+        $checks = $probe->checks();
+        $status = HealthProbe::worstStatus($checks);
+
+        return response()->json([
+            'status' => $status,
+            'version' => config('app.version'),
+            'checked_at' => now()->toIso8601String(),
+            'checks' => collect($checks)->mapWithKeys(fn (array $check): array => [
+                $check['key'] => ['status' => $check['status'], 'detail' => $check['detail']],
+            ]),
+        ], $status === HealthProbe::StatusCritical ? 503 : 200)
+            ->header('Cache-Control', 'no-store');
+    }
+}

app/Http/Controllers/SyS/SystemStatusController.php

@@ -0,0 +1,59 @@
+<?php
+
+namespace App\Http\Controllers\SyS;
+
+use App\Http\Controllers\Controller;
+use App\Services\Monitoring\HealthProbe;
+use Illuminate\Contracts\View\View;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Str;
+use Throwable;
+
+/**
+ * Systemstatus im Sysadmin-Bereich: dieselben Prüfungen wie `/health`,
+ * dazu Umgebung und die zuletzt fehlgeschlagenen Jobs.
+ */
+class SystemStatusController extends Controller
+{
+    public function index(HealthProbe $probe): View
+    {
+        $checks = $probe->checks();
+
+        return view('sys.status.index', [
+            'checks' => $checks,
+            'overall' => HealthProbe::worstStatus($checks),
+            'environment' => [
+                'Version' => config('app.version') ?? 'unbekannt',
+                'Umgebung' => app()->environment(),
+                'PHP' => PHP_VERSION,
+                'Laravel' => app()->version(),
+                'Queue-Treiber' => config('queue.default'),
+                'Cache-Treiber' => config('cache.default'),
+            ],
+            'healthTokenConfigured' => filled(config('monitoring.health.token')),
+            'horizonAvailable' => config('queue.default') === 'redis',
+            'failedJobs' => $this->recentFailedJobs(),
+        ]);
+    }
+
+    /**
+     * @return Collection<int, array{uuid: ?string, job: string, queue: string, failed_at: string, error: string}>
+     */
+    private function recentFailedJobs(): Collection
+    {
+        try {
+            $rows = DB::table('failed_jobs')->latest('failed_at')->limit(10)->get();
+        } catch (Throwable) {
+            return collect();
+        }
+
+        return $rows->map(fn (object $row): array => [
+            'uuid' => $row->uuid ?? null,
+            'job' => json_decode($row->payload, true)['displayName'] ?? 'unbekannt',
+            'queue' => $row->connection.' / '.$row->queue,
+            'failed_at' => (string) $row->failed_at,
+            'error' => Str::limit(Str::before((string) $row->exception, "\n"), 200),
+        ]);
+    }
+}

app/Providers/RouteServiceProvider.php

@@ -4,6 +4,7 @@ namespace App\Providers;
 
 // use App\Domain\DomainContext; // HOTFIX: Temporär deaktiviert
 use App\Domain\EarlyDomainParser;
+use App\Http\Controllers\HealthController;
 use Illuminate\Cache\RateLimiting\Limit;
 use Illuminate\Foundation\Support\Providers\RouteServiceProvider as ServiceProvider;
 use Illuminate\Http\Request;
@@ -36,8 +37,14 @@ class RouteServiceProvider extends ServiceProvider
         $this->configureRateLimiting();
 
         $this->routes(function () {
+            // Tiefer Health-Check für den externen Monitor: auf jeder Domain,
+            // ohne web-Gruppe (keine Session, kein CSRF, kein SubdomainResolver)
+            Route::get('/health', HealthController::class)
+                ->middleware('throttle:30,1')
+                ->name('health.deep');
+
             // API-Routen werden global geladen
-            Route::domain('api.' . config('app.domain') . config('app.tld_care'))
+            Route::domain('api.'.config('app.domain').config('app.tld_care'))
                 ->middleware('api')
                 ->namespace($this->namespace)
                 ->group(base_path('routes/api.php'));
@@ -74,7 +81,7 @@ class RouteServiceProvider extends ServiceProvider
             default => ['main'],
         };
         foreach ($routesToLoad as $routeType) {
-            $this->loadDomainRoutes($routeType, $routeType . '.php');
+            $this->loadDomainRoutes($routeType, $routeType.'.php');
         }
 
         if (config('app.debug')) {
@@ -91,7 +98,7 @@ class RouteServiceProvider extends ServiceProvider
      */
     protected function loadDomainRoutes(string $domainType, string $fileName): void
     {
-        $routePath = base_path('routes/domains/' . $fileName);
+        $routePath = base_path('routes/domains/'.$fileName);
 
         if (file_exists($routePath)) {
             Route::group([], $routePath);

app/Services/Monitoring/HealthProbe.php

@@ -0,0 +1,265 @@
+<?php
+
+namespace App\Services\Monitoring;
+
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Redis;
+use Illuminate\Support\Str;
+use Laravel\Horizon\Contracts\MasterSupervisorRepository;
+use Laravel\Horizon\WaitTimeCalculator;
+use Throwable;
+
+/**
+ * Die Prüfungen hinter `/health` (externer Monitor) und dem Systemstatus im
+ * Sysadmin-Bereich: Datenbank, Cache, Queue, fehlgeschlagene Jobs,
+ * Scheduler-Herzschlag, Plattenplatz. Bewusst billig — keine Zählungen über
+ * große Tabellen, damit `/health` selbst keine Last erzeugt.
+ */
+class HealthProbe
+{
+    public const StatusOk = 'ok';
+
+    public const StatusWarn = 'warn';
+
+    public const StatusCritical = 'critical';
+
+    /**
+     * Datei statt Datenbank/Cache: übersteht `cache:clear`, braucht keine
+     * Migration. Bei mehreren App-Servern stattdessen eine DB-Zeile nehmen.
+     */
+    public static function heartbeatPath(): string
+    {
+        return storage_path('framework/scheduler-heartbeat');
+    }
+
+    public function recordHeartbeat(): void
+    {
+        file_put_contents(self::heartbeatPath(), now()->toIso8601String(), LOCK_EX);
+    }
+
+    public function lastHeartbeat(): ?Carbon
+    {
+        $value = @file_get_contents(self::heartbeatPath());
+
+        return is_string($value) && trim($value) !== '' ? Carbon::parse(trim($value)) : null;
+    }
+
+    /**
+     * @return list<array{key: string, label: string, status: string, detail: string}>
+     */
+    public function checks(): array
+    {
+        return [
+            $this->database(),
+            $this->cache(),
+            $this->queue(),
+            $this->failedJobs(),
+            $this->scheduler(),
+            $this->disk(),
+        ];
+    }
+
+    /**
+     * @param  list<array{status: string}>  $checks
+     */
+    public static function worstStatus(array $checks): string
+    {
+        $statuses = array_column($checks, 'status');
+
+        return match (true) {
+            in_array(self::StatusCritical, $statuses, true) => self::StatusCritical,
+            in_array(self::StatusWarn, $statuses, true) => self::StatusWarn,
+            default => self::StatusOk,
+        };
+    }
+
+    /**
+     * @return array{key: string, label: string, status: string, detail: string}
+     */
+    public function scheduler(): array
+    {
+        $lastBeat = $this->lastHeartbeat();
+        $warnMinutes = (int) config('monitoring.scheduler.heartbeat_warn_minutes');
+        $criticalMinutes = (int) config('monitoring.scheduler.heartbeat_critical_minutes');
+        $minutesAgo = $lastBeat !== null ? (int) $lastBeat->diffInMinutes(now(), absolute: true) : null;
+
+        return $this->check(
+            'scheduler',
+            'Scheduler',
+            match (true) {
+                $minutesAgo === null, $minutesAgo >= $criticalMinutes => self::StatusCritical,
+                $minutesAgo >= $warnMinutes => self::StatusWarn,
+                default => self::StatusOk,
+            },
+            $lastBeat === null
+                ? 'Noch kein Herzschlag — läuft `schedule:run` minütlich per Cron?'
+                : 'Letzter Herzschlag '.$lastBeat->format('d.m.Y H:i:s').' (vor '.$minutesAgo.' Min.).',
+        );
+    }
+
+    /**
+     * @return array{key: string, label: string, status: string, detail: string}
+     */
+    public function disk(): array
+    {
+        $path = storage_path();
+        $free = @disk_free_space($path);
+        $total = @disk_total_space($path);
+
+        if ($free === false || $total === false || $total <= 0) {
+            return $this->check('disk', 'Plattenplatz', self::StatusWarn, 'Plattenplatz nicht ermittelbar.');
+        }
+
+        $freePercent = round($free / $total * 100, 1);
+        $minPercent = (float) config('monitoring.health.disk_min_free_percent');
+
+        return $this->check(
+            'disk',
+            'Plattenplatz',
+            match (true) {
+                $freePercent < $minPercent => self::StatusCritical,
+                $freePercent < $minPercent * 2 => self::StatusWarn,
+                default => self::StatusOk,
+            },
+            sprintf('%s %% frei (%s GB von %s GB).', $freePercent, round($free / 1024 ** 3, 1), round($total / 1024 ** 3, 1)),
+        );
+    }
+
+    /**
+     * @return array{key: string, label: string, status: string, detail: string}
+     */
+    private function database(): array
+    {
+        try {
+            DB::select('select 1');
+
+            return $this->check('database', 'Datenbank', self::StatusOk, 'Verbindung steht.');
+        } catch (Throwable $exception) {
+            return $this->check('database', 'Datenbank', self::StatusCritical, 'Keine Verbindung: '.Str::limit($exception->getMessage(), 120));
+        }
+    }
+
+    /**
+     * @return array{key: string, label: string, status: string, detail: string}
+     */
+    private function cache(): array
+    {
+        try {
+            $value = Str::random(8);
+            Cache::put('monitoring:health-probe', $value, 60);
+
+            return Cache::get('monitoring:health-probe') === $value
+                ? $this->check('cache', 'Cache', self::StatusOk, 'Schreiben und Lesen funktionieren ('.config('cache.default').').')
+                : $this->check('cache', 'Cache', self::StatusCritical, 'Geschriebener Wert kommt nicht zurück.');
+        } catch (Throwable $exception) {
+            return $this->check('cache', 'Cache', self::StatusCritical, 'Nicht erreichbar: '.Str::limit($exception->getMessage(), 120));
+        }
+    }
+
+    /**
+     * Je nach QUEUE_DRIVER: `sync` braucht keinen Worker, `database` wird am
+     * ältesten wartenden Job gemessen, `redis` an Horizon (läuft es, wie lange
+     * warten die Jobs).
+     *
+     * @return array{key: string, label: string, status: string, detail: string}
+     */
+    private function queue(): array
+    {
+        return match (config('queue.default')) {
+            'sync' => $this->check('queue', 'Queue', self::StatusOk, 'Treiber sync — Jobs laufen direkt im Request, kein Worker nötig.'),
+            'database' => $this->databaseQueue(),
+            'redis' => $this->horizonQueue(),
+            default => $this->check('queue', 'Queue', self::StatusWarn, 'Treiber '.config('queue.default').' — hier nicht prüfbar.'),
+        };
+    }
+
+    /**
+     * @return array{key: string, label: string, status: string, detail: string}
+     */
+    private function databaseQueue(): array
+    {
+        try {
+            $now = now()->getTimestamp();
+            $oldestAvailable = DB::table('jobs')
+                ->whereNull('reserved_at')
+                ->where('available_at', '<=', $now)
+                ->min('available_at');
+        } catch (Throwable $exception) {
+            return $this->check('queue', 'Queue', self::StatusCritical, 'Jobs-Tabelle nicht lesbar: '.Str::limit($exception->getMessage(), 120));
+        }
+
+        $waitingSeconds = $oldestAvailable !== null ? max(0, $now - (int) $oldestAvailable) : 0;
+        $criticalSeconds = (int) config('monitoring.health.queue_backlog_critical_seconds');
+
+        return $this->check(
+            'queue',
+            'Queue',
+            $waitingSeconds >= $criticalSeconds ? self::StatusCritical : self::StatusOk,
+            $oldestAvailable === null
+                ? 'Treiber database — kein wartender Job.'
+                : 'Treiber database — ältester wartender Job seit '.$waitingSeconds.' s.'.($waitingSeconds >= $criticalSeconds ? ' Worker gestoppt?' : ''),
+        );
+    }
+
+    /**
+     * @return array{key: string, label: string, status: string, detail: string}
+     */
+    private function horizonQueue(): array
+    {
+        try {
+            Redis::connection()->ping();
+            $masters = app(MasterSupervisorRepository::class)->all();
+        } catch (Throwable $exception) {
+            return $this->check('queue', 'Queue', self::StatusCritical, 'Redis nicht erreichbar: '.Str::limit($exception->getMessage(), 120));
+        }
+
+        if ($masters === []) {
+            return $this->check('queue', 'Queue', self::StatusCritical, 'Treiber redis — Horizon läuft nicht. Daemon in Ploi prüfen.');
+        }
+
+        if (collect($masters)->every(fn (object $master): bool => $master->status === 'paused')) {
+            return $this->check('queue', 'Queue', self::StatusWarn, 'Treiber redis — Horizon ist pausiert.');
+        }
+
+        $longestWait = (int) collect(app(WaitTimeCalculator::class)->calculate())->max();
+        $criticalSeconds = (int) config('monitoring.health.queue_backlog_critical_seconds');
+
+        return $this->check(
+            'queue',
+            'Queue',
+            $longestWait >= $criticalSeconds ? self::StatusCritical : self::StatusOk,
+            'Treiber redis — Horizon läuft, geschätzte Wartezeit höchstens '.$longestWait.' s.',
+        );
+    }
+
+    /**
+     * @return array{key: string, label: string, status: string, detail: string}
+     */
+    private function failedJobs(): array
+    {
+        $hours = (int) config('monitoring.health.failed_jobs_window_hours');
+
+        try {
+            $count = DB::table('failed_jobs')->where('failed_at', '>=', now()->subHours($hours))->count();
+        } catch (Throwable $exception) {
+            return $this->check('failed_jobs', 'Fehlgeschlagene Jobs', self::StatusWarn, 'Tabelle failed_jobs nicht lesbar: '.Str::limit($exception->getMessage(), 120));
+        }
+
+        return $this->check(
+            'failed_jobs',
+            'Fehlgeschlagene Jobs',
+            $count > 0 ? self::StatusWarn : self::StatusOk,
+            $count > 0 ? $count.' in den letzten '.$hours.' Stunden.' : 'Keine in den letzten '.$hours.' Stunden.',
+        );
+    }
+
+    /**
+     * @return array{key: string, label: string, status: string, detail: string}
+     */
+    private function check(string $key, string $label, string $status, string $detail): array
+    {
+        return ['key' => $key, 'label' => $label, 'status' => $status, 'detail' => $detail];
+    }
+}