Systemstatus im Sysadmin-Bereich und /health fuer externe Ueberwachung
- HealthProbe: Datenbank, Cache, Queue (sync / database / Horizon), fehlgeschlagene Jobs (24 h), Scheduler-Herzschlag, Plattenplatz - /health auf jeder Domain, ohne web-Gruppe, nur mit MONITORING_HEALTH_TOKEN - monitoring:heartbeat minuetlich im Scheduler - Sysadmin -> Systemstatus (my.mivita.care/sysadmin/status) mit Version, Umgebung und zuletzt fehlgeschlagenen Jobs - Anleitung Abschnitt G Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
22c85fc746
commit
fec88ee265
17 changed files with 783 additions and 3 deletions
25
app/Console/Commands/MonitoringHeartbeat.php
Normal file
25
app/Console/Commands/MonitoringHeartbeat.php
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Services\Monitoring\HealthProbe;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
/**
|
||||
* Minütlicher Herzschlag des Schedulers. Bleibt er aus,
|
||||
* melden `/health` und der Systemstatus den Scheduler als ausgefallen —
|
||||
* `onFailure` der einzelnen Tasks greift in diesem Fall nicht mehr.
|
||||
*/
|
||||
class MonitoringHeartbeat extends Command
|
||||
{
|
||||
protected $signature = 'monitoring:heartbeat';
|
||||
|
||||
protected $description = 'Schreibt den Herzschlag des Schedulers (Grundlage für /health und den Systemstatus).';
|
||||
|
||||
public function handle(HealthProbe $probe): int
|
||||
{
|
||||
$probe->recordHeartbeat();
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
}
|
||||
|
|
@ -36,6 +36,9 @@ class Kernel extends ConsoleKernel
|
|||
*/
|
||||
protected function schedule(Schedule $schedule)
|
||||
{
|
||||
// Herzschlag fuer /health und den Systemstatus: bleibt er aus, laeuft der Cron nicht
|
||||
$schedule->command('monitoring:heartbeat')->everyMinute()->withoutOverlapping(2);
|
||||
|
||||
// Uptime-Check: PAYONE-Erreichbarkeit alle 5 Minuten prüfen
|
||||
$schedule->command('payment:check-uptime')
|
||||
->everyFiveMinutes()
|
||||
|
|
|
|||
39
app/Http/Controllers/HealthController.php
Normal file
39
app/Http/Controllers/HealthController.php
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
<?php
|
||||
|
||||
namespace App\Http\Controllers;
|
||||
|
||||
use App\Services\Monitoring\HealthProbe;
|
||||
use Illuminate\Http\JsonResponse;
|
||||
use Illuminate\Http\Request;
|
||||
|
||||
/**
|
||||
* Tiefer Health-Check für den externen Monitor.
|
||||
*
|
||||
* `/up` zeigt nur, dass die App bootet. `/health` prüft zusätzlich DB,
|
||||
* Cache, Queue, fehlgeschlagene Jobs, Scheduler-Herzschlag und Plattenplatz: 200 solange
|
||||
* nichts kritisch ist, sonst 503. Zugang nur mit MONITORING_HEALTH_TOKEN
|
||||
* (Bearer oder `?token=`); ohne konfiguriertes Token gibt es den Endpunkt nicht.
|
||||
*/
|
||||
class HealthController extends Controller
|
||||
{
|
||||
public function __invoke(Request $request, HealthProbe $probe): JsonResponse
|
||||
{
|
||||
$token = (string) config('monitoring.health.token');
|
||||
$given = (string) ($request->bearerToken() ?? $request->query('token', ''));
|
||||
|
||||
abort_if($token === '' || ! hash_equals($token, $given), 404);
|
||||
|
||||
$checks = $probe->checks();
|
||||
$status = HealthProbe::worstStatus($checks);
|
||||
|
||||
return response()->json([
|
||||
'status' => $status,
|
||||
'version' => config('app.version'),
|
||||
'checked_at' => now()->toIso8601String(),
|
||||
'checks' => collect($checks)->mapWithKeys(fn (array $check): array => [
|
||||
$check['key'] => ['status' => $check['status'], 'detail' => $check['detail']],
|
||||
]),
|
||||
], $status === HealthProbe::StatusCritical ? 503 : 200)
|
||||
->header('Cache-Control', 'no-store');
|
||||
}
|
||||
}
|
||||
59
app/Http/Controllers/SyS/SystemStatusController.php
Normal file
59
app/Http/Controllers/SyS/SystemStatusController.php
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
<?php
|
||||
|
||||
namespace App\Http\Controllers\SyS;
|
||||
|
||||
use App\Http\Controllers\Controller;
|
||||
use App\Services\Monitoring\HealthProbe;
|
||||
use Illuminate\Contracts\View\View;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Str;
|
||||
use Throwable;
|
||||
|
||||
/**
|
||||
* Systemstatus im Sysadmin-Bereich: dieselben Prüfungen wie `/health`,
|
||||
* dazu Umgebung und die zuletzt fehlgeschlagenen Jobs.
|
||||
*/
|
||||
class SystemStatusController extends Controller
|
||||
{
|
||||
public function index(HealthProbe $probe): View
|
||||
{
|
||||
$checks = $probe->checks();
|
||||
|
||||
return view('sys.status.index', [
|
||||
'checks' => $checks,
|
||||
'overall' => HealthProbe::worstStatus($checks),
|
||||
'environment' => [
|
||||
'Version' => config('app.version') ?? 'unbekannt',
|
||||
'Umgebung' => app()->environment(),
|
||||
'PHP' => PHP_VERSION,
|
||||
'Laravel' => app()->version(),
|
||||
'Queue-Treiber' => config('queue.default'),
|
||||
'Cache-Treiber' => config('cache.default'),
|
||||
],
|
||||
'healthTokenConfigured' => filled(config('monitoring.health.token')),
|
||||
'horizonAvailable' => config('queue.default') === 'redis',
|
||||
'failedJobs' => $this->recentFailedJobs(),
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Collection<int, array{uuid: ?string, job: string, queue: string, failed_at: string, error: string}>
|
||||
*/
|
||||
private function recentFailedJobs(): Collection
|
||||
{
|
||||
try {
|
||||
$rows = DB::table('failed_jobs')->latest('failed_at')->limit(10)->get();
|
||||
} catch (Throwable) {
|
||||
return collect();
|
||||
}
|
||||
|
||||
return $rows->map(fn (object $row): array => [
|
||||
'uuid' => $row->uuid ?? null,
|
||||
'job' => json_decode($row->payload, true)['displayName'] ?? 'unbekannt',
|
||||
'queue' => $row->connection.' / '.$row->queue,
|
||||
'failed_at' => (string) $row->failed_at,
|
||||
'error' => Str::limit(Str::before((string) $row->exception, "\n"), 200),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ namespace App\Providers;
|
|||
|
||||
// use App\Domain\DomainContext; // HOTFIX: Temporär deaktiviert
|
||||
use App\Domain\EarlyDomainParser;
|
||||
use App\Http\Controllers\HealthController;
|
||||
use Illuminate\Cache\RateLimiting\Limit;
|
||||
use Illuminate\Foundation\Support\Providers\RouteServiceProvider as ServiceProvider;
|
||||
use Illuminate\Http\Request;
|
||||
|
|
@ -36,8 +37,14 @@ class RouteServiceProvider extends ServiceProvider
|
|||
$this->configureRateLimiting();
|
||||
|
||||
$this->routes(function () {
|
||||
// Tiefer Health-Check für den externen Monitor: auf jeder Domain,
|
||||
// ohne web-Gruppe (keine Session, kein CSRF, kein SubdomainResolver)
|
||||
Route::get('/health', HealthController::class)
|
||||
->middleware('throttle:30,1')
|
||||
->name('health.deep');
|
||||
|
||||
// API-Routen werden global geladen
|
||||
Route::domain('api.' . config('app.domain') . config('app.tld_care'))
|
||||
Route::domain('api.'.config('app.domain').config('app.tld_care'))
|
||||
->middleware('api')
|
||||
->namespace($this->namespace)
|
||||
->group(base_path('routes/api.php'));
|
||||
|
|
@ -74,7 +81,7 @@ class RouteServiceProvider extends ServiceProvider
|
|||
default => ['main'],
|
||||
};
|
||||
foreach ($routesToLoad as $routeType) {
|
||||
$this->loadDomainRoutes($routeType, $routeType . '.php');
|
||||
$this->loadDomainRoutes($routeType, $routeType.'.php');
|
||||
}
|
||||
|
||||
if (config('app.debug')) {
|
||||
|
|
@ -91,7 +98,7 @@ class RouteServiceProvider extends ServiceProvider
|
|||
*/
|
||||
protected function loadDomainRoutes(string $domainType, string $fileName): void
|
||||
{
|
||||
$routePath = base_path('routes/domains/' . $fileName);
|
||||
$routePath = base_path('routes/domains/'.$fileName);
|
||||
|
||||
if (file_exists($routePath)) {
|
||||
Route::group([], $routePath);
|
||||
|
|
|
|||
265
app/Services/Monitoring/HealthProbe.php
Normal file
265
app/Services/Monitoring/HealthProbe.php
Normal file
|
|
@ -0,0 +1,265 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Monitoring;
|
||||
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Redis;
|
||||
use Illuminate\Support\Str;
|
||||
use Laravel\Horizon\Contracts\MasterSupervisorRepository;
|
||||
use Laravel\Horizon\WaitTimeCalculator;
|
||||
use Throwable;
|
||||
|
||||
/**
|
||||
* Die Prüfungen hinter `/health` (externer Monitor) und dem Systemstatus im
|
||||
* Sysadmin-Bereich: Datenbank, Cache, Queue, fehlgeschlagene Jobs,
|
||||
* Scheduler-Herzschlag, Plattenplatz. Bewusst billig — keine Zählungen über
|
||||
* große Tabellen, damit `/health` selbst keine Last erzeugt.
|
||||
*/
|
||||
class HealthProbe
|
||||
{
|
||||
public const StatusOk = 'ok';
|
||||
|
||||
public const StatusWarn = 'warn';
|
||||
|
||||
public const StatusCritical = 'critical';
|
||||
|
||||
/**
|
||||
* Datei statt Datenbank/Cache: übersteht `cache:clear`, braucht keine
|
||||
* Migration. Bei mehreren App-Servern stattdessen eine DB-Zeile nehmen.
|
||||
*/
|
||||
public static function heartbeatPath(): string
|
||||
{
|
||||
return storage_path('framework/scheduler-heartbeat');
|
||||
}
|
||||
|
||||
public function recordHeartbeat(): void
|
||||
{
|
||||
file_put_contents(self::heartbeatPath(), now()->toIso8601String(), LOCK_EX);
|
||||
}
|
||||
|
||||
public function lastHeartbeat(): ?Carbon
|
||||
{
|
||||
$value = @file_get_contents(self::heartbeatPath());
|
||||
|
||||
return is_string($value) && trim($value) !== '' ? Carbon::parse(trim($value)) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list<array{key: string, label: string, status: string, detail: string}>
|
||||
*/
|
||||
public function checks(): array
|
||||
{
|
||||
return [
|
||||
$this->database(),
|
||||
$this->cache(),
|
||||
$this->queue(),
|
||||
$this->failedJobs(),
|
||||
$this->scheduler(),
|
||||
$this->disk(),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<array{status: string}> $checks
|
||||
*/
|
||||
public static function worstStatus(array $checks): string
|
||||
{
|
||||
$statuses = array_column($checks, 'status');
|
||||
|
||||
return match (true) {
|
||||
in_array(self::StatusCritical, $statuses, true) => self::StatusCritical,
|
||||
in_array(self::StatusWarn, $statuses, true) => self::StatusWarn,
|
||||
default => self::StatusOk,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
public function scheduler(): array
|
||||
{
|
||||
$lastBeat = $this->lastHeartbeat();
|
||||
$warnMinutes = (int) config('monitoring.scheduler.heartbeat_warn_minutes');
|
||||
$criticalMinutes = (int) config('monitoring.scheduler.heartbeat_critical_minutes');
|
||||
$minutesAgo = $lastBeat !== null ? (int) $lastBeat->diffInMinutes(now(), absolute: true) : null;
|
||||
|
||||
return $this->check(
|
||||
'scheduler',
|
||||
'Scheduler',
|
||||
match (true) {
|
||||
$minutesAgo === null, $minutesAgo >= $criticalMinutes => self::StatusCritical,
|
||||
$minutesAgo >= $warnMinutes => self::StatusWarn,
|
||||
default => self::StatusOk,
|
||||
},
|
||||
$lastBeat === null
|
||||
? 'Noch kein Herzschlag — läuft `schedule:run` minütlich per Cron?'
|
||||
: 'Letzter Herzschlag '.$lastBeat->format('d.m.Y H:i:s').' (vor '.$minutesAgo.' Min.).',
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
public function disk(): array
|
||||
{
|
||||
$path = storage_path();
|
||||
$free = @disk_free_space($path);
|
||||
$total = @disk_total_space($path);
|
||||
|
||||
if ($free === false || $total === false || $total <= 0) {
|
||||
return $this->check('disk', 'Plattenplatz', self::StatusWarn, 'Plattenplatz nicht ermittelbar.');
|
||||
}
|
||||
|
||||
$freePercent = round($free / $total * 100, 1);
|
||||
$minPercent = (float) config('monitoring.health.disk_min_free_percent');
|
||||
|
||||
return $this->check(
|
||||
'disk',
|
||||
'Plattenplatz',
|
||||
match (true) {
|
||||
$freePercent < $minPercent => self::StatusCritical,
|
||||
$freePercent < $minPercent * 2 => self::StatusWarn,
|
||||
default => self::StatusOk,
|
||||
},
|
||||
sprintf('%s %% frei (%s GB von %s GB).', $freePercent, round($free / 1024 ** 3, 1), round($total / 1024 ** 3, 1)),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
private function database(): array
|
||||
{
|
||||
try {
|
||||
DB::select('select 1');
|
||||
|
||||
return $this->check('database', 'Datenbank', self::StatusOk, 'Verbindung steht.');
|
||||
} catch (Throwable $exception) {
|
||||
return $this->check('database', 'Datenbank', self::StatusCritical, 'Keine Verbindung: '.Str::limit($exception->getMessage(), 120));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
private function cache(): array
|
||||
{
|
||||
try {
|
||||
$value = Str::random(8);
|
||||
Cache::put('monitoring:health-probe', $value, 60);
|
||||
|
||||
return Cache::get('monitoring:health-probe') === $value
|
||||
? $this->check('cache', 'Cache', self::StatusOk, 'Schreiben und Lesen funktionieren ('.config('cache.default').').')
|
||||
: $this->check('cache', 'Cache', self::StatusCritical, 'Geschriebener Wert kommt nicht zurück.');
|
||||
} catch (Throwable $exception) {
|
||||
return $this->check('cache', 'Cache', self::StatusCritical, 'Nicht erreichbar: '.Str::limit($exception->getMessage(), 120));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Je nach QUEUE_DRIVER: `sync` braucht keinen Worker, `database` wird am
|
||||
* ältesten wartenden Job gemessen, `redis` an Horizon (läuft es, wie lange
|
||||
* warten die Jobs).
|
||||
*
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
private function queue(): array
|
||||
{
|
||||
return match (config('queue.default')) {
|
||||
'sync' => $this->check('queue', 'Queue', self::StatusOk, 'Treiber sync — Jobs laufen direkt im Request, kein Worker nötig.'),
|
||||
'database' => $this->databaseQueue(),
|
||||
'redis' => $this->horizonQueue(),
|
||||
default => $this->check('queue', 'Queue', self::StatusWarn, 'Treiber '.config('queue.default').' — hier nicht prüfbar.'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
private function databaseQueue(): array
|
||||
{
|
||||
try {
|
||||
$now = now()->getTimestamp();
|
||||
$oldestAvailable = DB::table('jobs')
|
||||
->whereNull('reserved_at')
|
||||
->where('available_at', '<=', $now)
|
||||
->min('available_at');
|
||||
} catch (Throwable $exception) {
|
||||
return $this->check('queue', 'Queue', self::StatusCritical, 'Jobs-Tabelle nicht lesbar: '.Str::limit($exception->getMessage(), 120));
|
||||
}
|
||||
|
||||
$waitingSeconds = $oldestAvailable !== null ? max(0, $now - (int) $oldestAvailable) : 0;
|
||||
$criticalSeconds = (int) config('monitoring.health.queue_backlog_critical_seconds');
|
||||
|
||||
return $this->check(
|
||||
'queue',
|
||||
'Queue',
|
||||
$waitingSeconds >= $criticalSeconds ? self::StatusCritical : self::StatusOk,
|
||||
$oldestAvailable === null
|
||||
? 'Treiber database — kein wartender Job.'
|
||||
: 'Treiber database — ältester wartender Job seit '.$waitingSeconds.' s.'.($waitingSeconds >= $criticalSeconds ? ' Worker gestoppt?' : ''),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
private function horizonQueue(): array
|
||||
{
|
||||
try {
|
||||
Redis::connection()->ping();
|
||||
$masters = app(MasterSupervisorRepository::class)->all();
|
||||
} catch (Throwable $exception) {
|
||||
return $this->check('queue', 'Queue', self::StatusCritical, 'Redis nicht erreichbar: '.Str::limit($exception->getMessage(), 120));
|
||||
}
|
||||
|
||||
if ($masters === []) {
|
||||
return $this->check('queue', 'Queue', self::StatusCritical, 'Treiber redis — Horizon läuft nicht. Daemon in Ploi prüfen.');
|
||||
}
|
||||
|
||||
if (collect($masters)->every(fn (object $master): bool => $master->status === 'paused')) {
|
||||
return $this->check('queue', 'Queue', self::StatusWarn, 'Treiber redis — Horizon ist pausiert.');
|
||||
}
|
||||
|
||||
$longestWait = (int) collect(app(WaitTimeCalculator::class)->calculate())->max();
|
||||
$criticalSeconds = (int) config('monitoring.health.queue_backlog_critical_seconds');
|
||||
|
||||
return $this->check(
|
||||
'queue',
|
||||
'Queue',
|
||||
$longestWait >= $criticalSeconds ? self::StatusCritical : self::StatusOk,
|
||||
'Treiber redis — Horizon läuft, geschätzte Wartezeit höchstens '.$longestWait.' s.',
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
private function failedJobs(): array
|
||||
{
|
||||
$hours = (int) config('monitoring.health.failed_jobs_window_hours');
|
||||
|
||||
try {
|
||||
$count = DB::table('failed_jobs')->where('failed_at', '>=', now()->subHours($hours))->count();
|
||||
} catch (Throwable $exception) {
|
||||
return $this->check('failed_jobs', 'Fehlgeschlagene Jobs', self::StatusWarn, 'Tabelle failed_jobs nicht lesbar: '.Str::limit($exception->getMessage(), 120));
|
||||
}
|
||||
|
||||
return $this->check(
|
||||
'failed_jobs',
|
||||
'Fehlgeschlagene Jobs',
|
||||
$count > 0 ? self::StatusWarn : self::StatusOk,
|
||||
$count > 0 ? $count.' in den letzten '.$hours.' Stunden.' : 'Keine in den letzten '.$hours.' Stunden.',
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{key: string, label: string, status: string, detail: string}
|
||||
*/
|
||||
private function check(string $key, string $label, string $status, string $detail): array
|
||||
{
|
||||
return ['key' => $key, 'label' => $label, 'status' => $status, 'detail' => $detail];
|
||||
}
|
||||
}
|
||||
30
config/monitoring.php
Normal file
30
config/monitoring.php
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
<?php
|
||||
|
||||
return [
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Health-Endpunkt und Scheduler-Herzschlag
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| `/health` antwortet nur mit gültigem Token (Bearer-Header oder
|
||||
| `?token=`); ohne konfiguriertes Token gibt es den Endpunkt nicht (404).
|
||||
| Der Scheduler schreibt jede Minute einen Herzschlag; bleibt er aus, ist
|
||||
| der Cron-Eintrag weg oder der Server hängt. Dieselben Prüfungen zeigt
|
||||
| der Systemstatus im Sysadmin-Bereich.
|
||||
|
|
||||
*/
|
||||
|
||||
'health' => [
|
||||
'token' => env('MONITORING_HEALTH_TOKEN'),
|
||||
'queue_backlog_critical_seconds' => (int) env('MONITORING_QUEUE_BACKLOG_SECONDS', 300),
|
||||
'disk_min_free_percent' => (float) env('MONITORING_DISK_MIN_FREE_PERCENT', 10),
|
||||
'failed_jobs_window_hours' => 24,
|
||||
],
|
||||
|
||||
'scheduler' => [
|
||||
'heartbeat_warn_minutes' => 3,
|
||||
'heartbeat_critical_minutes' => 10,
|
||||
],
|
||||
|
||||
];
|
||||
|
|
@ -221,6 +221,29 @@ grep -E '^(APP_ENV|QUEUE_DRIVER|REDIS_HOST)=' .env # APP_ENV=production
|
|||
Zurück: `QUEUE_DRIVER=sync` in `.env`, in `deploy.sh` `QUEUE_WORKER="none"`. Liegen gebliebene Jobs
|
||||
vorher mit `php artisan queue:work redis --stop-when-empty` abarbeiten.
|
||||
|
||||
## G — Überwachung: Systemstatus und `/health`
|
||||
|
||||
**Systemstatus** im CRM unter Sysadmin → „Systemstatus“ (`https://my.mivita.care/sysadmin/status`, nur admin ≥ 4):
|
||||
Datenbank, Cache, Queue (sync / database / Horizon), fehlgeschlagene Jobs der letzten 24 h,
|
||||
Scheduler-Herzschlag, Plattenplatz, Version, zuletzt fehlgeschlagene Jobs.
|
||||
|
||||
**`/health`** liefert dieselben Prüfungen als JSON für einen externen Monitor, auf jeder Domain
|
||||
(z. B. `https://mivita.care/health`). 200 = in Ordnung oder Warnung, 503 = kritisch. Ohne Token 404.
|
||||
|
||||
Einrichtung auf Live (einmalig):
|
||||
|
||||
```bash
|
||||
cd /home/ploi/mivita.care
|
||||
openssl rand -hex 24 # Wert als MONITORING_HEALTH_TOKEN in die .env
|
||||
curl -s -H "Authorization: Bearer <token>" https://mivita.care/health
|
||||
```
|
||||
|
||||
Der **Scheduler-Herzschlag** (`monitoring:heartbeat`, minütlich) zeigt, ob der Cron `schedule:run`
|
||||
wirklich läuft. Steht er nach dem Deploy auf „Kritisch“, ist in Ploi kein Scheduler-Eintrag
|
||||
(`php artisan schedule:run` jede Minute) aktiv.
|
||||
|
||||
Schwellen in `.env` (optional): `MONITORING_QUEUE_BACKLOG_SECONDS=300`, `MONITORING_DISK_MIN_FREE_PERCENT=10`.
|
||||
|
||||
## Fallstricke
|
||||
|
||||
- **FPM-Restart statt Reload** → 502 für Sekunden.
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ return [
|
|||
'customer_orders' => 'Kundenbestellungen',
|
||||
'external_orders' => 'Externe Bestellungen',
|
||||
'tools' => 'Tools',
|
||||
'system_status' => 'Systemstatus',
|
||||
'news_archive' => 'News Archiv',
|
||||
'incentive' => 'Incentive',
|
||||
'incentives' => 'Incentives',
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ return [
|
|||
'customer_orders' => 'Customer Orders',
|
||||
'external_orders' => 'External Orders',
|
||||
'tools' => 'Tools',
|
||||
'system_status' => 'System status',
|
||||
'news_archive' => 'News Archive',
|
||||
'incentive' => 'Incentive',
|
||||
'incentives' => 'Incentives',
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ return [
|
|||
'customer_orders' => 'Pedidos de clientes',
|
||||
'external_orders' => 'Pedidos externos',
|
||||
'tools' => 'Herramientas',
|
||||
'system_status' => 'Estado del sistema',
|
||||
'news_archive' => 'Archivo de Noticias',
|
||||
'incentive' => 'Incentivo',
|
||||
'incentives' => 'Incentivos',
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ return [
|
|||
'customer_orders' => 'Commandes clients',
|
||||
'external_orders' => 'Commandes externes',
|
||||
'tools' => 'Outils',
|
||||
'system_status' => 'État du système',
|
||||
'news_archive' => 'Archive des actualités',
|
||||
'incentive' => 'Incentive',
|
||||
'incentives' => 'Incentives',
|
||||
|
|
|
|||
|
|
@ -713,6 +713,12 @@
|
|||
<div>{{ __('navigation.tools') }}</div>
|
||||
</a>
|
||||
</li>
|
||||
<li class="sidenav-item{{ Request::is('sysadmin/status') ? ' active' : '' }}">
|
||||
<a href="{{ route('sysadmin_status') }}" class="sidenav-link"><i
|
||||
class="sidenav-icon ion ion-md-pulse"></i>
|
||||
<div>{{ __('navigation.system_status') }}</div>
|
||||
</a>
|
||||
</li>
|
||||
<li class="sidenav-item{{ Request::is('sysadmin/settings') ? ' active' : '' }}">
|
||||
<a href="{{ route('sysadmin_settings') }}" class="sidenav-link"><i
|
||||
class="sidenav-icon ion ion-ios-cog"></i>
|
||||
|
|
|
|||
109
resources/views/sys/status/index.blade.php
Normal file
109
resources/views/sys/status/index.blade.php
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
@extends('layouts.layout-2')
|
||||
|
||||
@php
|
||||
$badges = ['ok' => 'badge-success', 'warn' => 'badge-warning', 'critical' => 'badge-danger'];
|
||||
$labels = ['ok' => 'OK', 'warn' => 'Warnung', 'critical' => 'Kritisch'];
|
||||
@endphp
|
||||
|
||||
@section('content')
|
||||
<h4 class="font-weight-bold py-3 mb-2">
|
||||
Systemstatus
|
||||
<span class="badge {{ $badges[$overall] }} align-middle ml-2">{{ $labels[$overall] }}</span>
|
||||
<small class="text-muted ml-2">Stand {{ now()->format('d.m.Y H:i:s') }}</small>
|
||||
</h4>
|
||||
|
||||
<div class="card mb-4">
|
||||
<h6 class="card-header">Prüfungen</h6>
|
||||
<div class="table-responsive">
|
||||
<table class="table table-striped mb-0">
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="width: 220px;">Prüfung</th>
|
||||
<th style="width: 120px;">Status</th>
|
||||
<th>Detail</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@foreach ($checks as $check)
|
||||
<tr>
|
||||
<td>{{ $check['label'] }}</td>
|
||||
<td><span class="badge {{ $badges[$check['status']] }}">{{ $labels[$check['status']] }}</span></td>
|
||||
<td>{{ $check['detail'] }}</td>
|
||||
</tr>
|
||||
@endforeach
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-6">
|
||||
<div class="card mb-4">
|
||||
<h6 class="card-header">Umgebung</h6>
|
||||
<table class="table mb-0">
|
||||
<tbody>
|
||||
@foreach ($environment as $label => $value)
|
||||
<tr>
|
||||
<td style="width: 160px;">{{ $label }}</td>
|
||||
<td><code>{{ $value }}</code></td>
|
||||
</tr>
|
||||
@endforeach
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="card mb-4">
|
||||
<h6 class="card-header">Externe Überwachung</h6>
|
||||
<div class="card-body">
|
||||
<p class="mb-2">
|
||||
Endpunkt für den externen Monitor:
|
||||
<code>{{ url('/health') }}</code>
|
||||
</p>
|
||||
@if ($healthTokenConfigured)
|
||||
<p class="mb-2"><span class="badge badge-success">Token gesetzt</span>
|
||||
Abruf mit Header <code>Authorization: Bearer <MONITORING_HEALTH_TOKEN></code>.
|
||||
Antwort 200 = in Ordnung, 503 = kritisch.</p>
|
||||
@else
|
||||
<p class="mb-2"><span class="badge badge-warning">Kein Token</span>
|
||||
<code>MONITORING_HEALTH_TOKEN</code> ist in der <code>.env</code> nicht gesetzt — der Endpunkt
|
||||
antwortet mit 404.</p>
|
||||
@endif
|
||||
@if ($horizonAvailable)
|
||||
<a href="{{ url(config('horizon.path', 'horizon')) }}" class="btn btn-sm btn-outline-primary" target="_blank">Horizon öffnen</a>
|
||||
@endif
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card mb-4">
|
||||
<h6 class="card-header">Zuletzt fehlgeschlagene Jobs</h6>
|
||||
@if ($failedJobs->isEmpty())
|
||||
<div class="card-body text-muted">Keine fehlgeschlagenen Jobs gespeichert.</div>
|
||||
@else
|
||||
<div class="table-responsive">
|
||||
<table class="table table-striped mb-0">
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="width: 160px;">Zeitpunkt</th>
|
||||
<th>Job</th>
|
||||
<th>Queue</th>
|
||||
<th>Fehler</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@foreach ($failedJobs as $job)
|
||||
<tr>
|
||||
<td>{{ $job['failed_at'] }}</td>
|
||||
<td>{{ $job['job'] }}</td>
|
||||
<td>{{ $job['queue'] }}</td>
|
||||
<td><small>{{ $job['error'] }}</small></td>
|
||||
</tr>
|
||||
@endforeach
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
@endif
|
||||
</div>
|
||||
@endsection
|
||||
|
|
@ -570,6 +570,7 @@ Route::domain(config('app.pre_url_crm').config('app.domain').config('app.tld_car
|
|||
Route::post('/sysadmin/tool/store/{tool}', 'SyS\SysController@store')->name('sysadmin_tool_store');
|
||||
|
||||
Route::get('/sysadmin/settings', 'SyS\SettingController@index')->name('sysadmin_settings');
|
||||
Route::get('/sysadmin/status', 'SyS\SystemStatusController@index')->name('sysadmin_status');
|
||||
Route::post('/sysadmin/setting/store', 'SyS\SettingController@store')->name('sysadmin_setting_store');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
131
tests/Feature/Monitoring/HealthEndpointTest.php
Normal file
131
tests/Feature/Monitoring/HealthEndpointTest.php
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
<?php
|
||||
|
||||
use App\Services\Monitoring\HealthProbe;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Redis;
|
||||
use Laravel\Horizon\Contracts\MasterSupervisorRepository;
|
||||
use Tests\TestCase;
|
||||
|
||||
uses(TestCase::class, RefreshDatabase::class);
|
||||
|
||||
/**
|
||||
* Tiefer Health-Check und Scheduler-Herzschlag. Der Herzschlag liegt in
|
||||
* einer Datei unter storage/ — vor und nach jedem Test entfernen.
|
||||
*/
|
||||
beforeEach(function () {
|
||||
@unlink(HealthProbe::heartbeatPath());
|
||||
config()->set('cache.default', 'array');
|
||||
config()->set('monitoring.health.token', 'geheimes-monitor-token');
|
||||
config()->set('monitoring.health.disk_min_free_percent', 0);
|
||||
config()->set('queue.default', 'sync');
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
@unlink(HealthProbe::heartbeatPath());
|
||||
});
|
||||
|
||||
function healthHeaders(): array
|
||||
{
|
||||
return ['Authorization' => 'Bearer geheimes-monitor-token'];
|
||||
}
|
||||
|
||||
test('the endpoint does not exist without a configured token or with a wrong one', function () {
|
||||
app(HealthProbe::class)->recordHeartbeat();
|
||||
|
||||
$this->get('/health', healthHeaders())->assertOk();
|
||||
|
||||
$this->get('/health', ['Authorization' => 'Bearer falsch'])->assertNotFound();
|
||||
$this->get('/health')->assertNotFound();
|
||||
|
||||
config()->set('monitoring.health.token', null);
|
||||
$this->get('/health', healthHeaders())->assertNotFound();
|
||||
});
|
||||
|
||||
test('a healthy system answers 200 with every check', function () {
|
||||
config()->set('app.version', 'v1.1.0');
|
||||
$this->artisan('monitoring:heartbeat')->assertSuccessful();
|
||||
|
||||
$this->get('/health?token=geheimes-monitor-token')
|
||||
->assertOk()
|
||||
->assertHeader('Cache-Control', 'no-store, private')
|
||||
->assertJsonPath('status', 'ok')
|
||||
->assertJsonPath('version', 'v1.1.0')
|
||||
->assertJsonPath('checks.database.status', 'ok')
|
||||
->assertJsonPath('checks.cache.status', 'ok')
|
||||
->assertJsonPath('checks.queue.status', 'ok')
|
||||
->assertJsonPath('checks.failed_jobs.status', 'ok')
|
||||
->assertJsonPath('checks.scheduler.status', 'ok')
|
||||
->assertJsonStructure(['checks' => ['disk' => ['status', 'detail']]]);
|
||||
});
|
||||
|
||||
test('a missing or stale heartbeat makes the endpoint fail', function () {
|
||||
$this->get('/health', healthHeaders())
|
||||
->assertStatus(503)
|
||||
->assertJsonPath('checks.scheduler.status', 'critical');
|
||||
|
||||
app(HealthProbe::class)->recordHeartbeat();
|
||||
|
||||
$this->travel(4)->minutes();
|
||||
$this->get('/health', healthHeaders())->assertOk()->assertJsonPath('checks.scheduler.status', 'warn');
|
||||
|
||||
$this->travel(7)->minutes();
|
||||
$this->get('/health', healthHeaders())->assertStatus(503)->assertJsonPath('checks.scheduler.status', 'critical');
|
||||
});
|
||||
|
||||
test('a database queue backlog older than five minutes makes the endpoint fail', function () {
|
||||
config()->set('queue.default', 'database');
|
||||
app(HealthProbe::class)->recordHeartbeat();
|
||||
|
||||
DB::table('jobs')->insert([
|
||||
'queue' => 'default',
|
||||
'payload' => '{}',
|
||||
'attempts' => 0,
|
||||
'reserved_at' => null,
|
||||
'available_at' => now()->subMinutes(6)->getTimestamp(),
|
||||
'created_at' => now()->subMinutes(6)->getTimestamp(),
|
||||
]);
|
||||
|
||||
$this->get('/health', healthHeaders())
|
||||
->assertStatus(503)
|
||||
->assertJsonPath('checks.queue.status', 'critical');
|
||||
});
|
||||
|
||||
test('with the redis driver a stopped horizon makes the endpoint fail', function () {
|
||||
config()->set('queue.default', 'redis');
|
||||
app(HealthProbe::class)->recordHeartbeat();
|
||||
|
||||
Redis::shouldReceive('connection->ping')->andReturn(true);
|
||||
$this->mock(MasterSupervisorRepository::class)->shouldReceive('all')->andReturn([]);
|
||||
|
||||
$this->get('/health', healthHeaders())
|
||||
->assertStatus(503)
|
||||
->assertJsonPath('checks.queue.status', 'critical')
|
||||
->assertJsonPath('checks.queue.detail', 'Treiber redis — Horizon läuft nicht. Daemon in Ploi prüfen.');
|
||||
});
|
||||
|
||||
test('recently failed jobs are a warning, not an outage', function () {
|
||||
app(HealthProbe::class)->recordHeartbeat();
|
||||
|
||||
DB::table('failed_jobs')->insert([
|
||||
'uuid' => 'a1b2c3',
|
||||
'connection' => 'redis',
|
||||
'queue' => 'default',
|
||||
'payload' => json_encode(['displayName' => 'App\\Jobs\\SyncStripeInvoiceReference']),
|
||||
'exception' => "RuntimeException: Stripe nicht erreichbar\n#0 ...",
|
||||
'failed_at' => now()->subHour(),
|
||||
]);
|
||||
|
||||
$this->get('/health', healthHeaders())
|
||||
->assertOk()
|
||||
->assertJsonPath('status', 'warn')
|
||||
->assertJsonPath('checks.failed_jobs.status', 'warn');
|
||||
});
|
||||
|
||||
test('the endpoint skips the session and the web middleware group', function () {
|
||||
app(HealthProbe::class)->recordHeartbeat();
|
||||
|
||||
$this->get('/health', healthHeaders())
|
||||
->assertOk()
|
||||
->assertCookieMissing(config('session.cookie'));
|
||||
});
|
||||
78
tests/Feature/Monitoring/SystemStatusPageTest.php
Normal file
78
tests/Feature/Monitoring/SystemStatusPageTest.php
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
<?php
|
||||
|
||||
use App\Models\UserAccount;
|
||||
use App\Services\Monitoring\HealthProbe;
|
||||
use App\User;
|
||||
use Tests\TestCase;
|
||||
|
||||
uses(TestCase::class);
|
||||
|
||||
/**
|
||||
* Systemstatus unter Sysadmin (my.mivita.care/sysadmin/status). Die
|
||||
* CRM-Routen werden host-abhängig geladen, daher startet die App mit dem
|
||||
* CRM-Host neu.
|
||||
*/
|
||||
beforeEach(function () {
|
||||
$this->previousHttpHost = $_SERVER['HTTP_HOST'] ?? null;
|
||||
$_SERVER['HTTP_HOST'] = config('domains.domains.crm.host');
|
||||
$this->refreshApplication();
|
||||
$this->artisan('migrate');
|
||||
config()->set('cache.default', 'array');
|
||||
config()->set('queue.default', 'sync');
|
||||
@unlink(HealthProbe::heartbeatPath());
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
@unlink(HealthProbe::heartbeatPath());
|
||||
|
||||
if ($this->previousHttpHost === null) {
|
||||
unset($_SERVER['HTTP_HOST']);
|
||||
} else {
|
||||
$_SERVER['HTTP_HOST'] = $this->previousHttpHost;
|
||||
}
|
||||
});
|
||||
|
||||
function statusPageUser(int $admin): User
|
||||
{
|
||||
$account = UserAccount::create([
|
||||
'first_name' => 'Sys',
|
||||
'last_name' => 'Admin',
|
||||
'm_first_name' => 'Sys',
|
||||
'm_last_name' => 'Admin',
|
||||
'm_account' => 'M'.random_int(1000, 9999),
|
||||
]);
|
||||
|
||||
return User::forceCreate([
|
||||
'email' => 'sys-'.uniqid('', true).'@example.com',
|
||||
'password' => bcrypt('secret'),
|
||||
'lang' => 'de',
|
||||
'active' => 1,
|
||||
'blocked' => 0,
|
||||
'wizard' => 100,
|
||||
'admin' => $admin,
|
||||
'payment_account' => now()->addYear(),
|
||||
'account_id' => $account->id,
|
||||
])->fresh();
|
||||
}
|
||||
|
||||
test('system admins see the checks, the version and the menu entry', function () {
|
||||
config()->set('app.version', 'v1.0.2');
|
||||
app(HealthProbe::class)->recordHeartbeat();
|
||||
|
||||
$this->actingAs(statusPageUser(4))
|
||||
->get(route('sysadmin_status'))
|
||||
->assertOk()
|
||||
->assertSee('Systemstatus')
|
||||
->assertSee('Datenbank')
|
||||
->assertSee('Scheduler')
|
||||
->assertSee('Fehlgeschlagene Jobs')
|
||||
->assertSee('v1.0.2')
|
||||
->assertSee(route('sysadmin_status'), false)
|
||||
->assertSee('Kein Token');
|
||||
});
|
||||
|
||||
test('other admins are not allowed to open the page', function () {
|
||||
$this->actingAs(statusPageUser(3))
|
||||
->get(route('sysadmin_status'))
|
||||
->assertForbidden();
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue