web: Crawler-Abruf-Protokoll und Panel „Auslieferung" ohne Google-Link (Anhang 1: 1c.0, 1c.1, 1c.6)

Abrufe öffentlicher Meldungen durch bekannte Such- und KI-Dienste werden je Meldung mit Dienst, gekürztem User-Agent und Zeitpunkt protokolliert — keine IP. Dienste und Muster stehen in config/crawlers.php. Aufzeichnung wie der Aufruf-Zähler per defer() und fehlertolerant; Bereinigung hängt an press-releases:purge-events. VisibilityStats liefert die Abrufe je Dienst (letzter Abruf, Anzahl) und verliert den site:-Suchlink (Stufe 1 verspricht nichts, was von Dritten abhängt). Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 14:21:35 +00:00 · 2026-09-17 14:21:35 +00:00 · 7088d092d8
commit 7088d092d8
parent 6446e19727
10 changed files with 440 additions and 7 deletions
--- a/tests/Feature/Web/CrawlerHitLogTest.php
+++ b/tests/Feature/Web/CrawlerHitLogTest.php
@ -0,0 +1,90 @@
+<?php
+
+use App\Console\Commands\AggregatePressReleaseViews;
+use App\Console\Commands\PurgePressReleaseEvents;
+use App\Enums\Portal;
+use App\Models\AppSetting;
+use App\Models\Category;
+use App\Models\Company;
+use App\Models\PressRelease;
+use App\Models\PressReleaseCrawlerHit;
+use App\Support\CrawlerIdentifier;
+use Tests\TestCase;
+
+/**
+ * Crawler-Abruf-Protokoll (Roadmap 1c.0): Abrufe durch bekannte Such- und
+ * KI-Dienste werden je Meldung mit Dienst, User-Agent und Zeitpunkt
+ * gespeichert — ohne IP; Browser und unbekannte Bots nicht.
+ */
+function makeCrawlerLogRelease(): PressRelease
+{
+    $category = Category::factory()->create(['portal' => Portal::Presseecho]);
+    $category->translations()->create(['locale' => 'de', 'name' => 'Handel', 'slug' => 'handel']);
+    $company = Company::factory()->create(['name' => 'Crawler GmbH', 'portal' => Portal::Presseecho]);
+
+    return PressRelease::factory()
+        ->published()
+        ->forPortal(Portal::Presseecho)
+        ->for($category)
+        ->for($company)
+        ->create(['slug' => 'crawler-protokoll-pm', 'published_at' => now()->subDay()]);
+}
+
+test('the identifier maps known user agents to a service and ignores browsers', function (?string $userAgent, ?string $expected) {
+    expect(CrawlerIdentifier::identify($userAgent))->toBe($expected);
+})->with([
+    ['Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)', 'openai'],
+    ['Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)', 'openai'],
+    ['Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)', 'anthropic'],
+    ['Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)', 'perplexity'],
+    ['Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', 'google'],
+    ['Mozilla/5.0 (compatible; Google-Extended)', 'google-extended'],
+    ['Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)', 'bing'],
+    ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/128.0 Safari/537.36', null],
+    ['curl/8.4.0', null],
+    ['', null],
+    [null, null],
+]);
+
+test('a fetch by a known crawler is logged for the release without an IP address', function () {
+    /** @var TestCase $this */
+    $release = makeCrawlerLogRelease();
+
+    $this->withHeaders(['User-Agent' => 'Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)'])
+        ->get('https://presseecho.test/de/press-release/crawler-protokoll-pm')
+        ->assertSuccessful();
+
+    $hit = PressReleaseCrawlerHit::query()->sole();
+
+    expect($hit->press_release_id)->toBe($release->id)
+        ->and($hit->service)->toBe('anthropic')
+        ->and($hit->user_agent)->toContain('ClaudeBot')
+        ->and($hit->getAttributes())->not->toHaveKey('ip');
+});
+
+test('a browser visit is counted as a view but not as a crawler fetch', function () {
+    /** @var TestCase $this */
+    makeCrawlerLogRelease();
+
+    $this->withHeaders(['User-Agent' => 'Mozilla/5.0 (Macintosh) AppleWebKit/537.36 Chrome/128.0 Safari/537.36'])
+        ->get('https://presseecho.test/de/press-release/crawler-protokoll-pm')
+        ->assertSuccessful();
+
+    expect(PressReleaseCrawlerHit::query()->count())->toBe(0);
+});
+
+test('the purge command removes crawler hits older than the retention period', function () {
+    /** @var TestCase $this */
+    $release = makeCrawlerLogRelease();
+    AppSetting::setValue(AggregatePressReleaseViews::WatermarkKey, '0');
+
+    $old = PressReleaseCrawlerHit::factory()->create(['press_release_id' => $release->id]);
+    $old->forceFill(['created_at' => now()->subDays(400)])->saveQuietly();
+    PressReleaseCrawlerHit::factory()->create(['press_release_id' => $release->id]);
+
+    $this->artisan(PurgePressReleaseEvents::class, ['--days' => 395])
+        ->expectsOutputToContain('Crawler-Abruf-Protokoll bereinigt: 1 Einträge gelöscht.')
+        ->assertSuccessful();
+
+    expect(PressReleaseCrawlerHit::query()->count())->toBe(1);
+});