From 951b6c47fa872bc87ce6f466e3cb5ead41cd9a91 Mon Sep 17 00:00:00 2001 From: Kevin Adametz Date: Thu, 17 Sep 2026 14:21:35 +0000 Subject: [PATCH] =?UTF-8?q?web:=20Crawler-Abruf-Protokoll=20und=20Panel=20?= =?UTF-8?q?=E2=80=9EAuslieferung"=20ohne=20Google-Link=20(Anhang=201:=201c?= =?UTF-8?q?.0,=201c.1,=201c.6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Abrufe öffentlicher Meldungen durch bekannte Such- und KI-Dienste werden je Meldung mit Dienst, gekürztem User-Agent und Zeitpunkt protokolliert — keine IP. Dienste und Muster stehen in config/crawlers.php. Aufzeichnung wie der Aufruf-Zähler per defer() und fehlertolerant; Bereinigung hängt an press-releases:purge-events. VisibilityStats liefert die Abrufe je Dienst (letzter Abruf, Anzahl) und verliert den site:-Suchlink (Stufe 1 verspricht nichts, was von Dritten abhängt). Co-Authored-By: Claude Opus 5 (1M context) --- .../Commands/PurgePressReleaseEvents.php | 17 +++ app/Models/PressRelease.php | 29 +++++ app/Models/PressReleaseCrawlerHit.php | 35 ++++++ app/Services/PressRelease/VisibilityStats.php | 53 +++++++-- app/Services/Web/ReleaseDetailPageData.php | 10 ++ app/Support/CrawlerIdentifier.php | 41 +++++++ config/crawlers.php | 104 ++++++++++++++++++ .../PressReleaseCrawlerHitFactory.php | 33 ++++++ ...reate_press_release_crawler_hits_table.php | 35 ++++++ tests/Feature/Web/CrawlerHitLogTest.php | 90 +++++++++++++++ 10 files changed, 440 insertions(+), 7 deletions(-) create mode 100644 app/Models/PressReleaseCrawlerHit.php create mode 100644 app/Support/CrawlerIdentifier.php create mode 100644 config/crawlers.php create mode 100644 database/factories/PressReleaseCrawlerHitFactory.php create mode 100644 database/migrations/2026_09_18_090000_create_press_release_crawler_hits_table.php create mode 100644 tests/Feature/Web/CrawlerHitLogTest.php diff --git a/app/Console/Commands/PurgePressReleaseEvents.php b/app/Console/Commands/PurgePressReleaseEvents.php index e5774cca..2468a653 100644 --- a/app/Console/Commands/PurgePressReleaseEvents.php +++ b/app/Console/Commands/PurgePressReleaseEvents.php @@ -3,6 +3,7 @@ namespace App\Console\Commands; use App\Models\AppSetting; +use App\Models\PressReleaseCrawlerHit; use App\Models\PressReleaseEvent; use Illuminate\Console\Command; @@ -94,6 +95,22 @@ class PurgePressReleaseEvents extends Command $days, )); + // Crawler-Abruf-Protokoll (1c.0): gleiche Aufbewahrung, keine + // Wasserstandsmarke nötig — es wird nirgends aufsummiert. + $crawlerDeleted = 0; + + while (true) { + $removed = PressReleaseCrawlerHit::query()->where('created_at', '<', $cutoff)->limit($chunk)->delete(); + + if ($removed === 0) { + break; + } + + $crawlerDeleted += $removed; + } + + $this->info(sprintf('Crawler-Abruf-Protokoll bereinigt: %d Einträge gelöscht.', $crawlerDeleted)); + return self::SUCCESS; } } diff --git a/app/Models/PressRelease.php b/app/Models/PressRelease.php index e0186fd2..e9014658 100644 --- a/app/Models/PressRelease.php +++ b/app/Models/PressRelease.php @@ -407,6 +407,35 @@ class PressRelease extends Model } } + /** + * Crawler-Abruf-Protokoll (Roadmap 1c.0). + * + * @return HasMany + */ + public function crawlerHits(): HasMany + { + return $this->hasMany(PressReleaseCrawlerHit::class); + } + + /** + * Protokolliert den Abruf durch einen bekannten Such- oder KI-Dienst + * (Roadmap 1c.0) — wie der Aufruf-Zähler nach der Response und + * fehlertolerant: das Protokoll darf die Seite nie bremsen oder brechen. + * Gespeichert werden Dienst, gekürzter User-Agent und Zeitpunkt, keine IP. + */ + public static function recordDeferredCrawlerHit(int $pressReleaseId, string $service, string $userAgent): void + { + try { + PressReleaseCrawlerHit::create([ + 'press_release_id' => $pressReleaseId, + 'service' => $service, + 'user_agent' => mb_substr($userAgent, 0, 255), + ]); + } catch (\Throwable $e) { + report($e); + } + } + /** * Versionierter Audit-Trail der KI-Deklarationen (Art. 50 KI-VO) — * jeder Submit erzeugt einen neuen, unveränderlichen Eintrag. diff --git a/app/Models/PressReleaseCrawlerHit.php b/app/Models/PressReleaseCrawlerHit.php new file mode 100644 index 00000000..0007fd5d --- /dev/null +++ b/app/Models/PressReleaseCrawlerHit.php @@ -0,0 +1,35 @@ + */ + use HasFactory; + + public const ?string UPDATED_AT = null; + + protected $fillable = [ + 'press_release_id', + 'service', + 'user_agent', + ]; + + /** + * @return BelongsTo + */ + public function pressRelease(): BelongsTo + { + return $this->belongsTo(PressRelease::class); + } +} diff --git a/app/Services/PressRelease/VisibilityStats.php b/app/Services/PressRelease/VisibilityStats.php index 2c81b6b3..800487df 100644 --- a/app/Services/PressRelease/VisibilityStats.php +++ b/app/Services/PressRelease/VisibilityStats.php @@ -8,6 +8,7 @@ use App\Enums\PressReleaseStatus; use App\Models\Company; use App\Models\PressRelease; use App\Models\PressReleaseEvent; +use App\Support\CrawlerIdentifier; use Illuminate\Support\Carbon; use Illuminate\Support\Collection; @@ -25,9 +26,14 @@ use Illuminate\Support\Collection; * Tageswert älter als die Aufbewahrung existiert nicht mehr — die * Verlaufsgrafik darf ihn nicht als „0 Aufrufe" ausgeben. * - * Der Indexierungsstatus ist in dieser Stufe bewusst ohne Fremddienst - * (Entscheidung 16.09.2026, Variante B): „in der Sitemap seit" plus ein - * Link auf die Suche. Die Search-Console-Anbindung kommt als eigenes Paket. + * Stufe 1 der Absenderansicht (Anhang 17.09.2026) verspricht nichts, was + * von Dritten abhängt: Überschrift „Auslieferung", Daten aus eigener Hand — + * veröffentlicht am, in der Sitemap seit, Aufrufe im Verlauf. Kein Link auf + * eine `site:`-Suche (unvollständig, ohne Einordnung). Der Indexierungs- + * status aus der Search Console (1c.4) läuft zunächst nur als Admin- + * Auswertung; die Kundenanzeige hängt an config search_console.customer_visible + * (E8). Abrufe durch Such- und KI-Dienste (1c.6) kommen aus dem + * Crawler-Abruf-Protokoll (1c.0) — ein Abruf ist kein Zitat. */ class VisibilityStats { @@ -124,9 +130,9 @@ class VisibilityStats } /** - * Einzelwerte einer Meldung für das Panel „Sichtbarkeit". + * Einzelwerte einer Meldung für das Panel „Auslieferung". * - * @return array{hits: int, first_view_at: Carbon|null, last_view_at: Carbon|null, shares: array, shares_total: int, in_sitemap_since: Carbon|null, search_url: string|null} + * @return array{hits: int, first_view_at: Carbon|null, last_view_at: Carbon|null, shares: array, shares_total: int, in_sitemap_since: Carbon|null, crawlers: list, crawlers_since: Carbon|null} */ public function releaseSummary(PressRelease $pressRelease): array { @@ -147,7 +153,6 @@ class VisibilityStats ->sortDesc(); $links = $pressRelease->isVisibleInArchive() ? $pressRelease->publicPortalLinks() : []; - $firstUrl = $links[0]['url'] ?? null; return [ 'hits' => (int) $pressRelease->hits, @@ -159,10 +164,44 @@ class VisibilityStats // Moment der Veröffentlichung. Das ist eine Aussage über uns, nicht // über Google. 'in_sitemap_since' => $links !== [] && $pressRelease->published_at !== null ? Carbon::instance($pressRelease->published_at) : null, - 'search_url' => $firstUrl !== null ? 'https://www.google.com/search?q='.rawurlencode('site:'.preg_replace('#^https?://#', '', $firstUrl)) : null, + 'crawlers' => $this->crawlerFetches($pressRelease), + 'crawlers_since' => $this->crawlerHitsSince(), ]; } + /** + * Abrufe durch Such- und KI-Dienste je Meldung (Roadmap 1c.6), zuletzt + * abgerufener Dienst zuerst. Belegt nur den Abruf durch den Crawler des + * Dienstes — nicht, dass die Meldung dort zitiert oder angezeigt wird. + * + * @return list + */ + public function crawlerFetches(PressRelease $pressRelease): array + { + return $pressRelease->crawlerHits() + ->selectRaw('service, count(*) as fetches, max(created_at) as last_at') + ->groupBy('service') + ->toBase() + ->get() + ->map(fn (object $row): array => [ + 'service' => (string) $row->service, + 'label' => CrawlerIdentifier::label((string) $row->service), + 'kind' => CrawlerIdentifier::kind((string) $row->service), + 'fetches' => (int) $row->fetches, + 'last_at' => Carbon::parse((string) $row->last_at), + ]) + ->sortByDesc(fn (array $row): int => $row['last_at']->getTimestamp()) + ->values() + ->all(); + } + + private function crawlerHitsSince(): ?Carbon + { + $since = config('crawlers.recorded_since'); + + return $since ? Carbon::parse((string) $since) : null; + } + /** * Ab wann es für diese Meldung überhaupt Tageswerte geben kann: der * spätere von Veröffentlichung und Beginn der Ereignisaufzeichnung. diff --git a/app/Services/Web/ReleaseDetailPageData.php b/app/Services/Web/ReleaseDetailPageData.php index bb109653..0f355da9 100644 --- a/app/Services/Web/ReleaseDetailPageData.php +++ b/app/Services/Web/ReleaseDetailPageData.php @@ -5,6 +5,7 @@ namespace App\Services\Web; use App\Models\Contact; use App\Models\PressRelease; use App\Scopes\PortalScope; +use App\Support\CrawlerIdentifier; use App\Support\WebPortalContext; use Illuminate\Database\Eloquent\Builder; @@ -86,6 +87,15 @@ class ReleaseDetailPageData $releaseId = $release->getKey(); defer(static fn () => PressRelease::recordDeferredView($releaseId)); + // Crawler-Abruf-Protokoll (Roadmap 1c.0): nur bekannte Such- und + // KI-Dienste, erkannt am User-Agent, ohne IP. + $userAgent = (string) request()->userAgent(); + $crawler = CrawlerIdentifier::identify($userAgent); + + if ($crawler !== null) { + defer(static fn () => PressRelease::recordDeferredCrawlerHit($releaseId, $crawler, $userAgent)); + } + // Die Antwort zeigt den aktuellen Aufruf bereits an; die atomaren // Schreibzugriffe erfolgen nach dem Senden der Response. $release->hits = ($release->hits ?? 0) + 1; diff --git a/app/Support/CrawlerIdentifier.php b/app/Support/CrawlerIdentifier.php new file mode 100644 index 00000000..b10499a9 --- /dev/null +++ b/app/Support/CrawlerIdentifier.php @@ -0,0 +1,41 @@ +}> $services */ + $services = config('crawlers.services', []); + + foreach ($services as $key => $service) { + foreach ($service['patterns'] as $pattern) { + if (stripos($userAgent, $pattern) !== false) { + return $key; + } + } + } + + return null; + } + + public static function label(string $service): string + { + return (string) (config('crawlers.services.'.$service.'.label') ?? $service); + } + + public static function kind(string $service): string + { + return (string) (config('crawlers.services.'.$service.'.kind') ?? 'ai'); + } +} diff --git a/config/crawlers.php b/config/crawlers.php new file mode 100644 index 00000000..f2d8b983 --- /dev/null +++ b/config/crawlers.php @@ -0,0 +1,104 @@ + env('CRAWLER_HITS_RECORDED_SINCE', '2026-09-18'), + + 'services' => [ + 'openai' => [ + 'label' => 'OpenAI / ChatGPT', + 'kind' => 'ai', + 'patterns' => ['GPTBot', 'ChatGPT-User', 'OAI-SearchBot'], + ], + 'anthropic' => [ + 'label' => 'Anthropic / Claude', + 'kind' => 'ai', + 'patterns' => ['ClaudeBot', 'Claude-User', 'Claude-SearchBot', 'Claude-Web', 'anthropic-ai'], + ], + 'perplexity' => [ + 'label' => 'Perplexity', + 'kind' => 'ai', + 'patterns' => ['PerplexityBot', 'Perplexity-User'], + ], + 'google-extended' => [ + 'label' => 'Google Gemini', + 'kind' => 'ai', + 'patterns' => ['Google-Extended'], + ], + 'google' => [ + 'label' => 'Google Suche', + 'kind' => 'search', + 'patterns' => ['Googlebot', 'GoogleOther', 'Google-InspectionTool'], + ], + 'bing' => [ + 'label' => 'Bing / Copilot', + 'kind' => 'search', + 'patterns' => ['bingbot', 'BingPreview'], + ], + 'apple' => [ + 'label' => 'Apple', + 'kind' => 'ai', + 'patterns' => ['Applebot'], + ], + 'meta' => [ + 'label' => 'Meta AI', + 'kind' => 'ai', + 'patterns' => ['meta-externalagent', 'meta-externalfetcher', 'FacebookBot'], + ], + 'amazon' => [ + 'label' => 'Amazon', + 'kind' => 'ai', + 'patterns' => ['Amazonbot'], + ], + 'mistral' => [ + 'label' => 'Mistral', + 'kind' => 'ai', + 'patterns' => ['MistralAI-User'], + ], + 'cohere' => [ + 'label' => 'Cohere', + 'kind' => 'ai', + 'patterns' => ['cohere-ai', 'cohere-training-data-crawler'], + ], + 'duckduckgo' => [ + 'label' => 'DuckDuckGo', + 'kind' => 'search', + 'patterns' => ['DuckDuckBot', 'DuckAssistBot'], + ], + 'bytedance' => [ + 'label' => 'ByteDance', + 'kind' => 'ai', + 'patterns' => ['Bytespider'], + ], + 'commoncrawl' => [ + 'label' => 'Common Crawl', + 'kind' => 'ai', + 'patterns' => ['CCBot'], + ], + 'yandex' => [ + 'label' => 'Yandex', + 'kind' => 'search', + 'patterns' => ['YandexBot'], + ], + ], + +]; diff --git a/database/factories/PressReleaseCrawlerHitFactory.php b/database/factories/PressReleaseCrawlerHitFactory.php new file mode 100644 index 00000000..2419d98f --- /dev/null +++ b/database/factories/PressReleaseCrawlerHitFactory.php @@ -0,0 +1,33 @@ + + */ +class PressReleaseCrawlerHitFactory extends Factory +{ + /** + * @return array + */ + public function definition(): array + { + return [ + 'press_release_id' => PressRelease::factory(), + 'service' => 'openai', + 'user_agent' => 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)', + ]; + } + + public function service(string $service, ?string $userAgent = null): static + { + return $this->state([ + 'service' => $service, + 'user_agent' => $userAgent ?? $service.'-crawler/1.0', + ]); + } +} diff --git a/database/migrations/2026_09_18_090000_create_press_release_crawler_hits_table.php b/database/migrations/2026_09_18_090000_create_press_release_crawler_hits_table.php new file mode 100644 index 00000000..0e8e32bc --- /dev/null +++ b/database/migrations/2026_09_18_090000_create_press_release_crawler_hits_table.php @@ -0,0 +1,35 @@ +id(); + $table->foreignId('press_release_id')->constrained()->cascadeOnDelete(); + $table->string('service', 40); + $table->string('user_agent', 255); + $table->timestamp('created_at')->nullable(); + + $table->index(['press_release_id', 'created_at'], 'prch_release_created_idx'); + $table->index(['service', 'created_at'], 'prch_service_created_idx'); + }); + } + + public function down(): void + { + Schema::dropIfExists('press_release_crawler_hits'); + } +}; diff --git a/tests/Feature/Web/CrawlerHitLogTest.php b/tests/Feature/Web/CrawlerHitLogTest.php new file mode 100644 index 00000000..7eeba034 --- /dev/null +++ b/tests/Feature/Web/CrawlerHitLogTest.php @@ -0,0 +1,90 @@ +create(['portal' => Portal::Presseecho]); + $category->translations()->create(['locale' => 'de', 'name' => 'Handel', 'slug' => 'handel']); + $company = Company::factory()->create(['name' => 'Crawler GmbH', 'portal' => Portal::Presseecho]); + + return PressRelease::factory() + ->published() + ->forPortal(Portal::Presseecho) + ->for($category) + ->for($company) + ->create(['slug' => 'crawler-protokoll-pm', 'published_at' => now()->subDay()]); +} + +test('the identifier maps known user agents to a service and ignores browsers', function (?string $userAgent, ?string $expected) { + expect(CrawlerIdentifier::identify($userAgent))->toBe($expected); +})->with([ + ['Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)', 'openai'], + ['Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)', 'openai'], + ['Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)', 'anthropic'], + ['Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)', 'perplexity'], + ['Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', 'google'], + ['Mozilla/5.0 (compatible; Google-Extended)', 'google-extended'], + ['Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)', 'bing'], + ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/128.0 Safari/537.36', null], + ['curl/8.4.0', null], + ['', null], + [null, null], +]); + +test('a fetch by a known crawler is logged for the release without an IP address', function () { + /** @var TestCase $this */ + $release = makeCrawlerLogRelease(); + + $this->withHeaders(['User-Agent' => 'Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)']) + ->get('https://presseecho.test/de/press-release/crawler-protokoll-pm') + ->assertSuccessful(); + + $hit = PressReleaseCrawlerHit::query()->sole(); + + expect($hit->press_release_id)->toBe($release->id) + ->and($hit->service)->toBe('anthropic') + ->and($hit->user_agent)->toContain('ClaudeBot') + ->and($hit->getAttributes())->not->toHaveKey('ip'); +}); + +test('a browser visit is counted as a view but not as a crawler fetch', function () { + /** @var TestCase $this */ + makeCrawlerLogRelease(); + + $this->withHeaders(['User-Agent' => 'Mozilla/5.0 (Macintosh) AppleWebKit/537.36 Chrome/128.0 Safari/537.36']) + ->get('https://presseecho.test/de/press-release/crawler-protokoll-pm') + ->assertSuccessful(); + + expect(PressReleaseCrawlerHit::query()->count())->toBe(0); +}); + +test('the purge command removes crawler hits older than the retention period', function () { + /** @var TestCase $this */ + $release = makeCrawlerLogRelease(); + AppSetting::setValue(AggregatePressReleaseViews::WatermarkKey, '0'); + + $old = PressReleaseCrawlerHit::factory()->create(['press_release_id' => $release->id]); + $old->forceFill(['created_at' => now()->subDays(400)])->saveQuietly(); + PressReleaseCrawlerHit::factory()->create(['press_release_id' => $release->id]); + + $this->artisan(PurgePressReleaseEvents::class, ['--days' => 395]) + ->expectsOutputToContain('Crawler-Abruf-Protokoll bereinigt: 1 Einträge gelöscht.') + ->assertSuccessful(); + + expect(PressReleaseCrawlerHit::query()->count())->toBe(1); +});