create(['portal' => Portal::Presseecho]); $category->translations()->create(['locale' => 'de', 'name' => 'Handel', 'slug' => 'handel']); $company = Company::factory()->create(['name' => 'Crawler GmbH', 'portal' => Portal::Presseecho]); return PressRelease::factory() ->published() ->forPortal(Portal::Presseecho) ->for($category) ->for($company) ->create(['slug' => 'crawler-protokoll-pm', 'published_at' => now()->subDay()]); } test('the identifier maps known user agents to a service and ignores browsers', function (?string $userAgent, ?string $expected) { expect(CrawlerIdentifier::identify($userAgent))->toBe($expected); })->with([ ['Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)', 'openai'], ['Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)', 'openai'], ['Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)', 'anthropic'], ['Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)', 'perplexity'], ['Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', 'google'], ['Mozilla/5.0 (compatible; Google-Extended)', 'google-extended'], ['Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)', 'bing'], ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/128.0 Safari/537.36', null], ['curl/8.4.0', null], ['', null], [null, null], ]); test('a fetch by a known crawler is logged for the release without an IP address', function () { /** @var TestCase $this */ $release = makeCrawlerLogRelease(); $this->withHeaders(['User-Agent' => 'Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)']) ->get('https://presseecho.test/de/press-release/crawler-protokoll-pm') ->assertSuccessful(); $hit = PressReleaseCrawlerHit::query()->sole(); expect($hit->press_release_id)->toBe($release->id) ->and($hit->service)->toBe('anthropic') ->and($hit->user_agent)->toContain('ClaudeBot') ->and($hit->getAttributes())->not->toHaveKey('ip'); }); test('a browser visit is counted as a view but not as a crawler fetch', function () { /** @var TestCase $this */ makeCrawlerLogRelease(); $this->withHeaders(['User-Agent' => 'Mozilla/5.0 (Macintosh) AppleWebKit/537.36 Chrome/128.0 Safari/537.36']) ->get('https://presseecho.test/de/press-release/crawler-protokoll-pm') ->assertSuccessful(); expect(PressReleaseCrawlerHit::query()->count())->toBe(0); }); test('the purge command removes crawler hits older than the retention period', function () { /** @var TestCase $this */ $release = makeCrawlerLogRelease(); AppSetting::setValue(AggregatePressReleaseViews::WatermarkKey, '0'); $old = PressReleaseCrawlerHit::factory()->create(['press_release_id' => $release->id]); $old->forceFill(['created_at' => now()->subDays(400)])->saveQuietly(); PressReleaseCrawlerHit::factory()->create(['press_release_id' => $release->id]); $this->artisan(PurgePressReleaseEvents::class, ['--days' => 395]) ->expectsOutputToContain('Crawler-Abruf-Protokoll bereinigt: 1 Einträge gelöscht.') ->assertSuccessful(); expect(PressReleaseCrawlerHit::query()->count())->toBe(1); }); test('the crawler report aggregates fetches by service, by day and by release', function () { /** @var TestCase $this */ $release = makeCrawlerLogRelease(); $other = PressRelease::factory()->published()->forPortal(Portal::Presseecho)->create(['title' => 'Zweite Meldung', 'published_at' => now()->subDay()]); PressReleaseCrawlerHit::factory()->count(3)->create(['press_release_id' => $release->id]); PressReleaseCrawlerHit::factory()->service('google', 'Googlebot/2.1')->create(['press_release_id' => $release->id]); PressReleaseCrawlerHit::factory()->service('google', 'Googlebot/2.1')->create(['press_release_id' => $other->id]); $old = PressReleaseCrawlerHit::factory()->service('bing', 'bingbot/2.0')->create(['press_release_id' => $other->id]); $old->forceFill(['created_at' => now()->subDays(40)])->saveQuietly(); $report = app(CrawlerHitReport::class); $byService = collect($report->byService(30))->keyBy('service'); expect($byService->keys()->all())->toBe(['openai', 'google', 'bing']) ->and($byService['openai']['fetches'])->toBe(3) ->and($byService['openai']['releases'])->toBe(1) ->and($byService['google']['fetches'])->toBe(2) ->and($byService['google']['releases'])->toBe(2) ->and($byService['google']['kind'])->toBe('search') ->and($byService['bing']['fetches'])->toBe(0) ->and($byService['bing']['fetches_total'])->toBe(1); $daily = $report->daily(14); expect($daily)->toHaveCount(14) ->and($daily[13]['ai'])->toBe(3) ->and($daily[13]['search'])->toBe(2) ->and($daily[0]['ai'] + $daily[0]['search'])->toBe(0); $top = $report->topReleases(30); expect($top[0]['press_release_id'])->toBe($release->id) ->and($top[0]['fetches'])->toBe(4) ->and($top[0]['services'])->toBe(2); });