presseportale/tests/Feature/Web/CrawlerHitLogTest.php

<?php

use App\Console\Commands\AggregatePressReleaseViews;
use App\Console\Commands\PurgePressReleaseEvents;
use App\Enums\Portal;
use App\Models\AppSetting;
use App\Models\Category;
use App\Models\Company;
use App\Models\PressRelease;
use App\Models\PressReleaseCrawlerHit;
use App\Services\PressRelease\CrawlerHitReport;
use App\Support\CrawlerIdentifier;
use Tests\TestCase;

/**
 * Crawler-Abruf-Protokoll (Roadmap 1c.0): Abrufe durch bekannte Such- und
 * KI-Dienste werden je Meldung mit Dienst, User-Agent und Zeitpunkt
 * gespeichert — ohne IP; Browser und unbekannte Bots nicht.
 */
function makeCrawlerLogRelease(): PressRelease
{
    $category = Category::factory()->create(['portal' => Portal::Presseecho]);
    $category->translations()->create(['locale' => 'de', 'name' => 'Handel', 'slug' => 'handel']);
    $company = Company::factory()->create(['name' => 'Crawler GmbH', 'portal' => Portal::Presseecho]);

    return PressRelease::factory()
        ->published()
        ->forPortal(Portal::Presseecho)
        ->for($category)
        ->for($company)
        ->create(['slug' => 'crawler-protokoll-pm', 'published_at' => now()->subDay()]);
}

test('the identifier maps known user agents to a service and ignores browsers', function (?string $userAgent, ?string $expected) {
    expect(CrawlerIdentifier::identify($userAgent))->toBe($expected);
})->with([
    ['Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)', 'openai'],
    ['Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)', 'openai'],
    ['Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)', 'anthropic'],
    ['Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)', 'perplexity'],
    ['Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', 'google'],
    ['Mozilla/5.0 (compatible; Google-Extended)', 'google-extended'],
    ['Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)', 'bing'],
    ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/128.0 Safari/537.36', null],
    ['curl/8.4.0', null],
    ['', null],
    [null, null],
]);

test('a fetch by a known crawler is logged for the release without an IP address', function () {
    /** @var TestCase $this */
    $release = makeCrawlerLogRelease();

    $this->withHeaders(['User-Agent' => 'Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)'])
        ->get('https://presseecho.test/de/press-release/crawler-protokoll-pm')
        ->assertSuccessful();

    $hit = PressReleaseCrawlerHit::query()->sole();

    expect($hit->press_release_id)->toBe($release->id)
        ->and($hit->service)->toBe('anthropic')
        ->and($hit->user_agent)->toContain('ClaudeBot')
        ->and($hit->getAttributes())->not->toHaveKey('ip');
});

test('a browser visit is counted as a view but not as a crawler fetch', function () {
    /** @var TestCase $this */
    makeCrawlerLogRelease();

    $this->withHeaders(['User-Agent' => 'Mozilla/5.0 (Macintosh) AppleWebKit/537.36 Chrome/128.0 Safari/537.36'])
        ->get('https://presseecho.test/de/press-release/crawler-protokoll-pm')
        ->assertSuccessful();

    expect(PressReleaseCrawlerHit::query()->count())->toBe(0);
});

test('the purge command removes crawler hits older than the retention period', function () {
    /** @var TestCase $this */
    $release = makeCrawlerLogRelease();
    AppSetting::setValue(AggregatePressReleaseViews::WatermarkKey, '0');

    $old = PressReleaseCrawlerHit::factory()->create(['press_release_id' => $release->id]);
    $old->forceFill(['created_at' => now()->subDays(400)])->saveQuietly();
    PressReleaseCrawlerHit::factory()->create(['press_release_id' => $release->id]);

    $this->artisan(PurgePressReleaseEvents::class, ['--days' => 395])
        ->expectsOutputToContain('Crawler-Abruf-Protokoll bereinigt: 1 Einträge gelöscht.')
        ->assertSuccessful();

    expect(PressReleaseCrawlerHit::query()->count())->toBe(1);
});

test('the crawler report aggregates fetches by service, by day and by release', function () {
    /** @var TestCase $this */
    $release = makeCrawlerLogRelease();
    $other = PressRelease::factory()->published()->forPortal(Portal::Presseecho)->create(['title' => 'Zweite Meldung', 'published_at' => now()->subDay()]);

    PressReleaseCrawlerHit::factory()->count(3)->create(['press_release_id' => $release->id]);
    PressReleaseCrawlerHit::factory()->service('google', 'Googlebot/2.1')->create(['press_release_id' => $release->id]);
    PressReleaseCrawlerHit::factory()->service('google', 'Googlebot/2.1')->create(['press_release_id' => $other->id]);
    $old = PressReleaseCrawlerHit::factory()->service('bing', 'bingbot/2.0')->create(['press_release_id' => $other->id]);
    $old->forceFill(['created_at' => now()->subDays(40)])->saveQuietly();

    $report = app(CrawlerHitReport::class);
    $byService = collect($report->byService(30))->keyBy('service');

    expect($byService->keys()->all())->toBe(['openai', 'google', 'bing'])
        ->and($byService['openai']['fetches'])->toBe(3)
        ->and($byService['openai']['releases'])->toBe(1)
        ->and($byService['google']['fetches'])->toBe(2)
        ->and($byService['google']['releases'])->toBe(2)
        ->and($byService['google']['kind'])->toBe('search')
        ->and($byService['bing']['fetches'])->toBe(0)
        ->and($byService['bing']['fetches_total'])->toBe(1);

    $daily = $report->daily(14);
    expect($daily)->toHaveCount(14)
        ->and($daily[13]['ai'])->toBe(3)
        ->and($daily[13]['search'])->toBe(2)
        ->and($daily[0]['ai'] + $daily[0]['search'])->toBe(0);

    $top = $report->topReleases(30);
    expect($top[0]['press_release_id'])->toBe($release->id)
        ->and($top[0]['fetches'])->toBe(4)
        ->and($top[0]['services'])->toBe(2);
});