Kevin Adametz
951b6c47fa
Abrufe öffentlicher Meldungen durch bekannte Such- und KI-Dienste werden je Meldung mit Dienst, gekürztem User-Agent und Zeitpunkt protokolliert — keine IP. Dienste und Muster stehen in config/crawlers.php. Aufzeichnung wie der Aufruf-Zähler per defer() und fehlertolerant; Bereinigung hängt an press-releases:purge-events. VisibilityStats liefert die Abrufe je Dienst (letzter Abruf, Anzahl) und verliert den site:-Suchlink (Stufe 1 verspricht nichts, was von Dritten abhängt). Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
90 lines
3.9 KiB
PHP
90 lines
3.9 KiB
PHP
<?php
|
|
|
|
use App\Console\Commands\AggregatePressReleaseViews;
|
|
use App\Console\Commands\PurgePressReleaseEvents;
|
|
use App\Enums\Portal;
|
|
use App\Models\AppSetting;
|
|
use App\Models\Category;
|
|
use App\Models\Company;
|
|
use App\Models\PressRelease;
|
|
use App\Models\PressReleaseCrawlerHit;
|
|
use App\Support\CrawlerIdentifier;
|
|
use Tests\TestCase;
|
|
|
|
/**
|
|
* Crawler-Abruf-Protokoll (Roadmap 1c.0): Abrufe durch bekannte Such- und
|
|
* KI-Dienste werden je Meldung mit Dienst, User-Agent und Zeitpunkt
|
|
* gespeichert — ohne IP; Browser und unbekannte Bots nicht.
|
|
*/
|
|
function makeCrawlerLogRelease(): PressRelease
|
|
{
|
|
$category = Category::factory()->create(['portal' => Portal::Presseecho]);
|
|
$category->translations()->create(['locale' => 'de', 'name' => 'Handel', 'slug' => 'handel']);
|
|
$company = Company::factory()->create(['name' => 'Crawler GmbH', 'portal' => Portal::Presseecho]);
|
|
|
|
return PressRelease::factory()
|
|
->published()
|
|
->forPortal(Portal::Presseecho)
|
|
->for($category)
|
|
->for($company)
|
|
->create(['slug' => 'crawler-protokoll-pm', 'published_at' => now()->subDay()]);
|
|
}
|
|
|
|
test('the identifier maps known user agents to a service and ignores browsers', function (?string $userAgent, ?string $expected) {
|
|
expect(CrawlerIdentifier::identify($userAgent))->toBe($expected);
|
|
})->with([
|
|
['Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)', 'openai'],
|
|
['Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)', 'openai'],
|
|
['Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)', 'anthropic'],
|
|
['Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)', 'perplexity'],
|
|
['Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', 'google'],
|
|
['Mozilla/5.0 (compatible; Google-Extended)', 'google-extended'],
|
|
['Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)', 'bing'],
|
|
['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/128.0 Safari/537.36', null],
|
|
['curl/8.4.0', null],
|
|
['', null],
|
|
[null, null],
|
|
]);
|
|
|
|
test('a fetch by a known crawler is logged for the release without an IP address', function () {
|
|
/** @var TestCase $this */
|
|
$release = makeCrawlerLogRelease();
|
|
|
|
$this->withHeaders(['User-Agent' => 'Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)'])
|
|
->get('https://presseecho.test/de/press-release/crawler-protokoll-pm')
|
|
->assertSuccessful();
|
|
|
|
$hit = PressReleaseCrawlerHit::query()->sole();
|
|
|
|
expect($hit->press_release_id)->toBe($release->id)
|
|
->and($hit->service)->toBe('anthropic')
|
|
->and($hit->user_agent)->toContain('ClaudeBot')
|
|
->and($hit->getAttributes())->not->toHaveKey('ip');
|
|
});
|
|
|
|
test('a browser visit is counted as a view but not as a crawler fetch', function () {
|
|
/** @var TestCase $this */
|
|
makeCrawlerLogRelease();
|
|
|
|
$this->withHeaders(['User-Agent' => 'Mozilla/5.0 (Macintosh) AppleWebKit/537.36 Chrome/128.0 Safari/537.36'])
|
|
->get('https://presseecho.test/de/press-release/crawler-protokoll-pm')
|
|
->assertSuccessful();
|
|
|
|
expect(PressReleaseCrawlerHit::query()->count())->toBe(0);
|
|
});
|
|
|
|
test('the purge command removes crawler hits older than the retention period', function () {
|
|
/** @var TestCase $this */
|
|
$release = makeCrawlerLogRelease();
|
|
AppSetting::setValue(AggregatePressReleaseViews::WatermarkKey, '0');
|
|
|
|
$old = PressReleaseCrawlerHit::factory()->create(['press_release_id' => $release->id]);
|
|
$old->forceFill(['created_at' => now()->subDays(400)])->saveQuietly();
|
|
PressReleaseCrawlerHit::factory()->create(['press_release_id' => $release->id]);
|
|
|
|
$this->artisan(PurgePressReleaseEvents::class, ['--days' => 395])
|
|
->expectsOutputToContain('Crawler-Abruf-Protokoll bereinigt: 1 Einträge gelöscht.')
|
|
->assertSuccessful();
|
|
|
|
expect(PressReleaseCrawlerHit::query()->count())->toBe(1);
|
|
});
|