mail: zentrale Bounce-Suppression mit Versand-Gate und Circuit Breaker (Roadmap 1b.6, Schritte 1–2)
email_suppressions/email_bounces als Zustand und Ereignis-Log; ein MessageSending-Listener streicht gesperrte Empfänger aus JEDER ausgehenden Mail und setzt den Return-Path. Zweiter Aufnahmepfad: SMTP-Ablehnungen im Worker — 5.1.x sperrt den Empfänger, Relay-Ablehnungen zählen für den Circuit Breaker, der den Versand anhält statt gesunde Adressen zu sperren. X-Pk-Ref-Header auf den Mails, API-Meldungen fallen auf die Firmen-E-Mail zurück, der Bounce-Import schreibt in die zentrale Liste. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
87f95fb7a7
commit
0a658bb34b
25 changed files with 1312 additions and 4 deletions
|
|
@ -4,9 +4,11 @@ namespace App\Console\Commands;
|
|||
|
||||
use App\Enums\ActivityType;
|
||||
use App\Enums\EmailNotificationType;
|
||||
use App\Enums\EmailSuppressionReason;
|
||||
use App\Models\NewsletterSubscription;
|
||||
use App\Models\User;
|
||||
use App\Services\ActivityLog;
|
||||
use App\Services\Mail\EmailSuppressionService;
|
||||
use App\Services\Newsletter\NewsletterSyncService;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Str;
|
||||
|
|
@ -42,6 +44,7 @@ class ImportNewsletterBounces extends Command
|
|||
public function __construct(
|
||||
private readonly ActivityLog $activityLog,
|
||||
private readonly NewsletterSyncService $newsletterSync,
|
||||
private readonly EmailSuppressionService $suppressions,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
|
@ -73,6 +76,13 @@ class ImportNewsletterBounces extends Command
|
|||
$unknown = [];
|
||||
|
||||
foreach ($emails as $email) {
|
||||
// Seit 1b.6 zuerst die zentrale Sperrliste: Sie wirkt auf JEDE
|
||||
// Mailstrecke, nicht nur auf Newsletter — und auch auf Adressen,
|
||||
// die weder Konto noch Anmeldung haben (Pressekontakte, Firmen).
|
||||
if (! $dryRun) {
|
||||
$this->suppressions->suppress($email, EmailSuppressionReason::HardBounce, 'import', 'Bounce-Liste '.basename($path));
|
||||
}
|
||||
|
||||
$touchedUsers = $this->suppressUsers($email, $dryRun, $optedOut, $alreadyOptedOut);
|
||||
$touchedSubscriptions = $this->suppressSubscriptions($email, $dryRun, $unsubscribed, $alreadyUnsubscribed);
|
||||
|
||||
|
|
@ -85,6 +95,7 @@ class ImportNewsletterBounces extends Command
|
|||
}
|
||||
|
||||
$prefix = $dryRun ? '[Dry-Run] ' : '';
|
||||
$this->info(sprintf('%s%d Adressen in die Sperrliste (email_suppressions) übernommen.', $prefix, count($emails)));
|
||||
$this->info(sprintf(
|
||||
'%s%d Adressen gelesen: %d abgemeldet, %d waren bereits abgemeldet, %d ohne User-Konto.',
|
||||
$prefix,
|
||||
|
|
|
|||
36
app/Console/Commands/MailCircuitCommand.php
Normal file
36
app/Console/Commands/MailCircuitCommand.php
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Services\Mail\MailCircuitBreaker;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
/**
|
||||
* Zustand des Mail-Circuit-Breakers anzeigen oder die Pause aufheben.
|
||||
*/
|
||||
class MailCircuitCommand extends Command
|
||||
{
|
||||
protected $signature = 'mail:circuit {--reset : Pause aufheben und Zähler zurücksetzen}';
|
||||
|
||||
protected $description = 'Zeigt, ob der Mailversand angehalten ist (Circuit Breaker), und hebt die Pause auf Wunsch auf.';
|
||||
|
||||
public function handle(MailCircuitBreaker $breaker): int
|
||||
{
|
||||
if ($this->option('reset')) {
|
||||
$breaker->reset();
|
||||
$this->info('Circuit Breaker zurückgesetzt — der Versand läuft wieder. Liegengebliebene Mails: php artisan queue:retry all');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$until = $breaker->openUntil();
|
||||
|
||||
if ($until !== null) {
|
||||
$this->warn(sprintf('Versand ANGEHALTEN bis %s.', $until->format('d.m.Y H:i')));
|
||||
} else {
|
||||
$this->info(sprintf('Versand läuft. Relay-Ablehnungen im aktuellen Fenster: %d.', $breaker->pendingRejections()));
|
||||
}
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
}
|
||||
|
|
@ -28,6 +28,7 @@ enum ActivityType: string
|
|||
case Anonymization = 'anonymization';
|
||||
case Correction = 'correction';
|
||||
case ContentRemoval = 'content_removal';
|
||||
case MailSuppressed = 'mail_suppressed';
|
||||
|
||||
public function label(): string
|
||||
{
|
||||
|
|
@ -51,6 +52,7 @@ enum ActivityType: string
|
|||
self::Anonymization => 'Anonymisierung',
|
||||
self::Correction => 'Korrektur',
|
||||
self::ContentRemoval => 'Entfernung (410)',
|
||||
self::MailSuppressed => 'Mail-Sperre',
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -74,6 +76,7 @@ enum ActivityType: string
|
|||
self::Anonymization => 'red',
|
||||
self::Correction => 'blue',
|
||||
self::ContentRemoval => 'red',
|
||||
self::MailSuppressed => 'orange',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
28
app/Enums/EmailBounceKind.php
Normal file
28
app/Enums/EmailBounceKind.php
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
<?php
|
||||
|
||||
namespace App\Enums;
|
||||
|
||||
/**
|
||||
* Art eines Rückläufers. Maßgeblich ist bei DSNs `Action`, nicht der
|
||||
* Statuscode allein (Konzept `bounce-mails-2026-09-15.md`).
|
||||
*/
|
||||
enum EmailBounceKind: string
|
||||
{
|
||||
/** `Action: failed`, Status 5.x.x — Postfach existiert nicht, sofort sperren */
|
||||
case Hard = 'hard';
|
||||
|
||||
/** `Action: delayed` oder Status 4.x.x — zählen, ab dem dritten in sieben Tagen sperren */
|
||||
case Soft = 'soft';
|
||||
|
||||
/** Relay lehnt UNS ab (5.7.x, 554 Spam-Pattern, Verbindung) — kein Empfängerproblem, Circuit Breaker */
|
||||
case Relay = 'relay';
|
||||
|
||||
public function label(): string
|
||||
{
|
||||
return match ($this) {
|
||||
self::Hard => 'hart',
|
||||
self::Soft => 'weich',
|
||||
self::Relay => 'Relay',
|
||||
};
|
||||
}
|
||||
}
|
||||
21
app/Enums/EmailSuppressionReason.php
Normal file
21
app/Enums/EmailSuppressionReason.php
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
<?php
|
||||
|
||||
namespace App\Enums;
|
||||
|
||||
enum EmailSuppressionReason: string
|
||||
{
|
||||
case HardBounce = 'hard_bounce';
|
||||
|
||||
case SoftBounces = 'soft_bounces';
|
||||
|
||||
case Manual = 'manual';
|
||||
|
||||
public function label(): string
|
||||
{
|
||||
return match ($this) {
|
||||
self::HardBounce => 'Postfach existiert nicht',
|
||||
self::SoftBounces => 'wiederholt unzustellbar',
|
||||
self::Manual => 'manuell gesperrt',
|
||||
};
|
||||
}
|
||||
}
|
||||
12
app/Exceptions/MailDeliveryPausedException.php
Normal file
12
app/Exceptions/MailDeliveryPausedException.php
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
<?php
|
||||
|
||||
namespace App\Exceptions;
|
||||
|
||||
use RuntimeException;
|
||||
|
||||
/**
|
||||
* Der Circuit Breaker hat den Versand angehalten. Im Queue-Worker lässt die
|
||||
* Ausnahme den Job fehlschlagen, damit er sich später nachholen lässt —
|
||||
* ein still verworfener Magic Link wäre verloren.
|
||||
*/
|
||||
class MailDeliveryPausedException extends RuntimeException {}
|
||||
103
app/Listeners/GateOutgoingMail.php
Normal file
103
app/Listeners/GateOutgoingMail.php
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
<?php
|
||||
|
||||
namespace App\Listeners;
|
||||
|
||||
use App\Enums\ActivityType;
|
||||
use App\Exceptions\MailDeliveryPausedException;
|
||||
use App\Mail\OperationalAlertMail;
|
||||
use App\Services\ActivityLog;
|
||||
use App\Services\Mail\EmailSuppressionService;
|
||||
use App\Services\Mail\MailCircuitBreaker;
|
||||
use Illuminate\Mail\Events\MessageSending;
|
||||
use Symfony\Component\Mime\Address;
|
||||
|
||||
/**
|
||||
* Das eine Gate vor jeder ausgehenden Mail (Roadmap 1b.6). Drei Aufgaben:
|
||||
*
|
||||
* 1. Gesperrte Empfänger streichen. Bleibt kein Empfänger, geht die Mail
|
||||
* nicht raus — mit Eintrag im Aktivitätsverlauf. Damit sind Magic Links,
|
||||
* Benachrichtigungen, Newsletter und alles Künftige an EINER Stelle von
|
||||
* der Suppression erfasst; keine Strecke kann es vergessen.
|
||||
* 2. Return-Path setzen, damit Rückläufer im Bounce-Postfach landen statt
|
||||
* im Posteingang (Konzept `bounce-mails-2026-09-15.md`).
|
||||
* 3. Circuit Breaker: Ist der Versand angehalten, schlägt die Mail im
|
||||
* Worker fehl (nachholbar) und wird im Web-Kontext still verworfen.
|
||||
*
|
||||
* Betriebs-Alerts gehen immer durch — sonst erführe niemand vom Stopp.
|
||||
*
|
||||
* Die Methode heißt bewusst NICHT `handle`: Laravels Event-Discovery würde
|
||||
* sie sonst doppelt registrieren (siehe RecordMailActivity).
|
||||
*/
|
||||
class GateOutgoingMail
|
||||
{
|
||||
public function __construct(
|
||||
private readonly EmailSuppressionService $suppressions,
|
||||
private readonly MailCircuitBreaker $breaker,
|
||||
private readonly ActivityLog $activityLog,
|
||||
) {}
|
||||
|
||||
public function gate(MessageSending $event): bool
|
||||
{
|
||||
$message = $event->message;
|
||||
$mailable = $event->data['__laravel_mailable'] ?? null;
|
||||
$isAlert = $mailable === OperationalAlertMail::class;
|
||||
|
||||
if ($returnPath = config('bounces.return_path')) {
|
||||
$message->returnPath((string) $returnPath);
|
||||
}
|
||||
|
||||
if (! $isAlert && $this->breaker->isOpen()) {
|
||||
if (app()->runningInConsole()) {
|
||||
throw new MailDeliveryPausedException(sprintf(
|
||||
'Mailversand angehalten bis %s (Circuit Breaker).',
|
||||
$this->breaker->openUntil()?->format('H:i') ?? '?',
|
||||
));
|
||||
}
|
||||
|
||||
$this->activityLog->record(
|
||||
ActivityType::MailSuppressed,
|
||||
sprintf('E-Mail „%s“ verworfen: Versand angehalten (Circuit Breaker).', (string) $message->getSubject()),
|
||||
meta: ['to' => $this->addresses($message->getTo()), 'mailable' => $mailable],
|
||||
);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
$to = $this->addresses($message->getTo());
|
||||
$cc = $this->addresses($message->getCc());
|
||||
$bcc = $this->addresses($message->getBcc());
|
||||
$suppressed = $this->suppressions->suppressedAmong([...$to, ...$cc, ...$bcc]);
|
||||
|
||||
if ($suppressed === []) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$keep = fn (Address $address): bool => ! in_array(EmailSuppressionService::normalise($address->getAddress()), $suppressed, true);
|
||||
|
||||
$message->to(...array_values(array_filter($message->getTo(), $keep)));
|
||||
$message->cc(...array_values(array_filter($message->getCc(), $keep)));
|
||||
$message->bcc(...array_values(array_filter($message->getBcc(), $keep)));
|
||||
|
||||
if ($message->getTo() !== []) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$this->activityLog->record(
|
||||
ActivityType::MailSuppressed,
|
||||
sprintf('E-Mail „%s“ an %s nicht versendet: Adresse gesperrt.', (string) $message->getSubject(), implode(', ', $suppressed)),
|
||||
actorLabel: $suppressed[0],
|
||||
meta: array_filter(['to' => $to, 'suppressed' => $suppressed, 'mailable' => $mailable]),
|
||||
);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Address[] $addresses
|
||||
* @return list<string>
|
||||
*/
|
||||
private function addresses(array $addresses): array
|
||||
{
|
||||
return array_values(array_map(fn (Address $address): string => $address->getAddress(), $addresses));
|
||||
}
|
||||
}
|
||||
105
app/Listeners/RecordSmtpFailure.php
Normal file
105
app/Listeners/RecordSmtpFailure.php
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
<?php
|
||||
|
||||
namespace App\Listeners;
|
||||
|
||||
use App\Enums\EmailBounceKind;
|
||||
use App\Exceptions\MailDeliveryPausedException;
|
||||
use App\Services\Mail\EmailSuppressionService;
|
||||
use App\Services\Mail\MailCircuitBreaker;
|
||||
use App\Services\Mail\SmtpFailureClassifier;
|
||||
use Illuminate\Mail\SendQueuedMailable;
|
||||
use Illuminate\Queue\Events\JobFailed;
|
||||
use Symfony\Component\Mailer\Exception\TransportExceptionInterface;
|
||||
use Throwable;
|
||||
|
||||
/**
|
||||
* Zweiter Pfad der Bounce-Behandlung: Der Queue-Job einer Mail ist an einer
|
||||
* SMTP-Ablehnung gescheitert. Harte Empfängerfehler (5.1.x) sperren die
|
||||
* Adresse sofort; Ablehnungen des Relays zählen für den Circuit Breaker.
|
||||
*
|
||||
* Die Methode heißt bewusst NICHT `handle` (Event-Discovery, siehe
|
||||
* RecordMailActivity).
|
||||
*/
|
||||
class RecordSmtpFailure
|
||||
{
|
||||
public function __construct(
|
||||
private readonly SmtpFailureClassifier $classifier,
|
||||
private readonly EmailSuppressionService $suppressions,
|
||||
private readonly MailCircuitBreaker $breaker,
|
||||
) {}
|
||||
|
||||
public function record(JobFailed $event): void
|
||||
{
|
||||
$exception = $event->exception;
|
||||
|
||||
if ($exception instanceof MailDeliveryPausedException || ! $this->isMailFailure($exception)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$recipients = $this->recipientsOf($event);
|
||||
$result = $this->classifier->classify($exception->getMessage(), $recipients);
|
||||
|
||||
if ($result['kind'] === EmailBounceKind::Relay) {
|
||||
$this->breaker->recordRelayRejection($exception->getMessage());
|
||||
|
||||
if ($result['email'] !== null) {
|
||||
$this->suppressions->recordBounce($result['email'], EmailBounceKind::Relay, 'smtp', 'smtp:'.$event->job->uuid(), [
|
||||
'status' => $result['status'],
|
||||
'diagnostic' => $exception->getMessage(),
|
||||
]);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$this->suppressions->recordBounce((string) $result['email'], $result['kind'], 'smtp', 'smtp:'.$event->job->uuid(), [
|
||||
'action' => 'failed',
|
||||
'status' => $result['status'],
|
||||
'diagnostic' => $exception->getMessage(),
|
||||
]);
|
||||
}
|
||||
|
||||
private function isMailFailure(Throwable $exception): bool
|
||||
{
|
||||
for ($e = $exception; $e !== null; $e = $e->getPrevious()) {
|
||||
if ($e instanceof TransportExceptionInterface) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Empfänger aus dem serialisierten Job — derselbe Payload, den der Worker
|
||||
* ohnehin entpackt hat. Nur der Rückfall, wenn die Fehlermeldung die
|
||||
* Adresse nicht nennt.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
private function recipientsOf(JobFailed $event): array
|
||||
{
|
||||
try {
|
||||
$command = $event->job->payload()['data']['command'] ?? null;
|
||||
|
||||
if (! is_string($command)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$job = unserialize($command);
|
||||
|
||||
if (! $job instanceof SendQueuedMailable) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$to = $job->mailable->to ?? [];
|
||||
|
||||
return array_values(array_filter(array_map(
|
||||
fn (mixed $entry): ?string => is_array($entry) ? ($entry['address'] ?? null) : null,
|
||||
is_array($to) ? $to : [],
|
||||
)));
|
||||
} catch (Throwable) {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -34,9 +34,20 @@ trait Unsubscribable
|
|||
|
||||
public function headers(): Headers
|
||||
{
|
||||
return new Headers(text: [
|
||||
return new Headers(text: array_filter([
|
||||
'List-Unsubscribe' => '<'.$this->unsubscribeUrl.'>',
|
||||
'List-Unsubscribe-Post' => 'List-Unsubscribe=One-Click',
|
||||
]);
|
||||
// Bounce-Referenz (Roadmap 1b.6), sofern die Mailable eine hat.
|
||||
(string) config('bounces.reference_header', 'X-Pk-Ref') => $this->bounceReference(),
|
||||
]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Ein DSN bringt die Original-Header zurück; hiermit ordnet der
|
||||
* Bounce-Parser den Rückläufer dem Datensatz zu. Mailables überschreiben.
|
||||
*/
|
||||
public function bounceReference(): ?string
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
|
|||
use Illuminate\Mail\Mailable;
|
||||
use Illuminate\Mail\Mailables\Content;
|
||||
use Illuminate\Mail\Mailables\Envelope;
|
||||
use Illuminate\Mail\Mailables\Headers;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
|
||||
class MagicLoginLink extends Mailable implements ShouldQueue
|
||||
|
|
@ -20,6 +21,15 @@ class MagicLoginLink extends Mailable implements ShouldQueue
|
|||
public string $expiresAt
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Eigener Header je Mail: Ein DSN bringt die Original-Header zurück, so
|
||||
* ordnet der Bounce-Parser den Rückläufer dem Datensatz zu (Roadmap 1b.6).
|
||||
*/
|
||||
public function headers(): Headers
|
||||
{
|
||||
return new Headers(text: [(string) config('bounces.reference_header', 'X-Pk-Ref') => 'user:'.$this->user->id]);
|
||||
}
|
||||
|
||||
public function envelope(): Envelope
|
||||
{
|
||||
return new Envelope(
|
||||
|
|
|
|||
|
|
@ -84,6 +84,16 @@ class PressReleasePublished extends Mailable
|
|||
return self::tagged(sprintf('%s/%s/newsroom/%s/verwalten', $host, $edition, $company->slug), $pressRelease->id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounce-Referenz: Ein DSN bringt die Original-Header zurück, so ordnet
|
||||
* der Parser den Rückläufer der Meldung zu (Roadmap 1b.6). Gesetzt über
|
||||
* den gemeinsamen headers() des Unsubscribable-Traits.
|
||||
*/
|
||||
public function bounceReference(): string
|
||||
{
|
||||
return sprintf('pm:%d:company:%s', $this->pressRelease->id, $this->pressRelease->company_id ?? '-');
|
||||
}
|
||||
|
||||
public function envelope(): Envelope
|
||||
{
|
||||
return new Envelope(
|
||||
|
|
|
|||
|
|
@ -46,6 +46,16 @@ class PressReleaseRejected extends Mailable
|
|||
$this->configureUnsubscribe($user, EmailNotificationType::PressReleaseRejected);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounce-Referenz: Ein DSN bringt die Original-Header zurück, so ordnet
|
||||
* der Parser den Rückläufer der Meldung zu (Roadmap 1b.6). Gesetzt über
|
||||
* den gemeinsamen headers() des Unsubscribable-Traits.
|
||||
*/
|
||||
public function bounceReference(): string
|
||||
{
|
||||
return sprintf('pm:%d:company:%s', $this->pressRelease->id, $this->pressRelease->company_id ?? '-');
|
||||
}
|
||||
|
||||
public function envelope(): Envelope
|
||||
{
|
||||
return new Envelope(
|
||||
|
|
|
|||
38
app/Models/EmailBounce.php
Normal file
38
app/Models/EmailBounce.php
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use App\Enums\EmailBounceKind;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
|
||||
/**
|
||||
* Ein Rückläufer-Ereignis (DSN aus dem Postfach, SMTP-Ablehnung im Worker,
|
||||
* Import). Idempotent über `message_id`.
|
||||
*/
|
||||
class EmailBounce extends Model
|
||||
{
|
||||
public const UPDATED_AT = null;
|
||||
|
||||
protected $fillable = [
|
||||
'message_id',
|
||||
'email',
|
||||
'kind',
|
||||
'source',
|
||||
'action',
|
||||
'status',
|
||||
'diagnostic',
|
||||
'reference',
|
||||
'received_at',
|
||||
];
|
||||
|
||||
/**
|
||||
* @return array<string, string>
|
||||
*/
|
||||
protected function casts(): array
|
||||
{
|
||||
return [
|
||||
'kind' => EmailBounceKind::class,
|
||||
'received_at' => 'datetime',
|
||||
];
|
||||
}
|
||||
}
|
||||
71
app/Models/EmailSuppression.php
Normal file
71
app/Models/EmailSuppression.php
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use App\Enums\EmailSuppressionReason;
|
||||
use Illuminate\Database\Eloquent\Builder;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Database\Eloquent\Relations\BelongsTo;
|
||||
|
||||
/**
|
||||
* Zustand „nicht erreichbar" je E-Mail-Adresse (Roadmap 1b.6). Geschrieben
|
||||
* nur über `EmailSuppressionService`, gelesen vom Versand-Gate, der
|
||||
* Claim-Seite und dem Admin.
|
||||
*/
|
||||
class EmailSuppression extends Model
|
||||
{
|
||||
protected $fillable = [
|
||||
'email',
|
||||
'reason',
|
||||
'source',
|
||||
'hard_bounce_count',
|
||||
'soft_bounce_count',
|
||||
'first_bounced_at',
|
||||
'last_bounced_at',
|
||||
'suppressed_at',
|
||||
'released_at',
|
||||
'released_by_user_id',
|
||||
'note',
|
||||
];
|
||||
|
||||
/**
|
||||
* @return array<string, string>
|
||||
*/
|
||||
protected function casts(): array
|
||||
{
|
||||
return [
|
||||
'reason' => EmailSuppressionReason::class,
|
||||
'first_bounced_at' => 'datetime',
|
||||
'last_bounced_at' => 'datetime',
|
||||
'suppressed_at' => 'datetime',
|
||||
'released_at' => 'datetime',
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return BelongsTo<User, $this>
|
||||
*/
|
||||
public function releasedBy(): BelongsTo
|
||||
{
|
||||
return $this->belongsTo(User::class, 'released_by_user_id');
|
||||
}
|
||||
|
||||
/**
|
||||
* Gesperrt heißt: gesetzt und nicht freigegeben. Eine Freigabe hebt die
|
||||
* Sperre auf; ein späterer Bounce setzt `suppressed_at` neu und löscht
|
||||
* die Freigabe wieder — der Verlauf steht im Aktivitätslog.
|
||||
*/
|
||||
public function isActive(): bool
|
||||
{
|
||||
return $this->suppressed_at !== null && $this->released_at === null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Builder<EmailSuppression> $query
|
||||
* @return Builder<EmailSuppression>
|
||||
*/
|
||||
public function scopeActive(Builder $query): Builder
|
||||
{
|
||||
return $query->whereNotNull('suppressed_at')->whereNull('released_at');
|
||||
}
|
||||
}
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
namespace App\Providers;
|
||||
|
||||
use App\Contracts\NewsletterSyncClient;
|
||||
use App\Exceptions\MailDeliveryPausedException;
|
||||
use App\Helpers\ThemeHelper;
|
||||
use App\Http\Middleware\BlockAdminDuringImpersonation;
|
||||
use App\Http\Middleware\EnsureUserIsAdmin;
|
||||
|
|
@ -10,8 +11,10 @@ use App\Http\Middleware\EnsureUserIsCustomer;
|
|||
use App\Http\Middleware\LogSlowAdminRequests;
|
||||
use App\Http\Middleware\SetEdition;
|
||||
use App\Listeners\ActivateUserAfterVerification;
|
||||
use App\Listeners\GateOutgoingMail;
|
||||
use App\Listeners\RecordAuthActivity;
|
||||
use App\Listeners\RecordMailActivity;
|
||||
use App\Listeners\RecordSmtpFailure;
|
||||
use App\Listeners\SyncCompanyMembershipsOnLogin;
|
||||
use App\Models\AdminPreset;
|
||||
use App\Models\Category;
|
||||
|
|
@ -41,6 +44,7 @@ use Illuminate\Auth\Notifications\ResetPassword;
|
|||
use Illuminate\Auth\Notifications\VerifyEmail;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Database\Events\QueryExecuted;
|
||||
use Illuminate\Mail\Events\MessageSending;
|
||||
use Illuminate\Mail\Events\MessageSent;
|
||||
use Illuminate\Notifications\Messages\MailMessage;
|
||||
use Illuminate\Queue\Events\JobFailed;
|
||||
|
|
@ -123,6 +127,17 @@ class AppServiceProvider extends ServiceProvider
|
|||
// laufen auf der `classification`-Queue — ein stiller Stau in
|
||||
// `failed_jobs` würde jede Einreichung in der Prüfung hängen lassen.
|
||||
Queue::failing(function (JobFailed $event): void {
|
||||
// Bounce-Pfad 2: SMTP-Ablehnung im Worker → Sperre bzw. Circuit
|
||||
// Breaker (Roadmap 1b.6). Läuft VOR dem Alert, damit der Breaker
|
||||
// seinen eigenen, einmaligen Alert setzen kann.
|
||||
app(RecordSmtpFailure::class)->record($event);
|
||||
|
||||
// Der Breaker hat den Versand angehalten: Das ist EIN Alert (beim
|
||||
// Öffnen), nicht einer je liegengebliebener Mail.
|
||||
if ($event->exception instanceof MailDeliveryPausedException) {
|
||||
return;
|
||||
}
|
||||
|
||||
OperationalAlert::send("Queue-Job fehlgeschlagen: {$event->job->resolveName()}", [
|
||||
'connection' => $event->connectionName,
|
||||
'queue' => $event->job->getQueue(),
|
||||
|
|
@ -139,6 +154,10 @@ class AppServiceProvider extends ServiceProvider
|
|||
Event::listen(Registered::class, [RecordAuthActivity::class, 'recordRegistered']);
|
||||
Event::listen(Login::class, [RecordAuthActivity::class, 'recordLogin']);
|
||||
|
||||
// Das eine Gate vor jeder ausgehenden Mail: Suppression, Return-Path,
|
||||
// Circuit Breaker (Roadmap 1b.6). Gibt false zurück → Mail geht nicht raus.
|
||||
Event::listen(MessageSending::class, [GateOutgoingMail::class, 'gate']);
|
||||
|
||||
// Jede tatsächlich versendete E-Mail landet ebenfalls im Verlauf.
|
||||
Event::listen(MessageSent::class, [RecordMailActivity::class, 'recordSent']);
|
||||
|
||||
|
|
|
|||
212
app/Services/Mail/EmailSuppressionService.php
Normal file
212
app/Services/Mail/EmailSuppressionService.php
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Mail;
|
||||
|
||||
use App\Enums\ActivityType;
|
||||
use App\Enums\EmailBounceKind;
|
||||
use App\Enums\EmailSuppressionReason;
|
||||
use App\Models\EmailBounce;
|
||||
use App\Models\EmailSuppression;
|
||||
use App\Models\User;
|
||||
use App\Services\ActivityLog;
|
||||
use Illuminate\Database\UniqueConstraintViolationException;
|
||||
use Illuminate\Support\Carbon;
|
||||
|
||||
/**
|
||||
* Die eine Stelle, die „nicht erreichbar" bedeutet (Roadmap 1b.6).
|
||||
*
|
||||
* Schreibt Rückläufer-Ereignisse nach `email_bounces` und leitet daraus den
|
||||
* Zustand in `email_suppressions` ab: harter Bounce → sofort gesperrt,
|
||||
* weiche → ab dem dritten in sieben Tagen. Gelesen vom Versand-Gate
|
||||
* (GateOutgoingMail), von der Claim-Seite und vom Admin.
|
||||
*
|
||||
* Adressen werden normalisiert wie überall im Projekt: Kleinschreibung,
|
||||
* getrimmt. Bounce-Listen kommen aus dem Mailserver-Log und geben die
|
||||
* Adresse so wieder, wie der Absender sie geschrieben hat.
|
||||
*/
|
||||
class EmailSuppressionService
|
||||
{
|
||||
public function __construct(private readonly ActivityLog $activityLog) {}
|
||||
|
||||
public static function normalise(?string $email): ?string
|
||||
{
|
||||
$email = mb_strtolower(trim((string) $email));
|
||||
|
||||
return $email === '' ? null : $email;
|
||||
}
|
||||
|
||||
public function isSuppressed(?string $email): bool
|
||||
{
|
||||
$key = self::normalise($email);
|
||||
|
||||
if ($key === null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return EmailSuppression::query()->active()->where('email', $key)->exists();
|
||||
}
|
||||
|
||||
/**
|
||||
* Welche dieser Adressen sind gesperrt? Eine Abfrage für alle Empfänger
|
||||
* einer Mail.
|
||||
*
|
||||
* @param list<string> $emails
|
||||
* @return list<string> normalisiert
|
||||
*/
|
||||
public function suppressedAmong(array $emails): array
|
||||
{
|
||||
$keys = array_values(array_filter(array_map(self::normalise(...), $emails)));
|
||||
|
||||
if ($keys === []) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return EmailSuppression::query()->active()->whereIn('email', $keys)->pluck('email')->all();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ein Rückläufer-Ereignis. Idempotent über die Message-ID: Kommt dieselbe
|
||||
* Mail ein zweites Mal (abgebrochener Abhol-Lauf), passiert nichts.
|
||||
*
|
||||
* @param array{action?: string|null, status?: string|null, diagnostic?: string|null, reference?: string|null, received_at?: Carbon|null} $details
|
||||
* @return EmailBounce|null null, wenn das Ereignis schon bekannt war
|
||||
*/
|
||||
public function recordBounce(string $email, EmailBounceKind $kind, string $source, string $messageId, array $details = []): ?EmailBounce
|
||||
{
|
||||
$key = self::normalise($email);
|
||||
|
||||
if ($key === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
$bounce = EmailBounce::query()->create([
|
||||
'message_id' => mb_substr($messageId, 0, 255),
|
||||
'email' => $key,
|
||||
'kind' => $kind,
|
||||
'source' => $source,
|
||||
'action' => isset($details['action']) ? mb_substr($details['action'], 0, 32) : null,
|
||||
'status' => isset($details['status']) ? mb_substr($details['status'], 0, 16) : null,
|
||||
'diagnostic' => isset($details['diagnostic']) ? mb_substr($details['diagnostic'], 0, 500) : null,
|
||||
'reference' => isset($details['reference']) ? mb_substr($details['reference'], 0, 120) : null,
|
||||
'received_at' => $details['received_at'] ?? now(),
|
||||
]);
|
||||
} catch (UniqueConstraintViolationException) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if ($kind === EmailBounceKind::Relay) {
|
||||
// Kein Empfängerproblem — nur protokolliert, der Circuit Breaker
|
||||
// zählt es separat.
|
||||
return $bounce;
|
||||
}
|
||||
|
||||
$this->applyBounce($key, $kind, $source, $bounce);
|
||||
|
||||
return $bounce;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sperre von Hand oder aus einer Bounce-Liste (Import).
|
||||
*/
|
||||
public function suppress(string $email, EmailSuppressionReason $reason, string $source, ?string $note = null): ?EmailSuppression
|
||||
{
|
||||
$key = self::normalise($email);
|
||||
|
||||
if ($key === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$suppression = EmailSuppression::query()->firstOrNew(['email' => $key]);
|
||||
$wasActive = $suppression->exists && $suppression->isActive();
|
||||
|
||||
$suppression->fill([
|
||||
'reason' => $reason,
|
||||
'source' => $source,
|
||||
'suppressed_at' => now(),
|
||||
'released_at' => null,
|
||||
'released_by_user_id' => null,
|
||||
'note' => $note !== null ? mb_substr($note, 0, 500) : $suppression->note,
|
||||
])->save();
|
||||
|
||||
if (! $wasActive) {
|
||||
$this->activityLog->record(
|
||||
ActivityType::MailSuppressed,
|
||||
sprintf('%s gesperrt: %s (%s)', $key, $reason->label(), $source),
|
||||
subject: $suppression,
|
||||
actorLabel: $source,
|
||||
);
|
||||
}
|
||||
|
||||
return $suppression;
|
||||
}
|
||||
|
||||
/**
|
||||
* Freigabe im Admin, wenn jemand die Adresse repariert hat. Ein späterer
|
||||
* Bounce setzt die Sperre neu.
|
||||
*/
|
||||
public function release(EmailSuppression $suppression, ?User $by = null, ?string $note = null): void
|
||||
{
|
||||
$suppression->forceFill([
|
||||
'released_at' => now(),
|
||||
'released_by_user_id' => $by?->getKey(),
|
||||
'note' => $note !== null ? mb_substr($note, 0, 500) : $suppression->note,
|
||||
])->save();
|
||||
|
||||
$this->activityLog->record(
|
||||
ActivityType::MailSuppressed,
|
||||
sprintf('%s freigegeben', $suppression->email),
|
||||
user: $by,
|
||||
subject: $suppression,
|
||||
);
|
||||
}
|
||||
|
||||
private function applyBounce(string $email, EmailBounceKind $kind, string $source, EmailBounce $bounce): void
|
||||
{
|
||||
$suppression = EmailSuppression::query()->firstOrNew(['email' => $email], [
|
||||
'reason' => EmailSuppressionReason::HardBounce,
|
||||
'source' => $source,
|
||||
]);
|
||||
$wasActive = $suppression->exists && $suppression->isActive();
|
||||
|
||||
$suppression->first_bounced_at ??= $bounce->received_at;
|
||||
$suppression->last_bounced_at = $bounce->received_at;
|
||||
|
||||
if ($kind === EmailBounceKind::Hard) {
|
||||
$suppression->hard_bounce_count++;
|
||||
$suppression->reason = EmailSuppressionReason::HardBounce;
|
||||
$suppression->source = $source;
|
||||
$suppression->suppressed_at = now();
|
||||
$suppression->released_at = null;
|
||||
$suppression->released_by_user_id = null;
|
||||
} else {
|
||||
$suppression->soft_bounce_count++;
|
||||
|
||||
$recentSoft = EmailBounce::query()
|
||||
->where('email', $email)
|
||||
->where('kind', EmailBounceKind::Soft->value)
|
||||
->where('received_at', '>=', now()->subDays((int) config('bounces.soft.window_days', 7)))
|
||||
->count();
|
||||
|
||||
if (! $wasActive && $recentSoft >= (int) config('bounces.soft.threshold', 3)) {
|
||||
$suppression->reason = EmailSuppressionReason::SoftBounces;
|
||||
$suppression->source = $source;
|
||||
$suppression->suppressed_at = now();
|
||||
$suppression->released_at = null;
|
||||
$suppression->released_by_user_id = null;
|
||||
}
|
||||
}
|
||||
|
||||
$suppression->save();
|
||||
|
||||
if (! $wasActive && $suppression->isActive()) {
|
||||
$this->activityLog->record(
|
||||
ActivityType::MailSuppressed,
|
||||
sprintf('%s gesperrt: %s (%s%s)', $email, $suppression->reason->label(), $source, $bounce->status ? ', '.$bounce->status : ''),
|
||||
subject: $suppression,
|
||||
actorLabel: $source,
|
||||
meta: array_filter(['reference' => $bounce->reference, 'diagnostic' => $bounce->diagnostic]),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
83
app/Services/Mail/MailCircuitBreaker.php
Normal file
83
app/Services/Mail/MailCircuitBreaker.php
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Mail;
|
||||
|
||||
use App\Support\OperationalAlert;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
|
||||
/**
|
||||
* Hält den Versand an, wenn das Relay UNS ablehnt (Konzept
|
||||
* `bounce-mails-2026-09-15.md`: „bleibt Pflicht").
|
||||
*
|
||||
* Eine 554-Spam-Pattern-Ablehnung oder eine abgerissene Verbindung ist kein
|
||||
* Empfängerproblem, sondern ein Absenderproblem — und jede weitere Mail in
|
||||
* dem Zustand macht es schlimmer. Ab `threshold` Ablehnungen in
|
||||
* `window_minutes` ist der Kreis für `pause_minutes` offen: Queue-Mails
|
||||
* schlagen fehl (und lassen sich danach mit `queue:retry` nachholen),
|
||||
* synchrone Mails werden still verworfen, der Admin bekommt genau eine Mail.
|
||||
*
|
||||
* Betriebs-Alerts gehen immer durch — sonst erführe niemand vom Stopp.
|
||||
*/
|
||||
class MailCircuitBreaker
|
||||
{
|
||||
private const CounterKey = 'mail.circuit.rejections';
|
||||
|
||||
private const OpenKey = 'mail.circuit.open_until';
|
||||
|
||||
public function recordRelayRejection(string $detail): void
|
||||
{
|
||||
$windowMinutes = max(1, (int) config('bounces.circuit_breaker.window_minutes', 15));
|
||||
$threshold = max(1, (int) config('bounces.circuit_breaker.threshold', 5));
|
||||
|
||||
Cache::add(self::CounterKey, 0, now()->addMinutes($windowMinutes));
|
||||
$count = (int) Cache::increment(self::CounterKey);
|
||||
|
||||
if ($count < $threshold || $this->isOpen()) {
|
||||
return;
|
||||
}
|
||||
|
||||
$pauseMinutes = max(1, (int) config('bounces.circuit_breaker.pause_minutes', 30));
|
||||
$until = now()->addMinutes($pauseMinutes);
|
||||
|
||||
Cache::put(self::OpenKey, $until->toIso8601String(), $until);
|
||||
Cache::forget(self::CounterKey);
|
||||
|
||||
OperationalAlert::send(
|
||||
sprintf('Mailversand angehalten bis %s: %d Relay-Ablehnungen in %d Minuten', $until->format('H:i'), $count, $windowMinutes),
|
||||
[
|
||||
'last_rejection' => $detail,
|
||||
'hint' => 'Absenderruf/Relay prüfen. Fehlgeschlagene Queue-Mails danach mit `php artisan queue:retry all` nachholen; `php artisan mail:circuit --reset` hebt die Pause vorzeitig auf.',
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
public function isOpen(): bool
|
||||
{
|
||||
return $this->openUntil() !== null;
|
||||
}
|
||||
|
||||
public function openUntil(): ?Carbon
|
||||
{
|
||||
$until = Cache::get(self::OpenKey);
|
||||
|
||||
if (! is_string($until)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$carbon = Carbon::parse($until);
|
||||
|
||||
return $carbon->isFuture() ? $carbon : null;
|
||||
}
|
||||
|
||||
public function reset(): void
|
||||
{
|
||||
Cache::forget(self::OpenKey);
|
||||
Cache::forget(self::CounterKey);
|
||||
}
|
||||
|
||||
public function pendingRejections(): int
|
||||
{
|
||||
return (int) Cache::get(self::CounterKey, 0);
|
||||
}
|
||||
}
|
||||
59
app/Services/Mail/SmtpFailureClassifier.php
Normal file
59
app/Services/Mail/SmtpFailureClassifier.php
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Mail;
|
||||
|
||||
use App\Enums\EmailBounceKind;
|
||||
|
||||
/**
|
||||
* Deutet die Fehlermeldung eines fehlgeschlagenen SMTP-Versands (Symfony
|
||||
* `TransportException`). Zweiter Pfad des Konzepts: Manche Server weisen
|
||||
* schon in der Sitzung ab statt einen DSN zu schicken.
|
||||
*
|
||||
* Konservativ, denn hier steht kein strukturierter Zustellbericht:
|
||||
*
|
||||
* - Nur `5.1.x` mit erkennbarer Empfängeradresse ist ein harter Bounce.
|
||||
* (5.1.1 = Postfach existiert nicht, 5.1.2 = Domain existiert nicht.)
|
||||
* - `4.x.x` mit Adresse ist ein weicher Bounce.
|
||||
* - Alles andere — 5.7.x Policy, 554 Spam-Pattern, Verbindungsabbruch — ist
|
||||
* eine Ablehnung des RELAYS gegen uns. Wer das als Bounce wertet, sperrt
|
||||
* bei einem Reputationsproblem reihenweise gesunde Adressen.
|
||||
*/
|
||||
class SmtpFailureClassifier
|
||||
{
|
||||
/**
|
||||
* @param list<string> $recipients Empfänger des Jobs — Rückfall, wenn die Meldung keine Adresse nennt und es genau einen gibt
|
||||
* @return array{kind: EmailBounceKind, email: string|null, status: string|null, code: int|null}
|
||||
*/
|
||||
public function classify(string $message, array $recipients = []): array
|
||||
{
|
||||
$code = preg_match('/\b([45]\d{2})\b/', $message, $m) ? (int) $m[1] : null;
|
||||
$status = preg_match('/\b([45]\.\d{1,3}\.\d{1,3})\b/', $message, $m) ? $m[1] : null;
|
||||
$email = $this->recipientIn($message) ?? (count($recipients) === 1 ? EmailSuppressionService::normalise($recipients[0]) : null);
|
||||
|
||||
$isMailboxFailure = $status !== null && str_starts_with($status, '5.1.')
|
||||
|| ($status === null && $code === 550 && preg_match('/user unknown|no such user|unknown user|does not exist|mailbox unavailable|recipient address rejected/i', $message) === 1);
|
||||
|
||||
if ($isMailboxFailure && $email !== null) {
|
||||
return ['kind' => EmailBounceKind::Hard, 'email' => $email, 'status' => $status, 'code' => $code];
|
||||
}
|
||||
|
||||
if (($status !== null && str_starts_with($status, '4.') || ($status === null && $code !== null && $code < 500)) && $email !== null) {
|
||||
return ['kind' => EmailBounceKind::Soft, 'email' => $email, 'status' => $status, 'code' => $code];
|
||||
}
|
||||
|
||||
return ['kind' => EmailBounceKind::Relay, 'email' => $email, 'status' => $status, 'code' => $code];
|
||||
}
|
||||
|
||||
private function recipientIn(string $message): ?string
|
||||
{
|
||||
if (preg_match('/<([^<>\s]+@[^<>\s]+)>/', $message, $m) === 1) {
|
||||
return EmailSuppressionService::normalise($m[1]);
|
||||
}
|
||||
|
||||
if (preg_match('/([A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,})/i', $message, $m) === 1) {
|
||||
return EmailSuppressionService::normalise($m[1]);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -21,6 +21,7 @@ use App\Models\User;
|
|||
use App\Scopes\PortalScope;
|
||||
use App\Services\ActivityLog;
|
||||
use App\Services\Admin\AdminPerformanceCache;
|
||||
use App\Services\Mail\EmailSuppressionService;
|
||||
use App\Support\WebCaches;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Auth;
|
||||
|
|
@ -40,6 +41,7 @@ class PressReleaseService
|
|||
private readonly BlacklistService $blacklist,
|
||||
private readonly AdminPerformanceCache $adminCache,
|
||||
private readonly ActivityLog $activityLog,
|
||||
private readonly EmailSuppressionService $suppressions,
|
||||
) {}
|
||||
|
||||
/**
|
||||
|
|
@ -702,12 +704,17 @@ class PressReleaseService
|
|||
$company = Company::withoutGlobalScopes()->find($pressRelease->company_id);
|
||||
$companyName = trim((string) $company?->name);
|
||||
|
||||
if ($contact) {
|
||||
// Gesperrte Adressen (Bounce-Suppression, 1b.6) überspringen:
|
||||
// Kontakt → Firmen-E-Mail → sonst gar nicht. Nie zurück an den
|
||||
// API-Einlieferer — der soll nicht zugespammt werden.
|
||||
if ($contact && ! $this->suppressions->isSuppressed($contact->email)) {
|
||||
$recipient = $contact->email;
|
||||
$recipientName = trim($contact->first_name.' '.$contact->last_name) ?: $companyName;
|
||||
} elseif ($company && filled($company->email)) {
|
||||
} elseif ($company && filled($company->email) && ! $this->suppressions->isSuppressed($company->email)) {
|
||||
$recipient = $company->email;
|
||||
$recipientName = $companyName;
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue