Skip to content

Commit 3f32a5b

Browse files
rmcdanielrmcdaniel
committed
fix: preserve full failure messages through portable additive storage
1 parent f620dc3 commit 3f32a5b

8 files changed

Lines changed: 258 additions & 10 deletions

‎CHANGELOG.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
11
# Changelog
22

3+
## 2.12.1 - Unreleased
4+
5+
- Preserve canonical failure messages beyond MySQL TEXT capacity and across
6+
NUL-bearing PostgreSQL text parameters. Add nullable encoded overflow storage
7+
without backfilling existing rows, retain the exact public message and keep
8+
compatibility previews bounded. Metadata-lock waits are bounded and a package
9+
rollback retains the added storage and failure evidence.
10+
311
## 2.12.0 - 2026-10-23
412

513
- Preserve UTC instants and microseconds in datetime search attributes and

‎composer.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@
8787
"dev-main": "2.0.x-dev"
8888
},
8989
"durable-workflow": {
90-
"product-train": "2.12.0",
90+
"product-train": "2.12.1",
9191
"laravel-embedded-upgrade-contract": "resources/laravel-embedded-upgrade-contract.json",
9292
"laravel-dependency-security-policy": "resources/laravel-dependency-security-policy.json"
9393
},

‎docs/failure-message-storage.md‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Failure message storage
2+
3+
Read a failure's canonical message through `WorkflowFailure::message`, retained
4+
history or the history export. The physical `workflow_failures.message` column
5+
can contain a compatibility preview for a new large message. Reading that
6+
column directly does not always return the full message.
7+
8+
The additive `2026_10_23_000100_add_full_failure_message_storage` migration adds
9+
nullable `message_full` storage. Existing rows are not rewritten or backfilled.
10+
Ordinary messages keep their existing storage. Messages beyond the old TEXT
11+
capacity, and messages containing NUL, retain their full bytes in a marked
12+
base64 representation in the new column. The model decodes that representation
13+
and hides the physical field from public serialization. Invalid encoded storage
14+
raises an error instead of presenting a preview as the canonical message.
15+
16+
Base64 is a storage representation, not encryption. Apply the same access and
17+
backup protections to both columns. SDK report limits and visible truncation
18+
markers still apply independently; this storage expansion does not increase an
19+
HTTP request limit or turn an already shortened report into its original error.
20+
21+
## Upgrades with live traffic
22+
23+
The migration adds a nullable column without a data backfill. MySQL and MariaDB
24+
must support its explicit `ALGORITHM=INSTANT` operation. Unsupported table
25+
layouts fail instead of silently copying the table. The migration bounds
26+
metadata-lock acquisition to one second on MySQL, MariaDB and PostgreSQL, then
27+
restores the connection's original lock-wait policy on success or failure.
28+
29+
If an active transaction prevents the schema change, finish that transaction
30+
and retry the normal migration. If the engine rejects the instant operation,
31+
use a separately qualified online schema-change procedure appropriate to that
32+
database and table layout. Do not substitute a copying alteration during a
33+
rolling deployment. The expected column is nullable LONGTEXT on MySQL/MariaDB
34+
and the equivalent unbounded text storage on PostgreSQL/SQLite.
35+
36+
Package rollback retains the nullable expansion. It does not drop stored
37+
messages or race with active writers to decide whether the column is empty.
38+
Older readers can show the labelled compatibility preview for new oversized or
39+
NUL-bearing failures. Keep an upgraded reader when the complete canonical
40+
message is needed. Existing retained history continues to carry its full public
41+
failure payload.

‎src/V2/Models/WorkflowFailure.php‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
use Illuminate\Database\Eloquent\Concerns\HasUlids;
99
use Illuminate\Database\Eloquent\Model;
1010
use Illuminate\Database\Eloquent\Relations\BelongsTo;
11+
use RuntimeException;
1112
use Workflow\Traits\ResolvesStorageConnection;
1213
use Workflow\V2\Enums\FailureCategory;
1314
use Workflow\V2\Support\ConfiguredV2Models;
@@ -22,6 +23,8 @@ class WorkflowFailure extends Model
2223

2324
private const OVERFLOW_MARKER = "\n[full message stored separately]";
2425

26+
private const FULL_MESSAGE_ENCODING = 'base64:';
27+
2528
public $incrementing = false;
2629

2730
protected $table = 'workflow_failures';
@@ -47,9 +50,12 @@ protected function message(): Attribute
4750
{
4851
return Attribute::make(
4952
get: static fn (?string $value, array $attributes): ?string =>
50-
is_string($attributes['message_full'] ?? null) ? $attributes['message_full'] : $value,
53+
($attributes['message_full'] ?? null) === null
54+
? $value
55+
: self::decodeFullMessage($attributes['message_full']),
5156
set: static function (?string $value): array {
52-
if ($value === null || strlen($value) <= self::LEGACY_MESSAGE_LIMIT_BYTES) {
57+
if ($value === null || (strlen($value) <= self::LEGACY_MESSAGE_LIMIT_BYTES
58+
&& ! str_contains($value, "\0"))) {
5359
return [
5460
'message' => $value,
5561
'message_full' => null,
@@ -60,17 +66,34 @@ protected function message(): Attribute
6066
// a valid TEXT preview without a copying column alteration.
6167
return [
6268
'message' => mb_strcut(
63-
$value,
69+
str_replace("\0", '\\0', $value),
6470
0,
6571
self::LEGACY_MESSAGE_LIMIT_BYTES - strlen(self::OVERFLOW_MARKER),
6672
'UTF-8',
6773
) . self::OVERFLOW_MARKER,
68-
'message_full' => $value,
74+
// PostgreSQL text parameters cannot carry literal NULs.
75+
// This is internal storage, not an SDK payload codec.
76+
'message_full' => self::FULL_MESSAGE_ENCODING . base64_encode($value),
6977
];
7078
},
7179
);
7280
}
7381

82+
private static function decodeFullMessage(mixed $stored): string
83+
{
84+
if (! is_string($stored) || ! str_starts_with($stored, self::FULL_MESSAGE_ENCODING)) {
85+
throw new RuntimeException('Invalid stored full failure-message encoding.');
86+
}
87+
88+
$encoded = substr($stored, strlen(self::FULL_MESSAGE_ENCODING));
89+
$decoded = base64_decode($encoded, true);
90+
if ($decoded === false || base64_encode($decoded) !== $encoded) {
91+
throw new RuntimeException('Invalid stored full failure-message encoding.');
92+
}
93+
94+
return $decoded;
95+
}
96+
7497
public function run(): BelongsTo
7598
{
7699
return $this->belongsTo(ConfiguredV2Models::resolve('run_model', WorkflowRun::class), 'workflow_run_id');

‎src/V2/Support/RunRecentFailures.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public static function forRun(WorkflowRun $run, int $limit = 10): array
3636
->get([
3737
'id', 'workflow_run_id', 'source_kind', 'source_id', 'propagation_kind',
3838
'failure_category', 'non_retryable', 'handled', 'exception_class',
39-
'message', 'file', 'line', 'trace_preview', 'created_at',
39+
'message', 'message_full', 'file', 'line', 'trace_preview', 'created_at',
4040
]);
4141
$hasMore = $rows->count() > $limit;
4242
$rows = $rows->take($limit);

‎tests/Feature/V2/V2ActivityExceptionCodecTest.php‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
use Workflow\V2\Support\DefaultHistoryProjectionRole;
3030
use Workflow\V2\Support\FailureFactory;
3131
use Workflow\V2\Support\FailureSnapshots;
32+
use Workflow\V2\Support\RunRecentFailures;
3233

3334
/**
3435
* TD-089 regression: activity exception rows must be encoded with the
@@ -58,7 +59,7 @@ public function testLargeTerminalFailureKeepsFullCanonicalMessageAndHistory(): v
5859
$this->assertSame(ActivityStatus::Failed, $execution->refresh()->status);
5960
$failure = WorkflowFailure::query()->where('workflow_run_id', $run->id)->firstOrFail();
6061
$this->assertSame($message, $failure->message);
61-
$this->assertSame($message, $failure->getRawOriginal('message_full'));
62+
$this->assertSame('base64:' . base64_encode($message), $failure->getRawOriginal('message_full'));
6263
$this->assertLessThanOrEqual(65_535, strlen($failure->getRawOriginal('message')));
6364
$this->assertArrayNotHasKey('message_full', $failure->toArray());
6465
$event = WorkflowHistoryEvent::query()
@@ -67,6 +68,7 @@ public function testLargeTerminalFailureKeepsFullCanonicalMessageAndHistory(): v
6768
->firstOrFail();
6869
$this->assertSame($message, $event->payload['message']);
6970
$this->assertSame($failure->id, $event->payload['failure_id']);
71+
$this->assertSame($message, RunRecentFailures::forRun($run)['failures'][0]['message']);
7072
$this->assertExceptionBytesDecodeAs(
7173
bytes: $execution->exception,
7274
runCodec: 'avro',

‎tests/Feature/V2/V2FailureMessageStorageMigrationTest.php‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Tests\Feature\V2;
6+
7+
use Illuminate\Database\QueryException;
8+
use Illuminate\Database\Schema\Blueprint;
9+
use Illuminate\Support\Facades\DB;
10+
use Illuminate\Support\Facades\Schema;
11+
use Illuminate\Support\Str;
12+
use Tests\TestCase;
13+
use Workflow\Support\WorkflowMigration;
14+
use Workflow\V2\Models\WorkflowFailure;
15+
16+
final class V2FailureMessageStorageMigrationTest extends TestCase
17+
{
18+
public function testUpgradeIsAdditiveAndPackageRollbackRetainsCanonicalEvidence(): void
19+
{
20+
$legacy = $this->legacyRow();
21+
$migration = $this->migration();
22+
$migration->up();
23+
$connection = Schema::getConnection();
24+
$row = $connection->table('workflow_failures')->where('id', $legacy['id'])->first();
25+
$this->assertNull($row->message_full);
26+
unset($row->message_full);
27+
$this->assertSame($legacy, (array) $row);
28+
29+
$message = str_repeat("\0界🦀", 10_000);
30+
$failure = WorkflowFailure::query()->findOrFail($legacy['id']);
31+
$failure->message = $message;
32+
$failure->save();
33+
$this->assertSame($message, $failure->fresh()->message);
34+
35+
$migration->down();
36+
$this->assertTrue(Schema::hasColumn('workflow_failures', 'message_full'));
37+
$this->assertSame($message, $failure->fresh()->message);
38+
$migration->up();
39+
$this->assertSame($message, $failure->fresh()->message);
40+
$this->assertSame(1, WorkflowFailure::query()->count());
41+
}
42+
43+
public function testHeldWriterBoundsMetadataWaitAndRestoresConnectionPolicy(): void
44+
{
45+
$connection = Schema::getConnection();
46+
$driver = $connection->getDriverName();
47+
if (! in_array($driver, ['mysql', 'mariadb', 'pgsql'], true)) {
48+
$this->markTestSkipped('Network database metadata-lock control.');
49+
}
50+
51+
$legacy = $this->legacyRow();
52+
$timeoutQuery = $driver === 'pgsql' ? 'SHOW lock_timeout' : 'SELECT @@SESSION.lock_wait_timeout AS value';
53+
$previous = (array) $connection->selectOne($timeoutQuery);
54+
$blockerName = 'failure_message_metadata_blocker';
55+
config()->set('database.connections.' . $blockerName, $connection->getConfig());
56+
DB::purge($blockerName);
57+
$blocker = DB::connection($blockerName);
58+
$blocker->beginTransaction();
59+
$blocker->table('workflow_failures')->where('id', $legacy['id'])->lockForUpdate()->first();
60+
$migration = $this->migration();
61+
$started = microtime(true);
62+
63+
try {
64+
try {
65+
$migration->up();
66+
$this->fail('The held writer must prevent an unbounded metadata upgrade.');
67+
} catch (QueryException) {
68+
$elapsed = microtime(true) - $started;
69+
$this->assertGreaterThanOrEqual(0.8, $elapsed);
70+
$this->assertLessThan(3.0, $elapsed);
71+
$this->assertSame($previous, (array) $connection->selectOne($timeoutQuery));
72+
$this->assertFalse(Schema::hasColumn('workflow_failures', 'message_full'));
73+
$this->assertSame($legacy, (array) $connection->table('workflow_failures')->first());
74+
}
75+
} finally {
76+
$blocker->rollBack();
77+
DB::purge($blockerName);
78+
$migration->up();
79+
}
80+
}
81+
82+
public function testUnsupportedMySqlLayoutNeverFallsBackToCopyingTheTable(): void
83+
{
84+
$connection = Schema::getConnection();
85+
if ($connection->getDriverName() !== 'mysql') {
86+
$this->markTestSkipped('MySQL compressed-layout control.');
87+
}
88+
89+
$version = (array) $connection->selectOne('SELECT VERSION() AS version');
90+
if (str_contains($version['version'], 'MariaDB')) {
91+
$this->markTestSkipped('MariaDB has a separate instant-DDL support contract.');
92+
}
93+
94+
$legacy = $this->legacyRow();
95+
$table = $connection->getQueryGrammar()->wrapTable('workflow_failures');
96+
// This is disposable fixture preparation, not a rolling-upgrade operation.
97+
$connection->statement("ALTER TABLE {$table} ROW_FORMAT=COMPRESSED");
98+
$migration = $this->migration();
99+
100+
try {
101+
try {
102+
$migration->up();
103+
$this->fail('The unsupported instant operation must fail without a copying fallback.');
104+
} catch (QueryException $error) {
105+
$this->assertContains($error->errorInfo[1], [1845, 1846]);
106+
$this->assertFalse(Schema::hasColumn('workflow_failures', 'message_full'));
107+
$this->assertSame($legacy, (array) $connection->table('workflow_failures')->first());
108+
}
109+
} finally {
110+
$connection->statement("ALTER TABLE {$table} ROW_FORMAT=DYNAMIC");
111+
$migration->up();
112+
}
113+
}
114+
115+
/**
116+
* @return array<string, mixed>
117+
*/
118+
private function legacyRow(): array
119+
{
120+
Schema::table('workflow_failures', static function (Blueprint $table): void {
121+
$table->dropColumn('message_full');
122+
});
123+
$connection = Schema::getConnection();
124+
$connection->table('workflow_failures')->insert([
125+
'id' => (string) Str::ulid(),
126+
'workflow_run_id' => (string) Str::ulid(),
127+
'source_kind' => 'activity_execution',
128+
'source_id' => 'original-attempt',
129+
'propagation_kind' => 'activity',
130+
'failure_category' => null,
131+
'non_retryable' => false,
132+
'handled' => false,
133+
'exception_class' => 'RuntimeException',
134+
'message' => 'unchanged legacy message',
135+
'file' => '/fixture/activity.php',
136+
'line' => 7,
137+
'trace_preview' => null,
138+
'created_at' => '2026-10-23 23:00:00',
139+
'updated_at' => '2026-10-23 23:00:00',
140+
]);
141+
142+
return (array) $connection->table('workflow_failures')->first();
143+
}
144+
145+
private function migration(): WorkflowMigration
146+
{
147+
return require __DIR__ . '/../../../src/migrations/2026_10_23_000100_add_full_failure_message_storage.php';
148+
}
149+
}

‎tests/Unit/V2/WorkflowFailureMessageStorageTest.php‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
use PHPUnit\Framework\Attributes\DataProvider;
88
use PHPUnit\Framework\TestCase;
9+
use RuntimeException;
910
use Workflow\V2\Models\WorkflowFailure;
1011

1112
final class WorkflowFailureMessageStorageTest extends TestCase
@@ -21,7 +22,7 @@ public function testOversizedMessagesKeepExactCanonicalBytesAndBoundLegacyStorag
2122

2223
$this->assertSame($message, $failure->message);
2324
$this->assertSame($message, $failure->toArray()['message']);
24-
$this->assertSame($message, $failure->getAttributes()['message_full']);
25+
$this->assertSame('base64:' . base64_encode($message), $failure->getAttributes()['message_full']);
2526
$this->assertLessThanOrEqual(65_535, strlen($failure->getAttributes()['message']));
2627
$this->assertTrue(mb_check_encoding($failure->getAttributes()['message'], 'UTF-8'));
2728
$this->assertStringContainsString('[full message stored separately]', $failure->getAttributes()['message']);
@@ -38,6 +39,7 @@ public static function messages(): iterable
3839
yield 'ASCII beyond TEXT' => [str_repeat('a', 70_000)];
3940
yield 'Unicode and NUL beyond TEXT' => [str_repeat("\0界🦀", 10_000)];
4041
yield 'large embedded exception' => [str_repeat('x', 3_000_000)];
42+
yield 'short NUL must not be silently truncated by PostgreSQL' => ["prefix\0suffix"];
4143
}
4244

4345
public function testColdReadRestoresFullMessageWithoutExposingPhysicalStorageField(): void
@@ -46,7 +48,7 @@ public function testColdReadRestoresFullMessageWithoutExposingPhysicalStorageFie
4648
$failure = new WorkflowFailure();
4749
$failure->setRawAttributes([
4850
'message' => 'legacy preview [full message stored separately]',
49-
'message_full' => $message,
51+
'message_full' => 'base64:' . base64_encode($message),
5052
], true);
5153

5254
$this->assertSame($message, $failure->message);
@@ -70,7 +72,7 @@ public function testReplacingLargeMessageClearsStaleOverflowBytes(): void
7072
$failure = new WorkflowFailure();
7173
$failure->setRawAttributes([
7274
'message' => 'old preview',
73-
'message_full' => str_repeat('old', 30_000),
75+
'message_full' => 'base64:' . base64_encode(str_repeat('old', 30_000)),
7476
], true);
7577
$failure->message = 'replacement';
7678

@@ -79,4 +81,27 @@ public function testReplacingLargeMessageClearsStaleOverflowBytes(): void
7981
$this->assertNull($failure->getAttributes()['message_full']);
8082
$this->assertSame(['message' => 'replacement'], $failure->toArray());
8183
}
84+
85+
#[DataProvider('invalidStorage')]
86+
public function testMalformedOverflowStorageNeverFabricatesCanonicalMessage(mixed $stored): void
87+
{
88+
$failure = new WorkflowFailure();
89+
$failure->setRawAttributes(['message' => 'diagnostic preview', 'message_full' => $stored], true);
90+
$this->expectException(RuntimeException::class);
91+
$this->expectExceptionMessage('Invalid stored full failure-message encoding.');
92+
$failure->message;
93+
}
94+
95+
/**
96+
* @return iterable<string, array{mixed}>
97+
*/
98+
public static function invalidStorage(): iterable
99+
{
100+
yield 'missing encoding marker' => ['raw text'];
101+
yield 'empty accidental default' => [''];
102+
yield 'invalid base64' => ['base64:!'];
103+
yield 'noncanonical missing padding' => ['base64:Zg'];
104+
yield 'noncanonical whitespace' => ["base64:Zg==\n"];
105+
yield 'wrong stored type' => [false];
106+
}
82107
}

0 commit comments

Comments
 (0)