fix(store): derive current-schema column expectations from the applied schema

docs/internal/storage-design.md

@@ -486,6 +486,10 @@ Store startup runs the embedded idempotent schema in one immediate transaction
 and ensures the root exists. This safely creates missing compatible tables and
 indexes, but it is not a general migration system.
 
+The embedded `schema.sql` is the sole authority for current-layout columns.
+Startup derives the six guarded table layouts by applying it to an isolated
+temporary database. Released layouts remain pinned by their versioned adapters.
+
 Metadata-v1 identity changes are vertical changes to the live store, ingest,
 reachability, and backup/restore paths. A parallel schema or codec that
 production code does not consume has no authority; shared metadata helpers

internal/store/upgrade.go

@@ -70,6 +70,10 @@ var (
 	removeInvalidUpgradeStage = removeUpgradeFileSet
 )
 
+var currentSchemaTables = [...]string{
+	"blobs", "blob_packs", "vault_metadata", "blob_stores", "blob_locations", "blob_pack_entries",
+}
+
 // prepareReleasedSchemaUpgrade recognizes only storage layouts that shipped in
 // a public release. Older layouts rebuild through the same deterministic JSONL
 // authority used by backup and restore; released schemas are never mutated in
@@ -103,7 +107,7 @@ func prepareReleasedSchemaUpgrade(path string, driver docsqlite.Driver) error {
 	if err != nil {
 		return fmt.Errorf("inspecting database schema with %s: %w", driver.Name(), err)
 	}
-	kind, classifyErr := classifyDatabaseSchema(db)
+	kind, classifyErr := classifyDatabaseSchema(driver, db)
 	closeErr := db.Close()
 	if classifyErr != nil || closeErr != nil {
 		return errors.Join(classifyErr, closeErr)
@@ -146,7 +150,7 @@ func validateReleasedStorageSchemas() error {
 	return nil
 }
 
-func classifyDatabaseSchema(db *sql.DB) (databaseSchema, error) {
+func classifyDatabaseSchema(driver docsqlite.Driver, db *sql.DB) (databaseSchema, error) {
 	blobs, err := tableColumns(db, "blobs")
 	if err != nil {
 		return databaseSchema{}, err
@@ -175,7 +179,7 @@ func classifyDatabaseSchema(db *sql.DB) (databaseSchema, error) {
 			)
 		}
 		if version == currentStorageSchemaVersion {
-			if err := validateCurrentSchemaColumns(db, blobs, packs, vaultMetadata); err != nil {
+			if err := validateCurrentSchemaColumns(driver, db, blobs, packs, vaultMetadata); err != nil {
 				return databaseSchema{}, err
 			}
 			return databaseSchema{version: version, current: true}, nil
@@ -214,45 +218,27 @@ func classifyDatabaseSchema(db *sql.DB) (databaseSchema, error) {
 }
 
 func validateCurrentSchemaColumns(
-	db *sql.DB,
+	driver docsqlite.Driver, db *sql.DB,
 	blobs, packs, vaultMetadata []string,
 ) error {
-	currentBlobColumns := []string{
-		metadataCreatedAtField, "hash", metadataSizeField,
-	}
-	currentPackColumns := []string{
-		metadataCreatedAtField, "entry_count", "live_entries", "live_raw_bytes", "live_stored_bytes",
-		"max_live_raw_len", "max_live_stored_len", "pack_id", "scan_hash", "store_id", "stored_bytes",
+	wantTables, err := canonicalCurrentSchemaColumns(driver)
+	if err != nil {
+		return fmt.Errorf("deriving current schema columns: %w", err)
 	}
-	currentVaultColumns := []string{"schema_version", "singleton", "vault_uid"}
-	if !slices.Equal(blobs, currentBlobColumns) || !slices.Equal(packs, currentPackColumns) ||
-		!slices.Equal(vaultMetadata, currentVaultColumns) {
+	if !slices.Equal(blobs, wantTables["blobs"]) || !slices.Equal(packs, wantTables["blob_packs"]) ||
+		!slices.Equal(vaultMetadata, wantTables["vault_metadata"]) {
 		return fmt.Errorf(
 			"opening database: schema version %d has an unexpected layout (blobs=%s blob_packs=%s vault_metadata=%s)",
 			currentStorageSchemaVersion,
 			strings.Join(blobs, ","), strings.Join(packs, ","), strings.Join(vaultMetadata, ","),
 		)
 	}
-	wantTables := map[string][]string{
-		"blob_stores": {
-			"binding", metadataCreatedAtField, "kind", "lifecycle", "name",
-			"ownership_epoch", "role", "store_id",
-		},
-		"blob_locations": {
-			columnBlobHash, "encoding", "generation", "kind", "pack_eligible",
-			"store_id", "stored_size",
-		},
-		"blob_pack_entries": {
-			columnBlobHash, "crc32c", "flags", "pack_id", "pack_offset",
-			"raw_len", "store_id", "stored_len",
-		},
-	}
-	for table, want := range wantTables {
+	for _, table := range []string{"blob_stores", "blob_locations", "blob_pack_entries"} {
 		got, err := tableColumns(db, table)
 		if err != nil {
 			return err
 		}
-		if !slices.Equal(got, want) {
+		if !slices.Equal(got, wantTables[table]) {
 			return fmt.Errorf(
 				"opening database: schema version %d has an unexpected %s layout (%s)",
 				currentStorageSchemaVersion, table, strings.Join(got, ","),
@@ -262,6 +248,48 @@ func validateCurrentSchemaColumns(
 	return nil
 }
 
+func canonicalCurrentSchemaColumns(driver docsqlite.Driver) (map[string][]string, error) {
+	return deriveCurrentSchemaColumns(driver)
+}
+
+func deriveCurrentSchemaColumns(driver docsqlite.Driver) (columns map[string][]string, err error) {
+	tempFile, err := os.CreateTemp("", "docbank-current-schema-*.db")
+	if err != nil {
+		return nil, fmt.Errorf("creating temporary database: %w", err)
+	}
+	tempPath := tempFile.Name()
+	var db *sql.DB
+	defer func() {
+		var closeErr error
+		if db != nil {
+			closeErr = db.Close()
+		}
+		err = errors.Join(err, closeErr, os.Remove(tempPath))
+	}()
+	if err := tempFile.Close(); err != nil {
+		return nil, fmt.Errorf("closing temporary database file: %w", err)
+	}
+
+	db, err = driver.Open(tempPath, docsqlite.OpenOptions{
+		Access: docsqlite.Create, TransactionMode: docsqlite.Immediate,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("opening temporary database with %s: %w", driver.Name(), err)
+	}
+	if _, err := db.Exec(schemaSQL); err != nil {
+		return nil, fmt.Errorf("applying current schema to temporary database: %w", err)
+	}
+	columns = make(map[string][]string, len(currentSchemaTables))
+	for _, table := range currentSchemaTables {
+		columns[table], err = tableColumns(db, table)
+		if err != nil {
+			return nil, err
+		}
+		slices.Sort(columns[table])
+	}
+	return columns, nil
+}
+
 func validateV2Schema(db *sql.DB, blobs, packs []string) error {
 	v2BlobColumns := []string{
 		metadataCreatedAtField, "hash", "loose_encoding", "loose_stored_size",
@@ -1004,7 +1032,7 @@ func validateUpgradeStage(path string, driver docsqlite.Driver) error {
 	if err != nil {
 		return fmt.Errorf("opening interrupted upgrade staging database: %w", err)
 	}
-	kind, classifyErr := classifyDatabaseSchema(db)
+	kind, classifyErr := classifyDatabaseSchema(driver, db)
 	closeErr := db.Close()
 	if classifyErr != nil || closeErr != nil {
 		return errors.Join(classifyErr, closeErr)

internal/store/upgrade_test.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -206,6 +207,135 @@ func TestFreshStoresRecordCurrentStorageSchemaVersion(t *testing.T) {
 	}
 }
 
+func TestOpenAcceptsCurrentSchemaColumnAddedToEmbeddedSchema(t *testing.T) {
+	originalSchema := schemaSQL
+	t.Cleanup(func() { schemaSQL = originalSchema })
+
+	for _, table := range currentSchemaTables {
+		for _, test := range v090UpgradeDrivers() {
+			t.Run(table+"/"+test.name, func(t *testing.T) {
+				schemaSQL = schemaSQLWithAddedColumn(t, originalSchema, table, "synthetic_schema_269")
+
+				dbPath := filepath.Join(t.TempDir(), "docbank.db")
+				s, err := Open(dbPath, test.driver)
+				require.NoError(t, err)
+				require.NoError(t, s.Close())
+
+				reopened, err := Open(dbPath, test.driver)
+				require.NoError(t, err)
+				require.NoError(t, reopened.Close())
+			})
+		}
+	}
+}
+
+func TestCanonicalCurrentSchemaDerivationDoesNotCacheErrors(t *testing.T) {
+	originalSchema := schemaSQL
+	t.Cleanup(func() { schemaSQL = originalSchema })
+	schemaSQL = schemaSQLWithAddedColumn(t, originalSchema, "blobs", "synthetic_derivation_retry_269")
+	driver := &flakySchemaDriver{Driver: DefaultSQLiteDriver()}
+
+	_, err := canonicalCurrentSchemaColumns(driver)
+	require.ErrorContains(t, err, "synthetic derivation failure")
+
+	columns, err := canonicalCurrentSchemaColumns(driver)
+	require.NoError(t, err)
+	assert.Contains(t, columns["blobs"], "synthetic_derivation_retry_269")
+}
+
+func TestCanonicalCurrentSchemaDerivationSeparatesSameNamedDrivers(t *testing.T) {
+	const extraColumn = "synthetic_driver_variant_269"
+	base := DefaultSQLiteDriver()
+	plain := &schemaVariantDriver{Driver: base}
+	variant := &schemaVariantDriver{Driver: base, extraColumn: extraColumn}
+
+	columns, err := canonicalCurrentSchemaColumns(plain)
+	require.NoError(t, err)
+	assert.NotContains(t, columns["blobs"], extraColumn)
+
+	columns, err = canonicalCurrentSchemaColumns(variant)
+	require.NoError(t, err)
+	assert.Contains(t, columns["blobs"], extraColumn)
+}
+
+type flakySchemaDriver struct {
+	docsqlite.Driver
+
+	failed bool
+}
+
+func (d *flakySchemaDriver) Open(path string, opts docsqlite.OpenOptions) (*sql.DB, error) {
+	if !d.failed {
+		d.failed = true
+		return nil, errors.New("synthetic derivation failure")
+	}
+	return d.Driver.Open(path, opts)
+}
+
+type schemaVariantDriver struct {
+	docsqlite.Driver
+
+	extraColumn string
+}
+
+func (d *schemaVariantDriver) Open(path string, opts docsqlite.OpenOptions) (*sql.DB, error) {
+	db, err := d.Driver.Open(path, opts)
+	if err != nil || d.extraColumn == "" || opts.Access != docsqlite.Create {
+		return db, err
+	}
+	if _, err := db.Exec(schemaSQL + "\nALTER TABLE blobs ADD COLUMN " + d.extraColumn + " TEXT"); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	return db, nil
+}
+
+func schemaSQLWithAddedColumn(t *testing.T, original, table, column string) string {
+	t.Helper()
+	lineEnding := "\n"
+	if strings.Contains(original, "\r\n") {
+		lineEnding = "\r\n"
+	}
+	result := strings.Replace(
+		original,
+		"CREATE TABLE IF NOT EXISTS "+table+" ("+lineEnding,
+		"CREATE TABLE IF NOT EXISTS "+table+" ("+lineEnding+
+			"    "+column+" TEXT,"+lineEnding,
+		1,
+	)
+	require.NotEqual(t, original, result)
+	return result
+}
+
+func TestOpenRejectsCurrentDatabaseWithForeignColumn(t *testing.T) {
+	for _, table := range []struct {
+		name, expected string
+	}{
+		{name: "blobs", expected: "schema version 4 has an unexpected layout"},
+		{name: "blob_locations", expected: "schema version 4 has an unexpected blob_locations layout"},
+	} {
+		for _, test := range v090UpgradeDrivers() {
+			t.Run(table.name+"/"+test.name, func(t *testing.T) {
+				dbPath := filepath.Join(t.TempDir(), "docbank.db")
+				s, err := Open(dbPath, test.driver)
+				require.NoError(t, err)
+				require.NoError(t, s.Close())
+
+				db, err := test.driver.Open(dbPath, docsqlite.OpenOptions{
+					Access: docsqlite.ReadWriteExisting, TransactionMode: docsqlite.Immediate,
+				})
+				require.NoError(t, err)
+				_, err = db.Exec(`ALTER TABLE ` + table.name + ` ADD COLUMN synthetic_unexpected_269 TEXT`)
+				require.NoError(t, err)
+				require.NoError(t, db.Close())
+
+				_, err = Open(dbPath, test.driver)
+				require.ErrorContains(t, err, table.expected)
+			})
+		}
+	}
+}
+
 func TestOpenCutsOverEveryReleasedSchemaV2LayoutThroughJSONL(t *testing.T) {
 	layouts := []struct {
 		name     string
@@ -261,7 +391,7 @@ func TestOpenCutsOverEveryReleasedSchemaV2LayoutThroughJSONL(t *testing.T) {
 					Access: docsqlite.ReadWriteExisting, TransactionMode: docsqlite.Deferred,
 				})
 				require.NoError(t, err)
-				kind, err := classifyDatabaseSchema(backup)
+				kind, err := classifyDatabaseSchema(driver.driver, backup)
 				require.NoError(t, err)
 				assert.Equal(t, 2, kind.version)
 				assert.NotNil(t, kind.source)
@@ -300,7 +430,7 @@ func TestOpenCutsOverReleasedSchemaV3ThroughJSONL(t *testing.T) {
 				Access: docsqlite.ReadWriteExisting, TransactionMode: docsqlite.Deferred,
 			})
 			require.NoError(t, err)
-			kind, err := classifyDatabaseSchema(backup)
+			kind, err := classifyDatabaseSchema(test.driver, backup)
 			require.NoError(t, err)
 			assert.Equal(t, 3, kind.version)
 			assert.NotNil(t, kind.source)
@@ -622,7 +752,7 @@ func TestV090CutoverPublicationFailureRestoresReleasedDatabase(t *testing.T) {
 		Access: docsqlite.ReadWriteExisting, TransactionMode: docsqlite.Deferred,
 	})
 	require.NoError(t, err)
-	kind, err := classifyDatabaseSchema(db)
+	kind, err := classifyDatabaseSchema(driver, db)
 	require.NoError(t, err)
 	assert.Equal(t, 1, kind.version, "the released source is restored after publication fails")
 	assert.NotNil(t, kind.source)