From a59cc7ccb77144cf4ab38c6f973b1e029e8a80ce Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Mon, 26 Oct 2026 10:49:23 +0300 Subject: [PATCH] fix: verify SQL migrations before recording and preserve sessions (#3) --- bible-local/03-database.md | 12 +++- internal/models/sql_migration_verify.go | 64 ++++++++++++++++++ internal/models/sql_migration_verify_test.go | 69 ++++++++++++++++++++ internal/models/sql_migrations.go | 12 +--- 4 files changed, 145 insertions(+), 12 deletions(-) create mode 100644 internal/models/sql_migration_verify.go create mode 100644 internal/models/sql_migration_verify_test.go diff --git a/bible-local/03-database.md b/bible-local/03-database.md index 29c99e1..17eb94f 100644 --- a/bible-local/03-database.md +++ b/bible-local/03-database.md @@ -439,8 +439,16 @@ MariaDB: `UPTO` as applied **without running them**, for a server whose schema is already current; - migrations use their own connection without read/write timeouts (`openMigrationDB`), not the app's 3s-timeout sync connection — a long backfill must not be cut mid-file; -- a file is recorded only after all its statements succeed, so each file must be safe to - re-run (`ADD COLUMN IF NOT EXISTS`, idempotent `UPDATE ... WHERE `); +- each file, its `-- verify: | ` queries and history insertion run + on one database session, preserving session variables; +- a file is recorded only after statements and all verify checks succeed. Verify + queries return rows only for broken invariants; a row or SQL error stops the run + without marking that file applied. Malformed verify headers fail before execution; +- only duplicate column/key names and already-existing objects are tolerated. + FK creation errors (including errno 150/121) remain fatal; +- each file must be safe to re-run after a partial failure (`ADD COLUMN IF NOT EXISTS`, + idempotent `UPDATE ... WHERE `). The current statement splitter assumes + semicolons occur only between statements; do not add semicolons inside literals; - there is no GORM `AutoMigrate` against MariaDB: it retypes production columns to match the running binary's models. On 2026-09-24 a stale v1.18 binary run with the old AutoMigrate `-migrate` did exactly that, which is why it was removed. diff --git a/internal/models/sql_migration_verify.go b/internal/models/sql_migration_verify.go new file mode 100644 index 0000000..201e81b --- /dev/null +++ b/internal/models/sql_migration_verify.go @@ -0,0 +1,64 @@ +package models + +import ( + "fmt" + "strings" + + "gorm.io/gorm" +) + +type migrationVerification struct { + description string + query string +} + +func parseMigrationVerifications(script string) ([]migrationVerification, error) { + var checks []migrationVerification + for _, line := range strings.Split(script, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "-- verify:") { + continue + } + description, query, ok := strings.Cut(strings.TrimPrefix(line, "-- verify:"), "|") + description, query = strings.TrimSpace(description), strings.TrimSpace(query) + if !ok || description == "" || query == "" { + return nil, fmt.Errorf("invalid migration verify header") + } + checks = append(checks, migrationVerification{description: description, query: query}) + } + return checks, nil +} + +func applySQLMigrationFile(db *gorm.DB, filename, content string) error { + checks, err := parseMigrationVerifications(content) + if err != nil { + return fmt.Errorf("parse migration %s: %w", filename, err) + } + return db.Connection(func(conn *gorm.DB) error { + if err := executeMigrationStatements(conn, filename, splitSQLStatements(content)); err != nil { + return err + } + for _, check := range checks { + rows, err := conn.Raw(check.query).Rows() + if err != nil { + return fmt.Errorf("verify migration %s (%s): %w", filename, check.description, err) + } + failed := rows.Next() + rowErr := rows.Err() + closeErr := rows.Close() + if rowErr != nil { + return fmt.Errorf("read migration verification %s (%s): %w", filename, check.description, rowErr) + } + if closeErr != nil { + return fmt.Errorf("close migration verification %s (%s): %w", filename, check.description, closeErr) + } + if failed { + return fmt.Errorf("verify migration %s failed: %s", filename, check.description) + } + } + if err := conn.Create(&SQLSchemaMigration{Filename: filename}).Error; err != nil { + return fmt.Errorf("record migration %s: %w", filename, err) + } + return nil + }) +} diff --git a/internal/models/sql_migration_verify_test.go b/internal/models/sql_migration_verify_test.go new file mode 100644 index 0000000..e358db1 --- /dev/null +++ b/internal/models/sql_migration_verify_test.go @@ -0,0 +1,69 @@ +package models + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func TestMigrationVerification(t *testing.T) { + for _, tt := range []struct{ name, content, wantError string }{ + {"valid", "-- verify: bad value | SELECT 1 FROM example WHERE value <> 1\nCREATE TEMP TABLE example (value INT); INSERT INTO example VALUES (1);", ""}, + {"failed", "-- verify: bad value | SELECT 1 FROM example WHERE value <> 1\nCREATE TEMP TABLE example (value INT); INSERT INTO example VALUES (2);", "bad value"}, + {"query error", "-- verify: missing table | SELECT 1 FROM missing\nCREATE TEMP TABLE example (value INT);", "missing table"}, + {"malformed", "-- verify: no query\nCREATE TABLE example (value INT);", "invalid migration verify header"}, + {"second check fails", "-- verify: first | SELECT 1 WHERE 0\n-- verify: second | SELECT 1\nCREATE TEMP TABLE example (value INT);", "second"}, + {"execution failure", "-- verify: none | SELECT 1 WHERE 0\nINVALID SQL;", "exec migration"}, + } { + t.Run(tt.name, func(t *testing.T) { + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "migration.db")), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + sqlDB, _ := db.DB() + defer sqlDB.Close() + if err := db.AutoMigrate(&SQLSchemaMigration{}); err != nil { + t.Fatal(err) + } + err = applySQLMigrationFile(db, "001_example.sql", tt.content) + if tt.wantError != "" && (err == nil || !strings.Contains(err.Error(), tt.wantError)) { + t.Fatalf("want %q, got %v", tt.wantError, err) + } + if tt.wantError == "" && err != nil { + t.Fatal(err) + } + var count int64 + if err := db.Model(&SQLSchemaMigration{}).Count(&count).Error; err != nil { + t.Fatal(err) + } + if (count == 1) != (tt.wantError == "") { + t.Fatalf("incorrect history count=%d", count) + } + if tt.name == "malformed" && db.Migrator().HasTable("example") { + t.Fatal("malformed header mutated database") + } + }) + } +} + +func TestMigrationErrorAllowlist(t *testing.T) { + for _, tt := range []struct { + message string + want bool + }{ + {"Duplicate column name 'example'", true}, + {"Duplicate key name 'example'", true}, + {"Table 'example' already exists", true}, + {"Can't create table example (errno: 150 Foreign key constraint is incorrectly formed)", false}, + {"Can't create table example (errno: 121 Duplicate key)", false}, + {"Duplicate foreign key constraint name 'example'", false}, + {"Access denied", false}, + } { + if got := isIgnorableMigrationError(tt.message); got != tt.want { + t.Errorf("%s: got %v", tt.message, got) + } + } +} diff --git a/internal/models/sql_migrations.go b/internal/models/sql_migrations.go index 9f75136..6b71265 100644 --- a/internal/models/sql_migrations.go +++ b/internal/models/sql_migrations.go @@ -157,13 +157,8 @@ func RunSQLMigrations(db *gorm.DB, migrations fs.FS, onApply func(filename strin return applied, fmt.Errorf("read migration %s: %w", filename, err) } - if statements := splitSQLStatements(string(content)); len(statements) > 0 { - if err := executeMigrationStatements(db, filename, statements); err != nil { - return applied, err - } - } - if err := db.Create(&SQLSchemaMigration{Filename: filename}).Error; err != nil { - return applied, fmt.Errorf("record migration %s: %w", filename, err) + if err := applySQLMigrationFile(db, filename, string(content)); err != nil { + return applied, err } applied = append(applied, filename) } @@ -235,9 +230,6 @@ func isIgnorableMigrationError(message string) bool { "duplicate column name", "duplicate key name", "already exists", - "can't create table", - "duplicate foreign key constraint name", - "errno 121", } for _, pattern := range ignorable { if strings.Contains(lower, pattern) {