fix: verify server snapshots and recover whole migration sessions (#3)
This commit is contained in:
1 parent
3d4101ecc0
commit
8569871a17
29 files changed
+1923
-133
No files matched your search
@@ -0,0 +1,162 @@
|
||||
//go:build integration
|
||||
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
qfassets "git.mchus.pro/mchus/quoteforge"
|
||||
"git.mchus.pro/mchus/quoteforge/internal/mariadb"
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
// Input is read-only. All SQL runs in a new, disposable local container.
|
||||
func TestProductionDumpMigrationMariaDB(t *testing.T) {
|
||||
dump := os.Getenv("QFS_TEST_PRODUCTION_DUMP")
|
||||
if dump == "" {
|
||||
t.Skip("QFS_TEST_PRODUCTION_DUMP is not set")
|
||||
}
|
||||
input, err := os.Open(dump)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer input.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
image := os.Getenv("QFS_TEST_MARIADB_IMAGE")
|
||||
if image == "" {
|
||||
image = "mariadb:11.8.3"
|
||||
}
|
||||
out, err := exec.CommandContext(ctx, "docker", "run", "--detach", "--rm", "--pull=never", "-p", "127.0.0.1::3306",
|
||||
"-e", "MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1", "-e", "MARIADB_ROOT_HOST=%", "-e", "MARIADB_DATABASE=RFQ_LOG", image,
|
||||
"--skip-ssl", "--character-set-server=utf8mb4", "--collation-server=utf8mb4_uca1400_ai_ci", "--max-allowed-packet=256M").CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("start dump rehearsal MariaDB: %v", err)
|
||||
}
|
||||
id := strings.TrimSpace(string(out))
|
||||
t.Cleanup(func() {
|
||||
if _, err := exec.Command("docker", "stop", "-t", "2", id).CombinedOutput(); err != nil {
|
||||
t.Errorf("stop dump rehearsal MariaDB: %v", err)
|
||||
}
|
||||
})
|
||||
out, err = exec.CommandContext(ctx, "docker", "port", id, "3306/tcp").CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dsn := "root@tcp(" + strings.TrimSpace(string(out)) + ")/RFQ_LOG?parseTime=true&timeout=1s&tls=false"
|
||||
var db *gorm.DB
|
||||
deadline := time.Now().Add(45 * time.Second)
|
||||
for {
|
||||
db, err = gorm.Open(mysql.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("dump rehearsal database did not become ready")
|
||||
}
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
}
|
||||
pool, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
pool.SetMaxOpenConns(1)
|
||||
pool.SetMaxIdleConns(1)
|
||||
importer := exec.CommandContext(ctx, "docker", "exec", "-i", id, "mariadb", "--skip-ssl", "--binary-mode", "--default-character-set=utf8mb4", "RFQ_LOG")
|
||||
importer.Stdin = input
|
||||
if output, err := importer.CombinedOutput(); err != nil {
|
||||
// Import errors can include entire data rows; report only error code/line.
|
||||
location := regexp.MustCompile(`(?m)^ERROR[^\n]*`).Find(output)
|
||||
location = regexp.MustCompile(`'[^']*'`).ReplaceAll(location, []byte("'<redacted>'"))
|
||||
if len(location) > 250 {
|
||||
location = location[:250]
|
||||
}
|
||||
t.Fatalf("load dump into disposable database: %v (%s)", err, location)
|
||||
}
|
||||
var version, collation string
|
||||
if err := db.Raw("SELECT VERSION(), @@collation_database").Row().Scan(&version, &collation); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(version, "11.8.3-") || collation != "utf8mb4_uca1400_ai_ci" {
|
||||
t.Fatalf("unexpected rehearsal server: version=%s collation=%s", version, collation)
|
||||
}
|
||||
var tables, triggers int64
|
||||
if err := db.Raw("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE()").Scan(&tables).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Raw("SELECT COUNT(*) FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE()").Scan(&triggers).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("production dump imported: MariaDB=%s collation=%s tables=%d triggers=%d", version, collation, tables, triggers)
|
||||
if err := db.Exec("SET time_zone='+00:00'").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := t.TempDir()
|
||||
var original string
|
||||
if err := db.Connection(func(conn *gorm.DB) error {
|
||||
var err error
|
||||
original, err = mariadb.CreateSnapshot(conn, root, "production-dump")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return mariadb.VerifySnapshot(conn, original)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("full dump snapshot restore verified")
|
||||
const filename = "035_add_server_updated_at.sql"
|
||||
if err := db.Where("filename=?", filename).Delete(&SQLSchemaMigration{}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Reconstruct the real pre-035 schema only in this disposable clone.
|
||||
for _, table := range []string{"qt_configurations", "qt_projects"} {
|
||||
if err := db.Exec("ALTER TABLE " + table + " DROP INDEX IF EXISTS idx_" + table + "_server_updated_at, DROP COLUMN IF EXISTS server_updated_at").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var before string
|
||||
if err := db.Connection(func(conn *gorm.DB) error {
|
||||
var err error
|
||||
before, err = mariadb.CreateSnapshot(conn, root, "before-rehearsal")
|
||||
return err
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content, err := qfassets.MigrationFiles.ReadFile("migrations/" + filename)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files := fstest.MapFS{filename: &fstest.MapFile{Data: content}, "036_recovery_probe.sql": &fstest.MapFile{Data: []byte("-- Tables affected: qt_configurations\n-- verify: deliberate rehearsal failure | SELECT 1\nALTER TABLE qt_configurations ADD COLUMN qfs_recovery_probe INT;")}}
|
||||
if _, err := RunSQLMigrations(db, files, nil, root); err == nil || !strings.Contains(err.Error(), "all session changes restored") {
|
||||
t.Fatalf("dump rollback rehearsal: %v", err)
|
||||
}
|
||||
if err := mariadb.CheckSnapshot(db, before); err != nil {
|
||||
t.Fatalf("dump changed after rollback: %v", err)
|
||||
}
|
||||
t.Log("real migration plus later failed DDL/verify restored the complete dump")
|
||||
delete(files, "036_recovery_probe.sql")
|
||||
if applied, err := RunSQLMigrations(db, files, nil, root); err != nil || len(applied) != 1 {
|
||||
t.Fatalf("real migration rehearsal: applied=%v error=%v", applied, err)
|
||||
}
|
||||
states, err := SQLMigrationStatus(db, files)
|
||||
if err != nil || len(states) != 1 || states[0].AppliedAt == nil {
|
||||
t.Fatalf("rehearsal history invalid: %v", err)
|
||||
}
|
||||
if err := RestoreSQLMigrationSession(db, original); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := mariadb.CheckSnapshot(db, original); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("production dump rehearsal complete: %s, exact original restored", filename)
|
||||
}
|
||||
Reference in new issue
Block a user