diff --git a/Makefile b/Makefile index d811f8d..50cd704 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build build-release clean test run version install-hooks +.PHONY: build build-release clean test run version install-hooks migrate migrate-status migrate-baseline # Get version from git VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") @@ -55,6 +55,24 @@ clean: rm -rf bin/ rm -f $(BINARY) +# Server (MariaDB) migrations — migrations/*.sql tracked in qt_schema_migrations. +# Always built from the current source (go run), never a possibly stale bin/ binary. +# Uses the connection saved in the local app settings. + +# List migrations with applied/pending state (read-only) +migrate-status: + go run ./cmd/qfs -migrate-status + +# Apply pending migrations +migrate: + go run ./cmd/qfs -migrate + +# Record migrations up to UPTO as applied WITHOUT running them (one-time, for a server +# whose schema is already current): make migrate-baseline UPTO=033_add_price_quality_to_pricelist_items.sql +migrate-baseline: + @test -n "$(UPTO)" || (echo "usage: make migrate-baseline UPTO="; exit 1) + go run ./cmd/qfs -migrate-baseline "$(UPTO)" + # Run tests test: go test -v ./... diff --git a/README.md b/README.md index 31e6da7..7d43e68 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,8 @@ go run ./cmd/qfs Useful commands: ```bash -go run ./cmd/qfs -migrate +make migrate-status +make migrate go test ./... go vet ./... make build-release diff --git a/assets_embed.go b/assets_embed.go index 9a1fec1..ef6832e 100644 --- a/assets_embed.go +++ b/assets_embed.go @@ -19,3 +19,14 @@ var StaticFiles embed.FS func StaticFS() (fs.FS, error) { return fs.Sub(StaticFiles, "web/static") } + +// MigrationFiles contains the MariaDB server migrations (migrations/*.sql), so +// `qfs -migrate` works from any directory. Applied via models.RunSQLMigrations. +// +//go:embed migrations/*.sql +var MigrationFiles embed.FS + +// MigrationsFS returns a filesystem rooted at migrations/. +func MigrationsFS() (fs.FS, error) { + return fs.Sub(MigrationFiles, "migrations") +} diff --git a/assets_embed_test.go b/assets_embed_test.go new file mode 100644 index 0000000..6a7e9f4 --- /dev/null +++ b/assets_embed_test.go @@ -0,0 +1,16 @@ +package quoteforge + +import ( + "io/fs" + "testing" +) + +func TestMigrationsEmbedded(t *testing.T) { + migrations, err := MigrationsFS() + if err != nil { + t.Fatal(err) + } + if _, err := fs.Stat(migrations, "034_add_feature_flags_to_configurations.sql"); err != nil { + t.Fatalf("migration 034 not embedded: %v", err) + } +} diff --git a/bible-local/03-database.md b/bible-local/03-database.md index 3c17bc9..10fd1c6 100644 --- a/bible-local/03-database.md +++ b/bible-local/03-database.md @@ -57,7 +57,7 @@ Server-side only (not queried by client runtime): - `qt_lot_metadata` — component metadata / price settings; the Go server-side component/category management layer (`ComponentRepository`, `CategoryRepository`, `ComponentService`) was removed — no client code reads this table - `qt_component_usage_stats` — aggregated component popularity stats (written by server jobs) - `qt_pricing_alerts` — price anomaly alerts (models exist in Go; feature disabled in runtime) -- `qt_schema_migrations` — server migration history (applied via `go run ./cmd/qfs -migrate`) +- `qt_schema_migrations` — server migration history (applied via `make migrate`, see [Migrations](#migrations)) - `qt_scheduler_runs` — server background job tracking (no Go code references it in this repo) ### Competitor subsystem (server-side only, not used by QuoteForge Go code) @@ -427,5 +427,20 @@ SQLite: - local migration state is tracked in `local_schema_migrations`. MariaDB: -- SQL files live in `migrations/`; -- they are applied by `go run ./cmd/qfs -migrate`. +- SQL files live in `migrations/` (embedded into the binary), one file per change, + `NNN_short_topic.sql`, with the recovery/verify header used by the existing files; + filenames containing `local_` are SQLite-only and skipped; +- applied state is tracked per filename in `qt_schema_migrations`; +- commands (always from current source — never run a `bin/` binary of unknown age): + - `make migrate-status` (`qfs -migrate-status`) — list applied/pending, read-only; + - `make migrate` (`qfs -migrate`) — apply pending files in filename order, then exit; + refuses when `qt_schema_migrations` has no applied rows at all (every file would re-run); + - `make migrate-baseline UPTO=` (`qfs -migrate-baseline`) — record files up to + `UPTO` as applied **without running them**, for a server whose schema is already current; +- migrations use their own connection without read/write timeouts (`openMigrationDB`), + not the app's 3s-timeout sync connection — a long backfill must not be cut mid-file; +- a file is recorded only after all its statements succeed, so each file must be safe to + re-run (`ADD COLUMN IF NOT EXISTS`, idempotent `UPDATE ... WHERE `); +- there is no GORM `AutoMigrate` against MariaDB: it retypes production columns to match the + running binary's models. On 2026-09-24 a stale v1.18 binary run with the old AutoMigrate + `-migrate` did exactly that, which is why it was removed. diff --git a/bible-local/05-config.md b/bible-local/05-config.md index ee16995..28d546b 100644 --- a/bible-local/05-config.md +++ b/bible-local/05-config.md @@ -63,7 +63,9 @@ They are stored in SQLite and encrypted with `local_encryption.key` unless `QUOT | `-config ` | config file path | | `-localdb ` | SQLite path | | `-reset-localdb` | destructive local DB reset | -| `-migrate` | apply server migrations and exit | +| `-migrate` | apply pending server SQL migrations (`migrations/*.sql`) and exit | +| `-migrate-status` | list server SQL migrations with applied/pending state and exit | +| `-migrate-baseline ` | record server migrations up to `` as applied without running them, and exit | | `-version` | print app version and exit | ## First run diff --git a/bible-local/07-dev.md b/bible-local/07-dev.md index af6a22a..a17b4ae 100644 --- a/bible-local/07-dev.md +++ b/bible-local/07-dev.md @@ -4,7 +4,8 @@ ```bash go run ./cmd/qfs -go run ./cmd/qfs -migrate +make migrate-status # server SQL migrations: applied/pending +make migrate # apply pending server SQL migrations go run ./cmd/migrate_project_updated_at go test ./... go vet ./... diff --git a/cmd/qfs/main.go b/cmd/qfs/main.go index 309b872..54711a7 100644 --- a/cmd/qfs/main.go +++ b/cmd/qfs/main.go @@ -60,7 +60,9 @@ func main() { configPath := flag.String("config", "", "path to config file (default: user state dir or QFS_CONFIG_PATH)") localDBPath := flag.String("localdb", "", "path to local SQLite database (default: user state dir or QFS_DB_PATH)") resetLocalDB := flag.Bool("reset-localdb", false, "reset local SQLite data on startup (keeps connection settings)") - migrate := flag.Bool("migrate", false, "run database migrations") + migrate := flag.Bool("migrate", false, "apply pending server (MariaDB) SQL migrations and exit") + migrateStatus := flag.Bool("migrate-status", false, "list server SQL migrations with applied/pending state and exit") + migrateBaseline := flag.String("migrate-baseline", "", "record server SQL migrations up to and including this file as applied, without running them, and exit") version := flag.Bool("version", false, "show version information") flag.Parse() @@ -185,26 +187,25 @@ func main() { "online", false, ) - if *migrate { - mariaDB, err := connMgr.GetDB() + if *migrate || *migrateStatus || *migrateBaseline != "" { + mariaDB, err := openMigrationDB(local) if err != nil { slog.Error("cannot run migrations: database not available", "error", err) os.Exit(1) } - if mariaDB == nil { - slog.Error("cannot run migrations: database not available") + switch { + case *migrateStatus: + err = runMigrateStatus(mariaDB) + case *migrateBaseline != "": + err = runMigrateBaseline(mariaDB, *migrateBaseline) + default: + err = runMigrateApply(mariaDB) + } + if err != nil { + slog.Error("migration command failed", "error", err) os.Exit(1) } - slog.Info("running database migrations...") - if err := models.Migrate(mariaDB); err != nil { - slog.Error("migration failed", "error", err) - os.Exit(1) - } - if err := models.SeedCategories(mariaDB); err != nil { - slog.Error("seeding categories failed", "error", err) - os.Exit(1) - } - slog.Info("migrations completed") + os.Exit(0) } gin.SetMode(cfg.Server.Mode) diff --git a/cmd/qfs/migrate.go b/cmd/qfs/migrate.go new file mode 100644 index 0000000..aa42d4d --- /dev/null +++ b/cmd/qfs/migrate.go @@ -0,0 +1,123 @@ +package main + +import ( + "fmt" + "io/fs" + "time" + + qfassets "git.mchus.pro/mchus/quoteforge" + "git.mchus.pro/mchus/quoteforge/internal/localdb" + "git.mchus.pro/mchus/quoteforge/internal/models" + mysqlDriver "github.com/go-sql-driver/mysql" + "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +// Server (MariaDB) migration commands. Each runs only the embedded migrations/*.sql +// files tracked in qt_schema_migrations — never GORM AutoMigrate, which can silently +// retype production columns to match whatever models the running binary was built with. +// See bible-local/03-database.md "Migrations". + +// openMigrationDB opens a dedicated MariaDB connection for migrations. The app's shared +// connection (localdb.GetDSN) has 3s read/write timeouts for offline-first sync; a +// backfill UPDATE over qt_pricelist_items needs minutes, and a client-side timeout +// mid-statement leaves the file half-applied and unrecorded. +func openMigrationDB(local *localdb.LocalDB) (*gorm.DB, error) { + dsn, err := local.GetDSN() + if err != nil { + return nil, fmt.Errorf("server connection settings: %w", err) + } + cfg, err := mysqlDriver.ParseDSN(dsn) + if err != nil { + return nil, fmt.Errorf("parse DSN: %w", err) + } + cfg.Timeout = 10 * time.Second + cfg.ReadTimeout = 0 + cfg.WriteTimeout = 0 + db, err := gorm.Open(mysql.Open(cfg.FormatDSN()), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + return nil, fmt.Errorf("connect to server: %w", err) + } + return db, nil +} + +// runMigrateStatus prints every server migration and whether it is applied. +func runMigrateStatus(db *gorm.DB) error { + migrations, err := qfassets.MigrationsFS() + if err != nil { + return err + } + states, err := models.SQLMigrationStatus(db, migrations) + if err != nil { + return err + } + pending := 0 + for _, st := range states { + if st.AppliedAt != nil { + fmt.Printf(" applied %s (%s)\n", st.Filename, st.AppliedAt.Format("2006-01-02 15:04")) + } else { + fmt.Printf(" PENDING %s\n", st.Filename) + pending++ + } + } + fmt.Printf("%d migrations, %d pending\n", len(states), pending) + return nil +} + +// runMigrateApply applies pending server migrations. Refuses on a server with no +// migration history at all: its schema was built by other means, so every file would be +// re-run — record a baseline with -migrate-baseline first. +func runMigrateApply(db *gorm.DB) error { + migrations, err := qfassets.MigrationsFS() + if err != nil { + return err + } + if err := requireMigrationHistory(db, migrations); err != nil { + return err + } + applied, err := models.RunSQLMigrations(db, migrations, func(filename string) { + fmt.Printf(" applying %s ...\n", filename) + }) + if err != nil { + return err + } + if len(applied) == 0 { + fmt.Println("server schema is up to date — nothing to apply") + return nil + } + fmt.Printf("applied %d migration(s)\n", len(applied)) + return nil +} + +// runMigrateBaseline records migrations up to and including upto as applied without +// running them. +func runMigrateBaseline(db *gorm.DB, upto string) error { + migrations, err := qfassets.MigrationsFS() + if err != nil { + return err + } + recorded, err := models.BaselineSQLMigrations(db, migrations, upto) + for _, f := range recorded { + fmt.Printf(" recorded %s (not executed)\n", f) + } + if err != nil { + return err + } + fmt.Printf("baseline: %d migration(s) recorded as applied, up to %s\n", len(recorded), upto) + return nil +} + +func requireMigrationHistory(db *gorm.DB, migrations fs.FS) error { + states, err := models.SQLMigrationStatus(db, migrations) + if err != nil { + return err + } + for _, st := range states { + if st.AppliedAt != nil { + return nil + } + } + return fmt.Errorf("qt_schema_migrations has no applied migrations, so all %d files would run against this server. "+ + "Record what the server already has first: qfs -migrate-baseline (make migrate-baseline UPTO=...)", len(states)) +} diff --git a/internal/models/models.go b/internal/models/models.go index e2a2a9b..cb79d5e 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -1,57 +1,6 @@ package models -import ( - "log/slog" - "strings" - - "gorm.io/gorm" -) - -// AllModels returns all models for auto-migration -func AllModels() []interface{} { - return []interface{}{ - &Category{}, - &Project{}, - &Configuration{}, - &Pricelist{}, - &PricelistItem{}, - } -} - -// Migrate runs auto-migration for all QuoteForge tables -// Handles MySQL constraint errors gracefully for existing tables -func Migrate(db *gorm.DB) error { - for _, model := range AllModels() { - if err := db.AutoMigrate(model); err != nil { - // Skip known MySQL constraint errors for existing tables - errStr := err.Error() - if strings.Contains(errStr, "Can't DROP") || - strings.Contains(errStr, "Duplicate key name") || - strings.Contains(errStr, "check that it exists") || - strings.Contains(errStr, "Cannot change column") || - strings.Contains(errStr, "used in a foreign key constraint") { - slog.Warn("migration warning (skipped)", "model", model, "error", errStr) - continue - } - return err - } - } - return nil -} - -// SeedCategories upserts default categories, updating display_order on existing rows. -func SeedCategories(db *gorm.DB) error { - for _, cat := range DefaultCategories { - var existing Category - if err := db.Where("code = ?", cat.Code).First(&existing).Error; err != nil { - if err := db.Create(&cat).Error; err != nil { - return err - } - } else { - if err := db.Model(&existing).Update("display_order", cat.DisplayOrder).Error; err != nil { - return err - } - } - } - return nil -} +// Server schema changes are SQL files in migrations/, applied by `qfs -migrate` +// (models.RunSQLMigrations). There is deliberately no GORM AutoMigrate against +// MariaDB: it retypes production columns to match whatever models the running binary +// was built with — an old binary run with -migrate once did exactly that. diff --git a/internal/models/sql_migrations.go b/internal/models/sql_migrations.go index 711fe04..9f75136 100644 --- a/internal/models/sql_migrations.go +++ b/internal/models/sql_migrations.go @@ -4,8 +4,7 @@ import ( "bufio" "errors" "fmt" - "os" - "path/filepath" + "io/fs" "sort" "strings" "time" @@ -24,10 +23,87 @@ func (SQLSchemaMigration) TableName() string { return "qt_schema_migrations" } -// NeedsSQLMigrations reports whether at least one SQL migration from migrationsDir +// SQLMigrationState is one server migration file and when it was recorded as applied +// (nil = pending). +type SQLMigrationState struct { + Filename string + AppliedAt *time.Time +} + +// SQLMigrationStatus lists every server migration in migrations with its applied state. +// Read-only: does not create qt_schema_migrations. +func SQLMigrationStatus(db *gorm.DB, migrations fs.FS) ([]SQLMigrationState, error) { + files, err := listSQLMigrationFiles(migrations) + if err != nil { + return nil, err + } + applied := map[string]time.Time{} + if db.Migrator().HasTable(&SQLSchemaMigration{}) { + var rows []SQLSchemaMigration + if err := db.Find(&rows).Error; err != nil { + return nil, fmt.Errorf("read qt_schema_migrations: %w", err) + } + for _, r := range rows { + applied[r.Filename] = r.AppliedAt + } + } + states := make([]SQLMigrationState, 0, len(files)) + for _, f := range files { + st := SQLMigrationState{Filename: f} + if at, ok := applied[f]; ok { + at := at + st.AppliedAt = &at + } + states = append(states, st) + } + return states, nil +} + +// BaselineSQLMigrations records every migration up to and including upto as applied +// WITHOUT executing it — for a server whose schema was brought up to date by other +// means before qt_schema_migrations was used. Returns the filenames newly recorded. +func BaselineSQLMigrations(db *gorm.DB, migrations fs.FS, upto string) ([]string, error) { + files, err := listSQLMigrationFiles(migrations) + if err != nil { + return nil, err + } + found := false + for _, f := range files { + if f == upto { + found = true + break + } + } + if !found { + return nil, fmt.Errorf("baseline migration %q not found", upto) + } + if err := ensureSQLMigrationsTable(db); err != nil { + return nil, fmt.Errorf("migrate qt_schema_migrations table: %w", err) + } + var recorded []string + for _, f := range files { + if f > upto { + break + } + var count int64 + if err := db.Model(&SQLSchemaMigration{}).Where("filename = ?", f).Count(&count).Error; err != nil { + return recorded, fmt.Errorf("check migration %s: %w", f, err) + } + if count > 0 { + continue + } + if err := db.Create(&SQLSchemaMigration{Filename: f}).Error; err != nil { + return recorded, fmt.Errorf("record baseline migration %s: %w", f, err) + } + recorded = append(recorded, f) + } + return recorded, nil +} + +// NeedsSQLMigrations reports whether at least one SQL migration from migrations // is not yet recorded in qt_schema_migrations. -func NeedsSQLMigrations(db *gorm.DB, migrationsDir string) (bool, error) { - files, err := listSQLMigrationFiles(migrationsDir) +func NeedsSQLMigrations(db *gorm.DB, migrations fs.FS) (bool, error) { + files, err := listSQLMigrationFiles(migrations) if err != nil { return false, err } @@ -48,50 +124,51 @@ func NeedsSQLMigrations(db *gorm.DB, migrationsDir string) (bool, error) { return count < int64(len(files)), nil } -// RunSQLMigrations applies SQL files from migrationsDir once and records them in qt_schema_migrations. -// Local SQLite-only scripts are skipped automatically. -func RunSQLMigrations(db *gorm.DB, migrationsDir string) error { +// RunSQLMigrations applies pending SQL files from migrations once each, in filename +// order, and records them in qt_schema_migrations. onApply (optional) is called with +// each filename before it runs. Local SQLite-only scripts are skipped automatically. +// Returns the filenames applied. +func RunSQLMigrations(db *gorm.DB, migrations fs.FS, onApply func(filename string)) ([]string, error) { if err := ensureSQLMigrationsTable(db); err != nil { - return fmt.Errorf("migrate qt_schema_migrations table: %w", err) + return nil, fmt.Errorf("migrate qt_schema_migrations table: %w", err) } - files, err := listSQLMigrationFiles(migrationsDir) + files, err := listSQLMigrationFiles(migrations) if err != nil { - return err + return nil, err } + var applied []string + for _, filename := range files { var count int64 if err := db.Model(&SQLSchemaMigration{}).Where("filename = ?", filename).Count(&count).Error; err != nil { - return fmt.Errorf("check migration %s: %w", filename, err) + return applied, fmt.Errorf("check migration %s: %w", filename, err) } if count > 0 { continue } + if onApply != nil { + onApply(filename) + } - path := filepath.Join(migrationsDir, filename) - content, err := os.ReadFile(path) + content, err := fs.ReadFile(migrations, filename) if err != nil { - return fmt.Errorf("read migration %s: %w", filename, err) + return applied, fmt.Errorf("read migration %s: %w", filename, err) } - statements := splitSQLStatements(string(content)) - if len(statements) == 0 { - if err := db.Create(&SQLSchemaMigration{Filename: filename}).Error; err != nil { - return fmt.Errorf("record empty migration %s: %w", filename, err) + if statements := splitSQLStatements(string(content)); len(statements) > 0 { + if err := executeMigrationStatements(db, filename, statements); err != nil { + return applied, err } - continue - } - - if err := executeMigrationStatements(db, filename, statements); err != nil { - return err } if err := db.Create(&SQLSchemaMigration{Filename: filename}).Error; err != nil { - return fmt.Errorf("record migration %s: %w", filename, err) + return applied, fmt.Errorf("record migration %s: %w", filename, err) } + applied = append(applied, filename) } - return nil + return applied, nil } // IsMigrationPermissionError returns true if err indicates insufficient privileges @@ -199,13 +276,13 @@ func splitSQLStatements(script string) []string { return stmts } -func listSQLMigrationFiles(migrationsDir string) ([]string, error) { - entries, err := os.ReadDir(migrationsDir) +func listSQLMigrationFiles(migrations fs.FS) ([]string, error) { + entries, err := fs.ReadDir(migrations, ".") if err != nil { - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { return nil, nil } - return nil, fmt.Errorf("read migrations dir %s: %w", migrationsDir, err) + return nil, fmt.Errorf("read migrations dir: %w", err) } files := make([]string, 0, len(entries)) diff --git a/internal/models/sql_migrations_test.go b/internal/models/sql_migrations_test.go new file mode 100644 index 0000000..ae555ff --- /dev/null +++ b/internal/models/sql_migrations_test.go @@ -0,0 +1,53 @@ +package models + +import ( + "path/filepath" + "testing" + "testing/fstest" + + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func TestSQLMigrationStatusAndListing(t *testing.T) { + migrations := fstest.MapFS{ + "002_b.sql": {Data: []byte("ALTER TABLE x ADD COLUMN b INT;")}, + "001_a.sql": {Data: []byte("ALTER TABLE x ADD COLUMN a INT;")}, + "006_local_only_versions.sql": {Data: []byte("-- sqlite only")}, + "README.md": {Data: []byte("not a migration")}, + } + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "m.db")), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + + // No history table yet: everything pending, nothing created. + states, err := SQLMigrationStatus(db, migrations) + if err != nil { + t.Fatal(err) + } + if len(states) != 2 || states[0].Filename != "001_a.sql" || states[1].Filename != "002_b.sql" { + t.Fatalf("unexpected listing %+v", states) + } + if states[0].AppliedAt != nil || db.Migrator().HasTable(&SQLSchemaMigration{}) { + t.Fatal("status must be read-only and report pending") + } + + if err := db.AutoMigrate(&SQLSchemaMigration{}); err != nil { + t.Fatal(err) + } + if err := db.Create(&SQLSchemaMigration{Filename: "001_a.sql"}).Error; err != nil { + t.Fatal(err) + } + states, _ = SQLMigrationStatus(db, migrations) + if states[0].AppliedAt == nil || states[1].AppliedAt != nil { + t.Fatalf("want 001 applied, 002 pending: %+v", states) + } +} + +func TestSplitSQLStatementsSkipsComments(t *testing.T) { + got := splitSQLStatements("-- header; with semicolon\nALTER TABLE a ADD COLUMN x INT;\n\nALTER TABLE b\n ADD COLUMN y INT;\n") + if len(got) != 2 || got[1] != "ALTER TABLE b\n ADD COLUMN y INT" { + t.Fatalf("unexpected statements %q", got) + } +}