feat: make migrate — серверные SQL-миграции вместо AutoMigrate

`qfs -migrate` теперь применяет только migrations/*.sql (встроены в бинарник)
с учётом в qt_schema_migrations и завершается, не запуская приложение.
Новые команды: make migrate-status (read-only список applied/pending),
make migrate, make migrate-baseline UPTO=<file> (пометить как применённые
без выполнения). migrate отказывается работать, если в истории нет ни одной
применённой миграции — иначе все файлы пошли бы заново.

Удалён GORM AutoMigrate против MariaDB (models.Migrate, SeedCategories):
устаревший бинарник v1.18 с -migrate прогнал по боевой базе AutoMigrate
своих моделей. Миграции идут через отдельное соединение без read/write
таймаутов — 3s-таймаут sync-соединения обрывал backfill в 032.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mikhail ChusavitinandClaude Opus 5.5 committed 2026-09-24 12:21:53 +03:00
1 parent 259bd48d2e
commit 86164e044c
12 files changed
+374 -107

No files matched your search

+19 -1
View File
@@ -1,4 +1,4 @@
.PHONY: build build-release clean test run version install-hooks
.PHONY: build build-release clean test run version install-hooks migrate migrate-status migrate-baseline
# Get version from git
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
@@ -55,6 +55,24 @@ clean:
rm -rf bin/
rm -f $(BINARY)
# Server (MariaDB) migrations — migrations/*.sql tracked in qt_schema_migrations.
# Always built from the current source (go run), never a possibly stale bin/ binary.
# Uses the connection saved in the local app settings.
# List migrations with applied/pending state (read-only)
migrate-status:
go run ./cmd/qfs -migrate-status
# Apply pending migrations
migrate:
go run ./cmd/qfs -migrate
# Record migrations up to UPTO as applied WITHOUT running them (one-time, for a server
# whose schema is already current): make migrate-baseline UPTO=033_add_price_quality_to_pricelist_items.sql
migrate-baseline:
@test -n "$(UPTO)" || (echo "usage: make migrate-baseline UPTO=<file.sql>"; exit 1)
go run ./cmd/qfs -migrate-baseline "$(UPTO)"
# Run tests
test:
go test -v ./...
+2 -1
View File
@@ -24,7 +24,8 @@ go run ./cmd/qfs
Useful commands:
```bash
go run ./cmd/qfs -migrate
make migrate-status
make migrate
go test ./...
go vet ./...
make build-release
+11
View File
@@ -19,3 +19,14 @@ var StaticFiles embed.FS
func StaticFS() (fs.FS, error) {
return fs.Sub(StaticFiles, "web/static")
}
// MigrationFiles contains the MariaDB server migrations (migrations/*.sql), so
// `qfs -migrate` works from any directory. Applied via models.RunSQLMigrations.
//
//go:embed migrations/*.sql
var MigrationFiles embed.FS
// MigrationsFS returns a filesystem rooted at migrations/.
func MigrationsFS() (fs.FS, error) {
return fs.Sub(MigrationFiles, "migrations")
}
+16
View File
@@ -0,0 +1,16 @@
package quoteforge
import (
"io/fs"
"testing"
)
func TestMigrationsEmbedded(t *testing.T) {
migrations, err := MigrationsFS()
if err != nil {
t.Fatal(err)
}
if _, err := fs.Stat(migrations, "034_add_feature_flags_to_configurations.sql"); err != nil {
t.Fatalf("migration 034 not embedded: %v", err)
}
}
+18 -3
View File
@@ -57,7 +57,7 @@ Server-side only (not queried by client runtime):
- `qt_lot_metadata` — component metadata / price settings; the Go server-side component/category management layer (`ComponentRepository`, `CategoryRepository`, `ComponentService`) was removed — no client code reads this table
- `qt_component_usage_stats` — aggregated component popularity stats (written by server jobs)
- `qt_pricing_alerts` — price anomaly alerts (models exist in Go; feature disabled in runtime)
- `qt_schema_migrations` — server migration history (applied via `go run ./cmd/qfs -migrate`)
- `qt_schema_migrations` — server migration history (applied via `make migrate`, see [Migrations](#migrations))
- `qt_scheduler_runs` — server background job tracking (no Go code references it in this repo)
### Competitor subsystem (server-side only, not used by QuoteForge Go code)
@@ -427,5 +427,20 @@ SQLite:
- local migration state is tracked in `local_schema_migrations`.
MariaDB:
- SQL files live in `migrations/`;
- they are applied by `go run ./cmd/qfs -migrate`.
- SQL files live in `migrations/` (embedded into the binary), one file per change,
`NNN_short_topic.sql`, with the recovery/verify header used by the existing files;
filenames containing `local_` are SQLite-only and skipped;
- applied state is tracked per filename in `qt_schema_migrations`;
- commands (always from current source — never run a `bin/` binary of unknown age):
- `make migrate-status` (`qfs -migrate-status`) — list applied/pending, read-only;
- `make migrate` (`qfs -migrate`) — apply pending files in filename order, then exit;
refuses when `qt_schema_migrations` has no applied rows at all (every file would re-run);
- `make migrate-baseline UPTO=<file.sql>` (`qfs -migrate-baseline`) — record files up to
`UPTO` as applied **without running them**, for a server whose schema is already current;
- migrations use their own connection without read/write timeouts (`openMigrationDB`),
not the app's 3s-timeout sync connection — a long backfill must not be cut mid-file;
- a file is recorded only after all its statements succeed, so each file must be safe to
re-run (`ADD COLUMN IF NOT EXISTS`, idempotent `UPDATE ... WHERE <not yet done>`);
- there is no GORM `AutoMigrate` against MariaDB: it retypes production columns to match the
running binary's models. On 2026-09-24 a stale v1.18 binary run with the old AutoMigrate
`-migrate` did exactly that, which is why it was removed.
+3 -1
View File
@@ -63,7 +63,9 @@ They are stored in SQLite and encrypted with `local_encryption.key` unless `QUOT
| `-config <path>` | config file path |
| `-localdb <path>` | SQLite path |
| `-reset-localdb` | destructive local DB reset |
| `-migrate` | apply server migrations and exit |
| `-migrate` | apply pending server SQL migrations (`migrations/*.sql`) and exit |
| `-migrate-status` | list server SQL migrations with applied/pending state and exit |
| `-migrate-baseline <file>` | record server migrations up to `<file>` as applied without running them, and exit |
| `-version` | print app version and exit |
## First run
+2 -1
View File
@@ -4,7 +4,8 @@
```bash
go run ./cmd/qfs
go run ./cmd/qfs -migrate
make migrate-status # server SQL migrations: applied/pending
make migrate # apply pending server SQL migrations
go run ./cmd/migrate_project_updated_at
go test ./...
go vet ./...
+16 -15
View File
@@ -60,7 +60,9 @@ func main() {
configPath := flag.String("config", "", "path to config file (default: user state dir or QFS_CONFIG_PATH)")
localDBPath := flag.String("localdb", "", "path to local SQLite database (default: user state dir or QFS_DB_PATH)")
resetLocalDB := flag.Bool("reset-localdb", false, "reset local SQLite data on startup (keeps connection settings)")
migrate := flag.Bool("migrate", false, "run database migrations")
migrate := flag.Bool("migrate", false, "apply pending server (MariaDB) SQL migrations and exit")
migrateStatus := flag.Bool("migrate-status", false, "list server SQL migrations with applied/pending state and exit")
migrateBaseline := flag.String("migrate-baseline", "", "record server SQL migrations up to and including this file as applied, without running them, and exit")
version := flag.Bool("version", false, "show version information")
flag.Parse()
@@ -185,26 +187,25 @@ func main() {
"online", false,
)
if *migrate {
mariaDB, err := connMgr.GetDB()
if *migrate || *migrateStatus || *migrateBaseline != "" {
mariaDB, err := openMigrationDB(local)
if err != nil {
slog.Error("cannot run migrations: database not available", "error", err)
os.Exit(1)
}
if mariaDB == nil {
slog.Error("cannot run migrations: database not available")
switch {
case *migrateStatus:
err = runMigrateStatus(mariaDB)
case *migrateBaseline != "":
err = runMigrateBaseline(mariaDB, *migrateBaseline)
default:
err = runMigrateApply(mariaDB)
}
if err != nil {
slog.Error("migration command failed", "error", err)
os.Exit(1)
}
slog.Info("running database migrations...")
if err := models.Migrate(mariaDB); err != nil {
slog.Error("migration failed", "error", err)
os.Exit(1)
}
if err := models.SeedCategories(mariaDB); err != nil {
slog.Error("seeding categories failed", "error", err)
os.Exit(1)
}
slog.Info("migrations completed")
os.Exit(0)
}
gin.SetMode(cfg.Server.Mode)
+123
View File
@@ -0,0 +1,123 @@
package main
import (
"fmt"
"io/fs"
"time"
qfassets "git.mchus.pro/mchus/quoteforge"
"git.mchus.pro/mchus/quoteforge/internal/localdb"
"git.mchus.pro/mchus/quoteforge/internal/models"
mysqlDriver "github.com/go-sql-driver/mysql"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
// Server (MariaDB) migration commands. Each runs only the embedded migrations/*.sql
// files tracked in qt_schema_migrations — never GORM AutoMigrate, which can silently
// retype production columns to match whatever models the running binary was built with.
// See bible-local/03-database.md "Migrations".
// openMigrationDB opens a dedicated MariaDB connection for migrations. The app's shared
// connection (localdb.GetDSN) has 3s read/write timeouts for offline-first sync; a
// backfill UPDATE over qt_pricelist_items needs minutes, and a client-side timeout
// mid-statement leaves the file half-applied and unrecorded.
func openMigrationDB(local *localdb.LocalDB) (*gorm.DB, error) {
dsn, err := local.GetDSN()
if err != nil {
return nil, fmt.Errorf("server connection settings: %w", err)
}
cfg, err := mysqlDriver.ParseDSN(dsn)
if err != nil {
return nil, fmt.Errorf("parse DSN: %w", err)
}
cfg.Timeout = 10 * time.Second
cfg.ReadTimeout = 0
cfg.WriteTimeout = 0
db, err := gorm.Open(mysql.Open(cfg.FormatDSN()), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
if err != nil {
return nil, fmt.Errorf("connect to server: %w", err)
}
return db, nil
}
// runMigrateStatus prints every server migration and whether it is applied.
func runMigrateStatus(db *gorm.DB) error {
migrations, err := qfassets.MigrationsFS()
if err != nil {
return err
}
states, err := models.SQLMigrationStatus(db, migrations)
if err != nil {
return err
}
pending := 0
for _, st := range states {
if st.AppliedAt != nil {
fmt.Printf(" applied %s (%s)\n", st.Filename, st.AppliedAt.Format("2006-01-02 15:04"))
} else {
fmt.Printf(" PENDING %s\n", st.Filename)
pending++
}
}
fmt.Printf("%d migrations, %d pending\n", len(states), pending)
return nil
}
// runMigrateApply applies pending server migrations. Refuses on a server with no
// migration history at all: its schema was built by other means, so every file would be
// re-run — record a baseline with -migrate-baseline first.
func runMigrateApply(db *gorm.DB) error {
migrations, err := qfassets.MigrationsFS()
if err != nil {
return err
}
if err := requireMigrationHistory(db, migrations); err != nil {
return err
}
applied, err := models.RunSQLMigrations(db, migrations, func(filename string) {
fmt.Printf(" applying %s ...\n", filename)
})
if err != nil {
return err
}
if len(applied) == 0 {
fmt.Println("server schema is up to date — nothing to apply")
return nil
}
fmt.Printf("applied %d migration(s)\n", len(applied))
return nil
}
// runMigrateBaseline records migrations up to and including upto as applied without
// running them.
func runMigrateBaseline(db *gorm.DB, upto string) error {
migrations, err := qfassets.MigrationsFS()
if err != nil {
return err
}
recorded, err := models.BaselineSQLMigrations(db, migrations, upto)
for _, f := range recorded {
fmt.Printf(" recorded %s (not executed)\n", f)
}
if err != nil {
return err
}
fmt.Printf("baseline: %d migration(s) recorded as applied, up to %s\n", len(recorded), upto)
return nil
}
func requireMigrationHistory(db *gorm.DB, migrations fs.FS) error {
states, err := models.SQLMigrationStatus(db, migrations)
if err != nil {
return err
}
for _, st := range states {
if st.AppliedAt != nil {
return nil
}
}
return fmt.Errorf("qt_schema_migrations has no applied migrations, so all %d files would run against this server. "+
"Record what the server already has first: qfs -migrate-baseline <last-applied-file.sql> (make migrate-baseline UPTO=...)", len(states))
}
+4 -55
View File
@@ -1,57 +1,6 @@
package models
import (
"log/slog"
"strings"
"gorm.io/gorm"
)
// AllModels returns all models for auto-migration
func AllModels() []interface{} {
return []interface{}{
&Category{},
&Project{},
&Configuration{},
&Pricelist{},
&PricelistItem{},
}
}
// Migrate runs auto-migration for all QuoteForge tables
// Handles MySQL constraint errors gracefully for existing tables
func Migrate(db *gorm.DB) error {
for _, model := range AllModels() {
if err := db.AutoMigrate(model); err != nil {
// Skip known MySQL constraint errors for existing tables
errStr := err.Error()
if strings.Contains(errStr, "Can't DROP") ||
strings.Contains(errStr, "Duplicate key name") ||
strings.Contains(errStr, "check that it exists") ||
strings.Contains(errStr, "Cannot change column") ||
strings.Contains(errStr, "used in a foreign key constraint") {
slog.Warn("migration warning (skipped)", "model", model, "error", errStr)
continue
}
return err
}
}
return nil
}
// SeedCategories upserts default categories, updating display_order on existing rows.
func SeedCategories(db *gorm.DB) error {
for _, cat := range DefaultCategories {
var existing Category
if err := db.Where("code = ?", cat.Code).First(&existing).Error; err != nil {
if err := db.Create(&cat).Error; err != nil {
return err
}
} else {
if err := db.Model(&existing).Update("display_order", cat.DisplayOrder).Error; err != nil {
return err
}
}
}
return nil
}
// Server schema changes are SQL files in migrations/, applied by `qfs -migrate`
// (models.RunSQLMigrations). There is deliberately no GORM AutoMigrate against
// MariaDB: it retypes production columns to match whatever models the running binary
// was built with — an old binary run with -migrate once did exactly that.
+107 -30
View File
@@ -4,8 +4,7 @@ import (
"bufio"
"errors"
"fmt"
"os"
"path/filepath"
"io/fs"
"sort"
"strings"
"time"
@@ -24,10 +23,87 @@ func (SQLSchemaMigration) TableName() string {
return "qt_schema_migrations"
}
// NeedsSQLMigrations reports whether at least one SQL migration from migrationsDir
// SQLMigrationState is one server migration file and when it was recorded as applied
// (nil = pending).
type SQLMigrationState struct {
Filename string
AppliedAt *time.Time
}
// SQLMigrationStatus lists every server migration in migrations with its applied state.
// Read-only: does not create qt_schema_migrations.
func SQLMigrationStatus(db *gorm.DB, migrations fs.FS) ([]SQLMigrationState, error) {
files, err := listSQLMigrationFiles(migrations)
if err != nil {
return nil, err
}
applied := map[string]time.Time{}
if db.Migrator().HasTable(&SQLSchemaMigration{}) {
var rows []SQLSchemaMigration
if err := db.Find(&rows).Error; err != nil {
return nil, fmt.Errorf("read qt_schema_migrations: %w", err)
}
for _, r := range rows {
applied[r.Filename] = r.AppliedAt
}
}
states := make([]SQLMigrationState, 0, len(files))
for _, f := range files {
st := SQLMigrationState{Filename: f}
if at, ok := applied[f]; ok {
at := at
st.AppliedAt = &at
}
states = append(states, st)
}
return states, nil
}
// BaselineSQLMigrations records every migration up to and including upto as applied
// WITHOUT executing it — for a server whose schema was brought up to date by other
// means before qt_schema_migrations was used. Returns the filenames newly recorded.
func BaselineSQLMigrations(db *gorm.DB, migrations fs.FS, upto string) ([]string, error) {
files, err := listSQLMigrationFiles(migrations)
if err != nil {
return nil, err
}
found := false
for _, f := range files {
if f == upto {
found = true
break
}
}
if !found {
return nil, fmt.Errorf("baseline migration %q not found", upto)
}
if err := ensureSQLMigrationsTable(db); err != nil {
return nil, fmt.Errorf("migrate qt_schema_migrations table: %w", err)
}
var recorded []string
for _, f := range files {
if f > upto {
break
}
var count int64
if err := db.Model(&SQLSchemaMigration{}).Where("filename = ?", f).Count(&count).Error; err != nil {
return recorded, fmt.Errorf("check migration %s: %w", f, err)
}
if count > 0 {
continue
}
if err := db.Create(&SQLSchemaMigration{Filename: f}).Error; err != nil {
return recorded, fmt.Errorf("record baseline migration %s: %w", f, err)
}
recorded = append(recorded, f)
}
return recorded, nil
}
// NeedsSQLMigrations reports whether at least one SQL migration from migrations
// is not yet recorded in qt_schema_migrations.
func NeedsSQLMigrations(db *gorm.DB, migrationsDir string) (bool, error) {
files, err := listSQLMigrationFiles(migrationsDir)
func NeedsSQLMigrations(db *gorm.DB, migrations fs.FS) (bool, error) {
files, err := listSQLMigrationFiles(migrations)
if err != nil {
return false, err
}
@@ -48,50 +124,51 @@ func NeedsSQLMigrations(db *gorm.DB, migrationsDir string) (bool, error) {
return count < int64(len(files)), nil
}
// RunSQLMigrations applies SQL files from migrationsDir once and records them in qt_schema_migrations.
// Local SQLite-only scripts are skipped automatically.
func RunSQLMigrations(db *gorm.DB, migrationsDir string) error {
// RunSQLMigrations applies pending SQL files from migrations once each, in filename
// order, and records them in qt_schema_migrations. onApply (optional) is called with
// each filename before it runs. Local SQLite-only scripts are skipped automatically.
// Returns the filenames applied.
func RunSQLMigrations(db *gorm.DB, migrations fs.FS, onApply func(filename string)) ([]string, error) {
if err := ensureSQLMigrationsTable(db); err != nil {
return fmt.Errorf("migrate qt_schema_migrations table: %w", err)
return nil, fmt.Errorf("migrate qt_schema_migrations table: %w", err)
}
files, err := listSQLMigrationFiles(migrationsDir)
files, err := listSQLMigrationFiles(migrations)
if err != nil {
return err
return nil, err
}
var applied []string
for _, filename := range files {
var count int64
if err := db.Model(&SQLSchemaMigration{}).Where("filename = ?", filename).Count(&count).Error; err != nil {
return fmt.Errorf("check migration %s: %w", filename, err)
return applied, fmt.Errorf("check migration %s: %w", filename, err)
}
if count > 0 {
continue
}
if onApply != nil {
onApply(filename)
}
path := filepath.Join(migrationsDir, filename)
content, err := os.ReadFile(path)
content, err := fs.ReadFile(migrations, filename)
if err != nil {
return fmt.Errorf("read migration %s: %w", filename, err)
return applied, fmt.Errorf("read migration %s: %w", filename, err)
}
statements := splitSQLStatements(string(content))
if len(statements) == 0 {
if err := db.Create(&SQLSchemaMigration{Filename: filename}).Error; err != nil {
return fmt.Errorf("record empty migration %s: %w", filename, err)
if statements := splitSQLStatements(string(content)); len(statements) > 0 {
if err := executeMigrationStatements(db, filename, statements); err != nil {
return applied, err
}
continue
}
if err := executeMigrationStatements(db, filename, statements); err != nil {
return err
}
if err := db.Create(&SQLSchemaMigration{Filename: filename}).Error; err != nil {
return fmt.Errorf("record migration %s: %w", filename, err)
return applied, fmt.Errorf("record migration %s: %w", filename, err)
}
applied = append(applied, filename)
}
return nil
return applied, nil
}
// IsMigrationPermissionError returns true if err indicates insufficient privileges
@@ -199,13 +276,13 @@ func splitSQLStatements(script string) []string {
return stmts
}
func listSQLMigrationFiles(migrationsDir string) ([]string, error) {
entries, err := os.ReadDir(migrationsDir)
func listSQLMigrationFiles(migrations fs.FS) ([]string, error) {
entries, err := fs.ReadDir(migrations, ".")
if err != nil {
if errors.Is(err, os.ErrNotExist) {
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
return nil, fmt.Errorf("read migrations dir %s: %w", migrationsDir, err)
return nil, fmt.Errorf("read migrations dir: %w", err)
}
files := make([]string, 0, len(entries))
+53
View File
@@ -0,0 +1,53 @@
package models
import (
"path/filepath"
"testing"
"testing/fstest"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
)
func TestSQLMigrationStatusAndListing(t *testing.T) {
migrations := fstest.MapFS{
"002_b.sql": {Data: []byte("ALTER TABLE x ADD COLUMN b INT;")},
"001_a.sql": {Data: []byte("ALTER TABLE x ADD COLUMN a INT;")},
"006_local_only_versions.sql": {Data: []byte("-- sqlite only")},
"README.md": {Data: []byte("not a migration")},
}
db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "m.db")), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
// No history table yet: everything pending, nothing created.
states, err := SQLMigrationStatus(db, migrations)
if err != nil {
t.Fatal(err)
}
if len(states) != 2 || states[0].Filename != "001_a.sql" || states[1].Filename != "002_b.sql" {
t.Fatalf("unexpected listing %+v", states)
}
if states[0].AppliedAt != nil || db.Migrator().HasTable(&SQLSchemaMigration{}) {
t.Fatal("status must be read-only and report pending")
}
if err := db.AutoMigrate(&SQLSchemaMigration{}); err != nil {
t.Fatal(err)
}
if err := db.Create(&SQLSchemaMigration{Filename: "001_a.sql"}).Error; err != nil {
t.Fatal(err)
}
states, _ = SQLMigrationStatus(db, migrations)
if states[0].AppliedAt == nil || states[1].AppliedAt != nil {
t.Fatalf("want 001 applied, 002 pending: %+v", states)
}
}
func TestSplitSQLStatementsSkipsComments(t *testing.T) {
got := splitSQLStatements("-- header; with semicolon\nALTER TABLE a ADD COLUMN x INT;\n\nALTER TABLE b\n ADD COLUMN y INT;\n")
if len(got) != 2 || got[1] != "ALTER TABLE b\n ADD COLUMN y INT" {
t.Fatalf("unexpected statements %q", got)
}
}