feat: make migrate — серверные SQL-миграции вместо AutoMigrate
`qfs -migrate` теперь применяет только migrations/*.sql (встроены в бинарник) с учётом в qt_schema_migrations и завершается, не запуская приложение. Новые команды: make migrate-status (read-only список applied/pending), make migrate, make migrate-baseline UPTO=<file> (пометить как применённые без выполнения). migrate отказывается работать, если в истории нет ни одной применённой миграции — иначе все файлы пошли бы заново. Удалён GORM AutoMigrate против MariaDB (models.Migrate, SeedCategories): устаревший бинарник v1.18 с -migrate прогнал по боевой базе AutoMigrate своих моделей. Миграции идут через отдельное соединение без read/write таймаутов — 3s-таймаут sync-соединения обрывал backfill в 032. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
259bd48d2e
commit
a2a1904d91
12 files changed
+374
-107
No files matched your search
@@ -1,4 +1,4 @@
|
|||||||
.PHONY: build build-release clean test run version install-hooks
|
.PHONY: build build-release clean test run version install-hooks migrate migrate-status migrate-baseline
|
||||||
|
|
||||||
# Get version from git
|
# Get version from git
|
||||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
|
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
|
||||||
@@ -55,6 +55,24 @@ clean:
|
|||||||
rm -rf bin/
|
rm -rf bin/
|
||||||
rm -f $(BINARY)
|
rm -f $(BINARY)
|
||||||
|
|
||||||
|
# Server (MariaDB) migrations — migrations/*.sql tracked in qt_schema_migrations.
|
||||||
|
# Always built from the current source (go run), never a possibly stale bin/ binary.
|
||||||
|
# Uses the connection saved in the local app settings.
|
||||||
|
|
||||||
|
# List migrations with applied/pending state (read-only)
|
||||||
|
migrate-status:
|
||||||
|
go run ./cmd/qfs -migrate-status
|
||||||
|
|
||||||
|
# Apply pending migrations
|
||||||
|
migrate:
|
||||||
|
go run ./cmd/qfs -migrate
|
||||||
|
|
||||||
|
# Record migrations up to UPTO as applied WITHOUT running them (one-time, for a server
|
||||||
|
# whose schema is already current): make migrate-baseline UPTO=033_add_price_quality_to_pricelist_items.sql
|
||||||
|
migrate-baseline:
|
||||||
|
@test -n "$(UPTO)" || (echo "usage: make migrate-baseline UPTO=<file.sql>"; exit 1)
|
||||||
|
go run ./cmd/qfs -migrate-baseline "$(UPTO)"
|
||||||
|
|
||||||
# Run tests
|
# Run tests
|
||||||
test:
|
test:
|
||||||
go test -v ./...
|
go test -v ./...
|
||||||
|
|||||||
@@ -24,7 +24,8 @@ go run ./cmd/qfs
|
|||||||
Useful commands:
|
Useful commands:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go run ./cmd/qfs -migrate
|
make migrate-status
|
||||||
|
make migrate
|
||||||
go test ./...
|
go test ./...
|
||||||
go vet ./...
|
go vet ./...
|
||||||
make build-release
|
make build-release
|
||||||
|
|||||||
@@ -19,3 +19,14 @@ var StaticFiles embed.FS
|
|||||||
func StaticFS() (fs.FS, error) {
|
func StaticFS() (fs.FS, error) {
|
||||||
return fs.Sub(StaticFiles, "web/static")
|
return fs.Sub(StaticFiles, "web/static")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MigrationFiles contains the MariaDB server migrations (migrations/*.sql), so
|
||||||
|
// `qfs -migrate` works from any directory. Applied via models.RunSQLMigrations.
|
||||||
|
//
|
||||||
|
//go:embed migrations/*.sql
|
||||||
|
var MigrationFiles embed.FS
|
||||||
|
|
||||||
|
// MigrationsFS returns a filesystem rooted at migrations/.
|
||||||
|
func MigrationsFS() (fs.FS, error) {
|
||||||
|
return fs.Sub(MigrationFiles, "migrations")
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package quoteforge
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMigrationsEmbedded(t *testing.T) {
|
||||||
|
migrations, err := MigrationsFS()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := fs.Stat(migrations, "034_add_feature_flags_to_configurations.sql"); err != nil {
|
||||||
|
t.Fatalf("migration 034 not embedded: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -57,7 +57,7 @@ Server-side only (not queried by client runtime):
|
|||||||
- `qt_lot_metadata` — component metadata / price settings; the Go server-side component/category management layer (`ComponentRepository`, `CategoryRepository`, `ComponentService`) was removed — no client code reads this table
|
- `qt_lot_metadata` — component metadata / price settings; the Go server-side component/category management layer (`ComponentRepository`, `CategoryRepository`, `ComponentService`) was removed — no client code reads this table
|
||||||
- `qt_component_usage_stats` — aggregated component popularity stats (written by server jobs)
|
- `qt_component_usage_stats` — aggregated component popularity stats (written by server jobs)
|
||||||
- `qt_pricing_alerts` — price anomaly alerts (models exist in Go; feature disabled in runtime)
|
- `qt_pricing_alerts` — price anomaly alerts (models exist in Go; feature disabled in runtime)
|
||||||
- `qt_schema_migrations` — server migration history (applied via `go run ./cmd/qfs -migrate`)
|
- `qt_schema_migrations` — server migration history (applied via `make migrate`, see [Migrations](#migrations))
|
||||||
- `qt_scheduler_runs` — server background job tracking (no Go code references it in this repo)
|
- `qt_scheduler_runs` — server background job tracking (no Go code references it in this repo)
|
||||||
|
|
||||||
### Competitor subsystem (server-side only, not used by QuoteForge Go code)
|
### Competitor subsystem (server-side only, not used by QuoteForge Go code)
|
||||||
@@ -427,5 +427,20 @@ SQLite:
|
|||||||
- local migration state is tracked in `local_schema_migrations`.
|
- local migration state is tracked in `local_schema_migrations`.
|
||||||
|
|
||||||
MariaDB:
|
MariaDB:
|
||||||
- SQL files live in `migrations/`;
|
- SQL files live in `migrations/` (embedded into the binary), one file per change,
|
||||||
- they are applied by `go run ./cmd/qfs -migrate`.
|
`NNN_short_topic.sql`, with the recovery/verify header used by the existing files;
|
||||||
|
filenames containing `local_` are SQLite-only and skipped;
|
||||||
|
- applied state is tracked per filename in `qt_schema_migrations`;
|
||||||
|
- commands (always from current source — never run a `bin/` binary of unknown age):
|
||||||
|
- `make migrate-status` (`qfs -migrate-status`) — list applied/pending, read-only;
|
||||||
|
- `make migrate` (`qfs -migrate`) — apply pending files in filename order, then exit;
|
||||||
|
refuses when `qt_schema_migrations` has no applied rows at all (every file would re-run);
|
||||||
|
- `make migrate-baseline UPTO=<file.sql>` (`qfs -migrate-baseline`) — record files up to
|
||||||
|
`UPTO` as applied **without running them**, for a server whose schema is already current;
|
||||||
|
- migrations use their own connection without read/write timeouts (`openMigrationDB`),
|
||||||
|
not the app's 3s-timeout sync connection — a long backfill must not be cut mid-file;
|
||||||
|
- a file is recorded only after all its statements succeed, so each file must be safe to
|
||||||
|
re-run (`ADD COLUMN IF NOT EXISTS`, idempotent `UPDATE ... WHERE <not yet done>`);
|
||||||
|
- there is no GORM `AutoMigrate` against MariaDB: it retypes production columns to match the
|
||||||
|
running binary's models. On 2026-09-24 a stale v1.18 binary run with the old AutoMigrate
|
||||||
|
`-migrate` did exactly that, which is why it was removed.
|
||||||
@@ -63,7 +63,9 @@ They are stored in SQLite and encrypted with `local_encryption.key` unless `QUOT
|
|||||||
| `-config <path>` | config file path |
|
| `-config <path>` | config file path |
|
||||||
| `-localdb <path>` | SQLite path |
|
| `-localdb <path>` | SQLite path |
|
||||||
| `-reset-localdb` | destructive local DB reset |
|
| `-reset-localdb` | destructive local DB reset |
|
||||||
| `-migrate` | apply server migrations and exit |
|
| `-migrate` | apply pending server SQL migrations (`migrations/*.sql`) and exit |
|
||||||
|
| `-migrate-status` | list server SQL migrations with applied/pending state and exit |
|
||||||
|
| `-migrate-baseline <file>` | record server migrations up to `<file>` as applied without running them, and exit |
|
||||||
| `-version` | print app version and exit |
|
| `-version` | print app version and exit |
|
||||||
|
|
||||||
## First run
|
## First run
|
||||||
|
|||||||
@@ -4,7 +4,8 @@
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
go run ./cmd/qfs
|
go run ./cmd/qfs
|
||||||
go run ./cmd/qfs -migrate
|
make migrate-status # server SQL migrations: applied/pending
|
||||||
|
make migrate # apply pending server SQL migrations
|
||||||
go run ./cmd/migrate_project_updated_at
|
go run ./cmd/migrate_project_updated_at
|
||||||
go test ./...
|
go test ./...
|
||||||
go vet ./...
|
go vet ./...
|
||||||
|
|||||||
+16
-15
@@ -60,7 +60,9 @@ func main() {
|
|||||||
configPath := flag.String("config", "", "path to config file (default: user state dir or QFS_CONFIG_PATH)")
|
configPath := flag.String("config", "", "path to config file (default: user state dir or QFS_CONFIG_PATH)")
|
||||||
localDBPath := flag.String("localdb", "", "path to local SQLite database (default: user state dir or QFS_DB_PATH)")
|
localDBPath := flag.String("localdb", "", "path to local SQLite database (default: user state dir or QFS_DB_PATH)")
|
||||||
resetLocalDB := flag.Bool("reset-localdb", false, "reset local SQLite data on startup (keeps connection settings)")
|
resetLocalDB := flag.Bool("reset-localdb", false, "reset local SQLite data on startup (keeps connection settings)")
|
||||||
migrate := flag.Bool("migrate", false, "run database migrations")
|
migrate := flag.Bool("migrate", false, "apply pending server (MariaDB) SQL migrations and exit")
|
||||||
|
migrateStatus := flag.Bool("migrate-status", false, "list server SQL migrations with applied/pending state and exit")
|
||||||
|
migrateBaseline := flag.String("migrate-baseline", "", "record server SQL migrations up to and including this file as applied, without running them, and exit")
|
||||||
version := flag.Bool("version", false, "show version information")
|
version := flag.Bool("version", false, "show version information")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
@@ -185,26 +187,25 @@ func main() {
|
|||||||
"online", false,
|
"online", false,
|
||||||
)
|
)
|
||||||
|
|
||||||
if *migrate {
|
if *migrate || *migrateStatus || *migrateBaseline != "" {
|
||||||
mariaDB, err := connMgr.GetDB()
|
mariaDB, err := openMigrationDB(local)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("cannot run migrations: database not available", "error", err)
|
slog.Error("cannot run migrations: database not available", "error", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
if mariaDB == nil {
|
switch {
|
||||||
slog.Error("cannot run migrations: database not available")
|
case *migrateStatus:
|
||||||
|
err = runMigrateStatus(mariaDB)
|
||||||
|
case *migrateBaseline != "":
|
||||||
|
err = runMigrateBaseline(mariaDB, *migrateBaseline)
|
||||||
|
default:
|
||||||
|
err = runMigrateApply(mariaDB)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
slog.Error("migration command failed", "error", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
slog.Info("running database migrations...")
|
os.Exit(0)
|
||||||
if err := models.Migrate(mariaDB); err != nil {
|
|
||||||
slog.Error("migration failed", "error", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
if err := models.SeedCategories(mariaDB); err != nil {
|
|
||||||
slog.Error("seeding categories failed", "error", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
slog.Info("migrations completed")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
gin.SetMode(cfg.Server.Mode)
|
gin.SetMode(cfg.Server.Mode)
|
||||||
|
|||||||
@@ -0,0 +1,123 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
qfassets "git.mchus.pro/mchus/quoteforge"
|
||||||
|
"git.mchus.pro/mchus/quoteforge/internal/localdb"
|
||||||
|
"git.mchus.pro/mchus/quoteforge/internal/models"
|
||||||
|
mysqlDriver "github.com/go-sql-driver/mysql"
|
||||||
|
"gorm.io/driver/mysql"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"gorm.io/gorm/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Server (MariaDB) migration commands. Each runs only the embedded migrations/*.sql
|
||||||
|
// files tracked in qt_schema_migrations — never GORM AutoMigrate, which can silently
|
||||||
|
// retype production columns to match whatever models the running binary was built with.
|
||||||
|
// See bible-local/03-database.md "Migrations".
|
||||||
|
|
||||||
|
// openMigrationDB opens a dedicated MariaDB connection for migrations. The app's shared
|
||||||
|
// connection (localdb.GetDSN) has 3s read/write timeouts for offline-first sync; a
|
||||||
|
// backfill UPDATE over qt_pricelist_items needs minutes, and a client-side timeout
|
||||||
|
// mid-statement leaves the file half-applied and unrecorded.
|
||||||
|
func openMigrationDB(local *localdb.LocalDB) (*gorm.DB, error) {
|
||||||
|
dsn, err := local.GetDSN()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("server connection settings: %w", err)
|
||||||
|
}
|
||||||
|
cfg, err := mysqlDriver.ParseDSN(dsn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse DSN: %w", err)
|
||||||
|
}
|
||||||
|
cfg.Timeout = 10 * time.Second
|
||||||
|
cfg.ReadTimeout = 0
|
||||||
|
cfg.WriteTimeout = 0
|
||||||
|
db, err := gorm.Open(mysql.Open(cfg.FormatDSN()), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("connect to server: %w", err)
|
||||||
|
}
|
||||||
|
return db, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// runMigrateStatus prints every server migration and whether it is applied.
|
||||||
|
func runMigrateStatus(db *gorm.DB) error {
|
||||||
|
migrations, err := qfassets.MigrationsFS()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
states, err := models.SQLMigrationStatus(db, migrations)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pending := 0
|
||||||
|
for _, st := range states {
|
||||||
|
if st.AppliedAt != nil {
|
||||||
|
fmt.Printf(" applied %s (%s)\n", st.Filename, st.AppliedAt.Format("2006-01-02 15:04"))
|
||||||
|
} else {
|
||||||
|
fmt.Printf(" PENDING %s\n", st.Filename)
|
||||||
|
pending++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf("%d migrations, %d pending\n", len(states), pending)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// runMigrateApply applies pending server migrations. Refuses on a server with no
|
||||||
|
// migration history at all: its schema was built by other means, so every file would be
|
||||||
|
// re-run — record a baseline with -migrate-baseline first.
|
||||||
|
func runMigrateApply(db *gorm.DB) error {
|
||||||
|
migrations, err := qfassets.MigrationsFS()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := requireMigrationHistory(db, migrations); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
applied, err := models.RunSQLMigrations(db, migrations, func(filename string) {
|
||||||
|
fmt.Printf(" applying %s ...\n", filename)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(applied) == 0 {
|
||||||
|
fmt.Println("server schema is up to date — nothing to apply")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("applied %d migration(s)\n", len(applied))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// runMigrateBaseline records migrations up to and including upto as applied without
|
||||||
|
// running them.
|
||||||
|
func runMigrateBaseline(db *gorm.DB, upto string) error {
|
||||||
|
migrations, err := qfassets.MigrationsFS()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
recorded, err := models.BaselineSQLMigrations(db, migrations, upto)
|
||||||
|
for _, f := range recorded {
|
||||||
|
fmt.Printf(" recorded %s (not executed)\n", f)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("baseline: %d migration(s) recorded as applied, up to %s\n", len(recorded), upto)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireMigrationHistory(db *gorm.DB, migrations fs.FS) error {
|
||||||
|
states, err := models.SQLMigrationStatus(db, migrations)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, st := range states {
|
||||||
|
if st.AppliedAt != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Errorf("qt_schema_migrations has no applied migrations, so all %d files would run against this server. "+
|
||||||
|
"Record what the server already has first: qfs -migrate-baseline <last-applied-file.sql> (make migrate-baseline UPTO=...)", len(states))
|
||||||
|
}
|
||||||
@@ -1,57 +1,6 @@
|
|||||||
package models
|
package models
|
||||||
|
|
||||||
import (
|
// Server schema changes are SQL files in migrations/, applied by `qfs -migrate`
|
||||||
"log/slog"
|
// (models.RunSQLMigrations). There is deliberately no GORM AutoMigrate against
|
||||||
"strings"
|
// MariaDB: it retypes production columns to match whatever models the running binary
|
||||||
|
// was built with — an old binary run with -migrate once did exactly that.
|
||||||
"gorm.io/gorm"
|
|
||||||
)
|
|
||||||
|
|
||||||
// AllModels returns all models for auto-migration
|
|
||||||
func AllModels() []interface{} {
|
|
||||||
return []interface{}{
|
|
||||||
&Category{},
|
|
||||||
&Project{},
|
|
||||||
&Configuration{},
|
|
||||||
&Pricelist{},
|
|
||||||
&PricelistItem{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Migrate runs auto-migration for all QuoteForge tables
|
|
||||||
// Handles MySQL constraint errors gracefully for existing tables
|
|
||||||
func Migrate(db *gorm.DB) error {
|
|
||||||
for _, model := range AllModels() {
|
|
||||||
if err := db.AutoMigrate(model); err != nil {
|
|
||||||
// Skip known MySQL constraint errors for existing tables
|
|
||||||
errStr := err.Error()
|
|
||||||
if strings.Contains(errStr, "Can't DROP") ||
|
|
||||||
strings.Contains(errStr, "Duplicate key name") ||
|
|
||||||
strings.Contains(errStr, "check that it exists") ||
|
|
||||||
strings.Contains(errStr, "Cannot change column") ||
|
|
||||||
strings.Contains(errStr, "used in a foreign key constraint") {
|
|
||||||
slog.Warn("migration warning (skipped)", "model", model, "error", errStr)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// SeedCategories upserts default categories, updating display_order on existing rows.
|
|
||||||
func SeedCategories(db *gorm.DB) error {
|
|
||||||
for _, cat := range DefaultCategories {
|
|
||||||
var existing Category
|
|
||||||
if err := db.Where("code = ?", cat.Code).First(&existing).Error; err != nil {
|
|
||||||
if err := db.Create(&cat).Error; err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if err := db.Model(&existing).Update("display_order", cat.DisplayOrder).Error; err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -4,8 +4,7 @@ import (
|
|||||||
"bufio"
|
"bufio"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"io/fs"
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -24,10 +23,87 @@ func (SQLSchemaMigration) TableName() string {
|
|||||||
return "qt_schema_migrations"
|
return "qt_schema_migrations"
|
||||||
}
|
}
|
||||||
|
|
||||||
// NeedsSQLMigrations reports whether at least one SQL migration from migrationsDir
|
// SQLMigrationState is one server migration file and when it was recorded as applied
|
||||||
|
// (nil = pending).
|
||||||
|
type SQLMigrationState struct {
|
||||||
|
Filename string
|
||||||
|
AppliedAt *time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// SQLMigrationStatus lists every server migration in migrations with its applied state.
|
||||||
|
// Read-only: does not create qt_schema_migrations.
|
||||||
|
func SQLMigrationStatus(db *gorm.DB, migrations fs.FS) ([]SQLMigrationState, error) {
|
||||||
|
files, err := listSQLMigrationFiles(migrations)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
applied := map[string]time.Time{}
|
||||||
|
if db.Migrator().HasTable(&SQLSchemaMigration{}) {
|
||||||
|
var rows []SQLSchemaMigration
|
||||||
|
if err := db.Find(&rows).Error; err != nil {
|
||||||
|
return nil, fmt.Errorf("read qt_schema_migrations: %w", err)
|
||||||
|
}
|
||||||
|
for _, r := range rows {
|
||||||
|
applied[r.Filename] = r.AppliedAt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
states := make([]SQLMigrationState, 0, len(files))
|
||||||
|
for _, f := range files {
|
||||||
|
st := SQLMigrationState{Filename: f}
|
||||||
|
if at, ok := applied[f]; ok {
|
||||||
|
at := at
|
||||||
|
st.AppliedAt = &at
|
||||||
|
}
|
||||||
|
states = append(states, st)
|
||||||
|
}
|
||||||
|
return states, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BaselineSQLMigrations records every migration up to and including upto as applied
|
||||||
|
// WITHOUT executing it — for a server whose schema was brought up to date by other
|
||||||
|
// means before qt_schema_migrations was used. Returns the filenames newly recorded.
|
||||||
|
func BaselineSQLMigrations(db *gorm.DB, migrations fs.FS, upto string) ([]string, error) {
|
||||||
|
files, err := listSQLMigrationFiles(migrations)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, f := range files {
|
||||||
|
if f == upto {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return nil, fmt.Errorf("baseline migration %q not found", upto)
|
||||||
|
}
|
||||||
|
if err := ensureSQLMigrationsTable(db); err != nil {
|
||||||
|
return nil, fmt.Errorf("migrate qt_schema_migrations table: %w", err)
|
||||||
|
}
|
||||||
|
var recorded []string
|
||||||
|
for _, f := range files {
|
||||||
|
if f > upto {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
var count int64
|
||||||
|
if err := db.Model(&SQLSchemaMigration{}).Where("filename = ?", f).Count(&count).Error; err != nil {
|
||||||
|
return recorded, fmt.Errorf("check migration %s: %w", f, err)
|
||||||
|
}
|
||||||
|
if count > 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := db.Create(&SQLSchemaMigration{Filename: f}).Error; err != nil {
|
||||||
|
return recorded, fmt.Errorf("record baseline migration %s: %w", f, err)
|
||||||
|
}
|
||||||
|
recorded = append(recorded, f)
|
||||||
|
}
|
||||||
|
return recorded, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NeedsSQLMigrations reports whether at least one SQL migration from migrations
|
||||||
// is not yet recorded in qt_schema_migrations.
|
// is not yet recorded in qt_schema_migrations.
|
||||||
func NeedsSQLMigrations(db *gorm.DB, migrationsDir string) (bool, error) {
|
func NeedsSQLMigrations(db *gorm.DB, migrations fs.FS) (bool, error) {
|
||||||
files, err := listSQLMigrationFiles(migrationsDir)
|
files, err := listSQLMigrationFiles(migrations)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
@@ -48,50 +124,51 @@ func NeedsSQLMigrations(db *gorm.DB, migrationsDir string) (bool, error) {
|
|||||||
return count < int64(len(files)), nil
|
return count < int64(len(files)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunSQLMigrations applies SQL files from migrationsDir once and records them in qt_schema_migrations.
|
// RunSQLMigrations applies pending SQL files from migrations once each, in filename
|
||||||
// Local SQLite-only scripts are skipped automatically.
|
// order, and records them in qt_schema_migrations. onApply (optional) is called with
|
||||||
func RunSQLMigrations(db *gorm.DB, migrationsDir string) error {
|
// each filename before it runs. Local SQLite-only scripts are skipped automatically.
|
||||||
|
// Returns the filenames applied.
|
||||||
|
func RunSQLMigrations(db *gorm.DB, migrations fs.FS, onApply func(filename string)) ([]string, error) {
|
||||||
if err := ensureSQLMigrationsTable(db); err != nil {
|
if err := ensureSQLMigrationsTable(db); err != nil {
|
||||||
return fmt.Errorf("migrate qt_schema_migrations table: %w", err)
|
return nil, fmt.Errorf("migrate qt_schema_migrations table: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
files, err := listSQLMigrationFiles(migrationsDir)
|
files, err := listSQLMigrationFiles(migrations)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var applied []string
|
||||||
|
|
||||||
for _, filename := range files {
|
for _, filename := range files {
|
||||||
var count int64
|
var count int64
|
||||||
if err := db.Model(&SQLSchemaMigration{}).Where("filename = ?", filename).Count(&count).Error; err != nil {
|
if err := db.Model(&SQLSchemaMigration{}).Where("filename = ?", filename).Count(&count).Error; err != nil {
|
||||||
return fmt.Errorf("check migration %s: %w", filename, err)
|
return applied, fmt.Errorf("check migration %s: %w", filename, err)
|
||||||
}
|
}
|
||||||
if count > 0 {
|
if count > 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if onApply != nil {
|
||||||
|
onApply(filename)
|
||||||
|
}
|
||||||
|
|
||||||
path := filepath.Join(migrationsDir, filename)
|
content, err := fs.ReadFile(migrations, filename)
|
||||||
content, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read migration %s: %w", filename, err)
|
return applied, fmt.Errorf("read migration %s: %w", filename, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
statements := splitSQLStatements(string(content))
|
if statements := splitSQLStatements(string(content)); len(statements) > 0 {
|
||||||
if len(statements) == 0 {
|
if err := executeMigrationStatements(db, filename, statements); err != nil {
|
||||||
if err := db.Create(&SQLSchemaMigration{Filename: filename}).Error; err != nil {
|
return applied, err
|
||||||
return fmt.Errorf("record empty migration %s: %w", filename, err)
|
|
||||||
}
|
}
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := executeMigrationStatements(db, filename, statements); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
if err := db.Create(&SQLSchemaMigration{Filename: filename}).Error; err != nil {
|
if err := db.Create(&SQLSchemaMigration{Filename: filename}).Error; err != nil {
|
||||||
return fmt.Errorf("record migration %s: %w", filename, err)
|
return applied, fmt.Errorf("record migration %s: %w", filename, err)
|
||||||
}
|
}
|
||||||
|
applied = append(applied, filename)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return applied, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsMigrationPermissionError returns true if err indicates insufficient privileges
|
// IsMigrationPermissionError returns true if err indicates insufficient privileges
|
||||||
@@ -199,13 +276,13 @@ func splitSQLStatements(script string) []string {
|
|||||||
return stmts
|
return stmts
|
||||||
}
|
}
|
||||||
|
|
||||||
func listSQLMigrationFiles(migrationsDir string) ([]string, error) {
|
func listSQLMigrationFiles(migrations fs.FS) ([]string, error) {
|
||||||
entries, err := os.ReadDir(migrationsDir)
|
entries, err := fs.ReadDir(migrations, ".")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("read migrations dir %s: %w", migrationsDir, err)
|
return nil, fmt.Errorf("read migrations dir: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
files := make([]string, 0, len(entries))
|
files := make([]string, 0, len(entries))
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
package models
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"github.com/glebarez/sqlite"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSQLMigrationStatusAndListing(t *testing.T) {
|
||||||
|
migrations := fstest.MapFS{
|
||||||
|
"002_b.sql": {Data: []byte("ALTER TABLE x ADD COLUMN b INT;")},
|
||||||
|
"001_a.sql": {Data: []byte("ALTER TABLE x ADD COLUMN a INT;")},
|
||||||
|
"006_local_only_versions.sql": {Data: []byte("-- sqlite only")},
|
||||||
|
"README.md": {Data: []byte("not a migration")},
|
||||||
|
}
|
||||||
|
db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "m.db")), &gorm.Config{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// No history table yet: everything pending, nothing created.
|
||||||
|
states, err := SQLMigrationStatus(db, migrations)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(states) != 2 || states[0].Filename != "001_a.sql" || states[1].Filename != "002_b.sql" {
|
||||||
|
t.Fatalf("unexpected listing %+v", states)
|
||||||
|
}
|
||||||
|
if states[0].AppliedAt != nil || db.Migrator().HasTable(&SQLSchemaMigration{}) {
|
||||||
|
t.Fatal("status must be read-only and report pending")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := db.AutoMigrate(&SQLSchemaMigration{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := db.Create(&SQLSchemaMigration{Filename: "001_a.sql"}).Error; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
states, _ = SQLMigrationStatus(db, migrations)
|
||||||
|
if states[0].AppliedAt == nil || states[1].AppliedAt != nil {
|
||||||
|
t.Fatalf("want 001 applied, 002 pending: %+v", states)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSplitSQLStatementsSkipsComments(t *testing.T) {
|
||||||
|
got := splitSQLStatements("-- header; with semicolon\nALTER TABLE a ADD COLUMN x INT;\n\nALTER TABLE b\n ADD COLUMN y INT;\n")
|
||||||
|
if len(got) != 2 || got[1] != "ALTER TABLE b\n ADD COLUMN y INT" {
|
||||||
|
t.Fatalf("unexpected statements %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user