186 lines
6.4 KiB
Go
186 lines
6.4 KiB
Go
package mariadb
|
|
|
|
import (
|
|
"archive/zip"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.mchus.pro/mchus/quoteforge/internal/appstate"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
type snapshotTable struct {
|
|
Name, Create, Entry, Digest string
|
|
Columns []string
|
|
Rows int64
|
|
}
|
|
|
|
type snapshotManifest struct {
|
|
Format int
|
|
Schema, Collation string
|
|
Tables []snapshotTable
|
|
Triggers []snapshotTrigger
|
|
}
|
|
|
|
func freshDB(db *gorm.DB) *gorm.DB { return db.Session(&gorm.Session{NewDB: true}) }
|
|
func quoteIdentifier(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" }
|
|
|
|
// CreateSnapshot writes a consistent InnoDB snapshot, without external DB tools.
|
|
// The caller supplies a dedicated connection and excludes concurrent writers
|
|
// for the entire migration session, including any recovery.
|
|
func CreateSnapshot(db *gorm.DB, root, label string) (string, error) {
|
|
if err := appstate.ValidateBackupLocation(root); err != nil {
|
|
return "", err
|
|
}
|
|
if filepath.Base(label) != label {
|
|
return "", fmt.Errorf("invalid snapshot label")
|
|
}
|
|
if err := os.MkdirAll(root, 0700); err != nil {
|
|
return "", fmt.Errorf("create snapshot directory: %w", err)
|
|
}
|
|
if err := appstate.ValidateBackupLocation(root); err != nil {
|
|
return "", err
|
|
}
|
|
file, err := os.CreateTemp(root, "qfs-mariadb-v1-"+label+"-"+time.Now().UTC().Format("20060102T150405Z")+"-*.zip")
|
|
if err != nil {
|
|
return "", fmt.Errorf("create snapshot: %w", err)
|
|
}
|
|
path := file.Name()
|
|
archive := zip.NewWriter(file)
|
|
err = freshDB(db).Transaction(func(tx *gorm.DB) error {
|
|
manifest, err := snapshotInventory(tx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for i := range manifest.Tables {
|
|
table := &manifest.Tables[i]
|
|
table.Entry = fmt.Sprintf("tables/%06d.jsonl", i)
|
|
out, err := archive.Create(table.Entry)
|
|
if err != nil {
|
|
return fmt.Errorf("create table entry %s: %w", table.Name, err)
|
|
}
|
|
if err := snapshotRows(tx, table, out); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
out, err := archive.Create("manifest.json")
|
|
if err != nil {
|
|
return fmt.Errorf("create snapshot manifest: %w", err)
|
|
}
|
|
if err := json.NewEncoder(out).Encode(manifest); err != nil {
|
|
return fmt.Errorf("write snapshot manifest: %w", err)
|
|
}
|
|
return nil
|
|
}, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
|
|
zipErr := archive.Close()
|
|
syncErr := file.Sync()
|
|
closeErr := file.Close()
|
|
for _, candidate := range []error{err, zipErr, syncErr, closeErr} {
|
|
if candidate != nil {
|
|
return "", fmt.Errorf("write snapshot %s: %w", path, candidate)
|
|
}
|
|
}
|
|
if _, err := readSnapshot(path, nil); err != nil {
|
|
return "", err
|
|
}
|
|
return path, nil
|
|
}
|
|
|
|
func snapshotInventory(db *gorm.DB) (*snapshotManifest, error) {
|
|
m := &snapshotManifest{Format: 1}
|
|
if err := freshDB(db).Raw("SELECT DATABASE(), @@collation_database").Row().Scan(&m.Schema, &m.Collation); err != nil {
|
|
return nil, fmt.Errorf("read snapshot schema: %w", err)
|
|
}
|
|
var unsupported int64
|
|
query := `SELECT
|
|
(SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=? AND (TABLE_TYPE<>'BASE TABLE' OR ENGINE<>'InnoDB')) +
|
|
(SELECT COUNT(*) FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA=?) +
|
|
(SELECT COUNT(*) FROM information_schema.EVENTS WHERE EVENT_SCHEMA=?) +
|
|
(SELECT COUNT(*) FROM information_schema.KEY_COLUMN_USAGE WHERE
|
|
(TABLE_SCHEMA=? AND REFERENCED_TABLE_SCHEMA<>?) OR (REFERENCED_TABLE_SCHEMA=? AND TABLE_SCHEMA<>?))`
|
|
if err := freshDB(db).Raw(query, m.Schema, m.Schema, m.Schema, m.Schema, m.Schema, m.Schema, m.Schema).Scan(&unsupported).Error; err != nil {
|
|
return nil, fmt.Errorf("check snapshot support: %w", err)
|
|
}
|
|
if unsupported != 0 {
|
|
return nil, fmt.Errorf("snapshot refused: views, non-InnoDB tables, routines, events or cross-schema foreign keys require supported recovery before migration")
|
|
}
|
|
var names []string
|
|
if err := freshDB(db).Raw("SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA=? ORDER BY TABLE_NAME", m.Schema).Scan(&names).Error; err != nil {
|
|
return nil, fmt.Errorf("list snapshot tables: %w", err)
|
|
}
|
|
for _, name := range names {
|
|
table := snapshotTable{Name: name}
|
|
var ignored string
|
|
if err := freshDB(db).Raw("SHOW CREATE TABLE "+quoteIdentifier(name)).Row().Scan(&ignored, &table.Create); err != nil {
|
|
return nil, fmt.Errorf("read schema for %s: %w", name, err)
|
|
}
|
|
if err := freshDB(db).Raw(`SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? AND EXTRA NOT LIKE '%VIRTUAL GENERATED%' AND EXTRA NOT LIKE '%STORED GENERATED%' ORDER BY ORDINAL_POSITION`, m.Schema, name).Scan(&table.Columns).Error; err != nil {
|
|
return nil, fmt.Errorf("read columns for %s: %w", name, err)
|
|
}
|
|
if len(table.Columns) == 0 {
|
|
return nil, fmt.Errorf("snapshot table %s has no writable columns", name)
|
|
}
|
|
m.Tables = append(m.Tables, table)
|
|
}
|
|
if err := readSnapshotTriggers(db, m); err != nil {
|
|
return nil, err
|
|
}
|
|
return m, nil
|
|
}
|
|
|
|
func snapshotRows(db *gorm.DB, table *snapshotTable, out io.Writer) error {
|
|
columns := make([]string, len(table.Columns))
|
|
for i, col := range table.Columns {
|
|
columns[i] = "CAST(" + quoteIdentifier(col) + " AS BINARY)"
|
|
}
|
|
rows, err := freshDB(db).Raw("SELECT " + strings.Join(columns, ",") + " FROM " + quoteIdentifier(table.Name)).Rows()
|
|
if err != nil {
|
|
return fmt.Errorf("read snapshot table %s: %w", table.Name, err)
|
|
}
|
|
defer rows.Close()
|
|
var digest [32]byte
|
|
encoder := json.NewEncoder(out)
|
|
table.Rows = 0
|
|
for rows.Next() {
|
|
values := make([]sql.RawBytes, len(columns))
|
|
args := make([]any, len(columns))
|
|
for i := range args {
|
|
args[i] = &values[i]
|
|
}
|
|
if err := rows.Scan(args...); err != nil {
|
|
return fmt.Errorf("read row in %s: %w", table.Name, err)
|
|
}
|
|
if err := encoder.Encode(values); err != nil {
|
|
return fmt.Errorf("write row in %s: %w", table.Name, err)
|
|
}
|
|
addRowDigest(&digest, values)
|
|
table.Rows++
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return fmt.Errorf("iterate %s: %w", table.Name, err)
|
|
}
|
|
if err := rows.Close(); err != nil {
|
|
return fmt.Errorf("close rows in %s: %w", table.Name, err)
|
|
}
|
|
table.Digest = hex.EncodeToString(digest[:])
|
|
return nil
|
|
}
|
|
|
|
func addRowDigest(sum *[32]byte, row []sql.RawBytes) {
|
|
encoded, _ := json.Marshal(row) // Raw bytes and nil always marshal successfully.
|
|
hash := sha256.Sum256(encoded)
|
|
carry := uint16(0)
|
|
for i := len(sum) - 1; i >= 0; i-- {
|
|
value := uint16(sum[i]) + uint16(hash[i]) + carry
|
|
sum[i], carry = byte(value), value>>8
|
|
}
|
|
}
|