Local-first runtime cleanup and recovery hardening

This commit is contained in:
Mikhail Chusavitin
2026-03-07 23:18:07 +03:00
parent 9f8e050349
commit 84013c9dc4
53 changed files with 1856 additions and 2080 deletions

View File

@@ -46,10 +46,6 @@ func ConfigurationToLocal(cfg *models.Configuration) *LocalConfiguration {
OriginalUsername: cfg.OwnerUsername,
}
if local.OriginalUsername == "" && cfg.User != nil {
local.OriginalUsername = cfg.User.Username
}
if cfg.ID > 0 {
serverID := cfg.ID
local.ServerID = &serverID

View File

@@ -7,19 +7,104 @@ import (
"crypto/sha256"
"encoding/base64"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"git.mchus.pro/mchus/quoteforge/internal/appstate"
)
// getEncryptionKey derives a 32-byte key from environment variable or machine ID
func getEncryptionKey() []byte {
const encryptionKeyFileName = "local_encryption.key"
// getEncryptionKey resolves the active encryption key.
// Preference order:
// 1. QUOTEFORGE_ENCRYPTION_KEY env var
// 2. application-managed random key file in the user state directory
func getEncryptionKey() ([]byte, error) {
key := os.Getenv("QUOTEFORGE_ENCRYPTION_KEY")
if key == "" {
// Fallback to a machine-based key (hostname + fixed salt)
hostname, _ := os.Hostname()
key = hostname + "quoteforge-salt-2024"
if key != "" {
hash := sha256.Sum256([]byte(key))
return hash[:], nil
}
// Hash to get exactly 32 bytes for AES-256
stateDir, err := resolveEncryptionStateDir()
if err != nil {
return nil, fmt.Errorf("resolve encryption state dir: %w", err)
}
return loadOrCreateEncryptionKey(filepath.Join(stateDir, encryptionKeyFileName))
}
func resolveEncryptionStateDir() (string, error) {
configPath, err := appstate.ResolveConfigPath("")
if err != nil {
return "", err
}
return filepath.Dir(configPath), nil
}
func loadOrCreateEncryptionKey(path string) ([]byte, error) {
if data, err := os.ReadFile(path); err == nil {
return parseEncryptionKeyFile(data)
} else if !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("read encryption key: %w", err)
}
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
return nil, fmt.Errorf("create encryption key dir: %w", err)
}
raw := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, raw); err != nil {
return nil, fmt.Errorf("generate encryption key: %w", err)
}
encoded := base64.StdEncoding.EncodeToString(raw)
if err := writeKeyFile(path, []byte(encoded+"\n")); err != nil {
if errors.Is(err, os.ErrExist) {
data, readErr := os.ReadFile(path)
if readErr != nil {
return nil, fmt.Errorf("read concurrent encryption key: %w", readErr)
}
return parseEncryptionKeyFile(data)
}
return nil, err
}
return raw, nil
}
func writeKeyFile(path string, data []byte) error {
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
if err != nil {
return err
}
defer file.Close()
if _, err := file.Write(data); err != nil {
return err
}
return file.Sync()
}
func parseEncryptionKeyFile(data []byte) ([]byte, error) {
trimmed := strings.TrimSpace(string(data))
decoded, err := base64.StdEncoding.DecodeString(trimmed)
if err != nil {
return nil, fmt.Errorf("decode encryption key file: %w", err)
}
if len(decoded) != 32 {
return nil, fmt.Errorf("invalid encryption key length: %d", len(decoded))
}
return decoded, nil
}
func getLegacyEncryptionKey() []byte {
hostname, _ := os.Hostname()
key := hostname + "quoteforge-salt-2024"
hash := sha256.Sum256([]byte(key))
return hash[:]
}
@@ -30,7 +115,10 @@ func Encrypt(plaintext string) (string, error) {
return "", nil
}
key := getEncryptionKey()
key, err := getEncryptionKey()
if err != nil {
return "", err
}
block, err := aes.NewCipher(key)
if err != nil {
return "", err
@@ -56,12 +144,50 @@ func Decrypt(ciphertext string) (string, error) {
return "", nil
}
key := getEncryptionKey()
data, err := base64.StdEncoding.DecodeString(ciphertext)
key, err := getEncryptionKey()
if err != nil {
return "", err
}
plaintext, legacy, err := decryptWithKeys(ciphertext, key, getLegacyEncryptionKey())
if err != nil {
return "", err
}
_ = legacy
return plaintext, nil
}
func DecryptWithMetadata(ciphertext string) (string, bool, error) {
if ciphertext == "" {
return "", false, nil
}
key, err := getEncryptionKey()
if err != nil {
return "", false, err
}
return decryptWithKeys(ciphertext, key, getLegacyEncryptionKey())
}
func decryptWithKeys(ciphertext string, primaryKey, legacyKey []byte) (string, bool, error) {
data, err := base64.StdEncoding.DecodeString(ciphertext)
if err != nil {
return "", false, err
}
plaintext, err := decryptWithKey(data, primaryKey)
if err == nil {
return plaintext, false, nil
}
legacyPlaintext, legacyErr := decryptWithKey(data, legacyKey)
if legacyErr == nil {
return legacyPlaintext, true, nil
}
return "", false, err
}
func decryptWithKey(data, key []byte) (string, error) {
block, err := aes.NewCipher(key)
if err != nil {
return "", err

View File

@@ -0,0 +1,97 @@
package localdb
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"io"
"os"
"path/filepath"
"testing"
)
func TestEncryptCreatesPersistentKeyFile(t *testing.T) {
stateDir := t.TempDir()
t.Setenv("QFS_STATE_DIR", stateDir)
t.Setenv("QUOTEFORGE_ENCRYPTION_KEY", "")
ciphertext, err := Encrypt("secret-password")
if err != nil {
t.Fatalf("encrypt: %v", err)
}
if ciphertext == "" {
t.Fatal("expected ciphertext")
}
keyPath := filepath.Join(stateDir, encryptionKeyFileName)
info, err := os.Stat(keyPath)
if err != nil {
t.Fatalf("stat key file: %v", err)
}
if info.Mode().Perm() != 0600 {
t.Fatalf("expected 0600 key file, got %v", info.Mode().Perm())
}
}
func TestDecryptMigratesLegacyCiphertext(t *testing.T) {
stateDir := t.TempDir()
t.Setenv("QFS_STATE_DIR", stateDir)
t.Setenv("QUOTEFORGE_ENCRYPTION_KEY", "")
legacyCiphertext := encryptWithKeyForTest(t, getLegacyEncryptionKey(), "legacy-password")
plaintext, migrated, err := DecryptWithMetadata(legacyCiphertext)
if err != nil {
t.Fatalf("decrypt legacy: %v", err)
}
if plaintext != "legacy-password" {
t.Fatalf("unexpected plaintext: %q", plaintext)
}
if !migrated {
t.Fatal("expected legacy ciphertext to require migration")
}
currentCiphertext, err := Encrypt("legacy-password")
if err != nil {
t.Fatalf("encrypt current: %v", err)
}
plaintext, migrated, err = DecryptWithMetadata(currentCiphertext)
if err != nil {
t.Fatalf("decrypt current: %v", err)
}
if migrated {
t.Fatal("did not expect current ciphertext to require migration")
}
}
func encryptWithKeyForTest(t *testing.T, key []byte, plaintext string) string {
t.Helper()
block, err := aes.NewCipher(key)
if err != nil {
t.Fatalf("new cipher: %v", err)
}
gcm, err := cipher.NewGCM(block)
if err != nil {
t.Fatalf("new gcm: %v", err)
}
nonce := make([]byte, gcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
t.Fatalf("read nonce: %v", err)
}
ciphertext := gcm.Seal(nonce, nonce, []byte(plaintext), nil)
return base64.StdEncoding.EncodeToString(ciphertext)
}
func TestLegacyEncryptionKeyRemainsDeterministic(t *testing.T) {
hostname, _ := os.Hostname()
expected := sha256.Sum256([]byte(hostname + "quoteforge-salt-2024"))
actual := getLegacyEncryptionKey()
if string(actual) != string(expected[:]) {
t.Fatal("legacy key derivation changed")
}
}

View File

@@ -5,7 +5,10 @@ import (
"testing"
"time"
"github.com/glebarez/sqlite"
"github.com/google/uuid"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
func TestRunLocalMigrationsBackfillsExistingConfigurations(t *testing.T) {
@@ -313,3 +316,280 @@ func TestRunLocalMigrationsBackfillsConfigurationLineNo(t *testing.T) {
t.Fatalf("expected line_no [10,20], got [%d,%d]", rows[0].Line, rows[1].Line)
}
}
func TestRunLocalMigrationsDeduplicatesCanonicalPartnumberCatalog(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "partnumber_catalog_dedup.db")
db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
firstLots := LocalPartnumberBookLots{
{LotName: "LOT-A", Qty: 1},
}
secondLots := LocalPartnumberBookLots{
{LotName: "LOT-B", Qty: 2},
}
if err := db.Exec(`
CREATE TABLE local_partnumber_book_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
partnumber TEXT NOT NULL,
lots_json TEXT NOT NULL,
description TEXT
)
`).Error; err != nil {
t.Fatalf("create dirty local_partnumber_book_items: %v", err)
}
if err := db.Create(&LocalPartnumberBookItem{
Partnumber: "PN-001",
LotsJSON: firstLots,
Description: "",
}).Error; err != nil {
t.Fatalf("insert first duplicate row: %v", err)
}
if err := db.Create(&LocalPartnumberBookItem{
Partnumber: "PN-001",
LotsJSON: secondLots,
Description: "Canonical description",
}).Error; err != nil {
t.Fatalf("insert second duplicate row: %v", err)
}
if err := migrateLocalPartnumberBookCatalog(db); err != nil {
t.Fatalf("migrate local partnumber catalog: %v", err)
}
var items []LocalPartnumberBookItem
if err := db.Order("partnumber ASC").Find(&items).Error; err != nil {
t.Fatalf("load migrated partnumber items: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 deduplicated item, got %d", len(items))
}
if items[0].Partnumber != "PN-001" {
t.Fatalf("unexpected partnumber: %s", items[0].Partnumber)
}
if items[0].Description != "Canonical description" {
t.Fatalf("expected merged description, got %q", items[0].Description)
}
if len(items[0].LotsJSON) != 2 {
t.Fatalf("expected merged lots from duplicates, got %d", len(items[0].LotsJSON))
}
var duplicateCount int64
if err := db.Model(&LocalPartnumberBookItem{}).
Where("partnumber = ?", "PN-001").
Count(&duplicateCount).Error; err != nil {
t.Fatalf("count deduplicated partnumber: %v", err)
}
if duplicateCount != 1 {
t.Fatalf("expected unique partnumber row after migration, got %d", duplicateCount)
}
}
func TestSanitizeLocalPartnumberBookCatalogRemovesRowsWithoutPartnumber(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "sanitize_partnumber_catalog.db")
db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := db.Exec(`
CREATE TABLE local_partnumber_book_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
partnumber TEXT NULL,
lots_json TEXT NOT NULL,
description TEXT
)
`).Error; err != nil {
t.Fatalf("create local_partnumber_book_items: %v", err)
}
if err := db.Exec(`
INSERT INTO local_partnumber_book_items (partnumber, lots_json, description) VALUES
(NULL, '[]', 'null pn'),
('', '[]', 'empty pn'),
('PN-OK', '[]', 'valid pn')
`).Error; err != nil {
t.Fatalf("seed local_partnumber_book_items: %v", err)
}
if err := sanitizeLocalPartnumberBookCatalog(db); err != nil {
t.Fatalf("sanitize local partnumber catalog: %v", err)
}
var items []LocalPartnumberBookItem
if err := db.Order("id ASC").Find(&items).Error; err != nil {
t.Fatalf("load sanitized items: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 valid item after sanitize, got %d", len(items))
}
if items[0].Partnumber != "PN-OK" {
t.Fatalf("expected remaining partnumber PN-OK, got %q", items[0].Partnumber)
}
}
func TestNewMigratesLegacyPartnumberBookCatalogBeforeAutoMigrate(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "legacy_partnumber_catalog.db")
db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := db.Exec(`
CREATE TABLE local_partnumber_book_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
partnumber TEXT NOT NULL UNIQUE,
lots_json TEXT NOT NULL,
is_primary_pn INTEGER NOT NULL DEFAULT 0,
description TEXT
)
`).Error; err != nil {
t.Fatalf("create legacy local_partnumber_book_items: %v", err)
}
if err := db.Exec(`
INSERT INTO local_partnumber_book_items (partnumber, lots_json, is_primary_pn, description)
VALUES ('PN-001', '[{"lot_name":"CPU_A","qty":1}]', 0, 'Legacy row')
`).Error; err != nil {
t.Fatalf("seed legacy local_partnumber_book_items: %v", err)
}
local, err := New(dbPath)
if err != nil {
t.Fatalf("open localdb with legacy catalog: %v", err)
}
t.Cleanup(func() { _ = local.Close() })
var columns []struct {
Name string `gorm:"column:name"`
}
if err := local.DB().Raw(`SELECT name FROM pragma_table_info('local_partnumber_book_items')`).Scan(&columns).Error; err != nil {
t.Fatalf("load local_partnumber_book_items columns: %v", err)
}
for _, column := range columns {
if column.Name == "is_primary_pn" {
t.Fatalf("expected legacy is_primary_pn column to be removed before automigrate")
}
}
var items []LocalPartnumberBookItem
if err := local.DB().Find(&items).Error; err != nil {
t.Fatalf("load migrated local_partnumber_book_items: %v", err)
}
if len(items) != 1 || items[0].Partnumber != "PN-001" {
t.Fatalf("unexpected migrated rows: %#v", items)
}
}
func TestNewRecoversBrokenPartnumberBookCatalogCache(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "broken_partnumber_catalog.db")
db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := db.Exec(`
CREATE TABLE local_partnumber_book_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
partnumber TEXT NOT NULL UNIQUE,
lots_json TEXT NOT NULL,
description TEXT
)
`).Error; err != nil {
t.Fatalf("create broken local_partnumber_book_items: %v", err)
}
if err := db.Exec(`
INSERT INTO local_partnumber_book_items (partnumber, lots_json, description)
VALUES ('PN-001', '{not-json}', 'Broken cache row')
`).Error; err != nil {
t.Fatalf("seed broken local_partnumber_book_items: %v", err)
}
local, err := New(dbPath)
if err != nil {
t.Fatalf("open localdb with broken catalog cache: %v", err)
}
t.Cleanup(func() { _ = local.Close() })
var count int64
if err := local.DB().Model(&LocalPartnumberBookItem{}).Count(&count).Error; err != nil {
t.Fatalf("count recovered local_partnumber_book_items: %v", err)
}
if count != 0 {
t.Fatalf("expected empty recovered local_partnumber_book_items, got %d rows", count)
}
var quarantineTables []struct {
Name string `gorm:"column:name"`
}
if err := local.DB().Raw(`
SELECT name
FROM sqlite_master
WHERE type = 'table' AND name LIKE 'local_partnumber_book_items_broken_%'
`).Scan(&quarantineTables).Error; err != nil {
t.Fatalf("load quarantine tables: %v", err)
}
if len(quarantineTables) != 1 {
t.Fatalf("expected one quarantined broken catalog table, got %d", len(quarantineTables))
}
}
func TestCleanupStaleReadOnlyCacheTempTablesDropsShadowTempWhenBaseExists(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "stale_cache_temp.db")
db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := db.Exec(`
CREATE TABLE local_pricelist_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
pricelist_id INTEGER NOT NULL,
partnumber TEXT,
brand TEXT NOT NULL DEFAULT '',
lot_name TEXT NOT NULL,
description TEXT,
price REAL NOT NULL DEFAULT 0,
quantity INTEGER NOT NULL DEFAULT 0,
reserve INTEGER NOT NULL DEFAULT 0,
available_qty REAL,
partnumbers TEXT,
lot_category TEXT,
created_at DATETIME,
updated_at DATETIME
)
`).Error; err != nil {
t.Fatalf("create local_pricelist_items: %v", err)
}
if err := db.Exec(`
CREATE TABLE local_pricelist_items__temp (
id INTEGER PRIMARY KEY AUTOINCREMENT,
legacy TEXT
)
`).Error; err != nil {
t.Fatalf("create local_pricelist_items__temp: %v", err)
}
if err := cleanupStaleReadOnlyCacheTempTables(db); err != nil {
t.Fatalf("cleanup stale read-only cache temp tables: %v", err)
}
if db.Migrator().HasTable("local_pricelist_items__temp") {
t.Fatalf("expected stale temp table to be dropped")
}
if !db.Migrator().HasTable("local_pricelist_items") {
t.Fatalf("expected base local_pricelist_items table to remain")
}
}

View File

@@ -1,6 +1,7 @@
package localdb
import (
"encoding/json"
"errors"
"fmt"
"log/slog"
@@ -42,6 +43,14 @@ type LocalDB struct {
path string
}
var localReadOnlyCacheTables = []string{
"local_pricelist_items",
"local_pricelists",
"local_components",
"local_partnumber_book_items",
"local_partnumber_books",
}
// ResetData clears local data tables while keeping connection settings.
// It does not drop schema or connection_settings.
func ResetData(dbPath string) error {
@@ -70,7 +79,6 @@ func ResetData(dbPath string) error {
"local_pricelists",
"local_pricelist_items",
"local_components",
"local_remote_migrations_applied",
"local_sync_guard_state",
"pending_changes",
"app_settings",
@@ -111,6 +119,12 @@ func New(dbPath string) (*LocalDB, error) {
if err := ensureLocalProjectsTable(db); err != nil {
return nil, fmt.Errorf("ensure local_projects table: %w", err)
}
if err := prepareLocalPartnumberBookCatalog(db); err != nil {
return nil, fmt.Errorf("prepare local partnumber book catalog: %w", err)
}
if err := cleanupStaleReadOnlyCacheTempTables(db); err != nil {
return nil, fmt.Errorf("cleanup stale read-only cache temp tables: %w", err)
}
// Preflight: ensure local_projects has non-null UUIDs before AutoMigrate rebuilds tables.
if db.Migrator().HasTable(&LocalProject{}) {
@@ -131,24 +145,28 @@ func New(dbPath string) (*LocalDB, error) {
}
// Auto-migrate all local tables
if err := db.AutoMigrate(
&ConnectionSettings{},
&LocalConfiguration{},
&LocalConfigurationVersion{},
&LocalPricelist{},
&LocalPricelistItem{},
&LocalComponent{},
&AppSetting{},
&LocalRemoteMigrationApplied{},
&LocalSyncGuardState{},
&PendingChange{},
&LocalPartnumberBook{},
&LocalPartnumberBookItem{},
); err != nil {
return nil, fmt.Errorf("migrating sqlite database: %w", err)
if err := autoMigrateLocalSchema(db); err != nil {
if recovered, recoveryErr := recoverFromReadOnlyCacheInitFailure(db, err); recoveryErr != nil {
return nil, fmt.Errorf("migrating sqlite database: %w (recovery failed: %v)", err, recoveryErr)
} else if !recovered {
return nil, fmt.Errorf("migrating sqlite database: %w", err)
}
if err := autoMigrateLocalSchema(db); err != nil {
return nil, fmt.Errorf("migrating sqlite database after cache recovery: %w", err)
}
}
if err := ensureLocalPartnumberBookItemTable(db); err != nil {
return nil, fmt.Errorf("ensure local partnumber book item table: %w", err)
}
if err := runLocalMigrations(db); err != nil {
return nil, fmt.Errorf("running local sqlite migrations: %w", err)
if recovered, recoveryErr := recoverFromReadOnlyCacheInitFailure(db, err); recoveryErr != nil {
return nil, fmt.Errorf("running local sqlite migrations: %w (recovery failed: %v)", err, recoveryErr)
} else if !recovered {
return nil, fmt.Errorf("running local sqlite migrations: %w", err)
}
if err := runLocalMigrations(db); err != nil {
return nil, fmt.Errorf("running local sqlite migrations after cache recovery: %w", err)
}
}
slog.Info("local SQLite database initialized", "path", dbPath)
@@ -191,6 +209,282 @@ CREATE TABLE local_projects (
return nil
}
func autoMigrateLocalSchema(db *gorm.DB) error {
return db.AutoMigrate(
&ConnectionSettings{},
&LocalConfiguration{},
&LocalConfigurationVersion{},
&LocalPricelist{},
&LocalPricelistItem{},
&LocalComponent{},
&AppSetting{},
&LocalSyncGuardState{},
&PendingChange{},
&LocalPartnumberBook{},
)
}
func sanitizeLocalPartnumberBookCatalog(db *gorm.DB) error {
if !db.Migrator().HasTable(&LocalPartnumberBookItem{}) {
return nil
}
// Old local databases may contain partially migrated catalog rows with NULL/empty
// partnumber values. SQLite table rebuild during AutoMigrate fails on such rows once
// the schema enforces NOT NULL, so remove them before AutoMigrate touches the table.
if err := db.Exec(`
DELETE FROM local_partnumber_book_items
WHERE partnumber IS NULL OR TRIM(partnumber) = ''
`).Error; err != nil {
return err
}
return nil
}
func prepareLocalPartnumberBookCatalog(db *gorm.DB) error {
if err := cleanupStaleLocalPartnumberBookCatalogTempTable(db); err != nil {
if recoveryErr := recoverLocalPartnumberBookCatalog(db, fmt.Errorf("cleanup stale temp table: %w", err)); recoveryErr != nil {
return recoveryErr
}
return nil
}
if err := sanitizeLocalPartnumberBookCatalog(db); err != nil {
if recoveryErr := recoverLocalPartnumberBookCatalog(db, fmt.Errorf("sanitize catalog: %w", err)); recoveryErr != nil {
return recoveryErr
}
return nil
}
if err := migrateLegacyPartnumberBookCatalogBeforeAutoMigrate(db); err != nil {
if recoveryErr := recoverLocalPartnumberBookCatalog(db, fmt.Errorf("migrate legacy catalog: %w", err)); recoveryErr != nil {
return recoveryErr
}
return nil
}
if err := ensureLocalPartnumberBookItemTable(db); err != nil {
if recoveryErr := recoverLocalPartnumberBookCatalog(db, fmt.Errorf("ensure canonical catalog table: %w", err)); recoveryErr != nil {
return recoveryErr
}
return nil
}
if err := validateLocalPartnumberBookCatalog(db); err != nil {
if recoveryErr := recoverLocalPartnumberBookCatalog(db, fmt.Errorf("validate canonical catalog: %w", err)); recoveryErr != nil {
return recoveryErr
}
}
return nil
}
func cleanupStaleReadOnlyCacheTempTables(db *gorm.DB) error {
for _, tableName := range localReadOnlyCacheTables {
tempName := tableName + "__temp"
if !db.Migrator().HasTable(tempName) {
continue
}
if db.Migrator().HasTable(tableName) {
if err := db.Exec(`DROP TABLE ` + tempName).Error; err != nil {
return err
}
continue
}
if err := quarantineSQLiteTable(db, tempName, localReadOnlyCacheQuarantineTableName(tableName, "temp")); err != nil {
return err
}
}
return nil
}
func cleanupStaleLocalPartnumberBookCatalogTempTable(db *gorm.DB) error {
if !db.Migrator().HasTable("local_partnumber_book_items__temp") {
return nil
}
if db.Migrator().HasTable(&LocalPartnumberBookItem{}) {
return db.Exec(`DROP TABLE local_partnumber_book_items__temp`).Error
}
return quarantineSQLiteTable(db, "local_partnumber_book_items__temp", localPartnumberBookCatalogQuarantineTableName("temp"))
}
func migrateLegacyPartnumberBookCatalogBeforeAutoMigrate(db *gorm.DB) error {
if !db.Migrator().HasTable(&LocalPartnumberBookItem{}) {
return nil
}
// Legacy databases may still have the pre-catalog shape (`book_id`/`lot_name`) or the
// intermediate canonical shape with obsolete columns like `is_primary_pn`. Let the
// explicit rebuild logic normalize this table before GORM AutoMigrate attempts a
// table-copy migration on its own.
return migrateLocalPartnumberBookCatalog(db)
}
func ensureLocalPartnumberBookItemTable(db *gorm.DB) error {
if db.Migrator().HasTable(&LocalPartnumberBookItem{}) {
return nil
}
if err := db.Exec(`
CREATE TABLE local_partnumber_book_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
partnumber TEXT NOT NULL UNIQUE,
lots_json TEXT NOT NULL,
description TEXT
)
`).Error; err != nil {
return err
}
return db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_local_partnumber_book_items_partnumber ON local_partnumber_book_items(partnumber)`).Error
}
func validateLocalPartnumberBookCatalog(db *gorm.DB) error {
if !db.Migrator().HasTable(&LocalPartnumberBookItem{}) {
return nil
}
type rawCatalogRow struct {
Partnumber string `gorm:"column:partnumber"`
LotsJSON string `gorm:"column:lots_json"`
Description string `gorm:"column:description"`
}
var rows []rawCatalogRow
if err := db.Raw(`
SELECT partnumber, lots_json, COALESCE(description, '') AS description
FROM local_partnumber_book_items
ORDER BY id ASC
`).Scan(&rows).Error; err != nil {
return fmt.Errorf("load canonical catalog rows: %w", err)
}
seen := make(map[string]struct{}, len(rows))
for _, row := range rows {
partnumber := strings.TrimSpace(row.Partnumber)
if partnumber == "" {
return errors.New("catalog contains empty partnumber")
}
if _, exists := seen[partnumber]; exists {
return fmt.Errorf("catalog contains duplicate partnumber %q", partnumber)
}
seen[partnumber] = struct{}{}
if strings.TrimSpace(row.LotsJSON) == "" {
return fmt.Errorf("catalog row %q has empty lots_json", partnumber)
}
var lots LocalPartnumberBookLots
if err := json.Unmarshal([]byte(row.LotsJSON), &lots); err != nil {
return fmt.Errorf("catalog row %q has invalid lots_json: %w", partnumber, err)
}
}
return nil
}
func recoverLocalPartnumberBookCatalog(db *gorm.DB, cause error) error {
slog.Warn("recovering broken local partnumber book catalog", "error", cause.Error())
if err := ensureLocalPartnumberBooksCatalogColumn(db); err != nil {
return fmt.Errorf("ensure local_partnumber_books.partnumbers_json during recovery: %w", err)
}
if db.Migrator().HasTable("local_partnumber_book_items__temp") {
if err := quarantineSQLiteTable(db, "local_partnumber_book_items__temp", localPartnumberBookCatalogQuarantineTableName("temp")); err != nil {
return fmt.Errorf("quarantine local_partnumber_book_items__temp: %w", err)
}
}
if db.Migrator().HasTable(&LocalPartnumberBookItem{}) {
if err := quarantineSQLiteTable(db, "local_partnumber_book_items", localPartnumberBookCatalogQuarantineTableName("broken")); err != nil {
return fmt.Errorf("quarantine local_partnumber_book_items: %w", err)
}
}
if err := ensureLocalPartnumberBookItemTable(db); err != nil {
return fmt.Errorf("recreate local_partnumber_book_items after recovery: %w", err)
}
slog.Warn("local partnumber book catalog reset to empty cache; next sync will rebuild it")
return nil
}
func recoverFromReadOnlyCacheInitFailure(db *gorm.DB, cause error) (bool, error) {
lowerCause := strings.ToLower(cause.Error())
recoveredAny := false
for _, tableName := range affectedReadOnlyCacheTables(lowerCause) {
if err := resetReadOnlyCacheTable(db, tableName); err != nil {
return recoveredAny, err
}
recoveredAny = true
}
if strings.Contains(lowerCause, "local_partnumber_book_items") || strings.Contains(lowerCause, "local_partnumber_books") {
if err := recoverLocalPartnumberBookCatalog(db, cause); err != nil {
return recoveredAny, err
}
recoveredAny = true
}
if recoveredAny {
slog.Warn("recovered read-only local cache tables after startup failure", "error", cause.Error())
}
return recoveredAny, nil
}
func affectedReadOnlyCacheTables(lowerCause string) []string {
affected := make([]string, 0, len(localReadOnlyCacheTables))
for _, tableName := range localReadOnlyCacheTables {
if tableName == "local_partnumber_book_items" || tableName == "local_partnumber_books" {
continue
}
if strings.Contains(lowerCause, tableName) {
affected = append(affected, tableName)
}
}
return affected
}
func resetReadOnlyCacheTable(db *gorm.DB, tableName string) error {
slog.Warn("resetting read-only local cache table", "table", tableName)
tempName := tableName + "__temp"
if db.Migrator().HasTable(tempName) {
if err := quarantineSQLiteTable(db, tempName, localReadOnlyCacheQuarantineTableName(tableName, "temp")); err != nil {
return fmt.Errorf("quarantine temp table %s: %w", tempName, err)
}
}
if db.Migrator().HasTable(tableName) {
if err := quarantineSQLiteTable(db, tableName, localReadOnlyCacheQuarantineTableName(tableName, "broken")); err != nil {
return fmt.Errorf("quarantine table %s: %w", tableName, err)
}
}
return nil
}
func ensureLocalPartnumberBooksCatalogColumn(db *gorm.DB) error {
if !db.Migrator().HasTable(&LocalPartnumberBook{}) {
return nil
}
if db.Migrator().HasColumn(&LocalPartnumberBook{}, "partnumbers_json") {
return nil
}
return db.Exec(`ALTER TABLE local_partnumber_books ADD COLUMN partnumbers_json TEXT NOT NULL DEFAULT '[]'`).Error
}
func quarantineSQLiteTable(db *gorm.DB, tableName string, quarantineName string) error {
if !db.Migrator().HasTable(tableName) {
return nil
}
if tableName == quarantineName {
return nil
}
if db.Migrator().HasTable(quarantineName) {
if err := db.Exec(`DROP TABLE ` + quarantineName).Error; err != nil {
return err
}
}
return db.Exec(`ALTER TABLE ` + tableName + ` RENAME TO ` + quarantineName).Error
}
func localPartnumberBookCatalogQuarantineTableName(kind string) string {
return "local_partnumber_book_items_" + kind + "_" + time.Now().UTC().Format("20060102150405")
}
func localReadOnlyCacheQuarantineTableName(tableName string, kind string) string {
return tableName + "_" + kind + "_" + time.Now().UTC().Format("20060102150405")
}
// HasSettings returns true if connection settings exist
func (l *LocalDB) HasSettings() bool {
var count int64
@@ -206,10 +500,23 @@ func (l *LocalDB) GetSettings() (*ConnectionSettings, error) {
}
// Decrypt password
password, err := Decrypt(settings.PasswordEncrypted)
password, migrated, err := DecryptWithMetadata(settings.PasswordEncrypted)
if err != nil {
return nil, fmt.Errorf("decrypting password: %w", err)
}
if migrated {
encrypted, encryptErr := Encrypt(password)
if encryptErr != nil {
return nil, fmt.Errorf("re-encrypting legacy password: %w", encryptErr)
}
if err := l.db.Model(&ConnectionSettings{}).
Where("id = ?", settings.ID).
Update("password_encrypted", encrypted).Error; err != nil {
return nil, fmt.Errorf("upgrading legacy password encryption: %w", err)
}
}
settings.PasswordEncrypted = password // Return decrypted password in this field
return &settings, nil
@@ -1235,42 +1542,6 @@ func (l *LocalDB) repairConfigurationChange(change *PendingChange) error {
return nil
}
// GetRemoteMigrationApplied returns a locally applied remote migration by ID.
func (l *LocalDB) GetRemoteMigrationApplied(id string) (*LocalRemoteMigrationApplied, error) {
var migration LocalRemoteMigrationApplied
if err := l.db.Where("id = ?", id).First(&migration).Error; err != nil {
return nil, err
}
return &migration, nil
}
// UpsertRemoteMigrationApplied writes applied migration metadata.
func (l *LocalDB) UpsertRemoteMigrationApplied(id, checksum, appVersion string, appliedAt time.Time) error {
record := &LocalRemoteMigrationApplied{
ID: id,
Checksum: checksum,
AppVersion: appVersion,
AppliedAt: appliedAt,
}
return l.db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "id"}},
DoUpdates: clause.Assignments(map[string]interface{}{
"checksum": checksum,
"app_version": appVersion,
"applied_at": appliedAt,
}),
}).Create(record).Error
}
// GetLatestAppliedRemoteMigrationID returns last applied remote migration id.
func (l *LocalDB) GetLatestAppliedRemoteMigrationID() (string, error) {
var record LocalRemoteMigrationApplied
if err := l.db.Order("applied_at DESC").First(&record).Error; err != nil {
return "", err
}
return record.ID, nil
}
// GetSyncGuardState returns the latest readiness guard state.
func (l *LocalDB) GetSyncGuardState() (*LocalSyncGuardState, error) {
var state LocalSyncGuardState

View File

@@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"log/slog"
"sort"
"strings"
"time"
@@ -113,6 +114,19 @@ var localMigrations = []localMigration{
name: "Add line_no to local_configurations and backfill ordering",
run: addLocalConfigurationLineNo,
},
{
id: "2026_03_07_local_partnumber_book_catalog",
name: "Convert local partnumber book cache to book membership + deduplicated PN catalog",
run: migrateLocalPartnumberBookCatalog,
},
}
type localPartnumberCatalogRow struct {
Partnumber string
LotsJSON LocalPartnumberBookLots
Description string
CreatedAt time.Time
ServerID int
}
func runLocalMigrations(db *gorm.DB) error {
@@ -865,3 +879,216 @@ WHERE id IN (SELECT id FROM ranked)
return nil
}
func migrateLocalPartnumberBookCatalog(tx *gorm.DB) error {
type columnInfo struct {
Name string `gorm:"column:name"`
}
hasBooksTable := tx.Migrator().HasTable(&LocalPartnumberBook{})
hasItemsTable := tx.Migrator().HasTable(&LocalPartnumberBookItem{})
if !hasItemsTable {
return nil
}
if hasBooksTable {
var bookCols []columnInfo
if err := tx.Raw(`SELECT name FROM pragma_table_info('local_partnumber_books')`).Scan(&bookCols).Error; err != nil {
return fmt.Errorf("load local_partnumber_books columns: %w", err)
}
hasPartnumbersJSON := false
for _, c := range bookCols {
if c.Name == "partnumbers_json" {
hasPartnumbersJSON = true
break
}
}
if !hasPartnumbersJSON {
if err := tx.Exec(`ALTER TABLE local_partnumber_books ADD COLUMN partnumbers_json TEXT NOT NULL DEFAULT '[]'`).Error; err != nil {
return fmt.Errorf("add local_partnumber_books.partnumbers_json: %w", err)
}
}
}
var itemCols []columnInfo
if err := tx.Raw(`SELECT name FROM pragma_table_info('local_partnumber_book_items')`).Scan(&itemCols).Error; err != nil {
return fmt.Errorf("load local_partnumber_book_items columns: %w", err)
}
hasBookID := false
hasLotName := false
hasLotsJSON := false
for _, c := range itemCols {
if c.Name == "book_id" {
hasBookID = true
}
if c.Name == "lot_name" {
hasLotName = true
}
if c.Name == "lots_json" {
hasLotsJSON = true
}
}
if !hasBookID && !hasLotName && !hasLotsJSON {
return nil
}
type legacyRow struct {
BookID uint
Partnumber string
LotName string
Description string
CreatedAt time.Time
ServerID int
}
bookPNs := make(map[uint]map[string]struct{})
catalog := make(map[string]*localPartnumberCatalogRow)
if hasBookID || hasLotName {
var rows []legacyRow
if err := tx.Raw(`
SELECT
i.book_id,
i.partnumber,
i.lot_name,
COALESCE(i.description, '') AS description,
b.created_at,
b.server_id
FROM local_partnumber_book_items i
INNER JOIN local_partnumber_books b ON b.id = i.book_id
ORDER BY b.created_at DESC, b.id DESC, i.partnumber ASC, i.id ASC
`).Scan(&rows).Error; err != nil {
return fmt.Errorf("load legacy local partnumber book items: %w", err)
}
for _, row := range rows {
if _, ok := bookPNs[row.BookID]; !ok {
bookPNs[row.BookID] = make(map[string]struct{})
}
bookPNs[row.BookID][row.Partnumber] = struct{}{}
entry, ok := catalog[row.Partnumber]
if !ok {
entry = &localPartnumberCatalogRow{
Partnumber: row.Partnumber,
Description: row.Description,
CreatedAt: row.CreatedAt,
ServerID: row.ServerID,
}
catalog[row.Partnumber] = entry
}
if row.CreatedAt.After(entry.CreatedAt) || (row.CreatedAt.Equal(entry.CreatedAt) && row.ServerID >= entry.ServerID) {
entry.Description = row.Description
entry.CreatedAt = row.CreatedAt
entry.ServerID = row.ServerID
}
found := false
for i := range entry.LotsJSON {
if entry.LotsJSON[i].LotName == row.LotName {
entry.LotsJSON[i].Qty += 1
found = true
break
}
}
if !found && row.LotName != "" {
entry.LotsJSON = append(entry.LotsJSON, LocalPartnumberBookLot{LotName: row.LotName, Qty: 1})
}
}
var books []LocalPartnumberBook
if err := tx.Find(&books).Error; err != nil {
return fmt.Errorf("load local partnumber books: %w", err)
}
for _, book := range books {
pnSet := bookPNs[book.ID]
partnumbers := make([]string, 0, len(pnSet))
for pn := range pnSet {
partnumbers = append(partnumbers, pn)
}
sort.Strings(partnumbers)
if err := tx.Model(&LocalPartnumberBook{}).
Where("id = ?", book.ID).
Update("partnumbers_json", LocalStringList(partnumbers)).Error; err != nil {
return fmt.Errorf("update partnumbers_json for local book %d: %w", book.ID, err)
}
}
} else {
var items []LocalPartnumberBookItem
if err := tx.Order("id DESC").Find(&items).Error; err != nil {
return fmt.Errorf("load canonical local partnumber book items: %w", err)
}
for _, item := range items {
entry, ok := catalog[item.Partnumber]
if !ok {
copiedLots := append(LocalPartnumberBookLots(nil), item.LotsJSON...)
catalog[item.Partnumber] = &localPartnumberCatalogRow{
Partnumber: item.Partnumber,
LotsJSON: copiedLots,
Description: item.Description,
}
continue
}
if entry.Description == "" && item.Description != "" {
entry.Description = item.Description
}
for _, lot := range item.LotsJSON {
merged := false
for i := range entry.LotsJSON {
if entry.LotsJSON[i].LotName == lot.LotName {
if lot.Qty > entry.LotsJSON[i].Qty {
entry.LotsJSON[i].Qty = lot.Qty
}
merged = true
break
}
}
if !merged {
entry.LotsJSON = append(entry.LotsJSON, lot)
}
}
}
}
return rebuildLocalPartnumberBookCatalog(tx, catalog)
}
func rebuildLocalPartnumberBookCatalog(tx *gorm.DB, catalog map[string]*localPartnumberCatalogRow) error {
if err := tx.Exec(`
CREATE TABLE local_partnumber_book_items_new (
id INTEGER PRIMARY KEY AUTOINCREMENT,
partnumber TEXT NOT NULL UNIQUE,
lots_json TEXT NOT NULL,
description TEXT
)
`).Error; err != nil {
return fmt.Errorf("create new local_partnumber_book_items table: %w", err)
}
orderedPartnumbers := make([]string, 0, len(catalog))
for pn := range catalog {
orderedPartnumbers = append(orderedPartnumbers, pn)
}
sort.Strings(orderedPartnumbers)
for _, pn := range orderedPartnumbers {
row := catalog[pn]
sort.Slice(row.LotsJSON, func(i, j int) bool {
return row.LotsJSON[i].LotName < row.LotsJSON[j].LotName
})
if err := tx.Table("local_partnumber_book_items_new").Create(&LocalPartnumberBookItem{
Partnumber: row.Partnumber,
LotsJSON: row.LotsJSON,
Description: row.Description,
}).Error; err != nil {
return fmt.Errorf("insert new local_partnumber_book_items row for %s: %w", pn, err)
}
}
if err := tx.Exec(`DROP TABLE local_partnumber_book_items`).Error; err != nil {
return fmt.Errorf("drop legacy local_partnumber_book_items: %w", err)
}
if err := tx.Exec(`ALTER TABLE local_partnumber_book_items_new RENAME TO local_partnumber_book_items`).Error; err != nil {
return fmt.Errorf("rename new local_partnumber_book_items table: %w", err)
}
if err := tx.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_local_partnumber_book_items_partnumber ON local_partnumber_book_items(partnumber)`).Error; err != nil {
return fmt.Errorf("create local_partnumber_book_items partnumber index: %w", err)
}
return nil
}

View File

@@ -203,18 +203,6 @@ func (LocalComponent) TableName() string {
return "local_components"
}
// LocalRemoteMigrationApplied tracks remote SQLite migrations received from server and applied locally.
type LocalRemoteMigrationApplied struct {
ID string `gorm:"primaryKey;size:128" json:"id"`
Checksum string `gorm:"size:128;not null" json:"checksum"`
AppVersion string `gorm:"size:64" json:"app_version,omitempty"`
AppliedAt time.Time `gorm:"not null" json:"applied_at"`
}
func (LocalRemoteMigrationApplied) TableName() string {
return "local_remote_migrations_applied"
}
// LocalSyncGuardState stores latest sync readiness decision for UI and preflight checks.
type LocalSyncGuardState struct {
ID uint `gorm:"primaryKey;autoIncrement" json:"id"`
@@ -248,25 +236,52 @@ func (PendingChange) TableName() string {
// LocalPartnumberBook stores a version snapshot of the PN→LOT mapping book (pull-only from PriceForge)
type LocalPartnumberBook struct {
ID uint `gorm:"primaryKey;autoIncrement" json:"id"`
ServerID int `gorm:"uniqueIndex;not null" json:"server_id"`
Version string `gorm:"not null" json:"version"`
CreatedAt time.Time `gorm:"not null" json:"created_at"`
IsActive bool `gorm:"not null;default:true" json:"is_active"`
ID uint `gorm:"primaryKey;autoIncrement" json:"id"`
ServerID int `gorm:"uniqueIndex;not null" json:"server_id"`
Version string `gorm:"not null" json:"version"`
CreatedAt time.Time `gorm:"not null" json:"created_at"`
IsActive bool `gorm:"not null;default:true" json:"is_active"`
PartnumbersJSON LocalStringList `gorm:"column:partnumbers_json;type:text" json:"partnumbers_json"`
}
func (LocalPartnumberBook) TableName() string {
return "local_partnumber_books"
}
// LocalPartnumberBookItem stores a single PN→LOT mapping within a book snapshot
type LocalPartnumberBookLot struct {
LotName string `json:"lot_name"`
Qty float64 `json:"qty"`
}
type LocalPartnumberBookLots []LocalPartnumberBookLot
func (l LocalPartnumberBookLots) Value() (driver.Value, error) {
return json.Marshal(l)
}
func (l *LocalPartnumberBookLots) Scan(value interface{}) error {
if value == nil {
*l = make(LocalPartnumberBookLots, 0)
return nil
}
var bytes []byte
switch v := value.(type) {
case []byte:
bytes = v
case string:
bytes = []byte(v)
default:
return errors.New("type assertion failed for LocalPartnumberBookLots")
}
return json.Unmarshal(bytes, l)
}
// LocalPartnumberBookItem stores the canonical PN composition pulled from PriceForge.
type LocalPartnumberBookItem struct {
ID uint `gorm:"primaryKey;autoIncrement" json:"id"`
BookID uint `gorm:"not null;index:idx_local_book_pn,priority:1" json:"book_id"`
Partnumber string `gorm:"not null;index:idx_local_book_pn,priority:2" json:"partnumber"`
LotName string `gorm:"not null" json:"lot_name"`
IsPrimaryPN bool `gorm:"not null;default:false" json:"is_primary_pn"`
Description string `json:"description,omitempty"`
ID uint `gorm:"primaryKey;autoIncrement" json:"id"`
Partnumber string `gorm:"not null" json:"partnumber"`
LotsJSON LocalPartnumberBookLots `gorm:"column:lots_json;type:text" json:"lots_json"`
Description string `json:"description,omitempty"`
}
func (LocalPartnumberBookItem) TableName() string {