Files
QuoteForge/internal/services/sync/incremental_pull_mariadb_test.go
T
Mikhail ChusavitinandClaude Opus 5.5 44a09d4420 fix(sync): stop MariaDB connection leak and cut per-cycle sync waste
- ConnectionManager reuses a stale-but-healthy pool instead of dialing a new
  one every 30s, and closes every pool it replaces (one client held ~30
  connections). Pool capped at 2 connections, idle released after 1 min.
- Background sync every 1 min with ±30% jitter, scheduled from cycle start;
  users-status online threshold = interval × 1.3. One heartbeat per cycle.
- Partnumber books: read headers only, fetch partnumbers_json per book when
  new/incomplete; empty book counts as complete; an incomplete book is
  re-pulled once per process (server book lists BOT_AGIBOT_G2_PERF missing
  from qt_partnumber_book_items).
- NeedSync in one UNION ALL query; SyncQtSettings skips unchanged rewrite.
- Migration 035: server-maintained server_updated_at on qt_configurations and
  qt_projects. Clients pull only changed rows after one full pull per process;
  servers without 035 and old clients keep full pulls.
- Migrator: fresh GORM statement per verify query and history insert; files
  with two verify checks failed to record.

ADL: bible-local/decisions/2026-10-26-mariadb-single-pool.md,
bible-local/decisions/2026-10-26-incremental-config-project-pull.md

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 20:46:59 +03:00

204 lines
6.8 KiB
Go

package sync_test
import (
"os"
"strings"
"testing"
"git.mchus.pro/mchus/quoteforge/internal/models"
syncsvc "git.mchus.pro/mchus/quoteforge/internal/services/sync"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
// Runs only against a disposable MariaDB that already has the qt_projects and
// qt_configurations tables (e.g. a container loaded with SHOW CREATE TABLE from
// production). The test truncates them. Example:
//
// QF_TEST_MARIADB_DSN='root:test@tcp(127.0.0.1:33306)/qf?parseTime=true' go test ./internal/services/sync -run MariaDB
func newMariaDBForPullTest(t *testing.T) *gorm.DB {
t.Helper()
dsn := os.Getenv("QF_TEST_MARIADB_DSN")
if dsn == "" {
t.Skip("QF_TEST_MARIADB_DSN not set")
}
db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
if err != nil {
t.Fatalf("open mariadb: %v", err)
}
mustExec(t, db, "SET FOREIGN_KEY_CHECKS=0")
mustExec(t, db, "TRUNCATE TABLE qt_configurations")
mustExec(t, db, "TRUNCATE TABLE qt_projects")
mustExec(t, db, "SET FOREIGN_KEY_CHECKS=1")
for _, table := range []string{"qt_configurations", "qt_projects"} {
mustExec(t, db, "ALTER TABLE "+table+" DROP INDEX IF EXISTS idx_"+table+"_server_updated_at")
mustExec(t, db, "ALTER TABLE "+table+" DROP COLUMN IF EXISTS server_updated_at")
}
return db
}
func mustExec(t *testing.T, db *gorm.DB, sql string, args ...any) {
t.Helper()
if err := db.Exec(sql, args...).Error; err != nil {
t.Fatalf("%s: %v", sql, err)
}
}
func applyServerUpdatedAtMigration(t *testing.T, db *gorm.DB) {
t.Helper()
raw, err := os.ReadFile("../../../migrations/035_add_server_updated_at.sql")
if err != nil {
t.Fatalf("read migration: %v", err)
}
var body []string
for _, line := range strings.Split(string(raw), "\n") {
if !strings.HasPrefix(strings.TrimSpace(line), "--") {
body = append(body, line)
}
}
for _, stmt := range strings.Split(strings.Join(body, "\n"), ";") {
if strings.TrimSpace(stmt) != "" {
mustExec(t, db, stmt)
}
}
}
// seedServerConfig writes the way any client version does: through the
// models.Configuration struct, which knows nothing about server_updated_at.
func seedServerConfig(t *testing.T, db *gorm.DB, uuid, name string) *models.Configuration {
t.Helper()
cfg := &models.Configuration{
UUID: uuid,
OwnerUsername: "tester",
Name: name,
Items: models.ConfigItems{{LotName: "CPU_PULL", Quantity: 1, UnitPrice: 100}},
ServerCount: 1,
Line: 10,
}
if err := db.Create(cfg).Error; err != nil {
t.Fatalf("seed config %s: %v", uuid, err)
}
return cfg
}
func TestIncrementalPullMariaDB(t *testing.T) {
serverDB := newMariaDBForPullTest(t)
local := newLocalDBForSyncTest(t)
if err := local.SaveSettings("127.0.0.1", 33306, "qf", "root", "test"); err != nil {
t.Fatalf("save settings: %v", err)
}
// Before migration 035: every pull is full, exactly as before.
seedServerConfig(t, serverDB, "cfg-a", "A")
seedServerConfig(t, serverDB, "cfg-b", "B")
svc := syncsvc.NewServiceWithDB(serverDB, local)
for i := 0; i < 2; i++ {
res, err := svc.ImportConfigurationsToLocal()
if err != nil {
t.Fatalf("pre-migration import: %v", err)
}
if res.Imported+res.Updated != 2 {
t.Fatalf("pre-migration pull %d: expected full pull of 2, got %+v", i, res)
}
}
applyServerUpdatedAtMigration(t, serverDB)
applyServerUpdatedAtMigration(t, serverDB) // re-runnable
// A client without the column in its model still inserts and updates fine.
seedServerConfig(t, serverDB, "cfg-c", "C")
// Age all rows past the overlap window.
mustExec(t, serverDB, "UPDATE qt_configurations SET server_updated_at = NOW(6) - INTERVAL 1 HOUR")
// First pull after the migration in this process: full, sets the cursor.
res, err := svc.ImportConfigurationsToLocal()
if err != nil {
t.Fatalf("first post-migration import: %v", err)
}
if res.Imported+res.Updated != 3 {
t.Fatalf("expected full pull of 3, got %+v", res)
}
// Nothing changed on the server: nothing is pulled.
res, err = svc.ImportConfigurationsToLocal()
if err != nil {
t.Fatalf("idle incremental import: %v", err)
}
if res.Imported+res.Updated != 0 {
t.Fatalf("expected empty incremental pull, got %+v", res)
}
// An old-style client update (gorm Save of the whole struct) bumps the marker.
var b models.Configuration
if err := serverDB.Where("uuid = ?", "cfg-b").First(&b).Error; err != nil {
t.Fatalf("load cfg-b: %v", err)
}
b.Name = "B renamed"
if err := serverDB.Save(&b).Error; err != nil {
t.Fatalf("save cfg-b: %v", err)
}
seedServerConfig(t, serverDB, "cfg-d", "D")
res, err = svc.ImportConfigurationsToLocal()
if err != nil {
t.Fatalf("incremental import: %v", err)
}
if res.Updated != 1 || res.Imported != 1 {
t.Fatalf("expected 1 updated + 1 imported, got %+v", res)
}
localB, err := local.GetConfigurationByUUID("cfg-b")
if err != nil || localB.Name != "B renamed" {
t.Fatalf("expected cfg-b renamed locally, got %+v err=%v", localB, err)
}
// A new process does one full pull again.
restarted := syncsvc.NewServiceWithDB(serverDB, local)
res, err = restarted.ImportConfigurationsToLocal()
if err != nil {
t.Fatalf("restart import: %v", err)
}
if res.Imported+res.Updated != 4 {
t.Fatalf("expected full pull of 4 after restart, got %+v", res)
}
// Switching to another server invalidates the cursor.
if err := local.SaveSettings("other-host", 3306, "qf", "root", "test"); err != nil {
t.Fatalf("save settings: %v", err)
}
res, err = restarted.ImportConfigurationsToLocal()
if err != nil {
t.Fatalf("import after server switch: %v", err)
}
if res.Imported+res.Updated != 4 {
t.Fatalf("expected full pull of 4 after server switch, got %+v", res)
}
}
func TestIncrementalProjectPullMariaDB(t *testing.T) {
serverDB := newMariaDBForPullTest(t)
local := newLocalDBForSyncTest(t)
if err := local.SaveSettings("127.0.0.1", 33306, "qf", "root", "test"); err != nil {
t.Fatalf("save settings: %v", err)
}
applyServerUpdatedAtMigration(t, serverDB)
code := "P-1"
project := &models.Project{UUID: "prj-1", OwnerUsername: "tester", Code: code, IsActive: true}
if err := serverDB.Create(project).Error; err != nil {
t.Fatalf("seed project: %v", err)
}
mustExec(t, serverDB, "UPDATE qt_projects SET server_updated_at = NOW(6) - INTERVAL 1 HOUR")
svc := syncsvc.NewServiceWithDB(serverDB, local)
if res, err := svc.ImportProjectsToLocal(); err != nil || res.Imported != 1 {
t.Fatalf("first project pull: %+v err=%v", res, err)
}
if res, err := svc.ImportProjectsToLocal(); err != nil || res.Imported+res.Updated != 0 {
t.Fatalf("idle project pull: %+v err=%v", res, err)
}
mustExec(t, serverDB, "UPDATE qt_projects SET name = 'renamed' WHERE uuid = 'prj-1'")
if res, err := svc.ImportProjectsToLocal(); err != nil || res.Updated != 1 {
t.Fatalf("incremental project pull: %+v err=%v", res, err)
}
}