Files
QuoteForge/internal/services/sync/pull_cursor.go
T
Mikhail ChusavitinandClaude Opus 5.5 44a09d4420 fix(sync): stop MariaDB connection leak and cut per-cycle sync waste
- ConnectionManager reuses a stale-but-healthy pool instead of dialing a new
  one every 30s, and closes every pool it replaces (one client held ~30
  connections). Pool capped at 2 connections, idle released after 1 min.
- Background sync every 1 min with ±30% jitter, scheduled from cycle start;
  users-status online threshold = interval × 1.3. One heartbeat per cycle.
- Partnumber books: read headers only, fetch partnumbers_json per book when
  new/incomplete; empty book counts as complete; an incomplete book is
  re-pulled once per process (server book lists BOT_AGIBOT_G2_PERF missing
  from qt_partnumber_book_items).
- NeedSync in one UNION ALL query; SyncQtSettings skips unchanged rewrite.
- Migration 035: server-maintained server_updated_at on qt_configurations and
  qt_projects. Clients pull only changed rows after one full pull per process;
  servers without 035 and old clients keep full pulls.
- Migrator: fresh GORM statement per verify query and history insert; files
  with two verify checks failed to record.

ADL: bible-local/decisions/2026-10-26-mariadb-single-pool.md,
bible-local/decisions/2026-10-26-incremental-config-project-pull.md

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 20:46:59 +03:00

106 lines
3.4 KiB
Go

package sync
import (
"log/slog"
"gorm.io/gorm"
)
// serverPullOverlap re-reads rows changed shortly before the cursor, covering
// transactions that stamped server_updated_at before the cursor but committed
// after the previous pull read it.
const serverPullOverlap = "INTERVAL 120 SECOND"
// serverPull describes how one pull of a server table should run. See
// bible-local/decisions/2026-10-26-incremental-config-project-pull.md.
type serverPull struct {
table string
scope string
since string // cursor; empty = full pull
startedAt string // server NOW(6) before reading; becomes the next cursor
trackable bool // server has server_updated_at (migration 035)
}
// beginServerPull decides between a full and an incremental pull of table.
// The first pull of each table per process is full, so rows lost locally are
// restored at least once per app start; later pulls read only rows whose
// server_updated_at moved since the stored cursor.
func (s *Service) beginServerPull(mariaDB *gorm.DB, table string) serverPull {
pull := serverPull{table: table}
if !s.serverHasChangeMarker(mariaDB, table) {
return pull
}
if err := mariaDB.Raw("SELECT DATE_FORMAT(NOW(6), '%Y-%m-%d %H:%i:%s.%f')").Scan(&pull.startedAt).Error; err != nil || pull.startedAt == "" {
slog.Warn("server pull: failed to read server time, falling back to full pull", "table", table, "error", err)
return serverPull{table: table}
}
scope, err := s.localDB.ServerPullScope()
if err != nil {
return serverPull{table: table}
}
pull.scope = scope
pull.trackable = true
s.pullMu.Lock()
fullDone := s.fullPullDone[table]
s.pullMu.Unlock()
if fullDone {
if since, ok := s.localDB.GetServerPullCursor(table, scope); ok {
pull.since = since
}
}
return pull
}
// incrementalScope narrows query to rows changed since the cursor.
func (p serverPull) incrementalScope(query *gorm.DB) *gorm.DB {
return query.Where("server_updated_at >= CAST(? AS DATETIME(6)) - "+serverPullOverlap, p.since)
}
// finishServerPull records a successful pull: the cursor moves to the server
// time taken before reading, and a full pull marks the table as verified for
// this process.
func (s *Service) finishServerPull(pull serverPull) {
if !pull.trackable {
return
}
if err := s.localDB.SetServerPullCursor(pull.table, pull.scope, pull.startedAt); err != nil {
slog.Warn("server pull: failed to store cursor", "table", pull.table, "error", err)
return
}
if pull.since == "" {
s.pullMu.Lock()
if s.fullPullDone == nil {
s.fullPullDone = map[string]bool{}
}
s.fullPullDone[pull.table] = true
s.pullMu.Unlock()
}
}
// serverHasChangeMarker reports whether table has server_updated_at. A positive
// answer is cached for the process; a negative one is re-checked next time so a
// migration applied while the client runs is picked up.
func (s *Service) serverHasChangeMarker(mariaDB *gorm.DB, table string) bool {
s.pullMu.Lock()
known := s.changeMarker[table]
s.pullMu.Unlock()
if known {
return true
}
var n int64
if err := mariaDB.Raw(`
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE table_schema = DATABASE() AND table_name = ? AND column_name = 'server_updated_at'
`, table).Scan(&n).Error; err != nil || n == 0 {
return false
}
s.pullMu.Lock()
if s.changeMarker == nil {
s.changeMarker = map[string]bool{}
}
s.changeMarker[table] = true
s.pullMu.Unlock()
return true
}