- ConnectionManager reuses a stale-but-healthy pool instead of dialing a new one every 30s, and closes every pool it replaces (one client held ~30 connections). Pool capped at 2 connections, idle released after 1 min. - Background sync every 1 min with ±30% jitter, scheduled from cycle start; users-status online threshold = interval × 1.3. One heartbeat per cycle. - Partnumber books: read headers only, fetch partnumbers_json per book when new/incomplete; empty book counts as complete; an incomplete book is re-pulled once per process (server book lists BOT_AGIBOT_G2_PERF missing from qt_partnumber_book_items). - NeedSync in one UNION ALL query; SyncQtSettings skips unchanged rewrite. - Migration 035: server-maintained server_updated_at on qt_configurations and qt_projects. Clients pull only changed rows after one full pull per process; servers without 035 and old clients keep full pulls. - Migrator: fresh GORM statement per verify query and history insert; files with two verify checks failed to record. ADL: bible-local/decisions/2026-10-26-mariadb-single-pool.md, bible-local/decisions/2026-10-26-incremental-config-project-pull.md Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
106 lines
3.4 KiB
Go
106 lines
3.4 KiB
Go
package sync
|
|
|
|
import (
|
|
"log/slog"
|
|
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// serverPullOverlap re-reads rows changed shortly before the cursor, covering
|
|
// transactions that stamped server_updated_at before the cursor but committed
|
|
// after the previous pull read it.
|
|
const serverPullOverlap = "INTERVAL 120 SECOND"
|
|
|
|
// serverPull describes how one pull of a server table should run. See
|
|
// bible-local/decisions/2026-10-26-incremental-config-project-pull.md.
|
|
type serverPull struct {
|
|
table string
|
|
scope string
|
|
since string // cursor; empty = full pull
|
|
startedAt string // server NOW(6) before reading; becomes the next cursor
|
|
trackable bool // server has server_updated_at (migration 035)
|
|
}
|
|
|
|
// beginServerPull decides between a full and an incremental pull of table.
|
|
// The first pull of each table per process is full, so rows lost locally are
|
|
// restored at least once per app start; later pulls read only rows whose
|
|
// server_updated_at moved since the stored cursor.
|
|
func (s *Service) beginServerPull(mariaDB *gorm.DB, table string) serverPull {
|
|
pull := serverPull{table: table}
|
|
if !s.serverHasChangeMarker(mariaDB, table) {
|
|
return pull
|
|
}
|
|
if err := mariaDB.Raw("SELECT DATE_FORMAT(NOW(6), '%Y-%m-%d %H:%i:%s.%f')").Scan(&pull.startedAt).Error; err != nil || pull.startedAt == "" {
|
|
slog.Warn("server pull: failed to read server time, falling back to full pull", "table", table, "error", err)
|
|
return serverPull{table: table}
|
|
}
|
|
scope, err := s.localDB.ServerPullScope()
|
|
if err != nil {
|
|
return serverPull{table: table}
|
|
}
|
|
pull.scope = scope
|
|
pull.trackable = true
|
|
|
|
s.pullMu.Lock()
|
|
fullDone := s.fullPullDone[table]
|
|
s.pullMu.Unlock()
|
|
if fullDone {
|
|
if since, ok := s.localDB.GetServerPullCursor(table, scope); ok {
|
|
pull.since = since
|
|
}
|
|
}
|
|
return pull
|
|
}
|
|
|
|
// incrementalScope narrows query to rows changed since the cursor.
|
|
func (p serverPull) incrementalScope(query *gorm.DB) *gorm.DB {
|
|
return query.Where("server_updated_at >= CAST(? AS DATETIME(6)) - "+serverPullOverlap, p.since)
|
|
}
|
|
|
|
// finishServerPull records a successful pull: the cursor moves to the server
|
|
// time taken before reading, and a full pull marks the table as verified for
|
|
// this process.
|
|
func (s *Service) finishServerPull(pull serverPull) {
|
|
if !pull.trackable {
|
|
return
|
|
}
|
|
if err := s.localDB.SetServerPullCursor(pull.table, pull.scope, pull.startedAt); err != nil {
|
|
slog.Warn("server pull: failed to store cursor", "table", pull.table, "error", err)
|
|
return
|
|
}
|
|
if pull.since == "" {
|
|
s.pullMu.Lock()
|
|
if s.fullPullDone == nil {
|
|
s.fullPullDone = map[string]bool{}
|
|
}
|
|
s.fullPullDone[pull.table] = true
|
|
s.pullMu.Unlock()
|
|
}
|
|
}
|
|
|
|
// serverHasChangeMarker reports whether table has server_updated_at. A positive
|
|
// answer is cached for the process; a negative one is re-checked next time so a
|
|
// migration applied while the client runs is picked up.
|
|
func (s *Service) serverHasChangeMarker(mariaDB *gorm.DB, table string) bool {
|
|
s.pullMu.Lock()
|
|
known := s.changeMarker[table]
|
|
s.pullMu.Unlock()
|
|
if known {
|
|
return true
|
|
}
|
|
var n int64
|
|
if err := mariaDB.Raw(`
|
|
SELECT COUNT(*) FROM information_schema.COLUMNS
|
|
WHERE table_schema = DATABASE() AND table_name = ? AND column_name = 'server_updated_at'
|
|
`, table).Scan(&n).Error; err != nil || n == 0 {
|
|
return false
|
|
}
|
|
s.pullMu.Lock()
|
|
if s.changeMarker == nil {
|
|
s.changeMarker = map[string]bool{}
|
|
}
|
|
s.changeMarker[table] = true
|
|
s.pullMu.Unlock()
|
|
return true
|
|
}
|