- ConnectionManager reuses a stale-but-healthy pool instead of dialing a new one every 30s, and closes every pool it replaces (one client held ~30 connections). Pool capped at 2 connections, idle released after 1 min. - Background sync every 1 min with ±30% jitter, scheduled from cycle start; users-status online threshold = interval × 1.3. One heartbeat per cycle. - Partnumber books: read headers only, fetch partnumbers_json per book when new/incomplete; empty book counts as complete; an incomplete book is re-pulled once per process (server book lists BOT_AGIBOT_G2_PERF missing from qt_partnumber_book_items). - NeedSync in one UNION ALL query; SyncQtSettings skips unchanged rewrite. - Migration 035: server-maintained server_updated_at on qt_configurations and qt_projects. Clients pull only changed rows after one full pull per process; servers without 035 and old clients keep full pulls. - Migrator: fresh GORM statement per verify query and history insert; files with two verify checks failed to record. ADL: bible-local/decisions/2026-10-26-mariadb-single-pool.md, bible-local/decisions/2026-10-26-incremental-config-project-pull.md Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
198 lines
7.3 KiB
Go
198 lines
7.3 KiB
Go
package sync
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"git.mchus.pro/mchus/quoteforge/internal/localdb"
|
|
"git.mchus.pro/mchus/quoteforge/internal/repository"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// PullPartnumberBooks synchronizes partnumber book snapshots from MariaDB to local SQLite.
|
|
// Append-only for headers; re-pulls items if a book header exists but has 0 items.
|
|
func (s *Service) PullPartnumberBooks() (int, error) {
|
|
slog.Info("starting partnumber book pull")
|
|
|
|
mariaDB, err := s.getDB()
|
|
if err != nil {
|
|
return 0, fmt.Errorf("database not available: %w", err)
|
|
}
|
|
|
|
localBookRepo := repository.NewPartnumberBookRepository(s.localDB.DB())
|
|
|
|
// Headers only: partnumbers_json is several MB in total and the worker runs
|
|
// every minute, so it is fetched per book only when that book needs items.
|
|
type serverBook struct {
|
|
ID int `gorm:"column:id"`
|
|
Version string `gorm:"column:version"`
|
|
CreatedAt time.Time `gorm:"column:created_at"`
|
|
IsActive bool `gorm:"column:is_active"`
|
|
}
|
|
var serverBooks []serverBook
|
|
if err := mariaDB.Raw("SELECT id, version, created_at, is_active FROM qt_partnumber_books ORDER BY created_at DESC, id DESC").Scan(&serverBooks).Error; err != nil {
|
|
return 0, fmt.Errorf("querying server partnumber books: %w", err)
|
|
}
|
|
slog.Info("partnumber books found on server", "count", len(serverBooks))
|
|
|
|
pulled := 0
|
|
for _, sb := range serverBooks {
|
|
var existing localdb.LocalPartnumberBook
|
|
err := s.localDB.DB().Where("server_id = ?", sb.ID).First(&existing).Error
|
|
if err == nil {
|
|
if existing.Version != sb.Version || existing.IsActive != sb.IsActive {
|
|
existing.Version = sb.Version
|
|
existing.CreatedAt = sb.CreatedAt
|
|
existing.IsActive = sb.IsActive
|
|
if err := localBookRepo.SaveBook(&existing); err != nil {
|
|
slog.Error("failed to update local partnumber book header", "server_id", sb.ID, "error", err)
|
|
continue
|
|
}
|
|
}
|
|
|
|
if localBookRepo.HasAllBookItems(existing.ID) {
|
|
slog.Debug("partnumber book already synced, skipping", "server_id", sb.ID, "version", sb.Version)
|
|
continue
|
|
}
|
|
// A server book may list partnumbers that qt_partnumber_book_items
|
|
// lacks; re-pulling cannot fix that, so try once per process.
|
|
if !s.markPartnumberBookRepair(sb.ID, sb.Version) {
|
|
slog.Debug("partnumber book still incomplete after re-pull this session, skipping", "server_id", sb.ID, "version", sb.Version)
|
|
continue
|
|
}
|
|
slog.Info("partnumber book header exists but catalog items are missing, re-pulling items", "server_id", sb.ID, "version", sb.Version)
|
|
partnumbers, err := fetchServerBookPartnumbers(mariaDB, sb.ID)
|
|
if err != nil {
|
|
slog.Error("failed to fetch server partnumbers_json", "server_id", sb.ID, "error", err)
|
|
continue
|
|
}
|
|
existing.PartnumbersJSON = partnumbers
|
|
if err := localBookRepo.SaveBook(&existing); err != nil {
|
|
slog.Error("failed to update local partnumber book header", "server_id", sb.ID, "error", err)
|
|
continue
|
|
}
|
|
n, err := pullBookItems(mariaDB, localBookRepo, existing.PartnumbersJSON)
|
|
if err != nil {
|
|
slog.Error("failed to re-pull items for existing book", "server_id", sb.ID, "error", err)
|
|
} else {
|
|
slog.Info("re-pulled items for existing book", "server_id", sb.ID, "version", sb.Version, "items_saved", n)
|
|
pulled++
|
|
if missing := localBookRepo.MissingBookItems(existing.ID); len(missing) > 0 {
|
|
slog.Warn("partnumber book lists partnumbers absent from server qt_partnumber_book_items", "server_id", sb.ID, "version", sb.Version, "missing", missing)
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
|
|
slog.Info("pulling new partnumber book", "server_id", sb.ID, "version", sb.Version, "is_active", sb.IsActive)
|
|
|
|
partnumbers, err := fetchServerBookPartnumbers(mariaDB, sb.ID)
|
|
if err != nil {
|
|
slog.Error("failed to fetch server partnumbers_json", "server_id", sb.ID, "error", err)
|
|
continue
|
|
}
|
|
localBook := &localdb.LocalPartnumberBook{
|
|
ServerID: sb.ID,
|
|
Version: sb.Version,
|
|
CreatedAt: sb.CreatedAt,
|
|
IsActive: sb.IsActive,
|
|
PartnumbersJSON: partnumbers,
|
|
}
|
|
if err := localBookRepo.SaveBook(localBook); err != nil {
|
|
slog.Error("failed to save local partnumber book", "server_id", sb.ID, "error", err)
|
|
continue
|
|
}
|
|
|
|
n, err := pullBookItems(mariaDB, localBookRepo, localBook.PartnumbersJSON)
|
|
if err != nil {
|
|
slog.Error("failed to pull items for new book", "server_id", sb.ID, "error", err)
|
|
continue
|
|
}
|
|
|
|
slog.Info("partnumber book saved locally", "server_id", sb.ID, "version", sb.Version, "items_saved", n)
|
|
pulled++
|
|
}
|
|
|
|
slog.Info("partnumber book pull completed", "new_books_pulled", pulled, "total_on_server", len(serverBooks))
|
|
return pulled, nil
|
|
}
|
|
|
|
// markPartnumberBookRepair records a re-pull attempt for a book version and
|
|
// reports whether this is the first attempt in the current process.
|
|
func (s *Service) markPartnumberBookRepair(serverID int, version string) bool {
|
|
s.pnRepairMu.Lock()
|
|
defer s.pnRepairMu.Unlock()
|
|
if s.pnRepairAttempts == nil {
|
|
s.pnRepairAttempts = map[int]string{}
|
|
}
|
|
if v, ok := s.pnRepairAttempts[serverID]; ok && v == version {
|
|
return false
|
|
}
|
|
s.pnRepairAttempts[serverID] = version
|
|
return true
|
|
}
|
|
|
|
// fetchServerBookPartnumbers loads and decodes partnumbers_json of one server book.
|
|
func fetchServerBookPartnumbers(mariaDB *gorm.DB, serverID int) (localdb.LocalStringList, error) {
|
|
var raw string
|
|
if err := mariaDB.Raw("SELECT partnumbers_json FROM qt_partnumber_books WHERE id = ?", serverID).Scan(&raw).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeServerPartnumbers(raw)
|
|
}
|
|
|
|
// pullBookItems fetches catalog items for a partnumber list from MariaDB and saves them to SQLite.
|
|
// Returns the number of items saved.
|
|
func pullBookItems(mariaDB *gorm.DB, repo *repository.PartnumberBookRepository, partnumbers localdb.LocalStringList) (int, error) {
|
|
if len(partnumbers) == 0 {
|
|
return 0, nil
|
|
}
|
|
|
|
type serverItem struct {
|
|
Partnumber string `gorm:"column:partnumber"`
|
|
LotsJSON string `gorm:"column:lots_json"`
|
|
Description string `gorm:"column:description"`
|
|
}
|
|
var serverItems []serverItem
|
|
err := mariaDB.Raw("SELECT partnumber, lots_json, description FROM qt_partnumber_book_items WHERE partnumber IN ?", []string(partnumbers)).Scan(&serverItems).Error
|
|
if err != nil {
|
|
return 0, fmt.Errorf("querying items from server: %w", err)
|
|
}
|
|
slog.Info("partnumber book items fetched from server", "count", len(serverItems), "requested_partnumbers", len(partnumbers))
|
|
|
|
if len(serverItems) == 0 {
|
|
slog.Warn("server returned 0 partnumber book items")
|
|
return 0, nil
|
|
}
|
|
|
|
localItems := make([]localdb.LocalPartnumberBookItem, 0, len(serverItems))
|
|
for _, si := range serverItems {
|
|
var lots localdb.LocalPartnumberBookLots
|
|
if err := json.Unmarshal([]byte(si.LotsJSON), &lots); err != nil {
|
|
return 0, fmt.Errorf("decode lots_json for %s: %w", si.Partnumber, err)
|
|
}
|
|
localItems = append(localItems, localdb.LocalPartnumberBookItem{
|
|
Partnumber: si.Partnumber,
|
|
LotsJSON: lots,
|
|
Description: si.Description,
|
|
})
|
|
}
|
|
if err := repo.SaveBookItems(localItems); err != nil {
|
|
return 0, fmt.Errorf("saving items to local db: %w", err)
|
|
}
|
|
return len(localItems), nil
|
|
}
|
|
|
|
func decodeServerPartnumbers(raw string) (localdb.LocalStringList, error) {
|
|
if raw == "" {
|
|
return localdb.LocalStringList{}, nil
|
|
}
|
|
var items []string
|
|
if err := json.Unmarshal([]byte(raw), &items); err != nil {
|
|
return nil, err
|
|
}
|
|
return localdb.LocalStringList(items), nil
|
|
}
|