Files
QuoteForge/internal/services/sync/partnumber_books.go
T
Mikhail ChusavitinandClaude Opus 5.5 44a09d4420 fix(sync): stop MariaDB connection leak and cut per-cycle sync waste
- ConnectionManager reuses a stale-but-healthy pool instead of dialing a new
  one every 30s, and closes every pool it replaces (one client held ~30
  connections). Pool capped at 2 connections, idle released after 1 min.
- Background sync every 1 min with ±30% jitter, scheduled from cycle start;
  users-status online threshold = interval × 1.3. One heartbeat per cycle.
- Partnumber books: read headers only, fetch partnumbers_json per book when
  new/incomplete; empty book counts as complete; an incomplete book is
  re-pulled once per process (server book lists BOT_AGIBOT_G2_PERF missing
  from qt_partnumber_book_items).
- NeedSync in one UNION ALL query; SyncQtSettings skips unchanged rewrite.
- Migration 035: server-maintained server_updated_at on qt_configurations and
  qt_projects. Clients pull only changed rows after one full pull per process;
  servers without 035 and old clients keep full pulls.
- Migrator: fresh GORM statement per verify query and history insert; files
  with two verify checks failed to record.

ADL: bible-local/decisions/2026-10-26-mariadb-single-pool.md,
bible-local/decisions/2026-10-26-incremental-config-project-pull.md

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 20:46:59 +03:00

198 lines
7.3 KiB
Go

package sync
import (
"encoding/json"
"fmt"
"log/slog"
"time"
"git.mchus.pro/mchus/quoteforge/internal/localdb"
"git.mchus.pro/mchus/quoteforge/internal/repository"
"gorm.io/gorm"
)
// PullPartnumberBooks synchronizes partnumber book snapshots from MariaDB to local SQLite.
// Append-only for headers; re-pulls items if a book header exists but has 0 items.
func (s *Service) PullPartnumberBooks() (int, error) {
slog.Info("starting partnumber book pull")
mariaDB, err := s.getDB()
if err != nil {
return 0, fmt.Errorf("database not available: %w", err)
}
localBookRepo := repository.NewPartnumberBookRepository(s.localDB.DB())
// Headers only: partnumbers_json is several MB in total and the worker runs
// every minute, so it is fetched per book only when that book needs items.
type serverBook struct {
ID int `gorm:"column:id"`
Version string `gorm:"column:version"`
CreatedAt time.Time `gorm:"column:created_at"`
IsActive bool `gorm:"column:is_active"`
}
var serverBooks []serverBook
if err := mariaDB.Raw("SELECT id, version, created_at, is_active FROM qt_partnumber_books ORDER BY created_at DESC, id DESC").Scan(&serverBooks).Error; err != nil {
return 0, fmt.Errorf("querying server partnumber books: %w", err)
}
slog.Info("partnumber books found on server", "count", len(serverBooks))
pulled := 0
for _, sb := range serverBooks {
var existing localdb.LocalPartnumberBook
err := s.localDB.DB().Where("server_id = ?", sb.ID).First(&existing).Error
if err == nil {
if existing.Version != sb.Version || existing.IsActive != sb.IsActive {
existing.Version = sb.Version
existing.CreatedAt = sb.CreatedAt
existing.IsActive = sb.IsActive
if err := localBookRepo.SaveBook(&existing); err != nil {
slog.Error("failed to update local partnumber book header", "server_id", sb.ID, "error", err)
continue
}
}
if localBookRepo.HasAllBookItems(existing.ID) {
slog.Debug("partnumber book already synced, skipping", "server_id", sb.ID, "version", sb.Version)
continue
}
// A server book may list partnumbers that qt_partnumber_book_items
// lacks; re-pulling cannot fix that, so try once per process.
if !s.markPartnumberBookRepair(sb.ID, sb.Version) {
slog.Debug("partnumber book still incomplete after re-pull this session, skipping", "server_id", sb.ID, "version", sb.Version)
continue
}
slog.Info("partnumber book header exists but catalog items are missing, re-pulling items", "server_id", sb.ID, "version", sb.Version)
partnumbers, err := fetchServerBookPartnumbers(mariaDB, sb.ID)
if err != nil {
slog.Error("failed to fetch server partnumbers_json", "server_id", sb.ID, "error", err)
continue
}
existing.PartnumbersJSON = partnumbers
if err := localBookRepo.SaveBook(&existing); err != nil {
slog.Error("failed to update local partnumber book header", "server_id", sb.ID, "error", err)
continue
}
n, err := pullBookItems(mariaDB, localBookRepo, existing.PartnumbersJSON)
if err != nil {
slog.Error("failed to re-pull items for existing book", "server_id", sb.ID, "error", err)
} else {
slog.Info("re-pulled items for existing book", "server_id", sb.ID, "version", sb.Version, "items_saved", n)
pulled++
if missing := localBookRepo.MissingBookItems(existing.ID); len(missing) > 0 {
slog.Warn("partnumber book lists partnumbers absent from server qt_partnumber_book_items", "server_id", sb.ID, "version", sb.Version, "missing", missing)
}
}
continue
}
slog.Info("pulling new partnumber book", "server_id", sb.ID, "version", sb.Version, "is_active", sb.IsActive)
partnumbers, err := fetchServerBookPartnumbers(mariaDB, sb.ID)
if err != nil {
slog.Error("failed to fetch server partnumbers_json", "server_id", sb.ID, "error", err)
continue
}
localBook := &localdb.LocalPartnumberBook{
ServerID: sb.ID,
Version: sb.Version,
CreatedAt: sb.CreatedAt,
IsActive: sb.IsActive,
PartnumbersJSON: partnumbers,
}
if err := localBookRepo.SaveBook(localBook); err != nil {
slog.Error("failed to save local partnumber book", "server_id", sb.ID, "error", err)
continue
}
n, err := pullBookItems(mariaDB, localBookRepo, localBook.PartnumbersJSON)
if err != nil {
slog.Error("failed to pull items for new book", "server_id", sb.ID, "error", err)
continue
}
slog.Info("partnumber book saved locally", "server_id", sb.ID, "version", sb.Version, "items_saved", n)
pulled++
}
slog.Info("partnumber book pull completed", "new_books_pulled", pulled, "total_on_server", len(serverBooks))
return pulled, nil
}
// markPartnumberBookRepair records a re-pull attempt for a book version and
// reports whether this is the first attempt in the current process.
func (s *Service) markPartnumberBookRepair(serverID int, version string) bool {
s.pnRepairMu.Lock()
defer s.pnRepairMu.Unlock()
if s.pnRepairAttempts == nil {
s.pnRepairAttempts = map[int]string{}
}
if v, ok := s.pnRepairAttempts[serverID]; ok && v == version {
return false
}
s.pnRepairAttempts[serverID] = version
return true
}
// fetchServerBookPartnumbers loads and decodes partnumbers_json of one server book.
func fetchServerBookPartnumbers(mariaDB *gorm.DB, serverID int) (localdb.LocalStringList, error) {
var raw string
if err := mariaDB.Raw("SELECT partnumbers_json FROM qt_partnumber_books WHERE id = ?", serverID).Scan(&raw).Error; err != nil {
return nil, err
}
return decodeServerPartnumbers(raw)
}
// pullBookItems fetches catalog items for a partnumber list from MariaDB and saves them to SQLite.
// Returns the number of items saved.
func pullBookItems(mariaDB *gorm.DB, repo *repository.PartnumberBookRepository, partnumbers localdb.LocalStringList) (int, error) {
if len(partnumbers) == 0 {
return 0, nil
}
type serverItem struct {
Partnumber string `gorm:"column:partnumber"`
LotsJSON string `gorm:"column:lots_json"`
Description string `gorm:"column:description"`
}
var serverItems []serverItem
err := mariaDB.Raw("SELECT partnumber, lots_json, description FROM qt_partnumber_book_items WHERE partnumber IN ?", []string(partnumbers)).Scan(&serverItems).Error
if err != nil {
return 0, fmt.Errorf("querying items from server: %w", err)
}
slog.Info("partnumber book items fetched from server", "count", len(serverItems), "requested_partnumbers", len(partnumbers))
if len(serverItems) == 0 {
slog.Warn("server returned 0 partnumber book items")
return 0, nil
}
localItems := make([]localdb.LocalPartnumberBookItem, 0, len(serverItems))
for _, si := range serverItems {
var lots localdb.LocalPartnumberBookLots
if err := json.Unmarshal([]byte(si.LotsJSON), &lots); err != nil {
return 0, fmt.Errorf("decode lots_json for %s: %w", si.Partnumber, err)
}
localItems = append(localItems, localdb.LocalPartnumberBookItem{
Partnumber: si.Partnumber,
LotsJSON: lots,
Description: si.Description,
})
}
if err := repo.SaveBookItems(localItems); err != nil {
return 0, fmt.Errorf("saving items to local db: %w", err)
}
return len(localItems), nil
}
func decodeServerPartnumbers(raw string) (localdb.LocalStringList, error) {
if raw == "" {
return localdb.LocalStringList{}, nil
}
var items []string
if err := json.Unmarshal([]byte(raw), &items); err != nil {
return nil, err
}
return localdb.LocalStringList(items), nil
}