fix(sync): stop MariaDB connection leak and cut per-cycle sync waste
- ConnectionManager reuses a stale-but-healthy pool instead of dialing a new one every 30s, and closes every pool it replaces (one client held ~30 connections). Pool capped at 2 connections, idle released after 1 min. - Background sync every 1 min with ±30% jitter, scheduled from cycle start; users-status online threshold = interval × 1.3. One heartbeat per cycle. - Partnumber books: read headers only, fetch partnumbers_json per book when new/incomplete; empty book counts as complete; an incomplete book is re-pulled once per process (server book lists BOT_AGIBOT_G2_PERF missing from qt_partnumber_book_items). - NeedSync in one UNION ALL query; SyncQtSettings skips unchanged rewrite. - Migration 035: server-maintained server_updated_at on qt_configurations and qt_projects. Clients pull only changed rows after one full pull per process; servers without 035 and old clients keep full pulls. - Migrator: fresh GORM statement per verify query and history insert; files with two verify checks failed to record. ADL: bible-local/decisions/2026-10-26-mariadb-single-pool.md, bible-local/decisions/2026-10-26-incremental-config-project-pull.md Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8a08270b79
commit
44a09d4420
20 files changed
+901
-75
No files matched your search
+67
-18
@@ -22,9 +22,10 @@ const (
|
||||
// goroutines must never pay the MySQL dial/read timeout themselves.
|
||||
defaultStatusCheckInterval = 15 * time.Second
|
||||
|
||||
maxOpenConns = 10
|
||||
maxOpenConns = 2
|
||||
maxIdleConns = 2
|
||||
connMaxLifetime = 5 * time.Minute
|
||||
connMaxIdleTime = time.Minute
|
||||
)
|
||||
|
||||
// ConnectionStatus represents the current status of the database connection
|
||||
@@ -49,11 +50,13 @@ type ConnectionManager struct {
|
||||
|
||||
statusCheckInterval time.Duration // background prober cadence (default: 15s)
|
||||
stopStatusLoop chan struct{} // closed by Stop() to end the background loop
|
||||
|
||||
dialFn func() (*gorm.DB, error) // opens a new pool; cm.dial unless overridden in tests
|
||||
}
|
||||
|
||||
// NewConnectionManager creates a new ConnectionManager instance
|
||||
func NewConnectionManager(localDB *localdb.LocalDB) *ConnectionManager {
|
||||
return &ConnectionManager{
|
||||
cm := &ConnectionManager{
|
||||
localDB: localDB,
|
||||
connectTimeout: defaultConnectTimeout,
|
||||
pingInterval: defaultPingInterval,
|
||||
@@ -63,6 +66,8 @@ func NewConnectionManager(localDB *localdb.LocalDB) *ConnectionManager {
|
||||
lastError: nil,
|
||||
lastCheck: time.Time{},
|
||||
}
|
||||
cm.dialFn = cm.dial
|
||||
return cm
|
||||
}
|
||||
|
||||
// Start launches a background goroutine that keeps the online-status cache
|
||||
@@ -137,25 +142,66 @@ func (cm *ConnectionManager) GetDB() (*gorm.DB, error) {
|
||||
return db, err
|
||||
}
|
||||
|
||||
newDB, err := cm.dial()
|
||||
// The cached pool is only stale, not broken: ping it instead of dialing a
|
||||
// new one. Re-dialing here used to orphan the old *sql.DB together with its
|
||||
// idle connections, leaking a pool every pingInterval during sync.
|
||||
cm.mu.RLock()
|
||||
existing := cm.db
|
||||
cm.mu.RUnlock()
|
||||
if existing != nil {
|
||||
if err := cm.ping(existing); err == nil {
|
||||
cm.mu.Lock()
|
||||
cm.lastError = nil
|
||||
cm.lastCheck = time.Now()
|
||||
cm.mu.Unlock()
|
||||
return existing, nil
|
||||
}
|
||||
}
|
||||
|
||||
newDB, err := cm.dialFn()
|
||||
|
||||
cm.mu.Lock()
|
||||
old := cm.db
|
||||
if err != nil {
|
||||
// Drop stale handle so callers don't treat it as an active connection.
|
||||
cm.db = nil
|
||||
cm.lastError = err
|
||||
cm.lastCheck = time.Now()
|
||||
cm.mu.Unlock()
|
||||
closeDB(old)
|
||||
return nil, err
|
||||
}
|
||||
cm.db = newDB
|
||||
cm.lastError = nil
|
||||
cm.lastCheck = time.Now()
|
||||
cm.mu.Unlock()
|
||||
closeDB(old)
|
||||
|
||||
return newDB, nil
|
||||
}
|
||||
|
||||
// ping checks an existing pool with the connect timeout. No locks held.
|
||||
func (cm *ConnectionManager) ping(db *gorm.DB) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cm.connectTimeout)
|
||||
defer cancel()
|
||||
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return sqlDB.PingContext(ctx)
|
||||
}
|
||||
|
||||
// closeDB releases a pool that is no longer referenced by the manager.
|
||||
func closeDB(db *gorm.DB) {
|
||||
if db == nil {
|
||||
return
|
||||
}
|
||||
if sqlDB, err := db.DB(); err == nil {
|
||||
sqlDB.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// cachedResult returns (db, err, true) if the cached state is still fresh
|
||||
// enough to answer without a new network round-trip: either a live
|
||||
// connection within pingInterval, or a recent failure still within
|
||||
@@ -200,15 +246,17 @@ func (cm *ConnectionManager) dial() (*gorm.DB, error) {
|
||||
return nil, fmt.Errorf("getting sql.DB: %w", err)
|
||||
}
|
||||
|
||||
// Ping with timeout
|
||||
if err = sqlDB.PingContext(ctx); err != nil {
|
||||
return nil, fmt.Errorf("pinging database: %w", err)
|
||||
}
|
||||
|
||||
// Set connection pool settings
|
||||
sqlDB.SetMaxOpenConns(maxOpenConns)
|
||||
sqlDB.SetMaxIdleConns(maxIdleConns)
|
||||
sqlDB.SetConnMaxLifetime(connMaxLifetime)
|
||||
sqlDB.SetConnMaxIdleTime(connMaxIdleTime)
|
||||
|
||||
// Ping with timeout
|
||||
if err = sqlDB.PingContext(ctx); err != nil {
|
||||
sqlDB.Close()
|
||||
return nil, fmt.Errorf("pinging database: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
@@ -264,20 +312,17 @@ func (cm *ConnectionManager) checkOnlineNow() bool {
|
||||
}
|
||||
|
||||
// Perform ping with timeout — no locks held here.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cm.connectTimeout)
|
||||
defer cancel()
|
||||
|
||||
sqlDB, err := db.DB()
|
||||
if err == nil {
|
||||
err = sqlDB.PingContext(ctx)
|
||||
}
|
||||
err := cm.ping(db)
|
||||
|
||||
cm.mu.Lock()
|
||||
defer cm.mu.Unlock()
|
||||
if err != nil {
|
||||
cm.lastError = err
|
||||
cm.lastCheck = time.Now()
|
||||
cm.db = nil
|
||||
if cm.db == db {
|
||||
cm.db = nil
|
||||
}
|
||||
closeDB(db)
|
||||
return false
|
||||
}
|
||||
cm.lastCheck = time.Now()
|
||||
@@ -292,19 +337,23 @@ func (cm *ConnectionManager) TryConnect() error {
|
||||
cm.connMu.Lock()
|
||||
defer cm.connMu.Unlock()
|
||||
|
||||
newDB, err := cm.dial()
|
||||
newDB, err := cm.dialFn()
|
||||
|
||||
cm.mu.Lock()
|
||||
defer cm.mu.Unlock()
|
||||
old := cm.db
|
||||
if err != nil {
|
||||
cm.db = nil
|
||||
cm.lastError = err
|
||||
cm.lastCheck = time.Now()
|
||||
cm.mu.Unlock()
|
||||
closeDB(old)
|
||||
return err
|
||||
}
|
||||
cm.db = newDB
|
||||
cm.lastError = nil
|
||||
cm.lastCheck = time.Now()
|
||||
cm.mu.Unlock()
|
||||
closeDB(old)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user