Add Phase 2: Local SQLite database with sync functionality

Implements complete offline-first architecture with SQLite caching and MariaDB synchronization.

Key features:
- Local SQLite database for offline operation (data/quoteforge.db)
- Connection settings with encrypted credentials
- Component and pricelist caching with auto-sync
- Sync API endpoints (/api/sync/status, /components, /pricelists, /all)
- Real-time sync status indicator in UI with auto-refresh
- Offline mode detection middleware
- Migration tool for database initialization
- Setup wizard for initial configuration

New components:
- internal/localdb: SQLite repository layer (components, pricelists, sync)
- internal/services/sync: Synchronization service
- internal/handlers/sync: Sync API handlers
- internal/handlers/setup: Setup wizard handlers
- internal/middleware/offline: Offline detection
- cmd/migrate: Database migration tool

UI improvements:
- Setup page for database configuration
- Sync status indicator with online/offline detection
- Warning icons for pending synchronization
- Auto-refresh every 30 seconds

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
2026-02-01 11:00:32 +03:00
parent 8b8d2f18f9
commit 143d217397
30 changed files with 3697 additions and 887 deletions

View File

@@ -0,0 +1,268 @@
package localdb
import (
"fmt"
"log/slog"
"strings"
"time"
"gorm.io/gorm"
)
// ComponentSyncResult contains statistics from component sync
type ComponentSyncResult struct {
TotalSynced int
NewCount int
UpdateCount int
Duration time.Duration
}
// SyncComponents loads components from MariaDB (lot + qt_lot_metadata) into local_components
func (l *LocalDB) SyncComponents(mariaDB *gorm.DB) (*ComponentSyncResult, error) {
startTime := time.Now()
// Query to join lot with qt_lot_metadata
// Use LEFT JOIN to include lots without metadata
type componentRow struct {
LotName string
LotDescription string
Category *string
Model *string
CurrentPrice *float64
}
var rows []componentRow
err := mariaDB.Raw(`
SELECT
l.lot_name,
l.lot_description,
COALESCE(c.code, SUBSTRING_INDEX(l.lot_name, '_', 1)) as category,
m.model,
m.current_price
FROM lot l
LEFT JOIN qt_lot_metadata m ON l.lot_name = m.lot_name
LEFT JOIN qt_categories c ON m.category_id = c.id
WHERE m.is_hidden = FALSE OR m.is_hidden IS NULL
ORDER BY l.lot_name
`).Scan(&rows).Error
if err != nil {
return nil, fmt.Errorf("querying components from MariaDB: %w", err)
}
if len(rows) == 0 {
slog.Warn("no components found in MariaDB")
return &ComponentSyncResult{
Duration: time.Since(startTime),
}, nil
}
// Get existing local components for comparison
existingMap := make(map[string]bool)
var existing []LocalComponent
if err := l.db.Find(&existing).Error; err != nil {
return nil, fmt.Errorf("reading existing local components: %w", err)
}
for _, c := range existing {
existingMap[c.LotName] = true
}
// Prepare components for batch insert/update
syncTime := time.Now()
components := make([]LocalComponent, 0, len(rows))
newCount := 0
for _, row := range rows {
category := ""
if row.Category != nil {
category = *row.Category
} else {
// Parse category from lot_name (e.g., "CPU_AMD_9654" -> "CPU")
parts := strings.SplitN(row.LotName, "_", 2)
if len(parts) >= 1 {
category = parts[0]
}
}
model := ""
if row.Model != nil {
model = *row.Model
}
comp := LocalComponent{
LotName: row.LotName,
LotDescription: row.LotDescription,
Category: category,
Model: model,
CurrentPrice: row.CurrentPrice,
SyncedAt: syncTime,
}
components = append(components, comp)
if !existingMap[row.LotName] {
newCount++
}
}
// Use transaction for bulk upsert
err = l.db.Transaction(func(tx *gorm.DB) error {
// Delete all existing and insert new (simpler than upsert for SQLite)
if err := tx.Where("1=1").Delete(&LocalComponent{}).Error; err != nil {
return fmt.Errorf("clearing local components: %w", err)
}
// Batch insert
batchSize := 500
for i := 0; i < len(components); i += batchSize {
end := i + batchSize
if end > len(components) {
end = len(components)
}
if err := tx.CreateInBatches(components[i:end], batchSize).Error; err != nil {
return fmt.Errorf("inserting components batch: %w", err)
}
}
return nil
})
if err != nil {
return nil, err
}
// Update last sync time
if err := l.SetComponentSyncTime(syncTime); err != nil {
slog.Warn("failed to update component sync time", "error", err)
}
result := &ComponentSyncResult{
TotalSynced: len(components),
NewCount: newCount,
UpdateCount: len(components) - newCount,
Duration: time.Since(startTime),
}
slog.Info("components synced",
"total", result.TotalSynced,
"new", result.NewCount,
"updated", result.UpdateCount,
"duration", result.Duration)
return result, nil
}
// SearchLocalComponents searches components in local cache by query string
// Searches in lot_name, lot_description, category, and model fields
func (l *LocalDB) SearchLocalComponents(query string, limit int) ([]LocalComponent, error) {
if limit <= 0 {
limit = 50
}
var components []LocalComponent
if query == "" {
// Return all components with limit
err := l.db.Order("lot_name").Limit(limit).Find(&components).Error
return components, err
}
// Search with LIKE on multiple fields
searchPattern := "%" + strings.ToLower(query) + "%"
err := l.db.Where(
"LOWER(lot_name) LIKE ? OR LOWER(lot_description) LIKE ? OR LOWER(category) LIKE ? OR LOWER(model) LIKE ?",
searchPattern, searchPattern, searchPattern, searchPattern,
).Order("lot_name").Limit(limit).Find(&components).Error
return components, err
}
// SearchLocalComponentsByCategory searches components by category and optional query
func (l *LocalDB) SearchLocalComponentsByCategory(category string, query string, limit int) ([]LocalComponent, error) {
if limit <= 0 {
limit = 50
}
var components []LocalComponent
db := l.db.Where("LOWER(category) = ?", strings.ToLower(category))
if query != "" {
searchPattern := "%" + strings.ToLower(query) + "%"
db = db.Where(
"LOWER(lot_name) LIKE ? OR LOWER(lot_description) LIKE ? OR LOWER(model) LIKE ?",
searchPattern, searchPattern, searchPattern,
)
}
err := db.Order("lot_name").Limit(limit).Find(&components).Error
return components, err
}
// GetLocalComponent returns a single component by lot_name
func (l *LocalDB) GetLocalComponent(lotName string) (*LocalComponent, error) {
var component LocalComponent
err := l.db.Where("lot_name = ?", lotName).First(&component).Error
if err != nil {
return nil, err
}
return &component, nil
}
// GetLocalComponentCategories returns distinct categories from local components
func (l *LocalDB) GetLocalComponentCategories() ([]string, error) {
var categories []string
err := l.db.Model(&LocalComponent{}).
Distinct("category").
Where("category != ''").
Order("category").
Pluck("category", &categories).Error
return categories, err
}
// CountLocalComponents returns the total number of local components
func (l *LocalDB) CountLocalComponents() int64 {
var count int64
l.db.Model(&LocalComponent{}).Count(&count)
return count
}
// CountLocalComponentsByCategory returns component count by category
func (l *LocalDB) CountLocalComponentsByCategory(category string) int64 {
var count int64
l.db.Model(&LocalComponent{}).Where("LOWER(category) = ?", strings.ToLower(category)).Count(&count)
return count
}
// GetComponentSyncTime returns the last component sync timestamp
func (l *LocalDB) GetComponentSyncTime() *time.Time {
var setting struct {
Value string
}
if err := l.db.Table("app_settings").
Where("key = ?", "last_component_sync").
First(&setting).Error; err != nil {
return nil
}
t, err := time.Parse(time.RFC3339, setting.Value)
if err != nil {
return nil
}
return &t
}
// SetComponentSyncTime sets the last component sync timestamp
func (l *LocalDB) SetComponentSyncTime(t time.Time) error {
return l.db.Exec(`
INSERT INTO app_settings (key, value, updated_at)
VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at
`, "last_component_sync", t.Format(time.RFC3339), time.Now().Format(time.RFC3339)).Error
}
// NeedComponentSync checks if component sync is needed (older than specified hours)
func (l *LocalDB) NeedComponentSync(maxAgeHours int) bool {
syncTime := l.GetComponentSyncTime()
if syncTime == nil {
return true
}
return time.Since(*syncTime).Hours() > float64(maxAgeHours)
}

View File

@@ -0,0 +1,87 @@
package localdb
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"errors"
"io"
"os"
)
// getEncryptionKey derives a 32-byte key from environment variable or machine ID
func getEncryptionKey() []byte {
key := os.Getenv("QUOTEFORGE_ENCRYPTION_KEY")
if key == "" {
// Fallback to a machine-based key (hostname + fixed salt)
hostname, _ := os.Hostname()
key = hostname + "quoteforge-salt-2024"
}
// Hash to get exactly 32 bytes for AES-256
hash := sha256.Sum256([]byte(key))
return hash[:]
}
// Encrypt encrypts plaintext using AES-256-GCM
func Encrypt(plaintext string) (string, error) {
if plaintext == "" {
return "", nil
}
key := getEncryptionKey()
block, err := aes.NewCipher(key)
if err != nil {
return "", err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", err
}
nonce := make([]byte, gcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return "", err
}
ciphertext := gcm.Seal(nonce, nonce, []byte(plaintext), nil)
return base64.StdEncoding.EncodeToString(ciphertext), nil
}
// Decrypt decrypts ciphertext that was encrypted with Encrypt
func Decrypt(ciphertext string) (string, error) {
if ciphertext == "" {
return "", nil
}
key := getEncryptionKey()
data, err := base64.StdEncoding.DecodeString(ciphertext)
if err != nil {
return "", err
}
block, err := aes.NewCipher(key)
if err != nil {
return "", err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", err
}
nonceSize := gcm.NonceSize()
if len(data) < nonceSize {
return "", errors.New("ciphertext too short")
}
nonce, ciphertextBytes := data[:nonceSize], data[nonceSize:]
plaintext, err := gcm.Open(nil, nonce, ciphertextBytes, nil)
if err != nil {
return "", err
}
return string(plaintext), nil
}

339
internal/localdb/localdb.go Normal file
View File

@@ -0,0 +1,339 @@
package localdb
import (
"fmt"
"log/slog"
"os"
"path/filepath"
"time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
// ConnectionSettings stores MariaDB connection credentials
type ConnectionSettings struct {
ID uint `gorm:"primaryKey"`
Host string `gorm:"not null"`
Port int `gorm:"not null;default:3306"`
Database string `gorm:"not null"`
User string `gorm:"not null"`
PasswordEncrypted string `gorm:"not null"` // AES encrypted
UpdatedAt time.Time `gorm:"autoUpdateTime"`
}
func (ConnectionSettings) TableName() string {
return "connection_settings"
}
// LocalDB manages the local SQLite database for settings
type LocalDB struct {
db *gorm.DB
path string
}
// New creates a new LocalDB instance
func New(dbPath string) (*LocalDB, error) {
// Ensure directory exists
dir := filepath.Dir(dbPath)
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, fmt.Errorf("creating data directory: %w", err)
}
db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
return nil, fmt.Errorf("opening sqlite database: %w", err)
}
// Auto-migrate all local tables
if err := db.AutoMigrate(
&ConnectionSettings{},
&LocalConfiguration{},
&LocalPricelist{},
&LocalPricelistItem{},
&LocalComponent{},
&AppSetting{},
); err != nil {
return nil, fmt.Errorf("migrating sqlite database: %w", err)
}
slog.Info("local SQLite database initialized", "path", dbPath)
return &LocalDB{
db: db,
path: dbPath,
}, nil
}
// HasSettings returns true if connection settings exist
func (l *LocalDB) HasSettings() bool {
var count int64
l.db.Model(&ConnectionSettings{}).Count(&count)
return count > 0
}
// GetSettings retrieves the connection settings with decrypted password
func (l *LocalDB) GetSettings() (*ConnectionSettings, error) {
var settings ConnectionSettings
if err := l.db.First(&settings).Error; err != nil {
return nil, fmt.Errorf("getting settings: %w", err)
}
// Decrypt password
password, err := Decrypt(settings.PasswordEncrypted)
if err != nil {
return nil, fmt.Errorf("decrypting password: %w", err)
}
settings.PasswordEncrypted = password // Return decrypted password in this field
return &settings, nil
}
// SaveSettings saves connection settings with encrypted password
func (l *LocalDB) SaveSettings(host string, port int, database, user, password string) error {
// Encrypt password
encrypted, err := Encrypt(password)
if err != nil {
return fmt.Errorf("encrypting password: %w", err)
}
settings := ConnectionSettings{
ID: 1, // Always use ID=1 for single settings row
Host: host,
Port: port,
Database: database,
User: user,
PasswordEncrypted: encrypted,
}
// Upsert: create or update
result := l.db.Save(&settings)
if result.Error != nil {
return fmt.Errorf("saving settings: %w", result.Error)
}
slog.Info("connection settings saved", "host", host, "port", port, "database", database, "user", user)
return nil
}
// DeleteSettings removes all connection settings
func (l *LocalDB) DeleteSettings() error {
return l.db.Where("1=1").Delete(&ConnectionSettings{}).Error
}
// GetDSN returns the MariaDB DSN string
func (l *LocalDB) GetDSN() (string, error) {
settings, err := l.GetSettings()
if err != nil {
return "", err
}
dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local",
settings.User,
settings.PasswordEncrypted, // Contains decrypted password after GetSettings
settings.Host,
settings.Port,
settings.Database,
)
return dsn, nil
}
// DB returns the underlying gorm.DB for advanced operations
func (l *LocalDB) DB() *gorm.DB {
return l.db
}
// Close closes the database connection
func (l *LocalDB) Close() error {
sqlDB, err := l.db.DB()
if err != nil {
return err
}
return sqlDB.Close()
}
// GetDBUser returns the database username from settings
func (l *LocalDB) GetDBUser() string {
settings, err := l.GetSettings()
if err != nil {
return ""
}
return settings.User
}
// Configuration methods
// SaveConfiguration saves a configuration to local SQLite
func (l *LocalDB) SaveConfiguration(config *LocalConfiguration) error {
return l.db.Save(config).Error
}
// GetConfigurations returns all local configurations
func (l *LocalDB) GetConfigurations() ([]LocalConfiguration, error) {
var configs []LocalConfiguration
err := l.db.Order("created_at DESC").Find(&configs).Error
return configs, err
}
// GetConfigurationByUUID returns a configuration by UUID
func (l *LocalDB) GetConfigurationByUUID(uuid string) (*LocalConfiguration, error) {
var config LocalConfiguration
err := l.db.Where("uuid = ?", uuid).First(&config).Error
return &config, err
}
// DeleteConfiguration deletes a configuration by UUID
func (l *LocalDB) DeleteConfiguration(uuid string) error {
return l.db.Where("uuid = ?", uuid).Delete(&LocalConfiguration{}).Error
}
// CountConfigurations returns the number of local configurations
func (l *LocalDB) CountConfigurations() int64 {
var count int64
l.db.Model(&LocalConfiguration{}).Count(&count)
return count
}
// Pricelist methods
// GetLastSyncTime returns the last sync timestamp
func (l *LocalDB) GetLastSyncTime() *time.Time {
var setting struct {
Value string
}
if err := l.db.Table("app_settings").
Where("key = ?", "last_pricelist_sync").
First(&setting).Error; err != nil {
return nil
}
t, err := time.Parse(time.RFC3339, setting.Value)
if err != nil {
return nil
}
return &t
}
// SetLastSyncTime sets the last sync timestamp
func (l *LocalDB) SetLastSyncTime(t time.Time) error {
// Using raw SQL for upsert since SQLite doesn't have native UPSERT in all versions
return l.db.Exec(`
INSERT INTO app_settings (key, value, updated_at)
VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at
`, "last_pricelist_sync", t.Format(time.RFC3339), time.Now().Format(time.RFC3339)).Error
}
// CountLocalPricelists returns the number of local pricelists
func (l *LocalDB) CountLocalPricelists() int64 {
var count int64
l.db.Model(&LocalPricelist{}).Count(&count)
return count
}
// GetLatestLocalPricelist returns the most recently synced pricelist
func (l *LocalDB) GetLatestLocalPricelist() (*LocalPricelist, error) {
var pricelist LocalPricelist
if err := l.db.Order("created_at DESC").First(&pricelist).Error; err != nil {
return nil, err
}
return &pricelist, nil
}
// GetLocalPricelistByServerID returns a local pricelist by its server ID
func (l *LocalDB) GetLocalPricelistByServerID(serverID uint) (*LocalPricelist, error) {
var pricelist LocalPricelist
if err := l.db.Where("server_id = ?", serverID).First(&pricelist).Error; err != nil {
return nil, err
}
return &pricelist, nil
}
// GetLocalPricelistByID returns a local pricelist by its local ID
func (l *LocalDB) GetLocalPricelistByID(id uint) (*LocalPricelist, error) {
var pricelist LocalPricelist
if err := l.db.First(&pricelist, id).Error; err != nil {
return nil, err
}
return &pricelist, nil
}
// SaveLocalPricelist saves a pricelist to local SQLite
func (l *LocalDB) SaveLocalPricelist(pricelist *LocalPricelist) error {
return l.db.Save(pricelist).Error
}
// GetLocalPricelists returns all local pricelists
func (l *LocalDB) GetLocalPricelists() ([]LocalPricelist, error) {
var pricelists []LocalPricelist
if err := l.db.Order("created_at DESC").Find(&pricelists).Error; err != nil {
return nil, err
}
return pricelists, nil
}
// CountLocalPricelistItems returns the number of items for a pricelist
func (l *LocalDB) CountLocalPricelistItems(pricelistID uint) int64 {
var count int64
l.db.Model(&LocalPricelistItem{}).Where("pricelist_id = ?", pricelistID).Count(&count)
return count
}
// SaveLocalPricelistItems saves pricelist items to local SQLite
func (l *LocalDB) SaveLocalPricelistItems(items []LocalPricelistItem) error {
if len(items) == 0 {
return nil
}
// Batch insert
batchSize := 500
for i := 0; i < len(items); i += batchSize {
end := i + batchSize
if end > len(items) {
end = len(items)
}
if err := l.db.CreateInBatches(items[i:end], batchSize).Error; err != nil {
return err
}
}
return nil
}
// GetLocalPricelistItems returns items for a local pricelist
func (l *LocalDB) GetLocalPricelistItems(pricelistID uint) ([]LocalPricelistItem, error) {
var items []LocalPricelistItem
if err := l.db.Where("pricelist_id = ?", pricelistID).Find(&items).Error; err != nil {
return nil, err
}
return items, nil
}
// GetLocalPriceForLot returns the price for a lot from a local pricelist
func (l *LocalDB) GetLocalPriceForLot(pricelistID uint, lotName string) (float64, error) {
var item LocalPricelistItem
if err := l.db.Where("pricelist_id = ? AND lot_name = ?", pricelistID, lotName).
First(&item).Error; err != nil {
return 0, err
}
return item.Price, nil
}
// MarkPricelistAsUsed marks a pricelist as used by a configuration
func (l *LocalDB) MarkPricelistAsUsed(pricelistID uint, isUsed bool) error {
return l.db.Model(&LocalPricelist{}).Where("id = ?", pricelistID).
Update("is_used", isUsed).Error
}
// DeleteLocalPricelist deletes a pricelist and its items
func (l *LocalDB) DeleteLocalPricelist(id uint) error {
// Delete items first
if err := l.db.Where("pricelist_id = ?", id).Delete(&LocalPricelistItem{}).Error; err != nil {
return err
}
// Delete pricelist
return l.db.Delete(&LocalPricelist{}, id).Error
}

122
internal/localdb/models.go Normal file
View File

@@ -0,0 +1,122 @@
package localdb
import (
"database/sql/driver"
"encoding/json"
"errors"
"time"
)
// AppSetting stores application settings in local SQLite
type AppSetting struct {
Key string `gorm:"primaryKey" json:"key"`
Value string `gorm:"not null" json:"value"`
UpdatedAt time.Time `json:"updated_at"`
}
func (AppSetting) TableName() string {
return "app_settings"
}
// LocalConfigItem represents an item in a configuration
type LocalConfigItem struct {
LotName string `json:"lot_name"`
Quantity int `json:"quantity"`
UnitPrice float64 `json:"unit_price"`
}
// LocalConfigItems is a slice of LocalConfigItem that can be stored as JSON
type LocalConfigItems []LocalConfigItem
func (c LocalConfigItems) Value() (driver.Value, error) {
return json.Marshal(c)
}
func (c *LocalConfigItems) Scan(value interface{}) error {
if value == nil {
*c = make(LocalConfigItems, 0)
return nil
}
var bytes []byte
switch v := value.(type) {
case []byte:
bytes = v
case string:
bytes = []byte(v)
default:
return errors.New("type assertion failed for LocalConfigItems")
}
return json.Unmarshal(bytes, c)
}
func (c LocalConfigItems) Total() float64 {
var total float64
for _, item := range c {
total += item.UnitPrice * float64(item.Quantity)
}
return total
}
// LocalConfiguration stores configurations in local SQLite
type LocalConfiguration struct {
ID uint `gorm:"primaryKey;autoIncrement" json:"id"`
UUID string `gorm:"uniqueIndex;not null" json:"uuid"`
ServerID *uint `json:"server_id"` // ID on MariaDB server, NULL if local only
Name string `gorm:"not null" json:"name"`
Items LocalConfigItems `gorm:"type:text" json:"items"` // JSON stored as text in SQLite
TotalPrice *float64 `json:"total_price"`
CustomPrice *float64 `json:"custom_price"`
Notes string `json:"notes"`
IsTemplate bool `gorm:"default:false" json:"is_template"`
ServerCount int `gorm:"default:1" json:"server_count"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
SyncedAt *time.Time `json:"synced_at"`
SyncStatus string `gorm:"default:'local'" json:"sync_status"` // 'local', 'synced', 'modified'
OriginalUserID uint `json:"original_user_id"` // UserID from MariaDB for reference
}
func (LocalConfiguration) TableName() string {
return "local_configurations"
}
// LocalPricelist stores cached pricelists from server
type LocalPricelist struct {
ID uint `gorm:"primaryKey;autoIncrement" json:"id"`
ServerID uint `gorm:"not null" json:"server_id"` // ID on MariaDB server
Version string `gorm:"uniqueIndex;not null" json:"version"`
Name string `json:"name"`
CreatedAt time.Time `json:"created_at"`
SyncedAt time.Time `json:"synced_at"`
IsUsed bool `gorm:"default:false" json:"is_used"` // Used by any local configuration
}
func (LocalPricelist) TableName() string {
return "local_pricelists"
}
// LocalPricelistItem stores pricelist items
type LocalPricelistItem struct {
ID uint `gorm:"primaryKey;autoIncrement" json:"id"`
PricelistID uint `gorm:"not null;index" json:"pricelist_id"`
LotName string `gorm:"not null" json:"lot_name"`
Price float64 `gorm:"not null" json:"price"`
}
func (LocalPricelistItem) TableName() string {
return "local_pricelist_items"
}
// LocalComponent stores cached components for offline search
type LocalComponent struct {
LotName string `gorm:"primaryKey" json:"lot_name"`
LotDescription string `json:"lot_description"`
Category string `json:"category"`
Model string `json:"model"`
CurrentPrice *float64 `json:"current_price"`
SyncedAt time.Time `json:"synced_at"`
}
func (LocalComponent) TableName() string {
return "local_components"
}