From 7e1e2ac18d67aa152129676c04fca6278f794150 Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Wed, 15 Apr 2026 18:40:34 +0300 Subject: [PATCH] Fix storage sync and configurator category visibility --- bible-local/03-database.md | 14 +-- internal/localdb/components.go | 37 ++++-- internal/services/sync/service.go | 108 ------------------ ...ervice_pricelist_category_backfill_test.go | 101 ---------------- web/templates/index.html | 35 ++++-- 5 files changed, 63 insertions(+), 232 deletions(-) diff --git a/bible-local/03-database.md b/bible-local/03-database.md index e28ed79..40fcf68 100644 --- a/bible-local/03-database.md +++ b/bible-local/03-database.md @@ -29,17 +29,15 @@ Rules: ## MariaDB -MariaDB is the central sync database (`RFQ_LOG`). Final schema as of 2026-03-21. +MariaDB is the central sync database (`RFQ_LOG`). Final schema as of 2026-04-15. -### QuoteForge tables (qt_* and stock_*) +### QuoteForge tables (qt_*) Runtime read: - `qt_categories` — pricelist categories - `qt_lot_metadata` — component metadata, price settings - `qt_pricelists` — pricelist headers (source: estimate / warehouse / competitor) - `qt_pricelist_items` — pricelist rows -- `stock_log` — raw supplier price log, source for pricelist generation -- `stock_ignore_rules` — patterns to skip during stock import - `qt_partnumber_books` — partnumber book headers - `qt_partnumber_book_items` — PN→LOT catalog payload @@ -69,18 +67,20 @@ QuoteForge references competitor pricelists only via `qt_pricelists` (source='co ### Legacy RFQ tables (pre-QuoteForge, no Go code references) - `lot` — original component registry (data preserved; superseded by `qt_lot_metadata`) -- `lot_log` — original supplier price log (superseded by `stock_log`) +- `lot_log` — original supplier price log - `supplier` — supplier registry (FK target for lot_log and machine_log) - `machine` — device model registry - `machine_log` — device price/quote log +- `parts_log` — supplier partnumber log used by server-side import/pricing workflows, not by QuoteForge runtime These tables are retained for historical data. QuoteForge does not read or write them at runtime. Rules: - QuoteForge runtime must not depend on any legacy RFQ tables; -- stock enrichment happens during sync and is persisted into SQLite; +- QuoteForge sync reads prices and categories from `qt_pricelists` / `qt_pricelist_items` only; +- QuoteForge does not enrich local pricelist rows from `parts_log` or any other raw supplier log table; - normal UI requests must not query MariaDB tables directly; -- `qt_client_local_migrations` was removed from the schema on 2026-03-21 (was in earlier drafts). +- `qt_client_local_migrations` exists in the 2026-04-15 schema dump, but runtime sync does not depend on it. ## MariaDB Table Structures diff --git a/internal/localdb/components.go b/internal/localdb/components.go index 013edb6..fbd0077 100644 --- a/internal/localdb/components.go +++ b/internal/localdb/components.go @@ -71,18 +71,25 @@ func (l *LocalDB) SyncComponents(mariaDB *gorm.DB) (*ComponentSyncResult, error) existingMap[c.LotName] = true } - // Prepare components for batch insert/update + // Prepare components for batch insert/update. + // Source joins may duplicate the same lot_name, so collapse them before insert. syncTime := time.Now() components := make([]LocalComponent, 0, len(rows)) + componentIndex := make(map[string]int, len(rows)) newCount := 0 for _, row := range rows { + lotName := strings.TrimSpace(row.LotName) + if lotName == "" { + continue + } + category := "" if row.Category != nil { - category = *row.Category + category = strings.TrimSpace(*row.Category) } else { // Parse category from lot_name (e.g., "CPU_AMD_9654" -> "CPU") - parts := strings.SplitN(row.LotName, "_", 2) + parts := strings.SplitN(lotName, "_", 2) if len(parts) >= 1 { category = parts[0] } @@ -90,18 +97,34 @@ func (l *LocalDB) SyncComponents(mariaDB *gorm.DB) (*ComponentSyncResult, error) model := "" if row.Model != nil { - model = *row.Model + model = strings.TrimSpace(*row.Model) } comp := LocalComponent{ - LotName: row.LotName, - LotDescription: row.LotDescription, + LotName: lotName, + LotDescription: strings.TrimSpace(row.LotDescription), Category: category, Model: model, } + + if idx, exists := componentIndex[lotName]; exists { + // Keep the first row, but fill any missing metadata from duplicates. + if components[idx].LotDescription == "" && comp.LotDescription != "" { + components[idx].LotDescription = comp.LotDescription + } + if components[idx].Category == "" && comp.Category != "" { + components[idx].Category = comp.Category + } + if components[idx].Model == "" && comp.Model != "" { + components[idx].Model = comp.Model + } + continue + } + + componentIndex[lotName] = len(components) components = append(components, comp) - if !existingMap[row.LotName] { + if !existingMap[lotName] { newCount++ } } diff --git a/internal/services/sync/service.go b/internal/services/sync/service.go index 96d7a1a..3f8a067 100644 --- a/internal/services/sync/service.go +++ b/internal/services/sync/service.go @@ -789,9 +789,6 @@ func (s *Service) fetchServerPricelistItems(serverPricelistID uint) ([]localdb.L for i, item := range serverItems { localItems[i] = *localdb.PricelistItemToLocal(&item, 0) } - if err := s.enrichLocalPricelistItemsWithStock(mariaDB, localItems); err != nil { - slog.Warn("pricelist stock enrichment skipped", "server_pricelist_id", serverPricelistID, "error", err) - } return localItems, nil } @@ -805,111 +802,6 @@ func (s *Service) SyncPricelistItemsByServerID(serverPricelistID uint) (int, err return s.SyncPricelistItems(localPL.ID) } -func (s *Service) enrichLocalPricelistItemsWithStock(mariaDB *gorm.DB, items []localdb.LocalPricelistItem) error { - if len(items) == 0 { - return nil - } - - bookRepo := repository.NewPartnumberBookRepository(s.localDB.DB()) - book, err := bookRepo.GetActiveBook() - if err != nil || book == nil { - return nil - } - - bookItems, err := bookRepo.GetBookItems(book.ID) - if err != nil { - return err - } - if len(bookItems) == 0 { - return nil - } - - partnumberToLots := make(map[string][]string, len(bookItems)) - for _, item := range bookItems { - pn := strings.TrimSpace(item.Partnumber) - if pn == "" { - continue - } - seenLots := make(map[string]struct{}, len(item.LotsJSON)) - for _, lot := range item.LotsJSON { - lotName := strings.TrimSpace(lot.LotName) - if lotName == "" { - continue - } - key := strings.ToLower(lotName) - if _, exists := seenLots[key]; exists { - continue - } - seenLots[key] = struct{}{} - partnumberToLots[pn] = append(partnumberToLots[pn], lotName) - } - } - if len(partnumberToLots) == 0 { - return nil - } - - type stockRow struct { - Partnumber string `gorm:"column:partnumber"` - Qty *float64 `gorm:"column:qty"` - } - rows := make([]stockRow, 0) - if err := mariaDB.Raw(` - SELECT s.partnumber, s.qty - FROM stock_log s - INNER JOIN ( - SELECT partnumber, MAX(date) AS max_date - FROM stock_log - GROUP BY partnumber - ) latest ON latest.partnumber = s.partnumber AND latest.max_date = s.date - WHERE s.qty IS NOT NULL - `).Scan(&rows).Error; err != nil { - return err - } - - lotTotals := make(map[string]float64, len(items)) - lotPartnumbers := make(map[string][]string, len(items)) - seenPartnumbers := make(map[string]map[string]struct{}, len(items)) - - for _, row := range rows { - pn := strings.TrimSpace(row.Partnumber) - if pn == "" || row.Qty == nil { - continue - } - lots := partnumberToLots[pn] - if len(lots) == 0 { - continue - } - for _, lotName := range lots { - lotTotals[lotName] += *row.Qty - if _, ok := seenPartnumbers[lotName]; !ok { - seenPartnumbers[lotName] = make(map[string]struct{}, 4) - } - key := strings.ToLower(pn) - if _, exists := seenPartnumbers[lotName][key]; exists { - continue - } - seenPartnumbers[lotName][key] = struct{}{} - lotPartnumbers[lotName] = append(lotPartnumbers[lotName], pn) - } - } - - for i := range items { - lotName := strings.TrimSpace(items[i].LotName) - if qty, ok := lotTotals[lotName]; ok { - qtyCopy := qty - items[i].AvailableQty = &qtyCopy - } - if partnumbers := lotPartnumbers[lotName]; len(partnumbers) > 0 { - sort.Slice(partnumbers, func(a, b int) bool { - return strings.ToLower(partnumbers[a]) < strings.ToLower(partnumbers[b]) - }) - items[i].Partnumbers = append(localdb.LocalStringList{}, partnumbers...) - } - } - - return nil -} - // GetLocalPriceForLot returns the price for a lot from a local pricelist func (s *Service) GetLocalPriceForLot(localPricelistID uint, lotName string) (float64, error) { return s.localDB.GetLocalPriceForLot(localPricelistID, lotName) diff --git a/internal/services/sync/service_pricelist_category_backfill_test.go b/internal/services/sync/service_pricelist_category_backfill_test.go index ad1bcb9..ff2291f 100644 --- a/internal/services/sync/service_pricelist_category_backfill_test.go +++ b/internal/services/sync/service_pricelist_category_backfill_test.go @@ -17,7 +17,6 @@ func TestSyncPricelists_BackfillsLotCategoryForUsedPricelistItems(t *testing.T) &models.Pricelist{}, &models.PricelistItem{}, &models.Lot{}, - &models.StockLog{}, ); err != nil { t.Fatalf("migrate server tables: %v", err) } @@ -103,103 +102,3 @@ func TestSyncPricelists_BackfillsLotCategoryForUsedPricelistItems(t *testing.T) t.Fatalf("expected lot_category backfilled to CPU, got %q", items[0].LotCategory) } } - -func TestSyncPricelistItems_EnrichesStockFromLocalPartnumberBook(t *testing.T) { - local := newLocalDBForSyncTest(t) - serverDB := newServerDBForSyncTest(t) - - if err := serverDB.AutoMigrate( - &models.Pricelist{}, - &models.PricelistItem{}, - &models.Lot{}, - &models.StockLog{}, - ); err != nil { - t.Fatalf("migrate server tables: %v", err) - } - - serverPL := models.Pricelist{ - Source: "warehouse", - Version: "2026-03-07-001", - Notification: "server", - CreatedBy: "tester", - IsActive: true, - CreatedAt: time.Now().Add(-1 * time.Hour), - } - if err := serverDB.Create(&serverPL).Error; err != nil { - t.Fatalf("create server pricelist: %v", err) - } - if err := serverDB.Create(&models.PricelistItem{ - PricelistID: serverPL.ID, - LotName: "CPU_A", - LotCategory: "CPU", - Price: 10, - }).Error; err != nil { - t.Fatalf("create server pricelist item: %v", err) - } - qty := 7.0 - if err := serverDB.Create(&models.StockLog{ - Partnumber: "CPU-PN-1", - Date: time.Now(), - Price: 100, - Qty: &qty, - }).Error; err != nil { - t.Fatalf("create stock log: %v", err) - } - - if err := local.SaveLocalPricelist(&localdb.LocalPricelist{ - ServerID: serverPL.ID, - Source: serverPL.Source, - Version: serverPL.Version, - Name: serverPL.Notification, - CreatedAt: serverPL.CreatedAt, - SyncedAt: time.Now(), - IsUsed: false, - }); err != nil { - t.Fatalf("seed local pricelist: %v", err) - } - localPL, err := local.GetLocalPricelistByServerID(serverPL.ID) - if err != nil { - t.Fatalf("get local pricelist: %v", err) - } - - if err := local.DB().Create(&localdb.LocalPartnumberBook{ - ServerID: 1, - Version: "2026-03-07-001", - CreatedAt: time.Now(), - IsActive: true, - PartnumbersJSON: localdb.LocalStringList{"CPU-PN-1"}, - }).Error; err != nil { - t.Fatalf("create local partnumber book: %v", err) - } - if err := local.DB().Create(&localdb.LocalPartnumberBookItem{ - Partnumber: "CPU-PN-1", - LotsJSON: localdb.LocalPartnumberBookLots{ - {LotName: "CPU_A", Qty: 1}, - }, - Description: "CPU PN", - }).Error; err != nil { - t.Fatalf("create local partnumber book item: %v", err) - } - - svc := syncsvc.NewServiceWithDB(serverDB, local) - if _, err := svc.SyncPricelistItems(localPL.ID); err != nil { - t.Fatalf("sync pricelist items: %v", err) - } - - items, err := local.GetLocalPricelistItems(localPL.ID) - if err != nil { - t.Fatalf("load local items: %v", err) - } - if len(items) != 1 { - t.Fatalf("expected 1 local item, got %d", len(items)) - } - if items[0].AvailableQty == nil { - t.Fatalf("expected available_qty to be set") - } - if *items[0].AvailableQty != 7 { - t.Fatalf("expected available_qty=7, got %v", *items[0].AvailableQty) - } - if len(items[0].Partnumbers) != 1 || items[0].Partnumbers[0] != "CPU-PN-1" { - t.Fatalf("expected partnumbers [CPU-PN-1], got %v", items[0].Partnumbers) - } -} diff --git a/web/templates/index.html b/web/templates/index.html index f07024a..95dd6dc 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -1155,17 +1155,34 @@ function switchTab(tab) { const ALWAYS_VISIBLE_TABS = new Set(['base', 'storage', 'pci']); // Storage-only categories — hidden for server configs -const STORAGE_ONLY_BASE_CATEGORIES = ['ENC', 'DKC', 'CTL']; +const STORAGE_ONLY_BASE_CATEGORIES = ['DKC', 'CTL', 'ENC']; +// Server-only categories — hidden for storage configs +const SERVER_ONLY_BASE_CATEGORIES = ['MB', 'CPU', 'MEM']; function applyConfigTypeToTabs() { - if (configType === 'storage') return; // storage sees everything - // Remove ENC/DKC/CTL from Base - TAB_CONFIG.base.categories = TAB_CONFIG.base.categories.filter( - c => !STORAGE_ONLY_BASE_CATEGORIES.includes(c) - ); - // Remove HIC from PCI tab - TAB_CONFIG.pci.categories = TAB_CONFIG.pci.categories.filter(c => c !== 'HIC'); - TAB_CONFIG.pci.sections = TAB_CONFIG.pci.sections.filter(s => s.title !== 'HIC'); + const baseCategories = ['MB', 'CPU', 'MEM', 'DKC', 'CTL', 'ENC']; + const pciCategories = ['GPU', 'DPU', 'NIC', 'HCA', 'HBA', 'HIC']; + const pciSections = [ + { title: 'GPU / DPU', categories: ['GPU', 'DPU'] }, + { title: 'NIC / HCA', categories: ['NIC', 'HCA'] }, + { title: 'HBA', categories: ['HBA'] }, + { title: 'HIC', categories: ['HIC'] } + ]; + + TAB_CONFIG.base.categories = baseCategories.filter(c => { + if (configType === 'storage') { + return !SERVER_ONLY_BASE_CATEGORIES.includes(c); + } + return !STORAGE_ONLY_BASE_CATEGORIES.includes(c); + }); + + TAB_CONFIG.pci.categories = pciCategories.filter(c => { + return configType === 'storage' ? true : c !== 'HIC'; + }); + TAB_CONFIG.pci.sections = pciSections.filter(section => { + return configType === 'storage' ? true : section.title !== 'HIC'; + }); + // Rebuild assigned categories index ASSIGNED_CATEGORIES = Object.values(TAB_CONFIG) .flatMap(t => t.categories)