diff --git a/bible/architecture.md b/bible/architecture.md index a592a9c..2e1304e 100644 --- a/bible/architecture.md +++ b/bible/architecture.md @@ -174,7 +174,7 @@ Configurator, projects, and export links must **not** appear in the menu. | `lot_partnumbers` | partnumber → LOT mappings (canonical contract) | | `qt_lot_bundles` | Bundle definitions | | `qt_lot_bundle_items` | Bundle composition: `(bundle_lot_name, lot_name, qty)` | -| `qt_vendor_partnumber_seen` | Seen registry + `is_ignored` flag | +| `qt_vendor_partnumber_seen` | Seen registry (unique by `partnumber`) + `is_ignored` flag | --- diff --git a/bible/history.md b/bible/history.md index fe4b97b..b84b0a5 100644 --- a/bible/history.md +++ b/bible/history.md @@ -5,6 +5,33 @@ --- +## 2026-02-20: Seen Registry Deduplication by Partnumber + +### Decision + +Changed `qt_vendor_partnumber_seen` semantics to one row per `partnumber` (vendor/source are no longer part of uniqueness). + +### Rationale + +- Eliminates duplicate seen rows when the same partnumber appears both with vendor and without vendor. +- Keeps ignore behavior consistent regardless of vendor presence. +- Simplifies operational cleanup and prevents re-creation of vendor/no-vendor duplicates. + +### Constraints + +- `partnumber` is now the unique key in seen registry. +- Ignore checks are resolved by `partnumber` only. +- Stock provenance must be preserved (`source_type='stock'`) when stock data exists for the partnumber. + +### Files + +- Migration: `migrations/025_dedup_vendor_seen_by_partnumber.sql` +- Service: `internal/services/stock_import.go` +- Service: `internal/services/vendor_mapping.go` +- Model: `internal/models/lot.go` + +--- + ## 2026-02-18: Global Vendor Partnumber Mapping ### Decision diff --git a/bible/vendor-mapping.md b/bible/vendor-mapping.md index 5b87320..49a12be 100644 --- a/bible/vendor-mapping.md +++ b/bible/vendor-mapping.md @@ -41,7 +41,9 @@ qt_lot_bundle_items: bundle_lot_name, lot_name, qty ### 4. Ignore-логика - **Не использовать** `stock_ignore_rules` для новой логики. -- Использовать `qt_vendor_partnumber_seen.is_ignored` (cross-source флаг). +- Использовать `qt_vendor_partnumber_seen.is_ignored`. +- `qt_vendor_partnumber_seen` хранится в формате **1 строка на partnumber** (vendor/source не участвуют в уникальности). +- Ignore применяется по `partnumber` (одинаково для записей с vendor и без vendor). ### 5. Клиентская совместимость @@ -65,9 +67,11 @@ qt_lot_bundle_items: bundle_lot_name, lot_name, qty | `lot_partnumbers` | Канонические маппинги `(vendor, partnumber)` → `lot_name` | | `qt_lot_bundles` | Определения бандлов (bundle LOT → описание) | | `qt_lot_bundle_items` | Состав бандла: `(bundle_lot_name, lot_name, qty)` | -| `qt_vendor_partnumber_seen` | Реестр seen-записей + флаг `is_ignored` | +| `qt_vendor_partnumber_seen` | Реестр seen-записей (уникально по `partnumber`) + флаг `is_ignored` | -Миграция: `migrations/023_vendor_partnumber_global_mapping.sql` +Миграции: +- `migrations/023_vendor_partnumber_global_mapping.sql` +- `migrations/025_dedup_vendor_seen_by_partnumber.sql` --- @@ -76,6 +80,7 @@ qt_lot_bundle_items: bundle_lot_name, lot_name, qty | Роль | Файл | |------|------| | Миграция | `migrations/023_vendor_partnumber_global_mapping.sql` | +| Миграция | `migrations/025_dedup_vendor_seen_by_partnumber.sql` | | Резолвер | `internal/lotmatch/matcher.go` | | Сервис | `internal/services/vendor_mapping.go` | | API | `internal/handlers/pricing.go` | diff --git a/internal/models/lot.go b/internal/models/lot.go index 849cdbf..89e2cc2 100644 --- a/internal/models/lot.go +++ b/internal/models/lot.go @@ -93,9 +93,9 @@ func (LotBundleItem) TableName() string { type VendorPartnumberSeen struct { ID uint64 `gorm:"column:id;primaryKey;autoIncrement" json:"id"` - SourceType string `gorm:"column:source_type;size:32;not null;index:uq_qt_vendor_partnumber_seen_source_key,unique" json:"source_type"` - Vendor string `gorm:"column:vendor;size:255;not null;default:'';index:uq_qt_vendor_partnumber_seen_source_key,unique;index:idx_qt_vendor_partnumber_seen_vendor_partnumber" json:"vendor"` - Partnumber string `gorm:"column:partnumber;size:255;not null;index:uq_qt_vendor_partnumber_seen_source_key,unique;index:idx_qt_vendor_partnumber_seen_vendor_partnumber" json:"partnumber"` + SourceType string `gorm:"column:source_type;size:32;not null" json:"source_type"` + Vendor string `gorm:"column:vendor;size:255;not null;default:'';index:idx_qt_vendor_partnumber_seen_vendor_partnumber" json:"vendor"` + Partnumber string `gorm:"column:partnumber;size:255;not null;uniqueIndex:uq_qt_vendor_partnumber_seen_partnumber;index:idx_qt_vendor_partnumber_seen_vendor_partnumber" json:"partnumber"` Description *string `gorm:"column:description;size:10000" json:"description,omitempty"` LastSeenAt time.Time `gorm:"column:last_seen_at;not null" json:"last_seen_at"` IsIgnored bool `gorm:"column:is_ignored;not null;default:false;index:idx_qt_vendor_partnumber_seen_ignored" json:"is_ignored"` diff --git a/internal/services/stock_import.go b/internal/services/stock_import.go index f60e8ce..3cc9ab1 100644 --- a/internal/services/stock_import.go +++ b/internal/services/stock_import.go @@ -4,6 +4,7 @@ import ( "archive/zip" "bytes" "encoding/xml" + "errors" "fmt" "io" "path/filepath" @@ -182,7 +183,10 @@ func (s *StockImportService) Import( } partnumber := strings.TrimSpace(row.Article) vendorRaw := strings.TrimSpace(row.Vendor) - seenKey := strings.ToLower(vendorRaw) + "|" + normalizeKey(partnumber) + seenKey := normalizeKey(partnumber) + if seenKey == "" { + continue + } seen := models.VendorPartnumberSeen{ SourceType: "stock", Vendor: vendorRaw, @@ -192,7 +196,18 @@ func (s *StockImportService) Import( if trimmed := strings.TrimSpace(row.Description); trimmed != "" { seen.Description = &trimmed } - seenRowsToUpsert[seenKey] = seen + if existing, ok := seenRowsToUpsert[seenKey]; ok { + if strings.TrimSpace(existing.Vendor) == "" && strings.TrimSpace(seen.Vendor) != "" { + existing.Vendor = strings.TrimSpace(seen.Vendor) + } + if (existing.Description == nil || strings.TrimSpace(*existing.Description) == "") && + seen.Description != nil && strings.TrimSpace(*seen.Description) != "" { + existing.Description = seen.Description + } + seenRowsToUpsert[seenKey] = existing + } else { + seenRowsToUpsert[seenKey] = seen + } if isIgnoredBySeenIndex(ignoredSeenIndex, vendorRaw, partnumber) { ignored++ @@ -628,15 +643,29 @@ func (s *StockImportService) UpsertIgnoreRule(target, matchType, pattern string) return fmt.Errorf("partnumber is required") } now := time.Now() - return s.db.Model(&models.VendorPartnumberSeen{}). - Where("vendor = ? AND partnumber = ?", vendor, partnumber). - Assign(map[string]any{ - "source_type": "manual", - "last_seen_at": now, - "is_ignored": true, - "ignored_at": now, - }). - FirstOrCreate(&models.VendorPartnumberSeen{ + return s.db.Transaction(func(tx *gorm.DB) error { + var existing models.VendorPartnumberSeen + err := tx.Where("LOWER(TRIM(partnumber)) = LOWER(TRIM(?))", partnumber).First(&existing).Error + if err == nil { + updates := map[string]any{ + "last_seen_at": now, + "is_ignored": true, + "ignored_at": now, + } + if strings.TrimSpace(existing.SourceType) != "stock" { + updates["source_type"] = "manual" + } + if strings.TrimSpace(existing.Vendor) == "" && vendor != "" { + updates["vendor"] = vendor + } + return tx.Model(&models.VendorPartnumberSeen{}). + Where("id = ?", existing.ID). + Updates(updates).Error + } + if !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + return tx.Create(&models.VendorPartnumberSeen{ SourceType: "manual", Vendor: vendor, Partnumber: partnumber, @@ -644,6 +673,7 @@ func (s *StockImportService) UpsertIgnoreRule(target, matchType, pattern string) IsIgnored: true, IgnoredAt: &now, }).Error + }) } func (s *StockImportService) DeleteIgnoreRule(id uint) (int64, error) { @@ -697,28 +727,26 @@ func collectSortedSuggestions(src map[string]StockMappingSuggestion, limit int) func (s *StockImportService) loadIgnoredSeenIndex() (map[string]struct{}, error) { var rows []struct { - Vendor string Partnumber string } if err := s.db.Model(&models.VendorPartnumberSeen{}). - Select("vendor, partnumber"). + Select("partnumber"). Where("is_ignored = ?", true). Scan(&rows).Error; err != nil { return nil, err } index := make(map[string]struct{}, len(rows)) for _, row := range rows { - vendor := strings.ToLower(strings.TrimSpace(row.Vendor)) partnumber := normalizeKey(row.Partnumber) if partnumber == "" { continue } - index[vendor+"|"+partnumber] = struct{}{} + index[partnumber] = struct{}{} } return index, nil } -func isIgnoredBySeenIndex(index map[string]struct{}, vendor, partnumber string) bool { +func isIgnoredBySeenIndex(index map[string]struct{}, _ string, partnumber string) bool { if len(index) == 0 { return false } @@ -726,16 +754,8 @@ func isIgnoredBySeenIndex(index map[string]struct{}, vendor, partnumber string) if pn == "" { return false } - vk := strings.ToLower(strings.TrimSpace(vendor)) - if _, ok := index[vk+"|"+pn]; ok { - return true - } - if vk != "" { - if _, ok := index["|"+pn]; ok { - return true - } - } - return false + _, ok := index[pn] + return ok } func (s *StockImportService) upsertSeenRows(rows map[string]models.VendorPartnumberSeen) error { @@ -750,32 +770,57 @@ func (s *StockImportService) upsertSeenRows(rows map[string]models.VendorPartnum if partnumber == "" { continue } - assign := map[string]any{ - "last_seen_at": now, - "updated_at": now, + trimmedDesc := func() *string { + if row.Description == nil { + return nil + } + v := strings.TrimSpace(*row.Description) + if v == "" { + return nil + } + return &v + }() + + var existing models.VendorPartnumberSeen + err := tx.Where("LOWER(TRIM(partnumber)) = LOWER(TRIM(?))", partnumber).First(&existing).Error + if err == nil { + updates := map[string]any{ + "last_seen_at": now, + "updated_at": now, + } + // Preserve stock provenance for unmapped analysis. + if strings.TrimSpace(existing.SourceType) != "stock" && strings.TrimSpace(row.SourceType) != "" { + updates["source_type"] = strings.TrimSpace(row.SourceType) + } + if strings.TrimSpace(existing.Vendor) == "" && vendor != "" { + updates["vendor"] = vendor + } + if trimmedDesc != nil { + updates["description"] = *trimmedDesc + } + if err := tx.Model(&models.VendorPartnumberSeen{}). + Where("id = ?", existing.ID). + Updates(updates).Error; err != nil { + return err + } + continue } - if row.Description != nil && strings.TrimSpace(*row.Description) != "" { - assign["description"] = strings.TrimSpace(*row.Description) + if !errors.Is(err, gorm.ErrRecordNotFound) { + return err } - if err := tx.Model(&models.VendorPartnumberSeen{}). - Where("source_type = ? AND vendor = ? AND partnumber = ?", row.SourceType, vendor, partnumber). - Assign(assign). - FirstOrCreate(&models.VendorPartnumberSeen{ - SourceType: row.SourceType, - Vendor: vendor, - Partnumber: partnumber, - LastSeenAt: now, - Description: func() *string { - if row.Description == nil { - return nil - } - v := strings.TrimSpace(*row.Description) - if v == "" { - return nil - } - return &v - }(), - }).Error; err != nil { + if err := tx.Create(&models.VendorPartnumberSeen{ + SourceType: row.SourceType, + Vendor: vendor, + Partnumber: partnumber, + LastSeenAt: now, + Description: func() *string { + if trimmedDesc == nil { + return nil + } + v := *trimmedDesc + return &v + }(), + }).Error; err != nil { return err } } diff --git a/internal/services/stock_import_test.go b/internal/services/stock_import_test.go index e0220ef..fd0cc8d 100644 --- a/internal/services/stock_import_test.go +++ b/internal/services/stock_import_test.go @@ -347,6 +347,64 @@ func TestPartnumberMappings_WildcardMatch(t *testing.T) { } } +func TestUpsertSeenRows_DeduplicatesByPartnumber(t *testing.T) { + db := openTestDB(t) + if err := db.AutoMigrate(&models.VendorPartnumberSeen{}); err != nil { + t.Fatalf("automigrate seen: %v", err) + } + svc := NewStockImportService(db, nil) + + firstDesc := "first" + if err := svc.upsertSeenRows(map[string]models.VendorPartnumberSeen{ + "cpu123": { + SourceType: "stock", + Vendor: "", + Partnumber: "CPU-123", + Description: &firstDesc, + }, + }); err != nil { + t.Fatalf("upsert first: %v", err) + } + + secondDesc := "second" + if err := svc.upsertSeenRows(map[string]models.VendorPartnumberSeen{ + "cpu123": { + SourceType: "manual", + Vendor: "Dell", + Partnumber: "CPU-123", + Description: &secondDesc, + }, + }); err != nil { + t.Fatalf("upsert second: %v", err) + } + + var rows []models.VendorPartnumberSeen + if err := db.Where("partnumber = ?", "CPU-123").Find(&rows).Error; err != nil { + t.Fatalf("query seen: %v", err) + } + if len(rows) != 1 { + t.Fatalf("expected 1 seen row for partnumber, got %d", len(rows)) + } + if rows[0].SourceType != "stock" { + t.Fatalf("expected source_type stock to be preserved, got %s", rows[0].SourceType) + } + if rows[0].Vendor != "Dell" { + t.Fatalf("expected non-empty vendor to be promoted, got %q", rows[0].Vendor) + } +} + +func TestIsIgnoredBySeenIndex_ByPartnumberOnly(t *testing.T) { + index := map[string]struct{}{ + normalizeKey("CPU-123"): {}, + } + if !isIgnoredBySeenIndex(index, "Dell", "CPU-123") { + t.Fatalf("expected ignore match by partnumber") + } + if isIgnoredBySeenIndex(index, "Dell", "CPU-999") { + t.Fatalf("expected no ignore match for different partnumber") + } +} + func openTestDB(t *testing.T) *gorm.DB { t.Helper() db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) diff --git a/internal/services/vendor_mapping.go b/internal/services/vendor_mapping.go index a163b0c..1b126a1 100644 --- a/internal/services/vendor_mapping.go +++ b/internal/services/vendor_mapping.go @@ -132,18 +132,40 @@ func (s *VendorMappingService) UpsertMapping(vendor, partnumber, lotName, descri } now := time.Now() + var existing models.VendorPartnumberSeen + err = tx.Where("LOWER(TRIM(partnumber)) = LOWER(TRIM(?))", partnumber).First(&existing).Error + if err == nil { + updates := map[string]any{ + "last_seen_at": now, + } + if strings.TrimSpace(existing.SourceType) != "stock" { + updates["source_type"] = "manual" + } + if strings.TrimSpace(existing.Vendor) == "" && vendor != "" { + updates["vendor"] = vendor + } + if descPtr != nil { + updates["description"] = *descPtr + } + return tx.Model(&models.VendorPartnumberSeen{}).Where("id = ?", existing.ID).Updates(updates).Error + } + if !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } seen := models.VendorPartnumberSeen{ SourceType: "manual", Vendor: vendor, Partnumber: partnumber, LastSeenAt: now, + Description: func() *string { + if descPtr == nil { + return nil + } + v := *descPtr + return &v + }(), } - if descPtr != nil { - seen.Description = descPtr - } - return tx.Where("source_type = ? AND vendor = ? AND partnumber = ?", "manual", vendor, partnumber). - Assign(seen). - FirstOrCreate(&seen).Error + return tx.Create(&seen).Error }) } @@ -168,7 +190,9 @@ func (s *VendorMappingService) SetIgnore(vendor, partnumber, ignoredBy string, i updates["ignored_at"] = nil updates["ignored_by"] = nil } - res := tx.Model(&models.VendorPartnumberSeen{}).Where("vendor = ? AND partnumber = ?", vendor, partnumber).Updates(updates) + res := tx.Model(&models.VendorPartnumberSeen{}). + Where("LOWER(TRIM(partnumber)) = LOWER(TRIM(?))", partnumber). + Updates(updates) if res.Error != nil { return res.Error } @@ -242,51 +266,71 @@ func (s *VendorMappingService) List(page, perPage int, search string, unmappedOn offset := (page - 1) * perPage base := s.db.Table(`( - SELECT - COALESCE(TRIM(vendor), '') AS vendor, - TRIM(partnumber) AS partnumber + SELECT TRIM(partnumber) AS partnumber FROM lot_partnumbers WHERE TRIM(COALESCE(partnumber, '')) <> '' UNION - SELECT - COALESCE(TRIM(vendor), '') AS vendor, - TRIM(partnumber) AS partnumber + SELECT TRIM(partnumber) AS partnumber FROM qt_vendor_partnumber_seen WHERE TRIM(COALESCE(partnumber, '')) <> '' ) k`). Select(` - k.vendor, + COALESCE(NULLIF(sa.vendor, ''), NULLIF(lp.vendor, ''), '') as vendor, k.partnumber, - COALESCE(lp_exact.lot_name, lp_fallback.lot_name, '') as lot_name, - COALESCE(lp_exact.description, lp_fallback.description, sa.description, '') as description, + COALESCE(lp.lot_name, '') as lot_name, + COALESCE(lp.description, sa.description, '') as description, COALESCE(sa.ignored, 0) as ignored, COALESCE(sa.sources, '') as sources, sa.last_seen_at as last_seen_at, - CASE WHEN COALESCE(lp_exact.lot_name, lp_fallback.lot_name, '') = '' THEN 1 ELSE 0 END as unmapped, + CASE WHEN COALESCE(lp.lot_name, '') = '' THEN 1 ELSE 0 END as unmapped, CASE WHEN b.bundle_lot_name IS NULL THEN 'single' ELSE 'bundle' END as mapping_type, - COALESCE((SELECT COUNT(*) FROM qt_lot_bundle_items bi WHERE bi.bundle_lot_name = COALESCE(lp_exact.lot_name, lp_fallback.lot_name, '')), 0) as bundle_item_count + COALESCE((SELECT COUNT(*) FROM qt_lot_bundle_items bi WHERE bi.bundle_lot_name = COALESCE(lp.lot_name, '')), 0) as bundle_item_count `). - Joins("LEFT JOIN lot_partnumbers lp_exact ON lp_exact.vendor = k.vendor AND lp_exact.partnumber = k.partnumber"). - Joins("LEFT JOIN lot_partnumbers lp_fallback ON lp_fallback.vendor = '' AND lp_fallback.partnumber = k.partnumber AND k.vendor <> ''"). Joins(`LEFT JOIN ( SELECT - vendor, partnumber, + SUBSTRING_INDEX( + GROUP_CONCAT(COALESCE(TRIM(vendor), '') ORDER BY (TRIM(COALESCE(vendor, '')) <> '') DESC, vendor ASC SEPARATOR '\n'), + '\n', + 1 + ) AS vendor, + SUBSTRING_INDEX( + GROUP_CONCAT(COALESCE(TRIM(lot_name), '') ORDER BY (TRIM(COALESCE(vendor, '')) <> '') DESC, vendor ASC SEPARATOR '\n'), + '\n', + 1 + ) AS lot_name, + SUBSTRING_INDEX( + GROUP_CONCAT(COALESCE(TRIM(description), '') ORDER BY (TRIM(COALESCE(vendor, '')) <> '') DESC, vendor ASC SEPARATOR '\n'), + '\n', + 1 + ) AS description + FROM lot_partnumbers + WHERE TRIM(COALESCE(partnumber, '')) <> '' + GROUP BY partnumber + ) lp ON lp.partnumber = k.partnumber`). + Joins(`LEFT JOIN ( + SELECT + partnumber, + SUBSTRING_INDEX( + GROUP_CONCAT(COALESCE(TRIM(vendor), '') ORDER BY (TRIM(COALESCE(vendor, '')) <> '') DESC, (source_type = 'stock') DESC, last_seen_at DESC, id DESC SEPARATOR '\n'), + '\n', + 1 + ) AS vendor, MAX(is_ignored) AS ignored, GROUP_CONCAT(DISTINCT source_type ORDER BY source_type SEPARATOR ',') AS sources, MAX(last_seen_at) AS last_seen_at, MAX(description) AS description FROM qt_vendor_partnumber_seen - GROUP BY vendor, partnumber - ) sa ON sa.vendor = k.vendor AND sa.partnumber = k.partnumber`). - Joins("LEFT JOIN qt_lot_bundles b ON b.bundle_lot_name = COALESCE(lp_exact.lot_name, lp_fallback.lot_name)") + GROUP BY partnumber + ) sa ON sa.partnumber = k.partnumber`). + Joins("LEFT JOIN qt_lot_bundles b ON b.bundle_lot_name = COALESCE(lp.lot_name, '')") if q := strings.TrimSpace(search); q != "" { like := "%" + q + "%" - base = base.Where("k.vendor LIKE ? OR k.partnumber LIKE ? OR COALESCE(lp_exact.lot_name, lp_fallback.lot_name, '') LIKE ? OR COALESCE(lp_exact.description, lp_fallback.description, sa.description, '') LIKE ?", like, like, like, like) + base = base.Where("COALESCE(NULLIF(sa.vendor, ''), NULLIF(lp.vendor, ''), '') LIKE ? OR k.partnumber LIKE ? OR COALESCE(lp.lot_name, '') LIKE ? OR COALESCE(lp.description, sa.description, '') LIKE ?", like, like, like, like) } if unmappedOnly { - base = base.Where("COALESCE(lp_exact.lot_name, lp_fallback.lot_name, '') = ''"). + base = base.Where("COALESCE(lp.lot_name, '') = ''"). Where("COALESCE(sa.ignored, 0) = 0") } if ignoredOnly { @@ -317,7 +361,7 @@ func (s *VendorMappingService) List(page, perPage int, search string, unmappedOn } var rows []row - if err := base.Order("k.partnumber ASC").Order("k.vendor ASC").Offset(offset).Limit(perPage).Scan(&rows).Error; err != nil { + if err := base.Order("k.partnumber ASC").Offset(offset).Limit(perPage).Scan(&rows).Error; err != nil { return nil, 0, err } items := make([]VendorMappingListRow, 0, len(rows)) @@ -375,15 +419,9 @@ func (s *VendorMappingService) GetDetail(vendor, partnumber string) (*VendorMapp } var seenRows []models.VendorPartnumberSeen - if err := s.db.Where("vendor = ? AND partnumber = ?", vendor, partnumber).Find(&seenRows).Error; err != nil { + if err := s.db.Where("LOWER(TRIM(partnumber)) = LOWER(TRIM(?))", partnumber).Find(&seenRows).Error; err != nil { return nil, err } - if vendor != "" { - var fallbackSeen []models.VendorPartnumberSeen - if err := s.db.Where("vendor = '' AND partnumber = ?", partnumber).Find(&fallbackSeen).Error; err == nil { - seenRows = append(seenRows, fallbackSeen...) - } - } ignored := false sourceSet := make(map[string]struct{}) desc := "" diff --git a/migrations/025_dedup_vendor_seen_by_partnumber.sql b/migrations/025_dedup_vendor_seen_by_partnumber.sql new file mode 100644 index 0000000..7003796 --- /dev/null +++ b/migrations/025_dedup_vendor_seen_by_partnumber.sql @@ -0,0 +1,132 @@ +-- Deduplicate seen registry by partnumber only and enforce unique partnumber key. +-- Note: MariaDB unique index on VARCHAR can treat trailing spaces as equal, so normalize first. + +UPDATE qt_vendor_partnumber_seen +SET partnumber = RTRIM(partnumber) +WHERE partnumber <> RTRIM(partnumber); + +CREATE TEMPORARY TABLE tmp_qt_seen_keep AS +SELECT + LOWER(TRIM(partnumber)) AS partnumber_key, + CAST(SUBSTRING_INDEX( + GROUP_CONCAT( + id + ORDER BY + (TRIM(COALESCE(vendor, '')) <> '') DESC, + (source_type = 'stock') DESC, + (is_ignored = 1) DESC, + last_seen_at DESC, + id DESC + SEPARATOR ',' + ), + ',', + 1 + ) AS UNSIGNED) AS keep_id, + MAX(source_type = 'stock') AS has_stock, + MAX(is_ignored) AS any_ignored, + MAX(last_seen_at) AS max_last_seen, + MAX(CASE WHEN is_ignored = 1 THEN ignored_at END) AS max_ignored_at, + SUBSTRING_INDEX( + GROUP_CONCAT( + CASE + WHEN is_ignored = 1 AND NULLIF(TRIM(ignored_by), '') IS NOT NULL THEN ignored_by + ELSE NULL + END + ORDER BY ignored_at DESC, updated_at DESC, id DESC + SEPARATOR '\n' + ), + '\n', + 1 + ) AS keep_ignored_by, + SUBSTRING_INDEX( + GROUP_CONCAT( + NULLIF(TRIM(description), '') + ORDER BY + (source_type = 'stock') DESC, + (TRIM(COALESCE(vendor, '')) <> '') DESC, + last_seen_at DESC, + id DESC + SEPARATOR '\n' + ), + '\n', + 1 + ) AS keep_description, + SUBSTRING_INDEX( + GROUP_CONCAT( + NULLIF(TRIM(vendor), '') + ORDER BY + (source_type = 'stock') DESC, + last_seen_at DESC, + id DESC + SEPARATOR '\n' + ), + '\n', + 1 + ) AS keep_vendor +FROM qt_vendor_partnumber_seen +WHERE TRIM(COALESCE(partnumber, '')) <> '' +GROUP BY LOWER(TRIM(partnumber)); + +UPDATE qt_vendor_partnumber_seen s +JOIN tmp_qt_seen_keep k ON s.id = k.keep_id +SET + s.source_type = CASE WHEN k.has_stock = 1 THEN 'stock' ELSE 'manual' END, + s.vendor = COALESCE(NULLIF(TRIM(s.vendor), ''), NULLIF(TRIM(k.keep_vendor), ''), ''), + s.partnumber = TRIM(s.partnumber), + s.description = COALESCE(NULLIF(TRIM(s.description), ''), NULLIF(TRIM(k.keep_description), ''), s.description), + s.last_seen_at = GREATEST(s.last_seen_at, k.max_last_seen), + s.is_ignored = k.any_ignored, + s.ignored_at = CASE + WHEN k.any_ignored = 1 THEN COALESCE(s.ignored_at, k.max_ignored_at) + ELSE NULL + END, + s.ignored_by = CASE + WHEN k.any_ignored = 1 THEN COALESCE(NULLIF(TRIM(s.ignored_by), ''), NULLIF(TRIM(k.keep_ignored_by), '')) + ELSE NULL + END; + +DELETE s +FROM qt_vendor_partnumber_seen s +LEFT JOIN tmp_qt_seen_keep k ON LOWER(TRIM(s.partnumber)) = k.partnumber_key +WHERE k.keep_id IS NULL OR s.id <> k.keep_id; + +DROP TEMPORARY TABLE IF EXISTS tmp_qt_seen_keep; + +-- Safety pass: remove any remaining duplicates by normalized partnumber key. +DELETE s1 +FROM qt_vendor_partnumber_seen s1 +INNER JOIN qt_vendor_partnumber_seen s2 + ON LOWER(TRIM(s1.partnumber)) = LOWER(TRIM(s2.partnumber)) + AND s1.id < s2.id; + +SET @seen_idx_old_exists := ( + SELECT COUNT(*) + FROM information_schema.statistics + WHERE table_schema = DATABASE() + AND table_name = 'qt_vendor_partnumber_seen' + AND index_name = 'uq_qt_vendor_partnumber_seen_source_key' +); +SET @seen_idx_old_sql := IF( + @seen_idx_old_exists > 0, + 'ALTER TABLE qt_vendor_partnumber_seen DROP INDEX uq_qt_vendor_partnumber_seen_source_key', + 'SELECT 1' +); +PREPARE stmt_seen_idx_old FROM @seen_idx_old_sql; +EXECUTE stmt_seen_idx_old; +DEALLOCATE PREPARE stmt_seen_idx_old; + +SET @seen_idx_new_exists := ( + SELECT COUNT(*) + FROM information_schema.statistics + WHERE table_schema = DATABASE() + AND table_name = 'qt_vendor_partnumber_seen' + AND index_name = 'uq_qt_vendor_partnumber_seen_partnumber' +); +SET @seen_idx_new_sql := IF( + @seen_idx_new_exists > 0, + 'SELECT 1', + 'ALTER TABLE qt_vendor_partnumber_seen ADD UNIQUE INDEX uq_qt_vendor_partnumber_seen_partnumber (partnumber)' +); +PREPARE stmt_seen_idx_new FROM @seen_idx_new_sql; +EXECUTE stmt_seen_idx_new; +DEALLOCATE PREPARE stmt_seen_idx_new;