refactor lot matching into shared module

This commit is contained in:
2026-02-07 06:22:56 +03:00
parent 7c741ff675
commit 86ed26fdd6
14 changed files with 1190 additions and 520 deletions

View File

@@ -10,6 +10,7 @@ import (
"git.mchus.pro/mchus/quoteforge/internal/models"
"git.mchus.pro/mchus/quoteforge/internal/repository"
"git.mchus.pro/mchus/quoteforge/internal/services/pricing"
"git.mchus.pro/mchus/quoteforge/internal/warehouse"
"gorm.io/gorm"
)
@@ -132,6 +133,22 @@ func (s *Service) CreateForSourceWithProgress(createdBy, source string, sourceIt
}
items := make([]models.PricelistItem, 0)
if len(sourceItems) == 0 && source == string(models.PricelistSourceWarehouse) {
warehouseItems, err := warehouse.ComputePricelistItemsFromStockLog(s.db)
if err != nil {
_ = s.repo.Delete(pricelist.ID)
return nil, fmt.Errorf("building warehouse pricelist from stock_log: %w", err)
}
sourceItems = make([]CreateItemInput, 0, len(warehouseItems))
for _, item := range warehouseItems {
sourceItems = append(sourceItems, CreateItemInput{
LotName: item.LotName,
Price: item.Price,
PriceMethod: item.PriceMethod,
})
}
}
if len(sourceItems) > 0 {
items = make([]models.PricelistItem, 0, len(sourceItems))
for _, srcItem := range sourceItems {

View File

@@ -0,0 +1,72 @@
package pricelist
import (
"math"
"testing"
"time"
"git.mchus.pro/mchus/quoteforge/internal/models"
"git.mchus.pro/mchus/quoteforge/internal/repository"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
)
func TestCreateWarehousePricelistFromStockLog(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := db.AutoMigrate(
&models.Pricelist{},
&models.PricelistItem{},
&models.StockLog{},
&models.Lot{},
&models.LotPartnumber{},
); err != nil {
t.Fatalf("automigrate: %v", err)
}
if err := db.Create(&models.Lot{LotName: "CPU_X", LotDescription: "CPU"}).Error; err != nil {
t.Fatalf("seed lot: %v", err)
}
if err := db.Create(&models.LotPartnumber{Partnumber: "PN-CPU-X", LotName: "CPU_X"}).Error; err != nil {
t.Fatalf("seed mapping: %v", err)
}
qty1 := 2.0
qty2 := 8.0
now := time.Now()
rows := []models.StockLog{
{Partnumber: "PN-CPU-X", Date: now, Price: 100, Qty: &qty1},
{Partnumber: "PN-CPU-X", Date: now, Price: 200, Qty: &qty2},
}
if err := db.Create(&rows).Error; err != nil {
t.Fatalf("seed stock log: %v", err)
}
repo := repository.NewPricelistRepository(db)
svc := NewService(db, repo, nil, nil)
pl, err := svc.CreateForSourceWithProgress("tester", string(models.PricelistSourceWarehouse), nil, nil)
if err != nil {
t.Fatalf("create warehouse pricelist: %v", err)
}
if pl.Source != string(models.PricelistSourceWarehouse) {
t.Fatalf("unexpected source: %s", pl.Source)
}
var items []models.PricelistItem
if err := db.Where("pricelist_id = ?", pl.ID).Find(&items).Error; err != nil {
t.Fatalf("load pricelist items: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 item, got %d", len(items))
}
if items[0].LotName != "CPU_X" {
t.Fatalf("unexpected lot name: %s", items[0].LotName)
}
if math.Abs(items[0].Price-200) > 0.001 {
t.Fatalf("expected weighted median price 200, got %f", items[0].Price)
}
}

View File

@@ -4,7 +4,6 @@ import (
"archive/zip"
"bytes"
"encoding/xml"
"errors"
"fmt"
"io"
"path/filepath"
@@ -14,8 +13,10 @@ import (
"strings"
"time"
"git.mchus.pro/mchus/quoteforge/internal/lotmatch"
"git.mchus.pro/mchus/quoteforge/internal/models"
pricelistsvc "git.mchus.pro/mchus/quoteforge/internal/services/pricelist"
"git.mchus.pro/mchus/quoteforge/internal/warehouse"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
@@ -137,7 +138,7 @@ func (s *StockImportService) Import(
Total: 100,
})
partnumberMappings, err := s.loadPartnumberMappings()
partnumberMatcher, err := lotmatch.NewMappingMatcherFromDB(s.db)
if err != nil {
return nil, err
}
@@ -173,7 +174,7 @@ func (s *StockImportService) Import(
}
partnumber := strings.TrimSpace(row.Article)
key := normalizeKey(partnumber)
mappedLots := partnumberMappings[key]
mappedLots := partnumberMatcher.MatchLots(partnumber)
if len(mappedLots) == 0 {
unmapped++
suggestionsByPN[key] = upsertSuggestion(suggestionsByPN[key], StockMappingSuggestion{
@@ -342,74 +343,22 @@ func (s *StockImportService) replaceStockLogs(records []models.StockLog) (int64,
}
func (s *StockImportService) buildWarehousePricelistItems() ([]pricelistsvc.CreateItemInput, error) {
var logs []models.StockLog
if err := s.db.Select("partnumber, price, qty").Where("price > 0").Find(&logs).Error; err != nil {
return nil, err
}
resolver, err := s.newLotResolver()
warehouseItems, err := warehouse.ComputePricelistItemsFromStockLog(s.db)
if err != nil {
return nil, err
}
grouped := make(map[string][]weightedPricePoint)
for _, l := range logs {
partnumber := strings.TrimSpace(l.Partnumber)
if partnumber == "" || l.Price <= 0 {
continue
}
lotName, _, err := resolver.resolve(partnumber)
if err != nil || strings.TrimSpace(lotName) == "" {
continue
}
weight := 0.0
if l.Qty != nil && *l.Qty > 0 {
weight = *l.Qty
}
grouped[lotName] = append(grouped[lotName], weightedPricePoint{
price: l.Price,
weight: weight,
})
}
items := make([]pricelistsvc.CreateItemInput, 0, len(grouped))
for lot, values := range grouped {
price := weightedMedian(values)
if price <= 0 {
continue
}
items := make([]pricelistsvc.CreateItemInput, 0, len(warehouseItems))
for _, item := range warehouseItems {
items = append(items, pricelistsvc.CreateItemInput{
LotName: lot,
Price: price,
PriceMethod: "weighted_median",
LotName: item.LotName,
Price: item.Price,
PriceMethod: item.PriceMethod,
})
}
sort.Slice(items, func(i, j int) bool {
return items[i].LotName < items[j].LotName
})
return items, nil
}
func (s *StockImportService) loadPartnumberMappings() (map[string][]string, error) {
var mappings []models.LotPartnumber
if err := s.db.Find(&mappings).Error; err != nil {
return nil, err
}
partnumberToLots := make(map[string][]string, len(mappings))
for _, m := range mappings {
pn := normalizeKey(m.Partnumber)
lot := strings.TrimSpace(m.LotName)
if pn == "" || lot == "" {
continue
}
partnumberToLots[pn] = append(partnumberToLots[pn], lot)
}
for key, lots := range partnumberToLots {
partnumberToLots[key] = uniqueStrings(lots)
}
return partnumberToLots, nil
}
func upsertSuggestion(prev StockMappingSuggestion, candidate StockMappingSuggestion) StockMappingSuggestion {
if strings.TrimSpace(prev.Partnumber) == "" {
return candidate
@@ -674,11 +623,9 @@ func normalizeIgnoreMatchType(v string) string {
}
var (
reISODate = regexp.MustCompile(`\b(20\d{2})-(\d{2})-(\d{2})\b`)
reRuDate = regexp.MustCompile(`\b([0-3]\d)\.([01]\d)\.(20\d{2})\b`)
mxlCellRe = regexp.MustCompile(`\{16,\d+,\s*\{1,1,\s*\{"ru","(.*?)"\}\s*\},0\},(\d+),`)
errResolveConflict = errors.New("multiple lot matches")
errResolveNotFound = errors.New("lot not found")
reISODate = regexp.MustCompile(`\b(20\d{2})-(\d{2})-(\d{2})\b`)
reRuDate = regexp.MustCompile(`\b([0-3]\d)\.([01]\d)\.(20\d{2})\b`)
mxlCellRe = regexp.MustCompile(`\{16,\d+,\s*\{1,1,\s*\{"ru","(.*?)"\}\s*\},0\},(\d+),`)
)
func parseStockRows(filename string, content []byte) ([]stockImportRow, error) {
@@ -1020,124 +967,8 @@ func weightedMedian(values []weightedPricePoint) float64 {
return items[len(items)-1].price
}
type lotResolver struct {
partnumberToLots map[string][]string
exactLots map[string]string
allLots []string
}
func (s *StockImportService) newLotResolver() (*lotResolver, error) {
var mappings []models.LotPartnumber
if err := s.db.Find(&mappings).Error; err != nil {
return nil, err
}
partnumberToLots := make(map[string][]string, len(mappings))
for _, m := range mappings {
p := normalizeKey(m.Partnumber)
if p == "" || strings.TrimSpace(m.LotName) == "" {
continue
}
partnumberToLots[p] = append(partnumberToLots[p], m.LotName)
}
var lots []models.Lot
if err := s.db.Select("lot_name").Find(&lots).Error; err != nil {
return nil, err
}
exactLots := make(map[string]string, len(lots))
allLots := make([]string, 0, len(lots))
for _, l := range lots {
name := strings.TrimSpace(l.LotName)
if name == "" {
continue
}
k := normalizeKey(name)
exactLots[k] = name
allLots = append(allLots, name)
}
sort.Slice(allLots, func(i, j int) bool {
li := len([]rune(allLots[i]))
lj := len([]rune(allLots[j]))
if li == lj {
return allLots[i] < allLots[j]
}
return li > lj
})
return &lotResolver{
partnumberToLots: partnumberToLots,
exactLots: exactLots,
allLots: allLots,
}, nil
}
func (r *lotResolver) resolve(article string) (string, string, error) {
key := normalizeKey(article)
if key == "" {
return "", "", errResolveNotFound
}
if mapped := r.partnumberToLots[key]; len(mapped) > 0 {
uniq := uniqueStrings(mapped)
if len(uniq) == 1 {
return uniq[0], "mapping_table", nil
}
return "", "", errResolveConflict
}
if lot, ok := r.exactLots[key]; ok {
return lot, "article_exact", nil
}
best := ""
bestLen := -1
tie := false
for _, lot := range r.allLots {
lotKey := normalizeKey(lot)
if lotKey == "" {
continue
}
if strings.HasPrefix(key, lotKey) {
l := len([]rune(lotKey))
if l > bestLen {
best = lot
bestLen = l
tie = false
} else if l == bestLen && !strings.EqualFold(best, lot) {
tie = true
}
}
}
if best == "" {
return "", "", errResolveNotFound
}
if tie {
return "", "", errResolveConflict
}
return best, "prefix", nil
}
func normalizeKey(v string) string {
return strings.ToLower(strings.TrimSpace(v))
}
func uniqueStrings(values []string) []string {
seen := make(map[string]bool, len(values))
out := make([]string, 0, len(values))
for _, v := range values {
v = strings.TrimSpace(v)
if v == "" {
continue
}
k := strings.ToLower(v)
if seen[k] {
continue
}
seen[k] = true
out = append(out, v)
}
sort.Strings(out)
return out
return lotmatch.NormalizeKey(v)
}
func readZipFile(zr *zip.Reader, name string) ([]byte, error) {

View File

@@ -7,6 +7,7 @@ import (
"testing"
"time"
"git.mchus.pro/mchus/quoteforge/internal/lotmatch"
"git.mchus.pro/mchus/quoteforge/internal/models"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
@@ -99,39 +100,42 @@ func TestParseXLSXRows(t *testing.T) {
}
func TestLotResolverPrecedenceAndConflicts(t *testing.T) {
r := &lotResolver{
partnumberToLots: map[string][]string{
"pn-1": {"LOT_MAPPED"},
"pn-conflict": {"LOT_A", "LOT_B"},
resolver := lotmatch.NewLotResolver(
[]models.LotPartnumber{
{Partnumber: "pn-1", LotName: "LOT_MAPPED"},
{Partnumber: "pn-conflict", LotName: "LOT_A"},
{Partnumber: "pn-conflict", LotName: "LOT_B"},
},
exactLots: map[string]string{
"cpu_a": "CPU_A",
[]models.Lot{
{LotName: "CPU_A_LONG"},
{LotName: "CPU_A"},
{LotName: "ABC "},
{LotName: "ABC\t"},
},
allLots: []string{"CPU_A_LONG", "CPU_A", "ABC ", "ABC\t"},
}
)
lot, typ, err := r.resolve("pn-1")
lot, typ, err := resolver.Resolve("pn-1")
if err != nil || lot != "LOT_MAPPED" || typ != "mapping_table" {
t.Fatalf("mapping_table mismatch: lot=%s typ=%s err=%v", lot, typ, err)
}
lot, typ, err = r.resolve("cpu_a")
lot, typ, err = resolver.Resolve("cpu_a")
if err != nil || lot != "CPU_A" || typ != "article_exact" {
t.Fatalf("article_exact mismatch: lot=%s typ=%s err=%v", lot, typ, err)
}
lot, typ, err = r.resolve("cpu_a_long_suffix")
lot, typ, err = resolver.Resolve("cpu_a_long_suffix")
if err != nil || lot != "CPU_A_LONG" || typ != "prefix" {
t.Fatalf("prefix mismatch: lot=%s typ=%s err=%v", lot, typ, err)
}
_, _, err = r.resolve("abx")
if err == nil {
t.Fatalf("expected not found error")
_, _, err = resolver.Resolve("abx")
if err == nil || err != lotmatch.ErrResolveNotFound {
t.Fatalf("expected not found error, got %v", err)
}
_, _, err = r.resolve("pn-conflict")
if err == nil || err != errResolveConflict {
_, _, err = resolver.Resolve("pn-conflict")
if err == nil || err != lotmatch.ErrResolveConflict {
t.Fatalf("expected conflict, got %v", err)
}
}
@@ -267,6 +271,42 @@ func TestBuildWarehousePricelistItems_UsesPrefixResolver(t *testing.T) {
}
}
func TestPartnumberMappings_WildcardMatch(t *testing.T) {
db := openTestDB(t)
if err := db.AutoMigrate(&models.LotPartnumber{}, &models.Lot{}); err != nil {
t.Fatalf("automigrate: %v", err)
}
mappings := []models.LotPartnumber{
{Partnumber: "R750*", LotName: "SERVER_R750"},
{Partnumber: "HDD-01", LotName: "HDD_01"},
}
if err := db.Create(&mappings).Error; err != nil {
t.Fatalf("seed mappings: %v", err)
}
if err := db.Create(&models.Lot{LotName: "MEM_DDR5_16G_4800"}).Error; err != nil {
t.Fatalf("seed lot: %v", err)
}
resolver, err := lotmatch.NewMappingMatcherFromDB(db)
if err != nil {
t.Fatalf("NewMappingMatcherFromDB: %v", err)
}
if got := resolver.MatchLots("R750XD"); len(got) != 1 || got[0] != "SERVER_R750" {
t.Fatalf("expected wildcard match SERVER_R750, got %#v", got)
}
if got := resolver.MatchLots("HDD-01"); len(got) != 1 || got[0] != "HDD_01" {
t.Fatalf("expected exact match HDD_01, got %#v", got)
}
if got := resolver.MatchLots("UNKNOWN"); len(got) != 0 {
t.Fatalf("expected no matches, got %#v", got)
}
if got := resolver.MatchLots("MEM_DDR5_16G_4800"); len(got) != 1 || got[0] != "MEM_DDR5_16G_4800" {
t.Fatalf("expected exact lot fallback, got %#v", got)
}
}
func openTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})

View File

@@ -322,6 +322,9 @@ func (s *Service) NeedSync() (bool, error) {
// SyncPricelists synchronizes all active pricelists from server to local SQLite
func (s *Service) SyncPricelists() (int, error) {
slog.Info("starting pricelist sync")
if _, err := s.EnsureReadinessForSync(); err != nil {
return 0, err
}
// Get database connection
mariaDB, err := s.getDB()
@@ -592,10 +595,14 @@ func (s *Service) SyncPricelistItems(localPricelistID uint) (int, error) {
// Convert and save locally
localItems := make([]localdb.LocalPricelistItem, len(serverItems))
for i, item := range serverItems {
partnumbers := make(localdb.LocalStringList, 0, len(item.Partnumbers))
partnumbers = append(partnumbers, item.Partnumbers...)
localItems[i] = localdb.LocalPricelistItem{
PricelistID: localPricelistID,
LotName: item.LotName,
Price: item.Price,
PricelistID: localPricelistID,
LotName: item.LotName,
Price: item.Price,
AvailableQty: item.AvailableQty,
Partnumbers: partnumbers,
}
}
@@ -672,6 +679,10 @@ func (s *Service) SyncPricelistsIfNeeded() error {
// PushPendingChanges pushes all pending changes to the server
func (s *Service) PushPendingChanges() (int, error) {
if _, err := s.EnsureReadinessForSync(); err != nil {
return 0, err
}
removed, err := s.localDB.PurgeOrphanConfigurationPendingChanges()
if err != nil {
slog.Warn("failed to purge orphan configuration pending changes", "error", err)