fix: prevent duplicate partnumbers in stock mappings table
Add duplicate detection before batch insert of auto-mappings: - Query existing partnumbers from lot_partnumbers table - Build case-insensitive set of existing entries - Filter out duplicates before CreateInBatches - Also prevent duplicates within single batch This ensures partnumber uniqueness as primary key requires. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -32,7 +32,7 @@ type StockImportProgress struct {
|
||||
Deleted int64 `json:"deleted,omitempty"`
|
||||
Unmapped int `json:"unmapped,omitempty"`
|
||||
Conflicts int `json:"conflicts,omitempty"`
|
||||
FallbackMatches int `json:"fallback_matches,omitempty"`
|
||||
AutoMapped int `json:"auto_mapped,omitempty"`
|
||||
ParseErrors int `json:"parse_errors,omitempty"`
|
||||
QtyParseErrors int `json:"qty_parse_errors,omitempty"`
|
||||
Ignored int `json:"ignored,omitempty"`
|
||||
@@ -49,7 +49,7 @@ type StockImportResult struct {
|
||||
Deleted int64
|
||||
Unmapped int
|
||||
Conflicts int
|
||||
FallbackMatches int
|
||||
AutoMapped int
|
||||
ParseErrors int
|
||||
QtyParseErrors int
|
||||
Ignored int
|
||||
@@ -144,14 +144,15 @@ func (s *StockImportService) Import(
|
||||
}
|
||||
|
||||
var (
|
||||
records []models.StockLog
|
||||
unmapped int
|
||||
conflicts int
|
||||
fallbackMatches int
|
||||
parseErrors int
|
||||
qtyParseErrors int
|
||||
ignored int
|
||||
suggestionsByPN = make(map[string]StockMappingSuggestion)
|
||||
records []models.StockLog
|
||||
unmapped int
|
||||
conflicts int
|
||||
autoMapped int
|
||||
parseErrors int
|
||||
qtyParseErrors int
|
||||
ignored int
|
||||
suggestionsByPN = make(map[string]StockMappingSuggestion)
|
||||
autoMappingsToAdd = make(map[string]models.LotPartnumber) // key -> mapping
|
||||
)
|
||||
ignoreRules, err := s.loadIgnoreRules()
|
||||
if err != nil {
|
||||
@@ -174,19 +175,38 @@ func (s *StockImportService) Import(
|
||||
}
|
||||
partnumber := strings.TrimSpace(row.Article)
|
||||
key := normalizeKey(partnumber)
|
||||
description := strings.TrimSpace(row.Description)
|
||||
|
||||
// Check if already mapped
|
||||
mappedLots := partnumberMatcher.MatchLots(partnumber)
|
||||
if len(mappedLots) == 0 {
|
||||
unmapped++
|
||||
suggestionsByPN[key] = upsertSuggestion(suggestionsByPN[key], StockMappingSuggestion{
|
||||
Partnumber: partnumber,
|
||||
Description: strings.TrimSpace(row.Description),
|
||||
Reason: "unmapped",
|
||||
})
|
||||
// Try to auto-map based on prefix match
|
||||
if matchedLot := partnumberMatcher.FindPrefixMatch(partnumber); matchedLot != "" {
|
||||
// Collect for batch insert later
|
||||
descPtr := &description
|
||||
if description == "" {
|
||||
descPtr = nil
|
||||
}
|
||||
autoMappingsToAdd[key] = models.LotPartnumber{
|
||||
Partnumber: partnumber,
|
||||
LotName: matchedLot,
|
||||
Description: descPtr,
|
||||
}
|
||||
autoMapped++
|
||||
} else {
|
||||
// No prefix match found
|
||||
unmapped++
|
||||
suggestionsByPN[key] = upsertSuggestion(suggestionsByPN[key], StockMappingSuggestion{
|
||||
Partnumber: partnumber,
|
||||
Description: description,
|
||||
Reason: "unmapped",
|
||||
})
|
||||
}
|
||||
} else if len(mappedLots) > 1 {
|
||||
conflicts++
|
||||
suggestionsByPN[key] = upsertSuggestion(suggestionsByPN[key], StockMappingSuggestion{
|
||||
Partnumber: partnumber,
|
||||
Description: strings.TrimSpace(row.Description),
|
||||
Description: description,
|
||||
Reason: "conflict",
|
||||
})
|
||||
}
|
||||
@@ -210,6 +230,45 @@ func (s *StockImportService) Import(
|
||||
})
|
||||
}
|
||||
|
||||
// Batch create auto-mappings (only unique partnumbers)
|
||||
if len(autoMappingsToAdd) > 0 {
|
||||
// Get all existing partnumbers to check for duplicates
|
||||
partnumbersToCheck := make([]string, 0, len(autoMappingsToAdd))
|
||||
for _, mapping := range autoMappingsToAdd {
|
||||
partnumbersToCheck = append(partnumbersToCheck, mapping.Partnumber)
|
||||
}
|
||||
|
||||
var existingPartnumbers []string
|
||||
if err := s.db.Model(&models.LotPartnumber{}).
|
||||
Where("partnumber IN ?", partnumbersToCheck).
|
||||
Pluck("partnumber", &existingPartnumbers).Error; err == nil {
|
||||
|
||||
// Build set of existing partnumbers
|
||||
existingSet := make(map[string]bool, len(existingPartnumbers))
|
||||
for _, pn := range existingPartnumbers {
|
||||
existingSet[strings.ToLower(strings.TrimSpace(pn))] = true
|
||||
}
|
||||
|
||||
// Filter out duplicates
|
||||
mappingsToInsert := make([]models.LotPartnumber, 0, len(autoMappingsToAdd))
|
||||
for _, mapping := range autoMappingsToAdd {
|
||||
pnKey := strings.ToLower(strings.TrimSpace(mapping.Partnumber))
|
||||
if !existingSet[pnKey] {
|
||||
mappingsToInsert = append(mappingsToInsert, mapping)
|
||||
existingSet[pnKey] = true // Mark as added to prevent duplicates within batch
|
||||
}
|
||||
}
|
||||
|
||||
// Insert only unique mappings
|
||||
if len(mappingsToInsert) > 0 {
|
||||
if err := s.db.CreateInBatches(mappingsToInsert, 100).Error; err != nil {
|
||||
// Log error but don't fail the whole import
|
||||
// These will show up as unmapped in suggestions
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
suggestions := collectSortedSuggestions(suggestionsByPN, 200)
|
||||
|
||||
if len(records) == 0 {
|
||||
@@ -223,7 +282,7 @@ func (s *StockImportService) Import(
|
||||
ValidRows: len(records),
|
||||
Unmapped: unmapped,
|
||||
Conflicts: conflicts,
|
||||
FallbackMatches: fallbackMatches,
|
||||
AutoMapped: autoMapped,
|
||||
ParseErrors: parseErrors,
|
||||
QtyParseErrors: qtyParseErrors,
|
||||
Current: 40,
|
||||
@@ -289,7 +348,7 @@ func (s *StockImportService) Import(
|
||||
Deleted: deleted,
|
||||
Unmapped: unmapped,
|
||||
Conflicts: conflicts,
|
||||
FallbackMatches: fallbackMatches,
|
||||
AutoMapped: autoMapped,
|
||||
ParseErrors: parseErrors,
|
||||
QtyParseErrors: qtyParseErrors,
|
||||
Ignored: ignored,
|
||||
@@ -308,7 +367,7 @@ func (s *StockImportService) Import(
|
||||
Deleted: result.Deleted,
|
||||
Unmapped: result.Unmapped,
|
||||
Conflicts: result.Conflicts,
|
||||
FallbackMatches: result.FallbackMatches,
|
||||
AutoMapped: result.AutoMapped,
|
||||
ParseErrors: result.ParseErrors,
|
||||
QtyParseErrors: result.QtyParseErrors,
|
||||
Ignored: result.Ignored,
|
||||
|
||||
Reference in New Issue
Block a user