perf(convert): parallelize batch conversion across worker pool
Batch convert processed files strictly sequentially. Parsing/converting is CPU-bound and stateless per file, so run it across a GOMAXPROCS-sized worker pool; zip writing and progress updates stay serialized in original file order.
This commit is contained in:
+67
-27
@@ -15,6 +15,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"runtime"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -1423,40 +1424,79 @@ func (s *Server) runConvertJob(jobID, tempDir string, inputFiles []convertInputF
|
|||||||
success := 0
|
success := 0
|
||||||
totalProcess := len(inputFiles)
|
totalProcess := len(inputFiles)
|
||||||
|
|
||||||
for i, in := range inputFiles {
|
type convertOutcome struct {
|
||||||
s.jobManager.AppendJobLog(jobID, fmt.Sprintf("Processing %s", in.Name))
|
name string
|
||||||
payload, err := os.ReadFile(in.Path)
|
data any
|
||||||
if err != nil {
|
fail string
|
||||||
failures = append(failures, fmt.Sprintf("%s: %v", in.Name, err))
|
}
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
result, _, _, err := s.analyzeUploadedFile(in.Name, in.MIMEType, payload)
|
workers := runtime.GOMAXPROCS(0)
|
||||||
if err != nil {
|
if workers > totalProcess {
|
||||||
failures = append(failures, fmt.Sprintf("%s: %v", in.Name, err))
|
workers = totalProcess
|
||||||
progress := ((i + 1) * 100) / totalProcess
|
}
|
||||||
s.jobManager.UpdateJobStatus(jobID, CollectStatusRunning, progress, "")
|
if workers < 1 {
|
||||||
continue
|
workers = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
jobs := make(chan int)
|
||||||
|
results := make([]convertOutcome, totalProcess)
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for w := 0; w < workers; w++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for i := range jobs {
|
||||||
|
in := inputFiles[i]
|
||||||
|
s.jobManager.AppendJobLog(jobID, fmt.Sprintf("Processing %s", in.Name))
|
||||||
|
|
||||||
|
payload, err := os.ReadFile(in.Path)
|
||||||
|
if err != nil {
|
||||||
|
results[i] = convertOutcome{name: in.Name, fail: fmt.Sprintf("%s: %v", in.Name, err)}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
result, _, _, err := s.analyzeUploadedFile(in.Name, in.MIMEType, payload)
|
||||||
|
if err != nil {
|
||||||
|
results[i] = convertOutcome{name: in.Name, fail: fmt.Sprintf("%s: %v", in.Name, err)}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if result == nil || result.Hardware == nil {
|
||||||
|
results[i] = convertOutcome{name: in.Name, fail: fmt.Sprintf("%s: no hardware data", in.Name)}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
reanimatorData, err := exporter.ConvertToReanimator(result)
|
||||||
|
if err != nil {
|
||||||
|
results[i] = convertOutcome{name: in.Name, fail: fmt.Sprintf("%s: %v", in.Name, err)}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
results[i] = convertOutcome{name: in.Name, data: reanimatorData}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer close(jobs)
|
||||||
|
for i := range inputFiles {
|
||||||
|
jobs <- i
|
||||||
}
|
}
|
||||||
if result == nil || result.Hardware == nil {
|
}()
|
||||||
failures = append(failures, fmt.Sprintf("%s: no hardware data", in.Name))
|
wg.Wait()
|
||||||
|
|
||||||
|
for i, out := range results {
|
||||||
|
if out.data == nil {
|
||||||
|
failures = append(failures, out.fail)
|
||||||
progress := ((i + 1) * 100) / totalProcess
|
progress := ((i + 1) * 100) / totalProcess
|
||||||
s.jobManager.UpdateJobStatus(jobID, CollectStatusRunning, progress, "")
|
s.jobManager.UpdateJobStatus(jobID, CollectStatusRunning, progress, "")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
reanimatorData, err := exporter.ConvertToReanimator(result)
|
entryPath := sanitizeZipPath(out.name)
|
||||||
if err != nil {
|
|
||||||
failures = append(failures, fmt.Sprintf("%s: %v", in.Name, err))
|
|
||||||
progress := ((i + 1) * 100) / totalProcess
|
|
||||||
s.jobManager.UpdateJobStatus(jobID, CollectStatusRunning, progress, "")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
entryPath := sanitizeZipPath(in.Name)
|
|
||||||
entry, err := zw.Create(entryPath + ".reanimator.json")
|
entry, err := zw.Create(entryPath + ".reanimator.json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
failures = append(failures, fmt.Sprintf("%s: %v", in.Name, err))
|
failures = append(failures, fmt.Sprintf("%s: %v", out.name, err))
|
||||||
progress := ((i + 1) * 100) / totalProcess
|
progress := ((i + 1) * 100) / totalProcess
|
||||||
s.jobManager.UpdateJobStatus(jobID, CollectStatusRunning, progress, "")
|
s.jobManager.UpdateJobStatus(jobID, CollectStatusRunning, progress, "")
|
||||||
continue
|
continue
|
||||||
@@ -1464,8 +1504,8 @@ func (s *Server) runConvertJob(jobID, tempDir string, inputFiles []convertInputF
|
|||||||
|
|
||||||
encoder := json.NewEncoder(entry)
|
encoder := json.NewEncoder(entry)
|
||||||
encoder.SetIndent("", " ")
|
encoder.SetIndent("", " ")
|
||||||
if err := encoder.Encode(reanimatorData); err != nil {
|
if err := encoder.Encode(out.data); err != nil {
|
||||||
failures = append(failures, fmt.Sprintf("%s: %v", in.Name, err))
|
failures = append(failures, fmt.Sprintf("%s: %v", out.name, err))
|
||||||
} else {
|
} else {
|
||||||
success++
|
success++
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user