app: don't let a renamed-away source path abort the whole blackbox/support-bundle copy
copyPath/copyPathFiltered treated any os.Stat/os.ReadDir/os.Open error on a source entry as fatal, aborting the entire tree copy. A source path is read from the live export dir while bee's own task runner concurrently renames task directories (e.g. "_pending" -> "_done") — an entry present in the parent's os.ReadDir a moment ago disappearing by the time it's individually Stat'd/Open'd is an expected race, not a real failure. Seen on a real crash bundle: blackbox got stuck in status "degraded" from early in the run (first hit during the CPU pack, well before the GPU tests) after exactly this race, and every syncBracket wait then timed out for the rest of the run — the discovery/wait plumbing from the previous fix works, but had nothing working under it to wait on. The target also accumulated stale "_pending" copies alongside "_done" ones with no cleanup, though fixing that dedup is left for a follow-up. os.IsNotExist(err) now skips the vanished entry instead of propagating. Added regression tests simulating the race directly (copy_path_race_test.go). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
fa618d5abb
commit
49979c4da4
@@ -752,6 +752,17 @@ func copyDirContentsFiltered(srcDir, dstDir string, keep func(rel string, info o
|
||||
func copyPath(src, dst string) error {
|
||||
info, err := os.Stat(src)
|
||||
if err != nil {
|
||||
// src is read from a live export dir that bee's own SAT/task runner
|
||||
// keeps writing to and renaming (e.g. a task dir's "_pending" ->
|
||||
// "_done" transition) concurrently with this copy — a path present
|
||||
// in the parent's os.ReadDir listing a moment ago disappearing by
|
||||
// the time we get here is an expected race, not a real failure.
|
||||
// Skip it; the next cycle will pick up wherever it landed under its
|
||||
// new name. Aborting the whole copy over one renamed-away entry is
|
||||
// what previously left blackbox stuck "degraded" indefinitely.
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
@@ -760,6 +771,9 @@ func copyPath(src, dst string) error {
|
||||
}
|
||||
entries, err := os.ReadDir(src)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
@@ -783,6 +797,9 @@ func copyPath(src, dst string) error {
|
||||
}
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
@@ -800,6 +817,12 @@ func copyPath(src, dst string) error {
|
||||
func copyPathFiltered(rootSrc, src, dst string, keep func(rel string, info os.FileInfo) bool) error {
|
||||
info, err := os.Stat(src)
|
||||
if err != nil {
|
||||
// See the matching comment in copyPath: src disappearing between the
|
||||
// parent's os.ReadDir and this Stat is an expected race against
|
||||
// bee's own live task-dir renames, not a real error.
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(rootSrc, src)
|
||||
@@ -815,6 +838,9 @@ func copyPathFiltered(rootSrc, src, dst string, keep func(rel string, info os.Fi
|
||||
}
|
||||
entries, err := os.ReadDir(src)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
|
||||
Reference in New Issue
Block a user