platform/app: stream SAT job output live and kick blackbox on job completion

A crash mid-command (e.g. the nvbandwidth reboot) previously lost that
job's entire output: streamExecOutput only buffered stdout/stderr in
memory and the job's log file was written once, after the process
exited. It now also streams each line straight to that file as it
arrives, so whatever printed before a crash survives.

Root filesystem here is a tmpfs overlay (toram boot), so the only real
persistence boundary is blackbox's mirror to removable media, not the
local write itself. platform.SetJobBoundaryHook lets app wire a touch
of a small kick-file after each job's output is written; blackboxWorker
now polls that file's mtime alongside its normal adaptive timer and
syncs immediately on a kick instead of waiting out the current flush
period (up to 30s).
This commit is contained in:
Mikhail Chusavitin
2026-07-27 17:40:22 +03:00
parent ced2175fb0
commit 781cf5dcbf
5 changed files with 365 additions and 24 deletions
+64 -8
View File
@@ -26,8 +26,43 @@ const (
blackboxMinFlushPeriod = 1 * time.Second
blackboxMaxFlushPeriod = 30 * time.Second
blackboxRecoveryFastCount = 5
// blackboxKickFileName is a marker under exportDir that SAT job
// execution touches after each job finishes (see platform.SetJobBoundaryHook
// wiring in New()). Workers poll its mtime so a just-finished job's
// output reaches removable media promptly instead of waiting out
// whatever the current adaptive flush period happens to be.
blackboxKickFileName = ".blackbox-kick"
)
// blackboxKickPollInterval is how often an idle worker checks the kick file
// for a pending out-of-band sync request. A package var (not a const) so
// tests can shrink it instead of waiting out the real interval.
var blackboxKickPollInterval = 250 * time.Millisecond
// requestBlackboxSync touches the kick file under exportDir, signalling any
// running blackbox worker to sync on its next poll instead of waiting out
// its current flush period. Best-effort: a failure here just means the next
// scheduled sync picks up the data instead of an early one.
func requestBlackboxSync(exportDir string) {
path := filepath.Join(exportDir, blackboxKickFileName)
now := blackboxNow()
if err := os.Chtimes(path, now, now); err == nil {
return
}
_ = os.WriteFile(path, []byte(now.Format(time.RFC3339Nano)+"\n"), 0644)
}
// blackboxKickModTime returns the kick file's mtime, or the zero Time if it
// doesn't exist yet (nothing has requested a sync since boot).
func blackboxKickModTime(exportDir string) time.Time {
info, err := os.Stat(filepath.Join(exportDir, blackboxKickFileName))
if err != nil {
return time.Time{}
}
return info.ModTime()
}
var DefaultBlackboxStatePath = DefaultExportDir + "/blackbox-state.json"
var (
@@ -96,8 +131,14 @@ type blackboxWorker struct {
flushPeriod time.Duration
lastError string
fastCycles int
lastKickSeen time.Time
stopCh chan struct{}
stoppedCh chan struct{}
// syncCycleFunc defaults to w.syncCycle; overridable in tests so run()'s
// wait/wake-on-kick logic can be exercised without a real removable-media
// mount and copy.
syncCycleFunc func() error
}
func RunBlackbox(ctx context.Context, exportDir, statePath string, system *platform.System) error {
@@ -309,7 +350,7 @@ func (rt *blackboxRuntime) discoverMarkedTargets() ([]discoveredBlackboxTarget,
}
func newBlackboxWorker(rt *blackboxRuntime, found discoveredBlackboxTarget) *blackboxWorker {
return &blackboxWorker{
w := &blackboxWorker{
runtime: rt,
enrollmentID: found.marker.EnrollmentID,
target: found.target,
@@ -319,24 +360,39 @@ func newBlackboxWorker(rt *blackboxRuntime, found discoveredBlackboxTarget) *bla
stopCh: make(chan struct{}),
stoppedCh: make(chan struct{}),
}
w.syncCycleFunc = w.syncCycle
return w
}
func (w *blackboxWorker) run() {
defer close(w.stoppedCh)
kickPoll := time.NewTicker(blackboxKickPollInterval)
defer kickPoll.Stop()
for {
start := time.Now()
err := w.syncCycle()
err := w.syncCycleFunc()
duration := time.Since(start)
w.finishCycle(duration, err)
w.lastKickSeen = blackboxKickModTime(w.runtime.exportDir)
wait := w.currentFlushPeriod()
timer := time.NewTimer(wait)
select {
case <-w.stopCh:
timer.Stop()
w.cleanup()
return
case <-timer.C:
waitLoop:
for {
select {
case <-w.stopCh:
timer.Stop()
w.cleanup()
return
case <-timer.C:
break waitLoop
case <-kickPoll.C:
if mtime := blackboxKickModTime(w.runtime.exportDir); mtime.After(w.lastKickSeen) {
timer.Stop()
break waitLoop
}
}
}
}
}