platform/app: stream SAT job output live and kick blackbox on job completion
A crash mid-command (e.g. the nvbandwidth reboot) previously lost that job's entire output: streamExecOutput only buffered stdout/stderr in memory and the job's log file was written once, after the process exited. It now also streams each line straight to that file as it arrives, so whatever printed before a crash survives. Root filesystem here is a tmpfs overlay (toram boot), so the only real persistence boundary is blackbox's mirror to removable media, not the local write itself. platform.SetJobBoundaryHook lets app wire a touch of a small kick-file after each job's output is written; blackboxWorker now polls that file's mtime alongside its normal adaptive timer and syncs immediately on a kick instead of waiting out the current flush period (up to 30s).
This commit is contained in:
@@ -26,8 +26,43 @@ const (
|
||||
blackboxMinFlushPeriod = 1 * time.Second
|
||||
blackboxMaxFlushPeriod = 30 * time.Second
|
||||
blackboxRecoveryFastCount = 5
|
||||
|
||||
// blackboxKickFileName is a marker under exportDir that SAT job
|
||||
// execution touches after each job finishes (see platform.SetJobBoundaryHook
|
||||
// wiring in New()). Workers poll its mtime so a just-finished job's
|
||||
// output reaches removable media promptly instead of waiting out
|
||||
// whatever the current adaptive flush period happens to be.
|
||||
blackboxKickFileName = ".blackbox-kick"
|
||||
)
|
||||
|
||||
// blackboxKickPollInterval is how often an idle worker checks the kick file
|
||||
// for a pending out-of-band sync request. A package var (not a const) so
|
||||
// tests can shrink it instead of waiting out the real interval.
|
||||
var blackboxKickPollInterval = 250 * time.Millisecond
|
||||
|
||||
// requestBlackboxSync touches the kick file under exportDir, signalling any
|
||||
// running blackbox worker to sync on its next poll instead of waiting out
|
||||
// its current flush period. Best-effort: a failure here just means the next
|
||||
// scheduled sync picks up the data instead of an early one.
|
||||
func requestBlackboxSync(exportDir string) {
|
||||
path := filepath.Join(exportDir, blackboxKickFileName)
|
||||
now := blackboxNow()
|
||||
if err := os.Chtimes(path, now, now); err == nil {
|
||||
return
|
||||
}
|
||||
_ = os.WriteFile(path, []byte(now.Format(time.RFC3339Nano)+"\n"), 0644)
|
||||
}
|
||||
|
||||
// blackboxKickModTime returns the kick file's mtime, or the zero Time if it
|
||||
// doesn't exist yet (nothing has requested a sync since boot).
|
||||
func blackboxKickModTime(exportDir string) time.Time {
|
||||
info, err := os.Stat(filepath.Join(exportDir, blackboxKickFileName))
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return info.ModTime()
|
||||
}
|
||||
|
||||
var DefaultBlackboxStatePath = DefaultExportDir + "/blackbox-state.json"
|
||||
|
||||
var (
|
||||
@@ -96,8 +131,14 @@ type blackboxWorker struct {
|
||||
flushPeriod time.Duration
|
||||
lastError string
|
||||
fastCycles int
|
||||
lastKickSeen time.Time
|
||||
stopCh chan struct{}
|
||||
stoppedCh chan struct{}
|
||||
|
||||
// syncCycleFunc defaults to w.syncCycle; overridable in tests so run()'s
|
||||
// wait/wake-on-kick logic can be exercised without a real removable-media
|
||||
// mount and copy.
|
||||
syncCycleFunc func() error
|
||||
}
|
||||
|
||||
func RunBlackbox(ctx context.Context, exportDir, statePath string, system *platform.System) error {
|
||||
@@ -309,7 +350,7 @@ func (rt *blackboxRuntime) discoverMarkedTargets() ([]discoveredBlackboxTarget,
|
||||
}
|
||||
|
||||
func newBlackboxWorker(rt *blackboxRuntime, found discoveredBlackboxTarget) *blackboxWorker {
|
||||
return &blackboxWorker{
|
||||
w := &blackboxWorker{
|
||||
runtime: rt,
|
||||
enrollmentID: found.marker.EnrollmentID,
|
||||
target: found.target,
|
||||
@@ -319,24 +360,39 @@ func newBlackboxWorker(rt *blackboxRuntime, found discoveredBlackboxTarget) *bla
|
||||
stopCh: make(chan struct{}),
|
||||
stoppedCh: make(chan struct{}),
|
||||
}
|
||||
w.syncCycleFunc = w.syncCycle
|
||||
return w
|
||||
}
|
||||
|
||||
func (w *blackboxWorker) run() {
|
||||
defer close(w.stoppedCh)
|
||||
kickPoll := time.NewTicker(blackboxKickPollInterval)
|
||||
defer kickPoll.Stop()
|
||||
|
||||
for {
|
||||
start := time.Now()
|
||||
err := w.syncCycle()
|
||||
err := w.syncCycleFunc()
|
||||
duration := time.Since(start)
|
||||
w.finishCycle(duration, err)
|
||||
w.lastKickSeen = blackboxKickModTime(w.runtime.exportDir)
|
||||
|
||||
wait := w.currentFlushPeriod()
|
||||
timer := time.NewTimer(wait)
|
||||
select {
|
||||
case <-w.stopCh:
|
||||
timer.Stop()
|
||||
w.cleanup()
|
||||
return
|
||||
case <-timer.C:
|
||||
waitLoop:
|
||||
for {
|
||||
select {
|
||||
case <-w.stopCh:
|
||||
timer.Stop()
|
||||
w.cleanup()
|
||||
return
|
||||
case <-timer.C:
|
||||
break waitLoop
|
||||
case <-kickPoll.C:
|
||||
if mtime := blackboxKickModTime(w.runtime.exportDir); mtime.After(w.lastKickSeen) {
|
||||
timer.Stop()
|
||||
break waitLoop
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user