webui/collector: strip ANSI escapes in nvidia-smi topo parsing, auto-prepare drives for RAID mirror creation

- nvidia-smi underlines the topo -m header with ANSI CSI codes even when
  writing to a file; both NVLink matrix parsers failed to find the header
  and /topo showed "No NVLink-bonded GPU pairs found" on bonded systems.
- raid-lsi-create-mirror failed with "resources already in use" (exit 11)
  on JBOD drives; the task now reads drive states and auto-converts
  JBOD/UBad (set good force), releases hotspares, refuses Frgn/Onln with
  actionable messages, and dumps preservedcache info when add vd fails.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-08 23:16:04 +03:00
co-authored by Claude Fable 5
parent 1175e6ccd8
commit 10557ec0f6
6 changed files with 184 additions and 3 deletions
+110 -1
View File
@@ -576,8 +576,107 @@ func runRAIDForeignImportTask(ctx context.Context, j *jobState, ctrl int) error
return streamCmdJob(j, cmd)
}
// raidPrepareAction says what (if anything) must be done to a drive in the
// given storcli state before it can join a new VD. Derived from the Broadcom
// StorCLI drive-state matrix: only UGood drives are accepted by "add vd";
// JBOD/UBad convert with "set good force"; hotspares must be released first;
// Frgn/Onln hold configuration data and must not be silently destroyed.
type raidPrepareAction int
const (
raidPrepNone raidPrepareAction = iota // UGood or unknown — try add vd as-is
raidPrepSetGood // JBOD, UBad — "set good force"
raidPrepHotspare // GHS, DHS — "delete hotsparedrive", then set good
raidPrepBlockedFrgn
raidPrepBlockedOnln
)
func classifyRAIDPrepareAction(state string) raidPrepareAction {
switch strings.TrimSpace(state) {
case "JBOD", "UBad":
return raidPrepSetGood
case "GHS", "DHS":
return raidPrepHotspare
case "Frgn":
return raidPrepBlockedFrgn
case "Onln", "Offln":
return raidPrepBlockedOnln
default: // "UGood", "" (state unknown — let add vd decide)
return raidPrepNone
}
}
// raidLSIDriveStates returns EID:Slt -> State for one controller, or nil if
// storcli/parsing fails (callers then fall back to unconditional prepare).
func raidLSIDriveStates(ctx context.Context, ctrl int) map[string]string {
out, err := exec.CommandContext(ctx, "storcli64",
fmt.Sprintf("/c%d/eall/sall", ctrl), "show", "all", "J").Output()
if err != nil {
return nil
}
var doc struct {
Controllers []struct {
ResponseData map[string]json.RawMessage `json:"Response Data"`
} `json:"Controllers"`
}
if err := json.Unmarshal(out, &doc); err != nil || len(doc.Controllers) == 0 {
return nil
}
states := map[string]string{}
for _, c := range doc.Controllers {
for _, d := range parseStorcliResponseDataDrives(c.ResponseData) {
states[strings.TrimSpace(d.EIDSlt)] = strings.TrimSpace(d.State)
}
}
return states
}
func runRAIDLSICreateMirrorTask(ctx context.Context, j *jobState, ctrl int, drives []string) error {
driveList := strings.Join(drives, ",")
states := raidLSIDriveStates(ctx, ctrl)
// Non-UGood drives cannot be added to a VD directly — storcli fails with
// "resources already in use" (exit 11) or similar. Fix what is safely
// fixable (JBOD/UBad/hotspare), refuse what holds data (Frgn/Onln).
for _, drive := range drives {
eid, slt, ok := parseRAIDSlot(drive)
if !ok {
return fmt.Errorf("invalid drive slot %q", drive)
}
state := states[drive]
slotPath := fmt.Sprintf("/c%d/e%d/s%d", ctrl, eid, slt)
switch classifyRAIDPrepareAction(state) {
case raidPrepBlockedFrgn:
return fmt.Errorf("drive %s carries a foreign configuration; run the RAID Foreign Clear (or Import) task first, then retry", drive)
case raidPrepBlockedOnln:
return fmt.Errorf("drive %s is part of an existing virtual drive (state %s); delete that VD first", drive, state)
case raidPrepHotspare:
j.append(fmt.Sprintf("Drive %s is a hotspare (%s); releasing it...", drive, state))
rel := exec.CommandContext(ctx, "storcli64", slotPath, "delete", "hotsparedrive")
if err := streamCmdJob(j, rel); err != nil {
return fmt.Errorf("release hotspare %s: %w", drive, err)
}
case raidPrepSetGood:
j.append(fmt.Sprintf("Drive %s is %s; converting to Unconfigured Good (set good force)...", drive, state))
prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force")
if err := streamCmdJob(j, prep); err != nil {
return fmt.Errorf("set good on %s: %w", drive, err)
}
case raidPrepNone:
if state == "" {
// Drive state unknown (storcli query failed) — attempt the
// conversion anyway; harmless on an already-UGood drive with
// force, and add vd below is the real verdict.
j.append(fmt.Sprintf("Preparing drive %s (set good, force)...", drive))
prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force")
if err := streamCmdJob(j, prep); err != nil {
j.append(fmt.Sprintf("note: set good on %s: %v (continuing)", drive, err))
}
}
}
}
j.append(fmt.Sprintf("Creating RAID 1 on controller %d with drives: %s", ctrl, driveList))
cmd := exec.CommandContext(ctx, "storcli64",
fmt.Sprintf("/c%d", ctrl),
@@ -585,7 +684,17 @@ func runRAIDLSICreateMirrorTask(ctx context.Context, j *jobState, ctrl int, driv
fmt.Sprintf("drives=%s", driveList),
"pdperarray=2",
)
return streamCmdJob(j, cmd)
if err := streamCmdJob(j, cmd); err != nil {
// A blocked add vd is often preserved cache from a dead VD
// ("controller has data in cache for offline or missing virtual
// drives"). Surface it so the log is actionable.
j.append("add vd failed; checking for preserved cache...")
pc := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d", ctrl), "show", "preservedcache")
_ = streamCmdJob(j, pc)
j.append(fmt.Sprintf("hint: if preserved cache is listed above, clear it with: storcli64 /c%d/vall delete preservedcache (invalidates cached data of dead VDs), then retry", ctrl))
return err
}
return nil
}
// parseRAIDSlot splits a storcli "EID:Slt" identifier (e.g. "252:0") into