mchus/logpile
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases 13 Wiki Activity

feat(collect): remove power-on/off, add skip-hung for Redfish collection

Browse Source 
Remove power-on and power-off functionality from the Redfish collector;
keep host power-state detection and show a warning in the UI when the
host is powered off before collection starts.

Add a "Пропустить зависшие" (skip hung) button that lets the user abort
stuck Redfish collection phases without losing already-collected data.
Introduces a two-level context model in Collect(): the outer job context
covers the full lifecycle including replay; an inner collectCtx covers
snapshot, prefetch, and plan-B phases only. Closing the skipCh cancels
collectCtx immediately — aborts all in-flight HTTP requests and exits
plan-B loops — then replay runs on whatever rawTree was collected.

Signal path: UI → POST /api/collect/{id}/skip → JobManager.SkipJob()
→ close(skipCh) → goroutine in Collect() → cancelCollect().

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-04-10 13:12:38 +03:00
parent 7e9af89c46
commit 9df13327aa
14 changed files with 240 additions and 771 deletions
Download Patch File Download Diff File Expand all files Collapse all files

389
internal/collector/redfish.go
View File

@@ -112,12 +112,11 @@ func (c *RedfishConnector) Probe(ctx context.Context, req Request) (*ProbeResult
}
powerState := redfishSystemPowerState(systemDoc)
return &ProbeResult{
Reachable: true,
Protocol: "redfish",
HostPowerState: powerState,
HostPoweredOn: isRedfishHostPoweredOn(powerState),
PowerControlAvailable: redfishResetActionTarget(systemDoc) != "",
SystemPath: primarySystem,
Reachable: true,
Protocol: "redfish",
HostPowerState: powerState,
HostPoweredOn: isRedfishHostPoweredOn(powerState),
SystemPath: primarySystem,
}, nil
}
@@ -160,17 +159,6 @@ func (c *RedfishConnector) Collect(ctx context.Context, req Request, emit Progre
systemPaths := c.discoverMemberPaths(discoveryCtx, snapshotClient, req, baseURL, "/redfish/v1/Systems", "/redfish/v1/Systems/1")
primarySystem := firstNonEmptyPath(systemPaths, "/redfish/v1/Systems/1")
if primarySystem != "" {
c.ensureHostPowerForCollection(ctx, snapshotClient, req, baseURL, primarySystem, emit)
}
defer func() {
if primarySystem == "" || !req.StopHostAfterCollect {
return
}
shutdownCtx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
c.restoreHostPowerAfterCollection(shutdownCtx, snapshotClient, req, baseURL, primarySystem, emit)
}()
chassisPaths := c.discoverMemberPaths(discoveryCtx, snapshotClient, req, baseURL, "/redfish/v1/Chassis", "/redfish/v1/Chassis/1")
managerPaths := c.discoverMemberPaths(discoveryCtx, snapshotClient, req, baseURL, "/redfish/v1/Managers", "/redfish/v1/Managers/1")
primaryChassis := firstNonEmptyPath(chassisPaths, "/redfish/v1/Chassis/1")
@@ -269,12 +257,35 @@ func (c *RedfishConnector) Collect(ctx context.Context, req Request, emit Progre
emit(Progress{Status: "running", Progress: 80, Message: "Redfish: подготовка расширенного snapshot...", CurrentPhase: "snapshot", ETASeconds: acquisitionPlan.Tuning.ETABaseline.SnapshotSeconds})
emit(Progress{Status: "running", Progress: 90, Message: "Redfish: сбор расширенного snapshot...", CurrentPhase: "snapshot", ETASeconds: acquisitionPlan.Tuning.ETABaseline.SnapshotSeconds})
}
// collectCtx covers all data-fetching phases (snapshot, prefetch, plan-B).
// Cancelling it via the skip signal aborts only the collection phases while
// leaving the replay phase intact so results from already-fetched data are preserved.
collectCtx, cancelCollect := context.WithCancel(ctx)
defer cancelCollect()
if req.SkipHungCh != nil {
go func() {
select {
case <-req.SkipHungCh:
if emit != nil {
emit(Progress{
Status: "running",
Progress: 97,
Message: "Redfish: пропуск зависших запросов, анализ уже собранных данных...",
})
}
log.Printf("redfish: skip-hung triggered, cancelling collection phases")
cancelCollect()
case <-ctx.Done():
}
}()
}
c.debugSnapshotf("snapshot crawl start host=%s port=%d", req.Host, req.Port)
rawTree, fetchErrors, postProbeMetrics, snapshotTimingSummary := c.collectRawRedfishTree(withRedfishTelemetryPhase(ctx, "snapshot"), snapshotClient, req, baseURL, seedPaths, acquisitionPlan.Tuning, emit)
rawTree, fetchErrors, postProbeMetrics, snapshotTimingSummary := c.collectRawRedfishTree(withRedfishTelemetryPhase(collectCtx, "snapshot"), snapshotClient, req, baseURL, seedPaths, acquisitionPlan.Tuning, emit)
c.debugSnapshotf("snapshot crawl done docs=%d", len(rawTree))
fetchErrMap := redfishFetchErrorListToMap(fetchErrors)
prefetchedCritical, prefetchMetrics := c.prefetchCriticalRedfishDocs(withRedfishTelemetryPhase(ctx, "prefetch"), prefetchClient, req, baseURL, criticalPaths, rawTree, fetchErrMap, acquisitionPlan.Tuning, emit)
prefetchedCritical, prefetchMetrics := c.prefetchCriticalRedfishDocs(withRedfishTelemetryPhase(collectCtx, "prefetch"), prefetchClient, req, baseURL, criticalPaths, rawTree, fetchErrMap, acquisitionPlan.Tuning, emit)
for p, doc := range prefetchedCritical {
if _, exists := rawTree[p]; exists {
continue
@@ -295,10 +306,10 @@ func (c *RedfishConnector) Collect(ctx context.Context, req Request, emit Progre
prefetchMetrics.Duration.Round(time.Millisecond),
firstNonEmpty(prefetchMetrics.SkipReason, "-"),
)
if recoveredN := c.recoverCriticalRedfishDocsPlanB(withRedfishTelemetryPhase(ctx, "critical_plan_b"), criticalClient, req, baseURL, criticalPaths, rawTree, fetchErrMap, acquisitionPlan.Tuning, emit); recoveredN > 0 {
if recoveredN := c.recoverCriticalRedfishDocsPlanB(withRedfishTelemetryPhase(collectCtx, "critical_plan_b"), criticalClient, req, baseURL, criticalPaths, rawTree, fetchErrMap, acquisitionPlan.Tuning, emit); recoveredN > 0 {
c.debugSnapshotf("critical plan-b recovered docs=%d", recoveredN)
}
if recoveredN := c.recoverProfilePlanBDocs(withRedfishTelemetryPhase(ctx, "profile_plan_b"), criticalClient, req, baseURL, acquisitionPlan, rawTree, emit); recoveredN > 0 {
if recoveredN := c.recoverProfilePlanBDocs(withRedfishTelemetryPhase(collectCtx, "profile_plan_b"), criticalClient, req, baseURL, acquisitionPlan, rawTree, emit); recoveredN > 0 {
c.debugSnapshotf("profile plan-b recovered docs=%d", recoveredN)
}
// Hide transient fetch errors for endpoints that were eventually recovered into rawTree.
@@ -485,230 +496,6 @@ func (c *RedfishConnector) Collect(ctx context.Context, req Request, emit Progre
return result, nil
}
func (c *RedfishConnector) ensureHostPowerForCollection(ctx context.Context, client *http.Client, req Request, baseURL, systemPath string, emit ProgressFn) (hostOn bool, poweredOnByCollector bool) {
systemDoc, err := c.getJSON(ctx, client, req, baseURL, systemPath)
if err != nil {
if emit != nil {
emit(Progress{Status: "running", Progress: 18, Message: "Redfish: не удалось проверить PowerState host, сбор продолжается без power-control"})
}
return false, false
}
powerState := redfishSystemPowerState(systemDoc)
if isRedfishHostPoweredOn(powerState) {
if emit != nil {
emit(Progress{Status: "running", Progress: 18, Message: fmt.Sprintf("Redfish: host включен (%s)", firstNonEmpty(powerState, "On"))})
}
return true, false
}
if emit != nil {
emit(Progress{Status: "running", Progress: 18, Message: fmt.Sprintf("Redfish: host выключен (%s)", firstNonEmpty(powerState, "Off"))})
}
if !req.PowerOnIfHostOff {
if emit != nil {
emit(Progress{Status: "running", Progress: 19, Message: "Redfish: включение host не запрошено, сбор продолжается на выключенном host"})
}
return false, false
}
// Invalidate all inventory CRC groups before powering on so the BMC accepts
// fresh inventory from the host after boot. Best-effort: failure is logged but
// does not block power-on.
c.invalidateRedfishInventory(ctx, client, req, baseURL, systemPath, emit)
resetTarget := redfishResetActionTarget(systemDoc)
resetType := redfishPickResetType(systemDoc, "On", "ForceOn")
if resetTarget == "" || resetType == "" {
if emit != nil {
emit(Progress{Status: "running", Progress: 19, Message: "Redfish: action ComputerSystem.Reset недоступен, сбор продолжается на выключенном host"})
}
return false, false
}
waitWindows := []time.Duration{5 * time.Second, 10 * time.Second, 30 * time.Second}
for i, waitFor := range waitWindows {
if emit != nil {
emit(Progress{Status: "running", Progress: 19, Message: fmt.Sprintf("Redfish: попытка включения host (%d/%d), ожидание %s", i+1, len(waitWindows), waitFor)})
}
if err := c.postJSON(ctx, client, req, baseURL, resetTarget, map[string]any{"ResetType": resetType}); err != nil {
if emit != nil {
emit(Progress{Status: "running", Progress: 19, Message: fmt.Sprintf("Redfish: включение host не удалось (%v)", err)})
}
continue
}
if c.waitForHostPowerState(ctx, client, req, baseURL, systemPath, true, waitFor) {
if !c.waitForStablePoweredOnHost(ctx, client, req, baseURL, systemPath, emit) {
if emit != nil {
emit(Progress{Status: "running", Progress: 20, Message: "Redfish: host включился, но не подтвердил стабильное состояние; сбор продолжается на выключенном host"})
}
return false, false
}
if emit != nil {
emit(Progress{Status: "running", Progress: 20, Message: "Redfish: host успешно включен и стабилен перед сбором"})
}
return true, true
}
if emit != nil {
emit(Progress{Status: "running", Progress: 20, Message: fmt.Sprintf("Redfish: host не включился за %s", waitFor)})
}
}
if emit != nil {
emit(Progress{Status: "running", Progress: 20, Message: "Redfish: host не удалось включить, сбор продолжается на выключенном host"})
}
return false, false
}
func (c *RedfishConnector) waitForStablePoweredOnHost(ctx context.Context, client *http.Client, req Request, baseURL, systemPath string, emit ProgressFn) bool {
stabilizationDelay := redfishPowerOnStabilizationDelay()
if stabilizationDelay > 0 {
if emit != nil {
emit(Progress{
Status: "running",
Progress: 20,
Message: fmt.Sprintf("Redfish: host включен, ожидание стабилизации %s перед началом сбора", stabilizationDelay),
})
}
timer := time.NewTimer(stabilizationDelay)
select {
case <-ctx.Done():
timer.Stop()
return false
case <-timer.C:
timer.Stop()
}
}
if emit != nil {
emit(Progress{
Status: "running",
Progress: 20,
Message: "Redfish: повторная проверка PowerState после стабилизации host",
})
}
if !c.waitForHostPowerState(ctx, client, req, baseURL, systemPath, true, 5*time.Second) {
return false
}
// After the initial stabilization wait, the BMC may still be populating its
// hardware inventory (PCIeDevices, memory summary). Poll readiness with
// increasing back-off (default: +60s, +120s), then warn and proceed.
readinessWaits := redfishBMCReadinessWaits()
for attempt, extraWait := range readinessWaits {
ready, reason := c.isBMCInventoryReady(ctx, client, req, baseURL, systemPath)
if ready {
if emit != nil {
emit(Progress{
Status: "running",
Progress: 20,
Message: fmt.Sprintf("Redfish: BMC готов (%s)", reason),
})
}
return true
}
if emit != nil {
emit(Progress{
Status: "running",
Progress: 20,
Message: fmt.Sprintf("Redfish: BMC не готов (%s), ожидание %s (попытка %d/%d)", reason, extraWait, attempt+1, len(readinessWaits)),
})
}
timer := time.NewTimer(extraWait)
select {
case <-ctx.Done():
timer.Stop()
return false
case <-timer.C:
timer.Stop()
}
if emit != nil {
emit(Progress{
Status: "running",
Progress: 20,
Message: fmt.Sprintf("Redfish: повторная проверка готовности BMC (%d/%d)...", attempt+1, len(readinessWaits)),
})
}
}
ready, reason := c.isBMCInventoryReady(ctx, client, req, baseURL, systemPath)
if !ready {
if emit != nil {
emit(Progress{
Status: "running",
Progress: 20,
Message: fmt.Sprintf("Redfish: WARNING — BMC не подтвердил готовность (%s), сбор может быть неполным", reason),
})
}
} else if emit != nil {
emit(Progress{
Status: "running",
Progress: 20,
Message: fmt.Sprintf("Redfish: BMC готов (%s)", reason),
})
}
return true
}
// isBMCInventoryReady checks whether the BMC has finished populating its
// hardware inventory after a power-on. Returns (ready, reason).
// It considers the BMC ready if either the system memory summary reports
// a non-zero total or the PCIeDevices collection is non-empty.
func (c *RedfishConnector) isBMCInventoryReady(ctx context.Context, client *http.Client, req Request, baseURL, systemPath string) (bool, string) {
systemDoc, err := c.getJSON(ctx, client, req, baseURL, systemPath)
if err != nil {
return false, "не удалось прочитать System"
}
if summary, ok := systemDoc["MemorySummary"].(map[string]interface{}); ok {
if asFloat(summary["TotalSystemMemoryGiB"]) > 0 {
return true, "MemorySummary заполнен"
}
}
pcieDoc, err := c.getJSON(ctx, client, req, baseURL, joinPath(systemPath, "/PCIeDevices"))
if err == nil {
if asInt(pcieDoc["Members@odata.count"]) > 0 {
return true, "PCIeDevices не пуст"
}
if members, ok := pcieDoc["Members"].([]interface{}); ok && len(members) > 0 {
return true, "PCIeDevices не пуст"
}
}
return false, "MemorySummary=0, PCIeDevices пуст"
}
func (c *RedfishConnector) restoreHostPowerAfterCollection(ctx context.Context, client *http.Client, req Request, baseURL, systemPath string, emit ProgressFn) {
systemDoc, err := c.getJSON(ctx, client, req, baseURL, systemPath)
if err != nil {
if emit != nil {
emit(Progress{Status: "running", Progress: 100, Message: "Redfish: не удалось повторно прочитать system перед выключением host"})
}
return
}
resetTarget := redfishResetActionTarget(systemDoc)
resetType := redfishPickResetType(systemDoc, "GracefulShutdown", "ForceOff", "PushPowerButton")
if resetTarget == "" || resetType == "" {
if emit != nil {
emit(Progress{Status: "running", Progress: 100, Message: "Redfish: выключение host после сбора недоступно"})
}
return
}
if emit != nil {
emit(Progress{Status: "running", Progress: 100, Message: "Redfish: выключаем host после завершения сбора"})
}
if err := c.postJSON(ctx, client, req, baseURL, resetTarget, map[string]any{"ResetType": resetType}); err != nil {
if emit != nil {
emit(Progress{Status: "running", Progress: 100, Message: fmt.Sprintf("Redfish: не удалось выключить host после сбора (%v)", err)})
}
return
}
if c.waitForHostPowerState(ctx, client, req, baseURL, systemPath, false, 20*time.Second) {
if emit != nil {
emit(Progress{Status: "running", Progress: 100, Message: "Redfish: host выключен после завершения сбора"})
}
return
}
if emit != nil {
emit(Progress{Status: "running", Progress: 100, Message: "Redfish: не удалось подтвердить выключение host после сбора"})
}
}
// collectDebugPayloads fetches vendor-specific diagnostic endpoints on a best-effort basis.
// Results are stored in rawPayloads["redfish_debug_payloads"] and exported with the bundle.
@@ -724,49 +511,6 @@ func (c *RedfishConnector) collectDebugPayloads(ctx context.Context, client *htt
return out
}
// invalidateRedfishInventory POSTs to the AMI/MSI InventoryCrc endpoint to zero out
// all known CRC groups before a host power-on. This causes the BMC to accept fresh
// inventory from the host after boot, preventing stale inventory (ghost GPUs, wrong
// BIOS version, etc.) from persisting across hardware changes.
// Best-effort: any error is logged and the call silently returns.
func (c *RedfishConnector) invalidateRedfishInventory(ctx context.Context, client *http.Client, req Request, baseURL, systemPath string, emit ProgressFn) {
crcPath := joinPath(systemPath, "/Oem/Ami/Inventory/Crc")
body := map[string]any{
"GroupCrcList": []map[string]any{
{"CPU": 0},
{"DIMM": 0},
{"PCIE": 0},
},
}
if err := c.postJSON(ctx, client, req, baseURL, crcPath, body); err != nil {
log.Printf("redfish: inventory invalidation skipped (not AMI/MSI or endpoint unavailable): %v", err)
return
}
log.Printf("redfish: inventory CRC groups invalidated at %s before host power-on", crcPath)
if emit != nil {
emit(Progress{Status: "running", Progress: 19, Message: "Redfish: инвентарь BMC инвалидирован перед включением host (все CRC группы сброшены)"})
}
}
func (c *RedfishConnector) waitForHostPowerState(ctx context.Context, client *http.Client, req Request, baseURL, systemPath string, wantOn bool, timeout time.Duration) bool {
deadline := time.Now().Add(timeout)
for {
systemDoc, err := c.getJSON(ctx, client, req, baseURL, systemPath)
if err == nil {
if isRedfishHostPoweredOn(redfishSystemPowerState(systemDoc)) == wantOn {
return true
}
}
if time.Now().After(deadline) {
return false
}
select {
case <-ctx.Done():
return false
case <-time.After(1 * time.Second):
}
}
}
func firstNonEmptyPath(paths []string, fallback string) string {
for _, p := range paths {
@@ -799,48 +543,6 @@ func redfishSystemPowerState(systemDoc map[string]interface{}) string {
return ""
}
func redfishResetActionTarget(systemDoc map[string]interface{}) string {
if systemDoc == nil {
return ""
}
actions, _ := systemDoc["Actions"].(map[string]interface{})
reset, _ := actions["#ComputerSystem.Reset"].(map[string]interface{})
target := strings.TrimSpace(asString(reset["target"]))
if target != "" {
return target
}
odataID := strings.TrimSpace(asString(systemDoc["@odata.id"]))
if odataID == "" {
return ""
}
return joinPath(odataID, "/Actions/ComputerSystem.Reset")
}
func redfishPickResetType(systemDoc map[string]interface{}, preferred ...string) string {
actions, _ := systemDoc["Actions"].(map[string]interface{})
reset, _ := actions["#ComputerSystem.Reset"].(map[string]interface{})
allowedRaw, _ := reset["ResetType@Redfish.AllowableValues"].([]interface{})
if len(allowedRaw) == 0 {
if len(preferred) > 0 {
return preferred[0]
}
return ""
}
allowed := make([]string, 0, len(allowedRaw))
for _, item := range allowedRaw {
if v := strings.TrimSpace(asString(item)); v != "" {
allowed = append(allowed, v)
}
}
for _, want := range preferred {
for _, have := range allowed {
if strings.EqualFold(want, have) {
return have
}
}
}
return ""
}
func (c *RedfishConnector) postJSON(ctx context.Context, client *http.Client, req Request, baseURL, resourcePath string, payload map[string]any) error {
body, err := json.Marshal(payload)
@@ -2597,33 +2299,6 @@ func redfishCriticalSlowGap() time.Duration {
return 1200 * time.Millisecond
}
func redfishPowerOnStabilizationDelay() time.Duration {
if v := strings.TrimSpace(os.Getenv("LOGPILE_REDFISH_POWERON_STABILIZATION")); v != "" {
if d, err := time.ParseDuration(v); err == nil && d >= 0 {
return d
}
}
return 60 * time.Second
}
// redfishBMCReadinessWaits returns the extra wait durations used when polling
// BMC inventory readiness after power-on. Defaults: [60s, 120s].
// Override with LOGPILE_REDFISH_BMC_READY_WAITS (comma-separated durations,
// e.g. "60s,120s").
func redfishBMCReadinessWaits() []time.Duration {
if v := strings.TrimSpace(os.Getenv("LOGPILE_REDFISH_BMC_READY_WAITS")); v != "" {
var out []time.Duration
for _, part := range strings.Split(v, ",") {
if d, err := time.ParseDuration(strings.TrimSpace(part)); err == nil && d >= 0 {
out = append(out, d)
}
}
if len(out) > 0 {
return out
}
}
return []time.Duration{60 * time.Second, 120 * time.Second}
}
func redfishSnapshotMemoryRequestTimeout() time.Duration {
if v := strings.TrimSpace(os.Getenv("LOGPILE_REDFISH_MEMORY_TIMEOUT")); v != "" {

222
internal/collector/redfish_test.go
View File

@@ -265,9 +265,6 @@ func TestRedfishConnectorProbe(t *testing.T) {
if got.HostPowerState != "Off" {
t.Fatalf("expected power state Off, got %q", got.HostPowerState)
}
if !got.PowerControlAvailable {
t.Fatalf("expected power control available")
}
}
func TestRedfishConnectorProbe_FallsBackToPowerSummary(t *testing.T) {
@@ -330,225 +327,6 @@ func TestRedfishConnectorProbe_FallsBackToPowerSummary(t *testing.T) {
if got.HostPowerState != "On" {
t.Fatalf("expected power state On, got %q", got.HostPowerState)
}
if !got.PowerControlAvailable {
t.Fatalf("expected power control available")
}
}
func TestEnsureHostPowerForCollection_WaitsForStablePowerOn(t *testing.T) {
t.Setenv("LOGPILE_REDFISH_POWERON_STABILIZATION", "1ms")
t.Setenv("LOGPILE_REDFISH_BMC_READY_WAITS", "1ms,1ms")
powerState := "Off"
resetCalls := 0
mux := http.NewServeMux()
mux.HandleFunc("/redfish/v1/Systems/1", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]interface{}{
"@odata.id": "/redfish/v1/Systems/1",
"PowerState": powerState,
"MemorySummary": map[string]interface{}{
"TotalSystemMemoryGiB": 128,
},
"Actions": map[string]interface{}{
"#ComputerSystem.Reset": map[string]interface{}{
"target": "/redfish/v1/Systems/1/Actions/ComputerSystem.Reset",
"ResetType@Redfish.AllowableValues": []interface{}{"On"},
},
},
})
})
mux.HandleFunc("/redfish/v1/Systems/1/Actions/ComputerSystem.Reset", func(w http.ResponseWriter, r *http.Request) {
resetCalls++
powerState = "On"
w.WriteHeader(http.StatusOK)
})
ts := httptest.NewTLSServer(mux)
defer ts.Close()
u, err := url.Parse(ts.URL)
if err != nil {
t.Fatalf("parse server url: %v", err)
}
port := 443
if u.Port() != "" {
fmt.Sscanf(u.Port(), "%d", &port)
}
c := NewRedfishConnector()
hostOn, changed := c.ensureHostPowerForCollection(context.Background(), c.httpClientWithTimeout(Request{TLSMode: "insecure"}, 5*time.Second), Request{
Host: u.Hostname(),
Protocol: "redfish",
Port: port,
Username: "admin",
AuthType: "password",
Password: "secret",
TLSMode: "insecure",
PowerOnIfHostOff: true,
}, ts.URL, "/redfish/v1/Systems/1", nil)
if !hostOn || !changed {
t.Fatalf("expected stable power-on result, got hostOn=%v changed=%v", hostOn, changed)
}
if resetCalls != 1 {
t.Fatalf("expected one reset call, got %d", resetCalls)
}
}
func TestEnsureHostPowerForCollection_FailsIfHostDoesNotStayOnAfterStabilization(t *testing.T) {
t.Setenv("LOGPILE_REDFISH_POWERON_STABILIZATION", "1ms")
t.Setenv("LOGPILE_REDFISH_BMC_READY_WAITS", "1ms,1ms")
powerState := "Off"
mux := http.NewServeMux()
mux.HandleFunc("/redfish/v1/Systems/1", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
current := powerState
if powerState == "On" {
powerState = "Off"
}
_ = json.NewEncoder(w).Encode(map[string]interface{}{
"@odata.id": "/redfish/v1/Systems/1",
"PowerState": current,
"Actions": map[string]interface{}{
"#ComputerSystem.Reset": map[string]interface{}{
"target": "/redfish/v1/Systems/1/Actions/ComputerSystem.Reset",
"ResetType@Redfish.AllowableValues": []interface{}{"On"},
},
},
})
})
mux.HandleFunc("/redfish/v1/Systems/1/Actions/ComputerSystem.Reset", func(w http.ResponseWriter, r *http.Request) {
powerState = "On"
w.WriteHeader(http.StatusOK)
})
ts := httptest.NewTLSServer(mux)
defer ts.Close()
u, err := url.Parse(ts.URL)
if err != nil {
t.Fatalf("parse server url: %v", err)
}
port := 443
if u.Port() != "" {
fmt.Sscanf(u.Port(), "%d", &port)
}
c := NewRedfishConnector()
hostOn, changed := c.ensureHostPowerForCollection(context.Background(), c.httpClientWithTimeout(Request{TLSMode: "insecure"}, 5*time.Second), Request{
Host: u.Hostname(),
Protocol: "redfish",
Port: port,
Username: "admin",
AuthType: "password",
Password: "secret",
TLSMode: "insecure",
PowerOnIfHostOff: true,
}, ts.URL, "/redfish/v1/Systems/1", nil)
if hostOn || changed {
t.Fatalf("expected unstable power-on result to fail, got hostOn=%v changed=%v", hostOn, changed)
}
}
func TestEnsureHostPowerForCollection_UsesPowerSummaryState(t *testing.T) {
t.Setenv("LOGPILE_REDFISH_POWERON_STABILIZATION", "1ms")
t.Setenv("LOGPILE_REDFISH_BMC_READY_WAITS", "1ms,1ms")
powerState := "On"
mux := http.NewServeMux()
mux.HandleFunc("/redfish/v1/Systems/1", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]interface{}{
"@odata.id": "/redfish/v1/Systems/1",
"PowerSummary": map[string]interface{}{
"PowerState": powerState,
},
"MemorySummary": map[string]interface{}{
"TotalSystemMemoryGiB": 128,
},
"Actions": map[string]interface{}{
"#ComputerSystem.Reset": map[string]interface{}{
"target": "/redfish/v1/Systems/1/Actions/ComputerSystem.Reset",
"ResetType@Redfish.AllowableValues": []interface{}{"On"},
},
},
})
})
ts := httptest.NewTLSServer(mux)
defer ts.Close()
u, err := url.Parse(ts.URL)
if err != nil {
t.Fatalf("parse server url: %v", err)
}
port := 443
if u.Port() != "" {
fmt.Sscanf(u.Port(), "%d", &port)
}
c := NewRedfishConnector()
hostOn, changed := c.ensureHostPowerForCollection(context.Background(), c.httpClientWithTimeout(Request{TLSMode: "insecure"}, 5*time.Second), Request{
Host: u.Hostname(),
Protocol: "redfish",
Port: port,
Username: "admin",
AuthType: "password",
Password: "secret",
TLSMode: "insecure",
PowerOnIfHostOff: true,
}, ts.URL, "/redfish/v1/Systems/1", nil)
if !hostOn || changed {
t.Fatalf("expected already-on host from PowerSummary, got hostOn=%v changed=%v", hostOn, changed)
}
}
func TestWaitForHostPowerState_UsesPowerSummaryState(t *testing.T) {
powerState := "Off"
mux := http.NewServeMux()
mux.HandleFunc("/redfish/v1/Systems/1", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
current := powerState
if powerState == "Off" {
powerState = "On"
}
_ = json.NewEncoder(w).Encode(map[string]interface{}{
"@odata.id": "/redfish/v1/Systems/1",
"PowerSummary": map[string]interface{}{
"PowerState": current,
},
})
})
ts := httptest.NewTLSServer(mux)
defer ts.Close()
u, err := url.Parse(ts.URL)
if err != nil {
t.Fatalf("parse server url: %v", err)
}
port := 443
if u.Port() != "" {
fmt.Sscanf(u.Port(), "%d", &port)
}
c := NewRedfishConnector()
ok := c.waitForHostPowerState(context.Background(), c.httpClientWithTimeout(Request{TLSMode: "insecure"}, 5*time.Second), Request{
Host: u.Hostname(),
Protocol: "redfish",
Port: port,
Username: "admin",
AuthType: "password",
Password: "secret",
TLSMode: "insecure",
}, ts.URL, "/redfish/v1/Systems/1", true, 3*time.Second)
if !ok {
t.Fatalf("expected waitForHostPowerState to use PowerSummary")
}
}
func TestParsePCIeDeviceSlot_FromNestedRedfishSlotLocation(t *testing.T) {

12
internal/collector/types.go
View File

@@ -15,9 +15,8 @@ type Request struct {
Password string
Token string
TLSMode string
PowerOnIfHostOff bool
StopHostAfterCollect bool
DebugPayloads bool
DebugPayloads bool
SkipHungCh <-chan struct{}
}
type Progress struct {
@@ -65,10 +64,9 @@ type PhaseTelemetry struct {
type ProbeResult struct {
Reachable bool
Protocol string
HostPowerState string
HostPoweredOn bool
PowerControlAvailable bool
SystemPath string
HostPowerState string
HostPoweredOn bool
SystemPath string
}
type Connector interface {

4
internal/server/collect_handlers_test.go
View File

@@ -24,6 +24,7 @@ func newCollectTestServer() (*Server, *httptest.Server) {
mux.HandleFunc("POST /api/collect", s.handleCollectStart)
mux.HandleFunc("GET /api/collect/{id}", s.handleCollectStatus)
mux.HandleFunc("POST /api/collect/{id}/cancel", s.handleCollectCancel)
mux.HandleFunc("POST /api/collect/{id}/skip", s.handleCollectSkip)
return s, httptest.NewServer(mux)
}
@@ -65,9 +66,6 @@ func TestCollectProbe(t *testing.T) {
if payload.HostPowerState != "Off" {
t.Fatalf("expected host power state Off, got %q", payload.HostPowerState)
}
if !payload.PowerControlAvailable {
t.Fatalf("expected power control to be available")
}
}
func TestCollectLifecycleToTerminal(t *testing.T) {

11
internal/server/collect_test_helpers_test.go
View File

@@ -26,12 +26,11 @@ func (c *mockConnector) Probe(ctx context.Context, req collector.Request) (*coll
hostPoweredOn = false
}
return &collector.ProbeResult{
Reachable: true,
Protocol: c.protocol,
HostPowerState: map[bool]string{true: "On", false: "Off"}[hostPoweredOn],
HostPoweredOn: hostPoweredOn,
PowerControlAvailable: true,
SystemPath: "/redfish/v1/Systems/1",
Reachable: true,
Protocol: c.protocol,
HostPowerState: map[bool]string{true: "On", false: "Off"}[hostPoweredOn],
HostPoweredOn: hostPoweredOn,
SystemPath: "/redfish/v1/Systems/1",
}, nil
}

14
internal/server/collect_types.go
View File

@@ -19,18 +19,15 @@ type CollectRequest struct {
Password string `json:"password,omitempty"`
Token string `json:"token,omitempty"`
TLSMode string `json:"tls_mode"`
PowerOnIfHostOff bool `json:"power_on_if_host_off,omitempty"`
StopHostAfterCollect bool `json:"stop_host_after_collect,omitempty"`
DebugPayloads bool `json:"debug_payloads,omitempty"`
DebugPayloads bool `json:"debug_payloads,omitempty"`
}
type CollectProbeResponse struct {
Reachable bool `json:"reachable"`
Protocol string `json:"protocol,omitempty"`
HostPowerState string `json:"host_power_state,omitempty"`
HostPoweredOn bool `json:"host_powered_on"`
PowerControlAvailable bool `json:"power_control_available"`
Message string `json:"message,omitempty"`
HostPowerState string `json:"host_power_state,omitempty"`
HostPoweredOn bool `json:"host_powered_on"`
Message string `json:"message,omitempty"`
}
type CollectJobResponse struct {
@@ -78,7 +75,8 @@ type Job struct {
CreatedAt time.Time
UpdatedAt time.Time
RequestMeta CollectRequestMeta
cancel func()
cancel func()
skipFn func()
}
type CollectModuleStatus struct {

54
internal/server/handlers.go
View File

@@ -18,6 +18,7 @@ import (
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
@@ -1674,34 +1675,28 @@ func (s *Server) handleCollectProbe(w http.ResponseWriter, r *http.Request) {
message := "Связь с BMC установлена"
if result != nil {
switch {
case !result.HostPoweredOn && result.PowerControlAvailable:
message = "Связь с BMC установлена, host выключен. Можно включить перед сбором."
case !result.HostPoweredOn:
message = "Связь с BMC установлена, host выключен."
default:
message = "Связь с BMC установлена, host включен."
if result.HostPoweredOn {
message = "Связь с BMC установлена, host включён."
} else {
message = "Связь с BMC установлена, host выключен. Данные инвентаря могут быть неполными."
}
}
hostPowerState := ""
hostPoweredOn := false
powerControlAvailable := false
reachable := false
if result != nil {
reachable = result.Reachable
hostPowerState = strings.TrimSpace(result.HostPowerState)
hostPoweredOn = result.HostPoweredOn
powerControlAvailable = result.PowerControlAvailable
}
jsonResponse(w, CollectProbeResponse{
Reachable: reachable,
Protocol: req.Protocol,
HostPowerState: hostPowerState,
HostPoweredOn: hostPoweredOn,
PowerControlAvailable: powerControlAvailable,
Message: message,
Reachable: reachable,
Protocol: req.Protocol,
HostPowerState: hostPowerState,
HostPoweredOn: hostPoweredOn,
Message: message,
})
}
@@ -1737,6 +1732,22 @@ func (s *Server) handleCollectCancel(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, job.toStatusResponse())
}
func (s *Server) handleCollectSkip(w http.ResponseWriter, r *http.Request) {
jobID := strings.TrimSpace(r.PathValue("id"))
if !isValidCollectJobID(jobID) {
jsonError(w, "Invalid collect job id", http.StatusBadRequest)
return
}
job, ok := s.jobManager.SkipJob(jobID)
if !ok {
jsonError(w, "Collect job not found", http.StatusNotFound)
return
}
jsonResponse(w, job.toStatusResponse())
}
func (s *Server) startCollectionJob(jobID string, req CollectRequest) {
ctx, cancel := context.WithCancel(context.Background())
if attached := s.jobManager.AttachJobCancel(jobID, cancel); !attached {
@@ -1744,6 +1755,11 @@ func (s *Server) startCollectionJob(jobID string, req CollectRequest) {
return
}
skipCh := make(chan struct{})
var skipOnce sync.Once
skipFn := func() { skipOnce.Do(func() { close(skipCh) }) }
s.jobManager.AttachJobSkip(jobID, skipFn)
go func() {
connector, ok := s.getCollector(req.Protocol)
if !ok {
@@ -1811,7 +1827,9 @@ func (s *Server) startCollectionJob(jobID string, req CollectRequest) {
}
}
result, err := connector.Collect(ctx, toCollectorRequest(req), emitProgress)
collectorReq := toCollectorRequest(req)
collectorReq.SkipHungCh = skipCh
result, err := connector.Collect(ctx, collectorReq, emitProgress)
if err != nil {
if ctx.Err() != nil {
return
@@ -2035,9 +2053,7 @@ func toCollectorRequest(req CollectRequest) collector.Request {
Password: req.Password,
Token: req.Token,
TLSMode: req.TLSMode,
PowerOnIfHostOff: req.PowerOnIfHostOff,
StopHostAfterCollect: req.StopHostAfterCollect,
DebugPayloads: req.DebugPayloads,
DebugPayloads: req.DebugPayloads,
}
}

38
internal/server/job_manager.go
View File

@@ -175,6 +175,43 @@ func (m *JobManager) UpdateJobDebugInfo(id string, info *CollectDebugInfo) (*Job
return cloned, true
}
func (m *JobManager) AttachJobSkip(id string, skipFn func()) bool {
m.mu.Lock()
defer m.mu.Unlock()
job, ok := m.jobs[id]
if !ok || job == nil || isTerminalCollectStatus(job.Status) {
return false
}
job.skipFn = skipFn
return true
}
func (m *JobManager) SkipJob(id string) (*Job, bool) {
m.mu.Lock()
job, ok := m.jobs[id]
if !ok || job == nil {
m.mu.Unlock()
return nil, false
}
if isTerminalCollectStatus(job.Status) {
cloned := cloneJob(job)
m.mu.Unlock()
return cloned, true
}
skipFn := job.skipFn
job.skipFn = nil
job.UpdatedAt = time.Now().UTC()
job.Logs = append(job.Logs, formatCollectLogLine(job.UpdatedAt, "Пропуск зависших запросов по команде пользователя"))
cloned := cloneJob(job)
m.mu.Unlock()
if skipFn != nil {
skipFn()
}
return cloned, true
}
func (m *JobManager) AttachJobCancel(id string, cancelFn context.CancelFunc) bool {
m.mu.Lock()
defer m.mu.Unlock()
@@ -229,5 +266,6 @@ func cloneJob(job *Job) *Job {
cloned.CurrentPhase = job.CurrentPhase
cloned.ETASeconds = job.ETASeconds
cloned.cancel = nil
cloned.skipFn = nil
return &cloned
}

1
internal/server/server.go
View File

@@ -99,6 +99,7 @@ func (s *Server) setupRoutes() {
s.mux.HandleFunc("POST /api/collect/probe", s.handleCollectProbe)
s.mux.HandleFunc("GET /api/collect/{id}", s.handleCollectStatus)
s.mux.HandleFunc("POST /api/collect/{id}/cancel", s.handleCollectCancel)
s.mux.HandleFunc("POST /api/collect/{id}/skip", s.handleCollectSkip)
}
func (s *Server) Run() error {

1
internal/server/upload_live_smoke_test.go
View File

@@ -24,6 +24,7 @@ func newFlowTestServer() (*Server, *httptest.Server) {
mux.HandleFunc("POST /api/collect", s.handleCollectStart)
mux.HandleFunc("GET /api/collect/{id}", s.handleCollectStatus)
mux.HandleFunc("POST /api/collect/{id}/cancel", s.handleCollectCancel)
mux.HandleFunc("POST /api/collect/{id}/skip", s.handleCollectSkip)
return s, httptest.NewServer(mux)
}