diff --git a/iso/overlay/usr/share/bee/scenarios/nvbandwidth-acs-ab.json b/iso/overlay/usr/share/bee/scenarios/nvbandwidth-acs-ab.json new file mode 100644 index 0000000..46b0f86 --- /dev/null +++ b/iso/overlay/usr/share/bee/scenarios/nvbandwidth-acs-ab.json @@ -0,0 +1,29 @@ +{ + "name": "nvbandwidth-acs-ab", + "description": "Two-phase A/B for the MSI CG480-S6053 cross-socket GPU P2P hard-reboot. Phase 1 disables PCIe ACS redirect at runtime via setpci (P2P goes device-direct instead of up to the root complex) then runs nvbandwidth across all 6 GPUs — the run expected to SURVIVE. Phase 2 restores ACS to the BIOS default and repeats — the run expected to REPRODUCE the reboot. Safe phase runs first so its full blackbox reaches the USB stick before the risky phase can reset the machine.", + "timeout_sec": 1800, + "jobs": [ + { + "name": "acs-disable", + "type": "command", + "cmd": ["bash", "-c", ": > /run/bee-acs-orig; for bdf in $(lspci -D | awk '{print $1}'); do setpci -s $bdf ECAP_ACS.w >/dev/null 2>&1 || continue; v=$(setpci -s $bdf ECAP_ACS+0x6.w 2>/dev/null) || continue; echo $bdf $v >> /run/bee-acs-orig; setpci -s $bdf ECAP_ACS+0x6.w=0000 >/dev/null 2>&1 || true; done; echo ACS disabled on $(wc -l < /run/bee-acs-orig) bridges; echo -n 'bridges still ReqRedir+: '; lspci -vvv 2>/dev/null | grep -c 'ReqRedir+' || true"] + }, + { + "name": "nvbandwidth-acs-off", + "type": "command", + "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], + "gpu_indices": [0, 1, 2, 3, 4, 5] + }, + { + "name": "acs-restore", + "type": "command", + "cmd": ["bash", "-c", "test -s /run/bee-acs-orig || { echo no saved ACS state; exit 0; }; while read -r bdf v; do setpci -s $bdf ECAP_ACS+0x6.w=$v >/dev/null 2>&1 || true; done < /run/bee-acs-orig; echo ACS restored on $(wc -l < /run/bee-acs-orig) bridges; echo -n 'bridges ReqRedir+ now: '; lspci -vvv 2>/dev/null | grep -c 'ReqRedir+' || true"] + }, + { + "name": "nvbandwidth-acs-on", + "type": "command", + "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], + "gpu_indices": [0, 1, 2, 3, 4, 5] + } + ] +} diff --git a/iso/overlay/usr/share/bee/scenarios/nvbandwidth-all-gpu-power-watch.json b/iso/overlay/usr/share/bee/scenarios/nvbandwidth-all-gpu-power-watch.json deleted file mode 100644 index f359e96..0000000 --- a/iso/overlay/usr/share/bee/scenarios/nvbandwidth-all-gpu-power-watch.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "nvbandwidth-all-gpu-power-watch", - "description": "Full nvbandwidth across all GPUs at once (the crash never reproduces on a single socket alone), with IPMI sensors and GPU power/temp sampled every 2s to check for a power-delivery correlation.", - "timeout_sec": 1800, - "jobs": [ - { - "name": "ipmi-sensors", - "type": "sampler", - "interval_sec": 2, - "cmd": ["ipmitool", "sensor"] - }, - { - "name": "gpu-power", - "type": "sampler", - "interval_sec": 2, - "cmd": ["nvidia-smi", "--query-gpu=index,power.draw,temperature.gpu,clocks.sm", "--format=csv"] - }, - { - "name": "nvbandwidth-all", - "type": "command", - "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], - "gpu_indices": [0, 1, 2, 3, 4, 5] - } - ] -} diff --git a/scenarios/nvbandwidth-acs-ab.json b/scenarios/nvbandwidth-acs-ab.json new file mode 100644 index 0000000..46b0f86 --- /dev/null +++ b/scenarios/nvbandwidth-acs-ab.json @@ -0,0 +1,29 @@ +{ + "name": "nvbandwidth-acs-ab", + "description": "Two-phase A/B for the MSI CG480-S6053 cross-socket GPU P2P hard-reboot. Phase 1 disables PCIe ACS redirect at runtime via setpci (P2P goes device-direct instead of up to the root complex) then runs nvbandwidth across all 6 GPUs — the run expected to SURVIVE. Phase 2 restores ACS to the BIOS default and repeats — the run expected to REPRODUCE the reboot. Safe phase runs first so its full blackbox reaches the USB stick before the risky phase can reset the machine.", + "timeout_sec": 1800, + "jobs": [ + { + "name": "acs-disable", + "type": "command", + "cmd": ["bash", "-c", ": > /run/bee-acs-orig; for bdf in $(lspci -D | awk '{print $1}'); do setpci -s $bdf ECAP_ACS.w >/dev/null 2>&1 || continue; v=$(setpci -s $bdf ECAP_ACS+0x6.w 2>/dev/null) || continue; echo $bdf $v >> /run/bee-acs-orig; setpci -s $bdf ECAP_ACS+0x6.w=0000 >/dev/null 2>&1 || true; done; echo ACS disabled on $(wc -l < /run/bee-acs-orig) bridges; echo -n 'bridges still ReqRedir+: '; lspci -vvv 2>/dev/null | grep -c 'ReqRedir+' || true"] + }, + { + "name": "nvbandwidth-acs-off", + "type": "command", + "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], + "gpu_indices": [0, 1, 2, 3, 4, 5] + }, + { + "name": "acs-restore", + "type": "command", + "cmd": ["bash", "-c", "test -s /run/bee-acs-orig || { echo no saved ACS state; exit 0; }; while read -r bdf v; do setpci -s $bdf ECAP_ACS+0x6.w=$v >/dev/null 2>&1 || true; done < /run/bee-acs-orig; echo ACS restored on $(wc -l < /run/bee-acs-orig) bridges; echo -n 'bridges ReqRedir+ now: '; lspci -vvv 2>/dev/null | grep -c 'ReqRedir+' || true"] + }, + { + "name": "nvbandwidth-acs-on", + "type": "command", + "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], + "gpu_indices": [0, 1, 2, 3, 4, 5] + } + ] +} diff --git a/scenarios/nvbandwidth-all-gpu-power-watch.json b/scenarios/nvbandwidth-all-gpu-power-watch.json deleted file mode 100644 index f359e96..0000000 --- a/scenarios/nvbandwidth-all-gpu-power-watch.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "nvbandwidth-all-gpu-power-watch", - "description": "Full nvbandwidth across all GPUs at once (the crash never reproduces on a single socket alone), with IPMI sensors and GPU power/temp sampled every 2s to check for a power-delivery correlation.", - "timeout_sec": 1800, - "jobs": [ - { - "name": "ipmi-sensors", - "type": "sampler", - "interval_sec": 2, - "cmd": ["ipmitool", "sensor"] - }, - { - "name": "gpu-power", - "type": "sampler", - "interval_sec": 2, - "cmd": ["nvidia-smi", "--query-gpu=index,power.draw,temperature.gpu,clocks.sm", "--format=csv"] - }, - { - "name": "nvbandwidth-all", - "type": "command", - "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], - "gpu_indices": [0, 1, 2, 3, 4, 5] - } - ] -}