{ "name": "nvbandwidth-all-gpu-power-watch", "description": "Full nvbandwidth across all GPUs at once (the crash never reproduces on a single socket alone), with IPMI sensors and GPU power/temp sampled every 2s to check for a power-delivery correlation.", "timeout_sec": 1800, "jobs": [ { "name": "ipmi-sensors", "type": "sampler", "interval_sec": 2, "cmd": ["ipmitool", "sensor"] }, { "name": "gpu-power", "type": "sampler", "interval_sec": 2, "cmd": ["nvidia-smi", "--query-gpu=index,power.draw,temperature.gpu,clocks.sm", "--format=csv"] }, { "name": "nvbandwidth-all", "type": "command", "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], "gpu_indices": [0, 1, 2, 3, 4, 5] } ] }