#!/bin/bash
# Detailed GPU monitoring during job execution

echo "=== GPU STATE MONITORING ==="
echo "Monitoring: clocks, memory, temperature, utilization"
echo "Press Ctrl+C to stop"
echo ""
echo "Time      | sclk (MHz) | mclk (MHz) | Temp (C) | GPU% | VRAM Used"
echo "----------|------------|------------|----------|------|------------"

while true; do
    timestamp=$(date +%H:%M:%S)
    
    # Get GPU stats
    stats=$(rocm-smi 2>/dev/null)
    
    # Extract values
    sclk=$(echo "$stats" | grep -A1 "sclk" | tail -1 | awk '{print $1}')
    mclk=$(echo "$stats" | grep -A1 "mclk" | tail -1 | awk '{print $1}')
    temp=$(echo "$stats" | grep "Temperature" | awk '{print $3}')
    gpu_use=$(echo "$stats" | grep "GPU use" | awk '{print $4}')
    vram=$(echo "$stats" | grep "GTT use" -A1 | tail -1 | grep -oP '\d+MiB')
    
    printf "%s | %10s | %10s | %8s | %4s | %s\n" \
        "$timestamp" "$sclk" "$mclk" "$temp" "$gpu_use" "$vram"
    
    sleep 0.5
done
