Memo‎ > ‎

Some slurm tricks

posted Jul 19, 2019, 7:31 AM by Teng-Yok Lee   [ updated Jul 19, 2019, 7:31 AM ]

How to get the stdout of a slurm job?

function slurm_get_job_stdout()
{
    jobid=$1
    scontrol show jobid ${jobid} | grep StdOut | cut -b 11-
}

How to list the CPU/GPU resources of all nodes?

function slurm_list_resources()
{
    extra_args=$@

    # This is a workaround for scontrol to query hidden nodes.
    w_hidden=""
    for a in ${extra_args}
    do
        if [ "-a" == "$a" ]
        then
            w_hidden=$a
        fi
    done

    for h in $(sinfo ${extra_args} -N -h | cut -d " " -f 1)
    do
        n_used_gpus=$(squeue ${extra_args} -t R -h -o "%8b" -w $h | grep gpu | cut -d : -f 2 | paste -s -d + - | bc)
        scontrol_output=$(scontrol ${w_hidden} show node $h | tr ' ' '\n')
        n_gpus=$(printf "%s" "${scontrol_output}" | grep Gres | cut -d : -f 2)
        n_used_cpus=$(printf "%s" "${scontrol_output}" | grep CPUAlloc | cut -d = -f 2)
        n_cpus=$(printf "%s" "${scontrol_output}" | grep CPUTot | cut -d = -f 2)
        cpu_load=$(printf "%s" "${scontrol_output}" | grep CPULoad | cut -d = -f 2)
        printf "[%s]\tGPU: %d/%d\tCPU: %3d/%3d\tLoad: %2.2f%%\n" "$h" "${n_used_gpus}" "${n_gpus}" "${n_used_cpus}" "${n_cpus}" ${cpu_load}
    done
}


Comments