代码
import os
import sys
import time
cmd = 'nohup python -u train_post_2d_aut.py > output1.log &'
def gpu_info():gpu_status = os.popen('nvidia-smi | grep %').read().split('|')gpu_memory = int(gpu_status[10].split('/')[0].split('M')[0].strip()) gpu_power = int(gpu_status[9].split(' ')[-1].split('/')[0].split('W')[0].strip()) power_all, memory_all = 250, 12288 power_free = power_all - gpu_powermemory_free = memory_all - gpu_memoryreturn power_free, memory_free
def narrow_setup(interval=2):power_free, memory_free = gpu_info()i = 0while not (memory_free > 7000 and power_free > 175): power_free, memory_free = gpu_info()i = i % 10symbol = 'monitoring: ' + '>' * i + ' ' * (10 - i - 1) + '|'gpu_power_str = 'gpu power_free:%d W |' % power_freegpu_memory_str = 'gpu memory_free:%d MiB |' % memory_freesys.stdout.write('\r' + gpu_memory_str + ' ' + gpu_power_str + ' ' + symbol)sys.stdout.flush()time.sleep(interval) i += 1print('\n' + cmd)os.system(cmd)if __name__ == '__main__':narrow_setup()