"""Launch one training process and retain its exit status for the local dashboard.""" import datetime import fcntl import json import os from pathlib import Path import subprocess import shutil import sys ROOT = Path(__file__).resolve().parent RUN = Path(os.environ.get('BORIS_RUN_DIR', '/media/joseph/Media/Boris-2-0929')) lock = open(RUN / 'training.lock', 'w') fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) status = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat()} def publish(**fields): status.update(fields) temp = RUN / 'process_status.json.tmp' temp.write_text(json.dumps(status, indent=2)) temp.replace(RUN / 'process_status.json') with open(RUN / 'train.log', 'a', buffering=1) as out: child = subprocess.Popen([sys.executable, '-u', str(ROOT / 'train.py'), '--config', str(RUN / 'config.json'), *sys.argv[1:]], cwd=ROOT, stdout=out, stderr=subprocess.STDOUT) (ROOT / 'logs/train.pid').write_text(str(child.pid)) publish(state='running', pid=child.pid) code = child.wait() publish(state='completed' if code == 0 else 'failed', exit_code=code, stopped_at=datetime.datetime.now(datetime.timezone.utc).isoformat()) if shutil.which('notify-send'): subprocess.run(['notify-send', '--urgency=critical', f'{RUN.name} stopped', f'Training exited with code {code}. See {RUN}/train.log'], check=False) sys.exit(code)