diff options
Diffstat (limited to 'supervise_run.py')
| -rw-r--r-- | supervise_run.py | 34 |
1 files changed, 34 insertions, 0 deletions
diff --git a/supervise_run.py b/supervise_run.py new file mode 100644 index 0000000..731ab59 --- /dev/null +++ b/supervise_run.py @@ -0,0 +1,34 @@ +"""Launch one training process and retain its exit status for the local dashboard.""" +import datetime +import fcntl +import json +import os +from pathlib import Path +import subprocess +import shutil +import sys + +ROOT = Path(__file__).resolve().parent +RUN = Path(os.environ.get('BORIS_RUN_DIR', '/media/joseph/Media/Boris-2-0929')) +lock = open(RUN / 'training.lock', 'w') +fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) +status = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat()} +def publish(**fields): + status.update(fields) + temp = RUN / 'process_status.json.tmp' + temp.write_text(json.dumps(status, indent=2)) + temp.replace(RUN / 'process_status.json') + +with open(RUN / 'train.log', 'a', buffering=1) as out: + child = subprocess.Popen([sys.executable, '-u', str(ROOT / 'train.py'), + '--config', str(RUN / 'config.json'), *sys.argv[1:]], + cwd=ROOT, stdout=out, stderr=subprocess.STDOUT) + (ROOT / 'logs/train.pid').write_text(str(child.pid)) + publish(state='running', pid=child.pid) + code = child.wait() + publish(state='completed' if code == 0 else 'failed', exit_code=code, + stopped_at=datetime.datetime.now(datetime.timezone.utc).isoformat()) + if shutil.which('notify-send'): + subprocess.run(['notify-send', '--urgency=critical', f'{RUN.name} stopped', + f'Training exited with code {code}. See {RUN}/train.log'], check=False) +sys.exit(code) |
