summaryrefslogtreecommitdiff
path: root/supervise_run.py
diff options
context:
space:
mode:
Diffstat (limited to 'supervise_run.py')
-rw-r--r--supervise_run.py34
1 files changed, 34 insertions, 0 deletions
diff --git a/supervise_run.py b/supervise_run.py
new file mode 100644
index 0000000..731ab59
--- /dev/null
+++ b/supervise_run.py
@@ -0,0 +1,34 @@
+"""Launch one training process and retain its exit status for the local dashboard."""
+import datetime
+import fcntl
+import json
+import os
+from pathlib import Path
+import subprocess
+import shutil
+import sys
+
+ROOT = Path(__file__).resolve().parent
+RUN = Path(os.environ.get('BORIS_RUN_DIR', '/media/joseph/Media/Boris-2-0929'))
+lock = open(RUN / 'training.lock', 'w')
+fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+status = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat()}
+def publish(**fields):
+ status.update(fields)
+ temp = RUN / 'process_status.json.tmp'
+ temp.write_text(json.dumps(status, indent=2))
+ temp.replace(RUN / 'process_status.json')
+
+with open(RUN / 'train.log', 'a', buffering=1) as out:
+ child = subprocess.Popen([sys.executable, '-u', str(ROOT / 'train.py'),
+ '--config', str(RUN / 'config.json'), *sys.argv[1:]],
+ cwd=ROOT, stdout=out, stderr=subprocess.STDOUT)
+ (ROOT / 'logs/train.pid').write_text(str(child.pid))
+ publish(state='running', pid=child.pid)
+ code = child.wait()
+ publish(state='completed' if code == 0 else 'failed', exit_code=code,
+ stopped_at=datetime.datetime.now(datetime.timezone.utc).isoformat())
+ if shutil.which('notify-send'):
+ subprocess.run(['notify-send', '--urgency=critical', f'{RUN.name} stopped',
+ f'Training exited with code {code}. See {RUN}/train.log'], check=False)
+sys.exit(code)