1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
|
"""Launch one training process and retain its exit status for the local dashboard."""
import datetime
import fcntl
import json
import os
from pathlib import Path
import subprocess
import shutil
import sys
ROOT = Path(__file__).resolve().parent
RUN = Path(os.environ.get('BORIS_RUN_DIR', '/media/joseph/Media/Boris-2-0929'))
lock = open(RUN / 'training.lock', 'w')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
status = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat()}
def publish(**fields):
status.update(fields)
temp = RUN / 'process_status.json.tmp'
temp.write_text(json.dumps(status, indent=2))
temp.replace(RUN / 'process_status.json')
with open(RUN / 'train.log', 'a', buffering=1) as out:
child = subprocess.Popen([sys.executable, '-u', str(ROOT / 'train.py'),
'--config', str(RUN / 'config.json'), *sys.argv[1:]],
cwd=ROOT, stdout=out, stderr=subprocess.STDOUT)
(ROOT / 'logs/train.pid').write_text(str(child.pid))
publish(state='running', pid=child.pid)
code = child.wait()
publish(state='completed' if code == 0 else 'failed', exit_code=code,
stopped_at=datetime.datetime.now(datetime.timezone.utc).isoformat())
if shutil.which('notify-send'):
subprocess.run(['notify-send', '--urgency=critical', f'{RUN.name} stopped',
f'Training exited with code {code}. See {RUN}/train.log'], check=False)
sys.exit(code)
|