"""Sample text from the newest Boris-2-0929 rolling checkpoint on the 3060 (no KV cache; recomputes the full prefix each token, fine for short samples).""" import glob, sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import torch, torch.nn.functional as F import eval as E from tokenizer import BorisTokenizer dev = "cuda:1" ck = sorted(glob.glob("/media/joseph/Media/Boris-2-0929/checkpoints/rolling_*.pt"))[-1] b = torch.load(ck, map_location="cpu", weights_only=False) tmp = "/home/joseph/boris-2-D112M-n15M/probe/_sample_export.pt" torch.save({k: b[k] for k in ("model", "config", "step", "tokens_seen", "ngram_fingerprint")}, tmp) del b model, cfg, blob = E.load_export(Path(tmp), dev) Path(tmp).unlink() tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer") W = model.head_weight().float() print(f"checkpoint step {blob['step']} tokens {blob['tokens_seen']/1e9:.2f}B\n") prompts = sys.argv[1:] or [ "The capital of France is", "Once upon a time, in a small village by the sea,", "Photosynthesis is the process by which", "To make a cup of tea, first", "Question: If I have 12 apples and give away 5, how many do I have left?\nAnswer:", "The most important invention of the 20th century was", ] torch.manual_seed(0) for p in prompts: ids = tok.encode(p) for _ in range(60): T = len(ids) x = torch.tensor([ids], device=dev) pos = torch.arange(T, device=dev)[None] bnd = torch.zeros(1, T, dtype=torch.bool, device=dev); bnd[0, 0] = True off = torch.tensor([0, T], device=dev) with torch.no_grad(), torch.autocast("cuda", dtype=torch.float16): h = model(x, pos, off, bnd, None, model._eval_theta) logits = F.linear(h[0, -1].float(), W) / 0.7 top = torch.topk(logits, 40) nxt = int(top.indices[torch.multinomial(F.softmax(top.values, -1), 1)]) if nxt == tok.eos_id: break ids.append(nxt) print("=" * 70); print(tok.decode(ids)); print()