summaryrefslogtreecommitdiff
path: root/scripts/sample.py
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/sample.py')
-rw-r--r--scripts/sample.py47
1 files changed, 47 insertions, 0 deletions
diff --git a/scripts/sample.py b/scripts/sample.py
new file mode 100644
index 0000000..b19a600
--- /dev/null
+++ b/scripts/sample.py
@@ -0,0 +1,47 @@
+"""Sample text from the newest Boris-2-0929 rolling checkpoint on the 3060 (no KV cache;
+recomputes the full prefix each token, fine for short samples)."""
+import glob, sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+import torch, torch.nn.functional as F
+import eval as E
+from tokenizer import BorisTokenizer
+
+dev = "cuda:1"
+ck = sorted(glob.glob("/media/joseph/Media/Boris-2-0929/checkpoints/rolling_*.pt"))[-1]
+b = torch.load(ck, map_location="cpu", weights_only=False)
+tmp = "/home/joseph/boris-2-D112M-n15M/probe/_sample_export.pt"
+torch.save({k: b[k] for k in ("model", "config", "step", "tokens_seen", "ngram_fingerprint")}, tmp)
+del b
+model, cfg, blob = E.load_export(Path(tmp), dev)
+Path(tmp).unlink()
+tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer")
+W = model.head_weight().float()
+print(f"checkpoint step {blob['step']} tokens {blob['tokens_seen']/1e9:.2f}B\n")
+
+prompts = sys.argv[1:] or [
+ "The capital of France is",
+ "Once upon a time, in a small village by the sea,",
+ "Photosynthesis is the process by which",
+ "To make a cup of tea, first",
+ "Question: If I have 12 apples and give away 5, how many do I have left?\nAnswer:",
+ "The most important invention of the 20th century was",
+]
+torch.manual_seed(0)
+for p in prompts:
+ ids = tok.encode(p)
+ for _ in range(60):
+ T = len(ids)
+ x = torch.tensor([ids], device=dev)
+ pos = torch.arange(T, device=dev)[None]
+ bnd = torch.zeros(1, T, dtype=torch.bool, device=dev); bnd[0, 0] = True
+ off = torch.tensor([0, T], device=dev)
+ with torch.no_grad(), torch.autocast("cuda", dtype=torch.float16):
+ h = model(x, pos, off, bnd, None, model._eval_theta)
+ logits = F.linear(h[0, -1].float(), W) / 0.7
+ top = torch.topk(logits, 40)
+ nxt = int(top.indices[torch.multinomial(F.softmax(top.values, -1), 1)])
+ if nxt == tok.eos_id:
+ break
+ ids.append(nxt)
+ print("=" * 70); print(tok.decode(ids)); print()