1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
|
"""Sample text from the newest Boris-2-0929 rolling checkpoint on the 3060 (no KV cache;
recomputes the full prefix each token, fine for short samples)."""
import glob, sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
import torch, torch.nn.functional as F
import eval as E
from tokenizer import BorisTokenizer
dev = "cuda:1"
ck = sorted(glob.glob("/media/joseph/Media/Boris-2-0929/checkpoints/rolling_*.pt"))[-1]
b = torch.load(ck, map_location="cpu", weights_only=False)
tmp = "/home/joseph/boris-2-D112M-n15M/probe/_sample_export.pt"
torch.save({k: b[k] for k in ("model", "config", "step", "tokens_seen", "ngram_fingerprint")}, tmp)
del b
model, cfg, blob = E.load_export(Path(tmp), dev)
Path(tmp).unlink()
tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer")
W = model.head_weight().float()
print(f"checkpoint step {blob['step']} tokens {blob['tokens_seen']/1e9:.2f}B\n")
prompts = sys.argv[1:] or [
"The capital of France is",
"Once upon a time, in a small village by the sea,",
"Photosynthesis is the process by which",
"To make a cup of tea, first",
"Question: If I have 12 apples and give away 5, how many do I have left?\nAnswer:",
"The most important invention of the 20th century was",
]
torch.manual_seed(0)
for p in prompts:
ids = tok.encode(p)
for _ in range(60):
T = len(ids)
x = torch.tensor([ids], device=dev)
pos = torch.arange(T, device=dev)[None]
bnd = torch.zeros(1, T, dtype=torch.bool, device=dev); bnd[0, 0] = True
off = torch.tensor([0, T], device=dev)
with torch.no_grad(), torch.autocast("cuda", dtype=torch.float16):
h = model(x, pos, off, bnd, None, model._eval_theta)
logits = F.linear(h[0, -1].float(), W) / 0.7
top = torch.topk(logits, 40)
nxt = int(top.indices[torch.multinomial(F.softmax(top.values, -1), 1)])
if nxt == tok.eos_id:
break
ids.append(nxt)
print("=" * 70); print(tok.decode(ids)); print()
|