summaryrefslogtreecommitdiff
path: root/scripts/sample.py
blob: b19a600d325bf49f2a5d248a2c4540688c856b08 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
"""Sample text from the newest Boris-2-0929 rolling checkpoint on the 3060 (no KV cache;
recomputes the full prefix each token, fine for short samples)."""
import glob, sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
import torch, torch.nn.functional as F
import eval as E
from tokenizer import BorisTokenizer

dev = "cuda:1"
ck = sorted(glob.glob("/media/joseph/Media/Boris-2-0929/checkpoints/rolling_*.pt"))[-1]
b = torch.load(ck, map_location="cpu", weights_only=False)
tmp = "/home/joseph/boris-2-D112M-n15M/probe/_sample_export.pt"
torch.save({k: b[k] for k in ("model", "config", "step", "tokens_seen", "ngram_fingerprint")}, tmp)
del b
model, cfg, blob = E.load_export(Path(tmp), dev)
Path(tmp).unlink()
tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer")
W = model.head_weight().float()
print(f"checkpoint step {blob['step']}  tokens {blob['tokens_seen']/1e9:.2f}B\n")

prompts = sys.argv[1:] or [
    "The capital of France is",
    "Once upon a time, in a small village by the sea,",
    "Photosynthesis is the process by which",
    "To make a cup of tea, first",
    "Question: If I have 12 apples and give away 5, how many do I have left?\nAnswer:",
    "The most important invention of the 20th century was",
]
torch.manual_seed(0)
for p in prompts:
    ids = tok.encode(p)
    for _ in range(60):
        T = len(ids)
        x = torch.tensor([ids], device=dev)
        pos = torch.arange(T, device=dev)[None]
        bnd = torch.zeros(1, T, dtype=torch.bool, device=dev); bnd[0, 0] = True
        off = torch.tensor([0, T], device=dev)
        with torch.no_grad(), torch.autocast("cuda", dtype=torch.float16):
            h = model(x, pos, off, bnd, None, model._eval_theta)
        logits = F.linear(h[0, -1].float(), W) / 0.7
        top = torch.topk(logits, 40)
        nxt = int(top.indices[torch.multinomial(F.softmax(top.values, -1), 1)])
        if nxt == tok.eos_id:
            break
        ids.append(nxt)
    print("=" * 70); print(tok.decode(ids)); print()