CUDA 위의 같은 토큰

#!/usr/bin/env python3
"""The same token loop on whatever accelerator this machine has. The one line that
differs between a Mac and a CUDA box is the .to(device): on a Mac it is a copy inside
one pool; on a discrete GPU it is the PCIe crossing, once, at load. Everything after it
is bytes per token against that device's own memory bandwidth."""
import time, torch
from transformers import AutoModelForCausalLM, AutoTokenizer

MODEL = "unsloth/Llama-3.2-1B-Instruct"
dev = "cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")
sync = {"cuda": lambda: torch.cuda.synchronize(), "mps": lambda: torch.mps.synchronize()}.get(dev, lambda: None)

tok = AutoTokenizer.from_pretrained(MODEL)
model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=torch.bfloat16).eval()      # step 1: file -> host pages (mmap, lazy)
nbytes = sum(p.numel() * p.element_size() for p in model.parameters())

t0 = time.perf_counter(); model.to(dev); sync(); t_move = time.perf_counter() - t0     # step 2: THE extra step
ids = tok.apply_chat_template([{"role": "user", "content": "Name three uses of unified memory."}],
                              add_generation_prompt=True, return_tensors="pt", return_dict=True)["input_ids"].to(dev)
with torch.no_grad():
    model.generate(ids, max_new_tokens=4, do_sample=False)                             # warm-up
    sync(); t0 = time.perf_counter(); model.generate(ids, max_new_tokens=1, do_sample=False); sync(); ttft = time.perf_counter() - t0
    sync(); t0 = time.perf_counter(); out = model.generate(ids, max_new_tokens=128, min_new_tokens=128, do_sample=False); sync(); dt = time.perf_counter() - t0
n = out.shape[1] - ids.shape[1]
print(f"device {dev}: {nbytes/1e9:.2f} GB bf16")
print(f"  step 2, .to({dev}): {t_move*1e3:.0f} ms = {nbytes/t_move/1e9:.1f} GB/s  <- one pool on a Mac; PCIe on a card (x16 Gen5 ~63 GB/s)")
print(f"  step 3, decode:     {n/(dt-ttft):.1f} tok/s = {nbytes*n/(dt-ttft)/1e9:.0f} GB/s effective; TTFT {ttft*1e3:.0f} ms on {ids.shape[1]} prompt tokens")

# office, M3 Ultra, torch 2.11.0, 2026-09-15 (a Mac: the step exists in the framework, not in the hardware):
#   step 2, .to(mps): 1133 ms in a cold process; 451 ms in a warm one; 110-115 ms on a repeat (22 GB/s) -- a CPU memcpy inside one pool
#   step 3, decode:   44.2 tok/s = 109 GB/s effective; TTFT 26 ms on 42 prompt tokens
#   for comparison, mlx-lm on the same bytes copies nothing at step 2 and decodes at 190 tok/s (previous lessons)

#!/usr/bin/env python3
"""The same 27B 4-bit checkpoint (14.42 GB per token, 64 KB of KV per token) on
five memory systems: vendor bandwidth figures in, derived ceilings and copy times out.
Nothing here is measured -- every input is a vendor number, every output a division."""
BYTES_PER_TOKEN = 14.42e9          # Qwen3.5-27B-4bit, read per token (measurements file)
KV_PER_TOKEN = 64e3                # 2 x 16 full-attention layers x 4 KV heads x 256 x 2 bytes
PCIE5_X16 = 63e9                   # ~63 GB/s per direction (T2 interface lesson)

systems = [  # name, memory GB, bandwidth GB/s, weights cross a bus at load?
    ("M3 Ultra (Mac Studio)",        512, 819,  False),
    ("M2 Ultra (Mac Studio)",        192, 800,  False),
    ("RTX 5090 (32 GB GDDR7)",        32, 1792, True),
    ("RTX PRO 6000 Blackwell (96 GB)", 96, 1792, True),
    ("H100 SXM (80 GB HBM3)",         80, 3350, True),
    ("DGX Spark GB10 (128 GB LPDDR5X)", 128, 273, False),
]
print(f"{'system':34} {'fits 27B+100k ctx?':20} {'ceiling tok/s':>14} {'load copy over PCIe5':>22}")
for name, mem, bw, bus in systems:
    need = (BYTES_PER_TOKEN + 1.6e9 + KV_PER_TOKEN * 100_000) / 1e9      # weights + vision/embed + 100k-token cache
    fits = need <= mem * 0.9
    ceiling = bw * 1e9 / BYTES_PER_TOKEN
    copy = f"{16.05e9 / PCIE5_X16:.2f} s" if bus else "none (one pool)"
    print(f"{name:34} {('yes' if fits else 'NO') + f' ({need:.1f} GB)':20} {ceiling:14.0f} {copy:>22}")

print("\nIf the weights do not fit the card, every token re-crosses the bus:")
print(f"  27B at 4-bit over PCIe 5 x16: {PCIE5_X16 / BYTES_PER_TOKEN:.1f} tok/s ceiling -- the bus becomes the memory")

# system                             fits 27B+100k ctx?    ceiling tok/s   load copy over PCIe5
# M3 Ultra (Mac Studio)              yes (22.4 GB)                    57        none (one pool)
# M2 Ultra (Mac Studio)              yes (22.4 GB)                    55        none (one pool)
# RTX 5090 (32 GB GDDR7)             yes (22.4 GB)                   124                 0.25 s
# RTX PRO 6000 Blackwell (96 GB)     yes (22.4 GB)                   124                 0.25 s
# H100 SXM (80 GB HBM3)              yes (22.4 GB)                   232                 0.25 s
# DGX Spark GB10 (128 GB LPDDR5X)    yes (22.4 GB)                    19        none (one pool)
# If the weights do not fit the card: 4.4 tok/s over PCIe 5 x16