학습은 NVIDIA의 땅이야

#!/usr/bin/env python3
"""Training is a different budget: bytes and FLOPs. Full fine-tuning in bf16 with Adam
needs about 16 bytes per parameter before activations; a training step costs ~6 FLOPs per
parameter per token. Compute rates are vendor peak figures (dense, no sparsity) and the
lab's measured MLX matmul on office; real utilization is a fraction of either."""
params = 7e9
tokens = 1e9
bytes_full = params * 16                           # bf16 weights 2 + grads 2 + Adam m,v fp32 8 + master fp32 4
flops = 6 * params * tokens
rates = [  # name, TFLOP/s, note
    ("M3 Ultra, MLX fp32 matmul (measured)",      19.5,   "this quest, CPU track"),
    ("M3 Ultra, PyTorch MPS bf16 matmul (measured)", 21.9, "this quest, journey track"),
    ("H100 SXM, BF16 tensor core, dense",          989.5,  "NVIDIA: 1,979 TFLOPS with sparsity, halved"),
    ("H100 SXM, FP8 tensor core, dense",           1979,   "NVIDIA: 3,958 TFLOPS with sparsity, halved"),
]
print(f"full fine-tune of a {params/1e9:.0f}B: {bytes_full/1e9:.0f} GB of optimizer state and weights before activations")
print(f"{tokens/1e9:.0f}B training tokens: {flops/1e18:.0f} EFLOP\n")
for name, tflops, note in rates:
    secs = flops / (tflops * 1e12)
    print(f"{name:52} {tflops:7.1f} TFLOP/s -> {secs/3600:8.1f} h at 100% utilization   [{note}]")
print("\nutilization is never 100%: published large runs report model-FLOPs utilization well below peak, and a laptop GPU through PyTorch far below that.")

# full fine-tune of a 7B: 112 GB of optimizer state and weights before activations
# 1B training tokens: 42 EFLOP
# M3 Ultra, MLX fp32 matmul (measured)                    19.5 TFLOP/s ->    598.3 h at 100% utilization
# M3 Ultra, PyTorch MPS bf16 matmul (measured)            21.9 TFLOP/s ->    532.7 h at 100% utilization
# H100 SXM, BF16 tensor core, dense                      989.5 TFLOP/s ->     11.8 h at 100% utilization
# H100 SXM, FP8 tensor core, dense                      1979.0 TFLOP/s ->      5.9 h at 100% utilization