KV 캐시는 대화하는 동안 자라

#!/usr/bin/env python3
"""KV cache bytes per token from a checkpoint's config: 2 (K and V) × layers
that keep a cache × kv_heads × head_dim × bytes. Hybrids cache only on the
full-attention layers (full_attention_interval). Latent-attention models
(MLA) need the latent dims instead; this script handles GQA and hybrids."""
import json, sys
from pathlib import Path
from huggingface_hub import snapshot_download

repos = sys.argv[1:] or ["mlx-community/Llama-3.2-1B-Instruct-4bit", "mlx-community/Qwen3.5-9B-4bit",
                         "mlx-community/Qwen3.5-27B-4bit", "mlx-community/Qwen3.5-35B-A3B-4bit"]
BYTES = 2   # bf16 / fp16 cache

for repo in repos:
    cfg = json.loads((Path(snapshot_download(repo, local_files_only=True)) / "config.json").read_text())
    tc = cfg.get("text_config", cfg)
    L = tc["num_hidden_layers"]
    interval = tc.get("full_attention_interval")           # Qwen3.5: every 4th layer is full attention
    cached_layers = L // interval if interval else L
    kv_heads = tc["num_key_value_heads"]
    head_dim = tc.get("head_dim") or tc["hidden_size"] // tc["num_attention_heads"]
    per_token = 2 * cached_layers * kv_heads * head_dim * BYTES
    print(f"{repo.split('/')[-1]:28} cache on {cached_layers:3}/{L:3} layers, {kv_heads}×{head_dim} -> "
          f"{per_token/1024:5.1f} KB/token; 32K = {per_token*32768/1e9:.2f} GB, 1M = {per_token*1_048_576/1e9:.1f} GB")

# office, 2026-09-15:
# Llama-3.2-1B-Instruct-4bit   cache on  16/ 16 layers, 8×64  ->  32.0 KB/token; 32K = 1.07 GB, 1M = 34.4 GB
# Qwen3.5-9B-4bit              cache on   8/ 32 layers, 4×256 ->  32.0 KB/token
# Qwen3.5-27B-4bit             cache on  16/ 64 layers, 4×256 ->  64.0 KB/token
# Qwen3.5-35B-A3B-4bit         cache on  10/ 40 layers, 2×256 ->  20.0 KB/token