운영 클라이언트 설계 — 재시도·백오프·회로 차단기

# Backoff 와 jitter 가진 retry — tenacity 라이브러리
from tenacity import retry, wait_exponential_jitter, stop_after_attempt, retry_if_exception_type
import httpx

# Idempotent method — transient 실패에 retry 안전
@retry(
    wait=wait_exponential_jitter(initial=1, max=30, jitter=1),
    stop=stop_after_attempt(5),
    retry=retry_if_exception_type((httpx.TransportError, httpx.HTTPStatusError)),
)
def fetch_user(uid: str) -> dict:
    resp = httpx.get(f'https://api.example.com/users/{uid}')
    if resp.status_code >= 500:
        resp.raise_for_status()  # retry trigger
    if resp.status_code == 429:
        # Retry 전 Retry-After 존중
        import time
        wait = int(resp.headers.get('Retry-After', 5))
        time.sleep(wait)
        raise httpx.HTTPStatusError('rate limited', request=resp.request, response=resp)
    resp.raise_for_status()
    return resp.json()

# 단순 circuit breaker — pybreaker 라이브러리
import pybreaker
import httpx

breaker = pybreaker.CircuitBreaker(
    fail_max=5,           # 5 연속 실패 후 OPEN
    reset_timeout=60,     # HALF-OPEN probe 전 60s cooldown
    exclude=[httpx.HTTPStatusError],  # 4xx 를 실패로 안 셈 (client 이슈)
)

@breaker
def call_upstream(url: str) -> dict:
    resp = httpx.get(url, timeout=10.0)
    resp.raise_for_status()
    return resp.json()

# 사용
try:
    data = call_upstream('https://api.example.com/users/42')
except pybreaker.CircuitBreakerError:
    # Circuit 이 OPEN — fail fast, cached 돌려줌, 나중 큐
    data = cached_or_fallback()
except httpx.HTTPError as e:
    # Upstream error 가 circuit threshold 향해 카운트
    raise

# Idempotency 인식 retry decorator (retry + idempotency 체크 결합)
import httpx
import time, random

IDEMPOTENT_METHODS = {'GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE'}

def call_with_resilience(method: str, url: str, idempotency_key: str | None = None, **kwargs):
    """Retry + backoff + idempotency 인식 + Retry-After 존중 결합."""
    method = method.upper()
    # POST/PATCH 는 caller 가 Idempotency-Key 제공할 때만 retry
    can_retry = method in IDEMPOTENT_METHODS or idempotency_key is not None
    if idempotency_key:
        kwargs.setdefault('headers', {})['Idempotency-Key'] = idempotency_key

    max_attempts = 5 if can_retry else 1
    for attempt in range(max_attempts):
        try:
            with httpx.Client(timeout=10.0) as c:
                resp = c.request(method, url, **kwargs)
            if resp.status_code == 429:
                wait = int(resp.headers.get('Retry-After', 1))
                time.sleep(wait)
                continue
            if 500 <= resp.status_code < 600:
                if not can_retry:
                    return resp
                wait = min(2 ** attempt, 30) + random.random()
                time.sleep(wait)
                continue
            return resp
        except httpx.TransportError:
            if not can_retry:
                raise
            wait = min(2 ** attempt, 30) + random.random()
            time.sleep(wait)
    return resp  # 모든 시도 소진