어디에나 있는 토큰은 아무것도 못 갈라

# A keyword only tells us something if it is rare. The stopword list is
# English-shaped, so Korean function words sailed straight into the
# profile: the heaviest negative weights were ordinary connective words,
# and one shelf was muting a restaurant piece and a drought story at
# -6.7 apiece for sharing them with a disliked article.
#
# Rather than chase stopwords per language forever, ignore any token
# that shows up in a large share of the shelf being filtered -- common
# words cannot discriminate, whatever language they are in.
MUTE_MAX_DOC_FREQ = 0.15
MUTE_DF_MIN_ROWS = 30      # DF over a handful of rows is noise


def _common_tokens(rows: list[dict]) -> set[str]:
    """Document frequency over the set BEING FILTERED, not the store.

    'Common' is relative to the choice being made: on a single-country
    shelf that country's name is in everything and discriminates
    nothing, while on a general shelf the same token is a strong signal.
    """
    if len(rows) < MUTE_DF_MIN_ROWS:
        return set()
    counts: dict[str, int] = {}
    for row in rows:
        for tok in tokens(row):
            counts[tok] = counts.get(tok, 0) + 1
    ceiling = len(rows) * MUTE_MAX_DOC_FREQ
    return {tok for tok, n in counts.items() if n > ceiling}