tidaldb/k8s/discover/schema.yaml

# The redgifs POSTS corpus schema, for the STANDALONE `tidaldb-discover`
# instance. Distinct from k8s/cluster/schema-configmap.yaml, which is the
# companions corpus and MUST NOT be changed by this feature.
#
# Read ONCE at boot: a ConfigMap edit needs a StatefulSet rollout. Only
# backward-compatible changes are safe against a populated data dir
# (docs/ops/recovery.md §4):
#   - adding a signal                          SAFE
#   - changing an existing signal's half_life  FORBIDDEN (define a new signal)
#   - changing an embedding slot's dimensions  requires deleting /data
#
# The byte-identical dev copy is thepeach's tidaldb_config/discover-schema.yaml.
# `diff` them whenever either changes; nothing else guards the fork.
signals:
  # Impression. Emitted on FOCUS, never on serve -- serving-time emission would
  # mark every item in the page `seen` and burn the corpus in one read. Every
  # context signal (one carrying a user_id) also marks the item seen for that
  # user, which is what advances the feed page to page.
  - name: view
    entity: item
    decay:
      exponential:
        half_life_seconds: 604800    # 7 days -- matches the companions schema
    windows: [one_hour, twenty_four_hours, seven_days]
    velocity: true
    positive_engagement: true
  # The heart of the ranking. Server-side tee off the existing post-reaction
  # path, so it cannot be forged by a client and cannot drift from the reaction
  # the user actually sees.
  - name: like
    entity: item
    decay:
      exponential:
        half_life_seconds: 1209600   # 14 days -- matches the companions schema
    windows: [twenty_four_hours, seven_days, thirty_days, all_time]
    velocity: false
    positive_engagement: true
  # Graded engagement depth in [0,1]. The signal WEIGHT *is* the ratio -- that
  # is the engine's documented contract for this exact signal name
  # (tidal/src/db/signals.rs:727-735), not a convention invented here.
  # Video: currentTime/duration. Image: dwell/3s, capped at 1.0.
  #
  # NOT present in any other shipped schema in either repo. Declared here for
  # the first time; a POST /signals with this name 400s against a schema that
  # omits it, inside a fire-and-forget path where nobody would see the 400.
  - name: completion
    entity: item
    decay:
      exponential:
        half_life_seconds: 1209600   # 14 days, matching `like`
    windows: [twenty_four_hours, seven_days, all_time]
    velocity: false
    positive_engagement: true
text_fields:
  # `title` carries the post caption, `category` the post kind. Declared so the
  # metadata the worker writes is indexed rather than inert. Nothing queries
  # /search on this instance yet.
  - name: title
    kind: text
  - name: category
    kind: keyword
embedding_slots:
  # DECLARED, NEVER WRITTEN in v1: 1,819 of 1,821 posts have an empty caption,
  # so an embedding would give one companion's 127 posts an identical vector.
  #
  # Both the NAME and the WIDTH are permanent once data lands, and a future ANN
  # profile must name this exact slot -- so both are fixed now to match
  # thepeach's embedder (text-embedding-3-small, 1536-D) and the name every
  # other peach schema already uses. The tidalDB standalone image bakes a 128-D
  # default; this file overrides it.
  - name: content_vector
    entity: item
    dimensions: 1536
profiles:
  # The v1 discover ranking. Deliberately NOT built-in `for_you`: that profile
  # hard-codes CandidateStrategy::Ann{slot:"content"}
  # (tidal/src/ranking/builtins.rs:305-311), which (a) names a slot this schema
  # does not declare -- every peach schema calls it `content_vector` -- and
  # (b) with no vectors written would fall through to the scan fallback and push
  # a warning on EVERY read, which thepeach's CLAUDE.md:57 forbids.
  #
  # `scan` reaches the same code path with no warning.
  - name: rg_discover
    version: 1
    # Takes max(limit * M, 200) ids in ASCENDING ENTITY-ID ORDER, BEFORE
    # seen-exclusion, where M is 10 with a user_id and 4 without
    # (tidal/src/query/executor/candidate_gen.rs:22-44). That ordering is why the
    # caller requests limit=200 rather than a page-size limit: a 2,000-wide pool
    # over a ~1,821-post corpus. See the endpoint's DISCOVER_TIDAL_FETCH const.
    candidate_strategy: scan
    # NO `sort:` — DECISION UNCHANGED, RATIONALE REWRITTEN. The score is
    # `sort_base + boost_sum`, then min-max normalized
    # (tidal/src/ranking/executor/{mod,helpers}.rs), so what decides whether
    # signals matter is the SPREAD of sort_base across the candidate set
    # relative to the spread of boost_sum (single digits here).
    #
    # The two bullets below previously argued from the PRE-CHANGE formulas and
    # cited `tidal/src/query/executor/scoring.rs`, a path that does not exist
    # (the real file is `tidal/src/ranking/executor/scoring.rs`). Both sorts are
    # now age-aware; here is what they actually do:
    #
    #   hot: IS age-aware now. `score_hot` reads this item's real `created_at`
    #        out of the item-metadata map and scores
    #        `log10(max(views,1)) / (age_hours + 2)^gravity`
    #        (`score_hot`, tidal/src/ranking/executor/scoring.rs). The old claim
    #        that it "treats EVERY candidate as exactly 24 hours old" and
    #        "contributes no recency whatsoever" is VOID.
    #        What still holds: the numerator is exactly 0.0 at 0 OR 1 views, so
    #        on the zero-signal rows that make up most of this corpus the age
    #        divisor has nothing to divide — every such candidate scores 0.0 and
    #        ties. `hot` would add recency only from an item's second view on.
    #   new: base is negated age in HOURS now, not `entity_id as f64`. The old
    #        "~1_821 on this corpus" figure is VOID.
    #        It is NOT automatically boost-comparable, though: the spread of
    #        `-age_hours` equals the corpus's age span expressed in hours, so a
    #        corpus accumulated over a year spans ~8_766 — the same order of
    #        magnitude as the entity-id spread it replaced, and still ~3 orders
    #        above a single-digit boost_sum. The domination concern SURVIVES the
    #        change; only its derivation moved from item COUNT to corpus AGE
    #        SPAN. On a corpus spanning under ~a day it would genuinely be
    #        boost-comparable, which is exactly why this needs measuring rather
    #        than asserting.
    #
    # With no sort the ranking is boost_sum alone — measured: 5 likes outranks 3
    # outranks 1, with the remainder in scan order. That IS the intent ("ranks
    # on signals alone").
    #
    # STATUS: PENDING RE-MEASUREMENT. Keeping `NO sort:` is the safe hold, but
    # it is no longer JUSTIFIED by the reasoning above — the reasoning it rested
    # on was about the old formulas. Before adding a sort back, measure on a
    # production snapshot:
    #   1. corpus age span in hours (max created_at - min created_at). That is
    #      the `new` sort_base spread.
    #   2. observed boost_sum spread (p1..p99) over the same candidate pool.
    #   3. fraction of the pool with >= 2 `view` signals. That is the fraction
    #      `hot` can differentiate at all.
    # Add `new` only if (1) is within roughly one order of magnitude of (2);
    # add `hot` only if (3) is a clear majority. Otherwise the sort still either
    # annihilates the signals or ties the set.
    #
    # Consequence, accepted and handled UPSTREAM — UNCHANGED by the age-aware
    # work, because with no sort no age term is read at all: on a zero-signal
    # corpus every score ties at the normalizer's neutral 0.5 and the order is
    # scan order, i.e. OLDEST-FIRST (the sweep allocates ids oldest-first). That
    # is why the endpoint routes a viewer below the impression threshold to its
    # `ranked` arm instead of reading this profile unpersonalized — an arbitrary
    # order is strictly worse than the recency ordering that already exists.
    # NOTE: `window` is INERT for `agg: decay_score`. The executor calls
    # read_decay_score_at(entity_id, signal, 0, now) -- window index hard-coded
    # to 0 (tidal/src/query/executor/helpers.rs:112-114). So `all_time` below is
    # not a claim that each signal declares an all_time window; built-in
    # `for_you` pairs view+decay_score+all_time against a schema whose `view`
    # has no all_time window, for the same reason. Do NOT "fix" this by adding
    # all_time to `view.windows` (a dead index) or by switching agg.
    boosts:
      - signal: view
        agg: decay_score
        window: all_time
        weight: 1.0
      - signal: like
        agg: decay_score
        window: all_time
        weight: 2.0
      - signal: completion
        agg: decay_score
        window: all_time
        weight: 2.5
    diversity:
      # 6, not built-in for_you's 2. One companion (`Nettie`) is 127 of 1,821
      # posts and the top 10 companions are 45% of the corpus, so 2 would
      # systematically starve the 1-post long tail; 6 still keeps any one
      # companion off two-thirds of a 24-item page.
      max_per_creator: 6
      # `format_mix_max_fraction` is DELIBERATELY ABSENT. It caps how much of a
      # page one `format` metadata value may occupy, but (a) this corpus is
      # 1,727 images to 94 videos (measured), so capping the dominant format
      # starves pages rather than diversifying them, and (b) the worker writes
      # the post kind to `category`, not `format`, so the key would govern a
      # field nothing populates. Two independent reasons; do not add it back
      # without changing both.
    # Exploration items are APPENDED at score 0.0
    # (tidal/src/query/executor/candidate_gen.rs:215-273) — appended, so they
    # rank last, not first. This does not solve cold start (see the no-sort note
    # above); it keeps a zero-signal post reachable once the head of the ranking
    # is signal-dominated, which is otherwise a closed loop: only signalled posts
    # rank, and only ranked posts get signalled.
    exploration: 0.1