tidaldb/k8s/discover/schema.yaml

# The redgifs POSTS corpus schema, for the STANDALONE `tidaldb-discover`
# instance. Distinct from k8s/cluster/schema-configmap.yaml, which is the
# companions corpus and MUST NOT be changed by this feature.
#
# Read ONCE at boot: a ConfigMap edit needs a StatefulSet rollout. Only
# backward-compatible changes are safe against a populated data dir
# (docs/ops/recovery.md §4):
#   - adding a signal                          SAFE
#   - changing an existing signal's half_life  FORBIDDEN (define a new signal)
#   - changing an embedding slot's dimensions  requires deleting /data
#
# The byte-identical dev copy is thepeach's tidaldb_config/discover-schema.yaml.
# `diff` them whenever either changes; nothing else guards the fork.
signals:
  # Impression. Emitted on FOCUS, never on serve -- serving-time emission would
  # mark every item in the page `seen` and burn the corpus in one read. Every
  # context signal (one carrying a user_id) also marks the item seen for that
  # user, which is what advances the feed page to page.
  - name: view
    entity: item
    decay:
      exponential:
        half_life_seconds: 604800    # 7 days -- matches the companions schema
    windows: [one_hour, twenty_four_hours, seven_days]
    velocity: true
    positive_engagement: true
  # The heart of the ranking. Server-side tee off the existing post-reaction
  # path, so it cannot be forged by a client and cannot drift from the reaction
  # the user actually sees.
  - name: like
    entity: item
    decay:
      exponential:
        half_life_seconds: 1209600   # 14 days -- matches the companions schema
    windows: [twenty_four_hours, seven_days, thirty_days, all_time]
    velocity: false
    positive_engagement: true
  # Graded engagement depth in [0,1]. The signal WEIGHT *is* the ratio -- that
  # is the engine's documented contract for this exact signal name
  # (tidal/src/db/signals.rs:727-735), not a convention invented here.
  # Video: currentTime/duration. Image: dwell/3s, capped at 1.0.
  #
  # NOT present in any other shipped schema in either repo. Declared here for
  # the first time; a POST /signals with this name 400s against a schema that
  # omits it, inside a fire-and-forget path where nobody would see the 400.
  - name: completion
    entity: item
    decay:
      exponential:
        half_life_seconds: 1209600   # 14 days, matching `like`
    windows: [twenty_four_hours, seven_days, all_time]
    velocity: false
    positive_engagement: true
text_fields:
  # `title` carries the post caption, `category` the post kind. Declared so the
  # metadata the worker writes is indexed rather than inert. Nothing queries
  # /search on this instance yet.
  - name: title
    kind: text
  - name: category
    kind: keyword
embedding_slots:
  # DECLARED, NEVER WRITTEN in v1: 1,819 of 1,821 posts have an empty caption,
  # so an embedding would give one companion's 127 posts an identical vector.
  #
  # Both the NAME and the WIDTH are permanent once data lands, and a future ANN
  # profile must name this exact slot -- so both are fixed now to match
  # thepeach's embedder (text-embedding-3-small, 1536-D) and the name every
  # other peach schema already uses. The tidalDB standalone image bakes a 128-D
  # default; this file overrides it.
  - name: content_vector
    entity: item
    dimensions: 1536
profiles:
  # The v1 discover ranking. Deliberately NOT built-in `for_you`: that profile
  # hard-codes CandidateStrategy::Ann{slot:"content"}
  # (tidal/src/ranking/builtins.rs:305-311), which (a) names a slot this schema
  # does not declare -- every peach schema calls it `content_vector` -- and
  # (b) with no vectors written would fall through to the scan fallback and push
  # a warning on EVERY read, which thepeach's CLAUDE.md:57 forbids.
  #
  # `scan` reaches the same code path with no warning.
  - name: rg_discover
    version: 1
    # Takes max(limit * M, 200) ids in ASCENDING ENTITY-ID ORDER, BEFORE
    # seen-exclusion, where M is 10 with a user_id and 4 without
    # (tidal/src/query/executor/candidate_gen.rs:22-44). That ordering is why the
    # caller requests limit=200 rather than a page-size limit: a 2,000-wide pool
    # over a ~1,821-post corpus. See the endpoint's DISCOVER_TIDAL_FETCH const.
    candidate_strategy: scan
    # NO `sort:` — VERIFIED DELIBERATE. Neither built-in sort is usable here, and
    # the score is `sort_base + boost_sum` then min-max normalized
    # (tidal/src/ranking/executor/{mod,helpers}.rs), so a sort's magnitude decides
    # whether signals matter at all:
    #
    #   hot: NOT age-aware. `score_hot` treats EVERY candidate as exactly 24
    #        hours old and ranks by `view` COUNT
    #        (tidal/src/ranking/executor/scoring.rs:263-271, which says so in a
    #        comment). It reads no created_at, so it adds a term the `view` boost
    #        below already covers, and contributes no recency whatsoever.
    #   new: base score is `entity_id as f64`
    #        (tidal/src/ranking/executor/scoring.rs:122-133), i.e. ~1_821 on this
    #        corpus, against a boost_sum of single digits. Recency would dominate
    #        by three orders of magnitude and the signals would be decorative.
    #
    # With no sort the ranking is boost_sum alone — measured: 5 likes outranks 3
    # outranks 1, with the remainder in scan order. That IS the intent ("ranks on
    # signals alone"); do not add a sort back without re-measuring both effects.
    #
    # Consequence, accepted and handled UPSTREAM: on a zero-signal corpus every
    # score ties at the normalizer's neutral 0.5 and the order is scan order,
    # i.e. OLDEST-FIRST (the sweep allocates ids oldest-first). That is why the
    # endpoint routes a viewer below the impression threshold to its `ranked`
    # arm instead of reading this profile unpersonalized — an arbitrary order is
    # strictly worse than the recency ordering that already exists.
    # NOTE: `window` is INERT for `agg: decay_score`. The executor calls
    # read_decay_score_at(entity_id, signal, 0, now) -- window index hard-coded
    # to 0 (tidal/src/query/executor/helpers.rs:112-114). So `all_time` below is
    # not a claim that each signal declares an all_time window; built-in
    # `for_you` pairs view+decay_score+all_time against a schema whose `view`
    # has no all_time window, for the same reason. Do NOT "fix" this by adding
    # all_time to `view.windows` (a dead index) or by switching agg.
    boosts:
      - signal: view
        agg: decay_score
        window: all_time
        weight: 1.0
      - signal: like
        agg: decay_score
        window: all_time
        weight: 2.0
      - signal: completion
        agg: decay_score
        window: all_time
        weight: 2.5
    diversity:
      # 6, not built-in for_you's 2. One companion (`Nettie`) is 127 of 1,821
      # posts and the top 10 companions are 45% of the corpus, so 2 would
      # systematically starve the 1-post long tail; 6 still keeps any one
      # companion off two-thirds of a 24-item page.
      max_per_creator: 6
      # `format_mix_max_fraction` is DELIBERATELY ABSENT. It caps how much of a
      # page one `format` metadata value may occupy, but (a) this corpus is
      # 1,727 images to 94 videos (measured), so capping the dominant format
      # starves pages rather than diversifying them, and (b) the worker writes
      # the post kind to `category`, not `format`, so the key would govern a
      # field nothing populates. Two independent reasons; do not add it back
      # without changing both.
    # Exploration items are APPENDED at score 0.0
    # (tidal/src/query/executor/candidate_gen.rs:215-273) — appended, so they
    # rank last, not first. This does not solve cold start (see the no-sort note
    # above); it keeps a zero-signal post reachable once the head of the ranking
    # is signal-dominated, which is otherwise a closed loop: only signalled posts
    # rank, and only ranked posts get signalled.
    exploration: 0.1