//! Per-peer circuit breaker for gRPC connections.
//!
//! State machine: Closed → Open → `HalfOpen` → Closed.
//! After `threshold` consecutive failures, the breaker opens for `reset_duration`.
//! The first [`check`](CircuitBreaker::check) after the reset period transitions
//! to `HalfOpen` and admits **exactly one** probe: that single `check()` returns
//! `Ok(())`, and every subsequent `check()` returns [`CircuitOpenError`] until the
//! outstanding probe resolves via [`record_success`](CircuitBreaker::record_success)
//! (→ Closed) or [`record_failure`](CircuitBreaker::record_failure) (→ Open again).
//! This single-probe invariant is enforced by the state machine itself, not by a
//! separate flag: `check()` returns `Ok(())` only on the `Open → HalfOpen`
//! transition and errors for every call while already in `HalfOpen`, so being in
//! the fieldless `HalfOpen` variant *is* the "a probe is outstanding" marker. No
//! `in_flight` field is needed (or present).
//!
//! Struct-shaped error: `CircuitOpenError` carries no fields and represents the single "breaker is open" outcome; an enum with one variant would be ceremony with no readability gain.

use std::{
    sync::Mutex,
    time::{Duration, Instant},
};

/// Error returned when the circuit breaker is open and not yet ready to probe.
#[derive(Debug, Clone, thiserror::Error)]
#[error("circuit breaker is open")]
pub struct CircuitOpenError;

/// A circuit breaker that tracks consecutive failures for a single peer.
pub struct CircuitBreaker {
    state: Mutex<CircuitState>,
    threshold: u32,
    reset_duration: Duration,
}

#[derive(Debug)]
enum CircuitState {
    Closed {
        consecutive_failures: u32,
    },
    Open {
        opened_at: Instant,
    },
    /// A single probe has been admitted and is in flight. `check()` returns
    /// `Ok(())` exactly once on the Open→`HalfOpen` transition; while in this
    /// state every further `check()` errors until `record_success` (→ Closed)
    /// or `record_failure` (→ Open) resolves the outstanding probe.
    HalfOpen,
}

impl CircuitBreaker {
    /// Create a new circuit breaker.
    #[must_use]
    pub const fn new(threshold: u32, reset_duration: Duration) -> Self {
        Self {
            state: Mutex::new(CircuitState::Closed {
                consecutive_failures: 0,
            }),
            threshold,
            reset_duration,
        }
    }

    /// Check if a request is allowed.
    ///
    /// Returns `Ok(())` if the breaker is closed, or on the single Open→`HalfOpen`
    /// transition that admits one probe, and `Ok(())` on lock poisoning (fail-open:
    /// allow the request through).
    ///
    /// # Half-open single-probe invariant
    ///
    /// The first `check()` after `reset_duration` elapses transitions Open→`HalfOpen`
    /// and returns `Ok(())` — that is the one admitted probe. While that probe is
    /// outstanding the breaker stays `HalfOpen`, and **every subsequent `check()`
    /// returns [`CircuitOpenError`]** until the probe resolves via
    /// [`record_success`](Self::record_success) (→ Closed) or
    /// [`record_failure`](Self::record_failure) (→ Open). N concurrent callers can
    /// therefore never all probe a peer that just failed; exactly one gets through.
    ///
    /// # Errors
    ///
    /// Returns [`CircuitOpenError`] if the breaker is open and the reset period
    /// has not yet elapsed, or if it is `HalfOpen` with a probe already in flight.
    pub fn check(&self) -> Result<(), CircuitOpenError> {
        let Ok(mut state) = self.state.lock() else {
            tracing::warn!("circuit breaker lock poisoned; failing open");
            return Ok(());
        };
        match *state {
            CircuitState::Closed { .. } => Ok(()),
            // A probe is already outstanding; admit no more until it resolves.
            CircuitState::HalfOpen => Err(CircuitOpenError),
            CircuitState::Open { opened_at } => {
                if opened_at.elapsed() >= self.reset_duration {
                    // Admit exactly one probe: the transition itself is the
                    // single Ok. The next `check()` will see `HalfOpen` and error
                    // until this probe's record_success/record_failure resolves it.
                    *state = CircuitState::HalfOpen;
                    Ok(())
                } else {
                    Err(CircuitOpenError)
                }
            }
        }
    }

    /// Record a successful request. Resets the failure count.
    pub fn record_success(&self) {
        let Ok(mut state) = self.state.lock() else {
            tracing::warn!("circuit breaker lock poisoned; ignoring success");
            return;
        };
        *state = CircuitState::Closed {
            consecutive_failures: 0,
        };
    }

    /// Record a follower backpressure reply (`accepted=false`).
    ///
    /// Backpressure is normal flow control from a healthy-but-busy follower whose
    /// inbound channel is full — the WAL is durable on the leader and the follower
    /// catches up once capacity frees. Counting it as a failure would let routine
    /// congestion open the breaker and convert transient backpressure into a
    /// self-inflicted `reset_duration` replication stall (see `error.rs`:
    /// `SegmentRejected` is classified transient). Only genuine transport/gRPC
    /// errors call [`record_failure`](Self::record_failure).
    ///
    /// Effect by state:
    /// - **Closed:** no-op. It neither opens the breaker nor resets a genuine
    ///   failure streak (a follower alternating real errors with backpressure must
    ///   still trip the breaker — see `backpressure_does_not_reset_failure_streak`).
    /// - **`HalfOpen`:** **resolves the outstanding probe by closing the breaker.**
    ///   A backpressure reply proves the *transport* is healthy (the RPC round-tripped;
    ///   only the follower's queue was full), so the half-open probe succeeded at the
    ///   layer the breaker protects. Resolving it is mandatory: under the single-probe
    ///   invariant a `HalfOpen` breaker admits no further `check()` until the probe is
    ///   resolved, so a no-op here would wedge the breaker permanently the first time a
    ///   probe drew a backpressure reply.
    /// - **Open:** no-op (no probe to resolve).
    pub fn record_backpressure(&self) {
        let Ok(mut state) = self.state.lock() else {
            tracing::warn!("circuit breaker lock poisoned; ignoring backpressure");
            return;
        };
        // Only a HalfOpen probe is resolved; Closed/Open are left untouched so
        // backpressure never opens the breaker nor resets a genuine failure streak.
        if matches!(*state, CircuitState::HalfOpen) {
            *state = CircuitState::Closed {
                consecutive_failures: 0,
            };
        }
    }

    /// Record a failed request. Increments the failure count and opens if threshold reached.
    pub fn record_failure(&self) {
        let Ok(mut state) = self.state.lock() else {
            tracing::warn!("circuit breaker lock poisoned; ignoring failure");
            return;
        };
        match *state {
            CircuitState::Closed {
                consecutive_failures,
            } => {
                let new_count = consecutive_failures + 1;
                if new_count >= self.threshold {
                    *state = CircuitState::Open {
                        opened_at: Instant::now(),
                    };
                } else {
                    *state = CircuitState::Closed {
                        consecutive_failures: new_count,
                    };
                }
            }
            CircuitState::HalfOpen => {
                // Probe failed; re-open.
                *state = CircuitState::Open {
                    opened_at: Instant::now(),
                };
            }
            CircuitState::Open { .. } => {
                // Already open, nothing to do.
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn closed_allows_requests() {
        let cb = CircuitBreaker::new(3, Duration::from_secs(30));
        assert!(cb.check().is_ok());
    }

    #[test]
    fn opens_after_threshold_failures() {
        let cb = CircuitBreaker::new(3, Duration::from_secs(30));
        cb.record_failure();
        cb.record_failure();
        assert!(cb.check().is_ok()); // 2 failures, threshold is 3
        cb.record_failure();
        assert!(cb.check().is_err()); // 3 failures, now open
    }

    #[test]
    fn success_resets_failure_count() {
        let cb = CircuitBreaker::new(3, Duration::from_secs(30));
        cb.record_failure();
        cb.record_failure();
        cb.record_success();
        cb.record_failure();
        cb.record_failure();
        assert!(cb.check().is_ok()); // reset happened, only 2 failures since
    }

    #[test]
    fn transitions_to_half_open_after_reset() {
        let cb = CircuitBreaker::new(2, Duration::from_millis(1));
        cb.record_failure();
        cb.record_failure();
        assert!(cb.check().is_err()); // open

        std::thread::sleep(Duration::from_millis(5));
        assert!(cb.check().is_ok()); // half-open after reset period
    }

    #[test]
    fn half_open_failure_reopens() {
        let cb = CircuitBreaker::new(2, Duration::from_millis(1));
        cb.record_failure();
        cb.record_failure();

        std::thread::sleep(Duration::from_millis(5));
        assert!(cb.check().is_ok()); // half-open
        cb.record_failure(); // probe failed
        assert!(cb.check().is_err()); // re-opened
    }

    #[test]
    fn half_open_admits_exactly_one_probe() {
        // Single-probe invariant: once `check()` transitions Open→HalfOpen and
        // returns Ok (the one admitted probe), every subsequent `check()` must
        // error until the probe resolves. Without this, N concurrent callers in
        // half-open would all probe a peer that just failed.
        let cb = CircuitBreaker::new(2, Duration::from_millis(1));
        cb.record_failure();
        cb.record_failure();
        assert!(cb.check().is_err(), "breaker should be open");

        std::thread::sleep(Duration::from_millis(5));

        // First check after reset admits the single probe.
        assert!(
            cb.check().is_ok(),
            "first half-open check admits exactly one probe"
        );
        // The probe is still outstanding: no success/failure recorded yet.
        assert!(
            cb.check().is_err(),
            "second half-open check must error while the probe is in flight"
        );
        assert!(
            cb.check().is_err(),
            "further half-open checks keep erroring until the probe resolves"
        );

        // Resolving the probe with success closes the breaker and re-admits traffic.
        cb.record_success();
        assert!(cb.check().is_ok(), "closed after probe succeeds");
    }

    #[test]
    fn half_open_probe_failure_then_check_stays_closed_to_traffic() {
        // The complementary resolution path: if the single probe fails, the
        // breaker re-opens and `check()` errors (now because it is Open, not
        // because a probe is in flight).
        let cb = CircuitBreaker::new(2, Duration::from_millis(1));
        cb.record_failure();
        cb.record_failure();

        std::thread::sleep(Duration::from_millis(5));
        assert!(cb.check().is_ok(), "single probe admitted");
        assert!(cb.check().is_err(), "second check errors: probe in flight");

        cb.record_failure(); // probe failed → re-open
        assert!(cb.check().is_err(), "re-opened after probe failure");
    }

    #[test]
    fn half_open_success_closes() {
        let cb = CircuitBreaker::new(2, Duration::from_millis(1));
        cb.record_failure();
        cb.record_failure();

        std::thread::sleep(Duration::from_millis(5));
        assert!(cb.check().is_ok()); // half-open
        cb.record_success(); // probe succeeded
        assert!(cb.check().is_ok()); // closed
    }

    #[test]
    fn backpressure_never_opens_breaker() {
        // CRITICAL-7 regression: a healthy-but-busy follower replying
        // `accepted=false` is the path `send_to` routes to `record_backpressure`.
        // Far more than `threshold` consecutive backpressure replies must leave
        // the breaker closed so the leader keeps shipping (no self-inflicted
        // replication stall).
        let cb = CircuitBreaker::new(5, Duration::from_secs(30));
        for _ in 0..10 {
            cb.record_backpressure();
        }
        assert!(
            cb.check().is_ok(),
            "backpressure must not open the breaker; a subsequent send must still be permitted"
        );
    }

    #[test]
    fn half_open_backpressure_resolves_probe_and_closes() {
        // A backpressure reply to the single half-open probe proves the transport
        // is healthy (the RPC round-tripped; only the follower's queue was full),
        // so it must resolve the probe by closing the breaker. Otherwise the
        // single-probe invariant would wedge the breaker in HalfOpen forever the
        // first time a probe drew `accepted=false`.
        let cb = CircuitBreaker::new(2, Duration::from_millis(1));
        cb.record_failure();
        cb.record_failure();

        std::thread::sleep(Duration::from_millis(5));
        assert!(cb.check().is_ok(), "single probe admitted");
        assert!(cb.check().is_err(), "second check errors: probe in flight");

        cb.record_backpressure(); // probe round-tripped, follower just busy
        assert!(
            cb.check().is_ok(),
            "backpressure resolves the half-open probe and re-admits traffic"
        );
    }

    #[test]
    fn genuine_failures_still_open_breaker() {
        // The opposite arm of `send_to` (`Err(status)`) calls `record_failure`,
        // which must still open the breaker after `threshold` consecutive
        // transport errors — backpressure handling must not weaken this.
        let cb = CircuitBreaker::new(5, Duration::from_secs(30));
        for _ in 0..5 {
            cb.record_failure();
        }
        assert!(
            cb.check().is_err(),
            "genuine transport errors must still open the breaker at threshold"
        );
    }

    #[test]
    fn backpressure_does_not_count_toward_open_among_failures() {
        // Backpressure must be neutral: interleaving it with genuine failures
        // neither advances the failure count toward the threshold nor resets it.
        // Here 4 real failures + many backpressures stays below the 5-failure
        // threshold (closed), and the 5th real failure opens it.
        let cb = CircuitBreaker::new(5, Duration::from_secs(30));
        for _ in 0..4 {
            cb.record_failure();
            cb.record_backpressure();
        }
        assert!(
            cb.check().is_ok(),
            "4 failures interleaved with backpressure must remain below threshold"
        );
        cb.record_failure(); // 5th genuine failure
        assert!(
            cb.check().is_err(),
            "the 5th genuine failure must open the breaker"
        );
    }

    #[test]
    fn backpressure_does_not_reset_failure_streak() {
        // Backpressure must not reset the consecutive-failure count the way
        // `record_success` does — otherwise a follower alternating real errors
        // with backpressure could never trip the breaker. 4 failures, a
        // backpressure, then a 5th failure must open it.
        let cb = CircuitBreaker::new(5, Duration::from_secs(30));
        for _ in 0..4 {
            cb.record_failure();
        }
        cb.record_backpressure();
        assert!(cb.check().is_ok()); // still 4 failures, below threshold
        cb.record_failure(); // 5th consecutive genuine failure
        assert!(
            cb.check().is_err(),
            "backpressure must not reset the failure streak; the 5th failure opens the breaker"
        );
    }
}
