{ "schema_version": "1.0.0", "bot_id": "6.4", "bot_name": "Backtester", "slug": "backtester", "layer": "Governance", "layer_key": "gov", "bot_class": "Governance Service", "authority": [ "Explain" ], "status": "beta", "readiness": "Limited live", "flagship": false, "is_reference": false, "public_export": false, "identity": { "layer": "Governance", "bot_class": "Governance Service", "authority": "Explain", "runs_before": "Nothing \u2014 Backtester is a governance simulation bot; it runs in replay mode against the report archive and never precedes live execution", "runs_after": "Historical report archive is populated; ReportEnvelope records for the target time window are available", "applies_to": "All strategy bots configured for replay; any historical OrderIntent window specified via start_ts/end_ts", "default_mode": "limited_live", "user_visible": "Advanced details only", "developer_owner": "Polytraders core \u2014 Governance pod" }, "purpose": "Backtester replays historical CLOB snapshots and the full report archive through the live execution path at tick resolution. It runs in replay mode (mode=replay), consuming archived ObservationReport, DecisionReport, RiskVote, ExecutionReport, and SettlementReport envelopes and re-executing the strategy under test against them. It emits replay-tagged OperationsReport records (and replay-tagged copies of every simulated report kind) to polytraders.reports.operations, partitioned by bot_slug+epoch, retained for 1 year. Backtester is used by the governance team to validate parameter changes, A/B test strategies, and produce audit-quality evidence before any strategy is promoted to live. Backtester never signs orders or touches the live CLOB.", "why_it_matters": [ { "failure": "Strategy promoted to live without backtesting", "consequence": "Untested parameter changes or new strategy logic may produce runaway losses or adverse fills on live capital. Backtesting is a mandatory gate before promotion." }, { "failure": "Backtester uses different execution path than live", "consequence": "Results are not comparable to live performance. Governance audit evidence is invalid and cannot be used to justify promotion decisions." }, { "failure": "Replay-tagged reports not emitted", "consequence": "Backtesting runs are not auditable. Governance cannot produce the required evidence trail for strategy promotion gates." }, { "failure": "Parameter sweep runs with non-deterministic inputs", "consequence": "Results are not reproducible. Successive backtests on the same window may produce different outputs, making comparison and audit impossible." } ], "polymarket_inputs": [ { "input": "Historical CLOB snapshots (order book + trades, tick-level)", "source": "internal", "required": true, "use": "Primary replay data source. Backtester reconstructs the order book state at each tick from the archived snapshot stream." }, { "input": "Archived ObservationReport envelopes", "source": "internal", "required": true, "use": "Replay pre-trade intelligence signals as they existed at the original observation time." }, { "input": "Archived DecisionReport envelopes", "source": "internal", "required": true, "use": "Reconstruct the original strategy decisions for comparison against replayed decisions." }, { "input": "Archived RiskVote envelopes", "source": "internal", "required": true, "use": "Reconstruct risk guardrail votes that were in effect during the replay window." }, { "input": "Archived ExecutionReport envelopes", "source": "internal", "required": false, "use": "Compare simulated fill outcomes against actual historical fills for slippage and fill-quality analysis." }, { "input": "Archived SettlementReport envelopes", "source": "internal", "required": false, "use": "Replay post-trade P&L settlement to reconstruct the full governance evidence trail." } ], "internal_inputs": [ { "input": "Strategy configuration under test", "source": "Config store", "required": true, "use": "Parameters (or parameter sweep grid) for the strategy being backtested." }, { "input": "KillSwitch active flag", "source": "KillSwitch", "required": false, "use": "When KillSwitch is active, suppress new backtesting run launches. Runs already in progress may continue." } ], "raw_params": [ "start_ts \u00b7 datetime", "end_ts \u00b7 datetime", "strategy \u00b7 enum", "parameter_sweep \u00b7 list" ], "parameters": [ { "name": "start_ts", "default": "now()-7d", "warning": null, "hard": null, "controls": "Start timestamp (ISO 8601, UTC) for the replay window.", "why_default_matters": "Defaulting to 7 days ago ensures at least one full trading week is covered by each backtest run.", "threshold_logic": [ { "condition": "start_ts < end_ts", "action": "Accept window and begin replay" }, { "condition": "start_ts >= end_ts", "action": "Reject with BACKTESTER_INVALID_WINDOW" } ], "dev_check": "if (p.start_ts >= p.end_ts) throw ConfigError('BACKTESTER_INVALID_WINDOW')", "user_facing": "The backtest covers historical data starting from this date and time." }, { "name": "end_ts", "default": "now()", "warning": null, "hard": null, "controls": "End timestamp (ISO 8601, UTC) for the replay window. Must be greater than start_ts.", "why_default_matters": "Defaulting to now() makes fresh backtests cover up to the most recent archived data.", "threshold_logic": [ { "condition": "end_ts > start_ts AND end_ts <= now()", "action": "Accept window" }, { "condition": "end_ts > now()", "action": "Clamp to now(); emit BACKTESTER_WINDOW_CLAMPED warn" } ], "dev_check": "if (p.end_ts > now()) { p.end_ts = now(); log.warn('BACKTESTER_WINDOW_CLAMPED') }", "user_facing": "The backtest covers historical data up to this date and time." }, { "name": "strategy", "default": "sum_to_one_arb", "warning": null, "hard": null, "controls": "The strategy bot slug to replay through the backtester execution path.", "why_default_matters": "Default to sum_to_one_arb as the most commonly backtested strategy. Must be explicitly set for other strategy types.", "threshold_logic": [ { "condition": "strategy in registered strategy registry", "action": "Load strategy configuration and begin replay" }, { "condition": "strategy not in registry", "action": "Reject with BACKTESTER_UNKNOWN_STRATEGY" } ], "dev_check": "if (!strategyRegistry.has(p.strategy)) throw ConfigError('BACKTESTER_UNKNOWN_STRATEGY')", "user_facing": "The trading strategy being evaluated in this backtest." }, { "name": "parameter_sweep", "default": [], "warning": 50, "hard": 200, "controls": "List of parameter override objects, each defining one variant run. Empty list means single run with defaults. Max 200 variants per backtest job.", "why_default_matters": "Empty list (single run) is safe as a default. Sweeps beyond 200 variants can exceed the archive query budget and should require explicit approval.", "threshold_logic": [ { "condition": "len(parameter_sweep) == 0", "action": "Run single backtest with strategy defaults" }, { "condition": "1 <= len(parameter_sweep) <= 50", "action": "Run all variants; emit summary OperationsReport per variant" }, { "condition": "51 <= len(parameter_sweep) <= 200", "action": "WARN BACKTESTER_LARGE_SWEEP; run all variants with resource throttle" }, { "condition": "len(parameter_sweep) > 200", "action": "Reject with PARAMETER_CHANGE_REQUIRES_APPROVAL" } ], "dev_check": "if (p.parameter_sweep.length > p.hard) throw ConfigError('PARAMETER_CHANGE_REQUIRES_APPROVAL')", "user_facing": "A list of parameter variations to test. Each entry produces a separate backtest result for comparison." } ], "default_config": { "bot_id": "gov.backtester", "version": "2.0.0", "mode": "replay", "defaults": { "start_ts": "now()-7d", "end_ts": "now()", "strategy": "sum_to_one_arb", "parameter_sweep": [] }, "locked": { "mode": { "immutable": true, "value": "replay" } } }, "implementation_flow": [ "On run start, validate start_ts < end_ts and strategy is in the registry; reject with BACKTESTER_INVALID_WINDOW or BACKTESTER_UNKNOWN_STRATEGY otherwise.", "Assign a replay_run_id (ULID) for this run; all emitted reports for this run carry replay_run_id and mode=replay.", "Fetch archived ObservationReport, DecisionReport, RiskVote, ExecutionReport, and SettlementReport envelopes for the window [start_ts, end_ts] from the report archive.", "For each tick in the replay window (tick = snapshot boundary from historical CLOB snapshots), reconstruct the order book state and available signals at that moment.", "Feed reconstructed state and archived signals into the strategy under test using the same execution path as live (same guardrails, same execution logic).", "For each simulated OrderIntent, run the full risk guardrail stack (replay mode \u2014 votes are simulated, not live); record each vote outcome.", "Simulate fill outcomes using the historical order book snapshot at the matching tick; compute simulated fill price, slippage, and fee using pUSD denomination.", "Emit replay-tagged OperationsReport after each simulated fill: includes replay_run_id, original_trace_id, simulated verdict, simulated P&L in pUSD, and comparison against original outcome.", "Emit replay-tagged copies of simulated DecisionReport and ExecutionReport envelopes with mode=replay and replay_run_id for archive and audit.", "At run completion, emit an aggregate OperationsReport summarising: total ticks replayed, simulated fills, simulated P&L, parameter variant results if sweep was configured, and comparison deltas vs original outcomes.", "Retain all replay-tagged reports in polytraders.reports.operations for 1 year, partitioned by bot_slug+epoch." ], "decision_logic": { "approve": "Not applicable \u2014 Backtester is a simulation bot in replay mode. It never approves live orders.", "reshape_required": "Not applicable.", "reject": "Not applicable as a live trading decision. Backtester will abort a replay run if the archive data is incomplete or the strategy is unknown.", "warning_only": "Parameter sweeps above 50 variants emit BACKTESTER_LARGE_SWEEP. Replay windows that extend beyond now() are clamped with BACKTESTER_WINDOW_CLAMPED warn." }, "decision_output_schema": "OperationsReport", "decision_output_example": { "report_id": "ops_backtester_replay_01HX9KZQ7E8VR5", "bot_id": "gov.backtester", "event_type": "BACKTESTER_REPLAY_COMPLETE", "replay_run_id": "replay_01HX9KZQ7E8VR5", "strategy": "sum_to_one_arb", "mode": "replay", "window_start": "2026-05-01T00:00:00Z", "window_end": "2026-05-08T00:00:00Z", "ticks_replayed": 483120, "simulated_fills": 214, "simulated_volume_pusd": 92450.0, "simulated_pnl_pusd": 1380.5, "simulated_net_fees_pusd": 231.1, "vs_original_pnl_delta_pusd": 42.0, "parameter_variants_run": 1, "report_kind": "OperationsReport", "topic": "polytraders.reports.operations", "partition": "backtester+2026-05-08T00:00Z", "retained_until": "2027-05-08" }, "developer_log": { "bot_id": "gov.backtester", "event_type": "BACKTESTER_TICK_PROCESSED", "replay_run_id": "replay_01HX9KZQ7E8VR5", "tick_ts_ms": 1746792060000, "market_id": "0x9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c", "simulated_intent_emitted": true, "simulated_fill_price": 0.621, "simulated_fill_size_pusd": 430.0, "simulated_slippage_bps": 0.8, "risk_votes": [ { "bot": "liquidityguard", "vote": "pass" }, { "bot": "portfolioguard", "vote": "reshape", "new_size_pusd": 430.0 } ], "mode": "replay" }, "user_explanations": [ { "situation": "Backtest run completed successfully", "message": "The strategy was replayed over the selected historical period. Results include simulated trades, P&L estimates, and a comparison against what actually happened." }, { "situation": "Parameter sweep completed", "message": "Multiple parameter variants were tested over the same historical data. A summary of results for each variant is available." }, { "situation": "Replay window clamped to present", "message": "The requested end date is in the future. The backtest was run up to the most recent available data." }, { "situation": "Archive data incomplete for window", "message": "Some historical data is unavailable for the requested window. The backtest covered only the periods with complete records." } ], "failure_modes": { "main_failure_mode": "The report archive is unavailable or has gaps for the requested replay window, causing the backtester to skip ticks and produce incomplete simulated results.", "false_positive_risk": "Simulated fills are computed against a historical order book snapshot that may not reflect the true fill price due to latency artifacts in the archive, inflating apparent P&L.", "false_negative_risk": "A strategy change that only manifests at rare market conditions (e.g. extreme spread) may not be detected if the replay window does not include those events.", "safe_fallback": "If archive data is incomplete for a tick, skip that tick and emit BACKTESTER_TICK_SKIPPED warn; continue replay from the next available tick. Never extrapolate or fill gaps synthetically. If the archive is entirely unavailable, abort the run and emit BACKTESTER_ARCHIVE_UNAVAILABLE.", "required_dependencies": [ "Historical report archive (ObservationReport, DecisionReport, RiskVote, ExecutionReport, SettlementReport)", "Historical CLOB snapshots (order book + trades)", "Strategy registry (strategy configuration under test)", "Internal message bus (OperationsReport emission)" ] }, "acceptance_tests": { "unit": [ { "test": "Replay window validation rejects start_ts >= end_ts", "setup": "start_ts='2026-05-09', end_ts='2026-05-01'", "expected": "BACKTESTER_INVALID_WINDOW ConfigError" }, { "test": "Unknown strategy slug raises error", "setup": "strategy='nonexistent_strategy'", "expected": "BACKTESTER_UNKNOWN_STRATEGY ConfigError" }, { "test": "Parameter sweep above hard limit (200) is rejected", "setup": "parameter_sweep = list of 201 variants", "expected": "PARAMETER_CHANGE_REQUIRES_APPROVAL ConfigError" }, { "test": "Replay assigns mode=replay on all emitted reports", "setup": "Run single-variant replay; inspect all emitted OperationsReport payloads", "expected": "All reports carry mode='replay' and replay_run_id" }, { "test": "Simulated fill denominated in pUSD (not USDC.e)", "setup": "Replay a fill event from historical archive", "expected": "simulated_fill output contains size_pusd field; no USDC.e references" }, { "test": "Archive gap causes tick skip, not crash", "setup": "Inject missing tick in archive at t=T; continue replay", "expected": "BACKTESTER_TICK_SKIPPED emitted; replay continues from T+1" } ], "integration": [ { "test": "End-to-end: 7-day replay of sum_to_one_arb produces aggregate OperationsReport", "expected": "Aggregate OperationsReport with ticks_replayed > 0, simulated_fills >= 0, mode=replay, retained_until = now+1y" }, { "test": "Parameter sweep of 3 variants produces 3 separate OperationsReport summaries", "expected": "3 reports, each with distinct parameter_variant field and independent simulated P&L" }, { "test": "KillSwitch active suppresses new run launches", "expected": "KILL_SWITCH_ACTIVE logged; backtest run not started; in-progress runs continue" } ], "property": [ { "property": "All emitted reports carry mode=replay and replay_run_id", "required": "Always true \u2014 replay mode is locked immutable in default_config" }, { "property": "No live CLOB calls are made during replay", "required": "Always true \u2014 all inputs come from the archive; no clob_auth or onchain surfaces are accessed" }, { "property": "Replay is deterministic: same inputs produce same outputs", "required": "Always true \u2014 no randomness; archive inputs are immutable" } ] }, "checklist_overrides": {}, "legacy_goal": "Tick-level replay over historical CLOB snapshots \u2014 same execution path as live.", "legacy_pm_signals": [ "Historical CLOB snapshots (book + trades)" ], "legacy_external_feeds": [ "Internal snapshot archive (replayed deterministically)" ], "reporting_groups": [ "governance_audit" ], "network": [ "polygon" ], "api_surface": [ "internal" ], "version": { "spec": "2.0.0", "implementation": "2.1.0", "schema": "2", "released": "2026-04-28" }, "migration_history": [ { "date": "2026-04-28", "from": "v1", "to": "v2", "reason": "CLOB V2 cutover", "action_taken": "Updated replay pipeline to consume V2 ReportEnvelope format (pUSD denomination, mode field, replay_run_id). Removed USDC.e references from simulated fill output. Replay now ingests archived OperationsReport, DecisionReport, ExecutionReport, and SettlementReport in V2 schema. All simulated fill sizes and P&L now denominated in pUSD. Switched to py-clob-client-v2 archive reader." } ], "polymarket_v2_compat": { "clob_version": "v2", "collateral": "pUSD", "eip712_domain_version": "2", "builder_code_aware": false, "negrisk_aware": true, "multichain_ready": false, "sdk_used": "py-clob-client-v2", "settlement_contract": "CTFExchangeV2 on Polygon", "notes": "Backtester consumes archived V2 ReportEnvelope records; all simulated fills are denominated in pUSD. Backtester is replay-only and never makes live CLOB calls. NegRisk market payoffs are replayed using the NegRiskAdapter path stored in the archived SettlementReport." }, "reference_implementation": { "summary": "Loads historical CLOB snapshots and archived report envelopes for the replay window, reconstructs the execution path tick by tick, emits replay-tagged OperationsReport per simulated fill and an aggregate report at run end. All outputs carry mode=replay and replay_run_id.", "language_note": "Pseudocode is language-agnostic. FETCH = read input. EMIT = produce output. Translate to TS/Python/Go/Rust.", "pseudocode": "// ---- STARTUP ----\nFUNCTION initReplay(config):\n IF config.start_ts >= config.end_ts:\n RAISE ConfigError('BACKTESTER_INVALID_WINDOW')\n IF NOT strategyRegistry.has(config.strategy):\n RAISE ConfigError('BACKTESTER_UNKNOWN_STRATEGY')\n IF config.end_ts > now():\n config.end_ts = now()\n alerting.emit('BACKTESTER_WINDOW_CLAMPED')\n replay_run_id = generateULID() // e.g. 'replay_01HX9KZQ7E8VR5'\n variants = config.parameter_sweep OR [config.strategy.defaults]\n\n// ---- PER-VARIANT LOOP ----\nFOR variant IN variants:\n strategy = strategyRegistry.load(config.strategy, params=variant)\n\n // Fetch archived signals for the full window\n archive = FETCH internal.reportArchive.GET({\n window: [config.start_ts, config.end_ts],\n kinds: ['ObservationReport', 'DecisionReport', 'RiskVote',\n 'ExecutionReport', 'SettlementReport']\n })\n\n // Fetch archived CLOB snapshots (tick-level)\n snapshots = FETCH internal.clobArchive.GET({\n window: [config.start_ts, config.end_ts],\n resolution: 'tick'\n })\n\n sim_fills = []\n sim_pnl_pusd = 0.0\n\n FOR tick IN snapshots:\n // Reconstruct order book at this tick\n book = reconstructBook(tick)\n\n // Replay archived signals at this tick timestamp\n signals = archive.signalsAt(tick.ts_ms)\n\n // Run strategy in replay mode\n intent = strategy.evaluate(book, signals, mode='replay')\n IF intent IS NULL:\n CONTINUE\n\n // Simulate risk guardrail stack (replay mode)\n votes = riskStack.simulate(intent, context={\n book: book, signals: signals, mode: 'replay'\n })\n shaped_intent = applyVotes(intent, votes)\n IF shaped_intent IS NULL:\n EMIT OperationsReport(event_type='BACKTESTER_INTENT_REJECTED',\n replay_run_id=replay_run_id, mode='replay', ...)\n CONTINUE\n\n // Simulate fill against historical book\n sim_fill = simulateFill(shaped_intent, book)\n sim_fill.size_pusd = toPusdUnits(sim_fill.size_usd)\n sim_fill.fee_pusd = sim_fill.size_pusd * sim_fill.fee_bps / 10_000\n sim_fill.pnl_pusd = computeRealisedPnL(sim_fill, costBasis='FIFO')\n sim_fills.append(sim_fill)\n sim_pnl_pusd += sim_fill.pnl_pusd\n\n // Emit per-fill replay OperationsReport\n EMIT OperationsReport({\n report_id: 'ops_backtester_' + replay_run_id + '_' + tick.ts_ms,\n bot_id: 'gov.backtester',\n event_type: 'BACKTESTER_TICK_PROCESSED',\n replay_run_id: replay_run_id,\n mode: 'replay',\n tick_ts_ms: tick.ts_ms,\n market_id: shaped_intent.market_id,\n simulated_fill: sim_fill,\n risk_votes: votes,\n topic: 'polytraders.reports.operations',\n partition: 'backtester+' + epochBucket(tick.ts_ms)\n })\n\n // Aggregate summary report at run end\n EMIT OperationsReport({\n event_type: 'BACKTESTER_REPLAY_COMPLETE',\n replay_run_id: replay_run_id,\n strategy: config.strategy,\n parameter_variant: variant,\n mode: 'replay',\n window_start: config.start_ts,\n window_end: config.end_ts,\n ticks_replayed: len(snapshots),\n simulated_fills: len(sim_fills),\n simulated_volume_pusd: SUM(f.size_pusd FOR f IN sim_fills),\n simulated_pnl_pusd: sim_pnl_pusd,\n simulated_net_fees_pusd: SUM(f.fee_pusd FOR f IN sim_fills),\n topic: 'polytraders.reports.operations',\n partition: 'backtester+' + epochBucket(config.end_ts),\n retained_until: config.end_ts + 1y\n })", "sdk_calls": [ "internal.reportArchive.GET({ window, kinds })", "internal.clobArchive.GET({ window, resolution: 'tick' })", "strategyRegistry.load(slug, params)", "riskStack.simulate(intent, context)", "toPusdUnits(raw_usd)", "alerting.emit('BACKTESTER_WINDOW_CLAMPED', metadata)" ], "complexity": "O(T * V) where T = ticks in replay window, V = parameter_sweep variants" }, "wire_examples": { "input": { "label": "Backtester run configuration", "source": "config_store", "payload": { "bot_id": "gov.backtester", "mode": "replay", "strategy": "sum_to_one_arb", "start_ts": "2026-05-01T00:00:00Z", "end_ts": "2026-05-08T00:00:00Z", "parameter_sweep": [ { "min_edge_bps": 12, "max_size_pusd": 1000 }, { "min_edge_bps": 15, "max_size_pusd": 1500 } ] } }, "output": { "label": "OperationsReport \u2014 BACKTESTER_REPLAY_COMPLETE (2-variant sweep)", "payload": { "report_id": "ops_backtester_replay_01HX9KZQ7E8VR5", "bot_id": "gov.backtester", "event_type": "BACKTESTER_REPLAY_COMPLETE", "replay_run_id": "replay_01HX9KZQ7E8VR5", "strategy": "sum_to_one_arb", "mode": "replay", "window_start": "2026-05-01T00:00:00Z", "window_end": "2026-05-08T00:00:00Z", "ticks_replayed": 483120, "parameter_variants_run": 2, "variants": [ { "variant_id": 0, "params": { "min_edge_bps": 12, "max_size_pusd": 1000 }, "simulated_fills": 214, "simulated_volume_pusd": 92450.0, "simulated_pnl_pusd": 1380.5, "simulated_net_fees_pusd": 231.1 }, { "variant_id": 1, "params": { "min_edge_bps": 15, "max_size_pusd": 1500 }, "simulated_fills": 178, "simulated_volume_pusd": 104200.0, "simulated_pnl_pusd": 1890.2, "simulated_net_fees_pusd": 260.5 } ], "report_kind": "OperationsReport", "topic": "polytraders.reports.operations", "partition": "backtester+2026-05-08T00:00Z", "retained_until": "2027-05-08" } } }, "reason_codes": [ { "code": "BACKTESTER_REPLAY_COMPLETE", "severity": "INFO", "meaning": "A replay run completed successfully; aggregate OperationsReport emitted.", "action": "No action \u2014 routine completion.", "user_message": "The backtest finished. Results are available in the governance audit report." }, { "code": "BACKTESTER_TICK_PROCESSED", "severity": "INFO", "meaning": "A single tick was processed during replay; per-tick OperationsReport emitted.", "action": "No action \u2014 operational heartbeat.", "user_message": "" }, { "code": "BACKTESTER_INVALID_WINDOW", "severity": "HARD_REJECT", "meaning": "start_ts >= end_ts; the replay window is invalid.", "action": "Reject the run configuration; emit alert.", "user_message": "The backtest start date must be before the end date." }, { "code": "BACKTESTER_UNKNOWN_STRATEGY", "severity": "HARD_REJECT", "meaning": "The strategy slug specified in the config is not in the strategy registry.", "action": "Reject the run; emit alert.", "user_message": "The requested strategy is not available for backtesting." }, { "code": "BACKTESTER_WINDOW_CLAMPED", "severity": "WARN", "meaning": "end_ts was in the future; clamped to now().", "action": "Clamp end_ts; emit WARN; continue replay.", "user_message": "The backtest end date was set to the most recent available data." }, { "code": "BACKTESTER_TICK_SKIPPED", "severity": "WARN", "meaning": "A tick was missing from the archive; the replay skipped that tick.", "action": "Log WARN; skip tick; continue from next available tick.", "user_message": "" }, { "code": "BACKTESTER_ARCHIVE_UNAVAILABLE", "severity": "HARD_REJECT", "meaning": "The report archive is entirely unavailable; the replay cannot start or has stalled.", "action": "Abort the run; emit alert.", "user_message": "" }, { "code": "BACKTESTER_LARGE_SWEEP", "severity": "WARN", "meaning": "Parameter sweep has more than 50 variants; resource throttle applied.", "action": "Emit WARN; apply run throttle; continue sweep.", "user_message": "" }, { "code": "KILL_SWITCH_ACTIVE", "severity": "WARN", "meaning": "KillSwitch is active; new backtesting run launches are suppressed.", "action": "Suppress new run; emit WARN; in-progress runs continue.", "user_message": "New backtests are paused while the system kill switch is active." } ], "metrics": { "emitted": [ { "name": "polytraders_gov_backtester_runs_total", "type": "counter", "unit": "count", "labels": [ "strategy", "status" ], "meaning": "Total backtester runs launched, labelled by strategy slug and completion status (complete/aborted)." }, { "name": "polytraders_gov_backtester_ticks_processed_total", "type": "counter", "unit": "count", "labels": [ "strategy", "replay_run_id" ], "meaning": "Total ticks processed across all replay runs." }, { "name": "polytraders_gov_backtester_simulated_fills_total", "type": "counter", "unit": "count", "labels": [ "strategy" ], "meaning": "Total simulated fills produced across all runs." }, { "name": "polytraders_gov_backtester_simulated_pnl_pusd", "type": "gauge", "unit": "usd", "labels": [ "strategy", "replay_run_id" ], "meaning": "Simulated P&L in pUSD for the most recent completed replay run per strategy." }, { "name": "polytraders_gov_backtester_ticks_skipped_total", "type": "counter", "unit": "count", "labels": [ "strategy" ], "meaning": "Ticks skipped due to archive gaps. Should remain near zero." }, { "name": "polytraders_gov_backtester_run_duration_ms", "type": "histogram", "unit": "ms", "labels": [ "strategy" ], "meaning": "Wall-clock duration of a complete backtest run." } ], "alerts": [ { "name": "BacktesterArchiveUnavailable", "condition": "rate(polytraders_gov_backtester_runs_total{status='aborted'}[15m]) > 0", "severity": "page", "runbook": "#runbook-backtester-archive-unavailable" }, { "name": "BacktesterHighTickSkipRate", "condition": "rate(polytraders_gov_backtester_ticks_skipped_total[10m]) / rate(polytraders_gov_backtester_ticks_processed_total[10m]) > 0.01", "severity": "warn", "runbook": "#runbook-backtester-tick-skip" }, { "name": "BacktesterNoRunsIn24h", "condition": "rate(polytraders_gov_backtester_runs_total[24h]) == 0", "severity": "warn", "runbook": "#runbook-backtester-no-runs" }, { "name": "BacktesterRunDurationHigh", "condition": "histogram_quantile(0.99, polytraders_gov_backtester_run_duration_ms) > 300000", "severity": "warn", "runbook": "#runbook-backtester-slow-run" } ], "dashboards": [ "Grafana \u2014 Governance / Backtester run history and simulated P&L by strategy", "Grafana \u2014 Governance / Parameter sweep results comparison" ], "log_level": "info" }, "state": { "store": "postgres", "shape": "replay_runs table: { replay_run_id, strategy, start_ts, end_ts, status, ticks_replayed, simulated_fills, simulated_pnl_pusd, created_at }. Per-tick replay events stored as OperationsReport records on the report bus.", "ttl": "1 year (governance_audit retention)", "recovery": "On restart, any in-progress replay run is marked as interrupted. A new run must be launched manually. No partial replay state is held in memory.", "size_estimate": "~500 B per replay run summary; ~200 B per tick event in the report bus; ~1 MB per 5,000-tick run" }, "concurrency": { "execution_model": "single-threaded event loop per replay run; multiple runs may execute in parallel up to max_in_flight", "max_in_flight": 5, "idempotency_key": "replay_run_id", "timeout_ms": 300000, "backpressure": "reject new run launches beyond max_in_flight; emit BACKTESTER_RUN_QUEUE_FULL warn", "locking": "Postgres unique constraint on replay_run_id; in-run state is per-goroutine" }, "dependencies": { "depends_on": [ { "bot_id": "internal.report_archive", "why": "All historical ObservationReport, DecisionReport, RiskVote, ExecutionReport, and SettlementReport envelopes are fetched from the report archive for replay reconstruction." }, { "bot_id": "internal.clob_archive", "why": "Tick-level CLOB snapshots (order book + trades) are the primary replay data source for simulated fill computation." } ], "emits_to": [ { "bot_id": "internal.governance_audit", "what": "Replay-tagged OperationsReport records (per-tick and aggregate) emitted to polytraders.reports.operations" } ], "sibling": [ { "bot_id": "gov.paper-trade-runner", "why": "Paper-Trade Runner uses live data in paper mode; Backtester uses archived data in replay mode. Both emit OperationsReport to the same topic." } ], "external": [ { "service": "Internal report archive", "sla": "99.9% (internal SRE target)", "fallback": "Abort replay run; emit BACKTESTER_ARCHIVE_UNAVAILABLE; retry after archive recovery." }, { "service": "Internal CLOB snapshot archive", "sla": "99.9% (internal SRE target)", "fallback": "Skip ticks with missing snapshots; emit BACKTESTER_TICK_SKIPPED per gap." } ] }, "security_surfaces": { "signs_orders": false, "private_key_access": "none", "abuse_vectors": [ "Submitting a large parameter_sweep (>200 variants) to exhaust archive query budget", "Injecting a crafted replay window that targets gaps in the archive to produce misleading P&L results", "Manipulating strategy registry to replay a strategy not approved for backtesting" ], "mitigations": [ "parameter_sweep hard limit of 200 enforced at config load; excess rejected with PARAMETER_CHANGE_REQUIRES_APPROVAL", "Backtester is locked to mode=replay; live CLOB and onchain surfaces are never accessed", "Strategy registry is read-only within the backtester; only registered strategies can be loaded", "All replay runs are audit-logged with replay_run_id and emitted to the governance report bus" ], "contract_calls": [] }, "failure_injection": [ { "scenario": "ARCHIVE_UNAVAILABLE", "how_to_inject": "Block all reads from internal.reportArchive and internal.clobArchive", "expected_behavior": "BACKTESTER_ARCHIVE_UNAVAILABLE raised; run aborted; OperationsReport with status=aborted emitted", "recovery": "Once archive is reachable, launch a new replay run. No partial state needs clearing." }, { "scenario": "TICK_GAPS_IN_ARCHIVE", "how_to_inject": "Delete 5% of tick records from clobArchive for the target window", "expected_behavior": "BACKTESTER_TICK_SKIPPED emitted per missing tick; replay continues; aggregate report notes skipped ticks", "recovery": "Automatic; skipped ticks are noted in the aggregate report." }, { "scenario": "INVALID_WINDOW", "how_to_inject": "Submit run with start_ts > end_ts", "expected_behavior": "BACKTESTER_INVALID_WINDOW raised; run rejected immediately", "recovery": "Correct the window configuration and resubmit." }, { "scenario": "LARGE_PARAMETER_SWEEP", "how_to_inject": "Submit parameter_sweep with 201 variants", "expected_behavior": "PARAMETER_CHANGE_REQUIRES_APPROVAL raised; run rejected", "recovery": "Reduce sweep size to <= 200 or request approval." }, { "scenario": "KILL_SWITCH_DURING_LAUNCH", "how_to_inject": "Activate KillSwitch; submit a new backtester run", "expected_behavior": "KILL_SWITCH_ACTIVE logged; new run not started; in-progress runs continue to completion", "recovery": "Deactivate KillSwitch; resubmit run." }, { "scenario": "STRATEGY_NOT_IN_REGISTRY", "how_to_inject": "Submit run with strategy='unknown_strategy'", "expected_behavior": "BACKTESTER_UNKNOWN_STRATEGY raised; run rejected", "recovery": "Use a registered strategy slug." } ], "runbook": { "summary": "Backtester incidents involve archive unavailability (run aborts), high tick-skip rates (data quality issue), or runs stuck in progress (goroutine leak or resource exhaustion). All incidents are low-urgency unless they block a promotion gate decision.", "oncall_actions": [ { "alert": "BacktesterArchiveUnavailable", "first_step": "Check internal report archive and CLOB archive health. Verify storage backend is reachable.", "escalation": "Governance pod lead if archive is down for > 30 minutes" }, { "alert": "BacktesterHighTickSkipRate", "first_step": "Check clobArchive completeness for the affected window. May indicate a historical data ingestion failure.", "escalation": "Governance pod lead; data engineering if archive ingestion is the root cause" }, { "alert": "BacktesterNoRunsIn24h", "first_step": "Check if any promotion-gate backtests are pending. If yes, investigate whether run launches are being suppressed by KillSwitch.", "escalation": "Governance pod lead" }, { "alert": "BacktesterRunDurationHigh", "first_step": "Check archive query latency and concurrent run count. Reduce max_in_flight if resource contention is the cause.", "escalation": "SRE on-call if duration > 10 minutes consistently" } ], "manual_overrides": [ { "command": "polytraders gov backtest abort --replay-run-id ", "effect": "Aborts a stuck replay run; use after investigating root cause. Originally: A replay run is stuck in progress and blocking the run queue." } ], "healthcheck": "/internal/health/backtester \u2192 200 Last run completed < 24h ago; archive reachable; max_in_flight not saturated; ticks_skipped_rate < 0.01.; red if Archive unreachable; all runs aborting; run queue saturated for > 5 minutes." }, "promotion_gates": { "to_shadow": [ { "gate": "Unit tests pass for window validation, strategy loading, and archive gap handling", "how_measured": "CI test run", "threshold": "100% pass" }, { "gate": "Postgres replay_runs schema migration verified", "how_measured": "Integration test", "threshold": "Pass" } ], "to_limited_live": [ { "gate": "7-day replay of sum_to_one_arb completes in < 5 minutes with < 0.1% tick skip rate", "how_measured": "polytraders_gov_backtester_run_duration_ms + ticks_skipped_rate", "threshold": "< 5 min, < 0.1% skips" }, { "gate": "All emitted OperationsReport records carry mode=replay and replay_run_id", "how_measured": "Report bus audit; sample 100 records", "threshold": "100% compliance" } ], "to_general_live": [ { "gate": "Parameter sweep of 5 variants completes deterministically: identical inputs produce identical outputs on two successive runs", "how_measured": "Determinism test: run same sweep twice; compare simulated_pnl_pusd per variant", "threshold": "0 delta between runs" }, { "gate": "Promotion-gate backtest evidence accepted for one strategy live promotion decision", "how_measured": "Governance pod review", "threshold": "Pass" } ] }, "reporting": { "emits_kinds": [ "OperationsReport" ], "topics": [ "polytraders.reports.operations" ], "cadence": "every-event", "retention_class": "1y", "sampling_rule": "batched-1/min", "bus_failure_action": "drop-after-buffer", "user_visible": "no", "consumes_kinds": [ "ObservationReport", "DecisionReport", "RiskVote", "ExecutionReport", "SettlementReport", "OperationsReport" ] }, "capital_impact": "Indirect", "v3_status": { "phase": 7, "phase_name": "Governance & replay", "docs": { "done": 27, "total": 27, "state": "done" }, "impl": { "done": 0, "total": 15, "state": "pending" }, "runtime": { "done": 0, "total": 8, "state": "pending" }, "overall": "pending" } }