{ "schema_version": "1.0.0", "bot_id": "1.2", "bot_name": "KillSwitch", "slug": "killswitch", "layer": "Risk", "layer_key": "risk", "bot_class": "Guardrail", "authority": [ "Pause", "Reject" ], "status": "live", "readiness": "General live", "flagship": false, "is_reference": true, "public_export": false, "identity": { "layer": "Risk", "bot_class": "Guardrail", "authority": "Pause, Reject", "runs_before": "All other guardrails and ExecutionPlan emit", "runs_after": "Any triggering event (drawdown breach, reject-rate spike, manual flag, feed loss)", "applies_to": "All OrderIntents \u2014 checked first in the guardrail pipeline", "default_mode": "general_live", "user_visible": "Yes", "developer_owner": "Polytraders core \u2014 Risk pod" }, "purpose": "KillSwitch is the top-level emergency stop for the entire trading system. It can be triggered automatically when intraday or weekly drawdown exceeds a threshold, when the order-reject rate spikes above a circuit-breaker level, or when a market data feed is lost with open positions. It can also be triggered manually via the Admin UI. Once active, KillSwitch rejects every incoming OrderIntent without exception until a manual reset is performed (if require_manual_reset=true) or the trigger condition clears. It does not modify orders \u2014 it only blocks them entirely.", "why_it_matters": [ { "failure": "Runaway loss not stopped automatically", "consequence": "Without a drawdown circuit breaker, a strategy that enters a losing streak continues to trade, compounding losses until a human intervenes \u2014 which may be too late.", "worked_example": { "setup": "Account drawdown crosses -5% intraday at 16:42 UTC. Three strategies are still open: maker_tight on 4 markets, cross_market_arb on 2 markets, news_materiality_trader on 1 market. Total notional 42,000 pUSD.", "without_bot": "Each strategy continues to submit OrderIntents on its own cadence. By 17:11 UTC drawdown is -8.4% and one strategy has averaged down twice. The team only sees the breach when the daily report runs at 18:00 UTC.", "with_bot": "KillSwitch crosses the -5% threshold at 16:42:09 and flips to active. Every subsequent OrderIntent across all three strategies is rejected with `KILLSWITCH_ACTIVE`. require_manual_reset=true, so the trip persists until an operator clears it." } }, { "failure": "API or wallet misbehaviour not detected", "consequence": "A spike in order rejections from the exchange or a wallet desync can indicate a broken execution path. Continuing to submit orders under these conditions risks unintended positions or double-orders." }, { "failure": "Data feed lost with open positions", "consequence": "If the CLOB WebSocket feed dies and positions remain open, the system cannot monitor or hedge those positions. The safe action is to halt new orders until the feed is restored." }, { "failure": "No single manual override path", "consequence": "In an incident, teams need a single, reliable mechanism to stop all trading immediately. Without a centralised kill signal, individual strategy shutdowns may be missed." } ], "polymarket_inputs": [ { "input": "CLOB WebSocket connection status", "source": "WebSocket", "required": true, "use": "Detect feed loss; trigger KillSwitch if WebSocket has been disconnected for more than the allowed dead window while positions are open." }, { "input": "Order reject events from the exchange", "source": "CLOB", "required": true, "use": "Count reject events per minute; if reject rate exceeds reject_rate_circuit threshold, trigger KillSwitch." } ], "internal_inputs": [ { "input": "Rolling intraday and weekly P&L", "source": "PortfolioGuard", "required": true, "use": "Monitor drawdown against intraday_drawdown_pct and weekly_drawdown_pct thresholds to trigger the circuit breaker." }, { "input": "Manual kill flag from operator dashboard", "source": "Admin UI", "required": true, "use": "Accept a manual activation signal that overrides all automatic conditions and immediately halts trading." }, { "input": "Open position count", "source": "PortfolioGuard", "required": false, "use": "Condition the WebSocket-dead trigger on whether any positions are currently open; if no positions exist, feed loss alone does not trigger KillSwitch." } ], "raw_params": [ "intraday_drawdown_pct \u00b7 0\u201350", "weekly_drawdown_pct \u00b7 0\u201350", "reject_rate_circuit \u00b7 0\u2013100", "require_manual_reset \u00b7 bool" ], "parameters": [ { "name": "intraday_drawdown_pct", "default": 12, "warning": 8, "hard": 12, "controls": "Intraday drawdown (as a percentage of start-of-day balance) at which KillSwitch triggers automatically.", "why_default_matters": "A 12% intraday drawdown is a significant adverse move that most strategies are not designed to continue trading through. Stopping at this level limits the maximum single-day loss.", "threshold_logic": [ { "condition": "Intraday drawdown \u2264 8%", "action": "No action" }, { "condition": "8\u201312%", "action": "WARN \u2014 log alert, no block yet" }, { "condition": "> 12%", "action": "ACTIVATE KillSwitch \u2014 reject all new orders" } ], "dev_check": "if (intradayDrawdownPct > p.hard) killswitch.activate('INTRADAY_DRAWDOWN_EXCEEDED');", "user_facing": "Trading has been paused because today's losses reached the daily safety limit. No new orders will be placed until the limit is reset." }, { "name": "weekly_drawdown_pct", "default": 20, "warning": 15, "hard": 20, "controls": "Rolling 7-day drawdown (as a percentage of start-of-week balance) at which KillSwitch triggers automatically.", "why_default_matters": "A 20% weekly drawdown indicates a sustained losing period. Halting trading at this level prevents a single bad week from damaging the account irreparably.", "threshold_logic": [ { "condition": "Weekly drawdown \u2264 15%", "action": "No action" }, { "condition": "15\u201320%", "action": "WARN \u2014 log alert, no block yet" }, { "condition": "> 20%", "action": "ACTIVATE KillSwitch \u2014 reject all new orders" } ], "dev_check": "if (weeklyDrawdownPct > p.hard) killswitch.activate('WEEKLY_DRAWDOWN_EXCEEDED');", "user_facing": "Trading has been paused because this week's total losses reached the weekly safety limit." }, { "name": "reject_rate_circuit", "default": 30, "warning": 20, "hard": 30, "controls": "Order reject rate (as a percentage of submitted orders in a rolling 5-minute window) at which KillSwitch triggers automatically.", "why_default_matters": "A 30% reject rate is far outside normal operating conditions and indicates a systemic problem \u2014 wallet desync, nonce collision, or exchange-side issue \u2014 that requires investigation before trading continues.", "threshold_logic": [ { "condition": "Reject rate \u2264 20% over 5 min", "action": "No action" }, { "condition": "20\u201330%", "action": "WARN \u2014 alert monitoring" }, { "condition": "> 30%", "action": "ACTIVATE KillSwitch \u2014 reject all new orders" } ], "dev_check": "if (rejectRate5min > p.hard) killswitch.activate('ORDER_BOOK_UNAVAILABLE');", "user_facing": "A high number of orders are being rejected by the exchange. Trading has been paused while the issue is investigated." }, { "name": "require_manual_reset", "default": true, "warning": null, "hard": null, "controls": "When true, KillSwitch remains active after the triggering condition clears and can only be deactivated by a manual action in the Admin UI.", "why_default_matters": "Automatic reactivation after a drawdown or reject-rate breach may resume trading before the root cause is understood. Requiring a manual reset ensures a human reviews the situation first.", "threshold_logic": [ { "condition": "require_manual_reset=true", "action": "KillSwitch stays active until Admin UI reset; condition clearing alone is insufficient" }, { "condition": "require_manual_reset=false", "action": "KillSwitch deactivates automatically when the triggering condition falls below the warning threshold" } ], "dev_check": "if (!p.require_manual_reset && triggerConditionCleared) killswitch.deactivate();", "user_facing": "Trading has been stopped for safety and requires a manual review before it can resume." } ], "default_config": { "bot_id": "risk.kill_switch", "version": "1.0.0", "mode": "hard_guard", "defaults": { "intraday_drawdown_pct": 12, "weekly_drawdown_pct": 20, "reject_rate_circuit": 30, "require_manual_reset": true }, "locked": { "require_manual_reset": { "immutable": true }, "intraday_drawdown_pct": { "max": 20 }, "weekly_drawdown_pct": { "max": 30 } } }, "implementation_flow": [ "On each OrderIntent received, check the KillSwitch active flag in shared state before any other processing.", "If active flag is set, immediately return REJECT with reason_code matching the trigger (e.g. STRATEGY_BUDGET_EXCEEDED for drawdown, ORDER_BOOK_UNAVAILABLE for reject-rate or feed loss) \u2014 do not consult any other data.", "In a parallel monitoring loop, continuously read rolling intraday and weekly drawdown from PortfolioGuard.", "If intraday_drawdown_pct or weekly_drawdown_pct hard limit is breached, atomically set the active flag and record the trigger reason, timestamp, and metric value.", "In the same monitoring loop, count order reject events from the CLOB over a rolling 5-minute window; if reject_rate_circuit hard limit is breached, set the active flag.", "Monitor the CLOB WebSocket connection status; if the connection has been dead for more than 30 seconds and open positions exist, set the active flag with reason ORDER_BOOK_UNAVAILABLE.", "Listen for manual kill signal from Admin UI; on receipt, set the active flag immediately with reason MANUAL_KILL regardless of metric levels.", "When KillSwitch is active, emit a system-level alert to the monitoring stack on every rejected order.", "If require_manual_reset=true, ignore all automatic condition-clearing events; wait for explicit Admin UI reset.", "On reset, clear the active flag, log the reset timestamp and operator action, and resume the normal guardrail pipeline." ], "decision_logic": { "approve": "KillSwitch active flag is false \u2014 no triggering conditions are met. All orders pass through to the rest of the guardrail pipeline.", "reshape_required": "Not applicable \u2014 KillSwitch does not reshape orders. It either passes them through or rejects them entirely.", "reject": "KillSwitch active flag is true for any reason: intraday drawdown exceeded, weekly drawdown exceeded, order reject rate too high, CLOB feed dead with open positions, or manual kill from Admin UI.", "warning_only": "Not used \u2014 KillSwitch has reject and pause authority. Warning-level metrics (approaching thresholds) are logged as alerts but do not activate the switch until the hard threshold is breached." }, "decision_output_schema": "RiskVote", "decision_output_example": { "guard_id": "risk.kill_switch", "decision": "REJECT", "severity": "HARD", "reason_code": "STRATEGY_BUDGET_EXCEEDED", "message": "KillSwitch is active. Intraday drawdown of 13.2% exceeded the 12% circuit breaker. All new orders are rejected. Manual reset required.", "constraints": {}, "trigger_reason": "INTRADAY_DRAWDOWN_EXCEEDED", "trigger_metric": 0.132, "activated_at": "2026-05-09T09:10:00Z", "inputs_used": [ "portfolio_guard.drawdown", "admin_ui.kill_flag" ], "checked_at": "2026-05-09T09:11:05Z" }, "developer_log": { "bot_id": "risk.kill_switch", "decision": "REJECT", "reason_code": "STRATEGY_BUDGET_EXCEEDED", "trigger_reason": "INTRADAY_DRAWDOWN_EXCEEDED", "inputs_used": [ "portfolio_guard.drawdown" ], "metrics": { "intraday_drawdown_pct": 13.2, "weekly_drawdown_pct": 8.4, "reject_rate_5min": 4.1, "ws_dead_seconds": 0, "open_positions": 3 }, "activated_at": "2026-05-09T09:10:00Z", "require_manual_reset": true, "checked_at": "2026-05-09T09:11:05Z" }, "user_explanations": [ { "situation": "Trading stopped \u2014 daily loss limit", "message": "All trading has been paused because today's total losses reached the daily safety limit. No new orders will be placed until you manually review and restart from the settings panel." }, { "situation": "Trading stopped \u2014 weekly loss limit", "message": "All trading has been paused because this week's total losses reached the weekly safety limit. A manual review is required before trading can resume." }, { "situation": "Trading stopped \u2014 order rejection spike", "message": "A large number of orders were rejected by the exchange in a short period. Trading has been paused while the issue is investigated to prevent placing orders that may not behave as expected." }, { "situation": "Trading stopped \u2014 market data connection lost", "message": "The live market data connection was interrupted while positions were open. Trading has been paused until the connection is restored to ensure decisions are made on current information." }, { "situation": "Trading stopped \u2014 manual action", "message": "Trading was manually paused. No new orders will be placed until the pause is lifted from the settings panel." } ], "failure_modes": { "main_failure_mode": "KillSwitch failing to activate during a genuine incident \u2014 for example, if the drawdown data feed from PortfolioGuard is itself unavailable, the circuit breaker never fires.", "false_positive_risk": "Triggering on a transient spike in the reject rate caused by a brief exchange-side issue that resolves in seconds, halting trading unnecessarily.", "false_negative_risk": "Not activating if the drawdown calculation is stale due to a P&L feed lag, allowing losses to accumulate beyond the stated hard limit before the trigger fires.", "safe_fallback": "If the drawdown or reject-rate data feed is unavailable for more than 60 seconds, KillSwitch activates automatically with reason STALE_MARKET_DATA. Uncertainty about system health defaults to halt, not continue.", "required_dependencies": [ "PortfolioGuard rolling P&L feed", "CLOB order reject event stream", "CLOB WebSocket connection health check", "Admin UI kill-flag channel" ] }, "acceptance_tests": { "unit": [ { "test": "Reject all orders when active flag is true", "setup": "killswitch.active=true, order=any", "expected": "REJECT with appropriate reason_code on every order" }, { "test": "Activate on intraday drawdown breach", "setup": "intraday_drawdown_pct=13, threshold=12", "expected": "active flag set to true, trigger_reason=INTRADAY_DRAWDOWN_EXCEEDED" }, { "test": "Activate on weekly drawdown breach", "setup": "weekly_drawdown_pct=22, threshold=20", "expected": "active flag set to true, trigger_reason=WEEKLY_DRAWDOWN_EXCEEDED" }, { "test": "Activate on reject-rate circuit breach", "setup": "reject_rate_5min=35, threshold=30", "expected": "active flag set to true, trigger_reason=ORDER_BOOK_UNAVAILABLE" }, { "test": "Activate on manual kill signal", "setup": "admin_ui.kill_flag=true", "expected": "active flag set to true, trigger_reason=MANUAL_KILL" }, { "test": "Does not auto-reset when require_manual_reset=true even after condition clears", "setup": "active=true, drawdown drops below warning, require_manual_reset=true", "expected": "active flag remains true; orders continue to be rejected" } ], "integration": [ { "test": "End-to-end: drawdown breach in PortfolioGuard activates KillSwitch and blocks next OrderIntent", "expected": "Next OrderIntent after drawdown breach returns REJECT without reaching LiquidityGuard or OracleRiskMonitor" }, { "test": "WebSocket dead with open positions activates KillSwitch within 30 seconds", "expected": "KillSwitch active flag set within 30 s of WebSocket disconnection when open_positions > 0" }, { "test": "Admin UI reset clears active flag and allows orders through again", "expected": "OrderIntents approved by other guardrails pass through normally after Admin UI reset" } ], "property": [ { "property": "No order is ever approved when KillSwitch active flag is true", "required": "Always true \u2014 active flag is checked atomically before any other processing" }, { "property": "KillSwitch activation is monotonic when require_manual_reset=true", "required": "Once active, the flag cannot be cleared by any automated process; only a manual reset counts" }, { "property": "Missing drawdown data triggers activation, never approval", "required": "Always true \u2014 data feed unavailability defaults to halt" } ] }, "checklist_overrides": {}, "legacy_goal": "One-button \u2014 or auto-triggered \u2014 flatten-all when something breaks.", "legacy_pm_signals": [ "Equity drawdown breaching circuit-breaker threshold", "Order-reject rate spike (API misbehaviour, wallet desync)", "CLOB WebSocket dead > 30s with open positions" ], "legacy_external_feeds": [ "Admin UI manual flag" ], "network": [ "polygon" ], "api_surface": [ "ws_market", "clob_auth" ], "reference_implementation": { "summary": "Runs two concurrent loops: a fast per-intent check that reads a single Redis key, and a background monitor that watches P&L, reject rate, and WS feed health. The Redis key is the single source of truth for the active flag.", "language_note": "Pseudocode is language-agnostic. FETCH = read input. EMIT = produce output. Translate to TS/Python/Go/Rust.", "pseudocode": "// ---- FAST PATH: called on every OrderIntent ----\nFUNCTION checkKillSwitch(intent):\n state = FETCH redis.GET('killswitch:state') // O(1) Redis read\n IF state.active:\n EMIT RiskVote(decision=HARD_REJECT,\n reason=KILL_SWITCH_ACTIVE,\n trigger_reason=state.trigger_reason,\n activated_at=state.activated_at)\n RETURN\n // Pass through \u2014 other guardrails take over\n EMIT RiskVote(decision=APPROVE)\n\n// ---- BACKGROUND MONITOR: runs every 5s ----\nFUNCTION monitorLoop():\n WHILE true:\n // 1. Intraday drawdown\n drawdown = FETCH internal.portfolio_guard.drawdown_pct()\n IF drawdown > params.intraday_drawdown_pct.hard:\n activate('INTRADAY_DRAWDOWN_EXCEEDED')\n\n // 2. Weekly drawdown\n weeklyDrawdown = FETCH internal.portfolio_guard.weekly_drawdown_pct()\n IF weeklyDrawdown > params.weekly_drawdown_pct.hard:\n activate('WEEKLY_DRAWDOWN_EXCEEDED')\n\n // 3. Order reject rate\n rejectRate = FETCH internal.clob_reject_counter.rate_5min()\n IF rejectRate > params.reject_rate_circuit.hard:\n activate('ORDER_BOOK_UNAVAILABLE')\n\n // 4. WebSocket feed health\n wsDead = FETCH ws_market.seconds_since_last_message()\n openPositions = FETCH internal.portfolio_guard.open_position_count()\n IF wsDead > 30 AND openPositions > 0:\n activate('ORDER_BOOK_UNAVAILABLE')\n\n // 5. Data feed staleness\n IF drawdown IS NULL AND wsDead > 60:\n activate('STALE_MARKET_DATA') // fail-safe: unknown system health\n\n SLEEP 5\n\n// ---- ACTIVATE / DEACTIVATE ----\nFUNCTION activate(triggerReason):\n state = { active: true, trigger_reason: triggerReason, activated_at: now_iso() }\n redis.SET('killswitch:state', state) // atomic write\n EMIT alert(P0, 'KillSwitchActivated', state)\n\nFUNCTION deactivate(operator):\n IF params.require_manual_reset:\n // Only admin UI can call this\n redis.SET('killswitch:state', { active: false, reset_by: operator, reset_at: now_iso() })\n ELSE:\n IF triggerConditionCleared():\n redis.SET('killswitch:state', { active: false })\n", "helpers": [ { "name": "isStale", "signature": "isStale(snapshot: any, maxAgeS: int) -> bool", "purpose": "Returns true if snapshot was produced more than maxAgeS seconds ago; used for data-feed staleness gate." }, { "name": "toUsdcUnits", "signature": "toUsdcUnits(rawUsd: float) -> int", "purpose": "Not used directly; imported for consistency with Risk pod SDK imports." }, { "name": "fetchClobPublic", "signature": "fetchClobPublic(path: str) -> JSON", "purpose": "Used by the background monitor to sanity-check CLOB reachability." }, { "name": "buildOrderTypedData", "signature": "buildOrderTypedData(intent, domain) -> TypedData", "purpose": "Not called by KillSwitch; referenced here because SmartRouter downstream calls it \u2014 KillSwitch's HARD_REJECT prevents it from running." } ], "sdk_calls": [ "redis.GET('killswitch:state')", "redis.SET('killswitch:state', state)", "internal.portfolio_guard.drawdown_pct()", "internal.portfolio_guard.weekly_drawdown_pct()", "ws_market.seconds_since_last_message()" ], "complexity": "O(1) per intent (single Redis read); background monitor O(1) per cycle" }, "wire_examples": { "input": [ { "label": "OrderIntent arriving while KillSwitch is active", "source": "internal", "payload": { "intent_id": "int_8e9f0a1b2c3d4e5f", "market_id": "0x4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d", "side": "BUY", "size_usd": 500, "generated_at": "2026-05-09T09:11:00Z" } }, { "label": "Redis KillSwitch state (active)", "source": "internal", "payload": { "active": true, "trigger_reason": "INTRADAY_DRAWDOWN_EXCEEDED", "trigger_metric": 0.132, "activated_at": "2026-05-09T09:10:00Z", "require_manual_reset": true } } ], "output": [ { "label": "RiskVote \u2014 HARD_REJECT (KillSwitch active, drawdown breach)", "payload": { "guard_id": "risk.kill_switch", "decision": "HARD_REJECT", "severity": "HARD", "reason_code": "KILL_SWITCH_ACTIVE", "message": "KillSwitch is active. Intraday drawdown of 13.2% exceeded the 12% circuit breaker. All new orders are rejected. Manual reset required.", "constraints": {}, "trigger_reason": "INTRADAY_DRAWDOWN_EXCEEDED", "trigger_metric": 0.132, "activated_at": "2026-05-09T09:10:00Z", "inputs_used": [ "redis.killswitch_state" ], "checked_at": "2026-05-09T09:11:05Z" } }, { "label": "RiskVote \u2014 APPROVE (KillSwitch not active)", "payload": { "guard_id": "risk.kill_switch", "decision": "APPROVE", "severity": "INFO", "reason_code": null, "inputs_used": [ "redis.killswitch_state" ], "checked_at": "2026-05-09T10:00:00Z" } } ], "curl": null }, "reason_codes": [ { "code": "KILL_SWITCH_ACTIVE", "severity": "HARD_REJECT", "meaning": "The KillSwitch active flag is set; all orders are rejected regardless of other conditions.", "action": "Return HARD_REJECT immediately on every intent.", "user_message": "Trading is currently paused. Please review the settings panel for details." }, { "code": "STRATEGY_BUDGET_EXCEEDED", "severity": "HARD_REJECT", "meaning": "Intraday or weekly drawdown has exceeded the hard circuit-breaker threshold.", "action": "Activate KillSwitch with trigger_reason=INTRADAY_DRAWDOWN_EXCEEDED or WEEKLY_DRAWDOWN_EXCEEDED.", "user_message": "All trading has been paused because today's or this week's losses reached the safety limit. A manual review is required before trading can resume." }, { "code": "STALE_MARKET_DATA", "severity": "HARD_REJECT", "meaning": "P&L or WS feed data is unavailable for more than 60s; system health is unknown.", "action": "Activate KillSwitch with trigger_reason=STALE_MARKET_DATA.", "user_message": "Trading has been paused because market data became unavailable. It will resume once the connection is restored." }, { "code": "KILL_SWITCH_INTRADAY_DRAWDOWN", "severity": "HARD_REJECT", "meaning": "Intraday drawdown exceeded intraday_drawdown_pct hard threshold.", "action": "Activate KillSwitch; set trigger_reason=INTRADAY_DRAWDOWN_EXCEEDED.", "user_message": "Today's losses reached the daily limit. Trading has been paused." }, { "code": "KILL_SWITCH_WEEKLY_DRAWDOWN", "severity": "HARD_REJECT", "meaning": "Rolling 7-day drawdown exceeded weekly_drawdown_pct hard threshold.", "action": "Activate KillSwitch; set trigger_reason=WEEKLY_DRAWDOWN_EXCEEDED.", "user_message": "This week's losses reached the weekly limit. Trading has been paused." }, { "code": "KILL_SWITCH_REJECT_RATE", "severity": "HARD_REJECT", "meaning": "Order reject rate from the CLOB exceeded reject_rate_circuit threshold in a 5-minute window.", "action": "Activate KillSwitch; set trigger_reason=ORDER_BOOK_UNAVAILABLE.", "user_message": "A high number of orders were rejected by the exchange. Trading has been paused while the issue is investigated." }, { "code": "KILL_SWITCH_FEED_DEAD", "severity": "HARD_REJECT", "meaning": "CLOB WebSocket feed has been dead for > 30s with open positions.", "action": "Activate KillSwitch; set trigger_reason=ORDER_BOOK_UNAVAILABLE.", "user_message": "The live market data connection was interrupted while positions were open. Trading has been paused." }, { "code": "KILL_SWITCH_MANUAL", "severity": "HARD_REJECT", "meaning": "Operator manually activated KillSwitch via Admin UI.", "action": "Activate KillSwitch; set trigger_reason=MANUAL_KILL.", "user_message": "Trading was manually paused. It can be resumed from the settings panel." } ], "metrics": { "emitted": [ { "name": "polytraders_risk_killswitch_active", "type": "gauge", "unit": "count", "labels": [ "trigger_reason" ], "meaning": "1 if KillSwitch is active, 0 otherwise. Label carries the triggering reason." }, { "name": "polytraders_risk_killswitch_activations_total", "type": "counter", "unit": "count", "labels": [ "trigger_reason" ], "meaning": "Cumulative count of KillSwitch activations by trigger reason." }, { "name": "polytraders_risk_killswitch_rejections_total", "type": "counter", "unit": "count", "labels": [ "trigger_reason" ], "meaning": "Cumulative count of OrderIntents rejected by KillSwitch while active." }, { "name": "polytraders_risk_killswitch_active_duration_seconds", "type": "histogram", "unit": "seconds", "labels": [ "trigger_reason" ], "meaning": "Duration of each KillSwitch activation from trigger to manual reset." }, { "name": "polytraders_risk_killswitch_check_latency_ms", "type": "histogram", "unit": "seconds", "labels": [], "meaning": "Wall-clock latency of the Redis read on the fast-path check." } ], "alerts": [ { "name": "KillSwitchActivated", "condition": "polytraders_risk_killswitch_active > 0", "severity": "P0", "runbook": "#runbook-killswitch-activated" }, { "name": "KillSwitchActivatedManual", "condition": "polytraders_risk_killswitch_active{trigger_reason='MANUAL_KILL'} > 0", "severity": "P0", "runbook": "#runbook-killswitch-manual" }, { "name": "KillSwitchRedisUnreachable", "condition": "up{job='killswitch_redis'} == 0", "severity": "P0", "runbook": "#runbook-killswitch-redis" }, { "name": "KillSwitchHighCheckLatency", "condition": "histogram_quantile(0.99, rate(polytraders_risk_killswitch_check_latency_ms_bucket[5m])) > 10", "severity": "P1", "runbook": "#runbook-killswitch-latency" } ], "dashboards": [ "Grafana \u2014 Risk overview / KillSwitch status", "Grafana \u2014 Incident timeline / KillSwitch activations" ], "log_levels": { "DEBUG": "Redis read result on every fast-path check.", "INFO": "KillSwitch activation and deactivation events with trigger_reason and metric values.", "WARN": "Drawdown or reject rate approaching warning threshold.", "ERROR": "Redis unreachable; drawdown data feed unavailable." } }, "state": { "summary": "Stores a single active flag and trigger metadata in Redis. This key is the authoritative source of truth for all guardrails.", "stores": [ { "name": "killswitch_state", "kind": "redis", "key": "killswitch:state", "value": "{ active: bool, trigger_reason: str, trigger_metric: float, activated_at: iso_ts, require_manual_reset: bool }", "ttl": "none", "durability": "strong" } ], "recovery": "On cold start, read Redis key. If key is missing (first deploy or Redis flush), default to active=false and log a WARNING.", "on_restart": "Redis state is re-read immediately. If Redis is unreachable on startup, the bot activates with trigger_reason=STALE_MARKET_DATA until Redis is reachable." }, "concurrency": { "execution_model": "single-threaded event loop (fast path) + background monitor goroutine", "max_in_flight": 1000, "idempotency_key": "intent_id", "replay_safe": true, "deduplication": "by intent_id within a 24h window", "ordering_guarantees": "no ordering \u2014 KillSwitch check is stateless per intent", "timeout_ms": 10, "backpressure": "drop newest", "locking": "none (Redis atomic SET)" }, "dependencies": { "depends_on": [ { "bot_id": "risk.portfolio_guard", "why": "Provides drawdown data that drives the automatic circuit-breaker trigger.", "contract": "Monitor reads portfolio_guard.drawdown_pct() and weekly_drawdown_pct()." } ], "emits_to": [ { "bot_id": "risk.liquidity_guard", "why": "Every Risk bot consults KillSwitch first.", "contract": "HARD_REJECT(KILL_SWITCH_ACTIVE) short-circuits LiquidityGuard." }, { "bot_id": "risk.oracle_risk_monitor", "why": "Every Risk bot consults KillSwitch first.", "contract": "HARD_REJECT(KILL_SWITCH_ACTIVE) short-circuits OracleRiskMonitor." }, { "bot_id": "risk.portfolio_guard", "why": "PortfolioGuard checks KillSwitch first.", "contract": "HARD_REJECT(KILL_SWITCH_ACTIVE) short-circuits PortfolioGuard." }, { "bot_id": "exec.smart_router", "why": "SmartRouter checks KillSwitch before emitting any ExecutionPlan.", "contract": "No ExecutionPlan emitted while KillSwitch is active." } ], "sibling": [], "external": [ { "service": "Redis", "endpoint": "internal Redis cluster", "sla": "99.99% / <5ms p99", "failure_mode": "Fail-safe: if Redis is unreachable, activate KillSwitch immediately." }, { "service": "WS market feed", "endpoint": "wss://ws-subscriptions-clob.polymarket.com/ws/market", "sla": "best-effort", "failure_mode": "If feed dead > 30s with open positions, activate KillSwitch." } ] }, "security_surfaces": { "summary": "KillSwitch holds no secrets and makes no signed calls. Its only write target is the internal Redis key. The Admin UI kill flag channel must be authenticated and audit-logged.", "signing": "This bot does NOT sign anything.", "secrets": [], "contract_calls": [], "abuse_vectors": [ "Unauthorized Redis write to clear the active flag and resume trading without operator review", "Race condition where two activations compete and one clears the trigger_reason" ], "mitigations": [ "Redis SET is atomic; activation is monotonic when require_manual_reset=true", "Admin UI kill-flag channel requires authenticated operator session and is audit-logged", "Deactivation via Admin UI requires explicit confirmation and logs operator identity" ] }, "polymarket_v2_compat": { "clob_version": "v2", "collateral": "pUSD", "eip712_domain_version": "2", "builder_code_aware": false, "negrisk_aware": false, "multichain_ready": false, "sdk_used": "@polymarket/clob-client-v2 ^2.x", "settlement_contract": "CTFExchangeV2 on Polygon", "notes": "KillSwitch logic is V2-agnostic \u2014 it operates above the order-construction layer. The reject_rate_circuit trigger now counts V2 order rejections from CTFExchangeV2 matchOrders() calls." }, "version": { "spec": "2.0.0", "implementation": "2.1.3", "schema": "2", "released": "2026-04-28" }, "migration_history": [ { "date": "2026-04-28", "from": "v1 (USDC.e + HMAC builder)", "to": "v2 (pUSD + builderCode field)", "reason": "Polymarket V2 cutover", "action_taken": "Updated reject-rate counter to track V2 CTFExchangeV2 rejections. No structural change to the kill-switch logic or Redis schema." } ], "failure_injection": [ { "scenario": "REDIS_UNREACHABLE", "how_to_inject": "Block TCP to Redis cluster", "expected_behavior": "KillSwitch activates immediately with trigger_reason=STALE_MARKET_DATA; all intents return HARD_REJECT", "recovery": "KillSwitch returns to normal (active=false) only after manual reset once Redis is reachable." }, { "scenario": "DRAWDOWN_BREACH", "how_to_inject": "Set portfolio_guard mock to return drawdown_pct=0.13", "expected_behavior": "Background monitor activates KillSwitch within one 5s cycle; next intent returns HARD_REJECT(KILL_SWITCH_ACTIVE)", "recovery": "Manual reset after drawdown is confirmed resolved." }, { "scenario": "REJECT_RATE_SPIKE", "how_to_inject": "Inject 35 reject events in a 5-minute window", "expected_behavior": "Background monitor activates KillSwitch with trigger_reason=ORDER_BOOK_UNAVAILABLE", "recovery": "Manual reset after root cause is investigated." }, { "scenario": "WS_FEED_DEAD_WITH_POSITIONS", "how_to_inject": "Disconnect WS market feed while open_positions > 0", "expected_behavior": "Background monitor activates KillSwitch within 30s", "recovery": "Reconnect WS; manual reset." }, { "scenario": "MANUAL_KILL", "how_to_inject": "Submit kill flag from Admin UI", "expected_behavior": "Redis state set to active=true with trigger_reason=MANUAL_KILL immediately", "recovery": "Manual reset from Admin UI only." } ], "runbook": { "summary": "KillSwitch is the most critical bot in the system. Every activation is a P0 incident requiring immediate operator attention. Never auto-reset without confirming root cause.", "oncall_actions": [ { "alert": "KillSwitchActivated", "first_step": "Identify trigger_reason from the alert label or Redis state.", "diagnosis": "For INTRADAY_DRAWDOWN_EXCEEDED: check PortfolioGuard drawdown gauge. For ORDER_BOOK_UNAVAILABLE: check WS feed and CLOB status. For STALE_MARKET_DATA: check Redis and Data API connectivity.", "mitigation": "Do NOT reset until root cause is confirmed. Notify Risk pod lead.", "escalation": "Risk pod lead immediately on every KillSwitch activation." }, { "alert": "KillSwitchRedisUnreachable", "first_step": "Check Redis cluster health immediately.", "diagnosis": "If Redis is down, KillSwitch is in fail-safe mode (active). All trading is halted.", "mitigation": "Restore Redis connectivity. KillSwitch will resume normal operation once Redis is reachable.", "escalation": "Infra on-call immediately." }, { "alert": "KillSwitchHighCheckLatency", "first_step": "Check Redis p99 latency and network path.", "diagnosis": "High latency on the fast-path Redis read will delay HARD_REJECT on every intent.", "mitigation": "Switch to replica Redis node if primary is degraded.", "escalation": "Infra on-call if latency > 10ms sustained." } ], "manual_overrides": [ { "command": "polytraders bot reset killswitch", "effect": "Clears the active flag in Redis after operator confirms root cause resolved. Requires Risk pod lead sign-off and is audit-logged." }, { "command": "polytraders bot status killswitch", "effect": "Prints current Redis state including trigger_reason, activated_at, and require_manual_reset flag." } ], "healthcheck": "GET /health \u2192 200 if Redis is reachable and key read latency < 5ms." }, "promotion_gates": { "to_shadow": [ { "gate": "Unit tests pass for all trigger conditions and deactivation logic", "how_measured": "CI test run", "threshold": "100% pass" }, { "gate": "Redis integration test: atomic SET and GET round-trip verified", "how_measured": "Integration test", "threshold": "Pass" } ], "to_limited_live": [ { "gate": "Fast-path check latency p99 < 10ms over 24h", "how_measured": "polytraders_risk_killswitch_check_latency_ms histogram", "threshold": "p99 < 10ms" }, { "gate": "Background monitor fires drawdown activation in staging within 5s of breach", "how_measured": "Failure injection test", "threshold": "Activation within 5s" } ], "to_general_live": [ { "gate": "Manual kill flow verified end-to-end including Admin UI auth and audit log", "how_measured": "E2E test in staging", "threshold": "Pass" }, { "gate": "Redis-unavailable fail-safe activates within 1s of Redis going down", "how_measured": "Failure injection test", "threshold": "Pass" } ] }, "reporting_groups": [ "risk_compliance" ], "capital_impact": "Critical", "mode_support": [ "quarantine" ], "v3_status": { "phase": 4, "phase_name": "Core risk", "docs": { "done": 27, "total": 27, "state": "done" }, "impl": { "done": 0, "total": 15, "state": "pending" }, "runtime": { "done": 0, "total": 8, "state": "pending" }, "overall": "pending" } }