{ "schema_version": "1.0.0", "bot_id": "2.3", "bot_name": "QueueWarden", "slug": "queuewarden", "layer": "Execution", "layer_key": "exec", "bot_class": "Execution Utility", "authority": [ "Reshape" ], "status": "live", "readiness": "General live", "flagship": false, "is_reference": false, "public_export": false, "identity": { "layer": "Execution", "bot_class": "Execution Utility", "authority": "Reshape", "runs_before": "Order submission to CLOB V2 (cancel-replace actions)", "runs_after": "NonceShepherd (nonce assigned) and SmartRouter (ExecutionPlan assembled)", "applies_to": "All resting maker orders on Polymarket CLOB V2 that are subject to quote drift or TTL management", "default_mode": "general_live", "user_visible": "Advanced details only", "developer_owner": "Polytraders core" }, "purpose": "QueueWarden polices the rate at which orders and cancel-replace operations are sent to the Polymarket CLOB V2 and manages the maker queue position of resting orders. It enforces a cap on cancel-replace operations per minute to avoid CLOB rate-limit rejections, monitors drift between the resting quote price and the best bid/ask, and cancels stale resting orders that have not been filled within the stale_ttl_s window. QueueWarden also injects builderCode (bytes32) into cancel-replace messages to maintain builder attribution continuity across order lifecycle events.", "why_it_matters": [ { "failure": "Excessive cancel-replace rate", "consequence": "Polymarket CLOB V2 enforces per-API-key rate limits. Exceeding them causes 429 rejections, breaking the order submission pipeline and potentially leaving the strategy with unintended open positions." }, { "failure": "Quote drift not corrected", "consequence": "A resting order priced more than drift_ticks_threshold ticks away from the current best bid/ask is no longer competitive; it accumulates adverse-selection risk without filling." }, { "failure": "Stale resting order not cancelled", "consequence": "An order sitting in the book beyond stale_ttl_s may fill at a price that no longer reflects the strategy's current view, creating unintended exposure." }, { "failure": "builderCode absent from cancel-replace message", "consequence": "Builder attribution is broken for the replacement order; maker rebates may not be correctly attributed." } ], "polymarket_inputs": [], "internal_inputs": [ { "input": "Active resting order registry", "source": "internal order state store", "required": true, "use": "Track all resting orders with their price, side, placed_at_ms, and queue position." }, { "input": "CLOB V2 order book snapshot (best bid/ask)", "source": "clob_auth (authenticated book endpoint)", "required": true, "use": "Compute drift between resting order price and current best bid/ask in ticks." }, { "input": "Cancel-replace rate counter", "source": "internal sliding-window counter (Redis)", "required": true, "use": "Enforce cancel_replace_per_min_cap by tracking ops in the last 60s." }, { "input": "ExecutionPlan for replacement order", "source": "exec.smart_router", "required": true, "use": "Receive the updated quote parameters when a cancel-replace is triggered." } ], "raw_params": [ "drift_ticks_threshold \u00b7 int", "stale_ttl_s \u00b7 int", "cancel_replace_per_min_cap \u00b7 int", "min_queue_position \u00b7 int" ], "parameters": [ { "name": "drift_ticks_threshold", "default": 2, "warning": 3, "hard": 5, "controls": "Maximum number of ticks a resting order's price may drift from the current best bid/ask before QueueWarden triggers a cancel-replace.", "why_default_matters": "A 2-tick tolerance prevents excessive cancel-replace churn on minor market movements while ensuring the quote remains competitive.", "threshold_logic": [ { "condition": "drift <= 2 ticks", "action": "No action \u2014 order remains in book" }, { "condition": "2 < drift <= 3 ticks", "action": "WARN \u2014 approaching drift limit; log for monitoring" }, { "condition": "3 < drift <= 5 ticks", "action": "Trigger cancel-replace (RESHAPE)" }, { "condition": "drift > 5 ticks (hard)", "action": "Immediate cancel-replace regardless of rate cap" } ], "dev_check": "if drift > params.hard: force_cancel_replace(order_id)", "user_facing": "Your resting order was updated to stay competitive with the current market price." }, { "name": "stale_ttl_s", "default": 300, "warning": 240, "hard": 600, "controls": "Maximum seconds a resting order may sit in the CLOB book without being filled or refreshed before QueueWarden cancels it.", "why_default_matters": "A 5-minute TTL ensures stale quotes based on outdated strategy signals are removed before they fill at an adverse price.", "threshold_logic": [ { "condition": "resting_time <= 300s", "action": "No action" }, { "condition": "300 < resting_time <= 240s warning", "action": "WARN \u2014 approaching stale TTL" }, { "condition": "resting_time > 300s default", "action": "Cancel order \u2014 QUEUE_WARDEN_STALE_ORDER" }, { "condition": "resting_time > 600s hard", "action": "Force cancel regardless of rate cap" } ], "dev_check": "if order.resting_s > params.hard: force_cancel(order_id)", "user_facing": "A resting order that had been waiting too long was cancelled to prevent it from filling at an outdated price." }, { "name": "cancel_replace_per_min_cap", "default": 30, "warning": 24, "hard": 30, "controls": "Maximum cancel-replace operations permitted per minute across all resting orders. Operations above this rate are queued and deferred to the next second.", "why_default_matters": "Polymarket CLOB V2 rate limits are per API key. Staying at or below 30 per minute provides headroom for fill acknowledgements and order status polling.", "threshold_logic": [ { "condition": "rate <= 24/min", "action": "Normal operation" }, { "condition": "24 < rate <= 30/min", "action": "WARN \u2014 approaching rate cap" }, { "condition": "rate > 30/min", "action": "Queue excess ops \u2014 QUEUE_WARDEN_RATE_CAP_HIT" } ], "dev_check": "if sliding_window_count(60) >= params.hard: defer_to_next_slot(op)", "user_facing": "" }, { "name": "min_queue_position", "default": 5, "warning": 8, "hard": 10, "controls": "Minimum acceptable maker queue position (1 = front of queue). If the resting order's queue position worsens beyond this threshold, a cancel-replace is triggered to re-enter the front of the book.", "why_default_matters": "Queue position 5 or better ensures the order is near the front of the maker queue, improving fill probability without triggering constant churn.", "threshold_logic": [ { "condition": "queue_position <= 5", "action": "No action \u2014 position acceptable" }, { "condition": "5 < queue_position <= 8", "action": "WARN \u2014 position degrading" }, { "condition": "8 < queue_position <= 10", "action": "Cancel-replace to re-enter queue at front" }, { "condition": "queue_position > 10", "action": "Immediate cancel-replace regardless of rate cap" } ], "dev_check": "if queue_pos > params.hard: force_cancel_replace(order_id)", "user_facing": "Your order was refreshed to maintain a competitive position in the market." } ], "default_config": { "bot_id": "exec.queue_warden", "version": "2.1.0", "mode": "general_live", "defaults": { "drift_ticks_threshold": 2, "stale_ttl_s": 300, "cancel_replace_per_min_cap": 30, "min_queue_position": 5 }, "locked": { "cancel_replace_per_min_cap": { "max": 30 }, "stale_ttl_s": { "max": 600 } } }, "implementation_flow": [ "On each evaluation tick (configurable, default 5s), load the active resting order registry from internal state.", "For each resting order, fetch the current best bid/ask from the CLOB V2 authenticated book endpoint.", "Compute price drift in ticks: drift = abs(order.price - best_bid_or_ask) / tick_size.", "Compute resting time: resting_s = (now_ms() - order.placed_at_ms) / 1000.", "Fetch queue position for the order from the CLOB V2 order status endpoint.", "Apply stale TTL check: if resting_s > stale_ttl_s (or > hard limit), mark order for cancellation.", "Apply drift check: if drift > drift_ticks_threshold (or > hard limit), mark order for cancel-replace.", "Apply queue position check: if queue_position > min_queue_position (or > hard limit), mark order for cancel-replace.", "For each marked order: check sliding-window cancel-replace rate counter; if < cancel_replace_per_min_cap, execute cancel-replace immediately.", "If rate cap reached: queue excess cancel-replace ops for the next available slot; emit WARN QUEUE_WARDEN_RATE_CAP_HIT.", "On cancel-replace execution: inject builderCode (bytes32) into the replacement order payload before forwarding to NonceShepherd.", "Update internal order registry with new order_id, price, placed_at_ms after successful cancel-replace.", "Emit QueueDecision record for each evaluated order with verdict (HOLD / CANCEL_REPLACE / CANCEL_STALE) and reason_code." ], "decision_logic": { "approve": "Order is within drift tolerance, below stale TTL, and queue position is acceptable. QueueDecision.HOLD emitted.", "reshape_required": "Drift exceeds threshold or queue position has degraded past min_queue_position. QueueDecision.CANCEL_REPLACE emitted; replacement order constructed with refreshed price and builderCode.", "reject": "Resting order has exceeded stale_ttl_s. QueueDecision.CANCEL_STALE emitted; order cancelled with no replacement.", "warning_only": "Drift or resting time is in the warning band but below the action threshold. WARN annotation on QueueDecision.HOLD." }, "decision_output_schema": "QueueDecision", "decision_output_example": { "warden_id": "exec.queue_warden", "order_id": "0xb2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3", "market_id": "0x9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c", "verdict": "CANCEL_REPLACE", "reason_code": "QUEUE_WARDEN_DRIFT_EXCEEDED", "drift_ticks": 3, "resting_s": 47, "queue_position": 4, "replacement_price": 0.68, "builder_code": "0x706f6c7974726164657273000000000000000000000000000000000000000000", "eip712_domain_version": "2", "evaluated_at_ms": 1746769200000 }, "developer_log": { "warden_id": "exec.queue_warden", "order_id": "0xb2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3", "market_id": "0x9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c", "order_price": 0.65, "best_ask": 0.68, "drift_ticks": 3, "drift_ticks_threshold": 2, "resting_s": 47, "stale_ttl_s": 300, "queue_position": 4, "min_queue_position": 5, "cancel_replace_rate_1m": 12, "cancel_replace_per_min_cap": 30, "verdict": "CANCEL_REPLACE", "reason_code": "QUEUE_WARDEN_DRIFT_EXCEEDED", "builder_code": "0x706f6c7974726164657273000000000000000000000000000000000000000000", "evaluated_at_ms": 1746769200000 }, "user_explanations": [ { "situation": "Resting order price updated", "message": "Your resting order was refreshed to stay aligned with the current market price." }, { "situation": "Stale resting order cancelled", "message": "A resting order that had been waiting longer than the allowed window was cancelled to prevent it from filling at an outdated price." }, { "situation": "Order update queued \u2014 high activity", "message": "A minor order update is queued briefly due to high activity. It will be applied within seconds." } ], "failure_modes": { "main_failure_mode": "Rate cap hit during a volatile period causing a backlog of cancel-replace ops. If the backlog is not drained before stale TTL expires, stale orders may fill at adverse prices before they can be cancelled.", "false_positive_risk": "Triggering a cancel-replace on a minor drift caused by a momentary price spike that immediately reverts, resulting in unnecessary churn and order management cost.", "false_negative_risk": "Failing to detect drift because the CLOB book snapshot is stale (e.g. book endpoint latency spike), allowing an order to rest at an uncompetitive price beyond the threshold.", "safe_fallback": "If the CLOB book endpoint is unreachable, cancel all resting orders as a precaution rather than leaving them with unknown drift. If the internal state store is unavailable, pause all evaluations and emit QUEUE_WARDEN_STATE_UNAVAILABLE.", "required_dependencies": [ "CLOB V2 authenticated book endpoint", "Internal order state store (Redis)", "Sliding-window rate counter (Redis)", "NonceShepherd (for cancel-replace nonce assignment)" ] }, "acceptance_tests": { "unit": [ { "test": "Emit HOLD when drift is within threshold", "setup": "order.price=0.65, best_ask=0.66, tick_size=0.01, drift_ticks_threshold=2", "expected": "QueueDecision.verdict=HOLD" }, { "test": "Trigger CANCEL_REPLACE when drift exceeds threshold", "setup": "order.price=0.65, best_ask=0.68, tick_size=0.01, drift_ticks_threshold=2", "expected": "QueueDecision.verdict=CANCEL_REPLACE, reason_code=QUEUE_WARDEN_DRIFT_EXCEEDED" }, { "test": "Cancel stale order when resting_s > stale_ttl_s", "setup": "order.resting_s=310, stale_ttl_s=300", "expected": "QueueDecision.verdict=CANCEL_STALE, reason_code=QUEUE_WARDEN_STALE_ORDER" }, { "test": "Defer cancel-replace when rate cap is reached", "setup": "cancel_replace_rate_1m=30, cancel_replace_per_min_cap=30", "expected": "Op queued for next slot; WARN QUEUE_WARDEN_RATE_CAP_HIT emitted" }, { "test": "builderCode injected into replacement order", "setup": "Standard cancel-replace trigger", "expected": "QueueDecision.builder_code = 32-byte hex string; non-null" } ], "integration": [ { "test": "End-to-end: drift exceeds threshold \u2192 cancel-replace \u2192 new order accepted by CLOB", "expected": "Old order cancelled; replacement order with refreshed price and builderCode submitted and accepted" }, { "test": "Rate cap hit: excess ops queue and drain correctly after cap window resets", "expected": "Ops execute in order after rate window resets; no ops dropped" } ], "property": [ { "property": "builderCode (bytes32) is always non-null on every cancel-replace replacement order", "required": "Always true \u2014 builder_code_aware=true" }, { "property": "cancel_replace_per_min_cap is never exceeded in any 60-second window", "required": "Always true \u2014 sliding window enforced before any op submission" } ] }, "checklist_overrides": {}, "legacy_goal": "Manage maker queue position across price levels.", "legacy_pm_signals": [ "Drift of our quote vs. best bid/ask in ticks", "Cancellations of better-priced orders ahead of us", "Stale-quote risk: order resting > T seconds with no book movement" ], "legacy_external_feeds": [], "reporting_groups": [ "execution" ], "network": [ "polygon" ], "api_surface": [ "clob_auth", "internal" ], "version": { "spec": "2.0.0", "implementation": "2.1.0", "schema": "2", "released": "2026-04-28" }, "migration_history": [ { "date": "2026-04-28", "from": "v1", "to": "v2", "reason": "CLOB V2 cutover \u2014 order lifecycle management updated for V2 signed-order schema", "action_taken": "Removed feeRateBps and nonce from cancel-replace order payloads (V2: operator-set fees, NonceShepherd manages nonces). Added builderCode (bytes32) injection into all replacement orders to maintain builder attribution across the full order lifecycle. Updated EIP-712 Exchange domain to version '2'. Collateral denomination updated from USDC.e to pUSD in all QueueDecision records." } ], "polymarket_v2_compat": { "clob_version": "v2", "collateral": "pUSD", "eip712_domain_version": "2", "builder_code_aware": true, "negrisk_aware": false, "multichain_ready": false, "sdk_used": "py-clob-client-v2", "settlement_contract": "CTFExchangeV2", "notes": "QueueWarden injects builderCode (bytes32) into every cancel-replace replacement order, maintaining builder attribution continuity across the order lifecycle. V2 cancel-replace does not include feeRateBps \u2014 fees are operator-set by CTFExchangeV2 at match time. EIP-712 Exchange domain version is '2'; ClobAuth domain version is '1'." }, "reference_implementation": { "summary": "On each evaluation tick, QueueWarden inspects every resting order in the internal registry, computes drift and resting time, and emits a QueueDecision (HOLD / CANCEL_REPLACE / CANCEL_STALE). Cancel-replace ops inject builderCode (bytes32) and are rate-limited by a sliding window counter.", "language_note": "Pseudocode is language-agnostic. FETCH = read input. EMIT = produce output. Translate to TS/Python/Go/Rust.", "pseudocode": "FUNCTION evaluateOrders():\n // --- 0. Load active orders ---\n orders = FETCH internal.order_registry.all_resting()\n IF orders IS NULL:\n EMIT WARN QUEUE_WARDEN_STATE_UNAVAILABLE\n RETURN\n\n // --- 1. Fetch rate counter ---\n rate_1m = FETCH internal.rate_counter.sliding_window_count(60) // ops in last 60s\n\n FOR order IN orders:\n // --- 2. Fetch best bid/ask from CLOB V2 ---\n book = FETCH clob_auth.GET('/book?token_id=' + order.token_id)\n IF book IS NULL:\n // Safe fallback: cancel rather than leave resting with unknown drift\n EMIT QueueDecision(order, CANCEL_STALE, QUEUE_WARDEN_BOOK_UNAVAILABLE)\n CONTINUE\n\n best_price = book.best_ask IF order.side == 'BUY' ELSE book.best_bid\n drift_ticks = abs(order.price - best_price) / order.tick_size\n resting_s = (now_ms() - order.placed_at_ms) / 1000\n\n // --- 3. Queue position check ---\n queue_pos = FETCH clob_auth.GET('/order/' + order.order_id + '/queue_position')\n\n // --- 4. Determine action ---\n force = (drift_ticks > params.drift_ticks_threshold_hard\n OR resting_s > params.stale_ttl_s_hard\n OR queue_pos > params.min_queue_position_hard)\n\n IF resting_s > params.stale_ttl_s:\n action = 'CANCEL_STALE'; reason = 'QUEUE_WARDEN_STALE_ORDER'\n ELSE IF drift_ticks > params.drift_ticks_threshold:\n action = 'CANCEL_REPLACE'; reason = 'QUEUE_WARDEN_DRIFT_EXCEEDED'\n ELSE IF queue_pos > params.min_queue_position:\n action = 'CANCEL_REPLACE'; reason = 'QUEUE_WARDEN_QUEUE_DEGRADED'\n ELSE:\n // HOLD \u2014 possibly with WARN annotation\n warn = (drift_ticks > params.drift_ticks_threshold - 1\n OR resting_s > params.stale_ttl_s * 0.8)\n EMIT QueueDecision(order, HOLD, None, warn=warn)\n CONTINUE\n\n // --- 5. Rate-cap check ---\n IF action == 'CANCEL_REPLACE' AND rate_1m >= params.cancel_replace_per_min_cap AND NOT force:\n internal.rate_queue.enqueue(order.order_id, action)\n EMIT WARN QUEUE_WARDEN_RATE_CAP_HIT\n CONTINUE\n\n // --- 6. Execute cancel ---\n clob_auth.DELETE('/order/' + order.order_id)\n internal.order_registry.remove(order.order_id)\n\n IF action == 'CANCEL_REPLACE':\n // --- 7. Build replacement with V2 fields + builderCode ---\n replacement = {\n market_id: order.market_id,\n side: order.side,\n price: best_price, // refreshed to current best\n size_usd: order.size_usd,\n builder_code: config.builder_code, // bytes32\n eip712_domain_version: '2'\n }\n // Nonce assigned by NonceShepherd before signing\n EMIT NonceAssignmentRequest(replacement)\n internal.rate_counter.increment()\n\n EMIT QueueDecision(order, action, reason,\n drift_ticks=drift_ticks, resting_s=resting_s,\n queue_position=queue_pos,\n builder_code=config.builder_code,\n evaluated_at_ms=now_ms())\n", "sdk_calls": [ "clob_auth.GET('/book?token_id=' + order.token_id)", "clob_auth.GET('/order/' + order.order_id + '/queue_position')", "clob_auth.DELETE('/order/' + order.order_id)", "clob_auth.POST('/order', signed_replacement_order)", "buildOrderTypedData(replacementParams, { name: 'CTFExchange', version: '2', chainId: 137 })" ], "complexity": "O(R) where R = number of resting orders in the registry per evaluation tick" }, "wire_examples": { "input": { "label": "Resting order evaluated for drift", "source": "internal order registry + CLOB V2 book snapshot", "payload": { "order_id": "0xb2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3", "market_id": "0x9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c", "side": "BUY", "outcome": "YES", "price": 0.65, "tick_size": 0.01, "size_usd": 200, "placed_at_ms": 1746769153000, "best_ask_at_eval": 0.68, "queue_position": 4 } }, "output": { "label": "QueueDecision \u2014 CANCEL_REPLACE due to drift", "payload": { "warden_id": "exec.queue_warden", "order_id": "0xb2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3", "market_id": "0x9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c", "verdict": "CANCEL_REPLACE", "reason_code": "QUEUE_WARDEN_DRIFT_EXCEEDED", "drift_ticks": 3, "resting_s": 47, "queue_position": 4, "replacement_price": 0.68, "builder_code": "0x706f6c7974726164657273000000000000000000000000000000000000000000", "eip712_domain_version": "2", "evaluated_at_ms": 1746769200000 } } }, "reason_codes": [ { "code": "QUEUE_WARDEN_HOLD", "severity": "INFO", "meaning": "Resting order is within drift tolerance, below stale TTL, and queue position is acceptable.", "action": "No action. Emit QueueDecision.HOLD.", "user_message": "" }, { "code": "QUEUE_WARDEN_DRIFT_EXCEEDED", "severity": "RESHAPE", "meaning": "Order price has drifted more than drift_ticks_threshold ticks from the current best bid/ask.", "action": "Cancel and replace order at current best price with refreshed builderCode.", "user_message": "Your resting order was updated to align with the current market price." }, { "code": "QUEUE_WARDEN_STALE_ORDER", "severity": "RESHAPE", "meaning": "Resting order has exceeded stale_ttl_s without being filled.", "action": "Cancel order without replacement. Emit QueueDecision.CANCEL_STALE.", "user_message": "A resting order was cancelled because it had been waiting too long without being filled." }, { "code": "QUEUE_WARDEN_QUEUE_DEGRADED", "severity": "RESHAPE", "meaning": "Maker queue position has worsened beyond min_queue_position threshold.", "action": "Cancel and replace to re-enter at the front of the queue.", "user_message": "Your order was refreshed to maintain a competitive position in the market." }, { "code": "QUEUE_WARDEN_RATE_CAP_HIT", "severity": "WARN", "meaning": "cancel_replace_per_min_cap has been reached; excess operations are queued for the next rate window.", "action": "Enqueue op in rate-deferred queue. Emit WARN. Do not block non-cancel-replace ops.", "user_message": "" }, { "code": "QUEUE_WARDEN_BOOK_UNAVAILABLE", "severity": "HARD_REJECT", "meaning": "CLOB V2 book endpoint is unreachable; drift cannot be computed for this order.", "action": "Cancel order as safe fallback. Emit QueueDecision.CANCEL_STALE.", "user_message": "A resting order was cancelled because market data was temporarily unavailable." }, { "code": "QUEUE_WARDEN_STATE_UNAVAILABLE", "severity": "HARD_REJECT", "meaning": "Internal order registry is unreachable; evaluation cannot proceed.", "action": "Pause all evaluation. Emit WARN. Resume when state store reconnects.", "user_message": "" }, { "code": "KILL_SWITCH_ACTIVE", "severity": "HARD_REJECT", "meaning": "Global kill switch is active; all resting orders are cancelled immediately.", "action": "Cancel all resting orders. Emit no replacement orders.", "user_message": "Trading is currently paused." } ], "metrics": { "emitted": [ { "name": "polytraders_exec_queuewarden_decisions_total", "type": "counter", "unit": "count", "labels": [ "verdict", "reason_code" ], "meaning": "Total QueueDecision records emitted per verdict and reason code." }, { "name": "polytraders_exec_queuewarden_cancel_replace_rate_1m", "type": "gauge", "unit": "count", "labels": [], "meaning": "Sliding 60-second cancel-replace operation rate; alert when approaching cap." }, { "name": "polytraders_exec_queuewarden_drift_ticks", "type": "histogram", "unit": "ticks", "labels": [ "market_id" ], "meaning": "Distribution of drift in ticks across all evaluated orders; >2 ticks triggers CANCEL_REPLACE." }, { "name": "polytraders_exec_queuewarden_resting_time_seconds", "type": "histogram", "unit": "seconds", "labels": [], "meaning": "Distribution of resting time across all evaluated orders; >stale_ttl_s triggers cancellation." }, { "name": "polytraders_exec_queuewarden_rate_queue_depth", "type": "gauge", "unit": "count", "labels": [], "meaning": "Number of cancel-replace ops currently deferred in the rate queue." }, { "name": "polytraders_exec_queuewarden_eval_latency_ms", "type": "histogram", "unit": "ms", "labels": [], "meaning": "Wall-clock time to complete one full evaluation tick across all resting orders." } ], "alerts": [ { "name": "QueueWardenRateCapPersistent", "condition": "polytraders_exec_queuewarden_cancel_replace_rate_1m >= 28", "severity": "warn", "runbook": "#runbook-queuewarden-rate-cap" }, { "name": "QueueWardenRateQueueDeep", "condition": "polytraders_exec_queuewarden_rate_queue_depth > 20", "severity": "warn", "runbook": "#runbook-queuewarden-rate-queue-deep" }, { "name": "QueueWardenBookUnavailable", "condition": "rate(polytraders_exec_queuewarden_decisions_total{reason_code='QUEUE_WARDEN_BOOK_UNAVAILABLE'}[5m]) > 0.1", "severity": "page", "runbook": "#runbook-queuewarden-book-unavailable" }, { "name": "QueueWardenHighDrift", "condition": "histogram_quantile(0.95, rate(polytraders_exec_queuewarden_drift_ticks_bucket[5m])) > 3", "severity": "warn", "runbook": "#runbook-queuewarden-high-drift" } ], "dashboards": [ "Grafana \u2014 Execution / QueueWarden", "Grafana \u2014 Order quality / drift distribution and stale order rate" ], "log_level": "info" }, "state": { "store": "redis", "shape": "Active resting order registry: hash keyed by order_id \u2192 { market_id, side, price, size_usd, placed_at_ms, tick_size, token_id }. Rate counter: sliding-window counter per 60s bucket.", "ttl": "Resting order entries auto-expire 3600s after placed_at_ms if not explicitly removed. Rate counter buckets expire after 120s.", "recovery": "On restart, order registry is replayed from Redis. Any entry older than stale_ttl_s_hard is immediately cancelled. Rate counter resets on restart.", "size_estimate": "~300 bytes per resting order entry; rate counter ~100 bytes" }, "concurrency": { "execution_model": "single-threaded event loop", "max_in_flight": 30, "idempotency_key": "order_id", "timeout_ms": 1000, "backpressure": "Excess cancel-replace ops queued in rate-deferred queue; evaluation tick skipped if previous tick has not completed", "locking": "Redis SETNX per order_id during cancel-replace execution to prevent double-cancel" }, "dependencies": { "depends_on": [ { "bot_id": "exec.smart_router", "why": "SmartRouter provides replacement ExecutionPlan parameters when a cancel-replace is triggered.", "contract": "Replacement order must preserve side, market_id, and outcome from original order." }, { "bot_id": "exec.nonce_shepherd", "why": "NonceShepherd assigns the nonce and injects builderCode for every replacement order.", "contract": "QueueWarden emits a NonceAssignmentRequest; NonceShepherd responds with a signed-ready payload." }, { "bot_id": "risk.kill_switch", "why": "KillSwitch active triggers immediate cancellation of all resting orders.", "contract": "On KillSwitch activation, QueueWarden cancels all resting orders and emits no replacement orders." } ], "emits_to": [ { "bot_id": "gov.builder_attribution", "what": "builderCode (bytes32) embedded in every cancel-replace replacement order for attribution continuity.", "contract": "builder_code non-null on every QueueDecision.CANCEL_REPLACE payload." } ], "sibling": [], "external": [ { "service": "Polymarket CLOB V2 (auth)", "endpoint": "https://clob.polymarket.com", "sla": "99.95% / 200ms p99", "fallback": "Cancel resting orders on book endpoint failure; do not leave orders with unknown drift." } ] }, "security_surfaces": { "signs_orders": true, "private_key_access": "signing-only", "abuse_vectors": [ "Manipulating the internal order registry to falsely report high queue positions, triggering excessive cancel-replace churn and burning through the rate cap", "Injecting a false book snapshot to inflate apparent drift and cause unnecessary order refresh operations", "Rate cap exhaustion attack: flooding QueueWarden with resting orders to exhaust cancel_replace_per_min_cap before legitimate orders can be managed" ], "mitigations": [ "Order registry write access restricted to QueueWarden and NonceShepherd service accounts via Redis ACL", "Book snapshots fetched directly from Polymarket CLOB V2 authenticated endpoint \u2014 not from an internal cache that could be tampered with", "Rate counter enforced per-key sliding window; max 30 ops/min per API key is a hard constraint on the CLOB side as well", "QueueDecision records HMAC-signed; consumers reject tampered records" ], "contract_calls": [] }, "failure_injection": [ { "scenario": "BOOK_ENDPOINT_DOWN", "how_to_inject": "Block TCP to clob.polymarket.com on the book endpoint", "expected_behaviour": "QueueWarden emits QUEUE_WARDEN_BOOK_UNAVAILABLE; all resting orders cancelled as safe fallback", "recovery": "Orders re-submitted by SmartRouter on next strategy evaluation cycle after CLOB connectivity restored" }, { "scenario": "RATE_CAP_EXHAUSTED", "how_to_inject": "Set cancel_replace_rate_1m counter to 30 (at cap) in Redis", "expected_behaviour": "New cancel-replace ops queued in rate-deferred queue; WARN QUEUE_WARDEN_RATE_CAP_HIT emitted", "recovery": "Queue drains automatically as rate window resets after 60s" }, { "scenario": "STALE_ORDER_MASS_EXPIRY", "how_to_inject": "Set placed_at_ms for all registry entries to now()-310s (beyond stale_ttl_s=300)", "expected_behaviour": "All resting orders cancelled with QUEUE_WARDEN_STALE_ORDER on next evaluation tick", "recovery": "SmartRouter re-submits fresh orders on next strategy cycle" }, { "scenario": "BUILDER_CODE_ABSENT", "how_to_inject": "Null out config.builder_code before a cancel-replace cycle", "expected_behaviour": "QueueWarden blocks cancel-replace; emits QUEUE_WARDEN_BUILDER_CODE_MISSING HARD_REJECT", "recovery": "Restore config.builder_code from secrets manager" }, { "scenario": "KILL_SWITCH_ON", "how_to_inject": "Activate global KillSwitch", "expected_behaviour": "All resting orders cancelled immediately; no replacement orders emitted", "recovery": "Orders re-submitted after manual KillSwitch reset and strategy confirmation" } ], "runbook": { "summary": "QueueWarden incidents are typically rate-cap exhaustion during volatile markets, CLOB book endpoint degradation causing precautionary mass cancellations, or KillSwitch-triggered mass cancellations. All require coordination with the Strategy pod.", "oncall_actions": [ { "alert": "QueueWardenBookUnavailable", "first_step": "Verify CLOB V2 book endpoint. Check Polymarket status page.", "diagnosis": "CLOB book endpoint unreachable; all resting orders cancelled as safe fallback. Orders need re-submission.", "mitigation": "Restore CLOB connectivity. Notify Strategy pod to re-submit resting orders.", "escalation": "Exec pod lead and Strategy pod \u2014 orders need re-submission after CLOB recovery" }, { "alert": "QueueWardenRateCapPersistent", "first_step": "Check cancel-replace rate in Grafana. If market volatility is causing rapid drift, reduce strategy aggressiveness temporarily.", "diagnosis": "High market volatility is triggering drift-based cancel-replace operations faster than the rate cap allows.", "mitigation": "Reduce strategy quote refresh aggressiveness or request a temporary rate cap increase.", "escalation": "Strategy pod lead if rate cap sustained > 5 minutes" }, { "alert": "QueueWardenRateQueueDeep", "first_step": "Rate queue depth > 20 means many updates are deferred. Check if rate cap can be increased temporarily.", "diagnosis": "Deferred cancel-replace ops are accumulating; some orders may become stale before they can be refreshed.", "mitigation": "Request rate cap increase from Exec pod lead. Run: polytraders bot flush-queue exec.queue_warden --rate after cap is raised.", "escalation": "Exec pod lead for rate cap increase approval" }, { "alert": "QueueWardenHighDrift", "first_step": "High drift p95 > 3 ticks indicates market is moving faster than the evaluation tick. Check evaluation_tick_s config and CLOB latency.", "diagnosis": "Evaluation tick interval may be too slow for current market volatility, or CLOB latency is causing stale book snapshots.", "mitigation": "Reduce evaluation_tick_s. Check CLOB book endpoint latency.", "escalation": "Strategy pod lead if sustained > 10 minutes" } ], "manual_overrides": [ { "command": "polytraders bot cancel-all exec.queue_warden", "effect": "Immediately cancels all resting orders without waiting for the next evaluation tick. Use in emergencies." }, { "command": "polytraders bot flush-queue exec.queue_warden --rate", "effect": "Force-drains the rate-deferred cancel-replace queue after the rate cap is raised or market calms." } ], "healthcheck": "GET /internal/health/queuewarden \u2192 200 if CLOB book endpoint reachable, cancel_replace_rate_1m < 24, rate queue depth < 10, state store connected. RED if CLOB unreachable, rate_1m >= 30, or state store disconnected." }, "promotion_gates": { "to_shadow": [ { "gate": "All 5 acceptance_tests.unit cases pass", "how_measured": "CI test run", "threshold": "100% pass" }, { "gate": "builderCode (bytes32) present on every cancel-replace replacement order", "how_measured": "Unit test: inspect QueueDecision.builder_code field", "threshold": "Non-null bytes32 on 100% of CANCEL_REPLACE decisions" } ], "to_limited_live": [ { "gate": "Rate cap enforcement: no cancel-replace rate ever exceeds 30/min in load test", "how_measured": "Load test: 50 resting orders with continuous drift triggers", "threshold": "Max observed rate <= 30/min" }, { "gate": "p99 eval latency < 1000ms over 24h", "how_measured": "polytraders_exec_queuewarden_eval_latency_ms histogram", "threshold": "p99 < 1000ms" } ], "to_general_live": [ { "gate": "E2E: drift-triggered cancel-replace \u2192 new order with builderCode fills on Polygon testnet", "how_measured": "E2E test on Polygon Amoy testnet", "threshold": "Pass with correct builderCode on replacement order" }, { "gate": "7-day production shadow: zero orders cancelled incorrectly (false stale detection)", "how_measured": "Audit log review against strategy fill records", "threshold": "Zero false stale cancellations" } ] }, "reporting": { "emits_kinds": [ "ExecutionReport" ], "topics": [ "polytraders.reports.exec" ], "cadence": "every-event", "retention_class": "7y", "sampling_rule": "emit-every", "bus_failure_action": "wal-then-retry", "user_visible": "summary-only", "consumes_kinds": [] }, "capital_impact": "Direct", "mode_support": [ "quarantine" ], "v3_status": { "phase": 5, "phase_name": "Execution rails", "docs": { "done": 27, "total": 27, "state": "done" }, "impl": { "done": 0, "total": 15, "state": "pending" }, "runtime": { "done": 0, "total": 8, "state": "pending" }, "overall": "pending" } }