{ "schema_version": "1.0.0", "bot_id": "2.7", "bot_name": "FillQualityAnalyzer", "slug": "fillqualityanalyzer", "layer": "Execution", "layer_key": "exec", "bot_class": "Execution Utility", "authority": [ "Reshape" ], "status": "planned", "readiness": "Spec started", "flagship": false, "is_reference": false, "public_export": false, "identity": { "layer": "Execution", "bot_class": "Execution Utility", "authority": "Reshape", "runs_before": "Post-trade governance pipeline", "runs_after": "OrderLifecycleManager (FILLED event received)", "applies_to": "Every completed fill event emitted by OrderLifecycleManager", "default_mode": "shadow_only", "user_visible": "summary-only", "developer_owner": "Polytraders core \u2014 Execution pod" }, "purpose": "FillQualityAnalyzer scores every completed fill on price slippage, execution latency, and post-fill toxicity relative to the original intent. Low-scoring fills generate WARN annotations; systematic poor scores feed back to strategy tuning.", "why_it_matters": [ { "failure": "Slippage not tracked per fill", "consequence": "Execution costs compound silently; strategy returns eroded without visibility into the cause." }, { "failure": "Post-fill toxicity not measured", "consequence": "The system continues submitting orders on markets with chronic adverse selection, accumulating losses." }, { "failure": "Fill latency not profiled", "consequence": "Slow fills on fast-moving markets indicate queue-position issues that smarter order type selection could fix." } ], "polymarket_inputs": [ { "input": "CLOB V2 trade tape \u2014 last 100 fills on market", "source": "ws_market", "required": true, "use": "Compute post-fill mark drift over toxicity_horizon_s for toxicity score." }, { "input": "Market mid at time of fill (top-of-book snapshot)", "source": "clob_public", "required": true, "use": "Compute arrival mid for slippage calculation." } ], "internal_inputs": [ { "input": "ExecutionReport (FILLED) from OrderLifecycleManager", "source": "exec.orderlifecyclemanager", "required": true, "use": "Source fill price, size, intent price, and submit_ms for all score dimensions." }, { "input": "Original DecisionReport (intent price)", "source": "strat layer via internal bus", "required": false, "use": "Compare fill price against decision-time price to compute decision-to-fill slippage." } ], "raw_params": [ "toxicity_horizon_s \u00b7 int", "min_score_for_repeat \u00b7 float", "publish_to \u00b7 list", "warn_threshold_bps \u00b7 int" ], "parameters": [ { "name": "toxicity_horizon_s", "default": 30, "warning": 60, "hard": 120, "controls": "Seconds after fill to measure post-fill mark drift as a toxicity proxy.", "why_default_matters": "30s captures the immediate adverse-selection window without confounding long-term price moves.", "threshold_logic": [ { "condition": "horizon <= 30s", "action": "Normal toxicity measurement" }, { "condition": "30 < horizon <= 60s", "action": "WARN \u2014 extended horizon may include non-toxic drift" }, { "condition": "horizon > 120s", "action": "Reject \u2014 PARAMETER_CHANGE_REQUIRES_APPROVAL" } ], "dev_check": "assert params.toxicity_horizon_s <= params.hard", "user_facing": "Fill quality is assessed over a short window after your order is executed." }, { "name": "min_score_for_repeat", "default": 0.6, "warning": 0.4, "hard": 0.2, "controls": "Minimum fill quality score (0\u20131) for the strategy to be permitted to repeat on the same market without a WARN annotation.", "why_default_matters": "A score below 0.6 indicates the fill underperformed the intent on at least two of three dimensions.", "threshold_logic": [ { "condition": "score >= 0.6", "action": "PASS \u2014 no annotation" }, { "condition": "0.4 <= score < 0.6", "action": "WARN \u2014 FILL_QUALITY_BELOW_THRESHOLD" }, { "condition": "score < 0.2 (hard)", "action": "HARD_REJECT \u2014 FILL_QUALITY_CRITICAL; forward to risk pipeline" } ], "dev_check": "if score < params.min_score_for_repeat: emit(FILL_QUALITY_BELOW_THRESHOLD)", "user_facing": "Your order was executed, but the fill quality was lower than expected." }, { "name": "warn_threshold_bps", "default": 20, "warning": 40, "hard": 100, "controls": "Slippage in bps relative to arrival mid above which a WARN annotation is added to the ExecutionReport.", "why_default_matters": "20 bps represents approximately one spread width on typical Polymarket markets; beyond this, slippage is notable.", "threshold_logic": [ { "condition": "slippage_bps <= 20", "action": "No annotation" }, { "condition": "20 < slippage_bps <= 40", "action": "WARN \u2014 FILL_SLIPPAGE_ELEVATED" }, { "condition": "slippage_bps > 100", "action": "HARD_REJECT \u2014 FILL_SLIPPAGE_CRITICAL" } ], "dev_check": "if slippage_bps > params.warn_threshold_bps: emit(FILL_SLIPPAGE_ELEVATED)", "user_facing": "Your order filled at a price slightly different from what was expected." }, { "name": "publish_to", "default": [ "polytraders.reports.execution" ], "warning": "\u2014", "hard": "\u2014", "controls": "List of bus topics to which the fill quality ExecutionReport is published.", "why_default_matters": "Publishing only to the execution topic keeps fill quality data within the exec layer; adding post-trade topics extends visibility to governance.", "threshold_logic": [ { "condition": "includes polytraders.reports.execution", "action": "Normal \u2014 exec layer receives score" }, { "condition": "empty list", "action": "WARN \u2014 fill quality scores not published; governance cannot audit" } ], "dev_check": "assert len(params.publish_to) > 0", "user_facing": "Your fill details are recorded and available in your trade summary." } ], "default_config": { "bot_id": "exec.fillqualityanalyzer", "version": "0.1.0", "mode": "shadow_only", "defaults": { "toxicity_horizon_s": 30, "min_score_for_repeat": 0.6, "warn_threshold_bps": 20, "publish_to": [ "polytraders.reports.execution" ] }, "locked": { "toxicity_horizon_s": { "max": 120 }, "warn_threshold_bps": { "max": 100 } } }, "implementation_flow": [ "Receive ExecutionReport(FILLED) from OrderLifecycleManager.", "Fetch CLOB V2 top-of-book snapshot from clob_public at fill_ts_ms to get arrival_mid.", "Compute slippage_bps = abs(fill_price - arrival_mid) / arrival_mid * 10000.", "Compute latency_ms = fill_ts_ms - submit_ms from the ExecutionReport.", "Wait toxicity_horizon_s; fetch trade tape from ws_market to compute post-fill drift_bps.", "Score dimensions: price_score = max(0, 1 - slippage_bps/100); latency_score = max(0, 1 - latency_ms/500); toxicity_score = max(0, 1 - drift_bps/100).", "Overall score = mean(price_score, latency_score, toxicity_score).", "Annotate ExecutionReport with scores and reason codes (FILL_QUALITY_BELOW_THRESHOLD if score < min_score_for_repeat).", "Emit annotated ExecutionReport to publish_to topics." ], "decision_logic": { "approve": "Fill quality score >= min_score_for_repeat; ExecutionReport annotated with FILL_QUALITY_PASS.", "reshape_required": "Not applicable \u2014 FillQualityAnalyzer is post-fill; it annotates but cannot reshape a completed fill.", "reject": "Score below hard threshold (0.2); FILL_QUALITY_CRITICAL emitted; alert forwarded to risk pipeline.", "warning_only": "Score between warn threshold and hard threshold; FILL_QUALITY_BELOW_THRESHOLD annotation added." }, "decision_output_schema": "ExecutionReport", "decision_output_example": { "report_id": "rep_2b3c4d5e6f7a8b9c", "trace_id": "trc_1a2b3c4d5e6f7a8b", "bot_id": "exec.fillqualityanalyzer", "order_id": "0xbbbb2222cccc3333dddd4444eeee5555ffff6666aaaa7777bbbb8888cccc9999", "market_id": "0x1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b", "fill_price": 0.621, "arrival_mid": 0.62, "slippage_bps": 1.6, "latency_ms": 85, "drift_bps": 4, "price_score": 0.98, "latency_score": 0.83, "toxicity_score": 0.96, "overall_score": 0.92, "verdict": "FILL_QUALITY_PASS", "collateral": "pUSD", "evaluated_at_ms": 1746770030000 }, "developer_log": { "report_id": "rep_2b3c4d5e6f7a8b9c", "slippage_bps": 1.6, "latency_ms": 85, "drift_bps": 4, "price_score": 0.98, "latency_score": 0.83, "toxicity_score": 0.96, "overall_score": 0.92, "min_score_for_repeat": 0.6, "warn_threshold_bps": 20 }, "user_explanations": [ { "situation": "Fill quality pass", "message": "Your order was filled at a good price with low slippage and no signs of adverse market conditions." }, { "situation": "Fill quality below threshold", "message": "Your order was filled, but the execution quality was lower than expected \u2014 the price moved slightly against you near the time of the fill." }, { "situation": "Fill quality critical", "message": "Your order filled with significant slippage or post-fill adverse movement. This market may have been experiencing unusual conditions at the time." } ], "failure_modes": { "main_failure_mode": "Post-fill trade tape unavailable because ws_market feed is lagged, causing toxicity_score to default to 1.0 (optimistic), masking genuine adverse selection.", "false_positive_risk": "Brief price spike at fill time causes high slippage_bps reading, triggering FILL_QUALITY_BELOW_THRESHOLD when the fill was actually at a reasonable price.", "false_negative_risk": "toxicity_horizon_s too short to capture slow-moving adverse drift, producing a high toxicity_score on a genuinely toxic fill.", "safe_fallback": "If ws_market tape is unavailable, set toxicity_score=None and emit WARN with FILL_QUALITY_TOXICITY_UNAVAILABLE; do not block ExecutionReport publication.", "required_dependencies": [ "ws_market trade tape", "clob_public top-of-book snapshot at fill time", "ExecutionReport(FILLED) from OrderLifecycleManager" ] }, "acceptance_tests": { "unit": [ { "test": "Slippage score computed correctly", "setup": "fill_price=0.63, arrival_mid=0.62, warn_threshold_bps=20", "expected": "slippage_bps=161 > 20 \u2192 FILL_SLIPPAGE_ELEVATED, price_score low" }, { "test": "Overall score passes when all dimensions good", "setup": "slippage_bps=2, latency_ms=50, drift_bps=3", "expected": "overall_score > 0.9; FILL_QUALITY_PASS" }, { "test": "Critical alert when score < 0.2", "setup": "slippage_bps=90, latency_ms=480, drift_bps=95", "expected": "overall_score < 0.2; FILL_QUALITY_CRITICAL emitted" } ], "integration": [ { "test": "End-to-end: FILLED event \u2192 score computed \u2192 annotated ExecutionReport published", "expected": "ExecutionReport includes all score fields; published to polytraders.reports.execution" }, { "test": "Toxicity measurement: wait toxicity_horizon_s, fetch tape, compute drift_bps", "expected": "drift_bps populated; toxicity_score reflects post-fill price move" } ], "property": [ { "property": "Overall score always in [0, 1]", "required": "Always true" }, { "property": "FillQualityAnalyzer never modifies fill_price or order_id in the ExecutionReport", "required": "Always true \u2014 read-only annotation" } ] }, "checklist_overrides": {}, "legacy_goal": "Score every fill on price, latency, and toxicity vs. the intent.", "legacy_pm_signals": [ "Slippage vs. arrival mid and decision price", "Queue position at time of fill", "Post-fill mark drift over N seconds (toxicity proxy)" ], "legacy_external_feeds": [], "reporting_groups": [ "execution", "post_trade" ], "network": [ "polygon" ], "api_surface": [ "clob_public", "ws_market", "internal" ], "version": { "spec": "2.0.0", "implementation": "0.1.0", "schema": "2", "released": null, "planned_release": "Q4-2026" }, "migration_history": [ { "date": "2026-04-28", "from": "n/a", "to": "v2-spec", "reason": "Spec drafted post-CLOB-V2 cutover; bot not yet implemented", "action_taken": "Designed against V2 schema (pUSD, builder codes, V2 EIP-712 domain)" } ], "polymarket_v2_compat": { "clob_version": "v2", "collateral": "pUSD", "eip712_domain_version": "2", "builder_code_aware": false, "negrisk_aware": false, "multichain_ready": false, "sdk_used": "py-clob-client-v2", "settlement_contract": "CTFExchangeV2", "notes": "All fill prices and slippage amounts are denominated in pUSD; no fee fields on signed orders \u2014 fees are operator-set at match time by CTFExchangeV2." }, "reference_implementation": { "pseudocode": "FUNCTION analyzeFill(executionReport):\n // 1. Fetch arrival mid at fill time\n book = FETCH clob_public.GET('/book?market=' + executionReport.market_id)\n arrivalMid = (book.best_bid + book.best_ask) / 2\n\n // 2. Slippage score\n slippageBps = abs(executionReport.fill_price - arrivalMid) / arrivalMid * 10000\n priceScore = max(0, 1 - slippageBps / 100)\n IF slippageBps > params.warn_threshold_bps:\n EMIT reason(FILL_SLIPPAGE_ELEVATED)\n\n // 3. Latency score\n latencyMs = executionReport.fill_ts_ms - executionReport.submit_ms\n latencyScore = max(0, 1 - latencyMs / 500)\n\n // 4. Toxicity score (deferred by toxicity_horizon_s)\n WAIT params.toxicity_horizon_s * 1000\n tape = FETCH ws_market.recent_fills(executionReport.market_id, last_n=100)\n IF tape IS NULL:\n toxicityScore = None\n EMIT reason(FILL_QUALITY_TOXICITY_UNAVAILABLE, WARN)\n ELSE:\n driftBps = computeDrift(tape, executionReport.fill_price, params.toxicity_horizon_s)\n toxicityScore = max(0, 1 - driftBps / 100)\n\n // 5. Overall score and verdict\n scores = [s FOR s IN [priceScore, latencyScore, toxicityScore] IF s IS NOT None]\n overallScore = mean(scores)\n IF overallScore < 0.2:\n verdict = 'FILL_QUALITY_CRITICAL'\n ELIF overallScore < params.min_score_for_repeat:\n verdict = 'FILL_QUALITY_BELOW_THRESHOLD'\n ELSE:\n verdict = 'FILL_QUALITY_PASS'\n\n // 6. Annotate and emit\n executionReport.fill_quality = {slippageBps, latencyMs, driftBps, overallScore, verdict}\n EMIT ExecutionReport(executionReport) TO params.publish_to", "sdk_calls": [ "clob_public.GET('/book?market=' + market_id)", "ws_market.recent_fills(market_id, last_n=100)" ], "complexity": "O(F) where F = tape size (capped at 100)" }, "wire_examples": { "input": [ { "label": "ExecutionReport (FILLED) from OrderLifecycleManager", "source": "exec.orderlifecyclemanager", "payload": { "order_id": "0xbbbb2222cccc3333dddd4444eeee5555ffff6666aaaa7777bbbb8888cccc9999", "status": "FILLED", "fill_price": 0.621, "fill_usd": 450, "submit_ms": 1746770000000, "fill_ts_ms": 1746770000085, "collateral": "pUSD" } } ], "output": [ { "label": "Annotated ExecutionReport with fill quality scores", "payload": { "report_id": "rep_2b3c4d5e6f7a8b9c", "bot_id": "exec.fillqualityanalyzer", "verdict": "FILL_QUALITY_PASS", "slippage_bps": 1.6, "latency_ms": 85, "drift_bps": 4, "overall_score": 0.92, "collateral": "pUSD", "evaluated_at_ms": 1746770030000 } } ] }, "reason_codes": [ { "code": "FILL_QUALITY_PASS", "severity": "INFO", "meaning": "Fill quality score >= min_score_for_repeat on all dimensions.", "action": "Emit annotated ExecutionReport; no further action.", "user_message": "" }, { "code": "FILL_QUALITY_BELOW_THRESHOLD", "severity": "WARN", "meaning": "Overall fill score below min_score_for_repeat.", "action": "Annotate ExecutionReport; emit WARN to strategy layer.", "user_message": "Your order was filled with below-average execution quality." }, { "code": "FILL_QUALITY_CRITICAL", "severity": "HARD_REJECT", "meaning": "Overall fill score below 0.2 \u2014 severe slippage, latency, or toxicity.", "action": "Emit HARD_REJECT; alert risk pipeline.", "user_message": "Your order filled with significant adverse conditions." }, { "code": "FILL_SLIPPAGE_ELEVATED", "severity": "WARN", "meaning": "Slippage exceeded warn_threshold_bps.", "action": "Annotate ExecutionReport.", "user_message": "Your fill price differed from the market mid more than expected." }, { "code": "FILL_QUALITY_TOXICITY_UNAVAILABLE", "severity": "WARN", "meaning": "Post-fill trade tape unavailable; toxicity_score not computed.", "action": "Omit toxicity_score from overall; note in ExecutionReport.", "user_message": "" } ], "metrics": { "emitted": [ { "name": "polytraders_exec_fillqualityanalyzer_scores", "type": "histogram", "unit": "score", "labels": [ "verdict" ], "meaning": "Distribution of overall fill quality scores per verdict." }, { "name": "polytraders_exec_fillqualityanalyzer_slippage_bps", "type": "histogram", "unit": "bps", "labels": [ "market_id" ], "meaning": "Distribution of slippage in bps across all scored fills." }, { "name": "polytraders_exec_fillqualityanalyzer_drift_bps", "type": "histogram", "unit": "bps", "labels": [], "meaning": "Distribution of post-fill drift bps over toxicity_horizon_s." }, { "name": "polytraders_exec_fillqualityanalyzer_critical_total", "type": "counter", "unit": "count", "labels": [], "meaning": "Total fills scoring below 0.2 (FILL_QUALITY_CRITICAL)." } ], "alerts": [ { "name": "FQACriticalFillRate", "condition": "rate(polytraders_exec_fillqualityanalyzer_critical_total[10m]) > 0.05", "severity": "P1", "runbook": "#runbook-fqa-critical" }, { "name": "FQAHighSlippage", "condition": "histogram_quantile(0.95, rate(polytraders_exec_fillqualityanalyzer_slippage_bps_bucket[5m])) > 40", "severity": "P2", "runbook": "#runbook-fqa-slippage" } ] }, "state": { "store": "in-memory with WAL", "shape": "Per fill record: {order_id, score_dimensions, overall_score, verdict, evaluated_at_ms}", "ttl": "24h in-memory; 7y in WAL", "recovery": "WAL replayed on restart; in-memory cache rebuilt from last 1000 fills.", "size_estimate": "~300 bytes per scored fill; ~300 KB for 1000 fills in memory" }, "concurrency": { "execution_model": "async per-fill coroutine (deferred by toxicity_horizon_s)", "max_in_flight": 100, "idempotency_key": "order_id + fill_ts_ms", "timeout_ms": 35000, "backpressure": "Drop oldest pending if > 100 fills awaiting toxicity measurement", "locking": "none \u2014 read-only annotation on immutable ExecutionReport" }, "dependencies": { "depends_on": [ { "bot_id": "exec.orderlifecyclemanager", "why": "Provides FILLED ExecutionReport with fill_price, size, submit_ms.", "contract": "FillQualityAnalyzer activates only on FILLED status events." } ], "emits_to": [ { "bot_id": "gov.posttradeexplainer", "why": "Annotated fill quality scores feed the post-trade explanation layer.", "contract": "ExecutionReport with quality annotations published to polytraders.reports.execution." } ], "sibling": [], "external": [ { "service": "CLOB V2 public API", "endpoint": "https://clob.polymarket.com", "sla": "99.9% / 200ms p99", "failure_mode": "arrivalMid set to None; price_score omitted from overall." }, { "service": "WS market feed", "endpoint": "wss://ws-subscriptions-clob.polymarket.com/ws/market", "sla": "best-effort", "failure_mode": "toxicity_score omitted; FILL_QUALITY_TOXICITY_UNAVAILABLE emitted." } ] }, "security_surfaces": { "signs_orders": false, "private_key_access": "none", "abuse_vectors": [ "Injecting a fabricated FILLED event with a favourable fill_price to suppress a poor-quality fill from scoring", "Replaying old ExecutionReports to inflate fill quality metrics" ], "mitigations": [ "ExecutionReport messages validated against originating bot HMAC before scoring", "Idempotency key (order_id + fill_ts_ms) prevents duplicate scoring" ] }, "failure_injection": [ { "scenario": "WS_MARKET_TAPE_UNAVAILABLE", "how_to_inject": "Disconnect ws_market feed before toxicity_horizon_s elapses", "expected_behaviour": "toxicity_score=None; FILL_QUALITY_TOXICITY_UNAVAILABLE emitted; overall_score computed from price and latency only", "recovery": "Feed reconnects; next fill scored normally" }, { "scenario": "HIGH_SLIPPAGE_FILL", "how_to_inject": "Inject fill_price=0.70 when arrival_mid=0.62", "expected_behaviour": "slippage_bps=1290; FILL_SLIPPAGE_ELEVATED emitted; overall_score < 0.2; FILL_QUALITY_CRITICAL", "recovery": "Automatic on next fill" }, { "scenario": "CLOB_PUBLIC_UNAVAILABLE", "how_to_inject": "Block clob_public at fill time", "expected_behaviour": "arrivalMid=None; price_score omitted; WARN emitted; overall_score from latency and toxicity only", "recovery": "Automatic on next fill" } ], "runbook": { "summary": "FillQualityAnalyzer incidents are typically high critical fill rates (systematic adverse selection) or stale market tape causing toxicity measurement failures.", "oncall_actions": [ { "alert": "FQACriticalFillRate", "first_step": "Check which markets have high critical rates in Grafana; cross-reference with AntiToxicFill cooldown history.", "diagnosis": "", "mitigation": "", "escalation": "Strategy pod lead for market selection review" }, { "alert": "FQAHighSlippage", "first_step": "Check arrival_mid availability and ws_market feed freshness.", "diagnosis": "", "mitigation": "", "escalation": "Exec pod lead if p95 slippage > 40 bps sustained > 10 min" } ], "manual_overrides": [ { "name": "rescore_fill", "how": "polytraders bot rescore exec.fillqualityanalyzer --order ", "when": "Rescore a fill after feed availability is restored to get accurate toxicity_score.", "command": "polytraders bot rescore exec.fillqualityanalyzer --order ", "effect": "Rescore a fill after feed availability is restored to get accurate toxicity_score." } ], "healthcheck": "GET /internal/health/fillqualityanalyzer \u2192 green if ws_market feed connected, clob_public reachable, critical_total rate < 0.05/min, pending toxicity evals < 100; red if ws_market disconnected, critical rate > 0.2/min, pending evals > 100" }, "promotion_gates": { "to_shadow": [ { "gate": "Score computation unit tests pass with known inputs", "how_measured": "CI test run", "threshold": "100% pass" } ], "to_limited_live": [ { "gate": "Mean slippage_bps < 20 over 48h shadow run", "how_measured": "polytraders_exec_fillqualityanalyzer_slippage_bps histogram", "threshold": "p50 < 20 bps" } ], "to_general_live": [ { "gate": "Critical fill rate < 0.5% over 7-day limited-live period", "how_measured": "polytraders_exec_fillqualityanalyzer_critical_total", "threshold": "< 0.5%" } ] }, "reporting": { "emits_kinds": [ "ExecutionReport" ], "topics": [ "polytraders.reports.execution" ], "partition_key": "trace_id", "cadence": "every-event", "retention_class": "7y", "sampling_rule": "emit-every", "bus_failure_action": "wal-then-retry", "user_visible": "yes", "consumes_kinds": [ "DecisionReport" ] }, "capital_impact": "Direct", "mode_support": [ "quarantine" ], "v3_status": { "phase": 5, "phase_name": "Execution rails", "docs": { "done": 27, "total": 27, "state": "done" }, "impl": { "done": 0, "total": 15, "state": "pending" }, "runtime": { "done": 0, "total": 8, "state": "pending" }, "overall": "pending" } }