{ "schema_version": "2.0.0", "bot_id": "4.1", "bot_name": "NewsIngest", "slug": "newsingest", "layer": "Intelligence", "layer_key": "intel", "bot_class": "Signal Service", "authority": [ "Read-only" ], "status": "live", "readiness": "General live", "flagship": false, "is_reference": false, "public_export": false, "identity": { "layer": "Intelligence", "bot_class": "Signal Service", "authority": "Read-only", "runs_before": "news-materiality-trader, contradictiondetector", "runs_after": "RSS / partner API fetch and dedup", "applies_to": "All live Polymarket markets with matching entities in watchlist", "default_mode": "general_live", "user_visible": "Advanced details only", "developer_owner": "Polytraders core \u2014 Intelligence pod" }, "purpose": "NewsIngest continuously pulls news from external feeds (RSS, partner APIs), resolves named entities to Polymarket market IDs via the Gamma API, and scores each story for materiality. High-materiality stories trigger immediate ObservationReport emission; routine items are sampled 1/10. NewsIngest is strictly read-only \u2014 it never submits, signs, or modifies orders. Output is consumed by news-materiality-trader and contradictiondetector.", "why_it_matters": [ { "failure": "High-materiality story missed", "consequence": "news-materiality-trader fails to re-price after a major event; positions held through adverse resolution." }, { "failure": "Entity resolution maps story to wrong market", "consequence": "Materiality signal sent to unrelated market, polluting downstream strategies with false signal." }, { "failure": "Duplicate story flooding the bus", "consequence": "Contradictiondetector and materiality-trader overwhelmed with redundant events, latency spikes." }, { "failure": "Stale feed accepted as fresh", "consequence": "Old news recycled as new; obsolete materiality score triggers trades on stale information." } ], "polymarket_inputs": [ { "input": "Market metadata including condition IDs, question text, and neg-risk flags", "source": "Gamma API", "required": true, "use": "Resolve entity mentions in news stories to specific Polymarket market IDs." }, { "input": "Market resolution source and rules text", "source": "Gamma API", "required": false, "use": "Determine whether a story is relevant to UMA-resolved vs partner-resolved markets." } ], "internal_inputs": [ { "input": "Entity watchlist and market mapping", "source": "StrategyRegistry / config", "required": true, "use": "Filter incoming stories to only those touching watched entities." }, { "input": "KillSwitch active flag", "source": "KillSwitch", "required": true, "use": "Suppress ObservationReport emissions when KillSwitch is active; continue ingesting passively." } ], "raw_params": [ "entity_watchlist \u00b7 list", "materiality_min \u00b7 0\u20131", "source_priority \u00b7 ordered list", "dedup_window_s \u00b7 int" ], "parameters": [ { "name": "materiality_min", "default": 0.3, "warning": 0.1, "hard": 0.05, "controls": "Minimum materiality score (0\u20131) for a story to be forwarded. Stories below this threshold are silently dropped.", "why_default_matters": "A threshold of 0.3 balances signal noise \u2014 lower values flood downstream bots with irrelevant items.", "threshold_logic": [ { "condition": "score \u2265 0.3", "action": "Emit ObservationReport" }, { "condition": "0.1\u20130.3", "action": "WARN \u2014 borderline; include with LOW_MATERIALITY flag" }, { "condition": "< 0.05", "action": "Hard drop \u2014 NEWSINGEST_BELOW_MATERIALITY_FLOOR" } ], "dev_check": "if (story.score < params.hard) drop('NEWSINGEST_BELOW_MATERIALITY_FLOOR');", "user_facing": "Only stories judged sufficiently relevant to your markets are forwarded." }, { "name": "dedup_window_s", "default": 300, "warning": 60, "hard": 10, "controls": "Seconds within which an identical story fingerprint is considered a duplicate and silently dropped.", "why_default_matters": "300 s suppresses wire-service re-transmissions without missing genuine updates.", "threshold_logic": [ { "condition": "window \u2265 300 s", "action": "Standard dedup" }, { "condition": "60\u2013300 s", "action": "WARN \u2014 tighter window; risk of some duplicates" }, { "condition": "< 10 s", "action": "Reject config change \u2014 PARAMETER_CHANGE_REQUIRES_APPROVAL" } ], "dev_check": "if (p.dedup_window_s < p.hard) throw ConfigError('PARAMETER_CHANGE_REQUIRES_APPROVAL');", "user_facing": "Duplicate news items are suppressed automatically." }, { "name": "source_priority", "default": [ "reuters", "ap", "bloomberg", "league_feeds", "official_handles" ], "warning": null, "hard": null, "controls": "Ordered list of feed sources; earlier entries get higher materiality weight.", "why_default_matters": "Wire services carry higher factual confidence than unofficial handles.", "threshold_logic": [], "dev_check": "", "user_facing": "Higher-confidence sources influence opportunity scores more than social feeds." }, { "name": "emit_every_above", "default": 0.7, "warning": 0.5, "hard": 0.4, "controls": "Materiality score above which every story is emitted (no sampling). Below this, 1/10 sampling applies.", "why_default_matters": "High-materiality stories (\u22650.7) must never be dropped by the sampler.", "threshold_logic": [ { "condition": "score \u2265 0.7", "action": "emit-every \u2014 no sampling" }, { "condition": "0.3\u20130.7", "action": "sample-1/10" }, { "condition": "< 0.4", "action": "Cannot lower emit_every threshold without Risk review" } ], "dev_check": "if (p.emit_every_above < p.hard) throw ConfigError('PARAMETER_CHANGE_REQUIRES_APPROVAL');", "user_facing": "Breaking news is always forwarded in full; routine updates are sampled." } ], "default_config": { "bot_id": "intel.newsingest", "version": "2.1.0", "mode": "general_live", "defaults": { "materiality_min": 0.3, "dedup_window_s": 300, "source_priority": [ "reuters", "ap", "bloomberg", "league_feeds", "official_handles" ], "emit_every_above": 0.7 }, "locked": { "dedup_window_s": { "min": 10 }, "emit_every_above": { "min": 0.4 } } }, "implementation_flow": [ "Subscribe to all configured external feeds (RSS polling + partner API long-poll or webhook).", "On each incoming story: compute a 64-bit fingerprint (source + headline hash). Check dedup_window_s ring buffer; if duplicate, drop silently.", "Perform entity extraction (NER); resolve entities against Gamma API market metadata to produce a list of candidate condition_ids.", "If no condition_id matched, drop story with NEWSINGEST_NO_MARKET_MATCH.", "Score materiality (0\u20131) based on: source priority weight \u00d7 entity confidence \u00d7 recency decay \u00d7 neg-risk amplifier if any matched market has neg_risk=true.", "If score < materiality_min hard floor, drop with NEWSINGEST_BELOW_MATERIALITY_FLOOR.", "Check KillSwitch; if active, ingest and score but suppress ObservationReport emissions.", "Apply sampling rule: if score \u2265 emit_every_above, emit every time; else sample 1/10.", "Emit ObservationReport with: report_id, trace_id, condition_ids, materiality_score, story_summary, source, pub_ts_ms, neg_risk_flag, and sampling_applied.", "Log ingest cycle summary: stories_received, deduped, no_match, below_floor, sampled_out, emitted." ], "decision_logic": { "approve": "Not applicable \u2014 NewsIngest does not issue approval votes. It emits ObservationReports for stories that pass dedup, entity resolution, and materiality thresholds.", "reshape_required": "Not applicable \u2014 NewsIngest is read-only.", "reject": "Stories are dropped (not emitted) for: duplicate fingerprint (NEWSINGEST_DEDUP_DROP), no market match (NEWSINGEST_NO_MARKET_MATCH), below materiality floor (NEWSINGEST_BELOW_MATERIALITY_FLOOR), or KillSwitch active (KILL_SWITCH_ACTIVE).", "warning_only": "Stories with materiality between warning and default thresholds are included with a LOW_MATERIALITY flag. Downstream bots decide whether to act." }, "decision_output_schema": "ObservationReport", "decision_output_example": { "report_id": "rep_ni_0xabc1_1746700800000", "trace_id": "trc_0xdeadbeef01020304", "bot_id": "intel.newsingest", "kind": "ObservationReport", "condition_ids": [ "0xabc1230000000000000000000000000000000000000000000000000000000000" ], "materiality_score": 0.82, "story_summary": "AP: Fed holds rates unchanged at May 2026 meeting", "source": "ap", "pub_ts_ms": 1746700800000, "neg_risk_flag": false, "sampling_applied": false, "warnings": [], "emitted_at_ms": 1746700800450 }, "developer_log": { "bot_id": "intel.newsingest", "cycle_ts_ms": 1746700800000, "stories_received": 47, "deduped": 12, "no_market_match": 8, "below_floor": 5, "sampled_out": 19, "emitted": 3, "top_materiality": 0.82, "killswitch_active": false }, "user_explanations": [ { "situation": "Fewer news signals than expected", "message": "Many stories were filtered out because they did not match any tracked market, fell below the materiality threshold, or were duplicates of items already processed." }, { "situation": "News signal delayed relative to headline time", "message": "Stories go through entity resolution and materiality scoring before being forwarded. High-materiality breaking news is forwarded immediately; routine items may be sampled." }, { "situation": "No signals during KillSwitch", "message": "News ingestion continues in the background, but no signals are forwarded while trading is paused system-wide." } ], "failure_modes": { "main_failure_mode": "Entity resolution incorrectly maps a high-materiality story to a wrong condition_id, causing downstream strategies to act on a market that is not actually affected.", "false_positive_risk": "A story with a borderline materiality score is forwarded with LOW_MATERIALITY flag and triggers a trade that would not have passed a stricter threshold.", "false_negative_risk": "A high-materiality story is dropped due to an entity resolution miss \u2014 the story mentions an entity in shorthand not in the watchlist dictionary.", "safe_fallback": "If Gamma API is unavailable, entity-to-market resolution halts and no ObservationReports are emitted for that feed cycle. Stories are buffered up to dedup_window_s; if API recovers within that window, they are re-processed.", "required_dependencies": [ "External feeds (RSS / partner APIs) reachable", "Gamma API for entity-to-market resolution", "KillSwitch active flag readable" ] }, "acceptance_tests": { "unit": [ { "test": "Duplicate story is dropped within dedup_window_s", "setup": "Inject same headline twice within 60 s", "expected": "Second occurrence silently dropped; NEWSINGEST_DEDUP_DROP logged" }, { "test": "Story below materiality hard floor is dropped", "setup": "story.score=0.03, hard=0.05", "expected": "Story dropped with NEWSINGEST_BELOW_MATERIALITY_FLOOR" }, { "test": "High-materiality story emitted without sampling", "setup": "story.score=0.82 \u2265 emit_every_above=0.7", "expected": "ObservationReport emitted with sampling_applied=false" }, { "test": "Routine story sampled 1/10", "setup": "story.score=0.4, emit_every_above=0.7; run 10 stories", "expected": "Approximately 1 ObservationReport emitted; sampling_applied=true on remainder" }, { "test": "KillSwitch suppresses emissions", "setup": "killswitch.active=true; high-materiality story arrives", "expected": "Story scored but no ObservationReport emitted; KILL_SWITCH_ACTIVE logged" }, { "test": "No market match drops story", "setup": "Story entities not in Gamma API watchlist", "expected": "Drop with NEWSINGEST_NO_MARKET_MATCH" } ], "integration": [ { "test": "AP wire story reaches contradictiondetector and news-materiality-trader", "expected": "ObservationReport with correct condition_id propagates to both downstream bots within 2 s" }, { "test": "Gamma API unavailability halts emission with buffering", "expected": "No ObservationReports emitted during outage; stories buffered; re-processed on recovery" }, { "test": "neg-risk market amplifies materiality score", "expected": "Story matched to neg-risk market has materiality_score elevated and neg_risk_flag=true in output" } ], "property": [ { "property": "NewsIngest never submits, signs, or modifies any order", "required": "Always true" }, { "property": "No ObservationReport emitted when KillSwitch is active", "required": "Always true" }, { "property": "No ObservationReport emitted when Gamma API is unavailable", "required": "Always true \u2014 missing entity resolution halts emissions" } ] }, "checklist_overrides": {}, "legacy_goal": "Real-time news ingestion + entity resolution + materiality scoring.", "legacy_pm_signals": [ "Watchlist of tracked markets & entity dictionary" ], "legacy_external_feeds": [ "Reuters & AP wires", "Bloomberg headlines", "League / federation feeds", "Official organisation handles (curated)" ], "reporting_groups": [ "pretrade_intel" ], "network": [ "polygon" ], "api_surface": [ "gamma", "internal" ], "version": { "spec": "2.0.0", "implementation": "2.1.0", "schema": "2", "released": "2026-04-28" }, "migration_history": [ { "date": "2026-04-28", "from": "v1", "to": "v2", "reason": "CLOB V2 cutover \u2014 collateral denomination change", "action_taken": "Materiality score and volume references updated from USDC.e to pUSD. Gamma API queries updated to include enableNegRisk flag for neg-risk market detection. No signed-order plumbing in this bot; no feeRateBps or nonce fields to remove." } ], "polymarket_v2_compat": { "clob_version": "v2", "collateral": "pUSD", "eip712_domain_version": "2", "builder_code_aware": false, "negrisk_aware": true, "multichain_ready": false, "sdk_used": "py-clob-client-v2", "settlement_contract": "CTFExchangeV2", "notes": "NewsIngest reads Gamma API negRisk and enableNegRisk fields to amplify materiality scores for stories touching multi-outcome negative-risk markets; volume figures in ObservationReport payloads are denominated in pUSD." }, "reference_implementation": { "summary": "Subscribes to external feeds, fingerprints each story for dedup, resolves entities to Polymarket condition_ids via Gamma API, scores materiality, applies sampling, and emits ObservationReport for stories that pass all gates.", "language_note": "Pseudocode is language-agnostic. FETCH = read input. EMIT = produce output.", "pseudocode": "FUNCTION ingestFeed(feed_event):\n // --- 0. Dedup ---\n fp = fingerprint(feed_event.source + feed_event.headline)\n IF fp IN dedup_ring_buffer(dedup_window_s):\n LOG DEBUG 'NEWSINGEST_DEDUP_DROP'\n RETURN\n dedup_ring_buffer.add(fp, ttl=dedup_window_s)\n\n // --- 1. KillSwitch gate ---\n ks = FETCH internal.killswitch.status\n killswitchActive = ks.active\n\n // --- 2. Entity extraction + market resolution ---\n entities = extractEntities(feed_event.body)\n condition_ids = []\n FOR entity IN entities:\n markets = gamma_api.GET('/markets?keyword=' + entity.canonical)\n FOR m IN markets:\n IF m.active AND NOT m.closed:\n condition_ids.append(m.condition_id)\n\n IF condition_ids IS EMPTY:\n LOG DEBUG 'NEWSINGEST_NO_MARKET_MATCH'\n RETURN\n\n // --- 3. Materiality scoring ---\n source_weight = SOURCE_WEIGHTS[feed_event.source] OR 0.5\n entity_conf = AVG(entity.confidence FOR entity IN entities)\n recency_decay = exp(-age_s / 600) // half-life 10 min\n neg_risk_amp = 1.2 IF any(m.neg_risk FOR m IN markets) ELSE 1.0\n score = source_weight * entity_conf * recency_decay * neg_risk_amp\n\n IF score < params.materiality_min.hard:\n LOG DEBUG 'NEWSINGEST_BELOW_MATERIALITY_FLOOR'\n RETURN\n\n // --- 4. Warning annotation ---\n warnings = []\n IF score < params.materiality_min.default:\n warnings.append('NEWSINGEST_LOW_MATERIALITY')\n\n // --- 5. Sampling ---\n IF score >= params.emit_every_above:\n sampling_applied = false\n ELSE:\n IF random() > 0.1: // sample-1/10\n RETURN\n sampling_applied = true\n\n // --- 6. KillSwitch suppress ---\n IF killswitchActive:\n LOG INFO 'KILL_SWITCH_ACTIVE \u2014 suppressing ObservationReport'\n RETURN\n\n // --- 7. Emit ---\n report = ObservationReport(\n report_id = 'rep_ni_' + condition_ids[0][:6] + '_' + now_ms(),\n trace_id = newTraceId(),\n bot_id = 'intel.newsingest',\n kind = 'ObservationReport',\n condition_ids = condition_ids,\n materiality_score= score,\n story_summary = feed_event.headline[:200],\n source = feed_event.source,\n pub_ts_ms = feed_event.pub_ts_ms,\n neg_risk_flag = neg_risk_amp > 1.0,\n sampling_applied = sampling_applied,\n warnings = warnings,\n emitted_at_ms = now_ms()\n )\n EMIT internal.bus.observations <- report\n", "sdk_calls": [ "gamma_api.GET('/markets?keyword=')", "gamma_api.GET('/market/')" ], "complexity": "O(E \u00d7 M) per story, where E = extracted entities, M = live markets per entity" }, "wire_examples": { "input": { "label": "Incoming AP wire story", "source": "ap", "payload": { "source": "ap", "headline": "Federal Reserve holds rates at 4.25\u20134.50% at May 2026 meeting", "body": "The Federal Open Market Committee voted unanimously to hold...", "pub_ts_ms": 1746700800000, "feed": "rss" } }, "output": { "label": "ObservationReport \u2014 high-materiality story", "payload": { "report_id": "rep_ni_0xabc1_1746700800000", "trace_id": "trc_0xdeadbeef01020304050607", "bot_id": "intel.newsingest", "kind": "ObservationReport", "condition_ids": [ "0xabc1230000000000000000000000000000000000000000000000000000000000" ], "materiality_score": 0.82, "story_summary": "Federal Reserve holds rates at 4.25\u20134.50% at May 2026 meeting", "source": "ap", "pub_ts_ms": 1746700800000, "neg_risk_flag": false, "sampling_applied": false, "warnings": [], "emitted_at_ms": 1746700800452 } } }, "reason_codes": [ { "code": "NEWSINGEST_DEDUP_DROP", "severity": "INFO", "meaning": "Story fingerprint matched a recent entry in the dedup ring buffer.", "action": "Silently drop; log at DEBUG level.", "user_message": "" }, { "code": "NEWSINGEST_NO_MARKET_MATCH", "severity": "INFO", "meaning": "Entity extraction produced no condition_ids from Gamma API.", "action": "Drop story; no ObservationReport emitted.", "user_message": "" }, { "code": "NEWSINGEST_BELOW_MATERIALITY_FLOOR", "severity": "INFO", "meaning": "Story materiality score below hard floor; irrelevant to any open position.", "action": "Drop story silently.", "user_message": "" }, { "code": "NEWSINGEST_LOW_MATERIALITY", "severity": "WARN", "meaning": "Story score between warning and default thresholds; forwarded with flag.", "action": "Emit ObservationReport with LOW_MATERIALITY warning; downstream bots decide.", "user_message": "" }, { "code": "KILL_SWITCH_ACTIVE", "severity": "HARD_REJECT", "meaning": "KillSwitch is active; ObservationReport emissions suppressed.", "action": "Continue ingesting and scoring but do not emit.", "user_message": "News signals are paused while trading is suspended system-wide." }, { "code": "STALE_DATA", "severity": "WARN", "meaning": "Gamma API unavailable; entity resolution halted for this cycle.", "action": "Buffer incoming stories up to dedup_window_s; halt emissions until API recovers.", "user_message": "" }, { "code": "MARKET_CLOSED", "severity": "EXPLAIN", "meaning": "Entity resolved to a market that is already closed or resolved.", "action": "Skip that condition_id; continue processing remaining matched markets.", "user_message": "" }, { "code": "NEWSINGEST_FEED_UNREACHABLE", "severity": "WARN", "meaning": "An external feed (RSS / partner API) is unreachable.", "action": "Log WARN; continue processing other feeds; alert if all feeds unreachable.", "user_message": "" } ], "metrics": { "emitted": [ { "name": "polytraders_intel_newsingest_stories_received_total", "type": "counter", "unit": "count", "labels": [ "source" ], "meaning": "Total raw stories received from all feeds, per source." }, { "name": "polytraders_intel_newsingest_observations_emitted_total", "type": "counter", "unit": "count", "labels": [ "source", "sampling_applied" ], "meaning": "ObservationReport items emitted to the internal bus." }, { "name": "polytraders_intel_newsingest_drop_total", "type": "counter", "unit": "count", "labels": [ "reason_code" ], "meaning": "Stories dropped, broken down by drop reason code." }, { "name": "polytraders_intel_newsingest_materiality_score", "type": "histogram", "unit": "ratio", "labels": [], "meaning": "Distribution of materiality scores for all stories that passed entity resolution." }, { "name": "polytraders_intel_newsingest_entity_resolution_latency_ms", "type": "histogram", "unit": "ms", "labels": [], "meaning": "Wall-clock time for Gamma API entity resolution per story." }, { "name": "polytraders_intel_newsingest_feed_lag_s", "type": "gauge", "unit": "seconds", "labels": [ "source" ], "meaning": "Age of the most recent story received from each feed source." } ], "alerts": [ { "name": "NewsIngestAllFeedsDown", "condition": "rate(polytraders_intel_newsingest_stories_received_total[5m]) == 0", "severity": "page", "runbook": "#runbook-newsingest-feeds-down" }, { "name": "NewsIngestGammaAPIDown", "condition": "rate(polytraders_intel_newsingest_entity_resolution_latency_ms_count[5m]) == 0 AND rate(polytraders_intel_newsingest_stories_received_total[5m]) > 0", "severity": "page", "runbook": "#runbook-newsingest-gamma-api-down" }, { "name": "NewsIngestHighDropRate", "condition": "rate(polytraders_intel_newsingest_drop_total[10m]{reason_code='NEWSINGEST_NO_MARKET_MATCH'}) / rate(polytraders_intel_newsingest_stories_received_total[10m]) > 0.95", "severity": "warn", "runbook": "#runbook-newsingest-entity-mismatch" }, { "name": "NewsIngestFeedLag", "condition": "polytraders_intel_newsingest_feed_lag_s{source='reuters'} > 120", "severity": "warn", "runbook": "#runbook-newsingest-feed-lag" } ], "dashboards": [ "Grafana \u2014 Intelligence / NewsIngest feed health", "Grafana \u2014 Intelligence / materiality score distribution" ], "log_level": "info" }, "state": { "store": "redis", "shape": "Dedup ring buffer keyed by story fingerprint (64-bit); TTL = dedup_window_s. Entity watchlist snapshot refreshed from config every 5 min.", "ttl": "dedup_window_s (default 300 s) per fingerprint", "recovery": "On cold start, dedup buffer is empty; the first dedup_window_s of operation may emit some near-duplicates from wire re-transmissions.", "size_estimate": "~50 KB for 1 000 active fingerprints at 300 s TTL" }, "concurrency": { "execution_model": "single-threaded event loop", "max_in_flight": 10, "idempotency_key": "story fingerprint", "timeout_ms": 3000, "backpressure": "drop-after-buffer \u2014 excess stories dropped when internal queue > 500 items", "locking": "none \u2014 dedup ring buffer access is single-threaded" }, "dependencies": { "depends_on": [ { "bot_id": "risk.kill_switch", "why": "KillSwitch gate determines whether ObservationReports are emitted." } ], "emits_to": [ { "bot_id": "strat.news_materiality_trader", "what": "ObservationReport with materiality_score and matched condition_ids" }, { "bot_id": "strat.contradiction_detector", "what": "ObservationReport for contradiction analysis across correlated markets" } ], "sibling": [], "external": [ { "service": "Reuters RSS / AP wire", "sla": "best-effort; typically < 5 s from event to wire", "fallback": "Continue processing other feeds; alert if silent > 120 s" }, { "service": "Bloomberg headline API", "sla": "99.5% / 1 s p99", "fallback": "Degrade gracefully; Reuters/AP fills coverage gap" }, { "service": "Gamma API", "endpoint": "https://gamma-api.polymarket.com", "sla": "99.9% / 500 ms p99", "fallback": "Buffer incoming stories up to dedup_window_s; halt emissions until API recovers" } ] }, "security_surfaces": { "signs_orders": false, "private_key_access": "none", "abuse_vectors": [ "Malicious external feed injecting a crafted headline that resolves to a high-value condition_id with artificially elevated materiality score", "Entity resolution poisoning \u2014 an entity alias added to the watchlist that maps news about an unrelated topic to a targeted market" ], "mitigations": [ "condition_id format validated against known 32-byte hex pattern before inclusion", "Gamma API responses cross-checked against local market ID whitelist", "All ObservationReports are recommendations only \u2014 downstream guardrails independently re-validate" ], "contract_calls": [] }, "failure_injection": [ { "scenario": "STALE_FEED", "how_to_inject": "Disconnect Reuters RSS for 120 s", "expected_behaviour": "Bot emits WARN NEWSINGEST_FEED_UNREACHABLE for Reuters; other feeds continue; NewsIngestFeedLag alert fires", "recovery": "Automatic when feed reconnects within dedup_window_s; buffered stories re-processed" }, { "scenario": "GAMMA_API_DOWN", "how_to_inject": "Block TCP to gamma-api.polymarket.com", "expected_behaviour": "Entity resolution halts; no ObservationReports emitted; STALE_DATA WARN logged; NewsIngestGammaAPIDown alert fires", "recovery": "Automatic on Gamma API recovery; buffered stories re-processed" }, { "scenario": "HIGH_VOLUME_FLOOD", "how_to_inject": "Inject 1 000 unique stories/s for 10 s", "expected_behaviour": "Internal queue fills; excess stories dropped via drop-after-buffer policy; no crash", "recovery": "Automatic when ingestion rate drops below queue drain rate" }, { "scenario": "KILL_SWITCH_ON", "how_to_inject": "Set killswitch.active=true", "expected_behaviour": "Stories ingested and scored but zero ObservationReports emitted; KILL_SWITCH_ACTIVE logged", "recovery": "Emissions resume on first event after KillSwitch reset" }, { "scenario": "ENTITY_RESOLUTION_MISS", "how_to_inject": "Submit story mentioning a real market entity not in the watchlist dictionary", "expected_behaviour": "NEWSINGEST_NO_MARKET_MATCH logged; no ObservationReport emitted", "recovery": "Add entity alias to watchlist; next occurrence resolved correctly" } ], "runbook": { "summary": "NewsIngest incidents are usually external feed outages or Gamma API slowness. The bot is read-only so incidents do not affect active positions \u2014 only new signal discovery.", "oncall_actions": [ { "alert": "NewsIngestAllFeedsDown", "first_action": "Check feed connectivity from the ingest host; verify RSS endpoint reachability.", "escalate_to": "Intelligence pod lead after 10 minutes" }, { "alert": "NewsIngestGammaAPIDown", "first_action": "Check https://gamma-api.polymarket.com status. Stories are buffered; no trade impact until buffer expires.", "escalate_to": "Intelligence pod lead after 15 minutes" }, { "alert": "NewsIngestHighDropRate", "first_action": "Inspect recent no_match drops in developer log. Check if entity watchlist is stale.", "escalate_to": "Intelligence pod lead if drop rate > 95% for > 30 min" }, { "alert": "NewsIngestFeedLag", "first_action": "Check feed_lag_s gauge. If Reuters lag > 120 s, verify partner API credentials and network connectivity.", "escalate_to": "Infra on-call if lag > 300 s" } ], "manual_overrides": [ { "name": "force_pass", "how": "Set config.force_emit=true for 60 s to bypass sampling; use only in debugging", "when": "Verifying that downstream bots receive ObservationReports correctly in staging" } ], "healthcheck": "GET /internal/health/newsingest -> 200 if Last story received < 60 s ago AND Gamma API responding AND dedup buffer writable. RED if All feeds silent > 120 s OR Gamma API errors > 50% for 5 min." }, "promotion_gates": { "to_shadow": [ { "gate": "Unit tests pass for dedup, materiality scoring, and KillSwitch suppression", "how_measured": "CI test run", "threshold": "100% pass" }, { "gate": "Gamma API entity resolution integration test: known entity resolves to correct condition_id", "how_measured": "Integration test", "threshold": "Pass" } ], "to_limited_live": [ { "gate": "Entity resolution latency p99 < 500 ms over 24 h", "how_measured": "polytraders_intel_newsingest_entity_resolution_latency_ms histogram", "threshold": "p99 < 500 ms" }, { "gate": "neg-risk materiality amplifier produces correct neg_risk_flag in output", "how_measured": "Integration test with known neg-risk market entity", "threshold": "Pass" } ], "to_general_live": [ { "gate": "Zero false NEWSINGEST_NO_MARKET_MATCH for entities in the curated watchlist over 7 days", "how_measured": "Grafana drop_total metric filtered by reason_code", "threshold": "< 1% miss rate on watchlist entities" }, { "gate": "KillSwitch suppression verified: zero ObservationReports emitted when KillSwitch active", "how_measured": "Integration test", "threshold": "Pass" } ] }, "reporting": { "emits_kinds": [ "ObservationReport" ], "topics": [ "polytraders.reports.intel" ], "cadence": "every-event", "retention_class": "30d", "retention_notes": "Full fidelity for 30 d; rolled-up summary retained for 1 y", "sampling_rule": "emit-every for materiality \u2265 emit_every_above; sample-1/10 for routine items", "bus_failure_action": "drop-after-buffer", "user_visible": "no", "consumes_kinds": [] }, "capital_impact": "Indirect", "v3_status": { "phase": 2, "phase_name": "Data normalisation", "docs": { "done": 27, "total": 27, "state": "done" }, "impl": { "done": 0, "total": 15, "state": "pending" }, "runtime": { "done": 0, "total": 8, "state": "pending" }, "overall": "pending" } }