{ "schema_version": "1.0.0", "bot_id": "6.6", "bot_name": "Cron Runner", "slug": "cron-runner", "layer": "Governance", "layer_key": "gov", "bot_class": "Governance Service", "authority": [ "Explain" ], "status": "live", "readiness": "General live", "flagship": false, "is_reference": false, "public_export": false, "identity": { "layer": "Governance", "bot_class": "Governance Service", "authority": "Explain", "runs_before": "Every scheduled bot task \u2014 CronRunner fires the trigger", "runs_after": "System startup", "applies_to": "All cron-scheduled tasks across all 97 production bots", "default_mode": "general_live", "user_visible": "Advanced details only", "developer_owner": "Polytraders core \u2014 Governance pod" }, "purpose": "CronRunner maintains the authoritative cron schedule for the entire Polytraders system. It reads a declarative cron manifest (cron_expression per task, UTC), fires each task at the correct wall-clock moment, serialises concurrent firings into an ordered queue, and emits an OperationsReport heartbeat after each dispatch. CronRunner never executes strategy logic or touches the CLOB directly; it is purely a scheduling and dispatch service. Internal-only \u2014 no API surface beyond the internal bus.", "why_it_matters": [ { "failure": "Cron task not fired on schedule", "consequence": "Downstream bots that rely on scheduled triggers (e.g. daily P&L reconciliation, pre-event quoting cycles) silently miss their window. Delayed reconciliation compounds drift." }, { "failure": "Two instances of CronRunner fire the same task simultaneously", "consequence": "Duplicate fills or double-counted reconciliation records corrupt ledgers and inflate reported P&L." }, { "failure": "CronRunner crashes and does not recover", "consequence": "All scheduled tasks across all 97 bots stop firing. The system drifts into an uncontrolled state without time-based governance." }, { "failure": "Missed quiet-hours suppression", "consequence": "Strategies fire during configured quiet hours (e.g. market-open volatility windows), creating unintended exposure." } ], "polymarket_inputs": [ { "input": "Market schedule metadata (event start times)", "source": "internal", "required": false, "use": "Align pre-event quoting cycles to Polymarket event start times when calendar feeds are configured." } ], "internal_inputs": [ { "input": "Cron manifest \u2014 list of tasks with cron_expression, enabled_strategies, quiet_hour windows", "source": "Config store", "required": true, "use": "Primary schedule definition. Each entry maps a cron expression to one or more strategy or governance task IDs." }, { "input": "KillSwitch active flag", "source": "KillSwitch", "required": false, "use": "When KillSwitch is active, suppress all non-critical scheduled tasks. Health and governance tasks continue." } ], "raw_params": [ "cron_expression \u00b7 string (UTC)", "enabled_strategies \u00b7 list", "disable_during_quiet_hours \u00b7 bool" ], "parameters": [ { "name": "cron_expression", "default": "0 * * * *", "warning": null, "hard": null, "controls": "Standard POSIX cron expression (UTC) defining when a scheduled task fires.", "why_default_matters": "Hourly default prevents unintentional high-frequency dispatch. Each task must explicitly set its own expression.", "threshold_logic": [ { "condition": "valid cron expression", "action": "Schedule task" }, { "condition": "invalid expression", "action": "Reject at config load; emit CRON_RUNNER_INVALID_EXPRESSION" } ], "dev_check": "parseCron(p.cron_expression) || throw ConfigError('CRON_RUNNER_INVALID_EXPRESSION')", "user_facing": "The system runs certain maintenance and reporting tasks on a fixed schedule." }, { "name": "enabled_strategies", "default": [], "warning": null, "hard": null, "controls": "List of strategy or bot IDs that this cron entry should trigger.", "why_default_matters": "Empty list by default \u2014 tasks must be explicitly enabled to prevent accidental firing of inactive strategies.", "threshold_logic": [ { "condition": "non-empty list", "action": "Dispatch trigger to each listed bot ID" }, { "condition": "empty list", "action": "Log warning CRON_RUNNER_NO_TARGETS and skip" } ], "dev_check": "if (!p.enabled_strategies.length) log.warn('CRON_RUNNER_NO_TARGETS')", "user_facing": "Scheduled tasks run only for active components." }, { "name": "disable_during_quiet_hours", "default": false, "warning": null, "hard": null, "controls": "When true, suppresses this task during configured system quiet hours.", "why_default_matters": "Off by default so critical governance tasks (health checks, reconciliation) always fire. Enable only for trading-adjacent tasks.", "threshold_logic": [ { "condition": "disable_during_quiet_hours=true AND now in quiet window", "action": "Skip task; emit CRON_RUNNER_QUIET_HOURS_SKIP" }, { "condition": "disable_during_quiet_hours=false", "action": "Fire regardless of quiet hours" } ], "dev_check": "if (p.disable_during_quiet_hours && isQuietHour(now)) { log.info('CRON_RUNNER_QUIET_HOURS_SKIP'); return; }", "user_facing": "Some automated tasks are paused during quiet periods." }, { "name": "max_concurrent_tasks", "default": 20, "warning": 40, "hard": 60, "controls": "Maximum number of cron tasks that may be dispatched concurrently in a single tick.", "why_default_matters": "At 20 concurrent tasks, the internal bus stays within throughput limits. Above 40, dispatch latency degrades.", "threshold_logic": [ { "condition": "max_concurrent_tasks <= 20", "action": "Normal dispatch" }, { "condition": "20\u201360", "action": "WARN \u2014 bus throughput may degrade" }, { "condition": "> 60", "action": "Reject config change \u2014 PARAMETER_CHANGE_REQUIRES_APPROVAL" } ], "dev_check": "if (p.max_concurrent_tasks > p.hard) throw ConfigError(\"PARAMETER_CHANGE_REQUIRES_APPROVAL\")", "user_facing": "" } ], "default_config": { "bot_id": "gov.cron_runner", "version": "2.0.0", "mode": "general_live", "defaults": { "cron_expression": "0 * * * *", "enabled_strategies": [], "disable_during_quiet_hours": false, "max_concurrent_tasks": 20 } }, "implementation_flow": [ "On startup, load the cron manifest from the config store; parse each entry's cron_expression (UTC) and validate.", "Register a ticker per unique cron expression; at each tick, collect all tasks whose expression matches the current UTC time.", "Before dispatching any task, check KillSwitch active flag; if active, suppress trading-adjacent tasks but allow health and governance tasks.", "For each eligible task, check disable_during_quiet_hours against the current UTC time window.", "Dispatch the task trigger to each enabled_strategies entry via the internal message bus; log dispatch with a trace_id.", "After dispatch, emit an OperationsReport heartbeat: { task_id, fired_at_ms, targets, suppressed_count, trace_id }.", "Detect and deduplicate concurrent firings using a distributed lock keyed on (task_id, scheduled_at_ms); if lock is held, skip and emit CRON_RUNNER_DUPLICATE_FIRE.", "Retain dispatch logs for 1 year for operational audit." ], "decision_logic": { "approve": "Not applicable \u2014 CronRunner dispatches triggers, it does not approve or reject trading decisions.", "reshape_required": "Not applicable.", "reject": "CronRunner will reject a task dispatch if KillSwitch is active for that task class, or if a duplicate lock is held.", "warning_only": "Invalid cron expressions and empty target lists emit warnings but do not crash the scheduler." }, "decision_output_schema": "OperationsReport", "decision_output_example": { "report_id": "ops_cronrunner_20260509T120000Z", "bot_id": "gov.cron_runner", "event_type": "CRON_TASK_DISPATCHED", "task_id": "daily_pnl_reconcile", "fired_at_ms": 1746792000000, "targets": [ "gov.pnl_reporter" ], "suppressed_count": 0, "trace_id": "tr_abc123def456", "report_kind": "OperationsReport" }, "developer_log": { "bot_id": "gov.cron_runner", "event_type": "CRON_TASK_DISPATCHED", "task_id": "hourly_health_sweep", "fired_at_ms": 1746788400000, "targets": [ "gov.health_heartbeat" ], "lock_acquired": true, "trace_id": "tr_00deadbeef01" }, "user_explanations": [ { "situation": "Scheduled maintenance task fired", "message": "The system ran a scheduled internal task on time. No action needed." }, { "situation": "Task skipped during quiet hours", "message": "A scheduled task was paused during a quiet period to avoid interference with market activity." }, { "situation": "Task suppressed by kill switch", "message": "Scheduled trading-related tasks are paused. Governance and health checks continue." } ], "failure_modes": { "main_failure_mode": "CronRunner process crashes or is partitioned from the internal bus, causing all scheduled tasks to miss their windows silently.", "false_positive_risk": "A task fires twice if two CronRunner instances start simultaneously and both acquire the dispatch lock before expiry.", "false_negative_risk": "A task is silently skipped if the system clock drifts more than the tick interval (1s), causing a cron boundary to be missed.", "safe_fallback": "On restart, CronRunner reloads the manifest and resumes from the current tick. Missed ticks in the recovery window are not retroactively fired \u2014 governance bots that depend on scheduled triggers include their own staleness checks.", "required_dependencies": [ "Config store (cron manifest)", "Internal message bus", "Distributed lock (Redis or equivalent)", "KillSwitch status feed" ] }, "acceptance_tests": { "unit": [ { "test": "Valid cron expression parses without error", "setup": "cron_expression='0 9 * * *'", "expected": "Task scheduled for 09:00 UTC daily" }, { "test": "Invalid cron expression raises ConfigError", "setup": "cron_expression='99 * * * *'", "expected": "ConfigError CRON_RUNNER_INVALID_EXPRESSION" }, { "test": "Task suppressed during quiet hours", "setup": "disable_during_quiet_hours=true, now=quiet window", "expected": "CRON_RUNNER_QUIET_HOURS_SKIP logged; no dispatch" }, { "test": "Duplicate lock prevents double-fire", "setup": "Two instances attempt to dispatch same task_id at same scheduled_at_ms", "expected": "Second instance emits CRON_RUNNER_DUPLICATE_FIRE and skips" }, { "test": "KillSwitch suppresses trading tasks", "setup": "KillSwitch active; task target=strat.some_strategy", "expected": "Task suppressed; governance and health tasks fire normally" } ], "integration": [ { "test": "Task dispatched end-to-end on cron boundary", "expected": "OperationsReport emitted; target bot receives trigger on internal bus" }, { "test": "Manifest reload after config change", "expected": "New cron expressions take effect on next tick without restart" } ], "property": [ { "property": "No task fires more than once per scheduled interval even under concurrent instances", "required": "Always true \u2014 distributed lock enforces single dispatch" }, { "property": "Every dispatch produces an OperationsReport heartbeat", "required": "Always true" } ] }, "checklist_overrides": {}, "legacy_goal": "Time-of-day strategy enable / disable; scheduled rebalancing; pre-event quoting cycles.", "legacy_pm_signals": [ "Polymarket market open / scheduled-event metadata" ], "legacy_external_feeds": [ "System clock (UTC)", "Calendar feeds for sport / political schedules" ], "reporting_groups": [ "governance_audit" ], "network": [ "polygon" ], "api_surface": [ "internal" ], "reference_implementation": { "summary": "Loads a cron manifest, registers tickers, dispatches task triggers at UTC boundaries, deduplicates via distributed lock, and emits an OperationsReport per dispatch.", "language_note": "Pseudocode is language-agnostic. FETCH = read input. EMIT = produce output. Translate to TS/Python/Go/Rust.", "pseudocode": "// ---- STARTUP ----\nFUNCTION init():\n manifest = FETCH config_store.GET('/cron-manifest')\n FOR entry IN manifest:\n validateCronExpression(entry.cron_expression)\n registerTicker(entry) // fires onTick() when expression matches UTC now\n\n// ---- PER-TICK HANDLER ----\nFUNCTION onTick(entry, scheduled_at_ms):\n lock_key = entry.task_id + ':' + scheduled_at_ms\n IF NOT redis.SET(lock_key, NX=true, PX=5000): // 5s TTL\n log.warn('CRON_RUNNER_DUPLICATE_FIRE', { task_id: entry.task_id })\n RETURN\n\n ks = FETCH killswitch.status()\n IF ks.active AND entry.task_class == 'trading':\n log.info('CRON_RUNNER_KILL_SWITCH_ACTIVE', { task_id: entry.task_id })\n RETURN\n\n IF entry.disable_during_quiet_hours AND isQuietHour(now()):\n log.info('CRON_RUNNER_QUIET_HOURS_SKIP', { task_id: entry.task_id })\n EMIT OperationsReport(event_type='CRON_TASK_SUPPRESSED', ...)\n RETURN\n\n IF NOT entry.enabled_strategies:\n log.warn('CRON_RUNNER_NO_TARGETS', { task_id: entry.task_id })\n RETURN\n\n trace_id = generateTraceId()\n FOR target IN entry.enabled_strategies:\n internal_bus.publish(topic='cron.trigger', payload={\n task_id: entry.task_id,\n target: target,\n scheduled_at: scheduled_at_ms,\n trace_id: trace_id\n })\n\n EMIT OperationsReport({\n report_id: 'ops_cronrunner_' + scheduled_at_ms,\n bot_id: 'gov.cron_runner',\n event_type: 'CRON_TASK_DISPATCHED',\n task_id: entry.task_id,\n fired_at_ms: scheduled_at_ms,\n targets: entry.enabled_strategies,\n suppressed_count: 0,\n trace_id: trace_id\n })\n\n// ---- MANIFEST RELOAD ----\nFUNCTION reloadManifest():\n // Called on config change event; validates and re-registers all tickers\n FOR entry IN FETCH config_store.GET('/cron-manifest'):\n validateCronExpression(entry.cron_expression)\n registerTickers(manifest) // replaces existing registrations atomically\n", "sdk_calls": [ "config_store.GET('/cron-manifest')", "redis.SET(lock_key, NX=true, PX=5000)", "killswitch.status()", "internal_bus.publish(topic='cron.trigger', payload=...)", "alerting.emit('CRON_RUNNER_DUPLICATE_FIRE', metadata)" ], "complexity": "O(T) per tick where T = tasks matching the current cron boundary" }, "wire_examples": { "input": { "label": "Cron manifest entry", "source": "config_store", "payload": { "task_id": "daily_pnl_reconcile", "cron_expression": "0 0 * * *", "enabled_strategies": [ "gov.pnl_reporter" ], "disable_during_quiet_hours": false, "task_class": "governance" } }, "output": { "label": "OperationsReport \u2014 CRON_TASK_DISPATCHED", "payload": { "report_id": "ops_cronrunner_1746792000000", "bot_id": "gov.cron_runner", "event_type": "CRON_TASK_DISPATCHED", "task_id": "daily_pnl_reconcile", "fired_at_ms": 1746792000000, "targets": [ "gov.pnl_reporter" ], "suppressed_count": 0, "trace_id": "tr_abc123def456", "report_kind": "OperationsReport" } } }, "reason_codes": [ { "code": "CRON_RUNNER_TASK_DISPATCHED", "severity": "INFO", "meaning": "A cron task fired on schedule and was dispatched to all configured targets.", "action": "No action \u2014 operational heartbeat.", "user_message": "" }, { "code": "CRON_RUNNER_INVALID_EXPRESSION", "severity": "HARD_REJECT", "meaning": "A cron expression in the manifest failed to parse.", "action": "Reject the config entry at load time; alert on-call.", "user_message": "" }, { "code": "CRON_RUNNER_DUPLICATE_FIRE", "severity": "WARN", "meaning": "A second CronRunner instance attempted to dispatch the same task at the same scheduled time; distributed lock was already held.", "action": "Skip dispatch; log warning.", "user_message": "" }, { "code": "CRON_RUNNER_QUIET_HOURS_SKIP", "severity": "INFO", "meaning": "A task with disable_during_quiet_hours=true was suppressed because the current UTC time falls within a configured quiet window.", "action": "No action \u2014 expected behaviour.", "user_message": "A scheduled task was paused during a quiet period." }, { "code": "CRON_RUNNER_NO_TARGETS", "severity": "WARN", "meaning": "A task entry has an empty enabled_strategies list; there is nothing to dispatch.", "action": "Log warning; skip dispatch.", "user_message": "" }, { "code": "KILL_SWITCH_ACTIVE", "severity": "WARN", "meaning": "KillSwitch is active; trading-class scheduled tasks are suppressed.", "action": "Suppress trading tasks; continue governance and health tasks.", "user_message": "Scheduled trading tasks are paused while the system kill switch is active." }, { "code": "CRON_RUNNER_BUS_PUBLISH_FAILED", "severity": "WARN", "meaning": "Internal bus publish for a task trigger failed after retries.", "action": "Log error; emit alert; retry on next tick.", "user_message": "" }, { "code": "CRON_RUNNER_MANIFEST_RELOAD_OK", "severity": "INFO", "meaning": "Cron manifest reloaded successfully after a config change event.", "action": "No action.", "user_message": "" } ], "metrics": { "emitted": [ { "name": "polytraders_gov_cronrunner_tasks_dispatched_total", "type": "counter", "unit": "count", "labels": [ "task_id", "task_class" ], "meaning": "Total cron tasks dispatched successfully." }, { "name": "polytraders_gov_cronrunner_tasks_suppressed_total", "type": "counter", "unit": "count", "labels": [ "reason", "task_id" ], "meaning": "Tasks suppressed due to quiet hours, kill switch, or no targets." }, { "name": "polytraders_gov_cronrunner_duplicate_fire_total", "type": "counter", "unit": "count", "labels": [ "task_id" ], "meaning": "Duplicate dispatch attempts blocked by distributed lock." }, { "name": "polytraders_gov_cronrunner_dispatch_latency_ms", "type": "histogram", "unit": "ms", "labels": [ "task_id" ], "meaning": "Wall-clock latency from cron boundary to internal bus publish." }, { "name": "polytraders_gov_cronrunner_manifest_entries", "type": "gauge", "unit": "count", "labels": [], "meaning": "Number of active entries in the loaded cron manifest." }, { "name": "polytraders_gov_cronrunner_bus_failures_total", "type": "counter", "unit": "count", "labels": [ "task_id" ], "meaning": "Internal bus publish failures for cron triggers." } ], "alerts": [ { "name": "CronRunnerNoDispatchIn15m", "condition": "rate(polytraders_gov_cronrunner_tasks_dispatched_total[15m]) == 0", "severity": "page", "runbook": "#runbook-cronrunner-no-dispatch" }, { "name": "CronRunnerDuplicateFireRate", "condition": "rate(polytraders_gov_cronrunner_duplicate_fire_total[5m]) > 0", "severity": "warn", "runbook": "#runbook-cronrunner-duplicate" }, { "name": "CronRunnerBusFailures", "condition": "rate(polytraders_gov_cronrunner_bus_failures_total[5m]) > 0", "severity": "page", "runbook": "#runbook-cronrunner-bus-failure" }, { "name": "CronRunnerDispatchLatencyHigh", "condition": "histogram_quantile(0.99, polytraders_gov_cronrunner_dispatch_latency_ms) > 500", "severity": "warn", "runbook": "#runbook-cronrunner-latency" } ], "dashboards": [ "Grafana \u2014 Governance / CronRunner task dispatch timeline", "Grafana \u2014 Governance / Scheduled task suppression rate" ], "log_level": "info" }, "state": { "store": "redis", "shape": "Distributed lock keys: {task_id}:{scheduled_at_ms} \u2192 TTL 5s. Cron manifest cached in-memory after load.", "ttl": "Lock TTL 5s; manifest cache refreshed on config change event", "recovery": "On restart, cron manifest is re-fetched from config store and tickers are re-registered. Locks expire naturally. No missed ticks are retroactively fired.", "size_estimate": "< 1 KB per active lock; manifest < 50 KB" }, "concurrency": { "execution_model": "single-threaded event loop", "max_in_flight": 100, "idempotency_key": "task_id + scheduled_at_ms (distributed lock)", "timeout_ms": 2000, "backpressure": "skip dispatch and emit CRON_RUNNER_BUS_PUBLISH_FAILED if bus is saturated", "locking": "Redis SET NX PX per (task_id, scheduled_at_ms)" }, "dependencies": { "depends_on": [ { "bot_id": "internal.config_store", "why": "Cron manifest is loaded from the config store on startup and on change events." }, { "bot_id": "internal.killswitch", "why": "KillSwitch status is checked before dispatching trading-class tasks." } ], "emits_to": [ { "bot_id": "gov.pnl_reporter", "what": "daily_pnl_reconcile trigger via internal bus" }, { "bot_id": "gov.health_heartbeat", "what": "hourly_health_sweep trigger via internal bus" }, { "bot_id": "gov.portfolio_sync", "what": "scheduled sync trigger via internal bus when interval-based sync is configured" } ], "sibling": [], "external": [ { "service": "Redis (distributed lock)", "sla": "99.99% (internal SRE target)", "fallback": "Fall back to in-process lock with warn; only safe for single-instance deployments." } ] }, "security_surfaces": { "signs_orders": false, "private_key_access": "none", "abuse_vectors": [ "Injecting a malformed cron expression to crash the scheduler", "Racing the distributed lock to trigger double-dispatch of financial tasks", "Modifying the cron manifest to fire tasks outside of approved windows" ], "mitigations": [ "All cron expressions are validated at manifest load; invalid entries are rejected with HARD_REJECT", "Distributed lock (Redis SET NX) enforces single-dispatch per (task_id, scheduled_at_ms)", "Config store changes are audit-logged; manifest reload events are emitted as OperationsReport entries", "CronRunner has no CLOB or on-chain access \u2014 it only publishes to the internal bus" ], "contract_calls": [] }, "polymarket_v2_compat": { "clob_version": "v2", "collateral": "pUSD", "eip712_domain_version": "2", "builder_code_aware": false, "negrisk_aware": false, "multichain_ready": false, "sdk_used": "internal-only", "settlement_contract": "none", "notes": "CronRunner is internal-only and has no direct CLOB or on-chain interface. It dispatches triggers to other bots that operate on V2 surfaces." }, "version": { "spec": "2.0.0", "implementation": "2.1.0", "schema": "2", "released": "2026-04-28" }, "migration_history": [ { "date": "2026-04-28", "from": "v1", "to": "v2", "reason": "CLOB V2 cutover", "action_taken": "No direct CLOB changes required \u2014 CronRunner has no CLOB surface. Updated OperationsReport schema to pUSD denomination in downstream task payloads. Removed legacy USDC.e references from task manifest documentation." } ], "failure_injection": [ { "scenario": "REDIS_LOCK_UNAVAILABLE", "how_to_inject": "Take Redis offline during a cron boundary", "expected_behavior": "CronRunner falls back to in-process lock with WARN; CRON_RUNNER_DUPLICATE_FIRE suppression is weakened for the outage window", "recovery": "Once Redis reconnects, distributed locking resumes. Any duplicate dispatches during outage are handled by downstream idempotency." }, { "scenario": "INVALID_MANIFEST_ENTRY", "how_to_inject": "Push a cron manifest entry with cron_expression='99 * * * *'", "expected_behavior": "CRON_RUNNER_INVALID_EXPRESSION raised; bad entry rejected; valid entries continue scheduling", "recovery": "Fix the manifest entry; reload triggers CRON_RUNNER_MANIFEST_RELOAD_OK." }, { "scenario": "BUS_PUBLISH_FAILURE", "how_to_inject": "Drop all packets to the internal bus for 60s", "expected_behavior": "Dispatch attempts fail; CRON_RUNNER_BUS_PUBLISH_FAILED alert fired; tasks missed for that window", "recovery": "Bus reconnects; next tick dispatches normally. Downstream bots handle missed ticks via staleness checks." }, { "scenario": "KILL_SWITCH_ACTIVE_DURING_TRADING_TASK", "how_to_inject": "Activate KillSwitch; observe behaviour at next hourly tick for trading-class tasks", "expected_behavior": "Trading tasks suppressed; governance and health tasks fire normally; KILL_SWITCH_ACTIVE logged", "recovery": "Deactivate KillSwitch; trading tasks resume at next tick." }, { "scenario": "DUPLICATE_CRONRUNNER_INSTANCES", "how_to_inject": "Start two CronRunner processes simultaneously", "expected_behavior": "One instance acquires lock and dispatches; second emits CRON_RUNNER_DUPLICATE_FIRE and skips", "recovery": "Shut down duplicate instance; primary continues normally." } ], "runbook": { "summary": "CronRunner incidents fall into three categories: no dispatch (process down or bus failure), duplicate fires (lock race or multiple instances), or manifest errors (invalid expressions). All three block downstream governance tasks.", "oncall_actions": [ { "alert": "CronRunnerNoDispatchIn15m", "first_step": "Check CronRunner process status and internal bus connectivity.", "escalation": "Governance pod lead" }, { "alert": "CronRunnerDuplicateFireRate", "first_step": "Check for multiple CronRunner instances; verify Redis lock is reachable.", "escalation": "SRE on-call" }, { "alert": "CronRunnerBusFailures", "first_step": "Check internal message bus health; review bus failure logs for task_id.", "escalation": "SRE on-call" }, { "alert": "CronRunnerDispatchLatencyHigh", "first_step": "Check Redis latency and internal bus throughput; reduce manifest size if overloaded.", "escalation": "Governance pod lead after 30 minutes" } ], "manual_overrides": [ { "name": "force_dispatch", "how": "polytraders gov cron force-dispatch --task-id ", "when": "Manually trigger a missed cron task after investigating root cause.", "command": "polytraders gov cron force-dispatch --task-id ", "effect": "Manually trigger a missed cron task after investigating root cause." } ], "healthcheck": "Endpoint: /internal/health/cron-runner | Green: Last dispatch < 65 minutes ago; manifest loaded; Redis lock reachable. | Red: No dispatch in 15 minutes; Redis unreachable; manifest parse errors." }, "promotion_gates": { "to_shadow": [ { "gate": "Unit tests pass for cron expression parsing and lock deduplication", "how_measured": "CI test run", "threshold": "100% pass" }, { "gate": "Redis lock integration test verified", "how_measured": "Integration test", "threshold": "Pass" } ], "to_limited_live": [ { "gate": "Dispatch latency p99 < 100ms under 20 simultaneous task firings", "how_measured": "polytraders_gov_cronrunner_dispatch_latency_ms histogram", "threshold": "< 100ms" }, { "gate": "Zero duplicate fires observed in 24h shadow run", "how_measured": "polytraders_gov_cronrunner_duplicate_fire_total", "threshold": "0" } ], "to_general_live": [ { "gate": "End-to-end: all 97 bots receive their scheduled triggers on time for 7 consecutive days", "how_measured": "Governance audit log + OperationsReport timeline", "threshold": "100% on-time dispatch" }, { "gate": "KillSwitch suppression verified", "how_measured": "Failure injection test", "threshold": "Pass" } ] }, "reporting": { "emits_kinds": [ "OperationsReport" ], "topics": [ "polytraders.reports.ops" ], "cadence": "every-event", "retention_class": "1y", "sampling_rule": "batched-1/min", "bus_failure_action": "drop-after-buffer", "user_visible": "no", "consumes_kinds": [] }, "capital_impact": "Indirect", "v3_status": { "phase": 7, "phase_name": "Governance & replay", "docs": { "done": 27, "total": 27, "state": "done" }, "impl": { "done": 0, "total": 15, "state": "pending" }, "runtime": { "done": 0, "total": 8, "state": "pending" }, "overall": "pending" } }