feat(performance): executor performance as a first-class read surface, plus an 18-item backlog burndown

.gitignore

@@ -190,7 +190,10 @@ conf/
 # IDE files
 .vscode/
 .idea/
-improvements
+# Local backlog of the /improvements and /design-feature workflows: working
+# notes for the burndown runs, not something the repo should carry.
+improvements/
+features/
 bots/gateway-files/
 .DS_Store
 Dockerfile.patched-hummingbot

README.md

@@ -170,9 +170,15 @@ The `.env` file contains all configuration. Key settings:
 USERNAME=admin              # API username
 PASSWORD=admin              # API password
 CONFIG_PASSWORD=admin       # Encrypts bot credentials
+DEBUG_MODE=false            # Verbose logging and reload
 DATABASE_URL=...            # PostgreSQL connection
 GATEWAY_URL=...             # Gateway URL (for DEX)
 
+# Performance snapshots and backtests
+PERFORMANCE_EXECUTOR_SNAPSHOT_INTERVAL=60   # Seconds between live executor snapshots
+PERFORMANCE_RETENTION_DAYS=0                # Delete snapshots older than N days; 0 keeps them forever
+BACKTESTING_MAX_CONCURRENT=1                # Backtests allowed to run at once (one core each)
+
 # Tailscale (recommended for production)
 TAILSCALE_ENABLED=true
 TAILSCALE_AUTH_KEY=tskey-auth-...
@@ -186,6 +192,11 @@ TAILSCALE_HOSTNAME=hummingbot-api   # MagicDNS hostname on your tailnet
 # DB_BIND=127.0.0.1
 ```
 
+These are the settings most deployments touch, not the full list: `config.py` is the authoritative
+list of every setting, its default and what it does. The `.env` that `setup.sh` generates also carries
+the optional `PERFORMANCE_`, `BACKTESTING_`, `MARKET_DATA_`, `CORS_` and `AWS_` groups as commented-out
+lines showing their defaults, so you can see and override them without leaving the file.
+
 Edit `.env` and restart with `make deploy` to apply changes.
 
 ## Secure Connection via Tailscale

bots/controllers/generic/lp_rebalancer/lp_rebalancer.py

@@ -93,6 +93,19 @@ class LPRebalancerConfig(ControllerConfigBase):
         description="Extra % to swap beyond deficit to account for slippage (e.g., 0.01 = 0.01%)"
     )
 
+    @field_validator("lp_provider")
+    @classmethod
+    def validate_lp_provider(cls, v: str) -> str:
+        """The trading type is never guessed: Gateway rejects a guessed one with a 400, so an
+        untyped provider has to fail at config load rather than mid-operation. The core's
+        parse_provider defaults an untyped provider to "router", which is the wrong branch
+        entirely for an LP controller, so the contract is enforced here instead."""
+        if "/" not in v:
+            raise ValueError(
+                f"Invalid lp_provider '{v}': expected 'name/type' (e.g. 'meteora/clmm')"
+            )
+        return v
+
     @field_validator("sell_price_min", "sell_price_max", "buy_price_min", "buy_price_max", mode="before")
     @classmethod
     def validate_price_limits(cls, v):
@@ -173,7 +186,9 @@ def __init__(self, config: LPRebalancerConfig, *args, **kwargs):
         super().__init__(config, *args, **kwargs)
         self.config: LPRebalancerConfig = config
 
-        # Parse lp_provider into dex_name and trading_type for gateway calls
+        # Parse lp_provider into dex_name and trading_type for gateway calls. The config
+        # validator guarantees the "name/type" form, so parse_provider's own default for an
+        # untyped provider is never reached.
         self.lp_dex_name, self.lp_trading_type = parse_provider(config.lp_provider)
 
         # Parse token symbols from trading pair

config.py

@@ -55,6 +55,22 @@ class MarketDataSettings(BaseSettings):
         default=60.0,
         description="Maximum allowed WebSocket subscription update interval in seconds"
     )
+    ws_executor_min_update_interval: float = Field(
+        default=0.5,
+        description="Minimum allowed /ws/executors subscription update interval in seconds. "
+                    "The floor is stricter than the market-data one because executor push loops "
+                    "hit the database (executors, performance reports, positions with per-position "
+                    "rate lookups) instead of reading in-memory candles and order books"
+    )
+    ws_executor_max_update_interval: float = Field(
+        default=60.0,
+        description="Maximum allowed /ws/executors subscription update interval in seconds"
+    )
+    ws_executor_default_update_interval: float = Field(
+        default=2.0,
+        description="Update interval applied to a /ws/executors subscription that does not "
+                    "request one, in seconds"
+    )
     ticker_update_interval: int = Field(
         default=30,
         description="How often to refresh tickers from connected exchanges in seconds"
@@ -211,13 +227,28 @@ class AppSettings(BaseSettings):
 
 
 class BacktestingSettings(BaseSettings):
-    """Backtest result retention.
+    """Backtest execution limits and result retention.
 
     A finished backtest is ~98% bulk arrays (processed_data, pnl_timeseries) and only a
     few KB of metrics, so full payloads are archived to disk and only metrics stay
     resident. Retention is therefore a count of results, not a memory budget.
+
+    A run executes in its own worker process and saturates a core for its whole duration,
+    so the two execution limits are about the box, not about memory: how many cores runs
+    may claim at once, and how long one is allowed to claim one before being abandoned.
     """
 
+    max_concurrent: int = Field(
+        default=1,
+        description=(
+            "How many backtests may run at once; further submissions queue. Runs are isolated "
+            "in separate processes, so this can be raised up to the cores you are willing to give them"
+        )
+    )
+    timeout_seconds: float = Field(
+        default=1800.0,
+        description="Wall-clock budget for one backtest; the worker is killed and the task fails past it"
+    )
     max_results: int = Field(
         default=100,
         description="How many finished backtests to retain before the oldest are reaped"
@@ -226,10 +257,48 @@ class BacktestingSettings(BaseSettings):
         default="bots/data/backtests",
         description="Directory holding archived backtest payloads (inside the bots volume, so it survives redeploys)"
     )
+    candles_cache_path: str = Field(
+        default="bots/data/backtests/candles",
+        description="Directory holding downloaded candle history shared by backtest workers"
+    )
+    candles_cache_entries: int = Field(
+        default=32,
+        description=(
+            "How many downloaded candle ranges to keep; the least recently used are dropped past it. "
+            "0 disables the cache and makes every run download its own history again"
+        )
+    )
+    candles_cache_ttl_seconds: float = Field(
+        default=3600.0,
+        description=(
+            "How long a downloaded candle range may be reused. A window ending near now is fetched "
+            "with its last candle still forming, so an entry is refetched once it is older than this"
+        )
+    )
 
     model_config = SettingsConfigDict(env_prefix="BACKTESTING_", extra="ignore")
 
 
+class PerformanceSettings(BaseSettings):
+    """Performance snapshot cadence and retention."""
+
+    executor_snapshot_interval: int = Field(
+        default=60,
+        description="How often a live executor's performance is snapshotted, in seconds. "
+                    "Finer than the controller dump because executors are short-lived: at "
+                    "a 5-minute grain a three-minute position executor gets one point."
+    )
+    retention_days: int = Field(
+        default=0,
+        description="Delete performance snapshots (executor AND controller) older than "
+                    "this many days. 0 keeps everything forever, which is what every "
+                    "existing deployment does today -- an upgrade must not start deleting "
+                    "an operator's history."
+    )
+
+    model_config = SettingsConfigDict(env_prefix="PERFORMANCE_", extra="ignore")
+
+
 class Settings(BaseSettings):
     """Combined application settings."""
 
@@ -242,6 +311,7 @@ class Settings(BaseSettings):
     cors: CORSSettings = Field(default_factory=CORSSettings)
     app: AppSettings = Field(default_factory=AppSettings)
     backtesting: BacktestingSettings = Field(default_factory=BacktestingSettings)
+    performance: PerformanceSettings = Field(default_factory=PerformanceSettings)
 
     # Direct banned_tokens field to handle env parsing
     banned_tokens: List[str] = Field(

database/__init__.py

@@ -4,6 +4,7 @@
     Base,
     BotRun,
     ControllerPerformanceSnapshot,
+    ExecutorPerformanceSnapshot,
     FundingPayment,
     GatewayCLMMEvent,
     GatewayCLMMPosition,
@@ -17,6 +18,7 @@
     AccountRepository,
     BotRunRepository,
     ControllerPerformanceRepository,
+    ExecutorPerformanceRepository,
     ExecutorRepository,
     FundingRepository,
     GatewayCLMMRepository,
@@ -28,10 +30,10 @@
 __all__ = [
     "AccountState", "TokenState", "Order", "Trade", "PositionSnapshot", "FundingPayment", "BotRun",
     "GatewaySwap", "GatewayCLMMPosition", "GatewayCLMMEvent",
-    "ControllerPerformanceSnapshot",
+    "ControllerPerformanceSnapshot", "ExecutorPerformanceSnapshot",
     "Base", "AsyncDatabaseManager",
     "AccountRepository", "BotRunRepository", "ControllerPerformanceRepository",
-    "ExecutorRepository",
+    "ExecutorPerformanceRepository", "ExecutorRepository",
     "OrderRepository", "TradeRepository", "FundingRepository",
     "GatewaySwapRepository", "GatewayCLMMRepository"
 ]

database/models.py

@@ -1,4 +1,4 @@
-from sqlalchemy import TIMESTAMP, Column, ForeignKey, Integer, Numeric, String, Text, UniqueConstraint, func
+from sqlalchemy import TIMESTAMP, Boolean, Column, ForeignKey, Index, Integer, Numeric, String, Text, UniqueConstraint, func
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import relationship
 
@@ -453,6 +453,52 @@ class ControllerPerformanceSnapshot(Base):
     custom_info = Column(Text, nullable=True)  # JSON dict of custom info
 
 
+class ExecutorPerformanceSnapshot(Base):
+    """Periodic snapshot of a live executor's performance, plus one terminal row.
+
+    Written by ExecutorService: on the snapshot tick for everything in
+    _active_executors, and once more at completion with is_terminal=True. The terminal
+    row is what makes a closed executor's series answerable from this table alone --
+    no join to ExecutorRecord, and no exposure to the two paths that leave that record's
+    metrics at their creation-time zeros (the startup reap, and the except branch in
+    _persist_executor_completed).
+
+    Deliberately narrow and typed, unlike controller_performance_snapshots: ExecutorInfo
+    is named field by field all over this repo, while the controller's PerformanceReport
+    is versioned by the core and has to be blobbed. Typing it also keeps the heavy
+    custom_info payloads (fill_events, levels_by_state, ...) out of a per-minute row.
+    """
+    __tablename__ = "executor_performance_snapshots"
+    __table_args__ = (
+        # The hot query is WHERE executor_id = ? ORDER BY timestamp DESC -- one executor's
+        # series, which is what both the history route and the reap lookup ask for.
+        Index("ix_exec_perf_executor_timestamp", "executor_id", "timestamp"),
+    )
+
+    id = Column(Integer, primary_key=True, index=True)
+    timestamp = Column(TIMESTAMP(timezone=True), server_default=func.now(), nullable=False, index=True)
+
+    # Identity, denormalized from ExecutorRecord so a series needs no join.
+    executor_id = Column(String, nullable=False, index=True)
+    executor_type = Column(String, nullable=False, index=True)
+    account_name = Column(String, nullable=False, index=True)
+    connector_name = Column(String, nullable=False)
+    trading_pair = Column(String, nullable=False)
+    controller_id = Column(String, nullable=False, default="main", index=True)
+
+    status = Column(String, nullable=False)  # RunnableStatus name
+    close_type = Column(String, nullable=True)  # only ever set on the terminal row
+    is_terminal = Column(Boolean, nullable=False, default=False, index=True)
+
+    # The four ExecutorInfo metrics, same precision as ExecutorRecord. There is NO
+    # separate volume column: filled_amount_quote IS the volume traded, on every executor
+    # type including LP -- see test_executor_volume_is_the_filled_amount.py.
+    net_pnl_quote = Column(Numeric(precision=30, scale=18), nullable=False, default=0)
+    net_pnl_pct = Column(Numeric(precision=10, scale=6), nullable=False, default=0)
+    cum_fees_quote = Column(Numeric(precision=30, scale=18), nullable=False, default=0)
+    filled_amount_quote = Column(Numeric(precision=30, scale=18), nullable=False, default=0)
+
+
 class ExecutorRecord(Base):
     """Database model for executor state persistence."""
     __tablename__ = "executors"

database/repositories/__init__.py

@@ -1,6 +1,7 @@
 from .account_repository import AccountRepository
 from .bot_run_repository import BotRunRepository
 from .controller_performance_repository import ControllerPerformanceRepository
+from .executor_performance_repository import ExecutorPerformanceRepository
 from .executor_repository import ExecutorRepository
 from .funding_repository import FundingRepository
 from .gateway_amm_repository import GatewayAMMRepository
@@ -13,6 +14,7 @@
     "AccountRepository",
     "BotRunRepository",
     "ControllerPerformanceRepository",
+    "ExecutorPerformanceRepository",
     "ExecutorRepository",
     "FundingRepository",
     "OrderRepository",