From cfd4e64abe00fabbee84a7508787df5e95e7dad7 Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Thu, 8 Oct 2026 12:05:17 -0700 Subject: [PATCH 1/3] fix(providers): cut per-call cost of CLI-provider scans Each LLM call with claude_cli starts a separate `claude -p` session. A user scanning 15 skills (five scans in parallel) started 434 sessions in about 6 minutes and used 98% of their 5-hour plan window, at roughly 34k tokens of startup context per session. - claude argv: add `--tools ""` so built-in tool schemas are not sent (`--allowed-tools ""` only denies use), replace the default agent system prompt with a short fixed one, and pass `--no-session-persistence` so calls stop leaving one ~/.claude/projects folder each. - Default LLM fan-out to 2 for CLI providers when SKILLSPECTOR_MAX_LLM_CONCURRENCY is unset (HTTP providers keep 10). - Warn once per process when a CLI provider is available that every LLM call is a separate session on the user's plan, and point to --no-llm. - skill-inspector: keep --no-llm unless the user asks, never pick a provider on the agent's own, and don't run CLI-provider scans in parallel. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Narendran Raghavan --- skills/skill-inspector/SKILL.md | 1 + src/skillspector/llm_analyzer_base.py | 32 ++++++++++++++++--- src/skillspector/providers/_agent_cli.py | 31 ++++++++++++++++++ src/skillspector/providers/_agent_cli_base.py | 31 ++++++++++++++++-- tests/nodes/test_llm_analyzer_base.py | 16 ++++++++++ tests/unit/test_agent_cli.py | 18 +++++++++++ tests/unit/test_providers.py | 30 +++++++++++++++++ 7 files changed, 152 insertions(+), 7 deletions(-) diff --git a/skills/skill-inspector/SKILL.md b/skills/skill-inspector/SKILL.md index 25f071e28..a968a585f 100644 --- a/skills/skill-inspector/SKILL.md +++ b/skills/skill-inspector/SKILL.md @@ -22,6 +22,7 @@ Do not rely on the numeric score alone. A low score can miss semantic risk, and - Run SkillSpector first when the `skillspector` CLI is available. - If `skillspector` is missing, say so clearly and continue with manual source review. - Do not install tools, dependencies, or runtimes silently. +- Keep `--no-llm` unless the user asks for LLM analysis. Do not choose or change `SKILLSPECTOR_PROVIDER` on your own. The `claude_cli`, `codex_cli`, `gemini_cli`, and `opencode_cli` providers start a separate agent session for every LLM call, billed to the user's own plan, so ask before using them and do not run several such scans in parallel. - Do not execute scripts from the target skill. - Use read-only inspection commands such as `find`, `rg`, `sed`, `jq`, `file`, and `git diff`. - Read source around every high-signal finding instead of trusting the scanner summary alone. diff --git a/src/skillspector/llm_analyzer_base.py b/src/skillspector/llm_analyzer_base.py index 6a353a928..3a7a3cf51 100644 --- a/src/skillspector/llm_analyzer_base.py +++ b/src/skillspector/llm_analyzer_base.py @@ -71,12 +71,15 @@ from skillspector.logging_config import get_logger from skillspector.model_info import get_max_input_tokens from skillspector.models import Finding -from skillspector.providers import get_active_provider +from skillspector.providers import UnknownProviderError, get_active_provider, has_cli_capability from skillspector.providers.gemini import GeminiProvider logger = get_logger(__name__) DEFAULT_MAX_LLM_CONCURRENCY = 10 +# A CLI-provider call is a whole agent session on the user's own plan, so the +# default fan-out is much smaller than for an HTTP API. +DEFAULT_CLI_MAX_LLM_CONCURRENCY = 2 API_CONNECTION_MAX_RETRIES = 3 API_CONNECTION_RETRY_DELAYS_SECONDS = (0.5, 1.0, 2.0) RATE_LIMIT_RETRY_DELAYS_SECONDS = (5.0, 15.0, 30.0) @@ -536,10 +539,28 @@ def _shared_limiter(limit: int) -> _GlobalLLMLimiter: return limiter +def _default_max_concurrency() -> int: + """Return the fan-out default for the active provider. + + CLI providers get :data:`DEFAULT_CLI_MAX_LLM_CONCURRENCY` because every + request starts a separate agent session that counts against the user's + plan. An unknown provider keeps the HTTP default; provider selection + reports that error itself. + """ + try: + provider = get_active_provider() + except UnknownProviderError: + return DEFAULT_MAX_LLM_CONCURRENCY + if has_cli_capability(provider): + return DEFAULT_CLI_MAX_LLM_CONCURRENCY + return DEFAULT_MAX_LLM_CONCURRENCY + + def resolve_max_concurrency() -> int: """Resolve the LLM fan-out concurrency from ``SKILLSPECTOR_MAX_LLM_CONCURRENCY``. - Defaults to :data:`DEFAULT_MAX_LLM_CONCURRENCY`. Users on rate-limited + Defaults to :data:`DEFAULT_MAX_LLM_CONCURRENCY`, or to + :data:`DEFAULT_CLI_MAX_LLM_CONCURRENCY` for CLI providers. Users on rate-limited providers (free tiers with a low RPM) can set it to ``1`` to serialize requests across every analyzer instead of bursting up to 10 in parallel — a burst that otherwise guarantees 429s, and 429'd batches are dropped from the result (see the @@ -548,16 +569,17 @@ def resolve_max_concurrency() -> int: """ raw = os.environ.get("SKILLSPECTOR_MAX_LLM_CONCURRENCY", "").strip() if not raw: - return DEFAULT_MAX_LLM_CONCURRENCY + return _default_max_concurrency() try: value = int(raw) except ValueError: + default = _default_max_concurrency() logger.warning( "Invalid SKILLSPECTOR_MAX_LLM_CONCURRENCY=%r (not an int); using %d", raw, - DEFAULT_MAX_LLM_CONCURRENCY, + default, ) - return DEFAULT_MAX_LLM_CONCURRENCY + return default if value < 1: logger.warning("SKILLSPECTOR_MAX_LLM_CONCURRENCY=%d < 1; clamping to 1", value) return 1 diff --git a/src/skillspector/providers/_agent_cli.py b/src/skillspector/providers/_agent_cli.py index a2f9e02b0..3fa313c04 100644 --- a/src/skillspector/providers/_agent_cli.py +++ b/src/skillspector/providers/_agent_cli.py @@ -183,6 +183,15 @@ def _validate_model_label(model: str) -> str: # --------------------------------------------------------------------------- +# Fixed text, never derived from untrusted input. Short on purpose: it replaces +# Claude Code's default agent prompt, which is sent again on every call. +_CLAUDE_SYSTEM_PROMPT = ( + "You are a non-interactive text analysis component. You have no tools. " + "Follow the instructions in the user message and reply only with the " + "requested output." +) + + def _build_claude_argv(binary: str, model: str, max_output_tokens: int) -> list[str]: """Build the argv list for a capability-stripped ``claude -p`` call. @@ -231,6 +240,23 @@ def _build_claude_argv(binary: str, model: str, max_output_tokens: int) -> list[ ``--disable-slash-commands`` Prevents skill/plugin invocations from within the sandboxed call. + ``--tools ""`` + Removes every built-in tool definition from the model's context. + ``--allowed-tools ""`` only denies tool *use*; without ``--tools ""`` + each call still carries the full tool schemas, which a scan pays for + once per LLM call. + + ``--system-prompt <_CLAUDE_SYSTEM_PROMPT>`` + Replaces Claude Code's default agent system prompt, which describes + tools and coding workflows this call cannot use. The analyzer's own + instructions still arrive on stdin. Together with ``--tools ""`` this + is most of the fixed per-call startup cost. + + ``--no-session-persistence`` + Do not write a session transcript. Each call runs in a fresh temporary + cwd, so without this every call leaves a new project folder under + ``~/.claude/projects``. + Deliberately NOT included: - ``--dangerously-skip-permissions`` / ``--allow-dangerously-skip-permissions`` — explicitly forbidden. @@ -256,6 +282,11 @@ def _build_claude_argv(binary: str, model: str, max_output_tokens: int) -> list[ "--strict-mcp-config", "--setting-sources=", "--disable-slash-commands", + "--tools", + "", + "--system-prompt", + _CLAUDE_SYSTEM_PROMPT, + "--no-session-persistence", ] diff --git a/src/skillspector/providers/_agent_cli_base.py b/src/skillspector/providers/_agent_cli_base.py index 624e503d0..b71e1414e 100644 --- a/src/skillspector/providers/_agent_cli_base.py +++ b/src/skillspector/providers/_agent_cli_base.py @@ -28,8 +28,28 @@ import os +from skillspector.logging_config import get_logger from skillspector.providers import _agent_cli, registry +logger = get_logger(__name__) + +# CLIs already warned about in this process, so a scan warns once, not per probe. +_cost_warned: set[str] = set() + + +def _warn_session_cost_once(binary_name: str) -> None: + if binary_name in _cost_warned: + return + _cost_warned.add(binary_name) + logger.warning( + "LLM analysis via the local %r CLI starts a separate %s session for every " + "LLM call, and each one counts against your own plan. A scan can make dozens " + "of calls per skill. Use --no-llm for a static-only scan, or lower " + "SKILLSPECTOR_MAX_LLM_CONCURRENCY to slow the fan-out.", + binary_name, + binary_name, + ) + class AgentCLIProviderBase: """Base for providers that drive a local agent CLI (no API key needed).""" @@ -52,8 +72,15 @@ def resolve_credentials(self) -> tuple[str, str | None] | None: # -- Availability -------------------------------------------------------- def is_available(self) -> tuple[bool, str | None]: - """Binary on PATH AND authenticated (delegates to the registry probe).""" - return _agent_cli.is_available(self.BINARY_NAME) + """Binary on PATH AND authenticated (delegates to the registry probe). + + Warns once per process when the CLI is usable, before any LLM call + fans out, because each call is a full agent session on the user's plan. + """ + available, error = _agent_cli.is_available(self.BINARY_NAME) + if available: + _warn_session_cost_once(self.BINARY_NAME) + return available, error # -- Transport ----------------------------------------------------------- diff --git a/tests/nodes/test_llm_analyzer_base.py b/tests/nodes/test_llm_analyzer_base.py index 8a53db58f..c212a3e25 100644 --- a/tests/nodes/test_llm_analyzer_base.py +++ b/tests/nodes/test_llm_analyzer_base.py @@ -64,6 +64,7 @@ from skillspector.inspection_ledger import LedgerOutcome, LedgerReason, finalize_ledger from skillspector.llm_analyzer_base import ( API_CONNECTION_MAX_RETRIES, + DEFAULT_CLI_MAX_LLM_CONCURRENCY, DEFAULT_MAX_LLM_CONCURRENCY, OUTPUT_LANGUAGE_MAX_LENGTH, PROVIDER_RETRY_AFTER_MAX_SECONDS, @@ -131,6 +132,21 @@ def test_below_one_clamps_to_one(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("SKILLSPECTOR_MAX_LLM_CONCURRENCY", "0") assert resolve_max_concurrency() == 1 + def test_cli_provider_unset_uses_cli_default(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("SKILLSPECTOR_MAX_LLM_CONCURRENCY", raising=False) + monkeypatch.setenv("SKILLSPECTOR_PROVIDER", "claude_cli") + assert resolve_max_concurrency() == DEFAULT_CLI_MAX_LLM_CONCURRENCY + + def test_cli_provider_explicit_value_wins(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SKILLSPECTOR_PROVIDER", "claude_cli") + monkeypatch.setenv("SKILLSPECTOR_MAX_LLM_CONCURRENCY", "6") + assert resolve_max_concurrency() == 6 + + def test_unknown_provider_uses_http_default(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("SKILLSPECTOR_MAX_LLM_CONCURRENCY", raising=False) + monkeypatch.setenv("SKILLSPECTOR_PROVIDER", "not-a-provider") + assert resolve_max_concurrency() == DEFAULT_MAX_LLM_CONCURRENCY + class TestOutputLanguage: def test_unset_is_disabled(self, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/unit/test_agent_cli.py b/tests/unit/test_agent_cli.py index f371c28fc..cd7ded312 100644 --- a/tests/unit/test_agent_cli.py +++ b/tests/unit/test_agent_cli.py @@ -42,6 +42,7 @@ from skillspector.providers import _agent_cli from skillspector.providers._agent_cli import ( + _CLAUDE_SYSTEM_PROMPT, MAX_INPUT_BYTES, AgentCLIError, _build_claude_argv, @@ -210,6 +211,23 @@ def test_disable_slash_commands_present(self) -> None: argv = _build_claude_argv(CLAUDE_BINARY, MODEL, 4096) assert "--disable-slash-commands" in argv + def test_tools_flag_removes_all_builtin_tools(self) -> None: + argv = _build_claude_argv(CLAUDE_BINARY, MODEL, 4096) + # --allowed-tools "" only denies use; --tools "" also drops the schemas + # from context, which every call would otherwise pay for. + assert argv[argv.index("--tools") + 1] == "" + + def test_system_prompt_replaced_with_fixed_text(self) -> None: + argv = _build_claude_argv(CLAUDE_BINARY, MODEL, 4096) + assert argv[argv.index("--system-prompt") + 1] == _CLAUDE_SYSTEM_PROMPT + assert "--append-system-prompt" not in argv + + def test_no_session_persistence(self) -> None: + argv = _build_claude_argv(CLAUDE_BINARY, MODEL, 4096) + # Each call runs in a fresh temp cwd; persisting would leave one + # ~/.claude/projects folder per call. + assert "--no-session-persistence" in argv + def test_dangerously_skip_permissions_never_in_argv(self) -> None: argv = _build_claude_argv(CLAUDE_BINARY, MODEL, 4096) # Neither the short nor any variation may appear. diff --git a/tests/unit/test_providers.py b/tests/unit/test_providers.py index c838cb5ed..c1150e687 100644 --- a/tests/unit/test_providers.py +++ b/tests/unit/test_providers.py @@ -32,6 +32,8 @@ from pydantic import SecretStr import skillspector.providers as providers_module +import skillspector.providers._agent_cli as agent_cli_module +import skillspector.providers._agent_cli_base as agent_cli_base_module import skillspector.providers.anthropic.provider as anthropic_provider_module from skillspector.inference_usage import chat_model_controls, chat_model_requested_controls from skillspector.providers import ( @@ -1068,6 +1070,34 @@ def test_is_available_reports_not_ready(self) -> None: assert reason +class TestAgentCLISessionCostWarning: + """A usable CLI provider warns once that every LLM call is a full session.""" + + @pytest.fixture(autouse=True) + def _reset_warned(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(agent_cli_base_module, "_cost_warned", set()) + + def test_warns_once_when_available( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ) -> None: + monkeypatch.setattr(agent_cli_module, "is_available", lambda name: (True, None)) + provider = ClaudeCLIProvider() + with caplog.at_level("WARNING", logger=agent_cli_base_module.logger.name): + assert provider.is_available() == (True, None) + assert provider.is_available() == (True, None) + warnings = [r for r in caplog.records if "separate claude session" in r.getMessage()] + assert len(warnings) == 1 + assert "--no-llm" in warnings[0].getMessage() + + def test_silent_when_unavailable( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ) -> None: + monkeypatch.setattr(agent_cli_module, "is_available", lambda name: (False, "not logged in")) + with caplog.at_level("WARNING", logger=agent_cli_base_module.logger.name): + assert ClaudeCLIProvider().is_available() == (False, "not logged in") + assert not [r for r in caplog.records if "separate claude session" in r.getMessage()] + + class TestAgentCLIProviderMetadata: """Shared model-registry behavior for supported agent CLI providers.""" From fb2b70d246e4bfe6f1a3f09749e40113bc1016c4 Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Thu, 8 Oct 2026 12:05:47 -0700 Subject: [PATCH 2/3] docs(changelog): note CLI-provider cost fix (#808) Co-Authored-By: Claude Opus 5.5 Signed-off-by: Narendran Raghavan --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90b1a88d6..b6cb5f231 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ### 2.12.0 (Unreleased) ### Features/Bug Fixes +* fix(providers): cut per-call cost of CLI-provider scans and default their LLM fan-out to 2 (#808) * fix(static): stop code comparisons after removal verbs from reading as unterminated tag-marker directives (#777) * fix(baseline): fingerprint every occurrence of a deduplicated finding so the next scan suppresses all of them (#633) * fix(security): retain incomplete coverage for runtime-selected commands and remeasure active Git clones strictly (#514) From 0f51020a5d2b36822aba7e80743d9f20ecf04100 Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Wed, 14 Oct 2026 09:18:51 -0700 Subject: [PATCH 3/3] fix(providers): warn about CLI session cost before the first LLM call skillspector scan only probes is_available() from the report node, after every semantic analyzer has run, so the session-cost warning arrived after the cost. get_chat_model() now warns (once per process) when it builds a CLI-provider chat model, before the analyzers fan out. The MCP server's up-front probe still warns through is_available(). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Narendran Raghavan --- src/skillspector/llm_utils.py | 6 ++++ src/skillspector/providers/_agent_cli_base.py | 16 ++++++++-- tests/unit/test_providers.py | 30 +++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/skillspector/llm_utils.py b/src/skillspector/llm_utils.py index 6cbed29e4..4792103fa 100644 --- a/src/skillspector/llm_utils.py +++ b/src/skillspector/llm_utils.py @@ -614,6 +614,12 @@ def get_chat_model( """ provider = get_active_provider() if has_cli_capability(provider): + # Every CLI call is a full agent session on the user's plan. Warn here, + # before the analyzers fan out; the scan path only probes availability + # from the report node, after the calls have run. + warn_session_cost = getattr(provider, "warn_session_cost", None) + if callable(warn_session_cost): + warn_session_cost() resolved_model = model or provider.resolve_model() chat_model = AgentCLIChatModel( provider, diff --git a/src/skillspector/providers/_agent_cli_base.py b/src/skillspector/providers/_agent_cli_base.py index b71e1414e..f10468498 100644 --- a/src/skillspector/providers/_agent_cli_base.py +++ b/src/skillspector/providers/_agent_cli_base.py @@ -74,14 +74,24 @@ def resolve_credentials(self) -> tuple[str, str | None] | None: def is_available(self) -> tuple[bool, str | None]: """Binary on PATH AND authenticated (delegates to the registry probe). - Warns once per process when the CLI is usable, before any LLM call - fans out, because each call is a full agent session on the user's plan. + Warns once per process when the CLI is usable (see + :meth:`warn_session_cost`), so callers that probe up front, such as + the MCP server, warn before any scan starts. """ available, error = _agent_cli.is_available(self.BINARY_NAME) if available: - _warn_session_cost_once(self.BINARY_NAME) + self.warn_session_cost() return available, error + def warn_session_cost(self) -> None: + """Warn once per process that each LLM call is a full agent session. + + ``get_chat_model()`` calls this before the analyzers fan out, because + ``skillspector scan`` only probes :meth:`is_available` from the report + node, after every LLM call has run. + """ + _warn_session_cost_once(self.BINARY_NAME) + # -- Transport ----------------------------------------------------------- def complete( diff --git a/tests/unit/test_providers.py b/tests/unit/test_providers.py index ada04ea37..164b7e2cc 100644 --- a/tests/unit/test_providers.py +++ b/tests/unit/test_providers.py @@ -36,6 +36,7 @@ import skillspector.providers._agent_cli_base as agent_cli_base_module import skillspector.providers.anthropic.provider as anthropic_provider_module from skillspector.inference_usage import chat_model_controls, chat_model_requested_controls +from skillspector.llm_utils import get_chat_model from skillspector.providers import ( NO_LLM_API_KEY_MESSAGE, chat_models, @@ -1110,6 +1111,35 @@ def test_silent_when_unavailable( assert ClaudeCLIProvider().is_available() == (False, "not logged in") assert not [r for r in caplog.records if "separate claude session" in r.getMessage()] + def test_warns_before_first_cli_call_without_availability_probe( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ) -> None: + # `skillspector scan` builds its chat model before any CLI call and only + # probes is_available() from the report node, after the calls have run. + def _no_probe(name: str) -> tuple[bool, str | None]: + raise AssertionError("the scan path must warn without probing availability") + + warned_before_call: list[bool] = [] + + def _run_agent_cli(binary: str, prompt: str, **kwargs: object) -> str: + warned_before_call.append( + any("separate claude session" in r.getMessage() for r in caplog.records) + ) + return "pong" + + monkeypatch.setattr(agent_cli_module, "is_available", _no_probe) + monkeypatch.setattr(agent_cli_module, "run_agent_cli", _run_agent_cli) + token = use_provider(ClaudeCLIProvider()) + try: + with caplog.at_level("WARNING", logger=agent_cli_base_module.logger.name): + assert get_chat_model().invoke("ping").content == "pong" + assert get_chat_model().invoke("ping").content == "pong" + finally: + reset_provider(token) + assert warned_before_call == [True, True] + warnings = [r for r in caplog.records if "separate claude session" in r.getMessage()] + assert len(warnings) == 1 + class TestAgentCLIProviderMetadata: """Shared model-registry behavior for supported agent CLI providers."""