diff --git a/.changeset/exec-tool-module-shape.md b/.changeset/exec-tool-module-shape.md new file mode 100644 index 00000000..e8e64870 --- /dev/null +++ b/.changeset/exec-tool-module-shape.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +The `exec` tool now tells the model to put module code in an `export default async function (input)` shape. diff --git a/.changeset/isolate-default-export.md b/.changeset/isolate-default-export.md new file mode 100644 index 00000000..d0af3571 --- /dev/null +++ b/.changeset/isolate-default-export.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +Isolate JavaScript modules now need a default export, and fail before running without one. diff --git a/.changeset/isolate-node-modules.md b/.changeset/isolate-node-modules.md new file mode 100644 index 00000000..560e8871 --- /dev/null +++ b/.changeset/isolate-node-modules.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +Isolate JavaScript code can now import Node.js built-ins such as `node:path`, `node:crypto`, and `node:zlib`. diff --git a/.changeset/isolate-root-directory.md b/.changeset/isolate-root-directory.md new file mode 100644 index 00000000..2a8d5bd1 --- /dev/null +++ b/.changeset/isolate-root-directory.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +The JavaScript backend now creates its root directory (`/workspace`) if the Workspace doesn't have one. diff --git a/.changeset/isolate-unhandled-rejections.md b/.changeset/isolate-unhandled-rejections.md new file mode 100644 index 00000000..341db68b --- /dev/null +++ b/.changeset/isolate-unhandled-rejections.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +Isolate JavaScript runs now fail on I/O at module scope or an unhandled rejection, instead of completing silently, with no output. diff --git a/.changeset/storage-without-cast.md b/.changeset/storage-without-cast.md new file mode 100644 index 00000000..26133ced --- /dev/null +++ b/.changeset/storage-without-cast.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +A DO's `ctx.storage` can now be passed to a Workspace without casting. diff --git a/.changeset/workspace-error-path-once.md b/.changeset/workspace-error-path-once.md new file mode 100644 index 00000000..650d1acd --- /dev/null +++ b/.changeset/workspace-error-path-once.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +Workspace filesystem errors no longer repeat the path. diff --git a/docs/17_isolate_javascript.md b/docs/17_isolate_javascript.md index f97412b6..bfae193e 100644 --- a/docs/17_isolate_javascript.md +++ b/docs/17_isolate_javascript.md @@ -24,6 +24,8 @@ const workspace = new Workspace({ }); ``` +`root`, `/workspace` by default, confines every path isolate code touches. A new Workspace doesn't have that directory yet, so a read-write backend creates it the first time it runs. + Execute a module through the common runtime entry point: ```ts @@ -49,7 +51,23 @@ const result = await handle.result(); // result.value = { value: 42, persisted: "42" } ``` -The source is a real ES module. Static imports, literal dynamic imports, and top-level await are supported. If the module default-exports a function, Workspace invokes it with `options.input`. Otherwise module evaluation completes with a `null` structured result. +The source is a real ES module, with static imports and literal dynamic imports. The module needs a default export, or the run fails before it starts. A default-exported function is called with `options.input`, and any other default value is the result. To run code that's already in a file, re-export it with `export { default } from "./main.js"`. + +Put the module's work in that function. Each run loads the module first, then calls its default export, and the Workers runtime doesn't allow I/O while a module loads. So `node:fs` and host module calls only work once the function is running: + +```js +import fs from "node:fs/promises"; + +// Fails: this runs while the module loads. +const early = await fs.readFile("/workspace/a.txt", "utf8"); + +export default async function () { + // Works: this runs when Workspace calls the function. + return fs.readFile("/workspace/a.txt", "utf8"); +} +``` + +A call made while the module loads fails the run with an error that names the call, even if the code catches the error, since the work it asked for never happened. The run also fails if a promise rejects and nothing has handled it by the time the function finishes. Top-level `await` is fine for anything that doesn't do I/O. The returned value becomes the result's `value` and must be JSON-compatible plain data. As with `JSON.stringify`, an `undefined` object field is left out, so `{ kept: 1, dropped: undefined }` completes as `{ kept: 1 }`, and returning `undefined` gives `null`. A function, a class instance such as a `Date`, an `undefined` array item, or a cycle fails the run. `options.input` is checked the same way. @@ -123,11 +141,12 @@ const handle = await workspace.runtime.exec( ## Modules -Caller source can import three kinds of module, and all of them are fixed when the backend is constructed: +Caller source can import four kinds of module, and all of them are fixed when the backend is constructed: | Kind | Configured with | Runs in | Example | | --- | --- | --- | --- | | Built in | Always installed | The isolate, backed by the Workspace | `node:fs`, `node:fs/promises` | +| Node.js | `nodejs_compat` in `compatibilityFlags`, the default | The isolate, provided by the runtime | `node:path`, `node:crypto` | | Source | `modules: { name: "source" }` | The isolate | a bundled library | | Host | `modules: { "ws:name": { fn } }`, or a factory | The Durable Object | `ws:git`, `ws:container`, your own | @@ -150,7 +169,9 @@ new WorkerJavaScriptBackend({ }); ``` -An import that is not built in, configured, or a relative or absolute Workspace path fails before the Worker is created. Caller source and durable files cannot shadow a configured or built-in module. +An import that is not built in, configured, one of the allowed Node.js modules, or a relative or absolute Workspace path fails before the Worker is created. Caller source and durable files cannot shadow a configured or built-in module. + +The allowed Node.js modules are the ones that work entirely inside the isolate: `node:path`, `node:url`, `node:util`, `node:events`, `node:buffer`, `node:assert`, `node:string_decoder`, `node:querystring`, `node:stream`, `node:crypto`, `node:zlib`, `node:timers`, `node:async_hooks`, and `node:diagnostics_channel`, with their subpaths such as `node:path/posix` and `node:timers/promises`. The runtime provides them, and imports of them are left as written. A bare name such as `path` works too and becomes `node:path`, unless a configured module has that name, in which case the configured module wins. The runtime has more Node.js modules, but they either duplicate what the Workspace provides, as `node:fs` does, reach outside the isolate, or are stubs that throw when called, so they stay unavailable. Any import that is not a path is resolved by name. The Worker Loader has no `node_modules` lookup and resolves a bare import next to the importing file, so Workspace stores each source and host module once, in a `__modules__` directory of the Worker's bundle, and rewrites every import of one into a relative path to it. Every file that imports `lodash` gets the same instance, however many directories the code spans. An absolute import is rewritten the same way. Relative paths are the only form the Worker Loader's legacy and new module registries resolve alike, so imports work whether or not `compatibilityFlags` includes `new_module_registry`. When a module fails to link, the error names it as the code wrote it. @@ -158,6 +179,7 @@ The backend describes its modules for a model in `backend.description`, which `w ```text `command` is ECMAScript module source, run in an isolated JavaScript runtime. Relative imports resolve from `cwd` in the workspace. +Put the work in `export default async function (input) { ... }` and call `node:fs` and the other modules below inside it, since the module's top level can't do I/O. To run a file you've already written, re-export it: `export { default } from "./main.js"`. Code has no direct network access. Modules code can import: @@ -166,6 +188,7 @@ Modules code can import: - `ws:git`: The workspace's Git repository tools: `status({ dir })`, ... - `ws:container`: Runs shell commands in a full Linux container that shares this workspace's files. ... - `ws:weather`: exports `forecast`. +- Node.js built-ins: `node:path`, `node:url`, ... Bare names such as `path` work too. ``` A factory adds its own text through a `description` property, as the prebuilt modules do. An object of functions is listed by its export names; say more about it in the `exec` tool's backend description if the model needs it. diff --git a/examples/artifacts/src/index.ts b/examples/artifacts/src/index.ts index 3e576fe5..f12d56d6 100644 --- a/examples/artifacts/src/index.ts +++ b/examples/artifacts/src/index.ts @@ -11,7 +11,6 @@ import { DurableObject } from "cloudflare:workers"; import { - type DurableObjectStorageLike, getWorkspace, sh, type WorkspaceClient, @@ -64,7 +63,7 @@ export class ArtifactCreator extends withWorkspace(class extends DurableObject { constructor(ctx: DurableObjectState, env: Env) { super(ctx, env); this.#workspace = new Workspace({ - // ctx.storage.sql.exec returns a narrower row type than - // DurableObjectStorageLike declares; the runtime shape - // matches. Cast through unknown to bypass invariance. - storage: ctx.storage as unknown as DurableObjectStorageLike, + storage: ctx.storage, // No backend: this workspace only needs its filesystem. The // shell half throws if touched, which we never do. sessionId: ctx.id.toString(), diff --git a/examples/celld/src/index.ts b/examples/celld/src/index.ts index ab9d23f1..7cd1e9f4 100644 --- a/examples/celld/src/index.ts +++ b/examples/celld/src/index.ts @@ -1,10 +1,5 @@ import { AIChatAgent, type OnChatMessageOptions } from "@cloudflare/ai-chat"; -import { - type DurableObjectStorageLike, - getWorkspace, - type WorkspaceRuntimeLoader, - withWorkspace, -} from "@cloudflare/computer"; +import { getWorkspace, type WorkspaceRuntimeLoader, withWorkspace } from "@cloudflare/computer"; import { createAITools } from "@cloudflare/computer/tools/ai-sdk"; import { routeAgentRequest } from "agents"; import { convertToModelMessages, isStepCount, streamText } from "ai"; @@ -35,7 +30,7 @@ class CelldAgentBase extends AIChatAgent { export class CelldAgent extends withWorkspace(CelldAgentBase, (self) => { const { ctx, env } = self as unknown as { ctx: DurableObjectState; env: CelldAgentEnv }; return { - storage: ctx.storage as unknown as DurableObjectStorageLike, + storage: ctx.storage, backends: env.LOADER ? [new CelldJavaScriptBackend(env.LOADER)] : [], }; }) { diff --git a/examples/container-legacy/src/index.ts b/examples/container-legacy/src/index.ts index 5d324402..346f2615 100644 --- a/examples/container-legacy/src/index.ts +++ b/examples/container-legacy/src/index.ts @@ -20,7 +20,6 @@ import { DurableObject, tracing } from "cloudflare:workers"; import { - type DurableObjectStorageLike, getWorkspace, R2Bucket, type WorkspaceOptions, @@ -63,10 +62,7 @@ class ContainerBase extends withLegacyWorkspaceContainer(class extends DurableOb function workspaceOptions(self: InstanceType): WorkspaceOptions { const { ctx, env } = self as unknown as { ctx: DurableObjectState; env: Env }; return { - // ctx.storage.sql.exec returns a narrower row type than - // DurableObjectStorageLike declares; the runtime shape - // matches. Cast through unknown to bypass invariance. - storage: ctx.storage as unknown as DurableObjectStorageLike, + storage: ctx.storage, backends: [self.backend], // Mount the Bucket binding at /workspace/r2. Seed it with // `npm run seed:r2` (uploads ./seed/data/hello.txt) so the diff --git a/examples/container/src/index.ts b/examples/container/src/index.ts index 02a025dc..df906e74 100644 --- a/examples/container/src/index.ts +++ b/examples/container/src/index.ts @@ -28,7 +28,6 @@ import { DurableObject, tracing } from "cloudflare:workers"; import { - type DurableObjectStorageLike, getWorkspace, type WorkspaceOptions, WorkspaceProxy, @@ -79,10 +78,7 @@ class ContainerBase extends withWorkspaceContainer(class extends DurableObject): WorkspaceOptions { const { ctx } = self as unknown as { ctx: DurableObjectState; env: Env }; return { - // ctx.storage.sql.exec returns a narrower row type than - // DurableObjectStorageLike declares; the runtime shape - // matches. Cast through unknown to bypass invariance. - storage: ctx.storage as unknown as DurableObjectStorageLike, + storage: ctx.storage, backends: [self.backend], // Route every workspace operation through the Cloudflare // runtime's user-tracing surface. The runtime owns the span diff --git a/examples/egress/src/index.ts b/examples/egress/src/index.ts index 4c704fac..809fdc2f 100644 --- a/examples/egress/src/index.ts +++ b/examples/egress/src/index.ts @@ -1,7 +1,6 @@ import { DurableObject, WorkerEntrypoint } from "cloudflare:workers"; import { - type DurableObjectStorageLike, getWorkspace, type WorkspaceClient, type WorkspaceOptions, @@ -62,7 +61,7 @@ function workspaceOptions(self: InstanceType): Works const workspace = { binding: "EgressExample", id: ctx.id.toString() }; return { - storage: ctx.storage as unknown as DurableObjectStorageLike, + storage: ctx.storage, backends: [ self.containerBackend, new WorkerShellBackend({ diff --git a/examples/mcp/src/index.ts b/examples/mcp/src/index.ts index 4394367d..355cddd4 100644 --- a/examples/mcp/src/index.ts +++ b/examples/mcp/src/index.ts @@ -1,6 +1,5 @@ import { DurableObject } from "cloudflare:workers"; import { - type DurableObjectStorageLike, getWorkspace, type WorkspaceOptions, WorkspaceProxy, @@ -47,7 +46,7 @@ class ComputerMCPBase extends withWorkspaceContainer(ComputerMCPDurableObject) { function workspaceOptions(self: InstanceType): WorkspaceOptions { const { ctx } = self as unknown as { ctx: DurableObjectState }; return { - storage: ctx.storage as unknown as DurableObjectStorageLike, + storage: ctx.storage, sessionId: ctx.id.toString(), git: createGitClient(), backends: [self.workerShell, self.containerShell], diff --git a/examples/pi-ai/src/index.ts b/examples/pi-ai/src/index.ts index 3980702e..3de719e2 100644 --- a/examples/pi-ai/src/index.ts +++ b/examples/pi-ai/src/index.ts @@ -7,12 +7,7 @@ import { DurableObject } from "cloudflare:workers"; -import { - type DurableObjectStorageLike, - Workspace, - WorkspaceServiceProxy, - type WorkspaceStub, -} from "@cloudflare/computer"; +import { Workspace, WorkspaceServiceProxy, type WorkspaceStub } from "@cloudflare/computer"; import { WorkerShellBackend } from "@cloudflare/computer/backends/worker-shell"; import { createPiTools } from "@cloudflare/computer/tools/pi-ai"; import { createModels, type Message } from "@earendil-works/pi-ai"; @@ -30,7 +25,7 @@ const MAX_TURNS = 10; export class PiAgent extends DurableObject { workspace = new Workspace({ - storage: this.ctx.storage as unknown as DurableObjectStorageLike, + storage: this.ctx.storage, backends: [ new WorkerShellBackend({ id: "shell", diff --git a/examples/rlm/worker/executor-agent.ts b/examples/rlm/worker/executor-agent.ts index 7e245ea5..ac0a2aeb 100644 --- a/examples/rlm/worker/executor-agent.ts +++ b/examples/rlm/worker/executor-agent.ts @@ -1,9 +1,5 @@ import { AIChatAgent, type OnChatMessageOptions } from "@cloudflare/ai-chat"; -import { - type DurableObjectStorageLike, - Workspace, - type WorkspaceRuntimeLoader, -} from "@cloudflare/computer"; +import { Workspace, type WorkspaceRuntimeLoader } from "@cloudflare/computer"; import { WorkerJavaScriptBackend } from "@cloudflare/computer/backends/worker-javascript"; import type { Connection } from "agents"; import { isStepCount, streamText, type ToolSet } from "ai"; @@ -58,7 +54,7 @@ export class ExecutorAgent extends AIChatAgent { maxTimeoutMs: 360_000, }); this.#workspace = new Workspace({ - storage: ctx.storage as unknown as DurableObjectStorageLike, + storage: ctx.storage, backends: [backend], }); this.#tools = createExecutorTool(this.#workspace, EXECUTOR_BACKEND); diff --git a/examples/rlm/worker/rlm-agent.ts b/examples/rlm/worker/rlm-agent.ts index a3fd3d3f..f254a420 100644 --- a/examples/rlm/worker/rlm-agent.ts +++ b/examples/rlm/worker/rlm-agent.ts @@ -1,9 +1,5 @@ import { AIChatAgent, type OnChatMessageOptions } from "@cloudflare/ai-chat"; -import { - type DurableObjectStorageLike, - Workspace, - type WorkspaceRuntimeLoader, -} from "@cloudflare/computer"; +import { Workspace, type WorkspaceRuntimeLoader } from "@cloudflare/computer"; import { WorkerJavaScriptBackend } from "@cloudflare/computer/backends/worker-javascript"; import type { Connection } from "agents"; import { isStepCount, streamText, type ToolSet } from "ai"; @@ -121,7 +117,7 @@ export class RlmAgent extends AIChatAgent { maxTimeoutMs: 360_000, }); this.#workspace = new Workspace({ - storage: ctx.storage as unknown as DurableObjectStorageLike, + storage: ctx.storage, backends: [backend], }); this.#tools = createExecutorTool(this.#workspace, RLM_BACKEND); diff --git a/examples/tanstack-ai/src/index.ts b/examples/tanstack-ai/src/index.ts index 55d3e63f..fda5ba74 100644 --- a/examples/tanstack-ai/src/index.ts +++ b/examples/tanstack-ai/src/index.ts @@ -6,12 +6,7 @@ import { DurableObject } from "cloudflare:workers"; -import { - type DurableObjectStorageLike, - Workspace, - WorkspaceServiceProxy, - type WorkspaceStub, -} from "@cloudflare/computer"; +import { Workspace, WorkspaceServiceProxy, type WorkspaceStub } from "@cloudflare/computer"; import { WorkerShellBackend } from "@cloudflare/computer/backends/worker-shell"; import { createTanStackTools } from "@cloudflare/computer/tools/tanstack-ai"; import { chat, maxIterations, streamToText } from "@tanstack/ai"; @@ -25,7 +20,7 @@ const MODEL = "@cf/meta/llama-3.3-70b-instruct-fp8-fast"; export class TanStackAgent extends DurableObject { workspace = new Workspace({ - storage: this.ctx.storage as unknown as DurableObjectStorageLike, + storage: this.ctx.storage, backends: [ new WorkerShellBackend({ id: "shell", diff --git a/examples/think-compare-runtimes/worker/think/agents.ts b/examples/think-compare-runtimes/worker/think/agents.ts index dee845ea..f9f0ee22 100644 --- a/examples/think-compare-runtimes/worker/think/agents.ts +++ b/examples/think-compare-runtimes/worker/think/agents.ts @@ -1,5 +1,4 @@ import { - type DurableObjectStorageLike, Workspace, WorkspaceProxy, WorkspaceServiceProxy, @@ -320,7 +319,7 @@ export class WorkspaceThinkAgent extends RuntimeThinkAgent { containerEnv: this.env.FUSE_MOUNT ? { FUSE_MOUNT: this.env.FUSE_MOUNT } : undefined, }); const workspace = new Workspace({ - storage: this.#ctx.storage as unknown as DurableObjectStorageLike, + storage: this.#ctx.storage, backends: [ new WorkerShellBackend({ id: "shell", diff --git a/examples/think/src/agent.ts b/examples/think/src/agent.ts index 1fc2ddc6..496719ab 100644 --- a/examples/think/src/agent.ts +++ b/examples/think/src/agent.ts @@ -25,7 +25,6 @@ */ import { - type DurableObjectStorageLike, type ThinkWorkspaceCompatibility, Workspace, WorkspaceProxy, @@ -90,7 +89,7 @@ export class Assistant extends withWorkspaceContainer(AssistantBase) { * the Cloudflare Container. */ override workspace = new Workspace({ - storage: this.ctx.storage as unknown as DurableObjectStorageLike, + storage: this.ctx.storage, backends: [ new WorkerShellBackend({ id: "shell", diff --git a/examples/tutorial/README.md b/examples/tutorial/README.md index ee903c5b..64f7741f 100644 --- a/examples/tutorial/README.md +++ b/examples/tutorial/README.md @@ -143,7 +143,6 @@ import { } from "@cloudflare/computer/backends/container-legacy"; import { Think } from "@cloudflare/think"; import { - type DurableObjectStorageLike, type ThinkWorkspaceCompatibility, Workspace, } from "@cloudflare/computer"; @@ -158,7 +157,7 @@ export class RecipeAgent extends withLegacyWorkspaceContainer(RecipeBase) { }); override workspace = new Workspace({ - storage: this.ctx.storage as unknown as DurableObjectStorageLike, + storage: this.ctx.storage, backends: [this.#backend], useThink: true, }) as Workspace & ThinkWorkspaceCompatibility; diff --git a/examples/tutorial/src/index.ts b/examples/tutorial/src/index.ts index 1a988d0f..530979b7 100644 --- a/examples/tutorial/src/index.ts +++ b/examples/tutorial/src/index.ts @@ -17,12 +17,7 @@ // // README.md walks through building this file from an empty directory. -import { - type DurableObjectStorageLike, - type ThinkWorkspaceCompatibility, - Workspace, - WorkspaceProxy, -} from "@cloudflare/computer"; +import { type ThinkWorkspaceCompatibility, Workspace, WorkspaceProxy } from "@cloudflare/computer"; import { createAssets } from "@cloudflare/computer/assets"; import { LegacyContainerBackend, @@ -52,7 +47,7 @@ export class RecipeAgent extends withLegacyWorkspaceContainer(RecipeBase) { }); override workspace = new Workspace({ - storage: this.ctx.storage as unknown as DurableObjectStorageLike, + storage: this.ctx.storage, backends: [this.#backend], useThink: true, }) as Workspace & ThinkWorkspaceCompatibility; diff --git a/examples/worker-javascript/src/index.ts b/examples/worker-javascript/src/index.ts index 5cd06032..9cef7d63 100644 --- a/examples/worker-javascript/src/index.ts +++ b/examples/worker-javascript/src/index.ts @@ -1,7 +1,6 @@ import { DurableObject } from "cloudflare:workers"; import { - type DurableObjectStorageLike, getWorkspace, R2Bucket, type WorkspaceRuntimeValue, @@ -12,7 +11,7 @@ import { WorkerJavaScriptBackend } from "@cloudflare/computer/backends/worker-ja export class ContainerExample extends withWorkspace(class extends DurableObject {}, (self) => { const { ctx, env } = self as unknown as { ctx: DurableObjectState; env: Env }; return { - storage: ctx.storage as unknown as DurableObjectStorageLike, + storage: ctx.storage, backends: [new WorkerJavaScriptBackend({ loader: env.LOADER })], mounts: { "/workspace/r2": R2Bucket(env.Bucket), diff --git a/examples/worker-shell/src/index.ts b/examples/worker-shell/src/index.ts index 31874d7a..b3f155af 100644 --- a/examples/worker-shell/src/index.ts +++ b/examples/worker-shell/src/index.ts @@ -28,13 +28,7 @@ import { DurableObject } from "cloudflare:workers"; -import { - type DurableObjectStorageLike, - getWorkspace, - R2Bucket, - WorkspaceServiceProxy, - withWorkspace, -} from "@cloudflare/computer"; +import { getWorkspace, R2Bucket, WorkspaceServiceProxy, withWorkspace } from "@cloudflare/computer"; import { WorkerShellBackend } from "@cloudflare/computer/backends/worker-shell"; // Opt-in shell commands. Each import pulls one command group into // this Worker's bundle; a group you do not import is unreachable @@ -59,10 +53,7 @@ export { WorkspaceServiceProxy }; export class ContainerExample extends withWorkspace(class extends DurableObject {}, (self) => { const { ctx, env } = self as unknown as { ctx: DurableObjectState; env: Env }; return { - // ctx.storage.sql.exec returns a narrower row type than - // DurableObjectStorageLike declares; the runtime shape - // matches. Cast through unknown to bypass invariance. - storage: ctx.storage as unknown as DurableObjectStorageLike, + storage: ctx.storage, backends: [ new WorkerShellBackend({ loader: env.LOADER, diff --git a/packages/computer/src/backends/worker-javascript/module-graph.ts b/packages/computer/src/backends/worker-javascript/module-graph.ts index ddc3ede5..98edd9c2 100644 --- a/packages/computer/src/backends/worker-javascript/module-graph.ts +++ b/packages/computer/src/backends/worker-javascript/module-graph.ts @@ -26,6 +26,53 @@ const MODULES_DIRECTORY = "__modules__"; // Installed in every execution and backed by the Workspace. No module // in the `modules` option may use these names. const BUILT_IN_MODULES = ["node:fs", "node:fs/promises"] as const; + +// Node.js built-ins the Dynamic Worker provides itself under +// `nodejs_compat`. They only compute: none reaches the network, the +// process, or a filesystem, so code imports them unchanged. Workerd has +// many more, but those either duplicate what the Workspace provides, +// such as `node:fs`, or are stubs that throw when called. +const NODE_MODULES = [ + "path", + "path/posix", + "url", + "util", + "util/types", + "events", + "buffer", + "assert", + "assert/strict", + "string_decoder", + "querystring", + "stream", + "stream/promises", + "stream/web", + "crypto", + "zlib", + "timers", + "timers/promises", + "async_hooks", + "diagnostics_channel", +] as const; + +/** Whether `flags` give a Dynamic Worker the Node.js built-ins. */ +export function hasNodeModules(flags: readonly string[]): boolean { + return flags.includes("nodejs_compat") || flags.includes("nodejs_compat_v2"); +} + +const TOP_LEVEL_NODE_MODULES = NODE_MODULES.filter((name) => !name.includes("/")) + .map((name) => `\`node:${name}\``) + .join(", "); + +/** One markdown bullet listing the Node.js built-ins, for a model. */ +export const NODE_MODULES_DESCRIPTION = `- Node.js built-ins: ${TOP_LEVEL_NODE_MODULES}, with subpaths such as \`node:stream/promises\`. Bare names such as \`path\` work too.`; + +// The `node:` form of a Node.js built-in the backend allows, from either +// spelling, or undefined for anything else. +function nodeModule(specifier: string): string | undefined { + const name = specifier.startsWith("node:") ? specifier.slice("node:".length) : specifier; + return NODE_MODULES.some((allowed) => allowed === name) ? `node:${name}` : undefined; +} const HOST_SPECIFIER = /^ws:[A-Za-z0-9][A-Za-z0-9._-]*$/; const EXPORT_NAME = /^[A-Za-z_$][A-Za-z0-9_$]*$/; // `default` would turn the function into the default export, and a @@ -136,6 +183,8 @@ export interface BuildModuleGraphOptions { capability: WorkspaceRuntimeCapability; configuredModules: Readonly>; hostModules: ReadonlyMap; + /** Whether the Dynamic Worker has the Node.js built-ins. */ + nodeModules: boolean; maxSourceBytes: number; maxCapabilityBytes: number; maxModules?: number; @@ -233,6 +282,13 @@ export async function buildModuleGraph(options: BuildModuleGraphOptions) { edits.push({ ...site, specifier: relativeSpecifier(name, storedName(specifier)) }); continue; } + // A bare name becomes its node: form, which both module registries + // resolve as a built-in. + const node = options.nodeModules ? nodeModule(specifier) : undefined; + if (node !== undefined) { + if (node !== specifier) edits.push({ ...site, specifier: node }); + continue; + } if (specifier === CAPABILITIES_MODULE) { throw new Error(`Module ${JSON.stringify(specifier)} is reserved for Workspace internals.`); } @@ -277,6 +333,7 @@ export async function buildModuleGraph(options: BuildModuleGraphOptions) { modules[name] = rewriteImports(source, edits); } + assertDefaultExport(options.source); await visit(entryPath, options.source, 0); // node:* specifiers are resolved by name in both registries, so they @@ -297,6 +354,44 @@ export async function buildModuleGraph(options: BuildModuleGraphOptions) { return { entryName, modules }; } +// The run's result comes from the entry module's default export, so a +// module without one would complete with nothing to show. Fail before +// loading it, and name any other exports, since a model that writes +// `export function main` meant one of them. +function assertDefaultExport(source: string): void { + const ast = parse(source, { ecmaVersion: "latest", sourceType: "module" }); + const named: string[] = []; + for (const node of ast.body) { + if (node.type === "ExportDefaultDeclaration") return; + if (node.type !== "ExportNamedDeclaration") continue; + for (const specifier of node.specifiers) { + const exported = specifier.exported; + const name = exported.type === "Identifier" ? exported.name : String(exported.value); + if (name === "default") return; + named.push(name); + } + const declaration = node.declaration; + if (declaration?.type === "VariableDeclaration") { + for (const variable of declaration.declarations) { + if (variable.id.type === "Identifier") named.push(variable.id.name); + } + } else if (declaration?.id) { + named.push(declaration.id.name); + } + } + const shape = + 'Put the work in `export default async function (input) { ... }`, or re-export one with `export { default } from "./main.js"`.'; + if (named.length === 0) { + throw new Error(`The module has no default export, so there is nothing to run. ${shape}`); + } + const list = named.map((name) => `\`${name}\``); + const exports = + list.length === 1 ? list[0] : `${list.slice(0, -1).join(", ")}, and ${list[list.length - 1]}`; + throw new Error( + `The module exports ${exports} but no default, so there is nothing to run. ${shape}`, + ); +} + // An import written as a string literal, and where that literal sits in // the source, quotes included. interface ImportSite { @@ -421,8 +516,13 @@ function capabilitiesModule(maxCapabilityBytes: number) { "readFile", "readFileBytes", "writeFile", "mkdir", "rm", "chmod", "symlink", "readlink", "readdir", "readdirWithFileTypes", "stat", "lstat", "exists" ]); + // Calls the Workers runtime refused because they ran at module scope. + // The runner fails the run if any are left, even when the module + // caught the error, since the work it asked for never happened. + export const moduleScopeRefusals = []; export function install(value) { host = value; + moduleScopeRefusals.length = 0; globalThis[callKey] = filesystemCall; } async function filesystemCall(namespace, method, args) { @@ -439,7 +539,24 @@ function capabilitiesModule(maxCapabilityBytes: number) { if (approximateBytes(args) > ${maxCapabilityBytes}) { throw new Error(${JSON.stringify(requestTooLargeMessage)}); } - const payload = await host.call(namespace + "." + method, args); + let payload; + try { + payload = await host.call(namespace + "." + method, args); + } catch (error) { + // A module is evaluated outside any request, and the runtime + // refuses I/O there with an error about request handlers, which + // this code doesn't have. Name the call and the fix instead. + if (error instanceof Error && error.message.startsWith("Disallowed operation called within global scope")) { + const name = namespace === "fs" ? "node:fs" : namespace.slice("host/".length); + const refused = new Error( + name + " " + method + " can't run at module scope, where the Workers runtime refuses I/O. " + + "Call it from inside the default-exported function." + ); + moduleScopeRefusals.push(refused); + throw refused; + } + throw error; + } if (payload.error !== undefined) { const error = new Error(payload.error.message); if (payload.error.code !== undefined) error.code = payload.error.code; diff --git a/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts b/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts index 94a8fb1c..a7d95e23 100644 --- a/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts +++ b/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts @@ -1048,6 +1048,32 @@ describe("WorkerJavaScriptBackend", () => { expect(backend.description).toContain("- `ws:plain`: a host module."); }); + it("lists the Node.js built-ins code can import", () => { + const backend = new WorkerJavaScriptBackend({ loader: throwingLoader("must not load") }); + + expect(backend.description).toContain( + "- Node.js built-ins: `node:path`, `node:url`, `node:util`,", + ); + expect(backend.description).toContain("Bare names such as `path` work too."); + }); + + it("lists no Node.js built-ins without the nodejs_compat flag", () => { + const backend = new WorkerJavaScriptBackend({ + loader: throwingLoader("must not load"), + compatibilityFlags: [], + }); + + expect(backend.description).not.toContain("Node.js built-ins"); + }); + + it("tells a model to put the work in a default-exported function", () => { + const backend = new WorkerJavaScriptBackend({ loader: throwingLoader("must not load") }); + + expect(backend.description).toContain( + "Put the work in `export default async function (input) { ... }` and call `node:fs` and the other modules below inside it, since the module's top level can't do I/O. To run a file you've already written, re-export it: `export { default } from \"./main.js\"`.", + ); + }); + it("builds host modules from the Workspace services when it connects", async () => { const db = new Database(new SQLiteTestStorage()); initializeSchema(db, () => 0); @@ -1200,6 +1226,46 @@ describe("WorkerJavaScriptBackend", () => { ).toThrow(/reserved module name/); }, ); + describe("root directory", () => { + const loader = { + load: () => ({ + getEntrypoint: () => ({ + evaluate: ( + _input: unknown, + host: { + assertResult(value: unknown): Promise; + attachOutput(readable: ReadableStream): Promise; + }, + ) => evaluateResult(host, null), + }), + }), + }; + + it("is created on the first run in a fresh Workspace", async () => { + const workspace = new Workspace({ + storage: new SQLiteTestStorage(), + backends: [new WorkerJavaScriptBackend({ loader })], + }); + + const execution = await workspace.runtime.exec("export default () => null;"); + await expect(execution.result()).resolves.toMatchObject({ status: "completed" }); + + await expect(workspace.fs.stat("/workspace")).resolves.toMatchObject({ isDirectory: true }); + }); + + it("is left alone by a read-only backend", async () => { + const workspace = new Workspace({ + storage: new SQLiteTestStorage(), + backends: [new WorkerJavaScriptBackend({ loader, access: "read" })], + }); + + const execution = await workspace.runtime.exec("export default () => null;"); + await expect(execution.result()).resolves.toMatchObject({ status: "completed" }); + + await expect(workspace.fs.stat("/workspace")).rejects.toMatchObject({ code: "ENOENT" }); + }); + }); + describe("module resolution", () => { function completingLoader() { return vi.fn((_code: { modules: Record }) => ({ @@ -1241,7 +1307,7 @@ describe("WorkerJavaScriptBackend", () => { await workspace.fs.writeFile("/workspace/a/b/two.js", `import "large"; import "ws:echo";`); const execution = await workspace.runtime.exec( - `import "large"; import "ws:echo"; import "./a/one.js";`, + `import "large"; import "ws:echo"; import "./a/one.js"; export default null;`, ); await expect(execution.result()).resolves.toMatchObject({ status: "completed" }); @@ -1257,6 +1323,122 @@ describe("WorkerJavaScriptBackend", () => { ); }); + it("leaves a Node.js built-in import for the runtime to resolve", async () => { + const load = completingLoader(); + const workspace = new Workspace({ + storage: new SQLiteTestStorage(), + backends: [new WorkerJavaScriptBackend({ loader: { load } })], + }); + + const execution = await workspace.runtime.exec( + `import { join } from "node:path"; import { createHash } from "node:crypto"; export default null;`, + ); + await expect(execution.result()).resolves.toMatchObject({ status: "completed" }); + + const modules = load.mock.calls[0]?.[0].modules ?? {}; + expect(source(modules["workspace/__workspace_entry__.js"])).toBe( + `import { join } from "node:path"; import { createHash } from "node:crypto"; export default null;`, + ); + }); + + it("points a bare Node.js built-in name at its node: module", async () => { + const load = completingLoader(); + const workspace = new Workspace({ + storage: new SQLiteTestStorage(), + backends: [new WorkerJavaScriptBackend({ loader: { load } })], + }); + + const execution = await workspace.runtime.exec( + `import path from "path"; import { inspect } from "util"; export default null;`, + ); + await expect(execution.result()).resolves.toMatchObject({ status: "completed" }); + + const modules = load.mock.calls[0]?.[0].modules ?? {}; + expect(source(modules["workspace/__workspace_entry__.js"])).toBe( + `import path from "node:path"; import { inspect } from "node:util"; export default null;`, + ); + }); + + it("prefers a configured module over a Node.js built-in of the same name", async () => { + const load = completingLoader(); + const workspace = new Workspace({ + storage: new SQLiteTestStorage(), + backends: [ + new WorkerJavaScriptBackend({ + loader: { load }, + modules: { events: "export const mine = true;" }, + }), + ], + }); + + const execution = await workspace.runtime.exec( + `import { mine } from "events"; export default mine;`, + ); + await expect(execution.result()).resolves.toMatchObject({ status: "completed" }); + + const modules = load.mock.calls[0]?.[0].modules ?? {}; + expect(source(modules["workspace/__workspace_entry__.js"])).toBe( + `import { mine } from "../__modules__/events"; export default mine;`, + ); + }); + + it.each(["node:child_process", "node:net", "node:process", "child_process"])( + "rejects %s, which is not one of the Node.js built-ins it allows", + async (specifier) => { + const workspace = new Workspace({ + storage: new SQLiteTestStorage(), + backends: [new WorkerJavaScriptBackend({ loader: completingLoader() })], + }); + + await expect( + workspace.runtime.exec(`import ${JSON.stringify(specifier)}; export default null;`), + ).rejects.toThrow(`Module ${JSON.stringify(specifier)} is not configured`); + }, + ); + + it("rejects Node.js built-ins without the nodejs_compat flag", async () => { + const workspace = new Workspace({ + storage: new SQLiteTestStorage(), + backends: [ + new WorkerJavaScriptBackend({ loader: completingLoader(), compatibilityFlags: [] }), + ], + }); + + await expect( + workspace.runtime.exec(`import "node:path"; export default null;`), + ).rejects.toThrow('Module "node:path" is not configured'); + }); + + it("rejects a module with no default export before loading it", async () => { + const workspace = new Workspace({ + storage: new SQLiteTestStorage(), + backends: [new WorkerJavaScriptBackend({ loader: throwingLoader("must not load") })], + }); + + await expect(workspace.runtime.exec(`const x = 1;`)).rejects.toThrow( + 'The module has no default export, so there is nothing to run. Put the work in `export default async function (input) { ... }`, or re-export one with `export { default } from "./main.js"`.', + ); + await expect( + workspace.runtime.exec(`export function main() {} export const a = 1, b = 2;`), + ).rejects.toThrow("The module exports `main`, `a`, and `b` but no default"); + }); + + it.each([ + ["a default export", `export default 1;`], + ["a named default", `function main() {} export { main as default };`], + ["a re-exported default", `export { default } from "./main.js";`], + ])("runs a module with %s", async (_label, source) => { + const workspace = new Workspace({ + storage: new SQLiteTestStorage(), + backends: [new WorkerJavaScriptBackend({ loader: { load: completingLoader() } })], + }); + await workspace.fs.mkdir("/workspace", { recursive: true }); + await workspace.fs.writeFile("/workspace/main.js", "export default 1;"); + + const execution = await workspace.runtime.exec(source); + await expect(execution.result()).resolves.toMatchObject({ status: "completed" }); + }); + it("rewrites an absolute import to a path relative to its importer", async () => { const load = completingLoader(); const workspace = new Workspace({ diff --git a/packages/computer/src/backends/worker-javascript/worker-javascript.ts b/packages/computer/src/backends/worker-javascript/worker-javascript.ts index 3317aede..0f4b9624 100644 --- a/packages/computer/src/backends/worker-javascript/worker-javascript.ts +++ b/packages/computer/src/backends/worker-javascript/worker-javascript.ts @@ -1,3 +1,4 @@ +import type { WorkspaceBackendHost } from "../../backend.js"; import { WorkspaceRuntimeBridge } from "../../runtime/bridge.js"; import { assertRuntimeValue, WorkspaceRuntimeCapability } from "../../runtime/capability.js"; import { dynamicWorkerEgress, type WorkspaceEgressPolicy } from "../../runtime/egress.js"; @@ -18,6 +19,8 @@ import { decodeRuntimeFrames, type RuntimeFrame } from "./frames.js"; import { assertHostModuleExports, buildModuleGraph, + hasNodeModules, + NODE_MODULES_DESCRIPTION, type ParsedModules, parseModules, prepareSourceModules, @@ -240,11 +243,13 @@ export class WorkerJavaScriptBackend implements WorkspaceModuleBackend { }; this.description = [ "`command` is ECMAScript module source, run in an isolated JavaScript runtime. Relative imports resolve from `cwd` in the workspace.", + "Put the work in `export default async function (input) { ... }` and call `node:fs` and the other modules below inside it, since the module's top level can't do I/O. To run a file you've already written, re-export it: `export { default } from \"./main.js\"`.", ...(resolvedEgress.mode === "none" ? ["Code has no direct network access."] : []), ...(this.#options.access === "read" ? ["The workspace is read-only here."] : []), "", "Modules code can import:", this.#options.modules.description, + ...(hasNodeModules(this.#options.compatibilityFlags) ? [NODE_MODULES_DESCRIPTION] : []), ].join("\n"); } @@ -378,6 +383,8 @@ class JavaScriptBackendHandle implements WorkspaceModuleBackendHandle { this.#pendingStarts += 1; this.#pendingIds.add(id); try { + if (this.#options.access === "read-write") + await ensureDirectory(this.#host.fs, this.#options.root); const capability = new WorkspaceRuntimeCapability( this.#host.fs, this.#options.root, @@ -391,6 +398,7 @@ class JavaScriptBackendHandle implements WorkspaceModuleBackendHandle { capability, configuredModules: this.#preparedSourceModules(), hostModules: this.#hostModuleFunctions, + nodeModules: hasNodeModules(this.#options.compatibilityFlags), maxSourceBytes: this.#options.maxSourceBytes, maxCapabilityBytes: this.#options.maxCapabilityBytes, }); @@ -1052,7 +1060,19 @@ function startJavaScriptExecution(options: { function runtimeWorkerModule(entryName: string, maxStdioBytes: number) { return ` import { WorkerEntrypoint } from "cloudflare:workers"; - import { install } from "workspace-capabilities.js"; + import { install, moduleScopeRefusals } from "workspace-capabilities.js"; + + // A promise nobody awaits would otherwise fail without a trace while + // the run reports success. Track the rejections nothing handled, and + // fail the run if any are left once it is done. The runtime doesn't + // report rejections from module evaluation this way, which is why + // capability calls refused at module scope are tracked separately. + const unhandled = new Map(); + addEventListener("unhandledrejection", (event) => { + unhandled.set(event.promise, event.reason); + event.preventDefault(); + }); + addEventListener("rejectionhandled", (event) => unhandled.delete(event.promise)); export default class extends WorkerEntrypoint { async evaluate(input, host, context) { @@ -1194,11 +1214,19 @@ function runtimeWorkerModule(entryName: string, maxStdioBytes: number) { // holds the host bridge stub alive for the whole run; frames // enqueue as output is produced. const drained = host.attachOutput(output.readable); + unhandled.clear(); try { const module = await import(${JSON.stringify(entryName)}); const result = typeof module.default === "function" ? await module.default(input) : module.default ?? null; + // Rejections are reported after the microtask queue drains. + await new Promise((resolve) => setTimeout(resolve, 0)); + if (moduleScopeRefusals.length > 0) throw moduleScopeRefusals[0]; + for (const reason of unhandled.values()) { + const message = reason instanceof Error ? reason.message : String(reason); + throw new Error("Unhandled rejection: " + message); + } const value = result ?? null; await host.assertResult(value); enqueue({ name: "exit", code: 0, result: value }); @@ -1220,6 +1248,19 @@ function runtimeWorkerModule(entryName: string, maxStdioBytes: number) { `; } +// A new Workspace has no directories, not even the root that code reads +// and writes by default. Create it the first time it's needed. A +// recursive mkdir records a change even when the directory exists, so +// look first rather than calling it on every run. +async function ensureDirectory(fs: WorkspaceBackendHost["fs"], path: string): Promise { + try { + await fs.stat(path); + } catch (error) { + if ((error as { code?: unknown }).code !== "ENOENT") throw error; + await fs.mkdir(path, { recursive: true }); + } +} + function assertLoaderGraph( modules: Record, maxSourceBytes: number, diff --git a/packages/computer/tests/script-runner-worker.ts b/packages/computer/tests/script-runner-worker.ts index 0757679c..b2c0a148 100644 --- a/packages/computer/tests/script-runner-worker.ts +++ b/packages/computer/tests/script-runner-worker.ts @@ -6,11 +6,7 @@ import { type WorkerJavaScriptBackendOptions, } from "../src/backends/worker-javascript/index.js"; import { createGitClient } from "../src/git/index.js"; -import type { - DurableObjectStorageLike, - WorkspaceRuntimeValue, - WorkspaceStub, -} from "../src/index.js"; +import type { WorkspaceRuntimeValue, WorkspaceStub } from "../src/index.js"; import { Workspace } from "../src/index.js"; import { createArtifactsModule } from "../src/modules/artifacts.js"; import { createContainerModule } from "../src/modules/container.js"; @@ -125,7 +121,7 @@ export class HostDO extends DurableObject { constructor(ctx: DurableObjectState, env: Env) { super(ctx, env); this.#workspace = new Workspace({ - storage: ctx.storage as unknown as DurableObjectStorageLike, + storage: ctx.storage, waitUntil: ctx.waitUntil.bind(ctx), git: createGitClient(), backends: [ @@ -175,7 +171,6 @@ export class HostDO extends DurableObject { stdin?: string; backend?: string; }) { - await this.#workspace.fs.mkdir("/workspace", { recursive: true }); const handle = await this.#workspace.runtime.exec(input.source, { backend: input.backend ?? "worker-javascript", cwd: input.cwd, @@ -189,7 +184,6 @@ export class HostDO extends DurableObject { } async startRuntime(input: { source: string; id: string }) { - await this.#workspace.fs.mkdir("/workspace", { recursive: true }); const handle = await this.#workspace.runtime.exec(input.source, { backend: "worker-javascript", id: input.id, @@ -265,7 +259,9 @@ class StdioProbeBridge extends RpcTarget { export default class extends WorkerEntrypoint { override async fetch(request: Request) { const url = new URL(request.url); - const stub = this.env.HOST.get(this.env.HOST.idFromName("script-runner")); + // `object` picks a fresh Workspace; the default is shared by every test. + const name = url.searchParams.get("object") ?? "script-runner"; + const stub = this.env.HOST.get(this.env.HOST.idFromName(name)); try { if (url.pathname === "/module-probe") { diff --git a/packages/computer/tests/script-runner.test.ts b/packages/computer/tests/script-runner.test.ts index b3d4b018..7796e133 100644 --- a/packages/computer/tests/script-runner.test.ts +++ b/packages/computer/tests/script-runner.test.ts @@ -420,6 +420,118 @@ describe("WorkspaceRuntime", () => { expect(JSON.parse(text).result.value, text).toContain("acyclic"); }); + it("fails a run when a floating promise at module scope rejects", async () => { + const response = await runtime({ + source: ` + import fs from "node:fs/promises"; + (async () => { + await fs.writeFile("/workspace/floating.txt", "never"); + })(); + export default () => "returned"; + `, + cwd: "/workspace", + }); + const text = await response.text(); + expect(response.status, text).toBe(200); + const { result } = JSON.parse(text); + expect(result, text).toMatchObject({ status: "failed", exitCode: 1 }); + expect(result.stderr, text).toContain("node:fs writeFile can't run at module scope"); + }); + + it("fails a run that caught a call refused at module scope", async () => { + const response = await runtime({ + source: ` + import { echo } from "ws:test-host"; + (async () => { + try { + await echo("too early"); + } catch {} + })(); + export default () => "returned"; + `, + cwd: "/workspace", + }); + const text = await response.text(); + expect(response.status, text).toBe(200); + const { result } = JSON.parse(text); + expect(result, text).toMatchObject({ status: "failed", exitCode: 1 }); + expect(result.stderr, text).toContain("ws:test-host echo can't run at module scope"); + }); + + it("names the call when top-level await does I/O", async () => { + const response = await runtime({ + source: ` + import fs from "node:fs/promises"; + await fs.readdir("/workspace"); + export default () => "returned"; + `, + cwd: "/workspace", + }); + const text = await response.text(); + expect(response.status, text).toBe(200); + const { result } = JSON.parse(text); + expect(result, text).toMatchObject({ status: "failed", exitCode: 1 }); + expect(result.stderr, text).toContain("node:fs readdir can't run at module scope"); + }); + + it("fails a run when a promise the default export left behind rejects", async () => { + const response = await runtime({ + source: ` + export default async () => { + Promise.reject(new Error("left behind")); + return "returned"; + }; + `, + cwd: "/workspace", + }); + const text = await response.text(); + expect(response.status, text).toBe(200); + const { result } = JSON.parse(text); + expect(result, text).toMatchObject({ status: "failed", exitCode: 1 }); + expect(result.stderr, text).toContain("left behind"); + }); + + it("ignores a rejection the module handles", async () => { + const response = await runtime({ + source: ` + export default async () => { + const failing = Promise.reject(new Error("handled")); + await new Promise((resolve) => setTimeout(resolve, 0)); + return await failing.catch((error) => error.message); + }; + `, + cwd: "/workspace", + }); + const text = await response.text(); + expect(response.status, text).toBe(200); + expect(JSON.parse(text).result, text).toMatchObject({ status: "completed", value: "handled" }); + }); + + it("creates its root in a fresh Workspace", async () => { + const response = await SELF.fetch( + `https://example.test/runtime?object=${crypto.randomUUID()}`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + source: ` + import fs from "node:fs/promises"; + export default async () => { + await fs.writeFile("/workspace/first.txt", "written"); + return await fs.readdir("/workspace"); + }; + `, + }), + }, + ); + const text = await response.text(); + expect(response.status, text).toBe(200); + expect(JSON.parse(text).result, text).toMatchObject({ + status: "completed", + value: ["first.txt"], + }); + }); + it("drops undefined fields from a run result, as JSON does", async () => { const response = await runtime({ source: `export default () => ({ kept: 1, dropped: undefined, nested: { also: undefined } });`, @@ -719,6 +831,74 @@ describe.each([ .result; } + it("runs Node.js built-ins the runtime provides, by node: or bare name", async () => { + const result = await run(` + import { join } from "node:path"; + import { createHash } from "node:crypto"; + import { gzipSync, gunzipSync } from "node:zlib"; + import path from "path"; + export default () => ({ + joined: join("/workspace", "a", "..", "b.txt"), + bare: path.basename("/workspace/c.txt"), + sha: createHash("sha256").update("abc").digest("hex").slice(0, 8), + zipped: gunzipSync(gzipSync("round trip")).toString(), + }); + `); + expect(result).toMatchObject({ + status: "completed", + value: { joined: "/workspace/b.txt", bare: "c.txt", sha: "ba7816bf", zipped: "round trip" }, + }); + }); + + it("runs a default export re-exported from a workspace file", async () => { + await write( + "/workspace/app/main.js", + `import fs from "node:fs/promises"; + export default async (input) => { + await fs.writeFile("/workspace/app/out.txt", String(input.n * 21)); + return fs.readFile("/workspace/app/out.txt", "utf8"); + };`, + ); + const response = await runtime({ + source: `export { default } from "./main.js";`, + cwd: "/workspace/app", + backend, + value: { n: 2 }, + }); + const text = await response.text(); + expect(response.status, text).toBe(200); + expect(JSON.parse(text)).toMatchObject({ result: { status: "completed", value: "42" } }); + }); + + it("runs the Node.js timer, async context, and diagnostics built-ins", async () => { + const result = await run(` + import { setTimeout as fire } from "node:timers"; + import { setTimeout as sleep } from "node:timers/promises"; + import { AsyncLocalStorage } from "node:async_hooks"; + import diagnostics from "node:diagnostics_channel"; + export default async () => { + const fired = await new Promise((resolve) => fire(() => resolve("fired"), 1)); + const slept = await sleep(1, "slept"); + const storage = new AsyncLocalStorage(); + const stored = await storage.run(7, async () => { + await sleep(1); + return storage.getStore(); + }); + const channel = diagnostics.channel("probe"); + let published; + channel.subscribe((message) => { + published = message; + }); + channel.publish("sent"); + return { fired, slept, stored, published }; + }; + `); + expect(result).toMatchObject({ + status: "completed", + value: { fired: "fired", slept: "slept", stored: 7, published: "sent" }, + }); + }); + it("resolves bare, relative, and absolute imports from nested directories", async () => { await write( "/workspace/app/lib/util.js", diff --git a/packages/computer/tests/worker-backend-worker.ts b/packages/computer/tests/worker-backend-worker.ts index acfc0437..befc805e 100644 --- a/packages/computer/tests/worker-backend-worker.ts +++ b/packages/computer/tests/worker-backend-worker.ts @@ -20,11 +20,7 @@ import { DurableObject, WorkerEntrypoint } from "cloudflare:workers"; import curlModules from "@cloudflare/computer/shell/curl"; import { WorkerShellBackend } from "../src/backends/worker-shell/index.js"; -import type { - DurableObjectStorageLike, - WorkspaceRuntimeTruncation, - WorkspaceStub, -} from "../src/index.js"; +import type { WorkspaceRuntimeTruncation, WorkspaceStub } from "../src/index.js"; import { Workspace } from "../src/index.js"; export { WorkspaceServiceProxy } from "../src/proxy.js"; @@ -41,7 +37,7 @@ export class HostDO extends DurableObject { constructor(ctx: DurableObjectState, env: Env) { super(ctx, env); this.#workspace = new Workspace({ - storage: ctx.storage as unknown as DurableObjectStorageLike, + storage: ctx.storage, backends: [ new WorkerShellBackend({ loader: env.LOADER, diff --git a/packages/dofs/src/bench/counting-storage.ts b/packages/dofs/src/bench/counting-storage.ts index 70797ae9..9b9340c6 100644 --- a/packages/dofs/src/bench/counting-storage.ts +++ b/packages/dofs/src/bench/counting-storage.ts @@ -49,13 +49,10 @@ export class CountingStorage implements DurableObjectStorageLike { constructor(inner: DurableObjectStorageLike) { this.sql = { - exec: >( - query: string, - ...bindings: unknown[] - ): SQLCursorLike => { + exec: (query: string, ...bindings: unknown[]): SQLCursorLike => { this.statements += 1; this.classify(query); - const cursor = inner.sql.exec(query, ...bindings); + const cursor = inner.sql.exec(query, ...bindings); // Writes report rowsWritten eagerly after exec on the DO // backend; run() never iterates the cursor, so capture it here. const written = readNumber(cursor, "rowsWritten"); @@ -63,7 +60,7 @@ export class CountingStorage implements DurableObjectStorageLike { this.rowsWritten += written; } return { - toArray: (): Row[] => { + toArray: () => { const rows = cursor.toArray(); // rowsRead is only meaningful once the cursor is drained, // which all() does exactly once. diff --git a/packages/dofs/src/bench/fs-ops.bench.ts b/packages/dofs/src/bench/fs-ops.bench.ts index 38360b96..660393e0 100644 --- a/packages/dofs/src/bench/fs-ops.bench.ts +++ b/packages/dofs/src/bench/fs-ops.bench.ts @@ -53,7 +53,7 @@ async function withRealDb( ): Promise { const stub = freshStub(); return runInDurableObject(stub, async (_instance: unknown, state: DurableObjectState) => { - const storage = state.storage as unknown as DurableObjectStorageLike; + const storage = state.storage; const db = new Database(storage); initializeSchema(db, NOW); const provider = new SQLiteWorkspaceProvider(db, { now: NOW }); @@ -68,7 +68,7 @@ async function withCountingDb( ): Promise { const stub = freshStub(); return runInDurableObject(stub, async (_instance: unknown, state: DurableObjectState) => { - const counting = new CountingStorage(state.storage as unknown as DurableObjectStorageLike); + const counting = new CountingStorage(state.storage); const db = new Database(counting); initializeSchema(db, NOW); counting.reset(); diff --git a/packages/dofs/src/bench/sync-blocks.bench.ts b/packages/dofs/src/bench/sync-blocks.bench.ts index 3eada815..3309bda8 100644 --- a/packages/dofs/src/bench/sync-blocks.bench.ts +++ b/packages/dofs/src/bench/sync-blocks.bench.ts @@ -33,7 +33,6 @@ import { planBlock, selectMode } from "../sync/blocks.js"; import { decodeChangePack, encodeChangePack } from "../sync/change-pack.js"; import { MIN_BLOCK_PROFILE } from "../sync/operations.js"; import { currentRev } from "../sync/watermarks.js"; -import type { DurableObjectStorageLike } from "../types.js"; const NOW = (): number => 1000; const BIG_PROFILE = { maxEntries: 1_000_000, maxBytes: Number.MAX_SAFE_INTEGER }; @@ -45,7 +44,7 @@ function freshStub(): DurableObjectStub { async function withRealDB(fn: (db: Database) => Promise | T): Promise { return runInDurableObject(freshStub(), async (_i: unknown, state: DurableObjectState) => { - const db = new Database(state.storage as unknown as DurableObjectStorageLike); + const db = new Database(state.storage); initializeSchema(db, NOW); return await fn(db); }); diff --git a/packages/dofs/src/errors.test.ts b/packages/dofs/src/errors.test.ts new file mode 100644 index 00000000..5ee0213a --- /dev/null +++ b/packages/dofs/src/errors.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; + +import { createWorkspaceError } from "./errors.js"; +import { WorkspaceFilesystem } from "./fs/filesystem.js"; +import { withDB } from "./fs/with-db.js"; + +describe("createWorkspaceError", () => { + it("appends the path to a message that doesn't name it", () => { + expect(createWorkspaceError("ENOENT", "no such path", "/a/b").message).toBe( + "no such path: /a/b", + ); + }); + + it("names the path once when the message already ends with it", () => { + expect(createWorkspaceError("ENOENT", "no such path: /a/b", "/a/b").message).toBe( + "no such path: /a/b", + ); + }); + + it("keeps the path on the error either way", () => { + expect(createWorkspaceError("ENOENT", "no such path: /a/b", "/a/b")).toMatchObject({ + code: "ENOENT", + path: "/a/b", + }); + }); + + it("names the path once in a filesystem error", async () => { + await withDB(async (db) => { + const fs = new WorkspaceFilesystem(db); + await expect(fs.readdir("/missing")).rejects.toThrow(/^no such path: \/missing$/); + }); + }); +}); diff --git a/packages/dofs/src/errors.ts b/packages/dofs/src/errors.ts index 1c21257d..b36e37ad 100644 --- a/packages/dofs/src/errors.ts +++ b/packages/dofs/src/errors.ts @@ -24,7 +24,9 @@ export function createWorkspaceError( message: string, path?: string, ): WorkspaceFsError { - const error = new Error(path === undefined ? message : `${message}: ${path}`) as WorkspaceFsError; + // Many callers already end the message with the path. Name it once. + const named = path === undefined || message.endsWith(path) ? message : `${message}: ${path}`; + const error = new Error(named) as WorkspaceFsError; error.name = "WorkspaceFsError"; error.code = code; error.path = path; diff --git a/packages/dofs/src/fs/with-db.workers.ts b/packages/dofs/src/fs/with-db.workers.ts index addfaf7a..32495e12 100644 --- a/packages/dofs/src/fs/with-db.workers.ts +++ b/packages/dofs/src/fs/with-db.workers.ts @@ -7,7 +7,6 @@ import { env, runInDurableObject } from "cloudflare:test"; import type { TestBindings } from "../../tests/worker.js"; import { initializeSchema } from "../schema/index.js"; import { Database } from "../storage.js"; -import type { DurableObjectStorageLike } from "../types.js"; export interface WithDBOptions { now?: () => number; @@ -26,7 +25,7 @@ export async function withDB( ): Promise { const stub = freshStub(); return runInDurableObject(stub, async (_instance: unknown, state: DurableObjectState) => { - const db = new Database(state.storage as unknown as DurableObjectStorageLike); + const db = new Database(state.storage); initializeSchema(db, options.now ?? (() => 1000)); return await fn(db); }); @@ -55,13 +54,13 @@ export async function withTwoDBs( const captured = await runInDurableObject( stubA, async (_a: unknown, stateA: DurableObjectState) => { - const a = new Database(stateA.storage as unknown as DurableObjectStorageLike); + const a = new Database(stateA.storage); initializeSchema(a, now); return await snapshot(a); }, ); return runInDurableObject(stubB, async (_b: unknown, stateB: DurableObjectState) => { - const b = new Database(stateB.storage as unknown as DurableObjectStorageLike); + const b = new Database(stateB.storage); initializeSchema(b, now); return await apply(b, captured); }); diff --git a/packages/dofs/src/storage.ts b/packages/dofs/src/storage.ts index becce5a7..9362325c 100644 --- a/packages/dofs/src/storage.ts +++ b/packages/dofs/src/storage.ts @@ -78,7 +78,7 @@ export class Database { } all(query: string, ...bindings: unknown[]): Row[] { - const rows = this.sql.exec(query, ...bindings).toArray(); + const rows = this.sql.exec(query, ...bindings).toArray() as Row[]; return rows.map((row) => normalizeRow(row as Record)) as Row[]; } diff --git a/packages/dofs/src/sync/blobs.test.ts b/packages/dofs/src/sync/blobs.test.ts index e9568596..0ce5b4a5 100644 --- a/packages/dofs/src/sync/blobs.test.ts +++ b/packages/dofs/src/sync/blobs.test.ts @@ -36,11 +36,11 @@ describe("stageBlob", () => { const hash = sha256(bytes); const failingDb = new Database({ sql: { - exec: (query: string, ...bindings: unknown[]) => { + exec: (query: string, ...bindings: unknown[]) => { if (query.startsWith("INSERT INTO vfs_blob_bytes")) { throw new Error("injected bytes failure"); } - return db.sql.exec(query, ...bindings); + return db.sql.exec(query, ...bindings); }, }, transactionSync: (closure) => db.transactionSync(closure), diff --git a/packages/dofs/src/sync/fetch.test.ts b/packages/dofs/src/sync/fetch.test.ts index 08764ac2..39dd6cd2 100644 --- a/packages/dofs/src/sync/fetch.test.ts +++ b/packages/dofs/src/sync/fetch.test.ts @@ -136,11 +136,11 @@ describe("hasObjects", () => { await withDB((db) => { const limitedStorage: DurableObjectStorageLike = { sql: { - exec(query: string, ...bindings: unknown[]): SQLCursorLike { + exec(query: string, ...bindings: unknown[]): SQLCursorLike { if (bindings.length > 100) { throw new Error(`too many SQLite bindings: ${bindings.length}`); } - return db.sql.exec(query, ...bindings); + return db.sql.exec(query, ...bindings); }, }, transactionSync: (closure) => db.transactionSync(closure), diff --git a/packages/dofs/src/types.ts b/packages/dofs/src/types.ts index 2832820e..9c66fa7d 100644 --- a/packages/dofs/src/types.ts +++ b/packages/dofs/src/types.ts @@ -3,10 +3,10 @@ export interface SQLCursorLike> { } export interface SQLStorageLike { - exec>( - query: string, - ...bindings: unknown[] - ): SQLCursorLike; + // Not generic: the Workers runtime's `exec` constrains its row type in + // a way a generic signature here can't accept. Database casts rows to + // the shape each query selects. + exec(query: string, ...bindings: unknown[]): SQLCursorLike; } export interface DurableObjectStorageLike { @@ -14,3 +14,10 @@ export interface DurableObjectStorageLike { transaction?(closure: () => T | Promise): T | Promise; transactionSync?(closure: () => T): T; } + +// A Durable Object's `ctx.storage` has to satisfy DurableObjectStorageLike +// as it is, without a cast. This fails to compile if the types drift. +type Assert = T; +type _DurableObjectStorageFits = Assert< + DurableObjectStorage extends DurableObjectStorageLike ? true : false +>;