Skip to content

Commit 0f3d168

Browse files
kitlangtonkitlangton
authored
test(cli): subprocess integration test harness + regression suite for opencode run (anomalyco#28230)
1 parent ce09fc8 commit 0f3d168

4 files changed

Lines changed: 317 additions & 35 deletions

File tree

‎packages/opencode/test/cli/run/run-process.test.ts‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Subprocess integration tests for `opencode run` (non-interactive mode).
2+
// These exercise the real CLI binary against a TestLLMServer running in the
3+
// same process. See `test/lib/run-process.ts` for the harness — each test uses
4+
// `opencode.run(message, opts?)` to spawn `bun src/index.ts run ...` with
5+
// `OPENCODE_CONFIG_CONTENT` providing the test provider config inline.
6+
import { describe, expect } from "bun:test"
7+
import { Effect } from "effect"
8+
import { runIt } from "../../lib/run-process"
9+
10+
describe("opencode run (non-interactive subprocess)", () => {
11+
// Happy path: prompt completes, output reaches stdout, process exits 0.
12+
// If this fails, all the others likely will too — debug here first.
13+
runIt.live(
14+
"exits 0 and writes the response to stdout on a successful prompt",
15+
({ llm, opencode }) =>
16+
Effect.gen(function* () {
17+
yield* llm.text("hello from the test llm")
18+
const result = yield* opencode.run("say hi")
19+
opencode.expectExit(result, 0)
20+
expect(result.stdout).toContain("hello from the test llm")
21+
}),
22+
60_000,
23+
)
24+
25+
// Regression for #27371: an unknown model used to hang the process forever
26+
// waiting on a session.status === idle event that never arrived. The fix
27+
// makes the SDK call surface an error promptly so the process exits nonzero.
28+
// We assert nonzero exit AND wall-clock under the harness timeout — a hang
29+
// would expire the timeout and produce a different (signal-killed) failure.
30+
runIt.live(
31+
"exits nonzero promptly when the model is unknown (regression for #27371)",
32+
({ opencode }) =>
33+
Effect.gen(function* () {
34+
const result = yield* opencode.run("say hi", {
35+
model: "test/nonexistent-model",
36+
timeoutMs: 15_000,
37+
})
38+
expect(result.exitCode).not.toBe(0)
39+
expect(result.durationMs).toBeLessThan(15_000)
40+
}),
41+
30_000,
42+
)
43+
44+
// Locks in the current behavior: when the LLM stream errors mid-response
45+
// (the prompt was accepted, then the upstream provider failed), opencode
46+
// emits a session.error event and the process exits 0 today.
47+
//
48+
// This is debatable — a future cleanup might flip it to exit 1. If you're
49+
// changing this expectation, do it deliberately and say so in the PR.
50+
runIt.live(
51+
"mid-stream LLM error still exits 0 today (contract lock-in)",
52+
({ llm, opencode }) =>
53+
Effect.gen(function* () {
54+
yield* llm.fail("upstream provider exploded mid-stream")
55+
const result = yield* opencode.run("trigger midstream error", { timeoutMs: 30_000 })
56+
expect(result.exitCode).toBe(0)
57+
}),
58+
60_000,
59+
)
60+
61+
// --format json puts one JSON object per line on stdout for each emitted
62+
// event. Consumers (CI scripts, tooling) parse this stream. Asserts the
63+
// shape so a future event-emit change has to update this expectation.
64+
runIt.live(
65+
"--format json emits parseable line-delimited JSON to stdout",
66+
({ llm, opencode }) =>
67+
Effect.gen(function* () {
68+
yield* llm.text("structured output")
69+
const result = yield* opencode.run("say hi", { format: "json" })
70+
opencode.expectExit(result, 0)
71+
72+
const events = opencode.parseJsonEvents(result.stdout)
73+
expect(events.length).toBeGreaterThan(0)
74+
for (const evt of events) {
75+
expect(typeof evt.type).toBe("string")
76+
expect(typeof evt.sessionID).toBe("string")
77+
}
78+
// At least one `text` event should appear with the LLM's response.
79+
const text = events.find((e) => e.type === "text")
80+
expect(text).toBeDefined()
81+
}),
82+
60_000,
83+
)
84+
})

‎packages/opencode/test/lib/run-process.ts‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
// Subprocess test harness for the `opencode run` CLI.
2+
//
3+
// This is the missing test tier: every other `cli/run/*.test.ts` is a unit
4+
// test of an extracted helper. Nothing actually exercises the `RunCommand`
5+
// handler end-to-end. Bugs that span argv parsing → server boot → SDK call →
6+
// event consumption → exit code (like the original /event race or the
7+
// non-interactive hang #27371) are invisible to in-process tests.
8+
//
9+
// The harness uses opencode's built-in test affordances to spawn the real CLI
10+
// hermetically:
11+
// - OPENCODE_CONFIG_CONTENT : provider config inline, no files to find
12+
// - OPENCODE_TEST_HOME : pins os.homedir() → tmpdir
13+
// - OPENCODE_DISABLE_PROJECT_CONFIG : skip walking up for opencode.json
14+
// - OPENCODE_PURE : skip external plugin discovery + install
15+
// - OPENCODE_DISABLE_AUTOUPDATE / AUTOCOMPACT / MODELS_FETCH : no background work
16+
//
17+
// Plus HOME / XDG_* pointing at the tmpdir for belt-and-suspenders isolation.
18+
//
19+
// The custom `test` provider points at a TestLLMServer running in the same
20+
// process at a random port. The CLI subprocess talks to it over real HTTP.
21+
import type { TestOptions } from "bun:test"
22+
import * as Scope from "effect/Scope"
23+
import { Effect } from "effect"
24+
import path from "node:path"
25+
import fs from "node:fs/promises"
26+
import os from "node:os"
27+
import { Process } from "@/util/process"
28+
import { TestLLMServer } from "./llm-server"
29+
import { testProviderConfig } from "./test-provider"
30+
import { it } from "./effect"
31+
32+
const opencodeRoot = path.resolve(import.meta.dir, "../../")
33+
const cliEntry = path.join(opencodeRoot, "src/index.ts")
34+
35+
export const testModelID = "test/test-model"
36+
37+
function isolatedEnv(home: string, configJson: string): Record<string, string> {
38+
return {
39+
OPENCODE_TEST_HOME: home,
40+
HOME: home,
41+
XDG_CONFIG_HOME: path.join(home, ".config"),
42+
XDG_DATA_HOME: path.join(home, ".local/share"),
43+
XDG_STATE_HOME: path.join(home, ".local/state"),
44+
XDG_CACHE_HOME: path.join(home, ".cache"),
45+
OPENCODE_CONFIG_CONTENT: configJson,
46+
OPENCODE_DISABLE_PROJECT_CONFIG: "1",
47+
OPENCODE_PURE: "1",
48+
OPENCODE_DISABLE_AUTOUPDATE: "1",
49+
OPENCODE_DISABLE_AUTOCOMPACT: "1",
50+
OPENCODE_DISABLE_MODELS_FETCH: "1",
51+
OPENCODE_AUTH_CONTENT: "{}",
52+
}
53+
}
54+
55+
export type RunResult = {
56+
readonly exitCode: number
57+
readonly stdout: string
58+
readonly stderr: string
59+
readonly durationMs: number
60+
}
61+
62+
type SpawnOpts = { readonly timeoutMs?: number; readonly env?: Record<string, string> }
63+
64+
// A `RunOpts` is the typed equivalent of constructing argv for `opencode run`.
65+
// New flags should land here so tests stay grep-able and refactor-safe.
66+
export type RunOpts = SpawnOpts & {
67+
readonly model?: string
68+
readonly agent?: string
69+
readonly format?: "default" | "json"
70+
readonly command?: string
71+
readonly printLogs?: boolean
72+
readonly extraArgs?: string[]
73+
}
74+
75+
export type OpencodeCli = {
76+
// High-level: run a single prompt against the test model.
77+
readonly run: (message: string, opts?: RunOpts) => Effect.Effect<RunResult>
78+
// Escape hatch: any CLI invocation with full control over argv.
79+
readonly spawn: (args: string[], opts?: SpawnOpts) => Effect.Effect<RunResult>
80+
// Convenience assertion. Dumps captured stderr/stdout on mismatch so CI
81+
// failures are debuggable without re-running locally.
82+
readonly expectExit: (result: RunResult, expected: number, label?: string) => void
83+
// Parse `--format json` stdout into one event object per non-empty line.
84+
// The CLI writes `JSON.stringify({ type, sessionID, ... }) + EOL` for each
85+
// event (see src/cli/cmd/run.ts `emit`). Throws if any line is malformed
86+
// so tests fail loudly rather than silently skipping data.
87+
readonly parseJsonEvents: (stdout: string) => Array<Record<string, unknown>>
88+
}
89+
90+
export type RunFixture = {
91+
readonly llm: TestLLMServer["Service"]
92+
readonly home: string
93+
readonly opencode: OpencodeCli
94+
}
95+
96+
// `withRunFixture(fn)` provisions a TestLLMServer + tmpdir + spawn helper and
97+
// invokes fn. Cleans up the tmpdir on scope exit.
98+
//
99+
// Note on the R channel: TestLLMServer.layer is provided internally so the
100+
// caller doesn't need to wire it up. The fixture's lifetime is tied to the
101+
// surrounding Scope.
102+
export function withRunFixture<A, E>(
103+
fn: (input: RunFixture) => Effect.Effect<A, E>,
104+
): Effect.Effect<A, E | unknown, Scope.Scope> {
105+
return Effect.gen(function* () {
106+
const llm = yield* TestLLMServer
107+
108+
const home = path.join(os.tmpdir(), "oc-run-" + Math.random().toString(36).slice(2))
109+
yield* Effect.promise(() => fs.mkdir(home, { recursive: true }))
110+
yield* Effect.addFinalizer(() =>
111+
Effect.promise(() => fs.rm(home, { recursive: true, force: true }).catch(() => undefined)),
112+
)
113+
114+
const configJson = JSON.stringify(testProviderConfig(llm.url))
115+
const env = isolatedEnv(home, configJson)
116+
117+
const spawn = (
118+
args: string[],
119+
opts?: SpawnOpts,
120+
): Effect.Effect<RunResult> =>
121+
Effect.promise(async () => {
122+
const start = Date.now()
123+
// Process.run pipes stdout/stderr by default and returns them as Buffers.
124+
const result = await Process.run(["bun", "run", "--conditions=browser", cliEntry, ...args], {
125+
cwd: home,
126+
timeout: opts?.timeoutMs ?? 30_000,
127+
env: { ...process.env, ...env, ...opts?.env },
128+
nothrow: true,
129+
})
130+
return {
131+
exitCode: result.code,
132+
stdout: result.stdout.toString(),
133+
stderr: result.stderr.toString(),
134+
durationMs: Date.now() - start,
135+
}
136+
})
137+
138+
const run = (message: string, opts?: RunOpts): Effect.Effect<RunResult> => {
139+
const argv: string[] = ["run"]
140+
if (opts?.printLogs) argv.push("--print-logs")
141+
argv.push("--model", opts?.model ?? testModelID)
142+
if (opts?.agent) argv.push("--agent", opts.agent)
143+
if (opts?.format) argv.push("--format", opts.format)
144+
if (opts?.command) argv.push("--command", opts.command)
145+
if (opts?.extraArgs) argv.push(...opts.extraArgs)
146+
argv.push(message)
147+
return spawn(argv, opts)
148+
}
149+
150+
const opencode: OpencodeCli = { run, spawn, expectExit, parseJsonEvents }
151+
152+
return yield* fn({ llm, home, opencode })
153+
}).pipe(Effect.provide(TestLLMServer.layer))
154+
}
155+
156+
function parseJsonEvents(stdout: string): Array<Record<string, unknown>> {
157+
return stdout
158+
.split("\n")
159+
.map((line) => line.trim())
160+
.filter((line) => line.length > 0)
161+
.map((line) => JSON.parse(line) as Record<string, unknown>)
162+
}
163+
164+
// Convenience for the common assertion pattern. Dumps stderr/stdout when
165+
// the exit code doesn't match — saves debugging time on CI failures.
166+
function expectExit(result: RunResult, expected: number, label = "opencode") {
167+
if (result.exitCode === expected) return
168+
const tail = (s: string, n: number) => (s.length > n ? "..." + s.slice(-n) : s)
169+
// eslint-disable-next-line no-console
170+
console.error(
171+
`[${label}] expected exit ${expected}, got ${result.exitCode} after ${result.durationMs}ms`,
172+
)
173+
// eslint-disable-next-line no-console
174+
console.error(`[${label}] stderr (last 2000):\n${tail(result.stderr, 2000)}`)
175+
// eslint-disable-next-line no-console
176+
console.error(`[${label}] stdout (last 500):\n${tail(result.stdout, 500)}`)
177+
throw new Error(`${label}: expected exit ${expected}, got ${result.exitCode}`)
178+
}
179+
180+
// `runIt.live(name, fixture => effect)` is the same as
181+
// `it.live(name, () => withRunFixture(fixture))` — one fewer nesting level at
182+
// every call site. Use this for any test that needs the opencode CLI fixture.
183+
//
184+
// Only `.live` is exposed because subprocess tests must run against the real
185+
// clock — a TestClock-paused environment can't drive a child process. If you
186+
// need `.only` or `.skip`, fall back to `it.live` + `withRunFixture` directly.
187+
export const runIt = {
188+
live: <A, E>(
189+
name: string,
190+
body: (input: RunFixture) => Effect.Effect<A, E>,
191+
opts?: number | TestOptions,
192+
) => it.live(name, () => withRunFixture(body), opts),
193+
}

‎packages/opencode/test/lib/test-provider.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// Shared provider config for tests that need opencode to talk to a fake LLM
2+
// over a real HTTP endpoint. Registers a single provider `test` with a single
3+
// model `test-model` (i.e. `--model test/test-model`), pointed at the URL the
4+
// caller supplies (typically a TestLLMServer instance).
5+
//
6+
// Used by:
7+
// - test/lib/run-process.ts (subprocess CLI tests)
8+
// - test/server/httpapi-sdk.test.ts (in-process SDK tests)
9+
export function testProviderConfig(llmUrl: string) {
10+
return {
11+
formatter: false,
12+
lsp: false,
13+
provider: {
14+
test: {
15+
name: "Test",
16+
id: "test",
17+
env: [],
18+
npm: "@ai-sdk/openai-compatible",
19+
models: {
20+
"test-model": {
21+
id: "test-model",
22+
name: "Test Model",
23+
attachment: false,
24+
reasoning: false,
25+
temperature: false,
26+
tool_call: true,
27+
release_date: "2025-01-01",
28+
limit: { context: 100_000, output: 10_000 },
29+
cost: { input: 0, output: 0 },
30+
options: {},
31+
},
32+
},
33+
options: { apiKey: "test-key", baseURL: llmUrl },
34+
},
35+
},
36+
}
37+
}

‎packages/opencode/test/server/httpapi-sdk.test.ts‎

Lines changed: 3 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import path from "path"
2323
import { resetDatabase } from "../fixture/db"
2424
import { disposeAllInstances, TestInstance, tmpdirScoped } from "../fixture/fixture"
2525
import { awaitWithTimeout, testEffect } from "../lib/effect"
26+
import { testProviderConfig } from "../lib/test-provider"
2627

2728
const noopBootstrap = Layer.succeed(InstanceBootstrap.Service, InstanceBootstrap.Service.of({ run: Effect.void }))
2829
const it = testEffect(
@@ -99,39 +100,6 @@ function authorization(username: string, password: string) {
99100
return `Basic ${Buffer.from(`${username}:${password}`).toString("base64")}`
100101
}
101102

102-
function providerConfig(url: string) {
103-
return {
104-
formatter: false,
105-
lsp: false,
106-
provider: {
107-
test: {
108-
name: "Test",
109-
id: "test",
110-
env: [],
111-
npm: "@ai-sdk/openai-compatible",
112-
models: {
113-
"test-model": {
114-
id: "test-model",
115-
name: "Test Model",
116-
attachment: false,
117-
reasoning: false,
118-
temperature: false,
119-
tool_call: true,
120-
release_date: "2025-01-01",
121-
limit: { context: 100000, output: 10000 },
122-
cost: { input: 0, output: 0 },
123-
options: {},
124-
},
125-
},
126-
options: {
127-
apiKey: "test-key",
128-
baseURL: url,
129-
},
130-
},
131-
},
132-
}
133-
}
134-
135103
function call<T>(request: () => Promise<T>) {
136104
return Effect.promise(request)
137105
}
@@ -283,7 +251,7 @@ function withStandardProject<A, E>(
283251
function withFakeLlm<A, E>(serverPath: ServerPath, run: (input: LlmProjectFixture) => Effect.Effect<A, E, TestScope>) {
284252
return Effect.gen(function* () {
285253
const llm = yield* TestLLMServer
286-
return yield* withProject(serverPath, { config: providerConfig(llm.url) }, (input) => run({ ...input, llm }))
254+
return yield* withProject(serverPath, { config: testProviderConfig(llm.url) }, (input) => run({ ...input, llm }))
287255
}).pipe(Effect.provide(TestLLMServer.layer))
288256
}
289257

@@ -297,7 +265,7 @@ function withFakeLlmProject<A, E>(
297265
return yield* withProject(
298266
serverPath,
299267
{
300-
config: providerConfig(llm.url),
268+
config: testProviderConfig(llm.url),
301269
setup: options.setup,
302270
},
303271
(input) => run({ ...input, llm }),

0 commit comments

Comments
 (0)