Repository navigation
Expand file tree
/
Copy pathdev-coding-task.mjs
More file actions
159 lines (145 loc) · 7.55 KB
/
Copy pathdev-coding-task.mjs
File metadata and controls
159 lines (145 loc) · 7.55 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
#!/usr/bin/env node
/**
* dev-coding-task — End-to-end coding task via AgentHarness + Grok OAuth.
*
* Runs a single coding task that exercises the full tool loop:
* 1. Grok receives the prompt + tool catalog
* 2. Grok decides to call write_file → AgentHarness executes → returns result
* 3. Grok decides to call read_file → AgentHarness executes → returns result
* 4. Grok produces a final summary message
*
* Working dir is /tmp/zelari-coding-test (created if missing, isolated
* from the project).
*/
import { mkdirSync, rmSync, existsSync, readFileSync } from 'node:fs';
import { join } from 'node:path';
import { AgentHarness } from '../dist/main/core/AgentHarness.js';
import { createBuiltinToolRegistry } from '../dist/cli/toolRegistry.js';
import { openaiCompatibleProvider, providerConfigFor } from '../dist/cli/provider/openai-compatible.js';
// ── Setup ───────────────────────────────────────────────────────────────
const WORKDIR = '/tmp/zelari-coding-test';
if (existsSync(WORKDIR)) {
rmSync(WORKDIR, { recursive: true, force: true });
}
mkdirSync(WORKDIR, { recursive: true });
process.chdir(WORKDIR);
const taskPrompt = process.argv.slice(2).join(' ') ||
`Create a file called index.html in the current working directory with the following content:
- A <!DOCTYPE html> document
- <html lang="en"> with <head> containing <title>Zelari Code Test</title>
- <body> containing an <h1>Zelari Code works</h1>, a <p> paragraph explaining that this file was generated by the Zelari Code CLI as an end-to-end smoke test, and a <a href="https://github.com/N-THEM-Studio/zelari-code">link to the repo</a>
Then use read_file to verify the file was written correctly, and end your reply with a one-line summary.`;
// ── Resolve Grok config + build harness ────────────────────────────────
const providerId = 'grok';
const providerCfg = await providerConfigFor(providerId);
if (!providerCfg) {
console.error(`\x1b[31m✗ No config for provider "${providerId}". Run /login grok first.\x1b[0m`);
process.exit(1);
}
console.log(`\x1b[36m[setup]\x1b[0m Provider: ${providerCfg.providerId} Model: ${providerCfg.model} baseUrl: ${providerCfg.baseUrl}`);
const providerStream = openaiCompatibleProvider(providerCfg);
const { registry: toolRegistry, tools } = createBuiltinToolRegistry({ root: WORKDIR });
const toolList = tools.map((t) => `- ${t.name}: ${t.description}`).join('\n');
const systemPrompt = `You are Zelari Code, an interactive AI coding agent running in a test harness.
You have real filesystem tools. Use them. Working directory: ${WORKDIR}
Available tools:
${toolList}
When done, give a one-line summary.`;
console.log(`\x1b[36m[setup]\x1b[0m Tools registered: ${tools.map(t => t.name).join(', ')}`);
console.log(`\x1b[36m[setup]\x1b[0m Working dir: ${WORKDIR}`);
console.log(`\x1b[36m[setup]\x1b[0m Task: ${taskPrompt.slice(0, 100)}${taskPrompt.length > 100 ? '…' : ''}\n`);
const harness = new AgentHarness({
model: providerCfg.model,
provider: providerCfg.providerId,
messages: [
{ role: 'system', content: systemPrompt },
{ role: 'user', content: taskPrompt },
],
tools: toolRegistry.toOpenAITools().map((t) => ({
name: t.function.name,
description: t.function.description,
parameters: t.function.parameters,
})),
toolRegistry,
providerStream,
});
// ── Stream events ──────────────────────────────────────────────────────
const ts = () => new Date().toISOString().slice(11, 23);
let textBuf = '';
let toolCallCount = 0;
let totalUsage = null;
const startMs = Date.now();
for await (const event of harness.run()) {
switch (event.type) {
case 'agent_start':
console.log(`\x1b[34m[${ts()}]\x1b[0m \x1b[1magent_start\x1b[0m session=${event.sessionId} model=${event.model ?? providerCfg.model}`);
break;
case 'message_start':
console.log(`\x1b[34m[${ts()}]\x1b[0m message_start messageId=${event.messageId}`);
textBuf = '';
break;
case 'message_delta':
textBuf += event.delta;
process.stdout.write(event.delta);
break;
case 'thinking_delta':
process.stdout.write(`\x1b[2m${event.delta}\x1b[0m`); // dimmed
break;
case 'message_end': {
const usage = event.usage;
if (usage) totalUsage = usage;
console.log(); // newline after streaming
console.log(`\x1b[34m[${ts()}]\x1b[0m message_end reason=${event.finishReason} length=${event.totalLength} usage=${usage ? `${usage.totalTokens}tok (${usage.promptTokens}+${usage.completionTokens})` : 'n/a'}`);
break;
}
case 'tool_execution_start': {
toolCallCount++;
const argsStr = JSON.stringify(event.args).slice(0, 140);
console.log(`\n\x1b[33m[${ts()}]\x1b[0m \x1b[1m→ tool_call #${toolCallCount}\x1b[0m ${event.toolName}(${argsStr}${argsStr.length >= 140 ? '…' : ''})`);
break;
}
case 'tool_execution_end': {
const preview = (event.result || '').replace(/\n/g, ' ⏎ ').slice(0, 180);
console.log(`\x1b[33m[${ts()}]\x1b[0m ← tool_result ${event.isError ? '\x1b[31m✗\x1b[0m' : '\x1b[32m✓\x1b[0m'} ${event.durationMs}ms ${preview}${preview.length >= 180 ? '…' : ''}`);
break;
}
case 'agent_end': {
const elapsed = ((Date.now() - startMs) / 1000).toFixed(1);
console.log(`\n\x1b[1m\x1b[36m[${ts()}]\x1b[0m \x1b[1magent_end\x1b[0m duration=${elapsed}s tool_calls=${toolCallCount}`);
if (totalUsage) {
console.log(`\x1b[36m[usage]\x1b[0m prompt=${totalUsage.promptTokens} completion=${totalUsage.completionTokens} total=${totalUsage.totalTokens}`);
}
break;
}
case 'error':
console.error(`\n\x1b[31m[${ts()}]\x1b[0m ERROR: ${event.message}`);
break;
default:
console.log(`\x1b[34m[${ts()}]\x1b[0m ${event.type}`);
}
}
// ── Post-check ─────────────────────────────────────────────────────────
const indexPath = join(WORKDIR, 'index.html');
console.log(`\n\x1b[1m\x1b[36m═══════════════════════════════════════════════════════════════════\x1b[0m`);
console.log(`\x1b[1m\x1b[36m POST-RUN VERIFICATION\x1b[0m`);
console.log(`\x1b[1m\x1b[36m═══════════════════════════════════════════════════════════════════\x1b[0m`);
if (existsSync(indexPath)) {
const content = readFileSync(indexPath, 'utf8');
console.log(`\x1b[32m✓\x1b[0m ${indexPath} exists (${content.length} bytes)`);
console.log('---');
console.log(content);
console.log('---');
const checks = [
['<!DOCTYPE html>', '<!DOCTYPE html>'],
['<title>Zelari Code Test</title>', '<title>Zelari Code Test</title>'],
['<h1>Zelari Code works</h1>', '<h1>Zelari Code works</h1>'],
['GitHub link', 'github.com/N-THEM-Studio/zelari-code'],
];
for (const [name, needle] of checks) {
const present = content.includes(needle);
console.log(` ${present ? '\x1b[32m✓\x1b[0m' : '\x1b[31m✗\x1b[0m'} ${name}`);
}
} else {
console.log(`\x1b[31m✗\x1b[0m ${indexPath} does NOT exist — agent did not create the file`);
process.exit(1);
}