Skip to content

Commit 16a0a4b

Browse files
JavaZerooJavaZeroo
committed
feat: add deterministic benchmark workloads and Windows resource sampling
1 parent 8423264 commit 16a0a4b

15 files changed

Lines changed: 1037 additions & 18 deletions

File tree

‎CONTRIBUTING.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,11 @@
1414

1515
## 当前本地检查
1616

17-
当前仓库主要是文档和 JSON 模板，尚无客户端构建命令：
17+
当前已有 Python 基准工具，尚无客户端构建命令。安装 `benchmarks/requirements.txt` 后执行：
1818

1919
```sh
2020
python -m json.tool benchmarks/run-template.json
21+
python -m unittest discover -s benchmarks/tests -v
2122
git diff --check
2223
```
2324

‎benchmarks/README.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
# 性能基准
22

3-
已定义 [M0 实验协议 v0.1](PROTOCOL.md)、[机器可读场景参数](cases.json) 与 [记录模板](run-template.json)。工具及校准待任务 #2/#3 实现，尚无测量结果。执行以 PROTOCOL.md 为准，本页为导航摘要。
3+
已定义 [M0 实验协议 v0.1](PROTOCOL.md)、[机器可读场景参数](cases.json) 与 [记录模板](run-template.json)。[工具使用说明](tools/README.md) 包含已实现的生成器、播放器和 Windows 资源采样。#3 的画面/输入端点及正式开销校准仍未完成；尚无 WindTerm 实测结果。执行以 PROTOCOL.md 为准，本页为导航摘要。
4+
5+
[2026-10-20 工具自检记录](results/2026-10-20-tool-calibration/README.md) 保留了开销问题及修正后的诊断；校准门槛仍未通过。
46

57
## 测量矩阵
68

‎benchmarks/cases.json‎

Lines changed: 147 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"protocol_version": "0.1",
3-
"status": "specified_not_implemented",
3+
"status": "partially_implemented_not_measured",
44
"base_profile": {
55
"id": "C0",
66
"columns": 120,
@@ -25,19 +25,152 @@
2525
"input_timeout_seconds": 5,
2626
"settle_seconds_min": 30,
2727
"input_events_per_level": 200,
28-
"input_gap_ms": [250, 750]
28+
"input_gap_ms": [
29+
250,
30+
750
31+
]
2932
},
3033
"cases": [
31-
{"id": "T01", "kind": "ascii", "sizes_bytes": [8388608, 134217728], "variants": ["repeated", "pseudorandom"]},
32-
{"id": "T02", "kind": "ansi", "sizes_bytes": [8388608, 134217728], "style_interval_chars": 8, "palette_colors": 16},
33-
{"id": "T03", "kind": "unicode", "sizes_bytes": [8388608], "write_chunks_bytes": [65536, 1, 7, 4096]},
34-
{"id": "T04", "kind": "overwrite", "updates": 100000, "columns": 80},
35-
{"id": "T05", "kind": "screen_resize", "screen_switches": 1000, "resize_cycles": 100, "sizes_cells": [[120, 40], [80, 24]]},
36-
{"id": "H01", "kind": "history", "line_counts": [10000, 100000, 1000000], "scrollback_lines": 1000000, "variants": ["repeated", "pseudorandom"], "line_columns_max": 80, "search_repeats_per_target": 10, "scroll_repeats_per_target": 10},
37-
{"id": "I01", "kind": "input_latency", "background_bytes_per_second": [0, 1048576, 8388608]},
38-
{"id": "M01", "kind": "multi_session", "sessions": [1, 5, 20], "background_bytes_per_second_per_session": 1048576},
39-
{"id": "L01", "kind": "soak", "duration_seconds": 3600, "timeout_seconds": 3900, "bytes_per_second": 1048576},
40-
{"id": "L02", "kind": "lifecycle", "cycles": 20, "payload_bytes_per_cycle": 8388608, "settle_seconds_per_cycle": 30, "final_observation_seconds": 300},
41-
{"id": "F01", "kind": "sftp", "variants": [{"file_count": 1, "file_bytes": 1073741824}, {"file_count": 10000, "file_bytes": 4096}], "directions": ["upload", "download"], "warmups": 1, "repeats": 5}
42-
]
34+
{
35+
"id": "T01",
36+
"kind": "ascii",
37+
"sizes_bytes": [
38+
8388608,
39+
134217728
40+
],
41+
"variants": [
42+
"repeated",
43+
"pseudorandom"
44+
]
45+
},
46+
{
47+
"id": "T02",
48+
"kind": "ansi",
49+
"sizes_bytes": [
50+
8388608,
51+
134217728
52+
],
53+
"style_interval_chars": 8,
54+
"palette_colors": 16
55+
},
56+
{
57+
"id": "T03",
58+
"kind": "unicode",
59+
"sizes_bytes": [
60+
8388608
61+
],
62+
"write_chunks_bytes": [
63+
65536,
64+
1,
65+
7,
66+
4096
67+
]
68+
},
69+
{
70+
"id": "T04",
71+
"kind": "overwrite",
72+
"updates": 100000,
73+
"columns": 80
74+
},
75+
{
76+
"id": "T05",
77+
"kind": "screen_resize",
78+
"screen_switches": 1000,
79+
"resize_cycles": 100,
80+
"sizes_cells": [
81+
[
82+
120,
83+
40
84+
],
85+
[
86+
80,
87+
24
88+
]
89+
]
90+
},
91+
{
92+
"id": "H01",
93+
"kind": "history",
94+
"line_counts": [
95+
10000,
96+
100000,
97+
1000000
98+
],
99+
"scrollback_lines": 1000000,
100+
"variants": [
101+
"repeated",
102+
"pseudorandom"
103+
],
104+
"line_columns_max": 80,
105+
"search_repeats_per_target": 10,
106+
"scroll_repeats_per_target": 10
107+
},
108+
{
109+
"id": "I01",
110+
"kind": "input_latency",
111+
"background_bytes_per_second": [
112+
0,
113+
1048576,
114+
8388608
115+
]
116+
},
117+
{
118+
"id": "M01",
119+
"kind": "multi_session",
120+
"sessions": [
121+
1,
122+
5,
123+
20
124+
],
125+
"background_bytes_per_second_per_session": 1048576
126+
},
127+
{
128+
"id": "L01",
129+
"kind": "soak",
130+
"duration_seconds": 3600,
131+
"timeout_seconds": 3900,
132+
"bytes_per_second": 1048576
133+
},
134+
{
135+
"id": "L02",
136+
"kind": "lifecycle",
137+
"cycles": 20,
138+
"payload_bytes_per_cycle": 8388608,
139+
"settle_seconds_per_cycle": 30,
140+
"final_observation_seconds": 300
141+
},
142+
{
143+
"id": "F01",
144+
"kind": "sftp",
145+
"variants": [
146+
{
147+
"file_count": 1,
148+
"file_bytes": 1073741824
149+
},
150+
{
151+
"file_count": 10000,
152+
"file_bytes": 4096
153+
}
154+
],
155+
"directions": [
156+
"upload",
157+
"download"
158+
],
159+
"warmups": 1,
160+
"repeats": 5
161+
}
162+
],
163+
"tool_coverage": {
164+
"generator": [
165+
"T01",
166+
"T02",
167+
"T03",
168+
"T04",
169+
"H01"
170+
],
171+
"partial_generator": {
172+
"T05": "alternate-screen bytes only; resize/interactive acknowledgement pending"
173+
},
174+
"resources": "Windows process-tree polling implemented; presentation/input pending"
175+
}
43176
}

‎benchmarks/requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
psutil==7.2.2

‎benchmarks/results/2026-10-20-tool-calibration/README.md‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# 工具自检：2026-10-20
2+
3+
这是受控合成 CPU/内存负载的诊断记录，不是 WindTerm 基线，不代表终端吞吐、输入延迟或性能等价。工具尚未通过完整开销校准，#3 保持未完成。
4+
5+
## 检查范围
6+
7+
Python 3.12.10 / Windows 11 build 26200，psutil 7.2.2。自生成 worker 分配 64 MiB，并执行 20000 次 64 KiB SHA-256。一次预热、五对交错 on/off 运行。原始十个 worker 样本和工具源码 SHA-256 见 JSON。后台系统负载、电源/温度未控制到正式基准要求，因此不能将差异全部归因于采样。
8+
9+
| 指标 | 最初：每 100 ms 枚举子进程 | 改为每 1000 ms 发现子进程 |
10+
|---|---:|---:|
11+
| worker 开采样/关采样耗时中位数差异 | +12.15% | -5.65% |
12+
| 每次采样运行的采样器 CPU seconds 中位数 | 1.4375 | 0.09375 |
13+
| 每次运行内 collection_ns 中位数，再跨运行取中位数 | 48.12 ms | 0.24 ms |
14+
15+
资源读取仍为 100 ms。后者降低了枚举开销，但增加漏掉短命子进程的风险。采样器 CPU 时间为首次/最后一次观测之间的增量，不含完整启动与退出成本；collection_ns 不包含写盘，不能当作端到端开销。
16+
17+
负载时间差异绝对值均超过 5% 检查阈值。负向结果不是加速证明；该轮样本不支持“采样开销已合格”的结论。下一步需要控制环境并在 T01/实际终端路径下校准，不能继续挑选重复直到过线。
18+
19+
## 数据与重现
20+
21+
- [initial.json](initial.json)：最初诊断（子进程发现 100 ms）。
22+
- [decoupled.json](decoupled.json)：当前实现诊断（子进程发现 1000 ms）。
23+
- 执行：`python benchmarks/tools/calibrate.py --out benchmarks/local-results/<new-directory>`。
24+
- 初始版本来自本次开发过程中的修正前实现，源码身份记录在 JSON；当前工具不能在不改变参数/源码的情况下复现该旧 revision。对照参数差异明确保留，不将它作为严格产品 A/B。
25+
- JSONL 包含本地进程身份，保留于忽略目录；公开 JSON 仅有 worker 原始样本、采样器汇总、版本与源码哈希，未包含用户名、个人路径或连接数据。由于未公开逐条资源轨迹，公开读者可重算 worker 中位数，但不能独立重算采样器汇总；可用工具重新采集。
26+
27+
## 功能验证
28+
29+
8 项 unittest 通过：全部生成类型的确定性/哈希、独立文本 oracle、覆盖/备用屏幕序列结构、整记录边界/种子/防覆盖、部分写入/零进度、UTF-8 多种分块回放/损坏拒绝、生成器内存有界、Windows 受控子进程发现及 CPU/private bytes 变化。
30+
31+
8 MiB 伪随机 ASCII 负载实际生成 8,388,643 字节（包括完整记录与 marker），SHA-256 为 `ba1c11ddb887a6ec5a64de83fe54ad84eaa861d1e457c98e6491d10c03663181`。这是输入身份，不是终端正确性结果。
32+
33+
功能验证不包括实际终端呈现、完整 VT 兼容性、输入注入、capture、resize 驱动、SSH/SFTP 或长时间稳定性。

‎benchmarks/results/2026-10-20-tool-calibration/decoupled.json‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
{
2+
"kind": "synthetic_sampler_calibration",
3+
"samples": [
4+
{
5+
"elapsed_ns": 2389854600,
6+
"cpu_seconds": 2.375,
7+
"allocation_bytes": 67108864,
8+
"pair": 0,
9+
"sampler_enabled": false
10+
},
11+
{
12+
"elapsed_ns": 2647175800,
13+
"cpu_seconds": 2.53125,
14+
"allocation_bytes": 67108864,
15+
"pair": 0,
16+
"sampler_enabled": true
17+
},
18+
{
19+
"elapsed_ns": 2916522600,
20+
"cpu_seconds": 2.71875,
21+
"allocation_bytes": 67108864,
22+
"pair": 1,
23+
"sampler_enabled": true
24+
},
25+
{
26+
"elapsed_ns": 2736848400,
27+
"cpu_seconds": 2.671875,
28+
"allocation_bytes": 67108864,
29+
"pair": 1,
30+
"sampler_enabled": false
31+
},
32+
{
33+
"elapsed_ns": 2812220300,
34+
"cpu_seconds": 2.734375,
35+
"allocation_bytes": 67108864,
36+
"pair": 2,
37+
"sampler_enabled": false
38+
},
39+
{
40+
"elapsed_ns": 2475557800,
41+
"cpu_seconds": 2.453125,
42+
"allocation_bytes": 67108864,
43+
"pair": 2,
44+
"sampler_enabled": true
45+
},
46+
{
47+
"elapsed_ns": 2813582800,
48+
"cpu_seconds": 2.75,
49+
"allocation_bytes": 67108864,
50+
"pair": 3,
51+
"sampler_enabled": true
52+
},
53+
{
54+
"elapsed_ns": 2805566700,
55+
"cpu_seconds": 2.65625,
56+
"allocation_bytes": 67108864,
57+
"pair": 3,
58+
"sampler_enabled": false
59+
},
60+
{
61+
"elapsed_ns": 2882527600,
62+
"cpu_seconds": 2.6875,
63+
"allocation_bytes": 67108864,
64+
"pair": 4,
65+
"sampler_enabled": false
66+
},
67+
{
68+
"elapsed_ns": 2639383900,
69+
"cpu_seconds": 2.46875,
70+
"allocation_bytes": 67108864,
71+
"pair": 4,
72+
"sampler_enabled": true
73+
}
74+
],
75+
"python_version": "3.12.10",
76+
"os": "Windows-11-10.0.26200-SP0",
77+
"worker_source_sha256": "b944fe5ea5a983a86109542b521cc0cca8a353ebf135ef92fa73caf2aa025117",
78+
"sampler_source_sha256": "82fe0e9ceb01d4bafb300b7e0a4b7c1487596f44a70c1eef392e209f02ec8473",
79+
"calibrator_source_sha256": "ffc9a4f1ae3e457a52e758f338a7a770f4999d9479c122be6ebb37aeca4a3e2e",
80+
"median_sampler_off_ns": 2805566700,
81+
"median_sampler_on_ns": 2647175800,
82+
"median_elapsed_change_percent": -5.64559381176003,
83+
"above_protocol_5_percent_trigger": true,
84+
"limitations": [
85+
"Synthetic CPU/memory worker only; not T01, PTY, renderer or WindTerm.",
86+
"Five pairs are diagnostic, not a universal overhead bound.",
87+
"Sampling every 100 ms until worker exit (30 second cap); final CPU after exit may be missed.",
88+
"No capture/input tool or inter-process clock alignment calibrated."
89+
],
90+
"sampler_cpu_seconds_per_instrumented_run": [
91+
0.09375,
92+
0.125,
93+
0.046875,
94+
0.09375,
95+
0.0625
96+
],
97+
"median_collection_ns_per_instrumented_run": [
98+
200500.0,
99+
242500,
100+
276250.0,
101+
264400,
102+
224600.0
103+
],
104+
"discovery_interval_ms": 1000,
105+
"resource_interval_ms": 100,
106+
"interpretation": "Diagnostic only; overhead gate unresolved. Not a terminal benchmark."
107+
}

0 commit comments

Comments
 (0)