Repository navigation
Expand file tree
/
Copy pathvalidate.py
More file actions
1011 lines (880 loc) · 38.2 KB
/
Copy pathvalidate.py
File metadata and controls
1011 lines (880 loc) · 38.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env -S uv run --script
# /// script
# requires-python = ">=3.11"
# dependencies = ["numpy>=1.26"]
# ///
"""
validate.py - convention-driven numerical validation for speech.cpp's transcribe
runtime (the unified ABI C ABI ported from transcribe.cpp and wired through the
ArchAdapter into audio.cpp's model sessions).
Orchestrates the full validation flow: generate reference dumps, generate
C++ dumps, and compare tensors. All paths are derived from convention;
the only required input is the family name.
Usage:
uv run scripts/validate.py ref --family cohere
uv run scripts/validate.py cpp --family cohere
uv run scripts/validate.py compare --family cohere
uv run scripts/validate.py all --family cohere
# Override model source (local path instead of HF download)
uv run scripts/validate.py all --family cohere --model /local/path
# Override GGUF path
uv run scripts/validate.py cpp --family cohere --gguf models/cohere-transcribe-03-2026/cohere-transcribe-03-2026-BF16.gguf
Conventions:
Manifest: tests/golden/{family}/*.manifest.json
Dump script: manifest reference.entrypoint
Python env: scripts/envs/{family}/
Tolerances: tests/tolerances/{family}.json
Audio: samples/jfk.wav
GGUF: models/{slug}/ where slug starts with {family}
(prefers *-BF16.gguf > *-F32.gguf > *-F16.gguf > first match)
Ref output: build/validate/{family}/{variant}/{case}/ref/
C++ output: build/validate/{family}/{variant}/{case}/cpp/
"""
from __future__ import annotations
import argparse
import datetime as dt
import glob
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any
def normalize_text(text: str) -> str:
return " ".join(text.strip().lower().split())
def normalize_text_for_compare(text: str) -> str:
return " ".join(re.sub(r"[^\w\s]+", " ", text.lower()).split())
def find_repo_root(start: Path) -> Path:
p = start.resolve()
while p != p.parent:
if (p / "CMakeLists.txt").exists() and (p / "scripts").is_dir():
return p
p = p.parent
raise SystemExit("error: cannot locate speech.cpp repo root")
def load_manifest(
repo: Path, family: str, variant: str | None = None
) -> dict[str, Any]:
manifest_dir = repo / "tests" / "golden" / family
if variant:
path = manifest_dir / f"{variant}.manifest.json"
if not path.exists():
raise SystemExit(f"error: manifest not found: {path}")
return json.loads(path.read_text())
pattern = manifest_dir / "*.manifest.json"
matches = sorted(glob.glob(str(pattern)))
if not matches:
raise SystemExit(
f"error: no manifest found for family '{family}' "
f"(looked in tests/golden/{family}/)"
)
if len(matches) > 1:
names = [Path(m).stem.replace(".manifest", "") for m in matches]
raise SystemExit(
f"error: multiple manifests for '{family}': {names}\n"
f" Use --variant to pick one"
)
return json.loads(Path(matches[0]).read_text())
def manifest_source_model(manifest: dict[str, Any]) -> str:
source_model = manifest.get("source_model")
if not isinstance(source_model, dict):
raise SystemExit("error: manifest missing source_model object")
hf_repo = source_model.get("hf_repo")
if not hf_repo:
raise SystemExit("error: manifest missing source_model.hf_repo")
return str(hf_repo)
def manifest_reference(manifest: dict[str, Any]) -> dict[str, Any]:
reference = manifest.get("reference")
if not isinstance(reference, dict):
raise SystemExit("error: manifest missing reference object")
if not reference.get("kind"):
raise SystemExit("error: manifest missing reference.kind")
if not reference.get("source"):
raise SystemExit("error: manifest missing reference.source")
if not reference.get("entrypoint"):
raise SystemExit("error: manifest missing reference.entrypoint")
return reference
def manifest_dump_script(repo: Path, manifest: dict[str, Any]) -> Path:
reference = manifest_reference(manifest)
script = repo / str(reference["entrypoint"])
if not script.exists():
raise SystemExit(f"error: reference entrypoint not found: {script}")
return script
def case_audio(case) -> str:
"""A case is either a bare string (legacy) or a dict with at least
{audio: <stem>}. Returns the audio stem."""
if isinstance(case, str):
return case
if isinstance(case, dict) and "audio" in case:
return str(case["audio"])
raise SystemExit(f"error: malformed case in manifest: {case!r}")
def case_language(case) -> str | None:
"""Per-case language code.
Legacy bare-string cases and dict cases that omit the field keep the
historical default of "en". Dict cases may set language to "auto",
"detect", or null to exercise the model's native language-detection path;
callers should omit the language flag in that case.
"""
if isinstance(case, dict):
value = case.get("language", "en")
if value is None:
return None
language = str(value)
if language == "" or language.lower() in {"auto", "detect"}:
return None
return language
return "en"
def case_transcript_compare(manifest: dict[str, Any], case) -> str:
value = manifest.get("transcript_compare", "exact")
if isinstance(case, dict) and "transcript_compare" in case:
value = case["transcript_compare"]
if value is None:
value = "exact"
mode = str(value).lower()
if mode not in {"exact", "normalized", "dediarized"}:
raise SystemExit(
f"error: unsupported transcript_compare={value!r}; "
"expected 'exact', 'normalized', or 'dediarized'"
)
return mode
def dediarize_text(text: str) -> str:
"""Strip inline [start]/[Sxx]/[end] bracket spans from an emergent-diarization
transcript, then normalize. Retained for historical validation artifacts and
reference raw-text fields; current C++ `full_text` is already marker-free.
Mirrors the WER harness de-diarization."""
stripped = re.sub(r"\[[^\]]*\]", " ", text)
return normalize_text_for_compare(stripped)
def find_gguf(repo: Path, family: str, slug: str | None = None,
variant: str | None = None) -> Path:
"""Find a GGUF under models/.
Discovery order:
1. If `variant` is provided and `models/<variant>/<variant>-<Q>.gguf`
exists, prefer it. This disambiguates families like gigaam where
multiple manifest variants share a single hf_repo (and therefore
a single hf-derived slug) but ship as separate per-variant GGUF
directories.
2. If `slug` is provided (derived from the manifest's
source_model.hf_repo, e.g. "Qwen3-ASR-0.6B" from
"Qwen/Qwen3-ASR-0.6B"), look for
models/<slug>/<slug>-<QUANT>.gguf directly. This is the
converter's output convention and is case-accurate — which
matters for families whose HF slug does not case-fold to the
family key (e.g. family="qwen3_asr", slug="Qwen3-ASR-0.6B").
3. Legacy fallback: scan models/*/ for any GGUF whose stem
starts with `family`. Kept so older manifests (or manual
layouts) still work.
Preferred quant order: BF16 > F32 > F16 > first match. If multiple
variants match the fallback, the first sort-order wins; use
--gguf to pick explicitly.
"""
model_root = repo / "models"
if not model_root.is_dir():
raise SystemExit(f"error: model root not found: {model_root}")
preferred_quants = ["BF16", "F32", "F16"]
def lookup_in(name: str) -> Path | None:
variant_dir = model_root / name
if not variant_dir.is_dir():
return None
for quant in preferred_quants:
candidate = variant_dir / f"{name}-{quant}.gguf"
if candidate.exists():
return candidate
matches = sorted(variant_dir.glob(f"{name}-*.gguf"))
return matches[0] if matches else None
# 1. Manifest-variant-driven lookup (handles multi-variant families
# sharing a single hf_repo, e.g. gigaam).
if variant:
found = lookup_in(variant)
if found is not None:
return found
# 2. Manifest-slug-driven lookup.
if slug:
found = lookup_in(slug)
if found is not None:
return found
# 2. Legacy family-prefix fallback.
def for_family(paths: list[Path]) -> list[Path]:
return [p for p in paths if p.stem.startswith(family)]
for quant in preferred_quants:
matches = for_family(sorted(model_root.glob(f"*/*-{quant}.gguf")))
if matches:
return matches[0]
matches = for_family(sorted(model_root.glob("*/*.gguf")))
if matches:
return matches[0]
hint = f" (manifest slug '{slug}' also matched nothing)" if slug else ""
raise SystemExit(
f"error: no GGUF files found under {model_root} for family "
f"'{family}'{hint}.\n"
f" Convert a GGUF first or set --gguf."
)
def find_cli(repo: Path) -> Path:
# In speech.cpp, the Phase 4 transcribe-cli target produces the validation-
# hook-aware CLI. Until it lands, audiocpp_cli (the audio.cpp CLI) is the
# only built binary — it has a different arg set, so cmd_cpp will fail with
# an unrecognized-flag error rather than producing a transcript.
for candidate in [
repo / "build" / "bin" / "transcribe-cli",
repo / "build" / "transcribe-cli",
repo / "build" / "bin" / "audiocpp_cli",
repo / "build" / "audiocpp_cli",
]:
if candidate.exists():
return candidate
raise SystemExit(
"error: no CLI found in build/bin/ or build/.\n"
" The Phase 4 transcribe-cli carries validation hooks (TRANSCRIBE_DUMP_DIR,\n"
" --backend, --timestamps, --raw-tokens). Build it with:\n"
" cmake --build build --target transcribe-cli -DTRANSCRIBE_ENABLE_VALIDATION_HOOKS=ON\n"
" Until Phase 4 lands, audiocpp_cli (audio.cpp CLI) is available but does\n"
" not support the validate.py argument set."
)
def validation_hooks_enabled(repo: Path) -> bool:
"""True if build/ was configured with the validation hooks compiled in.
Reads build/CMakeCache.txt next to the CLI that find_cli() runs. Used to
hard-fail --mel-from-ref against a build that would otherwise silently
ignore it (the hook code is compiled out unless the flag is ON).
"""
cache = repo / "build" / "CMakeCache.txt"
try:
for line in cache.read_text().splitlines():
if line.startswith("TRANSCRIBE_ENABLE_VALIDATION_HOOKS:"):
return line.rstrip().endswith("=ON")
except OSError:
pass
return False
def run_cmd(cmd: list[str], repo: Path, label: str) -> None:
print(f"\n{'=' * 60}", file=sys.stderr)
print(f" {label}", file=sys.stderr)
print(f" {' '.join(cmd)}", file=sys.stderr)
print(f"{'=' * 60}", file=sys.stderr)
env = os.environ.copy()
# Prevent uv from inheriting an active venv.
if len(cmd) >= 2 and cmd[0] == "uv" and cmd[1] == "run":
env.pop("VIRTUAL_ENV", None)
result = subprocess.run(cmd, cwd=repo, env=env)
if result.returncode != 0:
raise SystemExit(
f"error: {label} failed with exit code {result.returncode}"
)
def parse_cli_transcript(output: str) -> str | None:
for line in reversed(output.splitlines()):
if line.startswith("text: "):
return line[len("text: "):]
return None
def write_cpp_transcript(
out_dir: Path,
*,
family: str,
variant: str,
case: str,
gguf: Path,
backend: str,
text: str,
) -> None:
path = out_dir / "transcript.json"
payload = {
"schema": "transcribe-cpp-transcript-v1",
"family": family,
"variant": variant,
"case": case,
"text": text,
"normalized_text": normalize_text(text),
"source": {
"kind": "speech.cpp",
"gguf": str(gguf),
"backend": backend,
},
}
path.write_text(json.dumps(payload, indent=2) + "\n")
print(f" wrote {path}", file=sys.stderr)
def cmd_ref(args: argparse.Namespace) -> int:
repo = find_repo_root(Path(__file__).parent)
manifest = load_manifest(repo, args.family, getattr(args, "variant", None))
variant = manifest["variant"]
reference = manifest_reference(manifest)
model = args.model or manifest_source_model(manifest)
if not model:
raise SystemExit("error: no model specified and none in manifest")
dump_script = manifest_dump_script(repo, manifest)
env_dir = repo / "scripts" / "envs" / args.family
cases = manifest.get("cases", ["jfk"])
for case in cases:
case_name = case_audio(case)
language = case_language(case)
audio = repo / "samples" / f"{case_name}.wav"
if not audio.exists():
raise SystemExit(f"error: audio not found: {audio}")
out_dir = repo / "build" / "validate" / args.family / variant / case_name / "ref"
if out_dir.exists():
shutil.rmtree(out_dir)
out_dir.mkdir(parents=True)
base_args = [
"uv", "run", "--project", str(env_dir),
str(dump_script),
]
common_args = [
"--model", str(model),
"--audio", str(audio),
"--out", str(out_dir),
"--torch-threads", "1",
]
if language is not None:
common_args += ["--language", language]
# Pin the HF revision when the manifest declares one and the
# family's dumper accepts --revision. The dumper itself ignores
# --revision when --model resolves to a local directory.
hf_revision = (manifest.get("source_model") or {}).get("hf_revision")
if hf_revision and args.family in ("qwen3_asr", "granite_nar"):
common_args += ["--revision", str(hf_revision)]
# Forward any manifest-declared dumper args verbatim. Used today
# for parakeet-unified-en-0.6b, which needs --offline-only to
# force att_context_style='regular' (its v1 C++ port targets
# offline / full-context mode). Cache-aware streaming variants
# like nemotron-speech-streaming-en-0.6b deliberately omit this
# so their native chunked_limited style is preserved.
extra_dump_args = reference.get("dump_args") or []
if not isinstance(extra_dump_args, list) or not all(
isinstance(a, str) for a in extra_dump_args
):
raise SystemExit(
f"error: manifest reference.dump_args must be a list of strings; "
f"got {extra_dump_args!r}"
)
common_args += list(extra_dump_args)
# Run both encoder and decode subcommands. Some families dump
# everything from decode (cohere); others split encoder and
# decoder intermediates across subcommands (parakeet). Running
# both is safe — if a tensor is dumped by both, the decode
# pass overwrites the encoder pass (same values).
#
# Sortformer is an encoder-diarizer: the offline forward (encoder
# subcommand) emits the offline gate tensors; the streaming `diarize`
# subcommand emits diar.probs (AOSC/FIFO path). VALIDATE_SORTFORMER_PRESET
# selects a matching streaming operating point on both the reference
# (--preset) and the C++ side (TRANSCRIBE_SORTFORMER_STREAM_PRESET in
# cmd_cpp); unset -> the checkpoint-shipped cfg (single chunk on the
# short oracle, i.e. diar.probs == diar.preds_offline).
if args.family == "sortformer":
stages = ["encoder", "diarize"]
else:
stages = ["encoder", "decode"]
sf_preset = os.environ.get("VALIDATE_SORTFORMER_PRESET")
for stage in stages:
stage_args = list(common_args)
if args.family == "sortformer" and stage == "diarize" and sf_preset:
stage_args += ["--preset", sf_preset]
cmd = base_args + [stage] + stage_args
run_cmd(
cmd,
repo,
f"ref {stage} [{args.family}/{variant}/{case_name}/{reference['kind']}]",
)
return 0
def cmd_cpp(args: argparse.Namespace) -> int:
repo = find_repo_root(Path(__file__).parent)
manifest = load_manifest(repo, args.family, getattr(args, "variant", None))
variant = manifest["variant"]
cli = find_cli(repo)
slug = manifest_source_model(manifest).split("/", 1)[-1]
gguf = Path(args.gguf) if args.gguf else find_gguf(
repo, args.family, slug, variant=variant)
cases = manifest.get("cases", ["jfk"])
for case in cases:
case_name = case_audio(case)
language = case_language(case)
audio = repo / "samples" / f"{case_name}.wav"
if not audio.exists():
raise SystemExit(f"error: audio not found: {audio}")
out_dir = repo / "build" / "validate" / args.family / variant / case_name / "cpp"
if out_dir.exists():
shutil.rmtree(out_dir)
out_dir.mkdir(parents=True)
env = os.environ.copy()
env["TRANSCRIBE_DUMP_DIR"] = str(out_dir)
# Sortformer: keep the C++ streaming operating point in lockstep with
# the reference `diarize --preset` (see cmd_ref) so the diar.probs
# tensors are comparable. Also enable the offline full-context forward
# so the enc.* / diar.preds_offline parity tensors are dumped (it is
# gated off by default because it is O(T^2) over the whole clip and
# would OOM on long DER audio).
if args.family == "sortformer":
env["TRANSCRIBE_SORTFORMER_OFFLINE_DUMP"] = "1"
sf_preset = os.environ.get("VALIDATE_SORTFORMER_PRESET")
if sf_preset:
env["TRANSCRIBE_SORTFORMER_STREAM_PRESET"] = sf_preset
# Whisper: by default, exercise the production C++ MelFrontend so
# the per-tensor compare covers the full mel→encoder→decoder
# pipeline. The env-var ref-mel injection is preserved as an
# opt-in debug knob (--mel-from-ref): when a regression fires,
# re-running with --mel-from-ref isolates whether the drift
# originates in the C++ mel or downstream in the graph. Defaulting
# to ref-mel hid a base.en regression once (the mel-precision
# change in 4613129); we don't want that blind spot back.
if args.family == "whisper" and getattr(args, "mel_from_ref", False):
if not validation_hooks_enabled(repo):
raise SystemExit(
"error: --mel-from-ref requires validation hooks compiled "
"in, but build/ has TRANSCRIBE_ENABLE_VALIDATION_HOOKS=OFF "
"(the hook is compiled out and would be silently ignored).\n"
" Reconfigure: cmake -B build -D SPEECHCPP_ENABLE_UNIFIED_ABI=ON "
"-D TRANSCRIBE_ENABLE_VALIDATION_HOOKS=ON && "
"cmake --build build --target transcribe-cli"
)
ref_dir = repo / "build" / "validate" / args.family / variant / case_name / "ref"
env["TRANSCRIBE_MEL_FROM_REF"] = str(ref_dir)
cmd = [
str(cli),
"--backend", args.backend,
# Default 1 thread for the documented reproducible gate; ggml CPU
# parallelizes over independent output rows (full per-element
# reductions), so VALIDATE_CPP_THREADS>1 only speeds the dump up —
# the compared tensors are bit-identical.
"--threads", os.environ.get("VALIDATE_CPP_THREADS", "1"),
"-m", str(gguf),
]
if language is not None:
cmd += ["--language", language]
if args.family == "whisper":
cmd += ["--timestamps", "none"]
if args.family in ("sensevoice", "parakeet"):
# The reference dumper emits the raw token stream including
# control / language tags (sensevoice: language / event /
# emotion / itn; multilingual parakeet: the <xx-XX> language
# tag from auto language detection). The production C++ path
# strips these by default, so the validate dump must pass
# --raw-tokens to keep them and match the reference exactly.
cmd += ["--raw-tokens"]
cmd.append(str(audio))
print(f"\n{'=' * 60}", file=sys.stderr)
print(f" cpp dump [{args.family}/{case_name}]", file=sys.stderr)
print(f" TRANSCRIBE_DUMP_DIR={out_dir}", file=sys.stderr)
print(f" {' '.join(cmd)}", file=sys.stderr)
print(f"{'=' * 60}", file=sys.stderr)
result = subprocess.run(
cmd,
cwd=repo,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
errors="replace",
)
if result.stdout:
print(result.stdout, end="")
if result.returncode != 0:
raise SystemExit(
f"error: cpp dump [{args.family}/{case_name}] failed "
f"with exit code {result.returncode}"
)
transcript = parse_cli_transcript(result.stdout or "")
if transcript is None:
raise SystemExit(
f"error: cpp dump [{args.family}/{case_name}] did not emit a transcript line"
)
write_cpp_transcript(
out_dir,
family=args.family,
variant=variant,
case=case_name,
gguf=gguf,
backend=args.backend,
text=transcript,
)
return 0
def cmd_compare(args: argparse.Namespace) -> int:
repo = find_repo_root(Path(__file__).parent)
manifest = load_manifest(repo, args.family, getattr(args, "variant", None))
variant = manifest["variant"]
# Prefer per-variant tolerances declared in the manifest; fall
# back to the family default. This lets larger variants (e.g.
# Qwen3-ASR-1.7B vs 0.6B) carry scaled limits without stomping on
# the family file.
manifest_tol = manifest.get("tolerance_file")
if manifest_tol:
tolerances = repo / manifest_tol
else:
tolerances = repo / "tests" / "tolerances" / f"{args.family}.json"
if not tolerances.exists():
print(
f"warning: no tolerance file at {tolerances}, using defaults",
file=sys.stderr,
)
tolerances = None
compare_script = repo / "scripts" / "compare_tensors.py"
report_mode = getattr(args, "report", False)
cases = manifest.get("cases", ["jfk"])
all_passed = True
compare_outputs: list[dict[str, Any]] = []
transcript_results: list[dict[str, Any]] = []
cmd_log: list[dict[str, Any]] = []
for case in cases:
case_name = case_audio(case)
cpp_dir = repo / "build" / "validate" / args.family / variant / case_name / "cpp"
ref_dir = repo / "build" / "validate" / args.family / variant / case_name / "ref"
if not cpp_dir.exists():
print(f"SKIP {case_name}: no C++ dumps at {cpp_dir}", file=sys.stderr)
all_passed = False
continue
if not ref_dir.exists():
print(f"SKIP {case_name}: no reference dumps at {ref_dir}", file=sys.stderr)
all_passed = False
continue
cmd = [
"uv", "run", str(compare_script),
str(cpp_dir), str(ref_dir),
]
if tolerances:
cmd += ["--tolerances", str(tolerances)]
print(f"\n{'=' * 60}", file=sys.stderr)
print(f" compare [{args.family}/{case_name}]", file=sys.stderr)
print(f"{'=' * 60}", file=sys.stderr)
if report_mode:
result = subprocess.run(cmd, cwd=repo, capture_output=True, text=True)
if result.stdout:
print(result.stdout)
if result.stderr:
print(result.stderr, file=sys.stderr)
compare_outputs.append({
"case": case_name,
"returncode": result.returncode,
"stdout": result.stdout or "",
"stderr": result.stderr or "",
})
else:
result = subprocess.run(cmd, cwd=repo)
cmd_log.append({"case": case_name, "cmd": cmd, "returncode": result.returncode})
if result.returncode != 0:
all_passed = False
# Transcript comparison: if the reference produced a transcript.json,
# verify the C++ transcript. Manifests can opt into normalized compare
# for models whose generation differs only in punctuation/casing.
#
# Sortformer is a diarizer: it has no text transcript (the C++ CLI
# emits speaker segments, not `text`). Its behavioral artifact is the
# diar.probs tensor, gated above; the `diarize` stage's segment lines
# are informational only, so skip the text-transcript comparison.
ref_transcript = ref_dir / "transcript.json"
if ref_transcript.exists() and args.family != "sortformer":
transcript_compare = case_transcript_compare(manifest, case)
ref_data = json.loads(ref_transcript.read_text())
ref_text = str(ref_data.get("text", ""))
cpp_transcript = cpp_dir / "transcript.json"
if not cpp_transcript.exists():
print(
f"FAIL transcript: missing C++ transcript artifact: {cpp_transcript}",
file=sys.stderr,
)
all_passed = False
transcript_results.append({
"case": case_name, "match": False,
"reason": "missing C++ transcript artifact",
})
continue
cpp_data = json.loads(cpp_transcript.read_text())
cpp_text = str(cpp_data.get("text", ""))
if transcript_compare == "exact":
ref_compare = ref_text
cpp_compare = cpp_text
elif transcript_compare == "dediarized":
ref_compare = dediarize_text(ref_text)
cpp_compare = dediarize_text(cpp_text)
else:
ref_compare = normalize_text_for_compare(ref_text)
cpp_compare = normalize_text_for_compare(cpp_text)
match = cpp_compare == ref_compare
transcript_results.append({
"case": case_name, "match": match,
"reference": ref_text, "cpp": cpp_text,
"mode": transcript_compare,
})
if not match:
print(f"\nFAIL transcript mismatch ({transcript_compare})")
print(f" reference: {ref_text!r}")
print(f" c++: {cpp_text!r}")
if transcript_compare != "exact":
print(f" reference normalized: {ref_compare!r}")
print(f" c++ normalized: {cpp_compare!r}")
all_passed = False
else:
print(f"\n Transcript: ok ({transcript_compare}) {cpp_text!r}")
if report_mode:
write_report_bundle(
repo=repo,
family=args.family,
variant=variant,
compare_outputs=compare_outputs,
transcript_results=transcript_results,
cmd_log=cmd_log,
overall_passed=all_passed,
)
return 0 if all_passed else 1
def cmd_mel(args: argparse.Namespace) -> int:
repo = find_repo_root(Path(__file__).parent)
manifest = load_manifest(repo, args.family, getattr(args, "variant", None))
variant = manifest["variant"]
if args.family != "whisper":
print("mel parity is currently only defined for whisper", file=sys.stderr)
return 0
cli = find_cli(repo)
slug = manifest_source_model(manifest).split("/", 1)[-1]
gguf = Path(args.gguf) if getattr(args, "gguf", None) else find_gguf(
repo, args.family, slug, variant=variant)
compare_script = repo / "scripts" / "compare_tensors.py"
cases = manifest.get("cases", ["jfk"])
all_passed = True
for case in cases:
case_name = case_audio(case)
language = case_language(case)
audio = repo / "samples" / f"{case_name}.wav"
if not audio.exists():
raise SystemExit(f"error: audio not found: {audio}")
out_dir = repo / "build" / "validate" / args.family / variant / case_name / "mel_cpp"
if out_dir.exists():
shutil.rmtree(out_dir)
out_dir.mkdir(parents=True)
env = os.environ.copy()
env["TRANSCRIBE_DUMP_DIR"] = str(out_dir)
env.pop("TRANSCRIBE_MEL_FROM_REF", None)
cmd = [
str(cli),
"--backend", getattr(args, "backend", "cpu"),
"--threads", "1",
"-m", str(gguf),
"--timestamps", "none",
str(audio),
]
if language is not None:
cmd[-1:-1] = ["--language", language]
print(f"\n{'=' * 60}", file=sys.stderr)
print(f" mel parity cpp dump [{args.family}/{case_name}]", file=sys.stderr)
print(f" TRANSCRIBE_DUMP_DIR={out_dir}", file=sys.stderr)
print(f" {' '.join(cmd)}", file=sys.stderr)
print(f"{'=' * 60}", file=sys.stderr)
result = subprocess.run(
cmd, cwd=repo, env=env,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
)
if result.stdout:
print(result.stdout, end="")
if result.returncode != 0:
all_passed = False
continue
ref_dir = repo / "build" / "validate" / args.family / variant / case_name / "ref"
with tempfile.TemporaryDirectory() as td:
tol = Path(td) / "mel-tolerances.json"
# fp32 STFT path: worst observed drift on librispeech-style
# speech is 3.4e-4 max_abs (german) at peak signal ~1.4 →
# ~2.5e-4 relative. 5e-4 leaves headroom for new clips
# without crossing into territory that would shift WER.
tol.write_text(json.dumps({
"enc.mel.in": {"max_abs": 5e-4, "mean_abs": 5e-6},
}) + "\n")
cmp_cmd = [
"uv", "run", str(compare_script),
str(out_dir), str(ref_dir),
"--max-abs", "1e9",
"--mean-abs", "1e9",
"--tolerances", str(tol),
"--quiet",
]
print(f"\n{'=' * 60}", file=sys.stderr)
print(f" mel parity compare [{args.family}/{case_name}]", file=sys.stderr)
print(f"{'=' * 60}", file=sys.stderr)
cmp = subprocess.run(cmp_cmd, cwd=repo)
if cmp.returncode != 0:
all_passed = False
return 0 if all_passed else 1
def git_head_sha(repo: Path) -> str:
try:
result = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=repo, capture_output=True, text=True, check=True,
)
return result.stdout.strip()
except (subprocess.CalledProcessError, FileNotFoundError):
return "unknown"
def write_report_bundle(
*,
repo: Path,
family: str,
variant: str,
compare_outputs: list[dict[str, Any]],
transcript_results: list[dict[str, Any]],
cmd_log: list[dict[str, Any]],
overall_passed: bool,
) -> Path:
"""Write an ephemeral validation report bundle.
The bundle captures what's not reproducible from git alone: the compare
stdout, exact command invocations, and transcript comparison at this
moment. The manifest, tolerance file, and compare_tensors.py live in the
repo — pin them via `validated_at_sha` (the git HEAD at bundle time)
rather than duplicating them here.
"""
now = dt.datetime.now(dt.UTC)
ts = now.strftime("%Y%m%dT%H%M%SZ")
sha = git_head_sha(repo)
bundle_id = f"{ts}-{sha[:7]}" if sha != "unknown" else ts
bundle_dir = repo / "reports" / "porting" / family / variant / bundle_id
bundle_dir.mkdir(parents=True, exist_ok=True)
(bundle_dir / "commands.json").write_text(
json.dumps({"validated_at_sha": sha, "commands": cmd_log}, indent=2) + "\n"
)
summary = [
f"# Validation Report — {family}/{variant}",
"",
f"- Generated: {now.isoformat().replace('+00:00', 'Z')}",
f"- Repo SHA: `{sha}`",
f"- Overall: **{'PASS' if overall_passed else 'FAIL'}**",
"",
]
for co in compare_outputs:
summary += [
f"## compare_tensors — {co['case']}",
"",
f"Exit code: `{co['returncode']}`",
"",
"```",
co["stdout"].rstrip() or "(no stdout)",
"```",
"",
]
if transcript_results:
summary += ["## Transcript comparison", ""]
for tr in transcript_results:
summary.append(f"### {tr['case']}")
summary.append("")
if tr["match"]:
summary.append(f"- Match: **yes**")
summary.append(f"- Mode: `{tr.get('mode', 'exact')}`")
summary.append(f"- text: `{tr.get('cpp', '')!r}`")
else:
summary.append(f"- Match: **no**")
summary.append(f"- Mode: `{tr.get('mode', 'exact')}`")
if "reason" in tr:
summary.append(f"- Reason: {tr['reason']}")
else:
summary.append(f"- reference: `{tr.get('reference', '')!r}`")
summary.append(f"- c++: `{tr.get('cpp', '')!r}`")
summary.append("")
(bundle_dir / "summary.md").write_text("\n".join(summary))
repro = f"""# Reproducing this validation run
Check out repo at SHA `{sha}`, then run:
```bash
uv run scripts/validate.py all --family {family} --variant {variant}
```
Or step by step:
```bash
uv run scripts/validate.py ref --family {family} --variant {variant}
uv run scripts/validate.py cpp --family {family} --variant {variant}
uv run scripts/validate.py compare --family {family} --variant {variant}
```
Inputs (all version-controlled; pinned by `validated_at_sha` in commands.json):
- Golden manifest: `tests/golden/{family}/{variant}.manifest.json`
- Tolerance file: `tests/tolerances/{family}.json`
- Comparator: `scripts/compare_tensors.py`
Bundle contents are ephemeral evidence — the compare stdout and transcript
check at validation time. The validated commit SHA is authoritative;
snapshots of repo-tracked files are intentionally omitted.
"""
(bundle_dir / "reproduce.md").write_text(repro)
print(f"\nReport bundle: {bundle_dir.relative_to(repo)}", file=sys.stderr)
return bundle_dir
def cmd_all(args: argparse.Namespace) -> int:
rc = cmd_ref(args)
if rc != 0:
return rc
if args.family == "whisper":
rc = cmd_mel(args)
if rc != 0:
return rc
rc = cmd_cpp(args)
if rc != 0:
return rc
return cmd_compare(args)
def main() -> int:
p = argparse.ArgumentParser(
description="Convention-driven numerical validation for speech.cpp's transcribe C ABI runtime.",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
sub = p.add_subparsers(dest="cmd", required=True)
# Shared arguments.
def add_common(sp: argparse.ArgumentParser) -> None:
sp.add_argument(
"--family", required=True,
help="Model family (e.g. parakeet, cohere)",
)
sp.add_argument(
"--variant",
help="Model variant (e.g. cohere-transcribe-03-2026). "
"Required when the family has multiple manifests.",
)
# ref
sp_ref = sub.add_parser("ref", help="Generate reference dumps")
add_common(sp_ref)
sp_ref.add_argument(
"--model",
help="HF model ID or local path (overrides manifest default)",
)
sp_ref.set_defaults(func=cmd_ref)
# cpp
sp_cpp = sub.add_parser("cpp", help="Generate C++ dumps")
add_common(sp_cpp)
sp_cpp.add_argument("--gguf", help="GGUF path (overrides auto-detection)")
sp_cpp.add_argument(
"--backend", default="cpu",
choices=["auto", "cpu", "cpu_accel", "metal", "vulkan"],
help="Compute backend (default: cpu)",
)
sp_cpp.add_argument(
"--mel-from-ref", action="store_true",
help="Whisper-only validation hook: inject the reference mel via "
"TRANSCRIBE_MEL_FROM_REF so enc.mel.in is bit-"
"identical to HF's WhisperFeatureExtractor. Use to isolate "
"graph drift from frontend drift when a regression fires. "
"Requires a build with -DTRANSCRIBE_ENABLE_VALIDATION_HOOKS=ON. "
"Default is the production C++ MelFrontend.",
)
sp_cpp.set_defaults(func=cmd_cpp)
# mel
sp_mel = sub.add_parser("mel", help="Compare production C++ mel vs reference mel")
add_common(sp_mel)
sp_mel.add_argument("--gguf", help="GGUF path (overrides auto-detection)")
sp_mel.add_argument(
"--backend", default="cpu",
choices=["auto", "cpu", "cpu_accel", "metal", "vulkan"],
help="Compute backend (default: cpu)",
)
sp_mel.set_defaults(func=cmd_mel)
# compare
sp_cmp = sub.add_parser("compare", help="Compare C++ vs reference dumps")
add_common(sp_cmp)
sp_cmp.add_argument(
"--report", action="store_true",
help="Emit a report bundle under reports/porting/<family>/<variant>/<ts>-<sha>/",
)
sp_cmp.set_defaults(func=cmd_compare)
# all
sp_all = sub.add_parser("all", help="Run ref + cpp + compare")
add_common(sp_all)
sp_all.add_argument("--model", help="HF model ID or local path")
sp_all.add_argument("--gguf", help="GGUF path")
sp_all.add_argument("--backend", default="cpu", choices=["auto", "cpu", "cpu_accel", "metal", "vulkan"])
sp_all.add_argument(
"--report", action="store_true",
help="Emit a report bundle after compare completes.",
)
sp_all.add_argument(
"--mel-from-ref", action="store_true",
help="Whisper-only validation hook: inject the reference mel via "
"TRANSCRIBE_MEL_FROM_REF for the cpp dump. Requires a build with "