Skip to content

test: Explain to the user how to regenerate expected outputs. #889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eyal0 merged 2 commits into from
Apr 4, 2026
+40 −20
Merged

test: Explain to the user how to regenerate expected outputs. #889

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cb50d2a
refactor: Rename overwrite_expected to regenerate_expected.
eyal0 Apr 4, 2026
df319e7
test: Print out the command to regenerate output.
eyal0 Apr 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ pcb2gcode_add_test_executable(
# Integration script tests live in tests/data/CMakeLists.txt (and gerbv_example/).
# They stay registered but are DISABLED unless PCB2GCODE_RUN_INTEGRATION_TESTS is ON
# (see above). CTest labels: integration_expect_success vs integration_expect_failure.
# Regenerate expected/ with tools/run_gerbv_example_test.py --overwrite-expected.
# Regenerate expected/ with tools/run_gerbv_example_test.py --regenerate-expected.
if(NOT PCB2GCODE_RUN_INTEGRATION_TESTS)
set(_geos_display "${PCB2GCODE_GEOS_VERSION}")
if(NOT _geos_display)
Expand Down
28 changes: 14 additions & 14 deletions tests/data/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ if(PCB2GCODE_RUN_INTEGRATION_TESTS)
endif()

# run_gerbv_example_test.py harness. Ctest label reflects expected pcb2gcode exit code.
# Also registers ${_name}_overwrite_expected. Build target integration_overwrite_all_expected
# Also registers ${_name}_regenerate_expected. Build target integration_regenerate_all_expected
# depends on every such target (one command refreshes all).
function(pcb2gcode_add_integration_script_test _name _input_dir_relpath)
cmake_parse_arguments(
Expand Down Expand Up @@ -42,22 +42,22 @@ function(pcb2gcode_add_integration_script_test _name _input_dir_relpath)
set_tests_properties(
"${_name}"
PROPERTIES LABELS "${_integration_label}" DISABLED "${_pcb2gcode_integration_disabled}")
set(_overwrite_cmd ${_base_cmd})
list(APPEND _overwrite_cmd "--overwrite-expected")
list(APPEND _overwrite_cmd ${_extra_args})
set(_regenerate_cmd ${_base_cmd})
list(APPEND _regenerate_cmd "--regenerate-expected")
list(APPEND _regenerate_cmd ${_extra_args})
add_custom_target(
${_name}_overwrite_expected
COMMAND ${_overwrite_cmd}
${_name}_regenerate_expected
COMMAND ${_regenerate_cmd}
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
DEPENDS pcb2gcode
COMMENT "Overwrite expected/ for ${_name}")
COMMENT "Regenerate expected/ for ${_name}")
set_property(
GLOBAL
APPEND
PROPERTY PCB2GCODE_INTEGRATION_OVERWRITE_TARGETS "${_name}_overwrite_expected")
PROPERTY PCB2GCODE_INTEGRATION_REGENERATE_TARGETS "${_name}_regenerate_expected")
endfunction()

set_property(GLOBAL PROPERTY PCB2GCODE_INTEGRATION_OVERWRITE_TARGETS "")
set_property(GLOBAL PROPERTY PCB2GCODE_INTEGRATION_REGENERATE_TARGETS "")
add_subdirectory(gerbv_example)

pcb2gcode_add_integration_script_test(
Expand Down Expand Up @@ -160,12 +160,12 @@ pcb2gcode_add_integration_script_test(
PCB2GCODE_ARGS "--zchange-absolute=false")

get_property(
_pcb2gcode_integration_overwrite_targets
_pcb2gcode_integration_regenerate_targets
GLOBAL
PROPERTY PCB2GCODE_INTEGRATION_OVERWRITE_TARGETS)
if(_pcb2gcode_integration_overwrite_targets)
PROPERTY PCB2GCODE_INTEGRATION_REGENERATE_TARGETS)
if(_pcb2gcode_integration_regenerate_targets)
add_custom_target(
integration_overwrite_all_expected
DEPENDS ${_pcb2gcode_integration_overwrite_targets}
integration_regenerate_all_expected
DEPENDS ${_pcb2gcode_integration_regenerate_targets}
COMMENT "Regenerate expected/ for all integration script tests")
endif()
30 changes: 25 additions & 5 deletions tools/run_gerbv_example_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@

Usage:
tools/run_gerbv_example_test.py <input_dir> <pcb2gcode_binary>
[--overwrite-expected] [--expected-exit-code N] [--pcb2gcode-arg ARG ...]
[--regenerate-expected] [--expected-exit-code N] [--pcb2gcode-arg ARG ...]

By default, pcb2gcode writes to a temporary directory that is removed after the test,
and the script compares that output to the existing expected/ tree. If expected/ is
missing, exit code 0 is accepted only when pcb2gcode produces no output files (same
idea as tools/integration_tests.py for --version / --help).

With --overwrite-expected, the script removes expected/ under the input directory
With --regenerate-expected, the script removes expected/ under the input directory
(if present), runs pcb2gcode with --output-dir set to that path, runs fix-up on the
new tree, and skips comparison (use this to refresh golden files).

Expand All @@ -32,6 +32,7 @@
import filecmp
import os
import re
import shlex
import shutil
import subprocess
import sys
Expand Down Expand Up @@ -74,6 +75,19 @@ def _directory_tree_has_files(path):
return False


def _print_regenerate_command_hint(input_dir, binary, pcb2gcode_args):
"""Print a copy-paste shell command (for ctest / CI logs)."""
script = os.path.abspath(sys.argv[0])
argv = [sys.executable, script, input_dir, binary, "--regenerate-expected"]
for a in pcb2gcode_args:
argv.append("--pcb2gcode-arg=" + a)
line = " ".join(shlex.quote(x) for x in argv)
print(
"To regenerate expected/ output, run from the repository root:\n " + line,
file=sys.stderr,
)


def compare_directories(left, right):
"""Return True if both trees exist, have the same relative files, and contents match."""
if not os.path.isdir(left) or not os.path.isdir(right):
Expand All @@ -96,13 +110,13 @@ def main():
parser = argparse.ArgumentParser(
description=(
"Run pcb2gcode on one gerbv_example case: compare to expected/, "
"or overwrite expected/ with fresh output (--overwrite-expected)."
"or overwrite expected/ with fresh output (--regenerate-expected)."
)
)
parser.add_argument("input_dir", help="Example directory (contains millproject, expected/, …)")
parser.add_argument("pcb2gcode_binary", help="Path to pcb2gcode executable")
parser.add_argument(
"--overwrite-expected",
"--regenerate-expected",
action="store_true",
help="Remove input_dir/expected/, regenerate it with pcb2gcode, and skip comparison",
)
Expand Down Expand Up @@ -185,14 +199,20 @@ def main():
+ effective_out,
file=sys.stderr,
)
_print_regenerate_command_hint(
args.input_dir, args.pcb2gcode_binary, args.pcb2gcode_arg
)
return 1
return 0
if not compare_directories(expected_path, effective_out):
if args.overwrite_expected:
if args.regenerate_expected:
# Delete expected_path and copy effective_out to it.
shutil.rmtree(expected_path, ignore_errors=True)
shutil.copytree(effective_out, expected_path)
return 0
_print_regenerate_command_hint(
args.input_dir, args.pcb2gcode_binary, args.pcb2gcode_arg
)
return 1
return 0
finally:
Expand Down
Loading