ANALYSIS, // Only output analysis results (default)

TEMPLATE, // Only output rendered template

BOTH // Output both

NONE, // Don't render any message scenarios (only analysis)

CONTENT_ONLY, // Simple assistant message with content

REASONING_CONTENT, // Message with reasoning_content + content

TOOL_CALL_ONLY, // Message with tool_calls only

CONTENT_TOOL_CALL, // Message with content + tool_calls

REASONING_TOOL_CALL, // Message with reasoning_content + tool_calls

CONTENT_FAKE_TOOL_CALL, // Message with content but no actual tool_calls (for testing)

ALL // Render all scenarios

std::string template_path;

bool with_tools = true;

bool generation_prompt = true;

bool enable_reasoning = true;

bool debug_jinja = false;

bool force_tool_call = false;

bool parallel_tool_calls = true;

output_mode mode = output_mode::BOTH;

input_message_type input_message = input_message_type::NONE;

std::ifstream fin(path, std::ios::binary);

if (!fin.is_open()) {

throw std::runtime_error("Could not open file: " + path);

}

std::ostringstream buf;

buf << fin.rdbuf();

return buf.str();

struct gguf_init_params params = { /*no_alloc =*/true, // We only need metadata, not tensor data

/*ctx=*/nullptr };

struct gguf_context * ctx = gguf_init_from_file(path.c_str(), params);

if (ctx == nullptr) {

throw std::runtime_error("Could not open GGUF file: " + path);

}

const char * key = "tokenizer.chat_template";

int64_t key_id = gguf_find_key(ctx, key);

if (key_id == -1) {

gguf_free(ctx);

throw std::runtime_error("GGUF file does not contain chat template key: " + std::string(key));

}

const char * template_str = gguf_get_val_str(ctx, key_id);

if (template_str == nullptr) {

gguf_free(ctx);

throw std::runtime_error("GGUF file contains chat template key but value is null");

}

std::string result = template_str;

gguf_free(ctx);

return result;

LOG_ERR("Test the chat template auto-parser; also usable as a debug tool that shows the generated PEG parser, GBNF grammar and triggers for a given template.\n");

LOG_ERR("\nUsage: %s [filter_regex] run the automated tests (default)\n", program_name);

LOG_ERR(" %s <template_or_gguf_path> [options] debug a single template\n", program_name);

LOG_ERR("\nDebug mode options:\n");

LOG_ERR(" --no-tools Disable tool definitions\n");

LOG_ERR(" --force-tool-call Set tool calls to forced\n");

LOG_ERR(" --parallel-tool-calls=0|1 Set parallel_tool_calls (default: 1)\n");

LOG_ERR(" --generation-prompt=0|1 Set add_generation_prompt (default: 1)\n");

LOG_ERR(" --enable-reasoning=0|1 Enable reasoning parsing (default: 1)\n");

LOG_ERR(" --output=MODE Output mode: analysis, template, both (default: both)\n");

LOG_ERR(" --debug-jinja Enable Jinja fine-grained debug\n");

LOG_ERR(" --input-message=TYPE Message type to render:\n");

LOG_ERR(" content_only, reasoning_content, tool_call_only,\n");

LOG_ERR(" content_tool_call, reasoning_tool_call,\n");

LOG_ERR(" content_fake_tool_call, all\n");

LOG_ERR("\nExamples:\n");

LOG_ERR(" %s template.jinja --input-message=all --generation-prompt=1\n", program_name);

LOG_ERR(" %s template.jinja --output=template --input-message=tool_call_only\n", program_name);

opts.template_path = argv[1];

for (int i = 2; i < argc; ++i) {

std::string arg = argv[i];

if (arg == "--force-tool-call") {

opts.force_tool_call = true;

} else if (arg == "--debug-jinja") {

opts.debug_jinja = true;

} else if (arg == "--no-tools") {

opts.with_tools = false;

} else if (arg.rfind("--parallel-tool-calls=", 0) == 0) {

opts.parallel_tool_calls = parse_bool_option(arg.substr(22));

} else if (arg.rfind("--generation-prompt=", 0) == 0) {

opts.generation_prompt = parse_bool_option(arg.substr(20));

} else if (arg.rfind("--enable-reasoning=", 0) == 0) {

opts.enable_reasoning = parse_bool_option(arg.substr(19));

} else if (arg.rfind("--output=", 0) == 0) {

std::string mode = arg.substr(9);

if (mode == "analysis") {

opts.mode = output_mode::ANALYSIS;

} else if (mode == "template") {

opts.mode = output_mode::TEMPLATE;

} else if (mode == "both") {

opts.mode = output_mode::BOTH;

} else {

LOG_ERR("Unknown output mode: %s\n", mode.c_str());

return false;

}

} else if (arg.rfind("--input-message=", 0) == 0) {

std::string type = arg.substr(16);

if (type == "content_only") {

opts.input_message = input_message_type::CONTENT_ONLY;

} else if (type == "reasoning_content") {

opts.input_message = input_message_type::REASONING_CONTENT;

} else if (type == "tool_call_only") {

opts.input_message = input_message_type::TOOL_CALL_ONLY;

} else if (type == "content_tool_call") {

opts.input_message = input_message_type::CONTENT_TOOL_CALL;

} else if (type == "reasoning_tool_call") {

opts.input_message = input_message_type::REASONING_TOOL_CALL;

} else if (type == "content_fake_tool_call") {

opts.input_message = input_message_type::CONTENT_FAKE_TOOL_CALL;

} else if (type == "all") {

opts.input_message = input_message_type::ALL;

} else {

LOG_ERR("Unknown input message type: %s\n", type.c_str());

return false;

}

} else {

LOG_ERR("Unknown option: %s\n", arg.c_str());

print_usage(argv[0]);

return false;

}

}

return true;

return json{

{ "role", "user" },

{ "content", "Hello, please help me with a task." }

};

return json{

{ "role", "assistant" },

{ "content", "Hello! I'm here to help you with your task." }

};

return json{

{ "role", "assistant" },

{ "content", "Hello! I'm here to help you with your task." },

{ "reasoning_content", "The user is greeting me and asking for help. I should respond politely." }

};

return json{

{ "role", "assistant" },

{ "content", nullptr },

{ "tool_calls",

json::array({ json{

{ "type", "function" },

{ "function", json{ { "name", "test_function_name" },

{ "arguments", json::object({ { "param1", "value1" }, { "param2", "value2" } }) } } },

{ "id", "123456789" } } }) }

};

return json{

{ "role", "assistant" },

{ "content", "I'll help you by calling a function." },

{ "tool_calls",

json::array({ json{

{ "type", "function" },

{ "function",

json{ { "name", "test_function_name" },

{ "arguments", json::object({ { "param1", "value1" }, { "param2", "value2" } }) } } } } }) }

};

return json{

{ "role", "assistant" },

{ "content", nullptr },

{ "reasoning_content", "I need to call a function to help with this task." },

{ "tool_calls",

json::array({ json{

{ "type", "function" },

{ "function",

json{ { "name", "test_function_name" },

{ "arguments", json::object({ { "param1", "value1" }, { "param2", "value2" } }) } } } } }) }

};

// This message has content but NO tool_calls field

// It's used to test if a template renders tool definitions but not tool calls

return json{

{ "role", "assistant" },

{ "content", "I'll help you by calling a function." }

};

const std::string & scenario_name,

const json & messages,

const json & tools,

bool add_generation_prompt,

bool enable_thinking) {

LOG_ERR("\n=== Scenario: %s ===\n", scenario_name.c_str());

LOG_ERR("add_generation_prompt: %s, enable_thinking: %s\n", add_generation_prompt ? "true" : "false",

enable_thinking ? "true" : "false");

// When add_generation_prompt is true, add a trailing user message to trigger the prompt

json final_messages = messages;

if (add_generation_prompt && !messages.empty() && messages.back().value("role", "") == "assistant") {

final_messages.push_back(json{

{ "role", "user" },

{ "content", "Now please continue with another response." }

});

}

LOG_ERR("Messages:\n%s\n", final_messages.dump(2).c_str());

try {

generation_params inputs;

inputs.messages = final_messages;

inputs.add_generation_prompt = add_generation_prompt;

inputs.extra_context["enable_thinking"] = enable_thinking;

if (!tools.is_null() && tools.is_array() && !tools.empty()) {

inputs.tools = tools;

}

std::string output = common_chat_template_direct_apply(tmpl, inputs);

LOG_ERR("\n--- Rendered Output ---\n");

LOG_ERR("%s\n", output.c_str());

LOG_ERR("--- End Output (length: %zu) ---\n", output.length());

} catch (const std::exception & e) {

LOG_ERR("Rendering failed: %s\n", e.what());

}

const json & tools,

bool add_generation_prompt,

bool enable_thinking,

input_message_type message_type) {

json user_msg = build_debug_user_message();

auto render_if = [&](input_message_type type, const std::string & name, const json & assistant_msg) {

if (message_type == input_message_type::ALL || message_type == type) {

json messages = json::array({ user_msg, assistant_msg });

render_scenario(tmpl, name, messages, tools, add_generation_prompt, enable_thinking);

}

};

render_if(input_message_type::CONTENT_ONLY, "content_only", build_content_only_message());

render_if(input_message_type::REASONING_CONTENT, "reasoning_content", build_reasoning_content_message());

render_if(input_message_type::TOOL_CALL_ONLY, "tool_call_only", build_tool_call_only_message());

render_if(input_message_type::CONTENT_TOOL_CALL, "content_tool_call", build_content_tool_call_message());

render_if(input_message_type::REASONING_TOOL_CALL, "reasoning_tool_call", build_reasoning_tool_call_message());

render_if(input_message_type::CONTENT_FAKE_TOOL_CALL, "content_fake_tool_call",

build_content_fake_tool_call_message());

// Also render with add_generation_prompt=true to show the prompt ending

if (message_type == input_message_type::ALL) {

LOG_ERR("\n\n=== Generation Prompt Scenarios (add_generation_prompt=true) ===\n");

json prompt_messages = json::array({ user_msg });

render_scenario(tmpl, "generation_prompt_only", prompt_messages, tools, true, enable_thinking);

// With enable_thinking toggled

render_scenario(tmpl, "generation_prompt_thinking_disabled", prompt_messages, tools, true, false);

}

generation_params params;

params.messages = json::array({ build_debug_user_message() });

params.reasoning_format = opts.enable_reasoning ? COMMON_REASONING_FORMAT_DEEPSEEK : COMMON_REASONING_FORMAT_NONE;

params.enable_thinking = opts.enable_reasoning;

params.add_generation_prompt = opts.generation_prompt;

if (opts.with_tools) {

params.tools = tools;

params.tool_choice = opts.force_tool_call ? COMMON_CHAT_TOOL_CHOICE_REQUIRED : COMMON_CHAT_TOOL_CHOICE_AUTO;

} else {

params.tools = json();

params.tool_choice = COMMON_CHAT_TOOL_CHOICE_NONE;

}

params.parallel_tool_calls = opts.parallel_tool_calls;

return params;

std::string template_source;

try {

// Check if the file is a GGUF file

if (opts.template_path.size() >= 5 &&

opts.template_path.compare(opts.template_path.size() - 5, 5, ".gguf") == 0) {

template_source = read_gguf_chat_template(opts.template_path);

} else {

template_source = read_file(opts.template_path);

}

} catch (const std::exception & e) {

LOG_ERR("Error reading template: %s\n", e.what());

return 1;

}

LOG_ERR("Analyzing template: %s\n", opts.template_path.c_str());

LOG_ERR("Options: with_tools=%s, generation_prompt=%s, enable_reasoning=%s\n", opts.with_tools ? "true" : "false",

opts.generation_prompt ? "true" : "false", opts.enable_reasoning ? "true" : "false");

try {

common_chat_template chat_template(template_source, "", "");

json tools = opts.with_tools ? build_tools_definition() : json();

generation_params params = prepare_debug_params(opts, tools);

common_chat_params parser_data;

if (std::optional<common_chat_params> spec_tmpl =

common_chat_try_specialized_template(chat_template, template_source, params)) {

LOG_ERR("\n");

LOG_ERR("This template uses a specialized parser, analysis results will not be available.\n");

parser_data = *spec_tmpl;

} else {

// Render template scenarios if requested

if (opts.input_message != input_message_type::NONE &&

(opts.mode == output_mode::TEMPLATE || opts.mode == output_mode::BOTH)) {

LOG_ERR("\n");

LOG_ERR("================================================================================\n");

LOG_ERR(" TEMPLATE RENDERING OUTPUT\n");

LOG_ERR("================================================================================\n");

render_all_scenarios(chat_template, tools, opts.generation_prompt, opts.enable_reasoning,

opts.input_message);

}

// Output analysis if requested

if (opts.mode == output_mode::ANALYSIS || opts.mode == output_mode::BOTH) {

LOG_ERR("\n");

LOG_ERR("================================================================================\n");

LOG_ERR(" TEMPLATE ANALYSIS\n");

LOG_ERR("================================================================================\n");

struct autoparser analysis;

analysis.analyze_template(chat_template);

// Generate Parser

parser_data = peg_generator::generate_parser(chat_template, params, analysis);

}

}

if (!std::empty(parser_data.parser)) {

LOG_ERR("\n=== Generated Parser ===\n");

common_peg_arena arena;

arena.load(parser_data.parser);

LOG_ERR("%s\n", arena.dump(arena.root()).c_str());

LOG_ERR("\n=== Generated Grammar ===\n");

LOG_ERR("%s\n", parser_data.grammar.c_str());

LOG_ERR("\n=== Generated Lazy Grammar ===\n");

LOG_ERR("%d\n", parser_data.grammar_lazy);

LOG_ERR("\n=== Generated Grammar Triggers ===\n");

for (const common_grammar_trigger & cgt : parser_data.grammar_triggers) {

LOG_ERR("Token: %d | Type: %d | Value: %s\n", cgt.token, cgt.type, cgt.value.c_str());

}

LOG_ERR("\n=== Preserved Tokens ===\n");

for (const std::string & token : parser_data.preserved_tokens) {

LOG_ERR(" '%s'\n", token.c_str());

}

}

} catch (const std::exception & e) {

LOG_ERR("Analysis failed: %s\n", e.what());

return 1;

}

return 0;

if (argc > 1) {

std::string arg = argv[1];

if (arg == "-h" || arg == "--help") {

common_log_set_verbosity_thold(99);

print_usage(argv[0]);

return 0;

}

// debug mode: if the first argument is an existing file, analyze that template instead of running the automated tests

if (std::filesystem::is_regular_file(arg)) {

common_log_set_verbosity_thold(99);

debug_options opts;

if (!parse_debug_options(argc, argv, opts)) {

return 1;

}

if (opts.debug_jinja || std::getenv("LLAMA_DEBUG_JINJA") != nullptr) {

jinja::enable_debug(true);

}

return debug_single_template(opts);

}

}

testing t(std::cout);

t.verbose = true;

// usage: test-chat-auto-parser [filter_regex]

if (argc > 1) {

t.set_filter(argv[1]);

}

t.test("diff_split", test_calculate_diff_split);

t.test("common_prefix", test_until_common_prefix);

t.test("common_suffix", test_after_common_suffix);

t.test("compare_variants", test_compare_variants);

t.test("segments", test_marker_separation);

t.test("seed_oss_diffs", test_seed_oss_tool_analysis);

t.test("cohere", test_cohere_analysis);

t.test("nemotron", test_nemotron_analysis);

t.test("laguna", test_laguna_analysis);

t.test("laguna-s", test_laguna_s_analysis);

t.test("laguna-xs2", test_laguna_xs2_analysis);

t.test("smollm3", test_smollm3_analysis);

t.test("standard_json_tools", test_standard_json_tools_formats);

t.test("normalize_quotes_to_json", test_normalize_quotes_to_json);

t.test("tagged_args_embedded_quotes", test_tagged_args_with_embedded_quotes);

t.test("bailing_v3", test_bailing_v3_tool_format);

t.test("role_markers_all_templates", test_role_markers_all_templates);

return t.summary();

auto single_square_marker = segmentize_markers("pre_marker[marker]post_marker");

auto single_diag_marker = segmentize_markers("pre_marker<marker>post_marker");

auto paired_markers = segmentize_markers("<hello>world</hello>");

auto double_different_markers = segmentize_markers("<hello>[hello]<world>[world]");

auto in_between = segmentize_markers("im<blue>daba<dee>da[hey]");

t.test("single_square_marker", [&] (testing & t) {

t.assert_equal("first is text", segment_type::TEXT, single_square_marker[0].type);

t.assert_equal("second is marker", segment_type::MARKER, single_square_marker[1].type);

t.assert_equal("last is text", segment_type::TEXT, single_square_marker[2].type);

t.assert_equal("first is 'pre_marker'", "pre_marker", single_square_marker[0].value);

t.assert_equal("second is '[marker]'", "[marker]", single_square_marker[1].value);

t.assert_equal("last is 'post_marker'", "post_marker", single_square_marker[2].value);

});

t.test("single_diagonal_marker", [&] (testing & t) {

t.assert_equal("first is text", segment_type::TEXT, single_diag_marker[0].type);

t.assert_equal("second is marker", segment_type::MARKER, single_diag_marker[1].type);

t.assert_equal("last is text", segment_type::TEXT, single_diag_marker[2].type);

t.assert_equal("first is 'pre_marker'", "pre_marker", single_diag_marker[0].value);

t.assert_equal("second is '<marker>'", "<marker>", single_diag_marker[1].value);

t.assert_equal("last is 'post_marker'", "post_marker", single_diag_marker[2].value);

});

t.test("paired_markers", [&] (testing & t) {

t.assert_equal("first is marker", segment_type::MARKER, paired_markers[0].type);

t.assert_equal("second is text", segment_type::TEXT, paired_markers[1].type);

t.assert_equal("third is marker", segment_type::MARKER, paired_markers[2].type);

t.assert_equal("first is '<hello>'", "<hello>", paired_markers[0].value);

t.assert_equal("second is 'world'", "world", paired_markers[1].value);

t.assert_equal("third is '</hello>'", "</hello>", paired_markers[2].value);

});

t.test("double_different_markers", [&] (testing & t) {

t.assert_equal("first is marker", segment_type::MARKER, double_different_markers[0].type);

t.assert_equal("second is marker", segment_type::MARKER, double_different_markers[1].type);

t.assert_equal("third is marker", segment_type::MARKER, double_different_markers[2].type);

t.assert_equal("fourth is marker", segment_type::MARKER, double_different_markers[3].type);

t.assert_equal("first is '<hello>'", "<hello>", double_different_markers[0].value);

t.assert_equal("second is '[hello]'", "[hello]", double_different_markers[1].value);

t.assert_equal("third is '<world>'", "<world>", double_different_markers[2].value);

t.assert_equal("fourth is '[world]'", "[world]", double_different_markers[3].value);

});

t.test("in_between", [&] (testing & t) {

t.assert_equal("first is text", segment_type::TEXT, in_between[0].type);

t.assert_equal("second is marker", segment_type::MARKER, in_between[1].type);

t.assert_equal("third is text", segment_type::TEXT, in_between[2].type);

t.assert_equal("fourth is marker", segment_type::MARKER, in_between[3].type);

t.assert_equal("fifth is text", segment_type::TEXT, in_between[4].type);

t.assert_equal("sixth is marker", segment_type::MARKER, in_between[5].type);

t.assert_equal("first is 'im'", "im", in_between[0].value);

t.assert_equal("second is '<blue>'", "<blue>", in_between[1].value);

t.assert_equal("third is 'daba'", "daba", in_between[2].value);

t.assert_equal("fourth is '<dee>'", "<dee>", in_between[3].value);

t.assert_equal("fifth is 'da'", "da", in_between[4].value);

t.assert_equal("sixth is '[hey]'", "[hey]", in_between[5].value);

});

t.test("calculate_diff_split basic", test_calculate_diff_split_basic);

t.test("calculate_diff_split identical", test_calculate_diff_split_identical);

t.test("calculate_diff_split common prefix", test_calculate_diff_split_common_prefix);

t.test("calculate_diff_split common suffix", test_calculate_diff_split_common_suffix);

t.test("calculate_diff_split common both", test_calculate_diff_split_common_both);

t.test("calculate_diff_split empty cases", test_calculate_diff_split_empty_cases);

t.test("calculate_diff_split no common", test_calculate_diff_split_no_common);

t.test("calculate_diff_split single char", test_calculate_diff_split_single_char);

t.test("calculate_diff_split overlaps", test_calculate_diff_split_overlaps);

t.test("calculate_diff_split tag boundaries", test_calculate_diff_split_tag_boundaries);

t.test("calculate_diff_split generation prompt", test_calculate_diff_split_generation_prompt);

diff_split result = calculate_diff_split("hello world", "hello test");

t.assert_equal("prefix should be 'hello '", "hello ", result.prefix);

t.assert_equal("left should be 'world'", "world", result.left);

t.assert_equal("right should be 'test'", "test", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

result = calculate_diff_split("abc", "xyz");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be 'abc'", "abc", result.left);

t.assert_equal("right should be 'xyz'", "xyz", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

result = calculate_diff_split("prefixA suffix", "prefixB suffix");

t.assert_equal("prefix should be 'prefix'", "prefix", result.prefix);

t.assert_equal("left should be 'A'", "A", result.left);

t.assert_equal("right should be 'B'", "B", result.right);

t.assert_equal("suffix should be ' suffix'", " suffix", result.suffix);

diff_split result = calculate_diff_split("hello", "hello");

t.assert_equal("prefix should be 'hello'", "hello", result.prefix);

t.assert_equal("left should be empty", "", result.left);

t.assert_equal("right should be empty", "", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

result = calculate_diff_split("", "");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be empty", "", result.left);

t.assert_equal("right should be empty", "", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

result = calculate_diff_split("a", "a");

t.assert_equal("prefix should be 'a'", "a", result.prefix);

t.assert_equal("left should be empty", "", result.left);

t.assert_equal("right should be empty", "", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

result = calculate_diff_split("<row><row><row><your><boat><gently>", "<row><row><row><your><boat><gently>");

t.assert_equal("prefix should be '<row><row><row><your><boat><gently>'", "<row><row><row><your><boat><gently>", result.prefix);

t.assert_equal("left should be empty", "", result.left);

t.assert_equal("right should be empty", "", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

diff_split result = calculate_diff_split("abcdef", "abcxyz");

t.assert_equal("prefix should be 'abc'", "abc", result.prefix);

t.assert_equal("left should be 'def'", "def", result.left);

t.assert_equal("right should be 'xyz'", "xyz", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

result = calculate_diff_split("same", "sameagain");

t.assert_equal("prefix should be 'same'", "same", result.prefix);

t.assert_equal("left should be empty", "", result.left);

t.assert_equal("right should be 'again'", "again", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

result = calculate_diff_split("test", "testing");

t.assert_equal("prefix should be 'test'", "test", result.prefix);

t.assert_equal("left should be empty", "", result.left);

t.assert_equal("right should be 'ing'", "ing", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

diff_split result = calculate_diff_split("123end", "456end");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be '123'", "123", result.left);

t.assert_equal("right should be '456'", "456", result.right);

t.assert_equal("suffix should be 'end'", "end", result.suffix);

result = calculate_diff_split("start", "end");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be 'start'", "start", result.left);

t.assert_equal("right should be 'end'", "end", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

result = calculate_diff_split("abcsuffix", "xyzsuffix");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be 'abc'", "abc", result.left);

t.assert_equal("right should be 'xyz'", "xyz", result.right);

t.assert_equal("suffix should be 'suffix'", "suffix", result.suffix);

diff_split result = calculate_diff_split("helloXworld", "helloYworld");

t.assert_equal("prefix should be 'hello'", "hello", result.prefix);

t.assert_equal("left should be 'X'", "X", result.left);

t.assert_equal("right should be 'Y'", "Y", result.right);

t.assert_equal("suffix should be 'world'", "world", result.suffix);

result = calculate_diff_split("ABCmiddleXYZ", "ABCdifferentXYZ");

t.assert_equal("prefix should be 'ABC'", "ABC", result.prefix);

t.assert_equal("left should be 'middle'", "middle", result.left);

t.assert_equal("right should be 'different'", "different", result.right);

t.assert_equal("suffix should be 'XYZ'", "XYZ", result.suffix);

result = calculate_diff_split("startAend", "startBend");

t.assert_equal("prefix should be 'start'", "start", result.prefix);

t.assert_equal("left should be 'A'", "A", result.left);

t.assert_equal("right should be 'B'", "B", result.right);

t.assert_equal("suffix should be 'end'", "end", result.suffix);

// Edge case: common prefix and suffix overlap

result = calculate_diff_split("aa", "ab");

t.assert_equal("prefix should be 'a'", "a", result.prefix);

t.assert_equal("left should be 'a'", "a", result.left);

t.assert_equal("right should be 'b'", "b", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

// Empty left, non-empty right

diff_split result = calculate_diff_split("", "hello");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be empty", "", result.left);

t.assert_equal("right should be 'hello'", "hello", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

// Non-empty left, empty right

result = calculate_diff_split("hello", "");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be 'hello'", "hello", result.left);

t.assert_equal("right should be empty", "", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

// Both empty

result = calculate_diff_split("", "");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be empty", "", result.left);

t.assert_equal("right should be empty", "", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

// Left single char, empty right

result = calculate_diff_split("a", "");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be 'a'", "a", result.left);

t.assert_equal("right should be empty", "", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

// Empty left, right single char

result = calculate_diff_split("", "a");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be empty", "", result.left);

t.assert_equal("right should be 'a'", "a", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

diff_split result = calculate_diff_split("abc", "xyz");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be 'abc'", "abc", result.left);

t.assert_equal("right should be 'xyz'", "xyz", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

result = calculate_diff_split("left", "right");

// The algorithm finds "t" as a common suffix since both strings end with 't'

// This is the algorithm's actual behavior - it finds maximal common suffix

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be 'lef'", "lef", result.left);

t.assert_equal("right should be 'righ'", "righ", result.right);

t.assert_equal("suffix should be 't'", "t", result.suffix);

result = calculate_diff_split("123", "456");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be '123'", "123", result.left);

t.assert_equal("right should be '456'", "456", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

diff_split result = calculate_diff_split("a", "b");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be 'a'", "a", result.left);

t.assert_equal("right should be 'b'", "b", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

result = calculate_diff_split("a", "a");

t.assert_equal("prefix should be 'a'", "a", result.prefix);

t.assert_equal("left should be empty", "", result.left);

t.assert_equal("right should be empty", "", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

result = calculate_diff_split("a", "ab");

t.assert_equal("prefix should be 'a'", "a", result.prefix);

t.assert_equal("left should be empty", "", result.left);

t.assert_equal("right should be 'b'", "b", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

result = calculate_diff_split("ab", "a");

t.assert_equal("prefix should be 'a'", "a", result.prefix);

t.assert_equal("left should be 'b'", "b", result.left);

t.assert_equal("right should be empty", "", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

// One string is substring of another

diff_split result = calculate_diff_split("test", "testing");

t.assert_equal("prefix should be 'test'", "test", result.prefix);

t.assert_equal("left should be empty", "", result.left);

t.assert_equal("right should be 'ing'", "ing", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

result = calculate_diff_split("testing", "test");

t.assert_equal("prefix should be 'test'", "test", result.prefix);

t.assert_equal("left should be 'ing'", "ing", result.left);

t.assert_equal("right should be empty", "", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

// Similar strings with one extra char at start

result = calculate_diff_split("Xtest", "Ytest");

// The algorithm finds "test" as a common suffix since both strings end with "test"

// This is the algorithm's actual behavior - it finds maximal common suffix

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be 'X'", "X", result.left);

t.assert_equal("right should be 'Y'", "Y", result.right);

t.assert_equal("suffix should be 'test'", "test", result.suffix);

// Similar strings with one extra char at end

result = calculate_diff_split("testX", "testY");

t.assert_equal("prefix should be 'test'", "test", result.prefix);

t.assert_equal("left should be 'X'", "X", result.left);

t.assert_equal("right should be 'Y'", "Y", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

// Strings that are reverses

result = calculate_diff_split("abc", "cba");

t.assert_equal("prefix should be empty", "", result.prefix);

t.assert_equal("left should be 'abc'", "abc", result.left);

t.assert_equal("right should be 'cba'", "cba", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

// Test with unclosed XML tags

diff_split result = calculate_diff_split("test<tag", "test>content");

// The fix_tag_boundaries should move incomplete tags appropriately

t.assert_true("prefix should start with 'test'", result.prefix.find("test") == 0);

t.assert_true("should handle tag boundaries", result.left != "" || result.right != "" || result.suffix != "");

// Test with unclosed brackets

result = calculate_diff_split("test[", "test]value");

t.assert_true("should handle bracket boundaries", result.left != "" || result.right != "" || result.suffix != "");

// Test with partial tags on both sides

result = calculate_diff_split("prefix<tag>", "prefix</tag>suffix");

// fix_tag_boundaries moves the incomplete '<' from prefix to left/right

t.assert_equal("prefix should be 'prefix'", "prefix", result.prefix);

t.assert_equal("left should be '<tag>'", "<tag>", result.left);

t.assert_equal("right should be '</tag>suffix'", "</tag>suffix", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

// Test with complex nested tags

result = calculate_diff_split("prefix<div>content</div>", "prefix<div>different</div>");

// Algorithm finds "ent</div>" as a common suffix because both strings end with it

// This is the actual algorithm behavior, though not semantically ideal

t.assert_equal("prefix should be 'prefix<div>'", "prefix<div>", result.prefix);

t.assert_equal("left should be 'cont'", "cont", result.left);

t.assert_equal("right should be 'differ'", "differ", result.right);

t.assert_equal("suffix should be 'ent</div>'", "ent</div>", result.suffix);

// Test with unclosed angle bracket

result = calculate_diff_split("Hello <world>", "Hello test");

t.assert_equal("prefix should be 'Hello '", "Hello ", result.prefix);

t.assert_true("left should contain '<world>'", result.left.find("<world>") != std::string::npos);

t.assert_equal("right should be 'test'", "test", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

// Test with unclosed square bracket

result = calculate_diff_split("test [array]", "test other");

t.assert_equal("prefix should be 'test '", "test ", result.prefix);

t.assert_true("left should contain '[array]'", result.left.find("[array]") != std::string::npos);

t.assert_equal("right should be 'other'", "other", result.right);

t.assert_equal("suffix should be empty", "", result.suffix);

// Test empty prefix and suffix with tags

result = calculate_diff_split("<tag>left</tag>", "<tag>righ</tag>");

t.assert_equal("prefix should be '<tag>'", "<tag>", result.prefix);

t.assert_equal("left should be 'left'", "left", result.left);

t.assert_equal("right should be 'righ'", "righ", result.right);

t.assert_equal("suffix should be '</tag>'", "</tag>", result.suffix);

{

// real case from template tests, simplified

std::string left = "PREFIX</think>Sure";

std::string right = "PREFIX<think>Lemme think</think>Sure";

result = calculate_diff_split(left, right);

t.assert_equal("prefix should be PREFIX", "PREFIX", result.prefix);

t.assert_equal("suffix should be </think>Sure", "</think>Sure", result.suffix);

t.assert_equal("left should be empty", "", result.left);

t.assert_equal("right should be <think>Lemme think", "<think>Lemme think", result.right);

}

{

// Real case: special tokens with |> boundary issue

// The suffix starts with |> which should be moved to complete <|END_RESPONSE and <|END_ACTION

std::string prefix = "SOME_PREFIX";

std::string suffix = "|><|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>";

std::string left_diff = "<|START_RESPONSE|>Let me help you.<|END_RESPONSE";

std::string right_diff =

"<|START_THINKING|><|END_THINKING|><|START_ACTION|>[\n"

" {\"tool_call_id\": \"0\", \"tool_name\": \"test_function_name\", "

"\"parameters\": {\"param1\": \"value1\", \"param2\": \"value2\"}}\n"

"]<|END_ACTION";

std::string left = prefix + left_diff + suffix;

std::string right = prefix + right_diff + suffix;

result = calculate_diff_split(left, right);

t.assert_equal("special token prefix", prefix, result.prefix);

// The |> should be moved from suffix to complete the tokens

t.assert_equal("special token left", "<|START_RESPONSE|>Let me help you.<|END_RESPONSE|>", result.left);

t.assert_true("special token right ends with |>", result.right.find("<|END_ACTION|>") != std::string::npos);

t.assert_equal("special token suffix", "<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",

result.suffix);

}

// ChatML thinking template: left is a prefix of right, generation_prompt is the appended part.

// The trailing \n in left matches the trailing \n in the generation_prompt, causing