forked from 0xShug0/audio.cpp
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_partial_text.cpp
More file actions
151 lines (133 loc) · 6.58 KB
/
Copy pathtest_partial_text.cpp
File metadata and controls
151 lines (133 loc) · 6.58 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
#include "engine/framework/runtime/partial_text.h"
#include "test_assert.h"
#include <iostream>
#include <string>
#include <vector>
namespace {
using engine::runtime::PartialTextPublisher;
using engine::test::require_eq;
// What a consumer assembles by appending every delta, which is the contract:
// partials concatenate into the transcript.
std::string appended(const std::vector<std::string> & decodes) {
PartialTextPublisher publisher;
std::string client;
for (const auto & decode : decodes) {
client += publisher.publish(decode);
}
return client;
}
void test_growing_transcript_yields_increments() {
PartialTextPublisher publisher;
require_eq(publisher.publish("Some call me nat"), std::string("Some call me nat"), "first");
require_eq(publisher.publish("Some call me nature. Others"), std::string("ure. Others"), "second");
require_eq(publisher.publish("Some call me nature. Others"), std::string(""), "unchanged");
}
// The reason this is shared rather than reimplemented per family: a tokenizer
// falls back to bytes for text its vocabulary does not cover, so a decode can
// stop part way through a character. Publishing that puts half a code point on
// the wire, where it reaches a JSON encoder as invalid UTF-8.
void test_partial_character_is_held_until_complete() {
const std::string cjk = "\xE4\xB8\x80"; // U+4E00, three bytes
PartialTextPublisher publisher;
require_eq(publisher.publish("ab" + cjk.substr(0, 1)), std::string("ab"), "lead byte held");
require_eq(publisher.publish("ab" + cjk.substr(0, 2)), std::string(""), "still incomplete");
require_eq(publisher.publish("ab" + cjk), cjk, "released whole");
require_eq(appended({"ab" + cjk.substr(0, 1), "ab" + cjk.substr(0, 2), "ab" + cjk}),
"ab" + cjk, "assembled");
}
void test_two_byte_characters_are_held_too() {
const std::string ru = "\xD0\x9F\xD1\x80\xD0\xB8"; // При
require_eq(appended({ru.substr(0, 1), ru.substr(0, 3), ru.substr(0, 5), ru}), ru, "cyrillic");
}
void test_four_byte_character_is_held_until_complete() {
const std::string emoji = "\xF0\x9F\x8E\xB5"; // U+1F3B5
PartialTextPublisher publisher;
require_eq(publisher.publish(emoji.substr(0, 3)), std::string(""), "incomplete");
require_eq(publisher.publish(emoji), emoji, "released whole");
}
// A revision cannot be retracted -- the delta has already gone out -- but the
// next one must still start on a character boundary rather than inside one.
void test_revision_resumes_on_a_character_boundary() {
const std::string cjk = "\xE4\xB8\x80";
PartialTextPublisher publisher;
require_eq(publisher.publish("ab" + cjk), "ab" + cjk, "published");
// Same first byte of the character, different continuation.
const std::string revised = "ab\xE4\xB8\x81";
const std::string delta = publisher.publish(revised);
require_eq(delta, std::string("\xE4\xB8\x81"), "whole character re-sent");
}
// A decode that truncates mid-character must not un-publish the character it
// cut: the decode that restores it would then send it twice, and a consumer
// that appends every delta would show it twice.
void test_truncated_decode_does_not_resend() {
const std::string cjk = "\xE4\xB8\x80";
PartialTextPublisher publisher;
require_eq(publisher.publish("ab" + cjk), "ab" + cjk, "published whole");
require_eq(publisher.publish("ab" + cjk.substr(0, 2)), std::string(""), "truncated");
require_eq(publisher.publish("ab" + cjk), std::string(""), "not resent");
require_eq(appended({"ab" + cjk, "ab" + cjk.substr(0, 2), "ab" + cjk}),
"ab" + cjk, "consumer sees it once");
}
// The agreement check is bounded, so state what that buys and what it gives up.
// A revision near the end -- where a streaming decode actually revises -- is
// seen, and the delta resumes from it.
void test_revision_within_the_window_is_seen() {
PartialTextPublisher publisher;
require_eq(publisher.publish("the quick brown fox"), std::string("the quick brown fox"), "first");
const std::string delta = publisher.publish("the quick brown dog");
require_eq(delta, std::string("dog"), "resumes at the divergence");
}
// A revision further back than the window is not seen, and cannot be: the text
// it would correct has already gone out and nothing can retract it. The
// publisher keeps going forward instead of re-sending a transcript the consumer
// cannot un-append.
void test_revision_behind_the_window_does_not_resend_history() {
PartialTextPublisher publisher;
std::string transcript(600, 'a');
require_eq(publisher.publish(transcript).size(), transcript.size(), "first");
std::string revised = transcript;
revised[0] = 'b'; // far behind the 256-byte window
revised += "tail";
require_eq(publisher.publish(revised), std::string("tail"), "only the new tail");
}
void test_shrinking_transcript_publishes_nothing() {
PartialTextPublisher publisher;
require_eq(publisher.publish("abcdef"), std::string("abcdef"), "first");
require_eq(publisher.publish("abc"), std::string(""), "shrunk");
// And the text it dropped is not re-sent when it comes back, because the
// consumer was never told to remove it.
require_eq(publisher.publish("abcdef"), std::string(""), "not resent");
}
void test_reset_forgets_the_published_prefix() {
PartialTextPublisher publisher;
require_eq(publisher.publish("hello"), std::string("hello"), "first");
publisher.reset();
require_eq(publisher.published(), std::string(""), "cleared");
require_eq(publisher.publish("hello"), std::string("hello"), "republished after reset");
}
void test_empty_and_ascii_edges() {
PartialTextPublisher publisher;
require_eq(publisher.publish(""), std::string(""), "empty");
require_eq(publisher.publish("a"), std::string("a"), "single byte");
}
} // namespace
int main() {
try {
test_growing_transcript_yields_increments();
test_partial_character_is_held_until_complete();
test_two_byte_characters_are_held_too();
test_four_byte_character_is_held_until_complete();
test_revision_resumes_on_a_character_boundary();
test_truncated_decode_does_not_resend();
test_revision_within_the_window_is_seen();
test_revision_behind_the_window_does_not_resend_history();
test_shrinking_transcript_publishes_nothing();
test_reset_forgets_the_published_prefix();
test_empty_and_ascii_edges();
std::cout << "partial_text_test passed\n";
} catch (const std::exception & ex) {
std::cerr << "partial_text_test failed: " << ex.what() << "\n";
return 1;
}
return 0;
}