Repository navigation
Expand file tree
/
Copy pathtest_sqlite_backend.cpp
More file actions
4552 lines (4003 loc) · 183 KB
/
Copy pathtest_sqlite_backend.cpp
File metadata and controls
4552 lines (4003 loc) · 183 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/**
* SQLite backend integration tests
*
* Requires ONNX model files at the configured model_dir.
* Uses a temp DB that's deleted after the test.
*/
#include "config.h"
#include "embedder.h"
#include "sqlite/backend.h"
#include "sqlite/status.h"
#include "sqlite/tables.h"
#include "user_store.h"
#include "auth.h"
#include "util/time.h"
#include "util/fs.h"
#include "util/sqlite.h"
#include "test_env.h"
#include <sqlite3.h>
#include "check.h"
#include <filesystem>
#include <iostream>
#include <print>
#include <regex>
#include <thread>
#include <chrono>
#include <ctime>
#include <stdexcept>
#include <set>
#include <vector>
namespace fs = std::filesystem;
static const std::string TEMP_DB = "/tmp/ragger_test_backend.db";
// Clean up temp DB and WAL/SHM files
static void cleanup() {
fs::remove(TEMP_DB);
fs::remove(TEMP_DB + "-wal");
fs::remove(TEMP_DB + "-shm");
// BM25 index table lives in same DB, no separate file
}
// -----------------------------------------------------------------------
// Tests
// -----------------------------------------------------------------------
void test_store_and_count(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
CHECK(db.count() == 0);
db.store("Hello world, this is a test memory.");
CHECK(db.count() == 1);
db.store("Another memory about something else.");
CHECK(db.count() == 2);
db.close();
cleanup();
}
void test_store_with_metadata(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
// PHASE8: notes are knowledge rows; a level other than "note" becomes
// a `level:<value>` tag (there is no level column any more).
ragger::json meta = {
{"level", "session"},
{"tags", {"test", "unit"}}
};
std::string id = db.store("Metadata round-trip test.", meta);
CHECK(!id.empty());
auto all = db.load_all();
CHECK(all.size() == 1);
// Tags stored as comma-separated string in the tags column, with the
// level folded in.
CHECK(all[0].metadata["tags"] == "test,unit,level:session");
CHECK(all[0].metadata["kind"] == "note");
db.close();
cleanup();
}
void test_search_basic(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
db.store("The capital of France is Paris.");
db.store("SQLite is a lightweight database engine.");
db.store("Machine learning uses neural networks for prediction.");
auto resp = db.search("What is the capital of France?", 3, 0.0f);
CHECK(!resp.results.empty());
// The France/Paris doc should be the top result
CHECK(resp.results[0].text.find("Paris") != std::string::npos);
db.close();
cleanup();
}
// FTS5 MATCH is built from arbitrary user text; punctuation/quotes/operators
// must not produce a syntax error. The keyword pass should degrade to "no
// keyword contribution" rather than throw, and vector search still returns.
void test_search_fts_special_chars(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
db.store("SQLite is a lightweight embedded database engine.");
db.store("The Eiffel Tower stands in Paris.");
// Queries laden with FTS5-significant characters — must not throw.
const char* nasty[] = {
"database \"engine\" AND (lightweight)",
"what is SQLite? -- punctuation: *,^,:",
"NEAR(\"paris", // unbalanced quote / bare operator
"***", // no usable tokens → keyword pass skipped
};
for (const char* q : nasty) {
auto resp = db.search(q, 5, 0.0f); // must not throw
(void)resp;
}
// A normal keyword still resolves through the hybrid path.
auto resp = db.search("lightweight database", 5, 0.0f);
CHECK(!resp.results.empty());
CHECK(resp.results[0].text.find("SQLite") != std::string::npos);
db.close();
cleanup();
}
void test_search_min_score(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
db.store("The weather in Los Angeles is sunny.");
db.store("Quantum physics studies subatomic particles.");
// Very high min_score should return no results for a vague query
auto resp = db.search("random unrelated gibberish xyzzy", 5, 0.99f);
CHECK(resp.results.empty());
db.close();
cleanup();
}
// Regression for bug M4: min_score must be applied BEFORE top-k truncation,
// not after. Previously, partial_sort picked the top `limit` candidates by
// blended score and only then dropped any whose raw cosine was below
// min_score — so a high-blend/low-cosine candidate could occupy a slot and
// get filtered out, leaving fewer results than qualifying candidates existed.
// Here we assert two invariants that the post-filter ordering can violate:
// (1) every returned result's score is >= min_score, and
// (2) results are sorted by blended score with no min_score "holes" — i.e.
// we never return fewer than min(limit, #qualifying) items.
void test_search_min_score_before_topk(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
// A cluster of clearly on-topic docs (high cosine to the query) plus some
// off-topic ones. With a moderate threshold the on-topic docs qualify.
db.store("Paris is the capital city of France.");
db.store("France's capital, Paris, sits on the Seine river.");
db.store("The capital of France is the city of Paris.");
db.store("Lyon and Marseille are large French cities, not the capital.");
db.store("Bananas are a tropical fruit rich in potassium.");
db.store("The mitochondria is the powerhouse of the cell.");
const float thr = 0.25f;
auto resp = db.search("What is the capital of France?", 5, thr);
// (1) Invariant: nothing below threshold is ever returned.
for (const auto& r : resp.results)
CHECK(r.score >= thr);
// (2) Count how many candidates actually clear the threshold by asking for
// a huge limit; with the fix, a smaller limit returns exactly
// min(limit, qualifying), never fewer due to top-k stealing.
auto all = db.search("What is the capital of France?", 100, thr);
size_t qualifying = all.results.size();
size_t expected = std::min<size_t>(5, qualifying);
CHECK(resp.results.size() == expected);
db.close();
cleanup();
}
void test_search_limit(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
for (int i = 0; i < 10; ++i) {
db.store("Test document number " + std::to_string(i) + " with some content.");
}
auto resp = db.search("test document", 3, 0.0f);
CHECK((int)resp.results.size() <= 3);
db.close();
cleanup();
}
void test_load_all(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
db.store("First memory.");
db.store("Second memory.");
db.store("Third memory.");
auto all = db.load_all();
CHECK(all.size() == 3);
db.close();
cleanup();
}
void test_rebuild_embeddings(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
// Store some test documents
db.store("Embedding rebuild test document one.");
db.store("Embedding rebuild test document two.");
db.store("Something completely different here.");
// Rebuild all embeddings
int count = db.rebuild_embeddings(emb);
CHECK(count == 3);
// Search should still work after rebuild
auto resp = db.search("embedding test", 5, 0.0f);
CHECK(!resp.results.empty());
db.close();
cleanup();
}
void test_rebuild_embeddings_empty_db(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
// Rebuild on empty DB should return 0
int count = db.rebuild_embeddings(emb);
CHECK(count == 0);
db.close();
cleanup();
}
void test_rebuild_embeddings_count_matches(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
// Store various numbers of documents
for (int i = 0; i < 7; ++i) {
db.store("Test document number " + std::to_string(i));
}
int total_count = db.count();
int rebuild_count = db.rebuild_embeddings(emb);
// Rebuild count should match total count
CHECK(rebuild_count == total_count);
CHECK(rebuild_count == 7);
db.close();
cleanup();
}
// search_text_only() is the fallback used when vector search is unusable —
// embedding drift at startup, and while a re-embed is running. It queries every
// FTS corpus including turn_summaries, which has no created_at column; naming
// the wrong timestamp column there threw out of the whole function, so the
// fallback returned nothing at all and the failure only showed up in a log line
// nobody was reading. Exercise every corpus so a column rename can't do it again.
void test_search_text_only(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
db.store("A summary about harpsichord tuning.");
ragger::DocumentChunk doc;
doc.text = "A document about harpsichord maintenance.";
doc.title = "Harpsichords";
doc.path = "harpsichord.md";
db.store_document(doc);
db.store_decision("Decided to buy a harpsichord.", "current", "music");
int turn_id = db.store_turn("What is a harpsichord?",
"A plucked keyboard instrument.", "test-model");
// An L2 turn summary is the corpus that carried the bug. It only matters
// if the query actually HITS it — the bad column sat in the per-row fetch,
// which never runs when a corpus has no matches, so an empty
// turn_summaries table would let the bug through.
db.finalize_turn_summary(turn_id, "User asked about the harpsichord.",
"test-model");
CHECK(db.turn_summary_exists(turn_id));
// Must not throw, and must actually find the keyword across corpora.
auto resp = db.search_text_only("harpsichord", 10);
CHECK(resp.timing.contains("text_only"));
CHECK(!resp.results.empty());
// Every hit carries a source label and no vector signal.
bool saw_turn_summary = false;
for (const auto& r : resp.results) {
CHECK(r.metadata.contains("source"));
CHECK(r.vec_score < 0.0f);
if (r.metadata["source"] == "turn_summary") {
saw_turn_summary = true;
CHECK(!r.timestamp.empty()); // resolved from turn_datetime
}
}
CHECK(saw_turn_summary);
// A query that matches nothing returns cleanly rather than throwing.
auto none = db.search_text_only("zzzzznotawordzzzzz", 5);
CHECK(none.results.empty());
db.close();
cleanup();
}
void test_search_timing(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
db.store("Timing test document.");
auto resp = db.search("timing", 5, 0.0f);
// Timing JSON should have keys
CHECK(resp.timing.contains("embedding_ms") || resp.timing.contains("total_ms") ||
!resp.timing.empty());
db.close();
cleanup();
}
void test_delete_memory(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
// Store a memory
std::string id1 = db.store("Memory to delete.");
std::string id2 = db.store("Memory to keep.");
CHECK(db.count() == 2);
// Delete the first memory
bool deleted = db.delete_memory(std::stoi(id1));
CHECK(deleted);
CHECK(db.count() == 1);
// Verify the correct one was deleted
auto all = db.load_all();
CHECK(all.size() == 1);
CHECK(all[0].text == "Memory to keep.");
// Try deleting non-existent ID
deleted = db.delete_memory(99999);
CHECK(!deleted);
CHECK(db.count() == 1);
db.close();
cleanup();
}
void test_delete_batch(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
// Store 3 memories
std::string id1 = db.store("First memory.");
std::string id2 = db.store("Second memory.");
std::string id3 = db.store("Third memory.");
CHECK(db.count() == 3);
// Delete 2 by ID
std::vector<int> to_delete = {std::stoi(id1), std::stoi(id3)};
int deleted_count = db.delete_batch(to_delete);
CHECK(deleted_count == 2);
CHECK(db.count() == 1);
// Verify the correct one remains
auto all = db.load_all();
CHECK(all.size() == 1);
CHECK(all[0].text == "Second memory.");
// Empty vector → returns 0
deleted_count = db.delete_batch({});
CHECK(deleted_count == 0);
// Delete with non-existent IDs
deleted_count = db.delete_batch({99999, 88888});
CHECK(deleted_count == 0);
db.close();
cleanup();
}
void test_search_by_metadata(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
// Lean v2: filterable columns are level / status / tags (+ time window).
db.store("Apple note.", {{"level", "turn"}, {"tags", {"fruit"}}});
db.store("Banana note.", {{"level", "turn"}, {"tags", {"fruit", "tropical"}}});
db.store("Car note.", {{"level", "session"}, {"tags", {"vehicle"}}});
// Filter by level (PHASE8: level lives as a `level:<value>` tag)
auto results = db.search_by_metadata({{"level", "turn"}});
CHECK(results.size() == 2);
for (auto& r : results)
CHECK(r.metadata["tags"].get<std::string>().find("level:turn") != std::string::npos);
// Filter by tag (LIKE on the tags column)
results = db.search_by_metadata({{"tags", "vehicle"}});
CHECK(results.size() == 1);
CHECK(results[0].text == "Car note.");
// AND across columns
results = db.search_by_metadata({{"level", "turn"}, {"tags", "tropical"}});
CHECK(results.size() == 1);
CHECK(results[0].text == "Banana note.");
// Limit respected
results = db.search_by_metadata({{"level", "turn"}}, 1);
CHECK(results.size() == 1);
// No matches → empty
results = db.search_by_metadata({{"level", "project"}});
CHECK(results.empty());
// Unsupported key under lean schema → empty
results = db.search_by_metadata({{"color", "red"}});
CHECK(results.empty());
db.close();
cleanup();
}
void test_user_management(ragger::Embedder& emb) {
cleanup();
// User management now uses sqlite::Backend (separate from storage backend)
ragger::UserStore umgr(TEMP_DB);
// create_user → returns valid ID
int user_id = umgr.create_user("testuser", "abc123hash");
CHECK(user_id > 0);
int admin_id = umgr.create_user("adminuser", "def456hash");
CHECK(admin_id > 0);
CHECK(admin_id != user_id);
// get_user_by_token_hash → finds created user
auto user_opt = umgr.get_user_by_token_hash("abc123hash");
CHECK(user_opt.has_value());
CHECK(user_opt->username == "testuser");
CHECK(user_opt->token_hash == "abc123hash");
// get_user_by_username → finds created user
user_opt = umgr.get_user_by_username("adminuser");
CHECK(user_opt.has_value());
CHECK(user_opt->username == "adminuser");
CHECK(user_opt->token_hash == "def456hash");
// get_user_by_token_hash with wrong hash → nullopt
user_opt = umgr.get_user_by_token_hash("wronghash");
CHECK(!user_opt.has_value());
// get_user_by_username with wrong name → nullopt
user_opt = umgr.get_user_by_username("nonexistent");
CHECK(!user_opt.has_value());
cleanup();
}
void test_delete_respects_keep(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
int initial_count = db.count();
// Store a memory with {"keep": true} in metadata
ragger::json meta = {{"keep", true}, {"collection", "memory"}};
auto id = db.store("protected memory", meta);
// Try to delete — should return false
CHECK(!db.delete_memory(std::stoi(id)));
// Memory should still exist
CHECK(db.count() == initial_count + 1);
// Store without keep
auto id2 = db.store("deletable memory", {{"collection", "memory"}});
// Delete should work
CHECK(db.delete_memory(std::stoi(id2)));
// Only the protected one should remain
CHECK(db.count() == initial_count + 1);
db.close();
cleanup();
}
void test_delete_batch_respects_keep(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
// Store mix of keep and non-keep
ragger::json keep_meta = {{"keep", true}, {"collection", "memory"}};
ragger::json normal_meta = {{"collection", "memory"}};
auto id1 = db.store("keep me", keep_meta);
auto id2 = db.store("delete me", normal_meta);
auto id3 = db.store("delete me too", normal_meta);
int initial_count = db.count();
// Batch delete all three
int deleted = db.delete_batch({std::stoi(id1), std::stoi(id2), std::stoi(id3)});
// Only 2 should be deleted (not the keep one)
CHECK(deleted == 2);
CHECK(db.count() == initial_count - 2);
// Verify the protected one remains
auto all = db.load_all();
bool found_protected = false;
for (const auto& mem : all) {
if (mem.text == "keep me") {
found_protected = true;
break;
}
}
CHECK(found_protected);
db.close();
cleanup();
}
void test_timestamp_format(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
db.store("Timestamp format test.");
auto all = db.load_all();
CHECK(all.size() == 1);
// PHASE8: load_all renders knowledge.created_at as local
// "YYYY-MM-DD HH:MM:SS" (same rendering as every other read path).
auto& ts = all[0].timestamp;
CHECK(!ts.empty());
std::tm tm{};
CHECK(strptime(ts.c_str(), "%Y-%m-%d %H:%M:%S", &tm) != nullptr);
tm.tm_isdst = -1; // let mktime resolve DST (strptime leaves it 0)
int64_t epoch_val = static_cast<int64_t>(mktime(&tm));
int64_t now_epoch = static_cast<int64_t>(std::time(nullptr));
CHECK(epoch_val > 0);
CHECK(epoch_val <= now_epoch + 5); // not in the future
CHECK(now_epoch - epoch_val < 300); // stored "just now"
db.close();
cleanup();
}
// Guards the read-path wiring: the generic store/read API operates on the
// unified `knowledge` table as notes (PHASE8; formerly the lean `summaries`
// table), not the dropped pre-v2 `memories` table. tags round-trip with the
// level folded in as a tag; there is no metadata blob.
void test_v2_summaries_backing(ragger::Embedder& emb) {
cleanup();
{
ragger::sqlite::Backend db(emb, TEMP_DB);
db.store("v2 summaries backing check.",
{{"level", "session"}, {"tags", {"note"}}});
auto all = db.load_all();
CHECK(all.size() == 1);
CHECK(all[0].metadata["kind"] == "note");
CHECK(all[0].metadata["tags"] == "note,level:session");
db.close();
}
// Inspect the raw DB: the row must be in `knowledge`, and the legacy
// `memories` table must not exist in a fresh database.
sqlite3* raw = nullptr;
CHECK(sqlite3_open(TEMP_DB.c_str(), &raw) == SQLITE_OK);
sqlite3_stmt* st = nullptr;
sqlite3_prepare_v2(raw, "SELECT COUNT(*) FROM knowledge WHERE kind='note'", -1, &st, nullptr);
CHECK(sqlite3_step(st) == SQLITE_ROW);
CHECK(sqlite3_column_int(st, 0) == 1);
sqlite3_finalize(st);
sqlite3_prepare_v2(raw,
"SELECT COUNT(*) FROM sqlite_master "
"WHERE type='table' AND name='memories'",
-1, &st, nullptr);
CHECK(sqlite3_step(st) == SQLITE_ROW);
CHECK(sqlite3_column_int(st, 0) == 0); // no legacy table in fresh DB
sqlite3_finalize(st);
sqlite3_close(raw);
cleanup();
}
// store_document() must write Level 5 RAG chunks to the lean `documents`
// table with title / tags (subject) / year / path in dedicated columns for
// grouping — none of which are folded into the embedded `text`.
void test_store_document(ragger::Embedder& emb) {
cleanup();
int doc_id = -1;
{
ragger::sqlite::Backend db(emb, TEMP_DB);
ragger::DocumentChunk doc;
doc.text = "The mitochondria is the powerhouse of the cell.";
doc.title = "Cell Biology Primer";
doc.tags = "biology,cells";
doc.year = 2021;
doc.path = "/tmp/primer.md";
doc.chunk_index = 1;
doc_id = db.store_document(doc);
CHECK(doc_id > 0);
db.close();
}
// Verify the dedicated columns round-trip. Per-document metadata
// (title/tags/year/path) lives in document_sources since the v0.15
// normalization migration; documents (chunks) hold only text/chunk_index
// + a document_source_id FK.
sqlite3* raw = nullptr;
CHECK(sqlite3_open(TEMP_DB.c_str(), &raw) == SQLITE_OK);
sqlite3_stmt* st = nullptr;
sqlite3_prepare_v2(raw,
"SELECT d.text, ds.title, ds.tags, ds.year, ds.path, d.chunk_index "
"FROM documents d JOIN document_sources ds "
" ON ds.document_source_id = d.document_source_id "
"WHERE d.document_id = ?",
-1, &st, nullptr);
sqlite3_bind_int(st, 1, doc_id);
CHECK(sqlite3_step(st) == SQLITE_ROW);
auto col = [&](int i) {
const char* s = reinterpret_cast<const char*>(sqlite3_column_text(st, i));
return std::string(s ? s : "");
};
std::string text = col(0);
CHECK(col(1) == "Cell Biology Primer");
CHECK(col(2) == "biology,cells");
CHECK(sqlite3_column_int(st, 3) == 2021);
CHECK(col(4) == "/tmp/primer.md");
CHECK(sqlite3_column_int(st, 5) == 1);
// Grouping metadata must NOT be embedded into the body text.
CHECK(text == "The mitochondria is the powerhouse of the cell.");
CHECK(text.find("Cell Biology Primer") == std::string::npos);
sqlite3_finalize(st);
sqlite3_close(raw);
cleanup();
}
// Uniform turn capture (issue #41): store_turn writes a raw L1 exchange into
// the `turns` table with a model reference; the partial/finalize flow embeds
// the exchange once the assistant reply arrives.
void test_store_turn(ragger::Embedder& emb) {
cleanup();
{
ragger::sqlite::Backend db(emb, TEMP_DB);
// Complete turn in one call — embedded immediately.
int t1 = db.store_turn("What is the capital of France?",
"The capital of France is Paris.", "test-model");
CHECK(t1 > 0);
// Partial turn (no assistant yet) → embedding NULL → finalize fills it.
int t2 = db.store_turn("Tell me about photosynthesis.", "", "test-model",
/*defer_embedding=*/true);
CHECK(t2 > 0 && t2 != t1);
CHECK(db.finalize_turn(t2, "Plants convert sunlight into energy.",
"test-model"));
db.close();
}
sqlite3* raw = nullptr;
CHECK(sqlite3_open(TEMP_DB.c_str(), &raw) == SQLITE_OK);
// Both rows present, both have assistant_text and a non-NULL embedding,
// and a model_id resolved to the 'test-model' models row.
sqlite3_stmt* st = nullptr;
sqlite3_prepare_v2(raw,
"SELECT t.user_text, t.assistant_text, "
" length(t.embedding), m.name "
"FROM turns t JOIN models m ON m.model_id = t.model_id "
"ORDER BY t.turn_id", -1, &st, nullptr);
int rows = 0;
while (sqlite3_step(st) == SQLITE_ROW) {
const char* u = reinterpret_cast<const char*>(sqlite3_column_text(st, 0));
const char* a = reinterpret_cast<const char*>(sqlite3_column_text(st, 1));
int emb_bytes = sqlite3_column_int(st, 2);
const char* model = reinterpret_cast<const char*>(sqlite3_column_text(st, 3));
CHECK(u && a && a[0] != '\0'); // assistant filled in
// Embedded as f16, payload-only (db_version 0.15+; no version byte —
// that tag lives in the sibling embedding_version column now):
// 384 dims * 2 bytes/dim.
CHECK(emb_bytes == 384 * 2);
CHECK(model && std::string(model) == "test-model");
++rows;
}
sqlite3_finalize(st);
CHECK(rows == 2);
// The model row was created exactly once, not duplicated.
sqlite3_prepare_v2(raw, "SELECT COUNT(*) FROM models WHERE name='test-model'",
-1, &st, nullptr);
CHECK(sqlite3_step(st) == SQLITE_ROW);
CHECK(sqlite3_column_int(st, 0) == 1);
sqlite3_finalize(st);
// finalize_turn on a non-existent id returns false.
{
ragger::sqlite::Backend db2(emb, TEMP_DB);
CHECK(!db2.finalize_turn(99999, "x", "test-model"));
db2.close();
}
sqlite3_close(raw);
cleanup();
}
// Retry/regeneration dedup: when the agent re-answers the SAME user prompt
// without an intervening new prompt, store_turn must keep-latest (update the
// prior row's assistant_text in place) instead of inserting a duplicate. The
// match is cross-session by design — a session change between two identical
// prompts is itself a breakage signal (TUI crash-restart minting fresh GUIDs).
void test_store_turn_dedup(ragger::Embedder& emb) {
cleanup();
int t1 = 0;
{
ragger::sqlite::Backend db(emb, TEMP_DB);
// First answer to a prompt, in session A.
t1 = db.store_turn("Explain the general_search recipe.",
"It searches summaries.", "model-a",
/*defer_embedding=*/false, "sessionA");
CHECK(t1 > 0);
// Sleep across a second boundary: local_timestamp() has second-level
// resolution, so without a real gap here old_ts would trivially equal
// new_ts and this test would not exercise the drift this guards
// against (a dedup that lands in the same second as the original).
std::this_thread::sleep_for(std::chrono::milliseconds(1100));
// Same prompt re-answered — DIFFERENT session (sessionB), different
// assistant text. Must update t1 in place, not insert a new row.
int t2 = db.store_turn("Explain the general_search recipe.",
"It searches summaries, documents, and decisions.",
"model-b", /*defer_embedding=*/false, "sessionB");
CHECK(t2 == t1); // same row id returned — keep-latest
// Cross a second boundary before the next insert: local_timestamp()
// has second-level resolution, and turn3 landing in the same second
// as t1's dedup-updated created_at is itself a separate known
// same-second collision (harmless for the summarizer's matching, but
// it would otherwise make this assertion block conflate two issues).
std::this_thread::sleep_for(std::chrono::milliseconds(1100));
// A genuinely different prompt still inserts a fresh row.
int t3 = db.store_turn("What about decisions?", "They are first-class.",
"model-b", /*defer_embedding=*/false, "sessionB");
CHECK(t3 != t1);
db.close();
}
sqlite3* raw = nullptr;
CHECK(sqlite3_open(TEMP_DB.c_str(), &raw) == SQLITE_OK);
// Exactly two turn rows: the deduped pair collapsed to one, plus the
// distinct third prompt.
sqlite3_stmt* st = nullptr;
sqlite3_prepare_v2(raw, "SELECT COUNT(*) FROM turns", -1, &st, nullptr);
CHECK(sqlite3_step(st) == SQLITE_ROW);
CHECK(sqlite3_column_int(st, 0) == 2);
sqlite3_finalize(st);
// The surviving deduped row carries the LATEST assistant_text and is not
// duplicated in the FTS index (one hit for the prompt's distinctive term).
sqlite3_prepare_v2(raw,
"SELECT assistant_text FROM turns "
"WHERE user_text = 'Explain the general_search recipe.'",
-1, &st, nullptr);
CHECK(sqlite3_step(st) == SQLITE_ROW);
{
const char* a = reinterpret_cast<const char*>(sqlite3_column_text(st, 0));
CHECK(a && std::string(a).find("decisions") != std::string::npos);
}
// Only one such row (no duplicate).
CHECK(sqlite3_step(st) != SQLITE_ROW);
sqlite3_finalize(st);
// Custom terms index agrees: the prompt's distinctive stemmed bigram term
// resolves to exactly one turn_id (dedup didn't leave a stale/duplicate
// turns_terms row behind). Snowball's English algorithm leaves "general"
// unstemmed (unlike the old Porter pipeline, which reduced it to
// "genera"), so the stored literal bigram token is "general_search".
sqlite3_prepare_v2(raw,
"SELECT COUNT(*) FROM turns_terms tt "
"JOIN terms t ON t.term_id = tt.term_id "
"WHERE t.term = 'general_search'",
-1, &st, nullptr);
CHECK(sqlite3_step(st) == SQLITE_ROW);
CHECK(sqlite3_column_int(st, 0) == 1);
sqlite3_finalize(st);
// Regression: a dedup that moves the turn's created_at must not leave a
// stray turn_summaries row for this turn. Since placeholders no longer
// exist (per Reid's design decision — no placeholder mechanism), the
// deduped turn should have ZERO turn_summaries rows until explicitly
// finalized.
sqlite3_prepare_v2(raw,
"SELECT COUNT(*) FROM turn_summaries ts WHERE ts.turn_id = ?",
-1, &st, nullptr);
sqlite3_bind_int(st, 1, t1);
CHECK(sqlite3_step(st) == SQLITE_ROW);
CHECK(sqlite3_column_int(st, 0) == 0);
sqlite3_finalize(st);
sqlite3_close(raw);
cleanup();
}
// PHASE-REVIEW-0.17 step 2: same-session regeneration dedup. The existing
// test_store_turn_dedup covers the cross-session path (crash-restart GUID
// churn); this pins the ordinary healthy-agent case: the re-answer arrives
// in the SAME session as the original prompt.
void test_store_turn_dedup_same_session(ragger::Embedder& emb) {
cleanup();
const std::string S = "dedup-same-session-guid";
int t1 = 0;
{
ragger::sqlite::Backend db(emb, TEMP_DB);
t1 = db.store_turn("U1", "A1", "test-model",
/*defer_embedding=*/false, S);
CHECK(t1 > 0);
// Same prompt, same session, new answer, inside the 5-minute window
// (no sleep needed: same-second is well within it). Must keep-latest.
int t2 = db.store_turn("U1", "A2", "test-model",
/*defer_embedding=*/false, S);
CHECK(t2 == t1); // same row id returned — dedup branch reached
db.close();
}
{
// Exactly one row for that session after the deduped pair, and the
// surviving row carries the LATEST assistant_text.
sqlite3* raw = nullptr;
CHECK(sqlite3_open(TEMP_DB.c_str(), &raw) == SQLITE_OK);
sqlite3_stmt* st = nullptr;
sqlite3_prepare_v2(raw,
"SELECT COUNT(*) FROM turns t "
"JOIN sessions s ON s.session_id = t.session_id WHERE s.guid = ?",
-1, &st, nullptr);
sqlite3_bind_text(st, 1, S.c_str(), -1, SQLITE_STATIC);
CHECK(sqlite3_step(st) == SQLITE_ROW);
CHECK(sqlite3_column_int(st, 0) == 1);
sqlite3_finalize(st);
sqlite3_prepare_v2(raw,
"SELECT assistant_text FROM turns WHERE turn_id = ?",
-1, &st, nullptr);
sqlite3_bind_int(st, 1, t1);
CHECK(sqlite3_step(st) == SQLITE_ROW);
const char* a = reinterpret_cast<const char*>(sqlite3_column_text(st, 0));
CHECK(a && std::string(a) == "A2");
sqlite3_finalize(st);
sqlite3_close(raw);
}
{
// A genuinely different prompt in the same session inserts a fresh row.
ragger::sqlite::Backend db(emb, TEMP_DB);
int t3 = db.store_turn("U2", "A2b", "test-model",
/*defer_embedding=*/false, S);
CHECK(t3 > 0 && t3 != t1);
db.close();
}
{
sqlite3* raw = nullptr;
CHECK(sqlite3_open(TEMP_DB.c_str(), &raw) == SQLITE_OK);
sqlite3_stmt* st = nullptr;
sqlite3_prepare_v2(raw,
"SELECT COUNT(*) FROM turns t "
"JOIN sessions s ON s.session_id = t.session_id WHERE s.guid = ?",
-1, &st, nullptr);
sqlite3_bind_text(st, 1, S.c_str(), -1, SQLITE_STATIC);
CHECK(sqlite3_step(st) == SQLITE_ROW);
CHECK(sqlite3_column_int(st, 0) == 2);
sqlite3_finalize(st);
sqlite3_close(raw);
}
cleanup();
}
// --- v0.12.0 turn_summaries regression tests -------------------------------
// The turn_id-FK-based turn_summaries table replaces the old
// (session_guid, source_timestamp)-keyed linkage. Per Reid's explicit design
// decision, there is no placeholder-row mechanism: store_turn() never
// creates a turn_summaries row. finalize_turn_summary()/mark_turn_summarized()
// are plain INSERTs (no upsert) — the unique partial index on turn_id
// guards against an accidental duplicate insert for the same turn_id.
// unsummarized_turns() finds turns to process via a LEFT JOIN "no matching
// row" check, not a NULL-text sentinel.
// store_turn() must NOT create any turn_summaries row, even for a complete
// (non-partial) turn — the row only appears once finalize_turn_summary() or
// mark_turn_summarized() is explicitly called.
void test_store_turn_creates_no_turn_summaries_row(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
int t1 = db.store_turn("What is the capital of Spain?",
"The capital of Spain is Madrid.", "test-model");
CHECK(t1 > 0);
CHECK(!db.turn_summary_exists(t1));
sqlite3* raw = nullptr;
CHECK(sqlite3_open(TEMP_DB.c_str(), &raw) == SQLITE_OK);
sqlite3_stmt* st = nullptr;
sqlite3_prepare_v2(raw, "SELECT COUNT(*) FROM turn_summaries WHERE turn_id = ?",
-1, &st, nullptr);
sqlite3_bind_int(st, 1, t1);
CHECK(sqlite3_step(st) == SQLITE_ROW);
CHECK(sqlite3_column_int(st, 0) == 0);
sqlite3_finalize(st);
sqlite3_close(raw);
db.close();
cleanup();
}
// A partial turn (no assistant_text yet) also gets no turn_summaries row —
// same as a complete turn, since store_turn() never creates one at all now.
void test_store_turn_partial_turn_creates_no_turn_summaries_row(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
int t1 = db.store_turn("Tell me about volcanoes.", "", "test-model",
/*defer_embedding=*/true);
CHECK(t1 > 0);
CHECK(!db.turn_summary_exists(t1));
db.close();
cleanup();
}
// finalize_turn_summary() is a plain INSERT now (no upsert): calling it
// twice on the same turn_id must fail on the second call (unique partial
// index on turn_id rejects the duplicate), leaving exactly ONE row with the
// FIRST call's text in place — not the second.
void test_finalize_turn_summary_rejects_duplicate(ragger::Embedder& emb) {
cleanup();
ragger::sqlite::Backend db(emb, TEMP_DB);
int t1 = db.store_turn("Summarize this exchange.",
"First finalize target.", "test-model");
CHECK(t1 > 0);
CHECK(db.finalize_turn_summary(t1, "First summary text.", "summarizer-model"));
CHECK(!db.finalize_turn_summary(t1, "Second summary text.", "summarizer-model"));
sqlite3* raw = nullptr;
CHECK(sqlite3_open(TEMP_DB.c_str(), &raw) == SQLITE_OK);
sqlite3_stmt* st = nullptr;
sqlite3_prepare_v2(raw, "SELECT COUNT(*) FROM turn_summaries WHERE turn_id = ?",
-1, &st, nullptr);
sqlite3_bind_int(st, 1, t1);
CHECK(sqlite3_step(st) == SQLITE_ROW);
CHECK(sqlite3_column_int(st, 0) == 1); // no duplicate row
sqlite3_finalize(st);
sqlite3_prepare_v2(raw, "SELECT text FROM turn_summaries WHERE turn_id = ?",
-1, &st, nullptr);
sqlite3_bind_int(st, 1, t1);
CHECK(sqlite3_step(st) == SQLITE_ROW);
{
const char* txt = reinterpret_cast<const char*>(sqlite3_column_text(st, 0));
CHECK(txt && std::string(txt) == "First summary text.");
}
sqlite3_finalize(st);