static const std::unordered_set<std::string> valid = {

"turns", "turn_summaries", "summaries", "documents", "decisions"};

if (valid.find(table) == valid.end())

throw std::runtime_error("Invalid table name: " + table);

if (table == "turns") return "turn_id";

if (table == "turn_summaries") return "turn_summary_id";

if (table == "summaries") return "summary_id";

if (table == "documents") return "document_id";

if (table == "decisions") return "decision_id";

throw std::runtime_error("Invalid table name: " + table);

const std::string id = id_col_of(table);

if (table == "documents")

return std::format("SELECT {}, {} AS text FROM documents ORDER BY {}",

id, kDocEmbedTextSQL, id);

if (table == "turns")

return "SELECT turn_id, user_text, assistant_text FROM turns ORDER BY turn_id";

return std::format("SELECT {}, text FROM {} ORDER BY {}", id, table, id);

auto exec = [&](const std::string& sql) {

char* err = nullptr;

if (sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &err) != SQLITE_OK) {

std::string msg = err ? err : "unknown error";

sqlite3_free(err);

throw std::runtime_error("TextIndex schema: " + msg);

}

};

exec(R"(

)");

// One junction table per text table. FK ON DELETE CASCADE means a deleted

// record's index rows go away automatically — nothing to maintain in the

// write path beyond delete-then-insert for updates.

struct T { const char* junc; const char* base; const char* pk; };

const T tables[] = {

{"turns_terms", "turns", "turn_id"},

{"turn_summaries_terms", "turn_summaries", "turn_summary_id"},

{"summaries_terms", "summaries", "summary_id"},

{"documents_terms", "documents", "document_id"},

{"decisions_terms", "decisions", "decision_id"},

};

for (const auto& t : tables) {

exec(std::format(

"CREATE TABLE IF NOT EXISTS {} (\n"

" {} INTEGER NOT NULL REFERENCES {}({}) ON DELETE CASCADE,\n"

" term_id INTEGER NOT NULL REFERENCES terms(term_id) ON DELETE CASCADE,\n"

" count INTEGER NOT NULL DEFAULT 1,\n"

" PRIMARY KEY ({}, term_id)\n)",

t.junc, t.pk, t.base, t.pk, t.pk));

exec(std::format(

"CREATE INDEX IF NOT EXISTS idx_{}_term ON {}(term_id)",

t.junc, t.junc));

}

// Drop-and-recreate instead of DELETE: on ~900K+ rows, DELETE FROM terms

// has to walk every row, update the UNIQUE index B-tree, and write a WAL

// frame per row -- O(n) work. DROP TABLE is a catalog-only operation

// (SQLite just forgets the table/index exist), so it's near-instant

// regardless of row count; CREATE TABLE then reuses create_schema()'s

// exact DDL, so the two paths can never drift apart.

//

// Wrapped in one transaction: a crash mid-sequence rolls back to the

// pre-reset state instead of leaving tables half-gone.

//

// No manual sqlite_sequence cleanup needed -- a freshly CREATE'd

// AUTOINCREMENT table has no sqlite_sequence row until its first insert.

//

// Any cached Stmt bound to `terms` or a `<table>_terms` junction (e.g.

// select_term_id_, insert_term_, the per-table junction insert/delete

// members) is prepared via sqlite3_prepare_v2, which SQLite transparently

// reprepares against the new schema on its next step() -- no code

// elsewhere needs to change for this.

char* err = nullptr;

auto exec = [&](const char* sql) {

if (sqlite3_exec(db_, sql, nullptr, nullptr, &err) != SQLITE_OK) {

std::string msg = err ? err : "unknown error";

sqlite3_free(err);

throw std::runtime_error("TextIndex reset_terms_table: " + msg);

}

};

exec("BEGIN IMMEDIATE");

try {

// Junction tables reference terms(term_id); drop them first so no

// statement can observe a junction row pointing at a gone table

// (though within one transaction this is moot for correctness, it

// keeps drop order mirroring the FK dependency direction).

exec("DROP TABLE IF EXISTS turns_terms");

exec("DROP TABLE IF EXISTS turn_summaries_terms");

exec("DROP TABLE IF EXISTS summaries_terms");

exec("DROP TABLE IF EXISTS documents_terms");

exec("DROP TABLE IF EXISTS decisions_terms");

exec("DROP TABLE IF EXISTS terms");

create_schema(); // CREATE TABLE/INDEX IF NOT EXISTS -- recreates all 6, empty

exec("COMMIT");

} catch (...) {

exec("ROLLBACK");

throw;

}

// SELECT-first, not INSERT-OR-IGNORE-first: AUTOINCREMENT allocates the

// next term_id BEFORE the uniqueness check runs, so an "ignored" insert

// (term already exists) still permanently burns an id -- with upsert_term

// called on every tokenize of every store/reindex/backfill, that leaked

// ~13M ids against ~900K real terms in the live DB. Checking first avoids

// ever attempting the redundant insert for an existing term.

Stmt& sel = select_term_id();

sel.bind(1, term);

int existing = sel.step() ? sel.column_int(0) : 0;

sqlite3_reset(sel.raw()); // release the read snapshot immediately, see below

if (existing != 0) return existing;

Stmt& ins = insert_term();

ins.bind(1, term);

ins.exec();

return static_cast<int>(sqlite3_last_insert_rowid(db_));

const std::string& table, int id,

const std::unordered_map<int, int>& term_counts) {

validate_table(table);

{

Stmt& del = delete_terms_stmt(table);

del.bind(1, id);

del.exec();

}

for (const auto& [term_id, count] : term_counts) {

Stmt& ins = insert_terms_stmt(table);

ins.bind(1, id).bind(2, term_id).bind(3, count);

ins.exec();

}

int unigram_count, int bigram_count) {

validate_table(table);

Stmt& upd = update_counts_stmt(table);

upd.bind(1, unigram_count).bind(2, bigram_count).bind(3, id);

upd.exec();

validate_table(table);

Stmt& s = count_stmt(table);

int result = s.step() ? s.column_int(0) : 0;

sqlite3_reset(s.raw()); // release the read snapshot immediately, see upsert_term

return result;

validate_table(table);

Stmt& s = doc_freq_stmt(table);

s.bind(1, term_id);

int result = s.step() ? s.column_int(0) : 0;

sqlite3_reset(s.raw()); // release the read snapshot immediately, see upsert_term

return result;

Stmt& s = select_term_id();

s.bind(1, term);

int result = s.step() ? s.column_int(0) : 0;

sqlite3_reset(s.raw()); // release the read snapshot immediately, see upsert_term

return result;

const std::string& table, int term_id) {

validate_table(table);

Stmt& s = postings_stmt(table);

s.bind(1, term_id);

std::vector<Posting> out;

while (s.step()) {

out.push_back({s.column_int(0), s.column_int(1),

s.column_int(2), s.column_int(3)});

}

return out;

validate_table(table);

const std::string id_col = id_col_of(table);

const std::string select_sql = text_select_of(table);

int total = 0;

if (progress) {

Stmt& c = count_stmt(table);

if (c.step()) total = c.column_int(0);

}

// Two-pass batches: read a bounded batch (finalize the SELECT), then write.

// Walking a SELECT while UPDATE-ing the same table on one connection can

// invalidate the cursor mid-scan (see rebuild_phon's note).

//

// This SELECT's shape is loop-invariant (select_sql doesn't change across

// outer iterations) but reindex_table() itself is a rare admin/CLI

// operation, not a per-row hot path -- so it's hoisted above the loop for

// clarity, not promoted to a cached instance member.

constexpr int kBatch = 200;

struct Row { int id; std::string text; };

int done = 0;

int last_id = 0;

for (;;) {

std::vector<Row> batch;

batch.reserve(kBatch);

{

// select_sql is ORDER BY id; skip ids already processed to page.

Stmt s(db_, select_sql);

while (s.step()) {

int id = s.column_int(0);

if (id <= last_id) continue;

std::string text = s.column_text(1);

if (table == "turns") {

std::string a = s.column_text(2);

if (!a.empty()) text += "\n" + a;

}

batch.push_back({id, std::move(text)});

if (static_cast<int>(batch.size()) >= kBatch) break;

}

}

if (batch.empty()) break;

for (const auto& row : batch) {

index_record(table, row.id, row.text);

last_id = row.id;

++done;

if (progress) {

std::cout << std::format("\rReindexing {}: {}/{}",

table, done, total);

std::cout.flush();

}

}

}

if (progress) std::cout << "\n";

return done;