Repository navigation
Expand file tree
/
Copy pathstats_logger.cpp
More file actions
192 lines (176 loc) · 7.26 KB
/
Copy pathstats_logger.cpp
File metadata and controls
192 lines (176 loc) · 7.26 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
/**
* StatsLogger implementation — see stats_logger.h.
*
* Compiled ONLY under -DRAGGER_STATS (CMake option RAGGER_STATS, default OFF).
* Best-effort, non-throwing: every method guards its own errors so a stats
* failure can never reach the search path.
*/
#ifdef RAGGER_STATS
#include "stats_logger.h"
#include <sqlite3.h>
#include <algorithm>
#include "config.h" // expand_path
#include "util/time.h" // db_timestamp
#include "util/sqlite.h" // Stmt (RAII)
#include "Logger.h"
namespace ragger {
namespace {
constexpr const char* kSchema = R"sql(
PRAGMA journal_mode=WAL;
CREATE TABLE IF NOT EXISTS lookups (
id INTEGER PRIMARY KEY,
ts TEXT NOT NULL,
query TEXT NOT NULL,
limit_n INTEGER NOT NULL,
min_score REAL NOT NULL,
result_count INTEGER NOT NULL,
elapsed_ms REAL
);
CREATE TABLE IF NOT EXISTS hits (
id INTEGER PRIMARY KEY,
lookup_id INTEGER NOT NULL REFERENCES lookups(id) ON DELETE CASCADE,
rank INTEGER NOT NULL,
memory_id INTEGER NOT NULL,
score REAL NOT NULL,
vec_score REAL,
bm25_score REAL,
phon_score REAL,
blended REAL,
collection TEXT,
snippet TEXT
);
CREATE INDEX IF NOT EXISTS idx_hits_lookup ON hits(lookup_id);
)sql";
// Pull a human-readable collection/source label out of a result's metadata,
// tolerating whatever shape the row carries (best-effort, never throws).
std::string collection_of(const SearchResult& r) {
try {
if (r.metadata.is_object()) {
for (const char* key : {"collection", "kind", "level", "source"}) {
auto it = r.metadata.find(key);
if (it != r.metadata.end() && it->is_string())
return it->get<std::string>();
}
}
} catch (...) {}
return {};
}
std::string snippet_of(const SearchResult& r, std::size_t max = 160) {
std::string s = r.text;
if (s.size() > max) s.resize(max);
return s;
}
} // namespace
StatsLogger::StatsLogger(const std::string& db_path) {
const std::string path =
db_path.empty() ? config().resolved_stats_db_path() : expand_path(db_path);
try {
if (sqlite3_open(path.c_str(), &db_) != SQLITE_OK) {
Diskerror::Logger::warn(std::string("stats: open failed: ") +
(db_ ? sqlite3_errmsg(db_) : "unknown"));
if (db_) { sqlite3_close(db_); db_ = nullptr; }
return;
}
sqlite3_busy_timeout(db_, 2000);
char* err = nullptr;
if (sqlite3_exec(db_, kSchema, nullptr, nullptr, &err) != SQLITE_OK) {
Diskerror::Logger::warn(std::string("stats: schema failed: ") +
(err ? err : "unknown"));
sqlite3_free(err);
sqlite3_close(db_);
db_ = nullptr;
return;
}
// In-place migration: a stats.db created before the per-signal breakdown
// has a `hits` table without the vec/bm25/phon/blended columns, and
// CREATE TABLE IF NOT EXISTS won't add them. Add each if missing so the
// INSERT keeps working. SQLite has no ADD COLUMN IF NOT EXISTS; probe
// pragma_table_info and swallow the "duplicate column" case.
for (const char* col : {"vec_score", "bm25_score", "phon_score", "blended"}) {
std::string probe = std::string(
"SELECT 1 FROM pragma_table_info('hits') WHERE name='") + col + "'";
bool exists = false;
sqlite3_stmt* ps = nullptr;
if (sqlite3_prepare_v2(db_, probe.c_str(), -1, &ps, nullptr) == SQLITE_OK) {
exists = (sqlite3_step(ps) == SQLITE_ROW);
}
sqlite3_finalize(ps);
if (!exists) {
std::string alter = std::string("ALTER TABLE hits ADD COLUMN ") + col + " REAL";
sqlite3_exec(db_, alter.c_str(), nullptr, nullptr, nullptr); // best-effort
}
}
Diskerror::Logger::debug("stats: logging enabled → " + path);
} catch (...) {
if (db_) { sqlite3_close(db_); db_ = nullptr; }
}
}
StatsLogger::~StatsLogger() {
if (db_) sqlite3_close(db_);
}
void StatsLogger::log_lookup(const std::string& query,
int limit,
float min_score,
const std::vector<SearchResult>& results,
double elapsed_ms,
int top_n) {
if (!db_) return;
try {
// One transaction per lookup keeps the lookup row + its hits atomic
// and cheap. A failure rolls back and is swallowed.
if (sqlite3_exec(db_, "BEGIN", nullptr, nullptr, nullptr) != SQLITE_OK)
return;
sqlite3_int64 lookup_id = 0;
{
Stmt s(db_,
"INSERT INTO lookups(ts,query,limit_n,min_score,result_count,elapsed_ms) "
"VALUES(?,?,?,?,?,?)");
s.bind(1, db_timestamp())
.bind(2, query)
.bind(3, limit)
.bind(4, static_cast<double>(min_score))
.bind(5, static_cast<int>(results.size()));
if (elapsed_ms < 0.0) s.bind_null(6); else s.bind(6, elapsed_ms);
if (!s.exec()) { sqlite3_exec(db_, "ROLLBACK", nullptr, nullptr, nullptr); return; }
lookup_id = sqlite3_last_insert_rowid(db_);
}
const int cap = (top_n <= 0)
? static_cast<int>(results.size())
: std::min<int>(top_n, static_cast<int>(results.size()));
for (int i = 0; i < cap; ++i) {
const SearchResult& r = results[i];
Stmt s(db_,
"INSERT INTO hits(lookup_id,rank,memory_id,score,"
"vec_score,bm25_score,phon_score,blended,collection,snippet) "
"VALUES(?,?,?,?,?,?,?,?,?,?)");
s.bind(1, static_cast<int64_t>(lookup_id))
.bind(2, i + 1)
.bind(3, r.id)
.bind(4, static_cast<double>(r.score));
// Per-signal breakdown. A -1 sentinel means the signal was inactive
// for this search (bm25 disabled) → store NULL so analysis queries
// can AVG()/filter without a magic number skewing results. A real
// 0.0 (present but no contribution) is preserved. phon_score is
// always -1/NULL now (v0.16: folded into bm25_score).
auto bind_signal = [&](int idx, float v) {
if (v < 0.0f) s.bind_null(idx); else s.bind(idx, static_cast<double>(v));
};
bind_signal(5, r.vec_score);
bind_signal(6, r.bm25_score);
bind_signal(7, r.phon_score);
s.bind(8, static_cast<double>(r.blended));
const std::string coll = collection_of(r);
if (coll.empty()) s.bind_null(9); else s.bind(9, coll);
s.bind(10, snippet_of(r));
s.exec();
}
sqlite3_exec(db_, "COMMIT", nullptr, nullptr, nullptr);
} catch (const std::exception& e) {
sqlite3_exec(db_, "ROLLBACK", nullptr, nullptr, nullptr);
Diskerror::Logger::debug(std::string("stats: log_lookup swallowed: ") + e.what());
} catch (...) {
sqlite3_exec(db_, "ROLLBACK", nullptr, nullptr, nullptr);
}
}
} // namespace ragger
#endif // RAGGER_STATS