Repository navigation
Expand file tree
/
Copy pathcodebase_query.py
More file actions
977 lines (850 loc) · 42.8 KB
/
Copy pathcodebase_query.py
File metadata and controls
977 lines (850 loc) · 42.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
from __future__ import annotations
import asyncio
import re
from dataclasses import dataclass
from loguru import logger
from pydantic_ai import Tool
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from rich.text import Text
from .. import constants as cs
from .. import exceptions as ex
from .. import logs as ls
from ..config import settings
from ..console_marks import status_mark
from ..constants import (
QUERY_CELL_CONTROL_CHARS,
QUERY_CELL_CONTROL_ESCAPE,
QUERY_NOT_AVAILABLE,
QUERY_RESULTS_PANEL_TITLE,
QUERY_SUMMARY_DB_ERROR,
QUERY_SUMMARY_SUCCESS,
QUERY_SUMMARY_TIMEOUT,
QUERY_SUMMARY_TRANSLATION_FAILED,
QUERY_SUMMARY_TRUNCATED,
)
from ..schemas import QueryGraphData
from ..services import ReadOnlyQueryProtocol
from ..services.graph_service import is_query_rejection
from ..services.llm import CypherGenerator
from ..types_defs import ResultRow
from ..utils.terminal_console import terminal_aware_console
from ..utils.token_utils import truncate_results_by_tokens
from . import tool_descriptions as td
# A PROJECTED PROPERTY: `n.qualified_name`, with nothing appended. The
# trailing boundary is what rejects `AS qualified_name_of_thing`, and the
# required `<identifier>.` prefix is what rejects a bare string literal.
_PROJECTED_QUALIFIED_NAME_RE = re.compile(
rf"[A-Z_][A-Z0-9_]*\.{cs.CYPHER_QUALIFIED_NAME_TOKEN}(?![A-Z0-9_])"
)
# A Cypher property read, `<entity>.<property>`. Grouping a projection by
# the entity half says which entities a row exposes, so an entity that
# contributes properties without its qualified name can be spotted.
_PROPERTY_READ_RE = re.compile(r"\b([A-Z_][A-Z0-9_]*)\.([A-Z_][A-Z0-9_]*)")
# A bare entity inside an aggregate, `count(b)`. Such an entity is
# MEASURED without any property being read, so it needs attributing too.
# `count(b.name)` is caught by the property-read pattern instead.
_AGGREGATED_ENTITY_RE = re.compile(
r"\b(?:COUNT|SUM|AVG|MIN|MAX|COLLECT)\(\s*(?:DISTINCT\s+)?([A-Z_][A-Z0-9_]*)\s*\)"
)
# Any entity MEASURED by an aggregate, in either spelling: `count(b)` and
# `count(b.qualified_name)` both report on `b`. The property suffix is an
# optional tail on one pattern rather than a second alternative, so the two
# forms cannot drift apart and the captured group is the entity either way.
_AGGREGATED_READ_RE = re.compile(
r"\b(?:COUNT|SUM|AVG|MIN|MAX|COLLECT)\(\s*(?:DISTINCT\s+)?"
r"([A-Z_][A-Z0-9_]*)(?:\.[A-Z_][A-Z0-9_]*)?\s*\)"
)
# An entity aggregated into a MAGNITUDE -- a number that names nobody.
# `COLLECT` is deliberately absent: it returns the matched entities
# themselves, so the restriction that bounds a count does not make a
# collect safe to exempt (issue #1843).
_MAGNITUDE_AGGREGATED_ENTITY_RE = re.compile(
r"\b(?:COUNT|SUM|AVG|MIN|MAX)\(\s*(?:DISTINCT\s+)?([A-Z_][A-Z0-9_]*)\s*\)"
)
# Every aggregate and whatever it is applied to, so the operand can be
# judged rather than pattern-matched against known-bad spellings. Listing the
# constant forms was the wrong shape: it caught `*` and digits and missed a
# QUOTED string, which binds no alias just as thoroughly (issue #1494).
# `DISTINCT` is stripped from the captured text rather than matched here.
# Writing it as `\(\s*(?:DISTINCT\s+)?([^)]*)\)` put `\s*` beside `[^)]*`,
# and both consume whitespace -- overlapping quantifiers whose failure path
# backtracks super-linearly (python:S8786). The capture already contains any
# DISTINCT, so removing the prefix removes the ambiguity rather than tuning
# around it. The sibling patterns do not share the defect: their captures are
# `[A-Z_][A-Z0-9_]*`, which cannot match whitespace, so nothing overlaps.
_AGGREGATE_OPERAND_RE = re.compile(r"\b(?:COUNT|SUM|AVG|MIN|MAX|COLLECT)\(([^)]*)\)")
# A term that is NOTHING BUT one aggregate call, alias already stripped. Used
# to reject an aggregate sharing its term with anything else.
_SOLE_AGGREGATE_RE = re.compile(r"\s*(?:COUNT|SUM|AVG|MIN|MAX|COLLECT)\([^)]*\)\s*")
# What an operand must BE for its contributors to be checkable: an alias this
# analysis can bind, optionally with one property. Cypher's boolean and null
# literals uppercase into that same shape, so they are excluded by name --
# sound because the language defines exactly three of them.
_BINDABLE_OPERAND_RE = re.compile(r"^[A-Z_][A-Z0-9_]*(?:\.[A-Z_][A-Z0-9_]*)?$")
# The trailing clauses that end a projection, matched on ANY whitespace.
# Built from the same constants rather than spelled again, so the two cannot
# drift. Matching them as literal single-spaced strings meant a NEWLINE
# before `ORDER BY` defeated the cut and left the sort key inside the parsed
# projection, where its `n.qualified_name` read as projected evidence for a
# query returning only `n.name` (issue #1494).
_POST_RETURN_RE = re.compile(
"|".join(
r"\s+".join(re.escape(word) for word in keyword.split())
for keyword in cs.CYPHER_POST_RETURN_KEYWORDS
).join((r"\s(?:", r")")),
)
# One allowed conjunct, and the AND that joins them. Word-boundary split so
# tabs and newlines separate as Cypher treats them, and so an identifier
# containing "and" is not mistaken for the operator.
_CONJUNCT_RE = re.compile(cs.CYPHER_CONJUNCT_PATTERN)
_CONJUNCTION_SEPARATOR_RE = re.compile(cs.CYPHER_CONJUNCTION_SEPARATOR)
# Constructs a scoped query may not use, matched as whole words.
_UNANALYSABLE_RE = re.compile(cs.CYPHER_UNANALYSABLE_PATTERN)
# Text that returns nothing: quoted strings (a constant, not a read) and
# comments (not executed). Blanked before textual analysis so neither is
# mistaken for a property read.
_INERT_TEXT_RE = re.compile(
r"'(?:[^'\\]|\\.)*'"
r'|"(?:[^"\\]|\\.)*"'
r"|//[^\n]*"
r"|/\*.*?\*/",
re.DOTALL,
)
# A project name appearing as a LITERAL in the query text, uppercased by the
# caller. Used to check that an aggregate's restriction names the requested
# project rather than some other one.
_PROJECT_LITERAL_RE = re.compile(
rf"[A-Z0-9_-]+{cs.PROJECT_NAME_DIGEST_MARKER}[0-9A-F]{{{cs.PROJECT_NAME_DIGEST_LEN}}}\.?"
)
# `derive_project_name` builds "<base>__<8 hex digits>". Requiring the whole
# shape, not just the "__" marker, is what stops `__init__` being read as a
# project-qualified name.
_PROJECT_NAME_RE = re.compile(
rf".+{cs.PROJECT_NAME_DIGEST_MARKER}[0-9a-f]{{{cs.PROJECT_NAME_DIGEST_LEN}}}"
)
def scope_rows_to_project(
rows: list[ResultRow], project_name: str | None
) -> list[ResultRow]:
"""`rows` restricted to one project, or unchanged when none is given.
Enforced HERE rather than in the generated Cypher, because the model
writes that query and may omit the filter -- which is precisely what
made the reported cross-project bleed intermittent (issue #1494). A
code-level filter is always-or-never.
Every string in the row is inspected -- values, list and tuple and set
members, and dict KEYS as well as dict values, at any depth. Not a
fixed list of column names: the repo's own queries return
`from_qn`/`to_qn`, and a generated query may label a column anything,
so keying on known names failed open for exactly the shapes that leak.
Enumerated rather than asserted. An earlier version of this sentence
said "every string VALUE", written while describing the recursion, and
dict keys were genuinely unreachable by it.
A row naming no project at all is kept: `RETURN count(n)` identifies
nobody, and discarding it would turn scoping into silent data loss.
Such rows are also unattributable, which is why
`requires_project_evidence` refuses a scoped query that produces them.
"""
if not project_name:
return rows
# The separator makes this a COMPONENT-boundary match, so one project
# cannot swallow another whose name extends it. Reachable: a directory
# named after an existing project's derived name derives again, giving
# `alpha__11912daa` and `alpha__11912daa_extra__f71503af` -- a genuine
# strict prefix, both in the generator's own shape.
prefix = f"{project_name}{cs.SEPARATOR_DOT}"
kept: list[ResultRow] = []
for row in rows:
if _row_is_outside(row, prefix):
continue
kept.append(row)
return kept
@dataclass(frozen=True)
class ProjectEvidence:
"""What `requires_project_evidence` found: truthy when the query can be
scoped, otherwise the check that refused it and the variables or
constructs it refused."""
refusal: cs.ScopeRefusal | None = None
subjects: tuple[str, ...] = ()
def __bool__(self) -> bool:
return self.refusal is None
def requires_project_evidence(
cypher_query: str, project_name: str | None = None
) -> ProjectEvidence:
"""Whether `cypher_query` returns something a project filter can judge.
The result is truthy when it does. When it does not, it names the check
that refused the query and the variables or constructs concerned, so the
refusal can say what would satisfy it (issue #2197).
`scope_rows_to_project` decides per row, from the values it is given. A
query like `RETURN n.name, n.path` hands it rows with no project
evidence at all, so it cannot tell one project's rows from another's --
and it keeps them, since it cannot prove them foreign either.
That gap is closed here rather than there: a SCOPED request whose query
projects no qualified name is refused, so the caller learns the scope
could not be honoured instead of silently receiving every project.
Evidence means a projected PROPERTY (`n.qualified_name`), not the token
appearing somewhere: an alias like `AS qualified_name_of_thing` returns
no qualified name at all.
An aggregate counts only when the query ITSELF restricts by qualified
name. `RETURN count(n)` exposes no names but does expose a MAGNITUDE
spanning every indexed project, so a scoped caller would learn the size
of projects they never asked about. Restricting the match keeps
counting usable without that leak.
"""
executable = _without_inert_text(cypher_query)
# The restriction check needs comments gone but quoted strings kept: a
# predicate inside a comment restricts nothing, while the project literal
# it looks for lives inside quotes (issue #1494).
restrictable = _without_comments(cypher_query)
# DEFAULT-DENY ON STRUCTURE. Four review rounds found four ways to
# satisfy a textual evidence check while returning unattributable data,
# and two more used UNION so only the final branch was inspected.
# Enumerating bypasses is endless -- every construct not yet considered
# is a candidate -- so a scoped query must have the shape the prompt
# already mandates ("MATCH, WHERE, RETURN, LIMIT" with plain aliased
# property reads) and anything else is refused rather than analysed.
if constructs := sorted(
{match.group(0) for match in _UNANALYSABLE_RE.finditer(executable.upper())}
):
return ProjectEvidence(cs.ScopeRefusal.UNANALYSABLE, tuple(constructs))
projection = _return_clause(executable)
if not projection:
return ProjectEvidence(cs.ScopeRefusal.NO_RETURN)
# A projection term must be a bare property read or an aggregate over
# one. `left(b.qualified_name, 3)` mentions a qualified name without
# attributing `b`, so a transformed term cannot count as evidence.
if not _every_term_is_plain(projection):
return ProjectEvidence(cs.ScopeRefusal.TRANSFORMED_TERM)
# EVERY AGGREGATED ENTITY MUST BE RESTRICTED, checked BEFORE the
# attributability branch below, which returns early and would make this
# unreachable for exactly the queries that need it.
#
# Attributability and restriction answer different questions, and only the
# second is the right one for an aggregate. Attributability asks "does this
# row say who it is about"; restriction asks "is this entity confined to my
# project". For a projection of NAMES those coincide. An aggregate returns
# no names, so `count(b.qualified_name)` satisfies attributability
# trivially -- `b` does project its own qualified name -- while the
# MAGNITUDE still counts every `b` in every indexed project. `collect` is
# worse again: it returns the other projects' names themselves.
# A WILDCARD aggregate binds no alias, so its contributors cannot be
# enumerated and none of them can be shown restricted. Grouping does not
# rescue it: `MATCH (a)-[:CALLS]->(b) WHERE a…alpha RETURN
# a.qualified_name, count(*)` returns an ALPHA-labelled row whose total
# counts beta callees, so the label passes the row filter and carries a
# foreign magnitude past it. Refused wherever it appears in a scoped
# projection, not only in an all-aggregate one (issue #1494).
if not _every_aggregate_operand_is_bindable(projection):
return ProjectEvidence(cs.ScopeRefusal.UNBOUND_AGGREGATE)
aggregated_only = _entities_only_ever_aggregated(projection)
if unrestricted := _unrestricted_aliases(
restrictable, project_name, aggregated_only
):
return ProjectEvidence(
cs.ScopeRefusal.UNRESTRICTED_AGGREGATE,
_as_written(executable, unrestricted),
)
# A PROJECTED PROPERTY (`x.qualified_name`), not the bare token: an
# ALIAS containing it -- `RETURN n.name AS qualified_name_of_thing` --
# returns no qualified name at all, yet satisfied a substring match.
#
# And EVERY entity whose properties are returned must carry its own,
# not just one of them: `RETURN a.qualified_name, b.name, b.path`
# yields a row that is partly in-project, where `b`'s name and path
# belong to an entity nothing attributes. One entity's qualified name
# does not vouch for another's properties.
if _PROJECTED_QUALIFIED_NAME_RE.search(projection):
# `aggregated_only` reaches here only when `_restricts_to_project`
# above returned True for it, so these aliases are proven confined.
if unattributed := _unattributed_entities(projection, aggregated_only):
return ProjectEvidence(
cs.ScopeRefusal.UNATTRIBUTED_ENTITY,
_as_written(executable, unattributed),
)
return ProjectEvidence()
# An aggregate exposes no NAMES but does expose a MAGNITUDE: a scoped
# caller receiving `count(n)` over every indexed project learns the
# size of projects they did not ask about. So it counts as evidence
# only when the QUERY ITSELF restricts by qualified name -- then the
# number is attributable and counting stays usable when scoped.
terms = [term.strip() for term in projection.split(cs.CHAR_COMMA)]
all_aggregates = bool(terms) and all(
any(agg in term for agg in cs.CYPHER_AGGREGATE_TOKENS) for term in terms
)
if not all_aggregates:
return ProjectEvidence(cs.ScopeRefusal.NO_QUALIFIED_NAME)
# The restriction must constrain the alias being COUNTED, not merely
# exist. `MATCH (a),(b) WHERE a.qualified_name STARTS WITH "alpha."
# RETURN count(b)` restricts `a` and counts `b`, which nothing bounds.
counted = set(_AGGREGATED_ENTITY_RE.findall(projection))
# `count(*)` binds no alias, so `counted` is EMPTY and the per-alias
# check below is vacuously true -- an all-aggregate projection then
# passed while measuring every node in every project. An aggregate this
# analysis cannot bind to an entity cannot be shown restricted, so it is
# refused (issue #1494). `RETURN n.qualified_name, count(*)` is a
# different shape and is unaffected: it is not all-aggregates, and its
# count is grouped by a column the row filter can judge.
if not counted:
return ProjectEvidence(cs.ScopeRefusal.UNBOUND_AGGREGATE)
if unrestricted := _unrestricted_aliases(restrictable, project_name, counted):
return ProjectEvidence(
cs.ScopeRefusal.UNRESTRICTED_AGGREGATE,
_as_written(executable, unrestricted),
)
return ProjectEvidence()
def unscopeable_message(evidence: ProjectEvidence, project_name: str) -> str:
"""The refusal a scoped caller receives: the check that fired, and what
would satisfy it, named for the variables it concerns (issue #2197)."""
reason = evidence.refusal or cs.ScopeRefusal.NO_QUALIFIED_NAME
subjects = cs.QUERY_SCOPE_LIST_SEPARATOR.join(
cs.QUERY_SCOPE_SUBJECT.format(name=name) for name in evidence.subjects
)
predicates = cs.QUERY_SCOPE_AND_SEPARATOR.join(
cs.QUERY_SCOPE_RESTRICTION.format(name=name, project=project_name)
for name in evidence.subjects
)
qualified_names = cs.QUERY_SCOPE_LIST_SEPARATOR.join(
cs.QUERY_SCOPE_QUALIFIED_NAME.format(name=name) for name in evidence.subjects
)
return cs.QUERY_SUMMARY_UNSCOPEABLE_PREFIX.format(
project=project_name
) + cs.QUERY_SUMMARY_UNSCOPEABLE_REASONS[reason].format(
project=project_name,
subjects=subjects,
predicates=predicates,
qualified_names=qualified_names,
)
def _unrestricted_aliases(
cypher_query: str, project_name: str | None, aliases: set[str]
) -> list[str]:
"""The `aliases` `_restricts_to_project` does not find confined to the
project, each judged alone so the refusal can name them."""
return sorted(
alias
for alias in aliases
if not _restricts_to_project(cypher_query, project_name, {alias})
)
def _as_written(cypher_query: str, names: list[str]) -> tuple[str, ...]:
"""`names`, found uppercased by the analysis, spelled as in the query."""
spelled: list[str] = []
for name in names:
match = re.search(rf"\b{re.escape(name)}\b", cypher_query, re.IGNORECASE)
spelled.append(match.group(0) if match else name)
return tuple(spelled)
def _without_comments(cypher_query: str) -> str:
"""`cypher_query` with COMMENTS blanked and quoted strings kept.
What the restriction check needs, and distinct from
`_without_inert_text`. A comment is not executed, so a project predicate
written inside one restricts nothing -- but a quoted string IS the
project literal the check looks for, so it has to survive.
Reuses `_INERT_TEXT_RE` rather than adding a second pattern: it already
alternates quoted strings BEFORE comments, so a string is consumed as a
string and `"http://x"` is never read as a line comment. Only the
decision differs, so it belongs in the callback, not in a new regex.
"""
def _blank(match: re.Match[str]) -> str:
text = match.group(0)
if text[:1] in ("'", '"'):
return text
return "".join("\n" if ch == "\n" else " " for ch in text)
return _INERT_TEXT_RE.sub(_blank, cypher_query)
def _without_inert_text(cypher_query: str) -> str:
"""`cypher_query` with quoted spans and comments blanked, same length.
Neither returns anything. A property read inside quotes is a CONSTANT
(`RETURN "n.qualified_name" AS lit` projects a string), and one inside
a comment is not executed at all -- but textual matching saw both as
reads. A trailing `// n.qualified_name` was the sharper case, since the
projection is taken from the text after the last RETURN and the comment
sits inside it.
Blanking rather than deleting keeps offsets stable, so the RETURN
marker still falls where it does in the original. Newlines are
preserved so a `//` comment does not swallow the line after it.
Deliberately NOT applied when looking for a literal PROJECT NAME,
which legitimately lives inside quotes.
"""
def _blank(match: re.Match[str]) -> str:
return "".join("\n" if ch == "\n" else " " for ch in match.group(0))
return _INERT_TEXT_RE.sub(_blank, cypher_query)
def _every_term_is_plain(projection: str) -> bool:
"""Whether every returned term is a bare property read or an aggregate.
`left(b.qualified_name, 3)` mentions a qualified name without
attributing `b` -- the value returned is a fragment, and the row filter
would judge the row on a string that is not a qualified name at all.
Aggregates are allowed because they name nobody; a bare alias with no
property read (a literal, a parameter) is allowed for the same reason.
"""
for term in projection.split(cs.CHAR_COMMA):
reads = _PROPERTY_READ_RE.findall(term)
if not reads:
continue
bare = term.split(cs.CYPHER_ALIAS_KEYWORD, 1)[0].strip()
if any(agg in term for agg in cs.CYPHER_AGGREGATE_TOKENS):
# An aggregate excuses ITSELF, not whatever shares its term. This
# skipped the whole term, so `count(a.qualified_name) +
# left(b.qualified_name, 3)` rode along and returned a truncated
# foreign name the row filter cannot recognise as a qualified
# name at all. The term must BE the aggregate (issue #1494).
if not _SOLE_AGGREGATE_RE.fullmatch(bare):
return False
continue
entity, prop = reads[0]
# The term must be exactly `<entity>.<prop>`, optionally aliased.
if bare != f"{entity}{cs.SEPARATOR_DOT}{prop}":
return False
return True
def _every_aggregate_operand_is_bindable(projection: str) -> bool:
"""Whether every aggregate applies to an alias this analysis can check.
`count(*)`, `count(1)`, `count('x')` and `count(true)` all bind nothing,
so their contributors cannot be enumerated and none of them can be shown
restricted. Grouping does not rescue them: the grouping key bounds which
ROWS survive the filter, while the aggregate ranges over MATCHES, so an
in-project label can carry a foreign magnitude past the row filter.
Asked as "is the operand an alias" rather than "is it one of the constant
forms I thought of". The first version listed `*` and digits and missed a
quoted string (issue #1494).
"""
for operand in _AGGREGATE_OPERAND_RE.findall(projection):
candidate = operand.strip()
if candidate.startswith(cs.CYPHER_DISTINCT_KEYWORD):
candidate = candidate[len(cs.CYPHER_DISTINCT_KEYWORD) :].strip()
if not _BINDABLE_OPERAND_RE.match(candidate):
return False
if candidate in cs.CYPHER_LITERAL_OPERANDS:
return False
return True
def _entities_only_ever_aggregated(projection: str) -> set[str]:
"""Entities the projection MEASURES but never returns a plain value for.
The distinction decides whether the row filter can do its job. An entity
also projected as a plain property (`RETURN a.qualified_name, count(a)`)
groups the aggregate by a value the filter judges per row, so a foreign
group is dropped whole. An entity appearing ONLY inside aggregates
contributes no such column, so its magnitude is computed across every
indexed project and arrives as a number nothing can attribute.
Both aggregate spellings count as measuring: `count(b)` and
`count(b.qualified_name)` report on `b` alike. That the second one also
mentions a qualified name is what made it look attributable -- the value
is consumed by the aggregate, not returned.
"""
plain: set[str] = set()
aggregated: set[str] = set()
for term in projection.split(cs.CHAR_COMMA):
if any(agg in term for agg in cs.CYPHER_AGGREGATE_TOKENS):
aggregated.update(_AGGREGATED_READ_RE.findall(term))
continue
for entity, prop in _PROPERTY_READ_RE.findall(term):
# Only the QUALIFIED NAME excuses an aggregate. Any other plain
# property is a grouping column the row filter cannot judge:
# `RETURN n.name, count(n.qualified_name)` returns a name and a
# number, neither attributable, while the count still spans every
# project. An earlier version of this accepted any plain read
# (issue #1494).
if prop == cs.CYPHER_QUALIFIED_NAME_TOKEN:
plain.add(entity)
return aggregated - plain
def _unattributed_entities(
projection: str, restricted: set[str] | None = None
) -> list[str]:
"""The entities read in `projection` that do not also project their qn.
Cypher property reads are `<entity>.<property>`, so grouping the
projection by entity identifier says which entities the row exposes.
An entity contributing properties WITHOUT its qualified name cannot be
attributed to a project, and the row filter -- which judges a row as a
whole -- will keep the row on the strength of some other entity's
qualified name.
Entities reading only their qualified name are fine, and so are
literals, which name no entity at all.
An AGGREGATED entity counts too. `RETURN a.qualified_name, count(b)`
was accepted because `b` appears as a bare identifier inside the
aggregate and never as a property read -- so it never entered this
map, while the count reported how many `b` exist across every project.
An aggregate names nobody but it still MEASURES someone.
`restricted` names the aggregated-only aliases the CALLER has already
proven confined to this project via `_restricts_to_project`. They are
exempt, and only they. Attributability asks "does this row say who it
is about"; the leak it defends against is a MAGNITUDE spanning projects
the caller never asked for. Once restriction has established that an
alias cannot range outside the requested project, that magnitude is
bounded, and refusing anyway costs `RETURN f.qualified_name,
count(c)` -- "how many callers does this function have" -- for no leak
prevented (issue #1843).
The exemption is narrow on purpose. It covers only aliases the caller
passes in, which are only ever `_entities_only_ever_aggregated` that
`_restricts_to_project` returned True for; an alias whose properties
are READ still needs its own qualified name, because that row names an
entity and the row filter must be able to judge it.
"""
reads: dict[str, set[str]] = {}
for entity, prop in _PROPERTY_READ_RE.findall(projection):
reads.setdefault(entity, set()).add(prop)
for entity in _AGGREGATED_ENTITY_RE.findall(projection):
reads.setdefault(entity, set())
# Only an alias that reads NO properties may be exempted: one appearing
# both as `c.name` and inside `count(c)` exposes a name that still has to
# be attributable, so restriction alone does not settle it.
# Only an alias aggregated into a MAGNITUDE may be exempted. `collect(c)`
# returns the entities themselves, so restriction does not make it
# attributable -- and the row filter, which cannot inspect a driver Node,
# drops the whole row, turning a valid query into an empty result rather
# than a refusal (issue #1843).
measured = set(_MAGNITUDE_AGGREGATED_ENTITY_RE.findall(projection))
vouched = {
entity
for entity in restricted or ()
if entity in reads and not reads[entity] and entity in measured
}
return sorted(
entity
for entity, props in reads.items()
if entity not in vouched and cs.CYPHER_QUALIFIED_NAME_TOKEN not in props
)
def _where_is_a_plain_conjunction(body: str) -> bool:
"""Whether `body`'s WHERE clause is only ANDed plain comparisons.
A WHITELIST, because the alternative cannot work. Blacklisting the
operators known to widen or invert a predicate answers "does this
contain a known-bad spelling", while the contract is "does the
restriction bind" -- and Cypher can break the binding without any of
those spellings, via `CASE`, `coalesce`, or any function returning a
boolean.
Everything the system prompt actually mandates is a conjunction of
`<entity>.<property> <op> <value>`, so refusing anything else costs
nothing the model should be emitting and closes the class rather than
enumerating members of it.
"""
marker = body.find(cs.CYPHER_WHERE_KEYWORD)
if marker < 0:
return False
clause = body[marker + len(cs.CYPHER_WHERE_KEYWORD) :]
conjuncts = _CONJUNCTION_SEPARATOR_RE.split(clause)
return all(_CONJUNCT_RE.match(conjunct) for conjunct in conjuncts)
def _restricts_to_project(
cypher_query: str, project_name: str | None, counted: set[str] | None = None
) -> bool:
"""Whether the query narrows `counted`'s rows to `project_name`.
Restricting by *a* qualified name is not restricting to *yours*: a query
filtering on another project's prefix would otherwise pass and return
that project's count to a scoped caller -- the magnitude leak again, one
level deeper.
So the LITERAL project name is required. A parameter's value is
invisible here, so treating "no literal" as "safely parameterised"
accepted a restriction to ANY project -- and, for
`n.qualified_name = m.qualified_name`, to another matched entity.
Nothing useful is lost: the caller knows their own project name and
the prompt already instructs the model to emit it, so the one shape
this can verify is also the one it should get. Interpolation is not
a concern here because the name is validated against `list_projects`
before it reaches the query.
"""
upper = cypher_query.upper()
marker = upper.rfind(cs.CYPHER_RETURN_KEYWORD)
body = upper if marker < 0 else upper[:marker]
# A RESTRICTIVE predicate, not merely a mention. `WHERE
# n.qualified_name IS NOT NULL` matches every indexed node in every
# project, so a count over it spans them all -- yet it satisfied a
# check that only looked for the property appearing before RETURN.
if (
not any(op in body for op in cs.CYPHER_PREFIX_PREDICATES)
or _PROJECTED_QUALIFIED_NAME_RE.search(body) is None
):
return False
if not project_name:
return False
# The WHERE clause must BE a conjunction of plain comparisons, not
# merely lack known-bad operators. Every other check here SEARCHES the
# text for a predicate, which cannot tell "the restriction binds" from
# "the restriction appears": `OR TRUE` makes it optional, `NOT (...)`
# inverts it, and `CASE WHEN <pred> THEN true ELSE true END` contains
# neither token while evaluating true for every row (issue #1494).
if not _where_is_a_plain_conjunction(body):
return False
# The LITERAL project name is required. "No literal" was read as
# "safely parameterised", but a parameter's VALUE is invisible here,
# so that accepted a restriction to any project at all -- and, for
# `n.qualified_name = m.qualified_name`, to another matched entity.
literals = _PROJECT_LITERAL_RE.findall(body)
if not literals:
return False
if not all(
literal.rstrip(cs.SEPARATOR_DOT) == project_name.upper() for literal in literals
):
return False
# Every COUNTED alias must itself be restricted. A restriction on `a`
# says nothing about `count(b)`, and the two differ whenever the query
# matches more than one entity.
return all(
_alias_is_restricted(body, alias, project_name.upper())
for alias in counted or ()
)
def _alias_is_restricted(body: str, alias: str, project_upper: str) -> bool:
"""Whether `body` binds `alias`'s qualified name to THIS project's prefix.
The operand is required, not just the operator. Checking only that
`<alias>.qualified_name` was followed by a prefix predicate accepted a
VACUOUS one -- `STARTS WITH ''` -- and the separate literal check was
satisfied by the project name appearing anywhere, including in a
tautology like `'proj' = 'proj'`. Together those two half-checks passed
a query whose aggregate spanned every project (issue #1494).
"""
# A REGEX cannot authorise a scoped aggregate. The operand check requires
# the value to begin with the project name, which is sound for a literal
# but not for a pattern: `=~ "alpha__aaaa1111|.*"` starts with the project
# and matches every foreign qualified name, satisfying the same textual
# test. Deciding whether an arbitrary pattern is prefix-limited is a
# language question this module cannot answer, so the whole operator is
# refused here rather than inspected (issue #1494).
operators = "|".join(
re.escape(op.strip())
for op in cs.CYPHER_PREFIX_PREDICATES
if op.strip() != cs.CYPHER_REGEX_PREDICATE
)
pattern = re.compile(
rf"\b{re.escape(alias)}\.{cs.CYPHER_QUALIFIED_NAME_TOKEN}\s*"
rf"(?:{operators})\s*['\"]{re.escape(project_upper)}"
)
return pattern.search(body) is not None
def _return_clause(cypher_query: str) -> str:
"""The uppercased RETURN projection, without trailing ORDER BY / LIMIT.
Evidence has to be in what the query RETURNS. A substring match over
the whole query counted `WHERE n.qualified_name STARTS WITH ...` as
evidence while the projection returned only `n.name` -- precisely the
unattributable rows this guard exists to refuse.
"""
upper = cypher_query.upper()
marker = upper.rfind(cs.CYPHER_RETURN_KEYWORD)
if marker < 0:
return ""
projection = upper[marker + len(cs.CYPHER_RETURN_KEYWORD) :]
if cut := _POST_RETURN_RE.search(projection):
projection = projection[: cut.start()]
return projection.strip()
def _looks_like_a_qualified_name(value: str) -> bool:
"""Whether `value` is a project-qualified name rather than free text.
`derive_project_name` produces `<base>__<8 hex digits>`, and the FULL
shape is required -- not merely the `__` marker. `__init__` contains
that marker, so a marker-only test discarded every row carrying
Python's most common method name: the too-aggressive direction, which
fails as badly as leaking.
Matching on dots alone would be worse still, since a docstring or a
path may contain them.
"""
head = value.split(cs.SEPARATOR_DOT, 1)[0]
return _PROJECT_NAME_RE.fullmatch(head) is not None
def _row_is_outside(row: ResultRow, prefix: str) -> bool:
"""Whether `row` names any project other than the one `prefix` selects.
Every string VALUE is inspected, not a fixed list of key names. The
repo's own queries return `from_qn`/`to_qn`/`caller_qualified_name`,
and a generated query may label a column anything at all, so keying on
known names would fail open for precisely the shapes that leak.
A row naming no project at all -- `RETURN count(n)` -- is kept, since
it identifies nothing belonging to anyone else.
"""
return any(_names_another_project(value, prefix) for value in row.values())
def _names_another_project(value: object, prefix: str) -> bool:
"""Whether `value`, at any depth, carries a foreign qualified name.
Cypher returns lists and maps -- `collect(m.qualified_name)`, a map
projection -- so inspecting only top-level strings let a whole list of
another project's names ride through untouched.
"""
if isinstance(value, str):
return _looks_like_a_qualified_name(value) and not value.startswith(prefix)
if isinstance(value, dict):
# KEYS as well as values: a map projection keyed BY qualified name
# carries foreign names in a position `.values()` never reaches.
return any(
_names_another_project(item, prefix)
for pair in value.items()
for item in pair
)
if isinstance(value, list | tuple | set | frozenset):
return any(_names_another_project(item, prefix) for item in value)
# Scalars carry no name and are safe to keep -- `RETURN count(n)` must
# survive scoping, and a number identifies nobody.
if value is None or isinstance(value, bool | int | float):
return False
# ANYTHING ELSE FAILS CLOSED. A graph driver's Node, Relationship or Path
# is neither a string nor a built-in container, so it fell through to a
# permissive `return False` and rode through the filter -- then rendered
# with `str()`, printing another project's properties under an active
# scope. Unreadable means unattributable, which is the same rule this
# module applies to unanalysable QUERIES; applying it to values too keeps
# the two halves consistent (issue #1494).
return True
_CONTROL_CHARS = re.compile(QUERY_CELL_CONTROL_CHARS)
def _plain_cell(value: str) -> Text:
"""A result cell that prints exactly what the graph holds (#2260).
Values come from the indexed code and from MCP `annotate` glosses, so
neither Rich markup (`list[int]` lost its subscript, `[/]` raised, a
`[link=...]` became a live hyperlink) nor terminal control sequences
may take effect. `Text` is never parsed as markup; control characters
are shown as `\\xNN` escapes.
"""
return Text(
_CONTROL_CHARS.sub(
lambda m: QUERY_CELL_CONTROL_ESCAPE.format(code=ord(m.group())), value
)
)
def _row_cells(row: ResultRow, encoding: str) -> list[Text]:
cells: list[Text] = []
for value in row.values():
if value is None:
cells.append(Text())
elif isinstance(value, bool):
cells.append(Text(status_mark(value, encoding)))
else:
cells.append(_plain_cell(str(value)))
return cells
def _print_results_table(results: list[ResultRow], console: Console) -> None:
table = Table(
show_header=True,
header_style="bold magenta",
)
for header in results[0].keys():
table.add_column(_plain_cell(str(header)))
for row in results:
table.add_row(*_row_cells(row, console.encoding))
console.print(
Panel(
table,
title=QUERY_RESULTS_PANEL_TITLE,
expand=False,
)
)
def _query_summary(
results: list[ResultRow],
total_count: int,
tokens_used: int,
*,
was_truncated: bool,
) -> str:
if was_truncated or total_count > len(results):
return QUERY_SUMMARY_TRUNCATED.format(
kept=len(results),
total=total_count,
tokens=tokens_used,
max_tokens=settings.QUERY_RESULT_MAX_TOKENS,
)
return QUERY_SUMMARY_SUCCESS.format(count=len(results))
def _scope_refusal(
cypher_query: str, project_name: str | None
) -> QueryGraphData | None:
# BEFORE execution. A query returning no qualified name yields rows the
# filter cannot attribute, so it keeps them, and answering a SCOPED
# request with those would ignore the scope silently. The guard reads
# only the generated Cypher, so nothing required it to wait for results
# -- and running first meant an unscopable request still performed a
# cross-project read against the shared graph. The rows never reached
# the caller, but the database served them, which is the part a caller
# can neither see nor undo.
if project_name is None or (
evidence := requires_project_evidence(cypher_query, project_name)
):
return None
message = unscopeable_message(evidence, project_name)
# `error` as well as `summary`: MCP callers distinguish a refusal from a
# genuine empty result by the error key.
return QueryGraphData(
query_used=cypher_query,
results=[],
summary=message,
error=message,
)
async def _fetch(ingestor: ReadOnlyQueryProtocol, cypher_query: str) -> list[ResultRow]:
return await asyncio.wait_for(
asyncio.to_thread(ingestor.fetch_read_only, cypher_query),
timeout=settings.QUERY_TIMEOUT_S,
)
@dataclass
class _QueryAttempt:
"""The query currently being tried, for the caller's error reports."""
query: str = QUERY_NOT_AVAILABLE
async def _run_query(
ingestor: ReadOnlyQueryProtocol,
cypher_gen: CypherGenerator,
question: str,
project_name: str | None,
attempt: _QueryAttempt,
) -> list[ResultRow] | QueryGraphData:
attempt.query = await cypher_gen.generate(question)
if refusal := _scope_refusal(attempt.query, project_name):
return refusal
try:
return await _fetch(ingestor, attempt.query)
except Exception as e:
# One regeneration, fed the engine's own error, for a query the
# engine refused (issue #2361). Anything else, such as a timeout or
# a lost connection, is not the query's fault.
if not is_query_rejection(e):
raise
rejection = str(e)
logger.warning(ls.TOOL_QUERY_REPAIRING.format(error=rejection))
attempt.query = await cypher_gen.repair(question, attempt.query, rejection)
if refusal := _scope_refusal(attempt.query, project_name):
return refusal
return await _fetch(ingestor, attempt.query)
def create_query_tool(
ingestor: ReadOnlyQueryProtocol,
cypher_gen: CypherGenerator,
console: Console | None = None,
project_name: str | None = None,
) -> Tool:
if console is None:
console = terminal_aware_console(stderr=True)
async def query_codebase_knowledge_graph(
natural_language_query: str,
) -> QueryGraphData:
logger.info(ls.TOOL_QUERY_RECEIVED.format(query=natural_language_query))
attempt = _QueryAttempt()
try:
outcome = await _run_query(
ingestor, cypher_gen, natural_language_query, project_name, attempt
)
if isinstance(outcome, QueryGraphData):
return outcome
cypher_query = attempt.query
# Before the row cap and the token truncation, so a scoped query
# spends its budget on rows the caller can actually use rather
# than on another project's rows that are about to be dropped.
results = scope_rows_to_project(outcome, project_name)
total_count = len(results)
if total_count > settings.QUERY_RESULT_ROW_CAP:
results = results[: settings.QUERY_RESULT_ROW_CAP]
results, tokens_used, was_truncated = truncate_results_by_tokens(
results,
max_tokens=settings.QUERY_RESULT_MAX_TOKENS,
original_total=total_count,
)
if results:
_print_results_table(results, console)
summary = _query_summary(
results, total_count, tokens_used, was_truncated=was_truncated
)
return QueryGraphData(
query_used=cypher_query, results=results, summary=summary
)
except ex.LLMGenerationError as e:
return QueryGraphData(
query_used=QUERY_NOT_AVAILABLE,
results=[],
summary=QUERY_SUMMARY_TRANSLATION_FAILED.format(error=e),
)
except TimeoutError:
logger.warning(
ls.TOOL_QUERY_TIMEOUT.format(
timeout=settings.QUERY_TIMEOUT_S, query=attempt.query
)
)
return QueryGraphData(
query_used=attempt.query,
results=[],
summary=QUERY_SUMMARY_TIMEOUT.format(timeout=settings.QUERY_TIMEOUT_S),
)
except Exception as e:
logger.exception(ls.TOOL_QUERY_ERROR.format(error=e))
return QueryGraphData(
query_used=attempt.query,
results=[],
summary=QUERY_SUMMARY_DB_ERROR.format(error=e),
)
return Tool(
function=query_codebase_knowledge_graph,
name=td.AgenticToolName.QUERY_GRAPH,
description=td.CODEBASE_QUERY,
)