Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -461,8 +461,9 @@ js-test-live:

# python-sdk-live runs every query example in the Python SDK guide
# (docs/content/docs/guides/python-sdk.mdx) against a live server, with the
# pins of the SDK's query compile rules, so a query form the engine rejects
# fails here rather than in a user's hands.
# pins of the SDK's compile rules (the shared meta-command fixture in
# packages/conformance and a literal round-trip fuzz included), so a form the
# engine rejects fails here rather than in a user's hands.
PY_LIVE_PORT ?= 8092
python-sdk-live:
@cargo build --release --bin inputlayer-server
Expand All @@ -480,7 +481,7 @@ python-sdk-live:
INPUTLAYER_TEST_SERVER=ws://127.0.0.1:$(PY_LIVE_PORT)/ws \
INPUTLAYER_TEST_USER=admin \
INPUTLAYER_TEST_PASSWORD=admin \
uv run --extra dev pytest tests/docs_queries_live.py -v; \
uv run --extra dev pytest tests/docs_queries_live.py tests/compile_rules_live.py -v; \
TEST_EXIT=$$?; \
cd - > /dev/null; \
kill $$SERVER_PID 2>/dev/null || true; \
Expand Down
2 changes: 1 addition & 1 deletion docs/content/docs/guides/python-sdk.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ for col in desc.columns:
| `bool` | `bool` | Boolean |
| `Vector[N]` | `vector(N)` | N-dimensional float32 vector |
| `VectorInt8[N]` | `vector_int8(N)` | N-dimensional int8 quantized vector |
| `Timestamp` | `timestamp` | Unix epoch milliseconds |
| `Timestamp`, `datetime` | `int` | Unix epoch milliseconds; a `datetime` is written as milliseconds (a naive one as UTC) and read back as a UTC `datetime` |

### Custom Relation Names

Expand Down
7 changes: 6 additions & 1 deletion gui/lib/docs-bundle.ts

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion packages/conformance/meta-commands.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"description": "Meta commands the SDKs send and the replies they parse. The JS SDK renders `commands` from its command table and must produce `text` exactly (the Python SDK is planned to do the same); a live test sends every command to the engine (in order, so each one has the state it needs) and expects `rejected` to fail. Replies are copied from the engine and pin the SDK parsers.",
"description": "Meta commands the SDKs send and the replies they parse. Both SDKs render `commands` from their command tables and must produce `text` exactly; a live test sends every command to the engine (in order, so each one has the state it needs) and expects `rejected` to fail. Replies are copied from the engine and pin the SDK parsers.",
"commands": [
{ "name": "session_list", "args": [], "text": ".session" },
{ "name": "session_drop", "args": [2], "text": ".session drop 2" },
Expand Down
2 changes: 1 addition & 1 deletion packages/inputlayer-py/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ class Document(Relation):
created_at: Timestamp
```

Supported types: `int`, `float`, `str`, `bool`, `Vector[N]`, `VectorInt8[N]`, `Timestamp`
Supported types: `int`, `float`, `str`, `bool`, `Vector[N]`, `VectorInt8[N]`, `Timestamp`, `datetime` (both stored as Unix milliseconds)

### Derived Relations (Rules)

Expand Down
5 changes: 5 additions & 0 deletions packages/inputlayer-py/src/inputlayer/_ast.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,13 +115,16 @@ class InExpr(BoolExpr):
"""Membership test: Column appears in another relation."""
column: Expr
target_column: Expr
#: Every column of the target relation, in order (R-IN needs the atom's arity).
target_columns: tuple[str, ...] | None = None


@dataclass(frozen=True)
class NegatedIn(BoolExpr):
"""Negated membership test."""
column: Expr
target_column: Expr
target_columns: tuple[str, ...] | None = None


@dataclass(frozen=True)
Expand All @@ -130,3 +133,5 @@ class MatchExpr(BoolExpr):
relation: str
bindings: dict[str, Expr] # target_col -> source expr
negated: bool = False
#: Every column of the relation, in order; unbound ones become ``_``.
columns: tuple[str, ...] | None = None
122 changes: 122 additions & 0 deletions packages/inputlayer-py/src/inputlayer/_literal.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
"""The one place program text is built from values (R-LIT).

Every value the SDK writes into IQL (an inserted fact, a condition's
constant, a function argument, an integration's bound parameter) goes
through :func:`encode`. It produces exactly the literal the engine's parser
reads back as the same value, and refuses what IQL cannot express instead
of sending text the engine would misread:

- strings are quoted with the engine's five escapes (``\\\\``, ``\\"``,
``\\n``, ``\\t``, ``\\r``; the parser keeps any other backslash pair
verbatim, so nothing else is escaped); a lone surrogate is refused;
- ``bool`` is ``true``/``false`` (checked before ``int``);
- ``int`` must fit the engine's 64-bit integers;
- ``float`` uses Python's shortest round-trip form; NaN and the infinities
are refused, IQL has no literal for them;
- ``datetime`` is Unix milliseconds (R-TYPE; a naive datetime is UTC);
- a list or tuple of numbers is a vector;
- ``None`` is refused: IQL has no null.

A fuzz test (``tests/compile_rules_live.py``) round-trips random values
through a live engine: encode, insert, read back, compare.
"""

from __future__ import annotations

import math
import numbers
from datetime import datetime, timedelta, timezone
from typing import Any

from inputlayer.exceptions import CompileError

I64_MIN = -(2**63)
I64_MAX = 2**63 - 1

_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
_MS = timedelta(milliseconds=1)

_ESCAPES = str.maketrans({"\\": "\\\\", '"': '\\"', "\n": "\\n", "\t": "\\t", "\r": "\\r"})


def encode(value: Any) -> str:
"""The IQL literal for *value*; raises :class:`CompileError` when IQL has none."""
if isinstance(value, str):
return encode_string(value)
if isinstance(value, bool):
return "true" if value else "false"
if isinstance(value, numbers.Integral):
return _encode_int(int(value))
if isinstance(value, numbers.Real):
return _encode_float(float(value))
if isinstance(value, datetime):
return _encode_int(datetime_to_ms(value))
if isinstance(value, (list, tuple)):
return "[" + ", ".join(_encode_vector_item(v) for v in value) + "]"
if value is None:
raise CompileError(
"IQL has no null literal",
hint="pass a value of the column's type, or leave the column out of the condition",
)
raise CompileError(
f"Cannot write a {type(value).__name__} value into IQL: {value!r}",
hint="use str, int, float, bool, datetime or a list of numbers",
)


def encode_string(value: str) -> str:
"""The quoted IQL string literal for *value*."""
return f'"{escape_string(value)}"'


def escape_string(value: str) -> str:
"""The body of the IQL string literal for *value* (without the quotes)."""
if not isinstance(value, str):
raise CompileError(f"Expected a str, got {type(value).__name__}: {value!r}")
try:
value.encode("utf-8")
except UnicodeEncodeError as err:
raise CompileError(
f"String {value!r} holds a lone surrogate, which is not valid text",
hint="decode the bytes it came from with errors='replace'",
) from err
return value.translate(_ESCAPES)


def datetime_to_ms(value: datetime) -> int:
"""Unix milliseconds for *value*; a naive datetime is taken as UTC."""
if value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
return (value - _EPOCH) // _MS


def ms_to_datetime(ms: int) -> datetime:
"""The UTC datetime *ms* Unix milliseconds stand for."""
return _EPOCH + timedelta(milliseconds=ms)


def _encode_int(value: int) -> str:
if not I64_MIN <= value <= I64_MAX:
raise CompileError(
f"Integer {value} does not fit the engine's 64-bit integers",
hint=f"keep integers within [{I64_MIN}, {I64_MAX}], or store it as a string",
)
return str(value)


def _encode_float(value: float) -> str:
if not math.isfinite(value):
raise CompileError(
f"IQL has no literal for {value!r}: it does not support infinity or NaN",
hint="use a finite float, or leave the value out",
)
return repr(value)


def _encode_vector_item(value: Any) -> str:
if isinstance(value, bool) or not isinstance(value, numbers.Real):
raise CompileError(
f"A vector holds numbers only, got {type(value).__name__}: {value!r}",
hint="pass a list of floats",
)
return _encode_float(float(value))
138 changes: 138 additions & 0 deletions packages/inputlayer-py/src/inputlayer/_meta.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
"""Meta commands the SDK sends, in one table (R-META).

The table is pinned by the shared fixture
``packages/conformance/meta-commands.json``, which the JS SDK renders its own
table against and a live test runs against the engine, so a command spelling
cannot be wrong in one SDK only (``.session list``, ``.session remove`` and
``.rule show`` were). The reply parsers read the grammar the fixture records.
"""

from __future__ import annotations

import re
from typing import Any


def session_list() -> str:
"""List session facts and rules. The reply is parsed by :func:`session_rules`."""
return ".session"


def session_drop(name_or_position: str | int) -> str:
"""Drop a session rule by name (every clause), or by 1-based position among all of them."""
return f".session drop {name_or_position}"


def session_clear() -> str:
"""Clear every session fact and rule."""
return ".session clear"


def rule_list() -> str:
"""List persistent rules. The reply is parsed by :func:`rule_infos`."""
return ".rule list"


def rule_def(name: str) -> str:
"""Show a persistent rule's clauses. The reply is parsed by :func:`rule_clauses`."""
return f".rule def {name}"


def rule_drop(name: str) -> str:
"""Drop every clause of a persistent rule."""
return f".rule drop {name}"


def rule_drop_prefix(prefix: str) -> str:
"""Drop every persistent rule whose name starts with *prefix*."""
return f".rule drop prefix {prefix}"


def rule_remove(name: str, index: int) -> str:
"""Remove one clause of a persistent rule, by 1-based index within that rule."""
return f".rule remove {name} {index}"


def rule_clear(name: str) -> str:
"""Clear a persistent rule's clauses."""
return f".rule clear {name}"


def subscribe(subscription_id: str, query: str) -> str:
"""Open a standing query."""
return f".subscribe {subscription_id} {query}"


def unsubscribe(subscription_id: str) -> str:
"""Close a standing query."""
return f".unsubscribe {subscription_id}"


def why(statement: str, full: bool = False) -> str:
"""Proof trees for a rule's derived rows."""
return f".why full {statement}" if full else f".why {statement}"


def why_not(atom: str) -> str:
"""Why a fact was not derived."""
return f".why_not {atom}"


#: Every command of the table by its fixture name.
COMMANDS = {
fn.__name__: fn
for fn in (
session_list,
session_drop,
session_clear,
rule_list,
rule_def,
rule_drop,
rule_drop_prefix,
rule_remove,
rule_clear,
subscribe,
unsubscribe,
why,
why_not,
)
}

_NUMBERED = re.compile(r"^\s+\d+\. (.*)$")
_RULE_INFO = re.compile(r"^\s+(\S+) \((\d+) clause\(s\)\)$")


def session_rules(rows: list[list[Any]]) -> list[str]:
"""Rule clauses in a ``.session`` reply, in definition order.

The engine answers with message lines: ``No session data defined.``, or
an optional ``Session facts (n):`` section, then ``Session rules (n):``
followed by one `` <position>. <clause>`` line per rule.
"""
lines = [str(row[0]) for row in rows if row]
start = next((i for i, line in enumerate(lines) if line.startswith("Session rules (")), None)
if start is None:
return []
return [m.group(1) for line in lines[start + 1 :] if (m := _NUMBERED.match(line))]


def rule_infos(rows: list[list[Any]]) -> list[tuple[str, int]]:
"""``(name, clause count)`` per rule in a ``.rule list`` reply: a ``Rules:``
line, then one `` <name> (<n> clause(s))`` line per rule."""
return [
(m.group(1), int(m.group(2)))
for row in rows
if row and (m := _RULE_INFO.match(str(row[0])))
]


def rule_clauses(rows: list[list[Any]]) -> list[str]:
"""Clauses in a ``.rule def`` reply, in order. The engine answers with one
message: ``Rule: <name>``, ``Clauses:``, then one `` <i>. <clause>`` line each."""
return [
m.group(1)
for row in rows
if row
for line in str(row[0]).split("\n")
if (m := _NUMBERED.match(line))
]
Loading
Loading