Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -192,13 +192,13 @@ ENGRAPHIS_DECAY_HALFLIFE_DAYS=7
#
# ── Client side (end-user machines) ─────────────────────────────────────────
# Paid keys require a revocable, machine-bound lease from the managed service. The
# built-in default is https://engraphis-production.up.railway.app; override only for a different vendor
# built-in default is https://team.engraphis.com; override only for a different vendor
# deployment. Free-tier use remains local and does not require the service.
# ENGRAPHIS_CLOUD_URL=https://engraphis-production.up.railway.app
# ENGRAPHIS_CLOUD_URL=https://team.engraphis.com

# Server-side license enforcement (vendor/fulfillment server only): when set, every key
# minted by the Polar webhook carries a signed enforce:"cloud" claim + this URL, and the
# client requires a live lease from it (revocable, seat-counted, useless offline).
# Set this to the canonical managed-service URL for newly issued keys. Older keys carrying
# the retired custom-domain hostname are migrated by the client without changing their signature.
# ENGRAPHIS_KEY_CLOUD_URL=https://engraphis-production.up.railway.app
# the retired Railway hostname are migrated by the client without changing their signature.
# ENGRAPHIS_KEY_CLOUD_URL=https://team.engraphis.com
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -571,7 +571,7 @@ All via environment (or `.env`):
| `ENGRAPHIS_LOOP_INTERVAL` | `60` | Background consolidation loop interval in seconds (0 = disabled) |
| `ENGRAPHIS_DECAY_HALFLIFE_DAYS` | `7` | Ebbinghaus decay half-life (higher = memories persist longer) |
| `ENGRAPHIS_FORWARDED_ALLOW_IPS` | `127.0.0.1` | Trusted reverse-proxy IPs for TLS termination (`*` = trust all) |
| `ENGRAPHIS_RELAY_URL` | `https://engraphis-production.up.railway.app` | Managed sync, license, trial, and invite relay (Pro/Team); the retired custom-domain URL is migrated automatically |
| `ENGRAPHIS_RELAY_URL` | `https://team.engraphis.com` | Managed sync, license, trial, and invite relay (Pro/Team); the retired Railway URL is migrated automatically |
| `ENGRAPHIS_AUTOSYNC_LOOP` | `1` | Kill switch for the in-process auto-sync loop (0 = off) |

See `.env.example` for the full list including commercial/vendor, email delivery, and
Expand Down
13 changes: 10 additions & 3 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@ services:
ports:
- "8700:8700"
env_file:
- .env
# Optional: a fresh clone has no .env (it's gitignored), and `docker compose up`
# must still work using the `environment:` block below plus the shell env.
- path: .env
required: false
environment:
ENGRAPHIS_HOST: 0.0.0.0
ENGRAPHIS_DB_PATH: /data/engraphis.db
Expand All @@ -34,10 +37,14 @@ services:
ports:
- "8701:8700"
env_file:
- .env
- path: .env
required: false
environment:
ENGRAPHIS_HOST: 0.0.0.0
ENGRAPHIS_DB_PATH: /data/engraphis.db
# The v1 server uses a DIFFERENT, incompatible memory schema from the v2 dashboard,
# so it MUST NOT share the dashboard's engraphis.db (doing so corrupts both). Give it
# its own file on the shared volume.
ENGRAPHIS_DB_PATH: /data/engraphis_v1.db
ENGRAPHIS_STATE_DIR: /data/.engraphis
volumes:
- engraphis-data:/data
Expand Down
13 changes: 12 additions & 1 deletion engraphis/autosync.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,18 @@ def due(policy: dict, *, now: Optional[float] = None) -> bool:
def _record(summary: dict, *, now: Optional[float] = None) -> None:
"""Stamp last_run + a compact last_result onto the persisted policy (best-effort)."""
now = time.time() if now is None else now
existing = _read()
# Best-effort telemetry must NEVER clobber the policy. Distinguish a fresh install (no
# file — safe to create with the default) from a TRANSIENT read failure of an existing
# file: normalize_policy({}) is the DISABLED default, so blindly writing it back after a
# read hiccup would silently turn auto-sync off (this runs after every pass).
path = policy_path()
if path.exists():
try:
existing = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return # exists but unreadable → preserve the saved policy, skip this record
else:
existing = {}
try:
_write({"policy": normalize_policy(existing.get("policy", existing)),
"last_run": float(now),
Expand Down
254 changes: 220 additions & 34 deletions engraphis/billing.py

Large diffs are not rendered by default.

61 changes: 41 additions & 20 deletions engraphis/cloud_license.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import json
import logging
import os
import threading
import urllib.error
import urllib.request
import uuid
Expand Down Expand Up @@ -106,6 +107,8 @@ def validate_cloud_base_url(value: str) -> str:
#: call within a run returns the SAME id — so the trial HMAC verifies and leases
#: bind consistently instead of churning a fresh uuid on every call.
_machine_id_cache: dict = {}
# Serializes first-run id generation so concurrent threads don't each mint a device id.
_machine_id_lock = threading.Lock()


def cloud_url() -> str:
Expand All @@ -124,29 +127,47 @@ def machine_id() -> str:
cached = _machine_id_cache.get(key)
if cached:
return cached
try:
mid = _MACHINE_ID_FILE.read_text(encoding="utf-8").strip()
if mid:
_machine_id_cache[key] = mid
return mid
except OSError:
pass
mid = uuid.uuid4().hex
try:
_MACHINE_ID_FILE.parent.mkdir(parents=True, exist_ok=True)
_MACHINE_ID_FILE.write_text(mid, encoding="utf-8")
with _machine_id_lock:
# Re-check under the lock: another thread may have generated + cached the id while
# we waited, so we don't mint a second one (which would burn an extra Team seat).
cached = _machine_id_cache.get(key)
if cached:
return cached
try:
os.chmod(_MACHINE_ID_FILE, 0o600)
mid = _MACHINE_ID_FILE.read_text(encoding="utf-8").strip()
if mid:
_machine_id_cache[key] = mid
return mid
except OSError:
pass
except OSError as exc:
logger.warning(
"machine_id: could not persist device id to %s (%s); using an in-process id "
"for this run. Trial and cloud-lease binding need a writable home directory "
"— mount a persistent volume for %s (or set HOME to a writable path).",
_MACHINE_ID_FILE, exc, _MACHINE_ID_FILE.parent)
_machine_id_cache[key] = mid # stable for the rest of the process regardless
return mid
mid = uuid.uuid4().hex
try:
_MACHINE_ID_FILE.parent.mkdir(parents=True, exist_ok=True)
# Atomic create-if-absent so two PROCESSES racing a first run don't each persist
# a DIFFERENT id (registering the same device twice and burning a seat). The
# exclusive create fails if another process already wrote one; adopt that value.
try:
fd = os.open(str(_MACHINE_ID_FILE),
os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, "w", encoding="utf-8") as fh:
fh.write(mid)
except FileExistsError:
existing = _MACHINE_ID_FILE.read_text(encoding="utf-8").strip()
if existing:
mid = existing
try:
os.chmod(_MACHINE_ID_FILE, 0o600)
except OSError:
pass
except OSError as exc:
logger.warning(
"machine_id: could not persist device id to %s (%s); using an in-process "
"id for this run. Trial and cloud-lease binding need a writable home "
"directory — mount a persistent volume for %s (or set HOME to a writable "
"path).",
_MACHINE_ID_FILE, exc, _MACHINE_ID_FILE.parent)
_machine_id_cache[key] = mid # stable for the rest of the process regardless
return mid


# ── lease token (same ENGR1-style envelope, distinct prefix) ─────────────────────────
Expand Down
22 changes: 13 additions & 9 deletions engraphis/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@

#: Vendor-hosted managed service for sync, leases, trials, and invite delivery. This is
#: the default target when ``ENGRAPHIS_RELAY_URL`` is not overridden.
DEFAULT_RELAY_URL = "https://engraphis-production.up.railway.app"
DEFAULT_RELAY_URL = "https://team.engraphis.com"

# Keys issued while the Cloudflare custom domain was the default carry that URL inside
# their signed payload. Preserve the signature, but route that unavailable host to the
# live Railway service. Arbitrary signed URLs remain authoritative.
# Keys issued before the custom domain migration carry this URL inside their signed
# payload. Preserve the signature, but route that one retired vendor host to the current
# managed service. Arbitrary signed URLs remain authoritative.
RETIRED_RELAY_URLS = frozenset({
"https://team.engraphis.com",
"https://engraphis-production.up.railway.app",
})


Expand Down Expand Up @@ -55,7 +55,8 @@ class Settings:
api_token: str = field(default_factory=lambda: _env("ENGRAPHIS_API_TOKEN", ""))
# Comma-separated CORS allow-list. Defaults to loopback only (local-first).
cors_origins: list = field(
default_factory=lambda: _parse_origins(_env("ENGRAPHIS_CORS_ORIGINS", ""))
default_factory=lambda: _parse_origins(_env("ENGRAPHIS_CORS_ORIGINS", ""),
_env_int("ENGRAPHIS_PORT", 8700))
)
# Optional server-side workspace binding — the hard multi-tenant isolation boundary.
# When non-empty, MemoryService refuses any read or write whose
Expand Down Expand Up @@ -151,10 +152,13 @@ def _parse_headers(raw: str) -> dict:
return {}


def _parse_origins(raw: str) -> list:
"""CORS allow-list. Empty -> loopback only (safe local-first default)."""
def _parse_origins(raw: str, port: int = 8700) -> list:
"""CORS allow-list. Empty -> loopback on the CONFIGURED port (safe local-first default).

Deriving the default from ``port`` means running the dashboard on a non-default
ENGRAPHIS_PORT doesn't lock its own origin out of the CORS allow-list."""
if not raw.strip():
return ["http://127.0.0.1:8700", "http://localhost:8700"]
return ["http://127.0.0.1:%d" % port, "http://localhost:%d" % port]
return [o.strip() for o in raw.split(",") if o.strip()]


Expand Down
124 changes: 121 additions & 3 deletions engraphis/core/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,115 @@ def memory_matches_filter(rec: MemoryRecord, flt: Optional[SearchFilter], *,
return True


class _SerializedConnection:
"""Serializes access to one sqlite3 connection shared across threads.

The Store opens a SINGLE connection with ``check_same_thread=False`` and shares it
across the threadpool FastAPI runs sync handlers on. A bare sqlite3 connection is not
safe for concurrent multi-thread use: interleaved statements corrupt cursors, and —
because a connection has ONE transaction — one thread's ``commit()``/``rollback()``
lands on another thread's uncommitted writes, so a rollback can silently discard them.
(Per-thread connections are not an option: the sqlite-vec extension and FTS state are
loaded into THIS connection, and a ``:memory:`` DB can't be shared across connections
at all.)

This wrapper holds a reentrant lock for the DURATION of each write transaction —
pinned on the first statement that opens one (detected via ``in_transaction``) and
released on commit/rollback — so transactions never interleave. Read-only statements
lock only for the individual call. Two safety nets keep a stuck transaction from
deadlocking the process: a statement that raises while a transaction is open rolls it
back and frees the pin, and lock acquisition times out (raising, not blocking forever).
Non-statement attributes/methods (``in_transaction``, ``enable_load_extension`` at
setup, ...) pass straight through.
"""

_ACQUIRE_TIMEOUT = 60.0

def __init__(self, raw) -> None:
object.__setattr__(self, "_raw", raw)
object.__setattr__(self, "_lock", threading.RLock())
object.__setattr__(self, "_pin", threading.local())

def __getattr__(self, name):
return getattr(self._raw, name)

def __setattr__(self, name, value):
setattr(self._raw, name, value)

def _pinned(self) -> bool:
return getattr(self._pin, "held", False)

def _acquire(self) -> None:
if not self._lock.acquire(timeout=self._ACQUIRE_TIMEOUT):
raise sqlite3.OperationalError(
"store write lock timeout — a transaction appears stuck")

def _run(self, fn, *a, **k):
self._acquire()
try:
result = fn(*a, **k)
except BaseException:
# Do NOT roll back here: a failed statement in sqlite leaves any open
# transaction intact, and callers depend on that — e.g. probing an optional
# table inside a larger write transaction, catching the error, and continuing.
# Only manage the lock (identically to the success path); the pin is released
# by the eventual commit/rollback, or by the acquire timeout as a backstop.
self._settle()
raise
self._settle()
return result

def _settle(self) -> None:
"""After a statement, hold exactly one pinned lock acquire for this thread while a
write transaction is open (released on commit/rollback); otherwise release this
call's acquire so read-only statements don't hold the lock."""
if self._raw.in_transaction:
if self._pinned():
self._lock.release() # already pinned; drop this call's acquire
else:
self._pin.held = True # keep this acquire as the transaction pin
else:
self._lock.release() # no open transaction; release now

def _finish(self, fn):
self._acquire()
try:
fn()
finally:
if self._pinned():
self._pin.held = False
self._lock.release() # release the transaction pin
self._lock.release() # release this call's acquire

def execute(self, *a, **k):
return self._run(self._raw.execute, *a, **k)

def executemany(self, *a, **k):
return self._run(self._raw.executemany, *a, **k)

def executescript(self, *a, **k):
return self._run(self._raw.executescript, *a, **k)

def commit(self):
self._finish(self._raw.commit)

def rollback(self):
self._finish(self._raw.rollback)

def close(self):
self._raw.close()

def __enter__(self):
return self

def __exit__(self, exc_type, exc, tb):
if exc_type is None:
self.commit()
else:
self.rollback()
return False


class Store:
"""A connection to one Engraphis v2 database (one file, or ``:memory:``)."""

Expand All @@ -178,10 +287,14 @@ def __init__(self, path: str = ":memory:", *,
if connect is not None:
# Injected connection factory (e.g. the SQLCipher encrypted backend). It owns
# opening + keying + row_factory; the core never imports the concrete driver.
self.conn = connect(path)
raw_conn = connect(path)
else:
self.conn = sqlite3.connect(path, timeout=30, check_same_thread=False)
self.conn.row_factory = sqlite3.Row
raw_conn = sqlite3.connect(path, timeout=30, check_same_thread=False)
raw_conn.row_factory = sqlite3.Row
# Serialize the shared connection so concurrent threadpool handlers can't interleave
# transactions on it (see _SerializedConnection). All Store/service/backend access
# goes through self.conn, so wrapping here covers every writer.
self.conn = _SerializedConnection(raw_conn)
self.conn.execute("PRAGMA journal_mode=WAL")
self.conn.execute("PRAGMA foreign_keys=ON")
self.conn.execute("PRAGMA synchronous=NORMAL")
Expand Down Expand Up @@ -297,6 +410,11 @@ def create_workspace(self, name: str, *, settings: Optional[dict] = None) -> str
return wid

def get_or_create_workspace(self, name: str) -> str:
# Authorize on the RETRIEVE path too, not just create — otherwise a workspace
# outside ENGRAPHIS_WORKSPACES that already exists in the DB (e.g. predating the
# allow-list, or arriving via sync) could be handed back, silently bypassing the
# isolation boundary _authorize_workspace is meant to enforce ("create or retrieve").
self._authorize_workspace(name)
row = self.conn.execute("SELECT id FROM workspaces WHERE name=?", (name,)).fetchone()
if row:
return row["id"]
Expand Down
24 changes: 22 additions & 2 deletions engraphis/core/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,24 @@ def _clamp_ts(v: Any, now: float) -> Optional[float]:
return max(0.0, min(f, now + TS_FUTURE_SKEW))


# World-time validity ceiling (year ~2100). ``valid_from``/``valid_to`` are WORLD time —
# a fact may legitimately be true until a future date — and, unlike the system timestamps,
# do NOT feed the LWW version key (see _version_key), so a future value can't pin content.
# Bound only to a sane far-future ceiling to reject absurd/overflow values.
_WORLD_TS_MAX = 4_102_444_800.0


def _clamp_world_ts(v: Any) -> Optional[float]:
"""Coerce a world-time validity timestamp, allowing legitimate FUTURE values (bounded
to a far-future ceiling). Clamping these to ``now + skew`` like the system timestamps
truncated real future validity, and the earliest-wins merge then spread the truncation
to every device."""
f = _as_float(v, None)
if f is None:
return None
return max(0.0, min(f, _WORLD_TS_MAX))


def _clamp_str(v: Any, n: int) -> str:
s = v if isinstance(v, str) else ("" if v is None else str(v))
return _CONTROL_RE.sub("", s)[:n]
Expand Down Expand Up @@ -345,8 +363,10 @@ def dict_to_record(d: dict) -> Optional[MemoryRecord]:
stability=_clamp_num(d.get("stability"), 0.0, MAX_STABILITY, 1.0),
access_count=min(MAX_ACCESS_COUNT, max(0, _as_int(d.get("access_count"), 0))),
last_access=_clamp_ts(d.get("last_access"), now),
valid_from=_clamp_ts(d.get("valid_from"), now),
valid_to=_clamp_ts(d.get("valid_to"), now),
# World-time validity may be in the future; system timestamps may not (they feed
# the version key / anti-poison defense).
valid_from=_clamp_world_ts(d.get("valid_from")),
valid_to=_clamp_world_ts(d.get("valid_to")),
ingested_at=_clamp_ts(d.get("ingested_at"), now),
expired_at=_clamp_ts(d.get("expired_at"), now),
pinned=bool(d.get("pinned")), sensitivity=sens,
Expand Down
Loading