Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions docs/content/docs/guides/websocket-api.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,25 @@ A command that cannot join the transaction (`.kg use`, `.rel drop`, `.index`, `.

An empty `errors` means every statement succeeded. Do not match message text to detect failures.

### Statement Counts

A program that commits fact statements lists them in `statements`, in program order, with the tuples each one changed. The list rides on the reply even when the program ends with a query, whose rows are the `rows`:

```json
{
"type": "result",
"columns": ["message"],
"rows": [["Inserted 1 fact(s) into 'task'."], ["Update: 0 deleted, 1 inserted."]],
"errors": [],
"statements": [
{"index": 0, "kind": "insert", "inserted": 1, "deleted": 0},
{"index": 1, "kind": "update", "inserted": 1, "deleted": 0}
]
}
```

`kind` is `insert`, `delete` or `update`. Counts are effective, as of the statements before it: inserting a fact that is present or deleting one that is absent is listed with zero counts. A conditional write that matched nothing is listed with zero counts too, so `inserted` of a guarded update says whether it landed. Schema and rule statements are not listed. `statements` is absent when the program committed no fact statement, including a program that failed and rolled back. Read counts from `statements`, not from the message rows.

## Knowledge Graph Operations

Use meta commands via execute:
Expand Down
33 changes: 33 additions & 0 deletions docs/spec/asyncapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -715,6 +715,27 @@ components:
message:
type: string

StatementCounts:
type: object
required: [index, kind, inserted, deleted]
description: |
What a committed fact statement changed. Counts are effective, as of
the statements before it in the same program: `inserted` counts tuples
that were absent and are now present, `deleted` tuples that were
present and are now absent. A statement that changed nothing is listed
with zero counts.
properties:
index:
type: integer
description: 0-based statement index in the program
kind:
type: string
enum: [insert, delete, update]
inserted:
type: integer
deleted:
type: integer

NoticeResponse:
type: object
required: [type, code, message]
Expand Down Expand Up @@ -1212,6 +1233,14 @@ components:
items:
$ref: '#/components/schemas/StatementError'
description: Failed statements of the program; empty when every statement succeeded
statements:
type: array
items:
$ref: '#/components/schemas/StatementCounts'
description: |
Fact statements the program committed, with their counts, in
program order. Absent when the program committed no fact statement
(a read, a failed program, or only schema and rule changes).
subscribed:
$ref: '#/components/schemas/Subscribed'
description: Set on the reply to `.subscribe`, whose `rows` are the snapshot
Expand Down Expand Up @@ -1312,6 +1341,10 @@ components:
type: array
items:
$ref: '#/components/schemas/StatementError'
statements:
type: array
items:
$ref: '#/components/schemas/StatementCounts'
subscribed:
$ref: '#/components/schemas/Subscribed'
examples:
Expand Down
1 change: 1 addition & 0 deletions src/bin/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,7 @@ impl WsClient {
proof_trees: start.proof_trees,
timing_breakdown: start.timing_breakdown,
errors: start.errors,
statements: start.statements,
subscribed: None,
});
}
Expand Down
26 changes: 21 additions & 5 deletions src/protocol/handler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -722,6 +722,7 @@ impl ProofSnapshot {
proof_us,
),
errors: Vec::new(),
statements: Vec::new(),
})
}

Expand Down Expand Up @@ -772,6 +773,7 @@ impl ProofSnapshot {
proof_trees: Some(vec![graph]),
timing_breakdown: proof_timing(timing_mode, start, query_us, "explanation", explain_us),
errors: Vec::new(),
statements: Vec::new(),
})
}
}
Expand Down Expand Up @@ -1463,6 +1465,7 @@ impl Handler {
proof_trees: None,
timing_breakdown: None,
errors: Vec::new(),
statements: Vec::new(),
})
}

Expand Down Expand Up @@ -2218,6 +2221,7 @@ impl Handler {
proof_trees: None,
timing_breakdown: None,
errors: Vec::new(),
statements: Vec::new(),
});
}

Expand Down Expand Up @@ -2282,6 +2286,7 @@ impl Handler {
proof_trees: None,
timing_breakdown: None,
errors: Vec::new(),
statements: Vec::new(),
});
}

Expand Down Expand Up @@ -3027,6 +3032,7 @@ impl QueryJob {
proof_trees: None,
timing_breakdown: None,
errors,
statements: Vec::new(),
});
}
MetaCommand::AgentStart(_)
Expand Down Expand Up @@ -3254,13 +3260,16 @@ impl QueryJob {
current_stmt.clear();
}
}
// Counts of the fact statements committed below.
let mut statement_counts = Vec::new();
if !write_run.is_empty() {
if errors.is_empty() {
if let Err(failure) =
self.commit_write_run(&storage, &kg_name, &mut write_run, &mut messages)
{
stmt_index = failure.index;
fail!(failure.code, failure.message);
match self.commit_write_run(&storage, &kg_name, &mut write_run, &mut messages) {
Ok(counts) => statement_counts = counts,
Err(failure) => {
stmt_index = failure.index;
fail!(failure.code, failure.message);
}
}
} else if let Some(note) = write_run.abandon(errors[0].index) {
// The loop stopped at the first failure, so it is the last row.
Expand Down Expand Up @@ -3296,6 +3305,7 @@ impl QueryJob {
return Ok(QueryResult {
switched_kg: switched_kg_result,
errors,
statements: statement_counts,
execution_time_ms: start.elapsed().as_millis() as u64,
..Handler::messages_result(messages)
});
Expand All @@ -3317,6 +3327,7 @@ impl QueryJob {
return Ok(QueryResult {
switched_kg: switched_kg_result,
errors,
statements: statement_counts,
execution_time_ms: start.elapsed().as_millis() as u64,
..Handler::messages_result(messages)
});
Expand Down Expand Up @@ -3511,6 +3522,7 @@ impl QueryJob {
proof_trees: None,
timing_breakdown,
errors,
statements: statement_counts,
})
}
}
Expand Down Expand Up @@ -3799,6 +3811,7 @@ impl Handler {
proof_trees: None,
timing_breakdown,
errors: Vec::new(),
statements: Vec::new(),
})
}

Expand Down Expand Up @@ -4438,6 +4451,7 @@ impl Handler {
proof_trees: None,
timing_breakdown: None,
errors: Vec::new(),
statements: Vec::new(),
}
}

Expand Down Expand Up @@ -4530,6 +4544,7 @@ impl Handler {
proof_trees: None,
timing_breakdown: None,
errors: Vec::new(),
statements: Vec::new(),
}
}

Expand Down Expand Up @@ -5158,6 +5173,7 @@ impl Handler {
proof_trees: None,
timing_breakdown: None,
errors: Vec::new(),
statements: Vec::new(),
})
}

Expand Down
11 changes: 10 additions & 1 deletion src/protocol/handler/fact_staging.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

use super::{format_body_pred, format_term, term_to_value};
use crate::ast::Term;
use crate::protocol::wire::ErrorCode;
use crate::protocol::wire::{ErrorCode, StatementKind};
use crate::statement::{DeleteOp, DeletePattern, InsertOp, UpdateOp};
use crate::storage_engine::{FactChange, KnowledgeGraphSnapshot};
use crate::value::{Tuple, Value};
Expand Down Expand Up @@ -105,6 +105,15 @@ impl FactStatement {
}
}

/// The form of this statement, as reported in a result's counts.
pub fn kind(&self) -> StatementKind {
match self {
Self::Insert(_) => StatementKind::Insert,
Self::Delete(_) => StatementKind::Delete,
Self::Update(_) => StatementKind::Update,
}
}

/// Whether staging this statement reads the knowledge graph.
pub fn reads_state(&self) -> bool {
matches!(
Expand Down
18 changes: 13 additions & 5 deletions src/protocol/handler/write_run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
use super::catalog_staging::CatalogStatement;
use super::fact_staging::{FactStatement, InsertLimits, StageError};
use super::{storage_error_code, QueryJob};
use crate::protocol::wire::ErrorCode;
use crate::protocol::wire::{ErrorCode, StatementCounts};
use crate::rule_catalog::RuleCatalog;
use crate::storage_engine::{
CommitError, KnowledgeGraphSnapshot, ProgramCommit, StagedChanges, StatementEffect,
Expand Down Expand Up @@ -117,8 +117,9 @@ impl WriteRun {
}

impl QueryJob {
/// Commit the statements queued in `run` to `kg` as one transaction and
/// fill their message rows in `messages`.
/// Commit the statements queued in `run` to `kg` as one transaction, fill
/// their message rows in `messages` and return the counts of its fact
/// statements.
///
/// A program whose staging read the KG is staged again, up to
/// [`MAX_STAGE_ATTEMPTS`] times with a short random pause, if a concurrent
Expand All @@ -129,7 +130,7 @@ impl QueryJob {
kg: &str,
run: &mut WriteRun,
messages: &mut [String],
) -> Result<(), RunFailure> {
) -> Result<Vec<StatementCounts>, RunFailure> {
let commit = match self.commit_queued(storage, kg, &run.queued) {
Ok(commit) => commit,
Err(mut failure) => {
Expand All @@ -145,10 +146,17 @@ impl QueryJob {

let queued = std::mem::take(&mut run.queued);
let mut inserted_total = 0;
let mut counts = Vec::new();
for (queued, outcome) in queued.iter().zip(&commit.statements) {
let message = match (&queued.statement, &outcome.effect) {
(WriteStatement::Facts(statement), StatementEffect::Facts(count)) => {
inserted_total += count.inserted;
counts.push(StatementCounts {
index: queued.index,
kind: statement.kind(),
inserted: count.inserted,
deleted: count.deleted,
});
statement.success_message(count.inserted, count.deleted)
}
(WriteStatement::Catalog(statement), StatementEffect::Catalog(outcome)) => {
Expand Down Expand Up @@ -179,7 +187,7 @@ impl QueryJob {
change.inserted + change.deleted,
);
}
Ok(())
Ok(counts)
}

/// Stage `queued` and commit it, re-staging while it goes stale.
Expand Down
3 changes: 2 additions & 1 deletion src/protocol/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ pub use error::{InputLayerError, InputLayerResult};

// Re-export wire types
pub use wire::{
ColumnDef, ErrorCode, QueryResult, StatementError, WireDataType, WireTuple, WireValue,
ColumnDef, ErrorCode, QueryResult, StatementCounts, StatementError, StatementKind,
WireDataType, WireTuple, WireValue,
};

// Re-export handler
Expand Down
2 changes: 2 additions & 0 deletions src/protocol/rest/handlers/ws/execute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ fn result_frame(id: Option<RequestId>, response: QueryResult) -> ServerFrame {
proof_trees,
timing_breakdown: response.timing_breakdown,
errors: response.errors,
statements: response.statements,
subscribed: None,
})
}
Expand Down Expand Up @@ -173,6 +174,7 @@ pub(super) fn subscription_reply(
proof_trees: None,
timing_breakdown: None,
errors: Vec::new(),
statements: Vec::new(),
subscribed,
}
}
Expand Down
2 changes: 2 additions & 0 deletions src/protocol/rest/handlers/ws/stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ fn stream_result(result: ResultFrame) -> Result<Vec<String>, Undeliverable> {
proof_trees,
timing_breakdown,
errors,
statements,
subscribed,
} = result;
let empty_chunk = ServerFrame::ResultChunk {
Expand Down Expand Up @@ -116,6 +117,7 @@ fn stream_result(result: ResultFrame) -> Result<Vec<String>, Undeliverable> {
proof_trees,
timing_breakdown,
errors,
statements,
subscribed,
}))?);
let mut rows = rows.into_iter();
Expand Down
18 changes: 16 additions & 2 deletions src/protocol/rest/handlers/ws/stream/tests.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use inputlayer_ws_protocol::{RequestId, Subscribed};
use inputlayer_ws_protocol::{RequestId, StatementCounts, StatementKind, Subscribed};
use serde_json::{json, Value};

use super::*;
Expand Down Expand Up @@ -30,6 +30,7 @@ fn result(rows: Vec<Row>) -> ResultFrame {
proof_trees: None,
timing_breakdown: None,
errors: Vec::new(),
statements: Vec::new(),
subscribed: Some(Subscribed {
subscription: "s".into(),
generation: 2,
Expand Down Expand Up @@ -61,12 +62,25 @@ fn a_small_result_is_one_frame_serialized_as_before() {
#[test]
fn a_large_result_streams_as_one_bounded_ordered_result() {
let rows: Vec<Row> = (0..3_000).map(|i| row(i, 1_000)).collect();
let frames = parsed(&result_frames(result(rows.clone())).unwrap());
let counts = vec![StatementCounts {
index: 0,
kind: StatementKind::Insert,
inserted: 3,
deleted: 0,
}];
let frames = parsed(
&result_frames(ResultFrame {
statements: counts.clone(),
..result(rows.clone())
})
.unwrap(),
);

let (_, ServerFrame::ResultStart(start)) = &frames[0] else {
panic!("{:?}", frames[0]);
};
assert_eq!(start.id.as_ref().map(RequestId::as_str), Some("q"));
assert_eq!(start.statements, counts, "counts ride on the header");
assert_eq!(start.subscribed.as_ref().map(|s| s.generation), Some(2));
let (mut got, mut provenance) = (Vec::new(), Vec::new());
for (index, (bytes, frame)) in frames[1..frames.len() - 1].iter().enumerate() {
Expand Down
Loading
Loading