Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
d812d52
bench-313: adding heartbeat functions
jamesrut1996 Oct 5, 2026
c6cd41b
bench-313: adding primary testing
jamesrut1996 Oct 5, 2026
415f4ee
bench-313: finished primary testing
jamesrut1996 Oct 5, 2026
9cdecca
bench-313: added schema tests
jamesrut1996 Oct 6, 2026
af247b9
bench-313: spell fix and renaming primary tests file
jamesrut1996 Oct 6, 2026
3f49779
bench-313: style fix
jamesrut1996 Oct 6, 2026
5799087
branch-313: readme file name fix in testing section
jamesrut1996 Oct 6, 2026
30dfdda
bench-311: primary can now start up secondaries
jamesrut1996 Oct 7, 2026
a0da46d
bench-344: adding primary status functions
jamesrut1996 Oct 7, 2026
46fa7dd
bench-344: adding status testing
jamesrut1996 Oct 8, 2026
7409d57
bench-311: updating secondaries oom_score
jamesrut1996 Oct 8, 2026
517eb68
bench-314: added task id function and testing
jamesrut1996 Oct 9, 2026
430642a
bench-314: task update functionality
jamesrut1996 Oct 13, 2026
17d6e9a
bench-314: task update functionality, style fix
jamesrut1996 Oct 13, 2026
2345a9e
bench-314: adding .primary.newtask and testing
jamesrut1996 Oct 13, 2026
39d7ffa
bench-323: approved functions to be run on secondaries
jamesrut1996 Oct 13, 2026
e3db8c7
bench-324: adding functions to run queued tasks
jamesrut1996 Oct 15, 2026
0d2acef
bench-324: function to run queued tasks
jamesrut1996 Oct 16, 2026
a575f53
bench-324: added docs on how to do a manual run of tasks
jamesrut1996 Oct 20, 2026
18250e8
bench-377: fixing secondary and tasks status for a dead secondary whi…
jamesrut1996 Oct 20, 2026
bff62ba
bench-377: add retry logic .primary.retry
jamesrut1996 Oct 22, 2026
5824a2e
bench-369: comparing pri/sec setup v running with threads
jamesrut1996 Oct 22, 2026
76bcbfe
bench-369: bug fixes found in bench marking
jamesrut1996 Oct 22, 2026
b6e0bd1
bench-322: adding memory stats from the kernal
jamesrut1996 Oct 23, 2026
7edea14
bench-322: memory stats from the kernal monitoring
jamesrut1996 Oct 23, 2026
f2a5f15
bench-390: adding seconary respawn functions
jamesrut1996 Oct 26, 2026
d6fb07e
bench-344: updated .primary.status to now have memory stats
jamesrut1996 Oct 26, 2026
2b149d1
bench-344: improved status readability
jamesrut1996 Oct 27, 2026
52b0b8b
bench-391: cgroups and oom functionality
jamesrut1996 Oct 27, 2026
9f681ad
bench-392: results now saved to disk
jamesrut1996 Oct 28, 2026
2802616
bench-392: adding results testing and how to get results
jamesrut1996 Oct 28, 2026
b660328
bench-392: added jobs for combining tasks
jamesrut1996 Oct 28, 2026
7f6b8f3
bench-392: changing to just return result path
jamesrut1996 Oct 29, 2026
9cebfa1
bench-392: removing .primary.resultpath as not needed now
jamesrut1996 Oct 29, 2026
896d1ac
bench-322: adding memory after .Q.gc stats
jamesrut1996 Oct 29, 2026
74e5050
bench-393: demo prep
jamesrut1996 Nov 2, 2026
a45bacd
bench-400: can only connect to the secondaries via the primary and ad…
jamesrut1996 Nov 2, 2026
ee4eb30
bench-401: job estimation functionality
jamesrut1996 Nov 3, 2026
62506bf
bench-401: changed mem cap comparison to be anything killed with oom cap
jamesrut1996 Nov 3, 2026
6709556
bench-401: now only send tasks whose memory estimate fits under the cap
jamesrut1996 Nov 3, 2026
dbe4f41
bench-401: primary checks secondarys memory once a task is complete
jamesrut1996 Nov 3, 2026
c4ebada
bench-300: todying up
jamesrut1996 Nov 4, 2026
644663b
bench-300 tidy up of memory tracking
jamesrut1996 Nov 4, 2026
6b223e8
bench-300 tidy up of memory tracking
jamesrut1996 Nov 4, 2026
eac1588
bench-300 consistent dict type in main files
jamesrut1996 Nov 4, 2026
fe22b1b
bench-300: style changes
jamesrut1996 Nov 4, 2026
8689b54
bench-317: adding a stop script and starting primarys in a cgroup
jamesrut1996 Nov 5, 2026
57ef86c
bench-300: happy path integration testing
jamesrut1996 Nov 6, 2026
7d9cfcb
bench-300: memory cap env vars added
jamesrut1996 Nov 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .vscode/extensions.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"recommendations": [
"ms-vscode-remote.remote-ssh",
"KX.kdb"
]
}
616 changes: 615 additions & 1 deletion README.md

Large diffs are not rendered by default.

21 changes: 18 additions & 3 deletions bin/startprimary.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,26 @@ export QHOME="$KDBX_HOME"
# used for creating logs
# todo: have it create the absolute path
mkdir -p logs
echo "start primary: port $PRIMARY_PORT, config $CONFIG"
LOG="logs/primary.$(date -u +%Y.%m.%dD%H.%M.%S).log"

# Claude
# the command, wrapped in systemd-run when starting under a cgroup
USE_CGROUP="${USE_CGROUP:-1}"
CMD=("$KDBX_HOME/bin/q" src/primary.q -p "$PRIMARY_PORT")
if [[ "$USE_CGROUP" == 1 && -n "${GLOBAL_MEM_LIMIT:-}" ]]; then
if ! systemctl --user show-environment > /dev/null 2>&1; then # checks if there is a systemd user manager so we can create a cgroup
echo "startprimary: USE_CGROUP=1 but there's no systemd user session to make a cgroup in; set USE_CGROUP=0 to start without the memory cap" >&2
exit 1
fi
# --scope runs the command itself in a new cgroup; MemorySwapMax=0 means kill rather than swap; --quiet keeps systemd-run's own message out of the log
CMD=(systemd-run --user --scope --quiet -p MemoryMax="$GLOBAL_MEM_LIMIT" -p MemorySwapMax=0 "${CMD[@]}")
CAP="memory cap $GLOBAL_MEM_LIMIT bytes"
else
CAP="no memory cap"
fi

# nohup to run in backgorund if want to run interactively just use x src/primary.q
nohup "$KDBX_HOME/bin/q" src/primary.q -p "$PRIMARY_PORT" < /dev/null >> "$LOG" 2>&1 &
nohup "${CMD[@]}" < /dev/null >> "$LOG" 2>&1 &

echo $! > logs/primary.pid
echo "startprimary: port $PRIMARY_PORT, config $CONFIG, pid $!, log $LOG"
echo "startprimary: port $PRIMARY_PORT, $CAP, config $CONFIG, pid $!, log $LOG"
3 changes: 3 additions & 0 deletions bin/startsecondary.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ LOG="logs/sec$ID.$(date -u +%Y.%m.%dD%H.%M.%S).log"

# nohup to run in bbackgorund if want to run interactively just use x src/primary.q
nohup "$KDBX_HOME/bin/q" src/secondary.q -p "$PORT" -id "$ID" < /dev/null >> "$LOG" 2>&1 &
# should we use secondaries
#nohup "$KDBX_HOME/bin/q" src/secondary.q -p "$PORT" -id "$ID" -s 8 < /dev/null >> "$LOG" 2>&1 &


echo $! > "logs/sec$ID.pid"
echo "start secondary: id $ID, port $PORT, config $CONFIG, pid $!, log $LOG"
60 changes: 60 additions & 0 deletions bin/stopall.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/usr/bin/env bash

# Claude
# stop the primary and every secondary started with one config

# forces script to exit as soon a the command fails, or an unset variable is used, or a pipe fails
set -euo pipefail

CONFIG="${1:-config/default.env}"

# set -a exports every variable the file sets, so q can read them with getenv
set -a
source "$CONFIG"
set +a

# this user's primary and secondary processes for this config: PRIMARY_PORT in their environment,
# or, for a primary started without the config loaded, -p PRIMARY_PORT on its command line
ourpids() {
for pid in $(pgrep -u "$USER" -f 'src/(primary|secondary)\.q' || true); do
if tr '\0' '\n' < "/proc/$pid/environ" 2>/dev/null | grep -qx "PRIMARY_PORT=$PRIMARY_PORT" \
|| tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | grep -q -- "-p $PRIMARY_PORT "; then
echo "$pid"
fi
done
}

before=$(ourpids)
if [[ -z "$before" ]]; then
echo "stopall: nothing running for PRIMARY_PORT $PRIMARY_PORT"
exit 0
fi
echo "stopall: running for PRIMARY_PORT $PRIMARY_PORT: pids $(echo $before)"

# 1. ask the primary to exit (2s to connect); if it isn't there or doesn't answer, go straight to the wait and kill
"$KDBX_HOME/bin/q" -q <<'EOF' || true
h:@[hopen; (`$"::",getenv`PRIMARY_PORT; 2000); 0Ni];
if[null h; -1 "stopall: could not connect to the primary, will kill instead"; exit 1];
-1 "stopall: asking the primary to exit";
neg[h]"exit 0"; neg[h][];
exit 0
EOF

# 2. wait for everything to exit: the primary's own grace period, plus 5s
wait=$(( ${STOP_GRACE_PERIOD_SECS:-5} + 5 ))
for ((i=0; i<wait; i++)); do
[[ -z "$(ourpids)" ]] && { echo "stopall: all stopped"; exit 0; }
sleep 1
done

# 3. anything still running: kill, then kill -9
left=$(ourpids)
echo "stopall: still running after ${wait}s, killing: pids $(echo $left)"
kill $left 2>/dev/null || true
sleep 2
left=$(ourpids)
if [[ -n "$left" ]]; then
echo "stopall: killing with -9: pids $(echo $left)"
kill -9 $left 2>/dev/null || true
fi
echo "stopall: all stopped"
39 changes: 36 additions & 3 deletions config/default.env
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,43 @@ KDBX_HOME=/opt/kdbx/5.0/2026.07.23

# process info
PRIMARY_PORT=45000
SECONDARY_BASE_PORT=45001
NUM_SECONDARYS=4

# 4GB currently for testing, need to run on 30GB eventually
GLOBAL_MEM_LIMIT=4294967296

TASK_TYPES=noop,sleep,sum,fail,bigTable
# for heartbeating
HEARTBEAT_MS=1000
STALE_MS=5000

# for spinning up secondarys via the primary
SPAWN_SECONDARY=1
NUM_SECONDARYS=8
STOP_GRACE_PERIOD_SECS=5
LOG_DIR=logs

# for tasks
TASK_TIMEOUT_MS=60000 # 60 seconds timeout
TASK_RETRIES_MAX=3
TASK_TYPES=sleep,sumtil,fail,vwap,hold
RETRY_REASONS=lost,sendfailed

# mem stats from kernal
MEM_HISTORY_MAX=3600

# how many times the primary restarts a spawned secondary that dies (timeout, crash, OOM); 0 = never
RESPAWN_MAX=3

# start the primary under a cgroup with GLOBAL_MEM_LIMIT as the kernel's hard cap (bin/startprimary.sh); 0 for none
USE_CGROUP=1

# this is the multiple of the spare memory, 0 means no headroom, 0.1 means keep 10% free
MIN_HEADROOM=0

# this is the multiple of how much memory a task will use, 1 means use the current estimate 1.25 would mean estimate each task to use 25% more than the max memory task so far
ESTIMATE_MARGIN=1

# where to store results
RESULTS_DIR=/home/jrutledge/internalWork/Parallelisation-Project/results

# test hdb dir
HDB_DIR=/home/jrutledge/internalWork/hdb
15 changes: 15 additions & 0 deletions config/integration.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# overrides for tests/integration.sh, sourced after config/default.env
# separate ports so a test run never clashes with a dev primary on 45000
PRIMARY_PORT=46000
NUM_SECONDARYS=2
SPAWN_SECONDARY=1
STOP_GRACE_PERIOD_SECS=1
STALE_MS=3000
LOG_DIR=logs/integration
TASK_TYPES=sumtil,sleep,fail,hold
RESULTS_DIR=/tmp/integrationtesting

# integration specs kill secondaries on purpose and expect them to stay closed
RESPAWN_MAX=0

USE_CGROUP=0
Binary file added image.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
77 changes: 54 additions & 23 deletions schemas/primarytables.q
Original file line number Diff line number Diff line change
@@ -1,37 +1,68 @@
// tables for the master process
// tables for the primary process

// monitor the secondary procs
secondaries : ([id: `long$()]
handle: `int$();
pid: `int$();
host: `$();
port: `int$();
status: `$(); // `idle`busy`stale`disconnected
lastHeartbeat: `timestamp$();
lastMemBytes: `long$();
consecutiveFails: `int$();
status: `$(); // `free`busy`stale`closed
lastheartbeat: `timestamp$();
consecutivefails: `int$();
healthy: `boolean$();
spawned: `boolean$(); // if the master created it or it was up from before
startTime: `timestamp$();
registeredTime: `timestamp$();
taskTypes: ();
currentTask: `long$()
spawned: `boolean$(); // if the primary created it or it was up from before
starttime: `timestamp$();
registeredtime: `timestamp$();
tasktypes: ();
currenttask: `long$();
rssanon: `long$(); // in bytes
vmrss: `long$(); // in bytes
memtime: `timestamp$()
);

// montiro the tasks that are being carried out
tasks : ([taskId: `long$()]
taskType: `$();
// monitor the tasks that are being carried out
tasks : ([taskid: `long$()]
tasktype: `$();
args: ();
priority: `int$();
submittedTime: `timestamp$();
timeoutMs: `long$();
retriesMax: `int$();
retryCount: `int$();
submittedtime: `timestamp$();
memestimate: `long$();
timeout: `time$();
retriesmax: `int$();
retrycount: `int$();
state: `$(); // `queued`dispatched`running`done`failed
secondaryId: `long$();
dispatchedTime: `timestamp$();
startedTime: `timestamp$();
finishedTime: `timestamp$();
secondaryid: `long$();
dispatchedtime: `timestamp$();
startedtime: `timestamp$();
finishedtime: `timestamp$();
membefore: `long$(); // RssAnon before running
memafter: `long$(); // peak during the task VmHWN; or a task killed for memory, the cap it was killed under
memaftergc: `long$(); // RssAnon after the task ends and .Q.gc
err: ();
reason: `$() // `timeout`unknown`oom
reason: `$(); // `timeout`oom`error`lost`sendfailed
resultpath: `$();
resultbytes: `long$();
jobid: `long$();
user: `$()
);

// one row per memory poll, bounded to the last .primary.memhistorymax rows
memhistory : ([]
time: `timestamp$();
total: `long$();
primary: `long$();
secondaries: `long$();
busy: `int$()
);

// groups of tasks submitted together; when every part is done their results are combined into one
jobs : ([jobid: `long$()]
tasktype: `$();
user: `$();
ntasks: `int$();
priority: `int$();
state: `$();
submittedtime: `timestamp$();
finishedtime: `timestamp$();
resultbytes: `long$();
err: () // why the job failed, "" when it hasn't
);
34 changes: 34 additions & 0 deletions src/approvedtasks.q
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// this is a file containing any approved task
// we define an empty dict at the top, then a task then after we add this to the approved task
// all tasks that needs several args should be passed as a dict

.secondary.approvedtasks:()!();

// these are the test tasks before we define ones that would actually be useful

.task.sumtil:{sum til x};
.secondary.approvedtasks[`sumtil]:.task.sumtil;

.task.sleep:{system"sleep ",string x};
.secondary.approvedtasks[`sleep]:.task.sleep;

.task.fail:{'`fail};
.secondary.approvedtasks[`fail]:.task.fail;

/claude for the kernal mem stuff
/ test task: take a`mb MB of memory and keep it for a`secs seconds, so the memory gate can see it; a is `mb`secs!(mb;secs)
/ 131072 longs = 1MB; filling with 0 writes every page, so the memory really is used (RssAnon)
// .task.hold ([mb:600; secs:10])
.task.hold:{[a] v:(a[`mb]*131072)#0; system "sleep ",string a`secs; count v};
.secondary.approvedtasks[`hold]:.task.hold;

// actual useful ones

// lets add in a vwap where each secondary will do it for a different month for a time period
// means if we start with 4 secondarys, the primary needs to spin up 8 more if we are doing it over a year
// we will also need to combine the results correctly
// params is just a dict with sdate and edate
.task.vwap:{[d]
select vwap: size wavg price by date,sym from trades where date = d
};
.secondary.approvedtasks[`vwap]:.task.vwap;
18 changes: 18 additions & 0 deletions src/common/cgroup.q
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
// Claude minor edits
// the cgroup v2 directory this primary and the secondaries it spawns run in, e.g. /sys/fs/cgroup/user.slice/.../qfw.scope
// "" if it can't be found. Its memory.events file counts kernel OOM kills (only meaningful under a MemoryMax limit)
.cgroup.dir:{
l:@[system; "cat /proc/",string[.z.i],"/cgroup"; {[e] ()}];
l:l where l like "0::*";
$[count l; "/sys/fs/cgroup",3_first l; ""]
}[];


// how many processes the kernel has killed for memory in cgroup directory d (the oom_kill line of its memory.events)
// 0N if it can't be read: no directory, not cgroup v2, or no memory controller
.cgroup.oomkills:{[dir]
if[not count dir; :0N];
l:@[system; "cat ",dir,"/memory.events 2>/dev/null"; {[e] ()}];
l:l where l like "oom_kill *";
$[count l; "J"$last " " vs first l; 0N]
};
44 changes: 44 additions & 0 deletions src/common/mem.q
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// the fields read from /proc/<pid>/status
// RssAnon = the process's own memory (q's heap as the kernel sees it), what the global cap adds up
// VmRSS = all resident memory (RssAnon + file pages), VmHWM = the high-water mark of VmRSS
.mem.fields:`RssAnon`VmRSS`VmHWM;

// what .mem.read returns when there is nothing to read
.mem.empty:([] pid:`int$(); rssanon:`long$(); vmrss:`long$(); vmhwm:`long$());

// turn the lines of one or more /proc/<pid>/status files into a table, one row per process
// each line looks like "VmRSS:\t 12345 kB": the name before the colon, then the first word after it
// every status file starts with a Name line, so a running count of Name lines says which process a line is from
.mem.parse:{[l]
k:`$first each ":" vs/: l;
v:{w:" " vs ssr[last ":" vs x;"\t";" "]; first w where 0<count each w} each l;
g:sums k=`Name;
i:where k in `Pid,.mem.fields;
if[not count i; :.mem.empty];
t:([] g:g i; k:k i; v:"J"$v i);
r:(`Pid,.mem.fields)#/:value exec k!v by g from t;
select pid:`int$Pid, rssanon:1024*RssAnon, vmrss:1024*VmRSS, vmhwm:1024*VmHWM from r
};

// memory figures for a list of pids, one row per pid that is still running
// reads every pid with one cat, so one process start (~4ms) however many pids; read0 can't be used as
// /proc files report size 0. A pid that has exited is left out (cat's error is discarded)
// example: .mem.read 2993539 2993543i
.mem.read:{[pids]
if[not count pids; :.mem.empty];
files:" " sv {"/proc/",string[x],"/status"} each pids;
.mem.parse system "sh -c 'cat ",files," 2>/dev/null; true'"
};

// reset this process's VmHWM to its current VmRSS, so the next reading of vmhwm is the peak since now
// written through sh as clear_refs is write-only and hopen can't open it; uses our own pid, as /proc/self in sh is the shell
.mem.clearhwm:{[]
@[system; "sh -c 'echo 5 > /proc/",string[.z.i],"/clear_refs'"; {[e] .log.warn "could not reset VmHWM: ",e}];
};

// this process's figures (a dict) and then reset VmHWM, in one shell call instead of two (~4ms saved per task)
// used just before a task: rssanon is the starting point, and vmhwm read after the task is the task's peak
.mem.readandclear:{[]
p:string .z.i;
first .mem.parse system "sh -c 'cat /proc/",p,"/status; echo 5 > /proc/",p,"/clear_refs 2>/dev/null; true'"
};
1 change: 1 addition & 0 deletions src/common/os.q
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
.os:use `di.os
Loading