{ "schema_version": "0.3.0", "dataset_version": "2.2.0", "post": "five-open-source-analytics-tools-load-tested", "generated_from": "ramp/2 rows imported from the split-harness stages.csv per tool by scripts/import-load-test-dataset.py; observed_outcome generated by scripts/build-post.py", "protocols": { "ramp/2": { "version": "2", "summary": "120 s constant-arrival stages at offered rates 25→1,600 requests/s (ClickStack →3,200; Traceway 10→400), one event per request, 500 distinct visitors, k6 v2.2.0 on a separate e2-standard-4 with generator CPU, achieved rate and dropped iterations recorded; reconcile after every stage once the stage's row count is unchanged across two consecutive 5 s readings (cap 180 s); a 600 s soak at 80 % of the ceiling; one docker kill (SIGKILL) of the accept process at t=20 s of a 90 s run at the soak rate, 10 s restart-policy watch, docker start by the driver if the policy did not fire, recovery = first healthy probe over ssh. Per-event id reconciliation where the tool stores a client-supplied id (Umami url_query, Rybbit querystring, ClickStack log attribute, ours event_id); count comparison for Matomo (a unique URL per event would change its workload) and Traceway (attributes not queryable per key). Run 2026-09-14; our stack re-run 2026-09-17 under the same protocol (second trial).", "ceiling_rule": "highest observed throughput = the highest achieved acknowledged rate across the ramp stages where acknowledged ids missing from storage are at most 2 % of acknowledged (id matching) or stored rows are at least 98 % of 2xx (count comparison); the stage that produced it is the ceiling stage; no latency or error threshold enters the rule", "reconciliation": "event_id_matching for Umami, Rybbit, ClickStack, ours; count_comparison for Matomo, Traceway" } }, "definitions": { "acknowledged": "requests that received the tool's success status (2xx; Matomo 204; Traceway 2xx) inside the window; in id mode, the number of distinct acknowledged ids", "stored_rows": "rows for the stage's run in the tool's own database after the settle rule was satisfied", "stored_distinct": "distinct stored event ids (id mode only)", "acknowledged_minus_stored_rows": "acknowledged − stored_rows, signed; a net count deficit, not an identified set", "acknowledged_missing_count": "acknowledged ids absent from storage after settle (id mode); typed null with reason count_based otherwise", "duplicate_count": "stored rows beyond distinct ids (id mode); the ramp/2 id-matched rows found none anywhere", "stored_without_ack_count": "stored ids whose success response never reached the client (id mode)", "hard_failures": "requests that ended in a client-visible error (connection refused/EOF, client timeout) inside the window; HTTP 503 sheds are counted here for Traceway", "achieved_rate": "acknowledged ÷ stage duration; the rate the tool actually accepted, not the offered rate", "rate_bound": "exact when the application was the limit; lower when the generator machine was near capacity (see generator_busy_max_pct)", "is_ceiling_stage": "the stage the protocol's ceiling rule selected; its achieved_rate is the row's headline throughput", "p95_ms": "k6 http_req_duration p95 over the whole stage window", "recovery": "seconds from the kill to the named condition; healthy_probe = the tool's health endpoint answered, which may precede successful ingestion", "restart_policy_fired": "whether the container's restart policy restarted it within a 10 s watch after docker kill (API SIGKILL does not trigger Docker restart policies)", "overload_behaviour": "what the accept path does under sustained overload, from source or documentation; distinct from crash survival", "superseded_by": "the observation that replaces this row for headline use, when one exists", "axis": "pageview rows are directly comparable with each other; telemetry_row (ClickStack) and error_envelope (Traceway) are annotated axes and not comparable with pageview rows", "acknowledgement_strategy": "S synchronous write; M in-memory batch; D durable queue or durable log — classified from source, evidence in classification_evidence" }, "qualification_markers": { "count_based_reconciliation": "stored rows were counted, not matched by event id; a net deficit cannot separate missing, duplicate and stored-without-ack events", "single_trial": "one run, one kill; a memory-buffer loss depends on where in the flush cycle the kill lands", "generator_near_capacity": "the separate generator machine reached 63–72 % CPU during the stage; the achieved rate is a lower bound", "application_bound": "the limit was on the tool's side (its containers, or a per-request disk barrier) while the generator machine stayed under a third busy; the ceiling is the application's", "healthy_probe_recovery": "recovery is timed to the health endpoint answering, not to a successfully ingested event", "overload_discard": "this stack deliberately discards low-value events after acknowledging them under sustained overload (overload_behaviour); not exercised here, recorded so a zero-loss kill is not read as a general guarantee" }, "notes": { "n-ours-second-trial": "Second trial of ramp/2 for this configuration, run 2026-09-17 on the same box class and build class. The first trial (2026-09-14) measured 459 events/s at 800 offered with p95 3.8 s and the same kill outcome (0 of 27,073 acknowledged missing, healthy 17 s after the kill); this stack's accept path serialises one fsync per request and its stall tail varies from run to run (analytics-platform stress/results/2026-09-15-ours-causes.md)." }, "observations": [ { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Umami", "subject_version": "3.3.0", "subject_digests": [ "ghcr.io/umami-software/umami@sha256:87312d334d009ee67ee0d2fba8fed01435547cc468e452243aef5133a9984d48" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "CLIENT_IP_HEADER=x-forwarded-for so the visitor pool is not one session" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored == acknowledged at every stage; no buffering tier in the source", "tracker_retry": "none", "tracker_retry_source": "umami-software/umami src/tracker/index.ts (3.3.1) @ 8efd5d72: one fetch, errors swallowed", "tracker_retry_measured": false, "configuration_id": "cfg-umami-ramp2", "record_file": "2026-09-14-split-umami.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-umami.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-umami-ramp2-r25", "run_id": "run-2026-09-14-umami-ramp2", "stage": "ramp:25", "offered_rate": 25.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 3001, "acknowledged": 3001, "stored_rows": 3001, "stored_distinct": 3001, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 22.1, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 5.0, "k6_cpu_max_pct": 8.5, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 25.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "25.0 acknowledged events/s achieved at 25 offered; p95 22 ms. 0 of 3,001 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "e668ebbe34b99159307974b6d33d8c332ba254fd8b233a325b2c473a7ea24087", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-umami.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Umami", "subject_version": "3.3.0", "subject_digests": [ "ghcr.io/umami-software/umami@sha256:87312d334d009ee67ee0d2fba8fed01435547cc468e452243aef5133a9984d48" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "CLIENT_IP_HEADER=x-forwarded-for so the visitor pool is not one session" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored == acknowledged at every stage; no buffering tier in the source", "tracker_retry": "none", "tracker_retry_source": "umami-software/umami src/tracker/index.ts (3.3.1) @ 8efd5d72: one fetch, errors swallowed", "tracker_retry_measured": false, "configuration_id": "cfg-umami-ramp2", "record_file": "2026-09-14-split-umami.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-umami.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-umami-ramp2-r50", "run_id": "run-2026-09-14-umami-ramp2", "stage": "ramp:50", "offered_rate": 50.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 6000, "acknowledged": 6000, "stored_rows": 6000, "stored_distinct": 6000, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 19.1, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 6.0, "k6_cpu_max_pct": 13.3, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 50.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "50.0 acknowledged events/s achieved at 50 offered; p95 19 ms. 0 of 6,000 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "e668ebbe34b99159307974b6d33d8c332ba254fd8b233a325b2c473a7ea24087", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-umami.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Umami", "subject_version": "3.3.0", "subject_digests": [ "ghcr.io/umami-software/umami@sha256:87312d334d009ee67ee0d2fba8fed01435547cc468e452243aef5133a9984d48" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "CLIENT_IP_HEADER=x-forwarded-for so the visitor pool is not one session" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored == acknowledged at every stage; no buffering tier in the source", "tracker_retry": "none", "tracker_retry_source": "umami-software/umami src/tracker/index.ts (3.3.1) @ 8efd5d72: one fetch, errors swallowed", "tracker_retry_measured": false, "configuration_id": "cfg-umami-ramp2", "record_file": "2026-09-14-split-umami.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-umami.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-umami-ramp2-r100", "run_id": "run-2026-09-14-umami-ramp2", "stage": "ramp:100", "offered_rate": 100.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 12000, "acknowledged": 12000, "stored_rows": 12000, "stored_distinct": 12000, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 67.9, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 8.0, "k6_cpu_max_pct": 25.1, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 100.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "100.0 acknowledged events/s achieved at 100 offered; p95 68 ms. 0 of 12,000 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "e668ebbe34b99159307974b6d33d8c332ba254fd8b233a325b2c473a7ea24087", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-umami.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Umami", "subject_version": "3.3.0", "subject_digests": [ "ghcr.io/umami-software/umami@sha256:87312d334d009ee67ee0d2fba8fed01435547cc468e452243aef5133a9984d48" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "CLIENT_IP_HEADER=x-forwarded-for so the visitor pool is not one session" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored == acknowledged at every stage; no buffering tier in the source", "tracker_retry": "none", "tracker_retry_source": "umami-software/umami src/tracker/index.ts (3.3.1) @ 8efd5d72: one fetch, errors swallowed", "tracker_retry_measured": false, "configuration_id": "cfg-umami-ramp2", "record_file": "2026-09-14-split-umami.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-umami.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-umami-ramp2-r200", "run_id": "run-2026-09-14-umami-ramp2", "stage": "ramp:200", "offered_rate": 200.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 14934, "acknowledged": 14934, "stored_rows": 14934, "stored_distinct": 14934, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 13532.3, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 11.0, "k6_cpu_max_pct": 47.3, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 124.5, "rate_bound": "exact", "dropped_iterations": 9066, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "124.5 acknowledged events/s achieved at 200 offered; p95 13.53 s. 0 of 14,934 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "e668ebbe34b99159307974b6d33d8c332ba254fd8b233a325b2c473a7ea24087", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-umami.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Umami", "subject_version": "3.3.0", "subject_digests": [ "ghcr.io/umami-software/umami@sha256:87312d334d009ee67ee0d2fba8fed01435547cc468e452243aef5133a9984d48" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "CLIENT_IP_HEADER=x-forwarded-for so the visitor pool is not one session" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored == acknowledged at every stage; no buffering tier in the source", "tracker_retry": "none", "tracker_retry_source": "umami-software/umami src/tracker/index.ts (3.3.1) @ 8efd5d72: one fetch, errors swallowed", "tracker_retry_measured": false, "configuration_id": "cfg-umami-ramp2", "record_file": "2026-09-14-split-umami.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-umami.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-umami-ramp2-r400", "run_id": "run-2026-09-14-umami-ramp2", "stage": "ramp:400", "offered_rate": 400.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 14987, "acknowledged": 14961, "stored_rows": 14961, "stored_distinct": 14961, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 26, "p95_ms": 19562.3, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 17.0, "k6_cpu_max_pct": 73.0, "qualifications": [ "single_trial", "application_bound" ], "record_type": "throughput_stage", "achieved_rate": 124.7, "rate_bound": "exact", "dropped_iterations": 33014, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": true, "observed_outcome": "124.7 acknowledged events/s achieved at 400 offered; p95 19.56 s. 0 of 14,961 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id). Selected as the highest observed throughput by the protocol's ceiling rule.", "record_sha256": "e668ebbe34b99159307974b6d33d8c332ba254fd8b233a325b2c473a7ea24087", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-umami.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Umami", "subject_version": "3.3.0", "subject_digests": [ "ghcr.io/umami-software/umami@sha256:87312d334d009ee67ee0d2fba8fed01435547cc468e452243aef5133a9984d48" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "CLIENT_IP_HEADER=x-forwarded-for so the visitor pool is not one session" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored == acknowledged at every stage; no buffering tier in the source", "tracker_retry": "none", "tracker_retry_source": "umami-software/umami src/tracker/index.ts (3.3.1) @ 8efd5d72: one fetch, errors swallowed", "tracker_retry_measured": false, "configuration_id": "cfg-umami-ramp2", "record_file": "2026-09-14-split-umami.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-umami.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-umami-ramp2-r800", "run_id": "run-2026-09-14-umami-ramp2", "stage": "ramp:800", "offered_rate": 800.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 14700, "acknowledged": 14590, "stored_rows": 14590, "stored_distinct": 14590, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 110, "p95_ms": 19453.1, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 22.0, "k6_cpu_max_pct": 90.0, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 121.6, "rate_bound": "exact", "dropped_iterations": 81300, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "121.6 acknowledged events/s achieved at 800 offered; p95 19.45 s. 0 of 14,590 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "e668ebbe34b99159307974b6d33d8c332ba254fd8b233a325b2c473a7ea24087", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-umami.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Umami", "subject_version": "3.3.0", "subject_digests": [ "ghcr.io/umami-software/umami@sha256:87312d334d009ee67ee0d2fba8fed01435547cc468e452243aef5133a9984d48" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "CLIENT_IP_HEADER=x-forwarded-for so the visitor pool is not one session" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored == acknowledged at every stage; no buffering tier in the source", "tracker_retry": "none", "tracker_retry_source": "umami-software/umami src/tracker/index.ts (3.3.1) @ 8efd5d72: one fetch, errors swallowed", "tracker_retry_measured": false, "configuration_id": "cfg-umami-ramp2", "record_file": "2026-09-14-split-umami.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-umami.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-umami-ramp2-r1600", "run_id": "run-2026-09-14-umami-ramp2", "stage": "ramp:1600", "offered_rate": 1600.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 14790, "acknowledged": 14648, "stored_rows": 14648, "stored_distinct": 14648, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 142, "p95_ms": 20498.2, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 20.0, "k6_cpu_max_pct": 84.2, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 122.1, "rate_bound": "exact", "dropped_iterations": 177210, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "122.1 acknowledged events/s achieved at 1,600 offered; p95 20.50 s. 0 of 14,648 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "e668ebbe34b99159307974b6d33d8c332ba254fd8b233a325b2c473a7ea24087", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-umami.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Umami", "subject_version": "3.3.0", "subject_digests": [ "ghcr.io/umami-software/umami@sha256:87312d334d009ee67ee0d2fba8fed01435547cc468e452243aef5133a9984d48" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "CLIENT_IP_HEADER=x-forwarded-for so the visitor pool is not one session" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored == acknowledged at every stage; no buffering tier in the source", "tracker_retry": "none", "tracker_retry_source": "umami-software/umami src/tracker/index.ts (3.3.1) @ 8efd5d72: one fetch, errors swallowed", "tracker_retry_measured": false, "configuration_id": "cfg-umami-ramp2", "record_file": "2026-09-14-split-umami.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-umami.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-umami-ramp2-soak", "run_id": "run-2026-09-14-umami-ramp2", "stage": "soak:99", "offered_rate": 99.0, "offered_unit": "events_per_s", "duration_s": 600, "sent": 59400, "acknowledged": 59400, "stored_rows": 59400, "stored_distinct": 59400, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 63.0, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 10.0, "k6_cpu_max_pct": 30.9, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 99.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "99.0 acknowledged events/s achieved at 99 offered; p95 63 ms. 0 of 59,400 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "e668ebbe34b99159307974b6d33d8c332ba254fd8b233a325b2c473a7ea24087", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-umami.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Umami", "subject_version": "3.3.0", "subject_digests": [ "ghcr.io/umami-software/umami@sha256:87312d334d009ee67ee0d2fba8fed01435547cc468e452243aef5133a9984d48" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "CLIENT_IP_HEADER=x-forwarded-for so the visitor pool is not one session" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored == acknowledged at every stage; no buffering tier in the source", "tracker_retry": "none", "tracker_retry_source": "umami-software/umami src/tracker/index.ts (3.3.1) @ 8efd5d72: one fetch, errors swallowed", "tracker_retry_measured": false, "configuration_id": "cfg-umami-ramp2", "record_file": "2026-09-14-split-umami.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-umami.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-umami-ramp2-kill", "run_id": "run-2026-09-14-umami-ramp2", "stage": "kill:umami", "offered_rate": 99.0, "offered_unit": "events_per_s", "duration_s": 90, "sent": 8186, "acknowledged": 4992, "stored_rows": 4994, "stored_distinct": 4994, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": -2, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 2, "hard_failures": 3194, "p95_ms": 13964.2, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 9.0, "k6_cpu_max_pct": 26.7, "qualifications": [ "single_trial", "healthy_probe_recovery" ], "record_type": "failure_test", "failure_scope": { "component": "umami (Node app)", "mechanism": "docker kill (API SIGKILL)", "accept_path_up": false }, "kill_at_s": 20, "restart_by_driver_at_s": 35.9, "restart_policy_text": "no", "restart_policy_fired": false, "restart_policy_observation": "api_kill_10s_watch", "recovery": [ { "definition": "healthy_probe", "seconds": 63 } ], "observed_outcome": "0 of 4,992 acknowledged requests were shown to be missing after killing umami (Node app). 3,194 requests failed while the component was down.", "record_sha256": "e668ebbe34b99159307974b6d33d8c332ba254fd8b233a325b2c473a7ea24087", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-umami.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Matomo", "subject_version": "5.12.0", "subject_digests": [ "matomo@sha256:b6745de05fae0284ecfd564b82dc4a722a195c13f4c3dd8927441162907b37ad", "mariadb@sha256:1fe78d53850250aa2560aa0059e3088b4cd230a5db2230f530c70e4b87bcc30c" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "proxy_client_headers[]=HTTP_X_FORWARDED_FOR", "browser_archiving_disabled_enforce=1", "MariaDB --max-allowed-packet=64MB" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored >= acknowledged at every stage (timed-out requests were written); tracker inserts synchronously without QueuedTracking", "tracker_retry": "none", "tracker_retry_source": "matomo-org/matomo js/piwik.js 5.14.0-alpha @ 623f2865: batching queue exists, a failed request is not resent", "tracker_retry_measured": false, "configuration_id": "cfg-matomo-ramp2", "record_file": "2026-09-14-split-matomo.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-matomo.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-matomo-ramp2-r25", "run_id": "run-2026-09-14-matomo-ramp2", "stage": "ramp:25", "offered_rate": 25.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 2718, "acknowledged": 2548, "stored_rows": 2718, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": -170, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 170, "p95_ms": 59999.4, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 4.0, "k6_cpu_max_pct": 20.1, "qualifications": [ "single_trial", "count_based_reconciliation" ], "record_type": "throughput_stage", "achieved_rate": 21.2, "rate_bound": "exact", "dropped_iterations": 282, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "21.2 acknowledged events/s achieved at 25 offered; p95 60.00 s. 170 more rows stored than acknowledged (2,548 acknowledged, 2,718 stored), consistent with responses the client never received; count-based.", "record_sha256": "16024ba1ccc1c58cfc10d5d09ed0eb1b53f125bc8e297c32528b3da8f6293889", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-matomo.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Matomo", "subject_version": "5.12.0", "subject_digests": [ "matomo@sha256:b6745de05fae0284ecfd564b82dc4a722a195c13f4c3dd8927441162907b37ad", "mariadb@sha256:1fe78d53850250aa2560aa0059e3088b4cd230a5db2230f530c70e4b87bcc30c" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "proxy_client_headers[]=HTTP_X_FORWARDED_FOR", "browser_archiving_disabled_enforce=1", "MariaDB --max-allowed-packet=64MB" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored >= acknowledged at every stage (timed-out requests were written); tracker inserts synchronously without QueuedTracking", "tracker_retry": "none", "tracker_retry_source": "matomo-org/matomo js/piwik.js 5.14.0-alpha @ 623f2865: batching queue exists, a failed request is not resent", "tracker_retry_measured": false, "configuration_id": "cfg-matomo-ramp2", "record_file": "2026-09-14-split-matomo.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-matomo.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-matomo-ramp2-r50", "run_id": "run-2026-09-14-matomo-ramp2", "stage": "ramp:50", "offered_rate": 50.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 4965, "acknowledged": 3365, "stored_rows": 4067, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": -702, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 1600, "p95_ms": 59999.7, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 10.0, "k6_cpu_max_pct": 41.3, "qualifications": [ "single_trial", "count_based_reconciliation", "application_bound" ], "record_type": "throughput_stage", "achieved_rate": 28.0, "rate_bound": "exact", "dropped_iterations": 845, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": true, "observed_outcome": "28.0 acknowledged events/s achieved at 50 offered; p95 60.00 s. 702 more rows stored than acknowledged (3,365 acknowledged, 4,067 stored), consistent with responses the client never received; count-based. Selected as the highest observed throughput by the protocol's ceiling rule.", "record_sha256": "16024ba1ccc1c58cfc10d5d09ed0eb1b53f125bc8e297c32528b3da8f6293889", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-matomo.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Matomo", "subject_version": "5.12.0", "subject_digests": [ "matomo@sha256:b6745de05fae0284ecfd564b82dc4a722a195c13f4c3dd8927441162907b37ad", "mariadb@sha256:1fe78d53850250aa2560aa0059e3088b4cd230a5db2230f530c70e4b87bcc30c" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "proxy_client_headers[]=HTTP_X_FORWARDED_FOR", "browser_archiving_disabled_enforce=1", "MariaDB --max-allowed-packet=64MB" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored >= acknowledged at every stage (timed-out requests were written); tracker inserts synchronously without QueuedTracking", "tracker_retry": "none", "tracker_retry_source": "matomo-org/matomo js/piwik.js 5.14.0-alpha @ 623f2865: batching queue exists, a failed request is not resent", "tracker_retry_measured": false, "configuration_id": "cfg-matomo-ramp2", "record_file": "2026-09-14-split-matomo.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-matomo.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-matomo-ramp2-r100", "run_id": "run-2026-09-14-matomo-ramp2", "stage": "ramp:100", "offered_rate": 100.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 7517, "acknowledged": 3250, "stored_rows": 4126, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": -876, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 4267, "p95_ms": 59999.5, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 17.0, "k6_cpu_max_pct": 71.4, "qualifications": [ "single_trial", "count_based_reconciliation" ], "record_type": "throughput_stage", "achieved_rate": 27.1, "rate_bound": "exact", "dropped_iterations": 4119, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "27.1 acknowledged events/s achieved at 100 offered; p95 60.00 s. 876 more rows stored than acknowledged (3,250 acknowledged, 4,126 stored), consistent with responses the client never received; count-based.", "record_sha256": "16024ba1ccc1c58cfc10d5d09ed0eb1b53f125bc8e297c32528b3da8f6293889", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-matomo.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Matomo", "subject_version": "5.12.0", "subject_digests": [ "matomo@sha256:b6745de05fae0284ecfd564b82dc4a722a195c13f4c3dd8927441162907b37ad", "mariadb@sha256:1fe78d53850250aa2560aa0059e3088b4cd230a5db2230f530c70e4b87bcc30c" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "proxy_client_headers[]=HTTP_X_FORWARDED_FOR", "browser_archiving_disabled_enforce=1", "MariaDB --max-allowed-packet=64MB" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored >= acknowledged at every stage (timed-out requests were written); tracker inserts synchronously without QueuedTracking", "tracker_retry": "none", "tracker_retry_source": "matomo-org/matomo js/piwik.js 5.14.0-alpha @ 623f2865: batching queue exists, a failed request is not resent", "tracker_retry_measured": false, "configuration_id": "cfg-matomo-ramp2", "record_file": "2026-09-14-split-matomo.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-matomo.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-matomo-ramp2-r200", "run_id": "run-2026-09-14-matomo-ramp2", "stage": "ramp:200", "offered_rate": 200.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 7956, "acknowledged": 3332, "stored_rows": 3958, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": -626, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 4624, "p95_ms": 59999.5, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 32.0, "k6_cpu_max_pct": 128.0, "qualifications": [ "single_trial", "count_based_reconciliation" ], "record_type": "throughput_stage", "achieved_rate": 27.8, "rate_bound": "exact", "dropped_iterations": 15931, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "27.8 acknowledged events/s achieved at 200 offered; p95 60.00 s. 626 more rows stored than acknowledged (3,332 acknowledged, 3,958 stored), consistent with responses the client never received; count-based.", "record_sha256": "16024ba1ccc1c58cfc10d5d09ed0eb1b53f125bc8e297c32528b3da8f6293889", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-matomo.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Matomo", "subject_version": "5.12.0", "subject_digests": [ "matomo@sha256:b6745de05fae0284ecfd564b82dc4a722a195c13f4c3dd8927441162907b37ad", "mariadb@sha256:1fe78d53850250aa2560aa0059e3088b4cd230a5db2230f530c70e4b87bcc30c" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "proxy_client_headers[]=HTTP_X_FORWARDED_FOR", "browser_archiving_disabled_enforce=1", "MariaDB --max-allowed-packet=64MB" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored >= acknowledged at every stage (timed-out requests were written); tracker inserts synchronously without QueuedTracking", "tracker_retry": "none", "tracker_retry_source": "matomo-org/matomo js/piwik.js 5.14.0-alpha @ 623f2865: batching queue exists, a failed request is not resent", "tracker_retry_measured": false, "configuration_id": "cfg-matomo-ramp2", "record_file": "2026-09-14-split-matomo.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-matomo.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-matomo-ramp2-r400", "run_id": "run-2026-09-14-matomo-ramp2", "stage": "ramp:400", "offered_rate": 400.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 7888, "acknowledged": 3003, "stored_rows": 3515, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": -512, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 4885, "p95_ms": 59999.6, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 40.0, "k6_cpu_max_pct": 162.0, "qualifications": [ "single_trial", "count_based_reconciliation" ], "record_type": "throughput_stage", "achieved_rate": 25.0, "rate_bound": "exact", "dropped_iterations": 40113, "stored_settle_s": 20.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "25.0 acknowledged events/s achieved at 400 offered; p95 60.00 s. 512 more rows stored than acknowledged (3,003 acknowledged, 3,515 stored), consistent with responses the client never received; count-based.", "record_sha256": "16024ba1ccc1c58cfc10d5d09ed0eb1b53f125bc8e297c32528b3da8f6293889", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-matomo.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Matomo", "subject_version": "5.12.0", "subject_digests": [ "matomo@sha256:b6745de05fae0284ecfd564b82dc4a722a195c13f4c3dd8927441162907b37ad", "mariadb@sha256:1fe78d53850250aa2560aa0059e3088b4cd230a5db2230f530c70e4b87bcc30c" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "proxy_client_headers[]=HTTP_X_FORWARDED_FOR", "browser_archiving_disabled_enforce=1", "MariaDB --max-allowed-packet=64MB" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored >= acknowledged at every stage (timed-out requests were written); tracker inserts synchronously without QueuedTracking", "tracker_retry": "none", "tracker_retry_source": "matomo-org/matomo js/piwik.js 5.14.0-alpha @ 623f2865: batching queue exists, a failed request is not resent", "tracker_retry_measured": false, "configuration_id": "cfg-matomo-ramp2", "record_file": "2026-09-14-split-matomo.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-matomo.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-matomo-ramp2-r800", "run_id": "run-2026-09-14-matomo-ramp2", "stage": "ramp:800", "offered_rate": 800.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 8016, "acknowledged": 3129, "stored_rows": 3641, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": -512, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 4887, "p95_ms": 59999.6, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 47.0, "k6_cpu_max_pct": 175.0, "qualifications": [ "single_trial", "count_based_reconciliation" ], "record_type": "throughput_stage", "achieved_rate": 26.1, "rate_bound": "exact", "dropped_iterations": 87985, "stored_settle_s": 20.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "26.1 acknowledged events/s achieved at 800 offered; p95 60.00 s. 512 more rows stored than acknowledged (3,129 acknowledged, 3,641 stored), consistent with responses the client never received; count-based.", "record_sha256": "16024ba1ccc1c58cfc10d5d09ed0eb1b53f125bc8e297c32528b3da8f6293889", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-matomo.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Matomo", "subject_version": "5.12.0", "subject_digests": [ "matomo@sha256:b6745de05fae0284ecfd564b82dc4a722a195c13f4c3dd8927441162907b37ad", "mariadb@sha256:1fe78d53850250aa2560aa0059e3088b4cd230a5db2230f530c70e4b87bcc30c" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "proxy_client_headers[]=HTTP_X_FORWARDED_FOR", "browser_archiving_disabled_enforce=1", "MariaDB --max-allowed-packet=64MB" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored >= acknowledged at every stage (timed-out requests were written); tracker inserts synchronously without QueuedTracking", "tracker_retry": "none", "tracker_retry_source": "matomo-org/matomo js/piwik.js 5.14.0-alpha @ 623f2865: batching queue exists, a failed request is not resent", "tracker_retry_measured": false, "configuration_id": "cfg-matomo-ramp2", "record_file": "2026-09-14-split-matomo.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-matomo.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-matomo-ramp2-soak", "run_id": "run-2026-09-14-matomo-ramp2", "stage": "soak:22", "offered_rate": 22.0, "offered_unit": "events_per_s", "duration_s": 600, "sent": 12921, "acknowledged": 12921, "stored_rows": 12921, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 0, "p95_ms": 5879.8, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 6.0, "k6_cpu_max_pct": 18.1, "qualifications": [ "single_trial", "count_based_reconciliation" ], "record_type": "throughput_stage", "achieved_rate": 21.5, "rate_bound": "exact", "dropped_iterations": 279, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "21.5 acknowledged events/s achieved at 22 offered; p95 5.88 s. No count deficit (12,921 acknowledged, 12,921 stored); count-based, so this does not rule out losses offset by duplicates.", "record_sha256": "16024ba1ccc1c58cfc10d5d09ed0eb1b53f125bc8e297c32528b3da8f6293889", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-matomo.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Matomo", "subject_version": "5.12.0", "subject_digests": [ "matomo@sha256:b6745de05fae0284ecfd564b82dc4a722a195c13f4c3dd8927441162907b37ad", "mariadb@sha256:1fe78d53850250aa2560aa0059e3088b4cd230a5db2230f530c70e4b87bcc30c" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "proxy_client_headers[]=HTTP_X_FORWARDED_FOR", "browser_archiving_disabled_enforce=1", "MariaDB --max-allowed-packet=64MB" ], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": null, "classification_evidence": "run records: stored >= acknowledged at every stage (timed-out requests were written); tracker inserts synchronously without QueuedTracking", "tracker_retry": "none", "tracker_retry_source": "matomo-org/matomo js/piwik.js 5.14.0-alpha @ 623f2865: batching queue exists, a failed request is not resent", "tracker_retry_measured": false, "configuration_id": "cfg-matomo-ramp2", "record_file": "2026-09-14-split-matomo.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-matomo.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-matomo-ramp2-kill", "run_id": "run-2026-09-14-matomo-ramp2", "stage": "kill:matomo", "offered_rate": 22.0, "offered_unit": "events_per_s", "duration_s": 90, "sent": 1748, "acknowledged": 1411, "stored_rows": 1411, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 337, "p95_ms": 34303.0, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 5.0, "k6_cpu_max_pct": 18.3, "qualifications": [ "single_trial", "count_based_reconciliation", "healthy_probe_recovery" ], "record_type": "failure_test", "failure_scope": { "component": "matomo (Apache/PHP)", "mechanism": "docker kill (API SIGKILL)", "accept_path_up": false }, "kill_at_s": 20, "restart_by_driver_at_s": 34.8, "restart_policy_text": "no", "restart_policy_fired": false, "restart_policy_observation": "api_kill_10s_watch", "recovery": [ { "definition": "healthy_probe", "seconds": 80 } ], "observed_outcome": "No count deficit was observed after killing matomo (Apache/PHP) (1,411 acknowledged, 1,411 stored). Because reconciliation compared totals, this does not rule out losses offset by duplicates. 337 requests failed while the component was down.", "record_sha256": "16024ba1ccc1c58cfc10d5d09ed0eb1b53f125bc8e297c32528b3da8f6293889", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-matomo.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Rybbit", "subject_version": "2.8.0 (source rev 4a01c7a)", "subject_digests": [ "ghcr.io/rybbit-io/rybbit-backend@sha256:f729f425…6ac7 (full digest in the kit compose)", "ghcr.io/rybbit-io/rybbit-client@sha256:f384c1f7…103a (full digest in the kit compose)" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "blockBots=false for the bench site", "HOST_BACKEND_PORT=0.0.0.0:3001:3001 (vendor default binds the backend to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "200 before insert; up to ~1 s in-process queue then batch insert", "classification_evidence": "rybbit server pageview queue source; kill loss by event id", "tracker_retry": "none", "tracker_retry_source": "rybbit-io/rybbit server/src/analytics-script/tracking.ts @ 8a22fd9f (2.9.1): one fetch, no retry", "tracker_retry_measured": false, "configuration_id": "cfg-rybbit-ramp2", "record_file": "2026-09-14-split-rybbit.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-rybbit.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-rybbit-ramp2-r25", "run_id": "run-2026-09-14-rybbit-ramp2", "stage": "ramp:25", "offered_rate": 25.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 3001, "acknowledged": 3001, "stored_rows": 3001, "stored_distinct": 3001, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 10.4, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 4.0, "k6_cpu_max_pct": 9.1, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 25.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "25.0 acknowledged events/s achieved at 25 offered; p95 10 ms. 0 of 3,001 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "d5d0784e93bb8cb9145aad33c111ea482b71c1125c9eb62bcb6543191a3c33e4", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-rybbit.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Rybbit", "subject_version": "2.8.0 (source rev 4a01c7a)", "subject_digests": [ "ghcr.io/rybbit-io/rybbit-backend@sha256:f729f425…6ac7 (full digest in the kit compose)", "ghcr.io/rybbit-io/rybbit-client@sha256:f384c1f7…103a (full digest in the kit compose)" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "blockBots=false for the bench site", "HOST_BACKEND_PORT=0.0.0.0:3001:3001 (vendor default binds the backend to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "200 before insert; up to ~1 s in-process queue then batch insert", "classification_evidence": "rybbit server pageview queue source; kill loss by event id", "tracker_retry": "none", "tracker_retry_source": "rybbit-io/rybbit server/src/analytics-script/tracking.ts @ 8a22fd9f (2.9.1): one fetch, no retry", "tracker_retry_measured": false, "configuration_id": "cfg-rybbit-ramp2", "record_file": "2026-09-14-split-rybbit.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-rybbit.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-rybbit-ramp2-r50", "run_id": "run-2026-09-14-rybbit-ramp2", "stage": "ramp:50", "offered_rate": 50.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 6001, "acknowledged": 6001, "stored_rows": 6001, "stored_distinct": 6001, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 12.1, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 4.0, "k6_cpu_max_pct": 12.5, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 50.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "50.0 acknowledged events/s achieved at 50 offered; p95 12 ms. 0 of 6,001 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "d5d0784e93bb8cb9145aad33c111ea482b71c1125c9eb62bcb6543191a3c33e4", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-rybbit.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Rybbit", "subject_version": "2.8.0 (source rev 4a01c7a)", "subject_digests": [ "ghcr.io/rybbit-io/rybbit-backend@sha256:f729f425…6ac7 (full digest in the kit compose)", "ghcr.io/rybbit-io/rybbit-client@sha256:f384c1f7…103a (full digest in the kit compose)" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "blockBots=false for the bench site", "HOST_BACKEND_PORT=0.0.0.0:3001:3001 (vendor default binds the backend to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "200 before insert; up to ~1 s in-process queue then batch insert", "classification_evidence": "rybbit server pageview queue source; kill loss by event id", "tracker_retry": "none", "tracker_retry_source": "rybbit-io/rybbit server/src/analytics-script/tracking.ts @ 8a22fd9f (2.9.1): one fetch, no retry", "tracker_retry_measured": false, "configuration_id": "cfg-rybbit-ramp2", "record_file": "2026-09-14-split-rybbit.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-rybbit.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-rybbit-ramp2-r100", "run_id": "run-2026-09-14-rybbit-ramp2", "stage": "ramp:100", "offered_rate": 100.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 12001, "acknowledged": 12001, "stored_rows": 12001, "stored_distinct": 12001, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 12.2, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 6.0, "k6_cpu_max_pct": 22.1, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 100.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "100.0 acknowledged events/s achieved at 100 offered; p95 12 ms. 0 of 12,001 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "d5d0784e93bb8cb9145aad33c111ea482b71c1125c9eb62bcb6543191a3c33e4", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-rybbit.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Rybbit", "subject_version": "2.8.0 (source rev 4a01c7a)", "subject_digests": [ "ghcr.io/rybbit-io/rybbit-backend@sha256:f729f425…6ac7 (full digest in the kit compose)", "ghcr.io/rybbit-io/rybbit-client@sha256:f384c1f7…103a (full digest in the kit compose)" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "blockBots=false for the bench site", "HOST_BACKEND_PORT=0.0.0.0:3001:3001 (vendor default binds the backend to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "200 before insert; up to ~1 s in-process queue then batch insert", "classification_evidence": "rybbit server pageview queue source; kill loss by event id", "tracker_retry": "none", "tracker_retry_source": "rybbit-io/rybbit server/src/analytics-script/tracking.ts @ 8a22fd9f (2.9.1): one fetch, no retry", "tracker_retry_measured": false, "configuration_id": "cfg-rybbit-ramp2", "record_file": "2026-09-14-split-rybbit.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-rybbit.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-rybbit-ramp2-r200", "run_id": "run-2026-09-14-rybbit-ramp2", "stage": "ramp:200", "offered_rate": 200.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 23381, "acknowledged": 23381, "stored_rows": 23381, "stored_distinct": 23381, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 2902.9, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 8.0, "k6_cpu_max_pct": 37.1, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 194.8, "rate_bound": "exact", "dropped_iterations": 620, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "194.8 acknowledged events/s achieved at 200 offered; p95 2.90 s. 0 of 23,381 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "d5d0784e93bb8cb9145aad33c111ea482b71c1125c9eb62bcb6543191a3c33e4", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-rybbit.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Rybbit", "subject_version": "2.8.0 (source rev 4a01c7a)", "subject_digests": [ "ghcr.io/rybbit-io/rybbit-backend@sha256:f729f425…6ac7 (full digest in the kit compose)", "ghcr.io/rybbit-io/rybbit-client@sha256:f384c1f7…103a (full digest in the kit compose)" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "blockBots=false for the bench site", "HOST_BACKEND_PORT=0.0.0.0:3001:3001 (vendor default binds the backend to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "200 before insert; up to ~1 s in-process queue then batch insert", "classification_evidence": "rybbit server pageview queue source; kill loss by event id", "tracker_retry": "none", "tracker_retry_source": "rybbit-io/rybbit server/src/analytics-script/tracking.ts @ 8a22fd9f (2.9.1): one fetch, no retry", "tracker_retry_measured": false, "configuration_id": "cfg-rybbit-ramp2", "record_file": "2026-09-14-split-rybbit.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-rybbit.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-rybbit-ramp2-r400", "run_id": "run-2026-09-14-rybbit-ramp2", "stage": "ramp:400", "offered_rate": 400.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 48002, "acknowledged": 48002, "stored_rows": 48002, "stored_distinct": 48002, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 176.2, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 16.0, "k6_cpu_max_pct": 63.2, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 400.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "400.0 acknowledged events/s achieved at 400 offered; p95 176 ms. 0 of 48,002 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "d5d0784e93bb8cb9145aad33c111ea482b71c1125c9eb62bcb6543191a3c33e4", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-rybbit.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Rybbit", "subject_version": "2.8.0 (source rev 4a01c7a)", "subject_digests": [ "ghcr.io/rybbit-io/rybbit-backend@sha256:f729f425…6ac7 (full digest in the kit compose)", "ghcr.io/rybbit-io/rybbit-client@sha256:f384c1f7…103a (full digest in the kit compose)" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "blockBots=false for the bench site", "HOST_BACKEND_PORT=0.0.0.0:3001:3001 (vendor default binds the backend to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "200 before insert; up to ~1 s in-process queue then batch insert", "classification_evidence": "rybbit server pageview queue source; kill loss by event id", "tracker_retry": "none", "tracker_retry_source": "rybbit-io/rybbit server/src/analytics-script/tracking.ts @ 8a22fd9f (2.9.1): one fetch, no retry", "tracker_retry_measured": false, "configuration_id": "cfg-rybbit-ramp2", "record_file": "2026-09-14-split-rybbit.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-rybbit.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-rybbit-ramp2-r800", "run_id": "run-2026-09-14-rybbit-ramp2", "stage": "ramp:800", "offered_rate": 800.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 90864, "acknowledged": 90864, "stored_rows": 90864, "stored_distinct": 90864, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 1971.8, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 22.0, "k6_cpu_max_pct": 89.2, "qualifications": [ "single_trial", "application_bound" ], "record_type": "throughput_stage", "achieved_rate": 757.2, "rate_bound": "exact", "dropped_iterations": 5137, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": true, "observed_outcome": "757.2 acknowledged events/s achieved at 800 offered; p95 1.97 s. 0 of 90,864 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id). Selected as the highest observed throughput by the protocol's ceiling rule.", "record_sha256": "d5d0784e93bb8cb9145aad33c111ea482b71c1125c9eb62bcb6543191a3c33e4", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-rybbit.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Rybbit", "subject_version": "2.8.0 (source rev 4a01c7a)", "subject_digests": [ "ghcr.io/rybbit-io/rybbit-backend@sha256:f729f425…6ac7 (full digest in the kit compose)", "ghcr.io/rybbit-io/rybbit-client@sha256:f384c1f7…103a (full digest in the kit compose)" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "blockBots=false for the bench site", "HOST_BACKEND_PORT=0.0.0.0:3001:3001 (vendor default binds the backend to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "200 before insert; up to ~1 s in-process queue then batch insert", "classification_evidence": "rybbit server pageview queue source; kill loss by event id", "tracker_retry": "none", "tracker_retry_source": "rybbit-io/rybbit server/src/analytics-script/tracking.ts @ 8a22fd9f (2.9.1): one fetch, no retry", "tracker_retry_measured": false, "configuration_id": "cfg-rybbit-ramp2", "record_file": "2026-09-14-split-rybbit.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-rybbit.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-rybbit-ramp2-r1600", "run_id": "run-2026-09-14-rybbit-ramp2", "stage": "ramp:1600", "offered_rate": 1600.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 89779, "acknowledged": 89256, "stored_rows": 89779, "stored_distinct": 89779, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": -523, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 523, "hard_failures": 523, "p95_ms": 2145.7, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 30.0, "k6_cpu_max_pct": 119.0, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 743.8, "rate_bound": "exact", "dropped_iterations": 102222, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "743.8 acknowledged events/s achieved at 1,600 offered; p95 2.15 s. 0 of 89,256 acknowledged ids were missing from storage, 0 stored rows were duplicates, 523 stored ids were never acknowledged to the client (matched by event id).", "record_sha256": "d5d0784e93bb8cb9145aad33c111ea482b71c1125c9eb62bcb6543191a3c33e4", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-rybbit.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Rybbit", "subject_version": "2.8.0 (source rev 4a01c7a)", "subject_digests": [ "ghcr.io/rybbit-io/rybbit-backend@sha256:f729f425…6ac7 (full digest in the kit compose)", "ghcr.io/rybbit-io/rybbit-client@sha256:f384c1f7…103a (full digest in the kit compose)" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "blockBots=false for the bench site", "HOST_BACKEND_PORT=0.0.0.0:3001:3001 (vendor default binds the backend to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "200 before insert; up to ~1 s in-process queue then batch insert", "classification_evidence": "rybbit server pageview queue source; kill loss by event id", "tracker_retry": "none", "tracker_retry_source": "rybbit-io/rybbit server/src/analytics-script/tracking.ts @ 8a22fd9f (2.9.1): one fetch, no retry", "tracker_retry_measured": false, "configuration_id": "cfg-rybbit-ramp2", "record_file": "2026-09-14-split-rybbit.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-rybbit.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-rybbit-ramp2-soak", "run_id": "run-2026-09-14-rybbit-ramp2", "stage": "soak:605", "offered_rate": 605.0, "offered_unit": "events_per_s", "duration_s": 600, "sent": 347012, "acknowledged": 347012, "stored_rows": 347012, "stored_distinct": 347012, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 767.9, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 21.0, "k6_cpu_max_pct": 77.4, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 578.4, "rate_bound": "exact", "dropped_iterations": 15988, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "578.4 acknowledged events/s achieved at 605 offered; p95 768 ms. 0 of 347,012 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "d5d0784e93bb8cb9145aad33c111ea482b71c1125c9eb62bcb6543191a3c33e4", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-rybbit.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Rybbit", "subject_version": "2.8.0 (source rev 4a01c7a)", "subject_digests": [ "ghcr.io/rybbit-io/rybbit-backend@sha256:f729f425…6ac7 (full digest in the kit compose)", "ghcr.io/rybbit-io/rybbit-client@sha256:f384c1f7…103a (full digest in the kit compose)" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [ "blockBots=false for the bench site", "HOST_BACKEND_PORT=0.0.0.0:3001:3001 (vendor default binds the backend to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "200 before insert; up to ~1 s in-process queue then batch insert", "classification_evidence": "rybbit server pageview queue source; kill loss by event id", "tracker_retry": "none", "tracker_retry_source": "rybbit-io/rybbit server/src/analytics-script/tracking.ts @ 8a22fd9f (2.9.1): one fetch, no retry", "tracker_retry_measured": false, "configuration_id": "cfg-rybbit-ramp2", "record_file": "2026-09-14-split-rybbit.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-rybbit.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-rybbit-ramp2-kill", "run_id": "run-2026-09-14-rybbit-ramp2", "stage": "kill:backend", "offered_rate": 605.0, "offered_unit": "events_per_s", "duration_s": 90, "sent": 50338, "acknowledged": 19310, "stored_rows": 18951, "stored_distinct": 18951, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 359, "acknowledged_missing_count": 359, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 31028, "p95_ms": 1091.8, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 24.0, "k6_cpu_max_pct": 82.4, "qualifications": [ "single_trial", "healthy_probe_recovery" ], "record_type": "failure_test", "failure_scope": { "component": "backend", "mechanism": "docker kill (API SIGKILL)", "accept_path_up": false }, "kill_at_s": 20, "restart_by_driver_at_s": 35.5, "restart_policy_text": "unless-stopped", "restart_policy_fired": false, "restart_policy_observation": "api_kill_10s_watch", "recovery": [ { "definition": "healthy_probe", "seconds": 75 } ], "observed_outcome": "359 of 19,310 acknowledged requests were shown to be missing after killing backend. 31,028 requests failed while the component was down.", "record_sha256": "d5d0784e93bb8cb9145aad33c111ea482b71c1125c9eb62bcb6543191a3c33e4", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-rybbit.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Jetboat lite stack (ours)", "subject_version": "build 297be96", "subject_digests": [ "app image built from source on the box", "clickhouse/clickhouse-server@sha256:b627d7a9bc0e0c1bac26cdbe9d2fc6316faa29c5d8a174f28f5abd57d0fa6ba2" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-17", "deviations": [ "APP_BIND=0.0.0.0 (the shipped compose binds the app to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "durable_log", "acknowledgement_strategy_note": "append-only log written to disk before the 202; a separate drain moves rows to ClickHouse", "classification_evidence": "backend/cmd/server/main.go durable-buffer branch; drain-stall alerts and replay lines in the run records", "tracker_retry": "persisted", "tracker_retry_source": "sdk/src/transport.ts @ cbec428: localStorage retry queue, 2000 ms × 2^n capped at 5 min, 48 h horizon, pagehide sendBeacon", "tracker_retry_measured": false, "configuration_id": "cfg-ours-ramp2", "record_file": "2026-09-17-split-ours.md", "record_path": "load-test-2026-09-14/results/2026-09-17-split-ours.md", "trials": 1, "note_ids": [], "overload_behaviour": "Past a soft cap the accept path returns 202, then deliberately drops low-value events and counts them (shed counters). Not exercised in these stages; recorded so that a zero-loss kill result is not read as a general no-loss guarantee.", "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-17-ours-ramp2-r25", "run_id": "run-2026-09-17-ours-ramp2", "stage": "ramp:25", "offered_rate": 25.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 3001, "acknowledged": 3001, "stored_rows": 3001, "stored_distinct": 3001, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 6.1, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 3.0, "k6_cpu_max_pct": 5.7, "qualifications": [ "single_trial", "overload_discard" ], "record_type": "throughput_stage", "achieved_rate": 25.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "25.0 acknowledged events/s achieved at 25 offered; p95 6 ms. 0 of 3,001 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "d5f83155007542535fc8a527879adb6d005246a5dc3a0850287c8ca83b0ba342", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-17-split-ours.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Jetboat lite stack (ours)", "subject_version": "build 297be96", "subject_digests": [ "app image built from source on the box", "clickhouse/clickhouse-server@sha256:b627d7a9bc0e0c1bac26cdbe9d2fc6316faa29c5d8a174f28f5abd57d0fa6ba2" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-17", "deviations": [ "APP_BIND=0.0.0.0 (the shipped compose binds the app to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "durable_log", "acknowledgement_strategy_note": "append-only log written to disk before the 202; a separate drain moves rows to ClickHouse", "classification_evidence": "backend/cmd/server/main.go durable-buffer branch; drain-stall alerts and replay lines in the run records", "tracker_retry": "persisted", "tracker_retry_source": "sdk/src/transport.ts @ cbec428: localStorage retry queue, 2000 ms × 2^n capped at 5 min, 48 h horizon, pagehide sendBeacon", "tracker_retry_measured": false, "configuration_id": "cfg-ours-ramp2", "record_file": "2026-09-17-split-ours.md", "record_path": "load-test-2026-09-14/results/2026-09-17-split-ours.md", "trials": 1, "note_ids": [], "overload_behaviour": "Past a soft cap the accept path returns 202, then deliberately drops low-value events and counts them (shed counters). Not exercised in these stages; recorded so that a zero-loss kill result is not read as a general no-loss guarantee.", "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-17-ours-ramp2-r50", "run_id": "run-2026-09-17-ours-ramp2", "stage": "ramp:50", "offered_rate": 50.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 6001, "acknowledged": 6001, "stored_rows": 6001, "stored_distinct": 6001, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 6.7, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 3.0, "k6_cpu_max_pct": 11.3, "qualifications": [ "single_trial", "overload_discard" ], "record_type": "throughput_stage", "achieved_rate": 50.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "50.0 acknowledged events/s achieved at 50 offered; p95 7 ms. 0 of 6,001 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "d5f83155007542535fc8a527879adb6d005246a5dc3a0850287c8ca83b0ba342", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-17-split-ours.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Jetboat lite stack (ours)", "subject_version": "build 297be96", "subject_digests": [ "app image built from source on the box", "clickhouse/clickhouse-server@sha256:b627d7a9bc0e0c1bac26cdbe9d2fc6316faa29c5d8a174f28f5abd57d0fa6ba2" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-17", "deviations": [ "APP_BIND=0.0.0.0 (the shipped compose binds the app to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "durable_log", "acknowledgement_strategy_note": "append-only log written to disk before the 202; a separate drain moves rows to ClickHouse", "classification_evidence": "backend/cmd/server/main.go durable-buffer branch; drain-stall alerts and replay lines in the run records", "tracker_retry": "persisted", "tracker_retry_source": "sdk/src/transport.ts @ cbec428: localStorage retry queue, 2000 ms × 2^n capped at 5 min, 48 h horizon, pagehide sendBeacon", "tracker_retry_measured": false, "configuration_id": "cfg-ours-ramp2", "record_file": "2026-09-17-split-ours.md", "record_path": "load-test-2026-09-14/results/2026-09-17-split-ours.md", "trials": 1, "note_ids": [], "overload_behaviour": "Past a soft cap the accept path returns 202, then deliberately drops low-value events and counts them (shed counters). Not exercised in these stages; recorded so that a zero-loss kill result is not read as a general no-loss guarantee.", "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-17-ours-ramp2-r100", "run_id": "run-2026-09-17-ours-ramp2", "stage": "ramp:100", "offered_rate": 100.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 11973, "acknowledged": 11973, "stored_rows": 11973, "stored_distinct": 11973, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 151.8, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 6.0, "k6_cpu_max_pct": 18.9, "qualifications": [ "single_trial", "overload_discard" ], "record_type": "throughput_stage", "achieved_rate": 99.8, "rate_bound": "exact", "dropped_iterations": 28, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "99.8 acknowledged events/s achieved at 100 offered; p95 152 ms. 0 of 11,973 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "d5f83155007542535fc8a527879adb6d005246a5dc3a0850287c8ca83b0ba342", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-17-split-ours.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Jetboat lite stack (ours)", "subject_version": "build 297be96", "subject_digests": [ "app image built from source on the box", "clickhouse/clickhouse-server@sha256:b627d7a9bc0e0c1bac26cdbe9d2fc6316faa29c5d8a174f28f5abd57d0fa6ba2" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-17", "deviations": [ "APP_BIND=0.0.0.0 (the shipped compose binds the app to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "durable_log", "acknowledgement_strategy_note": "append-only log written to disk before the 202; a separate drain moves rows to ClickHouse", "classification_evidence": "backend/cmd/server/main.go durable-buffer branch; drain-stall alerts and replay lines in the run records", "tracker_retry": "persisted", "tracker_retry_source": "sdk/src/transport.ts @ cbec428: localStorage retry queue, 2000 ms × 2^n capped at 5 min, 48 h horizon, pagehide sendBeacon", "tracker_retry_measured": false, "configuration_id": "cfg-ours-ramp2", "record_file": "2026-09-17-split-ours.md", "record_path": "load-test-2026-09-14/results/2026-09-17-split-ours.md", "trials": 1, "note_ids": [], "overload_behaviour": "Past a soft cap the accept path returns 202, then deliberately drops low-value events and counts them (shed counters). Not exercised in these stages; recorded so that a zero-loss kill result is not read as a general no-loss guarantee.", "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-17-ours-ramp2-r200", "run_id": "run-2026-09-17-ours-ramp2", "stage": "ramp:200", "offered_rate": 200.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 24001, "acknowledged": 24001, "stored_rows": 24001, "stored_distinct": 24001, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 5.4, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 8.0, "k6_cpu_max_pct": 30.7, "qualifications": [ "single_trial", "overload_discard" ], "record_type": "throughput_stage", "achieved_rate": 200.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "200.0 acknowledged events/s achieved at 200 offered; p95 5 ms. 0 of 24,001 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "d5f83155007542535fc8a527879adb6d005246a5dc3a0850287c8ca83b0ba342", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-17-split-ours.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Jetboat lite stack (ours)", "subject_version": "build 297be96", "subject_digests": [ "app image built from source on the box", "clickhouse/clickhouse-server@sha256:b627d7a9bc0e0c1bac26cdbe9d2fc6316faa29c5d8a174f28f5abd57d0fa6ba2" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-17", "deviations": [ "APP_BIND=0.0.0.0 (the shipped compose binds the app to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "durable_log", "acknowledgement_strategy_note": "append-only log written to disk before the 202; a separate drain moves rows to ClickHouse", "classification_evidence": "backend/cmd/server/main.go durable-buffer branch; drain-stall alerts and replay lines in the run records", "tracker_retry": "persisted", "tracker_retry_source": "sdk/src/transport.ts @ cbec428: localStorage retry queue, 2000 ms × 2^n capped at 5 min, 48 h horizon, pagehide sendBeacon", "tracker_retry_measured": false, "configuration_id": "cfg-ours-ramp2", "record_file": "2026-09-17-split-ours.md", "record_path": "load-test-2026-09-14/results/2026-09-17-split-ours.md", "trials": 1, "note_ids": [], "overload_behaviour": "Past a soft cap the accept path returns 202, then deliberately drops low-value events and counts them (shed counters). Not exercised in these stages; recorded so that a zero-loss kill result is not read as a general no-loss guarantee.", "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-17-ours-ramp2-r400", "run_id": "run-2026-09-17-ours-ramp2", "stage": "ramp:400", "offered_rate": 400.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 48001, "acknowledged": 48001, "stored_rows": 48001, "stored_distinct": 48001, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 18.6, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 13.0, "k6_cpu_max_pct": 55.8, "qualifications": [ "single_trial", "overload_discard" ], "record_type": "throughput_stage", "achieved_rate": 400.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "400.0 acknowledged events/s achieved at 400 offered; p95 19 ms. 0 of 48,001 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "d5f83155007542535fc8a527879adb6d005246a5dc3a0850287c8ca83b0ba342", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-17-split-ours.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Jetboat lite stack (ours)", "subject_version": "build 297be96", "subject_digests": [ "app image built from source on the box", "clickhouse/clickhouse-server@sha256:b627d7a9bc0e0c1bac26cdbe9d2fc6316faa29c5d8a174f28f5abd57d0fa6ba2" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-17", "deviations": [ "APP_BIND=0.0.0.0 (the shipped compose binds the app to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "durable_log", "acknowledgement_strategy_note": "append-only log written to disk before the 202; a separate drain moves rows to ClickHouse", "classification_evidence": "backend/cmd/server/main.go durable-buffer branch; drain-stall alerts and replay lines in the run records", "tracker_retry": "persisted", "tracker_retry_source": "sdk/src/transport.ts @ cbec428: localStorage retry queue, 2000 ms × 2^n capped at 5 min, 48 h horizon, pagehide sendBeacon", "tracker_retry_measured": false, "configuration_id": "cfg-ours-ramp2", "record_file": "2026-09-17-split-ours.md", "record_path": "load-test-2026-09-14/results/2026-09-17-split-ours.md", "trials": 1, "note_ids": [ "n-ours-second-trial" ], "overload_behaviour": "Past a soft cap the accept path returns 202, then deliberately drops low-value events and counts them (shed counters). Not exercised in these stages; recorded so that a zero-loss kill result is not read as a general no-loss guarantee.", "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-17-ours-ramp2-r800", "run_id": "run-2026-09-17-ours-ramp2", "stage": "ramp:800", "offered_rate": 800.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 74334, "acknowledged": 74334, "stored_rows": 74334, "stored_distinct": 74334, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 2674.9, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 17.0, "k6_cpu_max_pct": 70.6, "qualifications": [ "single_trial", "application_bound", "overload_discard" ], "record_type": "throughput_stage", "achieved_rate": 619.5, "rate_bound": "exact", "dropped_iterations": 21667, "stored_settle_s": 70.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": true, "observed_outcome": "619.5 acknowledged events/s achieved at 800 offered; p95 2.67 s. 0 of 74,334 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id). Selected as the highest observed throughput by the protocol's ceiling rule.", "record_sha256": "d5f83155007542535fc8a527879adb6d005246a5dc3a0850287c8ca83b0ba342", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-17-split-ours.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Jetboat lite stack (ours)", "subject_version": "build 297be96", "subject_digests": [ "app image built from source on the box", "clickhouse/clickhouse-server@sha256:b627d7a9bc0e0c1bac26cdbe9d2fc6316faa29c5d8a174f28f5abd57d0fa6ba2" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-17", "deviations": [ "APP_BIND=0.0.0.0 (the shipped compose binds the app to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "durable_log", "acknowledgement_strategy_note": "append-only log written to disk before the 202; a separate drain moves rows to ClickHouse", "classification_evidence": "backend/cmd/server/main.go durable-buffer branch; drain-stall alerts and replay lines in the run records", "tracker_retry": "persisted", "tracker_retry_source": "sdk/src/transport.ts @ cbec428: localStorage retry queue, 2000 ms × 2^n capped at 5 min, 48 h horizon, pagehide sendBeacon", "tracker_retry_measured": false, "configuration_id": "cfg-ours-ramp2", "record_file": "2026-09-17-split-ours.md", "record_path": "load-test-2026-09-14/results/2026-09-17-split-ours.md", "trials": 1, "note_ids": [], "overload_behaviour": "Past a soft cap the accept path returns 202, then deliberately drops low-value events and counts them (shed counters). Not exercised in these stages; recorded so that a zero-loss kill result is not read as a general no-loss guarantee.", "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-17-ours-ramp2-r1600", "run_id": "run-2026-09-17-ours-ramp2", "stage": "ramp:1600", "offered_rate": 1600.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 73126, "acknowledged": 73126, "stored_rows": 73126, "stored_distinct": 73126, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 2811.6, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 21.0, "k6_cpu_max_pct": 86.2, "qualifications": [ "single_trial", "overload_discard" ], "record_type": "throughput_stage", "achieved_rate": 609.4, "rate_bound": "exact", "dropped_iterations": 118873, "stored_settle_s": 65.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "609.4 acknowledged events/s achieved at 1,600 offered; p95 2.81 s. 0 of 73,126 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "d5f83155007542535fc8a527879adb6d005246a5dc3a0850287c8ca83b0ba342", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-17-split-ours.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Jetboat lite stack (ours)", "subject_version": "build 297be96", "subject_digests": [ "app image built from source on the box", "clickhouse/clickhouse-server@sha256:b627d7a9bc0e0c1bac26cdbe9d2fc6316faa29c5d8a174f28f5abd57d0fa6ba2" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-17", "deviations": [ "APP_BIND=0.0.0.0 (the shipped compose binds the app to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "durable_log", "acknowledgement_strategy_note": "append-only log written to disk before the 202; a separate drain moves rows to ClickHouse", "classification_evidence": "backend/cmd/server/main.go durable-buffer branch; drain-stall alerts and replay lines in the run records", "tracker_retry": "persisted", "tracker_retry_source": "sdk/src/transport.ts @ cbec428: localStorage retry queue, 2000 ms × 2^n capped at 5 min, 48 h horizon, pagehide sendBeacon", "tracker_retry_measured": false, "configuration_id": "cfg-ours-ramp2", "record_file": "2026-09-17-split-ours.md", "record_path": "load-test-2026-09-14/results/2026-09-17-split-ours.md", "trials": 1, "note_ids": [], "overload_behaviour": "Past a soft cap the accept path returns 202, then deliberately drops low-value events and counts them (shed counters). Not exercised in these stages; recorded so that a zero-loss kill result is not read as a general no-loss guarantee.", "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-17-ours-ramp2-soak", "run_id": "run-2026-09-17-ours-ramp2", "stage": "soak:495", "offered_rate": 495.0, "offered_unit": "events_per_s", "duration_s": 600, "sent": 293518, "acknowledged": 293518, "stored_rows": 293518, "stored_distinct": 293518, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 2722.6, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 13.0, "k6_cpu_max_pct": 58.2, "qualifications": [ "single_trial", "overload_discard" ], "record_type": "throughput_stage", "achieved_rate": 489.2, "rate_bound": "exact", "dropped_iterations": 3482, "stored_settle_s": 35.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "489.2 acknowledged events/s achieved at 495 offered; p95 2.72 s. 0 of 293,518 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "d5f83155007542535fc8a527879adb6d005246a5dc3a0850287c8ca83b0ba342", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-17-split-ours.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Jetboat lite stack (ours)", "subject_version": "build 297be96", "subject_digests": [ "app image built from source on the box", "clickhouse/clickhouse-server@sha256:b627d7a9bc0e0c1bac26cdbe9d2fc6316faa29c5d8a174f28f5abd57d0fa6ba2" ], "axis": "pageview", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-17", "deviations": [ "APP_BIND=0.0.0.0 (the shipped compose binds the app to loopback; the generator is a separate machine)" ], "acknowledgement_strategy": "durable_log", "acknowledgement_strategy_note": "append-only log written to disk before the 202; a separate drain moves rows to ClickHouse", "classification_evidence": "backend/cmd/server/main.go durable-buffer branch; drain-stall alerts and replay lines in the run records", "tracker_retry": "persisted", "tracker_retry_source": "sdk/src/transport.ts @ cbec428: localStorage retry queue, 2000 ms × 2^n capped at 5 min, 48 h horizon, pagehide sendBeacon", "tracker_retry_measured": false, "configuration_id": "cfg-ours-ramp2", "record_file": "2026-09-17-split-ours.md", "record_path": "load-test-2026-09-14/results/2026-09-17-split-ours.md", "trials": 1, "note_ids": [ "n-ours-second-trial" ], "overload_behaviour": "Past a soft cap the accept path returns 202, then deliberately drops low-value events and counts them (shed counters). Not exercised in these stages; recorded so that a zero-loss kill result is not read as a general no-loss guarantee.", "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-17-ours-ramp2-kill", "run_id": "run-2026-09-17-ours-ramp2", "stage": "kill:app", "offered_rate": 495.0, "offered_unit": "events_per_s", "duration_s": 90, "sent": 44072, "acknowledged": 36549, "stored_rows": 36566, "stored_distinct": 36566, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": -17, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 17, "hard_failures": 7523, "p95_ms": 1912.5, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 15.0, "k6_cpu_max_pct": 54.0, "qualifications": [ "single_trial", "healthy_probe_recovery", "overload_discard" ], "record_type": "failure_test", "failure_scope": { "component": "app", "mechanism": "docker kill (API SIGKILL)", "accept_path_up": false }, "kill_at_s": 20, "restart_by_driver_at_s": 34.6, "restart_policy_text": "unless-stopped", "restart_policy_fired": false, "restart_policy_observation": "api_kill_10s_watch", "recovery": [ { "definition": "healthy_probe", "seconds": 16 } ], "observed_outcome": "0 of 36,549 acknowledged requests were shown to be missing after killing app. 7,523 requests failed while the component was down.", "record_sha256": "d5f83155007542535fc8a527879adb6d005246a5dc3a0850287c8ca83b0ba342", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-17-split-ours.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "ClickStack / HyperDX", "subject_version": "2.35.0", "subject_digests": [ "docker.clickhouse.com/clickhouse/clickstack-otel-collector@sha256:046d88ecfde94076c6bd2835b82deaafc37036c665be5232b6eb88394a8d0a50", "clickhouse/clickhouse-server@sha256:35b419db86eed71ab1c41c03b4fd1f39be26f41eb38b0866268e2ca162445105" ], "axis": "telemetry_row", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "the OpenTelemetry collector acknowledges after receipt and batches in memory before exporting to ClickHouse", "classification_evidence": "vendor compose otel-collector configuration; collector-kill loss by event id", "tracker_retry": "memory", "tracker_retry_source": "@hyperdx/browser 0.26.0 → @opentelemetry/otlp-exporter-base retrying transport: 5 attempts, in memory", "tracker_retry_measured": false, "configuration_id": "cfg-clickstack-ramp2", "record_file": "2026-09-14-split-clickstack.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-clickstack.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-clickstack-ramp2-r25", "run_id": "run-2026-09-14-clickstack-ramp2", "stage": "ramp:25", "offered_rate": 25.0, "offered_unit": "requests_per_s", "duration_s": 120, "sent": 3001, "acknowledged": 3001, "stored_rows": 3001, "stored_distinct": 3001, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 2.7, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 4.0, "k6_cpu_max_pct": 7.7, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 25.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "25.0 acknowledged requests/s achieved at 25 offered; p95 3 ms. 0 of 3,001 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "6959e90547c5618ed9ff117adc46e028aeecd6ee594616640271a7f810dd9013", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-clickstack.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "ClickStack / HyperDX", "subject_version": "2.35.0", "subject_digests": [ "docker.clickhouse.com/clickhouse/clickstack-otel-collector@sha256:046d88ecfde94076c6bd2835b82deaafc37036c665be5232b6eb88394a8d0a50", "clickhouse/clickhouse-server@sha256:35b419db86eed71ab1c41c03b4fd1f39be26f41eb38b0866268e2ca162445105" ], "axis": "telemetry_row", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "the OpenTelemetry collector acknowledges after receipt and batches in memory before exporting to ClickHouse", "classification_evidence": "vendor compose otel-collector configuration; collector-kill loss by event id", "tracker_retry": "memory", "tracker_retry_source": "@hyperdx/browser 0.26.0 → @opentelemetry/otlp-exporter-base retrying transport: 5 attempts, in memory", "tracker_retry_measured": false, "configuration_id": "cfg-clickstack-ramp2", "record_file": "2026-09-14-split-clickstack.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-clickstack.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-clickstack-ramp2-r50", "run_id": "run-2026-09-14-clickstack-ramp2", "stage": "ramp:50", "offered_rate": 50.0, "offered_unit": "requests_per_s", "duration_s": 120, "sent": 6000, "acknowledged": 6000, "stored_rows": 6000, "stored_distinct": 6000, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 2.7, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 5.0, "k6_cpu_max_pct": 14.5, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 50.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "50.0 acknowledged requests/s achieved at 50 offered; p95 3 ms. 0 of 6,000 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "6959e90547c5618ed9ff117adc46e028aeecd6ee594616640271a7f810dd9013", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-clickstack.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "ClickStack / HyperDX", "subject_version": "2.35.0", "subject_digests": [ "docker.clickhouse.com/clickhouse/clickstack-otel-collector@sha256:046d88ecfde94076c6bd2835b82deaafc37036c665be5232b6eb88394a8d0a50", "clickhouse/clickhouse-server@sha256:35b419db86eed71ab1c41c03b4fd1f39be26f41eb38b0866268e2ca162445105" ], "axis": "telemetry_row", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "the OpenTelemetry collector acknowledges after receipt and batches in memory before exporting to ClickHouse", "classification_evidence": "vendor compose otel-collector configuration; collector-kill loss by event id", "tracker_retry": "memory", "tracker_retry_source": "@hyperdx/browser 0.26.0 → @opentelemetry/otlp-exporter-base retrying transport: 5 attempts, in memory", "tracker_retry_measured": false, "configuration_id": "cfg-clickstack-ramp2", "record_file": "2026-09-14-split-clickstack.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-clickstack.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-clickstack-ramp2-r100", "run_id": "run-2026-09-14-clickstack-ramp2", "stage": "ramp:100", "offered_rate": 100.0, "offered_unit": "requests_per_s", "duration_s": 120, "sent": 12000, "acknowledged": 12000, "stored_rows": 12000, "stored_distinct": 12000, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 3.2, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 7.0, "k6_cpu_max_pct": 27.9, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 100.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "100.0 acknowledged requests/s achieved at 100 offered; p95 3 ms. 0 of 12,000 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "6959e90547c5618ed9ff117adc46e028aeecd6ee594616640271a7f810dd9013", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-clickstack.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "ClickStack / HyperDX", "subject_version": "2.35.0", "subject_digests": [ "docker.clickhouse.com/clickhouse/clickstack-otel-collector@sha256:046d88ecfde94076c6bd2835b82deaafc37036c665be5232b6eb88394a8d0a50", "clickhouse/clickhouse-server@sha256:35b419db86eed71ab1c41c03b4fd1f39be26f41eb38b0866268e2ca162445105" ], "axis": "telemetry_row", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "the OpenTelemetry collector acknowledges after receipt and batches in memory before exporting to ClickHouse", "classification_evidence": "vendor compose otel-collector configuration; collector-kill loss by event id", "tracker_retry": "memory", "tracker_retry_source": "@hyperdx/browser 0.26.0 → @opentelemetry/otlp-exporter-base retrying transport: 5 attempts, in memory", "tracker_retry_measured": false, "configuration_id": "cfg-clickstack-ramp2", "record_file": "2026-09-14-split-clickstack.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-clickstack.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-clickstack-ramp2-r200", "run_id": "run-2026-09-14-clickstack-ramp2", "stage": "ramp:200", "offered_rate": 200.0, "offered_unit": "requests_per_s", "duration_s": 120, "sent": 24000, "acknowledged": 24000, "stored_rows": 24000, "stored_distinct": 24000, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 2.6, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 12.0, "k6_cpu_max_pct": 48.7, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 200.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "200.0 acknowledged requests/s achieved at 200 offered; p95 3 ms. 0 of 24,000 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "6959e90547c5618ed9ff117adc46e028aeecd6ee594616640271a7f810dd9013", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-clickstack.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "ClickStack / HyperDX", "subject_version": "2.35.0", "subject_digests": [ "docker.clickhouse.com/clickhouse/clickstack-otel-collector@sha256:046d88ecfde94076c6bd2835b82deaafc37036c665be5232b6eb88394a8d0a50", "clickhouse/clickhouse-server@sha256:35b419db86eed71ab1c41c03b4fd1f39be26f41eb38b0866268e2ca162445105" ], "axis": "telemetry_row", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "the OpenTelemetry collector acknowledges after receipt and batches in memory before exporting to ClickHouse", "classification_evidence": "vendor compose otel-collector configuration; collector-kill loss by event id", "tracker_retry": "memory", "tracker_retry_source": "@hyperdx/browser 0.26.0 → @opentelemetry/otlp-exporter-base retrying transport: 5 attempts, in memory", "tracker_retry_measured": false, "configuration_id": "cfg-clickstack-ramp2", "record_file": "2026-09-14-split-clickstack.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-clickstack.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-clickstack-ramp2-r400", "run_id": "run-2026-09-14-clickstack-ramp2", "stage": "ramp:400", "offered_rate": 400.0, "offered_unit": "requests_per_s", "duration_s": 120, "sent": 48000, "acknowledged": 48000, "stored_rows": 48000, "stored_distinct": 48000, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 2.4, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 20.0, "k6_cpu_max_pct": 81.8, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 400.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "400.0 acknowledged requests/s achieved at 400 offered; p95 2 ms. 0 of 48,000 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "6959e90547c5618ed9ff117adc46e028aeecd6ee594616640271a7f810dd9013", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-clickstack.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "ClickStack / HyperDX", "subject_version": "2.35.0", "subject_digests": [ "docker.clickhouse.com/clickhouse/clickstack-otel-collector@sha256:046d88ecfde94076c6bd2835b82deaafc37036c665be5232b6eb88394a8d0a50", "clickhouse/clickhouse-server@sha256:35b419db86eed71ab1c41c03b4fd1f39be26f41eb38b0866268e2ca162445105" ], "axis": "telemetry_row", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "the OpenTelemetry collector acknowledges after receipt and batches in memory before exporting to ClickHouse", "classification_evidence": "vendor compose otel-collector configuration; collector-kill loss by event id", "tracker_retry": "memory", "tracker_retry_source": "@hyperdx/browser 0.26.0 → @opentelemetry/otlp-exporter-base retrying transport: 5 attempts, in memory", "tracker_retry_measured": false, "configuration_id": "cfg-clickstack-ramp2", "record_file": "2026-09-14-split-clickstack.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-clickstack.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-clickstack-ramp2-r800", "run_id": "run-2026-09-14-clickstack-ramp2", "stage": "ramp:800", "offered_rate": 800.0, "offered_unit": "requests_per_s", "duration_s": 120, "sent": 96000, "acknowledged": 96000, "stored_rows": 96000, "stored_distinct": 96000, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 2.4, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 31.0, "k6_cpu_max_pct": 123.0, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 800.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "800.0 acknowledged requests/s achieved at 800 offered; p95 2 ms. 0 of 96,000 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "6959e90547c5618ed9ff117adc46e028aeecd6ee594616640271a7f810dd9013", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-clickstack.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "ClickStack / HyperDX", "subject_version": "2.35.0", "subject_digests": [ "docker.clickhouse.com/clickhouse/clickstack-otel-collector@sha256:046d88ecfde94076c6bd2835b82deaafc37036c665be5232b6eb88394a8d0a50", "clickhouse/clickhouse-server@sha256:35b419db86eed71ab1c41c03b4fd1f39be26f41eb38b0866268e2ca162445105" ], "axis": "telemetry_row", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "the OpenTelemetry collector acknowledges after receipt and batches in memory before exporting to ClickHouse", "classification_evidence": "vendor compose otel-collector configuration; collector-kill loss by event id", "tracker_retry": "memory", "tracker_retry_source": "@hyperdx/browser 0.26.0 → @opentelemetry/otlp-exporter-base retrying transport: 5 attempts, in memory", "tracker_retry_measured": false, "configuration_id": "cfg-clickstack-ramp2", "record_file": "2026-09-14-split-clickstack.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-clickstack.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-clickstack-ramp2-r1600", "run_id": "run-2026-09-14-clickstack-ramp2", "stage": "ramp:1600", "offered_rate": 1600.0, "offered_unit": "requests_per_s", "duration_s": 120, "sent": 189579, "acknowledged": 189579, "stored_rows": 189579, "stored_distinct": 189579, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 485.0, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 47.0, "k6_cpu_max_pct": 176.0, "qualifications": [ "single_trial" ], "record_type": "throughput_stage", "achieved_rate": 1579.8, "rate_bound": "exact", "dropped_iterations": 2421, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "1,579.8 acknowledged requests/s achieved at 1,600 offered; p95 485 ms. 0 of 189,579 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "6959e90547c5618ed9ff117adc46e028aeecd6ee594616640271a7f810dd9013", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-clickstack.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "ClickStack / HyperDX", "subject_version": "2.35.0", "subject_digests": [ "docker.clickhouse.com/clickhouse/clickstack-otel-collector@sha256:046d88ecfde94076c6bd2835b82deaafc37036c665be5232b6eb88394a8d0a50", "clickhouse/clickhouse-server@sha256:35b419db86eed71ab1c41c03b4fd1f39be26f41eb38b0866268e2ca162445105" ], "axis": "telemetry_row", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "the OpenTelemetry collector acknowledges after receipt and batches in memory before exporting to ClickHouse", "classification_evidence": "vendor compose otel-collector configuration; collector-kill loss by event id", "tracker_retry": "memory", "tracker_retry_source": "@hyperdx/browser 0.26.0 → @opentelemetry/otlp-exporter-base retrying transport: 5 attempts, in memory", "tracker_retry_measured": false, "configuration_id": "cfg-clickstack-ramp2", "record_file": "2026-09-14-split-clickstack.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-clickstack.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-clickstack-ramp2-r3200", "run_id": "run-2026-09-14-clickstack-ramp2", "stage": "ramp:3200", "offered_rate": 3200.0, "offered_unit": "requests_per_s", "duration_s": 120, "sent": 377076, "acknowledged": 377076, "stored_rows": 377076, "stored_distinct": 377076, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 385.3, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 72.0, "k6_cpu_max_pct": 264.0, "qualifications": [ "single_trial", "generator_near_capacity" ], "record_type": "throughput_stage", "achieved_rate": 3142.3, "rate_bound": "lower", "dropped_iterations": 6925, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": true, "observed_outcome": "at least 3,142.3 acknowledged requests/s achieved at 3,200 offered; p95 385 ms. 0 of 377,076 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id). Selected as the highest observed throughput by the protocol's ceiling rule.", "record_sha256": "6959e90547c5618ed9ff117adc46e028aeecd6ee594616640271a7f810dd9013", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-clickstack.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "ClickStack / HyperDX", "subject_version": "2.35.0", "subject_digests": [ "docker.clickhouse.com/clickhouse/clickstack-otel-collector@sha256:046d88ecfde94076c6bd2835b82deaafc37036c665be5232b6eb88394a8d0a50", "clickhouse/clickhouse-server@sha256:35b419db86eed71ab1c41c03b4fd1f39be26f41eb38b0866268e2ca162445105" ], "axis": "telemetry_row", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "the OpenTelemetry collector acknowledges after receipt and batches in memory before exporting to ClickHouse", "classification_evidence": "vendor compose otel-collector configuration; collector-kill loss by event id", "tracker_retry": "memory", "tracker_retry_source": "@hyperdx/browser 0.26.0 → @opentelemetry/otlp-exporter-base retrying transport: 5 attempts, in memory", "tracker_retry_measured": false, "configuration_id": "cfg-clickstack-ramp2", "record_file": "2026-09-14-split-clickstack.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-clickstack.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-clickstack-ramp2-soak", "run_id": "run-2026-09-14-clickstack-ramp2", "stage": "soak:2513", "offered_rate": 2513.0, "offered_unit": "requests_per_s", "duration_s": 600, "sent": 1477070, "acknowledged": 1477070, "stored_rows": 1477070, "stored_distinct": 1477070, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": 0, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 0, "p95_ms": 225.3, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 65.0, "k6_cpu_max_pct": 228.0, "qualifications": [ "single_trial", "generator_near_capacity" ], "record_type": "throughput_stage", "achieved_rate": 2461.8, "rate_bound": "lower", "dropped_iterations": 30733, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "at least 2,461.8 acknowledged requests/s achieved at 2,513 offered; p95 225 ms. 0 of 1,477,070 acknowledged ids were missing from storage, 0 stored rows were duplicates (matched by event id).", "record_sha256": "6959e90547c5618ed9ff117adc46e028aeecd6ee594616640271a7f810dd9013", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-clickstack.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "ClickStack / HyperDX", "subject_version": "2.35.0", "subject_digests": [ "docker.clickhouse.com/clickhouse/clickstack-otel-collector@sha256:046d88ecfde94076c6bd2835b82deaafc37036c665be5232b6eb88394a8d0a50", "clickhouse/clickhouse-server@sha256:35b419db86eed71ab1c41c03b4fd1f39be26f41eb38b0866268e2ca162445105" ], "axis": "telemetry_row", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "in_memory_batch", "acknowledgement_strategy_note": "the OpenTelemetry collector acknowledges after receipt and batches in memory before exporting to ClickHouse", "classification_evidence": "vendor compose otel-collector configuration; collector-kill loss by event id", "tracker_retry": "memory", "tracker_retry_source": "@hyperdx/browser 0.26.0 → @opentelemetry/otlp-exporter-base retrying transport: 5 attempts, in memory", "tracker_retry_measured": false, "configuration_id": "cfg-clickstack-ramp2", "record_file": "2026-09-14-split-clickstack.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-clickstack.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-clickstack-ramp2-kill", "run_id": "run-2026-09-14-clickstack-ramp2", "stage": "kill:otel-collector", "offered_rate": 2513.0, "offered_unit": "requests_per_s", "duration_s": 90, "sent": 225419, "acknowledged": 177246, "stored_rows": 173092, "stored_distinct": 173092, "reconciliation_method": "event_id_matching", "acknowledged_minus_stored_rows": 4154, "acknowledged_missing_count": 4154, "duplicate_count": 0, "stored_without_ack_count": 0, "hard_failures": 48173, "p95_ms": 72.4, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 63.0, "k6_cpu_max_pct": 227.0, "qualifications": [ "single_trial", "healthy_probe_recovery" ], "record_type": "failure_test", "failure_scope": { "component": "otel-collector (accept path)", "mechanism": "docker kill (API SIGKILL)", "accept_path_up": false }, "kill_at_s": 20, "restart_by_driver_at_s": 36.3, "restart_policy_text": "always", "restart_policy_fired": false, "restart_policy_observation": "api_kill_10s_watch", "recovery": [ { "definition": "healthy_probe", "seconds": 23 } ], "observed_outcome": "4,154 of 177,246 acknowledged requests were shown to be missing after killing otel-collector (accept path). 48,173 requests failed while the component was down.", "record_sha256": "6959e90547c5618ed9ff117adc46e028aeecd6ee594616640271a7f810dd9013", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-clickstack.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Traceway", "subject_version": "ClickHouse mode; no version label, digest is the identity", "subject_digests": [ "ghcr.io/tracewayapp/traceway@sha256:688eeb294a051da84f47e31f3f25769b8f13e85fe0d81f03dc40961b10b1c77c" ], "axis": "error_envelope", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": "acknowledges after a synchronous ClickHouse insert; sheds with HTTP 503 under overload", "classification_evidence": "traceway source: chdb.SendBatch with WithAsync(false); stored == acknowledged at every stage", "tracker_retry": "memory", "tracker_retry_source": "tracewayapp/js-client packages/frontend/src/transport.ts @ ed81989a: unbounded in-memory re-queue", "tracker_retry_measured": false, "configuration_id": "cfg-traceway-ramp2", "record_file": "2026-09-14-split-traceway.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-traceway.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-traceway-ramp2-r10", "run_id": "run-2026-09-14-traceway-ramp2", "stage": "ramp:10", "offered_rate": 10.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 1200, "acknowledged": 1200, "stored_rows": 1200, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 0, "p95_ms": 9.6, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 3.0, "k6_cpu_max_pct": 5.0, "qualifications": [ "single_trial", "count_based_reconciliation" ], "record_type": "throughput_stage", "achieved_rate": 10.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "10.0 acknowledged events/s achieved at 10 offered; p95 10 ms. No count deficit (1,200 acknowledged, 1,200 stored); count-based, so this does not rule out losses offset by duplicates.", "record_sha256": "3a0cf4a63fb4e2a6fca137fec25a16f55ab4a5a9688c2c4248c97abafeff957a", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-traceway.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Traceway", "subject_version": "ClickHouse mode; no version label, digest is the identity", "subject_digests": [ "ghcr.io/tracewayapp/traceway@sha256:688eeb294a051da84f47e31f3f25769b8f13e85fe0d81f03dc40961b10b1c77c" ], "axis": "error_envelope", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": "acknowledges after a synchronous ClickHouse insert; sheds with HTTP 503 under overload", "classification_evidence": "traceway source: chdb.SendBatch with WithAsync(false); stored == acknowledged at every stage", "tracker_retry": "memory", "tracker_retry_source": "tracewayapp/js-client packages/frontend/src/transport.ts @ ed81989a: unbounded in-memory re-queue", "tracker_retry_measured": false, "configuration_id": "cfg-traceway-ramp2", "record_file": "2026-09-14-split-traceway.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-traceway.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-traceway-ramp2-r25", "run_id": "run-2026-09-14-traceway-ramp2", "stage": "ramp:25", "offered_rate": 25.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 3000, "acknowledged": 3000, "stored_rows": 3000, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 0, "p95_ms": 9.6, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 4.0, "k6_cpu_max_pct": 9.2, "qualifications": [ "single_trial", "count_based_reconciliation" ], "record_type": "throughput_stage", "achieved_rate": 25.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "25.0 acknowledged events/s achieved at 25 offered; p95 10 ms. No count deficit (3,000 acknowledged, 3,000 stored); count-based, so this does not rule out losses offset by duplicates.", "record_sha256": "3a0cf4a63fb4e2a6fca137fec25a16f55ab4a5a9688c2c4248c97abafeff957a", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-traceway.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Traceway", "subject_version": "ClickHouse mode; no version label, digest is the identity", "subject_digests": [ "ghcr.io/tracewayapp/traceway@sha256:688eeb294a051da84f47e31f3f25769b8f13e85fe0d81f03dc40961b10b1c77c" ], "axis": "error_envelope", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": "acknowledges after a synchronous ClickHouse insert; sheds with HTTP 503 under overload", "classification_evidence": "traceway source: chdb.SendBatch with WithAsync(false); stored == acknowledged at every stage", "tracker_retry": "memory", "tracker_retry_source": "tracewayapp/js-client packages/frontend/src/transport.ts @ ed81989a: unbounded in-memory re-queue", "tracker_retry_measured": false, "configuration_id": "cfg-traceway-ramp2", "record_file": "2026-09-14-split-traceway.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-traceway.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-traceway-ramp2-r50", "run_id": "run-2026-09-14-traceway-ramp2", "stage": "ramp:50", "offered_rate": 50.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 6001, "acknowledged": 6001, "stored_rows": 6001, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 0, "p95_ms": 13.6, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 5.0, "k6_cpu_max_pct": 15.3, "qualifications": [ "single_trial", "count_based_reconciliation" ], "record_type": "throughput_stage", "achieved_rate": 50.0, "rate_bound": "exact", "dropped_iterations": 0, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "50.0 acknowledged events/s achieved at 50 offered; p95 14 ms. No count deficit (6,001 acknowledged, 6,001 stored); count-based, so this does not rule out losses offset by duplicates.", "record_sha256": "3a0cf4a63fb4e2a6fca137fec25a16f55ab4a5a9688c2c4248c97abafeff957a", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-traceway.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Traceway", "subject_version": "ClickHouse mode; no version label, digest is the identity", "subject_digests": [ "ghcr.io/tracewayapp/traceway@sha256:688eeb294a051da84f47e31f3f25769b8f13e85fe0d81f03dc40961b10b1c77c" ], "axis": "error_envelope", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": "acknowledges after a synchronous ClickHouse insert; sheds with HTTP 503 under overload", "classification_evidence": "traceway source: chdb.SendBatch with WithAsync(false); stored == acknowledged at every stage", "tracker_retry": "memory", "tracker_retry_source": "tracewayapp/js-client packages/frontend/src/transport.ts @ ed81989a: unbounded in-memory re-queue", "tracker_retry_measured": false, "configuration_id": "cfg-traceway-ramp2", "record_file": "2026-09-14-split-traceway.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-traceway.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-traceway-ramp2-r100", "run_id": "run-2026-09-14-traceway-ramp2", "stage": "ramp:100", "offered_rate": 100.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 11788, "acknowledged": 11742, "stored_rows": 11742, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 46, "p95_ms": 4177.8, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 6.0, "k6_cpu_max_pct": 28.7, "qualifications": [ "single_trial", "count_based_reconciliation" ], "record_type": "throughput_stage", "achieved_rate": 97.8, "rate_bound": "exact", "dropped_iterations": 213, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "97.8 acknowledged events/s achieved at 100 offered; p95 4.18 s. No count deficit (11,742 acknowledged, 11,742 stored); count-based, so this does not rule out losses offset by duplicates.", "record_sha256": "3a0cf4a63fb4e2a6fca137fec25a16f55ab4a5a9688c2c4248c97abafeff957a", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-traceway.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Traceway", "subject_version": "ClickHouse mode; no version label, digest is the identity", "subject_digests": [ "ghcr.io/tracewayapp/traceway@sha256:688eeb294a051da84f47e31f3f25769b8f13e85fe0d81f03dc40961b10b1c77c" ], "axis": "error_envelope", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": "acknowledges after a synchronous ClickHouse insert; sheds with HTTP 503 under overload", "classification_evidence": "traceway source: chdb.SendBatch with WithAsync(false); stored == acknowledged at every stage", "tracker_retry": "memory", "tracker_retry_source": "tracewayapp/js-client packages/frontend/src/transport.ts @ ed81989a: unbounded in-memory re-queue", "tracker_retry_measured": false, "configuration_id": "cfg-traceway-ramp2", "record_file": "2026-09-14-split-traceway.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-traceway.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-traceway-ramp2-r200", "run_id": "run-2026-09-14-traceway-ramp2", "stage": "ramp:200", "offered_rate": 200.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 23389, "acknowledged": 14311, "stored_rows": 14311, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 9078, "p95_ms": 5052.3, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 10.0, "k6_cpu_max_pct": 44.7, "qualifications": [ "single_trial", "count_based_reconciliation", "application_bound" ], "record_type": "throughput_stage", "achieved_rate": 119.3, "rate_bound": "exact", "dropped_iterations": 611, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": true, "observed_outcome": "119.3 acknowledged events/s achieved at 200 offered; p95 5.05 s. No count deficit (14,311 acknowledged, 14,311 stored); count-based, so this does not rule out losses offset by duplicates. Selected as the highest observed throughput by the protocol's ceiling rule.", "record_sha256": "3a0cf4a63fb4e2a6fca137fec25a16f55ab4a5a9688c2c4248c97abafeff957a", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-traceway.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Traceway", "subject_version": "ClickHouse mode; no version label, digest is the identity", "subject_digests": [ "ghcr.io/tracewayapp/traceway@sha256:688eeb294a051da84f47e31f3f25769b8f13e85fe0d81f03dc40961b10b1c77c" ], "axis": "error_envelope", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": "acknowledges after a synchronous ClickHouse insert; sheds with HTTP 503 under overload", "classification_evidence": "traceway source: chdb.SendBatch with WithAsync(false); stored == acknowledged at every stage", "tracker_retry": "memory", "tracker_retry_source": "tracewayapp/js-client packages/frontend/src/transport.ts @ ed81989a: unbounded in-memory re-queue", "tracker_retry_measured": false, "configuration_id": "cfg-traceway-ramp2", "record_file": "2026-09-14-split-traceway.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-traceway.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-traceway-ramp2-r400", "run_id": "run-2026-09-14-traceway-ramp2", "stage": "ramp:400", "offered_rate": 400.0, "offered_unit": "events_per_s", "duration_s": 120, "sent": 35931, "acknowledged": 13156, "stored_rows": 13156, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 22775, "p95_ms": 5051.3, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 19.0, "k6_cpu_max_pct": 77.4, "qualifications": [ "single_trial", "count_based_reconciliation" ], "record_type": "throughput_stage", "achieved_rate": 109.6, "rate_bound": "exact", "dropped_iterations": 12069, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "109.6 acknowledged events/s achieved at 400 offered; p95 5.05 s. No count deficit (13,156 acknowledged, 13,156 stored); count-based, so this does not rule out losses offset by duplicates.", "record_sha256": "3a0cf4a63fb4e2a6fca137fec25a16f55ab4a5a9688c2c4248c97abafeff957a", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-traceway.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Traceway", "subject_version": "ClickHouse mode; no version label, digest is the identity", "subject_digests": [ "ghcr.io/tracewayapp/traceway@sha256:688eeb294a051da84f47e31f3f25769b8f13e85fe0d81f03dc40961b10b1c77c" ], "axis": "error_envelope", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": "acknowledges after a synchronous ClickHouse insert; sheds with HTTP 503 under overload", "classification_evidence": "traceway source: chdb.SendBatch with WithAsync(false); stored == acknowledged at every stage", "tracker_retry": "memory", "tracker_retry_source": "tracewayapp/js-client packages/frontend/src/transport.ts @ ed81989a: unbounded in-memory re-queue", "tracker_retry_measured": false, "configuration_id": "cfg-traceway-ramp2", "record_file": "2026-09-14-split-traceway.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-traceway.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-traceway-ramp2-soak", "run_id": "run-2026-09-14-traceway-ramp2", "stage": "soak:95", "offered_rate": 95.0, "offered_unit": "events_per_s", "duration_s": 600, "sent": 56710, "acknowledged": 56005, "stored_rows": 56005, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 705, "p95_ms": 4943.4, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 9.0, "k6_cpu_max_pct": 25.5, "qualifications": [ "single_trial", "count_based_reconciliation" ], "record_type": "throughput_stage", "achieved_rate": 93.3, "rate_bound": "exact", "dropped_iterations": 291, "stored_settle_s": 15.0, "settle_rule": "row count for the stage's run id unchanged across two consecutive 5 s readings after an initial 10 s; cap 180 s", "is_ceiling_stage": false, "observed_outcome": "93.3 acknowledged events/s achieved at 95 offered; p95 4.94 s. No count deficit (56,005 acknowledged, 56,005 stored); count-based, so this does not rule out losses offset by duplicates.", "record_sha256": "3a0cf4a63fb4e2a6fca137fec25a16f55ab4a5a9688c2c4248c97abafeff957a", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-traceway.md" }, { "posts": [ "five-open-source-analytics-tools-load-tested" ], "subject": "Traceway", "subject_version": "ClickHouse mode; no version label, digest is the identity", "subject_digests": [ "ghcr.io/tracewayapp/traceway@sha256:688eeb294a051da84f47e31f3f25769b8f13e85fe0d81f03dc40961b10b1c77c" ], "axis": "error_envelope", "box": "e2-standard-2", "box_note": null, "generator_colocated": false, "protocol": "ramp", "protocol_version": "2", "run_date": "2026-09-14", "deviations": [], "acknowledgement_strategy": "synchronous_write", "acknowledgement_strategy_note": "acknowledges after a synchronous ClickHouse insert; sheds with HTTP 503 under overload", "classification_evidence": "traceway source: chdb.SendBatch with WithAsync(false); stored == acknowledged at every stage", "tracker_retry": "memory", "tracker_retry_source": "tracewayapp/js-client packages/frontend/src/transport.ts @ ed81989a: unbounded in-memory re-queue", "tracker_retry_measured": false, "configuration_id": "cfg-traceway-ramp2", "record_file": "2026-09-14-split-traceway.md", "record_path": "load-test-2026-09-14/results/2026-09-14-split-traceway.md", "trials": 1, "note_ids": [], "overload_behaviour": null, "supersedes": null, "superseded_by": null, "observation_id": "obs-2026-09-14-traceway-ramp2-kill", "run_id": "run-2026-09-14-traceway-ramp2", "stage": "kill:all-in-one container", "offered_rate": 95.0, "offered_unit": "events_per_s", "duration_s": 90, "sent": 8258, "acknowledged": 3424, "stored_rows": 3424, "stored_distinct": { "null": "count_based" }, "reconciliation_method": "count_comparison", "acknowledged_minus_stored_rows": 0, "acknowledged_missing_count": { "null": "count_based" }, "duplicate_count": { "null": "count_based" }, "stored_without_ack_count": { "null": "count_based" }, "hard_failures": 4834, "p95_ms": 5140.5, "p95_scope": "all requests in the stage window", "generator_box": "e2-standard-4", "generator_busy_max_pct": 7.0, "k6_cpu_max_pct": 24.3, "qualifications": [ "single_trial", "count_based_reconciliation", "healthy_probe_recovery" ], "record_type": "failure_test", "failure_scope": { "component": "all-in-one container (backend + ClickHouse + Postgres)", "mechanism": "docker kill (API SIGKILL)", "accept_path_up": false }, "kill_at_s": 20, "restart_by_driver_at_s": 36.4, "restart_policy_text": "no", "restart_policy_fired": false, "restart_policy_observation": "api_kill_10s_watch", "recovery": [ { "definition": "healthy_probe", "seconds": 26 } ], "observed_outcome": "No count deficit was observed after killing all-in-one container (backend + ClickHouse + Postgres) (3,424 acknowledged, 3,424 stored). Because reconciliation compared totals, this does not rule out losses offset by duplicates. 4,834 requests failed while the component was down.", "record_sha256": "3a0cf4a63fb4e2a6fca137fec25a16f55ab4a5a9688c2c4248c97abafeff957a", "record_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/records/2026-09-14-split-traceway.md" } ], "license": "https://creativecommons.org/licenses/by/4.0/", "license_name": "CC BY 4.0", "attribution": "Jetboat Analytics, \"We load-tested five open-source tools. Which accepted events survived a process kill?\", https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/, CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/). Attribution must include this link.", "source_url": "https://jetboatanalytics.com/blog/five-open-source-analytics-tools-load-tested/" }