{
 "schema_version": "0.2.0",
 "dataset_version": "1.1.0",
 "post": "self-hosted-analytics-kill-test",
 "generated_from": "values copied by hand from the run records named in record_path/record_file (kill-test/2 rows) and imported from the load-test post dataset (ramp/2 rows); observed_outcome generated by scripts/build-post.py",
 "protocols": {
  "kill-test/2": {
   "version": "2",
   "summary": "Fixed offered rate (50 requests/s pageview, 10/s error envelopes, 20/s for the Matomo variant) for 90 s, one event per request in the vendor tracker's shape, 500 distinct visitors; a 60 s pre-check at the same rate must hold p95 < 1 s with no errors; docker kill of the accept process at t=20 s; 10 s watch for the vendor's restart policy; docker start at t≈31 s if it did not fire; recovery timed to the first successful request; a second 90 s run kills the consumer or worker where a queue sits between acknowledgement and storage; reconcile once the row count has been stable for 30 s. Run 2026-09-14."
  },
  "ramp/2": {
   "version": "2",
   "summary": "120 s constant-arrival stages at offered rates 25→1,600 requests/s (ClickStack →3,200; Traceway 10→400), one event per request, 500 distinct visitors, k6 v2.2.0 on a separate e2-standard-4 with generator CPU, achieved rate and dropped iterations recorded; reconcile after every stage once the stage's row count is unchanged across two consecutive 5 s readings (cap 180 s); a 600 s soak at 80 % of the ceiling; one docker kill (SIGKILL) of the accept process at t=20 s of a 90 s run at the soak rate, 10 s restart-policy watch, docker start by the driver if the policy did not fire, recovery = first healthy probe over ssh. Per-event id reconciliation where the tool stores a client-supplied id (Umami url_query, Rybbit querystring, ClickStack log attribute, ours event_id); count comparison for Matomo (a unique URL per event would change its workload) and Traceway (attributes not queryable per key). Run 2026-09-14.",
   "ceiling_rule": "highest observed throughput = the highest achieved acknowledged rate across the ramp stages where acknowledged ids missing from storage are at most 2 % of acknowledged (id matching) or stored rows are at least 98 % of 2xx (count comparison); the stage that produced it is the ceiling stage; no latency or error threshold enters the rule",
   "reconciliation": "event_id_matching for Umami, Rybbit, ClickStack, ours; count_comparison for Matomo, Traceway"
  }
 },
 "definitions": {
  "acknowledged": "requests that received the tool's success status (2xx; Plausible 202; Matomo 204; GoatCounter 200) inside the window",
  "stored_rows": "rows in the tool's own database attributable to the window, counted after the settle rule",
  "stored_distinct": "stored rows after removing duplicates, when duplicates were checked; otherwise a typed null",
  "acknowledged_minus_stored_rows": "signed difference of the two measured totals; positive = fewer rows than acknowledgements. A count deficit, not an identified set of events",
  "acknowledged_missing_count": "acknowledged events shown to be absent by event-id matching or a documented correction; typed null 'count_based' when only totals were compared",
  "duplicate_count": "stored rows that duplicate an already-stored event, when checked",
  "stored_without_ack_count": "stored events whose success response never reached the client, when established",
  "hard_failures": "requests that got an error status or no response; lost only if the client does not resend them (see tracker_retry)",
  "recovery": "seconds from the kill to the named event; each entry carries its definition",
  "restart_policy_fired": "whether the container came back on its own within the 10 s watch after an API-initiated docker kill; this does not test a real crash, which Docker's policies do restart",
  "acknowledgement_strategy": "architecture, from source reading: what the tool does before it answers success",
  "tracker_retry": "what the vendor's browser tracker does with a failed send, from reading its code; not measured in any row",
  "axis": "pageview = one page view per request; error_envelope = one Sentry-style envelope per request; telemetry_row = one OpenTelemetry log row per request",
  "class_display": "S = synchronous_write, M = in_memory_batch, D = durable_queue or durable_log; a display label for the acknowledgement strategy, not an outcome"
 },
 "qualification_markers": {
  "count_based_reconciliation": "stored rows were counted, not matched by event id; a net deficit cannot separate missing, duplicate and stored-without-ack events",
  "single_trial": "one run, one kill; a memory-buffer loss depends on where in the flush cycle the kill lands",
  "generator_colocated": "k6 ran on the same box as the tool",
  "generator_saturated": "k6 dropped iterations; the achieved rate is a lower bound on the tool",
  "nonstandard_box": "not the e2-standard-2 floor box",
  "protocol_variant": "not the fixed-rate kill-test/2 window: a ramp/2 kill at the tool's soak rate with the generator on a separate machine, or the 20/s Matomo variant rate",
  "store_stopped_before_kill": "the store was stopped before the process was killed, so every acknowledged event was provably only in the buffer",
  "retry_inferred_from_code": "client retry behaviour read from the tracker's source, not measured",
  "acknowledged_while_dependency_down": "the server returned its success status while a dependency it needed was unavailable",
  "healthy_probe_recovery": "recovery is timed to the health endpoint answering, not to a successfully ingested event",
  "overload_discard": "this stack deliberately discards low-value events after acknowledging them under sustained overload (overload_behaviour); not exercised here, recorded so a zero-loss kill is not read as a general guarantee"
 },
 "notes": {
  "n-matomo-dup": "Redis was SIGKILLed with no shutdown save; the last RDB snapshot was 112–123 s old. On restart Redis replayed 24 request sets from the earlier app-kill window that had already been processed, and Matomo inserted them a second time with their original server_time (verified in MariaDB by idvisitor + server_time + idaction_url grouping).",
  "n-matomo-204": "220 of the 1,800 acknowledged requests were answered 204 by Matomo core (Tracker/Response.php) after logging a Redis connection error; 33 more were the queue contents at the kill instant. 220 + 33 = 253 missing; 1,571 stored rows minus 24 duplicates = 1,547 of this run's hits.",
  "n-openpanel-batch": "The worker accumulates up to 100 events or 10 ms in process memory before pushing to Redis (event-buffer.ts, confirmed in the shipped image); the one-event deficit is consistent with that window but was not matched by id.",
  "n-bugsink-handoff": "The 200 is returned after the envelope is written to an on-disk SQLite task queue inside the container; the queue hands a task to its worker with delete-before-run semantics, so a kill mid-task can lose the task in flight. A supplementary probe on a separate project (50 acknowledged, killed, restarted) stored all 50.",
  "n-glitchtip-batch": "GlitchTip 6 runs its task worker in the web process by default; the ingest queue is a batch queue (count 25 / timeout 2 s) whose fetch has no processing list, so the batch being accumulated dies with the process. Everything still on the Valkey list survived and drained after restart.",
  "n-posthog-box": "PostHog's hobby deploy ran on e2-standard-4 (4 vCPU / 16 GB), the vendor's stated minimum, twice the standard box. The stack had been materialised on 2026-09-07 and was revived from the same containers on 2026-09-14 with no re-pull.",
  "n-posthog-mode": "k6 sent the captured browser request with its own gzip (adds a Content-Encoding: gzip header the setup smoke proved capture accepts and stores) because gzip in k6's JavaScript starved the generator on this box.",
  "n-ours-strict": "Strict shape: ClickHouse stopped at t=10 s, app SIGKILLed at t=20 s, docker start at t=31 s, ClickHouse started at t=45 s. Recovery is the first 202 for a real event while ClickHouse was still stopped.",
  "n-goatcounter": "Hits are held in memory and persisted every 10 s (-store-every); the kill landed about 7.5 s after the last persist. Session ids live in the same memory, so all 500 visitors reappeared as new uniques after the restart.",
  "n-plausible": "Events are held in an in-process write buffer flushed every 5 s (CLICKHOUSE_FLUSH_INTERVAL_MS, default 5000); the last flush was 3.3 s before the SIGKILL. A graceful stop flushes the buffer; the loss class is crashes and kills."
 },
 "observations": [
  {
   "observation_id": "obs-2026-09-14-posthog-kill-capture",
   "run_id": "run-2026-09-14-posthog-kill-native",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-posthog-durability.md",
   "record_sha256": "b57cd5a28a69e5b6dddf26ebc784708ba09d67e87834ddb20fd1a3185cd00f07",
   "record_file": "2026-09-14-posthog-durability.md",
   "configuration_id": "cfg-posthog-hobby-master-2026-09-06",
   "subject": "PostHog hobby",
   "subject_version": "master@09c5fbd3 (2026-09-06), images pulled 2026-09-07",
   "subject_digests": [
    "ghcr.io/posthog/posthog/capture@sha256:7869522e4b65cf618a85af3d1dd04854c3b31dd9304af5ce8716020223ed30da",
    "posthog/posthog@sha256:9eface8e64d70695fa7c25fe84de12ad3b7331537250631ec2b4d3fd8fd23d70",
    "redpandadata/redpanda@sha256:8f7e9e4c1422baaa1a5e2a6c6c668cfe05442cb3cb476542c7dff61725e6fe31"
   ],
   "axis": "pageview",
   "box": "e2-standard-4",
   "box_note": "vendor's stated minimum; twice the standard box",
   "deviations": [
    "k6 native gzip (adds Content-Encoding: gzip) instead of the captured JS gzip"
   ],
   "acknowledgement_strategy": "durable_queue",
   "acknowledgement_strategy_note": "capture answers 200 after producing to Kafka (Redpanda on disk); plugin-server consumes and lands rows in ClickHouse",
   "classification_evidence": "capture container log ('v1 kafka producer connected'); PostHog hobby compose topology",
   "tracker_retry": "memory",
   "tracker_retry_source": "posthog-js 1.431.4 retry-queue.ts @ f93160db: up to 10 retries, 3000 ms × 2^n, pagehide flush via sendBeacon",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-posthog-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:capture",
   "failure_scope": {
    "component": "capture",
    "mechanism": "docker kill (API SIGKILL)",
    "accept_path_up": false
   },
   "offered_rate": 50,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 4501,
   "acknowledged": 3921,
   "stored_rows": 3923,
   "stored_distinct": {
    "null": "not_measured"
   },
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": -2,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": {
    "null": "not_measured"
   },
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 580,
   "p95_ms": 28.2,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "first_success",
     "seconds": 12,
     "note": "first 2xx for a real event via Caddy"
    }
   ],
   "restart_policy_text": "on-failure",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "observed_outcome": "No count deficit was observed after killing capture: 2 more rows were stored than requests acknowledged (3,921 acknowledged, 3,923 stored), consistent with responses the client never received. Because reconciliation compared totals, this does not rule out losses offset by duplicates. 580 requests failed while the component was down.",
   "observed_outcome_override": null,
   "qualifications": [
    "count_based_reconciliation",
    "single_trial",
    "generator_colocated",
    "retry_inferred_from_code",
    "nonstandard_box"
   ],
   "note_ids": [
    "n-posthog-box",
    "n-posthog-mode"
   ],
   "run_date": "2026-09-14"
  },
  {
   "observation_id": "obs-2026-09-14-posthog-kill-consumer",
   "run_id": "run-2026-09-14-posthog-kill-native",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-posthog-durability.md",
   "record_sha256": "b57cd5a28a69e5b6dddf26ebc784708ba09d67e87834ddb20fd1a3185cd00f07",
   "record_file": "2026-09-14-posthog-durability.md",
   "configuration_id": "cfg-posthog-hobby-master-2026-09-06",
   "subject": "PostHog hobby",
   "subject_version": "master@09c5fbd3 (2026-09-06), images pulled 2026-09-07",
   "subject_digests": [
    "ghcr.io/posthog/posthog/capture@sha256:7869522e4b65cf618a85af3d1dd04854c3b31dd9304af5ce8716020223ed30da",
    "posthog/posthog@sha256:9eface8e64d70695fa7c25fe84de12ad3b7331537250631ec2b4d3fd8fd23d70",
    "redpandadata/redpanda@sha256:8f7e9e4c1422baaa1a5e2a6c6c668cfe05442cb3cb476542c7dff61725e6fe31"
   ],
   "axis": "pageview",
   "box": "e2-standard-4",
   "box_note": "vendor's stated minimum; twice the standard box",
   "deviations": [
    "k6 native gzip (adds Content-Encoding: gzip) instead of the captured JS gzip"
   ],
   "acknowledgement_strategy": "durable_queue",
   "acknowledgement_strategy_note": "capture answers 200 after producing to Kafka (Redpanda on disk); plugin-server consumes and lands rows in ClickHouse",
   "classification_evidence": "capture container log ('v1 kafka producer connected'); PostHog hobby compose topology",
   "tracker_retry": "memory",
   "tracker_retry_source": "posthog-js 1.431.4 retry-queue.ts @ f93160db: up to 10 retries, 3000 ms × 2^n, pagehide flush via sendBeacon",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-posthog-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:consumer",
   "failure_scope": {
    "component": "ingestion-general (Kafka consumer)",
    "mechanism": "docker kill (API SIGKILL)",
    "accept_path_up": true
   },
   "offered_rate": 50,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 4501,
   "acknowledged": 4501,
   "stored_rows": 4501,
   "stored_distinct": {
    "null": "not_measured"
   },
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": 0,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": {
    "null": "not_measured"
   },
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 0,
   "p95_ms": 29.9,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "worker_consuming",
     "seconds": 30,
     "note": "plugin-server /_health OK; rows landed within the 40 s settle"
    }
   ],
   "restart_policy_text": "on-failure",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "observed_outcome": "No count deficit was observed after killing ingestion-general (Kafka consumer) (4,501 acknowledged, 4,501 stored). Because reconciliation compared totals, this does not rule out losses offset by duplicates. The accept path kept answering throughout.",
   "observed_outcome_override": null,
   "qualifications": [
    "count_based_reconciliation",
    "single_trial",
    "generator_colocated",
    "retry_inferred_from_code",
    "nonstandard_box"
   ],
   "note_ids": [
    "n-posthog-box"
   ],
   "run_date": "2026-09-14"
  },
  {
   "observation_id": "obs-2026-09-14-openpanel-kill-api",
   "run_id": "run-2026-09-14-openpanel",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-openpanel-durability.md",
   "record_sha256": "72b34772cb4db0bcfb6350a549574b00a576fde52fa050a11a0587d162296f16",
   "record_file": "2026-09-14-openpanel-durability.md",
   "configuration_id": "cfg-openpanel-images-2026-08-18",
   "subject": "Openpanel",
   "subject_version": "images built 2026-08-18 (no tagged releases)",
   "subject_digests": [
    "lindesvard/openpanel-api@sha256:cee4855cd715a7248c5c49090fe93a28ffc505541de20cb238d8884a6bcb9223",
    "lindesvard/openpanel-worker@sha256:b22e247fb6980b820c1a954d0e3d2fe88b3edf40a58aa53329467af239538e80"
   ],
   "axis": "pageview",
   "box": "e2-standard-2",
   "box_note": null,
   "deviations": [
    "org/project/client bootstrapped via psql (no signup UI)",
    "visitor IP sent in x-client-ip because the vendor's Caddy overwrites X-Forwarded-For",
    "TLS verification off against Caddy's internal localhost certificate",
    "marker properties bench_run/bench_n on each event"
   ],
   "acknowledgement_strategy": "durable_queue",
   "acknowledgement_strategy_note": "API answers after enqueueing to Redis (groupmq job + event_buffer list); the worker holds a ≤100-event / ≤10 ms micro-batch in memory before RPUSH",
   "classification_evidence": "apps/worker event-buffer.ts in the shipped image; Redis keys inspected on the box",
   "tracker_retry": "memory",
   "tracker_retry_source": "@openpanel/sdk 1.3.1 api.ts @ 32415d31: 3 retries, 500 ms × 2^n; 401 not retried",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-openpanel-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:api",
   "failure_scope": {
    "component": "op-api",
    "mechanism": "docker kill (API SIGKILL)",
    "accept_path_up": false
   },
   "offered_rate": 50,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 4501,
   "acknowledged": 2781,
   "stored_rows": 2781,
   "stored_distinct": {
    "null": "not_measured"
   },
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": 0,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": {
    "null": "not_measured"
   },
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 1720,
   "p95_ms": 21.3,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "first_success",
     "seconds": 35,
     "note": "includes ~15 s of migrate-on-start in the vendor's API start command"
    }
   ],
   "restart_policy_text": "always",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "observed_outcome": "No count deficit was observed after killing op-api (2,781 acknowledged, 2,781 stored). Because reconciliation compared totals, this does not rule out losses offset by duplicates. 1,720 requests failed while the component was down.",
   "observed_outcome_override": null,
   "qualifications": [
    "count_based_reconciliation",
    "single_trial",
    "generator_colocated",
    "retry_inferred_from_code"
   ],
   "note_ids": [],
   "run_date": "2026-09-14"
  },
  {
   "observation_id": "obs-2026-09-14-openpanel-kill-worker",
   "run_id": "run-2026-09-14-openpanel",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-openpanel-durability.md",
   "record_sha256": "72b34772cb4db0bcfb6350a549574b00a576fde52fa050a11a0587d162296f16",
   "record_file": "2026-09-14-openpanel-durability.md",
   "configuration_id": "cfg-openpanel-images-2026-08-18",
   "subject": "Openpanel",
   "subject_version": "images built 2026-08-18 (no tagged releases)",
   "subject_digests": [
    "lindesvard/openpanel-api@sha256:cee4855cd715a7248c5c49090fe93a28ffc505541de20cb238d8884a6bcb9223",
    "lindesvard/openpanel-worker@sha256:b22e247fb6980b820c1a954d0e3d2fe88b3edf40a58aa53329467af239538e80"
   ],
   "axis": "pageview",
   "box": "e2-standard-2",
   "box_note": null,
   "deviations": [
    "org/project/client bootstrapped via psql (no signup UI)",
    "visitor IP sent in x-client-ip because the vendor's Caddy overwrites X-Forwarded-For",
    "TLS verification off against Caddy's internal localhost certificate",
    "marker properties bench_run/bench_n on each event"
   ],
   "acknowledgement_strategy": "durable_queue",
   "acknowledgement_strategy_note": "API answers after enqueueing to Redis (groupmq job + event_buffer list); the worker holds a ≤100-event / ≤10 ms micro-batch in memory before RPUSH",
   "classification_evidence": "apps/worker event-buffer.ts in the shipped image; Redis keys inspected on the box",
   "tracker_retry": "memory",
   "tracker_retry_source": "@openpanel/sdk 1.3.1 api.ts @ 32415d31: 3 retries, 500 ms × 2^n; 401 not retried",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-openpanel-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:worker",
   "failure_scope": {
    "component": "op-worker",
    "mechanism": "docker kill (API SIGKILL)",
    "accept_path_up": true
   },
   "offered_rate": 50,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 4501,
   "acknowledged": 4501,
   "stored_rows": 4500,
   "stored_distinct": 4500,
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": 1,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": 0,
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 0,
   "p95_ms": 148.8,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "worker_consuming",
     "seconds": 20
    }
   ],
   "restart_policy_text": "always",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "observed_outcome": "1 fewer row was stored than requests acknowledged (4,501 acknowledged, 4,500 stored) after killing op-worker. Because reconciliation compared totals, this is a count deficit, not an identified set of lost events. The accept path kept answering throughout.",
   "observed_outcome_override": null,
   "qualifications": [
    "count_based_reconciliation",
    "single_trial",
    "generator_colocated",
    "retry_inferred_from_code"
   ],
   "note_ids": [
    "n-openpanel-batch"
   ],
   "run_date": "2026-09-14"
  },
  {
   "observation_id": "obs-2026-09-14-plausible-kill-app",
   "run_id": "run-2026-09-14-plausible",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-plausible-durability.md",
   "record_sha256": "d38fca09cd138c216bbeda6215be87eeebcc60430b5cd1e49d14538c3c1249b6",
   "record_file": "2026-09-14-plausible-durability.md",
   "configuration_id": "cfg-plausible-ce-3.2.1",
   "subject": "Plausible CE",
   "subject_version": "3.2.1",
   "subject_digests": [
    "ghcr.io/plausible/community-edition@sha256:33e60bfb40f2df5da00f8753b76fad04f67dba3abe6d73eb516e440e3fb62985",
    "clickhouse/clickhouse-server@sha256:cd450891db46cc6ffe313ca2b0fb7dbfb897a6873ca74a724cbe050a2cf62621"
   ],
   "axis": "pageview",
   "box": "e2-standard-2",
   "box_note": null,
   "deviations": [
    "site registered via bin/plausible rpc (CE registration is a LiveView form)"
   ],
   "acknowledgement_strategy": "in_memory_batch",
   "acknowledgement_strategy_note": "202 after validation; events sit in an in-process write buffer (flush_interval_ms 5000, max_buffer_size 100000) until inserted into ClickHouse",
   "classification_evidence": "Plausible.Ingestion.WriteBuffer config read from the running node via rpc",
   "tracker_retry": "none",
   "tracker_retry_source": "plausible/analytics tracker v2 networking.js @ cc5ca4b7: one fetch with keepalive, no retry",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-plausible-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:app",
   "failure_scope": {
    "component": "plausible (Elixir app)",
    "mechanism": "docker kill (API SIGKILL)",
    "accept_path_up": false
   },
   "offered_rate": 50,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 4501,
   "acknowledged": 3320,
   "stored_rows": 3157,
   "stored_distinct": {
    "null": "not_measured"
   },
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": 163,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": {
    "null": "not_measured"
   },
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 1181,
   "p95_ms": 307.7,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "first_success",
     "seconds": 25
    }
   ],
   "restart_policy_text": "always",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "observed_outcome": "163 fewer rows were stored than requests acknowledged (3,320 acknowledged, 3,157 stored) after killing plausible (Elixir app). Because reconciliation compared totals, this is a count deficit, not an identified set of lost events. 1,181 requests failed while the component was down.",
   "observed_outcome_override": null,
   "qualifications": [
    "count_based_reconciliation",
    "single_trial",
    "generator_colocated",
    "retry_inferred_from_code"
   ],
   "note_ids": [
    "n-plausible"
   ],
   "run_date": "2026-09-14"
  },
  {
   "observation_id": "obs-2026-09-14-goatcounter-kill-app",
   "run_id": "run-2026-09-14-goatcounter",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-goatcounter-durability.md",
   "record_sha256": "b1dae9ece79404ed815cb8ee652786663912c011904eea3bd4cfcfdec3d01be6",
   "record_file": "2026-09-14-goatcounter-durability.md",
   "configuration_id": "cfg-goatcounter-2.7.0",
   "subject": "GoatCounter",
   "subject_version": "2.7.0",
   "subject_digests": [
    "arp242/goatcounter@sha256:791df3317bf5ce379511181a4802aaf29127a318b372d4fe39ec59e1c5af44be"
   ],
   "axis": "pageview",
   "box": "e2-standard-2",
   "box_note": null,
   "deviations": [
    "individual pageview collection enabled for the bench site (default aggregates count visits, which cannot be reconciled against requests)",
    "visitor IPs from a public range (GoatCounter discards private X-Forwarded-For values)"
   ],
   "acknowledgement_strategy": "in_memory_batch",
   "acknowledgement_strategy_note": "200 + GIF immediately; hits held in memory and persisted to SQLite every 10 s (-store-every)",
   "classification_evidence": "goatcounter memstore/cron source; -store-every documented flag",
   "tracker_retry": "none",
   "tracker_retry_source": "arp242/goatcounter public/count.js @ 478f026e: sendBeacon, one <img> fallback, no retry",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-goatcounter-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:app",
   "failure_scope": {
    "component": "goatcounter (single binary)",
    "mechanism": "docker kill (API SIGKILL)",
    "accept_path_up": false
   },
   "offered_rate": 50,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 4500,
   "acknowledged": 3920,
   "stored_rows": 3547,
   "stored_distinct": {
    "null": "not_measured"
   },
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": 373,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": {
    "null": "not_measured"
   },
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 580,
   "p95_ms": 1.9,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "first_success",
     "seconds": 12
    }
   ],
   "restart_policy_text": "no (Docker default; none in the bench kit)",
   "restart_policy_fired": {
    "null": "not_applicable"
   },
   "restart_policy_observation": null,
   "observed_outcome": "373 fewer rows were stored than requests acknowledged (3,920 acknowledged, 3,547 stored) after killing goatcounter (single binary). Because reconciliation compared totals, this is a count deficit, not an identified set of lost events. 580 requests failed while the component was down.",
   "observed_outcome_override": null,
   "qualifications": [
    "count_based_reconciliation",
    "single_trial",
    "generator_colocated",
    "retry_inferred_from_code"
   ],
   "note_ids": [
    "n-goatcounter"
   ],
   "run_date": "2026-09-14"
  },
  {
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "subject": "Rybbit",
   "subject_version": "2.8.0 (source rev 4a01c7a)",
   "subject_digests": [
    "ghcr.io/rybbit-io/rybbit-backend@sha256:f729f425…6ac7 (full digest in the kit compose)",
    "ghcr.io/rybbit-io/rybbit-client@sha256:f384c1f7…103a (full digest in the kit compose)"
   ],
   "axis": "pageview",
   "box": "e2-standard-2",
   "box_note": null,
   "generator_colocated": false,
   "protocol": "ramp",
   "protocol_version": "2",
   "run_date": "2026-09-14",
   "deviations": [
    "blockBots=false for the bench site",
    "HOST_BACKEND_PORT=0.0.0.0:3001:3001 (vendor default binds the backend to loopback; the generator is a separate machine)"
   ],
   "acknowledgement_strategy": "in_memory_batch",
   "acknowledgement_strategy_note": "200 before insert; up to ~1 s in-process queue then batch insert",
   "classification_evidence": "rybbit server pageview queue source; kill losses in both protocols",
   "tracker_retry": "none",
   "tracker_retry_source": "rybbit-io/rybbit server/src/analytics-script/tracking.ts @ 8a22fd9f (2.9.1): one fetch, no retry",
   "tracker_retry_measured": false,
   "configuration_id": "cfg-rybbit-ramp2",
   "record_file": "2026-09-14-split-rybbit.md",
   "record_path": "load-test-2026-09-14/results/2026-09-14-split-rybbit.md",
   "trials": 1,
   "note_ids": [],
   "overload_behaviour": null,
   "observation_id": "obs-2026-09-14-rybbit-ramp2-kill",
   "run_id": "run-2026-09-14-rybbit-ramp2",
   "stage": "kill:backend",
   "offered_rate": 605.0,
   "offered_unit": "events_per_s",
   "duration_s": 90,
   "sent": 50338,
   "acknowledged": 19310,
   "stored_rows": 18951,
   "stored_distinct": 18951,
   "reconciliation_method": "event_id_matching",
   "acknowledged_minus_stored_rows": 359,
   "acknowledged_missing_count": 359,
   "duplicate_count": 0,
   "stored_without_ack_count": 0,
   "hard_failures": 31028,
   "p95_ms": 1091.8,
   "p95_scope": "all requests in the stage window",
   "generator_box": "e2-standard-4",
   "generator_busy_max_pct": 24.0,
   "k6_cpu_max_pct": 82.4,
   "qualifications": [
    "single_trial",
    "healthy_probe_recovery",
    "protocol_variant"
   ],
   "record_type": "failure_test",
   "failure_scope": {
    "component": "backend",
    "mechanism": "docker kill (SIGKILL)",
    "accept_path_up": false
   },
   "kill_at_s": 20,
   "restart_by_driver_at_s": 30,
   "restart_policy_text": "unless-stopped",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "recovery": [
    {
     "definition": "healthy_probe",
     "seconds": 75
    }
   ],
   "observed_outcome": "359 of 19,310 acknowledged requests were shown to be missing after killing backend. 31,028 requests failed while the component was down.",
   "record_sha256": "d5d0784e93bb8cb9145aad33c111ea482b71c1125c9eb62bcb6543191a3c33e4",
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-split-rybbit.md"
  },
  {
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "subject": "Umami",
   "subject_version": "3.3.0",
   "subject_digests": [
    "ghcr.io/umami-software/umami@sha256:87312d334d009ee67ee0d2fba8fed01435547cc468e452243aef5133a9984d48"
   ],
   "axis": "pageview",
   "box": "e2-standard-2",
   "box_note": null,
   "generator_colocated": false,
   "protocol": "ramp",
   "protocol_version": "2",
   "run_date": "2026-09-14",
   "deviations": [
    "CLIENT_IP_HEADER=x-forwarded-for so the visitor pool is not one session"
   ],
   "acknowledgement_strategy": "synchronous_write",
   "acknowledgement_strategy_note": null,
   "classification_evidence": "run records: stored == acknowledged at every stage under both protocols; no buffering tier in the source",
   "tracker_retry": "none",
   "tracker_retry_source": "umami-software/umami src/tracker/index.ts (3.3.1) @ 8efd5d72: one fetch, errors swallowed",
   "tracker_retry_measured": false,
   "configuration_id": "cfg-umami-ramp2",
   "record_file": "2026-09-14-split-umami.md",
   "record_path": "load-test-2026-09-14/results/2026-09-14-split-umami.md",
   "trials": 1,
   "note_ids": [],
   "overload_behaviour": null,
   "observation_id": "obs-2026-09-14-umami-ramp2-kill",
   "run_id": "run-2026-09-14-umami-ramp2",
   "stage": "kill:umami",
   "offered_rate": 99.0,
   "offered_unit": "events_per_s",
   "duration_s": 90,
   "sent": 8186,
   "acknowledged": 4992,
   "stored_rows": 4994,
   "stored_distinct": 4994,
   "reconciliation_method": "event_id_matching",
   "acknowledged_minus_stored_rows": -2,
   "acknowledged_missing_count": 0,
   "duplicate_count": 0,
   "stored_without_ack_count": 2,
   "hard_failures": 3194,
   "p95_ms": 13964.2,
   "p95_scope": "all requests in the stage window",
   "generator_box": "e2-standard-4",
   "generator_busy_max_pct": 9.0,
   "k6_cpu_max_pct": 26.7,
   "qualifications": [
    "single_trial",
    "healthy_probe_recovery",
    "protocol_variant"
   ],
   "record_type": "failure_test",
   "failure_scope": {
    "component": "umami (Node app)",
    "mechanism": "docker kill (SIGKILL)",
    "accept_path_up": false
   },
   "kill_at_s": 20,
   "restart_by_driver_at_s": 30,
   "restart_policy_text": "no (Docker default; none in the bench kit)",
   "restart_policy_fired": {
    "null": "not_applicable"
   },
   "restart_policy_observation": null,
   "recovery": [
    {
     "definition": "healthy_probe",
     "seconds": 63
    }
   ],
   "observed_outcome": "0 of 4,992 acknowledged requests were shown to be missing after killing umami (Node app). 3,194 requests failed while the component was down.",
   "record_sha256": "e668ebbe34b99159307974b6d33d8c332ba254fd8b233a325b2c473a7ea24087",
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-split-umami.md"
  },
  {
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "subject": "Matomo",
   "subject_version": "5.12.0",
   "subject_digests": [
    "matomo@sha256:b6745de05fae0284ecfd564b82dc4a722a195c13f4c3dd8927441162907b37ad",
    "mariadb@sha256:1fe78d53850250aa2560aa0059e3088b4cd230a5db2230f530c70e4b87bcc30c"
   ],
   "axis": "pageview",
   "box": "e2-standard-2",
   "box_note": null,
   "generator_colocated": false,
   "protocol": "ramp",
   "protocol_version": "2",
   "run_date": "2026-09-14",
   "deviations": [
    "proxy_client_headers[]=HTTP_X_FORWARDED_FOR",
    "browser_archiving_disabled_enforce=1",
    "MariaDB --max-allowed-packet=64MB"
   ],
   "acknowledgement_strategy": "synchronous_write",
   "acknowledgement_strategy_note": null,
   "classification_evidence": "run records: stored >= acknowledged at every stage (timed-out requests were written); tracker inserts synchronously without QueuedTracking",
   "tracker_retry": "none",
   "tracker_retry_source": "matomo-org/matomo js/piwik.js 5.14.0-alpha @ 623f2865: batching queue exists, a failed request is not resent",
   "tracker_retry_measured": false,
   "configuration_id": "cfg-matomo-ramp2",
   "record_file": "2026-09-14-split-matomo.md",
   "record_path": "load-test-2026-09-14/results/2026-09-14-split-matomo.md",
   "trials": 1,
   "note_ids": [],
   "overload_behaviour": null,
   "observation_id": "obs-2026-09-14-matomo-ramp2-kill",
   "run_id": "run-2026-09-14-matomo-ramp2",
   "stage": "kill:matomo",
   "offered_rate": 22.0,
   "offered_unit": "events_per_s",
   "duration_s": 90,
   "sent": 1748,
   "acknowledged": 1411,
   "stored_rows": 1411,
   "stored_distinct": {
    "null": "count_based"
   },
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": 0,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": {
    "null": "count_based"
   },
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 337,
   "p95_ms": 34303.0,
   "p95_scope": "all requests in the stage window",
   "generator_box": "e2-standard-4",
   "generator_busy_max_pct": 5.0,
   "k6_cpu_max_pct": 18.3,
   "qualifications": [
    "single_trial",
    "count_based_reconciliation",
    "healthy_probe_recovery",
    "protocol_variant"
   ],
   "record_type": "failure_test",
   "failure_scope": {
    "component": "matomo (Apache/PHP)",
    "mechanism": "docker kill (SIGKILL)",
    "accept_path_up": false
   },
   "kill_at_s": 20,
   "restart_by_driver_at_s": 30,
   "restart_policy_text": "no (Docker default; none in the bench kit)",
   "restart_policy_fired": {
    "null": "not_applicable"
   },
   "restart_policy_observation": null,
   "recovery": [
    {
     "definition": "healthy_probe",
     "seconds": 80
    }
   ],
   "observed_outcome": "No count deficit was observed after killing matomo (Apache/PHP) (1,411 acknowledged, 1,411 stored). Because reconciliation compared totals, this does not rule out losses offset by duplicates. 337 requests failed while the component was down.",
   "record_sha256": "16024ba1ccc1c58cfc10d5d09ed0eb1b53f125bc8e297c32528b3da8f6293889",
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-split-matomo.md"
  },
  {
   "observation_id": "obs-2026-09-14-matomo-queued-kill-app",
   "run_id": "run-2026-09-14-matomo-queued",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-matomo-queued-durability.md",
   "record_sha256": "89df2aef67d6f12a39a6eac5be83faf3e6d4f7cf4d2fcbf2b9349a85140c9b62",
   "record_file": "2026-09-14-matomo-queued-durability.md",
   "configuration_id": "cfg-matomo-5.13.0-queuedtracking-5.2.0-redis-7.4",
   "subject": "Matomo + QueuedTracking (Redis)",
   "subject_version": "Matomo 5.13.0, QueuedTracking 5.2.0, Redis 7.4.11",
   "subject_digests": [
    "matomo@sha256:8e6bdd396496a98e4011cfbdc4be996d9259cbd1f09e0651b6058bb9a5984e13",
    "redis@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf"
   ],
   "axis": "pageview",
   "box": "e2-standard-2",
   "box_note": null,
   "deviations": [
    "proxy_client_headers[]=HTTP_X_FORWARDED_FOR",
    "browser_archiving_disabled_enforce=1",
    "QueuedTracking queueEnabled=1, redisHost=redis (plugin defaults otherwise)",
    "restart: always on all services, from the vendor's compose example"
   ],
   "acknowledgement_strategy": "durable_queue",
   "acknowledgement_strategy_note": "204 after pushing the request to a Redis list; Redis persistence as shipped: RDB snapshots (save 3600 1 300 100 60 10000), no AOF",
   "classification_evidence": "plugin-QueuedTracking tracker handler; redis-cli config get save / appendonly on the box",
   "tracker_retry": "none",
   "tracker_retry_source": "matomo-org/matomo js/piwik.js 5.14.0-alpha @ 623f2865",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-matomo-queued-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:app",
   "failure_scope": {
    "component": "matomo (Apache/PHP)",
    "mechanism": "docker kill (API SIGKILL)",
    "accept_path_up": false
   },
   "offered_rate": 20,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 1801,
   "acknowledged": 1567,
   "stored_rows": 1568,
   "stored_distinct": {
    "null": "not_measured"
   },
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": -1,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": {
    "null": "not_measured"
   },
   "stored_without_ack_count": 1,
   "hard_failures": 234,
   "p95_ms": 16.2,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "first_success",
     "seconds": 12
    }
   ],
   "restart_policy_text": "always",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "observed_outcome": "No count deficit was observed after killing matomo (Apache/PHP): 1 more rows were stored than requests acknowledged (1,567 acknowledged, 1,568 stored), consistent with responses the client never received. Because reconciliation compared totals, this does not rule out losses offset by duplicates. 234 requests failed while the component was down.",
   "observed_outcome_override": null,
   "qualifications": [
    "count_based_reconciliation",
    "single_trial",
    "generator_colocated",
    "retry_inferred_from_code",
    "protocol_variant"
   ],
   "note_ids": [],
   "run_date": "2026-09-14"
  },
  {
   "observation_id": "obs-2026-09-14-matomo-queued-kill-redis",
   "run_id": "run-2026-09-14-matomo-queued",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-matomo-queued-durability.md",
   "record_sha256": "89df2aef67d6f12a39a6eac5be83faf3e6d4f7cf4d2fcbf2b9349a85140c9b62",
   "record_file": "2026-09-14-matomo-queued-durability.md",
   "configuration_id": "cfg-matomo-5.13.0-queuedtracking-5.2.0-redis-7.4",
   "subject": "Matomo + QueuedTracking (Redis)",
   "subject_version": "Matomo 5.13.0, QueuedTracking 5.2.0, Redis 7.4.11",
   "subject_digests": [
    "matomo@sha256:8e6bdd396496a98e4011cfbdc4be996d9259cbd1f09e0651b6058bb9a5984e13",
    "redis@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf"
   ],
   "axis": "pageview",
   "box": "e2-standard-2",
   "box_note": null,
   "deviations": [
    "proxy_client_headers[]=HTTP_X_FORWARDED_FOR",
    "browser_archiving_disabled_enforce=1",
    "QueuedTracking queueEnabled=1, redisHost=redis (plugin defaults otherwise)",
    "restart: always on all services, from the vendor's compose example"
   ],
   "acknowledgement_strategy": "durable_queue",
   "acknowledgement_strategy_note": "204 after pushing the request to a Redis list; Redis persistence as shipped: RDB snapshots (save 3600 1 300 100 60 10000), no AOF",
   "classification_evidence": "plugin-QueuedTracking tracker handler; redis-cli config get save / appendonly on the box",
   "tracker_retry": "none",
   "tracker_retry_source": "matomo-org/matomo js/piwik.js 5.14.0-alpha @ 623f2865",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-matomo-queued-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:redis",
   "failure_scope": {
    "component": "redis (the queue)",
    "mechanism": "docker kill (API SIGKILL)",
    "accept_path_up": true
   },
   "offered_rate": 20,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 1800,
   "acknowledged": 1800,
   "stored_rows": 1571,
   "stored_distinct": 1547,
   "reconciliation_method": "documented_correction",
   "acknowledged_minus_stored_rows": 229,
   "acknowledged_missing_count": 253,
   "duplicate_count": 24,
   "stored_without_ack_count": 0,
   "hard_failures": 0,
   "p95_ms": 182.6,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "first_success",
     "seconds": 11,
     "note": "redis-cli PONG immediately after docker start"
    }
   ],
   "restart_policy_text": "always",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "observed_outcome": "253 of 1,800 acknowledged requests were not stored: 220 were answered 204 while Redis was down, and 33 were the queue contents at the kill. On restart Redis replayed a stale snapshot and 24 already-stored hits were inserted a second time. The client saw no failures.",
   "observed_outcome_override": "253 of 1,800 acknowledged requests were not stored: 220 were answered 204 while Redis was down, and 33 were the queue contents at the kill. On restart Redis replayed a stale snapshot and 24 already-stored hits were inserted a second time. The client saw no failures.",
   "qualifications": [
    "single_trial",
    "generator_colocated",
    "protocol_variant",
    "acknowledged_while_dependency_down",
    "retry_inferred_from_code"
   ],
   "note_ids": [
    "n-matomo-dup",
    "n-matomo-204"
   ],
   "run_date": "2026-09-14"
  },
  {
   "observation_id": "obs-2026-09-14-ours-kill-app-plain",
   "run_id": "run-2026-09-14-ours-kill2",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-ours-durability.md",
   "record_sha256": "14769bb673605dead696cec5063993b812c44da8f89dba33354b06987f1a295e",
   "record_file": "2026-09-14-ours-durability.md",
   "configuration_id": "cfg-jetboat-lite-671180f",
   "subject": "Jetboat lite stack (ours)",
   "subject_version": "build 671180f",
   "subject_digests": [
    "app image sha256:1345dc38ba4cd18913f41e5487e9ad7a1424e332f7e32a30e233a3a915b6ebba",
    "clickhouse/clickhouse-server@sha256:b627d7a9bc0e0c1bac26cdbe9d2fc6316faa29c5d8a174f28f5abd57d0fa6ba2"
   ],
   "axis": "pageview",
   "box": "e2-standard-2",
   "box_note": null,
   "deviations": [],
   "acknowledgement_strategy": "durable_log",
   "acknowledgement_strategy_note": "append-only log fsync'd before the 202; a separate drain moves rows to ClickHouse",
   "classification_evidence": "backend/cmd/server/main.go WAL branch; W5a/W5b drills in the run record",
   "tracker_retry": "persisted",
   "tracker_retry_source": "sdk/src/transport.ts @ cbec428: localStorage retry queue, 2000 ms × 2^n capped at 5 min, 48 h horizon, pagehide sendBeacon",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-ours-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:app",
   "failure_scope": {
    "component": "app",
    "mechanism": "docker kill (API SIGKILL)",
    "accept_path_up": false
   },
   "offered_rate": 50,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 4501,
   "acknowledged": 3927,
   "stored_rows": 3927,
   "stored_distinct": 3927,
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": 0,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": 0,
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 574,
   "p95_ms": 9.9,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "first_success",
     "seconds": 11
    }
   ],
   "restart_policy_text": "unless-stopped",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "observed_outcome": "No count deficit was observed after killing app (3,927 acknowledged, 3,927 stored). Because reconciliation compared totals, this does not rule out losses offset by duplicates. 574 requests failed while the component was down.",
   "observed_outcome_override": null,
   "qualifications": [
    "count_based_reconciliation",
    "single_trial",
    "generator_colocated",
    "retry_inferred_from_code"
   ],
   "note_ids": [],
   "run_date": "2026-09-14"
  },
  {
   "observation_id": "obs-2026-09-14-ours-kill-app-store-down",
   "run_id": "run-2026-09-14-ours-kill2",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-ours-durability.md",
   "record_sha256": "14769bb673605dead696cec5063993b812c44da8f89dba33354b06987f1a295e",
   "record_file": "2026-09-14-ours-durability.md",
   "configuration_id": "cfg-jetboat-lite-671180f",
   "subject": "Jetboat lite stack (ours)",
   "subject_version": "build 671180f",
   "subject_digests": [
    "app image sha256:1345dc38ba4cd18913f41e5487e9ad7a1424e332f7e32a30e233a3a915b6ebba",
    "clickhouse/clickhouse-server@sha256:b627d7a9bc0e0c1bac26cdbe9d2fc6316faa29c5d8a174f28f5abd57d0fa6ba2"
   ],
   "axis": "pageview",
   "box": "e2-standard-2",
   "box_note": null,
   "deviations": [],
   "acknowledgement_strategy": "durable_log",
   "acknowledgement_strategy_note": "append-only log fsync'd before the 202; a separate drain moves rows to ClickHouse",
   "classification_evidence": "backend/cmd/server/main.go WAL branch; W5a/W5b drills in the run record",
   "tracker_retry": "persisted",
   "tracker_retry_source": "sdk/src/transport.ts @ cbec428: localStorage retry queue, 2000 ms × 2^n capped at 5 min, 48 h horizon, pagehide sendBeacon",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-ours-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:app+store_down",
   "failure_scope": {
    "component": "app, with ClickHouse already stopped",
    "mechanism": "SIGKILL via docker kill",
    "accept_path_up": false,
    "steps": [
     {
      "t_s": 10,
      "action": "docker stop clickhouse"
     },
     {
      "t_s": 20,
      "action": "docker kill --signal=KILL app"
     },
     {
      "t_s": 31,
      "action": "docker start app"
     },
     {
      "t_s": 45,
      "action": "docker start clickhouse"
     }
    ]
   },
   "offered_rate": 50,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 4501,
   "acknowledged": 3691,
   "stored_rows": 3691,
   "stored_distinct": 3691,
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": 0,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": 0,
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 810,
   "p95_ms": 6.9,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "first_success_store_down",
     "seconds": 17,
     "note": "202 for a real event while ClickHouse was still stopped; ClickHouse returned at +27 s"
    }
   ],
   "restart_policy_text": "unless-stopped",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "observed_outcome": "No count deficit was observed after killing app, with ClickHouse already stopped (3,691 acknowledged, 3,691 stored). Because reconciliation compared totals, this does not rule out losses offset by duplicates. 810 requests failed while the component was down.",
   "observed_outcome_override": null,
   "qualifications": [
    "count_based_reconciliation",
    "single_trial",
    "generator_colocated",
    "store_stopped_before_kill",
    "retry_inferred_from_code"
   ],
   "note_ids": [
    "n-ours-strict"
   ],
   "run_date": "2026-09-14"
  },
  {
   "observation_id": "obs-2026-09-14-ours-kill-store",
   "run_id": "run-2026-09-14-ours-kill2",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-ours-durability.md",
   "record_sha256": "14769bb673605dead696cec5063993b812c44da8f89dba33354b06987f1a295e",
   "record_file": "2026-09-14-ours-durability.md",
   "configuration_id": "cfg-jetboat-lite-671180f",
   "subject": "Jetboat lite stack (ours)",
   "subject_version": "build 671180f",
   "subject_digests": [
    "app image sha256:1345dc38ba4cd18913f41e5487e9ad7a1424e332f7e32a30e233a3a915b6ebba",
    "clickhouse/clickhouse-server@sha256:b627d7a9bc0e0c1bac26cdbe9d2fc6316faa29c5d8a174f28f5abd57d0fa6ba2"
   ],
   "axis": "pageview",
   "box": "e2-standard-2",
   "box_note": null,
   "deviations": [],
   "acknowledgement_strategy": "durable_log",
   "acknowledgement_strategy_note": "append-only log fsync'd before the 202; a separate drain moves rows to ClickHouse",
   "classification_evidence": "backend/cmd/server/main.go WAL branch; W5a/W5b drills in the run record",
   "tracker_retry": "persisted",
   "tracker_retry_source": "sdk/src/transport.ts @ cbec428: localStorage retry queue, 2000 ms × 2^n capped at 5 min, 48 h horizon, pagehide sendBeacon",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-ours-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:store",
   "failure_scope": {
    "component": "clickhouse (the store)",
    "mechanism": "docker kill (API SIGKILL)",
    "accept_path_up": true
   },
   "offered_rate": 50,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 4501,
   "acknowledged": 4501,
   "stored_rows": 4501,
   "stored_distinct": 4501,
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": 0,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": 0,
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 0,
   "p95_ms": 16.1,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "store_answering",
     "seconds": 15
    }
   ],
   "restart_policy_text": "unless-stopped",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "observed_outcome": "No count deficit was observed after killing clickhouse (the store) (4,501 acknowledged, 4,501 stored). Because reconciliation compared totals, this does not rule out losses offset by duplicates. The accept path kept answering throughout.",
   "observed_outcome_override": null,
   "qualifications": [
    "count_based_reconciliation",
    "single_trial",
    "generator_colocated",
    "retry_inferred_from_code"
   ],
   "note_ids": [],
   "run_date": "2026-09-14"
  },
  {
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "subject": "Jetboat lite stack (ours)",
   "subject_version": "build f1dcdee",
   "subject_digests": [
    "app image built from source on the box",
    "clickhouse/clickhouse-server@sha256:b627d7a9bc0e0c1bac26cdbe9d2fc6316faa29c5d8a174f28f5abd57d0fa6ba2"
   ],
   "axis": "pageview",
   "box": "e2-standard-2",
   "box_note": null,
   "generator_colocated": false,
   "protocol": "ramp",
   "protocol_version": "2",
   "run_date": "2026-09-14",
   "deviations": [
    "APP_BIND=0.0.0.0 (the shipped compose binds the app to loopback; the generator is a separate machine)"
   ],
   "acknowledgement_strategy": "durable_log",
   "acknowledgement_strategy_note": "append-only log written to disk before the 202; a separate drain moves rows to ClickHouse",
   "classification_evidence": "backend/cmd/server/main.go durable-buffer branch; drain-stall alerts and replay lines in the run records",
   "tracker_retry": "persisted",
   "tracker_retry_source": "sdk/src/transport.ts @ cbec428: localStorage retry queue, 2000 ms × 2^n capped at 5 min, 48 h horizon, pagehide sendBeacon",
   "tracker_retry_measured": false,
   "configuration_id": "cfg-ours-ramp2",
   "record_file": "2026-09-14-split-ours.md",
   "record_path": "load-test-2026-09-14/results/2026-09-14-split-ours.md",
   "trials": 1,
   "note_ids": [],
   "overload_behaviour": "Past a soft cap the accept path returns 202, then deliberately drops low-value events and counts them (shed counters). Not exercised in these stages; recorded so that a zero-loss kill result is not read as a general no-loss guarantee.",
   "observation_id": "obs-2026-09-14-ours-ramp2-kill",
   "run_id": "run-2026-09-14-ours-ramp2",
   "stage": "kill:app",
   "offered_rate": 366.0,
   "offered_unit": "events_per_s",
   "duration_s": 90,
   "sent": 32941,
   "acknowledged": 27073,
   "stored_rows": 27079,
   "stored_distinct": 27079,
   "reconciliation_method": "event_id_matching",
   "acknowledged_minus_stored_rows": -6,
   "acknowledged_missing_count": 0,
   "duplicate_count": 0,
   "stored_without_ack_count": 6,
   "hard_failures": 5868,
   "p95_ms": 133.1,
   "p95_scope": "all requests in the stage window",
   "generator_box": "e2-standard-4",
   "generator_busy_max_pct": 18.0,
   "k6_cpu_max_pct": 66.4,
   "qualifications": [
    "single_trial",
    "healthy_probe_recovery",
    "overload_discard",
    "protocol_variant"
   ],
   "record_type": "failure_test",
   "failure_scope": {
    "component": "app",
    "mechanism": "docker kill (SIGKILL)",
    "accept_path_up": false
   },
   "kill_at_s": 20,
   "restart_by_driver_at_s": 30,
   "restart_policy_text": "unless-stopped",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "recovery": [
    {
     "definition": "healthy_probe",
     "seconds": 17
    }
   ],
   "observed_outcome": "0 of 27,073 acknowledged requests were shown to be missing after killing app. 5,868 requests failed while the component was down.",
   "record_sha256": "f8117aad0985852affe3f3727a90c55cbe345402a90fb0f425373d0c47a9650a",
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-split-ours.md"
  },
  {
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "subject": "ClickStack / HyperDX",
   "subject_version": "2.35.0",
   "subject_digests": [
    "docker.clickhouse.com/clickhouse/clickstack-otel-collector@sha256:046d88ecfde94076c6bd2835b82deaafc37036c665be5232b6eb88394a8d0a50",
    "clickhouse/clickhouse-server@sha256:35b419db86eed71ab1c41c03b4fd1f39be26f41eb38b0866268e2ca162445105"
   ],
   "axis": "telemetry_row",
   "box": "e2-standard-2",
   "box_note": null,
   "generator_colocated": false,
   "protocol": "ramp",
   "protocol_version": "2",
   "run_date": "2026-09-14",
   "deviations": [],
   "acknowledgement_strategy": "in_memory_batch",
   "acknowledgement_strategy_note": "the OpenTelemetry collector acknowledges after receipt and batches in memory before exporting to ClickHouse",
   "classification_evidence": "vendor compose otel-collector configuration; collector-kill losses in both protocols",
   "tracker_retry": "memory",
   "tracker_retry_source": "@hyperdx/browser 0.26.0 → @opentelemetry/otlp-exporter-base retrying transport: 5 attempts, in memory",
   "tracker_retry_measured": false,
   "configuration_id": "cfg-clickstack-ramp2",
   "record_file": "2026-09-14-split-clickstack.md",
   "record_path": "load-test-2026-09-14/results/2026-09-14-split-clickstack.md",
   "trials": 1,
   "note_ids": [],
   "overload_behaviour": null,
   "observation_id": "obs-2026-09-14-clickstack-ramp2-kill",
   "run_id": "run-2026-09-14-clickstack-ramp2",
   "stage": "kill:otel-collector",
   "offered_rate": 2513.0,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 225419,
   "acknowledged": 177246,
   "stored_rows": 173092,
   "stored_distinct": 173092,
   "reconciliation_method": "event_id_matching",
   "acknowledged_minus_stored_rows": 4154,
   "acknowledged_missing_count": 4154,
   "duplicate_count": 0,
   "stored_without_ack_count": 0,
   "hard_failures": 48173,
   "p95_ms": 72.4,
   "p95_scope": "all requests in the stage window",
   "generator_box": "e2-standard-4",
   "generator_busy_max_pct": 63.0,
   "k6_cpu_max_pct": 227.0,
   "qualifications": [
    "single_trial",
    "healthy_probe_recovery",
    "protocol_variant"
   ],
   "record_type": "failure_test",
   "failure_scope": {
    "component": "otel-collector (accept path)",
    "mechanism": "docker kill (SIGKILL)",
    "accept_path_up": false
   },
   "kill_at_s": 20,
   "restart_by_driver_at_s": 30,
   "restart_policy_text": "always",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "recovery": [
    {
     "definition": "healthy_probe",
     "seconds": 23
    }
   ],
   "observed_outcome": "4,154 of 177,246 acknowledged requests were shown to be missing after killing otel-collector (accept path). 48,173 requests failed while the component was down.",
   "record_sha256": "6959e90547c5618ed9ff117adc46e028aeecd6ee594616640271a7f810dd9013",
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-split-clickstack.md"
  },
  {
   "observation_id": "obs-2026-09-14-bugsink-kill-container",
   "run_id": "run-2026-09-14-bugsink",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-bugsink-durability.md",
   "record_sha256": "26f8e12c01ba76a900be7d3bf5021a282295165d70a11d2c36d11d49b6a3fb7d",
   "record_file": "2026-09-14-bugsink-durability.md",
   "configuration_id": "cfg-bugsink-2.5.1",
   "subject": "Bugsink",
   "subject_version": "2.5.1 (source-available)",
   "subject_digests": [
    "bugsink/bugsink@sha256:ecdd845877464d70d61244b8adba20d623e1860e88d8338f2808d9cc2925745c"
   ],
   "axis": "error_envelope",
   "box": "e2-standard-2",
   "box_note": null,
   "deviations": [
    "four per-project / global 5-minute and hourly event quota env vars raised so the fixed 10/s is not answered 429",
    "/data named volume per the vendor docs",
    "vendor sample compose's restart policy and healthcheck carried into the documented single-container SQLite shape"
   ],
   "acknowledgement_strategy": "durable_queue",
   "acknowledgement_strategy_note": "200 after the envelope file and a task row are written to an on-disk SQLite queue (snappea) inside the same container; digested later by the in-container worker",
   "classification_evidence": "TASK_ALWAYS_EAGER=False in the image settings; /tmp/snappea.sqlite3 inspected on the box",
   "tracker_retry": "none",
   "tracker_retry_source": "@sentry/browser 10.67.0 transports/base.ts @ bc5324b0: default fetch transport does not retry; offline transport is opt-in",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-bugsink-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:container",
   "failure_scope": {
    "component": "bench-bugsink (web + in-container worker)",
    "mechanism": "docker kill (API SIGKILL)",
    "accept_path_up": false
   },
   "offered_rate": 10,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 896,
   "acknowledged": 701,
   "stored_rows": 700,
   "stored_distinct": {
    "null": "not_measured"
   },
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": 1,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": {
    "null": "not_measured"
   },
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 195,
   "p95_ms": 45.5,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "first_success",
     "seconds": 23,
     "note": "/health/ready 200 at +22 s; real envelope 200 at +23 s"
    }
   ],
   "restart_policy_text": "unless-stopped",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "observed_outcome": "1 fewer row was stored than requests acknowledged (701 acknowledged, 700 stored) after killing bench-bugsink (web + in-container worker). Because reconciliation compared totals, this is a count deficit, not an identified set of lost events. 195 requests failed while the component was down.",
   "observed_outcome_override": null,
   "qualifications": [
    "count_based_reconciliation",
    "single_trial",
    "generator_colocated",
    "retry_inferred_from_code"
   ],
   "note_ids": [
    "n-bugsink-handoff"
   ],
   "run_date": "2026-09-14"
  },
  {
   "observation_id": "obs-2026-09-14-glitchtip-kill-web",
   "run_id": "run-2026-09-14-glitchtip",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-glitchtip-durability.md",
   "record_sha256": "6ac443c5b68d688088ae664ff81e3d2953d0066f3b06f2db335859cb0644d3cf",
   "record_file": "2026-09-14-glitchtip-durability.md",
   "configuration_id": "cfg-glitchtip-6.2.6",
   "subject": "GlitchTip",
   "subject_version": "6.2.6",
   "subject_digests": [
    "glitchtip/glitchtip@sha256:a3d8eb1b36c1e1603d55ab32711ea3dd8115874742a781a57391a62a16e0dff6",
    "valkey/valkey@sha256:c123e3715db63d06d4ad6964884037aa0d5d4d703939b9929954112889708e1d"
   ],
   "axis": "error_envelope",
   "box": "e2-standard-2",
   "box_note": null,
   "deviations": [],
   "acknowledgement_strategy": "in_memory_batch",
   "acknowledgement_strategy_note": "200 after LPUSH to a Valkey list; the in-process vtasks worker pops a batch (count 25 / 2 s) into memory before writing to Postgres",
   "classification_evidence": "glitchtip settings.py + django-vtasks 3.1.0 batch queue; worker log window in the run record",
   "tracker_retry": "none",
   "tracker_retry_source": "@sentry/browser 10.67.0 transports/base.ts @ bc5324b0",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-glitchtip-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:web",
   "failure_scope": {
    "component": "web (vendor all-in-one: HTTP + in-process worker)",
    "mechanism": "docker kill (API SIGKILL)",
    "accept_path_up": false
   },
   "offered_rate": 10,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 901,
   "acknowledged": 665,
   "stored_rows": 650,
   "stored_distinct": {
    "null": "not_measured"
   },
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": 15,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": {
    "null": "not_measured"
   },
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 236,
   "p95_ms": 17.3,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "first_success",
     "seconds": 26
    }
   ],
   "restart_policy_text": "unless-stopped",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "observed_outcome": "15 fewer rows were stored than requests acknowledged (665 acknowledged, 650 stored) after killing web (vendor all-in-one: HTTP + in-process worker). Because reconciliation compared totals, this is a count deficit, not an identified set of lost events. 236 requests failed while the component was down.",
   "observed_outcome_override": null,
   "qualifications": [
    "count_based_reconciliation",
    "single_trial",
    "generator_colocated",
    "retry_inferred_from_code"
   ],
   "note_ids": [
    "n-glitchtip-batch"
   ],
   "run_date": "2026-09-14"
  },
  {
   "observation_id": "obs-2026-09-14-glitchtip-kill-worker",
   "run_id": "run-2026-09-14-glitchtip",
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-glitchtip-durability.md",
   "record_sha256": "6ac443c5b68d688088ae664ff81e3d2953d0066f3b06f2db335859cb0644d3cf",
   "record_file": "2026-09-14-glitchtip-durability.md",
   "configuration_id": "cfg-glitchtip-6.2.6",
   "subject": "GlitchTip",
   "subject_version": "6.2.6",
   "subject_digests": [
    "glitchtip/glitchtip@sha256:a3d8eb1b36c1e1603d55ab32711ea3dd8115874742a781a57391a62a16e0dff6",
    "valkey/valkey@sha256:c123e3715db63d06d4ad6964884037aa0d5d4d703939b9929954112889708e1d"
   ],
   "axis": "error_envelope",
   "box": "e2-standard-2",
   "box_note": null,
   "deviations": [],
   "acknowledgement_strategy": "in_memory_batch",
   "acknowledgement_strategy_note": "200 after LPUSH to a Valkey list; the in-process vtasks worker pops a batch (count 25 / 2 s) into memory before writing to Postgres",
   "classification_evidence": "glitchtip settings.py + django-vtasks 3.1.0 batch queue; worker log window in the run record",
   "tracker_retry": "none",
   "tracker_retry_source": "@sentry/browser 10.67.0 transports/base.ts @ bc5324b0",
   "record_path": "durability-kill-test-2026-09-14/results/2026-09-14-glitchtip-durability.md",
   "generator_colocated": true,
   "protocol": "kill-test",
   "protocol_version": "2",
   "tracker_retry_measured": false,
   "record_type": "failure_test",
   "stage": "kill:worker",
   "failure_scope": {
    "component": "worker (vendor's documented role split, SERVER_ROLE=worker)",
    "mechanism": "docker kill (API SIGKILL)",
    "accept_path_up": true
   },
   "offered_rate": 10,
   "offered_unit": "requests_per_s",
   "duration_s": 90,
   "sent": 901,
   "acknowledged": 901,
   "stored_rows": 899,
   "stored_distinct": {
    "null": "not_measured"
   },
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": 2,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": {
    "null": "not_measured"
   },
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 0,
   "p95_ms": 13.0,
   "p95_scope": "all requests in the 90 s window",
   "trials": 1,
   "kill_at_s": 20,
   "restart_by_driver_at_s": 31,
   "recovery": [
    {
     "definition": "worker_consuming",
     "seconds": 22,
     "note": "driver's log poll; the log shows consumption ~13 s after the kill"
    }
   ],
   "restart_policy_text": "unless-stopped",
   "restart_policy_fired": false,
   "restart_policy_observation": "api_kill_10s_watch",
   "observed_outcome": "2 fewer rows were stored than requests acknowledged (901 acknowledged, 899 stored) after killing worker (vendor's documented role split, SERVER_ROLE=worker). Because reconciliation compared totals, this is a count deficit, not an identified set of lost events. The accept path kept answering throughout.",
   "observed_outcome_override": null,
   "qualifications": [
    "count_based_reconciliation",
    "single_trial",
    "generator_colocated",
    "retry_inferred_from_code"
   ],
   "note_ids": [
    "n-glitchtip-batch"
   ],
   "run_date": "2026-09-14"
  },
  {
   "posts": [
    "self-hosted-analytics-kill-test"
   ],
   "subject": "Traceway",
   "subject_version": "ClickHouse mode; no version label, digest is the identity",
   "subject_digests": [
    "ghcr.io/tracewayapp/traceway@sha256:688eeb294a051da84f47e31f3f25769b8f13e85fe0d81f03dc40961b10b1c77c"
   ],
   "axis": "error_envelope",
   "box": "e2-standard-2",
   "box_note": null,
   "generator_colocated": false,
   "protocol": "ramp",
   "protocol_version": "2",
   "run_date": "2026-09-14",
   "deviations": [],
   "acknowledgement_strategy": "synchronous_write",
   "acknowledgement_strategy_note": "acknowledges after a synchronous ClickHouse insert; sheds with HTTP 503 under overload",
   "classification_evidence": "traceway source: chdb.SendBatch with WithAsync(false); stored == acknowledged at every stage",
   "tracker_retry": "memory",
   "tracker_retry_source": "tracewayapp/js-client packages/frontend/src/transport.ts @ ed81989a: unbounded in-memory re-queue",
   "tracker_retry_measured": false,
   "configuration_id": "cfg-traceway-ramp2",
   "record_file": "2026-09-14-split-traceway.md",
   "record_path": "load-test-2026-09-14/results/2026-09-14-split-traceway.md",
   "trials": 1,
   "note_ids": [],
   "overload_behaviour": null,
   "observation_id": "obs-2026-09-14-traceway-ramp2-kill",
   "run_id": "run-2026-09-14-traceway-ramp2",
   "stage": "kill:all-in-one container",
   "offered_rate": 95.0,
   "offered_unit": "events_per_s",
   "duration_s": 90,
   "sent": 8258,
   "acknowledged": 3424,
   "stored_rows": 3424,
   "stored_distinct": {
    "null": "count_based"
   },
   "reconciliation_method": "count_comparison",
   "acknowledged_minus_stored_rows": 0,
   "acknowledged_missing_count": {
    "null": "count_based"
   },
   "duplicate_count": {
    "null": "count_based"
   },
   "stored_without_ack_count": {
    "null": "count_based"
   },
   "hard_failures": 4834,
   "p95_ms": 5140.5,
   "p95_scope": "all requests in the stage window",
   "generator_box": "e2-standard-4",
   "generator_busy_max_pct": 7.0,
   "k6_cpu_max_pct": 24.3,
   "qualifications": [
    "single_trial",
    "count_based_reconciliation",
    "healthy_probe_recovery",
    "protocol_variant"
   ],
   "record_type": "failure_test",
   "failure_scope": {
    "component": "all-in-one container (backend + ClickHouse + Postgres)",
    "mechanism": "docker kill (SIGKILL)",
    "accept_path_up": false
   },
   "kill_at_s": 20,
   "restart_by_driver_at_s": 30,
   "restart_policy_text": "no (Docker default; none in the bench kit)",
   "restart_policy_fired": {
    "null": "not_applicable"
   },
   "restart_policy_observation": null,
   "recovery": [
    {
     "definition": "healthy_probe",
     "seconds": 26
    }
   ],
   "observed_outcome": "No count deficit was observed after killing all-in-one container (backend + ClickHouse + Postgres) (3,424 acknowledged, 3,424 stored). Because reconciliation compared totals, this does not rule out losses offset by duplicates. 4,834 requests failed while the component was down.",
   "record_sha256": "3a0cf4a63fb4e2a6fca137fec25a16f55ab4a5a9688c2c4248c97abafeff957a",
   "record_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/records/2026-09-14-split-traceway.md"
  }
 ],
 "license": "https://creativecommons.org/licenses/by/4.0/",
 "license_name": "CC BY 4.0",
 "attribution": "Jetboat Analytics, \"Self-hosted analytics data loss: 12 tools tested under a kill signal\", https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/, CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/). Attribution must include this link.",
 "source_url": "https://jetboatanalytics.com/blog/self-hosted-analytics-kill-test/"
}