Mål – lad være med at gætte

perf/slow-syscalls.bt

#!/usr/bin/env bpftrace
// Which syscalls is this process actually waiting on?
// Usage: slow-syscalls.bt <PID>. Runs for 30s, then prints a latency
// histogram (µs) per syscall. No instrumentation, no restart, no code change.
//
// Keys are syscall numbers, which differ per architecture:
// `ausyscall <n>` (from auditd) names them.

tracepoint:raw_syscalls:sys_enter
/pid == $1/
{
    @start[tid] = nsecs;
}

tracepoint:raw_syscalls:sys_exit
/pid == $1 && @start[tid]/
{
    @us[args->id] = hist((nsecs - @start[tid]) / 1000);
    delete(@start[tid]);
}

interval:s:30 { exit(); }

END { clear(@start); }

perf/findings.sql

-- The actual bottleneck, found by measuring rather than guessing.
-- 2.4s sequential scan on every request; nobody had run EXPLAIN.

EXPLAIN (ANALYZE, BUFFERS)
SELECT o.id, o.total, c.email
  FROM orders o
  JOIN customers c ON c.id = o.customer_id
 WHERE o.created_at >= now() - interval '30 days'
   AND o.status = 'pending';

--  Seq Scan on orders  (cost=0..48210 rows=1204 width=48)
--    Filter: created_at >= ... AND status = 'pending'
--    Rows Removed by Filter: 2841902
--    Buffers: shared read=31204
--  Execution Time: 2417.882 ms

CREATE INDEX CONCURRENTLY orders_pending_recent_idx
    ON orders (created_at DESC)
 WHERE status = 'pending';   -- partial: 3% of the table

-- After: Index Scan, Execution Time: 4.112 ms

monitoring/rules/slo.yaml

groups:
  - name: api-slo
    rules:
      # Symptom, not cause: users are seeing errors.
      - alert: ApiErrorBudgetBurn
        expr: |
          (
            sum(rate(http_requests_total{job="api",code=~"5.."}[5m]))
            /
            sum(rate(http_requests_total{job="api"}[5m]))
          ) > 0.02
        for: 10m
        labels:
          severity: page
        annotations:
          summary: 'API 5xx rate above 2% for 10m'
          # Every alert names its first debugging step.
          runbook: https://runbooks.example.com/api/5xx

      - alert: ApiLatencyP99
        expr: |
          histogram_quantile(0.99,
            sum by (le) (rate(http_request_duration_seconds_bucket{job="api"}[5m]))
          ) > 1.5
        for: 15m
        labels:
          severity: ticket