Scaling to One Million Users: The Infrastructure Playbook Nobody Shares

// At 100 users, this is your entire infrastructure
// One server, one database, one process
 
import express from 'express';
import { Pool } from 'pg';
 
const pool = new Pool({
  host: 'localhost', // yes, localhost. the db is on the same box
  max: 10,          // 10 connections is plenty
  idleTimeoutMillis: 30000,
});
 
const app = express();
 
app.get('/api/users/:id', async (req, res) => {
  const { rows } = await pool.query(
    'SELECT id, name, email FROM users WHERE id = $1',
    [req.params.id]
  );
  res.json(rows[0]);
});
 
app.listen(3000);
// This handles 100 users without breaking a sweat

// The query that kills you at 1K users
// This is what every ORM generates by default
 
// BAD: Full table scan, no pagination
app.get('/api/posts', async (req, res) => {
  const posts = await db.query('SELECT * FROM posts ORDER BY created_at DESC');
  // You are loading every post into memory
  // At 50K posts, this takes 3 seconds and uses 200MB of RAM
  res.json(posts.rows);
});
 
// GOOD: Indexed, paginated, selecting only what you need
app.get('/api/posts', async (req, res) => {
  const cursor = req.query.cursor;
  const limit = Math.min(Number(req.query.limit) || 20, 100);
 
  const { rows } = await db.query(
    `SELECT id, title, excerpt, author_id, created_at
     FROM posts
     WHERE ($1::timestamptz IS NULL OR created_at < $1)
     ORDER BY created_at DESC
     LIMIT $2`,
    [cursor || null, limit + 1]
  );
 
  const hasMore = rows.length > limit;
  const posts = hasMore ? rows.slice(0, -1) : rows;
 
  res.json({
    posts,
    nextCursor: hasMore ? posts[posts.length - 1].created_at : null,
  });
});
 
// And the index that makes it work
// CREATE INDEX idx_posts_created_at ON posts (created_at DESC);

// At this stage, you need proper connection pooling
// This is not optional — it is the difference between working and not
 
import { Pool } from 'pg';
 
const pool = new Pool({
  host: process.env.DB_HOST,      // separate DB server now
  port: 5432,
  database: process.env.DB_NAME,
  user: process.env.DB_USER,
  password: process.env.DB_PASSWORD,
  max: 20,                         // per-process limit
  min: 5,                          // keep connections warm
  idleTimeoutMillis: 30000,
  connectionTimeoutMillis: 5000,   // fail fast if pool exhausted
  statement_timeout: '10000',      // kill queries over 10 seconds
  // THIS IS THE ONE EVERYONE FORGETS
  // Without it, a single slow query can starve the pool
  query_timeout: 15000,
});
 
// Monitor your pool — you'll thank me later
setInterval(() => {
  const { totalCount, idleCount, waitingCount } = pool;
  console.log(JSON.stringify({
    level: 'info',
    msg: 'pool_stats',
    total: totalCount,
    idle: idleCount,
    waiting: waitingCount,
    active: totalCount - idleCount,
  }));
  if (waitingCount > 5) {
    console.log(JSON.stringify({
      level: 'warn',
      msg: 'pool_pressure',
      waiting: waitingCount,
    }));
  }
}, 10000);

// Database query routing between primary and replicas
// This is the pattern that lets you scale reads independently
 
import { Pool } from 'pg';
 
interface DatabaseCluster {
  primary: Pool;
  replicas: Pool[];
  currentReplica: number;
}
 
function createCluster(): DatabaseCluster {
  const primary = new Pool({
    host: process.env.DB_PRIMARY_HOST,
    max: 20,
    min: 5,
    idleTimeoutMillis: 30000,
    connectionTimeoutMillis: 5000,
  });
 
  const replicaHosts = (process.env.DB_REPLICA_HOSTS || '').split(',');
  const replicas = replicaHosts.map(host =>
    new Pool({
      host,
      max: 30, // replicas handle more connections (read-heavy)
      min: 5,
      idleTimeoutMillis: 30000,
      connectionTimeoutMillis: 5000,
    })
  );
 
  return { primary, replicas, currentReplica: 0 };
}
 
const cluster = createCluster();
 
// Round-robin replica selection
function getReadPool(): Pool {
  if (cluster.replicas.length === 0) return cluster.primary;
  const pool = cluster.replicas[cluster.currentReplica];
  cluster.currentReplica = (cluster.currentReplica + 1) % cluster.replicas.length;
  return pool;
}
 
// The key insight: route by operation type
export async function query(
  sql: string,
  params?: unknown[],
  options?: { forcePrimary?: boolean }
) {
  // Writes always go to primary
  // Reads go to replicas unless you need strong consistency
  const isWrite = /^\s*(INSERT|UPDATE|DELETE|CREATE|ALTER|DROP)/i.test(sql);
  const pool = isWrite || options?.forcePrimary
    ? cluster.primary
    : getReadPool();
 
  const start = performance.now();
  try {
    const result = await pool.query(sql, params);
    const duration = performance.now() - start;
 
    if (duration > 1000) {
      console.log(JSON.stringify({
        level: 'warn',
        msg: 'slow_query',
        sql: sql.substring(0, 200),
        duration_ms: Math.round(duration),
        target: isWrite ? 'primary' : 'replica',
      }));
    }
 
    return result;
  } catch (error) {
    // If a replica fails, fall back to primary
    if (!isWrite && !options?.forcePrimary) {
      console.log(JSON.stringify({
        level: 'warn',
        msg: 'replica_fallback',
        error: (error as Error).message,
      }));
      return cluster.primary.query(sql, params);
    }
    throw error;
  }
}

-- PostgreSQL configuration for a server handling 100K+ users
-- These are the settings that actually matter
 
-- Connection limits
max_connections = 300               -- Don't go higher; use a pooler instead
superuser_reserved_connections = 3
 
-- Memory (for a 16GB RAM server)
shared_buffers = '4GB'              -- 25% of RAM
effective_cache_size = '12GB'       -- 75% of RAM
work_mem = '64MB'                   -- Per-operation sort/hash memory
maintenance_work_mem = '512MB'      -- For VACUUM, CREATE INDEX
 
-- WAL and replication
wal_level = 'replica'
max_wal_senders = 10
max_replication_slots = 10
wal_buffers = '64MB'
checkpoint_completion_target = 0.9
 
-- Query planner
random_page_cost = 1.1             -- Assuming SSD storage
effective_io_concurrency = 200     -- SSD can handle parallel I/O
default_statistics_target = 200    -- Better query plans, slightly slower ANALYZE
 
-- Logging (you need this for debugging)
log_min_duration_statement = 500   -- Log queries over 500ms
log_checkpoints = on
log_connections = off              -- Too noisy at scale
log_disconnections = off
log_lock_waits = on
log_temp_files = 0                 -- Log ALL temp file usage

// Production connection pool with health checking and metrics
 
import { Pool, PoolClient, QueryResult } from 'pg';
 
interface PoolMetrics {
  totalQueries: number;
  failedQueries: number;
  slowQueries: number;
  poolExhausted: number;
  averageQueryTime: number;
}
 
class DatabasePool {
  private pool: Pool;
  private metrics: PoolMetrics = {
    totalQueries: 0,
    failedQueries: 0,
    slowQueries: 0,
    poolExhausted: 0,
    averageQueryTime: 0,
  };
  private queryTimes: number[] = [];
  private readonly SLOW_QUERY_MS = 1000;
  private readonly MAX_QUERY_TIME_SAMPLES = 1000;
 
  constructor(config: {
    host: string;
    port: number;
    database: string;
    user: string;
    password: string;
    maxConnections: number;
  }) {
    this.pool = new Pool({
      host: config.host,
      port: config.port,
      database: config.database,
      user: config.user,
      password: config.password,
      max: config.maxConnections,
      min: Math.max(2, Math.floor(config.maxConnections / 4)),
      idleTimeoutMillis: 30000,
      connectionTimeoutMillis: 5000,
      allowExitOnIdle: false,
    });
 
    this.pool.on('error', (err) => {
      // Idle client errors — connection dropped by server/network
      console.error(JSON.stringify({
        level: 'error',
        msg: 'pool_idle_error',
        error: err.message,
      }));
    });
 
    this.pool.on('connect', () => {
      // New connection established — useful for debugging pool behavior
    });
  }
 
  async query<T = unknown>(
    sql: string,
    params?: unknown[]
  ): Promise<QueryResult<T>> {
    const start = performance.now();
    this.metrics.totalQueries++;
 
    // Check pool pressure before acquiring
    if (this.pool.waitingCount > 10) {
      this.metrics.poolExhausted++;
      console.log(JSON.stringify({
        level: 'warn',
        msg: 'pool_backpressure',
        waiting: this.pool.waitingCount,
        sql: sql.substring(0, 100),
      }));
    }
 
    try {
      const result = await this.pool.query<T>(sql, params);
      const duration = performance.now() - start;
 
      this.recordQueryTime(duration);
 
      if (duration > this.SLOW_QUERY_MS) {
        this.metrics.slowQueries++;
        console.log(JSON.stringify({
          level: 'warn',
          msg: 'slow_query',
          duration_ms: Math.round(duration),
          sql: sql.substring(0, 200),
          rows: result.rowCount,
        }));
      }
 
      return result;
    } catch (error) {
      this.metrics.failedQueries++;
      const duration = performance.now() - start;
      console.error(JSON.stringify({
        level: 'error',
        msg: 'query_error',
        duration_ms: Math.round(duration),
        sql: sql.substring(0, 200),
        error: (error as Error).message,
      }));
      throw error;
    }
  }
 
  // Transaction support with automatic rollback
  async transaction<T>(
    fn: (client: PoolClient) => Promise<T>
  ): Promise<T> {
    const client = await this.pool.connect();
    try {
      await client.query('BEGIN');
      const result = await fn(client);
      await client.query('COMMIT');
      return result;
    } catch (error) {
      await client.query('ROLLBACK');
      throw error;
    } finally {
      client.release();
    }
  }
 
  getMetrics(): PoolMetrics {
    return {
      ...this.metrics,
      averageQueryTime: this.calculateAvgQueryTime(),
    };
  }
 
  getPoolStats() {
    return {
      total: this.pool.totalCount,
      idle: this.pool.idleCount,
      waiting: this.pool.waitingCount,
      active: this.pool.totalCount - this.pool.idleCount,
    };
  }
 
  private recordQueryTime(duration: number) {
    this.queryTimes.push(duration);
    if (this.queryTimes.length > this.MAX_QUERY_TIME_SAMPLES) {
      this.queryTimes.shift();
    }
  }
 
  private calculateAvgQueryTime(): number {
    if (this.queryTimes.length === 0) return 0;
    const sum = this.queryTimes.reduce((a, b) => a + b, 0);
    return Math.round(sum / this.queryTimes.length);
  }
 
  async healthCheck(): Promise<boolean> {
    try {
      const result = await this.pool.query('SELECT 1 as health');
      return result.rows[0]?.health === 1;
    } catch {
      return false;
    }
  }
 
  async shutdown(): Promise<void> {
    await this.pool.end();
  }
}
 
export const db = new DatabasePool({
  host: process.env.DB_HOST!,
  port: Number(process.env.DB_PORT) || 5432,
  database: process.env.DB_NAME!,
  user: process.env.DB_USER!,
  password: process.env.DB_PASSWORD!,
  maxConnections: Number(process.env.DB_MAX_CONNECTIONS) || 20,
});

// THE N+1 PROBLEM: What most APIs actually do
 
// A request for "list posts with authors" generates:
// 1 query: SELECT * FROM posts LIMIT 20
// 20 queries: SELECT * FROM users WHERE id = ?  (one per post)
// Total: 21 queries for 20 posts
 
// BAD: The ORM makes this look innocent
async function getPostsWithAuthors() {
  const posts = await Post.findAll({ limit: 20 });
  // This triggers 20 additional queries behind the scenes
  for (const post of posts) {
    post.author = await User.findByPk(post.authorId);
  }
  return posts;
}
 
// GOOD: One query with a JOIN
async function getPostsWithAuthors(db: DatabasePool) {
  const { rows } = await db.query(`
    SELECT
      p.id,
      p.title,
      p.excerpt,
      p.created_at,
      json_build_object(
        'id', u.id,
        'name', u.name,
        'avatar_url', u.avatar_url
      ) as author
    FROM posts p
    INNER JOIN users u ON u.id = p.author_id
    ORDER BY p.created_at DESC
    LIMIT 20
  `);
  return rows;
}
 
// EVEN BETTER: Batch loading with DataLoader pattern
// This is what Facebook built for GraphQL, and it works for REST too
 
class BatchLoader<K, V> {
  private batch: Map<K, {
    resolve: (value: V | null) => void;
    reject: (error: Error) => void;
  }[]> = new Map();
  private timer: ReturnType<typeof setTimeout> | null = null;
 
  constructor(
    private loadFn: (keys: K[]) => Promise<Map<K, V>>
  ) {}
 
  async load(key: K): Promise<V | null> {
    return new Promise((resolve, reject) => {
      const existing = this.batch.get(key) || [];
      existing.push({ resolve, reject });
      this.batch.set(key, existing);
 
      if (!this.timer) {
        // Batch all loads within the same tick
        this.timer = setTimeout(() => this.executeBatch(), 0);
      }
    });
  }
 
  private async executeBatch() {
    const batch = this.batch;
    this.batch = new Map();
    this.timer = null;
 
    const keys = Array.from(batch.keys());
    try {
      const results = await this.loadFn(keys);
      for (const [key, callbacks] of batch) {
        const value = results.get(key) ?? null;
        callbacks.forEach(cb => cb.resolve(value));
      }
    } catch (error) {
      for (const callbacks of batch.values()) {
        callbacks.forEach(cb => cb.reject(error as Error));
      }
    }
  }
}
 
// Usage
const userLoader = new BatchLoader<string, User>(async (ids) => {
  const { rows } = await db.query(
    `SELECT * FROM users WHERE id = ANY($1)`,
    [ids]
  );
  return new Map(rows.map(u => [u.id, u]));
});
 
// Now 20 calls to userLoader.load(id) result in 1 query
// SELECT * FROM users WHERE id = ANY('{id1,id2,...,id20}')

-- INDEXING STRATEGY FOR A SOCIAL MEDIA APP AT SCALE
 
-- Rule 1: Index what you filter, join, and order by
-- But ONLY those columns, and in the right order
 
-- Posts table
-- Primary query: "Get recent posts by user"
CREATE INDEX idx_posts_user_created
  ON posts (user_id, created_at DESC);
-- This covers: WHERE user_id = X ORDER BY created_at DESC
-- The column order matters! user_id first because it's the equality filter
 
-- Rule 2: Partial indexes for common filters
-- 90% of queries only want published posts
CREATE INDEX idx_posts_published_recent
  ON posts (created_at DESC)
  WHERE status = 'published';
-- This index is smaller and faster than indexing all posts
 
-- Rule 3: Covering indexes to avoid table lookups
-- If a query can be satisfied entirely from the index, PostgreSQL
-- never touches the heap (the actual table data)
CREATE INDEX idx_posts_feed_covering
  ON posts (user_id, created_at DESC)
  INCLUDE (title, excerpt, like_count);
-- Now "get feed items" never reads the table
 
-- Rule 4: Expression indexes for computed filters
-- You search by lowercase email everywhere
CREATE UNIQUE INDEX idx_users_email_lower
  ON users (LOWER(email));
-- Make sure your queries use LOWER(email) too
 
-- Rule 5: GIN indexes for full-text search and JSONB
CREATE INDEX idx_posts_search
  ON posts USING gin(to_tsvector('english', title || ' ' || body));
-- For JSONB columns
CREATE INDEX idx_user_preferences
  ON users USING gin(preferences jsonb_path_ops);
 
-- ANTI-PATTERNS: Indexes that hurt more than they help
 
-- BAD: Indexing boolean columns with low selectivity
-- CREATE INDEX idx_posts_active ON posts (is_active);
-- If 95% of posts are active, this index is useless
-- PostgreSQL will do a seq scan anyway because the index isn't selective
 
-- BAD: Too many indexes on a write-heavy table
-- Every INSERT/UPDATE/DELETE must update ALL indexes
-- 10 indexes on a table that gets 1000 writes/second = 10,000 index updates/second
 
-- DIAGNOSTIC: Find unused indexes (run periodically)
SELECT
  schemaname,
  relname AS table_name,
  indexrelname AS index_name,
  pg_size_pretty(pg_relation_size(indexrelid)) AS index_size,
  idx_scan AS times_used
FROM pg_stat_user_indexes
WHERE idx_scan = 0
  AND indexrelname NOT LIKE '%_pkey'
  AND indexrelname NOT LIKE '%_unique'
ORDER BY pg_relation_size(indexrelid) DESC;
 
-- DIAGNOSTIC: Find missing indexes (queries doing seq scans)
SELECT
  relname AS table_name,
  seq_scan,
  seq_tup_read,
  idx_scan,
  CASE WHEN seq_scan > 0
    THEN seq_tup_read / seq_scan
    ELSE 0
  END AS avg_rows_per_scan
FROM pg_stat_user_tables
WHERE seq_scan > 100
ORDER BY seq_tup_read DESC
LIMIT 20;

-- Time-based partitioning for an events/analytics table
-- This is the most common and most useful partitioning strategy
 
CREATE TABLE events (
  id          bigserial,
  user_id     uuid NOT NULL,
  event_type  text NOT NULL,
  payload     jsonb,
  created_at  timestamptz NOT NULL DEFAULT now()
) PARTITION BY RANGE (created_at);
 
-- Create monthly partitions
CREATE TABLE events_2026_01 PARTITION OF events
  FOR VALUES FROM ('2026-01-01') TO ('2026-02-01');
CREATE TABLE events_2026_02 PARTITION OF events
  FOR VALUES FROM ('2026-02-01') TO ('2026-03-01');
CREATE TABLE events_2026_03 PARTITION OF events
  FOR VALUES FROM ('2026-03-01') TO ('2026-04-01');
 
-- Indexes on partitions (inherited automatically since PG 11)
CREATE INDEX ON events (user_id, created_at DESC);
CREATE INDEX ON events (event_type, created_at DESC);
 
-- Auto-create future partitions with a cron job
-- This is the script you run monthly:
DO $$
DECLARE
  partition_date date;
  partition_name text;
  start_date text;
  end_date text;
BEGIN
  -- Create partitions for the next 3 months
  FOR i IN 0..2 LOOP
    partition_date := date_trunc('month', now() + (i || ' months')::interval);
    partition_name := 'events_' || to_char(partition_date, 'YYYY_MM');
    start_date := to_char(partition_date, 'YYYY-MM-DD');
    end_date := to_char(partition_date + '1 month'::interval, 'YYYY-MM-DD');
 
    IF NOT EXISTS (
      SELECT 1 FROM pg_class WHERE relname = partition_name
    ) THEN
      EXECUTE format(
        'CREATE TABLE %I PARTITION OF events FOR VALUES FROM (%L) TO (%L)',
        partition_name, start_date, end_date
      );
      RAISE NOTICE 'Created partition %', partition_name;
    END IF;
  END LOOP;
END $$;
 
-- Dropping old data is now instant — just drop the partition
-- Instead of: DELETE FROM events WHERE created_at < '2025-01-01'
-- (which locks the table and takes hours)
-- Do this:
DROP TABLE events_2025_01;
-- Instant. No locks. No vacuum needed.

-- Dashboard analytics that would take 30 seconds to compute live
CREATE MATERIALIZED VIEW mv_daily_stats AS
SELECT
  date_trunc('day', e.created_at) AS day,
  COUNT(DISTINCT e.user_id) AS unique_users,
  COUNT(*) AS total_events,
  COUNT(*) FILTER (WHERE e.event_type = 'purchase') AS purchases,
  COALESCE(SUM(
    CASE WHEN e.event_type = 'purchase'
    THEN (e.payload->>'amount')::numeric
    ELSE 0 END
  ), 0) AS revenue,
  COUNT(*) FILTER (WHERE e.event_type = 'signup') AS signups,
  ROUND(
    COUNT(*) FILTER (WHERE e.event_type = 'purchase')::numeric /
    NULLIF(COUNT(DISTINCT e.user_id), 0) * 100, 2
  ) AS conversion_rate
FROM events e
WHERE e.created_at >= now() - INTERVAL '90 days'
GROUP BY date_trunc('day', e.created_at)
ORDER BY day DESC
WITH DATA;
 
-- Unique index enables CONCURRENTLY refresh (no locks)
CREATE UNIQUE INDEX ON mv_daily_stats (day);
 
-- Refresh without blocking readers
-- Run this via cron every 15 minutes
REFRESH MATERIALIZED VIEW CONCURRENTLY mv_daily_stats;
 
-- Now your dashboard query goes from 30 seconds to 5ms:
SELECT * FROM mv_daily_stats WHERE day >= now() - INTERVAL '30 days';

// Automated materialized view refresh with error handling
 
interface MaterializedViewConfig {
  name: string;
  refreshIntervalMs: number;
  concurrent: boolean;
}
 
class MaterializedViewRefresher {
  private timers: Map<string, ReturnType<typeof setInterval>> = new Map();
 
  constructor(
    private db: DatabasePool,
    private views: MaterializedViewConfig[]
  ) {}
 
  start() {
    for (const view of this.views) {
      const timer = setInterval(
        () => this.refresh(view),
        view.refreshIntervalMs
      );
      this.timers.set(view.name, timer);
 
      // Also refresh immediately on start
      this.refresh(view).catch(() => {});
    }
  }
 
  private async refresh(view: MaterializedViewConfig) {
    const start = performance.now();
    const concurrent = view.concurrent ? 'CONCURRENTLY' : '';
 
    try {
      await this.db.query(
        `REFRESH MATERIALIZED VIEW ${concurrent} ${view.name}`
      );
      const duration = performance.now() - start;
      console.log(JSON.stringify({
        level: 'info',
        msg: 'mv_refreshed',
        view: view.name,
        duration_ms: Math.round(duration),
      }));
    } catch (error) {
      const duration = performance.now() - start;
      console.error(JSON.stringify({
        level: 'error',
        msg: 'mv_refresh_failed',
        view: view.name,
        duration_ms: Math.round(duration),
        error: (error as Error).message,
      }));
    }
  }
 
  stop() {
    for (const timer of this.timers.values()) {
      clearInterval(timer);
    }
    this.timers.clear();
  }
}
 
// Usage
const refresher = new MaterializedViewRefresher(db, [
  { name: 'mv_daily_stats', refreshIntervalMs: 15 * 60 * 1000, concurrent: true },
  { name: 'mv_user_activity', refreshIntervalMs: 5 * 60 * 1000, concurrent: true },
  { name: 'mv_popular_posts', refreshIntervalMs: 60 * 1000, concurrent: true },
]);
 
refresher.start();

import Redis from 'ioredis';
 
const redis = new Redis({
  host: process.env.REDIS_HOST,
  port: Number(process.env.REDIS_PORT) || 6379,
  password: process.env.REDIS_PASSWORD,
  maxRetriesPerRequest: 3,
  retryStrategy(times) {
    const delay = Math.min(times * 50, 2000);
    return delay;
  },
  // THIS IS CRITICAL: enable lazy connect so the app starts
  // even if Redis is down
  lazyConnect: true,
  enableReadyCheck: true,
});
 
// The naive approach everyone starts with
async function getUserNaive(id: string): Promise<User | null> {
  const cached = await redis.get(`user:${id}`);
  if (cached) return JSON.parse(cached);
 
  const { rows } = await db.query(
    'SELECT * FROM users WHERE id = $1', [id]
  );
  if (rows[0]) {
    await redis.set(`user:${id}`, JSON.stringify(rows[0]), 'EX', 3600);
  }
  return rows[0] || null;
}
 
// What is wrong with the naive approach:
// 1. No protection against cache stampede
// 2. No circuit breaker when Redis is down
// 3. No serialization error handling
// 4. No cache warming
// 5. Stale data served indefinitely if invalidation fails
 
// The production version:
 
interface CacheOptions {
  ttlSeconds: number;
  staleWhileRevalidateSeconds?: number;
  lockTimeoutMs?: number;
}
 
class CacheManager {
  private redis: Redis;
  private localCache: Map<string, { value: string; expiry: number }> = new Map();
  private readonly LOCAL_TTL_MS = 5000; // L1 cache: 5 seconds
  private redisHealthy = true;
 
  constructor(redis: Redis) {
    this.redis = redis;
 
    // Health monitoring
    this.redis.on('error', () => { this.redisHealthy = false; });
    this.redis.on('ready', () => { this.redisHealthy = true; });
 
    // Clean expired local cache entries every 30 seconds
    setInterval(() => this.cleanLocalCache(), 30000);
  }
 
  async get<T>(
    key: string,
    loader: () => Promise<T>,
    options: CacheOptions
  ): Promise<T> {
    // L1: Check local in-memory cache first (no network call)
    const local = this.localCache.get(key);
    if (local && local.expiry > Date.now()) {
      return JSON.parse(local.value) as T;
    }
 
    // L2: Check Redis
    if (this.redisHealthy) {
      try {
        const cached = await this.redis.get(key);
        if (cached !== null) {
          // Populate L1
          this.localCache.set(key, {
            value: cached,
            expiry: Date.now() + this.LOCAL_TTL_MS,
          });
 
          // Check if we should revalidate in background
          if (options.staleWhileRevalidateSeconds) {
            const ttl = await this.redis.ttl(key);
            const staleThreshold =
              options.ttlSeconds - options.staleWhileRevalidateSeconds;
            if (ttl > 0 && ttl < staleThreshold) {
              // Serve stale, refresh in background
              this.revalidateInBackground(key, loader, options);
            }
          }
 
          return JSON.parse(cached) as T;
        }
      } catch {
        // Redis failed, fall through to loader
      }
    }
 
    // Cache miss — load from source with stampede protection
    return this.loadWithLock(key, loader, options);
  }
 
  private async loadWithLock<T>(
    key: string,
    loader: () => Promise<T>,
    options: CacheOptions
  ): Promise<T> {
    const lockKey = `lock:${key}`;
    const lockTimeout = options.lockTimeoutMs || 5000;
 
    if (this.redisHealthy) {
      // Try to acquire lock (only one process loads from DB)
      const acquired = await this.redis.set(
        lockKey, '1', 'PX', lockTimeout, 'NX'
      );
 
      if (!acquired) {
        // Another process is loading — wait and retry from cache
        await new Promise(resolve => setTimeout(resolve, 100));
        const cached = await this.redis.get(key);
        if (cached) return JSON.parse(cached) as T;
        // Still no cache — load anyway (lock holder might have failed)
      }
    }
 
    try {
      const value = await loader();
      const serialized = JSON.stringify(value);
 
      // Populate both cache layers
      this.localCache.set(key, {
        value: serialized,
        expiry: Date.now() + this.LOCAL_TTL_MS,
      });
 
      if (this.redisHealthy) {
        await this.redis.set(key, serialized, 'EX', options.ttlSeconds);
      }
 
      return value;
    } finally {
      if (this.redisHealthy) {
        await this.redis.del(lockKey).catch(() => {});
      }
    }
  }
 
  private async revalidateInBackground<T>(
    key: string,
    loader: () => Promise<T>,
    options: CacheOptions
  ) {
    // Fire and forget — do not await
    this.loadWithLock(key, loader, options).catch((error) => {
      console.error(JSON.stringify({
        level: 'error',
        msg: 'background_revalidation_failed',
        key,
        error: (error as Error).message,
      }));
    });
  }
 
  async invalidate(...keys: string[]) {
    // Invalidate both layers
    for (const key of keys) {
      this.localCache.delete(key);
    }
    if (this.redisHealthy && keys.length > 0) {
      await this.redis.del(...keys);
    }
  }
 
  async invalidatePattern(pattern: string) {
    // Invalidate local cache entries matching pattern
    const regex = new RegExp(
      '^' + pattern.replace(/\*/g, '.*') + '$'
    );
    for (const key of this.localCache.keys()) {
      if (regex.test(key)) this.localCache.delete(key);
    }
 
    // Invalidate Redis entries
    if (this.redisHealthy) {
      let cursor = '0';
      do {
        const [nextCursor, keys] = await this.redis.scan(
          cursor, 'MATCH', pattern, 'COUNT', 100
        );
        cursor = nextCursor;
        if (keys.length > 0) {
          await this.redis.del(...keys);
        }
      } while (cursor !== '0');
    }
  }
 
  private cleanLocalCache() {
    const now = Date.now();
    for (const [key, entry] of this.localCache) {
      if (entry.expiry < now) {
        this.localCache.delete(key);
      }
    }
  }
}
 
// Usage
const cache = new CacheManager(redis);
 
async function getUser(id: string): Promise<User | null> {
  return cache.get(
    `user:${id}`,
    async () => {
      const { rows } = await db.query(
        'SELECT * FROM users WHERE id = $1', [id]
      );
      return rows[0] || null;
    },
    {
      ttlSeconds: 3600,
      staleWhileRevalidateSeconds: 300,
      lockTimeoutMs: 5000,
    }
  );
}
 
// On user update, invalidate
async function updateUser(id: string, data: Partial<User>) {
  await db.query(
    'UPDATE users SET name = $2, email = $3 WHERE id = $1',
    [id, data.name, data.email]
  );
  await cache.invalidate(`user:${id}`);
  // Also invalidate any list caches that might include this user
  await cache.invalidatePattern(`user-list:*`);
}

// The full caching stack, from fastest to slowest:
//
// L0: CDN Edge Cache     — 50ms globally, 0 server load
// L1: In-Memory Cache    — <1ms, per-process, small capacity
// L2: Redis Cache        — 1-5ms, shared across processes, large capacity
// L3: Database           — 5-50ms, persistent, source of truth
//
// Strategy: Popular data should "float up" to faster layers
 
interface MultiLayerCacheConfig {
  key: string;
  l1TtlMs: number;         // In-memory TTL
  l2TtlSeconds: number;     // Redis TTL
  cdnMaxAge?: number;       // CDN cache-control max-age
  cdnStaleWhileRevalidate?: number;
}
 
// HTTP response with appropriate cache headers for each layer
function setCacheHeaders(
  res: Response,
  config: { isPublic: boolean; maxAge: number; staleWhileRevalidate?: number }
) {
  const directives: string[] = [];
 
  if (config.isPublic) {
    directives.push('public');
  } else {
    directives.push('private');
  }
 
  directives.push(`max-age=${config.maxAge}`);
 
  if (config.staleWhileRevalidate) {
    directives.push(`stale-while-revalidate=${config.staleWhileRevalidate}`);
  }
 
  res.setHeader('Cache-Control', directives.join(', '));
 
  // Vary header prevents cache poisoning
  res.setHeader('Vary', 'Accept-Encoding, Accept-Language');
}
 
// Example: Product listing page
// CDN: 60s max-age, 300s stale-while-revalidate
// Redis: 300s TTL
// DB: Fallback source of truth
 
async function getProducts(req: Request, res: Response) {
  const page = Number(req.query.page) || 1;
  const category = req.query.category as string;
  const cacheKey = `products:${category}:page${page}`;
 
  const products = await cache.get(
    cacheKey,
    async () => {
      const { rows } = await db.query(
        `SELECT id, name, price, image_url, rating
         FROM products
         WHERE category = $1 AND active = true
         ORDER BY popularity DESC
         LIMIT 20 OFFSET $2`,
        [category, (page - 1) * 20]
      );
      return rows;
    },
    { ttlSeconds: 300, staleWhileRevalidateSeconds: 60 }
  );
 
  // Tell CDN and browser how to cache this response
  setCacheHeaders(res, {
    isPublic: true,      // no user-specific data
    maxAge: 60,          // CDN serves cached version for 60s
    staleWhileRevalidate: 300,  // serve stale for up to 300s while refreshing
  });
 
  res.json(products);
}
 
// Example: User profile (private, no CDN)
async function getUserProfile(req: Request, res: Response) {
  const userId = req.user.id;
 
  const profile = await cache.get(
    `profile:${userId}`,
    async () => {
      const { rows } = await db.query(
        `SELECT u.*, COUNT(p.id) as post_count
         FROM users u
         LEFT JOIN posts p ON p.user_id = u.id
         WHERE u.id = $1
         GROUP BY u.id`,
        [userId]
      );
      return rows[0];
    },
    { ttlSeconds: 600, staleWhileRevalidateSeconds: 120 }
  );
 
  setCacheHeaders(res, {
    isPublic: false,      // user-specific data — no CDN
    maxAge: 0,            // browser always revalidates
  });
 
  res.json(profile);
}

// Pattern 1: Event-driven invalidation
// When data changes, publish an event, and cache subscribers handle it
 
import { EventEmitter } from 'events';
 
class CacheInvalidationBus {
  private emitter = new EventEmitter();
 
  // Register invalidation rules
  constructor(private cache: CacheManager) {
    // When a user changes, invalidate user cache and any derived caches
    this.on('user:updated', async (userId: string) => {
      await cache.invalidate(`user:${userId}`, `profile:${userId}`);
      await cache.invalidatePattern(`user-list:*`);
      await cache.invalidatePattern(`leaderboard:*`);
    });
 
    // When a post changes, invalidate the post and related feeds
    this.on('post:updated', async (postId: string, authorId: string) => {
      await cache.invalidate(`post:${postId}`);
      await cache.invalidatePattern(`feed:*`);
      await cache.invalidatePattern(`posts:author:${authorId}:*`);
    });
 
    // When a comment is added, invalidate the post's comment count
    this.on('comment:created', async (postId: string) => {
      await cache.invalidate(`post:${postId}`, `comments:${postId}`);
    });
  }
 
  private on(event: string, handler: (...args: string[]) => Promise<void>) {
    this.emitter.on(event, (...args: string[]) => {
      handler(...args).catch(err => {
        console.error(JSON.stringify({
          level: 'error',
          msg: 'cache_invalidation_failed',
          event,
          error: (err as Error).message,
        }));
      });
    });
  }
 
  emit(event: string, ...args: string[]) {
    this.emitter.emit(event, ...args);
  }
}
 
const invalidation = new CacheInvalidationBus(cache);
 
// In your update handlers:
async function updateUserHandler(req: Request, res: Response) {
  const userId = req.params.id;
  await db.query(
    'UPDATE users SET name = $2 WHERE id = $1',
    [userId, req.body.name]
  );
  invalidation.emit('user:updated', userId);
  res.json({ success: true });
}
 
// Pattern 2: Versioned cache keys
// Instead of invalidating, bump a version number
// This is simpler and avoids the "did invalidation succeed?" problem
 
class VersionedCache {
  constructor(
    private redis: Redis,
    private cacheManager: CacheManager
  ) {}
 
  private versionKey(entity: string, id: string): string {
    return `v:${entity}:${id}`;
  }
 
  private cacheKey(entity: string, id: string, version: number): string {
    return `${entity}:${id}:v${version}`;
  }
 
  async get<T>(
    entity: string,
    id: string,
    loader: () => Promise<T>,
    ttlSeconds: number
  ): Promise<T> {
    // Get current version
    const version = Number(await this.redis.get(this.versionKey(entity, id))) || 1;
    const key = this.cacheKey(entity, id, version);
 
    return this.cacheManager.get(key, loader, { ttlSeconds });
  }
 
  async invalidate(entity: string, id: string): Promise<void> {
    // Just bump the version — old cache entries expire naturally
    await this.redis.incr(this.versionKey(entity, id));
    // No need to find and delete old entries!
  }
}
 
const versionedCache = new VersionedCache(redis, cache);
 
// Usage — invalidation is now just an INCR, never fails
async function getUserVersioned(id: string): Promise<User | null> {
  return versionedCache.get('user', id, async () => {
    const { rows } = await db.query(
      'SELECT * FROM users WHERE id = $1', [id]
    );
    return rows[0] || null;
  }, 3600);
}
 
async function updateUserVersioned(id: string, data: Partial<User>) {
  await db.query('UPDATE users SET name = $2 WHERE id = $1', [id, data.name]);
  await versionedCache.invalidate('user', id);
}

# /etc/nginx/nginx.conf
# Production load balancer configuration
 
# Worker processes = CPU cores
worker_processes auto;
worker_rlimit_nofile 65535;
 
events {
    worker_connections 65535;
    multi_accept on;
    use epoll;
}
 
http {
    # Basic settings
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    keepalive_requests 1000;
 
    # Buffer sizes
    client_body_buffer_size 16k;
    client_header_buffer_size 1k;
    client_max_body_size 50m;
    large_client_header_buffers 4 8k;
 
    # Gzip compression
    gzip on;
    gzip_vary on;
    gzip_proxied any;
    gzip_comp_level 5;  # 5 is the sweet spot (6+ uses too much CPU)
    gzip_min_length 256;
    gzip_types
        text/plain
        text/css
        text/javascript
        application/json
        application/javascript
        application/xml
        image/svg+xml;
 
    # Rate limiting
    limit_req_zone $binary_remote_addr zone=api:10m rate=30r/s;
    limit_req_zone $binary_remote_addr zone=login:10m rate=5r/m;
 
    # Connection limiting
    limit_conn_zone $binary_remote_addr zone=addr:10m;
 
    # Upstream (your app servers)
    upstream app_servers {
        # Least connections — best for varying response times
        least_conn;
 
        # App servers with health checks
        server 10.0.1.10:3000 max_fails=3 fail_timeout=30s;
        server 10.0.1.11:3000 max_fails=3 fail_timeout=30s;
        server 10.0.1.12:3000 max_fails=3 fail_timeout=30s;
 
        # Keep connections alive to app servers
        keepalive 32;
    }
 
    # WebSocket upstream (if needed)
    upstream ws_servers {
        # IP hash for sticky sessions (WebSocket requires it)
        ip_hash;
 
        server 10.0.1.10:3001;
        server 10.0.1.11:3001;
        server 10.0.1.12:3001;
    }
 
    server {
        listen 80;
        listen 443 ssl http2;
        server_name example.com;
 
        # SSL (use certbot to generate)
        ssl_certificate /etc/ssl/certs/example.com.pem;
        ssl_certificate_key /etc/ssl/private/example.com.key;
        ssl_protocols TLSv1.2 TLSv1.3;
        ssl_prefer_server_ciphers on;
 
        # Security headers
        add_header X-Frame-Options DENY always;
        add_header X-Content-Type-Options nosniff always;
        add_header X-XSS-Protection "1; mode=block" always;
        add_header Strict-Transport-Security "max-age=63072000" always;
 
        # Static assets — served directly by Nginx
        location /_next/static/ {
            alias /var/www/app/.next/static/;
            expires 365d;
            add_header Cache-Control "public, immutable";
        }
 
        location /static/ {
            alias /var/www/app/public/static/;
            expires 30d;
            add_header Cache-Control "public";
        }
 
        # API routes with rate limiting
        location /api/ {
            limit_req zone=api burst=50 nodelay;
            limit_conn addr 30;
 
            proxy_pass http://app_servers;
            proxy_http_version 1.1;
            proxy_set_header Connection "";  # Enable keepalive
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
 
            # Timeouts
            proxy_connect_timeout 5s;
            proxy_send_timeout 30s;
            proxy_read_timeout 30s;
        }
 
        # Stricter rate limit for auth endpoints
        location /api/auth/ {
            limit_req zone=login burst=3 nodelay;
 
            proxy_pass http://app_servers;
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
        }
 
        # WebSocket
        location /ws/ {
            proxy_pass http://ws_servers;
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_read_timeout 3600s;  # 1 hour for WebSocket
            proxy_send_timeout 3600s;
        }
 
        # Health check endpoint (no rate limit, no logging)
        location /api/health {
            access_log off;
            proxy_pass http://app_servers;
            proxy_http_version 1.1;
            proxy_set_header Connection "";
        }
 
        # Everything else
        location / {
            proxy_pass http://app_servers;
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
        }
    }
}

// Graceful shutdown handler — use this in EVERY production Node.js app
 
import http from 'http';
 
interface ShutdownConfig {
  server: http.Server;
  db: DatabasePool;
  redis: Redis;
  gracePeriodMs: number; // How long to wait for in-flight requests
}
 
function setupGracefulShutdown(config: ShutdownConfig) {
  let isShuttingDown = false;
  let activeRequests = 0;
 
  // Track active requests
  config.server.on('request', (_req, res) => {
    if (isShuttingDown) {
      res.writeHead(503, { 'Retry-After': '30' });
      res.end('Server is shutting down');
      return;
    }
    activeRequests++;
    res.on('finish', () => { activeRequests--; });
  });
 
  async function shutdown(signal: string) {
    if (isShuttingDown) return;
    isShuttingDown = true;
 
    console.log(JSON.stringify({
      level: 'info',
      msg: 'shutdown_initiated',
      signal,
      activeRequests,
    }));
 
    // Step 1: Stop accepting new connections
    config.server.close();
 
    // Step 2: Wait for in-flight requests to complete
    const deadline = Date.now() + config.gracePeriodMs;
 
    while (activeRequests > 0 && Date.now() < deadline) {
      await new Promise(resolve => setTimeout(resolve, 100));
    }
 
    if (activeRequests > 0) {
      console.log(JSON.stringify({
        level: 'warn',
        msg: 'forced_shutdown',
        remainingRequests: activeRequests,
      }));
    }
 
    // Step 3: Close database connections
    try {
      await config.db.shutdown();
    } catch (error) {
      console.error('DB shutdown error:', (error as Error).message);
    }
 
    // Step 4: Close Redis connections
    try {
      config.redis.disconnect();
    } catch (error) {
      console.error('Redis shutdown error:', (error as Error).message);
    }
 
    console.log(JSON.stringify({
      level: 'info',
      msg: 'shutdown_complete',
    }));
 
    process.exit(0);
  }
 
  process.on('SIGTERM', () => shutdown('SIGTERM'));
  process.on('SIGINT', () => shutdown('SIGINT'));
 
  // Prevent crashes from taking down the process without cleanup
  process.on('uncaughtException', (error) => {
    console.error(JSON.stringify({
      level: 'fatal',
      msg: 'uncaught_exception',
      error: error.message,
      stack: error.stack,
    }));
    shutdown('uncaughtException');
  });
 
  process.on('unhandledRejection', (reason) => {
    console.error(JSON.stringify({
      level: 'fatal',
      msg: 'unhandled_rejection',
      reason: String(reason),
    }));
    // Don't shutdown on unhandled rejections — log and continue
    // But DO fix them. Every unhandled rejection is a bug.
  });
}
 
// Usage
const server = app.listen(3000, () => {
  console.log('Server started on port 3000');
});
 
setupGracefulShutdown({
  server,
  db,
  redis,
  gracePeriodMs: 30000, // 30 seconds to finish in-flight requests
});

// Health check endpoint with dependency checking
 
interface HealthStatus {
  status: 'healthy' | 'degraded' | 'unhealthy';
  checks: Record<string, {
    status: 'pass' | 'fail';
    latency_ms: number;
    message?: string;
  }>;
  uptime_seconds: number;
  version: string;
}
 
const startTime = Date.now();
 
async function healthCheck(): Promise<HealthStatus> {
  const checks: HealthStatus['checks'] = {};
 
  // Check database
  const dbStart = performance.now();
  try {
    await db.query('SELECT 1');
    checks.database = {
      status: 'pass',
      latency_ms: Math.round(performance.now() - dbStart),
    };
  } catch (error) {
    checks.database = {
      status: 'fail',
      latency_ms: Math.round(performance.now() - dbStart),
      message: (error as Error).message,
    };
  }
 
  // Check Redis
  const redisStart = performance.now();
  try {
    await redis.ping();
    checks.redis = {
      status: 'pass',
      latency_ms: Math.round(performance.now() - redisStart),
    };
  } catch (error) {
    checks.redis = {
      status: 'fail',
      latency_ms: Math.round(performance.now() - redisStart),
      message: (error as Error).message,
    };
  }
 
  // Check disk space (for apps that write to disk)
  // Check memory usage
  const memUsage = process.memoryUsage();
  const heapUsedPct = (memUsage.heapUsed / memUsage.heapTotal) * 100;
  checks.memory = {
    status: heapUsedPct < 90 ? 'pass' : 'fail',
    latency_ms: 0,
    message: `Heap: ${Math.round(heapUsedPct)}% (${Math.round(memUsage.heapUsed / 1024 / 1024)}MB)`,
  };
 
  // Determine overall status
  const allPassing = Object.values(checks).every(c => c.status === 'pass');
  const anyFailing = Object.values(checks).some(c => c.status === 'fail');
 
  // Database down = unhealthy (remove from rotation)
  // Redis down = degraded (keep serving, but warn)
  let status: HealthStatus['status'] = 'healthy';
  if (checks.database?.status === 'fail' || checks.memory?.status === 'fail') {
    status = 'unhealthy';
  } else if (!allPassing) {
    status = 'degraded';
  }
 
  return {
    status,
    checks,
    uptime_seconds: Math.round((Date.now() - startTime) / 1000),
    version: process.env.APP_VERSION || 'unknown',
  };
}
 
// Endpoint
app.get('/api/health', async (_req, res) => {
  const health = await healthCheck();
 
  const statusCode = health.status === 'unhealthy' ? 503 : 200;
  res.status(statusCode).json(health);
});
 
// Lightweight liveness probe (for Kubernetes)
// This should ALWAYS return 200 unless the process is completely broken
app.get('/api/health/live', (_req, res) => {
  res.status(200).json({ status: 'alive' });
});
 
// Readiness probe (for Kubernetes)
// Returns 503 during startup and shutdown
app.get('/api/health/ready', async (_req, res) => {
  if (isShuttingDown) {
    return res.status(503).json({ status: 'shutting_down' });
  }
 
  // Check critical dependencies
  try {
    await db.query('SELECT 1');
    res.status(200).json({ status: 'ready' });
  } catch {
    res.status(503).json({ status: 'not_ready' });
  }
});

// Production job queue with BullMQ
// This is the pattern I use for every application past 1K users
 
import { Queue, Worker, QueueScheduler, Job } from 'bullmq';
import { Redis } from 'ioredis';
 
// Shared Redis connection for all queues
const connection = new Redis({
  host: process.env.REDIS_HOST,
  port: Number(process.env.REDIS_PORT) || 6379,
  password: process.env.REDIS_PASSWORD,
  maxRetriesPerRequest: null, // Required by BullMQ
});
 
// Define job types
interface JobTypes {
  'email:welcome': { userId: string; email: string };
  'email:password-reset': { userId: string; token: string };
  'image:resize': { imageId: string; sizes: number[] };
  'report:generate': { userId: string; reportType: string; dateRange: [string, string] };
  'analytics:process': { eventBatch: Array<{ type: string; data: unknown }> };
  'webhook:deliver': { url: string; payload: unknown; attemptNumber: number };
}
 
type JobName = keyof JobTypes;
 
// Queue factory
function createQueue(name: string) {
  return new Queue(name, {
    connection,
    defaultJobOptions: {
      removeOnComplete: {
        age: 24 * 3600,  // Keep completed jobs for 24 hours
        count: 1000,      // Keep last 1000 completed jobs
      },
      removeOnFail: {
        age: 7 * 24 * 3600, // Keep failed jobs for 7 days
      },
      attempts: 3,
      backoff: {
        type: 'exponential',
        delay: 1000, // 1s, 2s, 4s
      },
    },
  });
}
 
// Create queues with different priorities
const emailQueue = createQueue('email');
const imageQueue = createQueue('image-processing');
const reportQueue = createQueue('report-generation');
const analyticsQueue = createQueue('analytics');
const webhookQueue = createQueue('webhook-delivery');
 
// Type-safe job enqueue
async function enqueueJob<T extends JobName>(
  name: T,
  data: JobTypes[T],
  options?: { priority?: number; delay?: number; deduplicationId?: string }
): Promise<string> {
  const queueMap: Record<string, Queue> = {
    'email:welcome': emailQueue,
    'email:password-reset': emailQueue,
    'image:resize': imageQueue,
    'report:generate': reportQueue,
    'analytics:process': analyticsQueue,
    'webhook:deliver': webhookQueue,
  };
 
  const queue = queueMap[name];
  if (!queue) throw new Error(`No queue for job type: ${name}`);
 
  // Deduplication — prevent the same job from being enqueued twice
  const jobId = options?.deduplicationId || undefined;
 
  const job = await queue.add(name, data, {
    priority: options?.priority || 0,
    delay: options?.delay || 0,
    jobId,
  });
 
  console.log(JSON.stringify({
    level: 'info',
    msg: 'job_enqueued',
    queue: queue.name,
    jobName: name,
    jobId: job.id,
  }));
 
  return job.id!;
}
 
// Example: Webhook delivery with exponential backoff and dead letter queue
const webhookWorker = new Worker(
  'webhook-delivery',
  async (job: Job) => {
    const { url, payload, attemptNumber } = job.data;
 
    const controller = new AbortController();
    const timeout = setTimeout(() => controller.abort(), 10000); // 10s timeout
 
    try {
      const response = await fetch(url, {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
          'X-Webhook-Signature': computeHmac(JSON.stringify(payload)),
          'X-Webhook-Attempt': String(attemptNumber),
        },
        body: JSON.stringify(payload),
        signal: controller.signal,
      });
 
      if (!response.ok) {
        throw new Error(`Webhook returned ${response.status}: ${response.statusText}`);
      }
 
      console.log(JSON.stringify({
        level: 'info',
        msg: 'webhook_delivered',
        url,
        attempt: attemptNumber,
        status: response.status,
      }));
    } finally {
      clearTimeout(timeout);
    }
  },
  {
    connection,
    concurrency: 10, // Process 10 webhooks in parallel
    limiter: {
      max: 100,      // Max 100 jobs per...
      duration: 1000, // ...second (rate limiting)
    },
  }
);
 
// Handle failed jobs — move to dead letter queue after all retries exhausted
webhookWorker.on('failed', async (job, error) => {
  if (job && job.attemptsMade >= (job.opts.attempts || 3)) {
    console.error(JSON.stringify({
      level: 'error',
      msg: 'webhook_dead_letter',
      jobId: job.id,
      url: job.data.url,
      attempts: job.attemptsMade,
      error: error.message,
    }));
    // Move to dead letter queue for manual review
    await enqueueJob('webhook:deliver', {
      ...job.data,
      attemptNumber: job.attemptsMade,
    }, { priority: 10 }); // low priority in DLQ
  }
});
 
function computeHmac(payload: string): string {
  const crypto = require('crypto');
  return crypto
    .createHmac('sha256', process.env.WEBHOOK_SECRET!)
    .update(payload)
    .digest('hex');
}
 
// Example API handler that returns immediately
app.post('/api/reports', async (req, res) => {
  const { reportType, startDate, endDate } = req.body;
  const userId = req.user.id;
 
  // Enqueue the job — takes < 5ms
  const jobId = await enqueueJob('report:generate', {
    userId,
    reportType,
    dateRange: [startDate, endDate],
  });
 
  // Return immediately — the report generates in the background
  res.status(202).json({
    message: 'Report generation started',
    jobId,
    statusUrl: `/api/reports/status/${jobId}`,
  });
});
 
// Status endpoint for long-running jobs
app.get('/api/reports/status/:jobId', async (req, res) => {
  const job = await reportQueue.getJob(req.params.jobId);
  if (!job) {
    return res.status(404).json({ error: 'Job not found' });
  }
 
  const state = await job.getState();
  const progress = job.progress;
 
  res.json({
    jobId: job.id,
    state,      // 'waiting' | 'active' | 'completed' | 'failed'
    progress,   // 0-100
    result: state === 'completed' ? job.returnvalue : undefined,
    failedReason: state === 'failed' ? job.failedReason : undefined,
  });
});