LLMs in Production Web Apps: Streaming, Caching, Cost Control, and What the Tutorials Skip

// src/lib/ai/provider.ts
import { createOpenAI } from "@ai-sdk/openai";
import { createAnthropic } from "@ai-sdk/anthropic";
 
const providers = {
  openai: createOpenAI({
    apiKey: process.env.OPENAI_API_KEY,
    compatibility: "strict",
  }),
  anthropic: createAnthropic({
    apiKey: process.env.ANTHROPIC_API_KEY,
  }),
};
 
export type ModelConfig = {
  provider: keyof typeof providers;
  model: string;
  maxTokens: number;
  temperature: number;
  fallback?: ModelConfig;
};
 
export const models = {
  fast: {
    provider: "openai" as const,
    model: "gpt-4o-mini",
    maxTokens: 1024,
    temperature: 0.3,
    fallback: {
      provider: "anthropic" as const,
      model: "claude-3-5-haiku-20241022",
      maxTokens: 1024,
      temperature: 0.3,
    },
  },
  quality: {
    provider: "anthropic" as const,
    model: "claude-sonnet-4-20250514",
    maxTokens: 4096,
    temperature: 0.4,
    fallback: {
      provider: "openai" as const,
      model: "gpt-4o",
      maxTokens: 4096,
      temperature: 0.4,
    },
  },
  reasoning: {
    provider: "anthropic" as const,
    model: "claude-opus-4-20250514",
    maxTokens: 8192,
    temperature: 0.2,
  },
} satisfies Record<string, ModelConfig>;

// src/app/api/ai/generate/route.ts
import { streamText } from "ai";
import { getModel } from "@/lib/ai/provider";
import { getPrompt } from "@/lib/ai/prompts";
import { rateLimit } from "@/lib/rate-limit";
import { estimateTokens } from "@/lib/ai/tokens";
 
export const runtime = "nodejs";
export const maxDuration = 60;
 
export async function POST(req: Request) {
  const ip = req.headers.get("x-forwarded-for") ?? "unknown";
  const limiter = await rateLimit(ip, { max: 20, window: 60 });
  if (!limiter.success) {
    return new Response("Rate limit exceeded", {
      status: 429,
      headers: { "Retry-After": String(limiter.retryAfter) },
    });
  }
 
  const { input, feature, context } = await req.json();
 
  // Token budget enforcement BEFORE calling the model
  const estimatedInputTokens = estimateTokens(input + (context ?? ""));
  if (estimatedInputTokens > 8000) {
    return new Response(JSON.stringify({ error: "Input too long", maxTokens: 8000 }), { status: 400 });
  }
 
  const prompt = getPrompt(feature, { input, context });
  const modelConfig = selectModel(feature, estimatedInputTokens);
 
  try {
    const result = streamText({
      model: getModel(modelConfig),
      messages: prompt.messages,
      maxTokens: modelConfig.maxTokens,
      temperature: modelConfig.temperature,
      abortSignal: req.signal,
    });
 
    return result.toDataStreamResponse();
  } catch (error) {
    return handleAIError(error);
  }
}

// src/hooks/useAIGeneration.ts
"use client";
 
import { useChat } from "ai/react";
import { useCallback, useRef, useState } from "react";
 
export function useAIGeneration(feature: string) {
  const [isGenerating, setIsGenerating] = useState(false);
  const abortRef = useRef<(() => void) | null>(null);
 
  const { messages, append, isLoading, error, stop } = useChat({
    api: "/api/ai/generate",
    body: { feature },
    onResponse(response) {
      if (!response.ok) {
        setIsGenerating(false);
      }
    },
    onFinish() {
      setIsGenerating(false);
    },
    onError(err) {
      setIsGenerating(false);
      console.error(`AI generation failed for ${feature}:`, err);
    },
  });
 
  const generate = useCallback(
    async (input: string, context?: string) => {
      setIsGenerating(true);
      await append({
        role: "user",
        content: input,
      });
    },
    [append],
  );
 
  const cancel = useCallback(() => {
    stop();
    setIsGenerating(false);
  }, [stop]);
 
  return {
    messages,
    generate,
    cancel,
    isGenerating,
    isStreaming: isLoading,
    error,
  };
}

// Raw SSE streaming without the AI SDK
export async function POST(req: Request) {
  const encoder = new TextEncoder();
 
  const stream = new ReadableStream({
    async start(controller) {
      try {
        const response = await fetch("https://api.anthropic.com/v1/messages", {
          method: "POST",
          headers: {
            "Content-Type": "application/json",
            "x-api-key": process.env.ANTHROPIC_API_KEY!,
            "anthropic-version": "2023-06-01",
          },
          body: JSON.stringify({
            model: "claude-sonnet-4-20250514",
            max_tokens: 2048,
            stream: true,
            messages: [{ role: "user", content: "..." }],
          }),
        });
 
        const reader = response.body!.getReader();
        const decoder = new TextDecoder();
        let buffer = "";
 
        while (true) {
          const { done, value } = await reader.read();
          if (done) break;
 
          buffer += decoder.decode(value, { stream: true });
          const lines = buffer.split("\n");
          buffer = lines.pop() ?? "";
 
          for (const line of lines) {
            if (line.startsWith("data: ")) {
              const data = line.slice(6);
              if (data === "[DONE]") {
                controller.close();
                return;
              }
              try {
                const parsed = JSON.parse(data);
                if (parsed.type === "content_block_delta") {
                  const text = parsed.delta?.text ?? "";
                  controller.enqueue(encoder.encode(`data: ${JSON.stringify({ text })}\n\n`));
                }
              } catch {
                // Incomplete JSON in buffer, will be completed
                // on next chunk
              }
            }
          }
        }
        controller.close();
      } catch (error) {
        controller.enqueue(encoder.encode(`data: ${JSON.stringify({ error: "Generation failed" })}\n\n`));
        controller.close();
      }
    },
  });
 
  return new Response(stream, {
    headers: {
      "Content-Type": "text/event-stream",
      "Cache-Control": "no-cache",
      Connection: "keep-alive",
    },
  });
}

// src/lib/ai/prompts.ts
type PromptTemplate = {
  system: string;
  userTemplate: (vars: Record<string, string>) => string;
  version: string;
  model: "fast" | "quality" | "reasoning";
};
 
const prompts: Record<string, PromptTemplate> = {
  "tool-description": {
    system: `You are a technical writer for a developer tools website.
Write concise, accurate descriptions of web-based tools.
 
Rules:
- Maximum 2 sentences
- Focus on what the tool DOES, not how it works
- No marketing language ("powerful", "revolutionary", "amazing")
- No first person
- Include the primary use case`,
    userTemplate: ({ toolName, toolCategory, existingDescription }) =>
      `Write a description for the tool "${toolName}" in the ${toolCategory} category.
${existingDescription ? `Current description (improve this): ${existingDescription}` : ""}`,
    version: "2.1",
    model: "fast",
  },
 
  "code-review": {
    system: `You are a senior software engineer reviewing code.
Provide specific, actionable feedback. No generic advice.
 
Focus on:
- Bugs and logic errors (critical)
- Security issues (critical)
- Performance problems (important)
- Readability improvements (nice to have)
 
Format: Use markdown. List issues by severity.
Do NOT suggest stylistic changes unless they impact readability.
Do NOT rewrite the entire code — point out specific lines.`,
    userTemplate: ({ code, language, context }) =>
      `Review this ${language} code:
\`\`\`${language}
${code}
\`\`\`
${context ? `Context: ${context}` : ""}`,
    version: "3.0",
    model: "quality",
  },
 
  "content-moderation": {
    system: `You are a content moderation system. Classify the input text.
 
Respond with ONLY a JSON object:
{
  "safe": boolean,
  "category": "safe" | "harassment" | "hate" | "sexual" | "violence" | "self-harm" | "illegal",
  "confidence": number (0-1),
  "reason": string (brief explanation, max 20 words)
}
 
No other text. No markdown. No explanation outside the JSON.`,
    userTemplate: ({ text }) => `Classify this text:\n\n${text}`,
    version: "1.4",
    model: "fast",
  },
};
 
export function getPrompt(
  feature: string,
  vars: Record<string, string>,
): { messages: Array<{ role: string; content: string }> } {
  const template = prompts[feature];
  if (!template) throw new Error(`Unknown prompt: ${feature}`);
 
  return {
    messages: [
      { role: "system", content: template.system },
      { role: "user", content: template.userTemplate(vars) },
    ],
  };
}

// src/lib/ai/safety.ts
const INJECTION_PATTERNS = [
  /ignore\s+(all\s+)?previous\s+instructions/i,
  /you\s+are\s+now\s+/i,
  /system\s*:\s*/i,
  /\[INST\]/i,
  /<<SYS>>/i,
  /forget\s+(everything|all|your)/i,
  /new\s+instructions?\s*:/i,
  /override\s+(system|instructions|prompt)/i,
];
 
export function detectPromptInjection(input: string): {
  suspicious: boolean;
  patterns: string[];
} {
  const matches = INJECTION_PATTERNS.filter((p) => p.test(input)).map((p) => p.source);
 
  return {
    suspicious: matches.length > 0,
    patterns: matches,
  };
}
 
export function sanitizeInput(input: string, maxLength = 4000): string {
  return input
    .slice(0, maxLength)
    .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "") // control chars
    .trim();
}

// src/lib/ai/cache.ts
import { createHash } from "crypto";
 
function hashPrompt(messages: Array<{ role: string; content: string }>, model: string, temperature: number): string {
  const input = JSON.stringify({ messages, model, temperature });
  return createHash("sha256").update(input).digest("hex");
}
 
export async function getCachedResponse(
  messages: Array<{ role: string; content: string }>,
  model: string,
  temperature: number,
): Promise<string | null> {
  // Only cache deterministic requests
  if (temperature > 0.1) return null;
 
  const key = `ai:cache:${hashPrompt(messages, model, temperature)}`;
 
  try {
    const cached = await redis.get(key);
    if (cached) {
      await redis.hincrby("ai:stats", "cache_hits", 1);
      return cached;
    }
    await redis.hincrby("ai:stats", "cache_misses", 1);
    return null;
  } catch {
    // Cache failures should never block AI responses
    return null;
  }
}
 
export async function cacheResponse(
  messages: Array<{ role: string; content: string }>,
  model: string,
  temperature: number,
  response: string,
  ttl = 86400, // 24 hours default
): Promise<void> {
  if (temperature > 0.1) return;
 
  const key = `ai:cache:${hashPrompt(messages, model, temperature)}`;
 
  try {
    await redis.setex(key, ttl, response);
  } catch {
    // Silent failure — caching is optimization, not requirement
  }
}

// src/lib/ai/semantic-cache.ts
import { embed } from "ai";
import { openai } from "@ai-sdk/openai";
 
const SIMILARITY_THRESHOLD = 0.92;
 
export async function getSemanticallyCachedResponse(query: string): Promise<string | null> {
  const { embedding } = await embed({
    model: openai.embedding("text-embedding-3-small"),
    value: query,
  });
 
  // Search for similar cached queries using cosine similarity
  // In production, use a vector DB (pgvector, Pinecone, etc.)
  const candidates = await vectorStore.search(embedding, {
    topK: 1,
    threshold: SIMILARITY_THRESHOLD,
  });
 
  if (candidates.length > 0) {
    return candidates[0].metadata.response;
  }
 
  return null;
}

// src/lib/ai/budget.ts
type FeatureBudget = {
  maxInputTokens: number;
  maxOutputTokens: number;
  maxRequestsPerUser: number;
  maxRequestsPerDay: number;
  costPerRequest: number; // estimated, in cents
  dailyBudgetCents: number;
};
 
const budgets: Record<string, FeatureBudget> = {
  "tool-description": {
    maxInputTokens: 500,
    maxOutputTokens: 200,
    maxRequestsPerUser: 50,
    maxRequestsPerDay: 5000,
    costPerRequest: 0.02,
    dailyBudgetCents: 100, // $1/day
  },
  "code-review": {
    maxInputTokens: 8000,
    maxOutputTokens: 2000,
    maxRequestsPerUser: 10,
    maxRequestsPerDay: 1000,
    costPerRequest: 0.8,
    dailyBudgetCents: 800, // $8/day
  },
  "content-moderation": {
    maxInputTokens: 2000,
    maxOutputTokens: 100,
    maxRequestsPerUser: 200,
    maxRequestsPerDay: 50000,
    costPerRequest: 0.005,
    dailyBudgetCents: 250, // $2.50/day
  },
};
 
export async function checkBudget(feature: string, userId: string): Promise<{ allowed: boolean; reason?: string }> {
  const budget = budgets[feature];
  if (!budget) return { allowed: false, reason: "Unknown feature" };
 
  const today = new Date().toISOString().slice(0, 10);
 
  // Check per-user limit
  const userKey = `ai:usage:${feature}:user:${userId}:${today}`;
  const userCount = await redis.incr(userKey);
  if (userCount === 1) await redis.expire(userKey, 86400);
 
  if (userCount > budget.maxRequestsPerUser) {
    return { allowed: false, reason: "Daily user limit reached" };
  }
 
  // Check global daily limit
  const globalKey = `ai:usage:${feature}:global:${today}`;
  const globalCount = await redis.incr(globalKey);
  if (globalCount === 1) await redis.expire(globalKey, 86400);
 
  if (globalCount > budget.maxRequestsPerDay) {
    return { allowed: false, reason: "Feature daily limit reached" };
  }
 
  // Check cost budget
  const costKey = `ai:cost:${feature}:${today}`;
  const currentCost = parseFloat((await redis.get(costKey)) ?? "0");
  if (currentCost > budget.dailyBudgetCents) {
    return { allowed: false, reason: "Daily cost budget exceeded" };
  }
 
  return { allowed: true };
}
 
export async function recordUsage(
  feature: string,
  inputTokens: number,
  outputTokens: number,
  model: string,
): Promise<void> {
  const cost = calculateCost(model, inputTokens, outputTokens);
  const today = new Date().toISOString().slice(0, 10);
  const costKey = `ai:cost:${feature}:${today}`;
 
  await redis.incrbyfloat(costKey, cost);
 
  // Also track aggregate stats
  await redis.hincrby("ai:stats:tokens", `${feature}:input`, inputTokens);
  await redis.hincrby("ai:stats:tokens", `${feature}:output`, outputTokens);
}

function selectModel(feature: string, estimatedInputTokens: number): ModelConfig {
  const budget = budgets[feature];
  const today = new Date().toISOString().slice(0, 10);
 
  // If we're over 80% of daily budget, downgrade to cheaper model
  const currentCost = getCachedCost(feature, today);
  if (currentCost > budget.dailyBudgetCents * 0.8) {
    return models.fast; // Always fall back to cheapest
  }
 
  // Short inputs with simple tasks -> fast model
  if (estimatedInputTokens < 500 && isSimpleTask(feature)) {
    return models.fast;
  }
 
  return models[prompts[feature].model];
}

// src/lib/ai/errors.ts
export async function withAIRetry<T>(
  fn: () => Promise<T>,
  options: {
    maxRetries?: number;
    feature: string;
    fallback?: () => Promise<T>;
  },
): Promise<T> {
  const maxRetries = options.maxRetries ?? 3;
  let lastError: Error | null = null;
 
  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    try {
      return await fn();
    } catch (error: any) {
      lastError = error;
 
      // Don't retry on client errors (except rate limits)
      if (error.status >= 400 && error.status < 500 && error.status !== 429) {
        break;
      }
 
      // Rate limit: respect Retry-After header
      if (error.status === 429) {
        const retryAfter = parseInt(error.headers?.["retry-after"] ?? "5", 10);
        await sleep(retryAfter * 1000);
        continue;
      }
 
      // Exponential backoff for other errors
      if (attempt < maxRetries) {
        await sleep(Math.pow(2, attempt) * 1000);
      }
    }
  }
 
  // All retries failed — try fallback model
  if (options.fallback) {
    try {
      return await options.fallback();
    } catch (fallbackError) {
      // Fallback also failed — log and throw
      console.error(`AI fallback failed for ${options.feature}:`, fallbackError);
    }
  }
 
  throw lastError;
}
 
function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

// src/lib/ai/fallback.ts
import { generateText, streamText } from "ai";
 
export async function generateWithFallback(
  config: ModelConfig,
  params: {
    messages: Array<{ role: string; content: string }>;
    maxTokens: number;
    temperature: number;
  },
) {
  try {
    return await generateText({
      model: getModel(config),
      ...params,
    });
  } catch (error: any) {
    if (config.fallback && shouldFallback(error)) {
      console.warn(`Primary model ${config.model} failed, falling back to ${config.fallback.model}`);
 
      return await generateText({
        model: getModel(config.fallback),
        ...params,
      });
    }
    throw error;
  }
}
 
function shouldFallback(error: any): boolean {
  // Fallback on server errors, timeouts, and rate limits
  // Don't fallback on client errors (bad prompt, content policy)
  return error.status >= 500 || error.status === 429 || error.code === "ETIMEDOUT" || error.code === "ECONNRESET";
}

// src/lib/ai/parse.ts
import { z } from "zod";
 
export function extractJSON<T>(
  text: string,
  schema: z.ZodSchema<T>,
): { success: true; data: T } | { success: false; error: string } {
  // Try direct parse first
  try {
    const parsed = JSON.parse(text);
    const result = schema.safeParse(parsed);
    if (result.success) return { success: true, data: result.data };
  } catch {
    // Not direct JSON, try extraction
  }
 
  // Extract from markdown code blocks
  const codeBlockMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/);
  if (codeBlockMatch) {
    try {
      const parsed = JSON.parse(codeBlockMatch[1].trim());
      const result = schema.safeParse(parsed);
      if (result.success) return { success: true, data: result.data };
    } catch {
      // Malformed JSON in code block
    }
  }
 
  // Try to find JSON object/array in text
  const jsonMatch = text.match(/(\{[\s\S]*\}|\[[\s\S]*\])/);
  if (jsonMatch) {
    try {
      // Fix common LLM JSON mistakes
      const fixed = jsonMatch[1]
        .replace(/,\s*}/g, "}") // trailing commas in objects
        .replace(/,\s*]/g, "]") // trailing commas in arrays
        .replace(/'/g, '"'); // single quotes to double
 
      const parsed = JSON.parse(fixed);
      const result = schema.safeParse(parsed);
      if (result.success) return { success: true, data: result.data };
    } catch {
      // Still not valid
    }
  }
 
  return {
    success: false,
    error: `Failed to extract valid JSON from response: ${text.slice(0, 200)}...`,
  };
}

import { generateObject } from "ai";
import { z } from "zod";
 
const moderationSchema = z.object({
  safe: z.boolean(),
  category: z.enum(["safe", "harassment", "hate", "sexual", "violence", "self-harm", "illegal"]),
  confidence: z.number().min(0).max(1),
  reason: z.string().max(100),
});
 
const { object } = await generateObject({
  model: getModel(models.fast),
  schema: moderationSchema,
  prompt: `Classify this text: ${sanitizedInput}`,
});
 
// object is fully typed and validated
console.log(object.safe, object.category);

// src/lib/ai/rag.ts
import { embed } from "ai";
import { openai } from "@ai-sdk/openai";
 
type Document = {
  id: string;
  content: string;
  metadata: Record<string, string>;
  similarity: number;
};
 
export async function queryWithRAG(
  query: string,
  options: {
    collection: string;
    topK?: number;
    minSimilarity?: number;
  },
): Promise<{ answer: string; sources: Document[] }> {
  const { topK = 5, minSimilarity = 0.7 } = options;
 
  // 1. Embed the query
  const { embedding } = await embed({
    model: openai.embedding("text-embedding-3-small"),
    value: query,
  });
 
  // 2. Vector search (using pgvector here, but Pinecone/Weaviate work too)
  const documents = await prisma.$queryRaw<Document[]>`
    SELECT
      id,
      content,
      metadata,
      1 - (embedding <=> ${embedding}::vector) as similarity
    FROM documents
    WHERE collection = ${options.collection}
      AND 1 - (embedding <=> ${embedding}::vector) > ${minSimilarity}
    ORDER BY embedding <=> ${embedding}::vector
    LIMIT ${topK}
  `;
 
  if (documents.length === 0) {
    return {
      answer: "I don't have enough information to answer that question accurately.",
      sources: [],
    };
  }
 
  // 3. Build context-augmented prompt
  const context = documents.map((d, i) => `[Source ${i + 1}]: ${d.content}`).join("\n\n");
 
  const { text } = await generateText({
    model: getModel(models.quality),
    messages: [
      {
        role: "system",
        content: `Answer the user's question based ONLY on the provided sources.
If the sources don't contain enough information, say so.
Cite sources using [Source N] notation.
Do not make up information not present in the sources.`,
      },
      {
        role: "user",
        content: `Sources:\n${context}\n\nQuestion: ${query}`,
      },
    ],
  });
 
  return { answer: text, sources: documents };
}

// Bad: sequential calls (6 seconds total)
const summary = await generateText({ ... });
const keywords = await generateText({ ... });
const sentiment = await generateText({ ... });
 
// Good: parallel calls (2 seconds total)
const [summary, keywords, sentiment] = await Promise.all([
  generateText({ model: getModel(models.fast), prompt: summaryPrompt }),
  generateText({ model: getModel(models.fast), prompt: keywordsPrompt }),
  generateText({ model: getModel(models.fast), prompt: sentimentPrompt }),
]);

// When user starts typing code, speculatively begin analysis
const debouncedAnalysis = useMemo(
  () =>
    debounce(async (code: string) => {
      if (code.length > 50) {
        // Start analysis in background
        speculativeResultRef.current = generateText({
          model: getModel(models.fast),
          prompt: buildAnalysisPrompt(code),
        });
      }
    }, 2000),
  [],
);

// scripts/prefetch-descriptions.ts
// Run this as a cron job, not on every request
async function prefetchToolDescriptions() {
  const tools = await getAllTools();
 
  for (const tool of tools) {
    const cacheKey = `ai:desc:${tool.slug}`;
    const cached = await redis.get(cacheKey);
    if (cached) continue;
 
    const { text } = await generateText({
      model: getModel(models.fast),
      prompt: buildDescriptionPrompt(tool),
    });
 
    await redis.setex(cacheKey, 604800, text); // 7 days
 
    // Respect rate limits
    await sleep(200);
  }
}

// src/components/ai/StreamingMarkdown.tsx
"use client";
 
import { memo, useMemo } from "react";
import ReactMarkdown from "react-markdown";
import { Prism as SyntaxHighlighter } from "react-syntax-highlighter";
 
interface StreamingMarkdownProps {
  content: string;
  isStreaming: boolean;
}
 
export const StreamingMarkdown = memo(function StreamingMarkdown({
  content,
  isStreaming,
}: StreamingMarkdownProps) {
  // Avoid re-parsing markdown on every token
  const stableContent = useMemo(() => {
    if (!isStreaming) return content;
 
    // While streaming, ensure we don't render incomplete
    // markdown that causes layout shifts
    const lines = content.split("\n");
    const lastLine = lines[lines.length - 1];
 
    // If the last line looks like an incomplete code block, don't render it yet
    if (lastLine?.startsWith("```") && !content.endsWith("```")) {
      return lines.slice(0, -1).join("\n");
    }
 
    return content;
  }, [content, isStreaming]);
 
  return (
    <div className="prose prose-neutral dark:prose-invert max-w-none">
      <ReactMarkdown
        components={{
          code({ className, children, ...props }) {
            const match = /language-(\w+)/.exec(className ?? "");
            if (match) {
              return (
                <SyntaxHighlighter language={match[1]}>
                  {String(children).replace(/\n$/, "")}
                </SyntaxHighlighter>
              );
            }
            return (
              <code className={className} {...props}>
                {children}
              </code>
            );
          },
        }}
      >
        {stableContent}
      </ReactMarkdown>
      {isStreaming && (
        <span className="inline-block w-2 h-4 bg-current animate-pulse ml-0.5" />
      )}
    </div>
  );
});

function getErrorMessage(error: any, feature: string): string {
  if (error.status === 429) {
    return "This feature is temporarily busy. Please try again in a few seconds.";
  }
  if (error.status === 413 || error.message?.includes("token")) {
    return "Your input is too long for AI analysis. Try with a shorter text.";
  }
  if (error.code === "ETIMEDOUT") {
    return "The AI took too long to respond. This sometimes happens with complex requests. Please try again.";
  }
  if (error.message?.includes("content_policy")) {
    return "This content cannot be processed by our AI. Please modify your input.";
  }
  return "AI analysis is temporarily unavailable. Your other tools still work fine.";
}

// src/components/ai/AILoadingState.tsx
export function AILoadingState({ feature }: { feature: string }) {
  const estimates: Record<string, string> = {
    "code-review": "10-20 seconds",
    "tool-description": "2-5 seconds",
    summarize: "5-10 seconds",
  };
 
  return (
    <div className="flex items-center gap-3 text-muted-foreground">
      <div className="flex gap-1">
        <span className="w-2 h-2 bg-current rounded-full animate-bounce" />
        <span className="w-2 h-2 bg-current rounded-full animate-bounce [animation-delay:0.1s]" />
        <span className="w-2 h-2 bg-current rounded-full animate-bounce [animation-delay:0.2s]" />
      </div>
      <span>
        Analyzing... This usually takes {estimates[feature] ?? "a few seconds"}.
      </span>
    </div>
  );
}

// src/lib/ai/__tests__/parse.test.ts
import { describe, it, expect } from "vitest";
import { extractJSON } from "../parse";
import { z } from "zod";
 
const schema = z.object({
  safe: z.boolean(),
  category: z.string(),
});
 
describe("extractJSON", () => {
  it("parses direct JSON", () => {
    const result = extractJSON('{"safe": true, "category": "safe"}', schema);
    expect(result.success).toBe(true);
  });
 
  it("extracts JSON from markdown code block", () => {
    const input = `Here is my analysis:\n\`\`\`json\n{"safe": true, "category": "safe"}\n\`\`\``;
    const result = extractJSON(input, schema);
    expect(result.success).toBe(true);
  });
 
  it("handles trailing commas", () => {
    const input = '{"safe": true, "category": "safe",}';
    const result = extractJSON(input, schema);
    expect(result.success).toBe(true);
  });
 
  it("fails gracefully on garbage input", () => {
    const result = extractJSON("I cannot help with that.", schema);
    expect(result.success).toBe(false);
  });
});

describe("prompts", () => {
  it("tool-description prompt matches snapshot", () => {
    const prompt = getPrompt("tool-description", {
      toolName: "JSON Formatter",
      toolCategory: "developer",
      existingDescription: "",
    });
 
    expect(prompt.messages[0].content).toMatchSnapshot();
  });
});

// src/lib/ai/__tests__/integration.test.ts
// Only runs with AI_INTEGRATION_TESTS=true
describe.skipIf(!process.env.AI_INTEGRATION_TESTS)("AI integration", () => {
  it("moderation correctly flags harmful content", async () => {
    const result = await moderate("I will hurt someone");
    expect(result.safe).toBe(false);
    expect(result.confidence).toBeGreaterThan(0.8);
  }, 30000);
 
  it("moderation allows benign content", async () => {
    const result = await moderate("How do I center a div in CSS?");
    expect(result.safe).toBe(true);
  }, 30000);
 
  it("structured output matches schema", async () => {
    const result = await analyzeCode("function add(a, b) { return a + b; }", "javascript");
    expect(result).toHaveProperty("issues");
    expect(Array.isArray(result.issues)).toBe(true);
  }, 60000);
});

const evaluationSet = [
  {
    input: "A React hook for fetching data",
    expectedCategory: "developer",
    expectedContains: ["hook", "fetch", "React"],
    expectedNotContains: ["revolutionary", "powerful"],
  },
  {
    input: "Convert PDF to Word document",
    expectedCategory: "converter",
    expectedContains: ["PDF", "Word"],
  },
];
 
async function runEvaluation() {
  const results = [];
 
  for (const testCase of evaluationSet) {
    const response = await generateDescription(testCase.input);
 
    const passed =
      testCase.expectedContains?.every((word) => response.toLowerCase().includes(word.toLowerCase())) &&
      !testCase.expectedNotContains?.some((word) => response.toLowerCase().includes(word.toLowerCase()));
 
    results.push({
      input: testCase.input,
      response,
      passed,
    });
  }
 
  const passRate = results.filter((r) => r.passed).length / results.length;
  console.log(`Evaluation pass rate: ${(passRate * 100).toFixed(1)}%`);
 
  if (passRate < 0.85) {
    // Alert: prompt may have regressed
    await sendAlert(`AI evaluation pass rate dropped to ${passRate}`);
  }
}

// src/lib/ai/moderation.ts
export async function moderateInput(input: string): Promise<{
  allowed: boolean;
  reason?: string;
}> {
  // Layer 1: Keyword blocklist (fast, catches obvious cases)
  const blocklist = getBlocklist();
  for (const term of blocklist) {
    if (input.toLowerCase().includes(term)) {
      return { allowed: false, reason: "blocked_content" };
    }
  }
 
  // Layer 2: Prompt injection detection
  const injection = detectPromptInjection(input);
  if (injection.suspicious) {
    return { allowed: false, reason: "prompt_injection" };
  }
 
  // Layer 3: AI-based moderation (most accurate, but costs money)
  const moderation = await classifyContent(input);
  if (!moderation.safe && moderation.confidence > 0.8) {
    return { allowed: false, reason: moderation.category };
  }
 
  return { allowed: true };
}

export function sanitizeOutput(text: string): string {
  // Remove any system prompt leakage
  const systemLeakPatterns = [
    /you are a .* assistant/gi,
    /as an ai/gi,
    /i'?m (just )?an? ai/gi,
    /my (system )?instructions/gi,
  ];
 
  let cleaned = text;
  for (const pattern of systemLeakPatterns) {
    cleaned = cleaned.replace(pattern, "");
  }
 
  return cleaned.trim();
}

// src/lib/ai/embeddings.ts
import { embed, embedMany } from "ai";
import { openai } from "@ai-sdk/openai";
 
const embeddingModel = openai.embedding("text-embedding-3-small");
 
export async function embedDocument(content: string, metadata: Record<string, string>): Promise<void> {
  const chunks = splitIntoChunks(content, {
    maxTokens: 600,
    overlap: 100,
  });
 
  const { embeddings } = await embedMany({
    model: embeddingModel,
    values: chunks.map((c) => c.text),
  });
 
  // Store in database with pgvector
  for (let i = 0; i < chunks.length; i++) {
    await prisma.embedding.create({
      data: {
        content: chunks[i].text,
        embedding: embeddings[i],
        metadata,
        chunkIndex: i,
        totalChunks: chunks.length,
      },
    });
  }
}
 
function splitIntoChunks(
  text: string,
  options: { maxTokens: number; overlap: number },
): Array<{ text: string; startOffset: number }> {
  const { maxTokens, overlap } = options;
  const chunks: Array<{ text: string; startOffset: number }> = [];
 
  // Split on paragraph boundaries when possible
  const paragraphs = text.split(/\n\n+/);
  let currentChunk = "";
  let currentOffset = 0;
  let chunkStartOffset = 0;
 
  for (const paragraph of paragraphs) {
    const combined = currentChunk ? `${currentChunk}\n\n${paragraph}` : paragraph;
    const estimatedTokens = Math.ceil(combined.length / 4);
 
    if (estimatedTokens > maxTokens && currentChunk) {
      chunks.push({
        text: currentChunk.trim(),
        startOffset: chunkStartOffset,
      });
 
      // Overlap: keep last portion of previous chunk
      const overlapText = currentChunk.slice(-(overlap * 4));
      currentChunk = overlapText + "\n\n" + paragraph;
      chunkStartOffset = currentOffset - overlapText.length;
    } else {
      if (!currentChunk) chunkStartOffset = currentOffset;
      currentChunk = combined;
    }
 
    currentOffset += paragraph.length + 2;
  }
 
  if (currentChunk.trim()) {
    chunks.push({
      text: currentChunk.trim(),
      startOffset: chunkStartOffset,
    });
  }
 
  return chunks;
}

// src/lib/ai/telemetry.ts
type AITelemetry = {
  feature: string;
  model: string;
  inputTokens: number;
  outputTokens: number;
  latencyMs: number;
  cached: boolean;
  fallbackUsed: boolean;
  error?: string;
  userSatisfaction?: "positive" | "negative";
};
 
export async function trackAICall(telemetry: AITelemetry): Promise<void> {
  // Real-time metrics for alerting
  await redis.lpush("ai:telemetry:recent", JSON.stringify({ ...telemetry, timestamp: Date.now() }));
  await redis.ltrim("ai:telemetry:recent", 0, 9999);
 
  // Aggregate stats for dashboards
  const today = new Date().toISOString().slice(0, 10);
  const prefix = `ai:metrics:${today}:${telemetry.feature}`;
 
  await Promise.all([
    redis.hincrby(prefix, "requests", 1),
    redis.hincrby(prefix, "input_tokens", telemetry.inputTokens),
    redis.hincrby(prefix, "output_tokens", telemetry.outputTokens),
    redis.hincrby(prefix, "total_latency_ms", telemetry.latencyMs),
    redis.hincrby(prefix, telemetry.cached ? "cache_hits" : "cache_misses", 1),
    telemetry.error && redis.hincrby(prefix, "errors", 1),
    telemetry.fallbackUsed && redis.hincrby(prefix, "fallbacks", 1),
  ]);
 
  redis.expire(prefix, 2592000); // 30 days
}