라우팅과 디스패치

type RouteResult = {
  target: string
  confidence: number
  reason: string
}

async function dispatchRequest(input: string) {
  if (matchesPolicyRule(input)) return sendTo('policy-gate')

  const route: RouteResult = await classify(input)

  if (route.confidence >= 0.85) return sendTo(route.target)
  if (route.confidence >= 0.6) return sendTo('generalist-review')

  return sendTo('human-triage')
}

from sklearn.metrics import precision_recall_curve
import numpy as np

def find_optimal_threshold(
    y_true: list[int],
    y_scores: list[float],
    min_precision: float = 0.90,
) -> float:
    """
    최소 precision을 보장하면서 가장 높은 recall을 달성하는
    threshold를 찾는다.
    """
    precisions, recalls, thresholds = precision_recall_curve(
        y_true, y_scores
    )

    # min_precision 이상인 구간에서 recall이 최대인 threshold
    valid = precisions[:-1] >= min_precision
    if not valid.any():
        return float(thresholds[-1])  # 가장 보수적인 값

    best_idx = np.where(valid)[0][np.argmax(recalls[:-1][valid])]
    return float(thresholds[best_idx])


# 사용 예시
# y_true: 정답 라벨 (1=해당 클래스, 0=아님)
# y_scores: 분류기가 반환한 confidence 값
threshold = find_optimal_threshold(y_true, y_scores, min_precision=0.90)
print(f"최적 threshold: {threshold:.3f}")

const THRESHOLDS: Record<string, { high: number; low: number }> = {
  'payment-cancel':  { high: 0.90, low: 0.75 },  // 고위험: 높은 기준
  'general-inquiry': { high: 0.70, low: 0.50 },  // 저위험: 낮은 기준
  'account-delete':  { high: 0.92, low: 0.80 },  // 고위험
  'faq':             { high: 0.65, low: 0.40 },  // 저위험
}

function routeWithCalibration(category: string, confidence: number) {
  const t = THRESHOLDS[category] ?? { high: 0.85, low: 0.60 }

  if (confidence >= t.high) return 'direct-dispatch'
  if (confidence >= t.low)  return 'generalist-review'
  return 'human-triage'
}

interface DriftMetrics {
  windowStart: Date
  windowEnd: Date
  accuracy: number
  avgConfidence: number
  fallbackRate: number
  totalRequests: number
}

function detectDrift(
  baseline: DriftMetrics,
  current: DriftMetrics,
): { drifted: boolean; reasons: string[] } {
  const reasons: string[] = []

  const accuracyDrop = baseline.accuracy - current.accuracy
  if (accuracyDrop > 0.05) {
    reasons.push(
      `accuracy ${accuracyDrop.toFixed(2)} 하락 ` +
      `(${baseline.accuracy.toFixed(2)} → ${current.accuracy.toFixed(2)})`
    )
  }

  const confidenceDrop = baseline.avgConfidence - current.avgConfidence
  if (confidenceDrop > 0.1) {
    reasons.push(
      `평균 confidence ${confidenceDrop.toFixed(2)} 하락`
    )
  }

  const fallbackIncrease = current.fallbackRate - baseline.fallbackRate
  if (fallbackIncrease > 0.1) {
    reasons.push(
      `fallback 비율 ${(fallbackIncrease * 100).toFixed(1)}%p 증가`
    )
  }

  return { drifted: reasons.length > 0, reasons }
}

interface RoutingTestCase {
  id: string
  input: string
  expectedTarget: string
  category: 'clear' | 'boundary' | 'ambiguous' | 'adversarial'
  addedAt: string        // ISO date
  source: 'production-log' | 'synthetic' | 'feedback'
  notes?: string
}

const goldenTestSet: RoutingTestCase[] = [
  {
    id: 'refund-001',
    input: '지난주 결제한 거 환불 가능한가요?',
    expectedTarget: 'refund-agent',
    category: 'clear',
    addedAt: '2026-03-01',
    source: 'production-log',
  },
  {
    id: 'boundary-001',
    input: '부분 환불 되나요? 배송비만 빼고요',
    expectedTarget: 'refund-agent',
    category: 'boundary',
    addedAt: '2026-03-01',
    source: 'production-log',
    notes: 'partial-refund와 refund 경계',
  },
  {
    id: 'multi-001',
    input: '배송 어디쯤 왔는지 확인하고, 안 오면 환불할게요',
    expectedTarget: 'needs_triage',
    category: 'ambiguous',
    addedAt: '2026-03-05',
    source: 'feedback',
    notes: '다중 의도: 배송조회 + 조건부 환불',
  },
]

async function runRoutingEval(
  router: (input: string) => Promise<{ target: string; confidence: number }>,
  testSet: RoutingTestCase[],
) {
  const results = await Promise.all(
    testSet.map(async (tc) => {
      const result = await router(tc.input)
      return {
        id: tc.id,
        category: tc.category,
        expected: tc.expectedTarget,
        actual: result.target,
        confidence: result.confidence,
        pass: result.target === tc.expectedTarget,
      }
    })
  )

  const total = results.length
  const passed = results.filter((r) => r.pass).length
  const byCategory = Object.groupBy(results, (r) => r.category)

  console.log(`전체: ${passed}/${total} (${((passed / total) * 100).toFixed(1)}%)`)
  for (const [cat, items] of Object.entries(byCategory)) {
    const catPassed = items!.filter((r) => r.pass).length
    console.log(`  ${cat}: ${catPassed}/${items!.length}`)
  }

  // 실패 케이스 상세
  results
    .filter((r) => !r.pass)
    .forEach((r) => {
      console.log(
        `  FAIL [${r.id}] expected=${r.expected} actual=${r.actual} ` +
        `confidence=${r.confidence.toFixed(3)}`
      )
    })

  return results
}

import numpy as np
from sklearn.metrics import f1_score

def find_similarity_threshold(
    similarities: list[float],
    labels: list[int],  # 1=정답 매칭, 0=오매칭
    candidates: np.ndarray | None = None,
) -> dict:
    """
    다양한 threshold에서 F1 score를 계산하고
    최적 threshold를 반환한다.
    """
    if candidates is None:
        candidates = np.arange(0.50, 0.96, 0.01)

    sims = np.array(similarities)
    labs = np.array(labels)
    best = {'threshold': 0.0, 'f1': 0.0}

    for t in candidates:
        preds = (sims >= t).astype(int)
        f1 = f1_score(labs, preds)
        if f1 > best['f1']:
            best = {'threshold': float(t), 'f1': float(f1)}

    return best


# 사용 예시
# 검증 세트에서 각 (입력, route) 쌍의 cosine similarity와 정답 여부를 수집
result = find_similarity_threshold(
    similarities=[0.89, 0.72, 0.91, 0.55, 0.83, 0.61],
    labels=      [1,    0,    1,    0,    1,    0],
)
print(f"최적 threshold: {result['threshold']:.2f}, F1: {result['f1']:.3f}")

interface SemanticRoute {
  name: string
  description: string          // route를 설명하는 텍스트
  examples: string[]           // 해당 route에 해당하는 예시 입력
  embedding?: number[]         // 사전 계산된 평균 임베딩
  threshold?: number           // route별 개별 threshold
}

async function semanticRoute(
  input: string,
  routes: SemanticRoute[],
  embed: (text: string) => Promise<number[]>,
  defaultThreshold = 0.78,
): Promise<{ route: string; similarity: number } | null> {
  const inputVec = await embed(input)

  let best: { route: string; similarity: number } | null = null

  for (const r of routes) {
    if (!r.embedding) continue
    const sim = cosineSimilarity(inputVec, r.embedding)
    const threshold = r.threshold ?? defaultThreshold

    if (sim >= threshold && (!best || sim > best.similarity)) {
      best = { route: r.name, similarity: sim }
    }
  }

  return best
}

function cosineSimilarity(a: number[], b: number[]): number {
  let dot = 0, normA = 0, normB = 0
  for (let i = 0; i < a.length; i++) {
    dot += a[i] * b[i]
    normA += a[i] * a[i]
    normB += b[i] * b[i]
  }
  return dot / (Math.sqrt(normA) * Math.sqrt(normB))
}