Simplified belief verification: single Opus agent + Exa search (replaces 8-agent adversarial pipeline)

belief-verifier.ts

/**
 * Simplified belief verification using a single Opus agent with Exa search tools.
 *
 * Replaces the 8-agent adversarial pipeline with one agent that can:
 * - Search for supporting AND contradicting evidence via Exa
 * - Distinguish first-person vs third-party sources
 * - Render per-field verdicts with citations
 *
 * Usage:
 *   EXA_API_KEY=... ANTHROPIC_API_KEY=... npx tsx belief-verifier.ts
 */

import { getModel } from "@earendil-works/pi-ai";
import { Agent, type AgentTool } from "@earendil-works/pi-agent-core";
import { Type } from "typebox";

// ---------------------------------------------------------------------------
// Exa search tool
// ---------------------------------------------------------------------------

const EXA_API_KEY = process.env.EXA_API_KEY;
if (!EXA_API_KEY) throw new Error("EXA_API_KEY is required");

interface ExaResult {
  title: string;
  url: string;
  publishedDate: string | null;
  author: string | null;
  text?: string;
  highlights?: string[];
  summary?: string;
}

const webSearch: AgentTool = {
  name: "web_search",
  label: "Web Search",
  description:
    "Search the web using Exa. Use this to find evidence for or against a belief. " +
    "You can run multiple queries in parallel. Results include page text when available.",
  parameters: Type.Object({
    queries: Type.Array(Type.String({ description: "Search queries to run" }), {
      minItems: 1,
      maxItems: 5,
      description: "One or more search queries. Use multiple queries to search for both supporting and contradicting evidence.",
    }),
    numResults: Type.Optional(
      Type.Number({ description: "Results per query (default 5, max 10)", minimum: 1, maximum: 10 })
    ),
    includeDomains: Type.Optional(
      Type.Array(Type.String(), { description: "Restrict to these domains (e.g. official sites)" })
    ),
  }),
  execute: async (_id, params) => {
    const numResults = params.numResults ?? 5;
    const allResults: Array<{ query: string; results: ExaResult[] }> = [];

    const fetches = params.queries.map(async (query: string) => {
      const body: Record<string, unknown> = {
        query,
        numResults,
        type: "auto",
        contents: { text: { maxCharacters: 3000 }, highlights: true },
      };
      if (params.includeDomains?.length) body.includeDomains = params.includeDomains;

      const res = await fetch("https://api.exa.ai/search", {
        method: "POST",
        headers: { "x-api-key": EXA_API_KEY!, "Content-Type": "application/json" },
        body: JSON.stringify(body),
      });
      if (!res.ok) throw new Error(`Exa search failed (${res.status}): ${await res.text()}`);
      const data = await res.json();
      return { query, results: data.results as ExaResult[] };
    });

    allResults.push(...(await Promise.all(fetches)));

    const text = allResults
      .map(({ query, results }) => {
        const entries = results
          .map(
            (r, i) =>
              `  [${i + 1}] ${r.title}\n      URL: ${r.url}\n      Published: ${r.publishedDate ?? "unknown"}\n      Author: ${r.author ?? "unknown"}` +
              (r.highlights?.length ? `\n      Highlights:\n${r.highlights.map((h) => `        - ${h}`).join("\n")}` : "") +
              (r.text ? `\n      Text (truncated):\n        ${r.text.slice(0, 1500)}` : "")
          )
          .join("\n\n");
        return `Query: "${query}"\n${entries}`;
      })
      .join("\n\n---\n\n");

    return { content: [{ type: "text", text }], details: { queryCount: allResults.length } };
  },
};

// ---------------------------------------------------------------------------
// Fetch content tool (for following specific URLs)
// ---------------------------------------------------------------------------

const fetchContent: AgentTool = {
  name: "fetch_content",
  label: "Fetch Page",
  description:
    "Fetch the full text content of a specific URL via Exa. " +
    "Use this to read a first-person source page (e.g. an official about page, interview transcript).",
  parameters: Type.Object({
    url: Type.String({ description: "URL to fetch" }),
  }),
  execute: async (_id, params) => {
    const body = {
      urls: [params.url],
      text: { maxCharacters: 8000 },
    };
    const res = await fetch("https://api.exa.ai/contents", {
      method: "POST",
      headers: { "x-api-key": EXA_API_KEY!, "Content-Type": "application/json" },
      body: JSON.stringify(body),
    });
    if (!res.ok) throw new Error(`Exa fetch failed (${res.status}): ${await res.text()}`);
    const data = await res.json();
    const page = data.results?.[0];
    if (!page) throw new Error("No content returned for URL");

    return {
      content: [{ type: "text", text: `URL: ${page.url}\nTitle: ${page.title}\n\n${page.text ?? "(no text)"}` }],
      details: { url: page.url },
    };
  },
};

// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------

interface BeliefRecord {
  entity_id: string;
  entity_name: string;
  entity_type: string;
  fields: Record<string, string | null>;
}

interface FieldVerdict {
  field: string;
  current_value: string | null;
  verdict: "confirm" | "correct" | "remove";
  proposed_value: string | null;
  confidence: "high" | "medium" | "low";
  source_url: string;
  citation: string;
  attribution_type: "first-person" | "third-party";
  reasoning: string;
}

// ---------------------------------------------------------------------------
// Terminating tool — structured output via tool call
// ---------------------------------------------------------------------------

let capturedVerdicts: FieldVerdict[] | null = null;

const submitVerdicts: AgentTool = {
  name: "submit_verdicts",
  label: "Submit Verdicts",
  description:
    "Submit your final per-field verdicts. Call this exactly once when you have finished " +
    "all research and are ready to deliver your structured results.",
  parameters: Type.Object({
    verdicts: Type.Array(
      Type.Object({
        field: Type.String({ description: "Field name from the record" }),
        current_value: Type.Union([Type.String(), Type.Null()], { description: "Current value in the record" }),
        verdict: Type.Union([Type.Literal("confirm"), Type.Literal("correct"), Type.Literal("remove")], {
          description: "confirm = evidence supports value, correct = first-person evidence contradicts it, remove = no evidence exists",
        }),
        proposed_value: Type.Union([Type.String(), Type.Null()], {
          description: "Replacement value (required for 'correct', null otherwise)",
        }),
        confidence: Type.Union([Type.Literal("high"), Type.Literal("medium"), Type.Literal("low")]),
        source_url: Type.String({ description: "URL of the best supporting source" }),
        citation: Type.String({ description: "Relevant quote from the source" }),
        attribution_type: Type.Union([Type.Literal("first-person"), Type.Literal("third-party")]),
        reasoning: Type.String({ description: "One sentence explaining the verdict" }),
      })
    ),
  }),
  execute: async (_id, params) => {
    capturedVerdicts = params.verdicts;
    return {
      content: [{ type: "text", text: "Verdicts recorded." }],
      details: {},
      terminate: true,
    };
  },
};

// ---------------------------------------------------------------------------
// System prompt
// ---------------------------------------------------------------------------

const SYSTEM_PROMPT = `You are a belief verification agent. Your job is to verify factual claims about entities (people, companies, organizations) by searching for evidence.

For each field in the record you receive, you must:

1. Search for BOTH supporting and contradicting evidence. Run queries that could confirm the value AND queries that could disprove it.
2. Prioritize first-person sources (official websites, interviews, press releases from the entity itself) over third-party characterizations (news articles, Wikipedia, blog posts about the entity).
3. Follow up on promising leads by fetching specific URLs when needed.
4. Render a verdict per field:
   - "confirm" — evidence supports the current value
   - "correct" — first-person evidence contradicts the current value (propose a replacement)
   - "remove" — no evidence exists for this field (set to null)

When you're done investigating, call the submit_verdicts tool with your per-field verdicts. Do NOT output verdicts as text — always use the tool.

Rules:
- A "correct" verdict MUST be backed by first-person evidence. Third-party sources alone can only support "confirm" or "remove".
- If you find conflicting evidence, weigh first-person sources over third-party ones.
- Be skeptical. Do not confirm a value just because one blog post repeats it.
- Search thoroughly — use 2-3 different query angles per field.`;

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------

async function verifyBeliefs(record: BeliefRecord): Promise<FieldVerdict[]> {
  const model = getModel("anthropic", "claude-opus-4-20250514");

  const agent = new Agent({
    initialState: {
      systemPrompt: SYSTEM_PROMPT,
      model,
      thinkingLevel: "medium",
      tools: [webSearch, fetchContent, submitVerdicts],
    },
    toolExecution: "parallel",
  });

  capturedVerdicts = null;

  agent.subscribe((event) => {
    if (event.type === "message_update" && event.assistantMessageEvent.type === "text_delta") {
      process.stdout.write(event.assistantMessageEvent.delta);
    }
    if (event.type === "tool_execution_start") {
      console.log(`\n🔧 ${event.toolName}(${JSON.stringify(event.args).slice(0, 120)}...)`);
    }
  });

  const fieldList = Object.entries(record.fields)
    .map(([k, v]) => `  - ${k}: ${v ?? "(null)"}`)
    .join("\n");

  const prompt = `Verify the following record:\n\nEntity: ${record.entity_name} (${record.entity_type}, id: ${record.entity_id})\n\nFields to verify:\n${fieldList}\n\nSearch for evidence and produce your per-field verdicts.`;

  await agent.prompt(prompt);

  if (!capturedVerdicts) throw new Error("Agent did not call submit_verdicts");
  return capturedVerdicts;
}

// ---------------------------------------------------------------------------
// Example: run on a sample record
// ---------------------------------------------------------------------------

const sampleRecord: BeliefRecord = {
  entity_id: "person-001",
  entity_name: "Dario Amodei",
  entity_type: "person",
  fields: {
    current_role: "CEO of Anthropic",
    education: "PhD in Computational Neuroscience from Princeton",
    previous_employer: "OpenAI",
    birth_year: "1983",
    nationality: "American",
  },
};

console.log("\n═══════════════════════════════════════════════");
console.log(" Belief Verification — Single Agent + Exa Search");
console.log("═══════════════════════════════════════════════\n");
console.log(`Verifying: ${sampleRecord.entity_name}\n`);

const verdicts = await verifyBeliefs(sampleRecord);

console.log("\n\n═══════════════════════════════════════════════");
console.log(" VERDICTS");
console.log("═══════════════════════════════════════════════\n");

for (const v of verdicts) {
  const icon = v.verdict === "confirm" ? "✅" : v.verdict === "correct" ? "🔄" : "❌";
  console.log(`${icon} ${v.field}: ${v.verdict.toUpperCase()}`);
  console.log(`   Current: ${v.current_value}`);
  if (v.proposed_value) console.log(`   Proposed: ${v.proposed_value}`);
  console.log(`   Confidence: ${v.confidence} | Source: ${v.attribution_type}`);
  console.log(`   Citation: "${v.citation}"`);
  console.log(`   URL: ${v.source_url}`);
  console.log(`   Reasoning: ${v.reasoning}\n`);
}

console.log("\nJSONL output:\n");
for (const v of verdicts) {
  console.log(JSON.stringify({ entity_id: sampleRecord.entity_id, ...v }));
}