find-affiliates.ts

import fs from "node:fs/promises"
import path from "node:path"
import { processBatchWithErrorHandling, sleep } from "@dirstack/utils"
import { generateText, Output } from "ai"
import { z } from "zod"
import { gateway } from "~/services/ai"
import { db } from "~/services/db"

// Known self-service affiliate platforms that host programs for many brands.
// A URL on one of these is valid when the path or subdomain references the brand.
const AFFILIATE_PLATFORMS = new Set([
  "tolt.io",
  "tolt.com",
  "getrewardful.com",
  "rewardful.com",
  "partnerstack.com",
  "dub.co",
  "firstpromoter.com",
  "refersion.com",
  "affonso.io",
  "reditus.com",
  "getreditus.com",
  "partneroapp.com",
  "lemonsqueezy.com",
  "tapfiliate.com",
  "promotekit.com",
  "leaddyno.com",
  "trackdesk.com",
  "gumroad.com",
])

// Traditional affiliate networks. Accept only with a brand-specific slug in the URL.
const AFFILIATE_NETWORKS = new Set(["cj.com", "shareasale.com", "impact.com", "awin.com"])

// Third-party directories listing affiliate programs (not the programs themselves).
const AGGREGATOR_HOSTS = new Set([
  "affililist.com",
  "affiliateotter.com",
  "commissiondex.com",
  "all-affiliates.com",
  "linkmydeals.com",
  "affiliate.watch",
  "affiliateprogramdb.com",
  "affpaying.com",
  "postaffiliatepro.com",
  "taprefer.com",
  "giverefer.com",
])

// Reason-text signals that the program is a reseller/integrator partnership, not affiliate.
const RESELLER_PATTERNS: RegExp[] = [
  /\bresell(?:ing|ers?)?\b/i,
  /\bdelivering\s+(?:the\s+)?(?:product|services|integrations?)/i,
  /\bintegration\s+partners?\b/i,
  /\btechnology\s+partners?\b/i,
  /\bchannel\s+partners?\b/i,
  /\bsolution\s+partners?\b/i,
  /\bimplementation\s+partners?\b/i,
  /\bdeal\s+registration\b/i,
  /\blicenses?\s+sold\b/i,
  /\bconsult(?:ing|ants?|ancy)\b/i,
  /\bfor\s+agencies\s+and\s+developers\b/i,
  /\bagency\s+partners?\b/i,
  /\bcontact\s+(?:our|the)\s+partner\s+team\b/i,
  /\bpartner\s+team\b/i,
  /\bcontributors?\s+program\b/i,
  /\b(?:join|become)\s+(?:a|an|our)\s+(?:solution|technology|integration|implementation|channel|reseller)\b/i,
]

// Reason-text signals that the entry is denying an affiliate program exists.
const NEGATION_PATTERNS: RegExp[] = [
  /\bno\s+(?:explicit\s+)?mentions?\s+of\s+['"]?(?:affiliate|referral)/i,
  /\bno\s+(?:customer-facing\s+)?affiliate\s+(?:or\s+referral\s+)?program\b/i,
  /\bnot\s+(?:an?\s+)?affiliate\s+program\b/i,
  /\bno\s+dedicated\s+affiliate\b/i,
]

// URL paths that typically indicate a non-affiliate page (generic partner/integration/legal).
const BAD_URL_PATH_PATTERNS: RegExp[] = [
  /\/partners?\/?$/i,
  /\/partner-programs?\/?$/i,
  /\/integrations?\//i,
  /\/cookies?\/?$/i,
  /\/user_admin\//i,
  /\/teams\/refer/i,
  /\/contact[-_]partner/i,
  /\/partners?\/ambassadors?\/?$/i,
  /\/legal\/partner/i,
  /\/contact[-_]sales/i,
]
const BAD_URL_QUERY = /[?&]template=/i
const AFFILIATE_URL_SIGNAL = /(?:aff?i?l+i?[ae]te?|refer(?:r|$|\/)|ambassadors?)/i

// Common 2-part TLDs for naive etld+1 extraction.
const TWO_PART_TLDS = new Set(["co.uk", "com.au", "co.nz", "co.jp", "com.br", "co.in", "co.za"])
const etld1 = (hostname: string) => {
  const h = hostname.toLowerCase().replace(/^www\./, "")
  const parts = h.split(".").filter(Boolean)
  if (parts.length <= 2) return h
  const lastTwo = parts.slice(-2).join(".")
  if (TWO_PART_TLDS.has(lastTwo) && parts.length >= 3) return parts.slice(-3).join(".")
  return lastTwo
}
const brandLabel = (hostname: string) => etld1(hostname).split(".")[0] || ""
const tryUrl = (s: string) => {
  try {
    return new URL(s)
  } catch {
    return null
  }
}
const slug = (s: string) => s.toLowerCase().replace(/[^a-z0-9]/g, "")

const brandInAffiliateUrl = (name: string, websiteUrl: string, affiliateUrl: string) => {
  const urlSlug = slug(affiliateUrl)
  const candidates = new Set<string>()
  const nameSlug = slug(name)
  if (nameSlug.length >= 3) candidates.add(nameSlug)

  const web = tryUrl(websiteUrl)
  if (web) {
    const webBrand = brandLabel(web.hostname)
    if (webBrand.length >= 3) candidates.add(slug(webBrand))
    for (const label of web.hostname.replace(/^www\./, "").split(".")) {
      const s = slug(label)
      if (s.length >= 4 && !["com", "net", "org", "www", "app", "dev", "ai", "io", "co"].includes(s)) {
        candidates.add(s)
      }
    }
  }

  return [...candidates].some(c => urlSlug.includes(c))
}

type Validation = { valid: true } | { valid: false; rejectReason: string }
type Candidate = { name: string; websiteUrl: string; reason: string; affiliateUrl: string }

/**
 * Filters out the common false positives produced by LLM + web search:
 * cross-domain hallucinations (mastra.ai → mastera.io), reseller partner
 * programs, aggregator listings, generic /partners pages, and affiliate
 * platform URLs that don't actually reference the brand.
 */
const validateAffiliateCandidate = ({
  name,
  websiteUrl,
  reason,
  affiliateUrl,
}: Candidate): Validation => {
  if (!affiliateUrl?.trim()) return { valid: false, rejectReason: "affiliate URL is empty" }
  const affUrl = tryUrl(affiliateUrl)
  const webUrl = tryUrl(websiteUrl)
  if (!affUrl || !webUrl) {
    return { valid: false, rejectReason: "affiliate or website URL is unparseable" }
  }

  const affEtld = etld1(affUrl.hostname)
  const webEtld = etld1(webUrl.hostname)

  if (AGGREGATOR_HOSTS.has(affEtld)) {
    return { valid: false, rejectReason: `affiliate URL is on third-party directory (${affEtld})` }
  }

  for (const p of RESELLER_PATTERNS) {
    if (p.test(reason)) {
      return { valid: false, rejectReason: `reason describes a reseller/integrator partnership` }
    }
  }
  for (const p of NEGATION_PATTERNS) {
    if (p.test(reason)) {
      return { valid: false, rejectReason: `reason explicitly denies an affiliate program` }
    }
  }

  // Affiliate platform URLs (e.g. webstudio.tolt.io, partners.dub.co/buffer) carry
  // the brand in subdomain or path — a bare "/" path is fine here.
  if (AFFILIATE_PLATFORMS.has(affEtld)) {
    if (brandInAffiliateUrl(name, websiteUrl, affiliateUrl)) return { valid: true }
    return {
      valid: false,
      rejectReason: `affiliate platform URL (${affEtld}) does not reference the brand '${name}'`,
    }
  }
  if (AFFILIATE_NETWORKS.has(affEtld)) {
    if (brandInAffiliateUrl(name, websiteUrl, affiliateUrl)) return { valid: true }
    return {
      valid: false,
      rejectReason: `generic affiliate-network URL (${affEtld}) without brand-specific identifier`,
    }
  }

  const isHomepageOnly =
    (affUrl.pathname === "/" || affUrl.pathname === "") && !affUrl.search && !affUrl.hash
  if (isHomepageOnly) {
    return {
      valid: false,
      rejectReason: "affiliate URL points at homepage with no dedicated affiliate page",
    }
  }

  for (const p of BAD_URL_PATH_PATTERNS) {
    if (p.test(affUrl.pathname)) {
      return { valid: false, rejectReason: `URL path does not point to an affiliate program page` }
    }
  }
  if (BAD_URL_QUERY.test(affUrl.search)) {
    return {
      valid: false,
      rejectReason: "URL is a form template or generic query, not an affiliate program page",
    }
  }

  if (affEtld === webEtld) return { valid: true }

  // Same brand on different TLD (openvpn.net ↔ openvpn.com, threema.ch ↔ threema.com).
  // Require an explicit affiliate/referral signal in the URL to avoid matching
  // unrelated products that happen to share a name.
  const affBrand = brandLabel(affUrl.hostname)
  const webBrand = brandLabel(webUrl.hostname)
  const pathHasAffiliateSignal = AFFILIATE_URL_SIGNAL.test(affUrl.hostname + affUrl.pathname)
  if (affBrand && affBrand === webBrand && affBrand.length >= 4 && pathHasAffiliateSignal) {
    return { valid: true }
  }

  return {
    valid: false,
    rejectReason: `affiliate domain '${affEtld}' differs from website domain '${webEtld}' and is not a known affiliate platform`,
  }
}

const checkAffiliateProgram = async (name: string, websiteUrl: string, retries = 3) => {
  const schema = z.object({
    hasAffiliate: z.boolean().describe("Whether the website has an affiliate program"),
    confidence: z.number().min(0).max(1).describe("Confidence level in the decision"),
    reason: z.string().describe("Brief explanation of the decision"),
    affiliateUrl: z
      .string()
      .describe("URL of the affiliate program signup or info page, or empty string"),
  })

  for (let i = 0; i < retries; i++) {
    try {
      const { output } = await generateText({
        model: gateway("perplexity/sonar"),
        output: Output.object({ schema }),
        providerOptions: {
          perplexity: { web_search_options: { search_context_size: "low" } },
        },
        system:
          "You determine whether the exact website specified by the user operates a referral-style affiliate program that pays a commission for customers referred via a tracking link or code. You must distinguish these from solution, integration, reseller, agency, or technology partner programs, which do NOT qualify. Never attribute a program found on a different company's domain to the target website unless you have strong direct evidence that the same company owns both domains.",
        prompt: `Does the website at ${websiteUrl} (brand name: "${name}") operate an affiliate program that pays commissions to anyone who refers paying customers via a tracking link or referral code?

CRITICAL domain rule: the affiliateUrl you return MUST be on one of:
  1. The same registrable domain as ${websiteUrl}, or a subdomain of it.
  2. A known self-service affiliate platform (tolt.io, rewardful.com, getrewardful.com, partnerstack.com, dub.co, firstpromoter.com, refersion.com, affonso.io, reditus.com, partneroapp.com, lemonsqueezy.com, tapfiliate.com, promotekit.com, trackdesk.com, gumroad.com) AND the URL path or subdomain clearly references "${name}" or the website's brand.
  3. A traditional affiliate network (cj.com, shareasale.com, impact.com, awin.com) with a URL that names the specific brand.

If you can't find a qualifying URL on one of those domains, set hasAffiliate to false. Do NOT return programs found on similarly-named but unrelated domains (e.g., a product at example.ai when asked about example.com).

Additionally, set hasAffiliate to true ONLY if ALL of these are true:
- The program pays a commission, revenue share, or recurring payout per referred paying customer.
- Anyone can sign up (self-serve or light approval), not just agencies, integrators, or resellers.
- The payout is driven by a tracking link, cookie, or referral code — not by contracts, deal registration, or services delivery.
- Typical signals: mentions of commission %, recurring commission, cookie duration, payout via PayPal/wire.

Set hasAffiliate to false if the program is actually:
- A solutions / implementation / consulting / services partner program.
- A reseller, channel, or distributor program.
- A technology or integration partner program (marketplace listing, building integrations, co-selling).
- An agency partner program where payout depends on delivering services rather than referring customers.
- An ambassador / influencer program with no explicit commission for referrals.

Do NOT point affiliateUrl at a generic /partners, /partners/, /integrations, /cookies, or /contact-sales page — only at a dedicated affiliate/referral signup or info page.

When unsure, set hasAffiliate to false and return an empty affiliateUrl. Put a one-sentence justification referencing the evidence in reason.`,
        temperature: 0.1,
      })

      return output
    } catch (error) {
      console.error(`Attempt ${i + 1} failed for ${name}:`, error)
      if (i === retries - 1) return null
      await sleep(5000 * (i + 1))
    }
  }
  return null
}

const ensureFile = async (filePath: string) => {
  try {
    await fs.access(filePath)
  } catch {
    await fs.writeFile(filePath, "[]", "utf-8")
  }
}

const readArrayFromFile = async <T>(filePath: string): Promise<T[]> => {
  const raw = await fs.readFile(filePath, "utf-8")
  return JSON.parse(raw) as T[]
}

const appendToArrayFile = async <T>(filePath: string, item: T) => {
  const arr = await readArrayFromFile<T>(filePath)
  arr.push(item)
  await fs.writeFile(filePath, JSON.stringify(arr, null, 2), "utf-8")
}

async function main() {
  const affiliateOutputPath = path.join(process.cwd(), "affiliates-programs.json")

  await ensureFile(affiliateOutputPath)

  const [tools, alternatives] = await Promise.all([
    db.tool.findMany({
      where: {
        AND: [
          { OR: [{ affiliateUrl: null }, { affiliateUrl: "" }] },
          { status: { notIn: ["Pending"] } },
        ],
      },
      select: { id: true, name: true, websiteUrl: true },
    }),
    db.alternative.findMany({
      where: { OR: [{ affiliateUrl: null }, { affiliateUrl: "" }] },
      select: { id: true, name: true, websiteUrl: true },
    }),
  ])

  console.log(`Found ${tools.length} tools and ${alternatives.length} alternatives`)

  const items = [...tools, ...alternatives].filter(item => item.websiteUrl)

  const processor = async ({ name, websiteUrl }: (typeof items)[0]) => {
    console.log(`Processing ${name}...`)

    const analysis = await checkAffiliateProgram(name, websiteUrl!)

    if (!analysis?.hasAffiliate) {
      console.log(`[-] No affiliate program found for ${name}`)
      return analysis
    }

    const validation = validateAffiliateCandidate({
      name,
      websiteUrl: websiteUrl!,
      reason: analysis.reason,
      affiliateUrl: analysis.affiliateUrl,
    })
    if (!validation.valid) {
      console.log(`[-] Rejected for ${name}: ${validation.rejectReason}`)
      return analysis
    }

    await appendToArrayFile(affiliateOutputPath, { name, websiteUrl, ...analysis })
    console.log(`[+] Found affiliate program for ${name}`)
    return analysis
  }

  await processBatchWithErrorHandling(items, processor, {
    batchSize: 5,
    delay: 1000,
    onError: (error, item) => console.error(`Error processing ${item.name}:`, error),
  })

  const affiliateFileArr = await readArrayFromFile<any>(affiliateOutputPath)
  console.log(`\nFound ${affiliateFileArr.length} alternatives with affiliate programs`)
  console.log(`Affiliate results saved to ${affiliateOutputPath}`)
}

main().catch(console.error)