// ToolPick engine: a compact index of every paid x402 resource listed on the public x402 discovery indexes
// (Coinbase CDP discovery API with 30-day call/payer counts, plus the PayAI facilitator discovery list), ranked
// search for "which paid API should my agent call for X", price/competition stats, and live 402 checks.
// Pure functions only (no storage); the IndexStore Durable Object (index-do.js) holds the rows.
import { withDeadline } from './deadline.js';
import { VERSION_STR } from './version.js';

export const ENGINE_VERSION = `toolpick-${VERSION_STR}`;
export const USER_AGENT = 'ToolPick/0.1 (+https://api.youraitools.co/toolpick/)';
export const CDP_URL = 'https://api.cdp.coinbase.com/platform/v2/x402/discovery/resources';
export const PAYAI_URL = 'https://facilitator.payai.network/discovery/resources';
export const DISCLAIMER = 'Rankings use public x402 index metadata and the index-reported 30-day call and payer counts; listings are self-described by sellers and are not endorsements. A live check only shows that an endpoint answers with x402 payment terms right now, not that its data is good or that the seller is safe. Check terms before paying. ToolPick never pays, holds funds or keys.';
export const LIMITS = [
  'Index: every resource on the CDP x402 discovery API (with 30-day calls and unique payers) merged with the PayAI discovery list (no usage counts); refreshed lazily at most every 6 hours, no cron.',
  'Usage counts are what the index reports: they include sellers testing their own endpoints and directory crawlers; repeat_buyers marks resources with 3+ payers and more calls than payers.',
  'Prices are the cheapest USDC (6 decimals) entry per resource on mainnet networks; testnet-only resources are excluded.',
  'Live checks (pick tier) send one unpaid request per candidate (3 s timeout) and read the 402 terms; nothing is ever paid. Endpoints with path parameters are not probed.',
  'Fail-closed: an index older than 72 hours or missing is refused (503, not charged); 12-72 hours old is flagged stale with lower confidence; a candidate whose live check fails is unverified (listed in sources_failed), never verified.',
];
export const PRIVACY = 'Seller pay-to addresses are not returned; only whether the live terms match the index.';

export class InputError extends Error { constructor(m, status = 400, code = 'bad_request') { super(m); this.status = status; this.code = code; } }
export class Incomplete extends Error { constructor(m, sources_failed = [], code = 'index_unavailable') { super(m); this.code = code; this.sources_failed = sources_failed; } }

// ---- networks / assets ----
const USDC = {
  '0x833589fcd6edb6e08f4c7c32d4f71b54bda02913': 'base',
  '0x3c499c542cef5e3811e1192ce70d8cc03d5c3359': 'polygon',
  '0xaf88d065e77c8cc2239327c5edb3a432268e5831': 'arbitrum',
  '0x0b2c639c533813f4aa9d7837caf62653d097ff85': 'optimism',
  '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48': 'ethereum',
  '0xb97ef9ef8734c71904d8002f8b6bc66dd9c48a6e': 'avalanche',
  'epjfwdd5aufqssqem2qn1xzybapc8g4weggkzwytdt1v': 'solana',
};
const NET = { 'eip155:8453': 'base', base: 'base', 'eip155:137': 'polygon', polygon: 'polygon', 'eip155:42161': 'arbitrum', arbitrum: 'arbitrum', 'arbitrum-one': 'arbitrum', 'eip155:10': 'optimism', optimism: 'optimism', 'eip155:1': 'ethereum', ethereum: 'ethereum', 'eip155:43114': 'avalanche', avalanche: 'avalanche', 'solana:5eykt4usfv8p8njdtrepy1vzqkqzkvdp': 'solana', solana: 'solana' };
const TESTNET = /sepolia|testnet|devnet|amoy|fuji|^eip155:84532$|^solana:etwtrabzayq6imfeykouru166vu2xqa1$/i;
export const NETWORKS = ['base', 'solana', 'polygon', 'arbitrum', 'optimism', 'ethereum', 'avalanche'];

/** Compact one discovery item. Returns null for testnet-only/unpriced items. */
export function compact(it, src) {
  const res = typeof it?.resource === 'string' ? it.resource : it?.resource?.url;
  if (!res || !/^https?:\/\//i.test(res)) return null;
  let price = null; const nets = new Set(); let testnetOnly = true;
  for (const a of Array.isArray(it.accepts) ? it.accepts : []) {
    const network = String(a?.network || '').toLowerCase();
    if (TESTNET.test(network)) continue;
    testnetOnly = false;
    const n = NET[network] || null;
    const asset = String(a?.asset || '').toLowerCase();
    const usdcNet = USDC[asset] || (USDC[asset.replace(/^0x/, '')] ?? null);
    if (!usdcNet) { if (n) nets.add(n); continue; }
    nets.add(n || usdcNet);
    const raw = a.amount ?? a.maxAmountRequired;
    const v = Number(raw) / 1e6;
    if (Number.isFinite(v) && v >= 0 && (price === null || v < price)) price = v;
  }
  if (testnetOnly) return null;
  const q = it.quality || {};
  const meta = it.metadata || {};
  const info = it.extensions?.bazaar?.info?.input || it.outputSchema?.input || {};
  let host = ''; try { host = new URL(res).host.toLowerCase(); } catch { return null; }
  const desc = String(it.description || meta.description || '').replace(/\s+/g, ' ').trim().slice(0, 240);
  const tags = (Array.isArray(it.tags) ? it.tags : []).map((t) => String(t).toLowerCase().slice(0, 30)).slice(0, 6);
  const last = q.lastCalledAt ? Math.round(Date.parse(q.lastCalledAt) / 1000) || null : null;
  return {
    u: res.slice(0, 300), h: host, s: String(it.serviceName || host).slice(0, 60), d: desc, t: tags,
    p: price === null ? null : Math.round(price * 1e6) / 1e6, n: [...nets],
    c: Number.isFinite(q.l30DaysTotalCalls) ? q.l30DaysTotalCalls : null, y: Number.isFinite(q.l30DaysUniquePayers) ? q.l30DaysUniquePayers : null,
    l: last, m: String(info.method || it.method || '').toUpperCase().slice(0, 6) || null, v: Number(it.x402Version) || null, src,
  };
}

/** Merge CDP rows (with quality) and PayAI rows (no quality); same URL keeps CDP data and gains src "both". */
export function mergeRows(cdp, payai) {
  const by = new Map();
  for (const r of cdp) if (r) by.set(r.u, r);
  for (const r of payai) {
    if (!r) continue;
    const e = by.get(r.u);
    if (e) { if (e.src === 'cdp') e.src = 'both'; for (const n of r.n) if (!e.n.includes(n)) e.n.push(n); if (e.p === null) e.p = r.p; }
    else by.set(r.u, r);
  }
  return [...by.values()];
}

// ---- index build (runs inside the Durable Object alarm) ----
async function pageFetch(fetchImpl, url, ms) {
  const r = await withDeadline(fetchImpl(url, { headers: { accept: 'application/json', 'user-agent': USER_AGENT } }), ms, 'index page');
  if (!r.ok) { try { await r.body?.cancel(); } catch { /* ignore */ } throw new Error(`HTTP ${r.status}`); }
  return r.json();
}
async function pullAll(fetchImpl, base, { pageSize = 1000, maxPages = 80, concurrency = 4, ms = 25000, src }) {
  const first = await pageFetch(fetchImpl, `${base}?limit=${pageSize}&offset=0`, ms);
  const total = Number(first?.pagination?.total) || (first.items || []).length;
  const rows = (first.items || []).map((x) => compact(x, src));
  let got = (first.items || []).length;
  const offsets = []; for (let o = pageSize; o < total && offsets.length < maxPages - 1; o += pageSize) offsets.push(o);
  let failed = 0;
  for (let i = 0; i < offsets.length; i += concurrency) {
    const batch = await Promise.allSettled(offsets.slice(i, i + concurrency).map(async (o) => {
      for (let a = 0; a < 3; a++) { try { return await pageFetch(fetchImpl, `${base}?limit=${pageSize}&offset=${o}`, ms); } catch (e) { if (a === 2) throw e; } }
      return null;
    }));
    for (const b of batch) {
      if (b.status !== 'fulfilled' || !b.value) { failed++; continue; }
      const it = b.value.items || []; got += it.length;
      for (const x of it) rows.push(compact(x, src));
    }
  }
  return { rows: rows.filter(Boolean), total, got, failedPages: failed };
}

/** Build the merged index. Throws if the primary (CDP) index is missing or < 80% complete (the caller keeps the last good one). */
export async function buildIndex({ fetchImpl = globalThis.fetch, now = Date.now(), pageSize = 1000 } = {}) {
  const failed = [];
  // Sequential sources, 2 pages in flight: keeps peak memory low inside the Durable Object (128 MB isolate).
  const [cdp] = await Promise.allSettled([pullAll(fetchImpl, CDP_URL, { pageSize, src: 'cdp', concurrency: 2 })]);
  const [payai] = cdp.status === 'fulfilled' ? await Promise.allSettled([pullAll(fetchImpl, PAYAI_URL, { pageSize: Math.min(pageSize, 1000), maxPages: 40, src: 'payai', concurrency: 2 })]) : [{ status: 'rejected', reason: 'skipped' }];
  if (cdp.status !== 'fulfilled') throw new Error(`cdp index: ${String(cdp.reason?.message || cdp.reason).slice(0, 120)}`);
  const c = cdp.value;
  if (!c.total || c.got < 0.8 * c.total) throw new Error(`cdp index incomplete: ${c.got}/${c.total}`);
  if (c.failedPages) failed.push('cdp_partial');
  let p = { rows: [], total: 0, got: 0 };
  if (payai.status === 'fulfilled') { p = payai.value; if (p.failedPages || p.got < 0.8 * p.total) failed.push('payai_partial'); } else failed.push('payai_index');
  const rows = mergeRows(c.rows, p.rows);
  return { at: now, rows, counts: { cdp_total: c.total, cdp_got: c.got, payai_total: p.total, payai_got: p.got, merged: rows.length }, sources_failed: failed };
}

// ---- search ----
const STOP = new Set('a an the and or for of to in on with by from my our your i we you me is are be can that this what which who how do does api apis tool tools service services paid x402 agent agents endpoint endpoints data get find need want call use using best cheap cheapest should would could will please some any there it its me give return returns via'.split(' '));
const SYN = {
  llm: ['inference', 'chat', 'completion'], inference: ['llm'], gpt: ['llm', 'inference'], scrape: ['extract', 'crawl', 'markdown'], scraping: ['scrape', 'extract', 'crawl'], crawl: ['scrape'],
  search: ['serp', 'query'], web: ['website', 'url', 'page'], honeypot: ['rug', 'token'], rug: ['honeypot', 'scam'], token: ['erc20', 'mint'], wallet: ['address'],
  price: ['quote', 'ticker'], ohlcv: ['candles', 'candle', 'kline'], candles: ['ohlcv', 'kline'], sentiment: ['fear', 'greed'], tweet: ['twitter'], twitter: ['tweet', 'x.com'],
  email: ['mail'], pdf: ['document'], image: ['picture', 'photo'], screenshot: ['capture'], sanctions: ['ofac', 'aml'], ofac: ['sanctions'], kyc: ['kyb', 'aml'],
  weather: ['forecast'], translate: ['translation'], transcribe: ['transcription', 'speech'], funding: ['perp', 'perps'], perp: ['perps', 'futures', 'funding'], perps: ['perp'],
  smart: ['whale'], whale: ['smart'], gas: ['fee'], swap: ['dex', 'quote'], nft: ['erc721'], ens: ['basename'],
};
export function stem(w) { return w.length > 4 ? w.replace(/(ings|ing|ies|es|s)$/, (m) => (m === 'ies' ? 'y' : '')) : w; }
export function terms(need) {
  const words = String(need || '').toLowerCase().replace(/[^a-z0-9.\- ]+/g, ' ').split(/\s+/).map((w) => w.replace(/^[.\-]+|[.\-]+$/g, '')).filter((w) => w.length >= 2 && !STOP.has(w));
  return [...new Set(words)].slice(0, 12);
}
const text = (r) => r._x || (r._x = `${r.s} ${r.h} ${r.u.replace(/^https?:\/\/[^/]+/, '')} ${r.t.join(' ')} ${r.d}`.toLowerCase());
/** Relevance 0..1 and matched term count for one row. */
export function relevance(r, ts) {
  if (!ts.length) return { rel: 1, matched: 0 };
  const hay = text(r); const tagStr = ` ${r.t.join(' ')} `; const head = `${r.s} ${r.h} ${r.u}`.toLowerCase();
  let score = 0; let matched = 0;
  for (const t of ts) {
    const st = stem(t);
    const alts = [st, ...(SYN[t] || [])];
    let hit = 0;
    for (const a of alts) {
      if (!hay.includes(a)) continue;
      const w = a === st ? 1 : 0.6;
      const tagHit = tagStr.includes(a) ? 1.4 : 0; const headHit = head.includes(a) ? 1 : 0;
      hit = Math.max(hit, w * (1 + tagHit + headHit));
    }
    if (hit) { matched++; score += hit; }
  }
  return { rel: score / (ts.length * 3.4), matched };
}
const pct = (arr, q) => { if (!arr.length) return null; const s = [...arr].sort((a, b) => a - b); return s[Math.min(s.length - 1, Math.floor(q * (s.length - 1) + 0.5))]; };
const r6 = (x) => (x === null || x === undefined ? null : Math.round(x * 1e6) / 1e6);
export const OUR_HOSTS = /(^|\.)agentgate\.workers\.dev$|^api\.youraitools\.co$/;

/**
 * Rank rows for a need. opts: { maxPrice, network, minPayers, limit, perHost, nowS }.
 * Score: relevance x demand (log payers, log calls) x recency, filtered by price/network/payers; at most perHost per host.
 */
export function search(rows, need, { maxPrice = null, network = null, minPayers = 0, limit = 10, perHost = 2, nowS = Math.floor(Date.now() / 1000) } = {}) {
  const ts = terms(need);
  const need_ = Math.max(1, Math.ceil(ts.length * 0.6));
  const hits = [];
  for (const r of rows) {
    if (maxPrice !== null && (r.p === null || r.p > maxPrice)) continue;
    if (network && !r.n.includes(network)) continue;
    if (minPayers && (r.y || 0) < minPayers) continue;
    const { rel, matched } = relevance(r, ts);
    if (ts.length && matched < need_) continue;
    const pay = r.y || 0; const calls = r.c || 0;
    const age = r.l ? (nowS - r.l) / 86400 : Infinity;
    const rec = age <= 7 ? 1 : age <= 30 ? 0.8 : 0.55;
    const demand = 1 + Math.log10(1 + pay) * 1.6 + Math.log10(1 + calls) * 0.4;
    hits.push({ r, rel, score: rel * demand * rec * (r.p === null ? 0.7 : 1) });
  }
  hits.sort((a, b) => b.score - a.score);
  const out = []; const perH = new Map();
  for (const h of hits) {
    const k = perH.get(h.r.h) || 0; if (k >= perHost) continue;
    perH.set(h.r.h, k + 1); out.push(h); if (out.length >= limit) break;
  }
  return { terms: ts, matches: hits.length, hosts: new Set(hits.map((h) => h.r.h)).size, top: out, all: hits };
}

export function view(h, rank) {
  const r = h.r;
  return {
    rank, service: r.s, host: r.h, resource: r.u, method: r.m || 'GET', description: r.d,
    price_usd: r.p, networks: r.n, calls_30d: r.c, payers_30d: r.y,
    repeat_buyers: r.y !== null && r.c !== null ? r.y >= 3 && r.c > r.y : null,
    last_called_at: r.l ? new Date(r.l * 1000).toISOString() : null,
    indexed_on: r.src === 'both' ? ['cdp', 'payai'] : [r.src], ours: OUR_HOSTS.test(r.h) || undefined,
    relevance: Math.round(h.rel * 100) / 100, score: Math.round(h.score * 1000) / 1000,
  };
}

/** Price / competition / demand stats over a set of hits. */
export function marketStats(hits) {
  const priced = hits.filter((h) => h.r.p !== null && h.r.p < 5).map((h) => h.r);
  const prices = priced.map((r) => r.p);
  const buyers = priced.filter((r) => (r.y || 0) >= 3);
  const bp = buyers.map((r) => r.p);
  // payer-weighted median price
  let pw = null; { const w = buyers.map((r) => [r.p, r.y]).sort((a, b) => a[0] - b[0]); const tot = w.reduce((s, x) => s + x[1], 0); let acc = 0; for (const [p, y] of w) { acc += y; if (acc >= tot / 2) { pw = p; break; } } }
  const calls = priced.reduce((s, r) => s + (r.c || 0), 0);
  const rev = priced.reduce((s, r) => s + (r.c || 0) * r.p, 0);
  const hostsAll = new Set(hits.map((h) => h.r.h)); const hostsPaid = new Set(buyers.map((r) => r.h));
  return {
    resources: hits.length, hosts: hostsAll.size, hosts_with_3plus_payers: hostsPaid.size, priced_resources: priced.length,
    price_usd: { min: r6(pct(prices, 0)), p25: r6(pct(prices, 0.25)), median: r6(pct(prices, 0.5)), p75: r6(pct(prices, 0.75)), max: r6(pct(prices, 1)) },
    price_with_buyers_usd: { p25: r6(pct(bp, 0.25)), median: r6(pct(bp, 0.5)), p75: r6(pct(bp, 0.75)), payer_weighted_median: r6(pw) },
    calls_30d: calls, est_revenue_30d_usd: Math.round(rev * 100) / 100,
    est_revenue_note: 'calls x listed price; an upper bound (includes sellers testing their own endpoints and crawler payments; resources priced $5+ excluded)',
  };
}

// ---- live 402 check ----
const b64json = (s) => { try { const bin = atob(s); const bytes = Uint8Array.from(bin, (c) => c.charCodeAt(0)); return JSON.parse(new TextDecoder().decode(bytes)); } catch { return null; } };
export function probeable(u) {
  let x; try { x = new URL(u); } catch { return false; }
  if (x.protocol !== 'https:') return false;
  if (/[:{}]/.test(decodeURIComponent(x.pathname)) || /\[|\]|^\d+\.\d+\.\d+\.\d+$|^localhost$|\.local$|\.internal$/i.test(x.hostname)) return false;
  return true;
}
/** One unpaid request; reads x402 v2 (PAYMENT-REQUIRED header) or v1 (JSON body) terms. Never pays. */
export async function liveCheck(row, { fetchImpl = globalThis.fetch, ms = 3000 } = {}) {
  if (!probeable(row.u)) return { status: 'not_probed', reason: 'path parameters or non-https URL' };
  const t0 = Date.now();
  const method = row.m === 'POST' ? 'POST' : 'GET';
  let res;
  try {
    res = await withDeadline(fetchImpl(row.u, { method, redirect: 'manual', headers: { accept: 'application/json', 'user-agent': USER_AGENT, ...(method === 'POST' ? { 'content-type': 'application/json' } : {}) }, ...(method === 'POST' ? { body: '{}' } : {}) }), ms, 'live check');
  } catch (e) { return { status: 'unreachable', error: String(e?.message || e).slice(0, 80), ms: Date.now() - t0 }; }
  const ms_ = Date.now() - t0;
  if (res.status !== 402) { try { await res.body?.cancel(); } catch { /* ignore */ } return { status: res.status >= 500 ? 'server_error' : 'no_402', http_status: res.status, ms: ms_ }; }
  let accepts = null; let version = null;
  const h = res.headers.get('payment-required');
  if (h) { const j = b64json(h); if (j && Array.isArray(j.accepts)) { accepts = j.accepts; version = 2; } }
  if (!accepts) {
    try { const txt = await withDeadline(res.text(), 1500, 'body'); const j = JSON.parse(txt.slice(0, 200_000)); if (Array.isArray(j?.accepts)) { accepts = j.accepts; version = j.x402Version || 1; } } catch { /* no body terms */ }
  } else { try { await res.body?.cancel(); } catch { /* ignore */ } }
  if (!accepts || !accepts.length) return { status: 'bad_terms', http_status: 402, ms: ms_ };
  const live = compact({ resource: row.u, accepts }, 'live');
  const price = live?.p ?? null;
  return {
    status: 'live', x402_version: version, http_status: 402, ms: ms_,
    live_price_usd: price, live_networks: live?.n || [], price_matches_index: row.p === null || price === null ? null : Math.abs(price - row.p) < 1e-9,
  };
}

agentgate-source: code mirror on Cloudflare Pages. All repos