// Privacy-safe demand log for ToolPick need= queries (0.1.2). Aggregate counts only, in the existing StatsCounter
// SQLite Durable Object (table `needs`): per UTC day and normalized need -> total, no_match, low_match, no_ours.
// Normalized = lowercase; emails, URLs, wallet/hex addresses, base58 keys, long tokens and long numbers removed;
// whitespace collapsed; capped at 120 chars. No IP, wallet, payment header or request id is ever stored.
// Fail-open and fire-and-forget: logging can never change or delay a response. Smoke requests are not logged.
export const NEED_MAX = 120;
export const LOW_REL = 0.5; // top relevance below this = a weak match
export const PUBLIC_MIN_COUNT = 3; // /v1/needs only shows needs asked at least this many times in the window

export function normalizeNeed(s) {
  let t = String(s ?? '').toLowerCase();
  t = t.replace(/[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}/g, ' ')              // emails
    .replace(/\b(?:https?|wss?|ftp|ipfs):\/\/\S+/g, ' ')                       // URLs
    .replace(/\bwww\.\S+/g, ' ')
    .replace(/\b[a-z0-9-]+(?:\.[a-z0-9-]+)*\.(?:com|io|xyz|org|net|co|ai|dev|app|so|sh|gg|me|fi|finance|eth|sol)\b\S*/g, ' ') // bare domains / ENS / SNS names
    .replace(/\b0x[0-9a-f]{6,}\b/g, ' ')                                       // EVM addresses, tx hashes, calldata
    .replace(/\b[1-9a-hj-np-z]{32,44}\b/g, ' ')                                // base58 (Solana, BTC) addresses/keys
    .replace(/\b[a-z0-9_\-+/=]{24,}\b/g, ' ')                                  // long tokens / keys
    .replace(/\d[\d.,:\-/ ]{4,}\d|\d{5,}/g, ' ')                               // long numbers (phones, ids, amounts)
    .replace(/[^\p{L}\p{N} .,'&+#\-/]+/gu, ' ')
    .replace(/\s+/g, ' ').trim();
  if (t.length > NEED_MAX) t = t.slice(0, NEED_MAX).replace(/\s+\S*$/, '').trim() || t.slice(0, NEED_MAX);
  return t.length >= 2 ? t : '';
}

/** Outcome flags from a ToolPick result (or a no_matches error). */
export function classify(result, err) {
  if (err) return err.code === 'no_matches' ? { no_match: 1, low_match: 0, no_ours: 1 } : null;
  const rs = result?.results || [];
  if (!rs.length) return { no_match: 1, low_match: 0, no_ours: 1 };
  const top = Math.max(...rs.map((x) => Number(x.relevance) || 0));
  return { no_match: 0, low_match: top < LOW_REL ? 1 : 0, no_ours: rs.some((x) => x.ours) ? 0 : 1 };
}

const day = (t) => new Date(t).toISOString().slice(0, 10);

export class NeedLog {
  constructor(counter, { flushMs = 60000, now = () => Date.now() } = {}) {
    this.counter = counter; this.flushMs = flushMs; this.now = now; this.pending = {}; this.lastFlush = 0; this.errors = 0;
  }
  stub() { return this.counter.get(this.counter.idFromName('global')); }
  /** Never throws. tier: lite | pick | preview (market is seller-side and not logged). */
  record(tier, need, result, err, waitUntil) {
    try {
      if (!['lite', 'pick', 'preview'].includes(tier)) return;
      const q = normalizeNeed(need); const f = classify(result, err);
      if (!q || !f) return;
      const bucket = (this.pending[day(this.now())] ||= {});
      if (!bucket[q] && Object.keys(bucket).length >= 2000) return; // bounded memory between flushes
      const b = (bucket[q] ||= { n: 0, no_match: 0, low_match: 0, no_ours: 0 });
      b.n++; b.no_match += f.no_match; b.low_match += f.low_match; b.no_ours += f.no_ours;
      if (this.counter && this.now() - this.lastFlush >= this.flushMs) {
        this.lastFlush = this.now();
        const p = this.flush().catch(() => { this.errors++; });
        try { waitUntil?.(p); } catch { /* not on Workers */ }
      }
    } catch { this.errors++; }
  }
  async flush() {
    if (!this.counter) return;
    const pending = this.pending; this.pending = {};
    if (!Object.keys(pending).length) return;
    await this.stub().addNeeds(pending);
  }
  /** Top needs in the last `days` days. unmet = no_match + low_match. Never throws. */
  async top({ days = 30, limit = 50, minCount = PUBLIC_MIN_COUNT, unmetOnly = true } = {}) {
    const since = day(this.now() - (days - 1) * 86400000);
    let rows = [];
    try { if (this.counter) rows = await this.stub().topNeeds(since, 500); } catch { return { enabled: true, degraded: true, needs: [] }; }
    const m = new Map();
    for (const r of rows) { const e = m.get(r.q) || { need: r.q, n: 0, no_match: 0, low_match: 0, no_ours: 0 }; e.n += r.n; e.no_match += r.no_match; e.low_match += r.low_match; e.no_ours += r.no_ours; m.set(r.q, e); }
    for (const [d, qs] of Object.entries(this.pending)) if (d >= since) for (const [q, c] of Object.entries(qs)) { const e = m.get(q) || { need: q, n: 0, no_match: 0, low_match: 0, no_ours: 0 }; e.n += c.n; e.no_match += c.no_match; e.low_match += c.low_match; e.no_ours += c.no_ours; m.set(q, e); }
    const needs = [...m.values()].map((e) => ({ ...e, unmet: e.no_match + e.low_match }))
      .filter((e) => e.n >= minCount && (!unmetOnly || e.unmet > 0 || e.no_ours > 0))
      .sort((a, b) => (b.unmet - a.unmet) || (b.no_ours - a.no_ours) || (b.n - a.n)).slice(0, limit);
    return { enabled: !!this.counter, window_days: days, min_count: minCount, needs };
  }
}

agentgate-source: code mirror on Cloudflare Pages. All repos