AI Visibility

// lib/dupes-ai-crawl.ts — drop this file into YOUR app.
import type { NextFetchEvent, NextRequest } from "next/server";

const ENDPOINT = "https://dupes-gilt.vercel.app//api/collect/server";

// Loose pre-filter so we don't POST on every human request. Dupes
// re-classifies server-side against the full crawler directory, so this only
// has to be wide enough not to miss anything.
const LIKELY_BOT = /bot|crawler|spider|GPT|Claude|Perplexity|OAI-|ChatGPT|CCBot|Bytespider/i;

export function trackAICrawlerRequest(
  request: NextRequest,
  event: NextFetchEvent,
  opts: { websiteId: string; token?: string },
) {
  const ua = request.headers.get("user-agent") || "";
  if (!LIKELY_BOT.test(ua)) return;

  // event.waitUntil — NOT a bare fetch. The runtime may tear the invocation
  // down as soon as you return a response, which would kill the report
  // mid-flight. waitUntil extends the lifetime until it settles; it still
  // never delays your response.
  event.waitUntil(
    fetch(ENDPOINT, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        ...(opts.token ? { Authorization: `Bearer ${opts.token}` } : {}),
      },
      body: JSON.stringify({
        websiteId: opts.websiteId,
        host: request.nextUrl.hostname,
        pathname: request.nextUrl.pathname, // path only — no query strings
        userAgent: ua,
        // Lets Dupes confirm the crawler is who it claims to be.
        ip:
          request.headers.get("x-real-ip") ??
          request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ??
          null,
      }),
    }).catch(() => {}),
  );
}

// proxy.ts (Next.js 16) or middleware.ts (Next.js 15 and below).
import { NextResponse, type NextFetchEvent, type NextRequest } from "next/server";
import { trackAICrawlerRequest } from "@/lib/dupes-ai-crawl";

// Note the second argument — you need it for waitUntil.
export function proxy(request: NextRequest, event: NextFetchEvent) {
  trackAICrawlerRequest(request, event, { websiteId: "dps_your_site_id" });

  // Crawler-facing files must pass straight through. If your app rewrites by
  // locale, /robots.txt would otherwise become /pt/robots.txt and break.
  const path = request.nextUrl.pathname;
  if (path === "/robots.txt" || path === "/sitemap.xml" || path === "/llms.txt") {
    return NextResponse.next();
  }

  // …your existing logic…
  return NextResponse.next();
}

export const config = {
  // Keep robots.txt / sitemap.xml / llms.txt IN the matcher — crawler hits on
  // those are the ones worth seeing.
  matcher: ["/((?!_next/static|_next/image|favicon.ico).*)"],
};

// Express / Node — same idea, any framework with a request hook.
app.use((req, res, next) => {
  const ua = req.get("user-agent") || "";
  if (/bot|crawler|spider|GPT|Claude|Perplexity|OAI-|ChatGPT|CCBot/i.test(ua)) {
    res.on("finish", () => {
      fetch("https://dupes-gilt.vercel.app//api/collect/server", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({
          websiteId: "dps_your_site_id",
          host: req.hostname,
          pathname: req.path,
          userAgent: ua,
          ip: req.ip,
          statusCode: res.statusCode, // 404s on cited URLs are worth knowing
        }),
      }).catch(() => {});
    });
  }
  next();
});

POST https://dupes-gilt.vercel.app//api/collect/server
Content-Type: application/json

{
  "websiteId":  "dps_your_site_id",
  "host":       "example.com",
  "pathname":   "/pricing",
  "userAgent":  "Mozilla/5.0 (compatible; GPTBot/1.1; +https://openai.com/gptbot)",
  "ip":         "203.0.113.10",
  "statusCode": 200
}