Workers · WEB Crawl

import { wrapFetchWithPayment } from "@x402/fetch";
import { privateKeyToAccount } from "viem/accounts";

const account = privateKeyToAccount(process.env.PRIVATE_KEY);
const fetchWithPay = wrapFetchWithPayment(fetch, account);

const res = await fetchWithPay("https://agentdata-api.sander-van-aard.workers.dev/web/crawl", {
  method: "POST",
  headers: { "Content-Type": "application/json" },
  body: JSON.stringify({
    "max_pages": 3,
    "url": "https://example.com/"
  }),
});
const data = await res.json();

curl -X POST \
  "https://agentdata-api.sander-van-aard.workers.dev/web/crawl" \
  -H "Content-Type: application/json" \
  -d '{"max_pages":3,"url":"https://example.com/"}' \
  -H "X-PAYMENT: <signed x402 payment>"

{
  "crawled_at": "2026-07-30T08:00:00.000Z",
  "pages": [
    {
      "content": "# Example Domain\n\nThis domain is for use in illustrative examples, with a [More information](https://example.com/more) link.",
      "content_chars": 120,
      "final_url": "https://example.com/",
      "title": "Example Domain",
      "truncated": false,
      "url": "https://example.com/"
    }
  ],
  "pages_charged": 3,
  "pages_crawled": 1,
  "skipped": [],
  "start_url": "https://example.com/"
}