Robots.txt check · Robots Check

import { wrapFetchWithPayment } from "@x402/fetch";
import { privateKeyToAccount } from "viem/accounts";

const account = privateKeyToAccount(process.env.PRIVATE_KEY);
const fetchWithPay = wrapFetchWithPayment(fetch, account);

const res = await fetchWithPay("https://agent402.tools/api/robots-check", {
  method: "POST",
  headers: { "Content-Type": "application/json" },
  body: JSON.stringify({
    "url": "https://www.wikipedia.org/wiki/Robots.txt",
    "userAgent": "MyAgent"
  }),
});
const data = await res.json();

curl -X POST \
  "https://agent402.tools/api/robots-check" \
  -H "Content-Type: application/json" \
  -d '{"url":"https://www.wikipedia.org/wiki/Robots.txt","userAgent":"MyAgent"}' \
  -H "X-PAYMENT: <signed x402 payment>"

{
  "allowed": true,
  "matchedRule": null,
  "sitemaps": [
    "https://example.com/sitemap.xml"
  ]
}