Toolvend · Parse Robots

import { wrapFetchWithPayment } from "@x402/fetch";
import { privateKeyToAccount } from "viem/accounts";

const account = privateKeyToAccount(process.env.PRIVATE_KEY);
const fetchWithPay = wrapFetchWithPayment(fetch, account);

const res = await fetchWithPay("https://toolvend.dev/parse/robots", {
  method: "GET",
});
const data = await res.json();

curl -X GET \
  "https://toolvend.dev/parse/robots?url=https%3A%2F%2Fexample.com" \
  -H "X-PAYMENT: <signed x402 payment>"

{
  "llmsTxt": {
    "present": false
  },
  "robots": {
    "aiAgentsBlocked": [
      "GPTBot"
    ],
    "groups": [],
    "sitemaps": []
  }
}