Netzhandwerker Agent Tools · Abtest Evaluate

import { wrapFetchWithPayment } from "@x402/fetch";
import { privateKeyToAccount } from "viem/accounts";

const account = privateKeyToAccount(process.env.PRIVATE_KEY);
const fetchWithPay = wrapFetchWithPayment(fetch, account);

const res = await fetchWithPay("https://tools.halowerk.com/v1/abtest/evaluate", {
  method: "POST",
  headers: { "Content-Type": "application/json" },
  body: JSON.stringify({
    "confidence_level": 0.95,
    "control": {
      "conversions": 241,
      "visitors": 4820
    },
    "looks_taken": 3,
    "minimum_detectable_effect_rel": 0.1,
    "variant": {
      "conversions": 302,
      "visitors": 4790
    }
  }),
});
const data = await res.json();

curl -X POST \
  "https://tools.halowerk.com/v1/abtest/evaluate" \
  -H "Content-Type: application/json" \
  -d '{"confidence_level":0.95,"control":{"conversions":241,"visitors":4820},"looks_taken":3,"minimum_detectable_effect_rel":0.1,"variant":{"conversions":302,"visitors":4790}}' \
  -H "X-PAYMENT: <signed x402 payment>"

{
  "bayesian": {
    "credible_interval_control": [
      0.044204,
      0.056523
    ],
    "credible_interval_variant": [
      0.056515,
      0.070291
    ],
    "expected_loss_choosing_control": 0.01169,
    "expected_loss_choosing_variant": 0.000002,
    "method": "closed-form Beta-Binomial, no sampling",
    "posterior_control": {
      "alpha": 242,
      "beta": 4580,
      "mean": 0.050187
    },
    "posterior_variant": {
      "alpha": 303,
      "beta": 4489,
      "mean": 0.06323
    },
    "prior": {
      "alpha": 1,
      "beta": 1
    },
    "probability_exact": true,
    "probability_variant_better": 0.997196
  },
  "control": {
    "conversions": 241,
    "rate": 0.05,
    "visitors": 4820
  },
  "deterministic": true,
  "frequentist": {
    "absolute_lift": 0.013048,
    "alpha": 0.05,
    "alternative": "two_sided",
    "confidence_interval_absolute": [
      0.003816,
      0.02228
    ],
    "confidence_level": 0.95,
    "note": null,
    "p_value": 0.0056071682,
    "relative_lift": 0.26096,
    "significant": true,
    "standard_error": 0.00471,
    "z": 2.769911
  },
  "peeking": {
    "adjusted_alpha": 0.01666667,
    "correction": "bonferroni",
    "effective_false_positive_rate_unadjusted": 0.142625,
    "looks_taken": 3,
    "significant_after_adjustment": true
  },
  "sample_size": {
    "current_control": 4820,
    "current_variant": 4790,
    "minimum_detectable_effect_rel": 0.1,
    "power": 0.8,
    "required_per_group": 30245,
    "sufficient": false
  },
  "stateless": true,
  "variant": {
    "conversions": 302,
    "rate": 0.063048,
    "visitors": 4790
  },
  "verdict": {
    "decision": "keep_running",
    "reasons": [
      "Fewer observations per group (4790) than the 30245 the requested minimum detectable effect needs."
    ]
  }
}