Get notified when a Batch job finishes

import requests

response = requests.post(
    "https://async.api.zenrows.com/v1/jobs",
    headers={"X-API-Key": "YOUR_ZENROWS_API_KEY"},
    json={
        "type": "regular",
        "status": "closed",
        "zenrows_params": {"mode": "auto", "response_type": "markdown"},
        "tasks": [{"url": "https://www.scrapingcourse.com/ecommerce/product/abominable-hoodie/"}],
        "webhook": {"url": "https://your-app.com/webhooks/zenrows", "signature": True},
    },
    timeout=60,
)
response.raise_for_status()
job = response.json()
print(job["job_id"], job["webhook"])

curl -X POST "https://async.api.zenrows.com/v1/jobs" \
  -H "X-API-Key: YOUR_ZENROWS_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "type": "regular",
    "status": "closed",
    "zenrows_params": { "mode": "auto" },
    "tasks": [{ "url": "https://www.scrapingcourse.com/ecommerce/product/abominable-hoodie/" }],
    "webhook": { "url": "https://your-app.com/webhooks/zenrows", "signature": true }
  }'

# pip install flask requests
import base64, hashlib, hmac, os, time
import requests
from flask import Flask, abort, request

app = Flask(__name__)

# kid -> base64 secret, as returned once by POST /v1/hmac/keys/rotate
SECRETS = {os.environ["ZENROWS_WEBHOOK_KID"]: os.environ["ZENROWS_WEBHOOK_SECRET"]}
API_KEY = os.environ["ZENROWS_API_KEY"]
seen_events = set()  # use a database or Redis in production

def verify(raw_body: bytes, header: str) -> bool:
    parts = dict(item.split("=", 1) for item in header.split(",") if "=" in item)
    secret = SECRETS.get(parts.get("kid"))
    if not secret or not parts.get("t", "").isdigit() or abs(time.time() - int(parts["t"])) > 300:
        return False
    expected = hmac.new(
        base64.b64decode(secret), f"{parts['t']}.".encode() + raw_body, hashlib.sha256
    ).hexdigest()
    return hmac.compare_digest(expected, parts.get("v1", ""))

@app.post("/webhooks/zenrows")
def zenrows_webhook():
    raw_body = request.get_data()  # the exact bytes Zenrows signed
    if not verify(raw_body, request.headers.get("X-Signature", "")):
        abort(401)

    event_id = request.headers.get("X-ZenRows-Event-Id")
    if event_id in seen_events:
        return "", 200  # a repeat delivery: already handled
    seen_events.add(event_id)

    event = request.get_json()
    if event["event_type"] == "run.completed":
        results = requests.get(
            f"https://async.api.zenrows.com/v1/jobs/{event['job_id']}/results",
            headers={"X-API-Key": API_KEY},
            params={"status": "successful"},
            timeout=30,
        ).json()["results"]
        print(f"Run {event['run_id']} done: {event['stats']['successful']} pages, {len(results)} on first results page")
    elif event["event_type"] == "run.failed":
        print(f"Run {event['run_id']} failed: {event.get('failure_reason')} {event.get('failure_detail')}")
    return "", 200

if __name__ == "__main__":
    app.run(port=8000)

import crypto from "node:crypto";
import http from "node:http";

// kid -> base64 secret, as returned once by POST /v1/hmac/keys/rotate
const SECRETS = { [process.env.ZENROWS_WEBHOOK_KID]: process.env.ZENROWS_WEBHOOK_SECRET };
const API_KEY = process.env.ZENROWS_API_KEY;
const seenEvents = new Set(); // use a database or Redis in production

function verify(rawBody, header = "") {
  const parts = Object.fromEntries(
    header
      .split(",")
      .filter((item) => item.includes("="))
      .map((item) => [item.slice(0, item.indexOf("=")), item.slice(item.indexOf("=") + 1)]),
  );
  const secret = SECRETS[parts.kid];
  if (!secret || !/^\d+$/.test(parts.t ?? "")) return false;
  if (Math.abs(Date.now() / 1000 - Number(parts.t)) > 300) return false;
  const expected = crypto
    .createHmac("sha256", Buffer.from(secret, "base64"))
    .update(Buffer.concat([Buffer.from(`${parts.t}.`), rawBody]))
    .digest("hex");
  const a = Buffer.from(expected);
  const b = Buffer.from(parts.v1 ?? "");
  return a.length === b.length && crypto.timingSafeEqual(a, b);
}

http
  .createServer(async (req, res) => {
    if (req.method !== "POST" || req.url !== "/webhooks/zenrows") return res.writeHead(404).end();

    const chunks = [];
    for await (const chunk of req) chunks.push(chunk);
    const rawBody = Buffer.concat(chunks); // the exact bytes Zenrows signed

    if (!verify(rawBody, req.headers["x-signature"])) return res.writeHead(401).end();

    const eventId = req.headers["x-zenrows-event-id"];
    if (seenEvents.has(eventId)) return res.writeHead(200).end(); // a repeat delivery
    seenEvents.add(eventId);
    res.writeHead(200).end(); // acknowledge first, then do the work

    const event = JSON.parse(rawBody);
    if (event.event_type === "run.completed") {
      const page = await (
        await fetch(`https://async.api.zenrows.com/v1/jobs/${event.job_id}/results?status=successful`, {
          headers: { "X-API-Key": API_KEY },
        })
      ).json();
      console.log(`Run ${event.run_id} done: ${event.stats.successful} pages, ${page.results.length} on first results page`);
    } else if (event.event_type === "run.failed") {
      console.log(`Run ${event.run_id} failed: ${event.failure_reason} ${event.failure_detail}`);
    }
  })
  .listen(8000);

curl -X POST "https://async.api.zenrows.com/v1/webhook/test" \
  -H "X-API-Key: YOUR_ZENROWS_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{ "url": "https://your-app.com/webhooks/zenrows", "signature": true }'

const response = await fetch("https://async.api.zenrows.com/v1/webhook/test", {
  method: "POST",
  headers: { "X-API-Key": "YOUR_ZENROWS_API_KEY", "Content-Type": "application/json" },
  body: JSON.stringify({ url: "https://your-app.com/webhooks/zenrows", signature: true }),
});
console.log(response.status, await response.json());