Capture the network requests behind a page

import json
import requests

params = {
    "apikey": "YOUR_ZENROWS_API_KEY",
    "url": "https://www.scrapingcourse.com/javascript-rendering",
    "js_render": "true",
    "wait_for": ".product-name",
    "json_response": "true",
}

response = requests.get("https://api.zenrows.com/v1/", params=params, timeout=180)
response.raise_for_status()

data = response.json()
for call in data["xhr"]:
    print(call["method"], call["status_code"], call["url"])

# Keep the call that returns the product list and parse its body
products_call = next(call for call in data["xhr"] if "/ajax/products" in call["url"])
products = json.loads(products_call["body"])
print(len(products), "products")
for product in products[:3]:
    print(product["sku"], product["name"], product["regular_price"])

const params = new URLSearchParams({
  apikey: "YOUR_ZENROWS_API_KEY",
  url: "https://www.scrapingcourse.com/javascript-rendering",
  js_render: "true",
  wait_for: ".product-name",
  json_response: "true",
});

const response = await fetch(`https://api.zenrows.com/v1/?${params}`, {
  signal: AbortSignal.timeout(180_000),
});
if (!response.ok) throw new Error(`${response.status}: ${await response.text()}`);

const data = await response.json();
for (const call of data.xhr) console.log(call.method, call.status_code, call.url);

// Keep the call that returns the product list and parse its body
const productsCall = data.xhr.find((call) => call.url.includes("/ajax/products"));
const products = JSON.parse(productsCall.body);
console.log(products.length, "products");
for (const p of products.slice(0, 3)) console.log(p.sku, p.name, p.regular_price);

curl "https://api.zenrows.com/v1/?apikey=YOUR_ZENROWS_API_KEY&url=https%3A%2F%2Fwww.scrapingcourse.com%2Fjavascript-rendering&js_render=true&wait_for=.product-name&json_response=true"

import asyncio

from playwright.async_api import async_playwright

CONNECTION_URL = "wss://browser.zenrows.com?apikey=YOUR_ZENROWS_API_KEY"

async def main():
    captured = []

    async def on_response(response):
        if response.request.resource_type in ("xhr", "fetch"):
            captured.append(response)

    async with async_playwright() as p:
        browser = await p.chromium.connect_over_cdp(CONNECTION_URL)
        try:
            page = await browser.new_page()
            page.on("response", on_response)
            await page.goto("https://www.scrapingcourse.com/javascript-rendering")
            await page.wait_for_selector(".product-name")

            for response in captured:
                print(response.request.method, response.status, response.url[:80])

            products_response = next(r for r in captured if "/ajax/products" in r.url)
            products = await products_response.json()
            print(len(products), "products")
            for product in products[:3]:
                print(product["sku"], product["name"], product["regular_price"])
        finally:
            await browser.close()

asyncio.run(main())

import puppeteer from "puppeteer-core";

const connectionURL = "wss://browser.zenrows.com?apikey=YOUR_ZENROWS_API_KEY";

const browser = await puppeteer.connect({ browserWSEndpoint: connectionURL });
try {
  const page = await browser.newPage();

  const captured = [];
  page.on("response", (response) => {
    const type = response.request().resourceType();
    if (type === "xhr" || type === "fetch") captured.push(response);
  });

  await page.goto("https://www.scrapingcourse.com/javascript-rendering");
  await page.waitForSelector(".product-name");

  for (const response of captured) {
    console.log(response.request().method(), response.status(), response.url().slice(0, 80));
  }

  const productsResponse = captured.find((r) => r.url().includes("/ajax/products"));
  const products = await productsResponse.json();
  console.log(products.length, "products");
  for (const p of products.slice(0, 3)) console.log(p.sku, p.name, p.regular_price);
} finally {
  await browser.close();
}