Turn search and listing pages into JSON

import json
import requests

params = {
    "apikey": "YOUR_ZENROWS_API_KEY",
    "url": "https://www.scrapingcourse.com/ecommerce/?s=jacket&post_type=product",
    "mode": "auto",
    "extract": "auto",
    "extract_fields": "products.name,products.price,products.product_url,page_context",
}

response = requests.get("https://api.zenrows.com/v1/", params=params, timeout=180)
response.raise_for_status()
print(json.dumps(response.json()["parsed"], indent=2))

const params = new URLSearchParams({
  apikey: "YOUR_ZENROWS_API_KEY",
  url: "https://www.scrapingcourse.com/ecommerce/?s=jacket&post_type=product",
  mode: "auto",
  extract: "auto",
  extract_fields: "products.name,products.price,products.product_url,page_context",
});

const response = await fetch(`https://api.zenrows.com/v1/?${params}`, {
  signal: AbortSignal.timeout(180_000),
});
if (!response.ok) throw new Error(`${response.status}: ${await response.text()}`);

const { parsed } = await response.json();
console.log(JSON.stringify(parsed, null, 2));

curl "https://api.zenrows.com/v1/?apikey=YOUR_ZENROWS_API_KEY&url=https%3A%2F%2Fwww.scrapingcourse.com%2Fecommerce%2F%3Fs%3Djacket%26post_type%3Dproduct&mode=auto&extract=auto&extract_fields=products.name%2Cproducts.price%2Cproducts.product_url%2Cpage_context"

zenrows extract "https://www.scrapingcourse.com/ecommerce/?s=jacket&post_type=product"

import json
import requests

SEARCH = "https://www.scrapingcourse.com/ecommerce/{page}?s=jacket&post_type=product"

def page_url(number):
    return SEARCH.format(page="" if number == 1 else f"page/{number}/")

def extract(url):
    params = {
        "apikey": "YOUR_ZENROWS_API_KEY",
        "url": url,
        "mode": "auto",
        "extract": "auto",
        "extract_fields": "products.name,products.price,products.product_url,page_context.total_pages",
    }
    response = requests.get("https://api.zenrows.com/v1/", params=params, timeout=180)
    response.raise_for_status()
    return response.json()["parsed"]

first = extract(page_url(1))
total_pages = first["page_context"]["total_pages"]
rows = {p["product_url"]: p for p in first["products"]}

for number in range(2, total_pages + 1):
    parsed = extract(page_url(number))
    for product in parsed["products"]:
        rows[product["product_url"]] = product
    print(f"page {number}/{total_pages}: {len(rows)} products so far")

dataset = list(rows.values())
with open("jackets.json", "w") as f:
    json.dump(dataset, f, indent=2)
print(f"Saved {len(dataset)} products from {total_pages} pages")

import { writeFile } from "node:fs/promises";

const pageUrl = (n) =>
  `https://www.scrapingcourse.com/ecommerce/${n === 1 ? "" : `page/${n}/`}?s=jacket&post_type=product`;

async function extract(url) {
  const params = new URLSearchParams({
    apikey: "YOUR_ZENROWS_API_KEY",
    url,
    mode: "auto",
    extract: "auto",
    extract_fields: "products.name,products.price,products.product_url,page_context.total_pages",
  });
  const response = await fetch(`https://api.zenrows.com/v1/?${params}`, {
    signal: AbortSignal.timeout(180_000),
  });
  if (!response.ok) throw new Error(`${response.status}: ${await response.text()}`);
  return (await response.json()).parsed;
}

const first = await extract(pageUrl(1));
const totalPages = first.page_context.total_pages;
const rows = new Map(first.products.map((p) => [p.product_url, p]));

for (let n = 2; n <= totalPages; n++) {
  const parsed = await extract(pageUrl(n));
  for (const product of parsed.products) rows.set(product.product_url, product);
  console.log(`page ${n}/${totalPages}: ${rows.size} products so far`);
}

const dataset = [...rows.values()];
await writeFile("jackets.json", JSON.stringify(dataset, null, 2));
console.log(`Saved ${dataset.length} products from ${totalPages} pages`);