Turn a whole catalog into a structured dataset

import csv
import json

from zenrows import ZenRowsBatchClient, ZenRowsCrawlClient

API_KEY = "YOUR_ZENROWS_API_KEY"
START_URL = "https://www.scrapingcourse.com/ecommerce/"

# 1. Crawl: discover every listing page of the catalog
crawler = ZenRowsCrawlClient(api_key=API_KEY)
crawl = crawler.create(
    START_URL,
    include_patterns=["/ecommerce/page/"],
    max_items=3,  # raise to cover the whole catalog
)
crawl = crawler.wait(crawl.crawl_id)
listing_urls = [START_URL] + [result.url for result in crawler.results(crawl.crawl_id)]
print(f"Crawl {crawl.status.value}: {len(listing_urls)} listing pages")

# 2. Batch: extract every listing page as structured JSON
batch = ZenRowsBatchClient(api_key=API_KEY)
job = batch.submit_regular(listing_urls, zenrows_params={"extract": "auto"})
run = job.run.wait(timeout=900)
print(f"Batch {run.data.status.value}: {run.stats.successful}/{run.stats.total} pages extracted")

# 3. Merge the products into one dataset, one row per product
products = {}
for task in run.results(status="successful"):
    parsed = json.loads(run.download_task_to_memory(task))["parsed"]
    for product in parsed.get("products", []):
        products[product["product_url"]] = product

rows = list(products.values())
fields = ["name", "price", "currency", "sku", "product_id", "product_url", "image_url"]

with open("catalog.csv", "w", newline="", encoding="utf-8") as f:
    writer = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore")
    writer.writeheader()
    writer.writerows(rows)

with open("catalog.json", "w", encoding="utf-8") as f:
    json.dump(rows, f, indent=2, ensure_ascii=False)

print(f"Saved {len(rows)} products to catalog.csv and catalog.json")

import { writeFile } from "node:fs/promises";
import { ZenRowsBatchClient, ZenRowsCrawlClient } from "zenrows";

const API_KEY = "YOUR_ZENROWS_API_KEY";
const START_URL = "https://www.scrapingcourse.com/ecommerce/";

// 1. Crawl: discover every listing page of the catalog
const crawler = new ZenRowsCrawlClient(API_KEY);
const crawl = await crawler.create({
  url: START_URL,
  includePatterns: ["/ecommerce/page/"],
  maxItems: 3, // raise to cover the whole catalog
});
const ended = await crawler.wait(crawl.crawl_id);
const listingUrls = [START_URL];
for await (const result of crawler.results(crawl.crawl_id)) listingUrls.push(result.url);
console.log(`Crawl ${ended.status}: ${listingUrls.length} listing pages`);

// 2. Batch: extract every listing page as structured JSON
const batch = new ZenRowsBatchClient(API_KEY);
const job = await batch.submitRegular(listingUrls, undefined, { zenrowsParams: { extract: "auto" } });
const run = await job.run.wait();
console.log(`Batch ${run.data.status}: ${run.data.stats.successful}/${run.data.stats.total} pages extracted`);

// 3. Merge the products into one dataset, one row per product
const products = new Map();
for await (const task of run.results({ status: "successful" })) {
  const { parsed } = JSON.parse(await run.downloadTaskToMemory(task));
  for (const product of parsed.products ?? []) products.set(product.product_url, product);
}

const rows = [...products.values()];
const fields = ["name", "price", "currency", "sku", "product_id", "product_url", "image_url"];
const csvCell = (value) => `"${String(value ?? "").replaceAll('"', '""')}"`;
const csv = [fields.join(","), ...rows.map((row) => fields.map((f) => csvCell(row[f])).join(","))].join("\n");

await writeFile("catalog.csv", csv);
await writeFile("catalog.json", JSON.stringify(rows, null, 2));
console.log(`Saved ${rows.length} products to catalog.csv and catalog.json`);