Run many requests concurrently

import re
from concurrent.futures import ThreadPoolExecutor, as_completed

import requests

API_KEY = "YOUR_ZENROWS_API_KEY"
SLUGS = [
    "abominable-hoodie", "adrienne-trek-jacket", "aeon-capri", "aero-daily-fitness-tee",
    "aether-gym-pant", "affirm-water-bottle", "aim-analog-watch", "ajax-full-zip-sweatshirt",
]
URLS = [f"https://www.scrapingcourse.com/ecommerce/product/{slug}/" for slug in SLUGS]

session = requests.Session()

def scrape(url):
    params = {"apikey": API_KEY, "url": url, "mode": "auto"}
    response = session.get("https://api.zenrows.com/v1/", params=params, timeout=180)
    response.raise_for_status()
    title = re.search(r"<h1[^>]*>(.*?)</h1>", response.text, re.S).group(1).strip()
    return title, response.headers

# Read your plan's limit from the first response, then size the pool to it.
first_title, headers = scrape(URLS[0])
limit = int(headers["Concurrency-Limit"])
print(f"Concurrency-Limit: {limit}")
print(first_title)

with ThreadPoolExecutor(max_workers=limit) as pool:
    futures = {pool.submit(scrape, url): url for url in URLS[1:]}
    for future in as_completed(futures):
        try:
            title, _ = future.result()
            print(title)
        except Exception as error:  # HTTP errors, timeouts, pages without an <h1>
            print(f"{futures[future]} failed: {error}")

const API_KEY = "YOUR_ZENROWS_API_KEY";
const slugs = [
  "abominable-hoodie", "adrienne-trek-jacket", "aeon-capri", "aero-daily-fitness-tee",
  "aether-gym-pant", "affirm-water-bottle", "aim-analog-watch", "ajax-full-zip-sweatshirt",
];
const urls = slugs.map((slug) => `https://www.scrapingcourse.com/ecommerce/product/${slug}/`);

async function scrape(url) {
  const params = new URLSearchParams({ apikey: API_KEY, url, mode: "auto" });
  const response = await fetch(`https://api.zenrows.com/v1/?${params}`, {
    signal: AbortSignal.timeout(180_000),
  });
  if (!response.ok) throw new Error(`${url} failed: ${response.status}`);
  const html = await response.text();
  const title = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/)[1].trim();
  return { title, limit: Number(response.headers.get("Concurrency-Limit")) };
}

// Read your plan's limit from the first response, then run that many workers.
const first = await scrape(urls[0]);
console.log(`Concurrency-Limit: ${first.limit}`);
console.log(first.title);

const queue = urls.slice(1);
async function worker() {
  while (queue.length > 0) {
    const url = queue.shift();
    try {
      const { title } = await scrape(url);
      console.log(title);
    } catch (error) {
      console.error(error.message);
    }
  }
}
await Promise.all(Array.from({ length: first.limit }, worker));

import asyncio
import re

from zenrows import ZenRowsClient

SLUGS = [
    "abominable-hoodie", "adrienne-trek-jacket", "aeon-capri", "aero-daily-fitness-tee",
    "aether-gym-pant", "affirm-water-bottle", "aim-analog-watch", "ajax-full-zip-sweatshirt",
]
URLS = [f"https://www.scrapingcourse.com/ecommerce/product/{slug}/" for slug in SLUGS]

# concurrency sets the requests in flight; retries re-sends transient failures.
client = ZenRowsClient("YOUR_ZENROWS_API_KEY", concurrency=5, retries=1)

async def main():
    responses = await asyncio.gather(
        *[client.fetch_async(url, params={"mode": "auto"}, timeout=180) for url in URLS]
    )
    for url, response in zip(URLS, responses):
        match = re.search(r"<h1[^>]*>(.*?)</h1>", response.text, re.S)
        print(response.status_code, match.group(1).strip() if match else url)

asyncio.run(main())

import { ZenRows } from "zenrows";

const slugs = [
  "abominable-hoodie", "adrienne-trek-jacket", "aeon-capri", "aero-daily-fitness-tee",
  "aether-gym-pant", "affirm-water-bottle", "aim-analog-watch", "ajax-full-zip-sweatshirt",
];
const urls = slugs.map((slug) => `https://www.scrapingcourse.com/ecommerce/product/${slug}/`);

// concurrency sets the requests in flight; retries re-sends transient failures.
const client = new ZenRows("YOUR_ZENROWS_API_KEY", { concurrency: 5, retries: 1 });

const results = await Promise.allSettled(urls.map((url) => client.fetch(url, { mode: "auto" })));

for (const [i, result] of results.entries()) {
  if (result.status === "rejected") {
    console.error(urls[i], result.reason);
    continue;
  }
  const html = await result.value.text();
  const title = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/)?.[1].trim();
  console.log(result.value.status, title);
}