Run browser sessions in parallel

import asyncio
from playwright.async_api import async_playwright

CONNECTION_URL = "wss://browser.zenrows.com?apikey=YOUR_ZENROWS_API_KEY"
MAX_SESSIONS = 2  # browser sessions open at the same time

URLS = [
    "https://www.scrapingcourse.com/ecommerce/product/abominable-hoodie/",
    "https://www.scrapingcourse.com/ecommerce/product/adrienne-trek-jacket/",
    "https://www.scrapingcourse.com/ecommerce/product/aeon-capri/",
]

async def scrape(p, semaphore, url):
    async with semaphore:
        # Each connect_over_cdp call opens its own browser session with its own IP
        browser = await p.chromium.connect_over_cdp(CONNECTION_URL)
        try:
            page = await browser.contexts[0].new_page()
            await page.route("**/*.{png,jpg,jpeg,webp,gif,svg}", lambda route: route.abort())
            await page.goto(url, wait_until="domcontentloaded")
            return {
                "url": url,
                "name": await page.inner_text("h1.product_title"),
                "price": await page.inner_text(".summary .price"),
            }
        finally:
            await browser.close()

async def main():
    semaphore = asyncio.Semaphore(MAX_SESSIONS)
    async with async_playwright() as p:
        results = await asyncio.gather(
            *(scrape(p, semaphore, url) for url in URLS), return_exceptions=True
        )
    for result in results:
        print(result)

asyncio.run(main())

import { chromium } from "playwright-core";

const connectionURL = "wss://browser.zenrows.com?apikey=YOUR_ZENROWS_API_KEY";
const MAX_SESSIONS = 2; // browser sessions open at the same time

const urls = [
  "https://www.scrapingcourse.com/ecommerce/product/abominable-hoodie/",
  "https://www.scrapingcourse.com/ecommerce/product/adrienne-trek-jacket/",
  "https://www.scrapingcourse.com/ecommerce/product/aeon-capri/",
];

async function scrape(url) {
  // Each connectOverCDP call opens its own browser session with its own IP
  const browser = await chromium.connectOverCDP(connectionURL);
  try {
    const page = await browser.contexts()[0].newPage();
    await page.route("**/*.{png,jpg,jpeg,webp,gif,svg}", (route) => route.abort());
    await page.goto(url, { waitUntil: "domcontentloaded" });
    return {
      url,
      name: await page.innerText("h1.product_title"),
      price: await page.innerText(".summary .price"),
    };
  } finally {
    await browser.close();
  }
}

// MAX_SESSIONS workers pull URLs from a shared queue
const queue = [...urls];
const results = [];
await Promise.all(
  Array.from({ length: MAX_SESSIONS }, async () => {
    while (queue.length) {
      const url = queue.shift();
      try {
        results.push(await scrape(url));
      } catch (error) {
        results.push({ url, error: error.message });
      }
    }
  }),
);
console.log(results);