Resolve common response codes

curl "https://api.zenrows.com/v1/?apikey=YOUR_ZENROWS_API_KEY&url=https%3A%2F%2Fhttpbin.io%2Fanything&mode=auto&js_render=true"

from concurrent.futures import ThreadPoolExecutor

import requests

API_KEY = "YOUR_ZENROWS_API_KEY"
urls = [f"https://www.scrapingcourse.com/ecommerce/page/{n}/" for n in range(1, 4)]

def fetch(url):
    params = {"apikey": API_KEY, "url": url, "mode": "auto"}
    response = requests.get("https://api.zenrows.com/v1/", params=params, timeout=180)
    return url, response.status_code, response.headers.get("Concurrency-Limit")

# Read your plan's limit from any response, then never run more requests at once
_, _, limit = fetch(urls[0])
workers = int(limit or 5)
print("Concurrency-Limit:", workers)

with ThreadPoolExecutor(max_workers=workers) as pool:
    for url, status, _ in pool.map(fetch, urls):
        print(status, url)

curl -H "X-API-Key: YOUR_ZENROWS_API_KEY" https://api.zenrows.com/v1/subscriptions/self/details

curl -i "https://api.zenrows.com/v1/?apikey=YOUR_ZENROWS_API_KEY&url=https%3A%2F%2Fwww.scrapingcourse.com%2Fecommerce%2Fproduct%2Fno-such-product%2F&allowed_status_codes=404,410"

import random
import time

import requests

API_KEY = "YOUR_ZENROWS_API_KEY"
RETRYABLE = {422, 429, 500, 502, 503, 504}

class FixNeeded(Exception):
    """A response that returns the same answer until you change something."""

def fetch(url, **extra):
    params = {
        "apikey": API_KEY,
        "url": url,
        "mode": "auto",                   # escalates rendering and proxies when a page needs it
        "allowed_status_codes": "404,410",  # return the target's own error pages as content
        **extra,
    }
    for attempt in range(4):
        try:
            response = requests.get("https://api.zenrows.com/v1/", params=params, timeout=180)
        except requests.Timeout:
            continue  # retry; keep the timeout at 180 s or more

        if response.ok:
            target_status = int(response.headers.get("Zr-Status-Code", 200))
            if target_status in (404, 410):
                return None  # the page is gone on the target site: drop it from your list
            return response.text

        if response.status_code in RETRYABLE:
            time.sleep(2 ** attempt + random.random())  # 1, 2, 4 s plus jitter
            continue
        error = response.json()
        # 400, 401, 402, 403, 404, 413: change the request, the key or the plan
        raise FixNeeded(f"{response.status_code} {error['code']}: {error['detail']} ({error['type']})")

    raise RuntimeError(f"Still failing after 4 attempts: {url}")

for url in [
    "https://www.scrapingcourse.com/ecommerce/",
    "https://www.scrapingcourse.com/ecommerce/product/no-such-product/",
]:
    body = fetch(url)
    print(url, "->", "gone" if body is None else f"{len(body)} bytes")

try:
    fetch("https://www.scrapingcourse.com/ecommerce/", js_render="true")
except FixNeeded as error:
    print(error)