Find broken links on a website

import json
import time
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
from html.parser import HTMLParser
from urllib.parse import urldefrag, urljoin, urlparse

import requests

API_KEY = "YOUR_ZENROWS_API_KEY"
START_URL = "https://www.scrapingcourse.com/ecommerce/"
CRAWLS = "https://api.zenrows.com/v1/crawls"
HEADERS = {"X-API-Key": API_KEY}
SKIP = ["add-to-cart"]  # links you don't want checked

class LinkParser(HTMLParser):
    def __init__(self):
        super().__init__()
        self.hrefs = []

    def handle_starttag(self, tag, attrs):
        if tag == "a":
            self.hrefs += [value for name, value in attrs if name == "href" and value]

# 1. Crawl the site and keep each page's HTML
response = requests.post(
    CRAWLS,
    headers=HEADERS,
    json={
        "url": START_URL,
        "include_patterns": ["/product/"],
        "max_items": 3,
        "max_pages": 6,
        "output_format": "html",
    },
    timeout=30,
)
response.raise_for_status()
crawl_id = response.json()["crawl_id"]
while requests.get(f"{CRAWLS}/{crawl_id}", headers=HEADERS, params={"limit": 1}, timeout=30).json()["status"] == "running":
    time.sleep(5)

# 2. Collect every link on the site's own domain, and the pages it appears on
linked_from = defaultdict(set)
download = requests.get(f"{CRAWLS}/{crawl_id}/download", headers=HEADERS, timeout=120)
for line in download.text.splitlines():
    page = json.loads(line)
    if page.get("content_status") != "fetched":
        continue
    parser = LinkParser()
    parser.feed(page["content"])
    for href in parser.hrefs:
        link = urldefrag(urljoin(page["url"], href)).url
        if urlparse(link).netloc == urlparse(START_URL).netloc and not any(s in link for s in SKIP):
            linked_from[link].add(page["url"])
print(f"{len(linked_from)} links found on the crawled pages")

# 3. Check each link: allowed_status_codes returns 404 and 410 pages with their status code
def check(link: str) -> tuple[str, int]:
    params = {"apikey": API_KEY, "url": link, "mode": "auto", "allowed_status_codes": "404,410"}
    # stream=True: the status is in the headers, so the body is never downloaded
    with requests.get("https://api.zenrows.com/v1/", params=params, timeout=180, stream=True) as response:
        return link, int(response.headers.get("Zr-Status-Code", response.status_code))

with ThreadPoolExecutor(max_workers=5) as pool:
    statuses = dict(pool.map(check, linked_from))

# 4. Report the broken links and where they appear
broken = {link: status for link, status in statuses.items() if status >= 400}
print(f"{len(statuses)} links checked, {len(broken)} broken")
for link, status in sorted(broken.items()):
    print(f"\n{status} {link}")
    for source in sorted(linked_from[link]):
        print(f"  linked from {source}")

curl -s -o /dev/null -D - "https://api.zenrows.com/v1/?apikey=YOUR_ZENROWS_API_KEY&url=https%3A%2F%2Fwww.scrapingcourse.com%2Fecommerce%2Fproduct%2Fdoes-not-exist%2F&mode=auto&allowed_status_codes=404,410" | grep -i "^zr-status-code"

Zr-Status-Code: 404

import time

import requests

BATCH = "https://async.api.zenrows.com/v1"
HEADERS = {"X-API-Key": "YOUR_ZENROWS_API_KEY"}
links = [
    "https://www.scrapingcourse.com/ecommerce/product/aeon-capri/",
    "https://www.scrapingcourse.com/ecommerce/product/does-not-exist/",
]

job = requests.post(
    f"{BATCH}/jobs",
    headers=HEADERS,
    json={
        "type": "regular",
        "status": "closed",
        "zenrows_params": {"mode": "auto"},
        "tasks": [{"url": link} for link in links],
    },
    timeout=60,
).json()

while requests.get(f"{BATCH}/jobs/{job['job_id']}", headers=HEADERS, timeout=30).json()["latest_run"]["status"] not in ("completed", "failed", "stopped"):
    time.sleep(5)

failed = requests.get(f"{BATCH}/jobs/{job['job_id']}/results", headers=HEADERS, params={"status": "failed"}, timeout=30).json()
for row in failed["results"]:
    print(row["error"]["status"], row["error"]["code"], row["url"])

404 RESP002 https://www.scrapingcourse.com/ecommerce/product/does-not-exist/