Build a resumable HTML scraping skill with bounded parallel fetching

"""Bounded Catalog108 HTML scraping skill example; Python 3.11+, no model calls.

This adapter is intentionally specific to the synthetic /blog listing markup.
It refuses changed required fields instead of silently returning partial records.
"""

import argparse
import concurrent.futures
import datetime as dt
from email.utils import parsedate_to_datetime
import hashlib
import html
import json
import os
from pathlib import Path
import re
import sqlite3
import threading
import time
from urllib.error import HTTPError, URLError
from urllib.parse import urljoin, urlparse
from urllib.request import Request, urlopen
from urllib.robotparser import RobotFileParser


ORIGIN = "https://practice.scrapingcentral.com"
START = ORIGIN + "/blog"
AGENT = "BIGCHANGE-guide-example/1.0 (bounded practice crawl)"
PARSER_VERSION = "catalog108-blog-html-v2"
POST_LIST = re.compile(r'<ul class="post-list">(.*?)</ul>', re.S)
CARD = re.compile(r'<li\b([^>]*)>(.*?)</li>', re.S)
CARD_CLASS = re.compile(r'\bclass="([^"]+)"')
TITLE = re.compile(r'<h2>\s*<a href="([^"]+)">(.*?)</a>\s*</h2>', re.S)
AUTHOR_DATE = re.compile(r'<p class="post-meta">\s*<a href="[^"]+">(.*?)</a>\s*·\s*<time datetime="(\d{4}-\d\d-\d\d)">', re.S)
EXCERPT = re.compile(r'<p class="post-excerpt">(.*?)</p>', re.S)
TAGS = re.compile(r'<p class="post-tags">(.*?)</p>', re.S)
TAG = re.compile(r'<a class="tag" href="[^"]+">(.*?)</a>', re.S)
NEXT = re.compile(r'<a rel="next" href="([^"]+)">', re.S)
PAGE_INFO = re.compile(r'<title>Blog page (\d+) of (\d+) - Catalog108</title>')


def page_url(number):
    return START if number == 1 else f"{ORIGIN}/blog/page/{number}"


def page_number(url):
    return 1 if url == START else int(urlparse(url).path.rsplit("/", 1)[-1])


class FetchGate:
    """One rate/cooldown clock shared by every HTTP worker and retry."""
    def __init__(self, interval, state_dir):
        self.interval = interval
        self.next_at = 0.0
        self.lock = threading.Lock()
        self.in_flight = 0
        self.maximum = 0
        self.guard_file = state_dir / "server-not-before.json"
        self.server_not_before = 0.0
        if self.guard_file.exists():
            value = json.loads(self.guard_file.read_text(encoding="utf-8"))["notBeforeUtc"]
            self.server_not_before = float(value)

    def slot(self, budget):
        while True:
            budget.check()
            with self.lock:
                now = time.monotonic()
                server_wait = self.server_not_before - time.time()
                if now >= self.next_at and server_wait <= 0:
                    self.next_at = now + self.interval
                    return
                wait = max(self.next_at - now, server_wait, 0)
            time.sleep(min(wait, budget.remaining()))  # Recheck shared cooldown.

    def cooldown(self, seconds):
        with self.lock:
            self.server_not_before = max(self.server_not_before, time.time() + seconds)
            temporary = self.guard_file.with_suffix(".tmp")
            temporary.write_text(json.dumps({"notBeforeUtc": self.server_not_before}), encoding="utf-8")
            os.replace(temporary, self.guard_file)

    def start(self):
        with self.lock:
            self.in_flight += 1
            self.maximum = max(self.maximum, self.in_flight)

    def end(self):
        with self.lock:
            self.in_flight -= 1


class DeadlineExpired(Exception):
    pass


class Budget:
    def __init__(self, deadline):
        self.deadline = deadline

    def remaining(self):
        if self.deadline is None:
            return float("inf")
        return max(0.0, (self.deadline - dt.datetime.now(dt.timezone.utc)).total_seconds())

    def check(self):
        if self.remaining() <= 0:
            raise DeadlineExpired("UTC collection deadline reached")


def retry_after_seconds(value):
    if value is None:
        return None
    value = value.strip()
    if value.isdigit():
        return float(value)
    try:
        date = parsedate_to_datetime(value)
        if date.tzinfo is None:
            return None
        return max(0.0, (date - dt.datetime.now(dt.timezone.utc)).total_seconds())
    except (TypeError, ValueError, OverflowError):
        return None


def clean(value):
    return html.unescape(re.sub(r"<[^>]*>", "", value)).strip()


def permitted_url(url):
    parsed = urlparse(url)
    if parsed.scheme != "https" or parsed.netloc != "practice.scrapingcentral.com":
        raise ValueError("Off-site or non-HTTPS cursor rejected")
    if not re.fullmatch(r"/blog(?:/page/[1-9]\d*)?/?", parsed.path) or parsed.query or parsed.fragment:
        raise ValueError("Cursor outside fixed listing path rejected")
    return url


def parse_card(fragment):
    title = TITLE.search(fragment)
    author_date = AUTHOR_DATE.search(fragment)
    excerpt = EXCERPT.search(fragment)
    tags = TAGS.search(fragment)
    if not (title and author_date and excerpt and tags):
        raise ValueError("Required post-summary markup changed")
    url = urljoin(ORIGIN, title.group(1))
    if urlparse(url).netloc != "practice.scrapingcentral.com" or not urlparse(url).path.startswith("/blog/"):
        raise ValueError("Post URL outside practice blog")
    date = dt.date.fromisoformat(author_date.group(2)).isoformat()
    result = {"id": url, "url": url, "title": clean(title.group(2)),
              "author": clean(author_date.group(1)), "publishedDate": date,
              "excerpt": clean(excerpt.group(1)),
              "tags": [clean(x).lstrip("#") for x in TAG.findall(tags.group(1))]}
    if not result["title"] or not result["author"] or not result["excerpt"]:
        raise ValueError("Empty required post field")
    return result


def parse_page(body, url, workers):
    text = body.decode("utf-8", errors="strict")
    page = PAGE_INFO.search(text)
    if not page or int(page.group(1)) != page_number(url) or not 1 <= int(page.group(1)) <= int(page.group(2)) <= 20:
        raise ValueError("Page number/total does not match fixed listing")
    listings = POST_LIST.findall(text)
    if len(listings) != 1:
        raise ValueError("Expected exactly one post-list")
    cards = list(CARD.finditer(listings[0]))
    if not cards or CARD.sub("", listings[0]).strip():
        raise ValueError("Unknown or malformed post-list child; no partial page allowed")
    fragments = []
    for card in cards:
        classes = CARD_CLASS.search(card.group(1))
        if not classes or "post-summary" not in classes.group(1).split():
            raise ValueError("Unknown post-list card class; no partial page allowed")
        fragments.append(card.group(2))
    if not fragments:
        raise ValueError("No post-summary rows; layout may have changed")
    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
        rows = list(pool.map(parse_card, fragments))
    ids = [row["id"] for row in rows]
    if len(ids) != len(set(ids)):
        raise ValueError("Duplicate record identity within one page")
    match = NEXT.search(text)
    next_url = permitted_url(urljoin(url, match.group(1))) if match else None
    if next_url == url:
        raise ValueError("Self-repeating next cursor")
    expected = page_url(int(page.group(1)) + 1) if int(page.group(1)) < int(page.group(2)) else None
    if next_url != expected:
        raise ValueError("Next link does not match numbered pagination")
    return rows, next_url, int(page.group(2))


def request_bytes(url, budget, max_bytes=1_000_000):
    budget.check()
    req = Request(url, headers={"User-Agent": AGENT, "Accept": "text/html"})
    with urlopen(req, timeout=min(10, budget.remaining())) as response:
        if response.status != 200 or permitted_url(response.url) != url:
            raise ValueError("Unexpected response or redirect")
        chunks = []
        size = 0
        while True:
            budget.check()
            chunk = response.read(min(16384, max_bytes + 1 - size))
            budget.check()
            if not chunk:
                break
            chunks.append(chunk)
            size += len(chunk)
            if size > max_bytes:
                raise ValueError("Response exceeds one-megabyte bound")
        data = b"".join(chunks)
    if len(data) > max_bytes:
        raise ValueError("Response exceeds one-megabyte bound")
    return data


def fetch_with_retry(url, gate, budget):
    for attempt in range(3):
        budget.check()
        gate.slot(budget)
        budget.check()
        gate.start()
        try:
            return request_bytes(url, budget)
        except HTTPError as exc:
            retry = retry_after_seconds(exc.headers.get("Retry-After", ""))
            if retry is not None:
                gate.cooldown(retry)  # Persist even on the final failed attempt.
            if exc.code not in (429, 500, 502, 503, 504) or attempt == 2:
                raise
            if retry is None:
                gate.cooldown(2 ** attempt)
        except (URLError, TimeoutError):
            if attempt == 2:
                raise
            gate.cooldown(2 ** attempt)
        finally:
            gate.end()
    raise RuntimeError("Retry budget exhausted")


def fetch_fixture(url, gate, directory, budget):
    budget.check()
    gate.slot(budget)
    budget.check()
    gate.start()
    try:
        time.sleep(0.08)  # Synthetic network latency demonstrates concurrent fetches offline.
        budget.check()
        body = (directory / f"page{page_number(url)}.html").read_bytes()
        budget.check()
        return body
    finally:
        gate.end()


def check_robots(budget, gate):
    url = ORIGIN + "/robots.txt"
    body = None
    for attempt in range(3):
        budget.check()
        gate.slot(budget)
        budget.check()
        gate.start()
        try:
            req = Request(url, headers={"User-Agent": AGENT})
            with urlopen(req, timeout=min(10, budget.remaining())) as response:
                if response.status != 200 or response.url != url:
                    raise ValueError("Robots policy unavailable")
                budget.check()
                body = response.read(65537)
                budget.check()
            break
        except HTTPError as exc:
            retry = retry_after_seconds(exc.headers.get("Retry-After", ""))
            if retry is not None:
                gate.cooldown(retry)
            if exc.code not in (429, 500, 502, 503, 504) or attempt == 2:
                raise
            if retry is None:
                gate.cooldown(2 ** attempt)
        except (URLError, TimeoutError):
            if attempt == 2:
                raise
            gate.cooldown(2 ** attempt)
        finally:
            gate.end()
    if body is None:
        raise RuntimeError("Robots retry budget exhausted")
    if len(body) > 65536:
        raise ValueError("Robots policy too large")
    robot = RobotFileParser()
    robot.parse(body.decode("utf-8", errors="strict").splitlines())
    if not robot.can_fetch(AGENT, START):
        raise ValueError("Robots policy disallows blog")
    return robot


def database(path, config_hash):
    db = sqlite3.connect(path)
    db.execute("CREATE TABLE IF NOT EXISTS posts (id TEXT PRIMARY KEY, payload TEXT NOT NULL, sha256 TEXT NOT NULL)")
    db.execute("CREATE TABLE IF NOT EXISTS pages (url TEXT PRIMARY KEY, sha256 TEXT NOT NULL, next_url TEXT, total INTEGER NOT NULL, row_count INTEGER NOT NULL)")
    db.execute("CREATE TABLE IF NOT EXISTS state (id INTEGER PRIMARY KEY CHECK(id=1), next_url TEXT, status TEXT NOT NULL, errors INTEGER NOT NULL, config_hash TEXT NOT NULL)")
    db.execute("INSERT OR IGNORE INTO state VALUES (1, ?, 'new', 0, ?)", (START, config_hash))
    db.commit()
    if db.execute("SELECT config_hash FROM state WHERE id=1").fetchone()[0] != config_hash:
        db.close()
        raise ValueError("Parser or date filter changed; use a new state directory after reviewing prior output")
    return db


def acquire_lock(path):
    try:
        fd = os.open(str(path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
    except FileExistsError as exc:
        raise RuntimeError("Another run holds the lock; inspect the process before manual removal") from exc
    os.write(fd, str(os.getpid()).encode())
    os.close(fd)


def main(argv=None):
    parser = argparse.ArgumentParser()
    parser.add_argument("--state-dir", type=Path, required=True)
    parser.add_argument("--max-pages", type=int, default=3)
    parser.add_argument("--max-records", type=int, default=200)
    parser.add_argument("--max-errors", type=int, default=1)
    parser.add_argument("--workers", type=int, default=4)
    parser.add_argument("--fetch-workers", type=int, default=1)
    parser.add_argument("--min-published", type=dt.date.fromisoformat)
    parser.add_argument("--stop-at-utc", type=lambda x: dt.datetime.fromisoformat(x.replace("Z", "+00:00")))
    parser.add_argument("--fixture-dir", type=Path, help="OFFLINE TEST ONLY: page1.html, page2.html ...")
    args = parser.parse_args(argv)
    if not (1 <= args.max_pages <= 20 and 1 <= args.max_records <= 200 and
            1 <= args.max_errors <= 3 and 1 <= args.workers <= 8 and 1 <= args.fetch_workers <= 2):
        parser.error("Bounds exceeded")
    if args.stop_at_utc and args.stop_at_utc.utcoffset() is None:
        parser.error("--stop-at-utc needs timezone or Z")
    args.state_dir.mkdir(parents=True, exist_ok=True)
    lock = args.state_dir / "run.lock"
    acquire_lock(lock)
    db = None
    try:
        budget = Budget(args.stop_at_utc)
        config = {"target": START, "parser": PARSER_VERSION,
                  "minPublished": args.min_published.isoformat() if args.min_published else None}
        config_hash = hashlib.sha256(json.dumps(config, sort_keys=True).encode()).hexdigest()
        db = database(args.state_dir / "scrape.sqlite3", config_hash)
        cursor, status, errors = db.execute("SELECT next_url,status,errors FROM state WHERE id=1").fetchone()
        if status == "complete":
            cursor, errors = START, 0  # A new run refreshes from page one.
        cursor = permitted_url(cursor or START)
        pages = 0
        seen_run = set()
        gate = FetchGate(0.02 if args.fixture_dir else 1.0, args.state_dir)
        fetch = (lambda url: fetch_fixture(url, gate, args.fixture_dir, budget)) if args.fixture_dir else (lambda url: fetch_with_retry(url, gate, budget))
        pending = {}
        reason = "max-pages"
        robot = None
        try:
            robot = None if args.fixture_dir else check_robots(budget, gate)
            budget.check()
        except DeadlineExpired:
            reason = "date-deadline"
        except (OSError, ValueError, UnicodeError, RuntimeError) as exc:
            if budget.remaining() <= 0:
                reason = "date-deadline"
            else:
                errors += 1
                with db:
                    db.execute("UPDATE state SET status='error',errors=? WHERE id=1", (errors,))
                reason = "robots-denied" if str(exc) == "Robots policy disallows blog" else f"error:{type(exc).__name__}:{str(exc)[:120]}"
        if reason == "max-pages":
            fetch_pool = concurrent.futures.ThreadPoolExecutor(max_workers=args.fetch_workers)
            try:
                while pages < args.max_pages:
                    try:
                        budget.check()
                        if cursor in seen_run:
                            reason = "duplicate-page-cursor"
                            break
                        if robot and not robot.can_fetch(AGENT, cursor):
                            reason = "robots-denied"
                            break
                        seen_run.add(cursor)
                        future = pending.pop(cursor, None) or fetch_pool.submit(fetch, cursor)
                        try:
                            body = future.result(timeout=None if budget.deadline is None else budget.remaining())
                        except concurrent.futures.TimeoutError as exc:
                            # concurrent.futures.TimeoutError is the builtin
                            # TimeoutError alias: a completed worker can raise
                            # that same type for an exhausted network failure.
                            if budget.remaining() <= 0 and not future.done():
                                raise DeadlineExpired("Fetch did not finish before UTC deadline") from exc
                            raise
                        budget.check()
                        content_hash = hashlib.sha256(body).hexdigest()
                        cached = db.execute("SELECT sha256,next_url,total,row_count FROM pages WHERE url=?", (cursor,)).fetchone()
                        unchanged = bool(cached and cached[0] == content_hash)
                        if unchanged:
                            eligible, next_url, total, row_count = [], cached[1], cached[2], cached[3]
                        else:
                            rows, next_url, total = parse_page(body, cursor, args.workers)
                            budget.check()
                            row_count = len(rows)
                            eligible = [r for r in rows if not args.min_published or r["publishedDate"] >= args.min_published.isoformat()]
                        budget.check()
                        stored_ids = {r[0] for r in db.execute("SELECT id FROM posts")}
                        if len(stored_ids | {r["id"] for r in eligible}) > args.max_records:
                            reason = "max-records-before-page"
                            break  # Never advance past a partly saved page.
                        # One writer; an expiry inside this transaction rolls it back.
                        with db:
                            for row in eligible:
                                budget.check()
                                payload = json.dumps(row, sort_keys=True, ensure_ascii=False)
                                db.execute("INSERT INTO posts VALUES(?,?,?) ON CONFLICT(id) DO UPDATE SET payload=excluded.payload,sha256=excluded.sha256", (row["id"], payload, hashlib.sha256(payload.encode()).hexdigest()))
                            budget.check()
                            db.execute("INSERT INTO pages(url,sha256,next_url,total,row_count) VALUES(?,?,?,?,?) ON CONFLICT(url) DO UPDATE SET sha256=excluded.sha256,next_url=excluded.next_url,total=excluded.total,row_count=excluded.row_count", (cursor, content_hash, next_url, total, row_count))
                            db.execute("UPDATE state SET next_url=?,status=?,errors=0 WHERE id=1", (next_url, "complete" if not next_url else "paused"))
                            budget.check()
                        pages += 1
                        errors = 0
                        reason = "max-pages"  # A recovered page clears the earlier transient stop.
                        if not next_url:
                            reason = "no-next"
                            break
                        if db.execute("SELECT COUNT(*) FROM posts").fetchone()[0] >= args.max_records:
                            reason = "max-records"
                            break
                        # Speculative numbered pages are never committed out of order.
                        for number in range(page_number(cursor) + 1, total + 1):
                            budget.check()
                            if len(pending) >= args.fetch_workers or pages + len(pending) >= args.max_pages:
                                break
                            candidate = page_url(number)
                            if candidate not in pending and candidate != cursor:
                                if robot and not robot.can_fetch(AGENT, candidate):
                                    break
                                pending[candidate] = fetch_pool.submit(fetch, candidate)
                        cursor = next_url
                    except DeadlineExpired:
                        reason = "date-deadline"
                        break
                    except (OSError, ValueError, UnicodeError) as exc:
                        if budget.remaining() <= 0:
                            reason = "date-deadline"
                            break
                        errors += 1
                        seen_run.discard(cursor)
                        with db:
                            db.execute("UPDATE state SET next_url=?,status='error',errors=? WHERE id=1", (cursor, errors))
                        reason = f"error:{type(exc).__name__}:{str(exc)[:120]}"
                        if errors >= args.max_errors:
                            break
            finally:
                for future in pending.values():
                    future.cancel()
                fetch_pool.shutdown(wait=True, cancel_futures=True)
        items = [json.loads(row[0]) for row in db.execute("SELECT payload FROM posts ORDER BY id LIMIT ?", (args.max_records,))]
        output = {"target": START, "sourceType": "synthetic HTML practice site", "reason": reason,
                  "pagesThisRun": pages, "recordsStored": db.execute("SELECT COUNT(*) FROM posts").fetchone()[0],
                  "nextCursor": db.execute("SELECT next_url FROM state WHERE id=1").fetchone()[0],
                  "workerScope": "bounded parallel HTTP fetch, shared global rate/cooldown, ordered single SQLite writer",
                  "fetchWorkers": args.fetch_workers, "maxFetchInFlight": gate.maximum,
                  "minPublished": args.min_published.isoformat() if args.min_published else None,
                  "items": items}
        print(json.dumps(output, ensure_ascii=False, sort_keys=True, indent=2))
        return 1 if reason.startswith("error:") or reason == "robots-denied" else 0
    finally:
        if db:
            db.close()
        lock.unlink(missing_ok=True)


if __name__ == "__main__":
    raise SystemExit(main())

---
name: catalog108-incremental-blog-crawl
description: Collect bounded dated cards from Catalog108's public HTML practice blog with a resumable cursor and checked JSON output.
disable-model-invocation: true
---

# Catalog108 incremental blog crawl

Invoke this skill manually as `/catalog108-incremental-blog-crawl`. `disable-model-invocation: true` prevents automatic invocation, but a Claude Code invocation still requires Claude Code authentication and consumes that account's usage; the Python script itself calls no model and needs no model account. Follow [Claude Code skills](https://code.claude.com/docs/en/skills), [setup](https://code.claude.com/docs/en/setup) and [authentication](https://code.claude.com/docs/en/authentication) for the agent host. Save this complete file and the complete `scrape_blog.py` printed with it in `.claude/skills/catalog108-incremental-blog-crawl/`. Those two files are sufficient for the bounded live run described here.

Use only for `https://practice.scrapingcentral.com/blog` and its `/blog/page/N` listing links. The site identifies itself as a synthetic scraping practice target. The script performs plain HTML requests; it does not call an AI model or accept a different host. Do not adapt its permission result to another site.

From the directory containing this file, on Python 3.11 or newer:

~~~powershell
$deadlineUtc = (Get-Date).ToUniversalTime().AddMinutes(10).ToString('yyyy-MM-ddTHH:mm:ssZ')
python .\scrape_blog.py --state-dir .\private-state --max-pages 3 --max-records 200 --fetch-workers 2 --workers 4 --stop-at-utc $deadlineUtc
~~~

When Claude invokes the skill from another working directory, use the host's documented `${CLAUDE_SKILL_DIR}` expansion to locate `scrape_blog.py`, and pass an explicit project-private state path. The commands above instead assume the shell's working directory is the skill directory.

For a date-bounded collection, add `--min-published 2026-01-01` to retain cards on or after that publication date and a dynamically computed future UTC deadline if a collection deadline is needed. For PowerShell, `$deadlineUtc = (Get-Date).ToUniversalTime().AddMinutes(10).ToString('yyyy-MM-ddTHH:mm:ssZ')`; then add `--stop-at-utc $deadlineUtc`. This site's cards are **not** in verified descending date order, so the publication cutoff filters cards while the crawler keeps following pages; it does not falsely stop at the first old card. Set `--max-pages`, `--max-records`, and `--stop-at-utc` before starting. The default is three pages, 200 total stored records, one HTTP request per second including retries, at most three attempts and a ten-second request timeout. The default single fetch worker is serial. `--fetch-workers 2` uses two in-flight requests after page one establishes numbered pagination. Robots retrieval and both workers share one request-start clock and Retry-After cooldown; the ordered single writer commits a page only after its actual predecessor validates the next link. `--workers` controls card parsing separately. No speed gain is claimed.

The script checks `robots.txt` on each online invocation, rejects off-host cursors, limits responses to 1 MB, handles 429 and 5xx with bounded retry count and full server `Retry-After` (delta seconds or HTTP date), and fails closed on required-markup changes. Its sole writer commits each page's records, content hash and next cursor in one SQLite transaction. A lock file prevents a second process using the same state directory. If a process crashes, inspect that process before removing a stale lock manually; never delete a live lock. SQLite keeps the last complete page. A resumed invocation starts at its next cursor. A completed invocation begins again at page one, skips card extraction for byte-identical previously validated pages using saved page metadata, and upserts stable post URLs when bytes change; it still fetches pages to check whether content changed. Changing the parser version or publication cutoff requires a fresh state directory so a cache cannot silently reuse the wrong extraction rules.

The JSON stdout contains `items` with URL ID, title, author, published date, excerpt and tags, plus stored count, next cursor, pages in this run, and exact stop reason. Save it privately with redirection if needed; quote/card text remains third-party content even though the practice site is synthetic. Success is a parseable JSON object, the expected bounded stop reason, nonempty required fields, unique IDs and an inspected sample against the current HTML. `error:*` (including exhausted transport timeouts or robots HTTP failures) or `robots-denied` returns exit code 1. A `max-records-before-page` stop keeps the entire next page pending; after inspecting the saved count, raise `--max-records` on the same state to resume. No output here is evidence that an AI model used the skill correctly; if an agent host invokes this script, inspect its actual call and result separately.

Our validation used three captured HTML pages and synthetic failure cases. For your own check, inspect the JSON fields and one card against the current listing, then rerun using the same state directory and a fresh future deadline. The deadline prevents new requests and late page commits; OS/network cleanup can finish after it, so the Python process is not guaranteed to exit by that instant. A valid server not-before time is saved locally and respected on a later run; if it exceeds the deadline, the run stops at `date-deadline` with an unchanged cursor. The offline test does not establish the site's future markup, online uptime or any external production scraper.

$deadlineUtc = (Get-Date).ToUniversalTime().AddMinutes(10).ToString('yyyy-MM-ddTHH:mm:ssZ')
python .\scrape_blog.py --state-dir .\private-state --max-pages 3 --max-records 200 --max-errors 1 --fetch-workers 2 --workers 4 --min-published 2026-01-01 --stop-at-utc $deadlineUtc > .\result.json
Get-Content .\result.json | ConvertFrom-Json | Select-Object reason,pagesThisRun,recordsStored,nextCursor