Polite Web Scraper

PythonScraping

Collects 100 book listings (title, price, rating, stock) from books.toscrape.com, a sandbox site made for scraping practice, with a delay between requests.

Run it

cd samples/book-scraper
python scrape_books.py 5   # pages; needs requests, beautifulsoup4

Files: books.csv · scrape_books.py

Output (first 10 of 100 rows)

title	price_gbp	rating	in_stock	url
A Light in the Attic	51.77	3	True	https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html
Tipping the Velvet	53.74	1	True	https://books.toscrape.com/catalogue/tipping-the-velvet_999/index.html
Soumission	50.1	1	True	https://books.toscrape.com/catalogue/soumission_998/index.html
Sharp Objects	47.82	4	True	https://books.toscrape.com/catalogue/sharp-objects_997/index.html
Sapiens: A Brief History of Humankind	54.23	5	True	https://books.toscrape.com/catalogue/sapiens-a-brief-history-of-humankind_996/index.html
The Requiem Red	22.65	1	True	https://books.toscrape.com/catalogue/the-requiem-red_995/index.html
The Dirty Little Secrets of Getting Your Dream Job	33.34	4	True	https://books.toscrape.com/catalogue/the-dirty-little-secrets-of-getting-your-dream-job_994/index.html
The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull	17.93	3	True	https://books.toscrape.com/catalogue/the-coming-woman-a-novel-based-on-the-life-of-the-infamous-feminist-victoria-woodhull_993/index.html
The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics	22.6	4	True	https://books.toscrape.com/catalogue/the-boys-in-the-boat-nine-americans-and-their-epic-quest-for-gold-at-the-1936-berlin-olympics_992/index.html
The Black Maria	52.15	1	True	https://books.toscrape.com/catalogue/the-black-maria_991/index.html

Code

"""Scrape books.toscrape.com (a sandbox site built for scraping practice) politely: title, price, rating, stock, URL."""
import csv, time, sys
from urllib.parse import urljoin
import requests
from bs4 import BeautifulSoup

BASE = "https://books.toscrape.com/catalogue/page-{}.html"
RATING = {"One": 1, "Two": 2, "Three": 3, "Four": 4, "Five": 5}

def scrape(pages=5, out="books.csv"):
    s = requests.Session(); s.headers["User-Agent"] = "KirkDaviesStudio-sample-scraper/1.0"
    rows = []
    for n in range(1, pages + 1):
        url = BASE.format(n)
        r = s.get(url, timeout=20); r.raise_for_status(); r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "html.parser")
        for b in soup.select("article.product_pod"):
            a = b.h3.a
            rows.append({
                "title": a["title"],
                "price_gbp": float(b.select_one(".price_color").text.strip().lstrip("£Â")),
                "rating": RATING[b.select_one(".star-rating")["class"][1]],
                "in_stock": "In stock" in b.select_one(".availability").text,
                "url": urljoin(url, a["href"]),
            })
        print(f"page {n}: {len(rows)} books so far"); time.sleep(1)
    with open(out, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=rows[0].keys()); w.writeheader(); w.writerows(rows)
    print(f"saved {len(rows)} rows to {out}")

if __name__ == "__main__":
    scrape(int(sys.argv[1]) if len(sys.argv) > 1 else 5)