2c  Web Scraping

# Python: Fetch a web page
import requests
from bs4 import BeautifulSoup

# Define a User-Agent (identifies your scraper)
# Be honest - some sites block generic Python requests
headers = {
    'User-Agent': 'Mozilla/5.0 (Research scraper for academic project; contact@university.edu)'
}

# Fetch the page
url = "https://example.com/data"
response = requests.get(url, headers=headers, timeout=30)

# Check if request was successful
if response.status_code == 200:
    print("Success! Page fetched.")
    # Parse the HTML
    soup = BeautifulSoup(response.text, 'html.parser')
else:
    print(f"Error: {response.status_code}")

# Python: Extract data from HTML

# Find a single element
title = soup.find('h1')
print(f"Title: {title.text}")

# Find by class name
price = soup.find('span', class_='price')

# Find by id
main_content = soup.find(id='main-content')

# Find ALL matching elements
all_prices = soup.find_all('span', class_='price')
for p in all_prices:
    print(p.text)

# Extract from a table
table = soup.find('table', class_='data-table')
rows = table.find_all('tr')
for row in rows[1:]:  # Skip header row
    cells = row.find_all('td')
    country = cells[0].text.strip()
    gdp = cells[1].text.strip()
    print(f"{country}: {gdp}")

# Extract attribute values (like href from links)
links = soup.find_all('a')
for link in links:
    href = link.get('href')
    print(href)

# Python: Handle pagination
import requests
from bs4 import BeautifulSoup
import time

# Store all results
all_data = []

# Loop through pages
for page_num in range(1, 11):  # Pages 1-10
    url = f"https://example.com/data?page={page_num}"

    # Be polite: wait between requests
    time.sleep(2)  # Wait 2 seconds

    response = requests.get(url, headers=headers)
    if response.status_code != 200:
        print(f"Error on page {page_num}, stopping")
        break

    soup = BeautifulSoup(response.text, 'html.parser')

    # Extract data from this page
    items = soup.find_all('div', class_='item')

    # Check if page is empty (end of data)
    if not items:
        print(f"No more data after page {page_num-1}")
        break

    for item in items:
        all_data.append({
            'name': item.find('h2').text,
            'value': item.find('span', class_='value').text
        })

    print(f"Page {page_num}: {len(items)} items")

print(f"Total: {len(all_data)} items collected")

# Python: Handle JavaScript with Selenium
# First: pip install selenium webdriver-manager

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup

# Set up Chrome in headless mode (no visible window)
options = webdriver.ChromeOptions()
options.add_argument('--headless')
driver = webdriver.Chrome(
    service=Service(ChromeDriverManager().install()),
    options=options
)

try:
    # Load the page
    driver.get("https://example.com/dynamic-data")

    # Wait for specific element to load (max 10 seconds)
    wait = WebDriverWait(driver, 10)
    wait.until(EC.presence_of_element_located((By.CLASS_NAME, "data-table")))

    # Now page is fully loaded - get the HTML
    html = driver.page_source
    soup = BeautifulSoup(html, 'html.parser')

    # Extract data as usual
    table = soup.find('table', class_='data-table')
    print("Data extracted successfully!")

finally:
    # Always close the browser
    driver.quit()

# Python: Polite scraping with rate limiting and caching
import requests
import time
import hashlib
import os
from pathlib import Path

class PoliteScraper:
    def __init__(self, delay=2, cache_dir="scraper_cache"):
        self.delay = delay
        self.cache_dir = Path(cache_dir)
        self.cache_dir.mkdir(exist_ok=True)
        self.last_request = 0
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Research Scraper (contact@university.edu)'
        })

    def _get_cache_path(self, url):
        # Create unique filename from URL
        url_hash = hashlib.md5(url.encode()).hexdigest()
        return self.cache_dir / f"{url_hash}.html"

    def fetch(self, url, use_cache=True):
        cache_path = self._get_cache_path(url)

        # Check cache first
        if use_cache and cache_path.exists():
            print(f"Using cached version of {url}")
            return cache_path.read_text()

        # Rate limiting
        elapsed = time.time() - self.last_request
        if elapsed < self.delay:
            time.sleep(self.delay - elapsed)

        # Make request
        response = self.session.get(url, timeout=30)
        self.last_request = time.time()

        if response.status_code == 200:
            # Cache the response
            cache_path.write_text(response.text)
            return response.text
        else:
            raise Exception(f"HTTP {response.status_code}")

# Usage
scraper = PoliteScraper(delay=3)
html = scraper.fetch("https://example.com/page1")
html = scraper.fetch("https://example.com/page2")

# Python: Robust error handling
import requests
import time
from requests.exceptions import RequestException

def fetch_with_retry(url, max_retries=3, base_delay=5):
    """Fetch URL with exponential backoff on failure."""

    for attempt in range(max_retries):
        try:
            response = requests.get(url, timeout=30)

            # Success
            if response.status_code == 200:
                return response.text

            # Rate limited - wait and retry
            elif response.status_code == 429:
                wait_time = int(response.headers.get('Retry-After', base_delay))
                print(f"Rate limited. Waiting {wait_time}s...")
                time.sleep(wait_time)

            # Server error - wait and retry
            elif response.status_code >= 500:
                delay = base_delay * (2 ** attempt)
                print(f"Server error. Retry in {delay}s...")
                time.sleep(delay)

            # Client error - don't retry
            else:
                raise Exception(f"HTTP {response.status_code}")

        except RequestException as e:
            print(f"Request failed: {e}")
            if attempt < max_retries - 1:
                delay = base_delay * (2 ** attempt)
                print(f"Retrying in {delay}s...")
                time.sleep(delay)

    raise Exception(f"Failed after {max_retries} attempts")

# Usage
html = fetch_with_retry("https://example.com/data")