How To Easily Scrape Web Content Using Python - No Tools Required

import requests
from bs4 import BeautifulSoup
import csv
import os
import time
def sanitize_filename(url):
    """ Sanitize the URL to be used as a filename """
    return url.replace("http://", "").replace("https://", "").replace("/", "_")
def format_tag_with_links(tag):
    """ Format tag text to include hyperlinks """
    for a in tag.find_all('a', href=True):
        link_text = a.text.strip()
        if link_text:
            a.replace_with(f"{link_text} (Link: {a['href']})")
    return tag.get_text(strip=True)
def extract_and_save_content(url, output_folder):
    try:
        response = requests.get(url)
        if response.status_code != 200:
            return f"HTTP Error: {response.status_code}"
        soup = BeautifulSoup(response.text, 'html.parser')
        # Exclude header and footer
        for header_footer in soup.find_all(['header', 'footer'], recursive=True):
            header_footer.decompose()
        for header_footer in soup.find_all(class_=['header', 'footer'], recursive=True):
            header_footer.decompose()
        for header_footer in soup.find_all(id=['header', 'footer'], recursive=True):
            header_footer.decompose()
        filename = f"{sanitize_filename(url)}.txt"
        output_path = os.path.join(output_folder, filename)
        with open(output_path, 'w', encoding='utf-8') as file:
            for tag in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p']):
                formatted_text = format_tag_with_links(tag)
                if formatted_text:
                    file.write(f"{tag.name.upper()}: {formatted_text}\n\n")
        if not os.path.getsize(output_path) > 0:
            return "No relevant content extracted"
        return True
    except requests.RequestException as e:
        return str(e)
def process_urls_from_csv(csv_file, output_folder, delay=2):
    urls_to_process = {}  # Dictionary to keep track of retries
    url_status = {}  # Dictionary to keep final status
    with open(csv_file, newline='', encoding='utf-8') as file:
        reader = csv.reader(file)
        for row in reader:
            url = row[0].strip()
            if url:
                urls_to_process[url] = 6  # Initialize retry count for each URL
    while urls_to_process:
        for url, attempts in list(urls_to_process.items()):
            status = extract_and_save_content(url, output_folder)
            if status is True:
                url_status[url] = "Crawled Successfully"
                del urls_to_process[url]
            else:
                if attempts > 1:
                    urls_to_process[url] -= 1
                else:
                    url_status[url] = "Failed to Crawl"
                    del urls_to_process[url]
            time.sleep(delay)
    # Write final URL status to a CSV file
    with open(os.path.join(output_folder, "final_url_status.csv"), 'w', newline='', encoding='utf-8') as file:
        writer = csv.writer(file)
        writer.writerow(['URL', 'Status'])
        for url, status in url_status.items():
            writer.writerow([url, status])
    print("URL processing status recorded in 'final_url_status.csv'.")
csv_file = 'urls.csv'
output_folder = 'extracted_contents'
os.makedirs(output_folder, exist_ok=True)
process_urls_from_csv(csv_file, output_folder)