Extract Pages as Markdown - From Sitemap URL Array

import os
import warnings
import requests
from bs4 import BeautifulSoup, XMLParsedAsHTMLWarning
from markdownify import markdownify as md
from urllib.parse import urljoin, urlparse

warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
}

"""
How to use this script:
1. You need python for obvious reasons
2. Install the imported dependencies above with the Powershell command: `python -m pip install requests beautifulsoup4 markdownify lxml`
3. Update the `BASE_URL` variable, it's the target website to scrap
4. Run the script with: `python name_of_file_you_pasted.py`
5. When finished, the terminal will tell you
"""


# ── Configuration ─────────────────────────────────────────────────────────────

# The output folder will be named after the domain automatically.
# Add or remove sitemap URLs from the list below as needed.
SITEMAPS = [
    "https://magnoliaaviation.com/post-sitemap.xml",
    "https://magnoliaaviation.com/page-sitemap.xml",
    "https://magnoliaaviation.com/product-sitemap.xml",
    "https://magnoliaaviation.com/product_cat-sitemap.xml",
    "https://magnoliaaviation.com/video-sitemap.xml",
    "https://magnoliaaviation.com/local-sitemap.xml",
]

# ── Internals (no need to edit below) ─────────────────────────────────────────

IGNORE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.pdf', '.mp4', '.zip', '.js', '.css', '.svg', '.xml', '.json']

def get_output_dir(sitemap_url):
    """Derive output dir from domain + sitemap name, e.g. magnoliaaviation.com/post"""
    parsed = urlparse(sitemap_url)
    domain = parsed.netloc.replace("www.", "")
    # Strip path separators and the '-sitemap.xml' / '.xml' suffix for a clean folder name
    sitemap_name = os.path.basename(parsed.path)           # e.g. post-sitemap.xml
    sitemap_name = sitemap_name.replace("-sitemap.xml", "").replace(".xml", "")
    return os.path.join(domain, sitemap_name)

def fetch_pages_from_sitemap(sitemap_url):
    """Fetch a sitemap XML and return all page URLs listed in <loc> tags."""
    try:
        response = requests.get(sitemap_url, headers=HEADERS, timeout=10)
        response.raise_for_status()
        soup = BeautifulSoup(response.content, "xml")
        urls = [loc.get_text(strip=True) for loc in soup.find_all("loc")]
        page_urls = [
            u for u in urls
            if not any(u.lower().endswith(ext) for ext in IGNORE_EXTENSIONS)
        ]
        print(f"  → {len(page_urls)} pages found")
        return page_urls
    except Exception as e:
        print(f"  Error fetching sitemap: {e}")
        return []

def save_page_as_markdown(url, output_dir):
    try:
        response = requests.get(url, headers=HEADERS, timeout=10)
        soup = BeautifulSoup(response.text, 'html.parser')

        tags_to_remove = [
            'nav', 'footer', 'header', 'script', 'style',
            'noscript', 'form', 'iframe', 'svg', 'button', 'aside', 'input'
        ]
        for tag in soup(tags_to_remove):
            tag.decompose()

        for div in soup.find_all("div", {"class": ["cookie", "banner", "popup", "modal", "advertisement"]}):
            div.decompose()

        content = soup.find('main')
        if not content:
            content = soup.find('body')

        if content:
            clean_md = md(
                str(content),
                heading_style="ATX",
                strip=['img', 'picture', 'figure'],
                newlines_after_headline=1
            )

            lines = [line.strip() for line in clean_md.splitlines() if line.strip()]
            final_text = "\n\n".join(lines)

            path_name = urlparse(url).path
            if not path_name or path_name == "/":
                path_name = "index"

            filename = f"{path_name.strip('/')}.md".replace('/', '_')
            filepath = os.path.join(output_dir, filename)

            with open(filepath, 'w', encoding='utf-8') as f:
                f.write(f"--- SOURCE_URL: {url} ---\n\n")
                f.write(final_text)

            print(f"    Saved: {filename}")

    except Exception as e:
        print(f"    Error processing {url}: {e}")

# ── Main ───────────────────────────────────────────────────────────────────────

if __name__ == "__main__":
    for sitemap_url in SITEMAPS:
        print(f"\nProcessing sitemap: {sitemap_url}")

        output_dir = get_output_dir(sitemap_url)
        os.makedirs(output_dir, exist_ok=True)

        pages = fetch_pages_from_sitemap(sitemap_url)
        for page in pages:
            save_page_as_markdown(page, output_dir)

    print("\nFinished.")