Extract Pages as Markdown - From Website URL

import os
import requests
from bs4 import BeautifulSoup
from markdownify import markdownify as md
from urllib.parse import urlparse
import xml.etree.ElementTree as ET

"""
How to use this script:

1. Install dependencies:
python -m pip install requests beautifulsoup4 markdownify

2. Change BASE_URL if needed

3. Run:
python extract-knowledge.py
"""

BASE_URL = "https://www.activepilot.com/"
SITEMAP_URL = BASE_URL + "sitemap.xml"

parsed_uri = urlparse(BASE_URL)
OUTPUT_DIR = parsed_uri.netloc.replace("www.", "")


def get_urls_from_sitemap():
    """Fetch all URLs listed in the sitemap."""
    try:
        response = requests.get(SITEMAP_URL, timeout=10)
        root = ET.fromstring(response.content)

        namespace = {"ns": "http://www.sitemaps.org/schemas/sitemap/0.9"}

        urls = []
        for url in root.findall("ns:url", namespace):
            loc = url.find("ns:loc", namespace)
            if loc is not None:
                urls.append(loc.text)

        return urls

    except Exception as e:
        print(f"Error reading sitemap: {e}")
        return []


def save_page_as_markdown(url):
    """Download a page and convert its main content to markdown."""
    try:
        response = requests.get(url, timeout=10)
        soup = BeautifulSoup(response.text, "html.parser")

        tags_to_remove = [
            "nav", "footer", "header", "script", "style",
            "noscript", "form", "iframe", "svg", "button", "aside", "input"
        ]

        for tag in soup(tags_to_remove):
            tag.decompose()

        for div in soup.find_all("div", {"class": ["cookie", "banner", "popup", "modal", "advertisement"]}):
            div.decompose()

        content = soup.find("main")

        if not content:
            content = soup.find("body")

        if content:
            clean_md = md(
                str(content),
                heading_style="ATX",
                strip=["img", "picture", "figure"],
                newlines_after_headline=1
            )

            lines = [line.strip() for line in clean_md.splitlines() if line.strip()]
            final_text = "\n\n".join(lines)

            path_name = urlparse(url).path

            if not path_name or path_name == "/":
                filename = "index.md"
            else:
                filename = path_name.strip("/").replace("/", "_") + ".md"

            filepath = os.path.join(OUTPUT_DIR, filename)

            with open(filepath, "w", encoding="utf-8") as f:
                f.write(f"--- SOURCE_URL: {url} ---\n\n")
                f.write(final_text)

            print(f"Saved: {filename}")

    except Exception as e:
        print(f"Error processing {url}: {e}")


if __name__ == "__main__":

    if not os.path.exists(OUTPUT_DIR):
        os.makedirs(OUTPUT_DIR)

    print(f"Fetching sitemap: {SITEMAP_URL}")
    all_pages = get_urls_from_sitemap()

    print(f"Found {len(all_pages)} pages in sitemap. Processing...\n")

    for page in all_pages:
        save_page_as_markdown(page)

    print(f"\nFinished. Files saved in: {OUTPUT_DIR}")