Extract Pages as Markdown - From Build Folder

import fs from 'fs';
import path from 'path';
import TurndownService from 'turndown';
import { JSDOM } from 'jsdom';

// How to run this script:
// 1. Add this script to a file on root folder (same level as src) named `extract-knowledge.mjs`
// 2. Run: npm install jsdom turndown -D
// 3. Run: node extract-knowledge.mjs 

const DIST_DIR = './dist';
const OUTPUT_DIR = './client_knowledge_base';

const turndownService = new TurndownService({
headingStyle: 'atx',
bullet: '*'
});

turndownService.remove(['img', 'picture', 'figure', 'video', 'a']);

function getHtmlFiles(dir, fileList = []) {
const files = fs.readdirSync(dir);
files.forEach(file => {
    const filePath = path.join(dir, file);
    if (fs.statSync(filePath).isDirectory()) {
    getHtmlFiles(filePath, fileList);
    } else if (file.endsWith('.index.html') || file.endsWith('.html')) {
    fileList.push(filePath);
    }
});
return fileList;
}

async function convert() {
if (!fs.existsSync(DIST_DIR)) {
    console.error("❌ Folder /dist not found. Run 'npm run build' first.");
    return;
}

if (!fs.existsSync(OUTPUT_DIR)) fs.mkdirSync(OUTPUT_DIR);

const files = getHtmlFiles(DIST_DIR);
console.log(`🚀 Processing ${files.length} pages from local build...`);

files.forEach(filePath => {
    const html = fs.readFileSync(filePath, 'utf8');
    const dom = new JSDOM(html);
    const document = dom.window.document;
    const tagsToRemove = ['nav', 'footer', 'header', 'script', 'style', 'noscript', 'form', 'aside'];
    tagsToRemove.forEach(tag => {
    document.querySelectorAll(tag).forEach(el => el.remove());
    });

    // Removes banners for cookies or popups
    const noiseClasses = ['.cookie', '.banner', '.popup', '.modal'];
    noiseClasses.forEach(selector => {
    document.querySelectorAll(selector).forEach(el => el.remove());
    });

    const content = document.querySelector('main') || document.querySelector('body');

    if (content) {
    let markdown = turndownService.turndown(content.innerHTML);
    
    markdown = markdown.split('\n').map(line => line.trim()).filter(line => line).join('\n\n');
    const relativePath = path.relative(DIST_DIR, filePath);
    const fileName = relativePath.replace(/\\/g, '_').replace('.html', '.md').replace('_index', 'index');
    
    const finalPath = path.join(OUTPUT_DIR, fileName);
    
    fs.writeFileSync(finalPath, `--- SOURCE: Local Build (${relativePath}) ---\n\n${markdown}`);
    console.log(`✅ Extracted: ${fileName}`);
    }
});

console.log(`\n✨ Done! Knowledge base created in: ${OUTPUT_DIR}`);
}

convert();