parser.py

from bs4 import BeautifulSoup
from backend.scraper.utils import clean_text

def parse_sections(html: str):
    soup = BeautifulSoup(html, "html.parser")
    sections = []
    page_title = soup.title.string.strip() if soup.title else "Untitled Page"
    meta_desc = ""
    meta = soup.find("meta", attrs={"name":"description"})
    if meta:
        meta_desc = meta.get("content","")
    meta_content=[page_title]
    if meta_desc: meta_content.append(meta_desc)
    sections.append({
        "title":"Page Metadata",
        "content":meta_content,
        "word_count":sum(len(c.split()) for c in meta_content)
    })
    current_title="Introduction"
    current_content=[]
    for tag in soup.find_all(["h1","h2","h3","p","li"]):
        if tag.name in ["h1","h2","h3"]:
            if current_content:
                sections.append({
                    "title":clean_text(current_title),
                    "content":current_content,
                    "word_count":sum(len(c.split()) for c in current_content)
                })
            current_title=tag.get_text()
            current_content=[]
        else:
            text=clean_text(tag.get_text())
            if text: current_content.append(text)
    if current_content:
        sections.append({
            "title":clean_text(current_title),
            "content":current_content,
            "word_count":sum(len(c.split()) for c in current_content)
        })
    return sections