HTML to clean Markdown chunks in Python, and spotting what really changed

class TreeBuilder(HTMLParser): """Build a small tree of {"tag", "attrs", "children"} dicts; text stays as plain strings.""" def __init__(self): super().__init__(convert_charrefs=True) self.root = {"tag": "#root", "attrs": {}, "children": []} self.stack = [self.root] def handle_starttag(self, tag, attrs): node = {"tag": tag, "attrs": {k: v or "" for k, v in attrs}, "children": []} self.stack[-1]["children"].append(node) if tag not in VOID: self.stack.append(node) def handle_endtag(self, tag): for depth in range(len(self.stack) - 1, 0, -1): # close the nearest open match, if any  if self.stack[depth]["tag"] == tag: del self.stack[depth:] break def handle_data(self, data): self.stack[-1]["children"].append(data) 

USER_AGENT = "html-to-chunks-example/1.0 (tutorial script; one request per page)" VOID = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"} SKIP = {"script", "style", "noscript", "template", "svg", "nav", "footer", "form", "button", "iframe", "aside"} NOISE_CLASSES = {"headerlink"} # e.g. the pilcrow permalinks Sphinx adds after headings BLOCKS = {"p", "div", "section", "article", "main", "blockquote", "pre", "ul", "ol", "table", "dl", "dt", "dd", "hr", "figure", "h1", "h2", "h3", "h4", "h5", "h6"} | SKIP 

def main_content(root): for test in (lambda n: n["tag"] == "main", lambda n: n["tag"] == "article", lambda n: n["attrs"].get("role") == "main", lambda n: n["tag"] == "body"): hit = find(root, test) if hit: return hit return root def ignored(node): return node["tag"] in SKIP or bool(NOISE_CLASSES & set(node["attrs"].get("class", "").split())) def collapse(text): return re.sub(r"\s+", " ", text).strip() 

def text_of(node): """Raw text with whitespace kept (for <pre>), minus ignored elements.""" if isinstance(node, str): return node return "" if ignored(node) else "".join(text_of(child) for child in node["children"]) def inline(node, base): """Render inline content as Markdown on one line.""" if isinstance(node, str): return node if ignored(node): return "" tag, attrs = node["tag"], node["attrs"] if tag == "br": return " " if tag == "img": return attrs.get("alt", "") inner = "".join(inline(child, base) for child in node["children"]) if tag == "code": return f"`{collapse(text_of(node))}`" if collapse(text_of(node)) else "" if tag in ("strong", "b", "em", "i") and collapse(inner): mark = "**" if tag in ("strong", "b") else "*" return f"{mark}{collapse(inner)}{mark}" href = attrs.get("href", "") if tag == "a" and collapse(inner) and href and not href.startswith(("#", "javascript:")): return f"[{collapse(inner)}]({urllib.parse.urljoin(base, href)})" return inner 

def render(node, base, out): """Append Markdown blocks (strings) for the children of `node` to `out`.""" run = [] def flush(): text = collapse("".join(run)) run.clear() if text: out.append(text) for child in node["children"]: if isinstance(child, str) or child["tag"] not in BLOCKS: run.append(inline(child, base)) continue flush() tag = child["tag"] if ignored(child) or tag == "hr": continue if re.fullmatch(r"h[1-6]", tag): title = collapse(inline(child, base)) if title: out.append("#" * int(tag[1]) + " " + title) elif tag == "pre": out.append("``` \n" + text_of(child).strip("\n") + "\n ```") elif tag in ("ul", "ol"): render_list(child, base, out) elif tag == "table": out.append(render_table(child, base)) elif tag == "dt": out.append(collapse(text_of(child))) elif tag == "blockquote": quoted = [] render(child, base, quoted) out.extend("> " + block.replace("\n", "\n> ") for block in quoted) else: render(child, base, out) flush() 

def render_table(node, base): rows = [] for row in find_all(node, "tr"): cells = [collapse(inline(cell, base)).replace("|", "\\|") for cell in row["children"] if isinstance(cell, dict) and cell["tag"] in ("th", "td")] if cells: rows.append(cells) if not rows: return "" width = max(len(row) for row in rows) rows = [row + [""] * (width - len(row)) for row in rows] lines = ["| " + " | ".join(rows[0]) + " |", "|" + " --- |" * width] return "\n".join(lines + ["| " + " | ".join(row) + " |" for row in rows[1:]]) 

def prose(text): """Text with link targets and whitespace differences removed.""" return collapse(re.sub(r"\]\([^)]*\)", "]", text)) def fingerprint(text): return hashlib.sha256(prose(text).encode("utf-8")).hexdigest()[:16] def chunk(blocks, max_level=3): """Group blocks under their nearest heading (h1..h{max_level}); one chunk per section.""" chunks, path, body, seen = [], [], [], {} def emit(): text = "\n\n".join(body).strip() body.clear() if text: name = " > ".join(path) or "(top)" seen[name] = seen.get(name, 0) + 1 chunk_id = name if seen[name] == 1 else f"{name} #{seen[name]}" chunks.append({"id": chunk_id, "words": len(text.split()), "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest()[:16], "fingerprint": fingerprint(text), "text": text}) for block in blocks: heading = re.match(r"(#{1,6}) (.*)", block) if heading and len(heading.group(1)) <= max_level: emit() del path[len(heading.group(1)) - 1:] path.append(heading.group(2)) body.append(block) emit() return chunks 

def compare(old, new, rename_threshold=0.6): """Classify chunks by id: added, removed, renamed, changed (prose), links-only, unchanged.""" before = {c["id"]: c for c in old} after = {c["id"]: c for cSource：https://dev.to/abdulwhab95/html-to-clean-markdown-chunks-in-python-and-spotting-what-really-changed-1ibm
My favorites