How to turn any RSS or Atom feed into JSON and CSV with Python

import re, requests from urllib.parse import urljoin, urlparse UA = {"User-Agent": "rss-tutorial/1.0"} def discover_feeds(page_url): html = requests.get(page_url, headers=UA, timeout=20).text found = [] for m in re.finditer(r'''<link[^>]+rel=["']alternate["'][^>]*>''', html, re.I): tag = m.group(0) if not re.search(r'type=["\']application/(rss|atom)\+xml["\']', tag, re.I): continue href = re.search(r'href=["\']([^"\']+)["\']', tag, re.I) if href: found.append(urljoin(page_url, href.group(1))) origin = f"{urlparse(page_url).scheme}://{urlparse(page_url).netloc}" for path in ("/feed", "/rss", "/atom.xml", "/feed.xml", "/rss.xml"): found.append(origin + path) return list(dict.fromkeys(found)) 

import gzip, xml.etree.ElementTree as ET from datetime import datetime, timezone from email.utils import parsedate_to_datetime from html import unescape def _local(tag): return tag.split("}", 1)[-1].lower() if tag else "" def _text(el): if el is None: return "" parts = [el.text or ""] + [ET.tostring(c, encoding="unicode") + (c.tail or "") for c in el] return unescape("".join(parts)).strip() def _strip_html(s): return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", s or "")).strip() def parse_date(s): if not s: return "" try: return parsedate_to_datetime(s).astimezone(timezone.utc).isoformat() except Exception: pass try: return datetime.fromisoformat(s.strip().replace("Z", "+00:00")).astimezone(timezone.utc).isoformat() except Exception: return s.strip() def fetch_xml(url): body = requests.get(url, headers=UA, timeout=30, allow_redirects=True).content if url.endswith(".gz") or body[:2] == b"\x1f\x8b": body = gzip.decompress(body) return body def parse_feed(body): root = ET.fromstring(body) items = [] if _local(root.tag) in ("rss", "rdf"): channel = root.find("channel") if _local(root.tag) == "rss" else root entries = (channel.findall("item") if channel is not None else []) or root.findall(".//item") for it in entries: get = lambda *names: _text(next((c for c in it if _local(c.tag) in names), None)) items.append({"title": get("title"), "link": get("link"), "published": parse_date(get("pubdate", "date")), "author": get("author", "creator"), "summary": _strip_html(get("description", "summary", "encoded"))[:300]}) elif _local(root.tag) == "feed": # Atom  for it in [c for c in root if _local(c.tag) == "entry"]: title = _text(next((c for c in it if _local(c.tag) == "title"), None)) link = next((c.attrib.get("href", "") for c in it if _local(c.tag) == "link" and c.attrib.get("rel", "alternate") in ("alternate", "")), "") pub = _text(next((c for c in it if _local(c.tag) in ("published", "updated")), None)) author_el = next((c for c in it if _local(c.tag) == "author"), None) author = _text(next((c for c in author_el if _local(c.tag) == "name"), None)) if author_el is not None else "" summary = _text(next((c for c in it if _local(c.tag) in ("summary", "content")), None)) items.append({"title": title, "link": link, "published": parse_date(pub), "author": author, "summary": _strip_html(summary)[:300]}) else: raise ValueError(f"Unknown feed root: {root.tag}") return items 

import csv rows = parse_feed(fetch_xml("https://hnrss.org/frontpage")) with open("feed.csv", "w", newline="", encoding="utf-8") as f: w = csv.DictWriter(f, fieldnames=["title", "link", "published", "author", "summary"]) w.writeheader() w.writerows(rows)