Meta Said a Claude Tool Browsed at 4x Human Speed. Here's a 124-Line Python Fix for Impolite AI Agents

"""polite_agent.py: keep an AI agent's HTTP traffic slow, honest and auditable. Standard library only. Per-host token bucket, jitter, backoff that honours Retry-After, robots.txt, an identifying User-Agent, a daily cap and a JSONL log. """ import email.utils, json, random, threading, time, urllib.error, urllib.request from datetime import datetime, timezone from urllib.parse import urlsplit from urllib.robotparser import RobotFileParser class Blocked(Exception): """Raised when a request is refused before it is sent (robots or cap).""" class TokenBucket: def __init__(self, rate_per_min, burst): self.rate, self.capacity = rate_per_min / 60.0, burst self.tokens, self.last = float(burst), time.monotonic() self.lock = threading.Lock() def take(self): with self.lock: now = time.monotonic() self.tokens = min(self.capacity, self.tokens + (now - self.last) * self.rate) self.last = now wait = 0.0 if self.tokens >= 1 else (1 - self.tokens) / self.rate self.tokens -= 1 # may go negative: the debt is paid by sleeping  if wait: time.sleep(wait) def utc_day(): return datetime.now(timezone.utc).date().isoformat() def retry_after_seconds(value): """Retry-After is either delta-seconds or an HTTP date.""" if not value: return None if value.strip().isdigit(): return float(value) try: when = email.utils.parsedate_to_datetime(value) return max(0.0, (when - datetime.now(timezone.utc)).total_seconds()) except (TypeError, ValueError): return None class PoliteClient: def __init__(self, user_agent, log_path="agent_actions.jsonl", rate_per_min=12, burst=2, jitter=(0.5, 2.0), daily_cap=500, max_retries=4, base_backoff=2.0, max_backoff=120.0, timeout=20): self.ua, self.log_path, self.timeout = user_agent, log_path, timeout self.rate_per_min, self.burst, self.jitter = rate_per_min, burst, jitter self.daily_cap, self.max_retries = daily_cap, max_retries self.base_backoff, self.max_backoff = base_backoff, max_backoff self.buckets, self.robots = {}, {} self.today = utc_day() self.count = self._count_today() def _count_today(self): # the cap survives restarts because it reads the log  try: with open(self.log_path) as f: events = [json.loads(line) for line in f if line.strip()] return sum(1 for e in events if e.get("day") == self.today and e.get("sent")) except FileNotFoundError: return 0 def _log(self, **event): event.update(ts=datetime.now(timezone.utc).isoformat(timespec="seconds"), day=self.today) with open(self.log_path, "a") as f: # append-only, one JSON object per line  f.write(json.dumps(event) + "\n") def _robots_for(self, url): parts = urlsplit(url) origin = f"{parts.scheme}://{parts.netloc}" if origin not in self.robots: rp = RobotFileParser() try: req = urllib.request.Request(origin + "/robots.txt", headers={"User-Agent": self.ua}) with urllib.request.urlopen(req, timeout=self.timeout) as r: rp.parse(r.read().decode("utf-8", "replace").splitlines()) except urllib.error.HTTPError as e: rp.disallow_all = e.code in (401, 403) # 404 means no rules  except OSError: rp.disallow_all = True # unreachable: be conservative  self.robots[origin] = rp delay = rp.crawl_delay(self.ua) rate = min(self.rate_per_min, 60.0 / float(delay)) if delay else self.rate_per_min self.buckets[parts.netloc] = TokenBucket(rate, self.burst) return self.robots[origin] def get(self, url): if utc_day() != self.today: # new day, fresh budget  self.today, self.count = utc_day(), 0 if not self._robots_for(url).can_fetch(self.ua, url): self._log(url=url, sent=False, reason="robots.txt disallow") raise Blocked(f"robots.txt disallows {url}") host = urlsplit(url).netloc for attempt in range(self.max_retries + 1): if self.count >= self.daily_cap: self._log(url=url, sent=False, reason="daily cap reached") raise Blocked(f"daily cap of {self.daily_cap} reached") self.buckets[host].take() time.sleep(random.uniform(*self.jitter)) self.count += 1 req = urllib.request.Request(url, headers={"User-Agent": self.ua}) try: with urllib.request.urlopen(req, timeout=self.timeout) as r: body = r.read() self._log(url=url, sent=True, status=r.status, attempt=attempt) return body except urllib.error.HTTPError as e: self._log(url=url, sent=True, status=e.code, attempt=attempt) if e.code not in (429, 503) or attempt == self.max_retries: raise wait = retry_after_seconds(e.headers.get("Retry-After")) if wait is None: # no hint: exponential backoff with jitter  wait = min(self.max_backoff, self.base_backoff * 2 ** attempt * random.uniform(0.8, 1.2)) elif wait > self.max_backoff: # never retry earlier than asked: give up  raise print(f" {e.code} on {url}, backing off {wait:.1f}s") time.sleep(wait) 

"""demo.py: run PoliteClient against a local mock server that pushes back.""" import os, threading, time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from polite_agent import Blocked, PoliteClient HITS = {"profile": 0, "busy": 0} class Mock(BaseHTTPRequestHandler): def do_GET(self): if self.path == "/robots.txt": return self.reply(200, b"User-agent: *\nDisallow: /private\nCrawl-delay: 1\n") if self.path.startswith("/profile/"): HITS["profile"] += 1 if HITS["profile"] % 3 == 0: # every third hit: slow down, try in 2s  return self.reply(429, b"slow down", {"Retry-After": "2"}) if self.path == "/busy": HITS["busy"] += 1 if HITS["busy"] == 1: # 503 with no Retry-After header  return self.reply(503, b"busy") self.reply(200, f"ok {self.path} ua={self.headers['User-Agent']}".encode()) def reply(self, code, body, headers=None): self.send_response(code) for k, v in (headers or {}).items(): self.send_header(k, v) self.end_headers() self.wfile.write(body) def log_message(self, *args): pass server = ThreadingHTTPServer(("127.0.0.1", 8765), Mock) threading.Thread(target=server.serve_forever, daemon=True).start() if os.path.exists("demo_log.jsonl"): os.remove("demo_log.jsonl") agent = PoliteClient( user_agent="ResearchBot/1.0 (+https://example.com/bot; bot@example.com)", log_path="demo_log.jsonl", rate_per_min=120, burst=1, jitter=(0.2, 0.6), daily_cap=8, base_backoff=1.0) start = time.monotonic() urls = [f"http://127.0.0.1:8765/profile/{i}" for i in range(1, 5)] urls += ["http://127.0.0.1:8765/private/inbox", "http://127.0.0.1:8765/busy", "http://127.0.0.1:8765/profile/5"] for url in urls: try: body = agent.get(url).decode() print(f"[{time.monotonic() - start:5.1f}s] 200 {body}") except Blocked as e: print(f"[{time.monotonic() - start:5.1f}s] BLOCKED {e}") print(f"requests sent today: {agent.count}/{agent.daily_cap}") server.shutdown() 

[ 0.6s] 200 ok /profile/1 ua=ResearchBot/1.0 (+https://example.com/bot; bot@example.com) [ 1.4s] 200 ok /profile/2 ua=ResearchBot/1.0 (+https://example.com/bot; bot@example.com) 429 on http://127.0.0.1:8765/profile/3, backing off 2.0s [ 4.6s] 200 ok /profile/3 ua=ResearchBot/1.0 (+https://example.com/bot; bot@example.com) [ 5.8s] 200 ok /profile/4 ua=ResearchBot/1.0 (+https://example.com/bot; bot@example.com) [ 5.8s] BLOCKED robots.txt disallows http://127.0.0.1:8765/private/inbox 503 on http://127.0.0.1:8765/busy, backing off 1.1s [ 8.6s] 200 ok /busy ua=ResearchBot/1.0 (+https://example.com/bot; bot@example.com) 429 on http://127.0.0.1:8765/profile/5, backing off 2.0s [ 11.5s] BLOCKED daily cap of 8 reached requests sent today: 8/8 

{"url": "http://127.0.0.1:8765/profile/3", "sent": true, "status": 429, "attempt": 0, "ts": "2026-10-19T12:09:22+00:00", "day": "2026-10-19"} {"url": "http://127.0.0.1:8765/private/inbox", "sent": false, "reason": "robots.txt disallow",Source：https://dev.to/neeraj_sharma_757fbb49aa7/meta-said-a-claude-tool-browsed-at-4x-human-speed-heres-a-124-line-python-fix-for-impolite-ai-58hh
My favorites