mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 04:11:35 +00:00
* refactor(skills): shipped-set slim — 15 skills to optional, github six-way merge, pdf absorbs OCR+nano-pdf, channel-gated teams pipeline
Maintainer-directed shipped-skills curation (skills index 1,900 -> ~1,400
tok/call on desktop; every session pays the index, so this is a per-call
diet on all installs):
- optional-skills moves (installable via skills hub, history preserved):
creative comfyui/ascii-art/excalidraw/pretext/sketch/touchdesigner-mcp;
ALL of mlops (huggingface-hub, llama-cpp, serving-llms-vllm,
weights-and-biases, evaluating-llms-harness — subcategory structure
kept); research-paper-writing (55 supporting files, 17.3K-tok load);
openhue; blogwatcher (first taught the cronjob monitor-field watch
pattern + web_extract instead of pre-cron manual workflows)
- DELETED session-librarian (Aug-12 'inspired by Perplexity Computer'
port, never maintainer-intended; session_search covers discovery)
- github: six skills (auth, issues, pr-workflow, issue-to-pr,
code-review, repo-management) merged into ONE software-development/
github skill — routing body + complete per-workflow references;
benbarclay authorship credited; codebase-inspection rides along;
discipline pins from test_github_issue_to_pr_skill.py preserved
against the reference body in the new test_github_skill.py
- pdf absorbs ocr-and-documents + nano-pdf as references/ + scripts
(extract_pymupdf, extract_marker converted to the argparse house
standard its contract test enforces)
- NEW session_platforms frontmatter gate (metadata.hermes): hides a
skill from the index on gateway channels it is not for; fail-open on
unknown platform; teams-meeting-pipeline gated to [teams, cron]
- blocked-page-recovery: research -> new web category; trigger-first
description ('Use when a fetch fails: 403/429, paywall, WAF, bot
wall.') so the model actually reaches for it on blocked fetches
- docs regenerated via generate-skill-docs.py (195 pages); related_skills
swept repo-wide; tests: 1672 passed (2 openclaw failures pre-existing
on clean main, Windows-local)
* chore: ignore .skills_prompt_snapshot.json (local index cache, accidentally committed)
241 lines
8.3 KiB
Python
241 lines
8.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Recover a blocked / paywalled / WAF'd page from third-party copies.
|
|
|
|
Ladder (cheapest first):
|
|
1. Wayback Machine "available" API -> dated snapshot (provenance: snapshot)
|
|
2. archive.today domain rotation -> dated snapshot (provenance: snapshot)
|
|
3. Jina Reader (JINA_API_KEY only) -> live re-render (provenance: live)
|
|
|
|
Every candidate body is validated before being declared a win: byte floors,
|
|
redirect-stub detection (meta-refresh/JS pointing back at the original host),
|
|
and interstitial-title rejection. Fake 200s are the norm in this space.
|
|
|
|
Stdlib only. Usage:
|
|
python3 recover_page.py URL [--json] [--out FILE] [--timeout N]
|
|
|
|
Exit codes: 0 recovered, 1 nothing worked, 2 bad invocation.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
USER_AGENT = (
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/126.0 Safari/537.36"
|
|
)
|
|
|
|
ARCHIVE_TODAY_HOSTS = ["archive.ph", "archive.md", "archive.li", "archive.is"]
|
|
|
|
# Titles that mean "this is not the page you asked for".
|
|
INTERSTITIAL_TITLES = (
|
|
"just a moment",
|
|
"redirecting",
|
|
"google search",
|
|
"attention required",
|
|
"access denied",
|
|
"are you a robot",
|
|
"one more step",
|
|
)
|
|
|
|
# Below these floors a body is a stub or an error page, not content.
|
|
MIN_BODY_BYTES = {"wayback": 3072, "archive_today": 3072, "jina": 512}
|
|
|
|
REDIRECT_STUB_RE = re.compile(
|
|
r'http-equiv=["\']?refresh|window\.location|location\.replace', re.IGNORECASE
|
|
)
|
|
TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
|
|
|
|
|
def _fetch(
|
|
url: str,
|
|
timeout: int,
|
|
headers: dict | None = None,
|
|
retries_on_429: int = 2,
|
|
) -> tuple[int, bytes]:
|
|
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, **(headers or {})})
|
|
for attempt in range(retries_on_429 + 1):
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
return resp.status, resp.read()
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code == 429 and attempt < retries_on_429:
|
|
time.sleep(5 * (attempt + 1))
|
|
continue
|
|
return exc.code, exc.read() if exc.fp else b""
|
|
except (urllib.error.URLError, OSError, ValueError):
|
|
return 0, b""
|
|
return 0, b""
|
|
|
|
|
|
def _fetch_follow(url: str, timeout: int) -> tuple[int, bytes, str]:
|
|
"""Like _fetch but also returns the final URL after redirects."""
|
|
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
return resp.status, resp.read(), resp.geturl()
|
|
except urllib.error.HTTPError as exc:
|
|
return exc.code, exc.read() if exc.fp else b"", exc.geturl() or url
|
|
except (urllib.error.URLError, OSError, ValueError):
|
|
return 0, b"", url
|
|
|
|
|
|
def _page_title(body: bytes) -> str:
|
|
m = TITLE_RE.search(body[:65536].decode("utf-8", "replace"))
|
|
return re.sub(r"\s+", " ", m.group(1)).strip().lower() if m else ""
|
|
|
|
|
|
def validate(body: bytes, route: str, target_url: str) -> str | None:
|
|
"""Return a rejection reason, or None if the body looks like real content."""
|
|
floor = MIN_BODY_BYTES.get(route, 3072)
|
|
if len(body) < floor:
|
|
return f"body_too_small:{len(body)}<{floor}"
|
|
title = _page_title(body)
|
|
for marker in INTERSTITIAL_TITLES:
|
|
if marker in title:
|
|
return f"interstitial_title:{marker!r}"
|
|
# Redirect stub: small-ish page whose only job is bouncing back to the
|
|
# original (blocked) host — the classic AMP-cache failure mode.
|
|
if len(body) < 8192 and REDIRECT_STUB_RE.search(body.decode("utf-8", "replace")):
|
|
target_host = urllib.parse.urlsplit(target_url).hostname or ""
|
|
if target_host and target_host.encode() in body:
|
|
return "redirect_stub_to_origin"
|
|
return None
|
|
|
|
|
|
def try_wayback(url: str, timeout: int) -> dict | None:
|
|
snap_url = None
|
|
snap_ts = None
|
|
discovery = "https://archive.org/wayback/available?url=" + urllib.parse.quote(url, safe="")
|
|
status, raw = _fetch(discovery, timeout)
|
|
if status == 200:
|
|
try:
|
|
closest = json.loads(raw).get("archived_snapshots", {}).get("closest", {})
|
|
except (json.JSONDecodeError, AttributeError):
|
|
closest = {}
|
|
if closest.get("available") and closest.get("url"):
|
|
snap_url = closest["url"].replace(
|
|
"http://web.archive.org", "https://web.archive.org"
|
|
)
|
|
snap_ts = closest.get("timestamp")
|
|
if snap_url is None:
|
|
# Discovery API is rate-limited far more aggressively than snapshot
|
|
# serving. Fall back to the redirect form: /web/2/<url> bounces to
|
|
# the newest snapshot if one exists (404 page otherwise).
|
|
snap_url = "https://web.archive.org/web/2/" + url
|
|
status, body, final_url = _fetch_follow(snap_url, timeout)
|
|
if status != 200 or validate(body, "wayback", url):
|
|
return None
|
|
if snap_ts is None:
|
|
m = re.search(r"/web/(\d{14})", final_url)
|
|
snap_ts = m.group(1) if m else None
|
|
return {
|
|
"route": "wayback",
|
|
"provenance": "snapshot",
|
|
"snapshot_timestamp": snap_ts,
|
|
"source_url": final_url,
|
|
"body": body,
|
|
}
|
|
|
|
|
|
def try_archive_today(url: str, timeout: int) -> dict | None:
|
|
for host in ARCHIVE_TODAY_HOSTS:
|
|
fetch_url = f"https://{host}/newest/{url}"
|
|
status, body = _fetch(fetch_url, timeout)
|
|
if status != 200:
|
|
continue
|
|
if validate(body, "archive_today", url):
|
|
continue # 429 bodies and interstitials land here
|
|
return {
|
|
"route": f"archive_today:{host}",
|
|
"provenance": "snapshot",
|
|
"snapshot_timestamp": None, # archive.today embeds the date in-page
|
|
"source_url": fetch_url,
|
|
"body": body,
|
|
}
|
|
return None
|
|
|
|
|
|
def try_jina(url: str, timeout: int) -> dict | None:
|
|
key = os.environ.get("JINA_API_KEY")
|
|
if not key:
|
|
return None
|
|
status, body = _fetch(
|
|
"https://r.jina.ai/" + url, timeout, headers={"Authorization": f"Bearer {key}"}
|
|
)
|
|
if status != 200 or validate(body, "jina", url):
|
|
return None
|
|
return {
|
|
"route": "jina_reader",
|
|
"provenance": "live",
|
|
"snapshot_timestamp": None,
|
|
"source_url": "https://r.jina.ai/" + url,
|
|
"body": body,
|
|
}
|
|
|
|
|
|
ROUTES = (try_wayback, try_archive_today, try_jina)
|
|
|
|
|
|
def recover(url: str, timeout: int = 25) -> dict | None:
|
|
for route_fn in ROUTES:
|
|
result = route_fn(url, timeout)
|
|
if result:
|
|
return result
|
|
return None
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(
|
|
description="Recover a blocked / paywalled / WAF'd page from third-party copies."
|
|
)
|
|
ap.add_argument("url")
|
|
ap.add_argument("--json", action="store_true", help="print metadata as JSON")
|
|
ap.add_argument("--out", help="write recovered body to this file")
|
|
ap.add_argument("--timeout", type=int, default=25)
|
|
args = ap.parse_args()
|
|
|
|
if not args.url.startswith(("http://", "https://")):
|
|
print("error: URL must start with http:// or https://", file=sys.stderr)
|
|
return 2
|
|
|
|
result = recover(args.url, args.timeout)
|
|
if not result:
|
|
msg = {"recovered": False, "url": args.url,
|
|
"hint": "No archive copy found. Try the API-first pivot or the browser tool."}
|
|
print(json.dumps(msg, indent=2) if args.json else msg["hint"], file=sys.stderr)
|
|
return 1
|
|
|
|
body = result.pop("body")
|
|
result.update({"recovered": True, "url": args.url, "body_bytes": len(body)})
|
|
if args.out:
|
|
with open(args.out, "wb") as fh:
|
|
fh.write(body)
|
|
result["saved_to"] = args.out
|
|
|
|
if args.json:
|
|
print(json.dumps(result, indent=2))
|
|
else:
|
|
for k, v in result.items():
|
|
print(f"{k}: {v}")
|
|
if not args.out:
|
|
print("\n--- body (first 2000 chars) ---")
|
|
print(body[:2000].decode("utf-8", "replace"))
|
|
if result["provenance"] == "snapshot":
|
|
print(
|
|
"\nNOTE: this is an ARCHIVED SNAPSHOT, not the live page. "
|
|
"Cite it with its timestamp.",
|
|
file=sys.stderr,
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|