download_zhihu_answers.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
下载/增量同步某知乎用户的全部回答，并渲染成一个 Bootstrap 5 单页。

数据流（默认输出到 answers/<user>/）:
    answers/<user>/answers.json  ← 原始数据源（每条回答的原始 API 对象；截断的存重拉后的全文）
    answers/<user>/images/       ← 本地化的图片
    answers/<user>/assets/       ← 本地化的 bootstrap.min.css
    answers/<user>/index.html    ← 由 answers.json 渲染出的单页（标题/时间/赞数/正文 + 搜索）

用法:
    1. 把浏览器复制的整条 Cookie 存到 cookies.txt（形如 'z_c0=...; d_c0=...; ...'）
    2. python3 download_zhihu_answers.py [--user <url_token>] [--out <dir>] [--cookie cookies.txt]
       - 首次运行：拉取全部回答。
       - 之后运行：增量同步，只补新增的回答（碰到整页都是已知 id 即提前停止）。

依赖:  pip install requests beautifulsoup4

策略:
    - cookie 认证 + 每请求生成知乎反爬签名头 x-zse-96（深层分页/部分接口强制要求，
      缺失会被 WAF 拦截并返回 code 10003「请求参数异常，请升级客户端」）。
    - 列表接口对长答案会截断，content_need_truncated=true 的重拉单答案接口取全文。
    - 每请求随机 sleep；遇 429/反爬指数退避。
    - 图片在渲染阶段按需下载（已存在的跳过），整个存档可离线打开。
"""

import argparse
import hashlib
import html
import json
import os
import random
import re
import sys
import time
from datetime import datetime, timezone
from urllib.parse import urlparse, parse_qs, unquote, urlsplit

try:
    import requests
    from bs4 import BeautifulSoup
except ImportError as e:
    sys.exit("缺少依赖：%s\n请先运行: pip install requests beautifulsoup4" % e)

try:
    from markdownify import markdownify as _html_to_md
except ImportError:
    _html_to_md = None  # 仅 render-md 子命令需要，缺失时在该命令里再报错


# ----------------------------------------------------------------------------- 常量
DEFAULT_USER = "ban-ma-ban-ma-30-2"

INCLUDE = (
    "data[*].is_normal,admin_closed_comment,reward_info,is_collapsed,"
    "annotation_action,annotation_detail,collapse_reason,collapsed_by,suggest_edit,"
    "comment_count,can_comment,content,editable_content,attachment,voteup_count,"
    "reshipment_settings,comment_permission,created_time,updated_time,review_info,"
    "excerpt,paid_info,reaction_instruction,is_labeled,label_info,"
    "relationship.is_authorized,voting,is_author,is_thanked,is_nothelp,reaction,"
    "vessay_info;data[*].author.badge[?(type=best_answerer)].topics;"
    "data[*].author.kvip_info;data[*].author.vip_info;"
    "data[*].question.has_publishing_draft,relationship"
)
ANSWER_INCLUDE = "content,excerpt,created_time,updated_time,voteup_count,comment_count,question"

UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36")

API_BASE = "https://www.zhihu.com/api/v4"
BOOTSTRAP_URL = "https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css"

DATA_FILE = "answers.json"
INDEX_FILE = "index.html"
MD_FILE = "answers.md"
IMAGES_DIR = "images"
ASSETS_DIR = "assets"


# ----------------------------------------------------------------------------- 工具
def log(msg):
    print("[%s] %s" % (datetime.now().strftime("%H:%M:%S"), msg), flush=True)


def load_cookie(path):
    if not os.path.exists(path):
        sys.exit("找不到 cookie 文件: %s\n请把浏览器复制的整条 Cookie 存进去。" % path)
    with open(path, encoding="utf-8") as f:
        raw = f.read().strip()
    raw = re.sub(r"^[Cc]ookie:\s*", "", raw).replace("\n", " ").strip()
    if "z_c0" not in raw:
        log("警告：cookie 里没有 z_c0（登录态 token），可能未登录，受限内容会拉不到。")
    return raw


def ts_to_date(ts):
    if not ts:
        return "0000-00-00"
    return datetime.fromtimestamp(ts, tz=timezone.utc).astimezone().strftime("%Y-%m-%d")


def ts_to_str(ts):
    if not ts:
        return ""
    return datetime.fromtimestamp(ts, tz=timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M")


def decode_zhihu_link(href):
    """知乎外链包了一层 link.zhihu.com/?target=...，解出真实地址。"""
    try:
        p = urlparse(href)
        if p.netloc.endswith("link.zhihu.com"):
            target = parse_qs(p.query).get("target")
            if target:
                return unquote(target[0])
    except Exception:
        pass
    return href


# ----------------------------------------------------------------------------- x-zse-96 反爬签名
# 知乎对深层分页/部分 v4 接口强制校验请求头 x-zse-96，缺失或错误会被 WAF 拦截
# （HTTP 403, code 10003「请求参数异常，请升级客户端」）。算法为：
#     md5("101_3_3.0+" + 请求的 path+query + "+" + cookie 里的 d_c0 值)
# 再过一层知乎自定义的 SM4 变体分组密码 + 自定义 base64，最终拼成 "2.0_<密文>"。
# 纯 Python 移植，无需 node/execjs。参照并对拍验证一致：
#     https://github.com/njzjz/zhihubackup/blob/main/zhihubackup/xzse96.js
ZSE93 = "101_3_3.0"
_M = 0xFFFFFFFF
_ZSE_ZK = [x & _M for x in (
    1170614578, 1024848638, 1413669199, -343334464, -766094290, -1373058082,
    -143119608, -297228157, 1933479194, -971186181, -406453910, 460404854,
    -547427574, -1891326262, -1679095901, 2119585428, -2029270069, 2035090028,
    -1521520070, -5587175, -77751101, -2094365853, -1243052806, 1579901135,
    1321810770, 456816404, -1391643889, -229302305, 330002838, -788960546,
    363569021, -1947871109,
)]
_ZSE_ZB = [
    20, 223, 245, 7, 248, 2, 194, 209, 87, 6, 227, 253, 240, 128, 222, 91, 237,
    9, 125, 157, 230, 93, 252, 205, 90, 79, 144, 199, 159, 197, 186, 167, 39,
    37, 156, 198, 38, 42, 43, 168, 217, 153, 15, 103, 80, 189, 71, 191, 97, 84,
    247, 95, 36, 69, 14, 35, 12, 171, 28, 114, 178, 148, 86, 182, 32, 83, 158,
    109, 22, 255, 94, 238, 151, 85, 77, 124, 254, 18, 4, 26, 123, 176, 232, 193,
    131, 172, 143, 142, 150, 30, 10, 146, 162, 62, 224, 218, 196, 229, 1, 192,
    213, 27, 110, 56, 231, 180, 138, 107, 242, 187, 54, 120, 19, 44, 117, 228,
    215, 203, 53, 239, 251, 127, 81, 11, 133, 96, 204, 132, 41, 115, 73, 55,
    249, 147, 102, 48, 122, 145, 106, 118, 74, 190, 29, 16, 174, 5, 177, 129,
    63, 113, 99, 31, 161, 76, 246, 34, 211, 13, 60, 68, 207, 160, 65, 111, 82,
    165, 67, 169, 225, 57, 112, 244, 155, 51, 236, 200, 233, 58, 61, 47, 100,
    137, 185, 64, 17, 70, 234, 163, 219, 108, 170, 166, 59, 149, 52, 105, 24,
    212, 78, 173, 45, 0, 116, 226, 119, 136, 206, 135, 175, 195, 25, 92, 121,
    208, 126, 139, 3, 75, 141, 21, 130, 98, 241, 40, 154, 66, 184, 49, 181, 46,
    243, 88, 101, 183, 8, 23, 72, 188, 104, 179, 210, 134, 250, 201, 164, 89,
    216, 202, 220, 50, 221, 152, 140, 33, 235, 214,
]
_ZSE_SALT = "6fpLRqJO8M/c3jnYxFkUVC4ZIG12SiH=5v0mXDazWBTsuw7QetbKdoPyAl+hN9rgE"
_ZSE_FIX = [48, 53, 57, 48, 53, 51, 102, 55, 100, 49, 53, 101, 48, 49, 100, 55]


def _zse_i(e, t, n):
    e &= _M
    t[n] = 0xFF & (e >> 24)
    t[n + 1] = 0xFF & (e >> 16)
    t[n + 2] = 0xFF & (e >> 8)
    t[n + 3] = 0xFF & e


def _zse_B(e, t):
    return ((0xFF & e[t]) << 24 | (0xFF & e[t + 1]) << 16 |
            (0xFF & e[t + 2]) << 8 | (0xFF & e[t + 3])) & _M


def _zse_Q(e, t):  # 循环左移 32 位
    e &= _M
    return ((e << t) | (e >> (32 - t))) & _M


def _zse_G(e):
    t = [0] * 4
    n = [0] * 4
    _zse_i(e, t, 0)
    n[0] = _ZSE_ZB[0xFF & t[0]]
    n[1] = _ZSE_ZB[0xFF & t[1]]
    n[2] = _ZSE_ZB[0xFF & t[2]]
    n[3] = _ZSE_ZB[0xFF & t[3]]
    r = _zse_B(n, 0)
    return (r ^ _zse_Q(r, 2) ^ _zse_Q(r, 10) ^ _zse_Q(r, 18) ^ _zse_Q(r, 24)) & _M


def _zse_r(e):
    t = [0] * 16
    n = [0] * 36
    n[0] = _zse_B(e, 0)
    n[1] = _zse_B(e, 4)
    n[2] = _zse_B(e, 8)
    n[3] = _zse_B(e, 12)
    for r in range(32):
        o = _zse_G((n[r + 1] ^ n[r + 2] ^ n[r + 3] ^ _ZSE_ZK[r]) & _M)
        n[r + 4] = (n[r] ^ o) & _M
    _zse_i(n[35], t, 0)
    _zse_i(n[34], t, 4)
    _zse_i(n[33], t, 8)
    _zse_i(n[32], t, 12)
    return t


def _zse_x(e, t):
    n = []
    r = len(e)
    idx = 0
    while r > 0:
        o = e[16 * idx: 16 * (idx + 1)]
        a = [(o[c] ^ t[c]) & 0xFF for c in range(16)]
        t = _zse_r(a)
        n = n + t
        idx += 1
        r -= 16
    return n


def _zse_encode(param):
    param &= _M
    out = ""
    for x in (0, 6, 12, 18):
        out += _ZSE_SALT[(param >> x) & 63]
    return out


def _zse_encrypt(md5hex, rand_byte):
    arr = [ord(ch) for ch in md5hex]
    arr.insert(0, 0)
    arr.insert(0, int(rand_byte))   # JS: Math.random()*127，^ 运算按 ToInt32 截断
    arr.extend([14] * 15)
    front = arr[0:16]
    new_md5 = [(front[i] ^ _ZSE_FIX[i] ^ 42) & _M for i in range(16)]
    gr = _zse_r(new_md5)
    processed = gr + _zse_x(arr[16:48], gr)
    current = 0
    out = ""
    for i in range(len(processed)):
        pop = processed[len(processed) - i - 1]
        c = (58 >> (8 * (i % 4))) & 255
        d = (pop ^ c) & _M
        current = (current | ((d << (8 * (i % 3))) & _M)) & _M
        if i % 3 == 2:
            out += _zse_encode(current)
            current = 0
    return out


def zse96_sign(api_path, dc0):
    """api_path 形如 /api/v4/...?a=b（请求实际发出的 path+query）；dc0 为 cookie 里 d_c0 的原值。
    返回 x-zse-96 头的值。"""
    f = "%s+%s+%s" % (ZSE93, api_path, dc0)
    md5hex = hashlib.md5(f.encode("utf-8")).hexdigest()
    return "2.0_" + _zse_encrypt(md5hex, int(random.random() * 127))


# ----------------------------------------------------------------------------- 客户端
class ZhihuClient:
    def __init__(self, cookie, min_delay, max_delay):
        self.s = requests.Session()
        self.s.headers.update({
            "User-Agent": UA,
            "Accept": "*/*",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Referer": "https://www.zhihu.com/",
            "x-requested-with": "fetch",
            "Cookie": cookie,
        })
        self.min_delay = min_delay
        self.max_delay = max_delay
        m = re.search(r"d_c0=([^;]+)", cookie)
        self.dc0 = m.group(1) if m else None
        if not self.dc0:
            log("警告：cookie 里没有 d_c0，无法生成 x-zse-96 签名，深层分页可能被拦截。")

    def _sleep(self):
        time.sleep(random.uniform(self.min_delay, self.max_delay))

    def _signed_get(self, url, params):
        """带 x-zse-96 签名发起 GET；签名必须匹配实际发出的 path+query，故先 prepare 再签。"""
        req = requests.Request("GET", url, params=params)
        prep = self.s.prepare_request(req)
        if self.dc0:
            sp = urlsplit(prep.url)
            api_path = sp.path + (("?" + sp.query) if sp.query else "")
            prep.headers["x-zse-93"] = ZSE93
            prep.headers["x-zse-96"] = zse96_sign(api_path, self.dc0)
        return self.s.send(prep, timeout=30)

    def get_json(self, url, params=None, _tries=0, _max_tries=6):
        self._sleep()
        try:
            r = self._signed_get(url, params)
        except requests.RequestException as e:
            if _tries >= _max_tries:
                raise
            back = min(60, 5 * 2 ** _tries) + random.uniform(0, 3)
            log("网络异常 %s，%.0fs 后重试 (%d/%d)" % (e, back, _tries + 1, _max_tries))
            time.sleep(back)
            return self.get_json(url, params, _tries + 1, _max_tries)

        if r.status_code == 200:
            try:
                return r.json()
            except ValueError:
                self._raise_blocked(r)
        if r.status_code in (429, 403, 401):
            return self._handle_blocked(r, url, params, _tries, _max_tries)
        if 500 <= r.status_code < 600 and _tries < _max_tries:
            back = min(60, 5 * 2 ** _tries) + random.uniform(0, 3)
            log("服务端 %d，%.0fs 后重试 (%d/%d)" % (r.status_code, back, _tries + 1, _max_tries))
            time.sleep(back)
            return self.get_json(url, params, _tries + 1, _max_tries)
        r.raise_for_status()
        return r.json()

    def _handle_blocked(self, r, url, params, _tries, _max_tries):
        # 429 限速 / 403 反爬（含签名校验）都退避重试——每次重试会用新随机数重新签名，
        # 知乎 WAF 的拦截带概率性，重试常能放行。
        if r.status_code in (429, 403) and _tries < _max_tries:
            back = min(120, 10 * 2 ** _tries) + random.uniform(0, 5)
            log("被拦截 %d，%.0fs 后退避重试 (%d/%d)" % (r.status_code, back, _tries + 1, _max_tries))
            time.sleep(back)
            return self.get_json(url, params, _tries + 1, _max_tries)
        self._raise_blocked(r)

    def _raise_blocked(self, r):
        snippet = (r.text or "")[:200].replace("\n", " ")
        raise SystemExit(
            "请求被知乎拦截 (HTTP %d)，多次退避重试仍失败。\n返回片段: %s\n\n"
            "脚本已携带 x-zse-96 反爬签名。仍被拦通常是：\n"
            "  1) cookies.txt 登录态失效——请重新从浏览器复制整条 Cookie（务必含 z_c0、d_c0）；\n"
            "  2) 请求过于频繁——调大 --min-delay/--max-delay 后重试；\n"
            "  3) 知乎再次升级了 x-zse-96 算法（x-zse-93 版本号变化）——需更新签名模块。"
            % (r.status_code, snippet)
        )

    def download_to(self, url, path, referer="https://www.zhihu.com/"):
        self._sleep()
        try:
            r = self.s.get(url, timeout=30, headers={"Referer": referer})
            if r.status_code != 200 or not r.content:
                return False
        except requests.RequestException:
            return False
        with open(path, "wb") as f:
            f.write(r.content)
        return True


# ----------------------------------------------------------------------------- 数据存取
def load_store(path):
    if os.path.exists(path):
        with open(path, encoding="utf-8") as f:
            store = json.load(f)
        if "answers" not in store:
            store = {"user": None, "answers": {}}
        return store
    return {"user": None, "answers": {}}


def save_store(path, store):
    store["updated_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    with open(path, "w", encoding="utf-8") as f:
        json.dump(store, f, ensure_ascii=False, indent=1)


# ----------------------------------------------------------------------------- 抓取（增量）
def fetch_new(client, user, store):
    """按发布时间倒序翻页，只补新增的回答；整页都是已知 id 即提前停。"""
    answers = store["answers"]
    offset, limit = 0, 20
    total = None
    new_ids = []

    while True:
        log("拉取列表 offset=%d …" % offset)
        data = client.get_json(
            "%s/members/%s/answers" % (API_BASE, user),
            params={"include": INCLUDE, "offset": offset, "limit": limit, "sort_by": "created"},
        )
        paging = data.get("paging", {})
        if total is None:
            total = paging.get("totals")
            if total is not None:
                store["total_reported"] = total
            log("该用户接口报告共约 %s 个回答；本地已有 %d 条。" % (total, len(answers)))

        batch = data.get("data", [])
        if not batch:
            break

        known_in_page = 0
        for ans in batch:
            aid = str(ans["id"])
            if aid in answers:
                known_in_page += 1
                continue
            title = (ans.get("question", {}) or {}).get("title", "")
            log("  新增: %s %s" % (aid, title[:30]))
            if ans.get("content_need_truncated"):
                log("    标记可能截断，重拉单答案接口取全文…")
                try:
                    full = client.get_json("%s/answers/%s" % (API_BASE, aid),
                                           params={"include": ANSWER_INCLUDE})
                    if full.get("content"):
                        ans["content"] = full["content"]
                        ans["content_need_truncated"] = False
                        ans["_refetched_full"] = True
                except SystemExit:
                    raise
                except Exception as e:
                    log("    重拉失败(%s)，沿用列表正文。" % e)
            answers[aid] = ans
            new_ids.append(aid)

        # 增量提前停：整页都是已知（按时间倒序，后面只会更旧）
        if known_in_page == len(batch) and len(answers) > len(new_ids):
            log("  整页均为已知回答，提前停止翻页。")
            break
        if paging.get("is_end"):
            break
        offset += limit

    return new_ids, total


def fetch_single(client, aid):
    """按 answer_id 直接抓单条回答（即使列表/动态流不返回也能取到）。"""
    a = client.get_json("%s/answers/%s" % (API_BASE, aid), params={"include": ANSWER_INCLUDE})
    if a.get("error"):
        return None
    return a


def add_answers(client, store, ids):
    """把指定 answer_id 直接抓取并注入 store（用于补列表接口不返回的回答）。"""
    answers = store["answers"]
    added = 0
    for aid in ids:
        aid = str(aid).strip()
        if not aid:
            continue
        if aid in answers:
            log("  %s 已在本地，跳过。" % aid)
            continue
        try:
            a = fetch_single(client, aid)
        except SystemExit:
            raise
        except Exception as e:
            log("  抓取 %s 失败: %s" % (aid, e))
            continue
        if not a:
            log("  %s 抓不到（可能已删除/无权限）。" % aid)
            continue
        title = (a.get("question", {}) or {}).get("title", "")
        a["_added_manually"] = True
        answers[aid] = a
        added += 1
        log("  已补入: %s 👍%s %s" % (aid, a.get("voteup_count"), title[:30]))
    return added


# ----------------------------------------------------------------------------- 图片本地化（渲染期）
def pick_img_url(img):
    for attr in ("data-original", "data-actualsrc", "data-default-watermark-src", "src"):
        v = img.get(attr)
        if v and not v.startswith("data:"):
            return v
    return None


def md5name(url):
    import hashlib
    ext = os.path.splitext(urlparse(url).path)[1].lower()
    if ext not in (".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg"):
        ext = ".jpg"
    return hashlib.md5(url.encode()).hexdigest()[:16] + ext


def localize_content(content_html, client, images_dir, stats):
    """改写外链、按需下载图片并改写为本地路径，返回处理后的 HTML。"""
    soup = BeautifulSoup(content_html or "", "html.parser")
    for ns in soup.find_all("noscript"):
        ns.decompose()

    for img in soup.find_all("img"):
        url = pick_img_url(img)
        if not url:
            continue
        if url.startswith("//"):
            url = "https:" + url
        name = md5name(url)
        path = os.path.join(images_dir, name)
        if not os.path.exists(path):
            if client and client.download_to(url, path):
                stats["downloaded"] += 1
            else:
                stats["failed"] += 1
                img["src"] = url  # 下载失败保留远程
                continue
        img["src"] = "%s/%s" % (IMAGES_DIR, name)
        for a in ("data-original", "data-actualsrc", "data-default-watermark-src",
                  "data-rawwidth", "data-rawheight", "srcset"):
            if img.has_attr(a):
                del img[a]

    for a in soup.find_all("a", href=True):
        a["href"] = decode_zhihu_link(a["href"])

    return str(soup)


# ----------------------------------------------------------------------------- 渲染单页
PAGE_TEMPLATE = """<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{title}</title>
<link rel="stylesheet" href="{assets}/bootstrap.min.css">
<style>
  body {{ background:#f5f6f8; }}
  .answer-content {{ font-size:1.02rem; line-height:1.8; word-break:break-word; }}
  .answer-content img {{ max-width:100%; height:auto; border-radius:6px; margin:.5rem 0; }}
  .answer-content figure {{ margin:1rem 0; }}
  .answer-content figcaption {{ color:#8590a6; font-size:.85rem; text-align:center; }}
  .answer-content blockquote {{ border-left:4px solid #dfe2e5; padding-left:1rem; color:#555; }}
  .answer-content pre {{ background:#f6f8fa; padding:1rem; border-radius:6px; overflow:auto; }}
  .sticky-top-bar {{ position:sticky; top:0; z-index:1020; }}
  .card-question a {{ text-decoration:none; }}
  .card-question a:hover {{ text-decoration:underline; }}
  .answer-card {{ scroll-margin-top:70px; }}
  #toc {{ max-height:45vh; overflow-y:auto; }}
  .toc-item {{ font-size:.92rem; }}
  #toTop {{ position:fixed; right:24px; bottom:24px; display:none; z-index:1030; }}
  mark {{ padding:0; }}
</style>
</head>
<body>
<nav class="navbar navbar-dark bg-dark sticky-top-bar shadow-sm">
  <div class="container">
    <span class="navbar-brand mb-0 h1">📚 {user} 的知乎回答</span>
    <span class="navbar-text text-light small">共 {count} 条{total_note} · 更新于 {updated}</span>
  </div>
</nav>

<div class="container py-4" style="max-width:860px;">
  <div class="mb-3">
    <input id="search" class="form-control form-control-lg" type="search"
           placeholder="🔍 搜索问题标题…（{count} 条回答）">
    <div id="searchInfo" class="form-text"></div>
  </div>

  <div class="card shadow-sm mb-4">
    <div id="tocHeader" class="card-header bg-white d-flex justify-content-between align-items-center"
         role="button">
      <span class="fw-bold">📑 目录 <span id="tocCaret">▾</span></span>
      <span id="tocCount" class="text-muted small">{count} 条</span>
    </div>
    <div id="tocWrap">
      <div id="toc" class="list-group list-group-flush">
        {toc}
      </div>
    </div>
  </div>

  {cards}
</div>

<button id="toTop" class="btn btn-dark btn-sm rounded-circle p-2" title="回到顶部"
        onclick="window.scrollTo({{top:0,behavior:'smooth'}})">↑</button>

<script>
const search = document.getElementById('search');
const cards = Array.from(document.querySelectorAll('.answer-card'));
const tocItems = Array.from(document.querySelectorAll('.toc-item'));
const info = document.getElementById('searchInfo');
const tocCount = document.getElementById('tocCount');
search.addEventListener('input', () => {{
  const q = search.value.trim().toLowerCase();
  let n = 0;
  cards.forEach(c => {{
    const hit = !q || c.dataset.title.toLowerCase().includes(q);
    c.style.display = hit ? '' : 'none';
    if (hit) n++;
  }});
  tocItems.forEach(t => {{
    t.style.display = (!q || t.dataset.title.toLowerCase().includes(q)) ? '' : 'none';
  }});
  info.textContent = q ? ('匹配 ' + n + ' 条') : '';
  tocCount.textContent = (q ? n : cards.length) + ' 条';
}});
const toTop = document.getElementById('toTop');
window.addEventListener('scroll', () => {{
  toTop.style.display = window.scrollY > 600 ? 'block' : 'none';
}});
// 目录折叠（原生 JS，无需 Bootstrap JS）
const tocHeader = document.getElementById('tocHeader');
const tocWrap = document.getElementById('tocWrap');
const tocCaret = document.getElementById('tocCaret');
tocHeader.addEventListener('click', () => {{
  const hidden = tocWrap.style.display === 'none';
  tocWrap.style.display = hidden ? '' : 'none';
  tocCaret.textContent = hidden ? '▾' : '▸';
}});
</script>
</body>
</html>
"""

TOC_ITEM_TEMPLATE = (
    '<a href="#a{aid}" class="list-group-item list-group-item-action toc-item '
    'd-flex justify-content-between align-items-center" data-title="{title_attr}">'
    '<span class="text-truncate me-2">{idx}. {title}</span>'
    '<small class="text-muted text-nowrap">{created_date} · 👍{voteup}</small></a>'
)

CARD_TEMPLATE = """
  <div class="card shadow-sm mb-4 answer-card" data-title="{title_attr}" id="a{aid}">
    <div class="card-header bg-white card-question">
      <h5 class="mb-1"><a href="{answer_url}" target="_blank" rel="noopener">{title}</a></h5>
      <div class="text-muted small">
        发布于 {created}{updated_html} ·
        👍 {voteup} · 💬 {comments} ·
        <a href="{answer_url}" target="_blank" rel="noopener" class="text-muted">原回答</a>
      </div>
    </div>
    <div class="card-body answer-content">
      {content}
    </div>
  </div>
"""


def render(store, out_dir, client, user):
    images_dir = os.path.join(out_dir, IMAGES_DIR)
    assets_dir = os.path.join(out_dir, ASSETS_DIR)
    os.makedirs(images_dir, exist_ok=True)
    os.makedirs(assets_dir, exist_ok=True)

    # 本地化 bootstrap
    css_path = os.path.join(assets_dir, "bootstrap.min.css")
    if not os.path.exists(css_path):
        log("下载 Bootstrap 5 到本地…")
        if not (client and client.download_to(BOOTSTRAP_URL, css_path, referer="")):
            log("⚠ Bootstrap 下载失败，页面样式可能缺失（可手动放 assets/bootstrap.min.css）。")

    answers = list(store["answers"].values())
    answers.sort(key=lambda a: a.get("created_time") or 0, reverse=True)

    stats = {"downloaded": 0, "failed": 0}
    cards = []
    toc = []
    for idx, ans in enumerate(answers, 1):
        aid = str(ans["id"])
        q = ans.get("question", {}) or {}
        qid = q.get("id")
        title = q.get("title", "无标题问题")
        created = ts_to_str(ans.get("created_time"))
        updated = ts_to_str(ans.get("updated_time"))
        updated_html = (" · 编辑于 %s" % updated) if updated and updated != created else ""
        content = localize_content(ans.get("content", ""), client, images_dir, stats)
        answer_url = "https://www.zhihu.com/question/%s/answer/%s" % (qid, aid)
        voteup = ans.get("voteup_count", 0)
        cards.append(CARD_TEMPLATE.format(
            aid=html.escape(aid),
            title=html.escape(title),
            title_attr=html.escape(title, quote=True),
            answer_url=html.escape(answer_url),
            created=created,
            updated_html=updated_html,
            voteup=voteup,
            comments=ans.get("comment_count", 0),
            content=content,
        ))
        toc.append(TOC_ITEM_TEMPLATE.format(
            aid=html.escape(aid),
            idx=idx,
            title=html.escape(title),
            title_attr=html.escape(title, quote=True),
            created_date=ts_to_date(ans.get("created_time")),
            voteup=voteup,
        ))

    total_reported = store.get("total_reported")
    if total_reported and total_reported != len(answers):
        total_note = ('（知乎接口计 %d，差 %d 条为被删/关闭/折叠的回答）'
                      % (total_reported, total_reported - len(answers)))
    else:
        total_note = ""

    page = PAGE_TEMPLATE.format(
        title="%s 的知乎回答" % user,
        user=html.escape(user),
        count=len(answers),
        total_note=html.escape(total_note),
        updated=store.get("updated_at", ""),
        assets=ASSETS_DIR,
        toc="\n".join(toc),
        cards="\n".join(cards),
    )
    with open(os.path.join(out_dir, INDEX_FILE), "w", encoding="utf-8") as f:
        f.write(page)
    if stats["downloaded"] or stats["failed"]:
        log("图片：新下载 %d 张，失败 %d 张。" % (stats["downloaded"], stats["failed"]))

    # 清理不再被引用的孤儿图片（URL 变体/已删回答留下的）
    referenced = set(re.findall(r"%s/([^\"'\s]+)" % re.escape(IMAGES_DIR), page))
    orphans = 0
    for fn in os.listdir(images_dir):
        if fn not in referenced:
            os.remove(os.path.join(images_dir, fn))
            orphans += 1
    if orphans:
        log("清理孤儿图片 %d 张。" % orphans)
    return len(answers)


# ----------------------------------------------------------------------------- 旧中间文件清理
def cleanup_legacy(out_dir):
    """删除旧版本生成的每条 .md / .html 中间文件（保留 index.html）。"""
    removed = 0
    for fn in os.listdir(out_dir):
        full = os.path.join(out_dir, fn)
        if not os.path.isfile(full):
            continue
        if fn == INDEX_FILE:
            continue
        if fn.endswith(".md") or (fn.endswith(".html") and re.match(r"^\d{4}-\d{2}-\d{2}_", fn)):
            os.remove(full)
            removed += 1
    if removed:
        log("已清理 %d 个旧中间文件（每条 md/html）。" % removed)


# ----------------------------------------------------------------------------- render 子命令
def discover_users(out_root):
    """扫描 out_root 下含 answers.json 的子目录，返回 user 列表。"""
    if not os.path.isdir(out_root):
        return []
    users = []
    for name in sorted(os.listdir(out_root)):
        if os.path.isfile(os.path.join(out_root, name, DATA_FILE)):
            users.append(name)
    return users


def cmd_render(args):
    """把所有（或指定）用户的 answers.json 重新渲染成 index.html，不联网抓取。"""
    out_root = args.out_root
    users = [args.user] if args.user else discover_users(out_root)
    if not users:
        sys.exit("在 %s 下没找到任何含 %s 的用户目录。" % (out_root, DATA_FILE))

    # 渲染只需在缺图片/bootstrap 时联网补；有 cookie 就建客户端，没有也能纯离线渲染
    client = None
    cookie = None
    try:
        cookie = load_cookie(args.cookie)
    except SystemExit:
        pass
    if cookie:
        client = ZhihuClient(cookie, args.min_delay, args.max_delay)

    log("render：共 %d 个用户 → %s" % (len(users), users))
    for u in users:
        out_dir = os.path.join(out_root, u)
        store = load_store(os.path.join(out_dir, DATA_FILE))
        if not store["answers"]:
            log("  跳过 %s（answers.json 为空）。" % u)
            continue
        n = render(store, out_dir, client, u)
        log("  ✓ %s → %s（%d 条）" % (u, os.path.join(out_dir, INDEX_FILE), n))


# ----------------------------------------------------------------------------- render-md 子命令（便于 LLM 阅读）
def content_to_markdown(content_html):
    """把回答正文 HTML 转成 Markdown，忽略图片（含 noscript 里的图片回退）、还原外链。"""
    soup = BeautifulSoup(content_html or "", "html.parser")
    for ns in soup.find_all("noscript"):  # noscript 常内嵌一份重复 <img>
        ns.decompose()
    for a in soup.find_all("a", href=True):
        a["href"] = decode_zhihu_link(a["href"])
    md = _html_to_md(str(soup), heading_style="ATX", bullets="-", strip=["img"])
    return re.sub(r"\n{3,}", "\n\n", md).strip()


def render_md(store, out_dir, user):
    """把 answers.json 渲染成单个 Markdown 文件（标题 / 回答时间 / 正文），忽略图片。"""
    answers = list(store["answers"].values())
    answers.sort(key=lambda a: a.get("created_time") or 0, reverse=True)

    parts = ["# %s 的知乎回答\n\n共 %d 条 · 更新于 %s\n"
             % (user, len(answers), store.get("updated_at", ""))]
    for ans in answers:
        q = ans.get("question", {}) or {}
        title = q.get("title", "无标题问题")
        created = ts_to_str(ans.get("created_time"))
        body = content_to_markdown(ans.get("content", ""))
        parts.append("\n---\n\n## %s\n\n回答时间：%s\n\n%s\n" % (title, created, body))

    text = "\n".join(parts).rstrip() + "\n"
    with open(os.path.join(out_dir, MD_FILE), "w", encoding="utf-8") as f:
        f.write(text)
    return len(answers)


def cmd_render_md(args):
    """把所有（或指定）用户的 answers.json 渲染成便于 LLM 阅读的 Markdown，忽略图片，不联网。"""
    if _html_to_md is None:
        sys.exit("render-md 需要 markdownify：请先运行 pip install markdownify")

    out_root = args.out_root
    users = [args.user] if args.user else discover_users(out_root)
    if not users:
        sys.exit("在 %s 下没找到任何含 %s 的用户目录。" % (out_root, DATA_FILE))

    log("render-md：共 %d 个用户 → %s" % (len(users), users))
    for u in users:
        out_dir = os.path.join(out_root, u)
        store = load_store(os.path.join(out_dir, DATA_FILE))
        if not store["answers"]:
            log("  跳过 %s（answers.json 为空）。" % u)
            continue
        n = render_md(store, out_dir, u)
        log("  ✓ %s → %s（%d 条）" % (u, os.path.join(out_dir, MD_FILE), n))


# ----------------------------------------------------------------------------- sync（默认）
def cmd_sync(args):
    out_dir = args.out or os.path.join("answers", args.user)
    os.makedirs(out_dir, exist_ok=True)
    data_path = os.path.join(out_dir, DATA_FILE)

    store = load_store(data_path)
    store["user"] = args.user

    client = None

    # --add-answer：按 id/URL 直接补抓指定回答（列表接口不返回的）
    if args.add_answer:
        cookie = load_cookie(args.cookie)
        client = ZhihuClient(cookie, args.min_delay, args.max_delay)
        ids = []
        for x in args.add_answer:
            m = re.search(r"/answer/(\d+)", x) or re.search(r"(\d{6,})", x)
            ids.append(m.group(1) if m else x)
        log("补抓指定回答: %s" % ids)
        added = add_answers(client, store, ids)
        save_store(data_path, store)
        log("补抓完成：新增 %d 条，本地共 %d 条。" % (added, len(store["answers"])))

    elif not args.render_only:
        cookie = load_cookie(args.cookie)
        client = ZhihuClient(cookie, args.min_delay, args.max_delay)
        log("目标: %s | 输出: %s | 间隔 %.1f-%.1fs" %
            (args.user, out_dir, args.min_delay, args.max_delay))
        try:
            new_ids, total = fetch_new(client, args.user, store)
            save_store(data_path, store)
            log("抓取完成：新增 %d 条，本地共 %d 条（接口报告约 %s）。" %
                (len(new_ids), len(store["answers"]), total))
            if total and len(store["answers"]) < total:
                log("注意：本地 %d < 接口计数 %d。差额通常是「问题被删/关闭」或"
                    "「被折叠/审核中」的回答——计数器算它、但列表接口不返回，属知乎固有偏差，"
                    "非漏抓。若你知道那条回答的 URL，可用 --add-answer <URL> 直接补。"
                    % (len(store["answers"]), total))
        except KeyboardInterrupt:
            save_store(data_path, store)
            log("已中断，已抓取的部分已存入 answers.json，下次可继续增量。")

    # answers.json 建好后，再清理旧版每条 md/html 中间文件（避免抓取失败时误删存档）
    if store["answers"]:
        cleanup_legacy(out_dir)

    # 渲染需要 client 来按需下载图片/bootstrap；render-only 时仍可联网下图，给个轻量 client
    if client is None:
        cookie = None
        try:
            cookie = load_cookie(args.cookie)
        except SystemExit:
            pass
        if cookie:
            client = ZhihuClient(cookie, args.min_delay, args.max_delay)

    n = render(store, out_dir, client, args.user)
    log("渲染完成：%s（%d 条回答）。" % (os.path.join(out_dir, INDEX_FILE), n))


# ----------------------------------------------------------------------------- 入口
def main():
    ap = argparse.ArgumentParser(description="下载/增量同步知乎用户回答并渲染成单页")
    ap.add_argument("--user", default=DEFAULT_USER, help="用户 url_token（默认 %s）" % DEFAULT_USER)
    ap.add_argument("--cookie", default="cookies.txt", help="cookie 文件路径（默认 cookies.txt）")
    ap.add_argument("--out", default=None, help="输出目录（默认 ./answers/<user>）")
    ap.add_argument("--min-delay", type=float, default=3.0, help="请求最小间隔秒（默认 3）")
    ap.add_argument("--max-delay", type=float, default=8.0, help="请求最大间隔秒（默认 8）")
    ap.add_argument("--render-only", action="store_true", help="只用本地 answers.json 重新渲染该用户，不联网抓取")
    ap.add_argument("--add-answer", nargs="+", metavar="ID_OR_URL", default=None,
                    help="按 answer_id 或回答 URL 直接补抓指定回答（用于列表接口不返回的回答），可多个")

    sub = ap.add_subparsers(dest="command")
    rp = sub.add_parser("render", help="把所有（或指定）用户的 answers.json 重新渲染成 index.html，不联网抓取")
    rp.add_argument("--user", default=None, help="只渲染该用户（默认渲染 out-root 下所有用户）")
    rp.add_argument("--out-root", default="answers", help="备份根目录（默认 ./answers）")
    rp.add_argument("--cookie", default="cookies.txt", help="cookie 文件（仅用于补下缺失图片，可选）")
    rp.add_argument("--min-delay", type=float, default=1.0)
    rp.add_argument("--max-delay", type=float, default=3.0)

    mp = sub.add_parser("render-md",
                        help="把所有（或指定）用户的 answers.json 渲染成便于 LLM 阅读的 Markdown（answers.md），忽略图片，不联网")
    mp.add_argument("--user", default=None, help="只渲染该用户（默认渲染 out-root 下所有用户）")
    mp.add_argument("--out-root", default="answers", help="备份根目录（默认 ./answers）")

    args = ap.parse_args()
    if args.command == "render":
        cmd_render(args)
    elif args.command == "render-md":
        cmd_render_md(args)
    else:
        cmd_sync(args)


if __name__ == "__main__":
    main()