#!/usr/bin/env python3
"""批量抓取法律文书模板内容"""
import os, re, json, time, subprocess, html as html_module

BASE = "https://www.court.gov.cn"
TEMPLATE_DIR = "/root/.openclaw/workspace/知识库/法律文书模板库/最高人民法院"
URL_FILE = "/root/.openclaw/workspace/知识库/法律文书模板库/全部模板URL列表.json"
DONE_FILE = "/root/.openclaw/workspace/知识库/法律文书模板库/已完成抓取.json"

# 简化分类映射
CATEGORY_MAP = {}

def get_html(url):
    try:
        r = subprocess.run(['curl', '-s', '-L', '-A', 'Mozilla/5.0', url], 
                          capture_output=True, text=True, timeout=30)
        return r.stdout
    except:
        return ""

def extract_content(html):
    # 提取正文 - 正确的class名
    m = re.search(r'<div[^>]*class="txt_txt"[^>]*id="zoom"[^>]*>(.*?)</div>\s*</div>\s*</div>', html, re.DOTALL)
    if not m:
        m = re.search(r'<div[^>]*class="txt_txt"[^>]*>(.*?)</div>', html, re.DOTALL)
    content = m.group(1) if m else ""
    
    # 清理HTML
    content = re.sub(r'<br\s*/?>', '\n', content)
    content = re.sub(r'</p>', '\n\n', content)
    content = re.sub(r'<[^>]+>', '', content)
    content = html_module.unescape(content)
    content = re.sub(r'&nbsp;', ' ', content)
    content = re.sub(r'\n{3,}', '\n\n', content)
    return content.strip()

def get_category(uid):
    if uid in CATEGORY_MAP:
        return CATEGORY_MAP[uid]
    return "其他类"

def main():
    with open(URL_FILE, 'r') as f:
        templates = json.load(f)
    
    done = set()
    if os.path.exists(DONE_FILE):
        with open(DONE_FILE, 'r') as f:
            done = set(json.load(f))
    
    total = len(templates)
    remaining = total - len(done)
    
    print(f"总共 {total} 个模板，已完成 {len(done)} 个，剩余 {remaining} 个")
    
    count = 0
    for url, title in sorted(templates.items(), key=lambda x: int(re.search(r'\d+', x[0]).group())):
        if url in done:
            continue
        
        uid = re.search(r'/xiangqing/(\d+)\.html', url)
        if not uid:
            continue
        uid = uid.group(1)
        
        category = get_category(uid)
        os.makedirs(os.path.join(TEMPLATE_DIR, category), exist_ok=True)
        
        html = get_html(BASE + url)
        if len(html) < 500:
            print(f"  [失败] {title} - HTML太短")
            done.add(url)
            continue
        
        content = extract_content(html)
        if len(content) < 50:
            print(f"  [失败] {title} - 内容太短({len(content)}字符)")
            done.add(url)
            continue
        
        safe_title = re.sub(r'[<>:"/\\|?*]', '', title)[:80]
        filename = f"{uid}_{safe_title}.txt"
        filepath = os.path.join(TEMPLATE_DIR, category, filename)
        
        with open(filepath, 'w', encoding='utf-8') as f:
            f.write(f"# {title}\n\n来源: {BASE + url}\n分类: {category}\n模板ID: {uid}\n\n{'='*60}\n\n{content}")
        
        done.add(url)
        count += 1
        
        if count % 50 == 0:
            with open(DONE_FILE, 'w') as f:
                json.dump(list(done), f)
            print(f"  >>> 已保存 {count} 个")
        
        if count % 10 == 0:
            print(f"  [进度] {len(done)}/{total} - {title[:30]}...")
        
        time.sleep(0.2)
    
    with open(DONE_FILE, 'w') as f:
        json.dump(list(done), f)
    
    print(f"\n完成！抓取 {count} 个模板")

if __name__ == "__main__":
    main()