#!/usr/bin/env python3
"""从页面75开始抓取所有模板URL"""
import subprocess
import re
import json

def get_html(url):
    try:
        result = subprocess.run(
            ['curl', '-s', '-L', '-A', 'Mozilla/5.0', url],
            capture_output=True, text=True, timeout=30
        )
        return result.stdout
    except:
        return ""

# 从页面75开始的所有列表页
pages = list(range(5, 100))  # 页面5-99
base_url = "https://www.court.gov.cn/susongyangshi/{}.html"

all_urls = {}

for page in pages:
    url = base_url.format(page)
    html = get_html(url)
    if len(html) < 1000 or "找不到" in html:
        continue
    
    # 提取模板URL
    urls = re.findall(r'/susongyangshi/xiangqing/(\d+)\.html', html)
    for uid in urls:
        full_url = f"/susongyangshi/xiangqing/{uid}.html"
        if full_url not in all_urls:
            # 提取标题
            title_match = re.search(rf'href="{re.escape(full_url)}"[^>]*>([^<]+)</a>', html)
            title = title_match.group(1).strip() if title_match else f"模板{uid}"
            all_urls[full_url] = title
    
    # 检查分页
    for i in range(2, 10):
        page_url = f"https://www.court.gov.cn/susongyangshi/{page}_{i}.html"
        page_html = get_html(page_url)
        if len(page_html) < 1000 or "找不到" in page_html:
            break
        urls = re.findall(r'/susongyangshi/xiangqing/(\d+)\.html', page_html)
        for uid in urls:
            full_url = f"/susongyangshi/xiangqing/{uid}.html"
            if full_url not in all_urls:
                title_match = re.search(rf'href="{re.escape(full_url)}"[^>]*>([^<]+)</a>', page_html)
                title = title_match.group(1).strip() if title_match else f"模板{uid}"
                all_urls[full_url] = title
    
    print(f"页面{page}: 发现{len(urls) if 'urls' in locals() else 0}个模板，总计{len(all_urls)}个")

print(f"\n总计找到 {len(all_urls)} 个模板")
with open('/root/.openclaw/workspace/知识库/法律文书模板库/全部模板URL列表.json', 'w') as f:
    json.dump(all_urls, f, ensure_ascii=False, indent=2)