#!/usr/bin/env python3
"""提取国家市场监管总局合同示范文本"""
import subprocess
import re
import os
import html as html_module

BASE_URL = "https://htsfwb.samr.gov.cn"
OUTPUT_DIR = "/root/.openclaw/workspace/知识库/法律文书模板库/市场监督管理局/国家总局"
os.makedirs(OUTPUT_DIR, exist_ok=True)

def get_html(url):
    try:
        r = subprocess.run(['curl', '-s', '-L', '-A', 'Mozilla/5.0', url], 
                          capture_output=True, text=True, timeout=30)
        return r.stdout
    except:
        return ""

def extract_content(html):
    """提取正文内容"""
    # 提取article标签内容
    m = re.search(r'<article[^>]*>(.*?)</article>', html, re.DOTALL)
    if not m:
        return None
    
    content = m.group(1)
    
    # 清理HTML标签但保留结构
    content = re.sub(r'<div[^>]*data-id="[^"]*"[^>]*>', '\n', content)
    content = re.sub(r'<p[^>]*data-id="[^"]*"[^>]*>', '\n', content)
    content = re.sub(r'<table[^>]*>.*?</table>', '', content, flags=re.DOTALL)
    content = re.sub(r'<[^>]+>', '', content)
    content = html_module.unescape(content)
    content = re.sub(r'\n{3,}', '\n\n', content)
    return content.strip()

def extract_title(html):
    """提取标题"""
    m = re.search(r'<title>([^<]+)</title>', html)
    if m:
        title = m.group(1)
        # 清理HTML实体
        title = html_module.unescape(title)
        return title
    return None

def main():
    # 读取URL列表
    with open(f"{OUTPUT_DIR}/urls.json", 'r') as f:
        import json
        urls = json.load(f)
    
    print(f"共 {len(urls)} 个模板待处理")
    
    # 处理前5个模板
    count = 0
    for url, title in urls[:5]:
        html = get_html(url)
        if not html:
            print(f"  [失败] {title} - 无法获取页面")
            continue
        
        content = extract_content(html)
        if not content or len(content) < 100:
            print(f"  [失败] {title} - 内容太短")
            continue
        
        # 清理标题
        title = html_module.unescape(title)
        
        # 保存文件
        safe_title = re.sub(r'[<>:"/\\|?*]', '', title)[:80]
        filename = f"{safe_title}.txt"
        filepath = os.path.join(OUTPUT_DIR, filename)
        
        with open(filepath, 'w', encoding='utf-8') as f:
            f.write(f"# {title}\n\n来源: {url}\n\n{'='*60}\n\n{content}")
        
        count += 1
        print(f"  [成功] {title[:50]}... ({len(content)}字符)")
    
    print(f"\n完成！成功提取 {count} 个模板")

if __name__ == "__main__":
    main()