#!/usr/bin/env python3
"""抓取国家市场监管总局合同示范文本库"""
import subprocess
import re
import os
import json

BASE_URL = "https://htsfwb.samr.gov.cn"
OUTPUT_DIR = "/root/.openclaw/workspace/知识库/法律文书模板库/市场监督管理局/国家总局"
os.makedirs(OUTPUT_DIR, exist_ok=True)

def get_html(url):
    try:
        r = subprocess.run(['curl', '-s', '-L', '-A', 'Mozilla/5.0', url], 
                          capture_output=True, text=True, timeout=30)
        return r.stdout
    except:
        return ""

def extract_template_links(html):
    """提取模板链接"""
    # 提取View页面的链接
    pattern = r'href="(/View\?id=[^"]+)"[^>]*title="([^"]+)"'
    matches = re.findall(pattern, html)
    return [(BASE_URL + m[0], m[1]) for m in matches]

def download_template(view_url, title):
    """下载模板"""
    html = get_html(view_url)
    if not html:
        return False
    
    # 查找下载链接 - 可能是PDF/DOC等
    download_patterns = [
        r'href="([^"]+\.pdf)"',
        r'href="([^"]+\.doc)"',
        r'href="([^"]+\.docx)"',
        r'src="([^"]+\.pdf)"',
        r'src="([^"]+\.doc)"',
        r'src="([^"]+\.docx)"',
    ]
    
    for pattern in download_patterns:
        matches = re.findall(pattern, html, re.IGNORECASE)
        for match in matches:
            if match.startswith('http'):
                return match
            elif match.startswith('/'):
                return BASE_URL + match
    return None

def main():
    print("开始抓取国家市场监管总局合同示范文本库...")
    
    # 获取首页
    html = get_html(BASE_URL)
    
    # 提取首页模板
    templates = extract_template_links(html)
    print(f"首页发现 {len(templates)} 个模板")
    
    # 保存URL列表
    with open(f"{OUTPUT_DIR}/urls.json", 'w', encoding='utf-8') as f:
        json.dump(templates, f, ensure_ascii=False, indent=2)
    
    print(f"\n已保存URL列表到 {OUTPUT_DIR}/urls.json")
    print(f"共 {len(templates)} 个模板链接")
    
    # 打印前10个
    for url, title in templates[:10]:
        print(f"  - {title}")

if __name__ == "__main__":
    main()