#!/usr/bin/env python3
"""
全面爬取上海律协各委员会业务指引
URL格式: https://www.lawyers.org.cn/studies/lawstudycommittee/{slug}/lawstudycommittee-{id}.businessguide
"""
import os
import re
import time
import json
import requests
from urllib.parse import urljoin

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9',
}

BASE_DIR = '/root/.openclaw/workspace/知识库/律师业务操作指引/上海律协'
os.makedirs(BASE_DIR, exist_ok=True)

session = requests.Session()
session.headers.update(HEADERS)

def fetch(url, timeout=30):
    try:
        r = session.get(url, timeout=timeout)
        r.raise_for_status()
        r.encoding = r.apparent_encoding or 'utf-8'
        return r.text
    except Exception as e:
        print(f"  [ERROR] {url}: {e}")
        return ""

def get_committee_id(html, slug):
    """从页面中提取committee id"""
    # 查找 lawstudycommittee-{数字}
    match = re.search(r'lawstudycommittee-(\d+)\.businessguide', html)
    if match:
        return match.group(1)
    # 也尝试其他格式
    match2 = re.search(r'lawstudycommittee[^0-9]*(\d+)', html)
    if match2:
        return match2.group(1)
    return None

def extract_guidelines(html, base_url):
    """从HTML中提取所有业务指引"""
    guidelines = []
    
    # 找所有 /info/xxx 链接
    # 格式: /info/32charhex
    pattern = r'<a\s+href=["\'](https://www\.lawyers\.org\.cn/info/[a-f0-9]{32,})["\']\s*([^>]*)>([^<]*)'
    
    seen = set()
    for match in re.finditer(pattern, html, re.DOTALL):
        url = match.group(1)
        attrs = match.group(2)
        text_raw = match.group(3)
        
        # 清理文本
        text = re.sub('<[^>]+>', '', text_raw).strip()
        
        if '指引' not in text and '操作' not in text and '规程' not in text and '规范' not in text:
            continue
        
        if url in seen:
            continue
        seen.add(url)
        
        # 提取日期
        date_match = re.search(r'(\d{4}-\d{2}-\d{2})', text_raw + text)
        
        guidelines.append({
            'url': url,
            'title': text.strip(),
            'date': date_match.group(1) if date_match else ''
        })
    
    return guidelines

def crawl_page_for_guidelines(url):
    """爬取单个页面，提取所有指引URL"""
    print(f"  获取页面: {url}")
    html = fetch(url)
    if not html:
        return [], None
    
    # 尝试提取 committee id
    cid_match = re.search(r'lawstudycommittee[^0-9]*(\d+)', url)
    cid = cid_match.group(1) if cid_match else None
    
    # 如果URL没有包含cid，尝试从页面中获取
    if not cid:
        cid = get_committee_id(html, '')
    
    guidelines = extract_guidelines(html, url)
    return guidelines, cid

def get_all_guidelines():
    """从所有委员会的业务指引页面获取全部指南"""
    
    # 各委员会的 slug -> id 映射
    # id需要从页面中动态获取，这里用slug方式访问
    committees = {
        'ESG': 'esg',
        '保险': 'baoxian',
        '并购与重组': 'binggouyuzhongzu',
        '财税与海关': 'caishui',
        '城市更新征收': 'budongchanzhengshou',
        '调解': 'diaojie',
        '房地产': 'fangdichan',
        '反垄断与反不正当竞争': 'jingzhengyufanlongduan',
        '反舞弊与刑事风控': 'fanwubiyuxingshifengkong',
        '非银行金融': 'rongzulin',
        '公司与商事': 'gongsiyushangshi',
        '国际法': 'guojifa',
        '国际贸易与自贸区': 'zimaoqu',
        '国际投资与一带一路': 'yidaiyilu',
        '国资国企': 'guoziguoqi',
        '海事海商': 'haishihaishang',
        '环境资源与能源': 'huanjingziyuanyunengyuan',
        '会展与旅游': 'huizhanyulvyou',
        '婚姻家事': 'hunyinjiashi',
        '建设工程与基础设施': 'jianshegongchengyujichusheshijianshe',
        '教育': 'jiaoyu',
        '基金': 'jijin',
        '金融工具与金融基础设施': 'jinronggongjuyujinrongjichusheshi',
        '劳动与社会保障': 'laodongyushehuibaozhang',
        '民商事诉讼': 'minshangshisusong',
        '民事': 'minshi',
        '破产与不良资产': 'pochanyubuliangzichan',
        '企业法律顾问': 'qiyefalvguwen',
        '企业合规': 'falvhegui',
        '社会公共服务': 'shehuigonggongfuwu',
        '社会公益与法律援助': 'shehuigongyifalvyuanzhu',
        '社会治理与社会矛盾化解': 'shehuizhiliyushehuimaodunhuajie',
        '数据合规与网络安全': 'sjhgywlaq',
        '数字科技与人工智能': 'shuzikejiyurengongzhineng',
        '体育': 'tiyu',
        '未成年人权益保护': 'weichengnianrenquanyibaohu',
        '文化传媒': 'wenhuachuanmei',
        '物业管理': 'wuyeguanli',
        '现代物流': 'xiandaiwuliu',
        '乡村振兴': 'xiangcunzhenxing',
        '刑法与刑事辩护': 'xingfayuxingshibianhu',
        '刑诉法与刑事辩护': 'xingsufayuxingshibianhu',
        '行政法': 'xingzhengfa',
        '银行': 'yinxing',
        '医药健康': 'yiyaojiankang',
        '政府法律顾问': 'zhengfufalvguwen',
        '证券合规与纠纷': 'zqhgyjf',
        '证券': 'zhengquan',
        '知识产权': 'zhishichanquan',
        '仲裁': 'zhongcai',
    }
    
    all_guidelines = {}
    committee_ids = {}
    
    # 首先获取每个委员会的ID
    print("第一步：获取各委员会ID...")
    for name, slug in committees.items():
        base_url = f'https://www.lawyers.org.cn/studies/lawstudycommittee/{slug}'
        html = fetch(base_url)
        if html:
            cid_match = re.search(r'lawstudycommittee[^0-9]*(\d+)', html)
            if cid_match:
                cid = cid_match.group(1)
                committee_ids[name] = cid
                print(f"  {name}: ID={cid}")
            else:
                print(f"  {name}: ID未找到")
        time.sleep(0.3)
    
    # 第二步：获取每个委员会的业务指引列表
    print("\n第二步：获取各委员会业务指引列表...")
    all_urls = set()
    
    for name, slug in committees.items():
        cid = committee_ids.get(name)
        if not cid:
            print(f"  {name}: 跳过（无ID）")
            continue
        
        url = f'https://www.lawyers.org.cn/studies/lawstudycommittee/{slug}/lawstudycommittee-{cid}.businessguide'
        html = fetch(url)
        if not html:
            print(f"  {name}: 获取失败")
            continue
        
        guidelines = extract_guidelines(html, url)
        
        for g in guidelines:
            all_urls.add(g['url'])
        
        print(f"  {name}: 找到 {len(guidelines)} 个指引")
        
        # 也检查分页
        page = 2
        while True:
            page_url = f'{url}?page={page}'
            page_html = fetch(page_url)
            if not page_html or page_html == html:
                break
            page_guidelines = extract_guidelines(page_html, page_url)
            if not page_guidelines:
                break
            for g in page_guidelines:
                all_urls.add(g['url'])
            print(f"    {name} 页{page}: +{len(page_guidelines)} 个")
            page += 1
            time.sleep(0.3)
        
        time.sleep(0.3)
    
    return list(all_urls)

def crawl_single_guideline(url, save_dir):
    """爬取单个指引页面的完整内容"""
    print(f"\n  抓取: {url}")
    
    html = fetch(url)
    if not html:
        return None
    
    # 提取正文内容
    # 移除脚本和样式
    html_clean = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL)
    html_clean = re.sub(r'<style[^>]*>.*?</style>', '', html_clean, flags=re.DOTALL)
    html_clean = re.sub(r'<nav[^>]*>.*?</nav>', '', html_clean, flags=re.DOTALL)
    html_clean = re.sub(r'<footer[^>]*>.*?</footer>', '', html_clean, flags=re.DOTALL)
    html_clean = re.sub(r'<header[^>]*>.*?</header>', '', html_clean, flags=re.DOTALL)
    
    # 提取标题
    title_match = re.search(r'<h[1-3][^>]*>\s*<[^>]*>\s*([^<]{2,100})', html_clean)
    if not title_match:
        title_match = re.search(r'<title>([^<]+)</title>', html)
    
    title = title_match.group(1).strip() if title_match else "未知标题"
    title = re.sub(r'<[^>]+>', '', title).strip()
    title = title[:150]
    
    # 提取日期
    date_match = re.search(r'(\d{4}-\d{2}-\d{2})', html)
    date = date_match.group(1) if date_match else ""
    
    # 提取正文 - 查找文章主体
    # 方式1: 找最大的文本区域
    article_match = re.search(r'<article[^>]*>(.*?)</article>', html_clean, re.DOTALL)
    if article_match:
        article_text = article_match.group(1)
        paragraphs = re.findall(r'<p[^>]*>([^<]+)</p>', article_text)
        content = '\n\n'.join([p.strip() for p in paragraphs if len(p.strip()) > 10])
    else:
        # 方式2: 查找class~=content或main的区域
        main_match = re.search(r'<div[^>]*(?:class|id)=["\'][^"\']*(?:content|main|article|body|text)[^"\']*["\'][^>]*>(.*?)</div>', html_clean, re.DOTALL | re.IGNORECASE)
        if main_match:
            paragraphs = re.findall(r'<p[^>]*>([^<]+)</p>', main_match.group(1))
            content = '\n\n'.join([p.strip() for p in paragraphs if len(p.strip()) > 10])
        else:
            # 方式3: 提取所有文字
            text = re.sub(r'<[^>]+>', ' ', html_clean)
            text = re.sub(r'\s+', ' ', text).strip()
            content = text
    
    # 清理内容
    content = re.sub(r'\s+', ' ', content)
    content = content[:500000]
    
    # 保存
    safe_title = re.sub(r'[^\u4e00-\u9fa5a-zA-Z0-9（）\(\)【】\[\]·\-]', '_', title)
    safe_title = re.sub(r'[（\(\)）\)]', '_', safe_title)
    safe_title = re.sub(r'_+', '_', safe_title).strip('_')[:80]
    
    filepath = os.path.join(save_dir, f'{safe_title}.md')
    i = 1
    while os.path.exists(filepath):
        filepath = os.path.join(save_dir, f'{safe_title}_{i}.md')
        i += 1
    
    with open(filepath, 'w', encoding='utf-8') as f:
        f.write(f"# {title}\n\n")
        if date:
            f.write(f"**发布日期：** {date}\n\n")
        f.write(f"**来源：** {url}\n\n")
        f.write("---\n\n")
        f.write(content)
    
    size = os.path.getsize(filepath)
    print(f"  保存: {filepath} ({size} bytes)")
    return filepath

def main():
    print("=" * 70)
    print("上海律协律师业务操作指引全面爬虫")
    print("=" * 70)
    
    # 第一步：获取所有指引URL
    all_urls = get_all_guidelines()
    print(f"\n\n总共发现 {len(all_urls)} 个指引URL")
    
    # 去重并保存URL列表
    url_file = os.path.join(BASE_DIR, '..', 'all_urls.json')
    with open(url_file, 'w', encoding='utf-8') as f:
        json.dump(all_urls, f, ensure_ascii=False, indent=2)
    
    # 第二步：爬取每个指引
    print("\n第三步：爬取每个指引内容...")
    for i, url in enumerate(all_urls):
        print(f"\n[{i+1}/{len(all_urls)}]")
        crawl_single_guideline(url, BASE_DIR)
        time.sleep(0.3)
    
    # 统计
    md_files = [f for f in os.listdir(BASE_DIR) if f.endswith('.md')]
    print(f"\n\n完成! 共保存 {len(md_files)} 个文件")

if __name__ == '__main__':
    main()