#!/usr/bin/env python3
"""
律师业务操作指引爬虫 - 全国律协 & 上海律协
"""
import os
import re
import time
import json
import requests
from urllib.parse import urljoin, urlparse

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}

BASE_DIR = '/root/.openclaw/workspace/知识库/律师业务操作指引'
SHANGHAI_DIR = f'{BASE_DIR}/上海律协'
QUANGUO_DIR = f'{BASE_DIR}/全国律协'

os.makedirs(SHANGHAI_DIR, exist_ok=True)
os.makedirs(QUANGUO_DIR, exist_ok=True)

session = requests.Session()
session.headers.update(HEADERS)

def fetch(url, timeout=30, max_chars=500000):
    """抓取页面内容"""
    try:
        r = session.get(url, timeout=timeout)
        r.raise_for_status()
        r.encoding = r.apparent_encoding or 'utf-8'
        return r.text[:max_chars]
    except Exception as e:
        print(f"  [ERROR] fetch {url}: {e}")
        return ""

def extract_text_from_html(html):
    """简单提取可读文本"""
    if not html:
        return ""
    # 移除脚本和样式
    html = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL)
    html = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL)
    # 移除标签
    text = re.sub(r'<[^>]+>', '', html)
    # 清理空白
    text = re.sub(r'\s+', ' ', text).strip()
    return text

def extract_links(html, base_url):
    """从HTML中提取链接"""
    links = []
    for match in re.finditer(r'href=["\']([^"\']+)["\']', html):
        href = match.group(1)
        full_url = urljoin(base_url, href)
        if urlparse(full_url).netloc:
            links.append(full_url)
    return links

def get_shanghai_committee_urls():
    """获取上海律协所有委员会的业务指引页面"""
    committees = {
        'ESG': 'esg',
        '保险': 'baoxian',
        '并购与重组': 'binggouyuzhongzu',
        '财税与海关': 'caishui',
        '城市更新征收': 'budongchanzhengshou',
        '调解': 'diaojie',
        '房地产': 'fangdichan',
        '反垄断与反不正当竞争': 'jingzhengyufanlongduan',
        '反舞弊与刑事风控': 'fanwubiyuxingshifengkong',
        '非银行金融': 'rongzizulin',
        '公司与商事': 'gongsiyushangshi',
        '国际法': 'guojifa',
        '国际贸易与自贸区': 'zimaoqu',
        '国际投资与一带一路': 'yidaiyilu',
        '国资国企': 'guoziguoqi',
        '海事海商': 'haishihaishang',
        '环境资源与能源': 'huanjingziyuanyunengyuan',
        '会展与旅游': 'huizhanyulvyou',
        '婚姻家事': 'hunyinjiashi',
        '建设工程与基础设施': 'jianshegongchengyujichusheshijianshe',
        '教育': 'jiaoyu',
        '基金': 'jijin',
        '金融工具与金融基础设施': 'jinronggongjuyujinrongjichusheshi',
        '劳动与社会保障': 'laodongyushehuibaozhang',
        '民商事诉讼': 'minshangshisusong',
        '民事': 'minshi',
        '破产与不良资产': 'pochanyubuliangzichan',
        '企业法律顾问': 'qiyefalvguwen',
        '企业合规': 'falvhegui',
        '社会公共服务': 'shehuigonggongfuwu',
        '社会公益与法律援助': 'shehuigongyifalvyuanzhu',
        '社会治理与社会矛盾化解': 'shehuizhiliyushehuimaodunhuajie',
        '数据合规与网络安全': 'sjhgywlaq',
        '数字科技与人工智能': 'shuzikejiyurengongzhineng',
        '体育': 'tiyu',
        '未成年人权益保护': 'weichengnianrenquanyibaohu',
        '文化传媒': 'wenhuachuanmei',
        '物业管理': 'wuyeguanli',
        '现代物流': 'xiandaiwuliu',
        '乡村振兴': 'xiangcunzhenxing',
        '刑法与刑事辩护': 'xingfayuxingshibianhu',
        '刑诉法与刑事辩护': 'xingsufayuxingshibianhu',
        '行政法': 'xingzhengfa',
        '银行': 'yinxing',
        '医药健康': 'yiyaojiankang',
        '政府法律顾问': 'zhengfufalvguwen',
        '证券合规与纠纷': 'zqhgyjf',
        '证券': 'zhengquan',
        '知识产权': 'zhishichanquan',
        '仲裁': 'zhongcai',
    }
    
    base = 'https://www.lawyers.org.cn/studies/lawstudycommittee'
    result = {}
    for name, slug in committees.items():
        result[name] = f'{base}/{slug}'
    return result

def crawl_shanghai_committee(committee_name, committee_url):
    """爬取上海律协单个委员会页面的所有指引"""
    print(f"\n{'='*60}")
    print(f"爬取上海律协: {committee_name}")
    print(f"URL: {committee_url}")
    
    html = fetch(committee_url)
    if not html:
        print(f"  [SKIP] 无法获取页面")
        return []
    
    # 查找所有 info/xxx 链接
    pattern = r'href=["\'](/info/[a-f0-9]{32,})["\'][^>]*>([^<]*)'
    # 也匹配 /info/xxx 格式
    pattern2 = r'href=["\'](https://www\.lawyers\.org\.cn/info/[a-fA-F0-9]+)["\']'
    
    found = []
    
    # 方式1：查找业务指引相关链接
    for match in re.finditer(r'href=["\'](/info/[a-f0-9]{32,})["\']', html):
        href = match.group(1)
        # 检查上下文是否有"操作指引"字样
        start = max(0, match.start() - 200)
        end = min(len(html), match.end() + 200)
        context = html[start:end]
        if '指引' in context or '操作' in context or '规程' in context or '规范' in context:
            full_url = urljoin('https://www.lawyers.org.cn', href)
            found.append(full_url)
    
    # 去重
    seen = set()
    unique = []
    for url in found:
        if url not in seen:
            seen.add(url)
            unique.append(url)
    
    print(f"  找到 {len(unique)} 个业务指引页面")
    return unique

def get_all_shanghai_guidelines():
    """获取上海律协所有指引URL"""
    all_urls = {}
    committees = get_shanghai_committee_urls()
    
    for name, url in committees.items():
        urls = crawl_shanghai_committee(name, url)
        if urls:
            all_urls[name] = urls
        time.sleep(0.5)
    
    return all_urls

def crawl_shanghai_guideline(url, save_dir):
    """爬取单个上海律协指引页面"""
    print(f"\n  抓取: {url}")
    
    html = fetch(url, max_chars=500000)
    if not html:
        return None
    
    # 提取正文内容 - 尝试多种方式
    # 方式1：查找 class="content" 或 id="content"
    content_match = re.search(r'<div[^>]*(?:class|id)=["\'][^"\']*content[^"\']*["\'][^>]*>(.*?)</div>', html, re.DOTALL)
    
    # 方式2：查找主要正文区域
    # 尝试找到包含正文的区域
    article_match = re.search(r'<article[^>]*>(.*?)</article>', html, re.DOTALL)
    
    # 方式3：查找最后一个大段文字
    text = extract_text_from_html(html)
    
    # 提取标题
    title_match = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', html)
    title = title_match.group(1).strip() if title_match else "未知标题"
    
    # 提取日期
    date_match = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', text)
    date = date_match.group(1) if date_match else ""
    
    # 提取正文 - 尝试找主要文本区域
    # 移除导航和脚注
    html_clean = re.sub(r'<nav[^>]*>.*?</nav>', '', html, flags=re.DOTALL)
    html_clean = re.sub(r'<footer[^>]*>.*?</footer>', '', html_clean, flags=re.DOTALL)
    html_clean = re.sub(r'<header[^>]*>.*?</header>', '', html_clean, flags=re.DOTALL)
    
    # 提取所有段落文本
    paragraphs = re.findall(r'<p[^>]*>([^<]+)</p>', html_clean)
    full_text = '\n\n'.join([p.strip() for p in paragraphs if len(p.strip()) > 20])
    
    if not full_text or len(full_text) < 200:
        # 使用整个页面的文本
        full_text = text
    
    result = {
        'title': title,
        'date': date,
        'url': url,
        'content': full_text
    }
    
    # 保存
    safe_title = re.sub(r'[^\u4e00-\u9fa5a-zA-Z0-9（）\(\)【】\[\]·\-]', '_', title)[:80]
    safe_title = safe_title.replace('（', '_').replace('）', '_').replace('(', '_').replace(')', '_')
    
    filepath = os.path.join(save_dir, f'{safe_title}.md')
    i = 1
    while os.path.exists(filepath):
        filepath = os.path.join(save_dir, f'{safe_title}_{i}.md')
        i += 1
    
    with open(filepath, 'w', encoding='utf-8') as f:
        f.write(f"# {title}\n\n")
        if date:
            f.write(f"**发布日期：** {date}\n\n")
        f.write(f"**来源：** {url}\n\n")
        f.write("---\n\n")
        f.write(full_text)
    
    print(f"  保存: {filepath} ({len(full_text)} 字)")
    return filepath

def crawl_quanguo_guideline(url, save_dir):
    """爬取单个全国律协指引页面"""
    print(f"\n  抓取: {url}")
    
    html = fetch(url, max_chars=500000)
    if not html:
        return None
    
    text = extract_text_from_html(html)
    
    # 提取标题
    title_match = re.search(r'<title>([^<]+)</title>', html)
    title = title_match.group(1).strip() if title_match else "未知标题"
    title = title.replace('_中国律师网', '').replace('-中国律师网', '').strip()
    
    # 提取日期
    date_match = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', text)
    date = date_match.group(1) if date_match else ""
    
    # 提取正文
    html_clean = re.sub(r'<nav[^>]*>.*?</nav>', '', html, flags=re.DOTALL)
    html_clean = re.sub(r'<footer[^>]*>.*?</footer>', '', html_clean, flags=re.DOTALL)
    
    paragraphs = re.findall(r'<p[^>]*>([^<]+)</p>', html_clean)
    full_text = '\n\n'.join([p.strip() for p in paragraphs if len(p.strip()) > 20])
    
    if not full_text or len(full_text) < 200:
        full_text = text
    
    result = {
        'title': title,
        'date': date,
        'url': url,
        'content': full_text
    }
    
    safe_title = re.sub(r'[^\u4e00-\u9fa5a-zA-Z0-9（）\(\)【】\[\]·\-]', '_', title)[:80]
    safe_title = safe_title.replace('（', '_').replace('）', '_').replace('(', '_').replace(')', '_')
    
    filepath = os.path.join(save_dir, f'{safe_title}.md')
    i = 1
    while os.path.exists(filepath):
        filepath = os.path.join(save_dir, f'{safe_title}_{i}.md')
        i += 1
    
    with open(filepath, 'w', encoding='utf-8') as f:
        f.write(f"# {title}\n\n")
        if date:
            f.write(f"**发布日期：** {date}\n\n")
        f.write(f"**来源：** {url}\n\n")
        f.write("---\n\n")
        f.write(full_text)
    
    print(f"  保存: {filepath} ({len(full_text)} 字)")
    return filepath

def main():
    print("=" * 60)
    print("律师业务操作指引爬虫启动")
    print("=" * 60)
    
    # ========== 第一步：爬取全国律协 ==========
    print("\n\n" + "="*60)
    print("第一步：爬取全国律协")
    print("="*60)
    
    # 已知的全国律协指引URL（从搜索结果获得）
    quanguo_urls = [
        # 婚姻家庭
        "https://www.acla.org.cn/info/db55cc94958f4f8eaebeae94e37a77a5",
        # 商业秘密
        "https://www.acla.org.cn/info/c8e2a90999394156a9bf57a459e74299",
        # 税法服务
        "https://www.acla.org.cn/info/6ac4bfc62cdf427a8bb9822bfe3ce1f7",
    ]
    
    # 尝试从"业务进阶"栏目获取更多
    print("\n探测全国律协业务进阶栏目...")
    yewu_urls = [
        "https://www.acla.org.cn/catalog/05fb418662344013b9fe273c025db721",
    ]
    
    for url in yewu_urls:
        html = fetch(url)
        if html:
            # 提取所有 info/xxx 链接
            for match in re.finditer(r'href=["\'](/info/[a-f0-9]{32,})["\']', html):
                href = match.group(1)
                full_url = urljoin('https://www.acla.org.cn', href)
                if full_url not in quanguo_urls:
                    quanguo_urls.append(full_url)
    
    print(f"\n全国律协待爬取: {len(quanguo_urls)} 个页面")
    for url in quanguo_urls:
        crawl_quanguo_guideline(url, QUANGUO_DIR)
        time.sleep(0.5)
    
    # ========== 第二步：爬取上海律协各委员会 ==========
    print("\n\n" + "="*60)
    print("第二步：爬取上海律协各委员会")
    print("="*60)
    
    committees = get_shanghai_committee_urls()
    all_shanghai_urls = []
    
    for name, url in committees.items():
        urls = crawl_shanghai_committee(name, url)
        all_shanghai_urls.extend(urls)
        time.sleep(0.3)
    
    print(f"\n\n上海律协共发现 {len(all_shanghai_urls)} 个指引")
    
    # 去重
    seen = set()
    unique_urls = []
    for url in all_shanghai_urls:
        if url not in seen:
            seen.add(url)
            unique_urls.append(url)
    
    print(f"去重后: {len(unique_urls)} 个")
    
    # 爬取每个指引
    for i, url in enumerate(unique_urls):
        print(f"\n[{i+1}/{len(unique_urls)}] 爬取上海律协指引")
        crawl_shanghai_guideline(url, SHANGHAI_DIR)
        time.sleep(0.3)
    
    print("\n\n" + "="*60)
    print("爬取完成!")
    print("="*60)
    
    # 统计
    shanghai_count = len([f for f in os.listdir(SHANGHAI_DIR) if f.endswith('.md')])
    quanguo_count = len([f for f in os.listdir(QUANGUO_DIR) if f.endswith('.md')])
    print(f"全国律协: {quanguo_count} 个文件")
    print(f"上海律协: {shanghai_count} 个文件")

if __name__ == '__main__':
    main()