#!/usr/bin/env python3
"""
上海律协 + 全国律协 律师业务操作指引 全面爬虫 v2
"""
import os, re, time, json, requests
from urllib.parse import urljoin

HEADERS = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'}

BASE_DIR = '/root/.openclaw/workspace/知识库/律师业务操作指引'
SHANGHAI_DIR = f'{BASE_DIR}/上海律协'
QUANGUO_DIR = f'{BASE_DIR}/全国律协'
os.makedirs(SHANGHAI_DIR, exist_ok=True)
os.makedirs(QUANGUO_DIR, exist_ok=True)

s = requests.Session()
s.headers.update(HEADERS)

# 各委员会 slug -> ID 映射
COMMITTEE_URLS = {
    'ESG': 'https://www.lawyers.org.cn/studies/lawstudycommittee/esg/lawstudycommittee-53.businessguide',
    '保险': 'https://www.lawyers.org.cn/studies/lawstudycommittee/baoxian/lawstudycommittee-01.businessguide',
    '并购与重组': 'https://www.lawyers.org.cn/studies/lawstudycommittee/binggouyuzhongzu/lawstudycommittee-02.businessguide',
    '财税与海关': 'https://www.lawyers.org.cn/studies/lawstudycommittee/caishui/lawstudycommittee-18.businessguide',
    '城市更新征收': 'https://www.lawyers.org.cn/studies/lawstudycommittee/budongchanzhengshou/lawstudycommittee-37.businessguide',
    '调解': 'https://www.lawyers.org.cn/studies/lawstudycommittee/diaojie/lawstudycommittee-35.businessguide',
    '房地产': 'https://www.lawyers.org.cn/studies/lawstudycommittee/fangdichan/lawstudycommittee-09.businessguide',
    '反垄断与反不正当竞争': 'https://www.lawyers.org.cn/studies/lawstudycommittee/jingzhengyufanlongduan/lawstudycommittee-32.businessguide',
    '反舞弊与刑事风控': 'https://www.lawyers.org.cn/studies/lawstudycommittee/fanwubiyuxingshifengkong/lawstudycommittee-46.businessguide',
    '非银行金融': 'https://www.lawyers.org.cn/studies/lawstudycommittee/rongzizulin/lawstudycommittee-33.businessguide',
    '公司与商事': 'https://www.lawyers.org.cn/studies/lawstudycommittee/gongsiyushangshi/lawstudycommittee-21.businessguide',
    '国际法': 'https://www.lawyers.org.cn/studies/lawstudycommittee/guojifa/lawstudycommittee-21.businessguide',
    '国际贸易与自贸区': 'https://www.lawyers.org.cn/studies/lawstudycommittee/zimaoqu/lawstudycommittee-52.businessguide',
    '国际投资与一带一路': 'https://www.lawyers.org.cn/studies/lawstudycommittee/yidaiyilu/lawstudycommittee-48.businessguide',
    '国资国企': 'https://www.lawyers.org.cn/studies/lawstudycommittee/guoziguoqi/lawstudycommittee-27.businessguide',
    '海事海商': 'https://www.lawyers.org.cn/studies/lawstudycommittee/haishihaishang/lawstudycommittee-13.businessguide',
    '环境资源与能源': 'https://www.lawyers.org.cn/studies/lawstudycommittee/huanjingziyuanyunengyuan/lawstudycommittee-15.businessguide',
    '会展与旅游': 'https://www.lawyers.org.cn/studies/lawstudycommittee/huizhanyulvyou/lawstudycommittee-19.businessguide',
    '婚姻家事': 'https://www.lawyers.org.cn/studies/lawstudycommittee/hunyinjiashi/lawstudycommittee-38.businessguide',
    '建设工程与基础设施': 'https://www.lawyers.org.cn/studies/lawstudycommittee/jianshegongchengyujichusheshijianshe/lawstudycommittee-08.businessguide',
    '教育': 'https://www.lawyers.org.cn/studies/lawstudycommittee/jiaoyu/lawstudycommittee-31.businessguide',
    '基金': 'https://www.lawyers.org.cn/studies/lawstudycommittee/jijin/lawstudycommittee-30.businessguide',
    '金融工具与金融基础设施': 'https://www.lawyers.org.cn/studies/lawstudycommittee/jinronggongjuyujinrongjichusheshi/lawstudycommittee-10.businessguide',
    '劳动与社会保障': 'https://www.lawyers.org.cn/studies/lawstudycommittee/laodongyushehuibaozhang/lawstudycommittee-25.businessguide',
    '民商事诉讼': 'https://www.lawyers.org.cn/studies/lawstudycommittee/minshangshisusong/lawstudycommittee-34.businessguide',
    '民事': 'https://www.lawyers.org.cn/studies/lawstudycommittee/minshi/lawstudycommittee-12.businessguide',
    '破产与不良资产': 'https://www.lawyers.org.cn/studies/lawstudycommittee/pochanyubuliangzichan/lawstudycommittee-22.businessguide',
    '企业法律顾问': 'https://www.lawyers.org.cn/studies/lawstudycommittee/qiyefalvguwen/lawstudycommittee-42.businessguide',
    '企业合规': 'https://www.lawyers.org.cn/studies/lawstudycommittee/falvhegui/lawstudycommittee-36.businessguide',
    '社会公共服务': 'https://www.lawyers.org.cn/studies/lawstudycommittee/shehuigonggongfuwu/lawstudycommittee-35.businessguide',
    '社会治理与社会矛盾化解': 'https://www.lawyers.org.cn/studies/lawstudycommittee/shehuizhiliyushehuimaodunhuajie/lawstudycommittee-20.businessguide',
    '数据合规与网络安全': 'https://www.lawyers.org.cn/studies/lawstudycommittee/sjhgywlaq/lawstudycommittee-54.businessguide',
    '数字科技与人工智能': 'https://www.lawyers.org.cn/studies/lawstudycommittee/shuzikejiyurengongzhineng/lawstudycommittee-03.businessguide',
    '体育': 'https://www.lawyers.org.cn/studies/lawstudycommittee/tiyu/lawstudycommittee-43.businessguide',
    '未成年人权益保护': 'https://www.lawyers.org.cn/studies/lawstudycommittee/weichengnianrenquanyibaohu/lawstudycommittee-44.businessguide',
    '文化传媒': 'https://www.lawyers.org.cn/studies/lawstudycommittee/wenhuachuanmei/lawstudycommittee-36.businessguide',
    '物业管理': 'https://www.lawyers.org.cn/studies/lawstudycommittee/wuyeguanli/lawstudycommittee-45.businessguide',
    '现代物流': 'https://www.lawyers.org.cn/studies/lawstudycommittee/xiandaiwuliu/lawstudycommittee-16.businessguide',
    '乡村振兴': 'https://www.lawyers.org.cn/studies/lawstudycommittee/xiangcunzhenxing/lawstudycommittee-40.businessguide',
    '刑法与刑事辩护': 'https://www.lawyers.org.cn/studies/lawstudycommittee/xingfayuxingshibianhu/lawstudycommittee-23.businessguide',
    '刑诉法与刑事辩护': 'https://www.lawyers.org.cn/studies/lawstudycommittee/xingsufayuxingshibianhu/lawstudycommittee-47.businessguide',
    '行政法': 'https://www.lawyers.org.cn/studies/lawstudycommittee/xingzhengfa/lawstudycommittee-05.businessguide',
    '银行': 'https://www.lawyers.org.cn/studies/lawstudycommittee/yinxing/lawstudycommittee-04.businessguide',
    '医药健康': 'https://www.lawyers.org.cn/studies/lawstudycommittee/yiyaojiankang/lawstudycommittee-26.businessguide',
    '政府法律顾问': 'https://www.lawyers.org.cn/studies/lawstudycommittee/zhengfufalvguwen/lawstudycommittee-49.businessguide',
    '证券合规与纠纷': 'https://www.lawyers.org.cn/studies/lawstudycommittee/zqhgyjf/lawstudycommittee-55.businessguide',
    '证券': 'https://www.lawyers.org.cn/studies/lawstudycommittee/zhengquan/lawstudycommittee-06.businessguide',
    '知识产权': 'https://www.lawyers.org.cn/studies/lawstudycommittee/zhishichanquan/lawstudycommittee-17.businessguide',
    '仲裁': 'https://www.lawyers.org.cn/studies/lawstudycommittee/zhongcai/lawstudycommittee-51.businessguide',
}

def fetch(url, timeout=30):
    try:
        r = s.get(url, timeout=timeout)
        r.raise_for_status()
        r.encoding = r.apparent_encoding or 'utf-8'
        return r.text
    except Exception as e:
        print(f'  [ERR] {url}: {e}')
        return ''

def extract_guidelines(html):
    """从页面HTML提取业务指引列表"""
    guidelines = []
    seen = set()
    
    # 匹配: https://www.lawyers.org.cn/info/32hex 后接标题文本
    for m in re.finditer(r'href="(https://www\.lawyers\.org\.cn/info/[a-f0-9]{32,})"[^>]*>(.*?)</a>', html, re.DOTALL):
        url = m.group(1)
        text_block = m.group(2)
        text = re.sub('<[^>]+>', '', text_block).strip()
        
        if '指引' not in text and '操作' not in text and '规程' not in text:
            continue
        if url in seen:
            continue
        seen.add(url)
        
        date_m = re.search(r'(\d{4}-\d{2}-\d{2})', text)
        guidelines.append({
            'url': url,
            'title': text[:200],
            'date': date_m.group(1) if date_m else ''
        })
    
    return guidelines

def crawl_guideline_page(url):
    """抓取单个指引页面正文"""
    print(f'  抓取: {url}')
    html = fetch(url)
    if not html:
        return None
    
    # 清理HTML
    html_c = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL)
    html_c = re.sub(r'<style[^>]*>.*?</style>', '', html_c, flags=re.DOTALL)
    html_c = re.sub(r'<nav[^>]*>.*?</nav>', '', html_c, flags=re.DOTALL)
    html_c = re.sub(r'<footer[^>]*>.*?</footer>', '', html_c, flags=re.DOTALL)
    html_c = re.sub(r'<header[^>]*>.*?</header>', '', html_c, flags=re.DOTALL)
    
    # 标题
    title_m = re.search(r'<title>([^<]+)</title>', html)
    if not title_m:
        title_m = re.search(r'<h[1-3][^>]*>\s*<[^>]*>\s*([^<]{2,100})', html_c)
    title = title_m.group(1).strip() if title_m else '未知标题'
    title = re.sub(r'<[^>]+>', '', title).strip()[:150]
    
    # 日期
    date_m = re.search(r'(\d{4}-\d{2}-\d{2})', html)
    date = date_m.group(1) if date_m else ''
    
    # 正文 - 找最大文本区域
    # 方式1: article
    art_m = re.search(r'<article[^>]*>(.*?)</article>', html_c, re.DOTALL)
    if art_m:
        paras = re.findall(r'<p[^>]*>([^<]+)</p>', art_m.group(1))
        content = '\n\n'.join([p.strip() for p in paras if len(p.strip()) > 10])
    
    if not content or len(content) < 200:
        # 方式2: 找content区域
        for cls in ['content', 'main', 'article', 'detail', 'text']:
            m = re.search(r'<div[^>]*(?:class|id)=["\'][^"\']*' + cls + r'[^"\']*["\'][^>]*>(.*?)</div>', html_c, re.DOTALL)
            if m and len(m.group(1)) > 200:
                paras = re.findall(r'<p[^>]*>([^<]+)</p>', m.group(1))
                content = '\n\n'.join([p.strip() for p in paras if len(p.strip()) > 10])
                if len(content) > 200:
                    break
    
    if not content or len(content) < 200:
        # 方式3: 所有文字
        text = re.sub(r'<[^>]+>', ' ', html_c)
        text = re.sub(r'\s+', ' ', text).strip()
        content = text
    
    content = re.sub(r'\s+', ' ', content).strip()[:500000]
    
    return {'title': title, 'date': date, 'content': content, 'url': url}

def save_guideline(g, save_dir, prefix=''):
    """保存指引到文件"""
    title = g['title']
    date = g.get('date', '')
    content = g.get('content', '')
    url = g.get('url', '')
    
    safe = re.sub(r'[^\u4e00-\u9fa5a-zA-Z0-9（）\(\)·\-]', '_', title)
    safe = re.sub(r'_+', '_', safe).strip('_')[:80]
    if prefix:
        safe = f'{prefix}_{safe}'
    
    filepath = os.path.join(save_dir, f'{safe}.md')
    i = 1
    while os.path.exists(filepath):
        filepath = os.path.join(save_dir, f'{safe}_{i}.md')
        i += 1
    
    with open(filepath, 'w', encoding='utf-8') as f:
        f.write(f"# {title}\n\n")
        if date:
            f.write(f"**发布日期：** {date}\n\n")
        f.write(f"**来源：** {url}\n\n")
        f.write("---\n\n")
        f.write(content)
    
    size = os.path.getsize(filepath)
    print(f'  保存: {os.path.basename(filepath)} ({size} bytes)')
    return filepath

def run_shanghai():
    """爬取上海律协"""
    print("="*70)
    print("第一步：从各委员会业务指引列表页提取URL")
    print("="*70)
    
    all_urls = []
    
    for name, url in COMMITTEE_URLS.items():
        print(f"\n[{name}] {url}")
        html = fetch(url)
        if not html:
            print(f'  [SKIP] 获取失败')
            continue
        
        guidelines = extract_guidelines(html)
        
        # 检查分页
        base_url = url.replace('.businessguide', '')
        for page in range(2, 20):
            page_url = f'{url}?page={page}'
            page_html = fetch(page_url)
            if page_html and page_html != html:
                p_gl = extract_guidelines(page_html)
                if p_gl:
                    guidelines.extend(p_gl)
                else:
                    break
            else:
                break
            time.sleep(0.2)
        
        print(f'  找到 {len(guidelines)} 个指引')
        for g in guidelines:
            all_urls.append(g)
        
        time.sleep(0.3)
    
    # 去重
    seen = set()
    unique = []
    for g in all_urls:
        if g['url'] not in seen:
            seen.add(g['url'])
            unique.append(g)
    
    print(f"\n去重后共 {len(unique)} 个指引")
    
    # 保存URL列表
    with open(f'{SHANGHAI_DIR}/url_list.json', 'w', encoding='utf-8') as f:
        json.dump(unique, f, ensure_ascii=False, indent=2)
    
    print("\n" + "="*70)
    print("第二步：爬取每个指引正文")
    print("="*70)
    
    saved = 0
    for i, g in enumerate(unique):
        print(f"\n[{i+1}/{len(unique)}]")
        data = crawl_guideline_page(g['url'])
        if data:
            save_guideline(data, SHANGHAI_DIR)
            saved += 1
        time.sleep(0.3)
    
    print(f"\n上海律协完成! 共保存 {saved} 个文件")

def run_quanguo():
    """爬取全国律协"""
    print("\n" + "="*70)
    print("第三步：爬取全国律协")
    print("="*70)
    
    # 从搜索结果和已知URLs
    quanguo_urls = [
        ('全国律协_律师从事税法服务业务操作指引', 'https://www.acla.org.cn/info/6ac4bfc62cdf427a8bb9822bfe3ce1f7'),
        ('全国律协_律师办理商业秘密法律业务操作指引', 'https://www.acla.org.cn/info/c8e2a90999394156a9bf57a459e74299'),
    ]
    
    # 探测全国律协业务操作指引栏目
    catalog_urls = [
        'https://www.acla.org.cn/catalog/05fb418662344013b9fe273c025db721',
        'https://www.acla.org.cn/catalog/cee5a15d387045ddac565bb107546cae',
    ]
    
    discovered = []
    for curl in catalog_urls:
        html = fetch(curl)
        if html:
            for m in re.finditer(r'href="(https://www\.acla\.org\.cn/info/[a-f0-9]{32,})"', html):
                url = m.group(1)
                if url not in [u[1] for u in quanguo_urls] and url not in discovered:
                    discovered.append(url)
        time.sleep(0.3)
    
    print(f"发现 {len(discovered)} 个全国律协页面")
    
    for url in discovered:
        print(f"\n抓取: {url}")
        html = fetch(url)
        if not html:
            continue
        
        html_c = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL)
        html_c = re.sub(r'<style[^>]*>.*?</style>', '', html_c, flags=re.DOTALL)
        
        title_m = re.search(r'<title>([^<]+)</title>', html)
        title = title_m.group(1).strip() if title_m else '未知'
        title = title.replace('_中国律师网', '').replace('-中国律师网', '').strip()
        
        date_m = re.search(r'(\d{4}-\d{2}-\d{2})', html)
        date = date_m.group(1) if date_m else ''
        
        art_m = re.search(r'<article[^>]*>(.*?)</article>', html_c, re.DOTALL)
        if art_m:
            paras = re.findall(r'<p[^>]*>([^<]+)</p>', art_m.group(1))
            content = '\n\n'.join([p.strip() for p in paras if len(p.strip()) > 10])
        else:
            text = re.sub(r'<[^>]+>', ' ', html_c)
            text = re.sub(r'\s+', ' ', text).strip()
            content = text
        
        g = {'title': title, 'date': date, 'content': content, 'url': url}
        save_guideline(g, QUANGUO_DIR)
        time.sleep(0.3)
    
    # 也爬取已知的URLs
    for name, url in quanguo_urls:
        html = fetch(url)
        if not html:
            continue
        
        html_c = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL)
        html_c = re.sub(r'<style[^>]*>.*?</style>', '', html_c, flags=re.DOTALL)
        
        date_m = re.search(r'(\d{4}-\d{2}-\d{2})', html)
        date = date_m.group(1) if date_m else ''
        
        art_m = re.search(r'<article[^>]*>(.*?)</article>', html_c, re.DOTALL)
        if art_m:
            paras = re.findall(r'<p[^>]*>([^<]+)</p>', art_m.group(1))
            content = '\n\n'.join([p.strip() for p in paras if len(p.strip()) > 10])
        else:
            text = re.sub(r'<[^>]+>', ' ', html_c)
            text = re.sub(r'\s+', ' ', text).strip()
            content = text
        
        g = {'title': name, 'date': date, 'content': content, 'url': url}
        save_guideline(g, QUANGUO_DIR)
        time.sleep(0.3)

if __name__ == '__main__':
    run_shanghai()
    run_quanguo()
    
    shanghai_count = len([f for f in os.listdir(SHANGHAI_DIR) if f.endswith('.md')])
    quanguo_count = len([f for f in os.listdir(QUANGUO_DIR) if f.endswith('.md')])
    print(f"\n\n{'='*70}")
    print(f"全部完成!")
    print(f"上海律协: {shanghai_count} 个文件")
    print(f"全国律协: {quanguo_count} 个文件")
    print(f"{'='*70}")