#!/usr/bin/env python3
"""Stage 2: enrich each app from its Bitrix24 detail page.
Extracts: support-text, emails, phones, whatsapp, telegram, external sites,
useful-links, description, version/changelog info.
"""
import re, os, json, time, sqlite3, urllib.request

UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36"
HERE = os.path.dirname(os.path.abspath(__file__))
CACHE = os.path.join(HERE, "cache_apps")
DB = os.path.join(HERE, "bitrix24_messenger_integrations.db")
os.makedirs(CACHE, exist_ok=True)

def fetch(url):
    fn = os.path.join(CACHE, re.sub(r'[^a-zA-Z0-9]+','_',url)[-110:]+".html")
    if os.path.exists(fn) and os.path.getsize(fn) > 5000:
        return open(fn, encoding="utf-8").read()
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "ru,en"})
    html = urllib.request.urlopen(req, timeout=60).read().decode("utf-8","replace")
    open(fn,"w",encoding="utf-8").write(html)
    time.sleep(1.0 + (time.time()%1))
    return html

def strip_tags(s):
    s = re.sub(r'<svg[\s\S]*?</svg>', ' ', s)
    s = re.sub(r'<script[\s\S]*?</script>', ' ', s)
    s = re.sub(r'<[^>]+>', ' ', s)
    s = s.replace('&nbsp;', ' ').replace('&amp;','&').replace('&rarr;','->')
    return re.sub(r'\s+', ' ', s).strip()

INFRA = re.compile(r'bitrix24|1c-bitrix|google|fonts\.|gstatic|w3\.org|schema\.org|youtube\.com/channel|rutube|zen\.yandex|cloudfront|mc\.yandex|apple\.com|facebook\.com/bitrix24|twitter\.com/bitrix24|x\.com/bitrix24|vk\.com/bitrix24|instagram\.com/bitrix')

def parse_detail(html):
    out = {}
    # --- support block ---
    i = html.find('market-app__scope-list_title-text')
    support_text, support_links = "", []
    # locate the block whose title is "Поддержка"
    for m in re.finditer(r'market-app__scope-list_title-text">([^<]+)<', html):
        if m.group(1).strip().lower().startswith('поддержк'):
            start = m.end()
            seg = html[start:start+6000]
            # cut at next scope-list_wrapper
            nxt = seg.find('market-app__scope-list_wrapper')
            # find end of content div
            endm = re.search(r'</div>\s*</div>\s*</div>', seg)
            chunk = seg[:endm.end() if endm else 3000]
            support_text = strip_tags(chunk)
            support_links = re.findall(r'href="([^"]+)"', chunk)
            break
    out['support_text'] = support_text
    # --- contacts anywhere on the page (support area is most reliable, fallback whole) ---
    emails = set(re.findall(r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}', support_text or html))
    emails = {e for e in emails if not e.lower().endswith(('png','jpg','jpeg','svg','webp','gif','css','js'))}
    out['emails'] = sorted(emails)
    wa = sorted(set(re.findall(r'https?://wa\.me/[0-9]+', html)))
    tg = sorted(set(re.findall(r'https?://t\.me/([A-Za-z0-9_]+)', html)))
    tg = sorted({t for t in tg if t.lower() != 'bitrix24_ru'})
    out['whatsapp'] = wa
    out['telegram'] = ['https://t.me/'+t for t in tg]
    phones = sorted(set(re.findall(r'\+?7[\s\-\(\)]?[0-9]{3}[\s\-\(\)]?[0-9]{3}[\s\-]?[0-9]{2}[\s\-]?[0-9]{2}', support_text)))
    out['phones'] = phones
    # --- developer / external sites (from support links + whole page) ---
    links = re.findall(r'href="(https?://[^"]+)"', html)
    ext = sorted({l for l in links if not INFRA.search(l) and not l.startswith('https://www.bitrix24')})
    out['external_sites'] = ext
    # --- useful links labels ---
    ul = re.findall(r'market-detail-useful-links-item__text">\s*<span>([^<]+)</span>', html)
    out['useful_links'] = ul
    # --- title/desc ---
    m = re.search(r'<meta property="og:title" content="([^"]*)"', html)
    out['og_title'] = m.group(1) if m else ""
    m = re.search(r'<meta property="og:description" content="([^"]*)"', html)
    out['og_desc'] = m.group(1) if m else ""
    return out

def main():
    con = sqlite3.connect(DB)
    cols = [r[1] for r in con.execute("PRAGMA table_info(apps)")]
    for c,t in [('emails','TEXT'),('phones','TEXT'),('telegram','TEXT'),('whatsapp','TEXT'),
                ('external_sites','TEXT'),('support_text','TEXT'),('useful_links','TEXT')]:
        if c not in cols:
            con.execute(f"ALTER TABLE apps ADD COLUMN {c} {t}")
    rows = con.execute("SELECT app_id, code FROM apps ORDER BY num_installs DESC").fetchall()
    for k,(app_id, code) in enumerate(rows,1):
        url = f"https://www.bitrix24.ru/apps/app/{code}/"
        try:
            html = fetch(url)
            info = parse_detail(html)
        except Exception as e:
            print(f"[{k}/{len(rows)}] {code}: ERROR {e}"); continue
        con.execute("""UPDATE apps SET emails=?,phones=?,telegram=?,whatsapp=?,
                       external_sites=?,support_text=?,useful_links=? WHERE app_id=?""",
                    (json.dumps(info['emails'],ensure_ascii=False),
                     json.dumps(info['phones'],ensure_ascii=False),
                     json.dumps(info['telegram'],ensure_ascii=False),
                     json.dumps(info['whatsapp'],ensure_ascii=False),
                     json.dumps(info['external_sites'],ensure_ascii=False),
                     info['support_text'][:2000],
                     json.dumps(info['useful_links'],ensure_ascii=False), app_id))
        con.commit()
        print(f"[{k}/{len(rows)}] {code}: emails={len(info['emails'])} tg={len(info['telegram'])} wa={len(info['whatsapp'])} sites={len(info['external_sites'])}")
    n_email = con.execute("SELECT COUNT(*) FROM apps WHERE emails!='[]'").fetchone()[0]
    n_tg = con.execute("SELECT COUNT(*) FROM apps WHERE telegram!='[]'").fetchone()[0]
    n_site = con.execute("SELECT COUNT(*) FROM apps WHERE external_sites!='[]'").fetchone()[0]
    print(f"\nWith email: {n_email}, tg: {n_tg}, external site: {n_site}")

if __name__ == "__main__":
    main()