#!/usr/bin/env bash
# DSP-daily 日报数据更新脚本
# 每小时被平台执行；UTC+8 过11点且当天未更新则补跑，查询 T-1(前一天)。
# 产出 data.json：
#   overall/cn/international : 每天一行，持续叠加
#   pkg_midas / pkg_ms       : {dt:{pkg:{account:rev}}} 保留14天(前端按模式聚合/算DoD)
#   cust_daily / cust_detail : 客户分类，保留14天
# ============================================================
set -euo pipefail
cd "$(dirname "$0")"

# ============ 凭证（会过期，过期后替换这两行即可）============
# Superset cookie（用于 Midas / 无_API account）
SUPERSET_COOKIE='SR-PROD=dea34a1f6d50f4426c0a9fa3d19d75a5; oidc=d86576ac9fae4782b20b16044ca089a0; session=.eJztkMFqAzEMRH8l-LwHWbaz1t4K-YsSgmRJzdJlA-v0FPLvNfQ3ehoG9IbRvMLNN-5362H5fIXTc0hQ3r_sCFP4aM16P639dLF9NQ3X9_R_dJ3GaIf1e1ict27DrhqWoK65ZtCcIEWPqK0QzAiVhbmm1oZKpUh5BkiKLOoulJgRFItkAyzguYKVVCl5xsqALatxKQk51aIyUEaspYmcudQyq0uV7Oc82rZ--O35-LZ99IFIJplJm9GslWb0AlGcyVFoBCM1oDkNbns03mwwA5zCT7fj76WIAOH9C_vZpvc.aqOlrQ.xmkrmwsNe89-wy3pSP20QhNNuC8'
# MS report/v2/list token（用于 MS / 带_API account）
MS_TOKEN='eyJhbGciOiJIUzI1NiJ9.eyJqdGkiOiI5OGNlN2YzZDY3NzE0NDFhOWQxM2I2YjJhOTY5M2UxZSIsImlhdCI6MTc4OTM1NTAzMywic3ViIjoidG9rZW4gYnkgdXNoYXJlaXQiLCJleHAiOjE3ODk0NDE0MzN9.6IupaUeP2gThNu0v7P5J8uA2aL44m8vvGDjFIhnmhKI'

SUPERSET_URL='https://ads-bi.ushareit.org/superset/explore_json/'
SLICE_ID=821
REPORT_URL='https://ms-api.sandssea.com/api/v1/report/v2/list'
DATA_FILE='data.json'
KEEP_DAYS=14

# ============ 执行时机（UTC+8，过11点且当天未更新则补跑）============
HOUR_CST=$(TZ='Asia/Shanghai' date +%H)
RUN_DATE=$(TZ='Asia/Shanghai' date '+%Y/%-m/%-d')
FORCE="${FORCE_RUN:-0}"
if [ "$FORCE" != "1" ]; then
  if [ "$((10#$HOUR_CST))" -lt 11 ]; then echo "[skip] UTC+8 $HOUR_CST 点，未到11点"; exit 0; fi
  if [ -f "$DATA_FILE" ]; then
    A=$(python3 -c "import json;d=json.load(open('$DATA_FILE',encoding='utf-8'));print('1' if any(r.get('date')=='$RUN_DATE' for r in d.get('overall',[])) else '0')" 2>/dev/null || echo 0)
    if [ "$A" = "1" ]; then echo "[skip] 今天($RUN_DATE)已更新"; exit 0; fi
  fi
fi

# T-1 日期（前一天）
PREV_YMD=$(TZ='Asia/Shanghai' date -v-1d '+%Y%m%d' 2>/dev/null || TZ='Asia/Shanghai' date -d 'yesterday' '+%Y%m%d')   # 20260910
PREV_DATE=$(TZ='Asia/Shanghai' date -v-1d '+%Y/%-m/%-d' 2>/dev/null || TZ='Asia/Shanghai' date -d 'yesterday' '+%Y/%-m/%-d')  # 2026/9/10
echo "[run] RUN_DATE=$RUN_DATE  T-1=$PREV_YMD($PREV_DATE)"

# ============ 主处理：Python 完成拉取+聚合+合并 ============
export SUPERSET_COOKIE MS_TOKEN SUPERSET_URL SLICE_ID REPORT_URL DATA_FILE KEEP_DAYS RUN_DATE PREV_YMD PREV_DATE
python3 <<'PYEOF'
import os,json,urllib.parse,urllib.request,ssl,datetime
from collections import defaultdict

SUPERSET_COOKIE=os.environ['SUPERSET_COOKIE']; MS_TOKEN=os.environ['MS_TOKEN']
SUPERSET_URL=os.environ['SUPERSET_URL']; SLICE_ID=int(os.environ['SLICE_ID'])
REPORT_URL=os.environ['REPORT_URL']; DATA_FILE=os.environ['DATA_FILE']
KEEP=int(os.environ['KEEP_DAYS']); RUN_DATE=os.environ['RUN_DATE']
PREV_YMD=os.environ['PREV_YMD']; PREV_DATE=os.environ['PREV_DATE']
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE

# ---- account 对照表 (advertiser_account_id -> _API名) ----
ACC={102:'nativex_API',99:'adsplay_API',98:'clickpuff_API',97:'affle_API',96:'mobhuge_API',95:'altosum_API',94:'seanear_API',93:'stridemobi_API',92:'volo_API',91:'strongmetrics_API',87:'aba_API',86:'hope_api',85:'stackyads_API',84:'isummer_API',83:'vertex_API',82:'viking_API',80:'acmemobi_API',79:'flymobi_API',78:'motilal_API',77:'flatmedia_API',76:'hypertap_API',75:'Influx_API',74:'sync_API',73:'c2c_API',72:'tecdo_API',71:'yomiads_API',70:'goodjoy_API',69:'apkpure_API',68:'flatads_API',67:'adnov_API',66:'dolemobi_API',65:'witgala_API',64:'mejoy_API',63:'adwisdom_ddj_API',62:'vfineads_ddj_API',60:'Iogerads_API',56:'quarklio_API',55:'quarklio_ddj_API',54:'bidneuron_API',52:'mstoko_API',51:'hopemobi_API',23:'Macan_API',18:'Midas_API',11:'Dreammobi_API',9:'Flowbox_API'}

# ---- CN / International 列头 account ----
CN_ACCS=['acshare','adsdeep','adwisdom','affinityad','alphadsp','altosum','apkpure','appfortis','appnextads','bidmatrix','bidneuron','bytemobi','cookeads','dolphin','dolphin2','flowbox','flymob','gleamx','gridads','imax','influxtec','iogerads','leapmob','lightspot','mbupps','mobgem','moblegend','mocaads','movablead','mstoko','nativexads','Ricefruit','shockit','smartconnectads','startym','tenads','verse','vertexs','vikingmedia','xoomads','yolo','youmi',
  'bidneuron_API','quarklio_API','quarklio_ddj_API','Iogerads_API','adwisdom_ddj_API','mejoy_API','witgala_API','dolemobi_API','flatads_API','adnov_API','goodjoy_API','yomiads_API','flymobi_API','vertex_api','isummer_API','viking_api','hope_api','aba_api','volo_api','seanear_API','mobhuge_API','altosum_API','clickpuff_API']
INTL_ACCS=['adsflourish','affleads','avowads','cubicom','keendigi','launchpad','mobavenue','stackyads','vserv','xapad',
  'sync_API','motilal_API','acmemobi_API','strongmetrics_API','c2c','adsplay_API']

# ---- 客户分类 base 明细 (以文字明细为准) ----
CUST_BASE_RAW="""头部直客渠道|appnextads|7422.98
头部直客渠道|xapad|4200.73
头部直客渠道|vserv|3551.89
头部直客渠道|affinityad|2099.18
头部直客渠道|avowads|381.55
头部直客渠道|alphadsp|0
厂商预算|flymob|1002.15
厂商预算|flymobi_API|0
厂商预算|dolphin|773.56
厂商预算|dolphin2|0
厂商预算|iogerads|371.16
厂商预算|Iogerads_API|0
厂商预算|shockit|292.4
中小客户|adwisdom|738.61
中小客户|adwisdom_ddj_API|0
中小客户|appfortis|257.96
中小客户|mobavenue|228.43
中小客户|verse|199.27
中小客户|startym|116.72
中小客户|mbupps|112.17
中小客户|nativexads|102
中小客户|mstoko|97.96
中小客户|flatmedia_API|72.42
中小客户|leapmob|60.46
中小客户|affleads|59.83
中小客户|moblegend|51.33
中小客户|vertexs|51.09
中小客户|vertex_api|0
中小客户|stackyads|43.7
中小客户|apkpure|37.73
中小客户|acshare|33.72
中小客户|bytemobi|33.55
中小客户|adsdeep|32.19
中小客户|Ricefruit|31.7
中小客户|abamobi|31.29
中小客户|aba_api|0
中小客户|imax|29.14
中小客户|vfineads|27.77
中小客户|influxtec|26.75
中小客户|isummer|24.8
中小客户|isummer_API|0
中小客户|smartconnectads|23.99
中小客户|macan|21.64
中小客户|xoomads|21.62
中小客户|flowbox|20.62
中小客户|bidmatrix|18.91
中小客户|seanear_API|17.48
中小客户|gridads|16.51
中小客户|yomiads_API|15.55
中小客户|gleamx|14.37
中小客户|bidneuron|12.89
中小客户|bidneuron_API|0
中小客户|quarklio_API|12.83
中小客户|altosum|11.95
中小客户|altosum_API|0
中小客户|lightspot|11.75
中小客户|vikingmedia|10.87
中小客户|viking_api|0
中小客户|mobgem|5.86
中小客户|launchpad|4.2
中小客户|adnov_API|0.24
中小客户|hopemobi_API|2.55
中小客户|hope_API|0
中小客户|volo|10.84
中小客户|volo_api|0
待抛弃客户|mobhugeads|9.95
待抛弃客户|wayne|7.86
待抛弃客户|c2c|6.3
待抛弃客户|tenads|4.45
待抛弃客户|movablead|4.22
待抛弃客户|tecdo|3.17
待抛弃客户|cookeads|1.52
待抛弃客户|mocaads|0.57
待抛弃客户|cubicom|0.55
待抛弃客户|sync_API|0.43
待抛弃客户|youmi|0.65"""
CUST_IDSP={'头部直客渠道':6,'厂商预算':4,'中小客户':42,'待抛弃客户':12}

# 解析客户分类明细
cust_items=[]  # (cat, acc, base)
for ln in CUST_BASE_RAW.strip().split('\n'):
    cat,acc,base=ln.split('|')
    cust_items.append((cat,acc,float(base) if base.strip() else 0.0))

def norm(s): return str(s).strip().lower()

# ============ 拉 Superset (Midas / 无_API account) ============
def fetch_superset():
    # 先用 slice_id 拿完整 form_data
    fd0=urllib.parse.quote(json.dumps({'slice_id':SLICE_ID}))
    req=urllib.request.Request(SUPERSET_URL+'?form_data='+fd0,headers={'cookie':SUPERSET_COOKIE,'accept':'application/json'})
    base=json.load(urllib.request.urlopen(req,timeout=120,context=ctx))
    fd=base['form_data']; fd['row_limit']=100000
    enc=urllib.parse.quote(json.dumps(fd))
    req2=urllib.request.Request(SUPERSET_URL+'?form_data='+enc,headers={'cookie':SUPERSET_COOKIE,'accept':'application/json'})
    d=json.load(urllib.request.urlopen(req2,timeout=180,context=ctx))
    return d['data']['records']

# ============ 拉 report/v2/list (MS / 带_API account) ============
def fetch_report(ymd):
    cols=[{"tCode":"adnouter_dws_report_san","tName":"dws_san_report_sum_hour","sqlType":"select","field":"action_dt","title":"Dt","sqlField":"action_dt","fieldType":1,"sorted":10,"checked":True},
          {"tCode":"adnouter_dws_report_san","tName":"dws_san_report_sum_hour","sqlType":"select","field":"advertiser_account_id","title":"Adv Account","sqlField":"advertiser_account_id","fieldType":1,"sorted":26,"checked":True},
          {"tCode":"adnouter_dws_report_san","tName":"dws_san_report_sum_hour","sqlType":"select","field":"package_name","title":"Pkg Name","sqlField":"pkg_name","fieldType":1,"sorted":60,"checked":True},
          {"tCode":"adnouter_dws_report_san","tName":"dws_san_report_sum_hour","sqlType":"select","field":"revenues","title":"Revenue","sqlField":"sum(revenue)","fieldType":2,"sorted":230,"checked":True}]
    body={"pageSize":100000,"pageIndex":1,"query":{"dt":[ymd,ymd],"pb_geo":[],"target_geo":[]},"columns":cols,"orderBy":{}}
    req=urllib.request.Request(REPORT_URL,data=json.dumps(body).encode(),
        headers={'authorization':MS_TOKEN,'content-type':'application/json;charset=UTF-8','origin':'https://ms.sandssea.com','referer':'https://ms.sandssea.com/'})
    d=json.load(urllib.request.urlopen(req,timeout=180,context=ctx))
    return d['data']['list']

# ---- 各源独立容错：一个失败不影响另一个 ----
src_status={'superset':'ok','report':'ok'}
sup=[]; rep=[]
try:
    print('[fetch] Superset...'); sup=fetch_superset()
except Exception as e:
    src_status['superset']='fail: '+str(e)[:120]
    print('[ERROR] Superset 拉取失败:', str(e)[:200])
try:
    print('[fetch] report/v2/list T-1...'); rep=fetch_report(PREV_YMD)
except Exception as e:
    src_status['report']='fail: '+str(e)[:120]
    print('[ERROR] report/v2/list 拉取失败(可能MS token过期):', str(e)[:200])

# 两个源都失败 -> 放弃本次更新，不写 data.json
if src_status['superset'].startswith('fail') and src_status['report'].startswith('fail'):
    print('[ABORT] 两个数据源都失败，本次不更新 data.json。')
    print('!!! 需检查凭证: Superset cookie / MS token 可能均已过期 !!!')
    import sys as _s; _s.exit(1)
# 单源失败：告警但继续（用另一个源的数据更新）
if src_status['report'].startswith('fail'):
    print('!!!!! 告警: MS(report/v2/list) 数据源失效，本次 MS/带_API account 数据将缺失。请尽快更新 MS_TOKEN !!!!!')
if src_status['superset'].startswith('fail'):
    print('!!!!! 告警: Superset 数据源失效，本次 Midas/无_API account 数据将缺失。请尽快更新 SUPERSET_COOKIE !!!!!')

# Superset: 只取 T-1 那天(dt格式20260910)
sup_prev=[r for r in sup if str(r.get('dt'))==PREV_YMD]
print(f'  superset T-1行={len(sup_prev)} (全部{len(sup)}) ; report行={len(rep)}')

# ---- 汇总：无_API account revenue (来自Superset attr_platform) ----
sup_acc_rev=defaultdict(float)      # account(小写) -> rev
sup_pkg_acc=defaultdict(lambda:defaultdict(float))  # pkg -> {account -> rev} (Midas, account=attr_platform)
for r in sup_prev:
    acc=norm(r.get('attr_platform'))
    rev=float(r.get('revenue(还原前)') or 0)
    sup_acc_rev[acc]+=rev
    pkg=r.get('pkg name(EN)') or ''
    if rev:
        sup_pkg_acc[pkg][r.get('attr_platform') or acc]+=rev

# ---- 汇总：带_API account revenue (来自report, 按account_id映射, 排除18) ----
rep_acc_rev=defaultdict(float)      # _API名(小写) -> rev
rep_pkg_acc=defaultdict(lambda:defaultdict(float))  # pkg -> {account -> rev} (MS, 排除account18)
for r in rep:
    aid=r.get('advertiser_account_id')
    rev=float(r.get('revenues') or 0)
    if aid==18:   # Midas_API 不记录
        continue
    name=ACC.get(aid)
    if name:
        rep_acc_rev[norm(name)]+=rev
    pkg=r.get('package_name') or ''
    disp = name if name else ('acct_%s'%aid)
    if rev:
        rep_pkg_acc[pkg][disp]+=rev

# 统一取某 account 收入：无_API走superset，带_API走report
def acc_rev(acc):
    a=norm(acc)
    if acc.endswith('_API') or acc.endswith('_api'):
        return round(rep_acc_rev.get(a,0.0),4)
    else:
        return round(sup_acc_rev.get(a,0.0),4)

# ============ 1.2 CN / 1.3 International ============
cn_row={'dt':PREV_DATE}; cn_total=0.0
for a in CN_ACCS:
    v=acc_rev(a); cn_row[a]=v; cn_total+=v
cn_row['DSP CN revenue']=round(cn_total,2)

intl_row={'dt':PREV_DATE}; intl_total=0.0
for a in INTL_ACCS:
    v=acc_rev(a); intl_row[a]=v; intl_total+=v
intl_row['DSP international']=round(intl_total,2)

# ============ 1.1 Overall ============
overall_row={'dt':PREV_DATE,'DSP total revenue':round(cn_total+intl_total,2),
             'DSP CN revenue':round(cn_total,2),'DSP international':round(intl_total,2)}

# ============ 2.1 PKG-Midas / 2.2 PKG-MS (当天 pkg -> {account: rev}) ============
pkg_midas={pkg:{a:round(v,4) for a,v in accs.items() if v} for pkg,accs in sup_pkg_acc.items() if accs}
pkg_ms={pkg:{a:round(v,4) for a,v in accs.items() if v} for pkg,accs in rep_pkg_acc.items() if accs}

# ============ 3.x 客户分类 (当天) ============
# 每个 account 的 T-1 收入
cust_detail_today=[]  # {cat,acc,base,t1}
cat_t1=defaultdict(float)
for cat,acc,base in cust_items:
    t1=acc_rev(acc)
    cust_detail_today.append({'cat':cat,'acc':acc,'base':base,'t1':round(t1,4)})
    cat_t1[cat]+=t1
cust_cat_today={cat:round(v,4) for cat,v in cat_t1.items()}  # 分类->T-1收入

# ============ 合并入 data.json ============
def dkey(s): return tuple(int(x) for x in s.split('/'))
old={'overall':[],'cn':[],'international':[],'pkg_midas':{},'pkg_ms':{},
     'cust_daily':{},'cust_detail':{},'cust_base':[],'cust_idsp':CUST_IDSP,'updated_at':''}
if os.path.exists(DATA_FILE):
    try: old=json.load(open(DATA_FILE,encoding='utf-8'))
    except Exception: pass
for k,dv in [('overall',[]),('cn',[]),('international',[]),('pkg_midas',{}),('pkg_ms',{}),('cust_daily',{}),('cust_detail',{})]:
    old.setdefault(k,dv)
old['cust_idsp']=CUST_IDSP
# 客户分类 base 明细(固定,每次覆盖为最新配置)
old['cust_base']=[{'cat':c,'acc':a,'base':b} for c,a,b in cust_items]

# 叠加型：overall/cn/international，按dt去重后append
def upsert_daily(lst,row):
    lst=[x for x in lst if x.get('dt')!=row['dt']]+[row]
    lst.sort(key=lambda x:dkey(x['dt']))
    return lst
old['overall']=upsert_daily(old['overall'],overall_row)
old['cn']=upsert_daily(old['cn'],cn_row)
old['international']=upsert_daily(old['international'],intl_row)

# 14天型：pkg_midas/pkg_ms 是 {dt: {pkg:rev}}；cust_daily {dt:{cat:t1}}；cust_detail {dt:[{cat,acc,base,t1}]}
def keep_recent(dct):
    dts=sorted(dct.keys(),key=dkey)
    for d in dts[:-KEEP]: dct.pop(d,None)
    return dct
old['pkg_midas'][PREV_DATE]=pkg_midas
old['pkg_ms'][PREV_DATE]=pkg_ms
old['cust_daily'][PREV_DATE]=cust_cat_today
old['cust_detail'][PREV_DATE]=cust_detail_today
for key in ['pkg_midas','pkg_ms','cust_daily','cust_detail']:
    old[key]=keep_recent(old[key])

old['updated_at']=datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=8))).strftime('%Y-%m-%d %H:%M:%S UTC+8')
old['source_status']=src_status
json.dump(old,open(DATA_FILE,'w',encoding='utf-8'),ensure_ascii=False,separators=(',',':'))
print(f"[done] overall={len(old['overall'])}天  CN合计={cn_row['DSP CN revenue']}  Intl合计={intl_row['DSP international']}")
print(f"       pkg_midas={len(pkg_midas)}个pkg  pkg_ms={len(pkg_ms)}个pkg  cust分类={cust_cat_today}")
PYEOF

echo "[ok] data.json 已更新"