309 lines
16 KiB
Bash
Executable File
309 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# DSP-daily 日报数据更新脚本
|
||
# 每小时被平台执行；UTC+8 过11点且当天未更新则补跑，查询 T-1(前一天)。
|
||
# 产出 data.json：
|
||
# overall/cn/international : 每天一行，持续叠加
|
||
# pkg_midas / pkg_ms : {dt:{pkg:{account:rev}}} 保留14天(前端按模式聚合/算DoD)
|
||
# cust_daily / cust_detail : 客户分类，保留14天
|
||
# ============================================================
|
||
set -euo pipefail
|
||
cd "$(dirname "$0")"
|
||
|
||
# ============ 凭证（会过期，过期后替换这两行即可）============
|
||
# Superset cookie（用于 Midas / 无_API account）
|
||
SUPERSET_COOKIE='SR-PROD=dea34a1f6d50f4426c0a9fa3d19d75a5; oidc=d86576ac9fae4782b20b16044ca089a0; session=.eJztkMFqAzEMRH8l-LwHWbaz1t4K-YsSgmRJzdJlA-v0FPLvNfQ3ehoG9IbRvMLNN-5362H5fIXTc0hQ3r_sCFP4aM16P639dLF9NQ3X9_R_dJ3GaIf1e1ict27DrhqWoK65ZtCcIEWPqK0QzAiVhbmm1oZKpUh5BkiKLOoulJgRFItkAyzguYKVVCl5xsqALatxKQk51aIyUEaspYmcudQyq0uV7Oc82rZ--O35-LZ99IFIJplJm9GslWb0AlGcyVFoBCM1oDkNbns03mwwA5zCT7fj76WIAOH9C_vZpvc.aqOlrQ.xmkrmwsNe89-wy3pSP20QhNNuC8'
|
||
# MS report/v2/list token（用于 MS / 带_API account）
|
||
MS_TOKEN='eyJhbGciOiJIUzI1NiJ9.eyJqdGkiOiI5OGNlN2YzZDY3NzE0NDFhOWQxM2I2YjJhOTY5M2UxZSIsImlhdCI6MTc4OTM1NTAzMywic3ViIjoidG9rZW4gYnkgdXNoYXJlaXQiLCJleHAiOjE3ODk0NDE0MzN9.6IupaUeP2gThNu0v7P5J8uA2aL44m8vvGDjFIhnmhKI'
|
||
|
||
SUPERSET_URL='https://ads-bi.ushareit.org/superset/explore_json/'
|
||
SLICE_ID=821
|
||
REPORT_URL='https://ms-api.sandssea.com/api/v1/report/v2/list'
|
||
DATA_FILE='data.json'
|
||
KEEP_DAYS=14
|
||
|
||
# ============ 执行时机（UTC+8，过11点且当天未更新则补跑）============
|
||
HOUR_CST=$(TZ='Asia/Shanghai' date +%H)
|
||
RUN_DATE=$(TZ='Asia/Shanghai' date '+%Y/%-m/%-d')
|
||
FORCE="${FORCE_RUN:-0}"
|
||
if [ "$FORCE" != "1" ]; then
|
||
if [ "$((10#$HOUR_CST))" -lt 11 ]; then echo "[skip] UTC+8 $HOUR_CST 点，未到11点"; exit 0; fi
|
||
if [ -f "$DATA_FILE" ]; then
|
||
A=$(python3 -c "import json;d=json.load(open('$DATA_FILE',encoding='utf-8'));print('1' if any(r.get('date')=='$RUN_DATE' for r in d.get('overall',[])) else '0')" 2>/dev/null || echo 0)
|
||
if [ "$A" = "1" ]; then echo "[skip] 今天($RUN_DATE)已更新"; exit 0; fi
|
||
fi
|
||
fi
|
||
|
||
# T-1 日期（前一天）
|
||
PREV_YMD=$(TZ='Asia/Shanghai' date -v-1d '+%Y%m%d' 2>/dev/null || TZ='Asia/Shanghai' date -d 'yesterday' '+%Y%m%d') # 20260910
|
||
PREV_DATE=$(TZ='Asia/Shanghai' date -v-1d '+%Y/%-m/%-d' 2>/dev/null || TZ='Asia/Shanghai' date -d 'yesterday' '+%Y/%-m/%-d') # 2026/9/10
|
||
echo "[run] RUN_DATE=$RUN_DATE T-1=$PREV_YMD($PREV_DATE)"
|
||
|
||
# ============ 主处理：Python 完成拉取+聚合+合并 ============
|
||
export SUPERSET_COOKIE MS_TOKEN SUPERSET_URL SLICE_ID REPORT_URL DATA_FILE KEEP_DAYS RUN_DATE PREV_YMD PREV_DATE
|
||
python3 <<'PYEOF'
|
||
import os,json,urllib.parse,urllib.request,ssl,datetime
|
||
from collections import defaultdict
|
||
|
||
SUPERSET_COOKIE=os.environ['SUPERSET_COOKIE']; MS_TOKEN=os.environ['MS_TOKEN']
|
||
SUPERSET_URL=os.environ['SUPERSET_URL']; SLICE_ID=int(os.environ['SLICE_ID'])
|
||
REPORT_URL=os.environ['REPORT_URL']; DATA_FILE=os.environ['DATA_FILE']
|
||
KEEP=int(os.environ['KEEP_DAYS']); RUN_DATE=os.environ['RUN_DATE']
|
||
PREV_YMD=os.environ['PREV_YMD']; PREV_DATE=os.environ['PREV_DATE']
|
||
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
|
||
|
||
# ---- account 对照表 (advertiser_account_id -> _API名) ----
|
||
ACC={102:'nativex_API',99:'adsplay_API',98:'clickpuff_API',97:'affle_API',96:'mobhuge_API',95:'altosum_API',94:'seanear_API',93:'stridemobi_API',92:'volo_API',91:'strongmetrics_API',87:'aba_API',86:'hope_api',85:'stackyads_API',84:'isummer_API',83:'vertex_API',82:'viking_API',80:'acmemobi_API',79:'flymobi_API',78:'motilal_API',77:'flatmedia_API',76:'hypertap_API',75:'Influx_API',74:'sync_API',73:'c2c_API',72:'tecdo_API',71:'yomiads_API',70:'goodjoy_API',69:'apkpure_API',68:'flatads_API',67:'adnov_API',66:'dolemobi_API',65:'witgala_API',64:'mejoy_API',63:'adwisdom_ddj_API',62:'vfineads_ddj_API',60:'Iogerads_API',56:'quarklio_API',55:'quarklio_ddj_API',54:'bidneuron_API',52:'mstoko_API',51:'hopemobi_API',23:'Macan_API',18:'Midas_API',11:'Dreammobi_API',9:'Flowbox_API'}
|
||
|
||
# ---- CN / International 列头 account ----
|
||
CN_ACCS=['acshare','adsdeep','adwisdom','affinityad','alphadsp','altosum','apkpure','appfortis','appnextads','bidmatrix','bidneuron','bytemobi','cookeads','dolphin','dolphin2','flowbox','flymob','gleamx','gridads','imax','influxtec','iogerads','leapmob','lightspot','mbupps','mobgem','moblegend','mocaads','movablead','mstoko','nativexads','Ricefruit','shockit','smartconnectads','startym','tenads','verse','vertexs','vikingmedia','xoomads','yolo','youmi',
|
||
'bidneuron_API','quarklio_API','quarklio_ddj_API','Iogerads_API','adwisdom_ddj_API','mejoy_API','witgala_API','dolemobi_API','flatads_API','adnov_API','goodjoy_API','yomiads_API','flymobi_API','vertex_api','isummer_API','viking_api','hope_api','aba_api','volo_api','seanear_API','mobhuge_API','altosum_API','clickpuff_API']
|
||
INTL_ACCS=['adsflourish','affleads','avowads','cubicom','keendigi','launchpad','mobavenue','stackyads','vserv','xapad',
|
||
'sync_API','motilal_API','acmemobi_API','strongmetrics_API','c2c']
|
||
|
||
# ---- 客户分类 base 明细 (以文字明细为准) ----
|
||
CUST_BASE_RAW="""头部直客渠道|appnextads|7422.98
|
||
头部直客渠道|xapad|4200.73
|
||
头部直客渠道|vserv|3551.89
|
||
头部直客渠道|affinityad|2099.18
|
||
头部直客渠道|avowads|381.55
|
||
头部直客渠道|alphadsp|0
|
||
厂商预算|flymob|1002.15
|
||
厂商预算|flymobi_API|0
|
||
厂商预算|dolphin|773.56
|
||
厂商预算|dolphin2|0
|
||
厂商预算|iogerads|371.16
|
||
厂商预算|Iogerads_API|0
|
||
厂商预算|shockit|292.4
|
||
中小客户|adwisdom|738.61
|
||
中小客户|adwisdom_ddj_API|0
|
||
中小客户|appfortis|257.96
|
||
中小客户|mobavenue|228.43
|
||
中小客户|verse|199.27
|
||
中小客户|startym|116.72
|
||
中小客户|mbupps|112.17
|
||
中小客户|nativexads|102
|
||
中小客户|mstoko|97.96
|
||
中小客户|flatmedia_API|72.42
|
||
中小客户|leapmob|60.46
|
||
中小客户|affleads|59.83
|
||
中小客户|moblegend|51.33
|
||
中小客户|vertexs|51.09
|
||
中小客户|vertex_api|0
|
||
中小客户|stackyads|43.7
|
||
中小客户|apkpure|37.73
|
||
中小客户|acshare|33.72
|
||
中小客户|bytemobi|33.55
|
||
中小客户|adsdeep|32.19
|
||
中小客户|Ricefruit|31.7
|
||
中小客户|abamobi|31.29
|
||
中小客户|aba_api|0
|
||
中小客户|imax|29.14
|
||
中小客户|vfineads|27.77
|
||
中小客户|influxtec|26.75
|
||
中小客户|isummer|24.8
|
||
中小客户|isummer_API|0
|
||
中小客户|smartconnectads|23.99
|
||
中小客户|macan|21.64
|
||
中小客户|xoomads|21.62
|
||
中小客户|flowbox|20.62
|
||
中小客户|bidmatrix|18.91
|
||
中小客户|seanear_API|17.48
|
||
中小客户|gridads|16.51
|
||
中小客户|yomiads_API|15.55
|
||
中小客户|gleamx|14.37
|
||
中小客户|bidneuron|12.89
|
||
中小客户|bidneuron_API|0
|
||
中小客户|quarklio_API|12.83
|
||
中小客户|altosum|11.95
|
||
中小客户|altosum_API|0
|
||
中小客户|lightspot|11.75
|
||
中小客户|vikingmedia|10.87
|
||
中小客户|viking_api|0
|
||
中小客户|mobgem|5.86
|
||
中小客户|launchpad|4.2
|
||
中小客户|adnov_API|0.24
|
||
中小客户|hopemobi_API|2.55
|
||
中小客户|hope_API|0
|
||
中小客户|volo|10.84
|
||
中小客户|volo_api|0
|
||
待抛弃客户|mobhugeads|9.95
|
||
待抛弃客户|wayne|7.86
|
||
待抛弃客户|c2c|6.3
|
||
待抛弃客户|tenads|4.45
|
||
待抛弃客户|movablead|4.22
|
||
待抛弃客户|tecdo|3.17
|
||
待抛弃客户|cookeads|1.52
|
||
待抛弃客户|mocaads|0.57
|
||
待抛弃客户|cubicom|0.55
|
||
待抛弃客户|sync_API|0.43
|
||
待抛弃客户|youmi|0.65"""
|
||
CUST_IDSP={'头部直客渠道':6,'厂商预算':4,'中小客户':42,'待抛弃客户':12}
|
||
|
||
# 解析客户分类明细
|
||
cust_items=[] # (cat, acc, base)
|
||
for ln in CUST_BASE_RAW.strip().split('\n'):
|
||
cat,acc,base=ln.split('|')
|
||
cust_items.append((cat,acc,float(base) if base.strip() else 0.0))
|
||
|
||
def norm(s): return str(s).strip().lower()
|
||
|
||
# ============ 拉 Superset (Midas / 无_API account) ============
|
||
def fetch_superset():
|
||
# 先用 slice_id 拿完整 form_data
|
||
fd0=urllib.parse.quote(json.dumps({'slice_id':SLICE_ID}))
|
||
req=urllib.request.Request(SUPERSET_URL+'?form_data='+fd0,headers={'cookie':SUPERSET_COOKIE,'accept':'application/json'})
|
||
base=json.load(urllib.request.urlopen(req,timeout=120,context=ctx))
|
||
fd=base['form_data']; fd['row_limit']=100000
|
||
enc=urllib.parse.quote(json.dumps(fd))
|
||
req2=urllib.request.Request(SUPERSET_URL+'?form_data='+enc,headers={'cookie':SUPERSET_COOKIE,'accept':'application/json'})
|
||
d=json.load(urllib.request.urlopen(req2,timeout=180,context=ctx))
|
||
return d['data']['records']
|
||
|
||
# ============ 拉 report/v2/list (MS / 带_API account) ============
|
||
def fetch_report(ymd):
|
||
cols=[{"tCode":"adnouter_dws_report_san","tName":"dws_san_report_sum_hour","sqlType":"select","field":"action_dt","title":"Dt","sqlField":"action_dt","fieldType":1,"sorted":10,"checked":True},
|
||
{"tCode":"adnouter_dws_report_san","tName":"dws_san_report_sum_hour","sqlType":"select","field":"advertiser_account_id","title":"Adv Account","sqlField":"advertiser_account_id","fieldType":1,"sorted":26,"checked":True},
|
||
{"tCode":"adnouter_dws_report_san","tName":"dws_san_report_sum_hour","sqlType":"select","field":"package_name","title":"Pkg Name","sqlField":"pkg_name","fieldType":1,"sorted":60,"checked":True},
|
||
{"tCode":"adnouter_dws_report_san","tName":"dws_san_report_sum_hour","sqlType":"select","field":"revenues","title":"Revenue","sqlField":"sum(revenue)","fieldType":2,"sorted":230,"checked":True}]
|
||
body={"pageSize":100000,"pageIndex":1,"query":{"dt":[ymd,ymd],"pb_geo":[],"target_geo":[]},"columns":cols,"orderBy":{}}
|
||
req=urllib.request.Request(REPORT_URL,data=json.dumps(body).encode(),
|
||
headers={'authorization':MS_TOKEN,'content-type':'application/json;charset=UTF-8','origin':'https://ms.sandssea.com','referer':'https://ms.sandssea.com/'})
|
||
d=json.load(urllib.request.urlopen(req,timeout=180,context=ctx))
|
||
return d['data']['list']
|
||
|
||
# ---- 各源独立容错：一个失败不影响另一个 ----
|
||
src_status={'superset':'ok','report':'ok'}
|
||
sup=[]; rep=[]
|
||
try:
|
||
print('[fetch] Superset...'); sup=fetch_superset()
|
||
except Exception as e:
|
||
src_status['superset']='fail: '+str(e)[:120]
|
||
print('[ERROR] Superset 拉取失败:', str(e)[:200])
|
||
try:
|
||
print('[fetch] report/v2/list T-1...'); rep=fetch_report(PREV_YMD)
|
||
except Exception as e:
|
||
src_status['report']='fail: '+str(e)[:120]
|
||
print('[ERROR] report/v2/list 拉取失败(可能MS token过期):', str(e)[:200])
|
||
|
||
# 两个源都失败 -> 放弃本次更新，不写 data.json
|
||
if src_status['superset'].startswith('fail') and src_status['report'].startswith('fail'):
|
||
print('[ABORT] 两个数据源都失败，本次不更新 data.json。')
|
||
print('!!! 需检查凭证: Superset cookie / MS token 可能均已过期 !!!')
|
||
import sys as _s; _s.exit(1)
|
||
# 单源失败：告警但继续（用另一个源的数据更新）
|
||
if src_status['report'].startswith('fail'):
|
||
print('!!!!! 告警: MS(report/v2/list) 数据源失效，本次 MS/带_API account 数据将缺失。请尽快更新 MS_TOKEN !!!!!')
|
||
if src_status['superset'].startswith('fail'):
|
||
print('!!!!! 告警: Superset 数据源失效，本次 Midas/无_API account 数据将缺失。请尽快更新 SUPERSET_COOKIE !!!!!')
|
||
|
||
# Superset: 只取 T-1 那天(dt格式20260910)
|
||
sup_prev=[r for r in sup if str(r.get('dt'))==PREV_YMD]
|
||
print(f' superset T-1行={len(sup_prev)} (全部{len(sup)}) ; report行={len(rep)}')
|
||
|
||
# ---- 汇总：无_API account revenue (来自Superset attr_platform) ----
|
||
sup_acc_rev=defaultdict(float) # account(小写) -> rev
|
||
sup_pkg_acc=defaultdict(lambda:defaultdict(float)) # pkg -> {account -> rev} (Midas, account=attr_platform)
|
||
for r in sup_prev:
|
||
acc=norm(r.get('attr_platform'))
|
||
rev=float(r.get('revenue(还原前)') or 0)
|
||
sup_acc_rev[acc]+=rev
|
||
pkg=r.get('pkg name(EN)') or ''
|
||
if rev:
|
||
sup_pkg_acc[pkg][r.get('attr_platform') or acc]+=rev
|
||
|
||
# ---- 汇总：带_API account revenue (来自report, 按account_id映射, 排除18) ----
|
||
rep_acc_rev=defaultdict(float) # _API名(小写) -> rev
|
||
rep_pkg_acc=defaultdict(lambda:defaultdict(float)) # pkg -> {account -> rev} (MS, 排除account18)
|
||
for r in rep:
|
||
aid=r.get('advertiser_account_id')
|
||
rev=float(r.get('revenues') or 0)
|
||
if aid==18: # Midas_API 不记录
|
||
continue
|
||
name=ACC.get(aid)
|
||
if name:
|
||
rep_acc_rev[norm(name)]+=rev
|
||
pkg=r.get('package_name') or ''
|
||
disp = name if name else ('acct_%s'%aid)
|
||
if rev:
|
||
rep_pkg_acc[pkg][disp]+=rev
|
||
|
||
# 统一取某 account 收入：无_API走superset，带_API走report
|
||
def acc_rev(acc):
|
||
a=norm(acc)
|
||
if acc.endswith('_API') or acc.endswith('_api'):
|
||
return round(rep_acc_rev.get(a,0.0),4)
|
||
else:
|
||
return round(sup_acc_rev.get(a,0.0),4)
|
||
|
||
# ============ 1.2 CN / 1.3 International ============
|
||
cn_row={'dt':PREV_DATE}; cn_total=0.0
|
||
for a in CN_ACCS:
|
||
v=acc_rev(a); cn_row[a]=v; cn_total+=v
|
||
cn_row['DSP CN revenue']=round(cn_total,2)
|
||
|
||
intl_row={'dt':PREV_DATE}; intl_total=0.0
|
||
for a in INTL_ACCS:
|
||
v=acc_rev(a); intl_row[a]=v; intl_total+=v
|
||
intl_row['DSP international']=round(intl_total,2)
|
||
|
||
# ============ 1.1 Overall ============
|
||
overall_row={'dt':PREV_DATE,'DSP total revenue':round(cn_total+intl_total,2),
|
||
'DSP CN revenue':round(cn_total,2),'DSP international':round(intl_total,2)}
|
||
|
||
# ============ 2.1 PKG-Midas / 2.2 PKG-MS (当天 pkg -> {account: rev}) ============
|
||
pkg_midas={pkg:{a:round(v,4) for a,v in accs.items() if v} for pkg,accs in sup_pkg_acc.items() if accs}
|
||
pkg_ms={pkg:{a:round(v,4) for a,v in accs.items() if v} for pkg,accs in rep_pkg_acc.items() if accs}
|
||
|
||
# ============ 3.x 客户分类 (当天) ============
|
||
# 每个 account 的 T-1 收入
|
||
cust_detail_today=[] # {cat,acc,base,t1}
|
||
cat_t1=defaultdict(float)
|
||
for cat,acc,base in cust_items:
|
||
t1=acc_rev(acc)
|
||
cust_detail_today.append({'cat':cat,'acc':acc,'base':base,'t1':round(t1,4)})
|
||
cat_t1[cat]+=t1
|
||
cust_cat_today={cat:round(v,4) for cat,v in cat_t1.items()} # 分类->T-1收入
|
||
|
||
# ============ 合并入 data.json ============
|
||
def dkey(s): return tuple(int(x) for x in s.split('/'))
|
||
old={'overall':[],'cn':[],'international':[],'pkg_midas':{},'pkg_ms':{},
|
||
'cust_daily':{},'cust_detail':{},'cust_base':[],'cust_idsp':CUST_IDSP,'updated_at':''}
|
||
if os.path.exists(DATA_FILE):
|
||
try: old=json.load(open(DATA_FILE,encoding='utf-8'))
|
||
except Exception: pass
|
||
for k,dv in [('overall',[]),('cn',[]),('international',[]),('pkg_midas',{}),('pkg_ms',{}),('cust_daily',{}),('cust_detail',{})]:
|
||
old.setdefault(k,dv)
|
||
old['cust_idsp']=CUST_IDSP
|
||
# 客户分类 base 明细(固定,每次覆盖为最新配置)
|
||
old['cust_base']=[{'cat':c,'acc':a,'base':b} for c,a,b in cust_items]
|
||
|
||
# 叠加型：overall/cn/international，按dt去重后append
|
||
def upsert_daily(lst,row):
|
||
lst=[x for x in lst if x.get('dt')!=row['dt']]+[row]
|
||
lst.sort(key=lambda x:dkey(x['dt']))
|
||
return lst
|
||
old['overall']=upsert_daily(old['overall'],overall_row)
|
||
old['cn']=upsert_daily(old['cn'],cn_row)
|
||
old['international']=upsert_daily(old['international'],intl_row)
|
||
|
||
# 14天型：pkg_midas/pkg_ms 是 {dt: {pkg:rev}}；cust_daily {dt:{cat:t1}}；cust_detail {dt:[{cat,acc,base,t1}]}
|
||
def keep_recent(dct):
|
||
dts=sorted(dct.keys(),key=dkey)
|
||
for d in dts[:-KEEP]: dct.pop(d,None)
|
||
return dct
|
||
old['pkg_midas'][PREV_DATE]=pkg_midas
|
||
old['pkg_ms'][PREV_DATE]=pkg_ms
|
||
old['cust_daily'][PREV_DATE]=cust_cat_today
|
||
old['cust_detail'][PREV_DATE]=cust_detail_today
|
||
for key in ['pkg_midas','pkg_ms','cust_daily','cust_detail']:
|
||
old[key]=keep_recent(old[key])
|
||
|
||
old['updated_at']=datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=8))).strftime('%Y-%m-%d %H:%M:%S UTC+8')
|
||
old['source_status']=src_status
|
||
json.dump(old,open(DATA_FILE,'w',encoding='utf-8'),ensure_ascii=False,separators=(',',':'))
|
||
print(f"[done] overall={len(old['overall'])}天 CN合计={cn_row['DSP CN revenue']} Intl合计={intl_row['DSP international']}")
|
||
print(f" pkg_midas={len(pkg_midas)}个pkg pkg_ms={len(pkg_ms)}个pkg cust分类={cust_cat_today}")
|
||
PYEOF
|
||
|
||
echo "[ok] data.json 已更新"
|