56 lines
2.8 KiB
Bash
Executable File
56 lines
2.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# AgentStage runs this hourly. The guard below refreshes once after 12:00 Asia/Shanghai.
|
|
export TZ=Asia/Shanghai
|
|
hour="$(date +%H)"
|
|
target_date="$(date -v-1d +%F 2>/dev/null || date -d yesterday +%F)"
|
|
if [ "$hour" -lt 12 ]; then exit 0; fi
|
|
|
|
: "${TABLEAU_COOKIE:?Set TABLEAU_COOKIE in the AgentStage runner secret environment}"
|
|
base_url="${TABLEAU_CSV_URL:-https://mars.ushareit.org/views/DSPSpend/DSPDailyData.csv}"
|
|
tmp_file="$(mktemp)"
|
|
trap 'rm -f "$tmp_file"' EXIT
|
|
|
|
pkg_csv="$(python3 -c 'import json; d=json.load(open("campaign_config.json")); print(",".join(sorted({r["pkg"] for r in d if r["status"].lower()=="active"})))')"
|
|
query="?:showVizHome=no&dt=${target_date}&pkg=${pkg_csv}&维度1=pkg&维度2=campaign_name&Source=展开&source=bigo,kwai,mintegral"
|
|
curl -fsSL --cookie "$TABLEAU_COOKIE" "${base_url}${query}" -o "$tmp_file"
|
|
|
|
python3 - "$tmp_file" "$target_date" <<'PY'
|
|
import csv, io, json, re, sys
|
|
from datetime import datetime
|
|
from zoneinfo import ZoneInfo
|
|
|
|
src, target = sys.argv[1:]
|
|
raw = open(src, 'rb').read()
|
|
if b'<html' in raw[:1000].lower(): raise SystemExit('Tableau authentication redirected to HTML')
|
|
text = raw.decode('utf-16') if raw.startswith((b'\xff\xfe', b'\xfe\xff')) else raw.decode('utf-8-sig')
|
|
delimiter = '\t' if text.count('\t') > text.count(',') else ','
|
|
incoming = list(csv.DictReader(io.StringIO(text), delimiter=delimiter))
|
|
config = json.load(open('campaign_config.json', encoding='utf-8'))
|
|
by_id = {str(r['adping_id']): r for r in config}
|
|
|
|
def num(v):
|
|
try: return float(str(v or '0').replace(',', '').replace('%', ''))
|
|
except ValueError: return 0
|
|
|
|
rows=[]
|
|
for r in incoming:
|
|
m=re.match(r'^(\d+)', r.get('维度2',''))
|
|
if not m or m.group(1) not in by_id: continue
|
|
c=by_id[m.group(1)]; group=c['group']
|
|
source=(r.get('Source') or c['channel']).lower()
|
|
rows.append({'date':target,'pkg':c['pkg'],'geo':c['geo'],'source':source,'group':group,
|
|
'adpingId':c['adping_id'],'campaign':'General campaign' if group=='对照组' else r.get('维度2',''),
|
|
'status':c['status'],'kpi':c['kpi'],'impression':num(r.get('impression')),'click':num(r.get('click')),
|
|
'conversions':num(r.get('conversions')),'spend':num(r.get('spend'))})
|
|
if not rows: raise SystemExit('No valid Tableau rows; data.json preserved')
|
|
old=json.load(open('data.json',encoding='utf-8')) if __import__('os').path.exists('data.json') else {'rows':[]}
|
|
old['rows']=[r for r in old.get('rows',[]) if r.get('date')!=target]+rows
|
|
old['latestDate']=max(r['date'] for r in old['rows'])
|
|
old['generatedAt']=datetime.now(ZoneInfo('Asia/Shanghai')).isoformat()
|
|
old['source']='DSP Daily Data / Campaign info'
|
|
with open('data.json.tmp','w',encoding='utf-8') as f: json.dump(old,f,ensure_ascii=False,separators=(',',':'))
|
|
__import__('os').replace('data.json.tmp','data.json')
|
|
PY
|