Classify campaigns by experiment adping IDs

index.html

@@ -41,7 +41,7 @@
     <div class="panel chart"><div class="title">Campaign 花费排名</div><div id="ranking"></div></div>
   </section>
   <section class="panel table-panel"><div class="table-head"><div class="title" style="margin:0">Detail</div><div class="table-note" id="count"></div></div><div class="table-wrap"><table><thead><tr><th>Date</th><th>组别</th><th>Campaign name</th><th class="num">Impression</th><th class="num">Click</th><th class="num">Conversions</th><th class="num">Spend</th><th class="num">CTR</th><th class="num">CVR</th><th class="num">CPA</th></tr></thead><tbody id="body"></tbody></table></div></section>
-  <div class="foot">数据源：Tableau · DSP Daily Data。组别、GEO、渠道、状态和 KPI 来自 Campaign info；订单状态以 G 列为准。对照组统一显示为 General campaign。比率均使用汇总后的分子与分母重新计算。</div>
+  <div class="foot">数据源：Tableau · DSP Daily Data。订单状态以 Campaign info 的 G 列为准。实验组通过 Campaign name 中的 adping ID 识别；同一 pkg、GEO、Platform 下，其余 Campaign 合并为对照组并显示为 General campaign。比率均使用汇总后的分子与分母重新计算。</div>
 </main><script>
 const $=id=>document.getElementById(id); let all=[];
 const n=(v,d=0)=>Number(v||0).toLocaleString('en-US',{minimumFractionDigits:d,maximumFractionDigits:d});

transform_tableau.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+import csv, io, json, os, re, sys
+from datetime import datetime
+from zoneinfo import ZoneInfo
+
+src, target = sys.argv[1:]
+raw = open(src, 'rb').read()
+if b'<html' in raw[:1000].lower():
+    raise SystemExit('Tableau authentication redirected to HTML')
+text = raw.decode('utf-16') if raw.startswith((b'\xff\xfe', b'\xfe\xff')) else raw.decode('utf-8-sig')
+delimiter = '\t' if text.count('\t') > text.count(',') else ','
+matrix = list(csv.reader(io.StringIO(text), delimiter=delimiter))
+
+aliases = {
+    'pkg': {'pkg', '维度1'}, 'source': {'source', 'platform'}, 'date': {'date', 'dt', '日期'},
+    'campaign': {'campaign name', 'campaign_name', '维度2'},
+    'impression': {'impression', 'impressions'}, 'click': {'click', 'clicks'},
+    'conversions': {'conversion', 'conversions'}, 'spend': {'spend', 'cost'},
+}
+header = [c.strip().lower() for c in matrix[0]] if matrix else []
+has_header = any(c in aliases['pkg'] for c in header) and any(c in aliases['campaign'] for c in header)
+
+def cell(row, field):
+    if has_header:
+        for i, name in enumerate(header):
+            if name in aliases[field]: return row[i] if i < len(row) else ''
+        return ''
+    positions = {'pkg':0, 'source':1, 'date':3, 'campaign':4, 'impression':5, 'click':6, 'conversions':7, 'spend':8}
+    i = positions[field]
+    return row[i] if i < len(row) else ''
+
+def num(v):
+    try: return float(str(v or '0').replace(',', '').replace('%', ''))
+    except ValueError: return 0
+
+config = [r for r in json.load(open('campaign_config.json', encoding='utf-8')) if r['status'].lower() == 'active']
+experiment = {}
+for r in config:
+    if r['group'] == '实验组':
+        experiment.setdefault((r['pkg'], r['channel'].lower()), []).append((str(r['adping_id']), r))
+scopes = {}
+for r in config:
+    scopes.setdefault((r['pkg'], r['channel'].lower(), r['geo'].upper()), r)
+
+def resolve_geo(pkg, source, campaign):
+    candidates = sorted({g for p, s, g in scopes if p == pkg and s == source}, key=len, reverse=True)
+    upper = campaign.upper()
+    # Some platform campaign names append a sequence directly to the GEO (for example MY2/TH1).
+    matches = [g for g in candidates if re.search(r'(^|[^A-Z0-9])' + re.escape(g) + r'(?=[^A-Z]|$)', upper)]
+    if len(matches) == 1: return matches[0]
+    return candidates[0] if len(candidates) == 1 else None
+
+rows, unresolved = [], []
+for raw_row in matrix[1:] if has_header else matrix:
+    pkg, source = cell(raw_row, 'pkg').strip(), cell(raw_row, 'source').strip().lower()
+    campaign = cell(raw_row, 'campaign').strip()
+    if not pkg or not source or not campaign: continue
+    exp = next((meta for adping, meta in experiment.get((pkg, source), [])
+                if re.search(r'(?<!\d)' + re.escape(adping) + r'(?!\d)', campaign)), None)
+    if exp:
+        meta, group, shown, adping = exp, '实验组', campaign, str(exp['adping_id'])
+    else:
+        geo = resolve_geo(pkg, source, campaign)
+        meta = scopes.get((pkg, source, geo)) if geo else None
+        if not meta:
+            unresolved.append(campaign)
+            continue
+        group, shown, adping = '对照组', 'General campaign', ''
+    rows.append({'date': cell(raw_row, 'date').strip() or target, 'pkg':pkg, 'geo':meta['geo'], 'source':source,
+      'group':group, 'adpingId':adping, 'campaign':shown, 'status':'Active', 'kpi':meta.get('kpi',''),
+      'impression':num(cell(raw_row,'impression')), 'click':num(cell(raw_row,'click')),
+      'conversions':num(cell(raw_row,'conversions')), 'spend':num(cell(raw_row,'spend'))})
+
+if not rows: raise SystemExit('No valid Tableau rows; data.json preserved')
+old = json.load(open('data.json',encoding='utf-8')) if os.path.exists('data.json') else {'rows':[]}
+old['rows'] = [r for r in old.get('rows',[]) if r.get('date') != target] + rows
+old['latestDate'] = max(r['date'] for r in old['rows'])
+old['generatedAt'] = datetime.now(ZoneInfo('Asia/Shanghai')).isoformat()
+old['source'] = 'DSP Daily Data / Campaign info'
+old['quality'] = {'unresolvedCount':len(unresolved), 'unresolvedExamples':unresolved[:20]}
+with open('data.json.tmp','w',encoding='utf-8') as f: json.dump(old,f,ensure_ascii=False,separators=(',',':'))
+os.replace('data.json.tmp','data.json')
+print(json.dumps({'date':target,'rows':len(rows),'unresolved':len(unresolved)},ensure_ascii=False))

update.sh

@@ -16,40 +16,4 @@ pkg_csv="$(python3 -c 'import json; d=json.load(open("campaign_config.json")); p
 query="?:showVizHome=no&dt=${target_date}&pkg=${pkg_csv}&维度1=pkg&维度2=campaign_name&Source=展开&source=bigo,kwai,mintegral"
 curl -fsSL --cookie "$TABLEAU_COOKIE" "${base_url}${query}" -o "$tmp_file"
 
-python3 - "$tmp_file" "$target_date" <<'PY'
-import csv, io, json, re, sys
-from datetime import datetime
-from zoneinfo import ZoneInfo
-
-src, target = sys.argv[1:]
-raw = open(src, 'rb').read()
-if b'<html' in raw[:1000].lower(): raise SystemExit('Tableau authentication redirected to HTML')
-text = raw.decode('utf-16') if raw.startswith((b'\xff\xfe', b'\xfe\xff')) else raw.decode('utf-8-sig')
-delimiter = '\t' if text.count('\t') > text.count(',') else ','
-incoming = list(csv.DictReader(io.StringIO(text), delimiter=delimiter))
-config = json.load(open('campaign_config.json', encoding='utf-8'))
-by_id = {str(r['adping_id']): r for r in config}
-
-def num(v):
-    try: return float(str(v or '0').replace(',', '').replace('%', ''))
-    except ValueError: return 0
-
-rows=[]
-for r in incoming:
-    m=re.match(r'^(\d+)', r.get('维度2',''))
-    if not m or m.group(1) not in by_id: continue
-    c=by_id[m.group(1)]; group=c['group']
-    source=(r.get('Source') or c['channel']).lower()
-    rows.append({'date':target,'pkg':c['pkg'],'geo':c['geo'],'source':source,'group':group,
-      'adpingId':c['adping_id'],'campaign':'General campaign' if group=='对照组' else r.get('维度2',''),
-      'status':c['status'],'kpi':c['kpi'],'impression':num(r.get('impression')),'click':num(r.get('click')),
-      'conversions':num(r.get('conversions')),'spend':num(r.get('spend'))})
-if not rows: raise SystemExit('No valid Tableau rows; data.json preserved')
-old=json.load(open('data.json',encoding='utf-8')) if __import__('os').path.exists('data.json') else {'rows':[]}
-old['rows']=[r for r in old.get('rows',[]) if r.get('date')!=target]+rows
-old['latestDate']=max(r['date'] for r in old['rows'])
-old['generatedAt']=datetime.now(ZoneInfo('Asia/Shanghai')).isoformat()
-old['source']='DSP Daily Data / Campaign info'
-with open('data.json.tmp','w',encoding='utf-8') as f: json.dump(old,f,ensure_ascii=False,separators=(',',':'))
-__import__('os').replace('data.json.tmp','data.json')
-PY
+python3 transform_tableau.py "$tmp_file" "$target_date"