Add KPI event and CAC data pipeline

transform_tableau.py

@@ -3,7 +3,10 @@ import csv, io, json, os, re, sys
 from datetime import datetime
 from zoneinfo import ZoneInfo
 
-src, target = sys.argv[1:]
+if len(sys.argv) not in (3, 4):
+    raise SystemExit('usage: transform_tableau.py SPEND.csv YYYY-MM-DD [EVENTS.csv]')
+src, target = sys.argv[1:3]
+events_path = sys.argv[3] if len(sys.argv) == 4 else None
 raw = open(src, 'rb').read()
 if b'<html' in raw[:1000].lower():
     raise SystemExit('Tableau authentication redirected to HTML')
@@ -33,6 +36,22 @@ def num(v):
     try: return float(str(v or '0').replace(',', '').replace('%', ''))
     except ValueError: return 0
 
+def event_names(value):
+    value = str(value or '').strip()
+    if not value or value == '-': return []
+    return [x.strip().lower() for x in re.split(r'[,，;/；\n]+', value) if x.strip()]
+
+events, covered_dates = {}, set()
+if events_path:
+    with open(events_path, encoding='utf-8-sig', newline='') as f:
+        for event_row in csv.DictReader(f):
+            dt = re.sub(r'[^0-9]', '', event_row.get('dt', ''))
+            if len(dt) == 8: dt = f'{dt[:4]}-{dt[4:6]}-{dt[6:]}'
+            covered_dates.add(dt)
+            key = (dt, event_row.get('pkg_name','').strip(), event_row.get('offer_name','').strip(),
+                   event_row.get('event_name','').strip().lower())
+            events[key] = events.get(key, 0) + num(event_row.get('event_num'))
+
 config = [r for r in json.load(open('campaign_config.json', encoding='utf-8')) if r['status'].lower() == 'active']
 experiment = {}
 for r in config:
@@ -66,8 +85,15 @@ for raw_row in matrix[1:] if has_header else matrix:
             unresolved.append(campaign)
             continue
         group, shown, adping = '对照组', 'General campaign', ''
+    date = cell(raw_row, 'date').strip() or target
+    required = event_names(meta.get('kpi',''))
+    if not required or source != 'mintegral' or date not in covered_dates:
+        event = None
+    else:
+        event = sum(events.get((date, pkg, campaign, name), 0) for name in required)
     rows.append({'date': cell(raw_row, 'date').strip() or target, 'pkg':pkg, 'geo':meta['geo'], 'source':source,
       'group':group, 'adpingId':adping, 'campaign':shown, 'status':'Active', 'kpi':meta.get('kpi',''),
+      'event':event,
       'impression':num(cell(raw_row,'impression')), 'click':num(cell(raw_row,'click')),
       'conversions':num(cell(raw_row,'conversions')), 'spend':num(cell(raw_row,'spend'))})