Update September 10 dashboard data and event sources

refresh_events.py

@@ -15,7 +15,7 @@ from tableauhyperapi import Connection, HyperProcess, Telemetry
 SERVER = os.environ.get("TABLEAU_SERVER", "https://mars.ushareit.org").rstrip("/")
 API_VERSION = os.environ.get("TABLEAU_API_VERSION", "3.15")
 SITE_CONTENT_URL = os.environ.get("TABLEAU_SITE_CONTENT_URL", "")
-WORKBOOK_ID = os.environ.get("TABLEAU_EVENT_WORKBOOK_ID", "ad567777-aa52-4c2f-89c3-5acfed4fdbba")
+WORKBOOK_ID = os.environ.get("TABLEAU_EVENT_WORKBOOK_ID", "5245cbe5-be36-46db-aaa5-51da9eba4f0b")
 
 
 def request_json(url, *, body=None, headers=None):
@@ -57,23 +57,25 @@ def main():
                 archive_path.write_bytes(response.read())
             with zipfile.ZipFile(archive_path) as archive:
                 members = [name for name in archive.namelist() if name.endswith(".hyper")]
-                if len(members) != 1:
-                    raise SystemExit(f"expected one Hyper extract, found {len(members)}")
-                archive.extract(members[0], temp_dir)
-            hyper_path = Path(temp_dir) / members[0]
+                if not members:
+                    raise SystemExit("no Hyper extracts found")
+                for member in members:
+                    archive.extract(member, temp_dir)
             query = '''
-                SELECT "dt", "pkg_name", "adping_id", "offer_name", "event_name",
+                SELECT "dt", "pkg_name", "adping_id", "offer_name", "second_channel", "event_name",
                        SUM("event_num")
                 FROM "Extract"."Extract"
-                GROUP BY "dt", "pkg_name", "adping_id", "offer_name", "event_name"
+                GROUP BY "dt", "pkg_name", "adping_id", "offer_name", "second_channel", "event_name"
                 ORDER BY "dt", "pkg_name", "offer_name", "event_name"
             '''
+            rows = []
             with HyperProcess(Telemetry.DO_NOT_SEND_USAGE_DATA_TO_TABLEAU) as process:
-                with Connection(process.endpoint, str(hyper_path)) as connection:
-                    rows = connection.execute_list_query(query)
+                for member in members:
+                    with Connection(process.endpoint, str(Path(temp_dir) / member)) as connection:
+                        rows.extend(connection.execute_list_query(query))
             with open(output_path, "w", encoding="utf-8", newline="") as handle:
                 writer = csv.writer(handle)
-                writer.writerow(("dt", "pkg_name", "adping_id", "offer_name", "event_name", "event_num"))
+                writer.writerow(("dt", "pkg_name", "adping_id", "offer_name", "second_channel", "event_name", "event_num"))
                 writer.writerows(rows)
             dates = sorted({str(row[0]) for row in rows})
             print(json.dumps({"rows": len(rows), "dates": dates}))

transform_tableau.py

@@ -41,14 +41,17 @@ def event_names(value):
     if not value or value == '-': return []
     return [x.strip().lower() for x in re.split(r'[,，;/；\n]+', value) if x.strip()]
 
-events, covered_dates = {}, set()
+events, covered_scopes = {}, set()
 if events_path:
     with open(events_path, encoding='utf-8-sig', newline='') as f:
         for event_row in csv.DictReader(f):
             dt = re.sub(r'[^0-9]', '', event_row.get('dt', ''))
             if len(dt) == 8: dt = f'{dt[:4]}-{dt[4:6]}-{dt[6:]}'
-            covered_dates.add(dt)
-            key = (dt, event_row.get('pkg_name','').strip(), event_row.get('offer_name','').strip(),
+            channel = event_row.get('second_channel','').lower()
+            event_source = next((s for s in ('mintegral','bigo','kwai') if s in channel), '')
+            if not event_source: continue
+            covered_scopes.add((dt, event_source))
+            key = (dt, event_row.get('pkg_name','').strip(), event_source, event_row.get('offer_name','').strip(),
                    event_row.get('event_name','').strip().lower())
             events[key] = events.get(key, 0) + num(event_row.get('event_num'))
 
@@ -87,10 +90,10 @@ for raw_row in matrix[1:] if has_header else matrix:
         group, shown, adping = '对照组', 'General campaign', ''
     date = cell(raw_row, 'date').strip() or target
     required = event_names(meta.get('kpi',''))
-    if not required or source != 'mintegral' or date not in covered_dates:
+    if not required or (date, source) not in covered_scopes:
         event = None
     else:
-        event = sum(events.get((date, pkg, campaign, name), 0) for name in required)
+        event = sum(events.get((date, pkg, source, campaign, name), 0) for name in required)
     rows.append({'date': cell(raw_row, 'date').strip() or target, 'pkg':pkg, 'geo':meta['geo'], 'source':source,
       'group':group, 'adpingId':adping, 'campaign':shown, 'status':'Active', 'kpi':meta.get('kpi',''),
       'event':event,