feat: add Bigo channel (backfill 08-26~09-01) + daily auto-update with token rotation

.gitignore

@@ -4,3 +4,6 @@ mtg_fetch.py
 mtg_backfill.py
 backfill_cache.json
 backfill.log
+bigo_test.py
+bigo_refresh.py
+bigo_backfill.py

bigo_token.json

@@ -0,0 +1 @@
+{"access_token": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IlA0SkRvOUx1emlkeUVoZlF6SG1qbHRKR0R5bC81UXNKVzBZMDhkTmkrTHM9In0.eyJhdWQiOlsib3BlbmFwaSJdLCJ1c2VyX25hbWUiOiJhZG1pbiIsInNjb3BlIjpbInJlYWQiXSwiZXhwIjoxNzg4OTQyNDA1LCJhdXRob3JpdGllcyI6WyJST0xFX0FETUlOIl0sImp0aSI6IjYzOTA5OTk5LTcwNDYtNDY1OC1iMzI5LTFhZWUyOGRmNTEyYiIsImNsaWVudF9pZCI6IjI2MDMyNTY0NDIyMTIwNDA2MTAwNDgifQ.Wj1KeLNE5DUXBkVhIYcFbVUaXevhy6g7fydyHennSmvKCBFFtZrsrq81EaAKJeST0fJdHVrfTj47wUAa_Z3VEQtCD7n6io50qo9o3VYjY6WetLlNWec--nYFiDoYbVN1vFjaPxi44BxUn32rIZScxQvZEhMxyxthn67KLnmlQi9_1smS5thnKdNUryC7iN_FmD4TwvD7dr4lU4P2VQJXyUDABFP7yXIf_zrhg22pn37zMIpBx9YDMxnhgZbxx-EiJ_kw_r3ANHvgN9Bl0QkXbj4pRLpeB0PLX0pjo9DKYZib4fSTP9Q6S_wq2eDd7_pLWYuxf8gc9bEV2SvnJGxC1w", "token_type": "bearer", "refresh_token": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IlA0SkRvOUx1emlkeUVoZlF6SG1qbHRKR0R5bC81UXNKVzBZMDhkTmkrTHM9In0.eyJhdWQiOlsib3BlbmFwaSJdLCJ1c2VyX25hbWUiOiJhZG1pbiIsInNjb3BlIjpbInJlYWQiXSwiYXRpIjoiNjM5MDk5OTktNzA0Ni00NjU4LWIzMjktMWFlZTI4ZGY1MTJiIiwiZXhwIjoxNzkxNDQ4MDA1LCJhdXRob3JpdGllcyI6WyJST0xFX0FETUlOIl0sImp0aSI6IjYxM2NlYjY2LWY4YmQtNGE0MS1hNDVjLTkyMzFiYTEzZTU2MyIsImNsaWVudF9pZCI6IjI2MDMyNTY0NDIyMTIwNDA2MTAwNDgifQ.N9-uw7G1tukQyVB0fRdvXWRC4s5o2BYzD1aUDX20XpITWJE4HKLICUkkVQFTL-RUy0Kpup14GcbpLXyE_W6vb9CJmvxsHQ8mdoDYe-QxOOeKGad2mmwNLl1UR8aIAXMs7GQE2yUfUC30FQnM8rC5fHXYWj00C-Q-YQ0xRgvK_p3SNpz9QIkoVp7JeGCpNT0cFVgMvroU5QC6KIUPILbHeYlrMYUzSOiD0CBvQkqfTusPXRj_qfW0CeMHFt0rizLaUKlTx1oXZQ9AtgSss28GSNXuPumTtG87LCIngBLWzdnB-jHepjfVhXg4k8hbE8RMhf2BYJ1lXAtE2sHUyDJIIA", "expires_in": 86399, "scope": "read", "jti": "63909999-7046-4658-b329-1aee28df512b"}

mtg_data.json

@@ -2767,6 +2767,22 @@
     "cvr": 0.0,
     "spend": 0.0
   },
+  {
+    "date": "2026-08-26",
+    "channel": "Bigo",
+    "campid": "91552556",
+    "offer_name": "91552556_ZombieWaves_US_GP_CPI_20260826",
+    "product": "Zombie",
+    "pkg": "com.ddup.zombiewaves.zw",
+    "location": "US",
+    "bid_type": "",
+    "impression": 600,
+    "clicks": 133,
+    "conversions": 3,
+    "ctr": 0.2216,
+    "cvr": 0.0225,
+    "spend": 4.61
+  },
   {
     "date": "2026-08-26",
     "channel": "Mintegral",
@@ -2895,6 +2911,22 @@
     "cvr": 1.0,
     "spend": 0.0
   },
+  {
+    "date": "2026-08-27",
+    "channel": "Bigo",
+    "campid": "91552556",
+    "offer_name": "91552556_ZombieWaves_US_GP_CPI_20260826",
+    "product": "Zombie",
+    "pkg": "com.ddup.zombiewaves.zw",
+    "location": "US",
+    "bid_type": "",
+    "impression": 2397,
+    "clicks": 845,
+    "conversions": 29,
+    "ctr": 0.3525,
+    "cvr": 0.0343,
+    "spend": 24.68
+  },
   {
     "date": "2026-08-27",
     "channel": "Mintegral",
@@ -2991,6 +3023,22 @@
     "cvr": 0.0,
     "spend": 0.0
   },
+  {
+    "date": "2026-08-28",
+    "channel": "Bigo",
+    "campid": "91552556",
+    "offer_name": "91552556_ZombieWaves_US_GP_CPI_20260826",
+    "product": "Zombie",
+    "pkg": "com.ddup.zombiewaves.zw",
+    "location": "US",
+    "bid_type": "",
+    "impression": 1143,
+    "clicks": 563,
+    "conversions": 32,
+    "ctr": 0.4925,
+    "cvr": 0.0568,
+    "spend": 24.31
+  },
   {
     "date": "2026-08-28",
     "channel": "Mintegral",
@@ -3119,6 +3167,22 @@
     "cvr": 0.0,
     "spend": 0.0
   },
+  {
+    "date": "2026-08-29",
+    "channel": "Bigo",
+    "campid": "91552556",
+    "offer_name": "91552556_ZombieWaves_US_GP_CPI_20260826",
+    "product": "Zombie",
+    "pkg": "com.ddup.zombiewaves.zw",
+    "location": "US",
+    "bid_type": "",
+    "impression": 776,
+    "clicks": 275,
+    "conversions": 25,
+    "ctr": 0.3543,
+    "cvr": 0.0909,
+    "spend": 23.9
+  },
   {
     "date": "2026-08-29",
     "channel": "Mintegral",
@@ -3215,6 +3279,22 @@
     "cvr": 0.0,
     "spend": 0.0
   },
+  {
+    "date": "2026-08-30",
+    "channel": "Bigo",
+    "campid": "91552556",
+    "offer_name": "91552556_ZombieWaves_US_GP_CPI_20260826",
+    "product": "Zombie",
+    "pkg": "com.ddup.zombiewaves.zw",
+    "location": "US",
+    "bid_type": "",
+    "impression": 1284,
+    "clicks": 368,
+    "conversions": 20,
+    "ctr": 0.2866,
+    "cvr": 0.0543,
+    "spend": 24.73
+  },
   {
     "date": "2026-08-30",
     "channel": "Mintegral",
@@ -3343,6 +3423,22 @@
     "cvr": 0.0,
     "spend": 0.0
   },
+  {
+    "date": "2026-08-31",
+    "channel": "Bigo",
+    "campid": "91552556",
+    "offer_name": "91552556_ZombieWaves_US_GP_CPI_20260826",
+    "product": "Zombie",
+    "pkg": "com.ddup.zombiewaves.zw",
+    "location": "US",
+    "bid_type": "",
+    "impression": 1385,
+    "clicks": 416,
+    "conversions": 12,
+    "ctr": 0.3003,
+    "cvr": 0.0288,
+    "spend": 23.74
+  },
   {
     "date": "2026-08-31",
     "channel": "Mintegral",
@@ -3439,6 +3535,22 @@
     "cvr": 0.0,
     "spend": 0.0
   },
+  {
+    "date": "2026-09-01",
+    "channel": "Bigo",
+    "campid": "91552556",
+    "offer_name": "91552556_ZombieWaves_US_GP_CPI_20260826",
+    "product": "Zombie",
+    "pkg": "com.ddup.zombiewaves.zw",
+    "location": "US",
+    "bid_type": "",
+    "impression": 964,
+    "clicks": 534,
+    "conversions": 5,
+    "ctr": 0.5539,
+    "cvr": 0.0093,
+    "spend": 16.05
+  },
   {
     "date": "2026-09-01",
     "channel": "Mintegral",

update.sh

@@ -98,12 +98,96 @@ existing = []
 if os.path.exists("mtg_data.json"):
     try: existing = json.load(open("mtg_data.json"))
     except: existing = []
-merged = {(r["date"],r["campid"],r["location"]): r for r in existing}
+merged = {(r["date"],r["campid"],r["location"],r.get("channel","")): r for r in existing}
 for r in new_rows:
-    merged[(r["date"],r["campid"],r["location"])] = r
+    merged[(r["date"],r["campid"],r["location"],r.get("channel",""))] = r
-out = sorted(merged.values(), key=lambda x:(x["date"], -x["impression"]))
+out = sorted(merged.values(), key=lambda x:(x["date"], x.get("channel",""), -x["impression"]))
 json.dump(out, open("mtg_data.json","w"), ensure_ascii=False, indent=2)
-print(f"[update] 合并后共 {len(out)} 条，已写入 mtg_data.json")
+print(f"[update] Mintegral 合并后共 {len(out)} 条")
+PYEOF
+
+# ============ Bigo 每日更新 ============
+python3 - <<'PYEOF'
+import json, os, base64, urllib.request, urllib.parse, urllib.error
+from datetime import date, timedelta
+
+CLIENT_ID = "2603256442212040610048"
+SECRET    = "14u0a4anx5a5"
+TOKEN_URL = "https://api.adsbigo.com/oauth/token"
+REPORT_URL = "https://api.adsbigo.com/openapi/report/list"
+STATE = "bigo_token.json"
+
+# 初始 refresh_token（首次运行用；之后从 STATE 文件读取轮转后的）
+INIT_REFRESH = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IlA0SkRvOUx1emlkeUVoZlF6SG1qbHRKR0R5bC81UXNKVzBZMDhkTmkrTHM9In0.eyJhdWQiOlsib3BlbmFwaSJdLCJ1c2VyX25hbWUiOiJhZG1pbiIsInNjb3BlIjpbInJlYWQiXSwiYXRpIjoiMzY2YjMxYTYtYzg2OS00ZDMzLTkyM2EtZjZlZDdjN2QxNzIzIiwiZXhwIjoxNzkxNDQ3NjgxLCJhdXRob3JpdGllcyI6WyJST0xFX0FETUlOIl0sImp0aSI6ImM4MmZiZmNmLTY2YzQtNDE5MS05ZjFkLWViYjVhYTVkNTk1MSIsImNsaWVudF9pZCI6IjI2MDMyNTY0NDIyMTIwNDA2MTAwNDgifQ.VB-ntRjhOvUKIF_BToHOXXT5SCsLKnIyYDGaQwiFICpLHQWJ8Nktyau8FMvbPjIe6DRcYshGCxNRXhP-W1H6R1Nd8Phv-5YW8IUT4ihhHdwarTjpXBU4B9HB9FIatomVr-OubN0RBHWRBy2LArnFIUDvnbD3d6eWr8K0XN8_eGm_5FIT81JhNNVi4jdaeqSMA0cUApQoPyLLZf_CjqyNMlLG3ph4vUm7xD0qM3-T9K7dZCNba2z4FuGZHZ3h_arsFx0nLk0WAIGymuitvY8RNZUbx9EqZdifF__ybtsD-Za2V7KTr37sMXmMnlTaPahBGl2kMfRizgVvwTm0WNw52Q"
+
+def get_refresh():
+    if os.path.exists(STATE):
+        try: return json.load(open(STATE)).get("refresh_token") or INIT_REFRESH
+        except: return INIT_REFRESH
+    return INIT_REFRESH
+
+# 1) 用 refresh_token 换新 access_token（并轮转保存新的 refresh_token）
+auth = base64.b64encode(f"{CLIENT_ID}:{SECRET}".encode()).decode()
+data = urllib.parse.urlencode({"grant_type":"refresh_token","refresh_token":get_refresh()}).encode()
+req = urllib.request.Request(TOKEN_URL, data=data, method="POST",
+    headers={"Content-Type":"application/x-www-form-urlencoded","Authorization":"Basic "+auth})
+try:
+    with urllib.request.urlopen(req, timeout=60) as r:
+        tok = json.loads(r.read().decode())
+except urllib.error.HTTPError as e:
+    print("[update] Bigo token 刷新失败:", e.read().decode()[:200]); raise SystemExit(0)
+
+access = tok.get("access_token")
+if not access:
+    print("[update] Bigo 未拿到 access_token，跳过"); raise SystemExit(0)
+# 保存轮转后的 token
+json.dump(tok, open(STATE,"w"))
+print("[update] Bigo token 已刷新")
+
+# 2) 拉前一天数据
+target = (date.today() - timedelta(days=1)).isoformat()
+camps = json.load(open("campaigns.json"))
+bigo = {c["pkg"]: c for c in camps if c.get("channel")=="Bigo"}
+our_pkgs = set(bigo.keys())
+
+body = {"timezone":8,"startDate":target,"endDate":target,
+    "indicators":["impression","click","ctr","real_result_conversion",
+                  "real_result_conversion_rate","totalCost","campaignName","cpm","cpc"],
+    "breakDowns":["campaignId","country","tgtPkgName"],
+    "aggregateType":2,"pageNo":1,"pageSize":100000}
+req = urllib.request.Request(REPORT_URL, data=json.dumps(body).encode(),
+    headers={"Content-Type":"application/json","Authorization":"Bearer "+access}, method="POST")
+try:
+    with urllib.request.urlopen(req, timeout=120) as r:
+        d = json.loads(r.read().decode())
+except urllib.error.HTTPError as e:
+    print("[update] Bigo 报表请求失败:", e.read().decode()[:200]); raise SystemExit(0)
+if d.get("retcode") != 0:
+    print("[update] Bigo API:", d.get("retmsg")); raise SystemExit(0)
+
+new_rows = []
+for r in d["result"].get("list") or []:
+    pkg = r.get("tgtPkgName")
+    if pkg not in our_pkgs: continue
+    rec = bigo[pkg]
+    new_rows.append({
+        "date": r["aggregateTime"], "channel":"Bigo", "campid": rec["campid"],
+        "offer_name": r.get("campaignName",""), "product": rec["product"], "pkg": pkg,
+        "location": (r.get("country") or "").upper(), "bid_type":"",
+        "impression": int(r.get("impression") or 0), "clicks": int(r.get("click") or 0),
+        "conversions": int(r.get("real_result_conversion") or 0),
+        "ctr": float(r.get("ctr") or 0), "cvr": float(r.get("real_result_conversion_rate") or 0),
+        "spend": round(float(r.get("totalCost") or 0)/100.0, 2),
+    })
+print(f"[update] Bigo 新增匹配 {len(new_rows)} 条 ({target})")
+
+existing = json.load(open("mtg_data.json")) if os.path.exists("mtg_data.json") else []
+merged = {(r["date"],r["campid"],r["location"],r.get("channel","")): r for r in existing}
+for r in new_rows:
+    merged[(r["date"],r["campid"],r["location"],r["channel"])] = r
+out = sorted(merged.values(), key=lambda x:(x["date"], x.get("channel",""), -x["impression"]))
+json.dump(out, open("mtg_data.json","w"), ensure_ascii=False, indent=2)
+print(f"[update] Bigo 合并后共 {len(out)} 条，已写入 mtg_data.json")
 PYEOF
 
 echo "[update] $(date '+%Y-%m-%d %H:%M:%S') 完成"