From 95e4849934ecc7dd415e06c9250418958d330bd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=BA=90?= Date: Wed, 9 Sep 2026 20:52:02 +0800 Subject: [PATCH] chore: switch to push-based sync (ads-center), remove server-side update.sh --- .gitignore | 5 ++ schedule.conf | 1 - update.sh | 245 -------------------------------------------------- 3 files changed, 5 insertions(+), 246 deletions(-) delete mode 100644 schedule.conf delete mode 100755 update.sh diff --git a/.gitignore b/.gitignore index 0c0468f..d7629fd 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,8 @@ mediago_probe.py mediago_backfill.py mediago_matches.json mediago_backfill.log +db_probe.py +db_fetch.py +db_fetch_built.py +mtg_data_new.json +adscenter_daily_sync.py diff --git a/schedule.conf b/schedule.conf deleted file mode 100644 index a45fd52..0000000 --- a/schedule.conf +++ /dev/null @@ -1 +0,0 @@ -24 diff --git a/update.sh b/update.sh deleted file mode 100755 index eae1562..0000000 --- a/update.sh +++ /dev/null @@ -1,245 +0,0 @@ -#!/bin/bash -# Moboost 报告 - 每日自动更新脚本 -# 服务器每天执行一次（schedule.conf=24）。 -# 逻辑：拉取"前一天"的 Mintegral 数据，增量合并进 mtg_data.json（去重，不覆盖历史）。 -# 工作目录 = 仓库根目录（同目录下有 mtg_data.json / campaigns.json）。 - -set -e -cd "$(dirname "$0")" - -python3 - <<'PYEOF' -import hashlib, time, json, os, urllib.request, urllib.parse, urllib.error -from datetime import date, timedelta - -# ---- Mintegral 认证（access-key=平台"Success key"，token secret=平台"API key"）---- -API_KEY = "22793ef96b076fbdefa7c1266d4ebccb" -SKEY = "c47267b67f9e4231325efe4283044721" -BASE = "https://ss-api.mintegral.com/api/v2/reports/data" - -def headers(): - ts = str(int(time.time())) - token = hashlib.md5((SKEY + hashlib.md5(ts.encode()).hexdigest()).encode()).hexdigest() - return {"access-key": API_KEY, "token": token, "timestamp": ts} - -def req(params): - url = BASE + "?" + urllib.parse.urlencode(params) - for attempt in range(6): - try: - r = urllib.request.Request(url, headers=headers()) - with urllib.request.urlopen(r, timeout=90) as resp: - return resp.status, resp.headers.get("Content-Type",""), resp.read() - except urllib.error.HTTPError as e: - return e.code, e.headers.get("Content-Type",""), e.read() - except Exception as e: - time.sleep(5*(attempt+1)) - raise RuntimeError("网络请求失败") - -# ---- 目标 campid（Mintegral 渠道），优先从 campaigns.json 读，读不到则用内置列表 ---- -FALLBACK_IDS = ["91530535","91530534","91530536","91530547","91530548", - "91530549","91530550","91530537","91530538","91551606", - "91534913","91533906"] -try: - camps = json.load(open("campaigns.json")) - mtg = {c["campid"]: c for c in camps if c.get("channel")=="Mintegral"} -except Exception: - mtg = {} -if not mtg: - mtg = {cid: {"campid":cid,"product":"","pkg":""} for cid in FALLBACK_IDS} -mtg_ids = list(mtg.keys()) - -# ---- 目标日期：前一天（UTC+8）---- -target = (date.today() - timedelta(days=1)).isoformat() -print(f"[update] 拉取 {target} 的 Mintegral 数据 ...") - -base = {"timezone":"+8","start_time":target,"end_time":target, - "dimension_option":"Offer,Location","time_granularity":"daily"} - -# type=1 生成 -for attempt in range(20): - p = dict(base); p["type"]="1" - s,ct,b = req(p) - try: code = json.loads(b.decode("utf-8","replace")).get("code") - except: code = None - if code == 200: break - time.sleep(6) -else: - print("[update] 数据生成超时，退出"); raise SystemExit(0) - -# type=2 下载 -p = dict(base); p["type"]="2" -s,ct,b = req(p) -if "octet-stream" not in ct: - print("[update] 下载失败:", b.decode("utf-8","replace")[:200]); raise SystemExit(0) - -lines = b.decode("utf-8","replace").splitlines() -header = lines[0].split("\t"); col = {n:i for i,n in enumerate(header)} -new_rows = [] -for line in lines[1:]: - f = line.split("\t") - if len(f) < len(header): continue - offer = f[col["Offer Name"]] - matched = next((cid for cid in mtg_ids if cid in offer), None) - if not matched: continue - rec = mtg[matched] - d = f[col["Date"]] - new_rows.append({ - "date": f"{d[:4]}-{d[4:6]}-{d[6:]}", - "channel":"Mintegral","campid":matched,"offer_name":offer, - "product":rec.get("product",""),"pkg":rec.get("pkg",""), - "location":f[col["Location"]],"bid_type":"", - "impression":int(f[col["Impression"]]),"clicks":int(f[col["Click"]]), - "conversions":int(f[col["Conversion"]]), - "ctr":float(f[col["Ctr"]]),"cvr":float(f[col["Cvr"]]),"spend":float(f[col["Spend"]]), - }) -print(f"[update] 新增匹配 {len(new_rows)} 条") - -# ---- 增量合并（按 date+campid+location 去重；新数据覆盖同键，历史保留）---- -existing = [] -if os.path.exists("mtg_data.json"): - try: existing = json.load(open("mtg_data.json")) - except: existing = [] -merged = {(r["date"],r["campid"],r["location"],r.get("channel","")): r for r in existing} -for r in new_rows: - merged[(r["date"],r["campid"],r["location"],r.get("channel",""))] = r -out = sorted(merged.values(), key=lambda x:(x["date"], x.get("channel",""), -x["impression"])) -json.dump(out, open("mtg_data.json","w"), ensure_ascii=False, indent=2) -print(f"[update] Mintegral 合并后共 {len(out)} 条") -PYEOF - -# ============ MediaGo 每日更新 ============ -python3 - <<'PYEOF' -import json, os, time, urllib.request, urllib.parse, urllib.error -from datetime import date, timedelta - -AUTH_URL = "https://api.mediago.com/manage/api/v1/access_token" -COST_URL = "https://api.mediago.com/report/api/v1/cost/daily" -TOKENS = [ - "YWNjb3VundF8zmu5wjlv1vcRwbpdqaCugRvQKslUg9kQH4XnrNXefCbtGDcC", - "YWNjb3VundF8XrY8TwaXlX6pJDLnmfqmGgZT654Hh6rBRH9Cj9GzPXpO48BW", - "YWNjb3VundF89iFLKQQHlvwxnhVbWvchTAz009Zcjw7jTkOPzCDC3oLK0RGO", - "YWNjb3VundF815Lb3KpFWB8pnajnDZwL8Aw0YHpquAqUPWl78NKSaSigRBsY", - "YWNjb3VundF8SKbXp3ceGoHmS9SyMBleK2bIglGThBhoHly8yPoYOP8jZtww", - "YWNjb3VundF80DXnMfSwD4LWsV1hVVSA2j5FZQsWMSTQbJ6lTZIUhhrQZ9c6", - "YWNjb3VundF8gK0LuXHCZkNyC4R14Adayu4j1xnNJMzxvu1hokiwUPNOPJxS", - "YWNjb3VundF82rld59cRZIn53Uf0NhkWdcGIdNzOSRCZrkyj81xemBTQrOmi", - "YWNjb3VundF86E54f6seADTvrVjgeY3mrFNYsTn4jKVH5G5UyixyKkJi7T1S", - "YWNjb3VundF8MCmSFzV7LO4VRrxpJME8YrZCb6MyLp5KhRz4Q3crCjsDxPvI", - "YWNjb3VundF8tKh2Lye5ehjWsrSQUBTq6L4caohAKvG71qJVSOkMKDCKcKGu", - "YWNjb3VundF8xZYwIg4AFeVlTmrjBovgjxCSsHxYenyTTNbnNo1a0gIgs75V", - "YWNjb3VundF8QcNhks2O2ObIVmVXoIr1fuPPGTQ34YsaE0idvLVqTM5bSjbr", - "YWNjb3VundF8JtqahIBkypktae5H98q3PYjFXYPzqukmwY2wIOYqLSrYg0JQ", - "YWNjb3VundF8x4GsJSnSBns6y5tVGZRlDdC0w40k1ApVlj8v9cme64KNHmKq", - "YWNjb3VundF8GbE7ycPaH0fSozZ80v38jC00qdJB5YJRJJeqdy4DomjKG2ns", - "YWNjb3VundF83KP9txx57PNlElIa61g7vi7QQ1BTw4nlyPn6Tw8P4bXfX5IT", - "YWNjb3VundF8CDELpJV0axNw1AH4Pb7N1yfoe9HzyL0u9XQEnW3tgXVj1rbB", - "YWNjb3VundF8AzmnXydKyjDthsIx7BaYn6xof2lRh4pdJRTJT8qmupaLi7pX", - "YWNjb3VundF8PwLbajYpTYLe2dVhm1EX8seXxsPJlbpkxT8LhSONHhffW48Z", - "YWNjb3VundF8OGELq8i2wxXDw2kniuY9wp9nRz6xvoDU5jfFo2vsiUeoO2MV", - "YWNjb3VundF8u5aQbeM0LrTQcxc3t4DcbvZp1kBoDueP0q0rMjDPvLefG3WR", -] - -camps = json.load(open("campaigns.json")) -mg = {c["campid"]: c for c in camps if c.get("channel") == "MediaGo"} -mg_ids = list(mg.keys()) -if not mg: - print("[update] MediaGo 无目标 campid，跳过"); raise SystemExit(0) -PKGS = ",".join(sorted({c["pkg"] for c in mg.values()})) -target = (date.today() - timedelta(days=1)).isoformat() - -def get_token(api_token): - req = urllib.request.Request(AUTH_URL, data=json.dumps({"api_token": api_token}).encode(), - headers={"Content-Type": "application/json"}, method="POST") - with urllib.request.urlopen(req, timeout=40) as r: - return json.loads(r.read().decode())["data"]["AccessToken"] - -def cost_daily(tok): - p = [("dimension","campaign"),("tz","UTC+8"),("app_ids",PKGS), - ("start_date",target),("end_date",target),("page_idx",0),("page_size",100)] - req = urllib.request.Request(COST_URL + "?" + urllib.parse.urlencode(p), - headers={"Authorization":"Bearer "+tok}, method="GET") - with urllib.request.urlopen(req, timeout=60) as r: - d = json.loads(r.read().decode()) - return d.get("data",{}).get("list") or [] if d.get("code")==0 else [] - -new_rows = [] -for api_token in TOKENS: - try: - tok = get_token(api_token) - lst = cost_daily(tok) - except Exception: - continue - for r in lst: - cn = r.get("campaign_name","") - cid = next((c for c in mg_ids if cn.startswith(c) or c in cn), None) - if not cid: continue - imp=int(r.get("imp") or 0); clk=int(r.get("click") or 0); cv=int(r.get("cv") or 0); sp=float(r.get("spend") or 0) - if imp==0 and clk==0 and sp==0: continue - rec = mg[cid] - new_rows.append({ - "date": r.get("date"), "channel":"MediaGo", "campid":cid, - "offer_name":cn, "product":rec["product"], "pkg":rec["pkg"], - "location":(r.get("geo") or "").upper(), "bid_type":"", - "impression":imp, "clicks":clk, "conversions":cv, - "ctr":round(clk/imp,5) if imp else 0.0, "cvr":round(cv/clk,5) if clk else 0.0, - "spend":round(sp,2), - }) - time.sleep(0.3) -print(f"[update] MediaGo 新增匹配 {len(new_rows)} 条 ({target})") - -existing = json.load(open("mtg_data.json")) if os.path.exists("mtg_data.json") else [] -merged = {(r["date"],r["campid"],r["location"],r.get("channel","")): r for r in existing} -for r in new_rows: - merged[(r["date"],r["campid"],r["location"],r["channel"])] = r -out = sorted(merged.values(), key=lambda x:(x["date"], x.get("channel",""), -x["impression"])) -json.dump(out, open("mtg_data.json","w"), ensure_ascii=False, indent=2) -print(f"[update] MediaGo 合并后共 {len(out)} 条，已写入 mtg_data.json") -PYEOF - -# ============ 重新生成渠道投放矩阵状态（matrix_overrides.json）============ -python3 - <<'PYEOF' -import json, os -from collections import defaultdict - -if not os.path.exists("mtg_data.json"): - raise SystemExit(0) -d = json.load(open("mtg_data.json")) -orders = json.load(open("orders.json")) if os.path.exists("orders.json") else [] - -# 产品状态：按 pkg -> status（"a / b" 合并形式拆开） -pkg_status = {} -for o in orders: - for p in str(o.get("pkg","")).replace(" ","").split("/"): - if p: pkg_status[p] = o.get("status","Pause") - -# 聚合每个 (channel, pkg) 的每日展示 -g = defaultdict(lambda: defaultdict(int)) -for r in d: - g[(r["channel"], r["pkg"])][r["date"]] += r.get("impression", 0) -alldates = sorted(set(r["date"] for r in d)) -if not alldates: - raise SystemExit(0) -lastday = alldates[-1] -last3 = alldates[-3:] - -overrides = {} -for (ch, pkg), days in g.items(): - prod_status = pkg_status.get(pkg, "Pause") - # 第一优先级：产品订单状态为 Pause -> 直接 Pause - if prod_status != "Active": - overrides[f"{ch}|{pkg}"] = "Pause" - continue - # 产品在投 -> 看数据判定该渠道 - total = sum(days.values()) - lastval = days.get(lastday, 0) - recent3 = sum(days.get(dt, 0) for dt in last3) - if total == 0: - overrides[f"{ch}|{pkg}"] = "none" # 零曝光=从未投放（前端留空） - elif lastval >= 10 and recent3 >= 30: - overrides[f"{ch}|{pkg}"] = "Active" # 末日两位数以上且近期持续 - else: - overrides[f"{ch}|{pkg}"] = "Pause" # 掉到个位数/断更=已停 -json.dump(overrides, open("matrix_overrides.json","w"), ensure_ascii=False, indent=2) -print(f"[update] 矩阵状态已刷新，共 {len(overrides)} 个渠道×产品组合") -PYEOF - -echo "[update] $(date '+%Y-%m-%d %H:%M:%S') 完成"