chore: switch to push-based sync (ads-center), remove server-side update.sh
This commit is contained in:
王源
@@ -11,3 +11,8 @@ mediago_probe.py
|
||||
mediago_backfill.py
|
||||
mediago_matches.json
|
||||
mediago_backfill.log
|
||||
db_probe.py
|
||||
db_fetch.py
|
||||
db_fetch_built.py
|
||||
mtg_data_new.json
|
||||
adscenter_daily_sync.py
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
24
|
||||
@@ -1,245 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Moboost 报告 - 每日自动更新脚本
|
||||
# 服务器每天执行一次（schedule.conf=24）。
|
||||
# 逻辑：拉取"前一天"的 Mintegral 数据，增量合并进 mtg_data.json（去重，不覆盖历史）。
|
||||
# 工作目录 = 仓库根目录（同目录下有 mtg_data.json / campaigns.json）。
|
||||
|
||||
set -e
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
python3 - <<'PYEOF'
|
||||
import hashlib, time, json, os, urllib.request, urllib.parse, urllib.error
|
||||
from datetime import date, timedelta
|
||||
|
||||
# ---- Mintegral 认证（access-key=平台"Success key"，token secret=平台"API key"）----
|
||||
API_KEY = "22793ef96b076fbdefa7c1266d4ebccb"
|
||||
SKEY = "c47267b67f9e4231325efe4283044721"
|
||||
BASE = "https://ss-api.mintegral.com/api/v2/reports/data"
|
||||
|
||||
def headers():
|
||||
ts = str(int(time.time()))
|
||||
token = hashlib.md5((SKEY + hashlib.md5(ts.encode()).hexdigest()).encode()).hexdigest()
|
||||
return {"access-key": API_KEY, "token": token, "timestamp": ts}
|
||||
|
||||
def req(params):
|
||||
url = BASE + "?" + urllib.parse.urlencode(params)
|
||||
for attempt in range(6):
|
||||
try:
|
||||
r = urllib.request.Request(url, headers=headers())
|
||||
with urllib.request.urlopen(r, timeout=90) as resp:
|
||||
return resp.status, resp.headers.get("Content-Type",""), resp.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.headers.get("Content-Type",""), e.read()
|
||||
except Exception as e:
|
||||
time.sleep(5*(attempt+1))
|
||||
raise RuntimeError("网络请求失败")
|
||||
|
||||
# ---- 目标 campid（Mintegral 渠道），优先从 campaigns.json 读，读不到则用内置列表 ----
|
||||
FALLBACK_IDS = ["91530535","91530534","91530536","91530547","91530548",
|
||||
"91530549","91530550","91530537","91530538","91551606",
|
||||
"91534913","91533906"]
|
||||
try:
|
||||
camps = json.load(open("campaigns.json"))
|
||||
mtg = {c["campid"]: c for c in camps if c.get("channel")=="Mintegral"}
|
||||
except Exception:
|
||||
mtg = {}
|
||||
if not mtg:
|
||||
mtg = {cid: {"campid":cid,"product":"","pkg":""} for cid in FALLBACK_IDS}
|
||||
mtg_ids = list(mtg.keys())
|
||||
|
||||
# ---- 目标日期：前一天（UTC+8）----
|
||||
target = (date.today() - timedelta(days=1)).isoformat()
|
||||
print(f"[update] 拉取 {target} 的 Mintegral 数据 ...")
|
||||
|
||||
base = {"timezone":"+8","start_time":target,"end_time":target,
|
||||
"dimension_option":"Offer,Location","time_granularity":"daily"}
|
||||
|
||||
# type=1 生成
|
||||
for attempt in range(20):
|
||||
p = dict(base); p["type"]="1"
|
||||
s,ct,b = req(p)
|
||||
try: code = json.loads(b.decode("utf-8","replace")).get("code")
|
||||
except: code = None
|
||||
if code == 200: break
|
||||
time.sleep(6)
|
||||
else:
|
||||
print("[update] 数据生成超时，退出"); raise SystemExit(0)
|
||||
|
||||
# type=2 下载
|
||||
p = dict(base); p["type"]="2"
|
||||
s,ct,b = req(p)
|
||||
if "octet-stream" not in ct:
|
||||
print("[update] 下载失败:", b.decode("utf-8","replace")[:200]); raise SystemExit(0)
|
||||
|
||||
lines = b.decode("utf-8","replace").splitlines()
|
||||
header = lines[0].split("\t"); col = {n:i for i,n in enumerate(header)}
|
||||
new_rows = []
|
||||
for line in lines[1:]:
|
||||
f = line.split("\t")
|
||||
if len(f) < len(header): continue
|
||||
offer = f[col["Offer Name"]]
|
||||
matched = next((cid for cid in mtg_ids if cid in offer), None)
|
||||
if not matched: continue
|
||||
rec = mtg[matched]
|
||||
d = f[col["Date"]]
|
||||
new_rows.append({
|
||||
"date": f"{d[:4]}-{d[4:6]}-{d[6:]}",
|
||||
"channel":"Mintegral","campid":matched,"offer_name":offer,
|
||||
"product":rec.get("product",""),"pkg":rec.get("pkg",""),
|
||||
"location":f[col["Location"]],"bid_type":"",
|
||||
"impression":int(f[col["Impression"]]),"clicks":int(f[col["Click"]]),
|
||||
"conversions":int(f[col["Conversion"]]),
|
||||
"ctr":float(f[col["Ctr"]]),"cvr":float(f[col["Cvr"]]),"spend":float(f[col["Spend"]]),
|
||||
})
|
||||
print(f"[update] 新增匹配 {len(new_rows)} 条")
|
||||
|
||||
# ---- 增量合并（按 date+campid+location 去重；新数据覆盖同键，历史保留）----
|
||||
existing = []
|
||||
if os.path.exists("mtg_data.json"):
|
||||
try: existing = json.load(open("mtg_data.json"))
|
||||
except: existing = []
|
||||
merged = {(r["date"],r["campid"],r["location"],r.get("channel","")): r for r in existing}
|
||||
for r in new_rows:
|
||||
merged[(r["date"],r["campid"],r["location"],r.get("channel",""))] = r
|
||||
out = sorted(merged.values(), key=lambda x:(x["date"], x.get("channel",""), -x["impression"]))
|
||||
json.dump(out, open("mtg_data.json","w"), ensure_ascii=False, indent=2)
|
||||
print(f"[update] Mintegral 合并后共 {len(out)} 条")
|
||||
PYEOF
|
||||
|
||||
# ============ MediaGo 每日更新 ============
|
||||
python3 - <<'PYEOF'
|
||||
import json, os, time, urllib.request, urllib.parse, urllib.error
|
||||
from datetime import date, timedelta
|
||||
|
||||
AUTH_URL = "https://api.mediago.com/manage/api/v1/access_token"
|
||||
COST_URL = "https://api.mediago.com/report/api/v1/cost/daily"
|
||||
TOKENS = [
|
||||
"YWNjb3VundF8zmu5wjlv1vcRwbpdqaCugRvQKslUg9kQH4XnrNXefCbtGDcC",
|
||||
"YWNjb3VundF8XrY8TwaXlX6pJDLnmfqmGgZT654Hh6rBRH9Cj9GzPXpO48BW",
|
||||
"YWNjb3VundF89iFLKQQHlvwxnhVbWvchTAz009Zcjw7jTkOPzCDC3oLK0RGO",
|
||||
"YWNjb3VundF815Lb3KpFWB8pnajnDZwL8Aw0YHpquAqUPWl78NKSaSigRBsY",
|
||||
"YWNjb3VundF8SKbXp3ceGoHmS9SyMBleK2bIglGThBhoHly8yPoYOP8jZtww",
|
||||
"YWNjb3VundF80DXnMfSwD4LWsV1hVVSA2j5FZQsWMSTQbJ6lTZIUhhrQZ9c6",
|
||||
"YWNjb3VundF8gK0LuXHCZkNyC4R14Adayu4j1xnNJMzxvu1hokiwUPNOPJxS",
|
||||
"YWNjb3VundF82rld59cRZIn53Uf0NhkWdcGIdNzOSRCZrkyj81xemBTQrOmi",
|
||||
"YWNjb3VundF86E54f6seADTvrVjgeY3mrFNYsTn4jKVH5G5UyixyKkJi7T1S",
|
||||
"YWNjb3VundF8MCmSFzV7LO4VRrxpJME8YrZCb6MyLp5KhRz4Q3crCjsDxPvI",
|
||||
"YWNjb3VundF8tKh2Lye5ehjWsrSQUBTq6L4caohAKvG71qJVSOkMKDCKcKGu",
|
||||
"YWNjb3VundF8xZYwIg4AFeVlTmrjBovgjxCSsHxYenyTTNbnNo1a0gIgs75V",
|
||||
"YWNjb3VundF8QcNhks2O2ObIVmVXoIr1fuPPGTQ34YsaE0idvLVqTM5bSjbr",
|
||||
"YWNjb3VundF8JtqahIBkypktae5H98q3PYjFXYPzqukmwY2wIOYqLSrYg0JQ",
|
||||
"YWNjb3VundF8x4GsJSnSBns6y5tVGZRlDdC0w40k1ApVlj8v9cme64KNHmKq",
|
||||
"YWNjb3VundF8GbE7ycPaH0fSozZ80v38jC00qdJB5YJRJJeqdy4DomjKG2ns",
|
||||
"YWNjb3VundF83KP9txx57PNlElIa61g7vi7QQ1BTw4nlyPn6Tw8P4bXfX5IT",
|
||||
"YWNjb3VundF8CDELpJV0axNw1AH4Pb7N1yfoe9HzyL0u9XQEnW3tgXVj1rbB",
|
||||
"YWNjb3VundF8AzmnXydKyjDthsIx7BaYn6xof2lRh4pdJRTJT8qmupaLi7pX",
|
||||
"YWNjb3VundF8PwLbajYpTYLe2dVhm1EX8seXxsPJlbpkxT8LhSONHhffW48Z",
|
||||
"YWNjb3VundF8OGELq8i2wxXDw2kniuY9wp9nRz6xvoDU5jfFo2vsiUeoO2MV",
|
||||
"YWNjb3VundF8u5aQbeM0LrTQcxc3t4DcbvZp1kBoDueP0q0rMjDPvLefG3WR",
|
||||
]
|
||||
|
||||
camps = json.load(open("campaigns.json"))
|
||||
mg = {c["campid"]: c for c in camps if c.get("channel") == "MediaGo"}
|
||||
mg_ids = list(mg.keys())
|
||||
if not mg:
|
||||
print("[update] MediaGo 无目标 campid，跳过"); raise SystemExit(0)
|
||||
PKGS = ",".join(sorted({c["pkg"] for c in mg.values()}))
|
||||
target = (date.today() - timedelta(days=1)).isoformat()
|
||||
|
||||
def get_token(api_token):
|
||||
req = urllib.request.Request(AUTH_URL, data=json.dumps({"api_token": api_token}).encode(),
|
||||
headers={"Content-Type": "application/json"}, method="POST")
|
||||
with urllib.request.urlopen(req, timeout=40) as r:
|
||||
return json.loads(r.read().decode())["data"]["AccessToken"]
|
||||
|
||||
def cost_daily(tok):
|
||||
p = [("dimension","campaign"),("tz","UTC+8"),("app_ids",PKGS),
|
||||
("start_date",target),("end_date",target),("page_idx",0),("page_size",100)]
|
||||
req = urllib.request.Request(COST_URL + "?" + urllib.parse.urlencode(p),
|
||||
headers={"Authorization":"Bearer "+tok}, method="GET")
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
d = json.loads(r.read().decode())
|
||||
return d.get("data",{}).get("list") or [] if d.get("code")==0 else []
|
||||
|
||||
new_rows = []
|
||||
for api_token in TOKENS:
|
||||
try:
|
||||
tok = get_token(api_token)
|
||||
lst = cost_daily(tok)
|
||||
except Exception:
|
||||
continue
|
||||
for r in lst:
|
||||
cn = r.get("campaign_name","")
|
||||
cid = next((c for c in mg_ids if cn.startswith(c) or c in cn), None)
|
||||
if not cid: continue
|
||||
imp=int(r.get("imp") or 0); clk=int(r.get("click") or 0); cv=int(r.get("cv") or 0); sp=float(r.get("spend") or 0)
|
||||
if imp==0 and clk==0 and sp==0: continue
|
||||
rec = mg[cid]
|
||||
new_rows.append({
|
||||
"date": r.get("date"), "channel":"MediaGo", "campid":cid,
|
||||
"offer_name":cn, "product":rec["product"], "pkg":rec["pkg"],
|
||||
"location":(r.get("geo") or "").upper(), "bid_type":"",
|
||||
"impression":imp, "clicks":clk, "conversions":cv,
|
||||
"ctr":round(clk/imp,5) if imp else 0.0, "cvr":round(cv/clk,5) if clk else 0.0,
|
||||
"spend":round(sp,2),
|
||||
})
|
||||
time.sleep(0.3)
|
||||
print(f"[update] MediaGo 新增匹配 {len(new_rows)} 条 ({target})")
|
||||
|
||||
existing = json.load(open("mtg_data.json")) if os.path.exists("mtg_data.json") else []
|
||||
merged = {(r["date"],r["campid"],r["location"],r.get("channel","")): r for r in existing}
|
||||
for r in new_rows:
|
||||
merged[(r["date"],r["campid"],r["location"],r["channel"])] = r
|
||||
out = sorted(merged.values(), key=lambda x:(x["date"], x.get("channel",""), -x["impression"]))
|
||||
json.dump(out, open("mtg_data.json","w"), ensure_ascii=False, indent=2)
|
||||
print(f"[update] MediaGo 合并后共 {len(out)} 条，已写入 mtg_data.json")
|
||||
PYEOF
|
||||
|
||||
# ============ 重新生成渠道投放矩阵状态（matrix_overrides.json）============
|
||||
python3 - <<'PYEOF'
|
||||
import json, os
|
||||
from collections import defaultdict
|
||||
|
||||
if not os.path.exists("mtg_data.json"):
|
||||
raise SystemExit(0)
|
||||
d = json.load(open("mtg_data.json"))
|
||||
orders = json.load(open("orders.json")) if os.path.exists("orders.json") else []
|
||||
|
||||
# 产品状态：按 pkg -> status（"a / b" 合并形式拆开）
|
||||
pkg_status = {}
|
||||
for o in orders:
|
||||
for p in str(o.get("pkg","")).replace(" ","").split("/"):
|
||||
if p: pkg_status[p] = o.get("status","Pause")
|
||||
|
||||
# 聚合每个 (channel, pkg) 的每日展示
|
||||
g = defaultdict(lambda: defaultdict(int))
|
||||
for r in d:
|
||||
g[(r["channel"], r["pkg"])][r["date"]] += r.get("impression", 0)
|
||||
alldates = sorted(set(r["date"] for r in d))
|
||||
if not alldates:
|
||||
raise SystemExit(0)
|
||||
lastday = alldates[-1]
|
||||
last3 = alldates[-3:]
|
||||
|
||||
overrides = {}
|
||||
for (ch, pkg), days in g.items():
|
||||
prod_status = pkg_status.get(pkg, "Pause")
|
||||
# 第一优先级：产品订单状态为 Pause -> 直接 Pause
|
||||
if prod_status != "Active":
|
||||
overrides[f"{ch}|{pkg}"] = "Pause"
|
||||
continue
|
||||
# 产品在投 -> 看数据判定该渠道
|
||||
total = sum(days.values())
|
||||
lastval = days.get(lastday, 0)
|
||||
recent3 = sum(days.get(dt, 0) for dt in last3)
|
||||
if total == 0:
|
||||
overrides[f"{ch}|{pkg}"] = "none" # 零曝光=从未投放（前端留空）
|
||||
elif lastval >= 10 and recent3 >= 30:
|
||||
overrides[f"{ch}|{pkg}"] = "Active" # 末日两位数以上且近期持续
|
||||
else:
|
||||
overrides[f"{ch}|{pkg}"] = "Pause" # 掉到个位数/断更=已停
|
||||
json.dump(overrides, open("matrix_overrides.json","w"), ensure_ascii=False, indent=2)
|
||||
print(f"[update] 矩阵状态已刷新，共 {len(overrides)} 个渠道×产品组合")
|
||||
PYEOF
|
||||
|
||||
echo "[update] $(date '+%Y-%m-%d %H:%M:%S') 完成"
|
||||
Reference in New Issue
Block a user