mediabuy-wh/build_data.py

#!/usr/bin/env python3
# 从 xlsx 重算 data.json。
#   平台消耗统计: B=Date, C=Adpingid, D=包名, E=地区, I=花费
#   投手:        B=Adpingid, D=pkg, L=Media Buyer投放人
# 规则：@某某 == 某某；未填投放人的 Adpingid 丢弃；
#       exclude.json 的 packages（按包名）与 adpings（按 Adpingid）永久剔除，均为精确匹配。
# 每个 package 明细：Geo 后缀 = 该 package 花费最高的地区；Adpingid = 该地区内花费最高的 Adping。
import openpyxl, collections, json, datetime, os, sys, glob

HERE = os.path.dirname(os.path.abspath(__file__))
# 源文件按月滚动（2608 -> 2609 -> ...）。全部读进来合并：
# 同一天出现在多个文件时，以文件名较新的为准（源表会修订历史数据）。
SRC_FILES = sorted(glob.glob(os.path.join(HERE, '*每日数据-武汉萌奇熊.xlsx')))
if not SRC_FILES:
    sys.exit('ERROR: 仓库里找不到 *每日数据-武汉萌奇熊.xlsx')
_cfg = json.load(open(os.path.join(HERE, 'exclude.json')))
EXCL      = set(_cfg['packages'])            # 按包名剔除
EXCL_ADP  = set(_cfg.get('adpings', {}))     # 按 Adpingid 剔除
NON_BUYER = {'GP下架'}
WINDOW_DAYS = 3

def key(v):
    if isinstance(v, float) and v.is_integer():
        return str(int(v))
    return str(v).strip()

books = [(f, openpyxl.load_workbook(f, data_only=True)) for f in SRC_FILES]

# --- 投手表：取最新一个带「投手」sheet 的文件（月度文件可能不含该表）---
roster_file, roster = None, None
for f, wb_ in reversed(books):
    if '投手' in wb_.sheetnames:
        roster_file, roster = f, wb_['投手']
        break
if roster is None:
    sys.exit('ERROR: 所有源文件都没有「投手」sheet，无法归属投放人')

amap = {}
dropped_ids = set(EXCL_ADP)   # 先纳入按 Adpingid 剔除的，再补上按包名命中的
for r in roster.iter_rows(min_row=2, values_only=True):
    adp, pkg, buyer, status = r[1], r[3], r[11], r[0]
    if adp is None or str(status).strip() == '示例':
        continue
    k = key(adp)
    if k in EXCL_ADP:
        continue
    if pkg is not None and str(pkg).strip() in EXCL:   # 精确匹配，避免误伤 in.swiggy.android
        dropped_ids.add(k)
        continue
    b = str(buyer).strip() if buyer is not None else ''
    if b in ('', 'None') or b.lstrip('@').strip() in NON_BUYER:
        continue
    amap[k] = b.lstrip('@').strip()

owned = collections.Counter(amap.values())   # 每位投手名下的 Adpingid 数（已扣除排除包名）

# --- 平台消耗统计：逐文件读取，按日期归组，新文件覆盖同一天的旧数据 ---
by_date = {}
for f, wb_ in books:
    if '平台消耗统计' not in wb_.sheetnames:
        continue
    day = collections.defaultdict(list)
    for r in wb_['平台消耗统计'].iter_rows(min_row=2, values_only=True):
        d, adp, pkg, geo, spend = r[1], r[2], r[3], r[4], r[8]
        if d is None or adp is None or not isinstance(d, datetime.datetime):
            continue
        try:
            s = float(spend) if spend is not None else 0.0
        except (TypeError, ValueError):
            s = 0.0
        day[d.date()].append((d.date(), key(adp), (str(pkg).strip() if pkg else '(未知)'),
                              (str(geo).strip() if geo else ''), s))
    by_date.update(day)          # books 已按文件名升序，后者覆盖前者

rows = [r for day_rows in by_date.values() for r in day_rows]
if not rows:
    sys.exit('ERROR: 平台消耗统计 无有效数据行')

end = max(r[0] for r in rows)
start = end - datetime.timedelta(days=WINDOW_DAYS - 1)

pkg_spend = collections.defaultdict(float)   # (buyer, pkg)             -> spend
geo_spend = collections.defaultdict(float)   # (buyer, pkg, geo)        -> spend
ga_spend  = collections.defaultdict(float)   # (buyer, pkg, geo, adping) -> spend
ids = collections.defaultdict(set)
unmatched = collections.defaultdict(float)
excluded_spend, excluded_rows = 0.0, 0

for d, adp, pkg, geo, s in rows:
    if not (start <= d <= end):
        continue
    if pkg in EXCL or adp in dropped_ids:
        excluded_spend += s; excluded_rows += 1
        continue
    b = amap.get(adp)
    if b is None:
        unmatched[(adp, pkg)] += s
        continue
    pkg_spend[(b, pkg)] += s
    geo_spend[(b, pkg, geo)] += s
    ga_spend[(b, pkg, geo, adp)] += s
    ids[b].add(adp)

by_buyer = collections.defaultdict(dict)
for (b, pkg), v in pkg_spend.items():
    by_buyer[b][pkg] = v

out = []
for b, pk in by_buyer.items():
    pkgs = []
    for pkg, v in sorted(pk.items(), key=lambda x: -x[1])[:5]:
        # 花费最高的地区，再取该地区内花费最高的 Adpingid
        top_geo = max((g for (bb, pp, g) in geo_spend if bb == b and pp == pkg),
                      key=lambda g: geo_spend[(b, pkg, g)])
        top_adp = max((a for (bb, pp, g, a) in ga_spend
                       if bb == b and pp == pkg and g == top_geo),
                      key=lambda a: ga_spend[(b, pkg, top_geo, a)])
        pkgs.append({
            "pkg": pkg,
            "geo": top_geo,
            "label": f"{pkg}_{top_geo}" if top_geo else pkg,
            "adping": top_adp,
            "spend": round(v, 4),
        })
    out.append({
        "name": b, "@name": b,
        "total": round(sum(pk.values()), 4),
        "active_ids": len(ids[b]),
        "owned_ids": owned[b],
        "pkgs": pkgs,
    })
out.sort(key=lambda x: -x['total'])

lbl = f"{start.month}/{start.day}-{end.month}/{end.day}"
data = {
    "meta": {
        "week": f"Last {WINDOW_DAYS} Days: {lbl}",
        "source": "萌奇熊日报",
        "generated_at": datetime.date.today().isoformat(),
        "date_range": f"{start:%Y/%m/%d}-{end:%Y/%m/%d}",
        "note": "投放人以「投手」表 Media Buyer 列为准，@某某 已与某某合并计算；"
                "未填写投放人的 Adpingid 不计入；exclude.json 所列包名已排除。"
                "Package 后缀为该 Package 花费最高的投放地区，Adpingid 为该地区内花费最高的 Adping。",
    },
    "grand_total": round(sum(b['total'] for b in out), 4),
    "week34_id_count": sum(len(v) for v in ids.values()),
    "buyers": out,
    "unmatched": [{"adping": k[0], "pkg": k[1], "spend": round(v, 4)}
                  for k, v in sorted(unmatched.items(), key=lambda x: -x[1])],
}
json.dump(data, open(os.path.join(HERE, 'data.json'), 'w'), ensure_ascii=False, indent=2)

print('源文件:', ', '.join(os.path.basename(f) for f in SRC_FILES))
print('投手表取自:', os.path.basename(roster_file))
print(f"窗口 {start} -> {end}")
print(f"排除包名命中 {excluded_rows} 行，剔除花费 {excluded_spend:.2f}")
print(f"合计 {data['grand_total']:.2f} / Adpingid {data['week34_id_count']}")
for b in out:
    print(f"  {b['name']:6s} {b['total']:10.2f}  在投{b['active_ids']:3d}/负责{b['owned_ids']:3d}")
    for p in b['pkgs']:
        print(f"      {p['label']:52s} {p['adping']:>10s} {p['spend']:9.2f}")