mediabuy-wh/build_data.py

#!/usr/bin/env python3
# 从 xlsx 重算 data.json。
#   平台消耗统计: B=Date, C=Adpingid, D=包名, E=地区, I=花费
#   投手:        B=Adpingid, D=pkg, L=Media Buyer投放人
# 规则：@某某 == 某某；未填投放人的 Adpingid 丢弃；
#       exclude.json 的 packages（按包名）与 adpings（按 Adpingid）永久剔除，均为精确匹配。
# 每个 package 明细：Geo 后缀 = 该 package 花费最高的地区；Adpingid = 该地区内花费最高的 Adping。
import openpyxl, collections, json, datetime, os, sys, glob

HERE = os.path.dirname(os.path.abspath(__file__))
# 源文件按月滚动（2608 -> 2609 -> ...）。全部读进来合并：
# 同一天出现在多个文件时，以文件名较新的为准（源表会修订历史数据）。
SRC_FILES = sorted(glob.glob(os.path.join(HERE, '*每日数据-武汉萌奇熊.xlsx')))
# 花名册 投手.xlsx 单独处理，不参与消耗数据合并
if not SRC_FILES:
    sys.exit('ERROR: 仓库里找不到 *每日数据-武汉萌奇熊.xlsx')
_cfg = json.load(open(os.path.join(HERE, 'exclude.json')))
EXCL      = set(_cfg['packages'])            # 按包名剔除
EXCL_ADP  = set(_cfg.get('adpings', {}))     # 按 Adpingid 剔除
NON_BUYER = {'GP下架'}
WINDOW_DAYS = 3

def key(v):
    if isinstance(v, float) and v.is_integer():
        return str(int(v))
    return str(v).strip()

books = [(f, openpyxl.load_workbook(f, data_only=True)) for f in SRC_FILES]

# --- 投手花名册 ---
# 优先用独立的 投手.xlsx（月度数据文件已不再带「投手」sheet）；
# 没有它时退回到最新一个仍带该 sheet 的月度文件。
# 花名册候选，按优先级从高到低；命中第一个就用它。
# 花名册统一放在 roster.xlsx —— 用户每次发来的新表（文件名各异）都覆盖到这个名字，
# 这样不必每换一个文件名就改一次代码。
ROSTER_PATTERNS = ['roster.xlsx']
roster_file, roster = None, None
for pat in ROSTER_PATTERNS:
    hits = sorted(glob.glob(os.path.join(HERE, pat)))
    if hits:
        roster_file = hits[-1]
        _rwb = openpyxl.load_workbook(roster_file, data_only=True)
        roster = _rwb['投手'] if '投手' in _rwb.sheetnames else _rwb[_rwb.sheetnames[0]]
        break
if roster is None:
    for f, wb_ in reversed(books):
        if '投手' in wb_.sheetnames:
            roster_file, roster = f, wb_['投手']
            break
if roster is None:
    sys.exit('ERROR: 既没有 投手.xlsx，月度文件里也没有「投手」sheet，无法归属投放人')

amap = {}
dropped_ids = set(EXCL_ADP)   # 先纳入按 Adpingid 剔除的，再补上按包名命中的
for r in roster.iter_rows(min_row=2, values_only=True):
    adp, pkg, buyer, status = r[1], r[3], r[11], r[0]
    if adp is None or str(status).strip() == '示例':
        continue
    k = key(adp)
    if k in EXCL_ADP:
        continue
    if pkg is not None and str(pkg).strip() in EXCL:   # 精确匹配，避免误伤 in.swiggy.android
        dropped_ids.add(k)
        continue
    b = str(buyer).strip() if buyer is not None else ''
    if b in ('', 'None') or b.lstrip('@').strip() in NON_BUYER:
        continue
    amap[k] = b.lstrip('@').strip()

# 花名册补充：只填花名册里查不到的 Adpingid（花名册优先）
EXTRA_PATH = os.path.join(HERE, 'roster_extra.json')
extra_used = []
if os.path.exists(EXTRA_PATH):
    for a, b in json.load(open(EXTRA_PATH)).get('adpings', {}).items():
        a = str(a).strip()
        if a in EXCL_ADP or a in dropped_ids:
            continue
        if a not in amap:
            amap[a] = str(b).strip().lstrip('@').strip()
            extra_used.append(a)

owned = collections.Counter(amap.values())   # 每位投手名下的 Adpingid 数（已扣除排除包名）

# --- 平台消耗统计：逐文件读取，按日期归组，新文件覆盖同一天的旧数据 ---
by_date = {}
for f, wb_ in books:
    if '平台消耗统计' not in wb_.sheetnames:
        continue
    day = collections.defaultdict(list)
    for r in wb_['平台消耗统计'].iter_rows(min_row=2, values_only=True):
        d, adp, pkg, geo, spend = r[1], r[2], r[3], r[4], r[8]
        if d is None or adp is None or not isinstance(d, datetime.datetime):
            continue
        try:
            s = float(spend) if spend is not None else 0.0
        except (TypeError, ValueError):
            s = 0.0
        day[d.date()].append((d.date(), key(adp), (str(pkg).strip() if pkg else '(未知)'),
                              (str(geo).strip() if geo else ''), s))
    by_date.update(day)          # books 已按文件名升序，后者覆盖前者

rows = [r for day_rows in by_date.values() for r in day_rows]
if not rows:
    sys.exit('ERROR: 平台消耗统计 无有效数据行')

end = max(r[0] for r in rows)
start = end - datetime.timedelta(days=WINDOW_DAYS - 1)

pkg_spend = collections.defaultdict(float)   # (buyer, pkg)             -> spend
geo_spend = collections.defaultdict(float)   # (buyer, pkg, geo)        -> spend
ga_spend  = collections.defaultdict(float)   # (buyer, pkg, geo, adping) -> spend
ids = collections.defaultdict(set)
unmatched = collections.defaultdict(float)
excluded_spend, excluded_rows = 0.0, 0

for d, adp, pkg, geo, s in rows:
    if not (start <= d <= end):
        continue
    if pkg in EXCL or adp in dropped_ids:
        excluded_spend += s; excluded_rows += 1
        continue
    b = amap.get(adp)
    if b is None:
        unmatched[(adp, pkg)] += s
        continue
    pkg_spend[(b, pkg)] += s
    geo_spend[(b, pkg, geo)] += s
    ga_spend[(b, pkg, geo, adp)] += s
    ids[b].add(adp)

by_buyer = collections.defaultdict(dict)
for (b, pkg), v in pkg_spend.items():
    by_buyer[b][pkg] = v

# --- 上一周期：紧邻本窗口之前的同样天数，口径完全一致 ---
prev_end = start - datetime.timedelta(days=1)
prev_start = prev_end - datetime.timedelta(days=WINDOW_DAYS - 1)
prev = collections.defaultdict(float)
prev_has_data = any(prev_start <= d <= prev_end for d in by_date)
for d, adp, pkg, geo, s_ in rows:
    if not (prev_start <= d <= prev_end):
        continue
    if pkg in EXCL or adp in dropped_ids:
        continue
    b = amap.get(adp)
    if b is not None:
        prev[b] += s_

out = []
for b, pk in by_buyer.items():
    pkgs = []
    for pkg, v in sorted(pk.items(), key=lambda x: -x[1])[:5]:
        # 花费最高的地区，再取该地区内花费最高的 Adpingid
        top_geo = max((g for (bb, pp, g) in geo_spend if bb == b and pp == pkg),
                      key=lambda g: geo_spend[(b, pkg, g)])
        top_adp = max((a for (bb, pp, g, a) in ga_spend
                       if bb == b and pp == pkg and g == top_geo),
                      key=lambda a: ga_spend[(b, pkg, top_geo, a)])
        pkgs.append({
            "pkg": pkg,
            "geo": top_geo,
            "label": f"{pkg}_{top_geo}" if top_geo else pkg,
            "adping": top_adp,
            "spend": round(v, 4),
        })
    out.append({
        "name": b, "@name": b,
        "total": round(sum(pk.values()), 4),
        "prev_total": round(prev.get(b, 0.0), 4),
        "active_ids": len(ids[b]),
        "owned_ids": owned[b],
        "pkgs": pkgs,
    })
out.sort(key=lambda x: -x['total'])

lbl = f"{start.month}/{start.day}-{end.month}/{end.day}"
data = {
    "meta": {
        "week": f"Last {WINDOW_DAYS} Days: {lbl}",
        "source": "萌奇熊日报",
        "generated_at": datetime.date.today().isoformat(),
        "date_range": f"{start:%Y/%m/%d}-{end:%Y/%m/%d}",
        "prev_range": (f"{prev_start:%Y/%m/%d}-{prev_end:%Y/%m/%d}" if prev_has_data else ""),
        "note": "投放人以「投手」表 Media Buyer 列为准，@某某 已与某某合并计算；"
                "未填写投放人的 Adpingid 不计入；exclude.json 所列包名已排除。"
                "Package 后缀为该 Package 花费最高的投放地区，Adpingid 为该地区内花费最高的 Adping。",
    },
    "grand_total": round(sum(b['total'] for b in out), 4),
    "week34_id_count": sum(len(v) for v in ids.values()),
    "buyers": out,
    "unmatched": [{"adping": k[0], "pkg": k[1], "spend": round(v, 4)}
                  for k, v in sorted(unmatched.items(), key=lambda x: -x[1])],
}
json.dump(data, open(os.path.join(HERE, 'data.json'), 'w'), ensure_ascii=False, indent=2)

print('源文件:', ', '.join(os.path.basename(f) for f in SRC_FILES))
print('投手表取自:', os.path.basename(roster_file))
if extra_used:
    print('roster_extra 补充生效:', ', '.join(f'{a}->{amap[a]}' for a in extra_used))
print(f"窗口 {start} -> {end}")
print(f"排除包名命中 {excluded_rows} 行，剔除花费 {excluded_spend:.2f}")
print(f"合计 {data['grand_total']:.2f} / Adpingid {data['week34_id_count']}")
print(f"上一周期 {prev_start} -> {prev_end}  合计 {sum(prev.values()):.2f}"
      + ('' if prev_has_data else '  (无数据)'))
for b in out:
    print(f"  {b['name']:6s} {b['total']:10.2f}  在投{b['active_ids']:3d}/负责{b['owned_ids']:3d}")
    for p in b['pkgs']:
        print(f"      {p['label']:52s} {p['adping']:>10s} {p['spend']:9.2f}")