From aa81fc7fa3f56f34ccb74ee65e7f4d8915e5e3d5 Mon Sep 17 00:00:00 2001 From: SHAREit Date: Fri, 16 Oct 2026 14:44:17 +0800 Subject: [PATCH] feat: add click gap pipeline and dashboard --- .gitignore | 12 + README.md | 62 +++ index.html | 20 + .../com.roasbloom.af-click-gap.plist.template | 12 + scripts/af_click_gap_pipeline.py | 366 ++++++++++++++++++ 5 files changed, 472 insertions(+) create mode 100755 .gitignore create mode 100755 README.md create mode 100755 index.html create mode 100755 launchd/com.roasbloom.af-click-gap.plist.template create mode 100755 scripts/af_click_gap_pipeline.py diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000..b58b66b --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +.DS_Store +.env +.env.* +*.db +*.sqlite +*.sqlite3 +*.log +__pycache__/ +*.py[cod] +.cache/ +.cookies/ +browser-profile/ diff --git a/README.md b/README.md new file mode 100755 index 0000000..8d2a4a2 --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +# AppsFlyer Click Gap + +运行目标是 **mac2**。流水线复用 `appsflyer/af_click_gap.py` 的 Superset 查询、AppsFlyer payload、特殊包名映射和差值公式，但新入口增加 timeout/重试、严格错误边界、AppsFlyer 重复键按 `dt+pkg_name` 求和，并将日志写到仓库外 `~/Library/Logs/af-click-gap/pipeline.log`。固定数据库：`/Users/ads-center/Documents/database/af_click_gap.db`；不复制或提交数据库、cookie、token、`.env`、日志和缓存。 + +## 文件和字段 + +- `scripts/af_click_gap_pipeline.py`：提取、校验、发布和 Git 操作。 +- `data/YYYYMMDD.json`：**直接 JSON 数组**；无数据日期也写 `[]`。每行字段为 `dt`（`YYYYMMDD` 字符串）、`attr_platform`、`pkg_name`（非空字符串）、`shareit_clicks`、`af_clicks`、`installs_ua`、`clicks_diff`（有限 number）、`clicks_diff_rate`（有限 number 或 `null`）。业务键为 `pid + dt + pkg_name`。 +- `data/index.json`：版本、生成时间及 `data/YYYYMMDD.json` 清单。 +- `index.html`：无构建依赖的静态看板。 + +默认窗口是 mac2 本地运行日及前 6 天，首尾包含，共 7 天；这是对旧脚本实际可能覆盖 8 个日期的修正。空 Superset 结果、无达标包或 AppsFlyer 合法空数组都是成功空结果（退出码 0），仍生成日期数组、同步清空 SQLite 窗口并可提交；HTTP、认证、响应格式、SQLite schema 或文件校验错误退出码 1，不覆盖正式文件。 + +SQLite 必须预先存在 `click_gap` 表，列类型为 `pid TEXT NOT NULL, dt TEXT NOT NULL, attr_platform TEXT NOT NULL, pkg_name TEXT NOT NULL, shareit_clicks REAL NOT NULL, af_clicks REAL NOT NULL, installs_ua REAL NOT NULL, clicks_diff REAL NOT NULL, clicks_diff_rate REAL, updated_at TEXT NOT NULL`，主键严格为 `(pid, dt, pkg_name)`。脚本不建表、不迁移；每次成功运行先删除窗口内 `shareit` 旧行再插入完整新结果，因此消失日期/业务键也会清零。 + +## 手工运行 + +在 mac2 先确认依赖和登录环境（`pandas`、`requests`、`requests_toolbelt`、`af_browser`、`superset_login`）： + +```sh +cd /Users/shareit./Library/CloudStorage/OneDrive-个人/ubuntu/Gitea/af-click-gap +python3 -m py_compile scripts/af_click_gap_pipeline.py +python3 scripts/af_click_gap_pipeline.py --help +python3 scripts/af_click_gap_pipeline.py --dry-run +# 正式写入并提交、push（默认行为） +python3 scripts/af_click_gap_pipeline.py --push +# 只写 data 和 SQLite，不操作 Git +python3 scripts/af_click_gap_pipeline.py --no-push +# 手工补数，日期首尾包含且最多 7 天 +python3 scripts/af_click_gap_pipeline.py --start-date 2026-10-10 --end-date 2026-10-16 --no-push +``` + +`--dry-run` 只生成仓库外临时 JSON 并校验，不写 data、数据库或执行 Git。正式文件先原子替换，再提交 SQLite，最后才 `git add/commit/push`；数据库提交失败时不执行 Git。commit/push 失败会保留本地文件和数据库，下一次可重跑或手工 `git add data index.html ... && git commit && git push`；不会伪造认证或 force push。工作区在 push 模式有非本任务改动时拒绝继续。 + +## 看板 + +直接打开 `index.html` 或运行 `python3 -m http.server` 后访问。页面读取并校验 `data/index.json` 及清单路径，只执行安全 JSON 解析；缺失/损坏文件显示“数据不完整”并跳过该文件，清单本身失败则不展示数据。默认最近 30 天，可选有效开始/结束日期和 `pkg_name`；默认 `dt` 降序、同日期 `shareit_clicks` 降序，点击表头切换排序。 + +## mac2 launchd + +`launchd/com.roasbloom.af-click-gap.plist.template` 明确为 `TEMPLATE_ONLY`。当前开发机不能证明自己是 mac2，因此没有伪造最终 plist、解释器或安装结果。必须在 mac2 执行并记录： + +```sh +hostname +command -v python3 +python3 --version +python3 -c 'import pandas,requests,requests_toolbelt,af_browser,superset_login; print("dependencies ok")' +cd /Users/shareit./Library/CloudStorage/OneDrive-个人/ubuntu/Gitea/af-click-gap +mkdir -p "$HOME/Library/Logs/af-click-gap" +PY=$(command -v python3); sed -e "s#__MAC2_PYTHON3__#$PY#" -e "s#__PROJECT_DIR__#$PWD#g" -e "s#__LOG_DIR__#$HOME/Library/Logs/af-click-gap#g" launchd/com.roasbloom.af-click-gap.plist.template > /tmp/com.roasbloom.af-click-gap.plist +plutil -lint /tmp/com.roasbloom.af-click-gap.plist +cp /tmp/com.roasbloom.af-click-gap.plist "$HOME/Library/LaunchAgents/" +launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.roasbloom.af-click-gap.plist" +launchctl print "gui/$(id -u)/com.roasbloom.af-click-gap" +# 卸载：launchctl bootout "gui/$(id -u)/com.roasbloom.af-click-gap"; rm "$HOME/Library/LaunchAgents/com.roasbloom.af-click-gap.plist" +``` + +定时任务失败不会触发成功 push；stdout/stderr 位于 mac2 用户日志目录。Git remote、SSH/HTTPS credential helper 或 Gitea token 必须由用户在 mac2 预先配置。若 `git remote -v` 为空，请按组织 Git 文档创建仓库并绑定 remote 后再运行；本项目不会猜测仓库名称或修改 git config。 + +## 安全和验证 + +允许提交的路径仅为脚本、README、看板、`data/*.json`、`data/index.json`、模板和 `.gitignore`。提交前禁止 `.env*`、SQLite、日志、浏览器状态，脚本还扫描 `cookie`、`authorization`、`password`、`api-key`、`token` 文本。建议验证：`python3 -m py_compile ...`、`--help`、mac2 上 `--dry-run`、`python3 -m json.tool data/index.json`，以及浏览器/静态 HTTP 打开页面。当前环境未验证 mac2 依赖、固定数据库、外部登录、launchd 安装和远程 push；这些不能以本机结果冒充。不要新增测试套件。 diff --git a/index.html b/index.html new file mode 100755 index 0000000..00a42c5 --- /dev/null +++ b/index.html @@ -0,0 +1,20 @@ + + + +AppsFlyer Click Gap + + +

AppsFlyer Click Gap

加载中…

+ diff --git a/launchd/com.roasbloom.af-click-gap.plist.template b/launchd/com.roasbloom.af-click-gap.plist.template new file mode 100755 index 0000000..922bccf --- /dev/null +++ b/launchd/com.roasbloom.af-click-gap.plist.template @@ -0,0 +1,12 @@ + + + + +Labelcom.roasbloom.af-click-gap +ProgramArguments__MAC2_PYTHON3____PROJECT_DIR__/scripts/af_click_gap_pipeline.py--push +WorkingDirectory__PROJECT_DIR__ +EnvironmentVariablesDB_PATH/Users/ads-center/Documents/database/af_click_gap.db +StartCalendarIntervalHour2Minute0 +StandardOutPath__LOG_DIR__/pipeline.stdout.logStandardErrorPath__LOG_DIR__/pipeline.stderr.log +RunAtLoad + diff --git a/scripts/af_click_gap_pipeline.py b/scripts/af_click_gap_pipeline.py new file mode 100755 index 0000000..ba1e329 --- /dev/null +++ b/scripts/af_click_gap_pipeline.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python3 +"""Extract AppsFlyer click-gap reports and publish validated static JSON.""" +from __future__ import annotations + +import argparse +import datetime as dt +import json +import logging +import math +import os +import re +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +DB_PATH = "/Users/ads-center/Documents/database/af_click_gap.db" +PID = "shareit" +CLICK_NUM = 300 +REPORT_COLUMNS = ["dt", "attr_platform", "pkg_name", "shareit_clicks", "af_clicks", "installs_ua", "clicks_diff", "clicks_diff_rate"] +SPECIAL_PKGS = { + "za.co.freeurl.easybetapp.fwdb": "za.co.freeurl.easybetapp.fwdb-Shareit", + "org.melbet.client": "org.melbet.client-Custom", "org.xbet.client1": "org.xbet.client1-Custom", + "org.betwinner_ng.client": "org.betwinner_ng.client-Custom", "org.betwinner_zm.client": "org.betwinner_zm.client-Custom", + "org.betwinner_cm.client": "org.betwinner_cm.client-Custom", "org.betwinner_cd.client": "org.betwinner_cd.client-Custom", +} +EXPECTED_TYPES = {"pid": "TEXT", "dt": "TEXT", "attr_platform": "TEXT", "pkg_name": "TEXT", "shareit_clicks": "REAL", "af_clicks": "REAL", "installs_ua": "REAL", "clicks_diff": "REAL", "clicks_diff_rate": "REAL", "updated_at": "TEXT"} +LOG_PATH = Path(os.path.expanduser("~/Library/Logs/af-click-gap/pipeline.log")) + + +def logger_setup() -> logging.Logger: + LOG_PATH.parent.mkdir(parents=True, exist_ok=True) + log = logging.getLogger("af_click_gap") + if not log.handlers: + log.setLevel(logging.INFO) + handler = logging.FileHandler(LOG_PATH, encoding="utf-8") + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s")) + log.addHandler(handler) + log.addHandler(logging.StreamHandler()) + return log + + +LOG = logger_setup() + + +def date8(value: str) -> str: + try: + parsed = dt.datetime.strptime(value, "%Y%m%d") + except ValueError as exc: + raise ValueError(f"非法日期: {value}") from exc + return parsed.strftime("%Y%m%d") + + +def date8_from_iso(value: str) -> str: + try: + return dt.datetime.strptime(value, "%Y-%m-%d").strftime("%Y%m%d") + except ValueError as exc: + raise ValueError(f"日期必须为 YYYY-MM-DD: {value}") from exc + + +def window(start: str | None, end: str | None) -> tuple[str, str]: + today = dt.date.today() + if start is None and end is None: + first, last = today - dt.timedelta(days=6), today + else: + if not start or not end: + raise ValueError("--start-date 与 --end-date 必须同时提供") + first = dt.datetime.strptime(start, "%Y-%m-%d").date() + last = dt.datetime.strptime(end, "%Y-%m-%d").date() + if first > last or (last - first).days > 6: + raise ValueError("日期区间必须为包含首尾且不超过 7 天") + return first.strftime("%Y%m%d"), last.strftime("%Y%m%d") + + +def finite_number(value: Any, field: str, allow_null: bool = False) -> int | float | None: + if value is None or (isinstance(value, float) and math.isnan(value)): + if allow_null: + return None + raise ValueError(f"{field} 不能为空") + try: + number = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field} 必须为数值") from exc + if not math.isfinite(number): + raise ValueError(f"{field} 必须为有限数值") + return int(number) if number.is_integer() else number + + +def validate_existing_schema(path: str) -> None: + db = Path(path) + if not db.is_file(): + raise RuntimeError(f"SQLite 文件不存在: {path}") + try: + with sqlite3.connect(f"file:{db}?mode=ro", uri=True) as conn: + columns = conn.execute("PRAGMA table_info(click_gap)").fetchall() + except sqlite3.Error as exc: + raise RuntimeError(f"SQLite 不可读: {exc}") from exc + if not columns: + raise RuntimeError("click_gap 表不存在") + by_name = {row[1]: row for row in columns} + missing = set(EXPECTED_TYPES) - set(by_name) + if missing: + raise RuntimeError(f"click_gap 缺少列: {sorted(missing)}") + for name, expected in EXPECTED_TYPES.items(): + if str(by_name[name][2]).upper() != expected: + raise RuntimeError(f"click_gap 列类型不兼容: {name}") + if [(row[1], row[5]) for row in columns if row[5]] != [("pid", 1), ("dt", 2), ("pkg_name", 3)]: + raise RuntimeError("click_gap 主键必须为 (pid, dt, pkg_name)") + nullable_required = {"pid", "dt", "pkg_name", "shareit_clicks", "af_clicks", "installs_ua", "clicks_diff", "updated_at"} + if any(by_name[name][3] != 1 for name in nullable_required): + raise RuntimeError("click_gap 必填列不得允许 NULL") + + +def read_existing_rows(path: str, start: str, end: str, pid: str = PID) -> list[dict[str, Any]]: + validate_existing_schema(path) + with sqlite3.connect(path) as conn: + conn.row_factory = sqlite3.Row + return [dict(row) for row in conn.execute("SELECT * FROM click_gap WHERE pid=? AND dt BETWEEN ? AND ?", (pid, start, end))] + + +def upsert_report(conn: sqlite3.Connection, rows: list[dict[str, Any]], start: str, end: str) -> int: + now = dt.datetime.now().astimezone().isoformat(timespec="seconds") + conn.execute("DELETE FROM click_gap WHERE pid=? AND dt BETWEEN ? AND ?", (PID, start, end)) + values = [] + for row in rows: + values.append((PID, row["dt"], row["attr_platform"], row["pkg_name"], row["shareit_clicks"], row["af_clicks"], row["installs_ua"], row["clicks_diff"], row["clicks_diff_rate"], now)) + conn.executemany("INSERT INTO click_gap (pid,dt,attr_platform,pkg_name,shareit_clicks,af_clicks,installs_ua,clicks_diff,clicks_diff_rate,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?)", values) + return len(values) + + +def superset_rows(start: str, end: str) -> list[dict[str, Any]]: + from superset_login import get_cookies + import requests + from requests_toolbelt import MultipartEncoder + session, csrf = get_cookies() + nextday = (dt.datetime.strptime(end, "%Y%m%d").date() + dt.timedelta(days=1)).strftime("%Y-%m-%d") + start_iso = dt.datetime.strptime(start, "%Y%m%d").strftime("%Y-%m-%d") + query = {"datasource": "114__table", "viz_type": "table", "slice_id": 1344, "url_params": {}, "time_range_endpoints": ["inclusive", "exclusive"], "granularity_sqla": "dt_as_time", "time_grain_sqla": "P1D", "time_range": f"{start_iso}T00:00:00 : {nextday}T00:00:00", "query_mode": "aggregate", "groupby": ["dt", "attr_platform", "pkg_name"], "metrics": ["clicks", "pb_install"], "all_columns": [], "array_join": [], "percent_metrics": [], "timeseries_limit_metric": ["dt_sort"], "order_by_cols": [], "row_limit": None, "order_desc": True, "adhoc_filters": [{"clause": "WHERE", "expressionType": "SIMPLE", "comparator": ["appsflyer"], "operator": "IN", "subject": "attr_platform"}, {"clause": "WHERE", "expressionType": "SIMPLE", "comparator": ["0"], "operator": "IN", "subject": "is_internal"}], "table_timestamp_format": "smart_date", "color_pn": True, "show_cell_bars": True} + encoder = MultipartEncoder(fields={"form_data": json.dumps(query)}) + response = session.post('https://ads-bi.ushareit.org/superset/explore_json/?form_data={"slice_id":1344}&force=true', headers={"x-csrftoken": csrf, "content-type": encoder.content_type}, data=encoder, timeout=(10, 60)) + if response.status_code != 200: + raise RuntimeError(f"Superset 请求失败: HTTP {response.status_code}") + payload = response.json() + data = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(data, dict) or not isinstance(data.get("records"), list) or not isinstance(data.get("columns"), list): + raise RuntimeError("Superset 响应格式错误") + return [dict(zip(data["columns"], row)) if isinstance(row, (list, tuple)) else row for row in data["records"]] + + +def pkg_list(rows: list[dict[str, Any]]) -> list[str]: + totals: dict[str, float] = {} + for row in rows: + pkg = str(row.get("pkg_name") or "").strip() + if pkg: + totals[pkg] = totals.get(pkg, 0) + float(row.get("clicks") or 0) + return sorted(pkg for pkg, clicks in totals.items() if clicks > CLICK_NUM) + + +def appsflyer_rows(packages: list[str], start: str, end: str, browser: Any) -> list[dict[str, Any]]: + import requests + cookie = browser.get_cookie_str() + if not cookie: + raise RuntimeError("AppsFlyer cookie 为空") + payload = {"dates": {"start": dt.datetime.strptime(start, "%Y%m%d").strftime("%Y-%m-%d"), "end": dt.datetime.strptime(end, "%Y%m%d").strftime("%Y-%m-%d")}, "filters": {"app-id": []}, "view-type": "unified", "localization": {"timezone": "UTC", "currency": "USD"}, "groupings": [{"dimension": "date", "limit": 200}], "summations": ["totals", "others"], "metrics": [{"metric-id": "clicks"}, {"metric-id": "installs-ua"}], "format": "json", "granularity": "days", "implicit-sorting": True} + result: list[dict[str, Any]] = [] + url = "https://hq1.appsflyer.com/platform/dashboard?widget=platform-table__performance:3" + for package in packages: + app_id = SPECIAL_PKGS.get(package, package) + payload["filters"]["app-id"] = [app_id] + last_error = None + for attempt in range(4): + try: + response = requests.post(url, data=json.dumps(payload), headers={"content-type": "application/json;charset=UTF-8", "cookie": cookie}, timeout=(10, 60)) + if response.status_code == 429 or response.status_code >= 500: + raise RuntimeError(f"HTTP {response.status_code}") + if response.status_code != 200: + raise RuntimeError(f"HTTP {response.status_code}") + data = response.json().get("data") + if not isinstance(data, list) or any(not isinstance(item, dict) for item in data): + raise RuntimeError("响应格式错误") + for item in data: + item["app_id"] = package + result.extend(data) + break + except Exception as exc: # do not leak request contents + last_error = exc + if attempt < 3: + time.sleep(2 ** (attempt + 1)) + else: + raise RuntimeError(f"AppsFlyer 查询失败 pkg={package}: {last_error}") + return result + + +def build_report(shareit: list[dict[str, Any]], af: list[dict[str, Any]], packages: list[str]) -> list[dict[str, Any]]: + shared: dict[tuple[str, str, str], float] = {} + for row in shareit: + if str(row.get("attr_platform") or "") != "appsflyer": + continue + pkg = str(row.get("pkg_name") or "").strip() + if not pkg or pkg not in packages: + continue + raw = str(row.get("dt") or "")[:10].replace("-", "") + date = date8(raw) + shared[(date, "appsflyer", pkg)] = shared.get((date, "appsflyer", pkg), 0) + float(row.get("clicks") or 0) + af_totals: dict[tuple[str, str], list[float]] = {} + for row in af: + raw = str(row.get("date") or "")[:10].replace("-", "") + date = date8(raw) + pkg = str(row.get("app_id") or "").strip() + key = (date, pkg) + values = af_totals.setdefault(key, [0.0, 0.0]) + values[0] += float(row.get("filtersGranularityMetricIdClicksPeriod", row.get("clicks", 0)) or 0) + values[1] += float(row.get("attributionSourceAppsflyerFiltersGranularityMetricIdInstallsUaPeriod", row.get("installs_ua", 0)) or 0) + result = [] + for (date, platform, pkg), share_clicks in shared.items(): + af_clicks, installs = af_totals.get((date, pkg), [0.0, 0.0]) + diff = share_clicks - af_clicks + result.append({"dt": date, "attr_platform": platform, "pkg_name": pkg, "shareit_clicks": finite_number(share_clicks, "shareit_clicks"), "af_clicks": finite_number(af_clicks, "af_clicks"), "installs_ua": finite_number(installs, "installs_ua"), "clicks_diff": finite_number(diff, "clicks_diff"), "clicks_diff_rate": None if share_clicks == 0 else finite_number(round(diff / share_clicks, 3), "clicks_diff_rate", True)}) + result.sort(key=lambda row: (row["dt"], row["shareit_clicks"]), reverse=True) + if len({(r["dt"], r["pkg_name"]) for r in result}) != len(result): + raise RuntimeError("报告存在重复业务键") + return result + + +def validate_rows(rows: list[dict[str, Any]], start: str, end: str) -> None: + for row in rows: + if list(row) != REPORT_COLUMNS or date8(row["dt"]) < start or date8(row["dt"]) > end or not str(row["pkg_name"]).strip(): + raise ValueError("报告字段、日期或包名不合法") + for field in REPORT_COLUMNS[3:]: + finite_number(row[field], field, field == "clicks_diff_rate") + + +def make_files(rows: list[dict[str, Any]], dates: list[str], out: Path) -> None: + out.mkdir(parents=True, exist_ok=True) + for date in dates: + content = [row for row in rows if row["dt"] == date] + target = out / f"{date}.json" + target.write_text(json.dumps(content, ensure_ascii=False, allow_nan=False, separators=(",", ":")) + "\n", encoding="utf-8") + index = {"schema_version": 1, "generated_at": dt.datetime.now().astimezone().isoformat(timespec="seconds"), "files": [{"date": date, "path": f"data/{date}.json"} for date in sorted(dates, reverse=True)]} + (out / "index.json").write_text(json.dumps(index, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def sensitive_check(paths: list[Path]) -> None: + pattern = re.compile(r"cookie|authorization|password|api[_-]?key|token", re.I) + for path in paths: + if path.name.startswith(".env") or path.suffix in {".db", ".log"}: + raise RuntimeError(f"禁止发布敏感文件: {path.name}") + if path.suffix in {".json", ".html", ".md"} and pattern.search(path.read_text(encoding="utf-8")): + raise RuntimeError(f"文件疑似包含敏感字段: {path.name}") + + +def publish_files(repo: Path, generated: Path, dates: list[str]) -> None: + data = repo / "data" + data.mkdir(exist_ok=True) + for date in dates: + source = generated / f"{date}.json" + tmp = data / f".{date}.tmp" + shutil.copyfile(source, tmp) + os.replace(tmp, data / f"{date}.json") + os.replace(generated / "index.json", data / "index.json") + + +def git_run(repo: Path, args: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run(["git", *args], cwd=repo, text=True, capture_output=True, timeout=120, check=False) + + +def git_publish(repo: Path, start: str, end: str, no_push: bool) -> str: + if not (repo / ".git").exists(): + raise RuntimeError("仓库尚未初始化；请先执行 git init 并配置 remote") + status = git_run(repo, ["status", "--porcelain"]).stdout.strip().splitlines() + allowed = {"README.md", ".gitignore", "index.html", "scripts/af_click_gap_pipeline.py", "launchd/com.roasbloom.af-click-gap.plist.template"} + allowed.update({f"data/{date}.json" for date in (repo / "data").glob("*.json")}) + unexpected = [line for line in status if line[3:] not in allowed] + if unexpected: + raise RuntimeError("工作区已有未预期改动: " + ", ".join(unexpected)) + paths = ["README.md", ".gitignore", "index.html", "scripts/af_click_gap_pipeline.py", "launchd/com.roasbloom.af-click-gap.plist.template", "data"] + add = git_run(repo, ["add", *paths]) + if add.returncode: + raise RuntimeError("git add 失败: " + add.stderr.strip()) + staged = git_run(repo, ["diff", "--cached", "--name-only"]) + if staged.returncode or not staged.stdout.strip(): + raise RuntimeError("没有可提交的文件") + commit = git_run(repo, ["commit", "-m", f"feat: publish click gap {start}-{end}"]) + if commit.returncode: + raise RuntimeError("git commit 失败: " + commit.stderr.strip()) + commit_hash = git_run(repo, ["rev-parse", "HEAD"]).stdout.strip() + if no_push: + return commit_hash + " (未 push)" + remote = git_run(repo, ["remote", "-v"]) + if remote.returncode or not remote.stdout.strip(): + raise RuntimeError("未配置 Git remote；本地 commit 已保留") + push = git_run(repo, ["push"]) + if push.returncode: + raise RuntimeError("git push 失败（本地 commit 已保留）: " + push.stderr.strip()) + return commit_hash + " (已 push)" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Extract and publish AppsFlyer click-gap JSON") + parser.add_argument("--start-date", help="补数开始日期 YYYY-MM-DD") + parser.add_argument("--end-date", help="补数结束日期 YYYY-MM-DD") + parser.add_argument("--dry-run", action="store_true", help="只提取并校验，不写正式文件、数据库或 Git") + group = parser.add_mutually_exclusive_group() + group.add_argument("--no-push", action="store_true", help="写文件和数据库但不 push") + group.add_argument("--push", action="store_true", help="写文件、数据库并 commit/push（默认）") + parser.add_argument("--db-path", default=DB_PATH, help=argparse.SUPPRESS) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + if args.dry_run and (args.no_push or args.push): + raise ValueError("--dry-run 不可与 --push/--no-push 同时使用") + start, end = window(args.start_date, args.end_date) + validate_existing_schema(args.db_path) + shareit = superset_rows(start, end) + packages = pkg_list(shareit) + af: list[dict[str, Any]] = [] + browser = None + if packages: + from af_browser import AFBrowser + browser = AFBrowser() + try: + af = appsflyer_rows(packages, start, end, browser) + finally: + try: + browser.close() + except Exception: + LOG.warning("关闭浏览器失败") + rows = build_report(shareit, af, packages) + validate_rows(rows, start, end) + dates = [(dt.datetime.strptime(start, "%Y%m%d").date() + dt.timedelta(days=i)).strftime("%Y%m%d") for i in range((dt.datetime.strptime(end, "%Y%m%d").date() - dt.datetime.strptime(start, "%Y%m%d").date()).days + 1)] + repo = Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory(prefix="af-click-gap-") as temp: + generated = Path(temp) / "data" + make_files(rows, dates, generated) + sensitive_check(list(generated.glob("*"))) + if args.dry_run: + LOG.info("dry-run 成功: %s-%s, %d 行", start, end, len(rows)) + return 0 + publish_files(repo, generated, dates) + with sqlite3.connect(args.db_path) as conn: + conn.execute("BEGIN") + upsert_report(conn, rows, start, end) + conn.commit() + if args.no_push: + LOG.info("已发布本地文件和 SQLite，未执行 Git") + else: + result = git_publish(repo, start, end, False) + LOG.info("Git 结果: %s", result) + return 0 + except ValueError as exc: + LOG.error("输入失败: %s", exc) + return 2 + except Exception as exc: + LOG.error("任务失败: %s", exc) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main())