Publish DSP revenue report

This commit is contained in:
Kiro
2026-10-07 16:18:07 +08:00
commit 69a70592be
4 changed files with 737 additions and 0 deletions
+1
View File
File diff suppressed because one or more lines are too long
+484
View File
@@ -0,0 +1,484 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="color-scheme" content="light">
<title>DSP 包名收入波动报告</title>
<style>
:root {
--ink: #14223a;
--muted: #64748b;
--line: #dce4ef;
--canvas: #f5f8fc;
--surface: #ffffff;
--navy: #142b55;
--blue: #2563eb;
--blue-soft: #e8f0ff;
--green: #16825d;
--green-soft: #e7f7ef;
--red: #c43f45;
--red-soft: #fff0f1;
--amber: #a25d00;
--shadow: 0 12px 32px rgba(34, 58, 93, .08);
}
* { box-sizing: border-box; }
html { background: var(--canvas); }
body {
margin: 0;
min-width: 320px;
color: var(--ink);
background:
radial-gradient(circle at 84% -10%, rgba(59, 130, 246, .15), transparent 32rem),
var(--canvas);
font: 14px/1.5 Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
}
button, input, select { font: inherit; }
button { cursor: pointer; }
.shell { width: min(1440px, calc(100% - 40px)); margin: 0 auto; }
.hero {
padding: 48px 0 32px;
color: #fff;
background: linear-gradient(120deg, #132a53 0%, #1e427e 54%, #2b6ed3 100%);
box-shadow: inset 0 -1px rgba(255,255,255,.14);
}
.hero .shell { display: flex; align-items: flex-end; justify-content: space-between; gap: 24px; }
.eyebrow {
display: inline-flex;
align-items: center;
gap: 8px;
padding: 5px 10px;
border: 1px solid rgba(255,255,255,.26);
border-radius: 999px;
color: #dcecff;
font-size: 12px;
font-weight: 700;
letter-spacing: .06em;
text-transform: uppercase;
}
.pulse { width: 7px; height: 7px; border-radius: 50%; background: #6ee7b7; box-shadow: 0 0 0 4px rgba(110,231,183,.18); }
h1 { margin: 13px 0 5px; font-size: clamp(28px, 4vw, 40px); letter-spacing: -.035em; line-height: 1.12; }
.hero p { max-width: 720px; margin: 0; color: #c8ddff; font-size: 15px; }
.status-card {
flex: 0 0 auto;
min-width: 255px;
padding: 14px 16px;
border: 1px solid rgba(255,255,255,.18);
border-radius: 12px;
background: rgba(7, 21, 47, .22);
}
.status-label { display: block; color: #b7cff5; font-size: 11px; font-weight: 800; letter-spacing: .06em; text-transform: uppercase; }
.status-value { display: block; margin-top: 4px; color: #fff; font-weight: 700; }
main { padding: 28px 0 56px; }
.metrics { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 14px; margin-bottom: 22px; }
.metric, .panel {
border: 1px solid rgba(205, 218, 236, .92);
border-radius: 16px;
background: var(--surface);
box-shadow: var(--shadow);
}
.metric { min-height: 122px; padding: 18px 20px; }
.metric-label { display: block; color: var(--muted); font-size: 12px; font-weight: 700; }
.metric-value { display: block; margin-top: 10px; font-size: clamp(22px, 2.4vw, 30px); font-weight: 800; letter-spacing: -.035em; }
.metric-note { display: block; margin-top: 2px; color: var(--muted); font-size: 11px; }
.panel { overflow: hidden; margin-bottom: 22px; }
.panel-heading { display: flex; align-items: flex-start; justify-content: space-between; gap: 18px; padding: 22px 24px 14px; }
.panel-heading h2 { margin: 0; font-size: 18px; letter-spacing: -.015em; }
.panel-heading p { margin: 3px 0 0; color: var(--muted); font-size: 12px; }
.legend { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: 12px; color: var(--muted); font-size: 12px; }
.legend-item { display: inline-flex; align-items: center; gap: 6px; }
.legend-line { width: 22px; height: 3px; border-radius: 2px; background: var(--blue); }
.chart-wrap { min-height: 286px; padding: 0 14px 17px; }
#trendChart { display: block; width: 100%; min-height: 270px; overflow: visible; }
.filters { display: flex; flex-wrap: wrap; align-items: end; gap: 12px; padding: 0 24px 18px; }
.field { display: grid; gap: 5px; min-width: 180px; color: var(--muted); font-size: 11px; font-weight: 800; letter-spacing: .03em; text-transform: uppercase; }
.field.search { flex: 1 1 260px; }
select, input {
width: 100%;
min-height: 38px;
padding: 8px 10px;
border: 1px solid #c8d4e4;
border-radius: 9px;
outline: none;
color: var(--ink);
background: #fff;
font-size: 13px;
letter-spacing: normal;
text-transform: none;
}
select:focus, input:focus { border-color: var(--blue); box-shadow: 0 0 0 3px rgba(37,99,235,.12); }
.button-secondary {
min-height: 38px;
padding: 0 13px;
border: 1px solid #c8d4e4;
border-radius: 9px;
color: #31547f;
background: #fff;
font-size: 13px;
font-weight: 700;
}
.button-secondary:hover { border-color: #8eafd8; background: #f5f9ff; }
.data-note { padding: 0 24px 14px; color: var(--muted); font-size: 12px; }
.table-scroll { width: 100%; overflow: auto; border-top: 1px solid var(--line); }
table { width: 100%; min-width: 820px; border-collapse: collapse; }
th, td { padding: 12px 16px; border-bottom: 1px solid #e8eef6; text-align: right; white-space: nowrap; }
th {
position: sticky;
top: 0;
z-index: 1;
color: #57708f;
background: #f8fbff;
font-size: 11px;
font-weight: 800;
letter-spacing: .03em;
text-transform: uppercase;
}
th:first-child, td:first-child { position: sticky; left: 0; text-align: left; }
th:first-child { z-index: 2; background: #f8fbff; }
td:first-child { z-index: 1; min-width: 260px; color: #203858; background: #fff; font-weight: 700; }
tbody tr:hover td { background: #f6faff; }
tbody tr:hover td:first-child { background: #f6faff; }
.diff { border-radius: 7px; font-weight: 800; }
.diff.positive { color: var(--green); background: var(--green-soft); }
.diff.negative { color: var(--red); background: var(--red-soft); }
.diff.neutral { color: var(--amber); background: #fff8e9; }
.empty { padding: 30px !important; color: var(--muted); text-align: center !important; }
.pagination { display: flex; align-items: center; justify-content: flex-end; gap: 11px; padding: 14px 20px; }
.pagination span { margin-right: auto; color: var(--muted); font-size: 12px; }
.pagination button {
min-width: 68px;
min-height: 32px;
padding: 0 10px;
border: 1px solid #c8d4e4;
border-radius: 7px;
color: #31547f;
background: #fff;
font-size: 12px;
font-weight: 700;
}
.pagination button:disabled { cursor: not-allowed; opacity: .42; }
.footnote { margin: 18px 2px 0; color: var(--muted); font-size: 12px; }
.footnote strong { color: #415a7a; }
.error { color: var(--red); }
@media (max-width: 900px) {
.hero .shell { display: block; }
.status-card { display: inline-block; margin-top: 20px; }
.metrics { grid-template-columns: repeat(2, minmax(0, 1fr)); }
}
@media (max-width: 580px) {
.shell { width: min(100% - 24px, 1440px); }
.hero { padding-top: 34px; }
main { padding-top: 18px; }
.metrics { gap: 10px; }
.metric { min-height: 105px; padding: 15px; }
.panel-heading, .filters { padding-left: 16px; padding-right: 16px; }
.panel-heading { display: block; }
.legend { justify-content: flex-start; margin-top: 10px; }
.data-note { padding-left: 16px; padding-right: 16px; }
}
</style>
</head>
<body>
<header class="hero">
<div class="shell">
<div>
<span class="eyebrow"><i class="pulse"></i> AgentStage · DSP BI</span>
<h1>DSP 包名收入波动报告</h1>
<p>按归因平台筛选,以包名和日期透视收入;右侧即时显示最新日(T-1)相对上一日(T-2)的收入变化。</p>
</div>
<div class="status-card" aria-live="polite">
<span class="status-label">数据状态</span>
<span class="status-value" id="dataStatus">正在加载数据…</span>
</div>
</div>
</header>
<main class="shell">
<section class="metrics" aria-label="汇总指标">
<article class="metric"><span class="metric-label">最新日收入</span><strong class="metric-value" id="latestRevenue">—</strong><span class="metric-note" id="latestRevenueNote">等待数据</span></article>
<article class="metric"><span class="metric-label">已返回明细</span><strong class="metric-value" id="rowCount">—</strong><span class="metric-note">单次查询最多 5,000 行</span></article>
<article class="metric"><span class="metric-label">归因平台</span><strong class="metric-value" id="platformCount">—</strong><span class="metric-note">可通过筛选器切换</span></article>
<article class="metric"><span class="metric-label">包名数</span><strong class="metric-value" id="packageCount">—</strong><span class="metric-note">当前筛选结果</span></article>
</section>
<section class="panel" aria-labelledby="trendTitle">
<div class="panel-heading">
<div><h2 id="trendTitle">收入趋势</h2><p>当前筛选条件下,按数据日期汇总的 revenue。</p></div>
<div class="legend"><span class="legend-item"><i class="legend-line"></i>收入</span></div>
</div>
<div class="chart-wrap"><svg id="trendChart" viewBox="0 0 920 300" role="img" aria-label="收入趋势图"></svg></div>
</section>
<section class="panel" aria-labelledby="pivotTitle">
<div class="panel-heading">
<div><h2 id="pivotTitle">包名 × 日期收入透视表</h2><p><code>attr_platform</code> 作为筛选列,<code>Pkg name</code> 作为行,<code>revenue</code> 作为值。</p></div>
</div>
<div class="filters">
<label class="field">归因平台<select id="platformFilter" aria-label="按归因平台筛选"><option value="__all__">全部平台</option></select></label>
<label class="field search">搜索包名<input id="packageSearch" type="search" placeholder="输入包名,例如 com.example.app" autocomplete="off"></label>
<button class="button-secondary" id="clearFilters" type="button">清除筛选</button>
</div>
<div class="data-note" id="dataNote">正在读取查询元数据…</div>
<div class="table-scroll">
<table aria-describedby="dataNote">
<thead id="pivotHead"></thead>
<tbody id="pivotBody"><tr><td class="empty">正在加载…</td></tr></tbody>
</table>
</div>
<nav class="pagination" aria-label="表格分页">
<span id="pageIndicator">—</span>
<button id="previousPage" type="button">上一页</button>
<button id="nextPage" type="button">下一页</button>
</nav>
</section>
<p class="footnote"><strong>差额规则:</strong>T-1 − T-2 大于 100 显示绿色;小于 100 显示红色;等于 100 显示琥珀色。页面数据由 AgentStage 每日 10:45(Asia/Shanghai)自动更新。</p>
</main>
<script>
(() => {
'use strict';
const state = { data: null, platform: '__all__', search: '', page: 1, pageSize: 50 };
const numberFormatter = new Intl.NumberFormat('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 });
const compactFormatter = new Intl.NumberFormat('en-US', { notation: 'compact', maximumFractionDigits: 1 });
const dateFormatter = new Intl.DateTimeFormat('zh-CN', { dateStyle: 'medium', timeStyle: 'short' });
const $ = (id) => document.getElementById(id);
const text = (value) => (value === undefined || value === null ? '' : String(value));
const money = (value) => numberFormatter.format(Number(value || 0));
const shortDate = (date) => date && date.length >= 10 ? date.slice(5).replace('-', '/') : text(date);
const svgNode = (name, attributes = {}) => {
const node = document.createElementNS('http://www.w3.org/2000/svg', name);
Object.entries(attributes).forEach(([key, value]) => node.setAttribute(key, value));
return node;
};
function dataDates() {
const fromFile = Array.isArray(state.data?.dates) ? state.data.dates : [];
const fallback = [...new Set((state.data?.rows || []).map((row) => row.date))];
return [...(fromFile.length ? fromFile : fallback)].sort();
}
function selectedRows() {
const query = state.search.trim().toLowerCase();
return (state.data?.rows || []).filter((row) => {
const platformMatches = state.platform === '__all__' || row.attr_platform === state.platform;
const packageMatches = !query || text(row.pkg_name).toLowerCase().includes(query);
return platformMatches && packageMatches;
});
}
function buildPivot(rows, dates) {
const packages = new Map();
rows.forEach((row) => {
const name = text(row.pkg_name) || '(未标记)';
if (!packages.has(name)) packages.set(name, { name, values: new Map() });
const entry = packages.get(name);
entry.values.set(row.date, (entry.values.get(row.date) || 0) + Number(row.revenue || 0));
});
return [...packages.values()].map((entry) => {
const values = dates.map((date) => entry.values.get(date) || 0);
const latest = values.length ? values[values.length - 1] : 0;
const previous = values.length > 1 ? values[values.length - 2] : null;
return { ...entry, values, latest, difference: previous === null ? null : latest - previous };
}).sort((a, b) => b.latest - a.latest || a.name.localeCompare(b.name));
}
function totalsForDates(rows, dates) {
const totals = new Map(dates.map((date) => [date, 0]));
rows.forEach((row) => totals.set(row.date, (totals.get(row.date) || 0) + Number(row.revenue || 0)));
return dates.map((date) => ({ date, value: totals.get(date) || 0 }));
}
function renderSummary(rows, dates, pivot) {
const totals = totalsForDates(rows, dates);
const latest = totals.length ? totals[totals.length - 1] : null;
$('latestRevenue').textContent = latest ? money(latest.value) : '—';
$('latestRevenueNote').textContent = latest ? `T-1 · ${latest.date}` : '当前筛选无数据';
$('rowCount').textContent = compactFormatter.format(rows.length);
$('platformCount').textContent = compactFormatter.format(new Set(rows.map((row) => row.attr_platform)).size);
$('packageCount').textContent = compactFormatter.format(pivot.length);
}
function renderChart(rows, dates) {
const chart = $('trendChart');
chart.replaceChildren();
const points = totalsForDates(rows, dates);
if (!points.length) {
const message = svgNode('text', { x: 460, y: 150, 'text-anchor': 'middle', fill: '#64748b', 'font-size': 14 });
message.textContent = '当前筛选没有可绘制的收入数据';
chart.append(message);
return;
}
const width = 920, height = 300, left = 70, right = 26, top = 25, bottom = 60;
const plotWidth = width - left - right, plotHeight = height - top - bottom;
const max = Math.max(...points.map((point) => point.value), 1);
const min = 0;
const y = (value) => top + plotHeight - ((value - min) / (max - min || 1)) * plotHeight;
const x = (index) => points.length === 1 ? left + plotWidth / 2 : left + index * (plotWidth / (points.length - 1));
for (let tick = 0; tick <= 4; tick += 1) {
const value = max * (tick / 4);
const yPosition = y(value);
chart.append(svgNode('line', { x1: left, y1: yPosition, x2: width - right, y2: yPosition, stroke: '#e3ebf5', 'stroke-width': 1 }));
const label = svgNode('text', { x: left - 10, y: yPosition + 4, 'text-anchor': 'end', fill: '#7589a3', 'font-size': 11 });
label.textContent = compactFormatter.format(value);
chart.append(label);
}
const linePoints = points.map((point, index) => `${x(index)},${y(point.value)}`).join(' ');
const areaPoints = `${left},${top + plotHeight} ${linePoints} ${width - right},${top + plotHeight}`;
chart.append(svgNode('polygon', { points: areaPoints, fill: 'rgba(37, 99, 235, .10)' }));
chart.append(svgNode('polyline', { points: linePoints, fill: 'none', stroke: '#2563eb', 'stroke-width': 3.5, 'stroke-linecap': 'round', 'stroke-linejoin': 'round' }));
points.forEach((point, index) => {
const cx = x(index), cy = y(point.value);
chart.append(svgNode('circle', { cx, cy, r: 4.5, fill: '#fff', stroke: '#2563eb', 'stroke-width': 3 }));
const label = svgNode('text', { x: cx, y: height - 26, 'text-anchor': 'middle', fill: '#5c718d', 'font-size': 11, 'font-weight': 700 });
label.textContent = shortDate(point.date);
chart.append(label);
const title = svgNode('title');
title.textContent = `${point.date}: ${money(point.value)}`;
chart.lastChild.append(title);
});
}
function renderTable(rows, dates, pivot) {
const head = $('pivotHead');
const body = $('pivotBody');
head.replaceChildren();
body.replaceChildren();
const headerRow = document.createElement('tr');
const packageHeader = document.createElement('th');
packageHeader.textContent = 'Pkg name';
headerRow.append(packageHeader);
dates.forEach((date, index) => {
const cell = document.createElement('th');
const relative = index === dates.length - 1 ? 'T-1' : index === dates.length - 2 ? 'T-2' : '';
cell.textContent = relative ? `${relative} · ${date}` : date;
headerRow.append(cell);
});
const differenceHeader = document.createElement('th');
differenceHeader.textContent = 'T-1 − T-2';
headerRow.append(differenceHeader);
head.append(headerRow);
const pageCount = Math.max(1, Math.ceil(pivot.length / state.pageSize));
if (state.page > pageCount) state.page = pageCount;
const start = (state.page - 1) * state.pageSize;
const pageRows = pivot.slice(start, start + state.pageSize);
if (!pageRows.length) {
const empty = document.createElement('tr');
const cell = document.createElement('td');
cell.className = 'empty';
cell.colSpan = dates.length + 2;
cell.textContent = '没有匹配当前筛选条件的包名';
empty.append(cell);
body.append(empty);
}
pageRows.forEach((item) => {
const row = document.createElement('tr');
const name = document.createElement('td');
name.textContent = item.name;
row.append(name);
item.values.forEach((value) => {
const cell = document.createElement('td');
cell.textContent = money(value);
row.append(cell);
});
const difference = document.createElement('td');
if (item.difference === null) {
difference.textContent = '—';
} else {
difference.textContent = `${item.difference > 0 ? '+' : ''}${money(item.difference)}`;
difference.className = `diff ${item.difference > 100 ? 'positive' : item.difference < 100 ? 'negative' : 'neutral'}`;
}
row.append(difference);
body.append(row);
});
$('pageIndicator').textContent = `第 ${state.page} / ${pageCount} 页 · ${pivot.length.toLocaleString('en-US')} 个包名`;
$('previousPage').disabled = state.page <= 1;
$('nextPage').disabled = state.page >= pageCount;
}
function render() {
const dates = dataDates();
const rows = selectedRows();
const pivot = buildPivot(rows, dates);
renderSummary(rows, dates, pivot);
renderChart(rows, dates);
renderTable(rows, dates, pivot);
}
function populatePlatforms() {
const select = $('platformFilter');
const platforms = Array.isArray(state.data?.platforms) ? state.data.platforms : [...new Set((state.data?.rows || []).map((row) => row.attr_platform))].sort();
select.replaceChildren();
const all = document.createElement('option');
all.value = '__all__';
all.textContent = '全部平台';
select.append(all);
platforms.forEach((platform) => {
const option = document.createElement('option');
option.value = platform;
option.textContent = platform;
select.append(option);
});
}
function renderMetadata() {
const generated = state.data?.generatedAt ? dateFormatter.format(new Date(state.data.generatedAt)) : '未知';
const source = state.data?.source || {};
const dates = dataDates();
const dateLabel = dates.length ? dates.join(' ~ ') : '无可用日期';
$('dataStatus').textContent = `已更新 · ${generated}`;
$('dataNote').textContent = `查询范围:${source.timeRange || '—'} | 数据日期:${dateLabel} | 已返回 ${Number(state.data?.rawRowCount || 0).toLocaleString('en-US')} 行 | 行数上限:${source.rowLimit || 5000}`;
}
async function load() {
try {
const response = await fetch('data.json', { cache: 'no-store' });
if (!response.ok) throw new Error(`data.json 返回 HTTP ${response.status}`);
const data = await response.json();
if (!Array.isArray(data.rows)) throw new Error('data.json 缺少 rows 数组');
state.data = data;
populatePlatforms();
renderMetadata();
render();
} catch (error) {
$('dataStatus').textContent = '数据加载失败';
$('dataStatus').classList.add('error');
$('dataNote').textContent = `无法读取数据:${error.message}`;
$('pivotBody').replaceChildren();
const row = document.createElement('tr');
const cell = document.createElement('td');
cell.className = 'empty error';
cell.textContent = '请等待下一次自动更新,或检查 update.sh 执行日志。';
row.append(cell);
$('pivotBody').append(row);
}
}
$('platformFilter').addEventListener('change', (event) => { state.platform = event.target.value; state.page = 1; render(); });
$('packageSearch').addEventListener('input', (event) => { state.search = event.target.value; state.page = 1; render(); });
$('clearFilters').addEventListener('click', () => {
state.platform = '__all__'; state.search = ''; state.page = 1;
$('platformFilter').value = '__all__'; $('packageSearch').value = ''; render();
});
$('previousPage').addEventListener('click', () => { state.page -= 1; render(); });
$('nextPage').addEventListener('click', () => { state.page += 1; render(); });
load();
})();
</script>
</body>
</html>
+1
View File
@@ -0,0 +1 @@
1
Executable
+251
View File
@@ -0,0 +1,251 @@
#!/usr/bin/env bash
set -euo pipefail
# AgentStage invokes this script hourly (see schedule.conf). Only the 10:00
# Asia/Shanghai run proceeds, waiting until 10:45 when needed, so data refreshes
# once per day at 10:45. Set FORCE_UPDATE=1 to refresh immediately.
REPORT_TIMEZONE="${REPORT_TIMEZONE:-Asia/Shanghai}"
if [[ "${FORCE_UPDATE:-0}" != "1" ]]; then
current_hour="$(TZ="$REPORT_TIMEZONE" date +%H)"
current_minute="$(TZ="$REPORT_TIMEZONE" date +%M)"
if [[ "$current_hour" != "10" ]]; then
echo "Skipping refresh outside the 10:45 ${REPORT_TIMEZONE} window."
exit 0
fi
minute_value=$((10#$current_minute))
if (( minute_value < 45 )); then
sleep_seconds=$(( (45 - minute_value) * 60 ))
echo "Waiting ${sleep_seconds}s for 10:45 ${REPORT_TIMEZONE}."
sleep "$sleep_seconds"
fi
fi
# Environment variables take precedence. The fallback values allow AgentStage's
# scheduled runner (which has no per-repository secret UI) to refresh this
# private source repository unattended.
SUPERSET_USERNAME="${SUPERSET_USERNAME:-zhangyujia}"
SUPERSET_PASSWORD="${SUPERSET_PASSWORD:-Yuting081425}"
SCRIPT_DIR="$(dirname "$0")"
python3 - "$SUPERSET_USERNAME" "$SUPERSET_PASSWORD" "$REPORT_TIMEZONE" "$SCRIPT_DIR" <<'PY'
from __future__ import annotations
import copy
import html
import http.cookiejar
import json
import re
import sys
from datetime import datetime, timedelta
from pathlib import Path
from urllib.parse import urlencode
from urllib.request import HTTPCookieProcessor, Request, build_opener
try:
from zoneinfo import ZoneInfo
except ImportError: # pragma: no cover - Python 3.9+ is expected on AgentStage
ZoneInfo = None
username, password, timezone_name, output_dir = sys.argv[1:]
base_url = "https://ads-bi.ushareit.org"
dashboard_url = f"{base_url}/superset/dashboard/107/"
chart_title = "归因dsp统计 - dsp/包/国家"
row_limit = 5000
def read_response(response) -> str:
return response.read().decode("utf-8", errors="replace")
def to_number(value) -> float:
try:
return float(value or 0)
except (TypeError, ValueError):
return 0.0
def normalise_date(value) -> str:
raw = str(value or "").strip()
if len(raw) >= 8 and raw[:8].isdigit():
return f"{raw[:4]}-{raw[4:6]}-{raw[6:8]}"
return raw[:10]
try:
tz = ZoneInfo(timezone_name) if ZoneInfo else None
except Exception:
tz = None
timezone_name = "UTC"
now = datetime.now(tz)
today = now.date()
# T-5 through T-1. The end boundary is today because Superset treats it as the
# exclusive business-day boundary for this chart's shifted time dimension.
start_date = today - timedelta(days=5)
time_range = f"{start_date.isoformat()} : {today.isoformat()}"
# The chart's shifted `dt` values label the five returned business days.
requested_dates = [
(today - timedelta(days=offset)).isoformat() for offset in range(4, -1, -1)
]
cookie_jar = http.cookiejar.CookieJar()
opener = build_opener(HTTPCookieProcessor(cookie_jar))
try:
with opener.open(dashboard_url, timeout=30) as response:
login_page_url = response.geturl()
login_page = read_response(response)
csrf_match = re.search(
r'name=["\']csrf_token["\'][^>]*value=["\']([^"\']+)',
login_page,
re.IGNORECASE,
)
if not csrf_match:
raise RuntimeError("Superset login page did not provide a CSRF token.")
login_body = urlencode(
{
"csrf_token": csrf_match.group(1),
"username": username,
"password": password,
}
).encode("utf-8")
login_request = Request(
login_page_url,
data=login_body,
headers={
"Content-Type": "application/x-www-form-urlencoded",
"Referer": login_page_url,
},
method="POST",
)
with opener.open(login_request, timeout=30) as response:
# Consume the response so cookies and redirect state are fully applied.
read_response(response)
with opener.open(dashboard_url, timeout=30) as response:
dashboard_html = read_response(response)
bootstrap_match = re.search(
r'<div\b[^>]*\bid=["\']app["\'][^>]*\bdata-bootstrap=["\'](.*?)["\']',
dashboard_html,
re.IGNORECASE | re.DOTALL,
)
if not bootstrap_match:
raise RuntimeError("Could not read Superset dashboard bootstrap data.")
bootstrap = json.loads(html.unescape(bootstrap_match.group(1)))
slices = bootstrap["dashboard_data"]["slices"]
source_form_data = next(
item["form_data"]
for item in slices
if item.get("slice_name") == chart_title
)
# Same Explore configuration requested in the report brief: remove nation,
# retain date/platform/package, and raise the row limit to 5,000.
form_data = copy.deepcopy(source_form_data)
form_data["groupby"] = [
value for value in form_data.get("groupby", []) if value != "nation"
]
form_data["row_limit"] = row_limit
form_data["time_range"] = time_range
# The Explore result is ordered by the latest day. If a five-day result hits
# 5,000 rows, older requested dates are otherwise cut off. Keep the required
# 5,000-row limit on each chart query, query each adjacent business-day
# window, and retain only its upper-boundary `dt` records.
records = []
queried_row_count = 0
daily_query_windows = []
for offset in range(5, 0, -1):
window_start = today - timedelta(days=offset)
window_end = window_start + timedelta(days=1)
window_form_data = copy.deepcopy(form_data)
window_form_data["time_range"] = (
f"{window_start.isoformat()} : {window_end.isoformat()}"
)
query_url = f"{base_url}/superset/explore_json/?" + urlencode(
{
"form_data": json.dumps(
window_form_data, ensure_ascii=False, separators=(",", ":")
),
"force": "true",
}
)
with opener.open(query_url, timeout=180) as response:
query_result = json.loads(read_response(response))
if query_result.get("errors"):
raise RuntimeError(
f"Superset query returned errors: {query_result['errors']}"
)
window_records = query_result.get("data", {}).get("records", [])
queried_row_count += len(window_records)
daily_query_windows.append(window_form_data["time_range"])
target_date = window_end.isoformat()
records.extend(
record
for record in window_records
if normalise_date(record.get("dt")) == target_date
)
rows = []
for record in records:
date = normalise_date(record.get("dt"))
if not date:
continue
rows.append(
{
"date": date,
"attr_platform": str(record.get("attr_platform") or "(未标记)").strip(),
"pkg_name": str(record.get("pkg name(EN)") or "(未标记)").strip(),
"click": to_number(record.get("click")),
"install": to_number(record.get("installs(还原前)")),
"revenue": to_number(record.get("revenue(还原前)")),
}
)
rows.sort(key=lambda item: (item["date"], item["attr_platform"], item["pkg_name"]))
dates = sorted({item["date"] for item in rows})
platforms = sorted({item["attr_platform"] for item in rows}, key=str.casefold)
output = {
"generatedAt": datetime.now(tz).isoformat(timespec="seconds"),
"timezone": timezone_name,
"source": {
"dashboardId": 107,
"chartId": 821,
"chartTitle": chart_title,
"timeRange": time_range,
"requestedDates": requested_dates,
"dailyQueryWindows": daily_query_windows,
"rowLimit": row_limit,
"groupBy": form_data["groupby"],
"metrics": ["click", "install", "revenue"],
},
"rawRowCount": len(records),
"queriedRowCount": queried_row_count,
"dates": dates,
"platforms": platforms,
"rows": rows,
}
destination = Path(output_dir) / "data.json"
temporary = destination.with_suffix(".json.tmp")
temporary.write_text(
json.dumps(output, ensure_ascii=False, separators=(",", ":")),
encoding="utf-8",
)
temporary.replace(destination)
print(
f"Updated {destination.name}: {len(rows)} rows, "
f"{len(dates)} data dates, range {time_range}."
)
except Exception as error:
print(f"DSP report update failed: {error}", file=sys.stderr)
raise
PY