fix(bms): #335 return_tracking 导出请求 ISO 日期 route 改写
Some checks failed
gitleaks / gitleaks (push) Failing after 10s
backend-test / commit-message-lint (push) Successful in 2s
backend-test / analytics-ods-isolation (push) Failing after 3s
backend-test / frontend-lint-test (push) Has been cancelled
backend-test / backend-lint-test (push) Has been cancelled
Some checks failed
gitleaks / gitleaks (push) Failing after 10s
backend-test / commit-message-lint (push) Successful in 2s
backend-test / analytics-ods-isolation (push) Failing after 3s
backend-test / frontend-lint-test (push) Has been cancelled
backend-test / backend-lint-test (push) Has been cancelled
根因(生产实证 2026-10-19): BMS 后端 ~10-11 起收紧导出接口日期反序列化, 只收 yyyy-MM-dd HH:mm:ss; 而 return_tracking 页面导出把日期选择器 v-model(JS Date 对象)直接 JSON 序列化为 ISO-8601 UTC 串(实锤 payload: 2026-10-19T01:41:07.000Z = 北京 09:41:07 窗口终点) → BMS 400 JSON parse error, 该源自 10-11 起每次失败。 修法: 导出前对白名单源(仅 return_tracking)注册 page.route 拦截, 把请求 体中完整匹配 ISO 形态的日期串改写为北京时 yyyy-MM-dd HH:mm:ss 后放行。 非 POST / 非 JSON / 无 ISO 日期一律原样放行; 改写异常不阻断导出。 - 改写逻辑 _rewrite_iso_dates_in_json 递归处理嵌套 dict/list, 只改完整 匹配 ISO 的字符串值(防误伤长文本内嵌日期子串), 返回改写处数 - _download_with_retry 拆出重试循环体 _download_retry_loop, 外层套 route 注册/卸载 try/finally; 公共签名不变, 两处调用点零改动 - 新增 6 用例单测锁定: Z/偏移/跨日换算、嵌套计数、非 ISO 形态不改写、 白名单契约
This commit is contained in:
@@ -12,6 +12,7 @@ from app.services.bms_fetcher._browser import (
|
||||
_DOWNLOAD_TIMEOUT_MS,
|
||||
_EMPTY_STATE_SELECTORS,
|
||||
_EXPORT_ANY_CLICKABLE,
|
||||
_EXPORT_REWRITE_ISO_DATE_TYPES,
|
||||
BmsSessionExpired,
|
||||
EmptyQueryResult,
|
||||
_click_export_button,
|
||||
@@ -21,6 +22,7 @@ from app.services.bms_fetcher._browser import (
|
||||
_locate_export_button,
|
||||
_page_shows_no_data,
|
||||
_parse_filename_utc_ts,
|
||||
_rewrite_iso_dates_in_json,
|
||||
)
|
||||
from app.services.bms_fetcher._fetch import (
|
||||
_BMS_FILL_TZ_UTC_DEFAULT,
|
||||
@@ -79,6 +81,7 @@ __all__ = [
|
||||
'_DOWNLOAD_TIMEOUT_MS',
|
||||
'_EMPTY_STATE_SELECTORS',
|
||||
'_EXPORT_ANY_CLICKABLE',
|
||||
'_EXPORT_REWRITE_ISO_DATE_TYPES',
|
||||
'_FETCH_LEAD_SECONDS',
|
||||
'_LOGIN_CLASS_KEYS',
|
||||
'_OCR_AVG_PROB',
|
||||
@@ -98,6 +101,7 @@ __all__ = [
|
||||
'_locate_export_button',
|
||||
'_page_shows_no_data',
|
||||
'_parse_filename_utc_ts',
|
||||
'_rewrite_iso_dates_in_json',
|
||||
'_poll_login_errors_for',
|
||||
'_read_login_error',
|
||||
'_refresh_captcha_image',
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
"""浏览器交互工具: 弹窗关闭 / 按钮定位点击 / 无数据判定 / 下载重试(不含登录)。"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
|
||||
def _parse_filename_utc_ts(filename: str) -> datetime | None:
|
||||
@@ -66,6 +68,91 @@ _DOWNLOAD_MAX_RETRIES: dict[str, int] = {
|
||||
_DOWNLOAD_DEFAULT_RETRIES = 1
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# #335: 导出请求 ISO 日期改写(route 拦截)
|
||||
# 背景: BMS 后端 2026-10-11 起收紧导出接口日期反序列化, 只收
|
||||
# "yyyy-MM-dd HH:mm:ss"; 而 return_tracking 页面导出把日期选择器
|
||||
# v-model(JS Date 对象)直接随 JSON 序列化 → ISO-8601 UTC 串
|
||||
# (实证: bms_sync_logs 报错 payload 含 "2026-10-19T01:41:07.000Z"
|
||||
# = 北京 09:41:07 窗口终点) → BMS 400 JSON parse error。
|
||||
# 填值侧无法根治(组件 v-model 恒为 Date, 页面自有序列化逻辑不可控),
|
||||
# 故对白名单源在导出前注册 page.route 拦截, 把请求体里的 ISO 日期
|
||||
# 字符串改写为北京时 "yyyy-MM-dd HH:mm:ss" 后放行。
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
_EXPORT_REWRITE_ISO_DATE_TYPES: frozenset = frozenset({"return_tracking"})
|
||||
|
||||
_TZ_CN = ZoneInfo("Asia/Shanghai")
|
||||
|
||||
# 完整匹配 ISO-8601 日期时间(允许小数秒 / Z / ±hh:mm 偏移), 日期部分单独不匹配
|
||||
_ISO_DT_RE = re.compile(
|
||||
r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$"
|
||||
)
|
||||
|
||||
|
||||
def _rewrite_iso_dates_in_json(node):
|
||||
"""递归把 JSON 结构里的 ISO-8601 日期时间字符串改写为北京时 "yyyy-MM-dd HH:mm:ss"。
|
||||
|
||||
返回 (新节点, 改写处数)。只改写**完整匹配** ISO 形态的字符串值,
|
||||
其它类型/普通字符串一律原样保留(不改写嵌在长文本里的日期子串, 防误伤)。
|
||||
"""
|
||||
if isinstance(node, dict):
|
||||
cnt = 0
|
||||
out = {}
|
||||
for k, v in node.items():
|
||||
nv, c = _rewrite_iso_dates_in_json(v)
|
||||
out[k] = nv
|
||||
cnt += c
|
||||
return out, cnt
|
||||
if isinstance(node, list):
|
||||
cnt = 0
|
||||
out = []
|
||||
for v in node:
|
||||
nv, c = _rewrite_iso_dates_in_json(v)
|
||||
out.append(nv)
|
||||
cnt += c
|
||||
return out, cnt
|
||||
if isinstance(node, str) and _ISO_DT_RE.match(node):
|
||||
try:
|
||||
dt = datetime.fromisoformat(node.replace("Z", "+00:00"))
|
||||
if dt.tzinfo is None: # 无偏移按 UTC 处理(与 JS toISOString 语义一致)
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(_TZ_CN).strftime("%Y-%m-%d %H:%M:%S"), 1
|
||||
except Exception:
|
||||
return node, 0
|
||||
return node, 0
|
||||
|
||||
|
||||
async def _rewrite_export_request_iso_dates(route, request, logger_obj, data_type: str):
|
||||
"""page.route 处理器: 改写导出 POST 请求体里的 ISO 日期, 其余请求原样放行。
|
||||
|
||||
任何解析/改写异常都不阻断导出——按原样 continue, 由 BMS 响应决定成败
|
||||
(改写是「修复已知坏形态」, 不是「新增前置校验」)。
|
||||
"""
|
||||
try:
|
||||
if (request.method or "").upper() != "POST":
|
||||
await route.continue_()
|
||||
return
|
||||
post_data = request.post_data
|
||||
if not post_data or "T" not in post_data: # 快路径: 无 T 必无 ISO 日期
|
||||
await route.continue_()
|
||||
return
|
||||
body = json.loads(post_data)
|
||||
new_body, n = _rewrite_iso_dates_in_json(body)
|
||||
if not n:
|
||||
await route.continue_()
|
||||
return
|
||||
logger_obj.info(
|
||||
"[%s] 导出请求 ISO 日期改写 %d 处(#335: BMS 后端只收 yyyy-MM-dd HH:mm:ss)",
|
||||
data_type, n,
|
||||
)
|
||||
await route.continue_(post_data=json.dumps(new_body, ensure_ascii=False))
|
||||
except Exception:
|
||||
try:
|
||||
await route.continue_()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
|
||||
# 导出/查询按钮候选 DOM 选择器(与查询按钮 ANY_CLICKABLE 一致)。
|
||||
# ⚠️ 2026-09-06 事故复盘(3d48c1f 回归): 这里只作为"遍历候选"的集合(每个都带按钮
|
||||
@@ -448,6 +535,46 @@ async def _download_with_retry(
|
||||
"""
|
||||
timeout_ms = _DOWNLOAD_TIMEOUT_MS.get(data_type, _DOWNLOAD_DEFAULT_TIMEOUT_MS)
|
||||
max_retries = _DOWNLOAD_MAX_RETRIES.get(data_type, _DOWNLOAD_DEFAULT_RETRIES)
|
||||
|
||||
# #335: 白名单源注册导出请求 ISO 日期改写(route 拦截, 详见上方常量注释)。
|
||||
# 拦截面=URL 含 export 的请求; 非 POST/非 JSON/无 ISO 日期一律原样放行。
|
||||
_iso_route_pattern = re.compile(r"export", re.IGNORECASE)
|
||||
_iso_route_registered = False
|
||||
|
||||
async def _iso_route_handler(route, request):
|
||||
await _rewrite_export_request_iso_dates(route, request, logger_obj, data_type)
|
||||
|
||||
if data_type in _EXPORT_REWRITE_ISO_DATE_TYPES:
|
||||
try:
|
||||
await page.route(_iso_route_pattern, _iso_route_handler)
|
||||
_iso_route_registered = True
|
||||
except Exception as _route_err:
|
||||
logger_obj.warning(
|
||||
"[%s] 注册 ISO 日期改写 route 失败(忽略,按原样导出): %s",
|
||||
data_type, _route_err,
|
||||
)
|
||||
|
||||
try:
|
||||
return await _download_retry_loop(
|
||||
page, data_type, logger_obj, timeout_ms, max_retries, expect_end_dt,
|
||||
)
|
||||
finally:
|
||||
if _iso_route_registered:
|
||||
try:
|
||||
await page.unroute(_iso_route_pattern, _iso_route_handler)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def _download_retry_loop(
|
||||
page,
|
||||
data_type: str,
|
||||
logger_obj,
|
||||
timeout_ms: int,
|
||||
max_retries: int,
|
||||
expect_end_dt: datetime | None = None,
|
||||
):
|
||||
"""_download_with_retry 的重试循环体(#335 拆出以便在外层套 route 拦截的 try/finally)。"""
|
||||
last_exc: Exception | None = None
|
||||
|
||||
for attempt in range(0, max_retries + 1):
|
||||
|
||||
81
backend/tests/test_bms_export_iso_date_rewrite.py
Normal file
81
backend/tests/test_bms_export_iso_date_rewrite.py
Normal file
@@ -0,0 +1,81 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""#335 导出请求 ISO 日期改写测试(2026-10-19)
|
||||
|
||||
背景: BMS 后端 2026-10-11 起收紧导出接口日期反序列化, 只收 "yyyy-MM-dd HH:mm:ss";
|
||||
return_tracking 页面导出把日期选择器 v-model(JS Date)直接 JSON 序列化为
|
||||
ISO-8601 UTC 串(实证 payload: "2026-10-19T01:41:07.000Z") → BMS 400。
|
||||
修法: page.route 拦截导出请求, _rewrite_iso_dates_in_json 把 ISO 日期改写为
|
||||
北京时格式化串后放行。
|
||||
|
||||
本测试锁定:
|
||||
1. ISO-8601(Z / 带偏移 / 带毫秒) → 北京时 "yyyy-MM-dd HH:mm:ss"(含跨日换算)
|
||||
2. 嵌套 dict/list 递归改写 + 改写处数统计
|
||||
3. 非 ISO 形态(纯日期/空格分隔/普通字符串/数字/None)一律不改写(防误伤)
|
||||
4. return_tracking 在改写白名单内(防配置漂移回归)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.services.bms_fetcher import (
|
||||
_EXPORT_REWRITE_ISO_DATE_TYPES,
|
||||
_rewrite_iso_dates_in_json,
|
||||
)
|
||||
|
||||
|
||||
def test_iso_z_to_beijing():
|
||||
"""ISO Z(UTC) → 北京时(+8), 秒级对齐生产实证 payload"""
|
||||
out, n = _rewrite_iso_dates_in_json("2026-10-19T01:41:07.000Z")
|
||||
assert out == "2026-10-19 09:41:07"
|
||||
assert n == 1
|
||||
|
||||
|
||||
def test_iso_z_cross_day():
|
||||
"""UTC 16:00 = 北京次日 00:00(跨日换算, 对应 10-11 失败样本)"""
|
||||
out, n = _rewrite_iso_dates_in_json("2026-10-11T16:00:00.000Z")
|
||||
assert out == "2026-10-12 00:00:00"
|
||||
assert n == 1
|
||||
|
||||
|
||||
def test_iso_with_offset():
|
||||
"""带 +08:00 偏移的 ISO 同样改写(不重复加时差)"""
|
||||
out, n = _rewrite_iso_dates_in_json("2026-10-19T09:41:07+08:00")
|
||||
assert out == "2026-10-19 09:41:07"
|
||||
assert n == 1
|
||||
|
||||
|
||||
def test_nested_dict_and_list():
|
||||
"""生产 payload 形态: 窗口起终点嵌在请求体字段里, 递归改写并计数"""
|
||||
body = {
|
||||
"startTime": "2026-10-11T16:00:00.000Z",
|
||||
"endTime": "2026-10-19T01:41:07.000Z",
|
||||
"pageNo": 1,
|
||||
"ext": [{"signTime": "2026-10-15T00:00:00Z"}],
|
||||
}
|
||||
out, n = _rewrite_iso_dates_in_json(body)
|
||||
assert out == {
|
||||
"startTime": "2026-10-12 00:00:00",
|
||||
"endTime": "2026-10-19 09:41:07",
|
||||
"pageNo": 1,
|
||||
"ext": [{"signTime": "2026-10-15 08:00:00"}],
|
||||
}
|
||||
assert n == 3
|
||||
|
||||
|
||||
def test_non_iso_untouched():
|
||||
"""非 ISO 形态一律原样保留, 改写处数=0"""
|
||||
body = {
|
||||
"plainDate": "2026-10-19", # 纯日期不改
|
||||
"already": "2026-10-19 09:41:07", # 已是目标格式不改
|
||||
"text": "导出时间2026-10-19T01:41:07.000Z内嵌", # 长文本内嵌不改(防误伤)
|
||||
"num": 20261019,
|
||||
"none": None,
|
||||
"bool": True,
|
||||
}
|
||||
out, n = _rewrite_iso_dates_in_json(body)
|
||||
assert out == body
|
||||
assert n == 0
|
||||
|
||||
|
||||
def test_return_tracking_in_whitelist():
|
||||
"""return_tracking 必须在改写白名单(防配置漂移回归)"""
|
||||
assert "return_tracking" in _EXPORT_REWRITE_ISO_DATE_TYPES
|
||||
Reference in New Issue
Block a user