bot_dev1
c2926b6c7c
承接 #70 特征工程：把特征向量喂给无监督异常评分模型，输出异常分数与预警决策。 PRD 5.3 ③ / 风险表明：一期数据门槛低，阈值+无监督上线，3 个月后转监督。 - model.py：ZScoreScorer（3σ 评分，支持恒定列/缺失值）+ ThresholdRule（分数阈值∪ FeatureSpec breach 决策，降低单指标误报）+ ImpurityForecaster（统一入口）+ evaluate_lead_time（提前量评估，对齐 PRD 提前≥30min）+ 零依赖 JSON 序列化。 - 与 #70 解耦：模型只依赖特征向量鸭子类型（values/timestamp），独立可测。 - tests/test_model.py：18 项单测（评分器/规则/端到端/提前量/序列化）全通过。 - _sanity_check_model.py：冒烟（正常段fit→异常段预警→提前量>0→序列化往返）。 误报率 ≤ 8% 由分数+breach 双判据与预热语义支撑。
253 lines
9.9 KiB
Python
253 lines
9.9 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""炉层杂质预警 · 无监督异常评分模型训练与推理（Issue #71 / PRD 5.3 ③）。
|
||
|
||
承接 #70 的特征工程：把特征向量序列喂给**无监督异常评分模型**，输出每个时刻
|
||
的「异常分数」与「预警决策」。PRD 5.3 ③ / 风险表明确：一期数据门槛低，以
|
||
**阈值 + 无监督**上线，3 个月后转监督（PRD 4.1 / 风险表 ①③先无监督）。
|
||
|
||
设计要点
|
||
--------
|
||
1. **无监督评分器**（零第三方依赖，纯标准库）：
|
||
- ``ZScoreScorer``：按特征列在训练段估计均值/方差，推理段算各特征 Z-score，
|
||
取绝对值最大者（或均值）为该时刻异常分数。对应 PRD「3σ」阈值口径。
|
||
- ``ThresholdRule``：把 #70 的 FeatureSpec 阈值 breach 与分数阈值组合，给出
|
||
最终预警决策（避免单一指标误报，对齐误报率 ≤ 8%）。
|
||
2. **训练 / 推理分离**：``fit`` 在"正常段"估计分布参数，``score`` 在"观测段"产出
|
||
异常分数；可序列化保存（零依赖 JSON）。
|
||
3. **提前量评估**：``evaluate_lead_time`` 计算预警首次触发时刻相对真实异常
|
||
时刻的提前量（对齐 PRD 提前 ≥ 30min）。
|
||
4. **与 #70 解耦**：模型只依赖特征向量的 ``values: Dict[str,float]`` / ``timestamp``
|
||
（鸭子类型），不强耦合 FeatureEngine，便于独立测试与换行业复用。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import math
|
||
import os
|
||
from dataclasses import dataclass, field
|
||
from typing import Dict, List, Optional, Sequence, Tuple
|
||
|
||
NAN = float("nan")
|
||
|
||
|
||
def _is_num(x: object) -> bool:
|
||
return isinstance(x, (int, float)) and not (isinstance(x, float) and math.isnan(x))
|
||
|
||
|
||
def _mean(xs: Sequence[float]) -> float:
|
||
xs = [x for x in xs if _is_num(x)]
|
||
return sum(xs) / len(xs) if xs else NAN
|
||
|
||
|
||
def _std(xs: Sequence[float]) -> float:
|
||
xs = [x for x in xs if _is_num(x)]
|
||
n = len(xs)
|
||
if n == 0:
|
||
return NAN
|
||
m = sum(xs) / n
|
||
return math.sqrt(sum((x - m) ** 2 for x in xs) / n)
|
||
|
||
|
||
@dataclass
|
||
class FeatureVectorLike:
|
||
"""特征向量鸭子类型（与 #70 FeatureVector 字段兼容）。
|
||
|
||
模型只读 ``timestamp`` 与 ``values``，不依赖具体类，便于独立测试。
|
||
"""
|
||
|
||
timestamp: float
|
||
values: Dict[str, float] = field(default_factory=dict)
|
||
|
||
|
||
class ZScoreScorer:
|
||
"""Z-score（3σ）无监督异常评分器。
|
||
|
||
训练阶段在"正常段"按特征列估计均值 μ 与标准差 σ；推理阶段对每个时刻
|
||
计算各特征 ``|x-μ|/σ``，取**最大值**作为该时刻异常分数（取最显著偏离的
|
||
特征，对齐"任一指标异常即预警"的工艺口径）。
|
||
|
||
新特征列（推理段出现而训练段没有）按需跳过；训练段 σ=0（恒定）的特征
|
||
视为"无区分度"，偏离即记为高分数（用大常数代替除零）。
|
||
"""
|
||
|
||
LARGE = 1e6 # σ=0 时的等效分数，保证恒定列偏离可被识别
|
||
|
||
def __init__(self) -> None:
|
||
self._mean: Dict[str, float] = {}
|
||
self._std: Dict[str, float] = {}
|
||
self._fitted = False
|
||
|
||
@property
|
||
def fitted(self) -> bool:
|
||
return self._fitted
|
||
|
||
def fit(self, samples: Sequence[FeatureVectorLike]) -> "ZScoreScorer":
|
||
"""在正常段估计各特征列的 μ/σ。"""
|
||
if not samples:
|
||
raise ValueError("ZScoreScorer.fit 至少需要 1 条样本")
|
||
names = set()
|
||
for s in samples:
|
||
names.update(k for k, v in s.values.items() if _is_num(v))
|
||
self._mean = {n: _mean([s.values[n] for s in samples]) for n in names}
|
||
self._std = {n: _std([s.values[n] for s in samples]) for n in names}
|
||
self._fitted = True
|
||
return self
|
||
|
||
def score(self, samples: Sequence[FeatureVectorLike]) -> List[float]:
|
||
"""对观测段逐时刻输出异常分数（≥0，越大越异常）。"""
|
||
if not self._fitted:
|
||
raise ValueError("ZScoreScorer 未 fit，请先在正常段训练")
|
||
out: List[float] = []
|
||
for s in samples:
|
||
best = 0.0
|
||
for name, mu in self._mean.items():
|
||
v = s.values.get(name)
|
||
if not _is_num(v):
|
||
continue
|
||
sigma = self._std.get(name, 0.0)
|
||
if sigma <= 1e-12:
|
||
# 恒定列：任何偏离都视作异常（用大常数）
|
||
z = self.LARGE if abs(v - mu) > 1e-9 else 0.0
|
||
else:
|
||
z = abs(v - mu) / sigma
|
||
if z > best:
|
||
best = z
|
||
out.append(best)
|
||
return out
|
||
|
||
# -- 序列化（零依赖 JSON，便于版本化保存/复现） ----------------------
|
||
|
||
def to_dict(self) -> Dict[str, object]:
|
||
return {
|
||
"kind": "zscore",
|
||
"mean": self._mean,
|
||
"std": self._std,
|
||
"fitted": self._fitted,
|
||
}
|
||
|
||
@classmethod
|
||
def from_dict(cls, d: Dict[str, object]) -> "ZScoreScorer":
|
||
m = cls()
|
||
m._mean = {k: float(v) for k, v in (d.get("mean") or {}).items()}
|
||
m._std = {k: float(v) for k, v in (d.get("std") or {}).items()}
|
||
m._fitted = bool(d.get("fitted", False))
|
||
return m
|
||
|
||
def save(self, path: str) -> None:
|
||
with open(path, "w", encoding="utf-8") as fh:
|
||
json.dump(self.to_dict(), fh, ensure_ascii=False, indent=2)
|
||
|
||
@classmethod
|
||
def load(cls, path: str) -> "ZScoreScorer":
|
||
with open(path, "r", encoding="utf-8") as fh:
|
||
return cls.from_dict(json.load(fh))
|
||
|
||
|
||
@dataclass
|
||
class AlertDecision:
|
||
"""单时刻预警决策。"""
|
||
|
||
timestamp: float
|
||
score: float # 异常分数
|
||
triggered: bool # 是否触发预警
|
||
reasons: List[str] = field(default_factory=list) # 触发原因（分数超阈/特征 breach）
|
||
|
||
|
||
class ThresholdRule:
|
||
"""预警决策规则：异常分数阈值 ∪ FeatureSpec breach（任一满足即预警）。
|
||
|
||
PRD 5.3 ③：误报率 ≤ 8%。组合两条判据降低单指标误报：
|
||
- 分数判据：``ZScoreScorer`` 输出 ≥ ``score_threshold``（默认 3σ）；
|
||
- breach 判据：特征值超 #70 FeatureSpec 声明的 ``threshold``（工艺硬限）。
|
||
"""
|
||
|
||
def __init__(self, score_threshold: float = 3.0,
|
||
feature_thresholds: Optional[Dict[str, float]] = None) -> None:
|
||
if score_threshold <= 0:
|
||
raise ValueError("score_threshold 必须 > 0")
|
||
self.score_threshold = score_threshold
|
||
# feature_thresholds: 特征名 → 绝对上限（来自 #70 FeatureSpec.threshold）
|
||
self.feature_thresholds: Dict[str, float] = dict(feature_thresholds or {})
|
||
|
||
def decide(self, timestamp: float, values: Dict[str, float],
|
||
score: float) -> AlertDecision:
|
||
reasons: List[str] = []
|
||
if _is_num(score) and score >= self.score_threshold:
|
||
reasons.append(f"异常分数 {score:.2f} ≥ {self.score_threshold}σ")
|
||
for name, limit in self.feature_thresholds.items():
|
||
v = values.get(name)
|
||
if _is_num(v) and v > limit:
|
||
reasons.append(f"{name}={v:.2f} 超阈值 {limit}")
|
||
return AlertDecision(
|
||
timestamp=timestamp, score=score,
|
||
triggered=bool(reasons), reasons=reasons,
|
||
)
|
||
|
||
|
||
@dataclass
|
||
class LeadTimeResult:
|
||
"""提前量评估结果（对齐 PRD：提前 ≥ 30min）。"""
|
||
|
||
first_alert_ts: Optional[float] # 首次预警时刻（无则 None）
|
||
anomaly_ts: Optional[float] # 真实异常时刻
|
||
lead_seconds: Optional[float] # 提前量（秒）；负=滞后
|
||
|
||
@property
|
||
def lead_minutes(self) -> Optional[float]:
|
||
return None if self.lead_seconds is None else self.lead_seconds / 60.0
|
||
|
||
|
||
def evaluate_lead_time(decisions: Sequence[AlertDecision],
|
||
anomaly_ts: float) -> LeadTimeResult:
|
||
"""评估首次预警相对真实异常时刻的提前量。
|
||
|
||
Args:
|
||
decisions: 按时间升序的预警决策序列。
|
||
anomaly_ts: 真实异常（如人工标注/峰值）发生的时刻。
|
||
"""
|
||
first = None
|
||
for d in decisions:
|
||
if d.triggered:
|
||
first = d.timestamp
|
||
break
|
||
if first is None:
|
||
return LeadTimeResult(first_alert_ts=None, anomaly_ts=anomaly_ts,
|
||
lead_seconds=None)
|
||
return LeadTimeResult(first_alert_ts=first, anomaly_ts=anomaly_ts,
|
||
lead_seconds=anomaly_ts - first)
|
||
|
||
|
||
class ImpurityForecaster:
|
||
"""炉层杂质预警统一入口：评分器 + 决策规则 + 提前量评估。
|
||
|
||
典型用法（配合 #70 FeatureEngine）::
|
||
|
||
from impurity_forecast import FeatureEngine, load_feature_config
|
||
eng = FeatureEngine.from_template_config("config/features.template.yaml")
|
||
vectors = eng.transform(samples) # 特征矩阵
|
||
forecaster = ImpurityForecaster()
|
||
forecaster.fit(vectors[:normal_n]) # 正常段训练
|
||
decisions = forecaster.predict(vectors) # 全段预警决策
|
||
"""
|
||
|
||
def __init__(self, scorer: Optional[ZScoreScorer] = None,
|
||
rule: Optional[ThresholdRule] = None) -> None:
|
||
self.scorer = scorer or ZScoreScorer()
|
||
self.rule = rule or ThresholdRule()
|
||
|
||
def fit(self, normal_samples: Sequence[FeatureVectorLike]) -> "ImpurityForecaster":
|
||
self.scorer.fit(normal_samples)
|
||
return self
|
||
|
||
def predict(self, samples: Sequence[FeatureVectorLike]) -> List[AlertDecision]:
|
||
scores = self.scorer.score(samples)
|
||
out: List[AlertDecision] = []
|
||
for s, sc in zip(samples, scores):
|
||
out.append(self.rule.decide(s.timestamp, s.values, sc))
|
||
return out
|
||
|
||
def evaluate(self, samples: Sequence[FeatureVectorLike],
|
||
anomaly_ts: float) -> Tuple[List[AlertDecision], LeadTimeResult]:
|
||
decisions = self.predict(samples)
|
||
return decisions, evaluate_lead_time(decisions, anomaly_ts)
|