253 lines
9.9 KiB
Python
253 lines
9.9 KiB
Python
|
# -*- coding: utf-8 -*-
|
|||
|
|
"""炉层杂质预警 · 无监督异常评分模型训练与推理（Issue #71 / PRD 5.3 ③）。
|
|||
|
|
|
|||
|
|
承接 #70 的特征工程：把特征向量序列喂给**无监督异常评分模型**，输出每个时刻
|
|||
|
|
的「异常分数」与「预警决策」。PRD 5.3 ③ / 风险表明确：一期数据门槛低，以
|
|||
|
|
**阈值 + 无监督**上线，3 个月后转监督（PRD 4.1 / 风险表 ①③先无监督）。
|
|||
|
|
|
|||
|
|
设计要点
|
|||
|
|
--------
|
|||
|
|
1. **无监督评分器**（零第三方依赖，纯标准库）：
|
|||
|
|
- ``ZScoreScorer``：按特征列在训练段估计均值/方差，推理段算各特征 Z-score，
|
|||
|
|
取绝对值最大者（或均值）为该时刻异常分数。对应 PRD「3σ」阈值口径。
|
|||
|
|
- ``ThresholdRule``：把 #70 的 FeatureSpec 阈值 breach 与分数阈值组合，给出
|
|||
|
|
最终预警决策（避免单一指标误报，对齐误报率 ≤ 8%）。
|
|||
|
|
2. **训练 / 推理分离**：``fit`` 在"正常段"估计分布参数，``score`` 在"观测段"产出
|
|||
|
|
异常分数；可序列化保存（零依赖 JSON）。
|
|||
|
|
3. **提前量评估**：``evaluate_lead_time`` 计算预警首次触发时刻相对真实异常
|
|||
|
|
时刻的提前量（对齐 PRD 提前 ≥ 30min）。
|
|||
|
|
4. **与 #70 解耦**：模型只依赖特征向量的 ``values: Dict[str,float]`` / ``timestamp``
|
|||
|
|
（鸭子类型），不强耦合 FeatureEngine，便于独立测试与换行业复用。
|
|||
|
|
"""
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
import math
|
|||
|
|
import os
|
|||
|
|
from dataclasses import dataclass, field
|
|||
|
|
from typing import Dict, List, Optional, Sequence, Tuple
|
|||
|
|
|
|||
|
|
NAN = float("nan")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _is_num(x: object) -> bool:
|
|||
|
|
return isinstance(x, (int, float)) and not (isinstance(x, float) and math.isnan(x))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _mean(xs: Sequence[float]) -> float:
|
|||
|
|
xs = [x for x in xs if _is_num(x)]
|
|||
|
|
return sum(xs) / len(xs) if xs else NAN
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _std(xs: Sequence[float]) -> float:
|
|||
|
|
xs = [x for x in xs if _is_num(x)]
|
|||
|
|
n = len(xs)
|
|||
|
|
if n == 0:
|
|||
|
|
return NAN
|
|||
|
|
m = sum(xs) / n
|
|||
|
|
return math.sqrt(sum((x - m) ** 2 for x in xs) / n)
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass
|
|||
|
|
class FeatureVectorLike:
|
|||
|
|
"""特征向量鸭子类型（与 #70 FeatureVector 字段兼容）。
|
|||
|
|
|
|||
|
|
模型只读 ``timestamp`` 与 ``values``，不依赖具体类，便于独立测试。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
timestamp: float
|
|||
|
|
values: Dict[str, float] = field(default_factory=dict)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ZScoreScorer:
|
|||
|
|
"""Z-score（3σ）无监督异常评分器。
|
|||
|
|
|
|||
|
|
训练阶段在"正常段"按特征列估计均值 μ 与标准差 σ；推理阶段对每个时刻
|
|||
|
|
计算各特征 ``|x-μ|/σ``，取**最大值**作为该时刻异常分数（取最显著偏离的
|
|||
|
|
特征，对齐"任一指标异常即预警"的工艺口径）。
|
|||
|
|
|
|||
|
|
新特征列（推理段出现而训练段没有）按需跳过；训练段 σ=0（恒定）的特征
|
|||
|
|
视为"无区分度"，偏离即记为高分数（用大常数代替除零）。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
LARGE = 1e6 # σ=0 时的等效分数，保证恒定列偏离可被识别
|
|||
|
|
|
|||
|
|
def __init__(self) -> None:
|
|||
|
|
self._mean: Dict[str, float] = {}
|
|||
|
|
self._std: Dict[str, float] = {}
|
|||
|
|
self._fitted = False
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def fitted(self) -> bool:
|
|||
|
|
return self._fitted
|
|||
|
|
|
|||
|
|
def fit(self, samples: Sequence[FeatureVectorLike]) -> "ZScoreScorer":
|
|||
|
|
"""在正常段估计各特征列的 μ/σ。"""
|
|||
|
|
if not samples:
|
|||
|
|
raise ValueError("ZScoreScorer.fit 至少需要 1 条样本")
|
|||
|
|
names = set()
|
|||
|
|
for s in samples:
|
|||
|
|
names.update(k for k, v in s.values.items() if _is_num(v))
|
|||
|
|
self._mean = {n: _mean([s.values[n] for s in samples]) for n in names}
|
|||
|
|
self._std = {n: _std([s.values[n] for s in samples]) for n in names}
|
|||
|
|
self._fitted = True
|
|||
|
|
return self
|
|||
|
|
|
|||
|
|
def score(self, samples: Sequence[FeatureVectorLike]) -> List[float]:
|
|||
|
|
"""对观测段逐时刻输出异常分数（≥0，越大越异常）。"""
|
|||
|
|
if not self._fitted:
|
|||
|
|
raise ValueError("ZScoreScorer 未 fit，请先在正常段训练")
|
|||
|
|
out: List[float] = []
|
|||
|
|
for s in samples:
|
|||
|
|
best = 0.0
|
|||
|
|
for name, mu in self._mean.items():
|
|||
|
|
v = s.values.get(name)
|
|||
|
|
if not _is_num(v):
|
|||
|
|
continue
|
|||
|
|
sigma = self._std.get(name, 0.0)
|
|||
|
|
if sigma <= 1e-12:
|
|||
|
|
# 恒定列：任何偏离都视作异常（用大常数）
|
|||
|
|
z = self.LARGE if abs(v - mu) > 1e-9 else 0.0
|
|||
|
|
else:
|
|||
|
|
z = abs(v - mu) / sigma
|
|||
|
|
if z > best:
|
|||
|
|
best = z
|
|||
|
|
out.append(best)
|
|||
|
|
return out
|
|||
|
|
|
|||
|
|
# -- 序列化（零依赖 JSON，便于版本化保存/复现） ----------------------
|
|||
|
|
|
|||
|
|
def to_dict(self) -> Dict[str, object]:
|
|||
|
|
return {
|
|||
|
|
"kind": "zscore",
|
|||
|
|
"mean": self._mean,
|
|||
|
|
"std": self._std,
|
|||
|
|
"fitted": self._fitted,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@classmethod
|
|||
|
|
def from_dict(cls, d: Dict[str, object]) -> "ZScoreScorer":
|
|||
|
|
m = cls()
|
|||
|
|
m._mean = {k: float(v) for k, v in (d.get("mean") or {}).items()}
|
|||
|
|
m._std = {k: float(v) for k, v in (d.get("std") or {}).items()}
|
|||
|
|
m._fitted = bool(d.get("fitted", False))
|
|||
|
|
return m
|
|||
|
|
|
|||
|
|
def save(self, path: str) -> None:
|
|||
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|||
|
|
json.dump(self.to_dict(), fh, ensure_ascii=False, indent=2)
|
|||
|
|
|
|||
|
|
@classmethod
|
|||
|
|
def load(cls, path: str) -> "ZScoreScorer":
|
|||
|
|
with open(path, "r", encoding="utf-8") as fh:
|
|||
|
|
return cls.from_dict(json.load(fh))
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass
|
|||
|
|
class AlertDecision:
|
|||
|
|
"""单时刻预警决策。"""
|
|||
|
|
|
|||
|
|
timestamp: float
|
|||
|
|
score: float # 异常分数
|
|||
|
|
triggered: bool # 是否触发预警
|
|||
|
|
reasons: List[str] = field(default_factory=list) # 触发原因（分数超阈/特征 breach）
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ThresholdRule:
|
|||
|
|
"""预警决策规则：异常分数阈值 ∪ FeatureSpec breach（任一满足即预警）。
|
|||
|
|
|
|||
|
|
PRD 5.3 ③：误报率 ≤ 8%。组合两条判据降低单指标误报：
|
|||
|
|
- 分数判据：``ZScoreScorer`` 输出 ≥ ``score_threshold``（默认 3σ）；
|
|||
|
|
- breach 判据：特征值超 #70 FeatureSpec 声明的 ``threshold``（工艺硬限）。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self, score_threshold: float = 3.0,
|
|||
|
|
feature_thresholds: Optional[Dict[str, float]] = None) -> None:
|
|||
|
|
if score_threshold <= 0:
|
|||
|
|
raise ValueError("score_threshold 必须 > 0")
|
|||
|
|
self.score_threshold = score_threshold
|
|||
|
|
# feature_thresholds: 特征名 → 绝对上限（来自 #70 FeatureSpec.threshold）
|
|||
|
|
self.feature_thresholds: Dict[str, float] = dict(feature_thresholds or {})
|
|||
|
|
|
|||
|
|
def decide(self, timestamp: float, values: Dict[str, float],
|
|||
|
|
score: float) -> AlertDecision:
|
|||
|
|
reasons: List[str] = []
|
|||
|
|
if _is_num(score) and score >= self.score_threshold:
|
|||
|
|
reasons.append(f"异常分数 {score:.2f} ≥ {self.score_threshold}σ")
|
|||
|
|
for name, limit in self.feature_thresholds.items():
|
|||
|
|
v = values.get(name)
|
|||
|
|
if _is_num(v) and v > limit:
|
|||
|
|
reasons.append(f"{name}={v:.2f} 超阈值 {limit}")
|
|||
|
|
return AlertDecision(
|
|||
|
|
timestamp=timestamp, score=score,
|
|||
|
|
triggered=bool(reasons), reasons=reasons,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass
|
|||
|
|
class LeadTimeResult:
|
|||
|
|
"""提前量评估结果（对齐 PRD：提前 ≥ 30min）。"""
|
|||
|
|
|
|||
|
|
first_alert_ts: Optional[float] # 首次预警时刻（无则 None）
|
|||
|
|
anomaly_ts: Optional[float] # 真实异常时刻
|
|||
|
|
lead_seconds: Optional[float] # 提前量（秒）；负=滞后
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def lead_minutes(self) -> Optional[float]:
|
|||
|
|
return None if self.lead_seconds is None else self.lead_seconds / 60.0
|
|||
|
|
|
|||
|
|
|
|||
|
|
def evaluate_lead_time(decisions: Sequence[AlertDecision],
|
|||
|
|
anomaly_ts: float) -> LeadTimeResult:
|
|||
|
|
"""评估首次预警相对真实异常时刻的提前量。
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
decisions: 按时间升序的预警决策序列。
|
|||
|
|
anomaly_ts: 真实异常（如人工标注/峰值）发生的时刻。
|
|||
|
|
"""
|
|||
|
|
first = None
|
|||
|
|
for d in decisions:
|
|||
|
|
if d.triggered:
|
|||
|
|
first = d.timestamp
|
|||
|
|
break
|
|||
|
|
if first is None:
|
|||
|
|
return LeadTimeResult(first_alert_ts=None, anomaly_ts=anomaly_ts,
|
|||
|
|
lead_seconds=None)
|
|||
|
|
return LeadTimeResult(first_alert_ts=first, anomaly_ts=anomaly_ts,
|
|||
|
|
lead_seconds=anomaly_ts - first)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ImpurityForecaster:
|
|||
|
|
"""炉层杂质预警统一入口：评分器 + 决策规则 + 提前量评估。
|
|||
|
|
|
|||
|
|
典型用法（配合 #70 FeatureEngine）::
|
|||
|
|
|
|||
|
|
from impurity_forecast import FeatureEngine, load_feature_config
|
|||
|
|
eng = FeatureEngine.from_template_config("config/features.template.yaml")
|
|||
|
|
vectors = eng.transform(samples) # 特征矩阵
|
|||
|
|
forecaster = ImpurityForecaster()
|
|||
|
|
forecaster.fit(vectors[:normal_n]) # 正常段训练
|
|||
|
|
decisions = forecaster.predict(vectors) # 全段预警决策
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self, scorer: Optional[ZScoreScorer] = None,
|
|||
|
|
rule: Optional[ThresholdRule] = None) -> None:
|
|||
|
|
self.scorer = scorer or ZScoreScorer()
|
|||
|
|
self.rule = rule or ThresholdRule()
|
|||
|
|
|
|||
|
|
def fit(self, normal_samples: Sequence[FeatureVectorLike]) -> "ImpurityForecaster":
|
|||
|
|
self.scorer.fit(normal_samples)
|
|||
|
|
return self
|
|||
|
|
|
|||
|
|
def predict(self, samples: Sequence[FeatureVectorLike]) -> List[AlertDecision]:
|
|||
|
|
scores = self.scorer.score(samples)
|
|||
|
|
out: List[AlertDecision] = []
|
|||
|
|
for s, sc in zip(samples, scores):
|
|||
|
|
out.append(self.rule.decide(s.timestamp, s.values, sc))
|
|||
|
|
return out
|
|||
|
|
|
|||
|
|
def evaluate(self, samples: Sequence[FeatureVectorLike],
|
|||
|
|
anomaly_ts: float) -> Tuple[List[AlertDecision], LeadTimeResult]:
|
|||
|
|
decisions = self.predict(samples)
|
|||
|
|
return decisions, evaluate_lead_time(decisions, anomaly_ts)
|