feat: 完成 issue #3 边缘采集网关模板化封装
- 点位字典 CSV schema/加载/自动校验(缺失字段/量纲/重复点号,PRD 5.1) - 协议可插拔只读驱动:OPC UA(S7-1200 适配)/S7/Modbus/称重/能源/模拟 - 周期采集引擎:只读+背压保护+健康度指标(丢失率/P99/可用性) - Kafka 流式上行 + 本地 spool 断点续传(丢失率≤0.02% 保障) - 模板配置外置(gateway.yaml + 点位字典 CSV),换行业零改码 - 18 个单元测试全绿;端到端运行 SLA 达标
This commit is contained in:
@@ -0,0 +1 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
@@ -0,0 +1,112 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""采集引擎 / 健康度指标 / spool 断点续传 端到端测试(模拟驱动)。"""
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from collector.engine import CollectorEngine
|
||||
from collector.metrics import HealthMetrics
|
||||
from collector.spool import SpoolStore
|
||||
from drivers import SimulatorDriver
|
||||
from point_dict.loader import Point, PointDict
|
||||
|
||||
|
||||
def make_point_dict(n: int = 10) -> PointDict:
|
||||
points = []
|
||||
for i in range(n):
|
||||
points.append(
|
||||
Point(
|
||||
device_id=f"CLF-{i // 5 + 1:02d}",
|
||||
point_id=f"CLF-{i // 5 + 1:02d}.P{i:03d}",
|
||||
name=f"测点{i}",
|
||||
unit="℃",
|
||||
data_type="float",
|
||||
sample_rate=1000,
|
||||
quality_code=True,
|
||||
row_number=i + 2,
|
||||
)
|
||||
)
|
||||
return PointDict(points)
|
||||
|
||||
|
||||
class EngineMetricsTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.mkdtemp()
|
||||
|
||||
def test_single_round_all_read(self):
|
||||
pd = make_point_dict(10)
|
||||
spool = SpoolStore(os.path.join(self._tmp, "spool"))
|
||||
metrics = HealthMetrics()
|
||||
engine = CollectorEngine(
|
||||
point_dict=pd,
|
||||
driver_slots=[("simulator", SimulatorDriver(), [])],
|
||||
spool=spool,
|
||||
metrics=metrics,
|
||||
interval_ms=1000,
|
||||
)
|
||||
got = engine.collect_once()
|
||||
self.assertEqual(got, 10)
|
||||
self.assertEqual(metrics.total_samples, 10)
|
||||
self.assertEqual(metrics.lost_samples, 0)
|
||||
self.assertLessEqual(metrics.loss_rate, 0.0002) # ≤0.02%
|
||||
self.assertLessEqual(metrics.p99_latency(), 1.8) # P99 ≤ 1.8s
|
||||
self.assertGreaterEqual(metrics.availability, 0.998)
|
||||
self.assertTrue(metrics.meets_sla())
|
||||
# spool 已落盘
|
||||
self.assertEqual(spool.total_pending(), 10)
|
||||
|
||||
def test_failed_driver_counts_round(self):
|
||||
"""驱动抛异常 → 该轮记为失败轮次,可用性下降。"""
|
||||
|
||||
class BoomDriver(SimulatorDriver):
|
||||
protocol = "boom"
|
||||
|
||||
def read_points(self, points):
|
||||
raise RuntimeError("simulated failure")
|
||||
|
||||
pd = make_point_dict(5)
|
||||
spool = SpoolStore(os.path.join(self._tmp, "spool"))
|
||||
metrics = HealthMetrics()
|
||||
engine = CollectorEngine(
|
||||
point_dict=pd,
|
||||
driver_slots=[("boom", BoomDriver(), [])],
|
||||
spool=spool,
|
||||
metrics=metrics,
|
||||
)
|
||||
got = engine.collect_once()
|
||||
self.assertEqual(got, 0)
|
||||
self.assertEqual(metrics.failed_rounds, 1)
|
||||
self.assertEqual(metrics.total_rounds, 1)
|
||||
self.assertEqual(metrics.availability, 0.0)
|
||||
|
||||
def test_spool_ack_removes_record(self):
|
||||
spool = SpoolStore(os.path.join(self._tmp, "spool"))
|
||||
spool.append("CLF-01", "CLF-01.TEMP", 85.5, time.time())
|
||||
self.assertEqual(spool.total_pending(), 1)
|
||||
spool.ack({"device_id": None, "point_id": "CLF-01.TEMP", "value": None, "ts": None})
|
||||
self.assertEqual(spool.total_pending(), 0)
|
||||
|
||||
def test_spool_pending_replay_after_restart(self):
|
||||
"""断点续传:spool 未确认记录重启后可全部重发。"""
|
||||
spool = SpoolStore(os.path.join(self._tmp, "spool"))
|
||||
ts = time.time()
|
||||
spool.append("CLF-01", "CLF-01.TEMP", 1.0, ts)
|
||||
spool.append("CLF-01", "CLF-01.PRES", 2.0, ts)
|
||||
# 模拟网关重启:新实例读取同一 spool 目录
|
||||
spool2 = SpoolStore(os.path.join(self._tmp, "spool"))
|
||||
pending = spool2.pending_records()
|
||||
self.assertEqual(len(pending), 2)
|
||||
ids = {r["point_id"] for r in pending}
|
||||
self.assertEqual(ids, {"CLF-01.TEMP", "CLF-01.PRES"})
|
||||
|
||||
def test_metrics_p99_window(self):
|
||||
metrics = HealthMetrics(window_size=4)
|
||||
for i in range(10):
|
||||
metrics.record_round(latency_sec=0.1 + i * 0.05, expected=10, got=10)
|
||||
self.assertLessEqual(metrics.p99_latency(), 0.6)
|
||||
self.assertEqual(metrics.total_samples, 100)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,59 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""点位字典加载器测试。"""
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from point_dict.loader import load_point_dict_csv
|
||||
|
||||
GOOD_CSV = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
|
||||
CLF-01,CLF-01.TEMP,炉温,℃,float,1000,true,ns=2;s=CLF.Temp
|
||||
CLF-01,CLF-01.PRES,炉压,kPa,float,1000,,ns=2;s=CLF.Pres
|
||||
"""
|
||||
|
||||
|
||||
class LoaderTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.mkdtemp()
|
||||
self.path = os.path.join(self._tmp, "points.csv")
|
||||
|
||||
def _write(self, content):
|
||||
with open(self.path, "w", encoding="utf-8") as fh:
|
||||
fh.write(content)
|
||||
return self.path
|
||||
|
||||
def test_load_valid_csv(self):
|
||||
path = self._write(GOOD_CSV)
|
||||
pd = load_point_dict_csv(path)
|
||||
self.assertEqual(len(pd), 2)
|
||||
p0 = pd.points[0]
|
||||
self.assertEqual(p0.device_id, "CLF-01")
|
||||
self.assertEqual(p0.point_id, "CLF-01.TEMP")
|
||||
self.assertEqual(p0.unit, "℃")
|
||||
self.assertEqual(p0.data_type, "float")
|
||||
self.assertEqual(p0.sample_rate, 1000)
|
||||
self.assertTrue(p0.quality_code)
|
||||
self.assertEqual(p0.opc_node, "ns=2;s=CLF.Temp")
|
||||
self.assertEqual(p0.row_number, 2)
|
||||
|
||||
def test_quality_code_default_true(self):
|
||||
"""qualityCode 列留空时默认 true。"""
|
||||
path = self._write(GOOD_CSV)
|
||||
pd = load_point_dict_csv(path)
|
||||
self.assertTrue(pd.points[1].quality_code)
|
||||
|
||||
def test_utf8_bom_tolerated(self):
|
||||
path = self._write("\ufeff" + GOOD_CSV)
|
||||
pd = load_point_dict_csv(path)
|
||||
self.assertEqual(len(pd), 2)
|
||||
|
||||
def test_grouping_by_device(self):
|
||||
path = self._write(GOOD_CSV)
|
||||
pd = load_point_dict_csv(path)
|
||||
grouped = pd.by_device_id()
|
||||
self.assertEqual(set(grouped.keys()), {"CLF-01"})
|
||||
self.assertEqual(len(grouped["CLF-01"]), 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,101 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""点位字典自动校验器测试(缺失字段/量纲/重复点号/采样率/表头)。"""
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from point_dict.validator import validate_point_dict_file
|
||||
|
||||
GOOD_CSV = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
|
||||
CLF-01,CLF-01.TEMP,炉温,℃,float,1000,true,ns=2;s=CLF.Temp
|
||||
CLF-01,CLF-01.PRES,炉压,kPa,float,1000,true,ns=2;s=CLF.Pres
|
||||
"""
|
||||
|
||||
|
||||
class ValidatorTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.mkdtemp()
|
||||
self.path = os.path.join(self._tmp, "points.csv")
|
||||
|
||||
def _write(self, content):
|
||||
with open(self.path, "w", encoding="utf-8") as fh:
|
||||
fh.write(content)
|
||||
return self.path
|
||||
|
||||
def test_valid_dict_passes(self):
|
||||
report = validate_point_dict_file(self._write(GOOD_CSV))
|
||||
self.assertTrue(report.ok, report.summary())
|
||||
self.assertEqual(report.issues, [])
|
||||
|
||||
def test_missing_required_field(self):
|
||||
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
|
||||
CLF-01,,炉温,℃,float,1000,true,ns=2
|
||||
"""
|
||||
report = validate_point_dict_file(self._write(csv))
|
||||
self.assertFalse(report.ok)
|
||||
codes = [i.code for i in report.issues]
|
||||
self.assertIn("missing_field", codes)
|
||||
|
||||
def test_bad_unit(self):
|
||||
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
|
||||
CLF-01,CLF-01.TEMP,炉温,摄氏度,float,1000,true,
|
||||
"""
|
||||
report = validate_point_dict_file(self._write(csv))
|
||||
codes = [i.code for i in report.issues]
|
||||
self.assertIn("bad_unit", codes)
|
||||
|
||||
def test_duplicate_point_id(self):
|
||||
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
|
||||
CLF-01,CLF-01.TEMP,炉温,℃,float,1000,true,
|
||||
CLF-02,CLF-01.TEMP,炉温2,℃,float,1000,true,
|
||||
"""
|
||||
report = validate_point_dict_file(self._write(csv))
|
||||
codes = [i.code for i in report.issues]
|
||||
self.assertIn("dup_point", codes)
|
||||
|
||||
def test_multiple_points_same_device_ok(self):
|
||||
"""同一设备下多个测点行是正常场景,不报错。"""
|
||||
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
|
||||
CLF-01,CLF-01.TEMP,炉温,℃,float,1000,true,
|
||||
CLF-01,CLF-01.PRES,炉压,kPa,float,1000,true,
|
||||
"""
|
||||
report = validate_point_dict_file(self._write(csv))
|
||||
self.assertTrue(report.ok, report.summary())
|
||||
|
||||
def test_bad_sample_rate(self):
|
||||
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
|
||||
CLF-01,CLF-01.TEMP,炉温,℃,float,0,true,
|
||||
"""
|
||||
report = validate_point_dict_file(self._write(csv))
|
||||
codes = [i.code for i in report.issues]
|
||||
self.assertIn("bad_sample_rate", codes)
|
||||
|
||||
def test_bad_data_type(self):
|
||||
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
|
||||
CLF-01,CLF-01.TEMP,炉温,℃,double,1000,true,
|
||||
"""
|
||||
report = validate_point_dict_file(self._write(csv))
|
||||
codes = [i.code for i in report.issues]
|
||||
self.assertIn("bad_data_type", codes)
|
||||
|
||||
def test_missing_header_column(self):
|
||||
csv = """device_id,point_id,name,dataType,sampleRate,qualityCode,opcNode
|
||||
CLF-01,CLF-01.TEMP,炉温,float,1000,true,
|
||||
"""
|
||||
report = validate_point_dict_file(self._write(csv))
|
||||
codes = [i.code for i in report.issues]
|
||||
self.assertIn("missing_column", codes)
|
||||
|
||||
def test_multiple_issues_aggregated(self):
|
||||
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
|
||||
CLF-01,CLF-01.TEMP,炉温,摄氏度,float,0,true,
|
||||
CLF-01,CLF-01.TEMP,炉温2,℃,float,1000,true,
|
||||
"""
|
||||
report = validate_point_dict_file(self._write(csv))
|
||||
self.assertGreaterEqual(len(report.issues), 3) # bad_unit + bad_sample_rate + dup_point
|
||||
self.assertFalse(report.ok)
|
||||
self.assertIn("点位字典校验失败", report.summary())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user