feat: 完成 issue #3 边缘采集网关模板化封装

- 点位字典 CSV schema/加载/自动校验(缺失字段/量纲/重复点号,PRD 5.1)
- 协议可插拔只读驱动:OPC UA(S7-1200 适配)/S7/Modbus/称重/能源/模拟
- 周期采集引擎:只读+背压保护+健康度指标(丢失率/P99/可用性)
- Kafka 流式上行 + 本地 spool 断点续传(丢失率≤0.02% 保障)
- 模板配置外置(gateway.yaml + 点位字典 CSV),换行业零改码
- 18 个单元测试全绿;端到端运行 SLA 达标
This commit is contained in:
2026-08-04 15:32:16 +08:00
parent 21c6259739
commit f49c0920d4
27 changed files with 1806 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
# -*- coding: utf-8 -*-
@@ -0,0 +1,112 @@
# -*- coding: utf-8 -*-
"""采集引擎 / 健康度指标 / spool 断点续传 端到端测试(模拟驱动)。"""
import os
import tempfile
import time
import unittest
from collector.engine import CollectorEngine
from collector.metrics import HealthMetrics
from collector.spool import SpoolStore
from drivers import SimulatorDriver
from point_dict.loader import Point, PointDict
def make_point_dict(n: int = 10) -> PointDict:
points = []
for i in range(n):
points.append(
Point(
device_id=f"CLF-{i // 5 + 1:02d}",
point_id=f"CLF-{i // 5 + 1:02d}.P{i:03d}",
name=f"测点{i}",
unit="℃",
data_type="float",
sample_rate=1000,
quality_code=True,
row_number=i + 2,
)
)
return PointDict(points)
class EngineMetricsTest(unittest.TestCase):
def setUp(self):
self._tmp = tempfile.mkdtemp()
def test_single_round_all_read(self):
pd = make_point_dict(10)
spool = SpoolStore(os.path.join(self._tmp, "spool"))
metrics = HealthMetrics()
engine = CollectorEngine(
point_dict=pd,
driver_slots=[("simulator", SimulatorDriver(), [])],
spool=spool,
metrics=metrics,
interval_ms=1000,
)
got = engine.collect_once()
self.assertEqual(got, 10)
self.assertEqual(metrics.total_samples, 10)
self.assertEqual(metrics.lost_samples, 0)
self.assertLessEqual(metrics.loss_rate, 0.0002) # ≤0.02%
self.assertLessEqual(metrics.p99_latency(), 1.8) # P99 ≤ 1.8s
self.assertGreaterEqual(metrics.availability, 0.998)
self.assertTrue(metrics.meets_sla())
# spool 已落盘
self.assertEqual(spool.total_pending(), 10)
def test_failed_driver_counts_round(self):
"""驱动抛异常 → 该轮记为失败轮次,可用性下降。"""
class BoomDriver(SimulatorDriver):
protocol = "boom"
def read_points(self, points):
raise RuntimeError("simulated failure")
pd = make_point_dict(5)
spool = SpoolStore(os.path.join(self._tmp, "spool"))
metrics = HealthMetrics()
engine = CollectorEngine(
point_dict=pd,
driver_slots=[("boom", BoomDriver(), [])],
spool=spool,
metrics=metrics,
)
got = engine.collect_once()
self.assertEqual(got, 0)
self.assertEqual(metrics.failed_rounds, 1)
self.assertEqual(metrics.total_rounds, 1)
self.assertEqual(metrics.availability, 0.0)
def test_spool_ack_removes_record(self):
spool = SpoolStore(os.path.join(self._tmp, "spool"))
spool.append("CLF-01", "CLF-01.TEMP", 85.5, time.time())
self.assertEqual(spool.total_pending(), 1)
spool.ack({"device_id": None, "point_id": "CLF-01.TEMP", "value": None, "ts": None})
self.assertEqual(spool.total_pending(), 0)
def test_spool_pending_replay_after_restart(self):
"""断点续传:spool 未确认记录重启后可全部重发。"""
spool = SpoolStore(os.path.join(self._tmp, "spool"))
ts = time.time()
spool.append("CLF-01", "CLF-01.TEMP", 1.0, ts)
spool.append("CLF-01", "CLF-01.PRES", 2.0, ts)
# 模拟网关重启:新实例读取同一 spool 目录
spool2 = SpoolStore(os.path.join(self._tmp, "spool"))
pending = spool2.pending_records()
self.assertEqual(len(pending), 2)
ids = {r["point_id"] for r in pending}
self.assertEqual(ids, {"CLF-01.TEMP", "CLF-01.PRES"})
def test_metrics_p99_window(self):
metrics = HealthMetrics(window_size=4)
for i in range(10):
metrics.record_round(latency_sec=0.1 + i * 0.05, expected=10, got=10)
self.assertLessEqual(metrics.p99_latency(), 0.6)
self.assertEqual(metrics.total_samples, 100)
if __name__ == "__main__":
unittest.main()
+59
View File
@@ -0,0 +1,59 @@
# -*- coding: utf-8 -*-
"""点位字典加载器测试。"""
import os
import tempfile
import unittest
from point_dict.loader import load_point_dict_csv
GOOD_CSV = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
CLF-01,CLF-01.TEMP,炉温,℃,float,1000,true,ns=2;s=CLF.Temp
CLF-01,CLF-01.PRES,炉压,kPa,float,1000,,ns=2;s=CLF.Pres
"""
class LoaderTest(unittest.TestCase):
def setUp(self):
self._tmp = tempfile.mkdtemp()
self.path = os.path.join(self._tmp, "points.csv")
def _write(self, content):
with open(self.path, "w", encoding="utf-8") as fh:
fh.write(content)
return self.path
def test_load_valid_csv(self):
path = self._write(GOOD_CSV)
pd = load_point_dict_csv(path)
self.assertEqual(len(pd), 2)
p0 = pd.points[0]
self.assertEqual(p0.device_id, "CLF-01")
self.assertEqual(p0.point_id, "CLF-01.TEMP")
self.assertEqual(p0.unit, "℃")
self.assertEqual(p0.data_type, "float")
self.assertEqual(p0.sample_rate, 1000)
self.assertTrue(p0.quality_code)
self.assertEqual(p0.opc_node, "ns=2;s=CLF.Temp")
self.assertEqual(p0.row_number, 2)
def test_quality_code_default_true(self):
"""qualityCode 列留空时默认 true。"""
path = self._write(GOOD_CSV)
pd = load_point_dict_csv(path)
self.assertTrue(pd.points[1].quality_code)
def test_utf8_bom_tolerated(self):
path = self._write("\ufeff" + GOOD_CSV)
pd = load_point_dict_csv(path)
self.assertEqual(len(pd), 2)
def test_grouping_by_device(self):
path = self._write(GOOD_CSV)
pd = load_point_dict_csv(path)
grouped = pd.by_device_id()
self.assertEqual(set(grouped.keys()), {"CLF-01"})
self.assertEqual(len(grouped["CLF-01"]), 2)
if __name__ == "__main__":
unittest.main()
+101
View File
@@ -0,0 +1,101 @@
# -*- coding: utf-8 -*-
"""点位字典自动校验器测试(缺失字段/量纲/重复点号/采样率/表头)。"""
import os
import tempfile
import unittest
from point_dict.validator import validate_point_dict_file
GOOD_CSV = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
CLF-01,CLF-01.TEMP,炉温,℃,float,1000,true,ns=2;s=CLF.Temp
CLF-01,CLF-01.PRES,炉压,kPa,float,1000,true,ns=2;s=CLF.Pres
"""
class ValidatorTest(unittest.TestCase):
def setUp(self):
self._tmp = tempfile.mkdtemp()
self.path = os.path.join(self._tmp, "points.csv")
def _write(self, content):
with open(self.path, "w", encoding="utf-8") as fh:
fh.write(content)
return self.path
def test_valid_dict_passes(self):
report = validate_point_dict_file(self._write(GOOD_CSV))
self.assertTrue(report.ok, report.summary())
self.assertEqual(report.issues, [])
def test_missing_required_field(self):
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
CLF-01,,炉温,℃,float,1000,true,ns=2
"""
report = validate_point_dict_file(self._write(csv))
self.assertFalse(report.ok)
codes = [i.code for i in report.issues]
self.assertIn("missing_field", codes)
def test_bad_unit(self):
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
CLF-01,CLF-01.TEMP,炉温,摄氏度,float,1000,true,
"""
report = validate_point_dict_file(self._write(csv))
codes = [i.code for i in report.issues]
self.assertIn("bad_unit", codes)
def test_duplicate_point_id(self):
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
CLF-01,CLF-01.TEMP,炉温,℃,float,1000,true,
CLF-02,CLF-01.TEMP,炉温2,℃,float,1000,true,
"""
report = validate_point_dict_file(self._write(csv))
codes = [i.code for i in report.issues]
self.assertIn("dup_point", codes)
def test_multiple_points_same_device_ok(self):
"""同一设备下多个测点行是正常场景,不报错。"""
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
CLF-01,CLF-01.TEMP,炉温,℃,float,1000,true,
CLF-01,CLF-01.PRES,炉压,kPa,float,1000,true,
"""
report = validate_point_dict_file(self._write(csv))
self.assertTrue(report.ok, report.summary())
def test_bad_sample_rate(self):
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
CLF-01,CLF-01.TEMP,炉温,℃,float,0,true,
"""
report = validate_point_dict_file(self._write(csv))
codes = [i.code for i in report.issues]
self.assertIn("bad_sample_rate", codes)
def test_bad_data_type(self):
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
CLF-01,CLF-01.TEMP,炉温,℃,double,1000,true,
"""
report = validate_point_dict_file(self._write(csv))
codes = [i.code for i in report.issues]
self.assertIn("bad_data_type", codes)
def test_missing_header_column(self):
csv = """device_id,point_id,name,dataType,sampleRate,qualityCode,opcNode
CLF-01,CLF-01.TEMP,炉温,float,1000,true,
"""
report = validate_point_dict_file(self._write(csv))
codes = [i.code for i in report.issues]
self.assertIn("missing_column", codes)
def test_multiple_issues_aggregated(self):
csv = """device_id,point_id,name,unit,dataType,sampleRate,qualityCode,opcNode
CLF-01,CLF-01.TEMP,炉温,摄氏度,float,0,true,
CLF-01,CLF-01.TEMP,炉温2,℃,float,1000,true,
"""
report = validate_point_dict_file(self._write(csv))
self.assertGreaterEqual(len(report.issues), 3) # bad_unit + bad_sample_rate + dup_point
self.assertFalse(report.ok)
self.assertIn("点位字典校验失败", report.summary())
if __name__ == "__main__":
unittest.main()