feat: 完成 issue #3 边缘采集网关模板化封装

- 点位字典 CSV schema/加载/自动校验（缺失字段/量纲/重复点号，PRD 5.1） - 协议可插拔只读驱动：OPC UA(S7-1200 适配)/S7/Modbus/称重/能源/模拟 - 周期采集引擎：只读+背压保护+健康度指标（丢失率/P99/可用性） - Kafka 流式上行 + 本地 spool 断点续传（丢失率≤0.02% 保障） - 模板配置外置（gateway.yaml + 点位字典 CSV），换行业零改码 - 18 个单元测试全绿；端到端运行 SLA 达标
2026-08-04 15:32:16 +08:00
parent 21c6259739
commit f49c0920d4
27 changed files with 1806 additions and 0 deletions
@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+"""iAOP 边缘采集网关 —— 主入口。
+
+用法：
+    python main.py --config config/gateway.example.yaml --point-dict point_dict.csv [--dry-run]
+
+流程（模板化封装，对齐 PRD 5.1 用户操作流程）：
+    实施工程师导入 DCS 点表 CSV → 自动校验（缺失字段/量纲/重复点号）
+    → 加载模板配置（协议/周期/背压阈值/Kafka）→ 网关启动只读采集
+    → Kafka 流式上行 + spool 断点续传 → 实时健康度上报。
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+import time
+from typing import List, Tuple
+
+import yaml
+
+# 允许直接以脚本方式运行（python main.py）时仍能解析包内模块
+from point_dict import load_point_dict_csv, validate_point_dict_file
+from point_dict.loader import PointDict
+
+logger = logging.getLogger("edge_gateway.main")
+
+
+def parse_args(argv: List[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="iAOP 边缘采集网关（模板化封装）")
+    parser.add_argument("--config", required=True, help="采集配置 YAML（模板参数化）")
+    parser.add_argument("--point-dict", required=True, help="点位字典 CSV（客户 DCS 点表）")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="仅加载配置并校验点位字典，不启动采集")
+    parser.add_argument("--rounds", type=int, default=0,
+                        help="采集轮数上限（0=无限，调试用）")
+    parser.add_argument("--verbose", action="store_true", help="输出调试日志")
+    return parser.parse_args(argv)
+
+
+def build_engine(config: dict, point_dict: PointDict):
+    """按模板配置组装采集引擎（驱动路由 → spool → metrics → 引擎）。"""
+    from collector import CollectorEngine, HealthMetrics, SpoolStore
+
+    collector_cfg = config.get("collector", {})
+    spool = SpoolStore(
+        spool_dir=collector_cfg.get("spool_dir", "./spool"),
+        cache_limit_bytes=int(collector_cfg.get("cache_limit_bytes", 512 * 1024 * 1024)),
+    )
+    metrics = HealthMetrics()
+
+    # 组装驱动插槽：[(protocol, driver, device_prefixes)]
+    driver_slots: List[Tuple[str, object, List[str]]] = []
+    from drivers import from_template
+
+    for item in collector_cfg.get("drivers", []):
+        protocol = item.get("protocol")
+        if not protocol:
+            raise ValueError("collector.drivers[].protocol 必填")
+        driver = from_template(protocol, item.get("config", {}))
+        driver_slots.append((protocol, driver, item.get("device_prefixes", [])))
+
+    if not driver_slots:
+        # 模板配置未声明驱动时默认走模拟驱动（本地联调），避免空跑
+        from drivers import SimulatorDriver
+
+        driver_slots.append(("simulator", SimulatorDriver(), []))
+
+    engine = CollectorEngine(
+        point_dict=point_dict,
+        driver_slots=driver_slots,
+        spool=spool,
+        metrics=metrics,
+        interval_ms=int(collector_cfg.get("interval_ms", 1000)),
+        max_pending=int(collector_cfg.get("max_pending", 100_000)),
+    )
+    return engine, spool, metrics
+
+
+def build_sink(config: dict, spool) -> object:
+    """按模板配置组装 Kafka 上行通道。"""
+    from upstream import KafkaSink
+
+    kafka_cfg = config.get("kafka", {})
+    return KafkaSink(
+        bootstrap_servers=kafka_cfg.get("bootstrap_servers", "127.0.0.1:9092"),
+        topic_prefix=kafka_cfg.get("topic_prefix", "iaop"),
+        spool=spool,
+        batch_size=int(kafka_cfg.get("batch_size", 500)),
+    )
+
+
+def main(argv: List[str]) -> int:
+    args = parse_args(argv)
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+
+    # 1) 加载模板配置
+    with open(args.config, "r", encoding="utf-8") as fh:
+        config = yaml.safe_load(fh) or {}
+
+    # 2) 加载 + 自动校验点位字典（缺失字段/量纲/重复点号）
+    report = validate_point_dict_file(args.point_dict)
+    logger.info("点位字典校验: %s", report.summary())
+    if not report.ok:
+        for issue in report.issues[:20]:
+            logger.error("  [%s] %s", issue.code, issue.message)
+        if len(report.issues) > 20:
+            logger.error("  ... 共 %d 条问题", len(report.issues))
+        print(f"点位字典校验失败：{report.summary()}")
+        return 2
+
+    point_dict = load_point_dict_csv(args.point_dict)
+    logger.info("点位字典加载完成：%d 个测点", len(point_dict))
+
+    if args.dry_run:
+        print("dry-run 通过：配置与点位字典校验 OK，未启动采集")
+        return 0
+
+    # 3) 组装并启动
+    engine, spool, metrics = build_engine(config, point_dict)
+    sink = build_sink(config, spool)
+
+    # 断点续传：启动时先重发上次未确认记录
+    pending = spool.pending_records(limit=10 ** 9)
+    if pending:
+        logger.info("检测到 %d 条未确认 spool 记录，启动续传", len(pending))
+        sink.publish(pending)
+
+    engine.start(sink=sink)
+    logger.info("采集网关已启动：%d 测点 @ %dms，Kafka=%s",
+                len(point_dict), engine.interval_ms, config.get("kafka", {}).get("bootstrap_servers"))
+
+    try:
+        rounds = 0
+        while True:
+            time.sleep(5)
+            rounds += 5
+            snap = metrics.snapshot()
+            logger.info("健康度: 轮次=%d 样本=%d 丢失率=%.4f%% P99=%.3fs 可用性=%.4f%%",
+                        snap["total_rounds"], snap["total_samples"],
+                        snap["loss_rate"] * 100, snap["p99_latency_sec"],
+                        snap["availability"] * 100)
+            if args.rounds and rounds >= args.rounds:
+                break
+    except KeyboardInterrupt:
+        pass
+    finally:
+        engine.stop()
+        sink.close()
+        snap = metrics.snapshot()
+        print(f"SLA 达标(P99≤1.8s/丢失率≤0.02%/可用性≥99.8%): {metrics.meets_sla()}")
+        print(f"健康度快照: {snap}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))