68 lines
2.4 KiB
Python
68 lines
2.4 KiB
Python
|
# -*- coding: utf-8 -*-
|
|||
|
|
"""树脂 RAG 知识库导出加载器 —— issue #83。
|
|||
|
|
|
|||
|
|
把已交付化工新材料AI平台（吸附树脂）的领域知识库**导出为模板资产**
|
|||
|
|
（`documents/*.md`），并提供与 `kb.resin.template.yaml` 清单一致的加载器：
|
|||
|
|
|
|||
|
|
- `resin_kb_loader(title)`：标题 → 文档文本（对齐 rag-kb 的 loader 约定，
|
|||
|
|
可直接用于 `RagKnowledgeBase.from_template_config(..., loader=resin_kb_loader)`）；
|
|||
|
|
- `validate_export()`：校验 kb 清单声明的文档均有对应文件且非空
|
|||
|
|
（导出完整性检查，防止清单与文档漂移）。
|
|||
|
|
|
|||
|
|
标题规范化：`GB/T 5475 ...` → 文件名 `GB-T-5475 ....md`（`/` 不允许出现在文件名）。
|
|||
|
|
"""
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import os
|
|||
|
|
from typing import List, Optional
|
|||
|
|
|
|||
|
|
import yaml
|
|||
|
|
|
|||
|
|
#: 知识库配置资产（与模板清单同目录）
|
|||
|
|
DEFAULT_KB_CONFIG = os.path.join(
|
|||
|
|
os.path.dirname(os.path.abspath(__file__)), "kb.resin.template.yaml")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def resin_documents_dir() -> str:
|
|||
|
|
"""文档集目录（documents/）。"""
|
|||
|
|
return os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|||
|
|
"documents")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _doc_path(title: str) -> str:
|
|||
|
|
"""标题 → 文档文件路径（规范化：/ 与空格 → -，对齐导出文件名）。"""
|
|||
|
|
safe = title.replace("/", "-").replace(" ", "-")
|
|||
|
|
return os.path.join(resin_documents_dir(), f"{safe}.md")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def resin_kb_loader(title: str) -> str:
|
|||
|
|
"""按标题返回文档文本（找不到返回空串，与 rag-kb loader 约定一致）。"""
|
|||
|
|
path = _doc_path(title)
|
|||
|
|
if not os.path.isfile(path):
|
|||
|
|
return ""
|
|||
|
|
with open(path, "r", encoding="utf-8") as fh:
|
|||
|
|
return fh.read()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def load_kb_config(kb_config_path: str = DEFAULT_KB_CONFIG) -> dict:
|
|||
|
|
"""加载树脂知识库配置资产。"""
|
|||
|
|
with open(kb_config_path, "r", encoding="utf-8") as fh:
|
|||
|
|
return yaml.safe_load(fh) or {}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def validate_export(kb_config_path: str = DEFAULT_KB_CONFIG) -> List[str]:
|
|||
|
|
"""校验导出完整性：清单声明的文档均有文件且非空。
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
缺失/为空的问题列表（空 = 导出完整）。
|
|||
|
|
"""
|
|||
|
|
problems: List[str] = []
|
|||
|
|
raw = load_kb_config(kb_config_path)
|
|||
|
|
for src in raw.get("sources", []):
|
|||
|
|
kind = src.get("kind")
|
|||
|
|
for doc in src.get("documents", []):
|
|||
|
|
text = resin_kb_loader(doc)
|
|||
|
|
if not text:
|
|||
|
|
problems.append(f"[{kind}] 文档缺失或为空: {doc}")
|
|||
|
|
return problems
|