first
This commit is contained in:
@@ -0,0 +1,424 @@
|
||||
#!/bin/bash
|
||||
# 外挂状态检测脚本：统一判断可选外挂的安装/环境/运行状态
|
||||
# 五种状态：not_installed / env_unavailable（仅 uv 依赖的来源） / runtime_failed / unsupported / empty_result
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SOURCE_REGISTRY_SCRIPT="$SCRIPT_DIR/source-registry.sh"
|
||||
# 微信工具 URL 从共享配置读取，与 install.sh 保持一致
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
source "$SCRIPT_DIR/runtime-context.sh"
|
||||
SKILL_ROOT_OVERRIDE=""
|
||||
LAYOUT_MODE_OVERRIDE=""
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/adapter-state.sh [--skill-root <path>] [--layout-mode <source_checkout|installed_skill|upgrade_target>] check <source_id>
|
||||
bash scripts/adapter-state.sh [--skill-root <path>] [--layout-mode <source_checkout|installed_skill|upgrade_target>] summary
|
||||
bash scripts/adapter-state.sh [--skill-root <path>] [--layout-mode <source_checkout|installed_skill|upgrade_target>] summary-human
|
||||
bash scripts/adapter-state.sh [--skill-root <path>] [--layout-mode <source_checkout|installed_skill|upgrade_target>] classify-run <source_id> <exit_code> <output_path>
|
||||
EOF
|
||||
}
|
||||
|
||||
resolve_optional_root() {
|
||||
resolve_optional_adapter_root "$PROJECT_ROOT" "$SKILL_ROOT_OVERRIDE" "$LAYOUT_MODE_OVERRIDE"
|
||||
}
|
||||
|
||||
dependency_installed() {
|
||||
local dependency_name="$1"
|
||||
local dependency_type="$2"
|
||||
local optional_root
|
||||
|
||||
case "$dependency_type" in
|
||||
bundled)
|
||||
optional_root="$(resolve_optional_root)"
|
||||
if [ -d "$optional_root/$dependency_name" ]; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
;;
|
||||
install_time)
|
||||
command -v "$dependency_name" >/dev/null 2>&1
|
||||
;;
|
||||
none)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
has_uv() {
|
||||
command -v uv >/dev/null 2>&1
|
||||
}
|
||||
|
||||
chrome_debug_ready() {
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
lsof -i :9222 -sTCP:LISTEN >/dev/null 2>&1
|
||||
return $?
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
print_header() {
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"source_id" \
|
||||
"source_label" \
|
||||
"state" \
|
||||
"state_label" \
|
||||
"detail" \
|
||||
"recovery_action" \
|
||||
"install_hint" \
|
||||
"fallback_hint"
|
||||
}
|
||||
|
||||
state_label() {
|
||||
case "$1" in
|
||||
available)
|
||||
printf '%s\n' "可用"
|
||||
;;
|
||||
not_installed)
|
||||
printf '%s\n' "未安装"
|
||||
;;
|
||||
env_unavailable)
|
||||
printf '%s\n' "环境不满足"
|
||||
;;
|
||||
runtime_failed)
|
||||
printf '%s\n' "运行失败"
|
||||
;;
|
||||
unsupported)
|
||||
printf '%s\n' "不支持自动提取"
|
||||
;;
|
||||
empty_result)
|
||||
printf '%s\n' "结果为空"
|
||||
;;
|
||||
*)
|
||||
printf '%s\n' "$1"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
default_install_hint() {
|
||||
local source_id="$1"
|
||||
local adapter_name="$2"
|
||||
|
||||
case "$source_id" in
|
||||
web_article|x_twitter|zhihu_article)
|
||||
printf '%s\n' "重新运行当前平台的 llm-wiki 安装命令，并追加 --with-optional-adapters，确认 ${adapter_name} 已准备到技能目录"
|
||||
;;
|
||||
wechat_article)
|
||||
printf '%s\n' "先安装 uv，再执行：uv tool install ${WECHAT_TOOL_URL}"
|
||||
;;
|
||||
youtube_video)
|
||||
printf '%s\n' "重新运行当前平台的 llm-wiki 安装命令，并追加 --with-optional-adapters，确认 ${adapter_name} 已准备到技能目录"
|
||||
;;
|
||||
*)
|
||||
printf '%s\n' "-"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
optional_hint() {
|
||||
local source_id="$1"
|
||||
|
||||
case "$source_id" in
|
||||
web_article|x_twitter|zhihu_article)
|
||||
printf '%s\n' '如需复用已登录的浏览器会话，可执行：open -na "Google Chrome" --args --remote-debugging-port=9222'
|
||||
;;
|
||||
wechat_article|youtube_video)
|
||||
printf '%s\n' "先安装 uv：brew install uv"
|
||||
;;
|
||||
*)
|
||||
printf '%s\n' "-"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
emit_state_row() {
|
||||
local source_id="$1"
|
||||
local source_label="$2"
|
||||
local state="$3"
|
||||
local detail="$4"
|
||||
local recovery_action="$5"
|
||||
local install_hint="$6"
|
||||
local fallback_hint="$7"
|
||||
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$source_id" \
|
||||
"$source_label" \
|
||||
"$state" \
|
||||
"$(state_label "$state")" \
|
||||
"$detail" \
|
||||
"$recovery_action" \
|
||||
"$install_hint" \
|
||||
"$fallback_hint"
|
||||
}
|
||||
|
||||
resolve_preflight_state() {
|
||||
local source_id="$1"
|
||||
local record
|
||||
local source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint
|
||||
local state detail recovery_action install_hint
|
||||
|
||||
record="$(bash "$SOURCE_REGISTRY_SCRIPT" get "$source_id")" || {
|
||||
echo "未知来源：$source_id" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
IFS=$'\t' read -r source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint <<EOF
|
||||
$record
|
||||
EOF
|
||||
|
||||
case "$source_category" in
|
||||
core_builtin)
|
||||
state="available"
|
||||
detail="核心主线可直接进入，不依赖外挂"
|
||||
recovery_action="直接继续主线"
|
||||
install_hint="-"
|
||||
;;
|
||||
manual_only)
|
||||
state="unsupported"
|
||||
detail="该来源当前只支持手动进入主线"
|
||||
recovery_action="直接走手动入口"
|
||||
install_hint="-"
|
||||
;;
|
||||
optional_adapter)
|
||||
case "$source_id" in
|
||||
wechat_article)
|
||||
if ! has_uv; then
|
||||
state="env_unavailable"
|
||||
detail="缺少 uv，当前无法准备微信公众号自动提取环境"
|
||||
recovery_action="先补环境；现在也可以直接走手动入口"
|
||||
install_hint="$(optional_hint "$source_id")"
|
||||
elif ! dependency_installed "$dependency_name" "$dependency_type"; then
|
||||
state="not_installed"
|
||||
detail="未找到 ${adapter_name}"
|
||||
recovery_action="先补安装；现在也可以直接走手动入口"
|
||||
install_hint="$(default_install_hint "$source_id" "$adapter_name")"
|
||||
else
|
||||
state="available"
|
||||
detail="${adapter_name} 已可用"
|
||||
recovery_action="继续自动提取"
|
||||
install_hint="-"
|
||||
fi
|
||||
;;
|
||||
web_article|x_twitter|zhihu_article)
|
||||
if ! dependency_installed "$dependency_name" "$dependency_type"; then
|
||||
state="not_installed"
|
||||
detail="未找到 ${adapter_name}"
|
||||
recovery_action="先补安装；现在也可以直接走手动入口"
|
||||
install_hint="$(default_install_hint "$source_id" "$adapter_name")"
|
||||
else
|
||||
state="available"
|
||||
if chrome_debug_ready; then
|
||||
detail="${adapter_name} 已可用，且已检测到可复用的 Chrome 调试会话"
|
||||
recovery_action="继续自动提取"
|
||||
install_hint="-"
|
||||
else
|
||||
detail="${adapter_name} 已可用；未检测到 9222，将在需要时自动拉起临时浏览器"
|
||||
recovery_action="继续自动提取；如需复用已登录会话，可先开启 Chrome 调试端口 9222"
|
||||
install_hint="$(optional_hint "$source_id")"
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
youtube_video)
|
||||
if ! dependency_installed "$dependency_name" "$dependency_type"; then
|
||||
state="not_installed"
|
||||
detail="未找到 ${adapter_name}"
|
||||
recovery_action="先补安装；现在也可以直接走手动入口"
|
||||
install_hint="$(default_install_hint "$source_id" "$adapter_name")"
|
||||
elif ! has_uv; then
|
||||
state="env_unavailable"
|
||||
detail="缺少 uv，当前无法运行 YouTube 字幕提取"
|
||||
recovery_action="先补环境；现在也可以直接走手动入口"
|
||||
install_hint="$(optional_hint "$source_id")"
|
||||
else
|
||||
state="available"
|
||||
detail="${adapter_name} 已可用"
|
||||
recovery_action="继续自动提取"
|
||||
install_hint="-"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
if ! dependency_installed "$dependency_name" "$dependency_type"; then
|
||||
state="not_installed"
|
||||
detail="未找到 ${adapter_name}"
|
||||
recovery_action="先补安装；现在也可以直接走手动入口"
|
||||
install_hint="$(default_install_hint "$source_id" "$adapter_name")"
|
||||
else
|
||||
state="available"
|
||||
detail="${adapter_name} 已可用"
|
||||
recovery_action="继续自动提取"
|
||||
install_hint="-"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
echo "未知来源分类：$source_category" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
emit_state_row \
|
||||
"$source_id" \
|
||||
"$source_label" \
|
||||
"$state" \
|
||||
"$detail" \
|
||||
"$recovery_action" \
|
||||
"$install_hint" \
|
||||
"$fallback_hint"
|
||||
}
|
||||
|
||||
classify_run_state() {
|
||||
local source_id="$1"
|
||||
local exit_code="$2"
|
||||
local output_path="$3"
|
||||
|
||||
# 校验 exit_code 为整数，防止 set -e 下非数字参数导致脚本崩溃
|
||||
case "$exit_code" in
|
||||
''|*[!0-9-]*) echo "exit_code 必须是整数，收到：$exit_code" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
local row
|
||||
local source_label state state_label_value detail recovery_action install_hint fallback_hint
|
||||
|
||||
row="$(resolve_preflight_state "$source_id")"
|
||||
IFS=$'\t' read -r _ source_label state state_label_value detail recovery_action install_hint fallback_hint <<EOF
|
||||
$row
|
||||
EOF
|
||||
|
||||
if [ "$state" != "available" ]; then
|
||||
emit_state_row \
|
||||
"$source_id" \
|
||||
"$source_label" \
|
||||
"$state" \
|
||||
"$detail" \
|
||||
"$recovery_action" \
|
||||
"$install_hint" \
|
||||
"$fallback_hint"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$exit_code" -ne 0 ]; then
|
||||
emit_state_row \
|
||||
"$source_id" \
|
||||
"$source_label" \
|
||||
"runtime_failed" \
|
||||
"自动提取执行失败" \
|
||||
"可以先重试一次；如果还不行，就改走手动入口" \
|
||||
"-" \
|
||||
"$fallback_hint"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ ! -f "$output_path" ] || ! grep -q '[^[:space:]]' "$output_path" 2>/dev/null; then
|
||||
emit_state_row \
|
||||
"$source_id" \
|
||||
"$source_label" \
|
||||
"empty_result" \
|
||||
"自动提取完成，但没有拿到有效正文" \
|
||||
"请手动补全文本后继续主线" \
|
||||
"-" \
|
||||
"$fallback_hint"
|
||||
return 0
|
||||
fi
|
||||
|
||||
emit_state_row \
|
||||
"$source_id" \
|
||||
"$source_label" \
|
||||
"available" \
|
||||
"自动提取已拿到有效正文" \
|
||||
"继续进入主线" \
|
||||
"-" \
|
||||
"$fallback_hint"
|
||||
}
|
||||
|
||||
print_summary() {
|
||||
local source_id
|
||||
|
||||
print_header
|
||||
|
||||
while IFS=$'\t' read -r source_id _; do
|
||||
[ -n "$source_id" ] || continue
|
||||
resolve_preflight_state "$source_id"
|
||||
done <<EOF
|
||||
$(bash "$SOURCE_REGISTRY_SCRIPT" list | awk -F '\t' 'NR > 1 && ($3 == "optional_adapter" || $3 == "manual_only") { print $1 "\t" $2 }')
|
||||
EOF
|
||||
}
|
||||
|
||||
print_summary_human() {
|
||||
local row
|
||||
local source_id source_label state state_label_value detail recovery_action install_hint fallback_hint
|
||||
|
||||
while IFS= read -r row; do
|
||||
[ -n "$row" ] || continue
|
||||
|
||||
IFS=$'\t' read -r source_id source_label state state_label_value detail recovery_action install_hint fallback_hint <<EOF
|
||||
$row
|
||||
EOF
|
||||
|
||||
printf '%s\n' "- ${source_label}：${state_label_value}。${detail}。"
|
||||
printf '%s\n' " 下一步：${recovery_action}。"
|
||||
if [ "$install_hint" != "-" ]; then
|
||||
if [ "$state" = "available" ]; then
|
||||
printf '%s\n' " 补充说明：${install_hint}。"
|
||||
else
|
||||
printf '%s\n' " 安装提示：${install_hint}。"
|
||||
fi
|
||||
fi
|
||||
printf '%s\n' " 回退方式：${fallback_hint}。"
|
||||
done <<EOF
|
||||
$(print_summary | tail -n +2)
|
||||
EOF
|
||||
}
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--skill-root)
|
||||
[ $# -ge 2 ] || { usage; exit 1; }
|
||||
SKILL_ROOT_OVERRIDE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--layout-mode)
|
||||
[ $# -ge 2 ] || { usage; exit 1; }
|
||||
LAYOUT_MODE_OVERRIDE="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
command_name="${1:-}"
|
||||
|
||||
case "$command_name" in
|
||||
check)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
print_header
|
||||
resolve_preflight_state "$2"
|
||||
;;
|
||||
summary)
|
||||
[ "$#" -eq 1 ] || { usage; exit 1; }
|
||||
print_summary
|
||||
;;
|
||||
summary-human)
|
||||
[ "$#" -eq 1 ] || { usage; exit 1; }
|
||||
print_summary_human
|
||||
;;
|
||||
classify-run)
|
||||
[ "$#" -eq 4 ] || { usage; exit 1; }
|
||||
print_header
|
||||
classify_run_state "$2" "$3" "$4"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Executable
+395
@@ -0,0 +1,395 @@
|
||||
#!/bin/bash
|
||||
# build-graph-data.sh — 扫描 wiki/ 生成交互式图谱所需的 graph-data.json
|
||||
#
|
||||
# 用法：bash scripts/build-graph-data.sh <wiki_root> [output_path]
|
||||
# wiki_root 包含 wiki/ 子目录的知识库根路径
|
||||
# output_path 可选，默认 <wiki_root>/wiki/graph-data.json
|
||||
#
|
||||
# 环境变量：
|
||||
# LLM_WIKI_TEST_MODE=1 启用稳定输出（nodes/edges 按 id 字典序 + 时间戳固定）
|
||||
#
|
||||
# 退出码：0 成功；1 路径/依赖错误；2 wiki 结构不完整
|
||||
|
||||
set -eu
|
||||
shopt -s nullglob
|
||||
|
||||
SCRIPT_DIR="${BASH_SOURCE[0]%/*}"
|
||||
[ "$SCRIPT_DIR" = "${BASH_SOURCE[0]}" ] && SCRIPT_DIR="."
|
||||
SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
|
||||
WIKI_ROOT="${1:-.}"
|
||||
DEFAULT_OUTPUT="$WIKI_ROOT/wiki/graph-data.json"
|
||||
OUTPUT="${2:-$DEFAULT_OUTPUT}"
|
||||
SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
HELPER="$SKILL_DIR/scripts/graph-analysis.js"
|
||||
MAX_CONTENT_BYTES=$((2 * 1024 * 1024))
|
||||
MAX_CONTENT_LINES=500
|
||||
MAX_INSIGHT_NODES=250
|
||||
MAX_INSIGHT_EDGES=1000
|
||||
|
||||
command -v jq >/dev/null 2>&1 || {
|
||||
echo "ERROR: jq is not installed. Install it via:" >&2
|
||||
print_install_hint jq
|
||||
exit 1
|
||||
}
|
||||
|
||||
command -v node >/dev/null 2>&1 || {
|
||||
echo "ERROR: node is not installed. Install it via:" >&2
|
||||
print_install_hint node
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -f "$HELPER" ] || {
|
||||
echo "ERROR: 找不到图谱分析 helper：$HELPER" >&2
|
||||
echo " 重装 skill 可修复（bash install.sh --platform claude）" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
WIKI_DIR="$WIKI_ROOT/wiki"
|
||||
[ -d "$WIKI_DIR" ] || {
|
||||
echo "ERROR: wiki 目录不存在：$WIKI_DIR" >&2
|
||||
echo " 请先运行 init-wiki.sh 初始化知识库。" >&2
|
||||
exit 2
|
||||
}
|
||||
WIKI_ROOT_ABS="$(cd "$WIKI_ROOT" && pwd)"
|
||||
|
||||
TMPDIR=$(mktemp -d -t llm-wiki-graph.XXXXXX)
|
||||
trap 'rm -rf "$TMPDIR"' EXIT
|
||||
|
||||
if [ "${LLM_WIKI_TEST_MODE:-0}" = "1" ]; then
|
||||
BUILD_DATE="2026-01-01T00:00:00Z"
|
||||
else
|
||||
BUILD_DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
fi
|
||||
|
||||
WIKI_TITLE=""
|
||||
if [ -f "$WIKI_ROOT/purpose.md" ]; then
|
||||
WIKI_TITLE=$(awk '/^# / { sub(/^# +/, ""); print; exit }' "$WIKI_ROOT/purpose.md")
|
||||
fi
|
||||
[ -n "$WIKI_TITLE" ] || WIKI_TITLE=$(basename "$(cd "$WIKI_ROOT" && pwd)")
|
||||
|
||||
NODES_TSV="$TMPDIR/nodes.tsv"
|
||||
: > "$NODES_TSV"
|
||||
|
||||
scan_kind() {
|
||||
local subdir="$1" type="$2"
|
||||
local dir="$WIKI_DIR/$subdir"
|
||||
[ -d "$dir" ] || return 0
|
||||
local f id label
|
||||
while IFS= read -r f; do
|
||||
[ -f "$f" ] || continue
|
||||
id=$(basename "$f" .md)
|
||||
case "$id" in
|
||||
index|log|purpose|.wiki-schema|README) continue ;;
|
||||
esac
|
||||
label=$(awk '/^# / { sub(/^# +/, ""); gsub(/[[:space:]]+$/, ""); print; exit }' "$f")
|
||||
[ -n "$label" ] || label="$id"
|
||||
printf '%s\t%s\t%s\t%s\n' "$id" "$label" "$type" "$f" >> "$NODES_TSV"
|
||||
done < <(find "$dir" -type f -name '*.md' | LC_ALL=C sort)
|
||||
}
|
||||
|
||||
scan_kind entities entity
|
||||
scan_kind topics topic
|
||||
scan_kind sources source
|
||||
scan_kind comparisons comparison
|
||||
scan_kind synthesis synthesis
|
||||
scan_kind queries query
|
||||
|
||||
if [ ! -s "$NODES_TSV" ]; then
|
||||
mkdir -p "$(dirname "$OUTPUT")"
|
||||
OUTPUT_TMP="$TMPDIR/graph-data.empty.json"
|
||||
jq -n \
|
||||
--arg build_date "$BUILD_DATE" \
|
||||
--arg wiki_title "$WIKI_TITLE" \
|
||||
'{
|
||||
meta: {
|
||||
build_date: $build_date,
|
||||
wiki_title: $wiki_title,
|
||||
total_nodes: 0,
|
||||
total_edges: 0,
|
||||
initial_view: [],
|
||||
degraded: false,
|
||||
insights_degraded: false
|
||||
},
|
||||
nodes: [],
|
||||
edges: [],
|
||||
insights: {
|
||||
surprising_connections: [],
|
||||
isolated_nodes: [],
|
||||
bridge_nodes: [],
|
||||
sparse_communities: [],
|
||||
meta: {
|
||||
degraded: false,
|
||||
node_count: 0,
|
||||
edge_count: 0,
|
||||
max_insight_nodes: 250,
|
||||
max_insight_edges: 1000
|
||||
}
|
||||
},
|
||||
learning: {
|
||||
version: 1,
|
||||
entry: {
|
||||
recommended_start_node_id: null,
|
||||
recommended_start_reason: null,
|
||||
default_mode: "global"
|
||||
},
|
||||
views: {
|
||||
path: { enabled: false, start_node_id: null, node_ids: [], degraded: true },
|
||||
community: { enabled: false, community_id: null, label: null, node_ids: [], is_weak: false, degraded: true },
|
||||
global: { enabled: true, node_ids: [], degraded: false }
|
||||
},
|
||||
communities: [],
|
||||
degraded: { path_to_community: true, community_to_global: true }
|
||||
}
|
||||
}' > "$OUTPUT_TMP"
|
||||
mv "$OUTPUT_TMP" "$OUTPUT"
|
||||
echo "空图谱已写入：${OUTPUT}（wiki/ 下无可纳入节点）"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
EDGES_RAW="$TMPDIR/edges_raw.tsv"
|
||||
: > "$EDGES_RAW"
|
||||
|
||||
while IFS=$'\t' read -r id label type path; do
|
||||
awk -v src="$id" '
|
||||
{
|
||||
line = $0
|
||||
conf = ""
|
||||
rel = ""
|
||||
if (match(line, /<!--[[:space:]]*confidence:[[:space:]]*[A-Z]+[[:space:]]*-->/)) {
|
||||
kind_str = substr(line, RSTART, RLENGTH)
|
||||
if (match(kind_str, /[A-Z]+/)) {
|
||||
conf = substr(kind_str, RSTART, RLENGTH)
|
||||
}
|
||||
}
|
||||
if (match(line, /<!--[[:space:]]*relation(_type)?:[[:space:]]*/)) {
|
||||
rel_str = substr(line, RSTART + RLENGTH)
|
||||
rel_end = index(rel_str, "-->")
|
||||
if (rel_end > 0) {
|
||||
rel = substr(rel_str, 1, rel_end - 1)
|
||||
gsub(/^[[:space:]]+|[[:space:]]+$/, "", rel)
|
||||
}
|
||||
}
|
||||
rest = line
|
||||
while (match(rest, /\[\[[^]]+\]\]/)) {
|
||||
inner = substr(rest, RSTART + 2, RLENGTH - 4)
|
||||
rest = substr(rest, RSTART + RLENGTH)
|
||||
n = index(inner, "|")
|
||||
if (n > 0) inner = substr(inner, 1, n - 1)
|
||||
gsub(/^[[:space:]]+|[[:space:]]+$/, "", inner)
|
||||
if (inner == "" || inner == src) continue
|
||||
print src "\t" NR "\t" inner "\t" conf "\t" rel
|
||||
}
|
||||
}
|
||||
' "$path" >> "$EDGES_RAW"
|
||||
done < "$NODES_TSV"
|
||||
|
||||
VALID_IDS="$TMPDIR/valid_ids.txt"
|
||||
cut -f1 "$NODES_TSV" | sort -u > "$VALID_IDS"
|
||||
|
||||
EDGES_TSV="$TMPDIR/edges.tsv"
|
||||
# 合并同一 from+to 的多条 raw edges：
|
||||
# - 第一次遇到时记录（有 conf 就用 conf，无 conf 就留空 → 最终默认 EXTRACTED）
|
||||
# - 后续遇到带显式 conf 的条目时 **升级**（覆盖之前的空值或 EXTRACTED 默认）
|
||||
# - 若后续遇到多条不同的非空 conf，保留首个非空（按首次显式标注优先）
|
||||
#
|
||||
# 这解决了"同一对节点被多次 [[]] 引用（正文 + 相关页面列表）时，
|
||||
# 首次出现的空 conf 会永久锁定 edge type 为 EXTRACTED"的问题。
|
||||
awk -F'\t' -v valids="$VALID_IDS" '
|
||||
BEGIN {
|
||||
while ((getline line < valids) > 0) valid[line] = 1
|
||||
close(valids)
|
||||
}
|
||||
{
|
||||
from = $1; to = $3; conf = $4; rel = $5
|
||||
if (!(to in valid)) next
|
||||
if (from == to) next
|
||||
key = from "\t" to
|
||||
if (!(key in seen)) {
|
||||
seen[key] = 1
|
||||
saved_conf[key] = conf # 可能为空，在 END 中兜底为 EXTRACTED
|
||||
saved_rel[key] = rel # 可能为空，在 END 中兜底为依赖
|
||||
order[++count] = key
|
||||
} else if (conf != "" && saved_conf[key] == "") {
|
||||
# 升级：之前未见显式 conf（留空），现在有，采用
|
||||
saved_conf[key] = conf
|
||||
}
|
||||
if (rel != "" && saved_rel[key] == "") {
|
||||
# 升级：之前未见显式 relation type，现在有，采用
|
||||
saved_rel[key] = rel
|
||||
}
|
||||
}
|
||||
END {
|
||||
for (i = 1; i <= count; i++) {
|
||||
split(order[i], parts, "\t")
|
||||
t = saved_conf[order[i]]
|
||||
if (t != "EXTRACTED" && t != "INFERRED" && t != "AMBIGUOUS") t = "EXTRACTED"
|
||||
r = saved_rel[order[i]]
|
||||
if (r == "") r = "依赖"
|
||||
print parts[1] "\t" parts[2] "\t" t "\t" r
|
||||
}
|
||||
}
|
||||
' "$EDGES_RAW" > "$EDGES_TSV"
|
||||
|
||||
TOTAL_SIZE=0
|
||||
while IFS=$'\t' read -r id label type path; do
|
||||
sz=$(wc -c < "$path" 2>/dev/null || echo 0)
|
||||
TOTAL_SIZE=$((TOTAL_SIZE + sz))
|
||||
done < "$NODES_TSV"
|
||||
|
||||
DEGRADE=0
|
||||
if [ "$TOTAL_SIZE" -gt "$MAX_CONTENT_BYTES" ]; then
|
||||
DEGRADE=1
|
||||
fi
|
||||
|
||||
NODES_JSONL="$TMPDIR/nodes.jsonl"
|
||||
: > "$NODES_JSONL"
|
||||
while IFS=$'\t' read -r id label type path; do
|
||||
abs_path=$(cd "$(dirname "$path")" && pwd)/$(basename "$path")
|
||||
rel_path="${abs_path#"$WIKI_ROOT_ABS"/}"
|
||||
jq -n \
|
||||
--arg id "$id" \
|
||||
--arg label "$label" \
|
||||
--arg type "$type" \
|
||||
--arg source_path "$abs_path" \
|
||||
--arg rel_path "$rel_path" \
|
||||
'{
|
||||
id: $id,
|
||||
label: $label,
|
||||
type: $type,
|
||||
source_path: $rel_path,
|
||||
_file_path: $source_path
|
||||
}' >> "$NODES_JSONL"
|
||||
done < "$NODES_TSV"
|
||||
|
||||
EDGES_JSONL="$TMPDIR/edges.jsonl"
|
||||
: > "$EDGES_JSONL"
|
||||
idx=0
|
||||
while IFS=$'\t' read -r from to etype relation_type; do
|
||||
idx=$((idx + 1))
|
||||
jq -n \
|
||||
--arg id "e$idx" \
|
||||
--arg from "$from" \
|
||||
--arg to "$to" \
|
||||
--arg etype "$etype" \
|
||||
--arg relation_type "$relation_type" \
|
||||
'{id: $id, from: $from, to: $to, type: $etype, confidence: $etype, relation_type: $relation_type}' >> "$EDGES_JSONL"
|
||||
done < "$EDGES_TSV"
|
||||
|
||||
if [ "${LLM_WIKI_TEST_MODE:-0}" = "1" ]; then
|
||||
jq -s 'sort_by(.id)' "$NODES_JSONL" > "$TMPDIR/nodes.raw.json"
|
||||
jq -s 'sort_by(.from, .to, .type)
|
||||
| to_entries
|
||||
| map(.value + {id: ("e" + ((.key + 1) | tostring))})' \
|
||||
"$EDGES_JSONL" > "$TMPDIR/edges.raw.json"
|
||||
else
|
||||
jq -s '.' "$NODES_JSONL" > "$TMPDIR/nodes.raw.json"
|
||||
jq -s '.' "$EDGES_JSONL" > "$TMPDIR/edges.raw.json"
|
||||
fi
|
||||
|
||||
ANALYSIS_JSON="$TMPDIR/analysis.json"
|
||||
if ! node "$HELPER" \
|
||||
"$TMPDIR/nodes.raw.json" \
|
||||
"$TMPDIR/edges.raw.json" \
|
||||
"$ANALYSIS_JSON" \
|
||||
"$DEGRADE" \
|
||||
"$MAX_CONTENT_LINES" \
|
||||
"$MAX_INSIGHT_NODES" \
|
||||
"$MAX_INSIGHT_EDGES"; then
|
||||
echo "ERROR: 图谱分析 helper 执行失败：$HELPER" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
jq -e '
|
||||
(.nodes | type) == "array" and
|
||||
(.edges | type) == "array" and
|
||||
(.insights | type) == "object" and
|
||||
(.insights.surprising_connections | type) == "array" and
|
||||
(.insights.isolated_nodes | type) == "array" and
|
||||
(.insights.bridge_nodes | type) == "array" and
|
||||
(.insights.sparse_communities | type) == "array" and
|
||||
(.learning | type) == "object"
|
||||
' "$ANALYSIS_JSON" > /dev/null 2>&1 || {
|
||||
echo "ERROR: 图谱分析 helper 返回坏 JSON：$ANALYSIS_JSON" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ "${LLM_WIKI_TEST_MODE:-0}" = "1" ]; then
|
||||
jq '.nodes | sort_by(.id)' "$ANALYSIS_JSON" > "$TMPDIR/nodes.sorted.json"
|
||||
jq '.edges | sort_by(.from, .to, .type)
|
||||
| to_entries
|
||||
| map(.value + {id: ("e" + ((.key + 1) | tostring))})' "$ANALYSIS_JSON" > "$TMPDIR/edges.sorted.json"
|
||||
else
|
||||
jq '.nodes' "$ANALYSIS_JSON" > "$TMPDIR/nodes.sorted.json"
|
||||
jq '.edges' "$ANALYSIS_JSON" > "$TMPDIR/edges.sorted.json"
|
||||
fi
|
||||
|
||||
INITIAL_VIEW=$(jq \
|
||||
--argjson nodes "$(cat "$TMPDIR/nodes.sorted.json")" \
|
||||
'
|
||||
. as $edges
|
||||
| (
|
||||
reduce $edges[] as $e (
|
||||
{};
|
||||
.[$e.from] = (.[$e.from] // 0) + 1 |
|
||||
.[$e.to] = (.[$e.to] // 0) + 1
|
||||
)
|
||||
) as $deg
|
||||
| ($nodes | group_by(.community // "_")) as $groups
|
||||
| ([ $groups[] | max_by(($deg[.id] // 0)) | .id ]) as $reps
|
||||
| (
|
||||
$nodes
|
||||
| sort_by(- ($deg[.id] // 0))
|
||||
| map(.id)
|
||||
| map(select(. as $x | $reps | index($x) | not))
|
||||
) as $rest
|
||||
| ($reps + $rest)[0:30]
|
||||
' \
|
||||
"$TMPDIR/edges.sorted.json")
|
||||
|
||||
NODE_COUNT=$(jq 'length' "$TMPDIR/nodes.sorted.json")
|
||||
EDGE_COUNT=$(jq 'length' "$TMPDIR/edges.sorted.json")
|
||||
INSIGHTS_DEGRADED=$(jq '.insights.meta.degraded == true' "$ANALYSIS_JSON")
|
||||
|
||||
mkdir -p "$(dirname "$OUTPUT")"
|
||||
OUTPUT_TMP="$TMPDIR/graph-data.final.json"
|
||||
|
||||
jq -n \
|
||||
--arg build_date "$BUILD_DATE" \
|
||||
--arg wiki_title "$WIKI_TITLE" \
|
||||
--argjson total_nodes "$NODE_COUNT" \
|
||||
--argjson total_edges "$EDGE_COUNT" \
|
||||
--argjson initial_view "$INITIAL_VIEW" \
|
||||
--argjson nodes "$(cat "$TMPDIR/nodes.sorted.json")" \
|
||||
--argjson edges "$(cat "$TMPDIR/edges.sorted.json")" \
|
||||
--argjson insights "$(jq '.insights' "$ANALYSIS_JSON")" \
|
||||
--argjson learning "$(jq '.learning' "$ANALYSIS_JSON")" \
|
||||
--argjson degraded "$DEGRADE" \
|
||||
--argjson insights_degraded "$INSIGHTS_DEGRADED" \
|
||||
'{
|
||||
meta: {
|
||||
build_date: $build_date,
|
||||
wiki_title: $wiki_title,
|
||||
total_nodes: $total_nodes,
|
||||
total_edges: $total_edges,
|
||||
initial_view: $initial_view,
|
||||
degraded: ($degraded == 1),
|
||||
insights_degraded: $insights_degraded
|
||||
},
|
||||
nodes: $nodes,
|
||||
edges: $edges,
|
||||
insights: $insights,
|
||||
learning: $learning
|
||||
}' > "$OUTPUT_TMP"
|
||||
|
||||
mv "$OUTPUT_TMP" "$OUTPUT"
|
||||
|
||||
echo "图谱数据已生成：$OUTPUT"
|
||||
echo " 节点：$NODE_COUNT"
|
||||
echo " 关联：$EDGE_COUNT"
|
||||
echo " 初始视图：$(echo "$INITIAL_VIEW" | jq 'length') 个节点"
|
||||
[ "$DEGRADE" = "1" ] && echo " ⚠ 降级模式：内嵌内容 > 2MB，每节点仅保留前 ${MAX_CONTENT_LINES} 行"
|
||||
[ "$INSIGHTS_DEGRADED" = "true" ] && echo " ⚠ 洞察降级：图规模超出预算，仅保留基础权重与社区"
|
||||
exit 0
|
||||
Executable
+459
@@ -0,0 +1,459 @@
|
||||
#!/bin/bash
|
||||
# build-graph-html.sh — 生成共享 graph-engine 驱动的离线知识图谱 HTML
|
||||
#
|
||||
# 用法：
|
||||
# bash scripts/build-graph-html.sh <wiki_root>
|
||||
#
|
||||
# 前置：需要先运行 build-graph-data.sh 生成 wiki/graph-data.json
|
||||
#
|
||||
# 行为：
|
||||
# 1. 读取 packages/graph-engine/dist/engine.iife.js
|
||||
# 2. 内嵌 graph-data.json 与可选 .wiki-graph-layout.json 钉位
|
||||
# 3. 注入离线启动脚本：创建 graph engine，持久化钉位到 localStorage
|
||||
# 4. 生成单文件 knowledge-graph.html
|
||||
#
|
||||
# 退出码：0 成功；1 依赖/文件缺失/参数错误
|
||||
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="${BASH_SOURCE[0]%/*}"
|
||||
[ "$SCRIPT_DIR" = "${BASH_SOURCE[0]}" ] && SCRIPT_DIR="."
|
||||
SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
|
||||
print_usage() {
|
||||
cat <<'USAGE'
|
||||
用法：
|
||||
bash scripts/build-graph-html.sh <wiki_root>
|
||||
|
||||
示例：
|
||||
bash scripts/build-graph-html.sh /path/to/wiki-root
|
||||
USAGE
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "ERROR: $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
ensure_file() {
|
||||
local file="$1"
|
||||
local label="${2:-文件}"
|
||||
[ -f "$file" ] || {
|
||||
echo "ERROR: 找不到${label} $file" >&2
|
||||
echo " 请先运行 npm run build -w @llm-wiki/graph-engine，或重装 skill。" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
json_for_script() {
|
||||
perl -pe 's|</script>|<\\/script>|gi' "$1"
|
||||
}
|
||||
|
||||
script_for_inline() {
|
||||
perl -pe 's|//# sourceMappingURL=.*$||' "$1"
|
||||
}
|
||||
|
||||
html_escape_text() {
|
||||
printf '%s' "$1" | perl -pe 's/&/&/g; s/</</g; s/>/>/g; s/"/"/g; s/'"'"'/'/g'
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
-h|--help)
|
||||
print_usage
|
||||
exit 0
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
-*)
|
||||
die "未知选项: $1"
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ "$#" -eq 1 ] || {
|
||||
print_usage >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
WIKI_ROOT="$1"
|
||||
|
||||
command -v jq >/dev/null 2>&1 || {
|
||||
echo "ERROR: jq is not installed. Install it via:" >&2
|
||||
print_install_hint jq
|
||||
exit 1
|
||||
}
|
||||
|
||||
SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
DATA="$WIKI_ROOT/wiki/graph-data.json"
|
||||
LAYOUT="$WIKI_ROOT/.wiki-graph-layout.json"
|
||||
ENGINE="$SKILL_DIR/packages/graph-engine/dist/engine.iife.js"
|
||||
MARKED="$SKILL_DIR/deps/marked.min.js"
|
||||
PURIFY="$SKILL_DIR/deps/purify.min.js"
|
||||
OUTPUT="$WIKI_ROOT/wiki/knowledge-graph.html"
|
||||
|
||||
[ -f "$DATA" ] || {
|
||||
echo "ERROR: 未找到 $DATA" >&2
|
||||
echo " 请先运行 build-graph-data.sh 生成图谱数据" >&2
|
||||
exit 1
|
||||
}
|
||||
ensure_file "$ENGINE" "graph-engine IIFE 产物"
|
||||
ensure_file "$MARKED" "marked vendor"
|
||||
ensure_file "$PURIFY" "purify vendor"
|
||||
|
||||
WIKI_TITLE=$(jq -r '.meta.wiki_title // "知识库"' "$DATA")
|
||||
NODE_COUNT=$(jq -r '.meta.total_nodes // 0' "$DATA")
|
||||
EDGE_COUNT=$(jq -r '.meta.total_edges // 0' "$DATA")
|
||||
BUILD_DATE=$(jq -r '.meta.build_date // ""' "$DATA")
|
||||
BUILD_DATE_SHORT="${BUILD_DATE:0:10}"
|
||||
[ -n "$BUILD_DATE_SHORT" ] || BUILD_DATE_SHORT="未知"
|
||||
WIKI_TITLE_HTML=$(html_escape_text "$WIKI_TITLE")
|
||||
NODE_COUNT_HTML=$(html_escape_text "$NODE_COUNT")
|
||||
EDGE_COUNT_HTML=$(html_escape_text "$EDGE_COUNT")
|
||||
BUILD_DATE_SHORT_HTML=$(html_escape_text "$BUILD_DATE_SHORT")
|
||||
|
||||
layout_json='{"version":2,"pins":{},"updatedAt":""}'
|
||||
if [ -f "$LAYOUT" ]; then
|
||||
if layout_json_candidate=$(jq -c '{version:(.version // 1), pins:(.pins // {}), updatedAt:(.updatedAt // "")}' "$LAYOUT" 2>/dev/null); then
|
||||
layout_json="$layout_json_candidate"
|
||||
else
|
||||
echo "WARN: 忽略损坏的钉位文件：$LAYOUT" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
output_dir="$(dirname "$OUTPUT")"
|
||||
mkdir -p "$output_dir"
|
||||
output_tmp="$OUTPUT.partial"
|
||||
output_next="$OUTPUT.next"
|
||||
rm -f "$output_tmp" "$output_next"
|
||||
|
||||
cat > "$output_tmp" <<HTML_HEAD
|
||||
<!doctype html>
|
||||
<html lang="zh-Hans">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>知识图谱 · ${WIKI_TITLE_HTML}</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--page-bg: #f7f1e5;
|
||||
--panel: rgba(255, 252, 244, .86);
|
||||
--ink: #2f2924;
|
||||
--muted: #766b5f;
|
||||
--rule: rgba(79, 64, 46, .2);
|
||||
--accent: #a83f35;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
html, body { margin: 0; min-height: 100%; }
|
||||
body {
|
||||
min-height: 100vh;
|
||||
color: var(--ink);
|
||||
background: var(--page-bg);
|
||||
font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
}
|
||||
.offline-shell {
|
||||
display: grid;
|
||||
grid-template-rows: auto minmax(0, 1fr);
|
||||
min-height: 100vh;
|
||||
}
|
||||
.offline-header {
|
||||
position: relative;
|
||||
z-index: 20;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
min-height: 64px;
|
||||
padding: 12px 18px;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
background: var(--panel);
|
||||
backdrop-filter: blur(10px);
|
||||
}
|
||||
.offline-title { min-width: 0; }
|
||||
.offline-title h1 {
|
||||
margin: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
font-family: Georgia, "Times New Roman", serif;
|
||||
font-size: 20px;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
.offline-title p {
|
||||
margin: 4px 0 0;
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
}
|
||||
.offline-badges {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
flex-wrap: wrap;
|
||||
justify-content: flex-end;
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
}
|
||||
.offline-badges span {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
min-height: 26px;
|
||||
padding: 4px 9px;
|
||||
border: 1px solid var(--rule);
|
||||
border-radius: 999px;
|
||||
background: rgba(255, 255, 255, .46);
|
||||
}
|
||||
.offline-theme-toggle {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
min-height: 26px;
|
||||
border: 1px solid var(--rule);
|
||||
border-radius: 999px;
|
||||
background: rgba(255, 255, 255, .52);
|
||||
color: var(--ink);
|
||||
padding: 4px 10px;
|
||||
font: inherit;
|
||||
cursor: pointer;
|
||||
}
|
||||
.offline-theme-toggle:hover {
|
||||
background: rgba(168, 63, 53, .08);
|
||||
}
|
||||
.offline-toolbar-host {
|
||||
position: relative;
|
||||
z-index: 5;
|
||||
flex: 1 1 320px;
|
||||
min-width: 240px;
|
||||
min-height: 38px;
|
||||
}
|
||||
.offline-toolbar-host .graph-toolbar {
|
||||
position: static;
|
||||
inset: auto;
|
||||
justify-items: center;
|
||||
}
|
||||
.offline-toolbar-host .graph-toolbar-panel {
|
||||
position: absolute;
|
||||
top: 38px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
}
|
||||
.offline-main {
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
min-height: 0;
|
||||
padding: 0;
|
||||
}
|
||||
#graph-root {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
min-height: 560px;
|
||||
}
|
||||
.offline-error {
|
||||
margin: 24px;
|
||||
padding: 16px;
|
||||
border: 1px solid rgba(168, 63, 53, .35);
|
||||
border-radius: 8px;
|
||||
background: rgba(168, 63, 53, .08);
|
||||
color: #7b2b24;
|
||||
font-size: 14px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
@media (max-width: 720px) {
|
||||
.offline-header { align-items: flex-start; flex-direction: column; }
|
||||
.offline-toolbar-host { width: 100%; flex-basis: auto; }
|
||||
.offline-toolbar-host .graph-toolbar { justify-items: start; }
|
||||
.offline-toolbar-host .graph-toolbar-panel { left: 0; transform: none; }
|
||||
.offline-badges { justify-content: flex-start; }
|
||||
#graph-root { min-height: 520px; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="offline-shell" data-llm-wiki-offline-graph="engine">
|
||||
<header class="offline-header">
|
||||
<div class="offline-title">
|
||||
<h1>${WIKI_TITLE_HTML} 知识舆图</h1>
|
||||
<p>国风知识库·数字山水图</p>
|
||||
</div>
|
||||
<div class="offline-toolbar-host" data-testid="offline-toolbar-host"></div>
|
||||
<div class="offline-badges" aria-label="图谱统计">
|
||||
<span>${NODE_COUNT_HTML} 节点</span>
|
||||
<span>${EDGE_COUNT_HTML} 关联</span>
|
||||
<span>${BUILD_DATE_SHORT_HTML}</span>
|
||||
<button class="offline-theme-toggle" type="button" data-testid="offline-theme-toggle" aria-label="切换墨夜主题">墨夜</button>
|
||||
</div>
|
||||
</header>
|
||||
<main class="offline-main">
|
||||
<div id="graph-root" data-testid="offline-graph-root"></div>
|
||||
</main>
|
||||
</div>
|
||||
<script id="graph-data" type="application/json">
|
||||
HTML_HEAD
|
||||
json_for_script "$DATA" >> "$output_tmp"
|
||||
cat >> "$output_tmp" <<'HTML_MID'
|
||||
</script>
|
||||
<script id="graph-layout" type="application/json">
|
||||
HTML_MID
|
||||
printf '%s\n' "$layout_json" | perl -pe 's|</script>|<\/script>|gi' >> "$output_tmp"
|
||||
cat >> "$output_tmp" <<'HTML_ENGINE'
|
||||
</script>
|
||||
<script>
|
||||
HTML_ENGINE
|
||||
script_for_inline "$MARKED" >> "$output_tmp"
|
||||
printf '\n' >> "$output_tmp"
|
||||
script_for_inline "$PURIFY" >> "$output_tmp"
|
||||
printf '\n' >> "$output_tmp"
|
||||
script_for_inline "$ENGINE" >> "$output_tmp"
|
||||
cat >> "$output_tmp" <<'HTML_BOOT'
|
||||
</script>
|
||||
<script>
|
||||
(function () {
|
||||
var root = document.getElementById("graph-root");
|
||||
var toolbarHost = document.querySelector("[data-testid='offline-toolbar-host']");
|
||||
var dataEl = document.getElementById("graph-data");
|
||||
var layoutEl = document.getElementById("graph-layout");
|
||||
function showError(message) {
|
||||
if (!root) return;
|
||||
root.innerHTML = "";
|
||||
var box = document.createElement("div");
|
||||
box.className = "offline-error";
|
||||
box.textContent = message;
|
||||
root.appendChild(box);
|
||||
}
|
||||
function parseJson(el, fallback) {
|
||||
try { return el && el.textContent ? JSON.parse(el.textContent) : fallback; }
|
||||
catch (err) { return fallback; }
|
||||
}
|
||||
function normalizeStorageSegment(value) {
|
||||
return String(value == null ? "" : value).trim().toLowerCase()
|
||||
.replace(/[^a-z0-9一-鿿]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 48);
|
||||
}
|
||||
function hashString(value) {
|
||||
var input = String(value == null ? "" : value);
|
||||
var hash = 0;
|
||||
for (var i = 0; i < input.length; i++) {
|
||||
hash = ((hash << 5) - hash + input.charCodeAt(i)) >>> 0;
|
||||
}
|
||||
return hash.toString(36);
|
||||
}
|
||||
function storageNamespace(meta, pathname) {
|
||||
var title = normalizeStorageSegment(meta && meta.wiki_title ? meta.wiki_title : "");
|
||||
var basis = typeof pathname === "string" && pathname ? pathname : (meta && meta.wiki_title) || title || "default";
|
||||
return "llm-wiki:" + (title || "default") + ":" + hashString(basis);
|
||||
}
|
||||
function readStoredPins(key) {
|
||||
try {
|
||||
var raw = window.localStorage && window.localStorage.getItem(key);
|
||||
var parsed = raw ? JSON.parse(raw) : null;
|
||||
return parsed && typeof parsed === "object" ? parsed : {};
|
||||
} catch (_) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
function writeStoredPins(key, pins) {
|
||||
try {
|
||||
if (window.localStorage) window.localStorage.setItem(key, JSON.stringify(pins || {}));
|
||||
} catch (_) {}
|
||||
}
|
||||
function normalizeBakedPins(layout) {
|
||||
return window.LlmWikiGraphEngine.normalizeGraphLayoutFile(layout).pins;
|
||||
}
|
||||
function normalizeStoredPins(rawPins) {
|
||||
return window.LlmWikiGraphEngine.normalizeGraphPinMap(rawPins);
|
||||
}
|
||||
if (!root || !dataEl || !window.LlmWikiGraphEngine || !window.LlmWikiGraphEngine.createGraphEngine) {
|
||||
showError("图谱引擎加载失败。请确认 HTML 文件完整生成。");
|
||||
return;
|
||||
}
|
||||
var graphData = parseJson(dataEl, null);
|
||||
if (!graphData || !Array.isArray(graphData.nodes) || !Array.isArray(graphData.edges)) {
|
||||
showError("图谱数据格式不完整。请重新运行 build-graph-data.sh 与 build-graph-html.sh。");
|
||||
return;
|
||||
}
|
||||
var bakedLayout = parseJson(layoutEl, { pins: {} });
|
||||
var key = storageNamespace(graphData.meta || {}, window.location && window.location.pathname) + ":graph-pins";
|
||||
var themeKey = storageNamespace(graphData.meta || {}, window.location && window.location.pathname) + ":graph-theme";
|
||||
var pins = Object.assign({}, normalizeBakedPins(bakedLayout), normalizeStoredPins(readStoredPins(key)));
|
||||
var themeToggle = document.querySelector("[data-testid='offline-theme-toggle']");
|
||||
function readStoredTheme() {
|
||||
try {
|
||||
var value = window.localStorage && window.localStorage.getItem(themeKey);
|
||||
return value === "mo-ye" ? "mo-ye" : "shan-shui";
|
||||
} catch (_) {
|
||||
return "shan-shui";
|
||||
}
|
||||
}
|
||||
function writeStoredTheme(theme) {
|
||||
try {
|
||||
if (window.localStorage) window.localStorage.setItem(themeKey, theme);
|
||||
} catch (_) {}
|
||||
}
|
||||
function syncThemeToggle(theme) {
|
||||
if (!themeToggle) return;
|
||||
var next = theme === "mo-ye" ? "shan-shui" : "mo-ye";
|
||||
themeToggle.textContent = theme === "mo-ye" ? "山水" : "墨夜";
|
||||
themeToggle.setAttribute("aria-label", next === "mo-ye" ? "切换墨夜主题" : "切换山水主题");
|
||||
}
|
||||
var currentTheme = readStoredTheme();
|
||||
var engine = window.LlmWikiGraphEngine.createGraphEngine(root, {
|
||||
data: graphData,
|
||||
pins: pins,
|
||||
theme: currentTheme,
|
||||
toolbarContainer: toolbarHost,
|
||||
capabilities: window.LlmWikiGraphEngine.createGraphOfflineCapabilities({
|
||||
persistPins: function (nextPins) {
|
||||
writeStoredPins(key, nextPins || {});
|
||||
return Promise.resolve();
|
||||
}
|
||||
}).capabilities
|
||||
});
|
||||
syncThemeToggle(currentTheme);
|
||||
if (themeToggle) {
|
||||
themeToggle.addEventListener("click", function () {
|
||||
currentTheme = currentTheme === "mo-ye" ? "shan-shui" : "mo-ye";
|
||||
engine.setTheme(currentTheme);
|
||||
writeStoredTheme(currentTheme);
|
||||
syncThemeToggle(currentTheme);
|
||||
});
|
||||
}
|
||||
window.__LLM_WIKI_GRAPH_ENGINE__ = engine;
|
||||
window.__LLM_WIKI_GRAPH_PINS_KEY__ = key;
|
||||
window.__LLM_WIKI_GRAPH_THEME_KEY__ = themeKey;
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
HTML_BOOT
|
||||
|
||||
mv "$output_tmp" "$output_next"
|
||||
mv "$output_next" "$OUTPUT"
|
||||
|
||||
rm -f \
|
||||
"$output_dir/d3.min.js" \
|
||||
"$output_dir/rough.min.js" \
|
||||
"$output_dir/marked.min.js" \
|
||||
"$output_dir/purify.min.js" \
|
||||
"$output_dir/graph-wash.js" \
|
||||
"$output_dir/graph-wash-helpers.js" \
|
||||
"$output_dir/LICENSE-d3.txt" \
|
||||
"$output_dir/LICENSE-roughjs.txt" \
|
||||
"$output_dir/LICENSE-marked.txt" \
|
||||
"$output_dir/LICENSE-purify.txt"
|
||||
|
||||
output_size=$(wc -c < "$OUTPUT" | tr -d ' ')
|
||||
output_kb=$((output_size / 1024))
|
||||
|
||||
echo "交互式图谱已生成："
|
||||
echo " - $OUTPUT (${output_kb} KB)"
|
||||
echo " 节点 $NODE_COUNT · 关联 $EDGE_COUNT"
|
||||
echo ""
|
||||
echo "查看方式："
|
||||
echo " 双击 $OUTPUT"
|
||||
Executable
+352
@@ -0,0 +1,352 @@
|
||||
#!/bin/bash
|
||||
# llm-wiki 缓存脚本
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/cache.sh check <file>
|
||||
bash scripts/cache.sh update <file> <source_page>
|
||||
bash scripts/cache.sh invalidate <file>
|
||||
EOF
|
||||
}
|
||||
|
||||
require_file() {
|
||||
local file_path="$1"
|
||||
|
||||
[ -n "$file_path" ] || {
|
||||
usage
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -f "$file_path" ] || {
|
||||
echo "文件不存在：$file_path" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
find_wiki_root() {
|
||||
local file_path="$1"
|
||||
local dir parent
|
||||
|
||||
dir="$(cd "$(dirname "$file_path")" && pwd)"
|
||||
|
||||
while true; do
|
||||
if [ -f "$dir/.wiki-cache.json" ] || [ -f "$dir/.wiki-schema.md" ]; then
|
||||
printf '%s\n' "$dir"
|
||||
return 0
|
||||
fi
|
||||
|
||||
parent="$(dirname "$dir")"
|
||||
[ "$parent" = "$dir" ] && return 1
|
||||
dir="$parent"
|
||||
done
|
||||
}
|
||||
|
||||
cache_file_path() {
|
||||
printf '%s/.wiki-cache.json\n' "$1"
|
||||
}
|
||||
|
||||
ensure_cache_file() {
|
||||
local cache_file="$1"
|
||||
|
||||
if [ ! -f "$cache_file" ]; then
|
||||
cat > "$cache_file" <<'EOF'
|
||||
{
|
||||
"version": 1,
|
||||
"entries": {}
|
||||
}
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
relative_path() {
|
||||
require_python_cmd
|
||||
|
||||
"$PYTHON_CMD" - "$1" "$2" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
|
||||
print(os.path.relpath(os.path.realpath(sys.argv[2]), os.path.realpath(sys.argv[1])))
|
||||
PY
|
||||
}
|
||||
|
||||
normalized_source_page() {
|
||||
local wiki_root="$1"
|
||||
local source_page="$2"
|
||||
|
||||
if [ -z "$source_page" ]; then
|
||||
printf '%s\n' ""
|
||||
return 0
|
||||
fi
|
||||
|
||||
case "$source_page" in
|
||||
/*)
|
||||
require_python_cmd
|
||||
|
||||
"$PYTHON_CMD" - "$wiki_root" "$source_page" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
|
||||
wiki_root = os.path.realpath(sys.argv[1])
|
||||
source_page = os.path.realpath(sys.argv[2])
|
||||
|
||||
try:
|
||||
common = os.path.commonpath([wiki_root, source_page])
|
||||
except ValueError:
|
||||
common = ""
|
||||
|
||||
if common == wiki_root:
|
||||
print(os.path.relpath(source_page, wiki_root))
|
||||
else:
|
||||
print(sys.argv[2])
|
||||
PY
|
||||
;;
|
||||
*)
|
||||
printf '%s\n' "$source_page"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
file_hash() {
|
||||
require_python_cmd
|
||||
|
||||
"$PYTHON_CMD" - "$1" "$2" <<'PY'
|
||||
import hashlib
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
relative_path = sys.argv[1].encode("utf-8")
|
||||
file_path = pathlib.Path(sys.argv[2])
|
||||
content = file_path.read_bytes()
|
||||
|
||||
digest = hashlib.sha256(relative_path + b"\0" + content).hexdigest()
|
||||
print(f"sha256:{digest}")
|
||||
PY
|
||||
}
|
||||
|
||||
cache_check() {
|
||||
local file_path="$1"
|
||||
local wiki_root cache_file relative_path_value current_hash result
|
||||
|
||||
require_file "$file_path"
|
||||
wiki_root="$(find_wiki_root "$file_path")" || {
|
||||
echo "未找到知识库根目录：$file_path" >&2
|
||||
exit 1
|
||||
}
|
||||
cache_file="$(cache_file_path "$wiki_root")"
|
||||
|
||||
if [ ! -f "$cache_file" ]; then
|
||||
printf 'MISS\n'
|
||||
return 0
|
||||
fi
|
||||
|
||||
require_python_cmd
|
||||
|
||||
relative_path_value="$(relative_path "$wiki_root" "$file_path")"
|
||||
current_hash="$(file_hash "$relative_path_value" "$file_path")"
|
||||
|
||||
result="$(
|
||||
"$PYTHON_CMD" - "$cache_file" "$wiki_root" "$relative_path_value" "$current_hash" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
cache_file, wiki_root, relative_path, current_hash = sys.argv[1:5]
|
||||
|
||||
with open(cache_file, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
|
||||
entry = data.get("entries", {}).get(relative_path)
|
||||
|
||||
# 无 cache entry → 尝试自愈（exact filename stem match + source_path 验证）
|
||||
if not entry:
|
||||
raw_stem = pathlib.Path(relative_path).stem
|
||||
sources_dir = os.path.join(wiki_root, "wiki", "sources")
|
||||
if os.path.isdir(sources_dir):
|
||||
for f in os.listdir(sources_dir):
|
||||
if pathlib.Path(f).stem == raw_stem and f.endswith(".md"):
|
||||
source_page = os.path.join("wiki", "sources", f)
|
||||
source_abs = os.path.join(wiki_root, source_page)
|
||||
# 验证 source 页面的 source_path frontmatter 是否指向当前 raw 文件
|
||||
source_path_match = False
|
||||
try:
|
||||
with open(source_abs, "r", encoding="utf-8") as sf:
|
||||
in_frontmatter = False
|
||||
for line in sf:
|
||||
stripped = line.strip()
|
||||
if stripped == "---":
|
||||
if in_frontmatter:
|
||||
break # end of frontmatter
|
||||
in_frontmatter = True
|
||||
continue
|
||||
if in_frontmatter and stripped.startswith("source_path:"):
|
||||
fm_value = stripped.split(":", 1)[1].strip()
|
||||
# 匹配相对路径的末尾部分
|
||||
if relative_path.endswith(fm_value) or fm_value.endswith(relative_path) or fm_value == relative_path:
|
||||
source_path_match = True
|
||||
break
|
||||
except (OSError, UnicodeDecodeError):
|
||||
pass
|
||||
if not source_path_match:
|
||||
# stem 匹配但 source_path 不一致 → 不信任，需要验证
|
||||
print("MISS:repaired_needs_verify")
|
||||
raise SystemExit(0)
|
||||
# stem + source_path 都匹配 → 安全自愈
|
||||
timestamp = __import__("datetime").datetime.now(__import__("datetime").timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
entries = data.setdefault("entries", {})
|
||||
entries[relative_path] = {
|
||||
"hash": current_hash,
|
||||
"ingested_at": timestamp,
|
||||
"source_page": source_page,
|
||||
}
|
||||
tmp_file = cache_file + ".tmp"
|
||||
with open(tmp_file, "w", encoding="utf-8") as fh2:
|
||||
json.dump(data, fh2, ensure_ascii=False, indent=2)
|
||||
fh2.write("\n")
|
||||
os.replace(tmp_file, cache_file)
|
||||
print("HIT(repaired)")
|
||||
raise SystemExit(0)
|
||||
print("MISS:no_entry")
|
||||
raise SystemExit(0)
|
||||
|
||||
if entry.get("hash") != current_hash:
|
||||
print("MISS:hash_changed")
|
||||
raise SystemExit(0)
|
||||
|
||||
source_page = entry.get("source_page")
|
||||
if not source_page:
|
||||
print("MISS:no_entry")
|
||||
raise SystemExit(0)
|
||||
|
||||
source_path = source_page
|
||||
if not os.path.isabs(source_path):
|
||||
source_path = os.path.join(wiki_root, source_path)
|
||||
|
||||
if not os.path.isfile(source_path):
|
||||
print("MISS:no_source")
|
||||
else:
|
||||
print("HIT")
|
||||
PY
|
||||
)"
|
||||
|
||||
printf '%s\n' "$result"
|
||||
}
|
||||
|
||||
cache_update() {
|
||||
local file_path="$1"
|
||||
local source_page="$2"
|
||||
local wiki_root cache_file relative_path_value current_hash normalized_source timestamp
|
||||
|
||||
require_file "$file_path"
|
||||
wiki_root="$(find_wiki_root "$file_path")" || {
|
||||
echo "未找到知识库根目录：$file_path" >&2
|
||||
exit 1
|
||||
}
|
||||
cache_file="$(cache_file_path "$wiki_root")"
|
||||
ensure_cache_file "$cache_file"
|
||||
|
||||
require_python_cmd
|
||||
|
||||
relative_path_value="$(relative_path "$wiki_root" "$file_path")"
|
||||
current_hash="$(file_hash "$relative_path_value" "$file_path")"
|
||||
normalized_source="$(normalized_source_page "$wiki_root" "$source_page")"
|
||||
timestamp="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
|
||||
"$PYTHON_CMD" - "$cache_file" "$relative_path_value" "$current_hash" "$timestamp" "$normalized_source" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
cache_file, relative_path, file_hash_value, timestamp, source_page = sys.argv[1:6]
|
||||
|
||||
with open(cache_file, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
|
||||
entries = data.setdefault("entries", {})
|
||||
entries[relative_path] = {
|
||||
"hash": file_hash_value,
|
||||
"ingested_at": timestamp,
|
||||
"source_page": source_page,
|
||||
}
|
||||
|
||||
tmp_file = cache_file + ".tmp"
|
||||
with open(tmp_file, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, ensure_ascii=False, indent=2)
|
||||
fh.write("\n")
|
||||
os.replace(tmp_file, cache_file)
|
||||
PY
|
||||
|
||||
printf 'UPDATED\n'
|
||||
}
|
||||
|
||||
cache_invalidate() {
|
||||
local file_path="$1"
|
||||
local wiki_root cache_file relative_path_value
|
||||
|
||||
# 不调用 require_file：文件可能已被删除（级联删除场景）
|
||||
# 直接通过路径查找缓存条目
|
||||
wiki_root="$(find_wiki_root "$file_path")" || {
|
||||
echo "未找到知识库根目录：$file_path" >&2
|
||||
exit 1
|
||||
}
|
||||
cache_file="$(cache_file_path "$wiki_root")"
|
||||
|
||||
if [ ! -f "$cache_file" ]; then
|
||||
printf 'INVALIDATED\n'
|
||||
return 0
|
||||
fi
|
||||
|
||||
require_python_cmd
|
||||
|
||||
relative_path_value="$(relative_path "$wiki_root" "$file_path")"
|
||||
|
||||
"$PYTHON_CMD" - "$cache_file" "$relative_path_value" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
cache_file, relative_path = sys.argv[1:3]
|
||||
|
||||
with open(cache_file, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
|
||||
data.setdefault("entries", {}).pop(relative_path, None)
|
||||
|
||||
tmp_file = cache_file + ".tmp"
|
||||
with open(tmp_file, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, ensure_ascii=False, indent=2)
|
||||
fh.write("\n")
|
||||
os.replace(tmp_file, cache_file)
|
||||
PY
|
||||
|
||||
printf 'INVALIDATED\n'
|
||||
}
|
||||
|
||||
command_name="${1:-}"
|
||||
|
||||
case "$command_name" in
|
||||
check)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
cache_check "$2"
|
||||
;;
|
||||
update)
|
||||
[ "$#" -eq 3 ] || { usage; exit 1; }
|
||||
cache_update "$2" "$3"
|
||||
;;
|
||||
invalidate)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
cache_invalidate "$2"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Executable
+100
@@ -0,0 +1,100 @@
|
||||
#!/bin/bash
|
||||
# llm-wiki source 页面写入脚本
|
||||
# 原子写入 source 页面 + 自动更新缓存，绑定为一项操作
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/create-source-page.sh <raw_file> <output_path> <content_file>
|
||||
|
||||
参数：
|
||||
raw_file : 原始素材文件路径（绝对或相对路径）
|
||||
output_path : 目标页面路径（相对于知识库根目录，如 wiki/sources/2026-04-16-rlhf.md）
|
||||
content_file : 包含待写入内容的临时文件路径
|
||||
EOF
|
||||
}
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
# 参数校验
|
||||
if [ "$#" -ne 3 ]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
raw_file="$1"
|
||||
output_path="$2"
|
||||
content_file="$3"
|
||||
|
||||
# raw_file 和 content_file 必须存在
|
||||
if [ ! -f "$raw_file" ]; then
|
||||
echo "ERROR: 原始素材文件不存在：$raw_file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$content_file" ]; then
|
||||
echo "ERROR: 内容文件不存在：$content_file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 通过 cache.sh 的 find_wiki_root 逻辑找到知识库根目录
|
||||
# 复用 cache.sh 里的函数
|
||||
source_cache_helpers() {
|
||||
# 内联 find_wiki_root（与 cache.sh 保持一致）
|
||||
find_wiki_root() {
|
||||
local file_path="$1"
|
||||
local dir parent
|
||||
|
||||
dir="$(cd "$(dirname "$file_path")" && pwd)"
|
||||
|
||||
while true; do
|
||||
if [ -f "$dir/.wiki-cache.json" ] || [ -f "$dir/.wiki-schema.md" ]; then
|
||||
printf '%s\n' "$dir"
|
||||
return 0
|
||||
fi
|
||||
|
||||
parent="$(dirname "$dir")"
|
||||
[ "$parent" = "$dir" ] && return 1
|
||||
dir="$parent"
|
||||
done
|
||||
}
|
||||
}
|
||||
|
||||
source_cache_helpers
|
||||
|
||||
wiki_root="$(find_wiki_root "$raw_file")" || {
|
||||
echo "ERROR: 未找到知识库根目录：$raw_file" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# 拼接完整目标路径
|
||||
full_output="$wiki_root/$output_path"
|
||||
|
||||
# 确保目标目录存在
|
||||
mkdir -p "$(dirname "$full_output")"
|
||||
|
||||
# 第一步：原子写入（临时文件 + rename，防止写一半崩溃）
|
||||
tmp_output="${full_output}.tmp.$$"
|
||||
if ! cp "$content_file" "$tmp_output"; then
|
||||
rm -f "$tmp_output" 2>/dev/null || true
|
||||
echo "ERROR: 写入临时文件失败" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! mv "$tmp_output" "$full_output"; then
|
||||
rm -f "$tmp_output" 2>/dev/null || true
|
||||
echo "ERROR: 原子重命名失败" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 第二步：更新缓存
|
||||
if ! bash "$SCRIPT_DIR/cache.sh" update "$raw_file" "$output_path"; then
|
||||
# 缓存更新失败 → 回滚：删除已写入的文件
|
||||
rm -f "$full_output"
|
||||
echo "ERROR: 缓存更新失败，已回滚写入" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "SUCCESS"
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/bin/bash
|
||||
# llm-wiki 删除辅助脚本
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/delete-helper.sh scan-refs <wiki_root> <素材文件名>
|
||||
EOF
|
||||
}
|
||||
|
||||
scan_refs() {
|
||||
local wiki_root="$1"
|
||||
local needle="$2"
|
||||
local wiki_dir="$wiki_root/wiki"
|
||||
|
||||
[ -n "$needle" ] || {
|
||||
echo "素材文件名不能为空" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -d "$wiki_dir" ] || {
|
||||
echo "知识库目录不存在：$wiki_dir" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_python_cmd
|
||||
|
||||
{
|
||||
grep -rlF --include='*.md' -- "$needle" "$wiki_dir" 2>/dev/null || true
|
||||
} | "$PYTHON_CMD" -c '
|
||||
import os
|
||||
import sys
|
||||
|
||||
wiki_root = os.path.realpath(sys.argv[1])
|
||||
seen = []
|
||||
|
||||
for line in sys.stdin:
|
||||
path = line.strip()
|
||||
if not path:
|
||||
continue
|
||||
real_path = os.path.realpath(path)
|
||||
if real_path in seen:
|
||||
continue
|
||||
seen.append(real_path)
|
||||
|
||||
for path in sorted(seen):
|
||||
print(os.path.relpath(path, wiki_root))
|
||||
' "$wiki_root"
|
||||
}
|
||||
|
||||
command_name="${1:-}"
|
||||
|
||||
case "$command_name" in
|
||||
scan-refs)
|
||||
[ "$#" -eq 3 ] || { usage; exit 1; }
|
||||
scan_refs "$2" "$3"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,732 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const { extractFrontmatter, parseSourcesFrontmatter, sortedUnique } = require("./lib/source-signal-eligibility");
|
||||
|
||||
function readJson(filePath) {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
||||
}
|
||||
|
||||
function writeJson(filePath, value) {
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function roundNumber(value, digits = 3) {
|
||||
const factor = 10 ** digits;
|
||||
return Math.round(value * factor) / factor;
|
||||
}
|
||||
|
||||
function clamp01(value) {
|
||||
return Math.max(0, Math.min(1, value));
|
||||
}
|
||||
|
||||
function sortedPairKey(a, b) {
|
||||
return a < b ? `${a}\t${b}` : `${b}\t${a}`;
|
||||
}
|
||||
|
||||
function normalizeBody(text, degraded, maxLines) {
|
||||
const { body } = extractFrontmatter(text);
|
||||
const normalized = body.replace(/^\s+/, "").replace(/\s+$/, "");
|
||||
if (!degraded) return normalized;
|
||||
return normalized.split(/\r?\n/).slice(0, maxLines).join("\n").replace(/\s+$/, "");
|
||||
}
|
||||
|
||||
function loadNodeDetails(nodes, degraded, maxLines) {
|
||||
const byId = {};
|
||||
|
||||
for (const node of nodes) {
|
||||
const filePath = node._file_path || node.source_path;
|
||||
const raw = fs.readFileSync(filePath, "utf8");
|
||||
const frontmatter = extractFrontmatter(raw);
|
||||
const parsedSources = parseSourcesFrontmatter(frontmatter.frontmatter);
|
||||
const normalizedNode = {
|
||||
...node,
|
||||
content: normalizeBody(raw, degraded, maxLines),
|
||||
_signals: {
|
||||
sources: parsedSources.sources,
|
||||
sourceSignalAvailable: parsedSources.signalAvailable,
|
||||
sourceFieldPresent: parsedSources.hasField,
|
||||
sourceFieldParsed: parsedSources.parsed
|
||||
}
|
||||
};
|
||||
byId[node.id] = normalizedNode;
|
||||
}
|
||||
|
||||
return byId;
|
||||
}
|
||||
|
||||
function buildInlinks(edges) {
|
||||
const inlinks = new Map();
|
||||
|
||||
for (const edge of edges) {
|
||||
if (!inlinks.has(edge.to)) inlinks.set(edge.to, new Set());
|
||||
inlinks.get(edge.to).add(edge.from);
|
||||
}
|
||||
|
||||
return inlinks;
|
||||
}
|
||||
|
||||
function intersectionCount(setA, setB) {
|
||||
if (!setA || !setB) return 0;
|
||||
let small = setA;
|
||||
let large = setB;
|
||||
if (setB.size < setA.size) {
|
||||
small = setB;
|
||||
large = setA;
|
||||
}
|
||||
|
||||
let count = 0;
|
||||
for (const value of small) {
|
||||
if (large.has(value)) count += 1;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function typeAffinity(typeA, typeB) {
|
||||
const pair = [typeA || "other", typeB || "other"].sort().join(":");
|
||||
switch (pair) {
|
||||
case "entity:entity":
|
||||
case "entity:topic":
|
||||
return 1;
|
||||
case "topic:topic":
|
||||
return 0.8;
|
||||
case "entity:source":
|
||||
return 0.6;
|
||||
case "source:source":
|
||||
return 0.3;
|
||||
default:
|
||||
return 0.5;
|
||||
}
|
||||
}
|
||||
|
||||
function computePairMetrics(nodesById, edges) {
|
||||
const inlinks = buildInlinks(edges);
|
||||
const pairMetrics = new Map();
|
||||
|
||||
for (const edge of edges) {
|
||||
const pairKey = sortedPairKey(edge.from, edge.to);
|
||||
if (pairMetrics.has(pairKey)) continue;
|
||||
|
||||
const fromNode = nodesById[edge.from];
|
||||
const toNode = nodesById[edge.to];
|
||||
if (!fromNode || !toNode) continue;
|
||||
|
||||
const fromInlinks = inlinks.get(edge.from) || new Set();
|
||||
const toInlinks = inlinks.get(edge.to) || new Set();
|
||||
const sharedInlinks = intersectionCount(fromInlinks, toInlinks);
|
||||
const coCitation = sharedInlinks / Math.max(fromInlinks.size, toInlinks.size, 1);
|
||||
const affinity = typeAffinity(fromNode.type, toNode.type);
|
||||
|
||||
const signals = [coCitation, affinity];
|
||||
let sourceOverlap = null;
|
||||
const sourceSignalAvailable = Boolean(
|
||||
fromNode._signals.sourceSignalAvailable && toNode._signals.sourceSignalAvailable
|
||||
);
|
||||
|
||||
if (sourceSignalAvailable) {
|
||||
const fromSources = new Set(fromNode._signals.sources);
|
||||
const toSources = new Set(toNode._signals.sources);
|
||||
const overlap = intersectionCount(fromSources, toSources);
|
||||
const minSize = Math.min(fromSources.size, toSources.size);
|
||||
sourceOverlap = minSize > 0 ? overlap / minSize : 0;
|
||||
signals.push(sourceOverlap);
|
||||
}
|
||||
|
||||
const weight = clamp01(signals.reduce((sum, value) => sum + value, 0) / signals.length);
|
||||
|
||||
pairMetrics.set(pairKey, {
|
||||
weight: roundNumber(weight),
|
||||
signals: {
|
||||
co_citation: roundNumber(coCitation),
|
||||
source_overlap: sourceOverlap == null ? null : roundNumber(sourceOverlap),
|
||||
type_affinity: roundNumber(affinity)
|
||||
},
|
||||
source_signal_available: sourceSignalAvailable
|
||||
});
|
||||
}
|
||||
|
||||
return pairMetrics;
|
||||
}
|
||||
|
||||
function buildUndirectedGraph(nodeIds, pairMetrics) {
|
||||
const adjacency = new Map();
|
||||
const degrees = new Map();
|
||||
|
||||
for (const nodeId of nodeIds) {
|
||||
adjacency.set(nodeId, new Map());
|
||||
degrees.set(nodeId, 0);
|
||||
}
|
||||
|
||||
for (const [pairKey, metrics] of pairMetrics.entries()) {
|
||||
const [left, right] = pairKey.split("\t");
|
||||
if (!adjacency.has(left) || !adjacency.has(right)) continue;
|
||||
const weight = metrics.weight;
|
||||
adjacency.get(left).set(right, weight);
|
||||
adjacency.get(right).set(left, weight);
|
||||
degrees.set(left, degrees.get(left) + weight);
|
||||
degrees.set(right, degrees.get(right) + weight);
|
||||
}
|
||||
|
||||
return { adjacency, degrees };
|
||||
}
|
||||
|
||||
function runLocalMove(graph) {
|
||||
const nodes = Array.from(graph.nodes.keys()).sort();
|
||||
const communities = new Map();
|
||||
const totals = new Map();
|
||||
let moved = false;
|
||||
|
||||
for (const nodeId of nodes) {
|
||||
communities.set(nodeId, nodeId);
|
||||
totals.set(nodeId, graph.degrees.get(nodeId) || 0);
|
||||
}
|
||||
|
||||
if (graph.m2 === 0) {
|
||||
return { communities, changed: false };
|
||||
}
|
||||
|
||||
let changedInPass = true;
|
||||
let passCount = 0;
|
||||
while (changedInPass && passCount < 50) {
|
||||
passCount++;
|
||||
changedInPass = false;
|
||||
|
||||
for (const nodeId of nodes) {
|
||||
const degree = graph.degrees.get(nodeId) || 0;
|
||||
const currentCommunity = communities.get(nodeId);
|
||||
const neighborCommunities = new Map();
|
||||
|
||||
for (const [neighborId, weight] of graph.nodes.get(nodeId).entries()) {
|
||||
const communityId = communities.get(neighborId);
|
||||
neighborCommunities.set(communityId, (neighborCommunities.get(communityId) || 0) + weight);
|
||||
}
|
||||
|
||||
totals.set(currentCommunity, (totals.get(currentCommunity) || 0) - degree);
|
||||
if ((neighborCommunities.get(currentCommunity) || 0) === 0) {
|
||||
neighborCommunities.set(currentCommunity, 0);
|
||||
}
|
||||
|
||||
let bestCommunity = currentCommunity;
|
||||
let bestGain = 0;
|
||||
|
||||
const candidates = Array.from(neighborCommunities.keys()).sort();
|
||||
for (const communityId of candidates) {
|
||||
const inWeight = neighborCommunities.get(communityId) || 0;
|
||||
const gain = inWeight - ((totals.get(communityId) || 0) * degree) / graph.m2;
|
||||
if (gain > bestGain + 1e-9) {
|
||||
bestGain = gain;
|
||||
bestCommunity = communityId;
|
||||
}
|
||||
}
|
||||
|
||||
communities.set(nodeId, bestCommunity);
|
||||
totals.set(bestCommunity, (totals.get(bestCommunity) || 0) + degree);
|
||||
|
||||
if (bestCommunity !== currentCommunity) {
|
||||
changedInPass = true;
|
||||
moved = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { communities, changed: moved };
|
||||
}
|
||||
|
||||
function aggregateGraph(graph, communities) {
|
||||
const communityIds = sortedUnique(Array.from(communities.values()));
|
||||
const aggregatedNodes = new Map();
|
||||
const aggregatedDegrees = new Map();
|
||||
const members = new Map();
|
||||
|
||||
for (const communityId of communityIds) {
|
||||
aggregatedNodes.set(communityId, new Map());
|
||||
aggregatedDegrees.set(communityId, 0);
|
||||
members.set(communityId, []);
|
||||
}
|
||||
|
||||
for (const [nodeId, communityId] of communities.entries()) {
|
||||
members.get(communityId).push(...(graph.members.get(nodeId) || [nodeId]));
|
||||
}
|
||||
|
||||
for (const [nodeId, neighbors] of graph.nodes.entries()) {
|
||||
const sourceCommunity = communities.get(nodeId);
|
||||
for (const [neighborId, weight] of neighbors.entries()) {
|
||||
if (nodeId > neighborId) continue;
|
||||
const targetCommunity = communities.get(neighborId);
|
||||
const current = aggregatedNodes.get(sourceCommunity).get(targetCommunity) || 0;
|
||||
aggregatedNodes.get(sourceCommunity).set(targetCommunity, current + weight);
|
||||
if (sourceCommunity !== targetCommunity) {
|
||||
const mirrored = aggregatedNodes.get(targetCommunity).get(sourceCommunity) || 0;
|
||||
aggregatedNodes.get(targetCommunity).set(sourceCommunity, mirrored + weight);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const [communityId, neighbors] of aggregatedNodes.entries()) {
|
||||
let degree = 0;
|
||||
for (const [neighborId, weight] of neighbors.entries()) {
|
||||
degree += neighborId === communityId ? weight * 2 : weight;
|
||||
}
|
||||
aggregatedDegrees.set(communityId, degree);
|
||||
}
|
||||
|
||||
return {
|
||||
nodes: aggregatedNodes,
|
||||
degrees: aggregatedDegrees,
|
||||
members,
|
||||
m2: Array.from(aggregatedDegrees.values()).reduce((sum, value) => sum + value, 0)
|
||||
};
|
||||
}
|
||||
|
||||
function runLouvain(nodeIds, pairMetrics) {
|
||||
const baseGraph = buildUndirectedGraph(nodeIds, pairMetrics);
|
||||
let graph = {
|
||||
nodes: baseGraph.adjacency,
|
||||
degrees: baseGraph.degrees,
|
||||
members: new Map(nodeIds.map((nodeId) => [nodeId, [nodeId]])),
|
||||
m2: Array.from(baseGraph.degrees.values()).reduce((sum, value) => sum + value, 0)
|
||||
};
|
||||
|
||||
let bestMembers = graph.members;
|
||||
|
||||
while (true) {
|
||||
const phase = runLocalMove(graph);
|
||||
const nextGraph = aggregateGraph(graph, phase.communities);
|
||||
bestMembers = nextGraph.members;
|
||||
|
||||
if (!phase.changed || nextGraph.nodes.size === graph.nodes.size) {
|
||||
break;
|
||||
}
|
||||
|
||||
graph = nextGraph;
|
||||
}
|
||||
|
||||
const finalCommunities = new Map();
|
||||
for (const [communityId, members] of bestMembers.entries()) {
|
||||
for (const nodeId of members) {
|
||||
finalCommunities.set(nodeId, communityId);
|
||||
}
|
||||
}
|
||||
|
||||
return finalCommunities;
|
||||
}
|
||||
|
||||
function buildDirectedDegree(edges) {
|
||||
const degree = new Map();
|
||||
for (const edge of edges) {
|
||||
degree.set(edge.from, (degree.get(edge.from) || 0) + 1);
|
||||
degree.set(edge.to, (degree.get(edge.to) || 0) + 1);
|
||||
}
|
||||
return degree;
|
||||
}
|
||||
|
||||
function chooseCommunityLabels(nodeIds, communityAssignments, nodesById, edges) {
|
||||
const groups = new Map();
|
||||
const degree = buildDirectedDegree(edges);
|
||||
|
||||
for (const nodeId of nodeIds) {
|
||||
const communityId = communityAssignments.get(nodeId) || nodeId;
|
||||
if (!groups.has(communityId)) groups.set(communityId, []);
|
||||
groups.get(communityId).push(nodeId);
|
||||
}
|
||||
|
||||
const labeledAssignments = new Map();
|
||||
|
||||
for (const members of groups.values()) {
|
||||
members.sort();
|
||||
if (members.length === 1) {
|
||||
labeledAssignments.set(members[0], null);
|
||||
continue;
|
||||
}
|
||||
|
||||
const memberNodes = members.map((memberId) => nodesById[memberId]);
|
||||
const topics = memberNodes.filter((node) => node && node.type === "topic");
|
||||
const candidates = topics.length ? topics : memberNodes;
|
||||
candidates.sort((left, right) => {
|
||||
const degreeDiff = (degree.get(right.id) || 0) - (degree.get(left.id) || 0);
|
||||
if (degreeDiff !== 0) return degreeDiff;
|
||||
return left.id.localeCompare(right.id);
|
||||
});
|
||||
|
||||
const label = candidates[0] ? candidates[0].id : members[0];
|
||||
for (const memberId of members) {
|
||||
labeledAssignments.set(memberId, label);
|
||||
}
|
||||
}
|
||||
|
||||
return labeledAssignments;
|
||||
}
|
||||
|
||||
function buildInsights(nodesById, edges, pairMetrics, communityAssignments, options) {
|
||||
const directedDegree = buildDirectedDegree(edges);
|
||||
const undirectedPairs = new Map();
|
||||
const adjacency = new Map();
|
||||
|
||||
for (const nodeId of Object.keys(nodesById)) {
|
||||
adjacency.set(nodeId, new Set());
|
||||
}
|
||||
|
||||
for (const edge of edges) {
|
||||
const pairKey = sortedPairKey(edge.from, edge.to);
|
||||
if (!undirectedPairs.has(pairKey)) {
|
||||
undirectedPairs.set(pairKey, {
|
||||
from: pairKey.split("\t")[0],
|
||||
to: pairKey.split("\t")[1],
|
||||
weight: pairMetrics.get(pairKey)?.weight || 0
|
||||
});
|
||||
}
|
||||
adjacency.get(edge.from)?.add(edge.to);
|
||||
adjacency.get(edge.to)?.add(edge.from);
|
||||
}
|
||||
|
||||
const isolatedNodes = Object.values(nodesById)
|
||||
.filter((node) => (directedDegree.get(node.id) || 0) <= 1)
|
||||
.sort((left, right) => left.id.localeCompare(right.id))
|
||||
.map((node) => ({
|
||||
id: node.id,
|
||||
label: node.label,
|
||||
degree: directedDegree.get(node.id) || 0,
|
||||
community: communityAssignments.get(node.id) || null
|
||||
}));
|
||||
|
||||
const bridgeNodes = [];
|
||||
for (const node of Object.values(nodesById).sort((left, right) => left.id.localeCompare(right.id))) {
|
||||
const ownCommunity = communityAssignments.get(node.id) || null;
|
||||
const connectedCommunities = sortedUnique(
|
||||
Array.from(adjacency.get(node.id) || [])
|
||||
.map((neighborId) => communityAssignments.get(neighborId) || null)
|
||||
.filter((c) => c && c !== ownCommunity)
|
||||
);
|
||||
|
||||
if (connectedCommunities.length >= 2) {
|
||||
bridgeNodes.push({
|
||||
id: node.id,
|
||||
label: node.label,
|
||||
community: ownCommunity,
|
||||
connected_communities: connectedCommunities,
|
||||
community_count: connectedCommunities.length
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const communityMembers = new Map();
|
||||
for (const node of Object.values(nodesById)) {
|
||||
const communityId = communityAssignments.get(node.id) || null;
|
||||
if (!communityId) continue;
|
||||
if (!communityMembers.has(communityId)) communityMembers.set(communityId, []);
|
||||
communityMembers.get(communityId).push(node.id);
|
||||
}
|
||||
|
||||
const sparseCommunities = [];
|
||||
for (const [communityId, members] of Array.from(communityMembers.entries()).sort((left, right) => left[0].localeCompare(right[0]))) {
|
||||
if (members.length < 3) continue;
|
||||
|
||||
const memberSet = new Set(members);
|
||||
let internalEdges = 0;
|
||||
for (const pair of undirectedPairs.values()) {
|
||||
if (memberSet.has(pair.from) && memberSet.has(pair.to)) internalEdges += 1;
|
||||
}
|
||||
|
||||
const possibleEdges = (members.length * (members.length - 1)) / 2;
|
||||
const density = possibleEdges === 0 ? 0 : internalEdges / possibleEdges;
|
||||
if (density < 0.15) {
|
||||
sparseCommunities.push({
|
||||
id: communityId,
|
||||
label: nodesById[communityId]?.label || communityId,
|
||||
node_count: members.length,
|
||||
density: roundNumber(density),
|
||||
members: members.sort(),
|
||||
internal_edges: internalEdges
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const surprisingConnections = Array.from(undirectedPairs.values())
|
||||
.filter((pair) => {
|
||||
const fromCommunity = communityAssignments.get(pair.from) || null;
|
||||
const toCommunity = communityAssignments.get(pair.to) || null;
|
||||
return fromCommunity && toCommunity && fromCommunity !== toCommunity && pair.weight >= 0.75;
|
||||
})
|
||||
.sort((left, right) => {
|
||||
if (right.weight !== left.weight) return right.weight - left.weight;
|
||||
if (left.from !== right.from) return left.from.localeCompare(right.from);
|
||||
return left.to.localeCompare(right.to);
|
||||
})
|
||||
.slice(0, 8)
|
||||
.map((pair) => ({
|
||||
from: pair.from,
|
||||
to: pair.to,
|
||||
weight: pair.weight,
|
||||
from_community: communityAssignments.get(pair.from) || null,
|
||||
to_community: communityAssignments.get(pair.to) || null
|
||||
}));
|
||||
|
||||
const degraded = options.nodeCount > options.maxInsightNodes || options.edgeCount > options.maxInsightEdges;
|
||||
if (degraded) {
|
||||
return {
|
||||
surprising_connections: [],
|
||||
isolated_nodes: isolatedNodes,
|
||||
bridge_nodes: [],
|
||||
sparse_communities: [],
|
||||
meta: {
|
||||
degraded: true,
|
||||
node_count: options.nodeCount,
|
||||
edge_count: options.edgeCount,
|
||||
max_insight_nodes: options.maxInsightNodes,
|
||||
max_insight_edges: options.maxInsightEdges
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
surprising_connections: surprisingConnections,
|
||||
isolated_nodes: isolatedNodes,
|
||||
bridge_nodes: bridgeNodes,
|
||||
sparse_communities: sparseCommunities,
|
||||
meta: {
|
||||
degraded: false,
|
||||
node_count: options.nodeCount,
|
||||
edge_count: options.edgeCount,
|
||||
max_insight_nodes: options.maxInsightNodes,
|
||||
max_insight_edges: options.maxInsightEdges
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildLearning(analyzedNodes, analyzedEdges) {
|
||||
const degreeMap = new Map();
|
||||
for (const edge of analyzedEdges) {
|
||||
degreeMap.set(edge.from, (degreeMap.get(edge.from) || 0) + 1);
|
||||
degreeMap.set(edge.to, (degreeMap.get(edge.to) || 0) + 1);
|
||||
}
|
||||
|
||||
const communityGroups = new Map();
|
||||
for (const node of analyzedNodes) {
|
||||
if (node.community == null) continue;
|
||||
if (!communityGroups.has(node.community)) communityGroups.set(node.community, []);
|
||||
communityGroups.get(node.community).push(node);
|
||||
}
|
||||
|
||||
const communities = [];
|
||||
for (const [cid, members] of communityGroups.entries()) {
|
||||
const memberIds = new Set(members.map(n => n.id));
|
||||
let totalWeight = 0;
|
||||
for (const edge of analyzedEdges) {
|
||||
if (memberIds.has(edge.from) && memberIds.has(edge.to)) totalWeight += edge.weight;
|
||||
}
|
||||
const isWeak = members.length < 3;
|
||||
const startNode = members.slice().sort((a, b) => {
|
||||
const degDiff = (degreeMap.get(b.id) || 0) - (degreeMap.get(a.id) || 0);
|
||||
if (degDiff !== 0) return degDiff;
|
||||
return a.id.localeCompare(b.id);
|
||||
})[0];
|
||||
|
||||
communities.push({
|
||||
id: cid,
|
||||
label: (members.find(n => n.id === cid) || members[0]).label,
|
||||
node_count: members.length,
|
||||
source_count: members.filter(n => n.type === "source").length,
|
||||
internal_edge_weight: roundNumber(totalWeight),
|
||||
is_primary: false,
|
||||
is_weak: isWeak,
|
||||
recommended_start_node_id: startNode.id
|
||||
});
|
||||
}
|
||||
|
||||
communities.sort((a, b) => {
|
||||
if (b.node_count !== a.node_count) return b.node_count - a.node_count;
|
||||
if (b.internal_edge_weight !== a.internal_edge_weight) return b.internal_edge_weight - a.internal_edge_weight;
|
||||
return a.id.localeCompare(b.id);
|
||||
});
|
||||
|
||||
if (communities.length > 0) communities[0].is_primary = true;
|
||||
|
||||
const primary = communities.length > 0 ? communities[0] : null;
|
||||
const startNodeId = primary ? primary.recommended_start_node_id : null;
|
||||
|
||||
let pathNodeIds = [];
|
||||
let pathDegraded = false;
|
||||
if (primary && !primary.is_weak && startNodeId) {
|
||||
const primaryMemberIds = new Set(communityGroups.get(primary.id).map(n => n.id));
|
||||
const neighbors = analyzedEdges
|
||||
.filter(e => (e.from === startNodeId && primaryMemberIds.has(e.to)) ||
|
||||
(e.to === startNodeId && primaryMemberIds.has(e.from)))
|
||||
.map(e => e.from === startNodeId ? e.to : e.from);
|
||||
pathNodeIds = [startNodeId, ...sortedUnique(neighbors).filter(id => id !== startNodeId)];
|
||||
if (pathNodeIds.length < 2) pathDegraded = true;
|
||||
} else {
|
||||
pathDegraded = true;
|
||||
}
|
||||
|
||||
let communityNodeIds = [];
|
||||
let communityDegraded = false;
|
||||
if (primary && !primary.is_weak) {
|
||||
communityNodeIds = communityGroups.get(primary.id).map(n => n.id).sort();
|
||||
} else {
|
||||
communityDegraded = true;
|
||||
}
|
||||
|
||||
const globalNodeIds = analyzedNodes.slice().sort((a, b) => {
|
||||
const degDiff = (degreeMap.get(b.id) || 0) - (degreeMap.get(a.id) || 0);
|
||||
if (degDiff !== 0) return degDiff;
|
||||
return a.id.localeCompare(b.id);
|
||||
}).map(n => n.id);
|
||||
|
||||
const defaultMode = "global";
|
||||
|
||||
return {
|
||||
version: 1,
|
||||
entry: {
|
||||
recommended_start_node_id: startNodeId,
|
||||
recommended_start_reason: startNodeId ? "community_hub" : null,
|
||||
default_mode: defaultMode
|
||||
},
|
||||
views: {
|
||||
path: {
|
||||
enabled: !pathDegraded,
|
||||
start_node_id: pathDegraded ? null : startNodeId,
|
||||
node_ids: pathDegraded ? [] : pathNodeIds,
|
||||
degraded: pathDegraded
|
||||
},
|
||||
community: {
|
||||
enabled: !communityDegraded,
|
||||
community_id: primary && !communityDegraded ? primary.id : null,
|
||||
label: primary && !communityDegraded ? primary.label : null,
|
||||
node_ids: communityDegraded ? [] : communityNodeIds,
|
||||
is_weak: primary ? primary.is_weak : false,
|
||||
degraded: communityDegraded
|
||||
},
|
||||
global: {
|
||||
enabled: true,
|
||||
node_ids: globalNodeIds,
|
||||
degraded: false
|
||||
}
|
||||
},
|
||||
communities,
|
||||
degraded: {
|
||||
path_to_community: pathDegraded,
|
||||
community_to_global: communityDegraded
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function analyzeGraph(nodes, edges, options = {}) {
|
||||
const degraded = options.degraded === true;
|
||||
const maxLines = options.maxLines || 500;
|
||||
const maxInsightNodes = options.maxInsightNodes || 250;
|
||||
const maxInsightEdges = options.maxInsightEdges || 1000;
|
||||
|
||||
const nodesById = loadNodeDetails(nodes, degraded, maxLines);
|
||||
const pairMetrics = computePairMetrics(nodesById, edges);
|
||||
const nodeIds = nodes.map((node) => node.id);
|
||||
const communityAssignments = chooseCommunityLabels(
|
||||
nodeIds,
|
||||
runLouvain(nodeIds, pairMetrics),
|
||||
nodesById,
|
||||
edges
|
||||
);
|
||||
|
||||
const analyzedNodes = nodes.map((node) => ({
|
||||
id: node.id,
|
||||
label: node.label,
|
||||
type: node.type,
|
||||
source_path: node.source_path,
|
||||
community: communityAssignments.get(node.id) || null,
|
||||
content: nodesById[node.id].content
|
||||
}));
|
||||
|
||||
const analyzedEdges = edges.map((edge) => {
|
||||
const pairKey = sortedPairKey(edge.from, edge.to);
|
||||
const metrics = pairMetrics.get(pairKey) || {
|
||||
weight: 0,
|
||||
signals: { co_citation: 0, source_overlap: null, type_affinity: 0.5 },
|
||||
source_signal_available: false
|
||||
};
|
||||
|
||||
return {
|
||||
id: edge.id,
|
||||
from: edge.from,
|
||||
to: edge.to,
|
||||
type: edge.type,
|
||||
confidence: edge.confidence || edge.type,
|
||||
relation_type: edge.relation_type || "依赖",
|
||||
weight: metrics.weight,
|
||||
source_signal_available: metrics.source_signal_available,
|
||||
signals: metrics.signals
|
||||
};
|
||||
});
|
||||
|
||||
const insights = buildInsights(nodesById, analyzedEdges, pairMetrics, communityAssignments, {
|
||||
nodeCount: analyzedNodes.length,
|
||||
edgeCount: analyzedEdges.length,
|
||||
maxInsightNodes,
|
||||
maxInsightEdges
|
||||
});
|
||||
|
||||
const learning = buildLearning(analyzedNodes, analyzedEdges);
|
||||
|
||||
return { nodes: analyzedNodes, edges: analyzedEdges, insights, learning };
|
||||
}
|
||||
|
||||
function main(argv) {
|
||||
if (argv.length < 7) {
|
||||
console.error("Usage: node graph-analysis.js <nodes.json> <edges.json> <output.json> <degraded:0|1> <max-lines> <max-insight-nodes> <max-insight-edges>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
console.error("ERROR: graph analysis timed out (120s)");
|
||||
process.exit(2);
|
||||
}, 120_000);
|
||||
timer.unref();
|
||||
|
||||
const [, , nodesPath, edgesPath, outputPath, degradedRaw, maxLinesRaw, maxInsightNodesRaw, maxInsightEdgesRaw] = argv;
|
||||
|
||||
for (const p of [nodesPath, edgesPath]) {
|
||||
if (!fs.existsSync(p)) {
|
||||
console.error(`ERROR: File not found: ${p}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
const analyzed = analyzeGraph(readJson(nodesPath), readJson(edgesPath), {
|
||||
degraded: degradedRaw === "1",
|
||||
maxLines: Number(maxLinesRaw) || 500,
|
||||
maxInsightNodes: Number(maxInsightNodesRaw) || 250,
|
||||
maxInsightEdges: Number(maxInsightEdgesRaw) || 1000
|
||||
});
|
||||
|
||||
writeJson(outputPath, analyzed);
|
||||
clearTimeout(timer);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
try {
|
||||
main(process.argv);
|
||||
} catch (error) {
|
||||
const code = error && error.code;
|
||||
if (code === "ENOENT") {
|
||||
console.error(`ERROR: File not found: ${error.path || "(unknown)"}`);
|
||||
} else if (error instanceof SyntaxError) {
|
||||
console.error(`ERROR: Invalid JSON in input: ${error.message}`);
|
||||
} else {
|
||||
console.error(`ERROR: ${error && error.message ? error.message : String(error)}`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
analyzeGraph,
|
||||
buildInsights,
|
||||
buildLearning,
|
||||
chooseCommunityLabels,
|
||||
computePairMetrics,
|
||||
extractFrontmatter,
|
||||
normalizeBody,
|
||||
parseSourcesFrontmatter,
|
||||
runLouvain,
|
||||
typeAffinity
|
||||
};
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/bin/bash
|
||||
# SessionStart hook: 会话开始时注入 wiki 上下文（只触发一次）
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
|
||||
WIKI_PATH=""
|
||||
|
||||
if [ -f "$HOME/.llm-wiki-path" ]; then
|
||||
WIKI_PATH="$(cat "$HOME/.llm-wiki-path")"
|
||||
fi
|
||||
|
||||
if [ -z "$WIKI_PATH" ] && [ -f .wiki-schema.md ]; then
|
||||
WIKI_PATH="$(pwd)"
|
||||
fi
|
||||
|
||||
if [ -z "$WIKI_PATH" ] || [ ! -f "$WIKI_PATH/.wiki-schema.md" ]; then
|
||||
printf '{}\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
require_python_cmd
|
||||
|
||||
"$PYTHON_CMD" - "$WIKI_PATH" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
# 防御性：即使上游 shared-config.sh 未设置 PYTHONIOENCODING，
|
||||
# 此处也强制 stdout 为 UTF-8，避免 Agent 接到 gbk 字节
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
wiki_path = os.path.realpath(sys.argv[1])
|
||||
message = f"[llm-wiki] 检测到知识库: {wiki_path}/index.md，回答问题时优先查阅 wiki 内容获取上下文"
|
||||
|
||||
print(json.dumps({
|
||||
"hookSpecificOutput": {
|
||||
"hookEventName": "SessionStart",
|
||||
"additionalContext": message,
|
||||
}
|
||||
}, ensure_ascii=False))
|
||||
PY
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/bin/bash
|
||||
# llm-wiki 初始化脚本
|
||||
# 自动创建知识库的目录结构
|
||||
# 用法：bash init-wiki.sh <知识库路径> <主题>
|
||||
|
||||
set -e
|
||||
|
||||
WIKI_ROOT="${1:-$HOME/Documents/我的知识库}"
|
||||
TOPIC="${2:-我的知识库}"
|
||||
LANGUAGE="${3:-中文}"
|
||||
DATE=$(date +%Y-%m-%d)
|
||||
SKILL_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
# 安全的模板变量替换函数（用 perl 替代 sed，避免中文/空格/特殊字符问题）
|
||||
replace_vars() {
|
||||
local input_file="$1"
|
||||
local output_file="$2"
|
||||
TOPIC_VALUE="$TOPIC" \
|
||||
DATE_VALUE="$DATE" \
|
||||
WIKI_ROOT_VALUE="$WIKI_ROOT" \
|
||||
LANGUAGE_VALUE="$LANGUAGE" \
|
||||
perl -pe '
|
||||
s/\{\{TOPIC\}\}/$ENV{TOPIC_VALUE}/g;
|
||||
s/\{\{DATE\}\}/$ENV{DATE_VALUE}/g;
|
||||
s/\{\{WIKI_ROOT\}\}/$ENV{WIKI_ROOT_VALUE}/g;
|
||||
s/\{\{LANGUAGE\}\}/$ENV{LANGUAGE_VALUE}/g;
|
||||
' "$input_file" > "$output_file"
|
||||
}
|
||||
|
||||
echo "正在创建知识库..."
|
||||
echo " 路径：$WIKI_ROOT"
|
||||
echo " 主题：$TOPIC"
|
||||
echo " 语言：$LANGUAGE"
|
||||
echo ""
|
||||
|
||||
# 创建目录结构（包含小红书和知乎）
|
||||
mkdir -p "$WIKI_ROOT"/raw/{articles,tweets,wechat,xiaohongshu,zhihu,pdfs,notes,assets}
|
||||
mkdir -p "$WIKI_ROOT"/wiki/{entities,topics,sources,comparisons,synthesis,synthesis/sessions,queries}
|
||||
|
||||
cat > "$WIKI_ROOT/.gitignore" <<'EOF'
|
||||
.wiki-tmp/
|
||||
EOF
|
||||
|
||||
echo "[完成] 目录结构已创建"
|
||||
|
||||
# 从模板生成文件
|
||||
replace_vars "$SKILL_DIR/templates/schema-template.md" "$WIKI_ROOT/.wiki-schema.md"
|
||||
echo "[完成] Schema 文件已生成"
|
||||
|
||||
replace_vars "$SKILL_DIR/templates/index-template.md" "$WIKI_ROOT/index.md"
|
||||
echo "[完成] 索引文件已生成"
|
||||
|
||||
replace_vars "$SKILL_DIR/templates/log-template.md" "$WIKI_ROOT/log.md"
|
||||
echo "[完成] 日志文件已生成"
|
||||
|
||||
replace_vars "$SKILL_DIR/templates/overview-template.md" "$WIKI_ROOT/wiki/overview.md"
|
||||
echo "[完成] 总览文件已生成"
|
||||
|
||||
if [ "$LANGUAGE" = "English" ]; then
|
||||
replace_vars "$SKILL_DIR/templates/purpose-en-template.md" "$WIKI_ROOT/purpose.md"
|
||||
else
|
||||
replace_vars "$SKILL_DIR/templates/purpose-template.md" "$WIKI_ROOT/purpose.md"
|
||||
fi
|
||||
echo "[完成] 研究方向文件已生成"
|
||||
|
||||
cat > "$WIKI_ROOT/.wiki-cache.json" <<'EOF'
|
||||
{
|
||||
"version": 1,
|
||||
"entries": {}
|
||||
}
|
||||
EOF
|
||||
echo "[完成] 缓存文件已生成"
|
||||
|
||||
echo ""
|
||||
echo "知识库创建完成！"
|
||||
echo ""
|
||||
echo "目录结构："
|
||||
echo " $WIKI_ROOT/"
|
||||
echo " ├── raw/ （原始素材）"
|
||||
echo " │ ├── articles/ 网页文章"
|
||||
echo " │ ├── tweets/ X/Twitter"
|
||||
echo " │ ├── wechat/ 微信公众号"
|
||||
echo " │ ├── xiaohongshu/ 小红书"
|
||||
echo " │ ├── zhihu/ 知乎"
|
||||
echo " │ ├── pdfs/ PDF"
|
||||
echo " │ ├── notes/ 笔记"
|
||||
echo " │ └── assets/ 图片等附件"
|
||||
echo " ├── wiki/ （知识库）"
|
||||
echo " ├── index.md （索引）"
|
||||
echo " ├── log.md （日志）"
|
||||
echo " ├── purpose.md （研究方向）"
|
||||
echo " ├── .wiki-cache.json （缓存）"
|
||||
echo " └── .wiki-schema.md （配置）"
|
||||
echo ""
|
||||
echo "下一步：给 agent 一个链接或文件，开始构建知识库！"
|
||||
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const SCAN_KINDS = [
|
||||
{ subdir: "entities", pageType: "entity", applicable: true },
|
||||
{ subdir: "topics", pageType: "topic", applicable: true },
|
||||
{ subdir: "sources", pageType: "source", applicable: true },
|
||||
{ subdir: "comparisons", pageType: "comparison", applicable: true },
|
||||
{ subdir: "queries", pageType: "query", applicable: false },
|
||||
{ subdir: "synthesis", pageType: "synthesis", applicable: false }
|
||||
];
|
||||
|
||||
function sortedUnique(values) {
|
||||
return Array.from(new Set(values)).sort();
|
||||
}
|
||||
|
||||
function extractFrontmatter(text) {
|
||||
if (!text.startsWith("---\n") && !text.startsWith("---\r\n")) {
|
||||
return { hasFrontmatter: false, frontmatter: "", body: text };
|
||||
}
|
||||
|
||||
const match = text.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)([\s\S]*)$/);
|
||||
if (!match) {
|
||||
return { hasFrontmatter: false, frontmatter: "", body: text };
|
||||
}
|
||||
|
||||
return {
|
||||
hasFrontmatter: true,
|
||||
frontmatter: match[1],
|
||||
body: match[2]
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeSourceToken(token) {
|
||||
const trimmed = String(token || "").trim();
|
||||
if (!trimmed) return null;
|
||||
|
||||
let value = trimmed;
|
||||
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
|
||||
value = value.slice(1, -1).trim();
|
||||
}
|
||||
|
||||
return value || null;
|
||||
}
|
||||
|
||||
function parseInlineSources(raw) {
|
||||
const trimmed = raw.trim();
|
||||
if (trimmed === "[]") return { ok: true, values: [] };
|
||||
if (!(trimmed.startsWith("[") && trimmed.endsWith("]"))) {
|
||||
return { ok: false, values: [] };
|
||||
}
|
||||
|
||||
const inner = trimmed.slice(1, -1).trim();
|
||||
if (!inner) return { ok: true, values: [] };
|
||||
|
||||
const values = inner
|
||||
.split(",")
|
||||
.map(normalizeSourceToken)
|
||||
.filter(Boolean);
|
||||
|
||||
return { ok: true, values };
|
||||
}
|
||||
|
||||
function parseSourcesFrontmatter(frontmatter) {
|
||||
if (!frontmatter) {
|
||||
return { hasField: false, parsed: false, sources: [], signalAvailable: false };
|
||||
}
|
||||
|
||||
const lines = frontmatter.split(/\r?\n/);
|
||||
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const match = lines[index].match(/^sources:\s*(.*)$/);
|
||||
if (!match) continue;
|
||||
|
||||
const rest = match[1].trim();
|
||||
if (rest) {
|
||||
if (!rest.startsWith("[")) {
|
||||
const single = normalizeSourceToken(rest);
|
||||
return {
|
||||
hasField: true,
|
||||
parsed: Boolean(single),
|
||||
sources: single ? [single] : [],
|
||||
signalAvailable: Boolean(single)
|
||||
};
|
||||
}
|
||||
|
||||
const parsedInline = parseInlineSources(rest);
|
||||
return {
|
||||
hasField: true,
|
||||
parsed: parsedInline.ok,
|
||||
sources: parsedInline.ok ? sortedUnique(parsedInline.values) : [],
|
||||
signalAvailable: parsedInline.ok && parsedInline.values.length > 0
|
||||
};
|
||||
}
|
||||
|
||||
const collected = [];
|
||||
let parsed = true;
|
||||
let consumed = 0;
|
||||
|
||||
for (let cursor = index + 1; cursor < lines.length; cursor += 1) {
|
||||
const line = lines[cursor];
|
||||
if (!line.trim()) {
|
||||
consumed += 1;
|
||||
continue;
|
||||
}
|
||||
if (/^[^\s-]/.test(line)) break;
|
||||
const itemMatch = line.match(/^\s*-\s*(.+)$/);
|
||||
if (!itemMatch) {
|
||||
parsed = false;
|
||||
consumed += 1;
|
||||
continue;
|
||||
}
|
||||
const token = normalizeSourceToken(itemMatch[1]);
|
||||
if (token) collected.push(token);
|
||||
consumed += 1;
|
||||
}
|
||||
|
||||
index += consumed;
|
||||
return {
|
||||
hasField: true,
|
||||
parsed,
|
||||
sources: parsed ? sortedUnique(collected) : [],
|
||||
signalAvailable: parsed && collected.length > 0
|
||||
};
|
||||
}
|
||||
|
||||
return { hasField: false, parsed: false, sources: [], signalAvailable: false };
|
||||
}
|
||||
|
||||
function evaluateSourceSignalEligibility({ pageType, frontmatter }) {
|
||||
const kind = SCAN_KINDS.find((k) => k.pageType === pageType);
|
||||
if (!kind || !kind.applicable) {
|
||||
return { eligible: false, reason: "not_applicable", sources: [] };
|
||||
}
|
||||
|
||||
const parsed = parseSourcesFrontmatter(frontmatter);
|
||||
|
||||
if (!parsed.hasField) {
|
||||
return { eligible: false, reason: "missing_sources", sources: [] };
|
||||
}
|
||||
if (!parsed.parsed) {
|
||||
return { eligible: false, reason: "invalid_sources", sources: [] };
|
||||
}
|
||||
if (parsed.sources.length === 0) {
|
||||
return { eligible: false, reason: "empty_sources", sources: [] };
|
||||
}
|
||||
|
||||
return { eligible: true, reason: "ok", sources: parsed.sources };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
SCAN_KINDS,
|
||||
extractFrontmatter,
|
||||
evaluateSourceSignalEligibility,
|
||||
normalizeSourceToken,
|
||||
parseSourcesFrontmatter,
|
||||
sortedUnique
|
||||
};
|
||||
Executable
+114
@@ -0,0 +1,114 @@
|
||||
#!/bin/bash
|
||||
# lint-fix.sh — 自动修复 lint 发现的低风险问题
|
||||
# 用法：bash scripts/lint-fix.sh <wiki_root> [--dry-run]
|
||||
# 修复范围：仅处理确定性修复（补 index 条目），不做高风险操作（删页面、改内容）
|
||||
# 退出码：0 = 完成，1 = 参数错误
|
||||
|
||||
set -u
|
||||
shopt -s nullglob
|
||||
|
||||
WIKI_ROOT="${1:-.}"
|
||||
DRY_RUN=false
|
||||
[ "${2:-}" = "--dry-run" ] && DRY_RUN=true
|
||||
|
||||
WIKI_DIR="$WIKI_ROOT/wiki"
|
||||
INDEX_FILE="$WIKI_ROOT/index.md"
|
||||
|
||||
if [ ! -d "$WIKI_DIR" ]; then
|
||||
echo "ERROR: wiki directory not found: $WIKI_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f "$INDEX_FILE" ]; then
|
||||
echo "ERROR: index.md not found: $INDEX_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FIXED=0
|
||||
|
||||
index_has_entry() {
|
||||
local entry="$1"
|
||||
grep -ohE "\[\[[^]]+\]\]" "$INDEX_FILE" 2>/dev/null | \
|
||||
sed -e 's/\[\[//g' -e 's/\]\]//g' -e 's/|.*//' | \
|
||||
grep -Fxq "$entry"
|
||||
}
|
||||
|
||||
# Insert a [[link]] entry after the matching section header in index.md.
|
||||
# If no matching section is found, appends to end of file as fallback.
|
||||
insert_under_section() {
|
||||
local index_file="$1"
|
||||
local section_pattern="$2"
|
||||
local entry="$3"
|
||||
|
||||
# Find the line number of the section header
|
||||
local line_num
|
||||
line_num=$(grep -n -i -E "^#.*($section_pattern)" "$index_file" 2>/dev/null | head -1 | cut -d: -f1)
|
||||
|
||||
if [ -n "$line_num" ]; then
|
||||
# Scan from section header to find insert point:
|
||||
# last "- [[" line before next "##" header or EOF
|
||||
local total_lines last_list_line offset
|
||||
total_lines=$(wc -l < "$index_file" | tr -d ' ')
|
||||
last_list_line="$line_num"
|
||||
offset=$((line_num + 1))
|
||||
while [ "$offset" -le "$total_lines" ]; do
|
||||
local cur_line
|
||||
cur_line=$(sed -n "${offset}p" "$index_file")
|
||||
case "$cur_line" in
|
||||
"##"*) break ;;
|
||||
"- [["*) last_list_line="$offset" ;;
|
||||
esac
|
||||
offset=$((offset + 1))
|
||||
done
|
||||
# Insert after the last list item
|
||||
local tmp_file
|
||||
tmp_file=$(mktemp "${index_file}.tmp.XXXXXX") || return 1
|
||||
awk -v insert_after="$last_list_line" -v entry="$entry" '
|
||||
{ print }
|
||||
NR == insert_after { print "- [[" entry "]]" }
|
||||
' "$index_file" > "$tmp_file" && mv "$tmp_file" "$index_file"
|
||||
else
|
||||
# Fallback: append to end of file
|
||||
printf '\n- [[%s]]\n' "$entry" >> "$index_file"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "=== lint-fix: low-risk auto-repair ==="
|
||||
echo ""
|
||||
|
||||
# Fix 1: Add unlisted pages to index.md
|
||||
# Only adds pages that exist in wiki/ but are not referenced in index.md
|
||||
# Skips derived pages (queries/, sessions/)
|
||||
echo "--- Checking for unlisted pages ---"
|
||||
for _subdir in entities topics sources comparisons synthesis; do
|
||||
for f in "$WIKI_DIR"/$_subdir/*.md; do
|
||||
[ -f "$f" ] || continue
|
||||
BASENAME=$(basename "$f" .md)
|
||||
# Skip derived pages
|
||||
case "$f" in
|
||||
*/queries/*|*/sessions/*) continue ;;
|
||||
esac
|
||||
if ! index_has_entry "$BASENAME"; then
|
||||
SECTION_PATTERN=""
|
||||
case "$_subdir" in
|
||||
entities) SECTION_PATTERN="实体页|Entities" ;;
|
||||
topics) SECTION_PATTERN="主题页|Topics" ;;
|
||||
sources) SECTION_PATTERN="素材摘要|Sources" ;;
|
||||
comparisons) SECTION_PATTERN="对比分析|Comparisons" ;;
|
||||
synthesis) SECTION_PATTERN="综合分析|Synthesis" ;;
|
||||
esac
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
echo " [dry-run] Would add [[$BASENAME]] under $_subdir section"
|
||||
else
|
||||
insert_under_section "$INDEX_FILE" "$SECTION_PATTERN" "$BASENAME"
|
||||
echo " Fixed: added [[$BASENAME]] under $_subdir section"
|
||||
fi
|
||||
FIXED=$((FIXED + 1))
|
||||
fi
|
||||
done
|
||||
done
|
||||
[ "$FIXED" -eq 0 ] && echo " (all pages already listed)"
|
||||
echo ""
|
||||
|
||||
echo "=== lint-fix complete: $FIXED fix(es) applied ==="
|
||||
[ "$DRY_RUN" = true ] && echo "(dry-run mode — no files were modified)"
|
||||
exit 0
|
||||
Executable
+217
@@ -0,0 +1,217 @@
|
||||
#!/bin/bash
|
||||
# lint-runner.sh — wiki 机械健康检查
|
||||
# 用法：bash scripts/lint-runner.sh <wiki_root>
|
||||
# 输出：结构化文本报告（供 AI 后续分析使用）
|
||||
# 退出码：0 = 运行完成，1 = 脚本错误（路径不存在、wiki 结构不完整）
|
||||
|
||||
set -u
|
||||
shopt -s nullglob
|
||||
|
||||
WIKI_ROOT="${1:-.}"
|
||||
WIKI_DIR="$WIKI_ROOT/wiki"
|
||||
INDEX_FILE="$WIKI_ROOT/index.md"
|
||||
|
||||
if [ ! -d "$WIKI_DIR" ]; then
|
||||
echo "ERROR: wiki 目录不存在：$WIKI_DIR" >&2
|
||||
echo " 请确认路径正确，或先运行 init 工作流初始化知识库。" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f "$INDEX_FILE" ]; then
|
||||
echo "ERROR: index.md 不存在：$INDEX_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
index_has_entry() {
|
||||
local entry="$1"
|
||||
grep -ohE "\[\[[^]]+\]\]" "$INDEX_FILE" 2>/dev/null | \
|
||||
sed -e 's/\[\[//g' -e 's/\]\]//g' -e 's/|.*//' | \
|
||||
grep -Fxq "$entry"
|
||||
}
|
||||
|
||||
echo "=== llm-wiki lint 报告 ==="
|
||||
echo "时间：$(date '+%Y-%m-%d %H:%M')"
|
||||
echo "检查路径：$WIKI_DIR"
|
||||
echo ""
|
||||
|
||||
# 检查 1：孤立页面
|
||||
# 定义：entities/、topics/、sources/ 下的页面，除了自己之外没有任何其他 wiki 页面用 [[名称]] 引用它
|
||||
echo "--- 孤立页面（没有被其他页面引用） ---"
|
||||
_ORPHANS=0
|
||||
for _subdir in entities topics sources; do
|
||||
for f in "$WIKI_DIR"/$_subdir/*.md; do
|
||||
[ -f "$f" ] || continue
|
||||
BASENAME=$(basename "$f" .md)
|
||||
if ! grep -rlF "[[$BASENAME]]" "$WIKI_DIR" 2>/dev/null | grep -vxF "$f" | grep -q .; then
|
||||
echo " 孤立: $_subdir/$BASENAME"
|
||||
_ORPHANS=$((_ORPHANS + 1))
|
||||
fi
|
||||
done
|
||||
done
|
||||
[ "$_ORPHANS" -eq 0 ] && echo " （无孤立页面）"
|
||||
echo ""
|
||||
|
||||
# 检查 2：断链
|
||||
# 定义：wiki/ 下的页面里有 [[X]] 链接（支持 [[X|别名]] 语法），但 wiki/ 任意子目录找不到 X.md
|
||||
echo "--- 断链（被链接但不存在的页面） ---"
|
||||
_TMP_BROKEN=$(mktemp)
|
||||
grep -rohE "\[\[[^]]+\]\]" "$WIKI_DIR" 2>/dev/null | \
|
||||
sed -e 's/\[\[//g' -e 's/\]\]//g' -e 's/|.*//' | \
|
||||
sort -u | \
|
||||
while read -r LINK; do
|
||||
[ -z "$LINK" ] && continue
|
||||
if ! find "$WIKI_DIR" -name "$LINK.md" 2>/dev/null | grep -q .; then
|
||||
echo " 断链: [[$LINK]]"
|
||||
echo "$LINK" >> "$_TMP_BROKEN"
|
||||
fi
|
||||
done
|
||||
if [ ! -s "$_TMP_BROKEN" ]; then
|
||||
echo " （无断链）"
|
||||
fi
|
||||
rm -f "$_TMP_BROKEN"
|
||||
echo ""
|
||||
|
||||
# 检查 3：index 一致性
|
||||
# 定义：index.md 里有 [[X]] 记录（去掉别名），但 wiki/ 任意子目录都找不到 X.md
|
||||
echo "--- index 一致性（index.md 有记录但文件缺失） ---"
|
||||
_TMP_MISSING=$(mktemp)
|
||||
grep -ohE "\[\[[^]]+\]\]" "$INDEX_FILE" 2>/dev/null | \
|
||||
sed -e 's/\[\[//g' -e 's/\]\]//g' -e 's/|.*//' | \
|
||||
sort -u | \
|
||||
while read -r ENTRY; do
|
||||
[ -z "$ENTRY" ] && continue
|
||||
if ! find "$WIKI_DIR" -name "$ENTRY.md" 2>/dev/null | grep -q .; then
|
||||
echo " index 有但文件缺失: $ENTRY"
|
||||
echo "$ENTRY" >> "$_TMP_MISSING"
|
||||
fi
|
||||
done
|
||||
if [ ! -s "$_TMP_MISSING" ]; then
|
||||
echo " （index 与文件一致）"
|
||||
fi
|
||||
rm -f "$_TMP_MISSING"
|
||||
echo ""
|
||||
|
||||
# 检查 4：反向 index 一致性
|
||||
# 定义：wiki/ 下实际存在的页面，但 index.md 里没有 [[页面名]] 记录
|
||||
# 排除 derived 页面（queries/、synthesis/sessions/）
|
||||
echo "--- 反向 index 一致性（文件存在但 index.md 未收录） ---"
|
||||
_TMP_UNLISTED=$(mktemp)
|
||||
for _subdir in entities topics sources comparisons synthesis; do
|
||||
for f in "$WIKI_DIR"/$_subdir/*.md; do
|
||||
[ -f "$f" ] || continue
|
||||
BASENAME=$(basename "$f" .md)
|
||||
# 跳过 derived 页面
|
||||
case "$f" in
|
||||
*/queries/*|*/sessions/*) continue ;;
|
||||
esac
|
||||
if ! index_has_entry "$BASENAME"; then
|
||||
echo " 未收录: $_subdir/$BASENAME"
|
||||
echo "$BASENAME" >> "$_TMP_UNLISTED"
|
||||
fi
|
||||
done
|
||||
done
|
||||
if [ ! -s "$_TMP_UNLISTED" ]; then
|
||||
echo " （所有页面均已收录）"
|
||||
fi
|
||||
rm -f "$_TMP_UNLISTED"
|
||||
echo ""
|
||||
|
||||
# 检查 5：图片资产一致性
|
||||
# 定义：source 页面 frontmatter 中 image_paths 列出的文件，在知识库中是否实际存在
|
||||
# 支持 block list 格式和 inline array 格式
|
||||
echo "--- 图片资产一致性（image_paths 声明但文件缺失） ---"
|
||||
_IMG_ISSUES=0
|
||||
for f in "$WIKI_DIR"/sources/*.md; do
|
||||
[ -f "$f" ] || continue
|
||||
_BASENAME=$(basename "$f" .md)
|
||||
# 提取 frontmatter 中 image_paths 的值
|
||||
_IN_FM=false
|
||||
_IN_IMG=false
|
||||
_INLINE_VAL=""
|
||||
while IFS= read -r line; do
|
||||
case "$line" in
|
||||
"---")
|
||||
if [ "$_IN_FM" = true ]; then break; fi
|
||||
_IN_FM=true
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
[ "$_IN_FM" = true ] || continue
|
||||
case "$line" in
|
||||
image_paths:*)
|
||||
# 检查是否有 inline value（如 image_paths: ["a.png", "b.jpg"]）
|
||||
_INLINE_VAL=$(echo "$line" | sed 's/^image_paths:[[:space:]]*//')
|
||||
if [ -n "$_INLINE_VAL" ] && [ "$_INLINE_VAL" != "[]" ]; then
|
||||
# 解析 inline array：去掉 []，按逗号分割
|
||||
echo "$_INLINE_VAL" | tr -d '[]' | tr ',' '\n' | while IFS= read -r _ITEM; do
|
||||
_PATH=$(echo "$_ITEM" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | tr -d '"' | tr -d "'")
|
||||
[ -z "$_PATH" ] && continue
|
||||
if [ ! -f "$WIKI_ROOT/$_PATH" ]; then
|
||||
echo " 缺失: $_BASENAME → $_PATH"
|
||||
fi
|
||||
done
|
||||
_INLINE_COUNT=$(echo "$_INLINE_VAL" | tr -d '[]' | tr ',' '\n' | while IFS= read -r _ITEM; do
|
||||
_P=$(echo "$_ITEM" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | tr -d '"' | tr -d "'")
|
||||
[ -z "$_P" ] && continue
|
||||
[ ! -f "$WIKI_ROOT/$_P" ] && echo "x"
|
||||
done | wc -l | tr -d ' ')
|
||||
_IMG_ISSUES=$((_IMG_ISSUES + _INLINE_COUNT))
|
||||
_IN_IMG=false
|
||||
else
|
||||
_IN_IMG=true
|
||||
fi
|
||||
continue
|
||||
;;
|
||||
" - "*)
|
||||
if [ "$_IN_IMG" = true ]; then
|
||||
_PATH=$(echo "$line" | sed 's/^[[:space:]]*- //' | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | tr -d '"' | tr -d "'")
|
||||
[ -z "$_PATH" ] && continue
|
||||
if [ ! -f "$WIKI_ROOT/$_PATH" ]; then
|
||||
echo " 缺失: $_BASENAME → $_PATH"
|
||||
_IMG_ISSUES=$((_IMG_ISSUES + 1))
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
*) _IN_IMG=false ;;
|
||||
esac
|
||||
done < "$f"
|
||||
done
|
||||
[ "$_IMG_ISSUES" -eq 0 ] && echo " （无缺失图片）"
|
||||
echo ""
|
||||
|
||||
# 检查 6：source-signal 覆盖情况
|
||||
echo "--- source-signal 覆盖情况 ---"
|
||||
_COVERAGE_SCRIPT="$(cd "$(dirname "$0")" && pwd)/source-signal-coverage.js"
|
||||
if [ -f "$_COVERAGE_SCRIPT" ] && command -v node >/dev/null 2>&1; then
|
||||
_COVERAGE_JSON=$(node "$_COVERAGE_SCRIPT" "$WIKI_ROOT" 2>/dev/null)
|
||||
if [ $? -eq 0 ] && [ -n "$_COVERAGE_JSON" ]; then
|
||||
node -e '
|
||||
const data = JSON.parse(require("fs").readFileSync("/dev/stdin", "utf8"));
|
||||
const s = data.summary;
|
||||
console.log(" 已参与：" + s.ok);
|
||||
console.log(" 缺少 sources 字段：" + s.missing_sources);
|
||||
console.log(" sources 为空：" + s.empty_sources);
|
||||
console.log(" sources 格式无效：" + s.invalid_sources);
|
||||
console.log(" 当前不参与：" + s.not_applicable);
|
||||
const issues = data.pages.filter(p => p.reason !== "ok" && p.reason !== "not_applicable");
|
||||
if (issues.length > 0) {
|
||||
const byReason = { missing_sources: [], empty_sources: [], invalid_sources: [] };
|
||||
for (const p of issues) { if (byReason[p.reason]) byReason[p.reason].push(p.path); }
|
||||
for (const [reason, paths] of Object.entries(byReason)) {
|
||||
if (paths.length === 0) continue;
|
||||
const label = { missing_sources: "缺少 sources 字段", empty_sources: "sources 为空", invalid_sources: "sources 格式无效" }[reason];
|
||||
console.log("");
|
||||
console.log(" " + label + "：");
|
||||
for (const p of paths) console.log(" - " + p);
|
||||
}
|
||||
}
|
||||
' <<< "$_COVERAGE_JSON"
|
||||
else
|
||||
echo " （coverage 脚本执行失败，跳过覆盖检查）"
|
||||
fi
|
||||
else
|
||||
echo " （coverage 脚本或 node 不可用，跳过覆盖检查）"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "=== 机械检查完成。矛盾检测、交叉引用、置信度抽查由 AI 继续执行 ==="
|
||||
exit 0
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
# 共享运行场景解析：供 install.sh 和 adapter-state.sh 复用
|
||||
|
||||
resolve_platform_skill_root() {
|
||||
case "$1" in
|
||||
claude)
|
||||
printf '%s\n' "$HOME/.claude/skills"
|
||||
;;
|
||||
codex)
|
||||
if [ -d "$HOME/.codex/skills" ] || [ ! -d "$HOME/.Codex/skills" ]; then
|
||||
printf '%s\n' "$HOME/.codex/skills"
|
||||
else
|
||||
printf '%s\n' "$HOME/.Codex/skills"
|
||||
fi
|
||||
;;
|
||||
openclaw)
|
||||
printf '%s\n' "$HOME/.openclaw/skills"
|
||||
;;
|
||||
hermes)
|
||||
printf '%s\n' "$HOME/.hermes/skills"
|
||||
;;
|
||||
*)
|
||||
echo "不支持的平台：$1" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
detect_layout_mode() {
|
||||
local bundle_root="$1"
|
||||
|
||||
if [ -e "$bundle_root/.git" ]; then
|
||||
printf '%s\n' "source_checkout"
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf '%s\n' "installed_skill"
|
||||
}
|
||||
|
||||
resolve_layout_mode() {
|
||||
local bundle_root="$1"
|
||||
local override_mode="${2:-}"
|
||||
|
||||
if [ -n "$override_mode" ]; then
|
||||
printf '%s\n' "$override_mode"
|
||||
return 0
|
||||
fi
|
||||
|
||||
detect_layout_mode "$bundle_root"
|
||||
}
|
||||
|
||||
resolve_optional_adapter_root() {
|
||||
local bundle_root="$1"
|
||||
local skill_root_override="${2:-}"
|
||||
local override_mode="${3:-}"
|
||||
local layout_mode
|
||||
|
||||
if [ -n "$skill_root_override" ]; then
|
||||
printf '%s\n' "$skill_root_override"
|
||||
return 0
|
||||
fi
|
||||
|
||||
layout_mode="$(resolve_layout_mode "$bundle_root" "$override_mode")"
|
||||
|
||||
case "$layout_mode" in
|
||||
source_checkout)
|
||||
printf '%s\n' "$bundle_root/deps"
|
||||
;;
|
||||
installed_skill|upgrade_target)
|
||||
printf '%s\n' "$(dirname "$bundle_root")"
|
||||
;;
|
||||
*)
|
||||
echo "未知运行模式：$layout_mode" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/bin/bash
|
||||
# 共享配置：被 install.sh / hook-session-start.sh / cache.sh / delete-helper.sh 等引用
|
||||
# 微信公众号提取工具的 Git 仓库地址
|
||||
WECHAT_TOOL_URL="git+https://github.com/jackwener/wechat-article-to-markdown.git"
|
||||
|
||||
# Python 命令检测：Windows 默认安装为 python.exe，不存在 python3 命令
|
||||
# （Microsoft Store 的 python3 是安装提示 stub，运行会失败）
|
||||
_python_version_check='import sys; sys.exit(0 if sys.version_info >= (3, 8) else 1)'
|
||||
|
||||
_python_cmd_is_valid() {
|
||||
local candidate="$1"
|
||||
|
||||
command -v "$candidate" >/dev/null 2>&1 && "$candidate" -c "$_python_version_check" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
_detect_python_cmd() {
|
||||
# 要求 Python 3.8+（见 README Windows 小节与下方错误消息）
|
||||
if _python_cmd_is_valid python3; then
|
||||
echo "python3"
|
||||
elif _python_cmd_is_valid python; then
|
||||
echo "python"
|
||||
else
|
||||
echo ""
|
||||
fi
|
||||
}
|
||||
|
||||
require_python_cmd() {
|
||||
local detected_cmd
|
||||
|
||||
if [ "${PYTHON_CMD_READY:-0}" = "1" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ -n "${PYTHON_CMD:-}" ] && _python_cmd_is_valid "$PYTHON_CMD"; then
|
||||
export PYTHON_CMD
|
||||
PYTHON_CMD_READY=1
|
||||
return 0
|
||||
fi
|
||||
|
||||
detected_cmd="$(_detect_python_cmd)"
|
||||
if [ -z "$detected_cmd" ]; then
|
||||
echo "[llm-wiki] 错误：找不到可用的 Python 3，请先安装 Python 3.8+ 并加入 PATH" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
PYTHON_CMD="$detected_cmd"
|
||||
export PYTHON_CMD
|
||||
PYTHON_CMD_READY=1
|
||||
}
|
||||
|
||||
# 统一 Python 子进程 stdout/stderr 编码为 UTF-8
|
||||
# Windows 中文环境下 Python 无 TTY 时 sys.stdout.encoding 默认 gbk (cp936)，
|
||||
# 会导致 Agent 通过 subprocess 读取的 JSON / 输出出现乱码 (issue #16)
|
||||
export PYTHONIOENCODING="${PYTHONIOENCODING:-utf-8}"
|
||||
|
||||
# 输出指定工具的跨平台安装提示，缩进 2 空格便于嵌套在 ERROR 消息下；
|
||||
# 输出走 stderr，与 ERROR 消息保持同一通道。
|
||||
print_install_hint() {
|
||||
local tool="$1"
|
||||
case "$tool" in
|
||||
jq)
|
||||
echo " macOS: brew install jq" >&2
|
||||
echo " Linux/WSL: sudo apt-get install jq (Debian/Ubuntu)" >&2
|
||||
echo " sudo dnf install jq (RHEL/Fedora)" >&2
|
||||
echo " Windows: winget install jqlang.jq (or choco install jq)" >&2
|
||||
;;
|
||||
node)
|
||||
echo " macOS: brew install node" >&2
|
||||
echo " Linux/WSL: sudo apt-get install nodejs npm" >&2
|
||||
echo " Windows: winget install OpenJS.NodeJS (or choco install nodejs)" >&2
|
||||
;;
|
||||
uv)
|
||||
echo " macOS/Linux: curl -LsSf https://astral.sh/uv/install.sh | sh (official)" >&2
|
||||
echo " brew install uv (alternative)" >&2
|
||||
echo " Windows: powershell -c \"irm https://astral.sh/uv/install.ps1 | iex\" (official)" >&2
|
||||
echo " winget install --id=astral-sh.uv -e (alternative)" >&2
|
||||
;;
|
||||
*)
|
||||
echo " unknown tool: $tool" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
field_name requiredness filled_by value_rule
|
||||
source_id required router 必须匹配 source-registry.tsv 中的 source_id
|
||||
source_label required router 必须匹配 source-registry.tsv 中的 source_label
|
||||
source_category required router 必须匹配 source-registry.tsv 中的 source_category
|
||||
input_mode required caller_or_router 只能是 url / file / text / asset
|
||||
raw_dir required router 必须是 raw/ 下的相对目录
|
||||
original_ref required caller 保存原始 URL、文件路径或用户粘贴说明
|
||||
ingest_text required adapter_or_user 进入主线前必须是非空文本
|
||||
adapter_name required_may_be_empty router_or_adapter 核心主线和手动入口留空；外挂来源写实际 adapter 名称
|
||||
fallback_hint required router 必须给出用户可执行的手动回退提示
|
||||
|
Executable
+349
@@ -0,0 +1,349 @@
|
||||
#!/bin/bash
|
||||
# 统一来源总表读取与验证脚本
|
||||
# 权威数据文件：source-registry.tsv（来源定义）、source-record-contract.tsv（字段契约）
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
CONTRACT_FILE="$SCRIPT_DIR/source-record-contract.tsv"
|
||||
REGISTRY_FILE="$SCRIPT_DIR/source-registry.tsv"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/source-registry.sh fields
|
||||
bash scripts/source-registry.sh list
|
||||
bash scripts/source-registry.sh get <source_id>
|
||||
bash scripts/source-registry.sh match-url <url>
|
||||
bash scripts/source-registry.sh match-file <path>
|
||||
bash scripts/source-registry.sh list-by-category <core_builtin|optional_adapter|manual_only>
|
||||
bash scripts/source-registry.sh unique-dependencies <bundled|install_time|none>
|
||||
bash scripts/source-registry.sh validate
|
||||
EOF
|
||||
}
|
||||
|
||||
require_file() {
|
||||
local file="$1"
|
||||
|
||||
[ -f "$file" ] || {
|
||||
echo "缺少文件：$file" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
expect_header() {
|
||||
local file="$1"
|
||||
local expected="$2"
|
||||
local actual
|
||||
|
||||
actual="$(head -n 1 "$file")"
|
||||
[ "$actual" = "$expected" ] || {
|
||||
echo "表头不匹配：$file" >&2
|
||||
echo "期望：$expected" >&2
|
||||
echo "实际：$actual" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
validate_contract() {
|
||||
require_file "$CONTRACT_FILE"
|
||||
expect_header "$CONTRACT_FILE" $'field_name\trequiredness\tfilled_by\tvalue_rule'
|
||||
|
||||
awk -F '\t' '
|
||||
BEGIN {
|
||||
required["source_id"] = 1
|
||||
required["source_label"] = 1
|
||||
required["source_category"] = 1
|
||||
required["input_mode"] = 1
|
||||
required["raw_dir"] = 1
|
||||
required["original_ref"] = 1
|
||||
required["ingest_text"] = 1
|
||||
required["adapter_name"] = 1
|
||||
required["fallback_hint"] = 1
|
||||
}
|
||||
NR == 1 { next }
|
||||
{
|
||||
if ($1 == "" || $2 == "" || $3 == "" || $4 == "") {
|
||||
printf("source-record-contract.tsv 第 %d 行存在空字段\n", NR) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
seen[$1] += 1
|
||||
}
|
||||
END {
|
||||
for (field in required) {
|
||||
if (seen[field] != 1) {
|
||||
printf("source-record-contract.tsv 缺少或重复字段：%s\n", field) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
}
|
||||
|
||||
exit failed ? 1 : 0
|
||||
}
|
||||
' "$CONTRACT_FILE"
|
||||
}
|
||||
|
||||
validate_registry() {
|
||||
require_file "$REGISTRY_FILE"
|
||||
expect_header "$REGISTRY_FILE" $'source_id\tsource_label\tsource_category\tinput_mode\tmatch_rule\traw_dir\tadapter_name\tdependency_name\tdependency_type\tfallback_hint'
|
||||
|
||||
awk -F '\t' '
|
||||
NR == 1 { next }
|
||||
{
|
||||
if ($1 == "" || $2 == "" || $3 == "" || $4 == "" || $5 == "" || $6 == "" || $10 == "") {
|
||||
printf("source-registry.tsv 第 %d 行存在空字段\n", NR) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($3 != "core_builtin" && $3 != "optional_adapter" && $3 != "manual_only") {
|
||||
printf("source-registry.tsv 第 %d 行存在未知分类：%s\n", NR, $3) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($4 != "url" && $4 != "file" && $4 != "text" && $4 != "asset") {
|
||||
printf("source-registry.tsv 第 %d 行存在未知输入模式：%s\n", NR, $4) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($4 == "url" && $5 !~ /^url_host:/) {
|
||||
printf("source-registry.tsv 第 %d 行 URL 来源必须声明 url_host 规则：%s\n", NR, $5) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($4 == "file" && $5 !~ /^file_ext:/) {
|
||||
printf("source-registry.tsv 第 %d 行文件来源必须声明 file_ext 规则：%s\n", NR, $5) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($4 == "text" && $5 !~ /^text:/) {
|
||||
printf("source-registry.tsv 第 %d 行文本来源必须声明 text 规则：%s\n", NR, $5) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($4 == "asset" && $5 !~ /^asset:/) {
|
||||
printf("source-registry.tsv 第 %d 行附件来源必须声明 asset 规则：%s\n", NR, $5) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($6 !~ /^raw\//) {
|
||||
printf("source-registry.tsv 第 %d 行 raw_dir 必须位于 raw/ 下：%s\n", NR, $6) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if (seen[$1]++) {
|
||||
printf("source-registry.tsv source_id 重复：%s\n", $1) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
category_seen[$3] = 1
|
||||
|
||||
if ($3 == "optional_adapter") {
|
||||
if ($7 == "-" || $8 == "-" || $9 == "none") {
|
||||
printf("source-registry.tsv 第 %d 行 optional_adapter 缺少依赖信息\n", NR) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
} else if ($7 != "-" || $8 != "-" || $9 != "none") {
|
||||
printf("source-registry.tsv 第 %d 行非外挂来源不应声明依赖\n", NR) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
}
|
||||
END {
|
||||
if (!category_seen["core_builtin"]) {
|
||||
print "source-registry.tsv 缺少 core_builtin 来源" > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if (!category_seen["optional_adapter"]) {
|
||||
print "source-registry.tsv 缺少 optional_adapter 来源" > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if (!category_seen["manual_only"]) {
|
||||
print "source-registry.tsv 缺少 manual_only 来源" > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
exit failed ? 1 : 0
|
||||
}
|
||||
' "$REGISTRY_FILE"
|
||||
}
|
||||
|
||||
print_contract() {
|
||||
validate_contract
|
||||
cat "$CONTRACT_FILE"
|
||||
}
|
||||
|
||||
print_registry() {
|
||||
validate_registry
|
||||
cat "$REGISTRY_FILE"
|
||||
}
|
||||
|
||||
get_source() {
|
||||
local source_id="$1"
|
||||
|
||||
validate_registry
|
||||
|
||||
awk -F '\t' -v source_id="$source_id" '
|
||||
NR == 1 { next }
|
||||
$1 == source_id {
|
||||
print
|
||||
found = 1
|
||||
}
|
||||
END {
|
||||
exit found ? 0 : 1
|
||||
}
|
||||
' "$REGISTRY_FILE"
|
||||
}
|
||||
|
||||
extract_url_host() {
|
||||
local url="$1"
|
||||
local rest host
|
||||
|
||||
rest="${url#*://}"
|
||||
if [ "$rest" = "$url" ]; then
|
||||
rest="$url"
|
||||
fi
|
||||
|
||||
rest="${rest#*@}"
|
||||
host="${rest%%/*}"
|
||||
host="${host%%\?*}"
|
||||
host="${host%%#*}"
|
||||
host="${host%%:*}"
|
||||
|
||||
printf '%s\n' "$host" | tr '[:upper:]' '[:lower:]'
|
||||
}
|
||||
|
||||
host_matches_pattern() {
|
||||
local host="$1"
|
||||
local pattern="$2"
|
||||
|
||||
case "$host" in
|
||||
"$pattern"|*."$pattern")
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
match_url() {
|
||||
local url="$1"
|
||||
local host row source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint
|
||||
local fallback_row=""
|
||||
local pattern pattern_list
|
||||
|
||||
validate_registry
|
||||
host="$(extract_url_host "$url")"
|
||||
|
||||
while IFS=$'\t' read -r source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint; do
|
||||
[ "$source_id" = "source_id" ] && continue
|
||||
[ "$input_mode" = "url" ] || continue
|
||||
|
||||
pattern_list="${match_rule#url_host:}"
|
||||
if [ "$pattern_list" = "*" ]; then
|
||||
fallback_row="$(printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$source_id" "$source_label" "$source_category" "$input_mode" "$match_rule" "$raw_dir" "$adapter_name" "$dependency_name" "$dependency_type" "$fallback_hint")"
|
||||
continue
|
||||
fi
|
||||
|
||||
for pattern in ${pattern_list//,/ }; do
|
||||
if host_matches_pattern "$host" "$pattern"; then
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$source_id" "$source_label" "$source_category" "$input_mode" "$match_rule" "$raw_dir" "$adapter_name" "$dependency_name" "$dependency_type" "$fallback_hint"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
done < "$REGISTRY_FILE"
|
||||
|
||||
[ -n "$fallback_row" ] || return 1
|
||||
printf '%s\n' "$fallback_row"
|
||||
}
|
||||
|
||||
match_file() {
|
||||
local path="$1"
|
||||
local lowered_path source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint
|
||||
local extension_list extension
|
||||
|
||||
validate_registry
|
||||
lowered_path="$(printf '%s\n' "$path" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
while IFS=$'\t' read -r source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint; do
|
||||
[ "$source_id" = "source_id" ] && continue
|
||||
[ "$input_mode" = "file" ] || continue
|
||||
|
||||
extension_list="${match_rule#file_ext:}"
|
||||
for extension in ${extension_list//,/ }; do
|
||||
case "$lowered_path" in
|
||||
*"$extension")
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$source_id" "$source_label" "$source_category" "$input_mode" "$match_rule" "$raw_dir" "$adapter_name" "$dependency_name" "$dependency_type" "$fallback_hint"
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
done < "$REGISTRY_FILE"
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
list_by_category() {
|
||||
local category="$1"
|
||||
|
||||
validate_registry
|
||||
|
||||
awk -F '\t' -v category="$category" '
|
||||
NR == 1 { next }
|
||||
$3 == category { print }
|
||||
' "$REGISTRY_FILE"
|
||||
}
|
||||
|
||||
list_unique_dependencies() {
|
||||
local dependency_type="$1"
|
||||
|
||||
validate_registry
|
||||
|
||||
awk -F '\t' -v dependency_type="$dependency_type" '
|
||||
NR == 1 { next }
|
||||
$9 == dependency_type && $8 != "-" { print $8 }
|
||||
' "$REGISTRY_FILE" | sort -u
|
||||
}
|
||||
|
||||
command_name="${1:-}"
|
||||
|
||||
case "$command_name" in
|
||||
fields)
|
||||
[ "$#" -eq 1 ] || { usage; exit 1; }
|
||||
print_contract
|
||||
;;
|
||||
list)
|
||||
[ "$#" -eq 1 ] || { usage; exit 1; }
|
||||
print_registry
|
||||
;;
|
||||
get)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
get_source "$2"
|
||||
;;
|
||||
match-url)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
match_url "$2"
|
||||
;;
|
||||
match-file)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
match_file "$2"
|
||||
;;
|
||||
list-by-category)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
list_by_category "$2"
|
||||
;;
|
||||
unique-dependencies)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
list_unique_dependencies "$2"
|
||||
;;
|
||||
validate)
|
||||
[ "$#" -eq 1 ] || { usage; exit 1; }
|
||||
validate_contract
|
||||
validate_registry
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,10 @@
|
||||
source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint
|
||||
local_pdf PDF / 本地 PDF core_builtin file file_ext:.pdf raw/pdfs - - none 直接提供文件路径即可进入主线
|
||||
local_document Markdown/文本/HTML core_builtin file file_ext:.md,.txt,.html raw/notes - - none 直接提供文件路径即可进入主线
|
||||
plain_text 纯文本粘贴 core_builtin text text:inline raw/notes - - none 直接粘贴正文即可进入主线
|
||||
x_twitter X/Twitter optional_adapter url url_host:x.com,twitter.com raw/tweets baoyu-url-to-markdown baoyu-url-to-markdown bundled 自动提取失败时，改为复制全文粘贴
|
||||
wechat_article 微信公众号 optional_adapter url url_host:mp.weixin.qq.com raw/wechat wechat-article-to-markdown wechat-article-to-markdown install_time 自动提取失败时，在浏览器打开后复制全文粘贴
|
||||
youtube_video YouTube optional_adapter url url_host:youtube.com,youtu.be raw/articles youtube-transcript youtube-transcript bundled 自动提取失败时，提供字幕文件或手动粘贴文本
|
||||
zhihu_article 知乎 optional_adapter url url_host:zhihu.com raw/zhihu baoyu-url-to-markdown baoyu-url-to-markdown bundled 自动提取失败时，改为复制全文粘贴
|
||||
xiaohongshu_post 小红书 manual_only url url_host:xiaohongshu.com,xhslink.com raw/xiaohongshu - - none 请先从 App 或网页复制内容，再粘贴进来
|
||||
web_article 网页文章 optional_adapter url url_host:* raw/articles baoyu-url-to-markdown baoyu-url-to-markdown bundled 自动提取失败时，改为复制全文或保存为本地文件后继续
|
||||
|
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const {
|
||||
SCAN_KINDS,
|
||||
extractFrontmatter,
|
||||
evaluateSourceSignalEligibility
|
||||
} = require("./lib/source-signal-eligibility");
|
||||
|
||||
function scanWiki(wikiRoot) {
|
||||
const wikiDir = path.join(wikiRoot, "wiki");
|
||||
if (!fs.existsSync(wikiDir)) {
|
||||
console.error(`ERROR: wiki 目录不存在：${wikiDir}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const pages = [];
|
||||
const summary = {
|
||||
applicable_total: 0,
|
||||
ok: 0,
|
||||
missing_sources: 0,
|
||||
empty_sources: 0,
|
||||
invalid_sources: 0,
|
||||
not_applicable: 0
|
||||
};
|
||||
|
||||
for (const kind of SCAN_KINDS) {
|
||||
const dir = path.join(wikiDir, kind.subdir);
|
||||
if (!fs.existsSync(dir)) continue;
|
||||
|
||||
const files = fs.readdirSync(dir)
|
||||
.filter((f) => f.endsWith(".md"))
|
||||
.sort();
|
||||
|
||||
for (const file of files) {
|
||||
const id = path.basename(file, ".md");
|
||||
if (["index", "log", "purpose", ".wiki-schema", "README"].includes(id)) continue;
|
||||
|
||||
const filePath = path.join(dir, file);
|
||||
const raw = fs.readFileSync(filePath, "utf8");
|
||||
const { frontmatter } = extractFrontmatter(raw);
|
||||
const result = evaluateSourceSignalEligibility({
|
||||
pageType: kind.pageType,
|
||||
frontmatter
|
||||
});
|
||||
|
||||
pages.push({
|
||||
path: path.relative(wikiRoot, filePath),
|
||||
id,
|
||||
pageType: kind.pageType,
|
||||
eligible: result.eligible,
|
||||
reason: result.reason,
|
||||
sourceCount: result.sources.length
|
||||
});
|
||||
|
||||
summary[result.reason] += 1;
|
||||
if (result.reason !== "not_applicable") {
|
||||
summary.applicable_total += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { summary, pages };
|
||||
}
|
||||
|
||||
function main(argv) {
|
||||
if (argv.length < 3) {
|
||||
console.error("Usage: node scripts/source-signal-coverage.js <wiki_root>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const wikiRoot = path.resolve(argv[2]);
|
||||
const result = scanWiki(wikiRoot);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main(process.argv);
|
||||
}
|
||||
|
||||
module.exports = { scanWiki };
|
||||
Executable
+144
@@ -0,0 +1,144 @@
|
||||
#!/bin/bash
|
||||
# 验证 ingest Step 1 的 JSON 输出格式
|
||||
# 用法：bash validate-step1.sh <json_file>
|
||||
# 返回：0 = 格式正确，1 = 格式有问题（触发回退）
|
||||
|
||||
SCRIPT_DIR="${BASH_SOURCE[0]%/*}"
|
||||
[ "$SCRIPT_DIR" = "${BASH_SOURCE[0]}" ] && SCRIPT_DIR="."
|
||||
SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
|
||||
JSON_FILE="$1"
|
||||
|
||||
# 参数检查
|
||||
[ -z "$1" ] && { echo "ERROR: usage: validate-step1.sh <json_file>"; exit 1; }
|
||||
|
||||
# 检查 jq 是否可用（必需依赖）
|
||||
command -v jq >/dev/null 2>&1 || {
|
||||
echo "ERROR: jq is not installed. Install it via:" >&2
|
||||
print_install_hint jq
|
||||
exit 1
|
||||
}
|
||||
|
||||
# 检查文件是否存在
|
||||
[ -f "$JSON_FILE" ] || { echo "ERROR: file not found: $JSON_FILE"; exit 1; }
|
||||
|
||||
# 检查是否是有效 JSON
|
||||
jq empty "$JSON_FILE" 2>/dev/null || { echo "ERROR: invalid JSON format"; exit 1; }
|
||||
|
||||
# 检查必需字段存在且类型正确
|
||||
jq -e '.entities | type == "array"' "$JSON_FILE" >/dev/null 2>&1 || { echo "ERROR: 'entities' must be an array"; exit 1; }
|
||||
jq -e '.topics | type == "array"' "$JSON_FILE" >/dev/null 2>&1 || { echo "ERROR: 'topics' must be an array"; exit 1; }
|
||||
jq -e '.connections | type == "array"' "$JSON_FILE" >/dev/null 2>&1 || { echo "ERROR: 'connections' must be an array"; exit 1; }
|
||||
jq -e '.contradictions | type == "array"' "$JSON_FILE" >/dev/null 2>&1 || { echo "ERROR: 'contradictions' must be an array"; exit 1; }
|
||||
jq -e '.new_vs_existing | type == "object"' "$JSON_FILE" >/dev/null 2>&1 || { echo "ERROR: 'new_vs_existing' must be an object"; exit 1; }
|
||||
|
||||
# 检查每个 entity 的必需子字段
|
||||
VALID_CONFIDENCE="EXTRACTED|INFERRED|AMBIGUOUS|UNVERIFIED"
|
||||
|
||||
ENTITY_COUNT=$(jq '.entities | length' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$ENTITY_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
NON_OBJECT_ENTITY_COUNT=$(jq '[.entities[] | select(type != "object")] | length' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$NON_OBJECT_ENTITY_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "ERROR: $NON_OBJECT_ENTITY_COUNT entity/entities must be objects"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# name, type, confidence 必须存在且非空
|
||||
BAD_ENTITY_COUNT=$(jq '
|
||||
[.entities[] | select(
|
||||
(.name // "" | length) == 0 or
|
||||
(.type // "" | length) == 0 or
|
||||
(.confidence // "" | length) == 0
|
||||
)] | length
|
||||
' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$BAD_ENTITY_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "ERROR: $BAD_ENTITY_COUNT entity/entities missing required fields (name/type/confidence)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# confidence 值必须是四个有效值之一
|
||||
INVALID=$(jq -r '.entities[]? | (.confidence // "MISSING")' "$JSON_FILE" 2>/dev/null | \
|
||||
grep -v -E "^($VALID_CONFIDENCE)$" | head -3)
|
||||
if [ -n "$INVALID" ]; then
|
||||
echo "ERROR: invalid entity confidence value(s): $INVALID"
|
||||
echo " Valid values: EXTRACTED | INFERRED | AMBIGUOUS | UNVERIFIED"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# EXTRACTED 和 INFERRED 必须提供 evidence 字段
|
||||
NO_EVIDENCE_COUNT=$(jq '
|
||||
[.entities[] | select(
|
||||
(.confidence == "EXTRACTED" or .confidence == "INFERRED") and
|
||||
((.evidence // "" | length) == 0)
|
||||
)] | length
|
||||
' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$NO_EVIDENCE_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "WARN: $NO_EVIDENCE_COUNT entity/entities with EXTRACTED/INFERRED confidence missing 'evidence' field"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 检查每个 topic 的必需子字段
|
||||
TOPIC_COUNT=$(jq '.topics | length' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$TOPIC_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
NON_OBJECT_TOPIC_COUNT=$(jq '[.topics[] | select(type != "object")] | length' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$NON_OBJECT_TOPIC_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "ERROR: $NON_OBJECT_TOPIC_COUNT topic(s) must be objects"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BAD_TOPIC_COUNT=$(jq '
|
||||
[.topics[] | select(
|
||||
(.name // "" | length) == 0
|
||||
)] | length
|
||||
' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$BAD_TOPIC_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "ERROR: $BAD_TOPIC_COUNT topic(s) missing required 'name' field"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# 检查每个 connection 的必需子字段（from, to, confidence）
|
||||
CONN_COUNT=$(jq '.connections | length' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$CONN_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
NON_OBJECT_CONN_COUNT=$(jq '[.connections[] | select(type != "object")] | length' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$NON_OBJECT_CONN_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "ERROR: $NON_OBJECT_CONN_COUNT connection(s) must be objects"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BAD_CONN_COUNT=$(jq '
|
||||
[.connections[] | select(
|
||||
(.from // "" | length) == 0 or
|
||||
(.to // "" | length) == 0 or
|
||||
(.confidence // "" | length) == 0
|
||||
)] | length
|
||||
' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$BAD_CONN_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "ERROR: $BAD_CONN_COUNT connection(s) missing required fields (from/to/confidence)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
INVALID_CONN_CONF=$(jq -r '.connections[]? | (.confidence // "MISSING")' "$JSON_FILE" 2>/dev/null | \
|
||||
grep -v -E "^($VALID_CONFIDENCE)$" | head -3)
|
||||
if [ -n "$INVALID_CONN_CONF" ]; then
|
||||
echo "ERROR: invalid connection confidence value(s): $INVALID_CONN_CONF"
|
||||
echo " Valid values: EXTRACTED | INFERRED | AMBIGUOUS | UNVERIFIED"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# EXTRACTED 和 INFERRED connections 必须提供 evidence
|
||||
NO_CONN_EVIDENCE=$(jq '
|
||||
[.connections[] | select(
|
||||
(.confidence == "EXTRACTED" or .confidence == "INFERRED") and
|
||||
((.evidence // "" | length) == 0)
|
||||
)] | length
|
||||
' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$NO_CONN_EVIDENCE" -gt 0 ] 2>/dev/null; then
|
||||
echo "WARN: $NO_CONN_EVIDENCE connection(s) with EXTRACTED/INFERRED confidence missing 'evidence' field"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "OK: Step 1 JSON validation passed"
|
||||
exit 0
|
||||
Executable
+267
@@ -0,0 +1,267 @@
|
||||
#!/bin/bash
|
||||
# 旧知识库兼容脚本：惰性默认、目录检查、按需创建
|
||||
# 原则：migration_required=no，只有确实无法兼容时才引入显式迁移
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SOURCE_REGISTRY_SCRIPT="$SCRIPT_DIR/source-registry.sh"
|
||||
|
||||
LEGACY_REQUIRED_RAW_DIRS=(
|
||||
"raw/articles"
|
||||
"raw/tweets"
|
||||
"raw/wechat"
|
||||
"raw/pdfs"
|
||||
"raw/notes"
|
||||
"raw/assets"
|
||||
)
|
||||
|
||||
REQUIRED_PATHS=(
|
||||
".wiki-schema.md"
|
||||
"index.md"
|
||||
"log.md"
|
||||
"raw"
|
||||
"wiki"
|
||||
"wiki/entities"
|
||||
"wiki/topics"
|
||||
"wiki/sources"
|
||||
"wiki/comparisons"
|
||||
"wiki/synthesis"
|
||||
"wiki/overview.md"
|
||||
)
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/wiki-compat.sh inspect <wiki_root>
|
||||
bash scripts/wiki-compat.sh validate <wiki_root>
|
||||
bash scripts/wiki-compat.sh ensure-source-dir <wiki_root> <source_id>
|
||||
EOF
|
||||
}
|
||||
|
||||
trim() {
|
||||
printf '%s' "$1" | awk '{ gsub(/^[[:space:]]+|[[:space:]]+$/, "", $0); printf "%s", $0 }'
|
||||
}
|
||||
|
||||
require_wiki_root() {
|
||||
local wiki_root="$1"
|
||||
|
||||
[ -n "$wiki_root" ] || {
|
||||
usage
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -d "$wiki_root" ] || {
|
||||
echo "知识库不存在：$wiki_root" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
schema_field_value() {
|
||||
local wiki_root="$1"
|
||||
local field_name="$2"
|
||||
local default_value="$3"
|
||||
local schema_path value
|
||||
|
||||
schema_path="$wiki_root/.wiki-schema.md"
|
||||
|
||||
if [ ! -f "$schema_path" ]; then
|
||||
printf '%s\n' "$default_value"
|
||||
return 0
|
||||
fi
|
||||
|
||||
value="$(
|
||||
awk -v field_name="$field_name" '
|
||||
$0 ~ "^-[[:space:]]*" field_name "[：:]" {
|
||||
line = $0
|
||||
sub("^-[[:space:]]*" field_name "[：:][[:space:]]*", "", line)
|
||||
print line
|
||||
exit
|
||||
}
|
||||
' "$schema_path"
|
||||
)"
|
||||
|
||||
value="$(trim "$value")"
|
||||
|
||||
if [ -n "$value" ]; then
|
||||
printf '%s\n' "$value"
|
||||
else
|
||||
printf '%s\n' "$default_value"
|
||||
fi
|
||||
}
|
||||
|
||||
resolved_language() {
|
||||
local wiki_root="$1"
|
||||
local raw_value
|
||||
|
||||
raw_value="$(schema_field_value "$wiki_root" "语言" "")"
|
||||
|
||||
case "$raw_value" in
|
||||
English|english|EN|en)
|
||||
printf 'en\n'
|
||||
;;
|
||||
*)
|
||||
printf 'zh\n'
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
resolved_schema_version() {
|
||||
local wiki_root="$1"
|
||||
|
||||
schema_field_value "$wiki_root" "版本" "1.0"
|
||||
}
|
||||
|
||||
is_legacy_required_raw_dir() {
|
||||
case "$1" in
|
||||
raw/articles|raw/tweets|raw/wechat|raw/pdfs|raw/notes|raw/assets)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
missing_optional_raw_dirs() {
|
||||
local wiki_root="$1"
|
||||
local raw_dir
|
||||
local missing=()
|
||||
|
||||
while IFS= read -r raw_dir; do
|
||||
[ -n "$raw_dir" ] || continue
|
||||
|
||||
if is_legacy_required_raw_dir "$raw_dir"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if [ ! -d "$wiki_root/$raw_dir" ]; then
|
||||
missing+=("$raw_dir")
|
||||
fi
|
||||
done < <(
|
||||
bash "$SOURCE_REGISTRY_SCRIPT" list | awk -F '\t' 'NR > 1 { print $6 }' | LC_ALL=C sort -u
|
||||
)
|
||||
|
||||
if [ "${#missing[@]}" -eq 0 ]; then
|
||||
printf '%s\n' '-'
|
||||
else
|
||||
local IFS=,
|
||||
printf '%s\n' "${missing[*]}"
|
||||
fi
|
||||
}
|
||||
|
||||
file_presence() {
|
||||
local wiki_root="$1"
|
||||
local relative_path="$2"
|
||||
|
||||
if [ -e "$wiki_root/$relative_path" ]; then
|
||||
printf 'present\n'
|
||||
else
|
||||
printf 'missing\n'
|
||||
fi
|
||||
}
|
||||
|
||||
validate_layout() {
|
||||
local wiki_root="$1"
|
||||
local failed=0
|
||||
local path
|
||||
|
||||
require_wiki_root "$wiki_root"
|
||||
|
||||
for path in "${REQUIRED_PATHS[@]}"; do
|
||||
if [ ! -e "$wiki_root/$path" ]; then
|
||||
echo "缺少必要路径：$path" >&2
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
for path in "${LEGACY_REQUIRED_RAW_DIRS[@]}"; do
|
||||
if [ ! -d "$wiki_root/$path" ]; then
|
||||
echo "缺少必要旧目录：$path" >&2
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$failed" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
source_raw_dir() {
|
||||
local source_id="$1"
|
||||
local record raw_dir
|
||||
|
||||
record="$(
|
||||
bash "$SOURCE_REGISTRY_SCRIPT" get "$source_id" 2>/dev/null
|
||||
)" || {
|
||||
echo "未知来源：$source_id" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
IFS=$'\t' read -r _ _ _ _ _ raw_dir _ _ _ _ <<EOF
|
||||
$record
|
||||
EOF
|
||||
|
||||
printf '%s\n' "$raw_dir"
|
||||
}
|
||||
|
||||
print_inspect() {
|
||||
local wiki_root="$1"
|
||||
local schema_version language optional_dirs legacy_mode purpose_file cache_file
|
||||
|
||||
validate_layout "$wiki_root"
|
||||
|
||||
schema_version="$(resolved_schema_version "$wiki_root")"
|
||||
language="$(resolved_language "$wiki_root")"
|
||||
optional_dirs="$(missing_optional_raw_dirs "$wiki_root")"
|
||||
purpose_file="$(file_presence "$wiki_root" "purpose.md")"
|
||||
cache_file="$(file_presence "$wiki_root" ".wiki-cache.json")"
|
||||
|
||||
if [ "$schema_version" = "1.0" ] || [ "$optional_dirs" != "-" ] || [ "$purpose_file" = "missing" ] || [ "$cache_file" = "missing" ]; then
|
||||
legacy_mode="yes"
|
||||
else
|
||||
legacy_mode="no"
|
||||
fi
|
||||
|
||||
printf 'wiki_root=%s\n' "$wiki_root"
|
||||
printf 'schema_version=%s\n' "$schema_version"
|
||||
printf 'language=%s\n' "$language"
|
||||
printf 'legacy_mode=%s\n' "$legacy_mode"
|
||||
printf 'migration_required=no\n'
|
||||
printf 'missing_optional_raw_dirs=%s\n' "$optional_dirs"
|
||||
printf 'purpose_file=%s\n' "$purpose_file"
|
||||
printf 'cache_file=%s\n' "$cache_file"
|
||||
}
|
||||
|
||||
ensure_source_dir() {
|
||||
local wiki_root="$1"
|
||||
local source_id="$2"
|
||||
local raw_dir
|
||||
|
||||
validate_layout "$wiki_root"
|
||||
raw_dir="$(source_raw_dir "$source_id")"
|
||||
|
||||
mkdir -p "$wiki_root/$raw_dir"
|
||||
printf '%s\n' "$wiki_root/$raw_dir"
|
||||
}
|
||||
|
||||
command_name="${1:-}"
|
||||
|
||||
case "$command_name" in
|
||||
inspect)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
print_inspect "$2"
|
||||
;;
|
||||
validate)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
print_inspect "$2" > /dev/null
|
||||
;;
|
||||
ensure-source-dir)
|
||||
[ "$#" -eq 3 ] || { usage; exit 1; }
|
||||
ensure_source_dir "$2" "$3"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user