#!/bin/bash # 统一来源总表读取与验证脚本 # 权威数据文件:source-registry.tsv(来源定义)、source-record-contract.tsv(字段契约) set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" CONTRACT_FILE="$SCRIPT_DIR/source-record-contract.tsv" REGISTRY_FILE="$SCRIPT_DIR/source-registry.tsv" usage() { cat <<'EOF' 用法: bash scripts/source-registry.sh fields bash scripts/source-registry.sh list bash scripts/source-registry.sh get bash scripts/source-registry.sh match-url bash scripts/source-registry.sh match-file bash scripts/source-registry.sh list-by-category bash scripts/source-registry.sh unique-dependencies bash scripts/source-registry.sh validate EOF } require_file() { local file="$1" [ -f "$file" ] || { echo "缺少文件:$file" >&2 exit 1 } } expect_header() { local file="$1" local expected="$2" local actual actual="$(head -n 1 "$file")" [ "$actual" = "$expected" ] || { echo "表头不匹配:$file" >&2 echo "期望:$expected" >&2 echo "实际:$actual" >&2 exit 1 } } validate_contract() { require_file "$CONTRACT_FILE" expect_header "$CONTRACT_FILE" $'field_name\trequiredness\tfilled_by\tvalue_rule' awk -F '\t' ' BEGIN { required["source_id"] = 1 required["source_label"] = 1 required["source_category"] = 1 required["input_mode"] = 1 required["raw_dir"] = 1 required["original_ref"] = 1 required["ingest_text"] = 1 required["adapter_name"] = 1 required["fallback_hint"] = 1 } NR == 1 { next } { if ($1 == "" || $2 == "" || $3 == "" || $4 == "") { printf("source-record-contract.tsv 第 %d 行存在空字段\n", NR) > "/dev/stderr" failed = 1 } seen[$1] += 1 } END { for (field in required) { if (seen[field] != 1) { printf("source-record-contract.tsv 缺少或重复字段:%s\n", field) > "/dev/stderr" failed = 1 } } exit failed ? 1 : 0 } ' "$CONTRACT_FILE" } validate_registry() { require_file "$REGISTRY_FILE" expect_header "$REGISTRY_FILE" $'source_id\tsource_label\tsource_category\tinput_mode\tmatch_rule\traw_dir\tadapter_name\tdependency_name\tdependency_type\tfallback_hint' awk -F '\t' ' NR == 1 { next } { if ($1 == "" || $2 == "" || $3 == "" || $4 == "" || $5 == "" || $6 == "" || $10 == "") { printf("source-registry.tsv 第 %d 行存在空字段\n", NR) > "/dev/stderr" failed = 1 } if ($3 != "core_builtin" && $3 != "optional_adapter" && $3 != "manual_only") { printf("source-registry.tsv 第 %d 行存在未知分类:%s\n", NR, $3) > "/dev/stderr" failed = 1 } if ($4 != "url" && $4 != "file" && $4 != "text" && $4 != "asset") { printf("source-registry.tsv 第 %d 行存在未知输入模式:%s\n", NR, $4) > "/dev/stderr" failed = 1 } if ($4 == "url" && $5 !~ /^url_host:/) { printf("source-registry.tsv 第 %d 行 URL 来源必须声明 url_host 规则:%s\n", NR, $5) > "/dev/stderr" failed = 1 } if ($4 == "file" && $5 !~ /^file_ext:/) { printf("source-registry.tsv 第 %d 行文件来源必须声明 file_ext 规则:%s\n", NR, $5) > "/dev/stderr" failed = 1 } if ($4 == "text" && $5 !~ /^text:/) { printf("source-registry.tsv 第 %d 行文本来源必须声明 text 规则:%s\n", NR, $5) > "/dev/stderr" failed = 1 } if ($4 == "asset" && $5 !~ /^asset:/) { printf("source-registry.tsv 第 %d 行附件来源必须声明 asset 规则:%s\n", NR, $5) > "/dev/stderr" failed = 1 } if ($6 !~ /^raw\//) { printf("source-registry.tsv 第 %d 行 raw_dir 必须位于 raw/ 下:%s\n", NR, $6) > "/dev/stderr" failed = 1 } if (seen[$1]++) { printf("source-registry.tsv source_id 重复:%s\n", $1) > "/dev/stderr" failed = 1 } category_seen[$3] = 1 if ($3 == "optional_adapter") { if ($7 == "-" || $8 == "-" || $9 == "none") { printf("source-registry.tsv 第 %d 行 optional_adapter 缺少依赖信息\n", NR) > "/dev/stderr" failed = 1 } } else if ($7 != "-" || $8 != "-" || $9 != "none") { printf("source-registry.tsv 第 %d 行非外挂来源不应声明依赖\n", NR) > "/dev/stderr" failed = 1 } } END { if (!category_seen["core_builtin"]) { print "source-registry.tsv 缺少 core_builtin 来源" > "/dev/stderr" failed = 1 } if (!category_seen["optional_adapter"]) { print "source-registry.tsv 缺少 optional_adapter 来源" > "/dev/stderr" failed = 1 } if (!category_seen["manual_only"]) { print "source-registry.tsv 缺少 manual_only 来源" > "/dev/stderr" failed = 1 } exit failed ? 1 : 0 } ' "$REGISTRY_FILE" } print_contract() { validate_contract cat "$CONTRACT_FILE" } print_registry() { validate_registry cat "$REGISTRY_FILE" } get_source() { local source_id="$1" validate_registry awk -F '\t' -v source_id="$source_id" ' NR == 1 { next } $1 == source_id { print found = 1 } END { exit found ? 0 : 1 } ' "$REGISTRY_FILE" } extract_url_host() { local url="$1" local rest host rest="${url#*://}" if [ "$rest" = "$url" ]; then rest="$url" fi rest="${rest#*@}" host="${rest%%/*}" host="${host%%\?*}" host="${host%%#*}" host="${host%%:*}" printf '%s\n' "$host" | tr '[:upper:]' '[:lower:]' } host_matches_pattern() { local host="$1" local pattern="$2" case "$host" in "$pattern"|*."$pattern") return 0 ;; *) return 1 ;; esac } match_url() { local url="$1" local host row source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint local fallback_row="" local pattern pattern_list validate_registry host="$(extract_url_host "$url")" while IFS=$'\t' read -r source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint; do [ "$source_id" = "source_id" ] && continue [ "$input_mode" = "url" ] || continue pattern_list="${match_rule#url_host:}" if [ "$pattern_list" = "*" ]; then fallback_row="$(printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$source_id" "$source_label" "$source_category" "$input_mode" "$match_rule" "$raw_dir" "$adapter_name" "$dependency_name" "$dependency_type" "$fallback_hint")" continue fi for pattern in ${pattern_list//,/ }; do if host_matches_pattern "$host" "$pattern"; then printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$source_id" "$source_label" "$source_category" "$input_mode" "$match_rule" "$raw_dir" "$adapter_name" "$dependency_name" "$dependency_type" "$fallback_hint" return 0 fi done done < "$REGISTRY_FILE" [ -n "$fallback_row" ] || return 1 printf '%s\n' "$fallback_row" } match_file() { local path="$1" local lowered_path source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint local extension_list extension validate_registry lowered_path="$(printf '%s\n' "$path" | tr '[:upper:]' '[:lower:]')" while IFS=$'\t' read -r source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint; do [ "$source_id" = "source_id" ] && continue [ "$input_mode" = "file" ] || continue extension_list="${match_rule#file_ext:}" for extension in ${extension_list//,/ }; do case "$lowered_path" in *"$extension") printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$source_id" "$source_label" "$source_category" "$input_mode" "$match_rule" "$raw_dir" "$adapter_name" "$dependency_name" "$dependency_type" "$fallback_hint" return 0 ;; esac done done < "$REGISTRY_FILE" return 1 } list_by_category() { local category="$1" validate_registry awk -F '\t' -v category="$category" ' NR == 1 { next } $3 == category { print } ' "$REGISTRY_FILE" } list_unique_dependencies() { local dependency_type="$1" validate_registry awk -F '\t' -v dependency_type="$dependency_type" ' NR == 1 { next } $9 == dependency_type && $8 != "-" { print $8 } ' "$REGISTRY_FILE" | sort -u } command_name="${1:-}" case "$command_name" in fields) [ "$#" -eq 1 ] || { usage; exit 1; } print_contract ;; list) [ "$#" -eq 1 ] || { usage; exit 1; } print_registry ;; get) [ "$#" -eq 2 ] || { usage; exit 1; } get_source "$2" ;; match-url) [ "$#" -eq 2 ] || { usage; exit 1; } match_url "$2" ;; match-file) [ "$#" -eq 2 ] || { usage; exit 1; } match_file "$2" ;; list-by-category) [ "$#" -eq 2 ] || { usage; exit 1; } list_by_category "$2" ;; unique-dependencies) [ "$#" -eq 2 ] || { usage; exit 1; } list_unique_dependencies "$2" ;; validate) [ "$#" -eq 1 ] || { usage; exit 1; } validate_contract validate_registry ;; *) usage exit 1 ;; esac