feat: 新增批量数据处理脚本并更新考勤脚本配置
- 更新考勤jupyter notebook的登录账号、密码与请求方式 - 新增Dify API调用脚本test.py用于测试对话请求 - 新增database.py实现批量格式化处理教学计划数据并生成更新SQL
This commit is contained in:
+135
@@ -0,0 +1,135 @@
|
||||
import json
|
||||
import time
|
||||
import requests
|
||||
from faker import Faker
|
||||
fake = Faker('zh_CN')
|
||||
import random
|
||||
|
||||
# Dify API 配置
|
||||
DIFY_API_KEY = "app-T0kpinzer52gZDGCtipsShVm"
|
||||
DIFY_API_URL = "http://net2.hxzhxy.cn:5093/v1/chat-messages"
|
||||
DIFY_USER = "xtlft@qq.com"
|
||||
|
||||
|
||||
def call_dify_format_data(data: dict) -> dict:
|
||||
"""
|
||||
调用 Dify 接口,传入数据,让 AI 格式化输出
|
||||
返回包含 subject 和 available 字段的字典
|
||||
"""
|
||||
payload = {
|
||||
"inputs": {},
|
||||
"query": json.dumps(data, ensure_ascii=False),
|
||||
"response_mode": "blocking",
|
||||
"conversation_id": "",
|
||||
"user": DIFY_USER,
|
||||
"files": []
|
||||
}
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {DIFY_API_KEY}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
response = requests.post(
|
||||
url=DIFY_API_URL,
|
||||
headers=headers,
|
||||
json=payload,
|
||||
timeout=300
|
||||
)
|
||||
response.raise_for_status()
|
||||
time.sleep(3)
|
||||
return response.json()
|
||||
|
||||
|
||||
def parse_dify_response(dify_resp: dict) -> dict:
|
||||
"""
|
||||
解析 Dify 返回的 JSON,提取 id, subject, available
|
||||
"""
|
||||
answer_str = dify_resp.get("answer", "{}")
|
||||
# 兼容 Dify 返回的字符串里有 ```json ... ``` 包裹
|
||||
answer_str = answer_str.strip()
|
||||
if answer_str.startswith("```"):
|
||||
answer_str = answer_str.strip("`")
|
||||
if answer_str.startswith("json"):
|
||||
answer_str = answer_str[4:]
|
||||
answer_str = answer_str.strip()
|
||||
|
||||
parsed = json.loads(answer_str)
|
||||
return {
|
||||
"id": parsed.get("id"),
|
||||
"subject": parsed.get("subject"),
|
||||
"available": parsed.get("available", [])
|
||||
}
|
||||
|
||||
|
||||
def build_update_sql(subject: str, available: list, record_id) -> str:
|
||||
"""
|
||||
生成 update temp_schoolwork_plan 的 SQL 语句
|
||||
"""
|
||||
available_json = json.dumps(available, ensure_ascii=False)
|
||||
# 用 mysql 的引号转义
|
||||
subject_escaped = subject.replace("'", "''") if subject else ""
|
||||
available_escaped = available_json.replace("'", "''")
|
||||
sql = f"UPDATE temp_schoolwork_plan SET subject='{subject_escaped}', available='{available_escaped}' WHERE id={record_id};"
|
||||
return sql
|
||||
|
||||
|
||||
def process_records(records: list, output_file: str = "update_statements.sql", limit: int = None):
|
||||
"""
|
||||
批量处理 records,调用 Dify 格式化,输出所有 update 语句到文件
|
||||
边处理边写入,避免全部跑完才发现问题
|
||||
"""
|
||||
if limit:
|
||||
records = records[:limit]
|
||||
|
||||
total = len(records)
|
||||
success_count = 0
|
||||
|
||||
# 先清空文件(如果存在)
|
||||
with open(output_file, "w", encoding="utf-8") as f:
|
||||
f.write("-- ===== temp_schoolwork_plan 更新语句 =====\n")
|
||||
|
||||
for idx, record in enumerate(records, 1):
|
||||
record_id = record.get("id")
|
||||
batch_name = record.get("batch_name", "")
|
||||
print(f"\n[{idx}/{total}] 开始处理 ID={record_id} batch_name={batch_name}")
|
||||
try:
|
||||
# 1. 调用 Dify 格式化
|
||||
print(f" → 调用 Dify 接口...")
|
||||
dify_resp = call_dify_format_data(record)
|
||||
|
||||
parsed = parse_dify_response(dify_resp)
|
||||
print(f" → 解析结果: id={parsed['id']}, subject={parsed['subject']}, available={parsed['available']}")
|
||||
|
||||
subject = parsed["subject"]
|
||||
available = parsed["available"]
|
||||
record_id = parsed["id"] or record.get("id")
|
||||
|
||||
# 有效性校验:subject 或 available 任一为空/空数组都视为无效
|
||||
if not subject or not available or len(available) == 0:
|
||||
print(f" ✗ ID={record_id} 无效: subject={subject!r}, available={available!r},跳过")
|
||||
continue
|
||||
|
||||
# 2. 生成 update 语句并立即写入文件
|
||||
sql = build_update_sql(subject, available, record_id)
|
||||
with open(output_file, "a", encoding="utf-8") as f:
|
||||
f.write(sql + "\n")
|
||||
success_count += 1
|
||||
print(f" ✓ 写入SQL: {sql}")
|
||||
except Exception as e:
|
||||
print(f" ✗ ID={record_id} 处理失败: {e}")
|
||||
continue
|
||||
|
||||
print(f"\n全部处理完成,共生成 {success_count} 条 update 语句")
|
||||
print(f"输出文件: {output_file}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 加载本地 table.json
|
||||
with open("table.json", "r", encoding="utf-8") as f:
|
||||
records = json.load(f)
|
||||
|
||||
print(f"加载到 {len(records)} 条记录")
|
||||
|
||||
# 批量处理,先只跑3条测试
|
||||
process_records(records, output_file="update_statements.sql", limit=3)
|
||||
Reference in New Issue
Block a user