Rain-Bus
21f16e6756
- Split summarizer into summary_generator and summary_persister modules - Refactor pdf_image_extractor to two-phase pipeline with PicoDet layout detection - Add layout_detector service for PicoDet-S_layout_3cls integration - Add exceptions module with ConflictError and NotFoundError - Improve admin dashboard with better statistics and task management - Add design review document with system optimization suggestions - Add new tests for crawler, pdf_downloader, pipeline, and summary_utils - Update dependencies and configuration - Clean up dead code and improve error handling
641 lines
20 KiB
Python
641 lines
20 KiB
Python
"""管理接口 — 仪表盘、抓取、总结、清理、删除、日志，需要登录鉴权。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import hmac
|
||
import json
|
||
import logging
|
||
from datetime import date
|
||
|
||
from fastapi import APIRouter, Depends, Form, HTTPException, Query, Request
|
||
from fastapi.responses import RedirectResponse
|
||
from pydantic import BaseModel, field_validator
|
||
from sqlalchemy import bindparam, func, select, text
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.config import settings
|
||
from app.database import get_db
|
||
from app.models import (
|
||
CrawlLog,
|
||
DataDeleteJob,
|
||
Paper,
|
||
PaperTag,
|
||
SummaryState,
|
||
SummaryStatus,
|
||
)
|
||
from app.services import admin as admin_svc
|
||
from app.services.admin import get_admin_stats
|
||
from app.services.cleaner import cleanup_tmp, delete_papers_by_date_range
|
||
from app.services.crawler import refresh_upvotes
|
||
from app.services.pipeline import run_crawl, run_pipeline
|
||
from app.services.scheduler import get_scheduler
|
||
from app.services.summarizer import summarize_batch, summarize_single
|
||
from app.utils import templates, today_str, utc_now
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
router = APIRouter(prefix="/admin", tags=["admin"])
|
||
|
||
|
||
# ── 认证 ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _check_password(password: str) -> bool:
|
||
"""校验密码，支持明文或 sha256 哈希（常量时间比较）。"""
|
||
stored = settings.ADMIN_PASSWORD
|
||
if not stored:
|
||
return False
|
||
if hmac.compare_digest(password, stored):
|
||
return True
|
||
# 也支持存 sha256 哈希
|
||
hashed = hashlib.sha256(password.encode()).hexdigest()
|
||
return hmac.compare_digest(hashed, stored)
|
||
|
||
|
||
async def verify_admin(request: Request) -> None:
|
||
"""检查 session 中的登录状态，未登录则重定向到登录页。"""
|
||
if not request.session.get("is_admin"):
|
||
raise HTTPException(status_code=303, headers={"Location": "/admin/login"})
|
||
|
||
|
||
# ── 登录 / 登出 ──────────────────────────────────────────────────────
|
||
|
||
|
||
@router.get("/login")
|
||
async def admin_login_page(request: Request):
|
||
"""显示登录页面。已登录则直接跳转管理页。"""
|
||
if request.session.get("is_admin"):
|
||
return RedirectResponse("/admin/", status_code=303)
|
||
return templates.TemplateResponse(request, "login.html", {"error": None})
|
||
|
||
|
||
@router.post("/login")
|
||
async def admin_login_submit(
|
||
request: Request,
|
||
username: str = Form(""),
|
||
password: str = Form(""),
|
||
):
|
||
"""处理登录表单提交。"""
|
||
if username == settings.ADMIN_USERNAME and _check_password(password):
|
||
request.session["is_admin"] = True
|
||
return RedirectResponse("/admin/", status_code=303)
|
||
return templates.TemplateResponse(
|
||
request, "login.html", {"error": "用户名或密码错误"}
|
||
)
|
||
|
||
|
||
@router.post("/logout")
|
||
async def admin_logout(request: Request):
|
||
"""退出登录，清除 session。"""
|
||
request.session.clear()
|
||
return RedirectResponse("/admin/login", status_code=303)
|
||
|
||
|
||
# ── 仪表盘 ──────────────────────────────────────────────────────────
|
||
|
||
|
||
@router.get("/")
|
||
async def admin_dashboard(
|
||
request: Request,
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
):
|
||
"""管理仪表盘 — 系统状态总览。"""
|
||
stats = get_admin_stats(db)
|
||
|
||
# 调度器历史（最近 10 条 task=scheduler 日志）
|
||
scheduler_history = (
|
||
db.execute(
|
||
select(CrawlLog)
|
||
.where(CrawlLog.task == "scheduler")
|
||
.order_by(CrawlLog.started_at.desc())
|
||
.limit(10)
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
|
||
return templates.TemplateResponse(
|
||
request,
|
||
"admin_dashboard.html",
|
||
{"stats": stats, "scheduler_history": scheduler_history},
|
||
)
|
||
|
||
|
||
# ── 调度器 ──────────────────────────────────────────────────────────
|
||
|
||
|
||
@router.get("/scheduler-status")
|
||
async def admin_scheduler_status(_admin: None = Depends(verify_admin)):
|
||
"""调度器运行状态（JSON）。"""
|
||
scheduler = get_scheduler()
|
||
next_run = None
|
||
upvote_next_run = None
|
||
if scheduler:
|
||
for job in scheduler.get_jobs():
|
||
if job.id == "daily_pipeline":
|
||
next_run = job.next_run_time
|
||
elif job.id == "upvote_refresh":
|
||
upvote_next_run = job.next_run_time
|
||
return {
|
||
"enabled": scheduler is not None,
|
||
"schedule_time": f"{settings.SCHEDULE_HOUR:02d}:{settings.SCHEDULE_MINUTE:02d}",
|
||
"timezone": settings.APP_TIMEZONE,
|
||
"next_run": next_run.isoformat() if next_run else None,
|
||
"upvote_next_run": upvote_next_run.isoformat() if upvote_next_run else None,
|
||
"upvote_refresh_days": settings.UPVOTE_REFRESH_DAYS,
|
||
}
|
||
|
||
|
||
@router.post("/trigger-pipeline")
|
||
async def admin_trigger_pipeline(
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
):
|
||
"""手动触发一次完整流水线（crawl → summarize → cleanup）。"""
|
||
today = today_str()
|
||
try:
|
||
result = await run_pipeline(db, today, owner="admin_trigger")
|
||
except RuntimeError as exc:
|
||
raise HTTPException(status_code=409, detail=str(exc))
|
||
|
||
if result["status"] == "failed":
|
||
raise HTTPException(status_code=500, detail=result.get("error"))
|
||
return {"status": "success", "message": "流水线执行完成"}
|
||
|
||
|
||
@router.post("/refresh-upvotes")
|
||
async def admin_refresh_upvotes(
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
days: int | None = Query(None, description="刷新最近 N 天，默认使用配置值"),
|
||
):
|
||
"""手动刷新最近 N 天论文的 upvotes。"""
|
||
result = await refresh_upvotes(db, days=days)
|
||
if result["status"] == "failed":
|
||
raise HTTPException(status_code=500, detail=result.get("error"))
|
||
return result
|
||
|
||
|
||
# ── 请求模型 ──────────────────────────────────────────────────────────
|
||
|
||
|
||
class DeleteRequest(BaseModel):
|
||
date_start: date
|
||
date_end: date
|
||
include_notes: bool = True
|
||
confirm: str
|
||
|
||
@field_validator("confirm")
|
||
@classmethod
|
||
def confirm_must_be_delete(cls, v: str) -> str:
|
||
if v != "DELETE":
|
||
raise ValueError("confirm must be 'DELETE' to proceed")
|
||
return v
|
||
|
||
|
||
# ── 抓取 ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
@router.post("/crawl")
|
||
async def admin_crawl(
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
date: str | None = Query(None, description="YYYY-MM-DD，默认今天"),
|
||
):
|
||
"""手动抓取指定日期，默认今天。"""
|
||
target_date = date or today_str()
|
||
try:
|
||
return await run_crawl(db, target_date, owner="admin_crawl")
|
||
except RuntimeError as exc:
|
||
raise HTTPException(status_code=409, detail=str(exc))
|
||
except Exception as exc:
|
||
raise HTTPException(status_code=500, detail=str(exc))
|
||
|
||
|
||
# ── 总结 ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
@router.post("/summarize")
|
||
async def admin_summarize_batch(
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
):
|
||
"""批量总结所有 pending 论文。"""
|
||
return await summarize_batch(db, pdf_mode=settings.SUMMARY_PDF_MODE)
|
||
|
||
|
||
@router.post("/summarize/{arxiv_id}")
|
||
async def admin_summarize_single(
|
||
arxiv_id: str,
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
):
|
||
"""总结或重跑单篇论文。"""
|
||
return await summarize_single(
|
||
db, arxiv_id, force=True, pdf_mode=settings.SUMMARY_PDF_MODE
|
||
)
|
||
|
||
|
||
# ── 清理 ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
@router.post("/cleanup")
|
||
async def admin_cleanup(
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
):
|
||
"""清理 data/tmp/ 中超过 24 小时的临时文件。"""
|
||
now = utc_now()
|
||
log_entry = CrawlLog(
|
||
task="cleanup",
|
||
status="running",
|
||
started_at=now,
|
||
)
|
||
db.add(log_entry)
|
||
db.commit()
|
||
|
||
try:
|
||
result = cleanup_tmp()
|
||
log_entry.status = "success"
|
||
log_entry.completed_at = utc_now()
|
||
log_entry.details_json = json.dumps(
|
||
{
|
||
"scanned": result.get("scanned", 0),
|
||
"removed": result.get("removed", 0),
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
if result.get("errors"):
|
||
log_entry.error = "; ".join(result["errors"])[:2000]
|
||
db.commit()
|
||
return result
|
||
except Exception as exc:
|
||
log_entry.status = "failed"
|
||
log_entry.error = str(exc)[:2000]
|
||
log_entry.completed_at = utc_now()
|
||
db.commit()
|
||
raise HTTPException(status_code=500, detail=str(exc))
|
||
|
||
|
||
# ── 删除 ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
@router.post("/delete")
|
||
async def admin_delete(
|
||
body: DeleteRequest,
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
):
|
||
"""删除指定日期范围内的论文（需要 confirm='DELETE' 二次确认）。"""
|
||
if body.date_start > body.date_end:
|
||
raise HTTPException(status_code=400, detail="date_start must be <= date_end")
|
||
|
||
result = await delete_papers_by_date_range(
|
||
db,
|
||
body.date_start,
|
||
body.date_end,
|
||
include_notes=body.include_notes,
|
||
)
|
||
return result
|
||
|
||
|
||
# ── 日志 ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
@router.get("/logs")
|
||
async def admin_logs(
|
||
request: Request,
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
page: int = Query(1, ge=1),
|
||
per_page: int = Query(20, ge=1, le=100),
|
||
):
|
||
"""查看任务日志（CrawlLog + DataDeleteJob）+ 总结状态统计。"""
|
||
crawl_logs = (
|
||
db.execute(
|
||
select(CrawlLog)
|
||
.order_by(CrawlLog.started_at.desc())
|
||
.limit(per_page)
|
||
.offset((page - 1) * per_page)
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
|
||
delete_jobs = (
|
||
db.execute(
|
||
select(DataDeleteJob)
|
||
.order_by(DataDeleteJob.started_at.desc())
|
||
.limit(per_page)
|
||
.offset((page - 1) * per_page)
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
|
||
# 总结状态统计概要
|
||
summary_total = db.scalar(select(func.count(Paper.id))) or 0
|
||
summary_done = (
|
||
db.scalar(
|
||
select(func.count(SummaryStatus.id)).where(
|
||
SummaryStatus.status == SummaryState.DONE
|
||
)
|
||
)
|
||
or 0
|
||
)
|
||
summary_pending = (
|
||
db.scalar(
|
||
select(func.count(SummaryStatus.id)).where(
|
||
SummaryStatus.status.in_(
|
||
[SummaryState.PENDING, SummaryState.PROCESSING]
|
||
)
|
||
)
|
||
)
|
||
or 0
|
||
)
|
||
summary_failed = (
|
||
db.scalar(
|
||
select(func.count(SummaryStatus.id)).where(
|
||
SummaryStatus.status.in_(
|
||
[SummaryState.FAILED, SummaryState.PERMANENT_FAILURE]
|
||
)
|
||
)
|
||
)
|
||
or 0
|
||
)
|
||
|
||
return templates.TemplateResponse(
|
||
request,
|
||
"admin_logs.html",
|
||
{
|
||
"crawl_logs": crawl_logs,
|
||
"delete_jobs": delete_jobs,
|
||
"page": page,
|
||
"per_page": per_page,
|
||
"summary_total": summary_total,
|
||
"summary_done": summary_done,
|
||
"summary_pending": summary_pending,
|
||
"summary_failed": summary_failed,
|
||
},
|
||
)
|
||
|
||
|
||
# ── 总结状态管理 ────────────────────────────────────────────────────
|
||
|
||
|
||
@router.get("/summary-status")
|
||
async def admin_summary_status(
|
||
request: Request,
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
status: str = Query("all"),
|
||
page: int = Query(1, ge=1),
|
||
per_page: int = Query(20, ge=1, le=100),
|
||
):
|
||
"""总结状态列表（HTMX 片段或 JSON）。"""
|
||
|
||
query = (
|
||
select(Paper, SummaryStatus)
|
||
.outerjoin(SummaryStatus, SummaryStatus.paper_id == Paper.id)
|
||
.order_by(Paper.paper_date.desc())
|
||
)
|
||
|
||
if status != "all":
|
||
if status == "none":
|
||
query = query.where(SummaryStatus.paper_id == None) # noqa: E711
|
||
else:
|
||
query = query.where(SummaryStatus.status == status)
|
||
|
||
total = db.scalar(select(func.count()).select_from(query.subquery()))
|
||
results = db.execute(query.offset((page - 1) * per_page).limit(per_page)).all()
|
||
|
||
# 判断是否 HTMX 请求
|
||
is_htmx = request.headers.get("HX-Request") == "true"
|
||
|
||
if is_htmx:
|
||
# 返回 HTML 片段
|
||
return templates.TemplateResponse(
|
||
request,
|
||
"partials/summary_list.html",
|
||
{
|
||
"results": results,
|
||
"total": total or 0,
|
||
"page": page,
|
||
"per_page": per_page,
|
||
"current_status": status,
|
||
},
|
||
)
|
||
|
||
# 非 HTMX 返回 JSON
|
||
items = []
|
||
for paper, ss in results:
|
||
item = {
|
||
"arxiv_id": paper.arxiv_id,
|
||
"title": paper.title_zh or paper.title_en,
|
||
"paper_date": str(paper.paper_date),
|
||
"summary_status": ss.status if ss else "none",
|
||
"retry_count": ss.retry_count if ss else 0,
|
||
"error_type": ss.error_type if ss else None,
|
||
"error": ss.error if ss else None,
|
||
}
|
||
items.append(item)
|
||
return {"items": items, "total": total or 0, "page": page, "per_page": per_page}
|
||
|
||
|
||
@router.post("/summary-retry-failed")
|
||
async def admin_summary_retry_failed(
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
):
|
||
"""重试所有失败状态的总结任务。"""
|
||
failed_ids = (
|
||
db.execute(
|
||
select(Paper.arxiv_id)
|
||
.join(SummaryStatus, SummaryStatus.paper_id == Paper.id)
|
||
.where(
|
||
SummaryStatus.status.in_(
|
||
[SummaryState.FAILED, SummaryState.PERMANENT_FAILURE]
|
||
)
|
||
)
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
|
||
if not failed_ids:
|
||
return {"status": "success", "message": "没有失败的任务需要重试", "count": 0}
|
||
|
||
# 重置失败任务的状态为 pending
|
||
db.execute(
|
||
SummaryStatus.__table__.update()
|
||
.where(
|
||
SummaryStatus.status.in_(
|
||
[SummaryState.FAILED, SummaryState.PERMANENT_FAILURE]
|
||
)
|
||
)
|
||
.values(status=SummaryState.PENDING, error=None, error_type=None)
|
||
)
|
||
db.commit()
|
||
|
||
return {
|
||
"status": "success",
|
||
"message": f"已重置 {len(failed_ids)} 个失败任务为待总结状态",
|
||
"count": len(failed_ids),
|
||
}
|
||
|
||
|
||
# ── 论文管理 ────────────────────────────────────────────────────────
|
||
|
||
|
||
@router.get("/papers")
|
||
async def admin_papers(
|
||
request: Request,
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
q: str = Query("", description="搜索标题/摘要"),
|
||
date_from: str | None = Query(None),
|
||
date_to: str | None = Query(None),
|
||
tag: str = Query(""),
|
||
summary_status: str = Query("all"),
|
||
sort: str = Query("date_desc"),
|
||
page: int = Query(1, ge=1),
|
||
per_page: int = Query(20, ge=1, le=100),
|
||
):
|
||
"""论文管理列表页面。"""
|
||
papers, total, statuses = admin_svc.query_papers(
|
||
db,
|
||
q=q,
|
||
date_from=date_from,
|
||
date_to=date_to,
|
||
tag=tag,
|
||
summary_status=summary_status,
|
||
sort=sort,
|
||
page=page,
|
||
per_page=per_page,
|
||
)
|
||
|
||
# 构建分页 URL 辅助函数
|
||
def pagination_url(p: int) -> str:
|
||
params = dict(request.query_params)
|
||
params["page"] = str(p)
|
||
return "/admin/papers?" + "&".join(f"{k}={v}" for k, v in params.items())
|
||
|
||
return templates.TemplateResponse(
|
||
request,
|
||
"admin_papers.html",
|
||
{
|
||
"papers": papers,
|
||
"paper_summary_statuses": statuses,
|
||
"total": total,
|
||
"page": page,
|
||
"per_page": per_page,
|
||
"current_status": summary_status,
|
||
"current_sort": sort,
|
||
"pagination_url": pagination_url,
|
||
},
|
||
)
|
||
|
||
|
||
@router.post("/paper-delete/{arxiv_id}")
|
||
async def admin_paper_delete(
|
||
arxiv_id: str,
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
):
|
||
"""删除单篇论文。"""
|
||
paper = db.scalar(select(Paper).where(Paper.arxiv_id == arxiv_id))
|
||
if not paper:
|
||
raise HTTPException(status_code=404, detail=f"Paper not found: {arxiv_id}")
|
||
|
||
# 删除相关数据（ORM cascade 自动处理关联表）
|
||
db.delete(paper)
|
||
db.commit()
|
||
|
||
# 清理 FTS 索引
|
||
try:
|
||
db.execute(
|
||
text("DELETE FROM papers_fts WHERE arxiv_id = :aid"), {"aid": arxiv_id}
|
||
)
|
||
db.commit()
|
||
except Exception:
|
||
logger.warning("Failed to clean FTS index for %s", arxiv_id, exc_info=True)
|
||
|
||
return {"status": "success", "message": f"已删除 {arxiv_id}"}
|
||
|
||
|
||
class BatchActionRequest(BaseModel):
|
||
action: str # "delete" or "summarize"
|
||
arxiv_ids: list[str]
|
||
|
||
@field_validator("action")
|
||
@classmethod
|
||
def action_must_be_valid(cls, v: str) -> str:
|
||
if v not in ("delete", "summarize"):
|
||
raise ValueError("action must be 'delete' or 'summarize'")
|
||
return v
|
||
|
||
|
||
@router.post("/papers-batch-action")
|
||
async def admin_papers_batch_action(
|
||
body: BatchActionRequest,
|
||
_admin: None = Depends(verify_admin),
|
||
db: Session = Depends(get_db),
|
||
):
|
||
"""批量操作论文（删除或总结）。"""
|
||
if not body.arxiv_ids:
|
||
raise HTTPException(status_code=400, detail="arxiv_ids 不能为空")
|
||
|
||
if body.action == "delete":
|
||
papers = (
|
||
db.execute(select(Paper).where(Paper.arxiv_id.in_(body.arxiv_ids)))
|
||
.scalars()
|
||
.all()
|
||
)
|
||
|
||
count = 0
|
||
for paper in papers:
|
||
db.delete(paper)
|
||
count += 1
|
||
db.commit()
|
||
|
||
# 清理 FTS 索引
|
||
try:
|
||
stmt = text("DELETE FROM papers_fts WHERE arxiv_id IN :ids").bindparams(
|
||
bindparam("ids", expanding=True)
|
||
)
|
||
db.execute(stmt, {"ids": body.arxiv_ids})
|
||
db.commit()
|
||
except Exception:
|
||
logger.warning("Failed to clean FTS index for batch delete", exc_info=True)
|
||
|
||
return {
|
||
"status": "success",
|
||
"message": f"已删除 {count} 篇论文",
|
||
"count": count,
|
||
}
|
||
|
||
elif body.action == "summarize":
|
||
# 将选中论文的总结状态重置为 pending
|
||
paper_ids = (
|
||
db.execute(select(Paper.id).where(Paper.arxiv_id.in_(body.arxiv_ids)))
|
||
.scalars()
|
||
.all()
|
||
)
|
||
|
||
if paper_ids:
|
||
# 删除旧的 status 记录让其重新进入 pipeline
|
||
db.execute(
|
||
SummaryStatus.__table__.delete().where(
|
||
SummaryStatus.paper_id.in_(paper_ids)
|
||
)
|
||
)
|
||
db.commit()
|
||
|
||
return {
|
||
"status": "success",
|
||
"message": f"已将 {len(paper_ids)} 篇论文重置为待总结",
|
||
"count": len(paper_ids),
|
||
}
|