"""管理接口 — 仪表盘、抓取、总结、清理、删除、日志,需要登录鉴权。""" from __future__ import annotations import hashlib import json import logging from datetime import date from fastapi import APIRouter, Depends, Form, HTTPException, Query, Request from fastapi.responses import RedirectResponse from pydantic import BaseModel, field_validator from sqlalchemy import func, select, text from sqlalchemy.orm import Session from app.config import settings from app.database import get_db from app.models import ( CrawlLog, DataDeleteJob, Paper, PaperTag, SummaryState, SummaryStatus, TaskLock, ) from app.services.admin import get_admin_stats from app.services.cleaner import cleanup_tmp, delete_papers_by_date_range from app.services.crawler import crawl_daily, refresh_upvotes from app.services.pipeline import run_pipeline from app.services.scheduler import get_scheduler from app.services.summarizer import summarize_batch, summarize_single from app.utils import release_lock, templates, today_str, utc_now logger = logging.getLogger(__name__) router = APIRouter(prefix="/admin", tags=["admin"]) # ── 认证 ────────────────────────────────────────────────────────────── def _check_password(password: str) -> bool: """校验密码,支持明文或 sha256 哈希。""" stored = settings.ADMIN_PASSWORD if not stored: return False if password == stored: return True # 也支持存 sha256 哈希 return hashlib.sha256(password.encode()).hexdigest() == stored async def verify_admin(request: Request) -> None: """检查 session 中的登录状态,未登录则重定向到登录页。""" if not request.session.get("is_admin"): raise HTTPException(status_code=303, headers={"Location": "/admin/login"}) # ── 登录 / 登出 ────────────────────────────────────────────────────── @router.get("/login") async def admin_login_page(request: Request): """显示登录页面。已登录则直接跳转管理页。""" if request.session.get("is_admin"): return RedirectResponse("/admin/", status_code=303) return templates.TemplateResponse(request, "login.html", {"error": None}) @router.post("/login") async def admin_login_submit( request: Request, username: str = Form(""), password: str = Form(""), ): """处理登录表单提交。""" if username == settings.ADMIN_USERNAME and _check_password(password): request.session["is_admin"] = True return RedirectResponse("/admin/", status_code=303) return templates.TemplateResponse( request, "login.html", {"error": "用户名或密码错误"} ) @router.post("/logout") async def admin_logout(request: Request): """退出登录,清除 session。""" request.session.clear() return RedirectResponse("/admin/login", status_code=303) # ── 仪表盘 ────────────────────────────────────────────────────────── @router.get("/") async def admin_dashboard( request: Request, _admin: None = Depends(verify_admin), db: Session = Depends(get_db), ): """管理仪表盘 — 系统状态总览。""" stats = get_admin_stats(db) # 调度器历史(最近 10 条 task=scheduler 日志) scheduler_history = ( db.execute( select(CrawlLog) .where(CrawlLog.task == "scheduler") .order_by(CrawlLog.started_at.desc()) .limit(10) ) .scalars() .all() ) return templates.TemplateResponse( request, "admin_dashboard.html", {"stats": stats, "scheduler_history": scheduler_history}, ) # ── 调度器 ────────────────────────────────────────────────────────── @router.get("/scheduler-status") async def admin_scheduler_status(_admin: None = Depends(verify_admin)): """调度器运行状态(JSON)。""" scheduler = get_scheduler() next_run = None upvote_next_run = None if scheduler: for job in scheduler.get_jobs(): if job.id == "daily_pipeline": next_run = job.next_run_time elif job.id == "upvote_refresh": upvote_next_run = job.next_run_time return { "enabled": scheduler is not None, "schedule_time": f"{settings.SCHEDULE_HOUR:02d}:{settings.SCHEDULE_MINUTE:02d}", "timezone": settings.APP_TIMEZONE, "next_run": next_run.isoformat() if next_run else None, "upvote_next_run": upvote_next_run.isoformat() if upvote_next_run else None, "upvote_refresh_days": settings.UPVOTE_REFRESH_DAYS, } @router.post("/trigger-pipeline") async def admin_trigger_pipeline( _admin: None = Depends(verify_admin), db: Session = Depends(get_db), ): """手动触发一次完整流水线(crawl → summarize → cleanup)。""" today = today_str() try: result = await run_pipeline(db, today, owner="admin_trigger") except RuntimeError as exc: raise HTTPException(status_code=409, detail=str(exc)) if result["status"] == "failed": raise HTTPException(status_code=500, detail=result.get("error")) return {"status": "success", "message": "流水线执行完成"} @router.post("/refresh-upvotes") async def admin_refresh_upvotes( _admin: None = Depends(verify_admin), db: Session = Depends(get_db), days: int | None = Query(None, description="刷新最近 N 天,默认使用配置值"), ): """手动刷新最近 N 天论文的 upvotes。""" result = await refresh_upvotes(db, days=days) if result["status"] == "failed": raise HTTPException(status_code=500, detail=result.get("error")) return result # ── 请求模型 ────────────────────────────────────────────────────────── class DeleteRequest(BaseModel): date_start: date date_end: date include_notes: bool = True confirm: str @field_validator("confirm") @classmethod def confirm_must_be_delete(cls, v: str) -> str: if v != "DELETE": raise ValueError("confirm must be 'DELETE' to proceed") return v # ── 抓取 ────────────────────────────────────────────────────────────── @router.post("/crawl") async def admin_crawl( _admin: None = Depends(verify_admin), db: Session = Depends(get_db), date: str | None = Query(None, description="YYYY-MM-DD,默认今天"), ): """手动抓取指定日期,默认今天。""" target_date = date or today_str() # TaskLock 防重入 now = utc_now() lock = TaskLock( task="crawl", lock_key=target_date, status="running", owner="admin_crawl", acquired_at=now, ) try: db.add(lock) db.commit() except Exception: db.rollback() raise HTTPException( status_code=409, detail=f"Crawl already running for {target_date}" ) try: result = await crawl_daily(db, target_date) return result except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) finally: release_lock(db, lock) # ── 总结 ────────────────────────────────────────────────────────────── @router.post("/summarize") async def admin_summarize_batch( _admin: None = Depends(verify_admin), db: Session = Depends(get_db), ): """批量总结所有 pending 论文。""" result = await summarize_batch(db, pdf_mode=settings.SUMMARY_PDF_MODE) if result.get("status") == "conflict": raise HTTPException( status_code=409, detail=result.get("error", "batch already running") ) return result @router.post("/summarize/{arxiv_id}") async def admin_summarize_single( arxiv_id: str, _admin: None = Depends(verify_admin), db: Session = Depends(get_db), ): """总结或重跑单篇论文。""" result = await summarize_single(db, arxiv_id, force=True, pdf_mode=settings.SUMMARY_PDF_MODE) if result.get("status") == "not_found": raise HTTPException(status_code=404, detail=f"Paper not found: {arxiv_id}") return result # ── 清理 ────────────────────────────────────────────────────────────── @router.post("/cleanup") async def admin_cleanup( _admin: None = Depends(verify_admin), db: Session = Depends(get_db), ): """清理 data/tmp/ 中超过 24 小时的临时文件。""" now = utc_now() log_entry = CrawlLog( task="cleanup", status="running", started_at=now, ) db.add(log_entry) db.commit() try: result = cleanup_tmp() log_entry.status = "success" log_entry.completed_at = utc_now() log_entry.details_json = json.dumps({ "scanned": result.get("scanned", 0), "removed": result.get("removed", 0), }, ensure_ascii=False) if result.get("errors"): log_entry.error = "; ".join(result["errors"])[:2000] db.commit() return result except Exception as exc: log_entry.status = "failed" log_entry.error = str(exc)[:2000] log_entry.completed_at = utc_now() db.commit() raise HTTPException(status_code=500, detail=str(exc)) # ── 删除 ────────────────────────────────────────────────────────────── @router.post("/delete") async def admin_delete( body: DeleteRequest, _admin: None = Depends(verify_admin), db: Session = Depends(get_db), ): """删除指定日期范围内的论文(需要 confirm='DELETE' 二次确认)。""" if body.date_start > body.date_end: raise HTTPException(status_code=400, detail="date_start must be <= date_end") result = await delete_papers_by_date_range( db, body.date_start, body.date_end, include_notes=body.include_notes, ) return result # ── 日志 ────────────────────────────────────────────────────────────── @router.get("/logs") async def admin_logs( request: Request, _admin: None = Depends(verify_admin), db: Session = Depends(get_db), page: int = Query(1, ge=1), per_page: int = Query(20, ge=1, le=100), ): """查看任务日志(CrawlLog + DataDeleteJob)+ 总结状态统计。""" crawl_logs = ( db.execute( select(CrawlLog) .order_by(CrawlLog.started_at.desc()) .limit(per_page) .offset((page - 1) * per_page) ) .scalars() .all() ) delete_jobs = ( db.execute( select(DataDeleteJob) .order_by(DataDeleteJob.started_at.desc()) .limit(per_page) .offset((page - 1) * per_page) ) .scalars() .all() ) # 总结状态统计概要 summary_total = db.scalar(select(func.count(Paper.id))) or 0 summary_done = db.scalar( select(func.count(SummaryStatus.id)).where(SummaryStatus.status == SummaryState.DONE) ) or 0 summary_pending = db.scalar( select(func.count(SummaryStatus.id)).where( SummaryStatus.status.in_([SummaryState.PENDING, SummaryState.PROCESSING]) ) ) or 0 summary_failed = db.scalar( select(func.count(SummaryStatus.id)).where( SummaryStatus.status.in_([SummaryState.FAILED, SummaryState.PERMANENT_FAILURE]) ) ) or 0 return templates.TemplateResponse( request, "admin_logs.html", { "crawl_logs": crawl_logs, "delete_jobs": delete_jobs, "page": page, "per_page": per_page, "summary_total": summary_total, "summary_done": summary_done, "summary_pending": summary_pending, "summary_failed": summary_failed, }, ) # ── 总结状态管理 ──────────────────────────────────────────────────── @router.get("/summary-status") async def admin_summary_status( request: Request, _admin: None = Depends(verify_admin), db: Session = Depends(get_db), status: str = Query("all"), page: int = Query(1, ge=1), per_page: int = Query(20, ge=1, le=100), ): """总结状态列表(HTMX 片段或 JSON)。""" query = ( select(Paper, SummaryStatus) .outerjoin(SummaryStatus, SummaryStatus.paper_id == Paper.id) .order_by(Paper.paper_date.desc()) ) if status != "all": if status == "none": query = query.where(SummaryStatus.paper_id == None) # noqa: E711 else: query = query.where(SummaryStatus.status == status) total = db.scalar( select(func.count()).select_from(query.subquery()) ) results = ( db.execute(query.offset((page - 1) * per_page).limit(per_page)) .all() ) # 判断是否 HTMX 请求 is_htmx = request.headers.get("HX-Request") == "true" if is_htmx: # 返回 HTML 片段 return templates.TemplateResponse( request, "partials/summary_list.html", { "results": results, "total": total or 0, "page": page, "per_page": per_page, "current_status": status, }, ) # 非 HTMX 返回 JSON items = [] for paper, ss in results: item = { "arxiv_id": paper.arxiv_id, "title": paper.title_zh or paper.title_en, "paper_date": str(paper.paper_date), "summary_status": ss.status if ss else "none", "retry_count": ss.retry_count if ss else 0, "error_type": ss.error_type if ss else None, "error": ss.error if ss else None, } items.append(item) return {"items": items, "total": total or 0, "page": page, "per_page": per_page} @router.post("/summary-retry-failed") async def admin_summary_retry_failed( _admin: None = Depends(verify_admin), db: Session = Depends(get_db), ): """重试所有失败状态的总结任务。""" failed_ids = ( db.execute( select(Paper.arxiv_id) .join(SummaryStatus, SummaryStatus.paper_id == Paper.id) .where(SummaryStatus.status.in_([SummaryState.FAILED, SummaryState.PERMANENT_FAILURE])) ) .scalars() .all() ) if not failed_ids: return {"status": "success", "message": "没有失败的任务需要重试", "count": 0} # 重置失败任务的状态为 pending db.execute( SummaryStatus.__table__.update() .where(SummaryStatus.status.in_([SummaryState.FAILED, SummaryState.PERMANENT_FAILURE])) .values(status=SummaryState.PENDING, error=None, error_type=None) ) db.commit() return { "status": "success", "message": f"已重置 {len(failed_ids)} 个失败任务为待总结状态", "count": len(failed_ids), } # ── 论文管理 ──────────────────────────────────────────────────────── # 排序映射 _SORT_MAP = { "date_desc": Paper.paper_date.desc(), "date_asc": Paper.paper_date.asc(), "upvotes_desc": Paper.upvotes.desc(), "title_asc": Paper.title_en.asc(), } @router.get("/papers") async def admin_papers( request: Request, _admin: None = Depends(verify_admin), db: Session = Depends(get_db), q: str = Query("", description="搜索标题/摘要"), date_from: str | None = Query(None), date_to: str | None = Query(None), tag: str = Query(""), summary_status: str = Query("all"), sort: str = Query("date_desc"), page: int = Query(1, ge=1), per_page: int = Query(20, ge=1, le=100), ): """论文管理列表页面。""" query = select(Paper) # 搜索 if q.strip(): query = query.where( Paper.title_en.ilike(f"%{q}%") | Paper.title_zh.ilike(f"%{q}%") | Paper.abstract.ilike(f"%{q}%") ) # 日期范围 if date_from: query = query.where(Paper.paper_date >= date_from) if date_to: query = query.where(Paper.paper_date <= date_to) # 标签筛选 if tag: query = query.join(PaperTag, PaperTag.paper_id == Paper.id).where( PaperTag.tag == tag ) # 总结状态筛选 if summary_status != "all": if summary_status == "none": query = query.outerjoin( SummaryStatus, SummaryStatus.paper_id == Paper.id ).where(SummaryStatus.paper_id == None) # noqa: E711 else: query = query.join( SummaryStatus, SummaryStatus.paper_id == Paper.id ).where(SummaryStatus.status == summary_status) # 排序 order = _SORT_MAP.get(sort, Paper.paper_date.desc()) query = query.order_by(order) # 计数 total = db.scalar(select(func.count()).select_from(query.subquery())) # 分页 papers = ( db.execute(query.offset((page - 1) * per_page).limit(per_page)) .scalars() .all() ) # 获取每篇论文的总结状态 paper_ids = [p.id for p in papers] statuses = {} if paper_ids: rows = db.execute( select(SummaryStatus.paper_id, SummaryStatus.status).where( SummaryStatus.paper_id.in_(paper_ids) ) ).all() paper_id_to_arxiv = {p.id: p.arxiv_id for p in papers} for pid, st in rows: statuses[paper_id_to_arxiv.get(pid, "")] = st # 构建分页 URL 辅助函数 def pagination_url(p: int) -> str: params = dict(request.query_params) params["page"] = str(p) return "/admin/papers?" + "&".join(f"{k}={v}" for k, v in params.items()) return templates.TemplateResponse( request, "admin_papers.html", { "papers": papers, "paper_summary_statuses": statuses, "total": total or 0, "page": page, "per_page": per_page, "current_status": summary_status, "current_sort": sort, "pagination_url": pagination_url, }, ) @router.post("/paper-delete/{arxiv_id}") async def admin_paper_delete( arxiv_id: str, _admin: None = Depends(verify_admin), db: Session = Depends(get_db), ): """删除单篇论文。""" paper = db.scalar(select(Paper).where(Paper.arxiv_id == arxiv_id)) if not paper: raise HTTPException(status_code=404, detail=f"Paper not found: {arxiv_id}") # 删除相关数据(ORM cascade 自动处理关联表) db.delete(paper) db.commit() # 清理 FTS 索引 try: db.execute(text("DELETE FROM papers_fts WHERE arxiv_id = :aid"), {"aid": arxiv_id}) db.commit() except Exception: logger.warning("Failed to clean FTS index for %s", arxiv_id, exc_info=True) return {"status": "success", "message": f"已删除 {arxiv_id}"} class BatchActionRequest(BaseModel): action: str # "delete" or "summarize" arxiv_ids: list[str] @field_validator("action") @classmethod def action_must_be_valid(cls, v: str) -> str: if v not in ("delete", "summarize"): raise ValueError("action must be 'delete' or 'summarize'") return v @router.post("/papers-batch-action") async def admin_papers_batch_action( body: BatchActionRequest, _admin: None = Depends(verify_admin), db: Session = Depends(get_db), ): """批量操作论文(删除或总结)。""" if not body.arxiv_ids: raise HTTPException(status_code=400, detail="arxiv_ids 不能为空") if body.action == "delete": papers = db.execute( select(Paper).where(Paper.arxiv_id.in_(body.arxiv_ids)) ).scalars().all() count = 0 for paper in papers: db.delete(paper) count += 1 db.commit() # 清理 FTS 索引 try: db.execute( text("DELETE FROM papers_fts WHERE arxiv_id IN :ids"), {"ids": tuple(body.arxiv_ids)}, ) db.commit() except Exception: logger.warning("Failed to clean FTS index for batch delete", exc_info=True) return {"status": "success", "message": f"已删除 {count} 篇论文", "count": count} elif body.action == "summarize": # 将选中论文的总结状态重置为 pending paper_ids = db.execute( select(Paper.id).where(Paper.arxiv_id.in_(body.arxiv_ids)) ).scalars().all() if paper_ids: # 删除旧的 status 记录让其重新进入 pipeline db.execute( SummaryStatus.__table__.delete().where( SummaryStatus.paper_id.in_(paper_ids) ) ) db.commit() return { "status": "success", "message": f"已将 {len(paper_ids)} 篇论文重置为待总结", "count": len(paper_ids), }