refactor: extract admin business logic to services, introduce job queue, add derived index helpers

- Move DB operations from routes/admin.py to services/admin.py (get_logs_context, query_summary_statuses, retry_failed, delete/reset operations)
- Add services/jobs.py with Job/JobEvent-based async job queue (create_job, run_job, enqueue_job)
- Add services/derived.py with FTS5 reindex and paper index deletion helpers
- Refactor scheduler to use job queue instead of direct pipeline calls
- Add heartbeat_at/expires_at to TaskLock for lock health tracking
- Remove DESIGN_REVIEW.md
- Update tests: remove redundant integration tests, add unit tests for new services
This commit is contained in:
2026-06-13 18:31:43 +08:00
parent 21f16e6756
commit 743d69efd0
20 changed files with 1391 additions and 1063 deletions
+60
View File
@@ -10,6 +10,7 @@ from app.services.crawler import (
_parse_paper,
crawl_daily,
fetch_daily,
refresh_upvotes,
upsert_papers,
)
@@ -187,3 +188,62 @@ class TestCrawlDaily:
assert result["status"] == "failed"
assert "network error" in result["error"]
class TestRefreshUpvotes:
@pytest.mark.asyncio
async def test_refresh_updates_existing_without_inserting_new(
self, db_session, sample_paper
):
sample_paper.arxiv_id = "1706.03762"
sample_paper.upvotes = 10
db_session.commit()
with patch(
"app.services.crawler.fetch_daily",
new_callable=AsyncMock,
return_value=[
{
"paper": {
"id": "1706.03762",
"upvotes": 999,
"authors": [],
"tags": [],
}
},
{
"paper": {
"id": "2010.11929",
"upvotes": 123,
"authors": [],
"tags": [],
}
},
],
):
result = await refresh_upvotes(db_session, days=1)
db_session.refresh(sample_paper)
assert result["status"] == "success"
assert result["updated"] == 1
assert sample_paper.upvotes == 999
assert db_session.query(type(sample_paper)).count() == 1
@pytest.mark.asyncio
async def test_refresh_returns_partial_when_one_day_fails(self, db_session):
async def _fetch_daily(target_date):
if target_date.endswith("01"):
raise ConnectionError("hf down")
return []
with (
patch(
"app.services.crawler.recent_date_strs",
return_value=["2024-01-01", "2024-01-02"],
),
patch("app.services.crawler.fetch_daily", side_effect=_fetch_daily),
):
result = await refresh_upvotes(db_session, days=2)
assert result["status"] == "partial"
assert result["errors"] == ["2024-01-01: hf down"]