refactor: improve PDF extraction, error handling, and proxy configuration

This commit is contained in:
2026-06-30 10:39:27 +08:00
parent 1ccac1f29a
commit a2e3dc398b
7 changed files with 438 additions and 71 deletions
+3 -2
View File
@@ -93,13 +93,14 @@ def release_lock(db, lock) -> None:
def make_http_client(
*, sync: bool = False, follow_redirects: bool = False, **kwargs
*, sync: bool = False, follow_redirects: bool = False, use_proxy: bool = True, **kwargs
) -> httpx.AsyncClient | httpx.Client:
"""创建带 proxy 和默认配置的 httpx 客户端。
Args:
sync: True 返回同步 ClientFalse 返回 AsyncClient
follow_redirects: 是否跟随重定向
use_proxy: 是否使用 HF_PROXY 代理(默认 True;国内 API 如 embedder 应传 False
**kwargs: 覆盖默认参数
"""
defaults: dict = {
@@ -107,7 +108,7 @@ def make_http_client(
"headers": {"User-Agent": settings.HTTP_USER_AGENT},
"follow_redirects": follow_redirects,
}
if settings.http_proxy:
if use_proxy and settings.http_proxy:
defaults["transport"] = (
httpx.HTTPTransport(proxy=settings.http_proxy)
if sync