refactor: improve PDF extraction, error handling, and proxy configuration
This commit is contained in:
+3
-2
@@ -93,13 +93,14 @@ def release_lock(db, lock) -> None:
|
||||
|
||||
|
||||
def make_http_client(
|
||||
*, sync: bool = False, follow_redirects: bool = False, **kwargs
|
||||
*, sync: bool = False, follow_redirects: bool = False, use_proxy: bool = True, **kwargs
|
||||
) -> httpx.AsyncClient | httpx.Client:
|
||||
"""创建带 proxy 和默认配置的 httpx 客户端。
|
||||
|
||||
Args:
|
||||
sync: True 返回同步 Client,False 返回 AsyncClient
|
||||
follow_redirects: 是否跟随重定向
|
||||
use_proxy: 是否使用 HF_PROXY 代理(默认 True;国内 API 如 embedder 应传 False)
|
||||
**kwargs: 覆盖默认参数
|
||||
"""
|
||||
defaults: dict = {
|
||||
@@ -107,7 +108,7 @@ def make_http_client(
|
||||
"headers": {"User-Agent": settings.HTTP_USER_AGENT},
|
||||
"follow_redirects": follow_redirects,
|
||||
}
|
||||
if settings.http_proxy:
|
||||
if use_proxy and settings.http_proxy:
|
||||
defaults["transport"] = (
|
||||
httpx.HTTPTransport(proxy=settings.http_proxy)
|
||||
if sync
|
||||
|
||||
Reference in New Issue
Block a user