85 lines
2.7 KiB
Python
85 lines
2.7 KiB
Python
"""Configuración del worker ARQ.
|
|
|
|
ARQ usa esta función `WorkerSettings` para saber qué funciones
|
|
manejar, cuántos jobs en paralelo y cómo conectarse a Redis.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
from typing import Any
|
|
|
|
from arq.connections import RedisSettings
|
|
|
|
from domain.config import settings
|
|
from domain.logging_config import configure_logging
|
|
from domain.service import InvoiceProcessingService
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def startup(ctx: dict[str, Any]) -> None:
|
|
"""Inicializa el servicio de extracción compartido entre corrutinas del worker."""
|
|
configure_logging()
|
|
logger.info("Worker ARQ iniciando...")
|
|
|
|
from infrastructure.database import init_db
|
|
from domain.config import extractor_llm_config
|
|
from domain.strategies import OpenAICompatibleExtractor
|
|
from domain.batch_processor import BatchProcessor
|
|
from domain.estimator import Estimator
|
|
init_db()
|
|
|
|
extractor_config = extractor_llm_config()
|
|
extractor = OpenAICompatibleExtractor(
|
|
api_key=str(extractor_config["api_key"]),
|
|
base_url=str(extractor_config["base_url"]),
|
|
model=str(extractor_config["model"]),
|
|
)
|
|
ctx["service"] = InvoiceProcessingService(
|
|
extractor,
|
|
batch_processor_factory=lambda s: BatchProcessor(s),
|
|
estimator_factory=lambda s: Estimator(s),
|
|
)
|
|
logger.info("Worker ARQ listo. concurrency=%d", int(os.getenv("WORKER_CONCURRENCY", "25")))
|
|
|
|
|
|
async def shutdown(ctx: dict[str, Any]) -> None:
|
|
logger.info("Worker ARQ deteniendo...")
|
|
from domain.rate_limiter import get_rate_limiter
|
|
try:
|
|
await get_rate_limiter().close()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _parse_redis_settings() -> RedisSettings:
|
|
url = settings.redis_url
|
|
# Extraer host/port/db de la URL redis://host:port/db
|
|
url = url.removeprefix("redis://")
|
|
host_port, *db_parts = url.split("/")
|
|
host, *port_parts = host_port.split(":")
|
|
port = int(port_parts[0]) if port_parts else 6379
|
|
db = int(db_parts[0]) if db_parts else 0
|
|
return RedisSettings(host=host, port=port, database=db)
|
|
|
|
|
|
class WorkerSettings:
|
|
"""Configuración declarativa que ARQ espera en el módulo del worker."""
|
|
from worker.tasks import process_document
|
|
functions = [process_document]
|
|
|
|
redis_settings = _parse_redis_settings()
|
|
|
|
# Cuántos jobs corre en paralelo ESTE proceso worker
|
|
max_jobs: int = int(os.getenv("WORKER_CONCURRENCY", "25"))
|
|
|
|
# Tiempo máximo por job antes de cancelarlo (segundos)
|
|
job_timeout: int = int(os.getenv("EXTRACTOR_TIMEOUT_S", "120")) * 3
|
|
|
|
# Retener resultados en Redis 1 hora para diagnóstico
|
|
keep_result: int = 3600
|
|
|
|
on_startup = startup
|
|
on_shutdown = shutdown
|