feat: Add .env.example configuration file and integrate Redis service into Docker Compose for improved task management. Update desktop application to support document indexing and extraction workflows, enhancing user experience with a new extraction wizard and stepper components. Revise README and documentation for clarity on setup and usage.

services/extraction/worker/__init__.py

@@ -0,0 +1 @@
+"""Worker ARQ para procesamiento paralelo de documentos."""

services/extraction/worker/main.py

@@ -0,0 +1,18 @@
+"""Punto de entrada del worker ARQ.
+
+Ejecutar con:
+    python -m worker.main
+"""
+import asyncio
+import logging
+
+from arq import run_worker
+from worker.settings import WorkerSettings
+from domain.logging_config import configure_logging
+
+configure_logging()
+logger = logging.getLogger(__name__)
+
+if __name__ == "__main__":
+    logger.info("Iniciando worker ARQ...")
+    asyncio.run(run_worker(WorkerSettings))

services/extraction/worker/settings.py

@@ -0,0 +1,84 @@
+"""Configuración del worker ARQ.
+
+ARQ usa esta función `WorkerSettings` para saber qué funciones
+manejar, cuántos jobs en paralelo y cómo conectarse a Redis.
+"""
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any
+
+from arq.connections import RedisSettings
+
+from domain.config import settings
+from domain.logging_config import configure_logging
+from domain.service import InvoiceProcessingService
+
+logger = logging.getLogger(__name__)
+
+
+async def startup(ctx: dict[str, Any]) -> None:
+    """Inicializa el servicio de extracción compartido entre corrutinas del worker."""
+    configure_logging()
+    logger.info("Worker ARQ iniciando...")
+
+    from infrastructure.database import init_db
+    from domain.config import extractor_llm_config
+    from domain.strategies import OpenAICompatibleExtractor
+    from domain.batch_processor import BatchProcessor
+    from domain.estimator import Estimator
+    init_db()
+
+    extractor_config = extractor_llm_config()
+    extractor = OpenAICompatibleExtractor(
+        api_key=str(extractor_config["api_key"]),
+        base_url=str(extractor_config["base_url"]),
+        model=str(extractor_config["model"]),
+    )
+    ctx["service"] = InvoiceProcessingService(
+        extractor,
+        batch_processor_factory=lambda s: BatchProcessor(s),
+        estimator_factory=lambda s: Estimator(s),
+    )
+    logger.info("Worker ARQ listo. concurrency=%d", int(os.getenv("WORKER_CONCURRENCY", "25")))
+
+
+async def shutdown(ctx: dict[str, Any]) -> None:
+    logger.info("Worker ARQ deteniendo...")
+    from domain.rate_limiter import get_rate_limiter
+    try:
+        await get_rate_limiter().close()
+    except Exception:
+        pass
+
+
+def _parse_redis_settings() -> RedisSettings:
+    url = settings.redis_url
+    # Extraer host/port/db de la URL redis://host:port/db
+    url = url.removeprefix("redis://")
+    host_port, *db_parts = url.split("/")
+    host, *port_parts = host_port.split(":")
+    port = int(port_parts[0]) if port_parts else 6379
+    db = int(db_parts[0]) if db_parts else 0
+    return RedisSettings(host=host, port=port, database=db)
+
+
+class WorkerSettings:
+    """Configuración declarativa que ARQ espera en el módulo del worker."""
+    from worker.tasks import process_document
+    functions = [process_document]
+
+    redis_settings = _parse_redis_settings()
+
+    # Cuántos jobs corre en paralelo ESTE proceso worker
+    max_jobs: int = int(os.getenv("WORKER_CONCURRENCY", "25"))
+
+    # Tiempo máximo por job antes de cancelarlo (segundos)
+    job_timeout: int = int(os.getenv("EXTRACTOR_TIMEOUT_S", "120")) * 3
+
+    # Retener resultados en Redis 1 hora para diagnóstico
+    keep_result: int = 3600
+
+    on_startup = startup
+    on_shutdown = shutdown

services/extraction/worker/tasks.py

@@ -0,0 +1,236 @@
+"""Tareas ARQ para procesamiento paralelo de documentos.
+
+Cada tarea recibe la ruta de un archivo temporal en el volumen compartido
+`/tmp/ips_queue/`, lo procesa y actualiza la tarea en la base de datos.
+El archivo se elimina al finalizar (éxito o error).
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from typing import Any
+
+from infrastructure.database import ExtractionRow, Task, session_scope, utcnow
+from infrastructure.serialization import flatten_extraction_result
+from api.desktop_ingest import (
+    _finalize_radicado_if_ready,
+    _paths_for_type,
+    mark_task_completed_if_done,
+)
+from domain.profiles import get_profile
+from domain.prompts import PromptBuilder
+from domain.normalizer import normalize_to_schema
+from domain.rate_limiter import get_rate_limiter
+
+logger = logging.getLogger(__name__)
+
+
+async def process_document(
+    ctx: dict[str, Any],
+    *,
+    task_id: str,
+    batch_id: str,
+    queue_path: str,
+    radicado: str,
+    filename: str,
+    doc_type: str | None,
+    instruction: str | None,
+    tiff_page_range: list[int] | None,
+) -> dict[str, Any]:
+    """Extrae datos de un documento y los persiste en la BD.
+
+    Args:
+        ctx: Contexto ARQ con 'service' (InvoiceProcessingService).
+        task_id: ID de la tarea parent.
+        batch_id: ID del lote de extracción.
+        queue_path: Ruta absoluta al archivo en el volumen compartido.
+        radicado: Radicado al que pertenece el documento.
+        filename: Nombre original del archivo.
+        doc_type: Tipo documental (None en modo carpeta general).
+        instruction: Instrucción de extracción personalizada.
+        tiff_page_range: Rango de páginas TIFF [inicio, fin].
+    """
+    service = ctx["service"]
+    rate_limiter = get_rate_limiter()
+    tiff_range = tuple(tiff_page_range) if tiff_page_range else None
+
+    logger.info("Procesando %s / %s (task=%s)", radicado, filename, task_id)
+
+    try:
+        # Obtener config de la tarea desde BD
+        def _load_task_config() -> dict[str, Any] | None:
+            with session_scope() as session:
+                task = session.get(Task, task_id)
+                if task is None:
+                    return None
+                return dict(task.request or {})
+
+        request = await asyncio.to_thread(_load_task_config)
+        if request is None:
+            logger.warning("Tarea %s no encontrada — saltando %s", task_id, filename)
+            return {"skipped": True, "reason": "task_not_found"}
+
+        fields: list[str] = list(request.get("fields") or [])
+        alias_map: dict[str, str] = dict(request.get("alias_map") or {})
+        structure_mode: str = str(request.get("structure_mode") or "radicados")
+        doc_types: list = list(request.get("doc_types") or [])
+        prof = get_profile("invoice")
+
+        selected_paths = _paths_for_type(
+            fields,
+            doc_type if structure_mode == "radicados" and doc_types else None,
+        )
+
+        if doc_type and structure_mode == "radicados" and doc_types:
+            try:
+                prompt = PromptBuilder.build_for_type(
+                    doc_type, selected_paths, {}, profile_name=prof.get("name", "invoice")
+                )
+            except Exception:
+                prompt = PromptBuilder.build(
+                    selected_paths, {}, profile_name=prof.get("name", "invoice")
+                )
+        else:
+            prompt = instruction or PromptBuilder.build(
+                selected_paths, {}, profile_name=prof.get("name", "invoice")
+            )
+
+        # Esperar turno en el rate limiter antes de llamar a OpenAI
+        estimated_tokens = max(len(selected_paths) * 80, 2000)
+        await rate_limiter.acquire(estimated_tokens=estimated_tokens)
+
+        # Extracción + normalización (CPU + IO bound, en thread pool)
+        def _extract_and_normalize() -> tuple[dict, dict]:
+            data, _usage = service.extract_full_from_pdf(
+                queue_path,
+                instruction=prompt,
+                tiff_page_range=tiff_range,
+            )
+            raw_data: dict = data if isinstance(data, dict) else {"data": data}
+            if isinstance(data, dict):
+                data["radicado"] = radicado
+            else:
+                data = {"radicado": radicado, "data": data}
+
+            norm = normalize_to_schema(
+                data,
+                paths=selected_paths,
+                profile_name=prof.get("name", "invoice"),
+            )
+            normalized_obj = data
+            if isinstance(norm, dict):
+                normalized_obj = norm.get("normalized") or data
+            return raw_data, normalized_obj
+
+        _raw, normalized_obj = await asyncio.to_thread(_extract_and_normalize)
+
+        # Persistir resultado en BD
+        def _persist() -> dict[str, Any]:
+            with session_scope() as session:
+                task = session.get(Task, task_id)
+                if task is None:
+                    return {"skipped": True, "reason": "task_not_found"}
+
+                result = dict(task.result or {})
+
+                if structure_mode == "radicados" and doc_types and doc_type:
+                    partials = dict(result.get("partials") or {})
+                    radicado_partials = dict(partials.get(radicado) or {})
+                    radicado_partials[doc_type] = normalized_obj or {}
+                    partials[radicado] = radicado_partials
+                    result["partials"] = partials
+                    task.result = result
+                    session.flush()
+                    finalized = _finalize_radicado_if_ready(
+                        session,
+                        task=task,
+                        batch_id=batch_id,
+                        radicado=radicado,
+                        request=request,
+                        fields=fields,
+                        alias_map=alias_map,
+                    )
+                    mark_task_completed_if_done(session, task, batch_id)
+                    return {
+                        "radicado": radicado,
+                        "filename": filename,
+                        "doc_type": doc_type,
+                        "finalized": finalized,
+                        "processed": int((task.result or {}).get("processed") or 0),
+                    }
+
+                fused = {"radicado": radicado, "_source_pdf": filename}
+                if doc_type:
+                    fused[doc_type] = normalized_obj or {}
+                else:
+                    fused.update(
+                        normalized_obj if isinstance(normalized_obj, dict) else {"data": normalized_obj}
+                    )
+
+                row_index = int(result.get("row_count") or 0)
+                for row in flatten_extraction_result(fused, fields, alias_map):
+                    row_index += 1
+                    session.add(
+                        ExtractionRow(
+                            batch_id=batch_id,
+                            radicado=radicado,
+                            source_pdf=filename,
+                            row_index=row_index,
+                            row_json=row,
+                            raw_json=fused,
+                        )
+                    )
+
+                result["row_count"] = row_index
+                result["processed"] = int(result.get("processed") or 0) + 1
+                task.result = result
+                task.updated_at = utcnow()
+                session.flush()
+                mark_task_completed_if_done(session, task, batch_id)
+                return {
+                    "radicado": radicado,
+                    "filename": filename,
+                    "doc_type": doc_type,
+                    "finalized": True,
+                    "processed": int(result.get("processed") or 0),
+                }
+
+        payload = await asyncio.to_thread(_persist)
+        logger.info("Completado %s / %s — payload=%s", radicado, filename, payload)
+        return payload
+
+    except Exception as exc:
+        logger.exception("Error procesando %s / %s: %s", radicado, filename, exc)
+        # Registrar error en la tarea sin fallar el worker
+        def _record_error() -> None:
+            try:
+                with session_scope() as session:
+                    task = session.get(Task, task_id)
+                    if task is None:
+                        return
+                    result = dict(task.result or {})
+                    errors = list(result.get("errors") or [])
+                    errors.append({
+                        "radicado": radicado,
+                        "filename": filename,
+                        "error": str(exc),
+                    })
+                    result["errors"] = errors
+                    result["processed"] = int(result.get("processed") or 0) + 1
+                    task.result = result
+                    task.updated_at = utcnow()
+                    mark_task_completed_if_done(session, task, batch_id)
+            except Exception as inner:
+                logger.error("No se pudo registrar el error en BD: %s", inner)
+
+        await asyncio.to_thread(_record_error)
+        return {"error": str(exc), "radicado": radicado, "filename": filename}
+
+    finally:
+        # Eliminar archivo temporal del volumen compartido
+        try:
+            if os.path.exists(queue_path):
+                os.unlink(queue_path)
+        except OSError as e:
+            logger.warning("No se pudo eliminar %s: %s", queue_path, e)