diff --git a/.env.example b/.env.example index acf84aa..7d76bc8 100644 --- a/.env.example +++ b/.env.example @@ -32,5 +32,16 @@ ASYNC_QUEUE=false # Solo necesario cuando ASYNC_QUEUE=true. Se levanta via docker-compose. REDIS_URL=redis://redis:6379/0 +# ── Carpeta de documentos ───────────────────────────────── +# Ruta local que Docker monta como /data dentro de los contenedores. +IPS_DOCUMENT_ROOT=./data + +# ── Servicio de conteo (count-api) ──────────────────────── +OCR_DPI=300 +OCR_LANG=spa +OCR_POOL_SIZE=4 +SYNC_MAX_FILES=50 +SYNC_PROCESS_CONCURRENCY=4 + # ── Misc ────────────────────────────────────────────────── ALLOWED_DOCUMENT_TYPES=CC,TI,CE,RC,PA,PPT,PEP,EPT,NUIP diff --git a/README.md b/README.md index fce1a94..e91aee8 100644 --- a/README.md +++ b/README.md @@ -1,276 +1,114 @@ -## IPSExtractor Microservices +# IPSExtractor -Monorepo con servicios separados como si fueran repos independientes. Cada -servicio tiene su propia API, dependencias, Dockerfile, codigo de dominio e -infraestructura local. No hay paquetes compartidos entre servicios. +Sistema de extracción y análisis de datos desde facturas médicas PDF. +Arquitectura de microservicios: cada servicio tiene su propia API, dependencias, Dockerfile y código de dominio. No hay paquetes compartidos entre servicios. -```text -services/ - extraction/ Repo/servicio de extraccion - api/ API FastAPI - domain/ Dominio propio de extraccion y normalizacion - core/ Utilidades propias para PDF, texto y filas - infrastructure/ Persistencia propia del servicio - requirements.txt - Dockerfile +--- - cleaning/ Repo/servicio de limpieza - api/ API FastAPI - domain/ Reglas propias de limpieza - infrastructure/ Persistencia propia del servicio - requirements.txt - Dockerfile +## Servicios - api_gateway/ API central opcional - api/ Proxy HTTP hacia extraction y cleaning - requirements.txt - Dockerfile +| Servicio | Puerto | Tecnología | Descripción | +|----------|--------|------------|-------------| +| `api_gateway` | 8000 | Python/FastAPI | Puerta de enlace única; enruta hacia los demás servicios | +| `extraction` | 8001 | Python/FastAPI | Extracción LLM de campos; orquestación de jobs y conteo | +| `cleaning` | 8002 | Python/FastAPI | Normalización y limpieza de datos extraídos | +| `count-api` | 8080 (→ 8081) | Go | Conteo de servicios en facturas con OCR + ONNX (sin estado) | +| `database` | 5432 | PostgreSQL 16 | Base de datos compartida | +| `redis` | 6379 | Redis 7 | Cola ARQ para workers de extracción | +| `desktop_app` | local | Electron + React | Aplicación de escritorio | - desktop_app/ App desktop Windows/Mac - electron/ Main process, preload y dialogos nativos - src/ React UI, cliente API y export CSV - package.json +--- + +## Flujo principal + +``` +Desktop app + │ + ├─ Conteo (OCR + ML) + │ ├─ POST /count/v1/count → count-api (lotes de 10 PDFs) + │ ├─ POST /count-jobs → extraction (guarda en PostgreSQL) + │ └─ PATCH /count-jobs → extraction (validación humana) + │ + └─ Extracción (LLM) + ├─ POST /tasks/extraction/from-count → extraction (lee desde /data) + ├─ POST /tasks/{id}/ingest-many → extraction (subida de PDFs) + └─ GET /tasks/{id}/rows → extraction (resultados) ``` -La base de datos Postgres es compartida como contrato de integracion, pero el -codigo de acceso a datos esta duplicado dentro de cada servicio para que puedan -vivir por separado. +--- -### Puertos - -| Servicio | Puerto | Uso | -| --- | ---: | --- | -| `api_gateway` | `8000` | API central opcional | -| `extraction` | `8001` | API propia de extraccion | -| `cleaning` | `8002` | API propia de limpieza | -| `database` | `5432` | Postgres | -| `desktop_app` | local | App desktop fuera de Docker | - -Documentacion interactiva: - -- `http://localhost:8000/docs` -- `http://localhost:8001/docs` -- `http://localhost:8002/docs` - -### Variables - -```env -IPS_DOCUMENT_ROOT=C:/ruta/a/documentos - -# Ejemplo: extraccion con Grok/xAI -EXTRACTOR_PROVIDER=xai -EXTRACTOR_BASE_URL=https://api.x.ai/v1 -EXTRACTOR_API_KEY=tu_xai_api_key -EXTRACTOR_MODEL=grok-tu-modelo -EXTRACTOR_TIMEOUT_S=120 - -# Ejemplo: normalizacion con OpenAI -NORMALIZER_PROVIDER=openai -NORMALIZER_BASE_URL=https://api.openai.com/v1 -NORMALIZER_API_KEY=tu_openai_api_key -NORMALIZER_MODEL=gpt-tu-modelo -NORMALIZER_TIMEOUT_S=120 - -PDF_DPI=600 -PDF_THREAD_COUNT=10 -CONCURRENCY_LIMIT=4 -ALLOWED_DOCUMENT_TYPES=CC,TI,CE,RC,PA,PPT,PEP,EPT,NUIP -``` - -Si `NORMALIZER_BASE_URL` o `NORMALIZER_API_KEY` se dejan vacios, el servicio usa -el mismo backend configurado en `EXTRACTOR_BASE_URL` y `EXTRACTOR_API_KEY`. - -Dentro de Docker, `IPS_DOCUMENT_ROOT` se monta como `/data`. - -### Levantar +## Arrancar el stack completo ```bash +# Clonar y configurar +cp .env.example .env +# Editar .env con las API keys del LLM y la ruta de documentos + docker compose up --build ``` Health checks: ```bash -curl http://localhost:8000/health -curl http://localhost:8001/health -curl http://localhost:8002/health +curl http://localhost:8000/health # gateway (agrega estado de los 3 servicios) +curl http://localhost:8081/health # count-api (directo) ``` -### Flujo Directo Por Servicio +--- -1. Ver modelos disponibles en extraccion: +## Variables de entorno principales -```bash -curl http://localhost:8001/models +| Variable | Descripción | +|----------|-------------| +| `IPS_DOCUMENT_ROOT` | Carpeta local con los documentos PDF (montada en `/data`) | +| `EXTRACTOR_API_KEY` | API key del LLM (OpenAI, xAI, etc.) | +| `EXTRACTOR_BASE_URL` | URL base del LLM (default: OpenAI) | +| `EXTRACTOR_MODEL` | Modelo de extracción (default: `gpt-5.4-mini`) | +| `WORKER_REPLICAS` | Réplicas del worker de extracción (default: `2`) | + +Ver `.env.example` para la lista completa. + +--- + +## Estructura del repositorio + +``` +services/ + api_gateway/ — proxy HTTP (Python) → README + extraction/ — LLM + jobs + conteo BD (Python) → README + cleaning/ — limpieza de datos (Python) → README + count-api/ — OCR + ONNX stateless (Go) → README + db/ — migraciones Alembic → README + desktop_app/ — app Electron + React → README +docs/ + database.dbml — esquema completo de la BD +docker-compose.yml — stack completo +.env.example — variables de entorno de referencia ``` -2. Descubrir campos: +--- -```bash -curl -X POST http://localhost:8001/schemas \ - -H "Content-Type: application/json" \ - -d '{"document_path": "/data/radicado-1/factura.pdf"}' +## Base de datos (esquema resumido) + +``` +jobs — contenedor de un trabajo del usuario + tasks — módulo ejecutado (counting / extraction / cleaning) + count_job_files — resultado de conteo por PDF + extraction_batches — lote de extracción + extraction_rows — fila extraída por documento + cleaning_jobs — job de limpieza + cleaned_rows — fila transformada ``` -3. Ejecutar extraccion: +Esquema completo: [`docs/database.dbml`](docs/database.dbml) -```bash -curl -X POST http://localhost:8001/batches \ - -H "Content-Type: application/json" \ - -d '{ - "root_path": "/data", - "keywords": ["factura"], - "fields": ["numero_factura", "items/*/codigo", "items/*/valor"], - "alias_map": { - "numero_factura": "factura", - "items/*/codigo": "codigo", - "items/*/valor": "valor" - }, - "structure_mode": "radicados" - }' -``` +--- -La respuesta devuelve `id`; ese valor es el `batch_id`. +## Documentación por servicio -4. Consultar filas extraidas: - -```bash -curl http://localhost:8001/batches/{batch_id}/rows -``` - -5. Ver reglas de limpieza: - -```bash -curl http://localhost:8002/rules -``` - -6. Analizar columnas extraidas: - -```bash -curl http://localhost:8002/batches/{batch_id}/analysis -``` - -7. Ejecutar limpieza: - -```bash -curl -X POST http://localhost:8002/jobs \ - -H "Content-Type: application/json" \ - -d '{ - "extraction_batch_id": "{batch_id}", - "rules_per_column": { - "valor": ["to_integer"], - "fecha": ["date_to_ddmmyyyy"] - } - }' -``` - -La respuesta devuelve `id`; ese valor es el `job_id`. - -8. Consultar filas limpias: - -```bash -curl http://localhost:8002/jobs/{job_id}/rows -``` - -### Flujo Por API Gateway - -El gateway no importa codigo interno de los servicios; solo enruta HTTP. - -```bash -curl http://localhost:8000/models -curl http://localhost:8000/config -curl http://localhost:8000/extraction/batches/{batch_id}/rows -curl http://localhost:8000/cleaning/jobs/{job_id}/rows -``` - -Para llamar cualquier endpoint propio de un servicio: - -- `http://localhost:8000/extraction/...` -- `http://localhost:8000/cleaning/...` - -### Desarrollo Independiente - -Extraccion como repo aparte: - -```bash -cd services/extraction -pip install -r requirements.txt -uvicorn api.main:app --host 0.0.0.0 --port 8001 -``` - -Limpieza como repo aparte: - -```bash -cd services/cleaning -pip install -r requirements.txt -uvicorn api.main:app --host 0.0.0.0 --port 8002 -``` - -Gateway como repo aparte: - -```bash -cd services/api_gateway -pip install -r requirements.txt -uvicorn api.main:app --host 0.0.0.0 --port 8000 -``` - -App desktop como repo aparte: - -```bash -cd services/desktop_app -npm install -npm run dev -``` - -La app desktop usa `http://localhost:8000` por defecto. **No monta la carpeta -en Docker**: indexa los documentos localmente y los sube uno a uno al API -(`POST /extraction/schemas/upload` y `POST /tasks/{id}/ingest`). Solo se -persisten las filas extraidas en Postgres; los PDFs permanecen en el equipo del -usuario. - -```bash -cd services/desktop_app -npm install -npm run dev -``` - -#### Flujo desktop (sin volumen `/data`) - -1. Seleccionar carpeta local en la app. -2. Descubrir campos (sube un PDF de muestra por tipo). -3. Ejecutar extraccion (crea tarea `source=desktop` e ingesta cada archivo). -4. Consultar tareas y exportar CSV opcional. - -#### Modo legacy API / curl - -Si llamas al API con `root_path` y rutas bajo `/data`, monta documentos al -levantar Docker: - -```bash -IPS_DOCUMENT_ROOT=C:/ruta/a/documentos docker compose up --build -``` - -### Error: No se encontraron documentos en /data - -Ese mensaje aparece en clientes legacy que envian `root_path=/data` sin montar -documentos. La app desktop no deberia usar ese modo. Verifica: - -```bash -docker exec ipsextractor-extraction ls -la /data -``` - -Para la app desktop, asegurate de usar Electron (`npm run dev`) y de que el -gateway responda en `http://localhost:8000`. - -### Contrato De Base De Datos - -Los servicios se integran por Postgres: - -- `extraction_batches` -- `extraction_rows` -- `tasks` -- `cleaning_jobs` -- `cleaned_rows` - -Cada servicio mantiene su propia copia del modelo SQLAlchemy para no depender -de un paquete comun. +- [`services/api_gateway/README.md`](services/api_gateway/README.md) +- [`services/extraction/README.md`](services/extraction/README.md) +- [`services/cleaning/README.md`](services/cleaning/README.md) +- [`services/services-count-facturas-medicas-serving/README.md`](services/services-count-facturas-medicas-serving/README.md) +- [`services/db/README.md`](services/db/README.md) +- [`services/desktop_app/README.md`](services/desktop_app/README.md) diff --git a/docker-compose.yml b/docker-compose.yml index f965d6b..dfaa886 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,18 @@ services: timeout: 3s retries: 10 + db-migrations: + build: + context: ./services/db + dockerfile: Dockerfile + container_name: ipsextractor-db-migrations + depends_on: + database: + condition: service_healthy + environment: + DATABASE_URL: postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor + restart: on-failure + extraction: build: context: ./services/extraction @@ -38,6 +50,8 @@ services: condition: service_healthy redis: condition: service_healthy + db-migrations: + condition: service_completed_successfully environment: DATABASE_URL: postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor REDIS_URL: redis://redis:6379/0 @@ -69,6 +83,8 @@ services: condition: service_healthy redis: condition: service_healthy + db-migrations: + condition: service_completed_successfully deploy: replicas: ${WORKER_REPLICAS:-2} environment: @@ -96,11 +112,39 @@ services: depends_on: database: condition: service_healthy + db-migrations: + condition: service_completed_successfully environment: DATABASE_URL: postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor ports: - "8002:8002" + count-api: + build: + context: ./services/services-count-facturas-medicas-serving + dockerfile: Dockerfile + container_name: ipsextractor-count-api + depends_on: + db-migrations: + condition: service_completed_successfully + environment: + PORT: "8080" + OCR_DPI: ${OCR_DPI:-300} + OCR_LANG: ${OCR_LANG:-spa} + OCR_POOL_SIZE: ${OCR_POOL_SIZE:-4} + SYNC_MAX_FILES: ${SYNC_MAX_FILES:-50} + SYNC_PROCESS_CONCURRENCY: ${SYNC_PROCESS_CONCURRENCY:-4} + MAX_PDF_SIZE_MB: ${MAX_PDF_SIZE_MB:-20} + LOG_PIPELINE_TIMING: ${LOG_PIPELINE_TIMING:-false} + ports: + - "8081:8080" + volumes: + - ./services/services-count-facturas-medicas-serving/models/latest:/app/models:ro + deploy: + resources: + limits: + memory: 1g + api_gateway: build: context: ./services/api_gateway @@ -109,9 +153,11 @@ services: depends_on: - extraction - cleaning + - count-api environment: EXTRACTION_URL: http://extraction:8001 CLEANING_URL: http://cleaning:8002 + COUNT_URL: http://count-api:8080 ports: - "8000:8000" diff --git a/docs/database.dbml b/docs/database.dbml index 09dd1bb..2e29b1b 100644 --- a/docs/database.dbml +++ b/docs/database.dbml @@ -1,6 +1,6 @@ // IPSExtractor — esquema de integración PostgreSQL -// Fuente: services/extraction/infrastructure/database.py -// services/cleaning/infrastructure/database.py +// Fuente autoritativa: services/extraction/infrastructure/database.py +// services/extraction/alembic/versions/ // // Visualizar en https://dbdiagram.io (importar este archivo) // o con la CLI de dbml: npx @dbml/cli dbml2sql docs/database.dbml @@ -8,13 +8,22 @@ Project ipsextractor { database_type: 'PostgreSQL' Note: ''' - Base de datos compartida entre los microservicios de extracción y limpieza. - Cada servicio define su propia copia del modelo SQLAlchemy y ejecuta - `init_db()` con bloqueo advisory (pg_advisory_lock) para crear tablas - sin condiciones de carrera. + Base de datos compartida entre todos los microservicios (extraction, cleaning, + count-facturas). Cada servicio accede directamente a las tablas que necesita. + Las migraciones se gestionan con Alembic (services/extraction/alembic/). + Cada servicio es desacoplado; comparten solo la DB como fuente de verdad. ''' } +// --------------------------------------------------------------------------- +// Grupos de tablas +// --------------------------------------------------------------------------- + +TableGroup orchestration { + jobs + tasks +} + TableGroup pipeline { extraction_batches extraction_rows @@ -22,8 +31,94 @@ TableGroup pipeline { cleaned_rows } -TableGroup async_jobs { - tasks +TableGroup counting { + count_job_files +} + +// --------------------------------------------------------------------------- +// Orchestration — trabajo multimodular del usuario +// --------------------------------------------------------------------------- + +Table jobs { + id varchar(36) [pk, note: 'UUID v4'] + status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed'] + title text [note: 'Nombre descriptivo del trabajo'] + metadata jsonb [not null, default: '{}', note: 'Datos adicionales específicos por tipo de trabajo'] + created_at timestamptz [not null] + updated_at timestamptz [not null] + + indexes { + status + } + + Note: ''' + Contenedor de trabajo del usuario. Un job puede tener múltiples tasks + (una por servicio/módulo utilizado). Permite agrupar conteo + extracción + + limpieza de un mismo conjunto de documentos. + ''' +} + +Table tasks { + id varchar(36) [pk, note: 'UUID v4'] + job_id varchar(36) [ref: > jobs.id, note: 'FK nullable — compat. con tasks anteriores'] + type varchar(64) [not null, note: 'counting | extraction | cleaning'] + status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed'] + title text + request jsonb [not null, default: '{}', note: 'Payload de la petición que originó la tarea'] + result jsonb [not null, default: '{}', note: 'Al completar incluye extraction_batch_id o conteos'] + error text + owner_id varchar(36) [note: 'Reservado para multi-usuario'] + created_at timestamptz [not null] + updated_at timestamptz [not null] + started_at timestamptz + finished_at timestamptz + + indexes { + job_id + type + status + owner_id + } + + Note: ''' + Módulo/resultado dentro de un Job. Una task representa la ejecución + de un servicio específico (conteo, extracción, limpieza). + job_id es nullable para compatibilidad con tasks creadas antes de jobs. + ''' +} + +// --------------------------------------------------------------------------- +// Counting — resultados del servicio de conteo de facturas médicas +// --------------------------------------------------------------------------- + +Table count_job_files { + id varchar(36) [pk, note: 'UUID v4'] + job_module_id varchar(36) [not null, ref: > tasks.id] + radicado varchar(255) [note: 'Identificador del radicado / carpeta de origen'] + filename varchar(512) [not null, note: 'Nombre del archivo PDF'] + file_path text [note: 'Ruta del archivo en el volumen montado (/data/...)'] + doc_type varchar(128) [note: 'Tipo de documento (ej. Factura, Historia_Clinica)'] + service_count integer [note: 'Número de servicios detectados por el modelo ML'] + services jsonb [not null, default: '[]', note: 'Lista de ServiceRow detectados por el modelo'] + status varchar(32) [not null, default: 'ok', note: 'ok | error'] + is_valid boolean [note: 'null=sin revisar | true=válido | false=rechazado por el usuario'] + error text [note: 'Mensaje de error si status=error'] + created_at timestamptz [not null] + + indexes { + job_module_id + radicado + status + is_valid + } + + Note: ''' + Resultado del servicio Go (services-count-facturas-medicas-serving) para + un PDF individual. El campo file_path permite que el servicio de extracción + procese el archivo directamente sin necesidad de re-subirlo. + is_valid=null hasta que el usuario revise; el umbral min_service_count + se aplica en el endpoint POST /tasks/extraction/from-count. + ''' } // --------------------------------------------------------------------------- @@ -32,18 +127,20 @@ TableGroup async_jobs { Table extraction_batches { id varchar(36) [pk, note: 'UUID v4'] + job_module_id varchar(36) [ref: > tasks.id, note: 'FK nullable — tarea que origino este lote'] status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed'] - root_path text [note: 'Ruta raíz de documentos procesados (IPS_DOCUMENT_ROOT)'] - request jsonb [not null, default: '{}', note: 'ExtractionRequest serializado (keywords, fields, doc_types, etc.)'] + root_path text [note: 'Ruta raíz de documentos (IPS_DOCUMENT_ROOT o "count://" si viene de conteo)'] + request jsonb [not null, default: '{}', note: 'ExtractionRequest serializado'] error text created_at timestamptz [not null] updated_at timestamptz [not null] indexes { + job_module_id status } - Note: 'Lote creado al iniciar una extracción; agrupa todas las filas extraídas de un procesamiento.' + Note: 'Lote creado al iniciar una extracción; agrupa todas las filas extraídas.' } Table extraction_rows { @@ -64,37 +161,6 @@ Table extraction_rows { Note: 'Una o más filas por radicado según flatten_extraction_result().' } -// --------------------------------------------------------------------------- -// Tareas asíncronas (API de extracción) -// --------------------------------------------------------------------------- - -Table tasks { - id varchar(36) [pk, note: 'UUID v4'] - type varchar(64) [not null, note: 'p. ej. extraction'] - status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed'] - title text - request jsonb [not null, default: '{}', note: 'Payload de la petición que originó la tarea'] - result jsonb [not null, default: '{}', note: 'Al completar incluye extraction_batch_id'] - error text - owner_id varchar(36) [note: 'Reservado para multi-usuario'] - created_at timestamptz [not null] - updated_at timestamptz [not null] - started_at timestamptz - finished_at timestamptz - - indexes { - type - status - owner_id - } - - Note: ''' - Cola de trabajos en segundo plano del servicio de extracción. - No hay FK a extraction_batches; el vínculo se guarda en - result.extraction_batch_id al finalizar la tarea. - ''' -} - // --------------------------------------------------------------------------- // Limpieza // --------------------------------------------------------------------------- @@ -103,7 +169,7 @@ Table cleaning_jobs { id varchar(36) [pk, note: 'UUID v4'] extraction_batch_id varchar(36) [not null, ref: > extraction_batches.id] status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed'] - request jsonb [not null, default: '{}', note: 'CleaningRequest (rules_per_column, cop_mod_*, etc.)'] + request jsonb [not null, default: '{}', note: 'CleaningRequest (rules_per_column, etc.)'] log jsonb [not null, default: '[]', note: 'Lista de mensajes de auditoría del job'] error text created_at timestamptz [not null] @@ -130,11 +196,5 @@ Table cleaned_rows { extraction_batch_id } - Note: 'Salida del servicio de limpieza; reemplaza las filas previas del mismo job en cada ejecución.' + Note: 'Salida del servicio de limpieza.' } - -// --------------------------------------------------------------------------- -// Relaciones lógicas (sin FK en BD) -// --------------------------------------------------------------------------- - -// tasks.result.extraction_batch_id -> extraction_batches.id (referencia en JSON) diff --git a/services/api_gateway/README.md b/services/api_gateway/README.md index de32f3b..7fcb649 100644 --- a/services/api_gateway/README.md +++ b/services/api_gateway/README.md @@ -1,29 +1,66 @@ -# IPSExtractor API Gateway +# api_gateway -Gateway HTTP opcional. Esta carpeta puede vivir como repo propio: solo depende -de HTTP para hablar con `extraction` y `cleaning`; no importa codigo interno de -esos servicios. +Puerta de enlace HTTP (FastAPI) que actúa como punto de entrada único para el sistema IPSExtractor. +Enruta las peticiones del cliente hacia los microservicios internos sin lógica de negocio propia. -## Estructura +--- -```text -api/ Endpoints FastAPI y proxy HTTP -requirements.txt +## Puerto + +**8000** + +--- + +## Enrutamiento + +| Ruta cliente | Servicio destino | Descripción | +|--------------|------------------|-------------| +| `GET /health` | Agrega `/health` de los tres servicios | Estado del sistema | +| `GET /models` | `extraction:8001/models` | Modelos LLM disponibles | +| `GET /config` | `extraction:8001/config` + `cleaning:8002/rules` | Configuración | +| `/extraction/{path}` | `extraction:8001/{path}` | Extracción (quita prefijo) | +| `/tasks/{path}` | `extraction:8001/tasks/{path}` | Tareas de extracción | +| `/jobs/{path}` | `extraction:8001/jobs/{path}` | Jobs | +| `/count-jobs/{path}` | `extraction:8001/count-jobs/{path}` | Jobs de conteo | +| `/cleaning/{path}` | `cleaning:8002/{path}` | Limpieza | +| `/count/{path}` | `count-api:8080/{path}` | Conteo OCR+ML (quita prefijo) | + +El proxy reenvía método, query string, headers y body sin modificaciones. +Timeout: ilimitado (para uploads grandes). + +--- + +## Variables de entorno + +| Variable | Default | Descripción | +|----------|---------|-------------| +| `EXTRACTION_URL` | `http://extraction:8001` | URL interna del servicio extraction | +| `CLEANING_URL` | `http://cleaning:8002` | URL interna del servicio cleaning | +| `COUNT_URL` | `http://count-api:8080` | URL interna del servicio count-api | + +--- + +## CORS + +Solo permite orígenes `localhost` y `127.0.0.1`. Sin autenticación. + +--- + +## Estructura del código + +``` +api/ + main.py — FastAPI app con proxy HTTP (httpx) Dockerfile +requirements.txt — fastapi, httpx, uvicorn ``` -## Desarrollo local +--- + +## Arrancar localmente ```bash -pip install -r requirements.txt -uvicorn api.main:app --host 0.0.0.0 --port 8000 +docker compose up --build api_gateway ``` -## Endpoints principales - -- `GET /health` -- `GET /models` -- `GET /config` -- `ANY /extraction/{path}` -- `ANY /cleaning/{path}` - +API disponible en `http://localhost:8000`. diff --git a/services/api_gateway/api/main.py b/services/api_gateway/api/main.py index 05f3cb5..949fa59 100644 --- a/services/api_gateway/api/main.py +++ b/services/api_gateway/api/main.py @@ -11,6 +11,7 @@ from fastapi.middleware.cors import CORSMiddleware EXTRACTION_URL = os.getenv("EXTRACTION_URL", "http://extraction:8001").rstrip("/") CLEANING_URL = os.getenv("CLEANING_URL", "http://cleaning:8002").rstrip("/") +COUNT_URL = os.getenv("COUNT_URL", "http://count-api:8080").rstrip("/") app = FastAPI(title="IPSExtractor API Gateway", version="1.0.0") @@ -28,11 +29,13 @@ async def health() -> dict[str, Any]: async with httpx.AsyncClient(timeout=10.0) as client: extraction = await _safe_health(client, EXTRACTION_URL) cleaning = await _safe_health(client, CLEANING_URL) + counting = await _safe_health(client, COUNT_URL) return { "status": "ok" if extraction["ok"] and cleaning["ok"] else "degraded", "services": { "extraction": extraction, "cleaning": cleaning, + "counting": counting, }, } @@ -73,6 +76,31 @@ async def cleaning_proxy(path: str, request: Request) -> Response: return await _proxy_request(request, f"{CLEANING_URL}/{path}") +@app.api_route("/count/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"]) +async def count_proxy(path: str, request: Request) -> Response: + return await _proxy_request(request, f"{COUNT_URL}/{path}") + + +@app.api_route("/count-jobs", methods=["GET", "POST"]) +async def count_jobs_root_proxy(request: Request) -> Response: + return await _proxy_request(request, f"{EXTRACTION_URL}/count-jobs") + + +@app.api_route("/count-jobs/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"]) +async def count_jobs_proxy(path: str, request: Request) -> Response: + return await _proxy_request(request, f"{EXTRACTION_URL}/count-jobs/{path}") + + +@app.api_route("/jobs", methods=["GET"]) +async def jobs_root_proxy(request: Request) -> Response: + return await _proxy_request(request, f"{EXTRACTION_URL}/jobs") + + +@app.api_route("/jobs/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"]) +async def jobs_proxy(path: str, request: Request) -> Response: + return await _proxy_request(request, f"{EXTRACTION_URL}/jobs/{path}") + + async def _safe_health(client: httpx.AsyncClient, base_url: str) -> dict[str, Any]: try: response = await client.get(f"{base_url}/health") diff --git a/services/cleaning/README.md b/services/cleaning/README.md index 0ec8191..e5b2b77 100644 --- a/services/cleaning/README.md +++ b/services/cleaning/README.md @@ -1,32 +1,96 @@ -# IPSExtractor Cleaning Service +# cleaning -Microservicio independiente de limpieza. Esta carpeta puede vivir como repo -propio: contiene su API REST, reglas de limpieza, infraestructura, dependencias -y Dockerfile sin importar codigo de otros servicios. +Microservicio Python (FastAPI) de limpieza y normalización de datos extraídos. +Recibe lotes de filas extraídas por el servicio `extraction` y aplica reglas configurables de transformación de columnas. -## Estructura +--- -```text -api/ Endpoints FastAPI -domain/ Reglas y analisis de limpieza -infrastructure/ Persistencia SQLAlchemy del servicio -requirements.txt -Dockerfile -``` +## Puerto -## Desarrollo local +**8002** + +--- + +## Endpoints + +| Método | Ruta | Descripción | +|--------|------|-------------| +| `GET` | `/health` | Liveness | +| `GET` | `/rules` | Lista de reglas de limpieza disponibles | +| `GET` | `/batches/{batch_id}/analysis` | Análisis de columnas de un lote de extracción | +| `POST` | `/jobs` | Crear job de limpieza | +| `GET` | `/jobs/{job_id}` | Estado y detalle del job | +| `GET` | `/jobs/{job_id}/rows` | Filas limpias del job | + +--- + +## Crear un job de limpieza ```bash +POST /jobs +{ + "extraction_batch_id": "...", + "rules_per_column": { + "valor": ["to_integer"], + "fecha": ["date_to_ddmmyyyy"], + "codigo": ["strip_spaces"] + } +} +``` + +Las reglas disponibles se listan en `GET /rules`. + +--- + +## Reglas de limpieza + +Definidas en `domain/csv_cleaning.py`. Cada regla es una función que transforma los valores de una columna. Se pueden encadenar múltiples reglas por columna. + +--- + +## Variables de entorno + +| Variable | Descripción | +|----------|-------------| +| `DATABASE_URL` | PostgreSQL (`postgresql+psycopg://...`) | + +--- + +## Modelo de datos + +``` +extraction_batches — lote de filas de extracción (leído, no modificado) + extraction_rows — filas originales + +cleaning_jobs — job de limpieza (FK → extraction_batches) + cleaned_rows — filas transformadas +``` + +--- + +## Estructura del código + +``` +api/ + main.py — FastAPI app con todos los endpoints +domain/ + csv_cleaning.py — reglas de limpieza y función de análisis +infrastructure/ + database.py — modelos SQLAlchemy y sesión +Dockerfile +requirements.txt +``` + +--- + +## Arrancar localmente + +```bash +# Con Docker Compose (desde la raíz del repositorio): +docker compose up --build cleaning + +# Sin Docker: +cd services/cleaning pip install -r requirements.txt uvicorn api.main:app --host 0.0.0.0 --port 8002 ``` - -## Endpoints principales - -- `GET /health` -- `GET /rules` -- `GET /batches/{batch_id}/analysis` -- `POST /jobs` -- `GET /jobs/{job_id}` -- `GET /jobs/{job_id}/rows` - diff --git a/services/db/Dockerfile b/services/db/Dockerfile new file mode 100644 index 0000000..e3cb30b --- /dev/null +++ b/services/db/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD ["python", "-m", "run_migrations"] diff --git a/services/db/README.md b/services/db/README.md new file mode 100644 index 0000000..ab96850 --- /dev/null +++ b/services/db/README.md @@ -0,0 +1,66 @@ +# db (migraciones) + +Contenedor de un solo uso que aplica las migraciones de base de datos con Alembic. +Se ejecuta al arrancar el stack y termina con código 0 al completar. El orquestador Docker reinicia en caso de error. + +--- + +## Comportamiento + +1. Conecta a PostgreSQL. +2. Adquiere un advisory lock para evitar ejecuciones concurrentes. +3. Aplica todas las migraciones pendientes. +4. Termina. + +Los demás servicios tienen `depends_on: db-migrations: condition: service_completed_successfully`, por lo que no arrancan hasta que las migraciones terminen. + +--- + +## Variables de entorno + +| Variable | Descripción | +|----------|-------------| +| `DATABASE_URL` | PostgreSQL (`postgresql+psycopg://...`) | + +--- + +## Esquema de base de datos + +``` +jobs — contenedor de un trabajo del usuario (multimódulo) +tasks — ejecución de un módulo dentro de un job + type: counting | extraction | cleaning +count_job_files — resultado de conteo por PDF (job_module_id → tasks.id) +extraction_batches — lote de extracción (job_module_id → tasks.id) +extraction_rows — fila extraída (batch_id → extraction_batches.id) +cleaning_jobs — job de limpieza (task_id → tasks.id) +cleaned_rows — fila limpiada (cleaning_job_id → cleaning_jobs.id) +``` + +Documentación completa en [`docs/database.dbml`](../../docs/database.dbml). + +--- + +## Estructura del código + +``` +Dockerfile — imagen Python slim + alembic + psycopg +alembic.ini — configuración de Alembic +alembic/ + env.py — entorno de migraciones con advisory lock + versions/ + 0001_initial_schema.py + 0002_add_jobs_and_count.py +run_migrations.py — script principal con manejo de errores +requirements.txt +``` + +--- + +## Añadir una migración + +```bash +# Generar una nueva revisión +docker run --rm -e DATABASE_URL=... ipsextractor-db-migrations \ + alembic revision --autogenerate -m "descripcion" +``` diff --git a/services/db/alembic.ini b/services/db/alembic.ini new file mode 100644 index 0000000..54c0ad6 --- /dev/null +++ b/services/db/alembic.ini @@ -0,0 +1,39 @@ +[alembic] +script_location = alembic +prepend_sys_path = . +version_path_separator = os +sqlalchemy.url = postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/services/db/alembic/env.py b/services/db/alembic/env.py new file mode 100644 index 0000000..4032c02 --- /dev/null +++ b/services/db/alembic/env.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import os +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +database_url = os.environ.get("DATABASE_URL") or config.get_main_option("sqlalchemy.url") +if database_url: + config.set_main_option("sqlalchemy.url", database_url) + +target_metadata = None + + +def run_migrations_offline() -> None: + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/services/db/alembic/script.py.mako b/services/db/alembic/script.py.mako new file mode 100644 index 0000000..b1f8b89 --- /dev/null +++ b/services/db/alembic/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from __future__ import annotations + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/services/db/alembic/versions/0001_initial_schema.py b/services/db/alembic/versions/0001_initial_schema.py new file mode 100644 index 0000000..26f2bde --- /dev/null +++ b/services/db/alembic/versions/0001_initial_schema.py @@ -0,0 +1,124 @@ +"""Schema inicial: extraction_batches, tasks, extraction_rows, cleaning_jobs, cleaned_rows. + +Revision ID: 0001 +Revises: +Create Date: 2026-07-05 + +""" +from __future__ import annotations + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0001" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + conn = op.get_bind() + inspector = sa.inspect(conn) + existing = set(inspector.get_table_names()) + + if "extraction_batches" not in existing: + op.create_table( + "extraction_batches", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("status", sa.String(32), nullable=False, server_default="pending"), + sa.Column("root_path", sa.Text, nullable=True), + sa.Column("request", sa.JSON, nullable=False, server_default="{}"), + sa.Column("error", sa.Text, nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index("ix_extraction_batches_status", "extraction_batches", ["status"]) + + if "tasks" not in existing: + op.create_table( + "tasks", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("type", sa.String(64), nullable=False), + sa.Column("status", sa.String(32), nullable=False, server_default="pending"), + sa.Column("title", sa.Text, nullable=True), + sa.Column("request", sa.JSON, nullable=False, server_default="{}"), + sa.Column("result", sa.JSON, nullable=False, server_default="{}"), + sa.Column("error", sa.Text, nullable=True), + sa.Column("owner_id", sa.String(36), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index("ix_tasks_type", "tasks", ["type"]) + op.create_index("ix_tasks_status", "tasks", ["status"]) + op.create_index("ix_tasks_owner_id", "tasks", ["owner_id"]) + + if "extraction_rows" not in existing: + op.create_table( + "extraction_rows", + sa.Column("id", sa.Integer, primary_key=True, autoincrement=True), + sa.Column( + "batch_id", sa.String(36), + sa.ForeignKey("extraction_batches.id"), nullable=False, + ), + sa.Column("radicado", sa.String(255), nullable=True), + sa.Column("source_pdf", sa.Text, nullable=True), + sa.Column("row_index", sa.Integer, nullable=False, server_default="0"), + sa.Column("row_json", sa.JSON, nullable=False, server_default="{}"), + sa.Column("raw_json", sa.JSON, nullable=False, server_default="{}"), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index("ix_extraction_rows_batch_id", "extraction_rows", ["batch_id"]) + op.create_index("ix_extraction_rows_radicado", "extraction_rows", ["radicado"]) + + if "cleaning_jobs" not in existing: + op.create_table( + "cleaning_jobs", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column( + "extraction_batch_id", sa.String(36), + sa.ForeignKey("extraction_batches.id"), nullable=False, + ), + sa.Column("status", sa.String(32), nullable=False, server_default="pending"), + sa.Column("request", sa.JSON, nullable=False, server_default="{}"), + sa.Column("log", sa.JSON, nullable=False, server_default="[]"), + sa.Column("error", sa.Text, nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index( + "ix_cleaning_jobs_extraction_batch_id", "cleaning_jobs", ["extraction_batch_id"] + ) + op.create_index("ix_cleaning_jobs_status", "cleaning_jobs", ["status"]) + + if "cleaned_rows" not in existing: + op.create_table( + "cleaned_rows", + sa.Column("id", sa.Integer, primary_key=True, autoincrement=True), + sa.Column( + "cleaning_job_id", sa.String(36), + sa.ForeignKey("cleaning_jobs.id"), nullable=False, + ), + sa.Column( + "extraction_batch_id", sa.String(36), + sa.ForeignKey("extraction_batches.id"), nullable=False, + ), + sa.Column("row_index", sa.Integer, nullable=False, server_default="0"), + sa.Column("row_json", sa.JSON, nullable=False, server_default="{}"), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index("ix_cleaned_rows_cleaning_job_id", "cleaned_rows", ["cleaning_job_id"]) + op.create_index( + "ix_cleaned_rows_extraction_batch_id", "cleaned_rows", ["extraction_batch_id"] + ) + + +def downgrade() -> None: + op.drop_table("cleaned_rows") + op.drop_table("cleaning_jobs") + op.drop_table("extraction_rows") + op.drop_table("tasks") + op.drop_table("extraction_batches") diff --git a/services/db/alembic/versions/0002_add_jobs_and_count.py b/services/db/alembic/versions/0002_add_jobs_and_count.py new file mode 100644 index 0000000..b3d5a8a --- /dev/null +++ b/services/db/alembic/versions/0002_add_jobs_and_count.py @@ -0,0 +1,104 @@ +"""Agregar tabla jobs, count_job_files, job_id en tasks, job_module_id en extraction_batches. + +Revision ID: 0002 +Revises: 0001 +Create Date: 2026-07-05 + +""" +from __future__ import annotations + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0002" +down_revision: Union[str, None] = "0001" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _col_exists(conn: sa.engine.Connection, table: str, column: str) -> bool: + inspector = sa.inspect(conn) + return any(c["name"] == column for c in inspector.get_columns(table)) + + +def _table_exists(conn: sa.engine.Connection, table: str) -> bool: + inspector = sa.inspect(conn) + return table in inspector.get_table_names() + + +def upgrade() -> None: + conn = op.get_bind() + + # ── tabla jobs ────────────────────────────────────────────────────────── + if not _table_exists(conn, "jobs"): + op.create_table( + "jobs", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("status", sa.String(32), nullable=False, server_default="pending"), + sa.Column("title", sa.Text, nullable=True), + sa.Column("metadata", sa.JSON, nullable=False, server_default="{}"), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index("ix_jobs_status", "jobs", ["status"]) + + # ── columna job_id en tasks ────────────────────────────────────────────── + if not _col_exists(conn, "tasks", "job_id"): + op.add_column( + "tasks", + sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True), + ) + op.create_index("ix_tasks_job_id", "tasks", ["job_id"]) + + # ── tabla count_job_files ──────────────────────────────────────────────── + if not _table_exists(conn, "count_job_files"): + op.create_table( + "count_job_files", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column( + "job_module_id", sa.String(36), + sa.ForeignKey("tasks.id"), nullable=False, + ), + sa.Column("radicado", sa.String(255), nullable=True), + sa.Column("filename", sa.String(512), nullable=False), + sa.Column("file_path", sa.Text, nullable=True), + sa.Column("doc_type", sa.String(128), nullable=True), + sa.Column("service_count", sa.Integer, nullable=True), + sa.Column("services", sa.JSON, nullable=False, server_default="[]"), + sa.Column("status", sa.String(32), nullable=False, server_default="ok"), + sa.Column("is_valid", sa.Boolean, nullable=True), + sa.Column("error", sa.Text, nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index("ix_count_job_files_job_module_id", "count_job_files", ["job_module_id"]) + op.create_index("ix_count_job_files_radicado", "count_job_files", ["radicado"]) + op.create_index("ix_count_job_files_status", "count_job_files", ["status"]) + + # ── columna job_module_id en extraction_batches ────────────────────────── + if not _col_exists(conn, "extraction_batches", "job_module_id"): + op.add_column( + "extraction_batches", + sa.Column( + "job_module_id", sa.String(36), + sa.ForeignKey("tasks.id"), nullable=True, + ), + ) + op.create_index( + "ix_extraction_batches_job_module_id", "extraction_batches", ["job_module_id"] + ) + + +def downgrade() -> None: + conn = op.get_bind() + if _col_exists(conn, "extraction_batches", "job_module_id"): + op.drop_index("ix_extraction_batches_job_module_id", "extraction_batches") + op.drop_column("extraction_batches", "job_module_id") + if _table_exists(conn, "count_job_files"): + op.drop_table("count_job_files") + if _col_exists(conn, "tasks", "job_id"): + op.drop_index("ix_tasks_job_id", "tasks") + op.drop_column("tasks", "job_id") + if _table_exists(conn, "jobs"): + op.drop_table("jobs") diff --git a/services/db/requirements.txt b/services/db/requirements.txt new file mode 100644 index 0000000..d629798 --- /dev/null +++ b/services/db/requirements.txt @@ -0,0 +1,3 @@ +alembic==1.16.2 +psycopg[binary]==3.2.3 +SQLAlchemy==2.0.36 diff --git a/services/db/run_migrations.py b/services/db/run_migrations.py new file mode 100644 index 0000000..365aad6 --- /dev/null +++ b/services/db/run_migrations.py @@ -0,0 +1,55 @@ +"""Punto de entrada del servicio de migraciones. + +Se ejecuta como init container antes de que arranquen los demás servicios. +Aplica todas las migraciones pendientes hasta 'head' con bloqueo advisory +para evitar condiciones de carrera en despliegues con múltiples réplicas. +""" +from __future__ import annotations + +import os +import sys +import logging + +from sqlalchemy import create_engine, text +from alembic.config import Config as AlembicConfig +from alembic import command as alembic_command + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") +log = logging.getLogger(__name__) + +DATABASE_URL = os.environ.get( + "DATABASE_URL", + "postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor", +) + + +def main() -> None: + log.info("Conectando a la base de datos…") + engine = create_engine(DATABASE_URL, pool_pre_ping=True) + + alembic_cfg = AlembicConfig() + alembic_cfg.set_main_option("script_location", "alembic") + alembic_cfg.set_main_option("sqlalchemy.url", DATABASE_URL) + + if DATABASE_URL.startswith("postgresql"): + with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn: + conn.execute(text("SELECT pg_advisory_lock(74512001)")) + log.info("Bloqueo advisory adquirido — ejecutando migraciones…") + try: + alembic_command.upgrade(alembic_cfg, "head") + log.info("Migraciones completadas.") + finally: + conn.execute(text("SELECT pg_advisory_unlock(74512001)")) + else: + alembic_command.upgrade(alembic_cfg, "head") + log.info("Migraciones completadas.") + + engine.dispose() + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + log.error("Migraciones fallidas: %s", exc) + sys.exit(1) diff --git a/services/desktop_app/README.md b/services/desktop_app/README.md index 97e4376..8472419 100644 --- a/services/desktop_app/README.md +++ b/services/desktop_app/README.md @@ -1,48 +1,106 @@ -# IPSExtractor Desktop App +# desktop_app -Aplicacion desktop Windows/Mac para ejecutar el flujo de extraccion usando el -API Gateway de IPSExtractor. Esta carpeta funciona como repo independiente. +Aplicación de escritorio Electron + React para interactuar con el stack IPSExtractor. +Permite seleccionar carpetas locales, contar servicios en facturas médicas, revisar resultados y lanzar extracciones LLM. -## Requisitos +--- -- Node.js 20+ -- Backend levantado con `docker compose up --build` -- Electron (incluido en `npm run dev`) +## Flujos principales -No necesitas montar la carpeta del usuario en Docker. La app indexa los archivos -localmente y los sube uno a uno al servicio de extraccion. +### Conteo de facturas -## Desarrollo +1. Seleccionar carpeta con PDFs. +2. Los PDFs se envían en lotes de 10 a `POST /count/v1/count` (count-api). +3. Los resultados se guardan en PostgreSQL vía `POST /count-jobs` (extraction). +4. El usuario revisa y marca archivos como válidos/inválidos. + +### Extracción desde conteo + +1. Desde el wizard de conteo, clic en "Extraer archivos válidos". +2. Se llama `POST /tasks/extraction/from-count` con el `counting_task_id`. +3. El servicio extraction lee los PDFs desde el volumen `/data` y los procesa con LLM. + +### Extracción directa + +1. Seleccionar carpeta → discovery de campos → seleccionar campos → estimar costo. +2. Subir PDFs en lotes a `/tasks/{id}/ingest-many`. +3. Polling de progreso vía `GET /tasks/{id}`. +4. Exportar resultados como CSV. + +--- + +## Arquitectura técnica + +| Capa | Tecnología | +|------|-----------| +| Shell | Electron 35+ | +| UI | React 19 + TypeScript | +| Bundler | Vite | +| Estilos | CSS custom (sin framework UI) | +| IPC | `contextBridge` (preload.cjs) | + +### Comunicación con el backend + +Toda la comunicación pasa por el API Gateway (`http://localhost:8000`). +El cliente HTTP es [`src/renderer/lib/apiClient.ts`](src/renderer/lib/apiClient.ts). + +### Indexación de carpetas + +El proceso principal (Electron) indexa carpetas locales con [`electron/folderIndex.cjs`](electron/folderIndex.cjs): +- Modo `general`: lista todos los PDFs/TIFFs de la carpeta. +- Modo `radicados`: busca un PDF por tipo de documento dentro de subdirectorios con nombre de radicado. + +--- + +## Estructura del código + +``` +electron/ + main.cjs — proceso principal Electron + preload.cjs — bridge seguro main ↔ renderer (contextBridge) + folderIndex.cjs — indexación de carpetas locales +src/ + renderer/ + App.tsx — root: navegación y estado global + components/ + CountingWizard.tsx — wizard de conteo en 3 pasos + CountResultsTable.tsx — tabla de resultados con validación + CountingStepper.tsx — indicador de progreso + ExtractionWizard.tsx — wizard de extracción en 4 pasos + ExtractionStepper.tsx — indicador de progreso + lib/ + apiClient.ts — cliente HTTP hacia el API Gateway + payloads.ts — builders de FormData y JSON + pathMapping.ts — mapeo rutas locales ↔ contenedor + csv.ts — exportación CSV + styles.css — estilos globales + shared/ + types.ts — tipos compartidos main/renderer +``` + +--- + +## Variables de configuración (runtime) + +| Variable | Default | Descripción | +|----------|---------|-------------| +| `API Gateway URL` | `http://localhost:8000` | Configurable en la pantalla de Configuración | +| Ruta local de documentos | — | Seleccionada con el explorador de archivos | +| Ruta en contenedor | `/data` | Prefijo donde Docker monta los documentos | + +--- + +## Arrancar en desarrollo ```bash +cd services/desktop_app npm install npm run dev ``` -El gateway por defecto es `http://localhost:8000`. - -## Tests y build +## Build para producción ```bash -npm test -npm run build -npm run dist +npm run build # compila el renderer +npm run electron:build # empaqueta la app ``` - -## Flujo desktop - -1. Verificar conexion al gateway. -2. Seleccionar una carpeta local (Electron indexa PDF/TIFF). -3. Configurar modo `radicados` o `general`. -4. Descubrir campos (`POST /extraction/schemas/upload` con un archivo de muestra). -5. Seleccionar campos y alias. -6. Ejecutar extraccion (`POST /tasks/extraction` con `source=desktop` + ingestas). -7. Consultar el historial de tareas. -8. Ver filas extraidas desde la base de datos. -9. Exportar CSV local de forma opcional. - -Los PDFs permanecen en el disco del usuario. El servidor solo usa archivos -temporales durante cada ingest y guarda JSON en Postgres. - -La app no incluye limpieza, preclasificacion, postproceso ni dashboard completo -en esta version. diff --git a/services/desktop_app/src/renderer/App.tsx b/services/desktop_app/src/renderer/App.tsx index a8587ac..4151482 100644 --- a/services/desktop_app/src/renderer/App.tsx +++ b/services/desktop_app/src/renderer/App.tsx @@ -1,6 +1,6 @@ -import { Activity, AlertCircle, CheckCircle2, Clock, Database, Download, FileSearch, Loader2, RefreshCw, Server, Settings, XCircle } from "lucide-react"; +import { Activity, AlertCircle, CheckCircle2, Clock, Database, Download, FileSearch, Hash, Loader2, RefreshCw, Server, Settings, XCircle } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; -import type { DocumentType, ExtractionFormState, FieldSelection, ManifestEntry, RowsResponse, SchemaResponse, Task } from "../shared/types"; +import type { DocumentType, ExtractionFormState, ExtractionFromCountPayload, FieldSelection, ManifestEntry, RowsResponse, SchemaResponse, Task } from "../shared/types"; import { ApiClient } from "./lib/apiClient"; import { rowsToCsv } from "./lib/csv"; import { @@ -16,6 +16,7 @@ import { defaultAlias } from "./lib/schemaDisplay"; import { splitRadicados } from "./lib/pathMapping"; import { ExtractionWizard } from "./components/ExtractionWizard"; import type { ExtractionStep } from "./components/ExtractionStepper"; +import { CountingWizard } from "./components/CountingWizard"; const defaultForm: ExtractionFormState = { apiBaseUrl: localStorage.getItem("apiBaseUrl") || "http://localhost:8000", @@ -36,7 +37,7 @@ const defaultForm: ExtractionFormState = { }; type Status = "idle" | "ok" | "error" | "busy"; -type View = "extraction" | "tasks" | "settings"; +type View = "extraction" | "counting" | "tasks" | "settings"; function App() { const [view, setView] = useState("extraction"); @@ -59,6 +60,8 @@ function App() { const [error, setError] = useState(null); const [extractionStep, setExtractionStep] = useState(1); const [maxReachedStep, setMaxReachedStep] = useState(1); + const [pendingCountTaskId, setPendingCountTaskId] = useState(null); + const [pendingValidCount, setPendingValidCount] = useState(0); const client = useMemo(() => new ApiClient(form.apiBaseUrl), [form.apiBaseUrl]); const flattened = useMemo(() => (schemaResponse ? flattenSchema(schemaResponse) : { schema: {}, sample: null }), [schemaResponse]); @@ -393,12 +396,19 @@ function App() { }); } - const currentTitle = view === "extraction" ? "Nueva extraccion" : view === "tasks" ? "Tareas y resultados" : "Configuracion"; - const currentDescription = view === "extraction" - ? "Asistente paso a paso para extraer datos de tus documentos." - : view === "tasks" - ? "Consulta el historial de ejecuciones y las filas guardadas en la base de datos." - : "Administra la conexion al API Gateway local."; + const currentTitle = + view === "extraction" ? "Nueva extraccion" + : view === "counting" ? "Contar servicios" + : view === "tasks" ? "Tareas y resultados" + : "Configuracion"; + const currentDescription = + view === "extraction" + ? "Asistente paso a paso para extraer datos de tus documentos." + : view === "counting" + ? "Cuenta servicios en facturas medicas y selecciona archivos validos para extraer." + : view === "tasks" + ? "Consulta el historial de ejecuciones y las filas guardadas en la base de datos." + : "Administra la conexion al API Gateway local."; return (
@@ -416,6 +426,10 @@ function App() { Extraccion + + + +
+
+ + setMinServiceCount(Number(e.target.value))} + className="w-16 px-2 py-1 text-xs rounded border border-indigo-300 dark:border-indigo-600 bg-white dark:bg-slate-800" + /> +
+ +
+ + {error && ( +

+ + {error} +

+ )} + + ); +} + export default App; diff --git a/services/desktop_app/src/renderer/components/CountResultsTable.tsx b/services/desktop_app/src/renderer/components/CountResultsTable.tsx new file mode 100644 index 0000000..cf28465 --- /dev/null +++ b/services/desktop_app/src/renderer/components/CountResultsTable.tsx @@ -0,0 +1,201 @@ +import { AlertCircle, CheckCircle2, Filter, XCircle } from "lucide-react"; +import { useMemo, useState } from "react"; +import type { CountJobFile } from "../../shared/types"; + +interface Props { + files: CountJobFile[]; + minServiceCount: number; + onMinServiceCountChange: (v: number) => void; + onToggleValid: (file: CountJobFile, isValid: boolean | null) => void; + loading?: boolean; +} + +export function CountResultsTable({ + files, + minServiceCount, + onMinServiceCountChange, + onToggleValid, + loading, +}: Props) { + const [filter, setFilter] = useState<"all" | "valid" | "invalid" | "unreviewed">("all"); + + const maxCount = useMemo(() => Math.max(0, ...files.map((f) => f.service_count ?? 0)), [files]); + + const filtered = useMemo(() => { + return files.filter((f) => { + if (filter === "valid") return f.is_valid === true || (f.is_valid === null && (f.service_count ?? 0) >= minServiceCount); + if (filter === "invalid") return f.is_valid === false || (f.is_valid === null && (f.service_count ?? 0) < minServiceCount); + if (filter === "unreviewed") return f.is_valid === null; + return true; + }); + }, [files, filter, minServiceCount]); + + const summary = useMemo(() => { + const valid = files.filter( + (f) => f.is_valid === true || (f.is_valid === null && (f.service_count ?? 0) >= minServiceCount && f.status === "ok"), + ).length; + const errors = files.filter((f) => f.status === "error").length; + return { valid, errors, total: files.length }; + }, [files, minServiceCount]); + + function validityClass(f: CountJobFile): string { + if (f.status === "error") return "bg-red-50 dark:bg-red-900/20"; + if (f.is_valid === true) return "bg-green-50 dark:bg-green-900/20"; + if (f.is_valid === false) return "bg-red-50 dark:bg-red-900/20"; + if ((f.service_count ?? 0) >= minServiceCount) return "bg-green-50 dark:bg-green-900/20"; + return "bg-yellow-50 dark:bg-yellow-900/20"; + } + + function validityIcon(f: CountJobFile) { + if (f.status === "error") return ; + if (f.is_valid === true) return ; + if (f.is_valid === false) return ; + if ((f.service_count ?? 0) >= minServiceCount) return ; + return ; + } + + return ( +
+ {/* Summary */} +
+
+

{summary.total}

+

Total archivos

+
+
+

{summary.valid}

+

Válidos para extracción

+
+
+

{summary.errors}

+

Errores de procesamiento

+
+
+ + {/* Threshold slider */} +
+
+ + + ≥ {minServiceCount} + +
+ 0 ? maxCount : 20} + value={minServiceCount} + onChange={(e) => onMinServiceCountChange(Number(e.target.value))} + className="w-full accent-indigo-600" + disabled={loading} + /> +

+ Solo se extraerán facturas con {minServiceCount === 0 ? "cualquier número" : `≥ ${minServiceCount}`} de servicio(s). + {minServiceCount > 0 && ` Se excluyen ${files.filter((f) => (f.service_count ?? 0) < minServiceCount && f.is_valid !== true).length} archivo(s).`} +

+
+ + {/* Filter tabs */} +
+ {(["all", "valid", "invalid", "unreviewed"] as const).map((f) => ( + + ))} +
+ + {/* Table */} +
+
+ + + + + + + + + + + {filtered.length === 0 && ( + + + + )} + {filtered.map((f) => ( + + + + + + + ))} + +
ArchivoServiciosEstadoValidez
+ No hay archivos que mostrar +
+
+ {f.filename} +
+ {f.radicado && f.radicado !== f.filename && ( +
{f.radicado}
+ )} + {f.status === "error" && ( +
{f.error}
+ )} + 		+ {f.status === "error" ? ( + — + ) : ( + + {f.service_count ?? "?"} + + )} + +
{validityIcon(f)}
+ 		+ {f.status !== "error" && ( +
+ + +
+ )} +
+
+
+
+ ); +} diff --git a/services/desktop_app/src/renderer/components/CountingStepper.tsx b/services/desktop_app/src/renderer/components/CountingStepper.tsx new file mode 100644 index 0000000..39e0c7c --- /dev/null +++ b/services/desktop_app/src/renderer/components/CountingStepper.tsx @@ -0,0 +1,44 @@ +import { Check } from "lucide-react"; + +export type CountingStep = 1 | 2 | 3; + +const STEPS: Array<{ id: CountingStep; label: string }> = [ + { id: 1, label: "Carpeta" }, + { id: 2, label: "Contar" }, + { id: 3, label: "Resultados" }, +]; + +interface CountingStepperProps { + current: CountingStep; + maxReached: CountingStep; + onStepClick?: (step: CountingStep) => void; +} + +export function CountingStepper({ current, maxReached, onStepClick }: CountingStepperProps) { + return ( + + ); +} diff --git a/services/desktop_app/src/renderer/components/CountingWizard.tsx b/services/desktop_app/src/renderer/components/CountingWizard.tsx new file mode 100644 index 0000000..e8d8d7b --- /dev/null +++ b/services/desktop_app/src/renderer/components/CountingWizard.tsx @@ -0,0 +1,477 @@ +import { + AlertCircle, + ArrowRight, + CheckCircle2, + FileText, + FolderOpen, + Hash, + Loader2, + Play, + RotateCcw, +} from "lucide-react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import type { + CountJobFile, + CountSyncResult, + CountingFormState, + CreateCountJobPayload, + DocumentType, + ManifestEntry, + Task, +} from "../../shared/types"; +import { ApiClient } from "../lib/apiClient"; +import { CountResultsTable } from "./CountResultsTable"; +import { CountingStepper, type CountingStep } from "./CountingStepper"; + +declare const desktopApi: import("../../shared/types").DesktopBridge; + +interface Props { + client: ApiClient; + localRoot: string; + containerRoot: string; + onStartExtraction: (countingTaskId: string, validCount: number) => void; +} + +interface CountProgress { + processed: number; + total: number; + failed: number; +} + +const BATCH_SIZE = 10; + +export function CountingWizard({ client, localRoot, containerRoot, onStartExtraction }: Props) { + const [step, setStep] = useState(1); + const [maxReached, setMaxReached] = useState(1); + const [form, setForm] = useState({ + localRoot, + containerRoot, + includeRejected: false, + minServiceCount: 1, + }); + const [docTypes] = useState([ + { name: "Factura", keyword: "Factura_O_Documento_Equivalente" }, + ]); + const [manifest, setManifest] = useState([]); + const [progress, setProgress] = useState({ processed: 0, total: 0, failed: 0 }); + const [savedTask, setSavedTask] = useState(null); + const [savedFiles, setSavedFiles] = useState([]); + const [error, setError] = useState(null); + const [running, setRunning] = useState(false); + const abortRef = useRef(false); + + const indexLocalFolder = useCallback( + async (dir: string) => { + const entries = await desktopApi.indexFolder(dir, "general", docTypes, []); + setManifest(entries); + if (entries.length === 0) { + setError("No se encontraron PDF en la carpeta seleccionada."); + } else { + setError(null); + } + return entries; + }, + [docTypes], + ); + + useEffect(() => { + setForm((f) => ({ + ...f, + localRoot: localRoot || f.localRoot, + containerRoot: containerRoot || f.containerRoot, + })); + }, [localRoot, containerRoot]); + + useEffect(() => { + if (!localRoot) return; + let cancelled = false; + indexLocalFolder(localRoot).catch((e) => { + if (!cancelled) setError(String(e)); + }); + return () => { + cancelled = true; + }; + }, [localRoot, indexLocalFolder]); + + function goToStep(target: CountingStep) { + setStep(target); + setMaxReached((prev) => (target > prev ? target : prev)); + } + + const selectFolder = useCallback(async () => { + const dir = await desktopApi.selectDirectory(); + if (!dir) return; + setForm((f) => ({ ...f, localRoot: dir })); + try { + await indexLocalFolder(dir); + } catch (e) { + setError(String(e)); + } + }, [indexLocalFolder]); + + function localToContainer(localPath: string): string { + const root = form.localRoot.replace(/\\/g, "/"); + const container = form.containerRoot.replace(/\\/g, "/"); + const norm = localPath.replace(/\\/g, "/"); + return norm.startsWith(root) ? container + norm.slice(root.length) : norm; + } + + async function runCounting() { + if (!manifest.length) { + setError("No hay archivos en la carpeta seleccionada."); + return; + } + abortRef.current = false; + setRunning(true); + setError(null); + setSavedTask(null); + setSavedFiles([]); + setProgress({ processed: 0, total: manifest.length, failed: 0 }); + goToStep(2); + + const allResults: CountSyncResult[] = []; + + for (let i = 0; i < manifest.length; i += BATCH_SIZE) { + if (abortRef.current) break; + const batch = manifest.slice(i, i + BATCH_SIZE); + try { + const formData = new FormData(); + for (const entry of batch) { + const fileData = await desktopApi.readFile(entry.localPath); + const blob = new Blob([fileData.data], { type: "application/pdf" }); + formData.append("files", blob, fileData.filename); + } + const response = await client.countFiles(formData, false); + allResults.push(...response.results); + setProgress((p) => ({ + ...p, + processed: Math.min(p.processed + batch.length, manifest.length), + failed: p.failed + response.failed, + })); + } catch (e) { + for (const entry of batch) { + allResults.push({ + filename: entry.filename, + service_count: 0, + status: "error", + services: [], + error: String(e), + }); + } + setProgress((p) => ({ + ...p, + processed: Math.min(p.processed + batch.length, manifest.length), + failed: p.failed + batch.length, + })); + } + } + + try { + const payload: CreateCountJobPayload = { + title: `Conteo — ${form.localRoot.split(/[\\/]/).pop()} (${allResults.length} archivos)`, + results: allResults.map((r) => { + const entry = manifest.find((m) => m.filename === r.filename); + return { + radicado: entry?.radicado ?? null, + filename: r.filename, + file_path: entry ? localToContainer(entry.localPath) : null, + doc_type: entry?.docType ?? null, + service_count: r.status === "ok" ? r.service_count : null, + services: r.services ?? [], + status: r.status === "ok" ? "ok" : "error", + error: r.error ?? null, + }; + }), + }; + const task = await client.createCountJob(payload); + const files = await client.countJobFiles(task.id); + setSavedTask(task); + setSavedFiles(files); + } catch (e) { + setError(`Resultados procesados pero no se pudieron guardar en BD: ${e}`); + } + + setRunning(false); + goToStep(3); + } + + async function handleToggleValid(file: CountJobFile, isValid: boolean | null) { + if (!savedTask) return; + try { + const updated = await client.markCountFileValid(savedTask.id, file.id, isValid); + setSavedFiles((prev) => prev.map((f) => (f.id === updated.id ? updated : f))); + } catch (e) { + setError(String(e)); + } + } + + function handleStartExtraction() { + if (!savedTask) return; + const validCount = savedFiles.filter( + (f) => + f.status === "ok" && + f.is_valid !== false && + (f.service_count ?? 0) >= form.minServiceCount, + ).length; + onStartExtraction(savedTask.id, validCount); + } + + function resetWizard() { + setStep(1); + setMaxReached(1); + setManifest([]); + setSavedTask(null); + setSavedFiles([]); + setError(null); + setProgress({ processed: 0, total: 0, failed: 0 }); + } + + const canExtract = + savedTask !== null && + savedFiles.some( + (f) => + f.status === "ok" && + f.is_valid !== false && + (f.service_count ?? 0) >= form.minServiceCount, + ); + + const progressPercent = + progress.total > 0 ? Math.round((progress.processed / progress.total) * 100) : 0; + + return ( +
+ + +
+ {/* ── PASO 1: Carpeta ── */} + {step === 1 && ( +
+
+

Origen de las facturas

+

+ Selecciona la carpeta local que contiene los PDF de facturas medicas a analizar. + El servicio contara cuantos servicios vienen en cada documento usando OCR y ML. +

+
+ +
+ +
+
+ {form.localRoot ? ( + <> + + {form.localRoot} + + ) : ( + Ninguna carpeta seleccionada + )} +
+ +
+ + {manifest.length > 0 && ( +
+ + + {manifest.length} documento(s) encontrados y listos para contar + +
+ )} + {form.localRoot && manifest.length === 0 && !error && ( +

Buscando PDF en la carpeta…

+ )} +
+ +
+ + setForm((f) => ({ ...f, containerRoot: e.target.value }))} + placeholder="/data" + /> +

+ Prefijo de ruta dentro del contenedor Docker donde estan montados los archivos. +

+
+
+ )} + + {/* ── PASO 2: Contando ── */} + {step === 2 && ( +
+
+
+ {running ? : } +
+

+ {running ? "Analizando facturas…" : "Analisis completado"} +

+

+ {running + ? "Enviando documentos al servicio de conteo. Se analiza cada PDF con OCR y ML para detectar servicios medicos." + : "Todos los documentos fueron procesados. Revisa los resultados en el siguiente paso."} +

+
+ +
+
+ + {running ? "Procesando documentos" : "Procesados"} + + + {progress.processed} / {progress.total} + {progress.failed > 0 && ( + + {progress.failed} error(es) + + )} + +
+
+
+
+
+ +
+
+ {progress.total} + Total +
+
+ {progress.processed - progress.failed} + Exitosos +
+
+ {progress.failed} + Errores +
+
+
+ )} + + {/* ── PASO 3: Resultados ── */} + {step === 3 && ( +
+
+
+

+ Resultados del conteo + {savedFiles.length > 0 && ( + {savedFiles.length} archivos + )} +

+ +
+ + {savedTask && ( + setForm((f) => ({ ...f, minServiceCount: v }))} + onToggleValid={handleToggleValid} + loading={running} + /> + )} +
+ +
+ + {canExtract && ( +

+ Se procesaran los archivos con ≥ {form.minServiceCount} servicio(s) +

+ )} + {!canExtract && savedFiles.length > 0 && ( +

+ Ajusta el umbral de servicios minimos para habilitar la extraccion. +

+ )} +
+
+ )} +
+ + {/* ── Status / Errores ── */} + {error && ( +
+ + {error} +
+ )} + + {/* ── Footer de navegacion ── */} +
+ + +
+ {([1, 2, 3] as CountingStep[]).map((id) => ( +
+ + {step === 1 && ( + + )} + {step === 2 && ( + + )} + {step === 3 && ( + + )} +
+
+ ); +} diff --git a/services/desktop_app/src/renderer/lib/apiClient.ts b/services/desktop_app/src/renderer/lib/apiClient.ts index 4bb64ec..93001dd 100644 --- a/services/desktop_app/src/renderer/lib/apiClient.ts +++ b/services/desktop_app/src/renderer/lib/apiClient.ts @@ -1,4 +1,14 @@ -import type { ExtractionBatch, RowsResponse, SchemaResponse, Task } from "../../shared/types"; +import type { + CountJobFile, + CountSyncResponse, + CreateCountJobPayload, + ExtractionBatch, + ExtractionFromCountPayload, + Job, + RowsResponse, + SchemaResponse, + Task, +} from "../../shared/types"; export class ApiClient { constructor(private readonly baseUrl: string) {} @@ -89,6 +99,59 @@ export class ApiClient { return this.getJson(`/extraction/batches/${encodeURIComponent(batchId)}/rows?limit=${limit}&offset=${offset}`); } + // ── Jobs (contenedores multimodulo) ──────────────────────────────────────── + + async jobs(): Promise { + return this.getJson("/jobs"); + } + + async job(jobId: string): Promise { + return this.getJson(`/jobs/${encodeURIComponent(jobId)}`); + } + + async jobTasks(jobId: string): Promise { + return this.getJson(`/jobs/${encodeURIComponent(jobId)}/tasks`); + } + + // ── Count Jobs ───────────────────────────────────────────────────────────── + + /** Envía PDFs al servicio de conteo (síncrono). */ + async countFiles(form: FormData, includeRejected = false): Promise { + const url = includeRejected ? "/count/v1/count?include_rejected=true" : "/count/v1/count"; + return this.postForm(url, form); + } + + /** Persiste los resultados del conteo en la BD compartida. */ + async createCountJob(payload: CreateCountJobPayload): Promise { + return this.postJson("/count-jobs", payload); + } + + async countJobs(): Promise { + return this.getJson("/count-jobs"); + } + + async countJob(taskId: string): Promise { + return this.getJson(`/count-jobs/${encodeURIComponent(taskId)}`); + } + + async countJobFiles(taskId: string): Promise { + return this.getJson(`/count-jobs/${encodeURIComponent(taskId)}/files`); + } + + async markCountFileValid(taskId: string, fileId: string, isValid: boolean | null): Promise { + const response = await fetch(this.url(`/count-jobs/${encodeURIComponent(taskId)}/files/${encodeURIComponent(fileId)}`), { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ is_valid: isValid }), + }); + return this.handle(response); + } + + /** Crea una tarea de extracción usando los archivos válidos de un job de conteo. */ + async createExtractionFromCount(payload: ExtractionFromCountPayload): Promise { + return this.postJson("/tasks/extraction/from-count", payload); + } + private async handle(response: Response): Promise { const text = await response.text(); let data: unknown = null; diff --git a/services/desktop_app/src/shared/types.ts b/services/desktop_app/src/shared/types.ts index 7e22db6..9a0a682 100644 --- a/services/desktop_app/src/shared/types.ts +++ b/services/desktop_app/src/shared/types.ts @@ -83,6 +83,79 @@ export interface RowsResponse { offset: number; } +export interface Job { + id: string; + status: "pending" | "running" | "completed" | "failed" | string; + title: string | null; + metadata: Record; + created_at: string; + updated_at: string; +} + +export interface CountJobFile { + id: string; + job_module_id: string; + radicado: string | null; + filename: string; + file_path: string | null; + doc_type: string | null; + service_count: number | null; + services: Array<{ text: string; probability: number; is_service?: boolean | null }>; + status: "ok" | "error" | string; + is_valid: boolean | null; + error: string | null; + created_at: string; +} + +export interface CountJobFileCreateEntry { + radicado?: string | null; + filename: string; + file_path?: string | null; + doc_type?: string | null; + service_count?: number | null; + services?: Array<{ text: string; probability: number; is_service?: boolean | null }>; + status?: "ok" | "error"; + error?: string | null; +} + +export interface CreateCountJobPayload { + title?: string; + results: CountJobFileCreateEntry[]; +} + +export interface ExtractionFromCountPayload { + counting_task_id: string; + min_service_count: number; + fields: string[]; + alias_map: Record; + doc_types?: Array<{ name: string; keyword: string }>; + structure_mode?: "general" | "radicados"; + instruction?: string | null; + tiff_page_range?: [number, number] | null; + model?: string | null; +} + +export interface CountSyncResult { + filename: string; + service_count: number; + status: "ok" | "error"; + services: Array<{ text: string; probability: number; is_service?: boolean | null }>; + error?: string; +} + +export interface CountSyncResponse { + results: CountSyncResult[]; + processed: number; + failed: number; +} + +export interface CountingFormState { + localRoot: string; + containerRoot: string; + includeRejected: boolean; + minServiceCount: number; +} + export interface RuntimeConfig { documentRootLocal: string; documentRootContainer: string; diff --git a/services/extraction/README.md b/services/extraction/README.md index cfa316a..10303b8 100644 --- a/services/extraction/README.md +++ b/services/extraction/README.md @@ -1,45 +1,139 @@ -# IPSExtractor Extraction Service +# extraction -Microservicio independiente de extraccion. Esta carpeta puede vivir como repo -propio: contiene su API REST, dominio, infraestructura, dependencias y -Dockerfile sin importar codigo de otros servicios. +Servicio Python (FastAPI) de extracción de datos estructurados desde documentos médicos PDF. +Usa LLMs (compatible con OpenAI API) para descubrir campos, estimar costos y extraer datos por lotes. -## Estructura +También actúa como capa de persistencia para el flujo de conteo: guarda los resultados del `count-api` en PostgreSQL y orquesta la extracción de documentos validados. -```text -api/ Endpoints FastAPI -domain/ Logica de extraccion, prompts, estimacion y normalizacion -core/ Utilidades propias de PDF, texto y CSV -infrastructure/ Persistencia SQLAlchemy del servicio -requirements.txt -Dockerfile -``` - -## Desarrollo local - -```bash -pip install -r requirements.txt -uvicorn api.main:app --host 0.0.0.0 --port 8001 -``` - -## Poppler - -El contenedor instala `poppler-utils` para que `pdf2image` encuentre -`pdfinfo` y `pdftoppm` desde el PATH del sistema, sin configurar variables de -entorno. - -En desarrollo local Windows, el servicio usa automaticamente la carpeta incluida -en `dependencies/poppler-24.08.0/Library/bin`. +--- ## Endpoints principales -- `GET /health` -- `GET /config` -- `GET /models` -- `POST /schemas` -- `POST /schemas/upload` -- `POST /estimate` -- `POST /batches` -- `POST /tasks/extraction` -- `POST /tasks/{task_id}/ingest` -- `GET /tasks` +Puerto: **8001** + +### Salud y configuración + +| Método | Ruta | Descripción | +|--------|------|-------------| +| `GET` | `/health` | Liveness | +| `GET` | `/config` | Configuración LLM activa | +| `GET` | `/models` | Modelos disponibles en el backend LLM | + +### Descubrimiento de esquema + +| Método | Ruta | Descripción | +|--------|------|-------------| +| `POST` | `/schemas` | Descubrir campos desde ruta en disco | +| `POST` | `/schemas/upload` | Descubrir campos desde PDF subido | + +### Estimación de costo + +| Método | Ruta | Descripción | +|--------|------|-------------| +| `POST` | `/estimate/upload` | Estimar tokens/costo desde PDF subido | + +### Tareas de extracción + +| Método | Ruta | Descripción | +|--------|------|-------------| +| `POST` | `/tasks/extraction` | Crear tarea de extracción | +| `POST` | `/tasks/{id}/ingest` | Subir un PDF a la tarea | +| `POST` | `/tasks/{id}/ingest-many` | Subir varios PDFs en bulk | +| `GET` | `/tasks` | Listar tareas | +| `GET` | `/tasks/{id}` | Detalle de tarea | +| `GET` | `/tasks/{id}/rows` | Filas extraídas de la tarea | +| `POST` | `/tasks/extraction/from-count` | Crear extracción desde conteo previo | + +### Jobs (contenedor de tareas) + +| Método | Ruta | Descripción | +|--------|------|-------------| +| `GET` | `/jobs` | Listar jobs | +| `GET` | `/jobs/{id}` | Detalle de job | +| `GET` | `/jobs/{id}/tasks` | Tareas del job | + +### Conteo (persistencia desde count-api) + +| Método | Ruta | Descripción | +|--------|------|-------------| +| `POST` | `/count-jobs` | Guardar resultados de conteo en BD | +| `GET` | `/count-jobs` | Listar jobs de conteo | +| `GET` | `/count-jobs/{task_id}` | Detalle de job de conteo | +| `GET` | `/count-jobs/{task_id}/files` | Archivos del conteo | +| `PATCH` | `/count-jobs/{task_id}/files/{file_id}` | Marcar archivo como válido/inválido | + +--- + +## Flujo de datos + +``` +PDF local (desktop) → POST /tasks/{id}/ingest-many → procesamiento LLM + ↓ + filas en extraction_rows (PG) + +count-api results → POST /count-jobs → count_job_files (PG) + → POST /tasks/extraction/from-count → lee file_path desde /data +``` + +--- + +## Variables de entorno + +| Variable | Descripción | +|----------|-------------| +| `DATABASE_URL` | PostgreSQL (`postgresql+psycopg://...`) | +| `REDIS_URL` | Redis para cola ARQ (`redis://...`) | +| `EXTRACTOR_BASE_URL` | URL base del LLM (compatible OpenAI) | +| `EXTRACTOR_API_KEY` | API key del LLM extractor | +| `EXTRACTOR_MODEL` | Modelo de extracción (ej. `gpt-5.4-mini`) | +| `NORMALIZER_BASE_URL` | URL base del LLM normalizador | +| `NORMALIZER_API_KEY` | API key del LLM normalizador | +| `NORMALIZER_MODEL` | Modelo normalizador (ej. `gpt-5.4-mini`) | +| `EXTRACTOR_TIMEOUT_S` | Timeout por documento (default `120`) | +| `PDF_DPI` | DPI para renderizado PDF (default `150`) | +| `PDF_THREAD_COUNT` | Hilos de renderizado PDF (default `4`) | +| `CONCURRENCY_LIMIT` | Máximo de extracciones paralelas (default `50`) | +| `ALLOWED_DOCUMENT_TYPES` | Tipos de documento permitidos | +| `ASYNC_QUEUE` | `true` para encolar en ARQ/Redis (default: sync) | + +--- + +## Modelo de datos (PostgreSQL) + +``` +jobs — contenedor de tareas del usuario + tasks — una tarea por módulo (counting / extraction / cleaning) + count_job_files — resultados de conteo por PDF + extraction_batches — lotes de extracción + extraction_rows — filas extraídas por documento +``` + +--- + +## Estructura del código + +``` +api/ + main.py — FastAPI app (~1400 líneas), todos los endpoints + desktop_ingest.py — lógica de ingesta desde desktop Electron +domain/ — lógica de negocio: LLM, prompts, extractor, batch +core/ — utilidades PDF, CSV, texto +infrastructure/ — SQLAlchemy, migraciones, uploads temporales +worker/ — workers ARQ (procesamiento async) + main.py — punto de entrada del worker + tasks.py — tareas ARQ + settings.py — configuración del worker +Dockerfile +requirements.txt +``` + +--- + +## Arrancar localmente + +```bash +# Desde la raíz del repositorio: +docker compose up --build extraction worker +``` + +API disponible en `http://localhost:8001`. diff --git a/services/extraction/api/main.py b/services/extraction/api/main.py index 314945c..8843952 100644 --- a/services/extraction/api/main.py +++ b/services/extraction/api/main.py @@ -28,8 +28,10 @@ from domain.estimator import Estimator from domain.service import InvoiceProcessingService from domain.strategies import OpenAICompatibleExtractor from infrastructure.database import ( + CountJobFile, ExtractionBatch, ExtractionRow, + Job, Task, init_db, session_scope, @@ -938,3 +940,420 @@ def get_rows(batch_id: str, limit: int = 500, offset: int = 0) -> dict[str, Any] "limit": limit, "offset": offset, } + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Jobs (contenedores multimodulo) +# ═══════════════════════════════════════════════════════════════════════════════ + +class JobRead(BaseModel): + id: str + status: str + title: str | None + metadata: dict[str, Any] + created_at: str + updated_at: str + + +def serialize_job(job: Job) -> dict[str, Any]: + return { + "id": job.id, + "status": job.status, + "title": job.title, + "metadata": job.job_metadata or {}, + "created_at": job.created_at.isoformat(), + "updated_at": job.updated_at.isoformat(), + } + + +@app.get("/jobs", response_model=list[JobRead]) +def list_jobs() -> list[dict[str, Any]]: + with session_scope() as session: + jobs = session.scalars(select(Job).order_by(Job.created_at.desc())).all() + return [serialize_job(j) for j in jobs] + + +@app.get("/jobs/{job_id}", response_model=JobRead) +def get_job(job_id: str) -> dict[str, Any]: + with session_scope() as session: + job = session.get(Job, job_id) + if job is None: + raise HTTPException(status_code=404, detail="Job no encontrado") + return serialize_job(job) + + +@app.get("/jobs/{job_id}/tasks", response_model=list[TaskRead]) +def get_job_tasks(job_id: str) -> list[dict[str, Any]]: + with session_scope() as session: + job = session.get(Job, job_id) + if job is None: + raise HTTPException(status_code=404, detail="Job no encontrado") + tasks = session.scalars( + select(Task).where(Task.job_id == job_id).order_by(Task.created_at) + ).all() + return [serialize_task(t) for t in tasks] + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Count Jobs — persistir y gestionar resultados del servicio de conteo +# ═══════════════════════════════════════════════════════════════════════════════ + +class CountJobFileCreate(BaseModel): + radicado: str | None = None + filename: str + file_path: str | None = None + doc_type: str | None = None + service_count: int | None = None + services: list[dict[str, Any]] = Field(default_factory=list) + status: str = "ok" + error: str | None = None + + +class CreateCountJobRequest(BaseModel): + title: str | None = None + results: list[CountJobFileCreate] + + +class CountJobFileRead(BaseModel): + id: str + job_module_id: str + radicado: str | None + filename: str + file_path: str | None + doc_type: str | None + service_count: int | None + services: list[dict[str, Any]] + status: str + is_valid: bool | None + error: str | None + created_at: str + + +class UpdateCountJobFileRequest(BaseModel): + is_valid: bool | None + + +class ExtractionFromCountRequest(BaseModel): + counting_task_id: str + min_service_count: int = 0 + fields: list[str] = Field(default_factory=list) + alias_map: dict[str, str] = Field(default_factory=dict) + doc_types: list[DocumentType] | None = None + structure_mode: str = "general" + instruction: str | None = None + tiff_page_range: tuple[int, int] | None = None + retry_on_empty: bool = False + model: str | None = None + + +def serialize_count_file(f: CountJobFile) -> dict[str, Any]: + return { + "id": f.id, + "job_module_id": f.job_module_id, + "radicado": f.radicado, + "filename": f.filename, + "file_path": f.file_path, + "doc_type": f.doc_type, + "service_count": f.service_count, + "services": f.services or [], + "status": f.status, + "is_valid": f.is_valid, + "error": f.error, + "created_at": f.created_at.isoformat(), + } + + +@app.post("/count-jobs", response_model=TaskRead) +def create_count_job(request: CreateCountJobRequest) -> dict[str, Any]: + """Persiste los resultados del servicio de conteo de facturas en la BD compartida.""" + total = len(request.results) + with session_scope() as session: + job = Job( + status="completed", + title=request.title or f"Conteo ({total} archivos)", + updated_at=utcnow(), + ) + session.add(job) + session.flush() + + task = Task( + job_id=job.id, + type="counting", + status="completed", + title=job.title, + request={}, + result={ + "total": total, + "processed": total, + "ok": sum(1 for r in request.results if r.status == "ok"), + "failed": sum(1 for r in request.results if r.status != "ok"), + }, + started_at=utcnow(), + finished_at=utcnow(), + updated_at=utcnow(), + ) + session.add(task) + session.flush() + + for r in request.results: + session.add( + CountJobFile( + job_module_id=task.id, + radicado=r.radicado, + filename=r.filename, + file_path=r.file_path, + doc_type=r.doc_type, + service_count=r.service_count, + services=r.services, + status=r.status, + error=r.error, + ) + ) + session.flush() + return serialize_task(task) + + +@app.get("/count-jobs", response_model=list[TaskRead]) +def list_count_jobs() -> list[dict[str, Any]]: + with session_scope() as session: + tasks = session.scalars( + select(Task) + .where(Task.type == "counting") + .order_by(Task.created_at.desc()) + ).all() + return [serialize_task(t) for t in tasks] + + +@app.get("/count-jobs/{task_id}", response_model=TaskRead) +def get_count_job(task_id: str) -> dict[str, Any]: + with session_scope() as session: + task = session.get(Task, task_id) + if task is None or task.type != "counting": + raise HTTPException(status_code=404, detail="Tarea de conteo no encontrada") + return serialize_task(task) + + +@app.get("/count-jobs/{task_id}/files", response_model=list[CountJobFileRead]) +def list_count_job_files(task_id: str) -> list[dict[str, Any]]: + with session_scope() as session: + task = session.get(Task, task_id) + if task is None or task.type != "counting": + raise HTTPException(status_code=404, detail="Tarea de conteo no encontrada") + files = session.scalars( + select(CountJobFile) + .where(CountJobFile.job_module_id == task_id) + .order_by(CountJobFile.created_at) + ).all() + return [serialize_count_file(f) for f in files] + + +@app.patch("/count-jobs/{task_id}/files/{file_id}", response_model=CountJobFileRead) +def update_count_job_file( + task_id: str, file_id: str, request: UpdateCountJobFileRequest +) -> dict[str, Any]: + with session_scope() as session: + f = session.get(CountJobFile, file_id) + if f is None or f.job_module_id != task_id: + raise HTTPException(status_code=404, detail="Archivo no encontrado") + f.is_valid = request.is_valid + session.flush() + return serialize_count_file(f) + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Extraccion desde resultados de conteo (sin re-subir archivos) +# ═══════════════════════════════════════════════════════════════════════════════ + +def run_from_count_task(task_id: str) -> None: + """Procesa archivos validos de un job de conteo en background (lee desde el volumen /data).""" + import logging as _logging + _log = _logging.getLogger("extraction.from_count") + + from api.desktop_ingest import _service_for_model, mark_task_completed_if_done + + with session_scope() as session: + task = session.get(Task, task_id) + if task is None: + return + files_data: list[dict[str, Any]] = list((task.result or {}).get("files", [])) + req = dict(task.request or {}) + batch_id = str((task.result or {}).get("extraction_batch_id") or "") + + model_name: str | None = req.get("model") + instruction: str | None = req.get("instruction") + tiff_page_range = req.get("tiff_page_range") + + for file_data in files_data: + file_path: str | None = file_data.get("file_path") + filename = str(file_data.get("filename") or "documento.pdf") + radicado = str(file_data.get("radicado") or file_data.get("filename") or "") + doc_type: str | None = file_data.get("doc_type") + + if not file_path or not __import__("os").path.exists(file_path): + _log.warning("Archivo no disponible: %s", file_path or filename) + with session_scope() as session: + task_obj = session.get(Task, task_id) + if task_obj is None: + return + result = dict(task_obj.result or {}) + errors = list(result.get("errors") or []) + errors.append(f"Archivo no encontrado: {filename}") + result["errors"] = errors + result["processed"] = int(result.get("processed") or 0) + 1 + task_obj.result = result + task_obj.updated_at = utcnow() + mark_task_completed_if_done(session, task_obj, batch_id) + continue + + try: + svc = _service_for_model(processing_service, model_name) + with session_scope() as session: + task_obj = session.get(Task, task_id, with_for_update=True) + if task_obj is None: + return + from api.desktop_ingest import process_desktop_ingest + process_desktop_ingest( + svc, session, + task=task_obj, + batch_id=batch_id, + temp_path=file_path, + radicado=radicado, + filename=filename, + doc_type=doc_type, + instruction=instruction, + tiff_page_range=tiff_page_range, + model=model_name, + ) + mark_task_completed_if_done(session, task_obj, batch_id) + except Exception as exc: + _log.exception("Error procesando %s: %s", filename, exc) + with session_scope() as session: + task_obj = session.get(Task, task_id) + if task_obj is None: + return + result = dict(task_obj.result or {}) + errors = list(result.get("errors") or []) + errors.append(f"{filename}: {exc}") + result["errors"] = errors + result["processed"] = int(result.get("processed") or 0) + 1 + task_obj.result = result + task_obj.updated_at = utcnow() + mark_task_completed_if_done(session, task_obj, batch_id) + + +@app.post("/tasks/extraction/from-count", response_model=TaskRead) +def create_extraction_from_count( + request: ExtractionFromCountRequest, background_tasks: BackgroundTasks +) -> dict[str, Any]: + """Crea una tarea de extraccion usando archivos validos de un job de conteo previo. + + No requiere subir archivos: usa las rutas almacenadas en count_job_files. + """ + if not request.fields: + raise HTTPException( + status_code=400, + detail="Envia fields. Usa POST /extraction/schemas/upload para descubrir campos.", + ) + + with session_scope() as session: + counting_task = session.get(Task, request.counting_task_id) + if counting_task is None or counting_task.type != "counting": + raise HTTPException(status_code=404, detail="Tarea de conteo no encontrada") + + all_files = session.scalars( + select(CountJobFile).where( + CountJobFile.job_module_id == request.counting_task_id + ) + ).all() + + valid_files = [ + f for f in all_files + if f.status == "ok" + and f.is_valid is not False + and (f.service_count is None or f.service_count >= request.min_service_count) + ] + + if not valid_files: + raise HTTPException( + status_code=400, + detail="No hay archivos validos para extraer con los criterios indicados.", + ) + + files_data = [ + { + "id": f.id, + "radicado": f.radicado or f.filename, + "filename": f.filename, + "file_path": f.file_path, + "doc_type": f.doc_type, + "service_count": f.service_count, + } + for f in valid_files + ] + total = len(files_data) + + job = Job( + status="running", + title=f"Extraccion desde conteo ({total} archivos)", + updated_at=utcnow(), + ) + session.add(job) + session.flush() + + batch = ExtractionBatch( + status="running", + root_path="count://", + request={ + "source": "from-count", + "counting_task_id": request.counting_task_id, + "fields": request.fields, + "alias_map": request.alias_map, + "min_service_count": request.min_service_count, + "structure_mode": request.structure_mode, + }, + updated_at=utcnow(), + ) + session.add(batch) + session.flush() + + task = Task( + job_id=job.id, + type="extraction", + status="running", + title=job.title, + request={ + "source": "from-count", + "counting_task_id": request.counting_task_id, + "fields": request.fields, + "alias_map": request.alias_map, + "doc_types": [dt.model_dump() for dt in request.doc_types or []], + "structure_mode": request.structure_mode, + "instruction": request.instruction, + "tiff_page_range": list(request.tiff_page_range) if request.tiff_page_range else None, + "min_service_count": request.min_service_count, + "model": request.model, + }, + result={ + "source": "from-count", + "extraction_batch_id": batch.id, + "processed": 0, + "total": total, + "row_count": 0, + "errors": [], + "files": files_data, + "partials": {}, + }, + started_at=utcnow(), + updated_at=utcnow(), + ) + session.add(task) + session.flush() + + batch.job_module_id = task.id + session.flush() + + serialized = serialize_task(task) + task_id = task.id + + background_tasks.add_task(run_from_count_task, task_id) + return serialized diff --git a/services/extraction/infrastructure/database.py b/services/extraction/infrastructure/database.py index 697580e..9f9c155 100644 --- a/services/extraction/infrastructure/database.py +++ b/services/extraction/infrastructure/database.py @@ -9,6 +9,7 @@ from typing import Any, Iterator from sqlalchemy import ( JSON, + Boolean, DateTime, ForeignKey, Integer, @@ -42,6 +43,20 @@ def new_id() -> str: return str(uuid.uuid4()) +class Job(Base): + """Contenedor de trabajo del usuario; puede abarcar múltiples módulos/servicios.""" + __tablename__ = "jobs" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + status: Mapped[str] = mapped_column(String(32), default="pending", index=True) + title: Mapped[str | None] = mapped_column(Text, nullable=True) + # "metadata" es reservado por DeclarativeBase; el atributo Python usa job_metadata + # pero la columna en la BD sigue llamándose "metadata". + job_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) + + class ExtractionBatch(Base): __tablename__ = "extraction_batches" @@ -50,14 +65,21 @@ class ExtractionBatch(Base): root_path: Mapped[str | None] = mapped_column(Text, nullable=True) request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) error: Mapped[str | None] = mapped_column(Text, nullable=True) + job_module_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("tasks.id"), nullable=True, index=True + ) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) class Task(Base): + """Módulo/resultado dentro de un Job; un job puede tener N tasks (una por servicio).""" __tablename__ = "tasks" id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + job_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("jobs.id"), nullable=True, index=True + ) type: Mapped[str] = mapped_column(String(64), index=True) status: Mapped[str] = mapped_column(String(32), default="pending", index=True) title: Mapped[str | None] = mapped_column(Text, nullable=True) @@ -71,6 +93,26 @@ class Task(Base): finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) +class CountJobFile(Base): + """Resultado del servicio de conteo para un PDF individual dentro de una tarea de conteo.""" + __tablename__ = "count_job_files" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + job_module_id: Mapped[str] = mapped_column( + String(36), ForeignKey("tasks.id"), index=True + ) + radicado: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True) + filename: Mapped[str] = mapped_column(String(512)) + file_path: Mapped[str | None] = mapped_column(Text, nullable=True) + doc_type: Mapped[str | None] = mapped_column(String(128), nullable=True) + service_count: Mapped[int | None] = mapped_column(Integer, nullable=True) + services: Mapped[list[Any]] = mapped_column(JSON, default=list) + status: Mapped[str] = mapped_column(String(32), default="ok", index=True) + is_valid: Mapped[bool | None] = mapped_column(Boolean, nullable=True) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) + + class ExtractionRow(Base): __tablename__ = "extraction_rows" @@ -109,16 +151,13 @@ class CleanedRow(Base): def init_db() -> None: - if DATABASE_URL.startswith("postgresql"): - with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection: - connection.execute(text("SELECT pg_advisory_lock(74512001)")) - try: - Base.metadata.create_all(bind=connection) - finally: - connection.execute(text("SELECT pg_advisory_unlock(74512001)")) - return + """Verifica conectividad con la BD. - Base.metadata.create_all(bind=engine) + Las migraciones son responsabilidad exclusiva de services/db (init container). + Este servicio asume que el esquema ya está actualizado al arrancar. + """ + with engine.connect() as conn: + conn.execute(text("SELECT 1")) @contextmanager diff --git a/services/services-count-facturas-medicas-serving/Dockerfile b/services/services-count-facturas-medicas-serving/Dockerfile index 8bae710..d82fbfe 100644 --- a/services/services-count-facturas-medicas-serving/Dockerfile +++ b/services/services-count-facturas-medicas-serving/Dockerfile @@ -1,51 +1,59 @@ # syntax=docker/dockerfile:1 -FROM golang:1.23-bookworm AS builder +# ── Etapa 1: Descargar ONNX Runtime ─────────────────────────────────────────── +FROM debian:bookworm-slim AS onnx-download + +ARG ONNX_VERSION=1.22.0 +RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates \ + && rm -rf /var/lib/apt/lists/* +RUN wget -q https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-x64-${ONNX_VERSION}.tgz \ + && tar -xzf onnxruntime-linux-x64-${ONNX_VERSION}.tgz \ + && mkdir -p /onnx/lib \ + && cp onnxruntime-linux-x64-${ONNX_VERSION}/lib/libonnxruntime.so* /onnx/lib/ \ + && rm -rf onnxruntime-linux-x64-${ONNX_VERSION}* + +# ── Etapa 2: Build ───────────────────────────────────────────────────────────── +FROM golang:1.25-bookworm AS builder RUN apt-get update && apt-get install -y --no-install-recommends \ gcc g++ pkg-config \ libtesseract-dev libleptonica-dev \ tesseract-ocr tesseract-ocr-spa \ - wget ca-certificates \ && rm -rf /var/lib/apt/lists/* -# ONNX Runtime -ARG ONNX_VERSION=1.22.0 -RUN wget -q https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-x64-${ONNX_VERSION}.tgz \ - && tar -xzf onnxruntime-linux-x64-${ONNX_VERSION}.tgz \ - && cp onnxruntime-linux-x64-${ONNX_VERSION}/lib/libonnxruntime.so* /usr/local/lib/ \ - && ldconfig +COPY --from=onnx-download /onnx/lib/ /usr/local/lib/ +RUN ldconfig ENV CGO_ENABLED=1 ENV ONNXRUNTIME_SHARED_LIBRARY=/usr/local/lib/libonnxruntime.so +# Limitar paralelismo CGO para evitar OOM en Docker Desktop +ENV GOMAXPROCS=2 WORKDIR /src COPY go.mod go.sum ./ RUN go mod download + COPY . . -RUN go build -o /out/api ./cmd/api && go build -o /out/worker ./cmd/worker +# -p 2 limita los paquetes compilados en paralelo (menor uso de RAM) +RUN go build -p 2 -o /out/api ./cmd/api +# ── Etapa 3: Runtime ─────────────────────────────────────────────────────────── FROM debian:bookworm-slim AS runtime RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates curl wget \ + ca-certificates curl \ tesseract-ocr tesseract-ocr-spa \ && rm -rf /var/lib/apt/lists/* -ARG ONNX_VERSION=1.22.0 -RUN wget -q https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-x64-${ONNX_VERSION}.tgz \ - && tar -xzf onnxruntime-linux-x64-${ONNX_VERSION}.tgz \ - && cp onnxruntime-linux-x64-${ONNX_VERSION}/lib/libonnxruntime.so* /usr/local/lib/ \ - && rm -rf onnxruntime-linux-x64-${ONNX_VERSION}* \ - && ldconfig +COPY --from=onnx-download /onnx/lib/ /usr/local/lib/ +RUN ldconfig ENV ONNXRUNTIME_SHARED_LIBRARY=/usr/local/lib/libonnxruntime.so WORKDIR /app -COPY --from=builder /out/api /app/api -COPY --from=builder /out/worker /app/worker -COPY api/openapi.yaml /app/openapi.yaml +COPY --from=builder /out/api /app/api +COPY api/openapi.yaml /app/openapi.yaml ENV OPENAPI_PATH=/app/openapi.yaml ENV MODEL_PATH=/app/models/model.onnx diff --git a/services/services-count-facturas-medicas-serving/Dockerfile.autoscaler b/services/services-count-facturas-medicas-serving/Dockerfile.autoscaler deleted file mode 100644 index 5e574d5..0000000 --- a/services/services-count-facturas-medicas-serving/Dockerfile.autoscaler +++ /dev/null @@ -1,14 +0,0 @@ -FROM golang:1.23-bookworm AS builder - -WORKDIR /src -COPY go.mod go.sum ./ -RUN go mod download -COPY . . -RUN go build -o /out/autoscaler ./cmd/autoscaler - -FROM docker:27-cli - -COPY --from=builder /out/autoscaler /usr/local/bin/autoscaler - -WORKDIR /project -ENTRYPOINT ["/usr/local/bin/autoscaler"] diff --git a/services/services-count-facturas-medicas-serving/README.md b/services/services-count-facturas-medicas-serving/README.md index ac5a9e3..81b18a0 100644 --- a/services/services-count-facturas-medicas-serving/README.md +++ b/services/services-count-facturas-medicas-serving/README.md @@ -1,339 +1,175 @@ -# Services Count — API de Facturas Médicas +# count-api -![Go](https://img.shields.io/badge/Go-1.23-blue) +Microservicio Go que cuenta servicios facturados en facturas médicas PDF. +Aplica OCR (Tesseract) + clustering espacial (DBSCAN) + clasificación ML (Random Forest vía ONNX Runtime). -Microservicio de inferencia en Go que cuenta servicios facturados en facturas médicas PDF. El entrenamiento del modelo vive en el repositorio hermano [services-count-facturas-medicas-ml](https://github.com/builker-col/services-count-facturas-medicas-ml); este repositorio despliega la API HTTP y los workers de procesamiento. +Es un servicio **sin estado**: recibe PDFs por HTTP, procesa en memoria y devuelve el resultado. No persiste datos ni depende de bases de datos, colas ni almacenamiento de objetos. -## Características +La persistencia de resultados la maneja el servicio `extraction` (Python) a través de los endpoints `/count-jobs`. -- Conteo **síncrono** (1–50 PDFs por petición) y **asíncrono** (hasta 50 000 PDFs por job). -- Pipeline de inferencia: OCR → clustering DBSCAN → Random Forest vía ONNX. -- Escalado horizontal de workers con Docker Compose. -- Especificación OpenAPI y descriptor de herramientas para agentes AI. +--- -## Arquitectura - -El sistema se compone de una API HTTP, workers consumidores y tres servicios de infraestructura (Redis, NATS JetStream y MinIO), definidos en [`docker-compose.yml`](docker-compose.yml). - -```mermaid -flowchart TB - Client[Cliente_o_Agente] - API[API_Go_8080] - Worker[Worker_Go_xN] - Redis[(Redis)] - NATS[NATS_JetStream] - MinIO[(MinIO)] - Pipeline[OCR_Tesseract_ONNX] - - Client -->|sync_POST_v1_count| API - Client -->|async_POST_v1_jobs| API - API --> Pipeline - API --> Redis - API --> MinIO - API -->|publish| NATS - NATS -->|consume| Worker - Worker --> MinIO - Worker --> Pipeline - Worker --> Redis -``` - -### Pipeline de inferencia - -Por cada PDF se procesa la primera página siguiendo este flujo (`internal/pipeline`): - -```mermaid -flowchart LR - PDF[PDF_primera_pagina] --> Render[Render_MuPDF] - Render --> OCR[Tesseract_spa] - OCR --> Cluster[DBSCAN_filas] - Cluster --> Features[8_features] - Features --> Infer[ONNX_RandomForest] - Infer --> Result[service_count_y_services] -``` - -1. **Render** — La primera página se rasteriza con MuPDF (go-fitz) a 300 DPI. -2. **OCR** — Tesseract extrae cajas de texto en español. -3. **Clustering** — DBSCAN agrupa palabras en filas candidatas. -4. **Features** — Se calculan 8 atributos por fila (códigos, moneda, posición, etc.). -5. **Inferencia** — Un Random Forest exportado a ONNX clasifica cada fila como servicio o no. - -## Componentes - -| Componente | Binario / servicio | Paquete | Responsabilidad | -|------------|-------------------|---------|-----------------| -| API HTTP | `cmd/api` | `internal/api` | Endpoints REST, gestión de jobs, health probes | -| Worker | `cmd/worker` | `internal/worker` | Consume NATS, descarga PDFs de MinIO, persiste resultados | -| Pipeline | — | `internal/pipeline` | Orquestación OCR → clustering → features → inferencia | -| Modelo ONNX | — | `internal/model` | Carga `model.onnx` e `inference.json` | -| OCR | — | `internal/ocr` | Render PDF (go-fitz) + Tesseract (gosseract) | -| Estado de jobs | Redis 7 | `internal/store` | Metadatos, resultados y progreso (TTL 72 h) | -| Cola | NATS JetStream | `internal/queue` | Stream `PDF_JOBS`, subject `pdf.process.*` | -| Object storage | MinIO | `internal/storage` | Almacenamiento de PDFs en jobs asíncronos | - -### Estructura del repositorio - -``` -├── cmd/ -│ ├── api/ # Punto de entrada HTTP -│ └── worker/ # Punto de entrada worker -├── internal/ -│ ├── api/ # Servidor HTTP (chi) -│ ├── config/ # Variables de entorno -│ ├── domain/ # Tipos de dominio -│ ├── model/ # Inferencia ONNX -│ ├── ocr/ # Extracción OCR -│ ├── pipeline/ # Procesamiento por PDF -│ ├── queue/ # NATS JetStream -│ ├── storage/ # Cliente MinIO -│ ├── store/ # Estado en Redis -│ └── worker/ # Runner del worker -├── api/openapi.yaml # Especificación OpenAPI 3.1 -├── mcp/tool.json # Descriptor de herramientas para agentes -├── web/ # Cliente demo (embebido en la API en /demo/) -├── scripts/ # Export ONNX, paridad con Python -└── models/latest/ # model.onnx + inference.json (montado en Docker) -``` - -## Modos de operación - -| Modo | Endpoint principal | Límite | Cuándo usarlo | -|------|-------------------|--------|---------------| -| Síncrono | `POST /v1/count` | 50 PDFs | Pruebas, integraciones en tiempo real, lotes pequeños | -| Asíncrono | `POST /v1/jobs` + poll | 50 000 PDFs/job | MinIO + NATS + workers (path producción) | - -**Flujo asíncrono (producción):** - -1. Crear job (`POST /v1/jobs`) — shell vacío, prefijo MinIO o lista de claves. -2. Subir PDFs por chunks (`POST /v1/jobs/{jobID}/upload`, máx. 500 por chunk) → **MinIO Put** → **NATS Publish**. -3. Workers consumen la cola, descargan de MinIO, OCR, guardan en Redis. -4. Consultar progreso (`GET /v1/jobs/{jobID}?wait_ms=30000` para long-polling). -5. Obtener resultados paginados (`GET /v1/jobs/{jobID}/results?cursor=&limit=`). - -## API REST - -Especificación completa: `GET /openapi.json` o [`api/openapi.yaml`](api/openapi.yaml). +## Endpoints | Método | Ruta | Descripción | |--------|------|-------------| -| `GET` | `/health` | Liveness probe | -| `GET` | `/ready` | Readiness probe | -| `POST` | `/v1/count` | Conteo síncrono (multipart `files`) | -| `POST` | `/v1/jobs` | Crear job asíncrono | -| `POST` | `/v1/jobs/{jobID}/upload` | Subir chunk de PDFs | -| `GET` | `/v1/jobs/{jobID}` | Estado y progreso del job | -| `GET` | `/v1/jobs/{jobID}/results` | Resultados paginados | -| `DELETE` | `/v1/jobs/{jobID}` | Cancelar job | +| `GET` | `/health` | Liveness — `{"status":"ok"}` | +| `GET` | `/ready` | Readiness — `{"status":"ready"}` | +| `GET` | `/openapi.json` | Especificación OpenAPI | +| `POST` | `/v1/count` | Conteo síncrono de PDFs | -**Parámetros relevantes:** +### POST /v1/count -- `include_rejected` (query) — Si es `true`, incluye todas las filas OCR con el campo `is_service`. -- Jobs asíncronos devuelven `202 Accepted`; la cola rechaza con `429` si supera `QUEUE_MAX_PENDING`. +**Request** -**Respuesta por PDF:** +``` +Content-Type: multipart/form-data +Query: include_rejected=true|false (default: false) +Field: files — uno o más archivos PDF (máx. SYNC_MAX_FILES) +``` + +**Response 200** ```json { - "filename": "factura.pdf", - "service_count": 2, - "status": "ok", - "services": [ - { "text": "890201 Consulta ...", "probability": 0.94 } - ] + "results": [ + { + "filename": "factura_001.pdf", + "service_count": 3, + "status": "ok", + "services": [ + { "text": "CONSULTA MEDICA", "probability": 0.91 }, + { "text": "LABORATORIO HEMOGRAMA", "probability": 0.87 }, + { "text": "RADIOGRAFIA TORAX", "probability": 0.83 } + ] + } + ], + "processed": 1, + "failed": 0 } ``` -## Stack tecnológico +**Response 4xx/5xx** -| Área | Tecnología | -|------|------------| -| Lenguaje | Go 1.23 | -| HTTP router | [go-chi/chi](https://github.com/go-chi/chi) v5 | -| OCR | go-fitz (MuPDF) + gosseract (Tesseract) | -| ML | ONNX Runtime + Random Forest exportado | -| Cola | NATS JetStream | -| Estado | Redis 7 | -| Storage | MinIO | -| Contenedores | Docker multi-stage ([`Dockerfile`](Dockerfile)) | - -## Inicio rápido - -### 1. Exportar el modelo ONNX - -Desde el repositorio de entrenamiento, generar los artefactos de inferencia: - -```bash -pip install -r requirements-export.txt -python scripts/export_onnx.py \ - --joblib ../services-count/models/latest/model.joblib \ - --metadata ../services-count/models/v1.0.0/model_metadata.json +```json +{ "error": "descripción del error" } ``` -Esto produce `models/latest/model.onnx` e `inference.json`. +--- -### 2. Levantar con Docker Compose +## Pipeline de procesamiento -```bash -docker compose up --build +``` +PDF bytes (memoria) + │ + ▼ gen2brain/go-fitz (MuPDF) +Render de la primera página a imagen (OCR_DPI, default 300) + │ + ▼ otiai10/gosseract (Tesseract) +OCR palabra a palabra con coordenadas (X, Y, W, H) + │ + ▼ DBSCAN 1D sobre coordenada Y +Agrupación de palabras en filas candidatas + │ + ▼ 8 features por fila +first_token_is_code · currency_count · integer_count · x_span_norm +y_norm · token_count · special_char_ratio · digit_ratio + │ + ▼ ONNX Runtime (Random Forest) +P(servicio) por fila → filtrar por threshold (inference.json) + │ Si el modelo no está disponible → heurística: todas las filas = servicio + ▼ +PDFResult { service_count, services[], status } ``` -| Servicio | Puerto / acceso | -|----------|-----------------| -| API | `http://localhost:8080` | -| MinIO Console | `http://localhost:9002` | -| Workers | 1 réplica inicial; autoscaler sube hasta 8 según cola | -| Autoscaler | Escala `worker` vía Docker socket según `PendingCount` en Redis | +--- -### 3. Probar +## Modelo ONNX -```bash -# Síncrono -curl -F "files=@factura.pdf" http://localhost:8080/v1/count +El modelo es opcional. Si no está disponible, el servicio arranca con una advertencia y usa heurística de conteo. -# Asíncrono (PDFs ya en MinIO bajo un prefijo) -curl -X POST http://localhost:8080/v1/jobs \ - -H "Content-Type: application/json" \ - -d '{"prefix":"jobs/batch-1/"}' +Los archivos deben montarse en `/app/models/`: + +``` +/app/models/model.onnx — modelo Random Forest exportado con skl2onnx +/app/models/inference.json — threshold y feature_columns ``` -## Cliente web de ejemplo - -Con la API en ejecución, abre **[http://localhost:8080/demo/](http://localhost:8080/demo/)** para probar la integración desde el navegador. Los archivos estáticos se sirven desde la propia API (mismo origen, sin CORS). - -| Pestaña | Demuestra | -|---------|-----------| -| **Síncrono** | `POST /v1/count` con subida multipart de hasta 50 PDFs | -| **Asíncrono** | `POST /v1/jobs` → upload → polling de estado → resultados paginados | - -El modo asíncrono requiere que workers, Redis, NATS y MinIO estén levantados (igual que con `docker compose up`). Código fuente en [`web/`](web/). - -## Desarrollo local - -Requisitos adicionales a Go 1.23+: - -- Tesseract con datos de idioma `spa` -- ONNX Runtime (la variable `ONNXRUNTIME_SHARED_LIBRARY` apunta a la librería compartida) -- Redis, NATS y MinIO en ejecución +Para generar el modelo desde un `model.joblib` existente: ```bash -go build -o api ./cmd/api -go build -o worker ./cmd/worker +cd scripts +pip install -r ../requirements-export.txt +python export_onnx.py --joblib /ruta/model.joblib --out ../models/latest ``` -Configurar las variables de entorno para apuntar a los servicios locales (ver tabla siguiente). +--- -## Configuración - -Variables definidas en [`internal/config/config.go`](internal/config/config.go): +## Variables de entorno | Variable | Default | Descripción | |----------|---------|-------------| -| `PORT` | `8080` | Puerto de la API | -| `REDIS_URL` | `redis://redis:6379/0` | Conexión Redis | -| `NATS_URL` | `nats://nats:4222` | Conexión NATS | -| `NATS_STREAM` | `PDF_JOBS` | Nombre del stream JetStream | -| `NATS_SUBJECT_PREFIX` | `pdf.process` | Prefijo de subjects | -| `MINIO_ENDPOINT` | `minio:9000` | Endpoint MinIO | -| `MINIO_ACCESS_KEY` | `minioadmin` | Access key | -| `MINIO_SECRET_KEY` | `minioadmin` | Secret key | -| `MINIO_BUCKET` | `pdf-count` | Bucket de PDFs | -| `MINIO_USE_SSL` | `false` | Usar SSL con MinIO | +| `PORT` | `8080` | Puerto HTTP | | `MODEL_PATH` | `/app/models/model.onnx` | Ruta al modelo ONNX | -| `INFERENCE_CONFIG` | `/app/models/inference.json` | Metadata de inferencia | -| `OCR_DPI` | `300` | DPI de renderizado PDF (150 reduce latencia ~2–4×) | +| `INFERENCE_CONFIG` | `/app/models/inference.json` | Threshold y metadatos del modelo | +| `OPENAPI_PATH` | `api/openapi.yaml` | Ruta a la especificación OpenAPI | +| `OCR_DPI` | `300` | DPI de renderizado PDF para OCR | | `OCR_LANG` | `spa` | Idioma Tesseract | -| `DBSCAN_EPS` | `10` | Parámetro epsilon de DBSCAN | -| `DBSCAN_MIN_SAMPLES` | `1` | Mínimo de muestras DBSCAN | -| `SYNC_MAX_FILES` | `50` | Máximo de PDFs en modo síncrono | -| `JOB_MAX_FILES` | `50000` | Máximo de PDFs por job | -| `QUEUE_MAX_PENDING` | `100000` | Límite de mensajes pendientes en cola | -| `WORKER_CONCURRENCY` | `4` | Goroutines paralelas por worker | -| `MAX_PDF_SIZE_MB` | `20` | Tamaño máximo por PDF | -| `PDF_PROCESS_TIMEOUT_SEC` | `120` | Timeout de procesamiento por PDF | -| `JOB_RESULT_TTL_HOURS` | `72` | TTL de resultados en Redis | -| `OCR_POOL_SIZE` | `4` (API) / `WORKER_CONCURRENCY` (worker) | Clientes Tesseract reutilizables en pool | -| `SYNC_PROCESS_CONCURRENCY` | `4` (o `OCR_POOL_SIZE`) | PDFs procesados en paralelo en `/v1/count` | -| `JOB_INLINE_PROCESSING` | `false` | `true` procesa uploads en la API sin MinIO (solo dev local) | -| `LOG_PIPELINE_TIMING` | `false` | Logs de duración por etapa del pipeline | -| `NATS_FETCH_MAX_WAIT_MS` | `300` | Tiempo máximo de espera del worker al leer NATS | +| `OCR_POOL_SIZE` | `0` (→ 4) | Tamaño del pool de clientes Tesseract | +| `DBSCAN_EPS` | `10` | Épsilon de clustering DBSCAN (píxeles en Y) | +| `DBSCAN_MIN_SAMPLES` | `1` | Mínimo de puntos por cluster DBSCAN | +| `SYNC_MAX_FILES` | `50` | Máximo de PDFs por request a `/v1/count` | +| `SYNC_PROCESS_CONCURRENCY` | `0` (→ OCR_POOL_SIZE) | Goroutines paralelas por request | +| `MAX_PDF_SIZE_MB` | `20` | Tamaño máximo permitido por PDF | +| `PDF_PROCESS_TIMEOUT_SEC` | `120` | Timeout por PDF | +| `LOG_PIPELINE_TIMING` | `false` | Loguear tiempos de cada etapa | -## Rendimiento +--- -El cuello de botella de `/v1/count` es el **OCR con Tesseract**, no la inferencia ONNX. Los clientes Tesseract se crean una vez al arranque y se reutilizan desde un pool (`internal/ocr/pool.go`). +## Dependencias en producción (Docker) -| Etapa | Tiempo típico (1 PDF) | -|-------|------------------------| -| Render PDF (MuPDF) | ~100–300 ms | -| OCR Tesseract | ~3–8 s | -| Clustering + ONNX | ~10–50 ms | +- `libonnxruntime.so` — ONNX Runtime (copiado en build multi-etapa) +- `tesseract-ocr tesseract-ocr-spa` — OCR en español +- `libmupdf` — via go-fitz, renderizado de PDFs -**Latencia esperada (síncrono):** ~3–10 s por PDF. Varios PDFs se procesan en paralelo (`SYNC_PROCESS_CONCURRENCY`, default 4). Ejemplo: 20 PDFs ≈ `ceil(20/4) × 2.85s ≈ 15s`. +Sin Redis, MinIO, NATS ni bases de datos. -**Latencia esperada (asíncrono por upload):** path producción MinIO → NATS → worker. Con `WORKER_CONCURRENCY=4` y autoscaler activo, 16 PDFs ≈ `ceil(16/4) × 2.85s` más overhead Put/Get (~15–25 s). 100 PDFs escala hasta 8 workers (32 slots OCR). +--- -**Asíncrono por prefix MinIO** (`POST /v1/jobs {"prefix":"..."}`): mismos workers; los PDFs ya están en el bucket. - -**Ajustes para reducir latencia:** - -- `OCR_DPI=150` — reduce tiempo de render y OCR (~2–4× más rápido) con posible ligera pérdida de precisión. Validar con `scripts/compare_parity.py` antes de usar en producción. -- `OCR_POOL_SIZE=4` y `WORKER_CONCURRENCY=4` — paralelismo por réplica worker. -- Upload pipeline publica en NATS tras cada Put (no al final del lote). -- `LOG_PIPELINE_TIMING=true` — logs por etapa: `ocr_render`, `ocr_tesseract`, `cluster`, `infer`, `total`. -- `NATS_FETCH_MAX_WAIT_MS=300` — el worker recoge mensajes con menor espera. -- `GET /v1/jobs/{id}?wait_ms=30000` — long-polling en lugar de poll cada 500 ms. - -## Escalado - -### Autoscaler (docker compose) - -El servicio `autoscaler` lee `PendingCount` en Redis cada 5 s y ajusta réplicas `worker`: +## Estructura del código ``` -desired = clamp(ceil(pending / WORKER_SLOTS), MIN_WORKERS, MAX_WORKERS) +cmd/api/ — punto de entrada del servidor HTTP +internal/ + api/ — router y handler countSync + config/ — carga de variables de entorno + domain/ — tipos PDFResult, CountResponse, ServiceRow + model/ — sesión ONNX Runtime (Predictor) + ocr/ — pool de clientes Tesseract + renderizado MuPDF + pipeline/ — orquestación OCR → DBSCAN → features → ML + processor.go — ProcessPDF por documento + clustering.go — DBSCAN 1D en Y + features.go — 8 features por fila +api/openapi.yaml — especificación OpenAPI 3.1 +models/latest/ — model.onnx + inference.json (gitignored, montar en Docker) +scripts/ — export_onnx.py para generar el modelo +Dockerfile — build multi-etapa: ONNX download + Go builder + runtime slim ``` -| Variable | Default | Descripción | -|----------|---------|-------------| -| `MIN_WORKERS` | `1` | Réplicas mínimas | -| `MAX_WORKERS` | `8` | Réplicas máximas | -| `WORKER_SLOTS` | `4` | PDFs en paralelo por réplica (`WORKER_CONCURRENCY`) | -| `SCALE_DOWN_IDLE_SEC` | `60` | Segundos sin cola antes de bajar 1 réplica | +--- -Requiere montar `/var/run/docker.sock` (solo entorno local). En Kubernetes usar HPA sobre lag de NATS o métricas custom. - -### Escalado manual +## Arrancar localmente (Docker) ```bash -docker compose up --scale worker=8 -d +# Desde la raíz del repositorio: +docker compose up --build count-api ``` -Cada worker procesa hasta `WORKER_CONCURRENCY` PDFs en paralelo. NATS JetStream distribuye los mensajes entre réplicas con política work-queue. +El servicio queda disponible en `http://localhost:8081`. -## Integración con agentes (MCP) - -[`mcp/tool.json`](mcp/tool.json) es un **descriptor declarativo** de herramientas para agentes AI (Cursor, etc.). No incluye un servidor MCP embebido. - -| Herramienta | Modo | Flujo | -|-------------|------|-------| -| `count_pdf_services` | Síncrono | `POST /v1/count` con multipart | -| `count_pdf_services_batch` | Asíncrono | Crear job → poll estado → resultados paginados | - -## Paridad con Python - -Scripts para validar que la inferencia Go reproduce los resultados del pipeline Python original: +Para probar el endpoint: ```bash -set SERVICES_COUNT_ROOT=../services-count -python scripts/parity_baseline.py -python scripts/compare_parity.py +curl -X POST http://localhost:8081/v1/count \ + -F "files=@factura.pdf" | jq . ``` - -## Troubleshooting - -| Síntoma | Causa probable | Acción | -|---------|------------------|--------| -| API no arranca | Modelo ONNX ausente | Ejecutar `scripts/export_onnx.py` y verificar `models/latest/` | -| `/ready` falla | Redis o NATS no disponibles | Comprobar healthchecks en `docker compose ps` | -| Jobs en `pending` sin avance | Workers no consumen cola | Verificar logs del worker y conectividad a NATS/MinIO | -| Job atascado en `running` con N/N procesados | Race en metadata del job (versiones antiguas) | Reconstruir con `docker compose up --build`; `GetJob` ahora finaliza jobs completados | -| `429` al crear job | Cola saturada | Escalar workers o esperar a que baje `QUEUE_MAX_PENDING` | -| Resultados vacíos o erróneos | OCR o modelo desactualizado | Revisar `OCR_LANG=spa`, reexportar ONNX desde el repo ML | diff --git a/services/services-count-facturas-medicas-serving/cmd/api/main.go b/services/services-count-facturas-medicas-serving/cmd/api/main.go index ceb6bb4..250212a 100644 --- a/services/services-count-facturas-medicas-serving/cmd/api/main.go +++ b/services/services-count-facturas-medicas-serving/cmd/api/main.go @@ -6,15 +6,11 @@ import ( "os" "os/signal" "syscall" - "time" "github.com/pys/pdf-count/internal/api" "github.com/pys/pdf-count/internal/config" "github.com/pys/pdf-count/internal/model" "github.com/pys/pdf-count/internal/ocr" - "github.com/pys/pdf-count/internal/queue" - "github.com/pys/pdf-count/internal/storage" - "github.com/pys/pdf-count/internal/store" ) func main() { @@ -24,9 +20,12 @@ func main() { predictor, err := model.NewPredictor(cfg.ModelPath, cfg.InferencePath) if err != nil { - log.Fatalf("model: %v", err) + log.Printf("WARN: modelo ML no disponible (%v) — usando heuristica de conteo", err) + predictor = nil + } else { + defer predictor.Close() + log.Printf("modelo ONNX cargado: %s", cfg.ModelPath) } - defer predictor.Close() poolSize := cfg.OCRPoolSize if poolSize <= 0 { @@ -38,37 +37,16 @@ func main() { } defer ocrEngine.Close() - st, err := store.New(cfg.RedisURL, cfg.JobResultTTL) - if err != nil { - log.Fatalf("redis: %v", err) - } - defer st.Close() - - stg, err := storage.New(cfg.MinIOEndpoint, cfg.MinIOAccessKey, cfg.MinIOSecretKey, cfg.MinIOBucket, cfg.MinIOUseSSL) - if err != nil { - log.Fatalf("minio: %v", err) - } - initCtx, initCancel := context.WithTimeout(ctx, 10*time.Second) - if err := stg.EnsureBucket(initCtx); err != nil { - initCancel() - log.Fatalf("minio bucket: %v", err) - } - initCancel() - - pub, err := queue.NewPublisher(cfg.NATSURL, cfg.StreamName, cfg.SubjectPrefix) - if err != nil { - log.Fatalf("nats: %v", err) - } - openAPIPath := os.Getenv("OPENAPI_PATH") if openAPIPath == "" { - openAPIPath = "api/openapi.json" + openAPIPath = "api/openapi.yaml" } - srv := api.NewServer(cfg, predictor, ocrEngine, st, stg, pub, api.LoadOpenAPI(openAPIPath)) + srv := api.NewServer(cfg, predictor, ocrEngine, api.LoadOpenAPI(openAPIPath)) + addr := ":" + cfg.Port - log.Printf("api listening on %s", addr) + log.Printf("count-api escuchando en %s", addr) if err := api.ListenAndServe(ctx, addr, srv.Router()); err != nil { - log.Fatalf("api: %v", err) + log.Fatalf("server: %v", err) } } diff --git a/services/services-count-facturas-medicas-serving/cmd/autoscaler/main.go b/services/services-count-facturas-medicas-serving/cmd/autoscaler/main.go deleted file mode 100644 index 9c89fca..0000000 --- a/services/services-count-facturas-medicas-serving/cmd/autoscaler/main.go +++ /dev/null @@ -1,157 +0,0 @@ -package main - -import ( - "context" - "log" - "math" - "os" - "os/exec" - "os/signal" - "strconv" - "strings" - "syscall" - "time" - - "github.com/pys/pdf-count/internal/store" -) - -func main() { - cfg := loadConfig() - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer cancel() - - st, err := store.New(cfg.redisURL, 72*time.Hour) - if err != nil { - log.Fatalf("redis: %v", err) - } - defer st.Close() - - log.Printf("autoscaler started min=%d max=%d slots=%d service=%s", - cfg.minWorkers, cfg.maxWorkers, cfg.workerSlots, cfg.composeService) - - var idleSince time.Time - - for { - select { - case <-ctx.Done(): - log.Println("autoscaler stopped") - return - default: - } - - pending, err := st.PendingCount(ctx) - if err != nil { - log.Printf("autoscaler pending count: %v", err) - time.Sleep(cfg.pollInterval) - continue - } - - current, err := countWorkers(cfg) - if err != nil { - log.Printf("autoscaler count workers: %v", err) - time.Sleep(cfg.pollInterval) - continue - } - - desired := cfg.minWorkers - if pending > 0 { - desired = int(math.Ceil(float64(pending) / float64(cfg.workerSlots))) - if desired < cfg.minWorkers { - desired = cfg.minWorkers - } - if desired > cfg.maxWorkers { - desired = cfg.maxWorkers - } - idleSince = time.Time{} - } else if current > cfg.minWorkers { - if idleSince.IsZero() { - idleSince = time.Now() - } else if time.Since(idleSince) >= cfg.scaleDownIdle { - desired = current - 1 - idleSince = time.Time{} - } else { - desired = current - } - } - - if desired != current { - log.Printf("autoscaler pending=%d current=%d desired=%d scaling", pending, current, desired) - if err := scaleWorkers(cfg, desired); err != nil { - log.Printf("autoscaler scale: %v", err) - } - } - - time.Sleep(cfg.pollInterval) - } -} - -type autoscalerConfig struct { - redisURL string - composeFile string - composeService string - minWorkers int - maxWorkers int - workerSlots int - pollInterval time.Duration - scaleDownIdle time.Duration -} - -func loadConfig() autoscalerConfig { - return autoscalerConfig{ - redisURL: getenv("REDIS_URL", "redis://redis:6379/0"), - composeFile: getenv("COMPOSE_FILE", "/project/docker-compose.yml"), - composeService: getenv("COMPOSE_SERVICE", "worker"), - minWorkers: getenvInt("MIN_WORKERS", 1), - maxWorkers: getenvInt("MAX_WORKERS", 8), - workerSlots: getenvInt("WORKER_SLOTS", 4), - pollInterval: time.Duration(getenvInt("POLL_INTERVAL_SEC", 5)) * time.Second, - scaleDownIdle: time.Duration(getenvInt("SCALE_DOWN_IDLE_SEC", 60)) * time.Second, - } -} - -func countWorkers(cfg autoscalerConfig) (int, error) { - cmd := exec.Command("docker", "compose", - "-f", cfg.composeFile, - "ps", cfg.composeService, - "--status", "running", - "-q", - ) - out, err := cmd.Output() - if err != nil { - return 0, err - } - trimmed := strings.TrimSpace(string(out)) - if trimmed == "" { - return 0, nil - } - return len(strings.Split(trimmed, "\n")), nil -} - -func scaleWorkers(cfg autoscalerConfig, n int) error { - scaleArg := cfg.composeService + "=" + strconv.Itoa(n) - cmd := exec.Command("docker", "compose", - "-f", cfg.composeFile, - "up", "-d", "--no-recreate", - "--scale", scaleArg, - cfg.composeService, - ) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} - -func getenv(key, fallback string) string { - if v := os.Getenv(key); v != "" { - return v - } - return fallback -} - -func getenvInt(key string, fallback int) int { - if v := os.Getenv(key); v != "" { - if n, err := strconv.Atoi(v); err == nil { - return n - } - } - return fallback -} diff --git a/services/services-count-facturas-medicas-serving/cmd/worker/main.go b/services/services-count-facturas-medicas-serving/cmd/worker/main.go deleted file mode 100644 index 5a0fdac..0000000 --- a/services/services-count-facturas-medicas-serving/cmd/worker/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "github.com/pys/pdf-count/internal/worker" - -func main() { - worker.RunMain() -} diff --git a/services/services-count-facturas-medicas-serving/go.mod b/services/services-count-facturas-medicas-serving/go.mod index 2e983b3..ce5fe54 100644 --- a/services/services-count-facturas-medicas-serving/go.mod +++ b/services/services-count-facturas-medicas-serving/go.mod @@ -1,40 +1,20 @@ module github.com/pys/pdf-count -go 1.23.0 - -toolchain go1.23.4 +go 1.25.0 require ( github.com/gen2brain/go-fitz v1.24.14 github.com/go-chi/chi/v5 v5.2.1 - github.com/google/uuid v1.6.0 - github.com/minio/minio-go/v7 v7.0.90 - github.com/nats-io/nats.go v1.39.1 github.com/otiai10/gosseract/v2 v2.4.1 - github.com/redis/go-redis/v9 v9.7.3 github.com/yalue/onnxruntime_go v1.21.0 gopkg.in/yaml.v3 v3.0.1 ) +require github.com/rogpeppe/go-internal v1.15.0 // indirect + require ( - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.8.0 // indirect - github.com/go-ini/ini v1.67.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect github.com/jupiterrider/ffi v0.2.0 // indirect - github.com/klauspost/compress v1.18.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect - github.com/minio/crc64nvme v1.0.1 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/nats-io/nkeys v0.4.9 // indirect - github.com/nats-io/nuid v1.0.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rs/xid v1.6.0 // indirect - golang.org/x/crypto v0.36.0 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.23.0 // indirect + github.com/kr/pretty v0.3.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/services/services-count-facturas-medicas-serving/go.sum b/services/services-count-facturas-medicas-serving/go.sum index ffa6fa2..2d50697 100644 --- a/services/services-count-facturas-medicas-serving/go.sum +++ b/services/services-count-facturas-medicas-serving/go.sum @@ -1,69 +1,33 @@ -github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= -github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= -github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= -github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/ebitengine/purego v0.8.0 h1:JbqvnEzRvPpxhCJzJJ2y0RbiZ8nyjccVUrSM3q+GvvE= github.com/ebitengine/purego v0.8.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/gen2brain/go-fitz v1.24.14 h1:09weRkjVtLYNGo7l0J7DyOwBExbwi8SJ9h8YPhw9WEo= github.com/gen2brain/go-fitz v1.24.14/go.mod h1:0KaZeQgASc20Yp5R/pFzyy7SmP01XcoHKNF842U2/S4= github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8= github.com/go-chi/chi/v5 v5.2.1/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jupiterrider/ffi v0.2.0 h1:tMM70PexgYNmV+WyaYhJgCvQAvtTCs3wXeILPutihnA= github.com/jupiterrider/ffi v0.2.0/go.mod h1:yqYqX5DdEccAsHeMn+6owkoI2llBLySVAF8dwCDZPVs= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= -github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.90 h1:TmSj1083wtAD0kEYTx7a5pFsv3iRYMsOJ6A4crjA1lE= -github.com/minio/minio-go/v7 v7.0.90/go.mod h1:uvMUcGrpgeSAAI6+sD3818508nUyMULw94j2Nxku/Go= -github.com/nats-io/nats.go v1.39.1 h1:oTkfKBmz7W047vRxV762M67ZdXeOtUgvbBaNoQ+3PPk= -github.com/nats-io/nats.go v1.39.1/go.mod h1:MgRb8oOdigA6cYpEPhXJuRVH6UE/V4jblJ2jQ27IXYM= -github.com/nats-io/nkeys v0.4.9 h1:qe9Faq2Gxwi6RZnZMXfmGMZkg3afLLOtrU+gDZJ35b0= -github.com/nats-io/nkeys v0.4.9/go.mod h1:jcMqs+FLG+W5YO36OX6wFIFcmpdAns+w1Wm6D3I/evE= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/otiai10/gosseract/v2 v2.4.1 h1:G8AyBpXEeSlcq8TI85LH/pM5SXk8Djy2GEXisgyblRw= github.com/otiai10/gosseract/v2 v2.4.1/go.mod h1:1gNWP4Hgr2o7yqWfs6r5bZxAatjOIdqWxJLWsTsembk= github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs= github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= -github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= -github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= -github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/yalue/onnxruntime_go v1.21.0 h1:DdtvfY7OP5gR8mwPDqAOAQckf+KcI30hPNJL8hQaYWI= github.com/yalue/onnxruntime_go v1.21.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/services-count-facturas-medicas-serving/internal/api/jobrunner.go b/services/services-count-facturas-medicas-serving/internal/api/jobrunner.go deleted file mode 100644 index de11cd8..0000000 --- a/services/services-count-facturas-medicas-serving/internal/api/jobrunner.go +++ /dev/null @@ -1,53 +0,0 @@ -package api - -import ( - "context" - - "github.com/pys/pdf-count/internal/pipeline" - "github.com/pys/pdf-count/internal/store" -) - -// FilePayload holds a PDF ready for inline job processing. -type FilePayload struct { - Filename string - Data []byte -} - -// JobRunner processes uploaded job files in-process using the API OCR pool. -type JobRunner struct { - store *store.Store - newProc func(includeRejected bool) *pipeline.Processor - sem chan struct{} -} - -func NewJobRunner(st *store.Store, newProc func(includeRejected bool) *pipeline.Processor, concurrency int) *JobRunner { - if concurrency < 1 { - concurrency = 1 - } - return &JobRunner{ - store: st, - newProc: newProc, - sem: make(chan struct{}, concurrency), - } -} - -// ProcessOne schedules a single PDF for inline OCR (non-blocking). -func (jr *JobRunner) ProcessOne(jobID string, file FilePayload, includeRejected bool) { - go func() { - jr.sem <- struct{}{} - defer func() { <-jr.sem }() - - ctx := context.Background() - proc := jr.newProc(includeRejected) - result := proc.ProcessPDF(file.Filename, file.Data) - _ = jr.store.SaveResult(ctx, jobID, result) - _ = jr.store.TryCompleteJob(ctx, jobID) - }() -} - -// ProcessFiles runs OCR in background goroutines and updates job progress in Redis. -func (jr *JobRunner) ProcessFiles(jobID string, files []FilePayload, includeRejected bool) { - for _, file := range files { - jr.ProcessOne(jobID, file, includeRejected) - } -} diff --git a/services/services-count-facturas-medicas-serving/internal/api/server.go b/services/services-count-facturas-medicas-serving/internal/api/server.go index f101cfd..e0c4baf 100644 --- a/services/services-count-facturas-medicas-serving/internal/api/server.go +++ b/services/services-count-facturas-medicas-serving/internal/api/server.go @@ -4,87 +4,47 @@ import ( "context" "encoding/json" "io" - "mime/multipart" "net/http" "os" "path/filepath" - "strconv" "strings" "sync" - "time" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" - "github.com/google/uuid" "github.com/pys/pdf-count/internal/config" "github.com/pys/pdf-count/internal/domain" "github.com/pys/pdf-count/internal/model" "github.com/pys/pdf-count/internal/ocr" "github.com/pys/pdf-count/internal/pipeline" - "github.com/pys/pdf-count/internal/queue" - "github.com/pys/pdf-count/internal/storage" - "github.com/pys/pdf-count/internal/store" - "github.com/pys/pdf-count/web" "gopkg.in/yaml.v3" ) +// Server holds the shared, stateless dependencies of the count-api. type Server struct { cfg config.Config - predictor *model.Predictor + predictor *model.Predictor // nil → heuristic fallback ocrEngine *ocr.Engine - store *store.Store - storage *storage.Client - publisher *queue.Publisher - jobRunner *JobRunner openAPI []byte - ready bool - readyMu sync.RWMutex } -func NewServer(cfg config.Config, predictor *model.Predictor, ocrEngine *ocr.Engine, st *store.Store, stg *storage.Client, pub *queue.Publisher, openAPI []byte) *Server { - srv := &Server{ +// NewServer constructs a Server. predictor may be nil when the ONNX model is +// unavailable; the pipeline will fall back to a counting heuristic. +func NewServer( + cfg config.Config, + predictor *model.Predictor, + ocrEngine *ocr.Engine, + openAPI []byte, +) *Server { + return &Server{ cfg: cfg, predictor: predictor, ocrEngine: ocrEngine, - store: st, - storage: stg, - publisher: pub, openAPI: openAPI, - ready: true, } - srv.jobRunner = NewJobRunner(st, srv.newProcessor, srv.ocrConcurrency()) - return srv -} - -func (s *Server) processConcurrency() int { - c := s.cfg.SyncProcessConcurrency - if c <= 0 { - c = s.cfg.OCRPoolSize - } - if c <= 0 { - c = 4 - } - return c -} - -func (s *Server) ocrConcurrency() int { - c := s.processConcurrency() - pool := s.cfg.OCRPoolSize - if pool <= 0 { - pool = 4 - } - if c > pool { - c = pool - } - return c -} - -func (s *Server) SetReady(v bool) { - s.readyMu.Lock() - s.ready = v - s.readyMu.Unlock() } +// Router registers all HTTP routes. func (s *Server) Router() http.Handler { r := chi.NewRouter() r.Use(middleware.RequestID) @@ -92,46 +52,41 @@ func (s *Server) Router() http.Handler { r.Use(middleware.Recoverer) r.Get("/health", s.health) - r.Get("/ready", s.readyHandler) + r.Get("/ready", s.ready) r.Get("/openapi.json", s.openAPIHandler) - r.Get("/demo", func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, "/demo/", http.StatusMovedPermanently) - }) - r.Handle("/demo/*", http.StripPrefix("/demo/", http.FileServer(http.FS(web.FS)))) - r.Route("/v1", func(r chi.Router) { r.Post("/count", s.countSync) - r.Post("/jobs", s.createJob) - r.Post("/jobs/{jobID}/upload", s.uploadChunk) - r.Get("/jobs/{jobID}", s.getJob) - r.Get("/jobs/{jobID}/results", s.getJobResults) - r.Delete("/jobs/{jobID}", s.cancelJob) }) + return r } -func (s *Server) health(w http.ResponseWriter, r *http.Request) { +// health returns a simple liveness response. +func (s *Server) health(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) } -func (s *Server) readyHandler(w http.ResponseWriter, r *http.Request) { - s.readyMu.RLock() - ok := s.ready - s.readyMu.RUnlock() - if !ok { - writeJSON(w, http.StatusServiceUnavailable, map[string]string{"status": "not_ready"}) - return - } +// ready returns 200 once the server is accepting traffic (always ready after init). +func (s *Server) ready(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, map[string]string{"status": "ready"}) } -func (s *Server) openAPIHandler(w http.ResponseWriter, r *http.Request) { +// openAPIHandler serves the OpenAPI specification as JSON. +func (s *Server) openAPIHandler(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write(s.openAPI) } +// countSync processes a multipart batch of PDFs synchronously and returns the +// counting results in a single JSON response. No state is persisted. +// +// POST /v1/count +// +// Query: include_rejected=true|false +// Body: multipart/form-data, field "files" (one or more PDFs) +// Limits: SYNC_MAX_FILES (default 50), MAX_PDF_SIZE_MB (default 20) func (s *Server) countSync(w http.ResponseWriter, r *http.Request) { includeRejected := r.URL.Query().Get("include_rejected") == "true" maxBytes := int64(s.cfg.MaxPDFSizeMB) * 1024 * 1024 @@ -151,61 +106,51 @@ func (s *Server) countSync(w http.ResponseWriter, r *http.Request) { return } - proc := s.newProcessor(includeRejected) type fileInput struct { filename string data []byte errRes *domain.PDFResult } - inputs := make([]fileInput, 0, len(files)) + inputs := make([]fileInput, 0, len(files)) for _, fh := range files { + name := filepath.Base(fh.Filename) f, err := fh.Open() if err != nil { - errRes := domain.PDFResult{ - Filename: filepath.Base(fh.Filename), - Status: domain.StatusError, - Error: err.Error(), - } + errRes := domain.PDFResult{Filename: name, Status: domain.StatusError, Error: err.Error()} inputs = append(inputs, fileInput{errRes: &errRes}) continue } data, err := io.ReadAll(io.LimitReader(f, maxBytes+1)) _ = f.Close() if err != nil { - errRes := domain.PDFResult{ - Filename: filepath.Base(fh.Filename), - Status: domain.StatusError, - Error: err.Error(), - } + errRes := domain.PDFResult{Filename: name, Status: domain.StatusError, Error: err.Error()} inputs = append(inputs, fileInput{errRes: &errRes}) continue } if int64(len(data)) > maxBytes { - errRes := domain.PDFResult{ - Filename: filepath.Base(fh.Filename), - Status: domain.StatusError, - Error: "file too large", - } + errRes := domain.PDFResult{Filename: name, Status: domain.StatusError, Error: "file too large"} inputs = append(inputs, fileInput{errRes: &errRes}) continue } - inputs = append(inputs, fileInput{ - filename: filepath.Base(fh.Filename), - data: data, - }) + inputs = append(inputs, fileInput{filename: name, data: data}) } + proc := s.newProcessor(includeRejected) results := make([]domain.PDFResult, len(inputs)) failed := 0 - sem := make(chan struct{}, s.ocrConcurrency()) + + concurrency := s.syncConcurrency() + sem := make(chan struct{}, concurrency) var wg sync.WaitGroup var mu sync.Mutex for i, in := range inputs { if in.errRes != nil { results[i] = *in.errRes + mu.Lock() failed++ + mu.Unlock() continue } wg.Add(1) @@ -231,314 +176,23 @@ func (s *Server) countSync(w http.ResponseWriter, r *http.Request) { }) } -type createJobRequest struct { - Files []string `json:"files"` - Prefix string `json:"prefix"` - IncludeRejected bool `json:"include_rejected"` - CallbackURL string `json:"callback_url"` -} - -func (s *Server) createJob(w http.ResponseWriter, r *http.Request) { - var req createJobRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - httpError(w, http.StatusBadRequest, "invalid json body") - return +// syncConcurrency returns the effective parallelism for /v1/count requests. +func (s *Server) syncConcurrency() int { + c := s.cfg.SyncProcessConcurrency + if c <= 0 { + c = s.cfg.OCRPoolSize } - - ctx := r.Context() - pending, err := s.store.PendingCount(ctx) - if err == nil && int(pending) > s.cfg.QueueMaxPending { - httpError(w, http.StatusTooManyRequests, "queue full") - return + if c <= 0 { + c = 4 } - - jobID := uuid.NewString() - var objectKeys []string - var filenames []string - - if req.Prefix != "" { - prefix := strings.TrimSuffix(req.Prefix, "/") - keys, err := s.storage.ListByPrefix(ctx, prefix) - if err != nil { - httpError(w, http.StatusBadRequest, err.Error()) - return - } - objectKeys = keys - for _, k := range keys { - filenames = append(filenames, filepath.Base(k)) - } - } else if len(req.Files) > 0 { - for _, f := range req.Files { - name := filepath.Base(f) - objectKeys = append(objectKeys, storage.JobObjectKey(jobID, name)) - filenames = append(filenames, name) - } - } else { - // Empty job shell for chunked upload flow. - if err := s.store.CreateJob(ctx, jobID, 0, req.IncludeRejected); err != nil { - httpError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, http.StatusAccepted, domain.JobStatus{ - JobID: jobID, - Status: domain.JobQueued, - Total: 0, - }) - return + pool := s.cfg.OCRPoolSize + if pool <= 0 { + pool = 4 } - - if len(objectKeys) > s.cfg.JobMaxFiles { - httpError(w, http.StatusBadRequest, "too many files in job") - return + if c > pool { + c = pool } - - if err := s.store.CreateJob(ctx, jobID, len(objectKeys), req.IncludeRejected); err != nil { - httpError(w, http.StatusInternalServerError, err.Error()) - return - } - if err := s.store.AddJobFiles(ctx, jobID, filenames); err != nil { - httpError(w, http.StatusInternalServerError, err.Error()) - return - } - _ = s.store.MarkRunning(ctx, jobID) - - for i, key := range objectKeys { - msg := domain.QueueMessage{ - JobID: jobID, - ObjectKey: key, - Filename: filenames[i], - IncludeRejected: req.IncludeRejected, - } - if err := s.publisher.Publish(ctx, msg); err != nil { - httpError(w, http.StatusInternalServerError, err.Error()) - return - } - } - - writeJSON(w, http.StatusAccepted, domain.JobStatus{ - JobID: jobID, - Status: domain.JobRunning, - Total: len(objectKeys), - }) -} - -func (s *Server) uploadChunk(w http.ResponseWriter, r *http.Request) { - jobID := chi.URLParam(r, "jobID") - includeRejected := r.URL.Query().Get("include_rejected") == "true" - maxBytes := int64(s.cfg.MaxPDFSizeMB) * 1024 * 1024 - - if err := r.ParseMultipartForm(maxBytes * 500); err != nil { - httpError(w, http.StatusBadRequest, "invalid multipart form") - return - } - files := r.MultipartForm.File["files"] - if len(files) == 0 { - httpError(w, http.StatusBadRequest, "field 'files' required") - return - } - if len(files) > 500 { - httpError(w, http.StatusBadRequest, "max 500 files per chunk") - return - } - - ctx := r.Context() - status, err := s.store.GetJob(ctx, jobID) - if err != nil { - _ = s.store.CreateJob(ctx, jobID, 0, includeRejected) - status = domain.JobStatus{JobID: jobID, Status: domain.JobQueued} - } - if status.Status == domain.JobCancelled { - httpError(w, http.StatusConflict, "job cancelled") - return - } - - var filenames []string - - if s.cfg.JobInlineProcessing { - _ = s.store.MarkRunning(ctx, jobID) - for _, fh := range files { - f, err := fh.Open() - if err != nil { - continue - } - data, err := io.ReadAll(io.LimitReader(f, maxBytes+1)) - _ = f.Close() - if err != nil || int64(len(data)) > maxBytes { - continue - } - name := filepath.Base(fh.Filename) - if err := s.store.AddJobFiles(ctx, jobID, []string{name}); err != nil { - httpError(w, http.StatusInternalServerError, err.Error()) - return - } - if err := s.store.IncrementTotal(ctx, jobID, 1); err != nil { - httpError(w, http.StatusInternalServerError, err.Error()) - return - } - filenames = append(filenames, name) - s.jobRunner.ProcessOne(jobID, FilePayload{Filename: name, Data: data}, includeRejected) - } - } else { - var err error - filenames, err = s.uploadChunkMinIO(ctx, jobID, files, includeRejected, maxBytes) - if err != nil { - httpError(w, http.StatusInternalServerError, err.Error()) - return - } - } - - if len(filenames) == 0 { - httpError(w, http.StatusBadRequest, "no valid files uploaded") - return - } - - writeJSON(w, http.StatusAccepted, map[string]interface{}{ - "job_id": jobID, - "uploaded": len(filenames), - }) -} - -func (s *Server) uploadChunkMinIO(ctx context.Context, jobID string, files []*multipart.FileHeader, includeRejected bool, maxBytes int64) ([]string, error) { - filenames := make([]string, 0, len(files)) - for _, fh := range files { - filenames = append(filenames, filepath.Base(fh.Filename)) - } - if len(filenames) == 0 { - return nil, nil - } - - if err := s.store.AddJobFiles(ctx, jobID, filenames); err != nil { - return nil, err - } - if err := s.store.IncrementTotal(ctx, jobID, len(filenames)); err != nil { - return nil, err - } - if err := s.store.MarkRunning(ctx, jobID); err != nil { - return nil, err - } - - ioLimit := 8 - if len(files) < ioLimit { - ioLimit = len(files) - } - ioSem := make(chan struct{}, ioLimit) - - var wg sync.WaitGroup - var mu sync.Mutex - uploadedNames := make([]string, 0, len(files)) - - for _, fh := range files { - wg.Add(1) - go func(fh *multipart.FileHeader) { - defer wg.Done() - ioSem <- struct{}{} - defer func() { <-ioSem }() - - name := filepath.Base(fh.Filename) - f, err := fh.Open() - if err != nil { - s.recordUploadError(ctx, jobID, name, err.Error()) - return - } - data, err := io.ReadAll(io.LimitReader(f, maxBytes+1)) - _ = f.Close() - if err != nil { - s.recordUploadError(ctx, jobID, name, err.Error()) - return - } - if int64(len(data)) > maxBytes { - s.recordUploadError(ctx, jobID, name, "file too large") - return - } - - key := storage.JobObjectKey(jobID, name) - if err := s.storage.Put(ctx, key, data, "application/pdf"); err != nil { - s.recordUploadError(ctx, jobID, name, err.Error()) - return - } - - msg := domain.QueueMessage{ - JobID: jobID, - ObjectKey: key, - Filename: name, - IncludeRejected: includeRejected, - } - if err := s.publisher.Publish(ctx, msg); err != nil { - s.recordUploadError(ctx, jobID, name, err.Error()) - return - } - - mu.Lock() - uploadedNames = append(uploadedNames, name) - mu.Unlock() - }(fh) - } - wg.Wait() - - if len(uploadedNames) == 0 { - return nil, nil - } - return uploadedNames, nil -} - -func (s *Server) recordUploadError(ctx context.Context, jobID, filename, errMsg string) { - _ = s.store.SaveResult(ctx, jobID, domain.PDFResult{ - Filename: filename, - Status: domain.StatusError, - Error: errMsg, - }) - _ = s.store.TryCompleteJob(ctx, jobID) -} - -func (s *Server) getJob(w http.ResponseWriter, r *http.Request) { - jobID := chi.URLParam(r, "jobID") - waitMs, _ := strconv.Atoi(r.URL.Query().Get("wait_ms")) - if waitMs > 30000 { - waitMs = 30000 - } - - deadline := time.Now() - if waitMs > 0 { - deadline = deadline.Add(time.Duration(waitMs) * time.Millisecond) - } - - for { - status, err := s.store.GetJob(r.Context(), jobID) - if err != nil { - httpError(w, http.StatusNotFound, "job not found") - return - } - if isTerminalJobStatus(status.Status) || waitMs <= 0 || !time.Now().Before(deadline) { - writeJSON(w, http.StatusOK, status) - return - } - time.Sleep(100 * time.Millisecond) - } -} - -func isTerminalJobStatus(status string) bool { - return status == domain.JobCompleted || status == domain.JobCancelled -} - -func (s *Server) getJobResults(w http.ResponseWriter, r *http.Request) { - jobID := chi.URLParam(r, "jobID") - cursor := r.URL.Query().Get("cursor") - limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) - page, err := s.store.GetResultsPage(r.Context(), jobID, cursor, limit) - if err != nil { - httpError(w, http.StatusNotFound, err.Error()) - return - } - writeJSON(w, http.StatusOK, page) -} - -func (s *Server) cancelJob(w http.ResponseWriter, r *http.Request) { - jobID := chi.URLParam(r, "jobID") - if err := s.store.CancelJob(r.Context(), jobID); err != nil { - httpError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, http.StatusOK, map[string]string{"job_id": jobID, "status": domain.JobCancelled}) + return c } func (s *Server) newProcessor(includeRejected bool) *pipeline.Processor { @@ -552,20 +206,24 @@ func (s *Server) newProcessor(includeRejected bool) *pipeline.Processor { }) } +// writeJSON encodes v as JSON and writes it with the given HTTP status. func writeJSON(w http.ResponseWriter, status int, v interface{}) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) _ = json.NewEncoder(w).Encode(v) } +// httpError writes a JSON error response. func httpError(w http.ResponseWriter, status int, msg string) { writeJSON(w, status, map[string]string{"error": msg}) } +// LoadOpenAPI reads an OpenAPI spec from path. +// YAML files are converted to JSON on the fly. Returns a minimal stub on error. func LoadOpenAPI(path string) []byte { data, err := os.ReadFile(path) if err != nil { - return []byte(`{"openapi":"3.1.0","info":{"title":"pdf-count","version":"1.0.0"}}`) + return []byte(`{"openapi":"3.1.0","info":{"title":"count-api","version":"1.0.0"}}`) } if strings.HasSuffix(strings.ToLower(path), ".yaml") || strings.HasSuffix(strings.ToLower(path), ".yml") { var doc interface{} @@ -578,6 +236,7 @@ func LoadOpenAPI(path string) []byte { return data } +// ListenAndServe starts the HTTP server and blocks until ctx is cancelled. func ListenAndServe(ctx context.Context, addr string, handler http.Handler) error { srv := &http.Server{Addr: addr, Handler: handler} go func() { diff --git a/services/services-count-facturas-medicas-serving/internal/config/config.go b/services/services-count-facturas-medicas-serving/internal/config/config.go index bfe3941..b62dc21 100644 --- a/services/services-count-facturas-medicas-serving/internal/config/config.go +++ b/services/services-count-facturas-medicas-serving/internal/config/config.go @@ -6,67 +6,39 @@ import ( "time" ) +// Config holds all runtime parameters for the count-api service. +// All values are loaded from environment variables with safe defaults. type Config struct { - Port string - RedisURL string - NATSURL string - MinIOEndpoint string - MinIOAccessKey string - MinIOSecretKey string - MinIOBucket string - MinIOUseSSL bool - ModelPath string - InferencePath string - OCRDPI int - OCRLang string - DBSCANEps float64 - DBSCANMinSamples int - SyncMaxFiles int - JobMaxFiles int - QueueMaxPending int - WorkerConcurrency int - MaxPDFSizeMB int - PDFProcessTimeout time.Duration - JobResultTTL time.Duration - StreamName string - SubjectPrefix string + Port string + ModelPath string + InferencePath string + OCRDPI int + OCRLang string OCRPoolSize int - LogPipelineTiming bool - NATSFetchMaxWait time.Duration - JobInlineProcessing bool + DBSCANEps float64 + DBSCANMinSamples int + SyncMaxFiles int + MaxPDFSizeMB int + PDFProcessTimeout time.Duration SyncProcessConcurrency int + LogPipelineTiming bool } func Load() Config { return Config{ - Port: getenv("PORT", "8080"), - RedisURL: getenv("REDIS_URL", "redis://redis:6379/0"), - NATSURL: getenv("NATS_URL", "nats://nats:4222"), - MinIOEndpoint: getenv("MINIO_ENDPOINT", "minio:9000"), - MinIOAccessKey: getenv("MINIO_ACCESS_KEY", "minioadmin"), - MinIOSecretKey: getenv("MINIO_SECRET_KEY", "minioadmin"), - MinIOBucket: getenv("MINIO_BUCKET", "pdf-count"), - MinIOUseSSL: getenvBool("MINIO_USE_SSL", false), - ModelPath: getenv("MODEL_PATH", "/app/models/model.onnx"), - InferencePath: getenv("INFERENCE_CONFIG", "/app/models/inference.json"), - OCRDPI: getenvInt("OCR_DPI", 300), - OCRLang: getenv("OCR_LANG", "spa"), - DBSCANEps: getenvFloat("DBSCAN_EPS", 10), - DBSCANMinSamples: getenvInt("DBSCAN_MIN_SAMPLES", 1), - SyncMaxFiles: getenvInt("SYNC_MAX_FILES", 50), - JobMaxFiles: getenvInt("JOB_MAX_FILES", 50000), - QueueMaxPending: getenvInt("QUEUE_MAX_PENDING", 100000), - WorkerConcurrency: getenvInt("WORKER_CONCURRENCY", 4), - MaxPDFSizeMB: getenvInt("MAX_PDF_SIZE_MB", 20), - PDFProcessTimeout: time.Duration(getenvInt("PDF_PROCESS_TIMEOUT_SEC", 120)) * time.Second, - JobResultTTL: time.Duration(getenvInt("JOB_RESULT_TTL_HOURS", 72)) * time.Hour, - StreamName: getenv("NATS_STREAM", "PDF_JOBS"), - SubjectPrefix: getenv("NATS_SUBJECT_PREFIX", "pdf.process"), + Port: getenv("PORT", "8080"), + ModelPath: getenv("MODEL_PATH", "/app/models/model.onnx"), + InferencePath: getenv("INFERENCE_CONFIG", "/app/models/inference.json"), + OCRDPI: getenvInt("OCR_DPI", 300), + OCRLang: getenv("OCR_LANG", "spa"), OCRPoolSize: getenvInt("OCR_POOL_SIZE", 0), - LogPipelineTiming: getenvBool("LOG_PIPELINE_TIMING", false), - NATSFetchMaxWait: time.Duration(getenvInt("NATS_FETCH_MAX_WAIT_MS", 300)) * time.Millisecond, - JobInlineProcessing: getenvBool("JOB_INLINE_PROCESSING", false), + DBSCANEps: getenvFloat("DBSCAN_EPS", 10), + DBSCANMinSamples: getenvInt("DBSCAN_MIN_SAMPLES", 1), + SyncMaxFiles: getenvInt("SYNC_MAX_FILES", 50), + MaxPDFSizeMB: getenvInt("MAX_PDF_SIZE_MB", 20), + PDFProcessTimeout: time.Duration(getenvInt("PDF_PROCESS_TIMEOUT_SEC", 120)) * time.Second, SyncProcessConcurrency: getenvInt("SYNC_PROCESS_CONCURRENCY", 0), + LogPipelineTiming: getenvBool("LOG_PIPELINE_TIMING", false), } } diff --git a/services/services-count-facturas-medicas-serving/internal/domain/types.go b/services/services-count-facturas-medicas-serving/internal/domain/types.go index 48a07bb..3b92ee7 100644 --- a/services/services-count-facturas-medicas-serving/internal/domain/types.go +++ b/services/services-count-facturas-medicas-serving/internal/domain/types.go @@ -1,13 +1,13 @@ package domain -// ServiceRow is one detected invoice line. +// ServiceRow represents a single detected invoice line in a PDF page. type ServiceRow struct { Text string `json:"text"` Probability float64 `json:"probability"` IsService *bool `json:"is_service,omitempty"` } -// PDFResult is the processing outcome for a single PDF. +// PDFResult is the counting outcome for a single PDF document. type PDFResult struct { Filename string `json:"filename"` ServiceCount int `json:"service_count"` @@ -16,44 +16,14 @@ type PDFResult struct { Error string `json:"error,omitempty"` } -// CountResponse wraps sync batch results. +// CountResponse wraps the results of a synchronous batch request to POST /v1/count. type CountResponse struct { Results []PDFResult `json:"results"` Processed int `json:"processed"` Failed int `json:"failed"` } -// JobStatus tracks async job progress. -type JobStatus struct { - JobID string `json:"job_id"` - Status string `json:"status"` - Total int `json:"total"` - Processed int `json:"processed"` - Failed int `json:"failed"` -} - -// JobResultsPage is a paginated slice of PDF results. -type JobResultsPage struct { - JobID string `json:"job_id"` - Results []PDFResult `json:"results"` - NextCursor string `json:"next_cursor,omitempty"` - HasMore bool `json:"has_more"` -} - -// QueueMessage is published per PDF for workers. -type QueueMessage struct { - JobID string `json:"job_id"` - ObjectKey string `json:"object_key"` - Filename string `json:"filename"` - IncludeRejected bool `json:"include_rejected"` -} - const ( StatusOK = "ok" StatusError = "error" - - JobQueued = "queued" - JobRunning = "running" - JobCompleted = "completed" - JobCancelled = "cancelled" ) diff --git a/services/services-count-facturas-medicas-serving/internal/pipeline/processor.go b/services/services-count-facturas-medicas-serving/internal/pipeline/processor.go index 7dc1a15..8828706 100644 --- a/services/services-count-facturas-medicas-serving/internal/pipeline/processor.go +++ b/services/services-count-facturas-medicas-serving/internal/pipeline/processor.go @@ -73,6 +73,21 @@ func (p *Processor) ProcessPDF(filename string, pdfData []byte) domain.PDFResult return result } + // Sin modelo ML: contar todas las filas agrupadas como servicios (heurística). + if p.predictor == nil { + services := make([]domain.ServiceRow, 0, len(rows)) + for _, row := range rows { + services = append(services, domain.ServiceRow{ + Text: row.TextoFila, + Probability: 1.0, + }) + } + result.Services = services + result.ServiceCount = len(rows) + p.logTiming(filename, renderDur, ocrDur, clusterDur, 0, time.Since(start)) + return result + } + features := make([][8]float32, len(rows)) for i, r := range rows { features[i] = r.Features diff --git a/services/services-count-facturas-medicas-serving/internal/queue/nats.go b/services/services-count-facturas-medicas-serving/internal/queue/nats.go deleted file mode 100644 index b590d9d..0000000 --- a/services/services-count-facturas-medicas-serving/internal/queue/nats.go +++ /dev/null @@ -1,109 +0,0 @@ -package queue - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - "github.com/pys/pdf-count/internal/domain" -) - -type Publisher struct { - js jetstream.JetStream - stream string - subject string -} - -func NewPublisher(natsURL, streamName, subjectPrefix string) (*Publisher, error) { - nc, err := nats.Connect(natsURL) - if err != nil { - return nil, fmt.Errorf("nats connect: %w", err) - } - js, err := jetstream.New(nc) - if err != nil { - nc.Close() - return nil, err - } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - _, err = js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ - Name: streamName, - Subjects: []string{subjectPrefix + ".>"}, - Retention: jetstream.WorkQueuePolicy, - MaxMsgs: 200000, - }) - if err != nil { - nc.Close() - return nil, fmt.Errorf("create stream: %w", err) - } - - return &Publisher{js: js, stream: streamName, subject: subjectPrefix}, nil -} - -func (p *Publisher) Publish(ctx context.Context, msg domain.QueueMessage) error { - data, err := json.Marshal(msg) - if err != nil { - return err - } - subject := fmt.Sprintf("%s.%s", p.subject, msg.JobID) - _, err = p.js.Publish(ctx, subject, data) - return err -} - -type Consumer struct { - js jetstream.JetStream - subject string - consumerName string -} - -func NewConsumer(natsURL, streamName, subjectPrefix, consumerName string) (*Consumer, jetstream.Consumer, *nats.Conn, error) { - nc, err := nats.Connect(natsURL) - if err != nil { - return nil, nil, nil, err - } - js, err := jetstream.New(nc) - if err != nil { - nc.Close() - return nil, nil, nil, err - } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - _, err = js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ - Name: streamName, - Subjects: []string{subjectPrefix + ".>"}, - Retention: jetstream.WorkQueuePolicy, - MaxMsgs: 200000, - }) - if err != nil { - nc.Close() - return nil, nil, nil, err - } - - cons, err := js.CreateOrUpdateConsumer(ctx, streamName, jetstream.ConsumerConfig{ - Name: consumerName, - FilterSubject: subjectPrefix + ".>", - AckPolicy: jetstream.AckExplicitPolicy, - MaxDeliver: 3, - AckWait: 2 * time.Minute, - MaxAckPending: 256, - }) - if err != nil { - nc.Close() - return nil, nil, nil, err - } - - return &Consumer{js: js, subject: subjectPrefix, consumerName: consumerName}, cons, nc, nil -} - -func DecodeMessage(data []byte) (domain.QueueMessage, error) { - var msg domain.QueueMessage - err := json.Unmarshal(data, &msg) - return msg, err -} diff --git a/services/services-count-facturas-medicas-serving/internal/storage/minio.go b/services/services-count-facturas-medicas-serving/internal/storage/minio.go deleted file mode 100644 index 45b73f2..0000000 --- a/services/services-count-facturas-medicas-serving/internal/storage/minio.go +++ /dev/null @@ -1,86 +0,0 @@ -package storage - -import ( - "bytes" - "context" - "fmt" - "io" - "strings" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" -) - -type Client struct { - client *minio.Client - bucket string -} - -func New(endpoint, accessKey, secretKey, bucket string, useSSL bool) (*Client, error) { - client, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKey, secretKey, ""), - Secure: useSSL, - }) - if err != nil { - return nil, err - } - return &Client{client: client, bucket: bucket}, nil -} - -func (c *Client) EnsureBucket(ctx context.Context) error { - exists, err := c.client.BucketExists(ctx, c.bucket) - if err != nil { - return err - } - if exists { - return nil - } - err = c.client.MakeBucket(ctx, c.bucket, minio.MakeBucketOptions{}) - if err == nil { - return nil - } - resp := minio.ToErrorResponse(err) - if resp.Code == "BucketAlreadyOwnedByYou" || resp.Code == "BucketAlreadyExists" { - return nil - } - return err -} - -func (c *Client) Put(ctx context.Context, objectKey string, data []byte, contentType string) error { - if contentType == "" { - contentType = "application/pdf" - } - _, err := c.client.PutObject(ctx, c.bucket, objectKey, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ - ContentType: contentType, - }) - return err -} - -func (c *Client) Get(ctx context.Context, objectKey string) ([]byte, error) { - obj, err := c.client.GetObject(ctx, c.bucket, objectKey, minio.GetObjectOptions{}) - if err != nil { - return nil, err - } - defer obj.Close() - return io.ReadAll(obj) -} - -func (c *Client) ListByPrefix(ctx context.Context, prefix string) ([]string, error) { - var keys []string - for obj := range c.client.ListObjects(ctx, c.bucket, minio.ListObjectsOptions{ - Prefix: prefix, - Recursive: true, - }) { - if obj.Err != nil { - return nil, obj.Err - } - if strings.HasSuffix(strings.ToLower(obj.Key), ".pdf") { - keys = append(keys, obj.Key) - } - } - return keys, nil -} - -func JobObjectKey(jobID, filename string) string { - return fmt.Sprintf("jobs/%s/%s", jobID, filename) -} diff --git a/services/services-count-facturas-medicas-serving/internal/store/redis.go b/services/services-count-facturas-medicas-serving/internal/store/redis.go deleted file mode 100644 index 253fc4b..0000000 --- a/services/services-count-facturas-medicas-serving/internal/store/redis.go +++ /dev/null @@ -1,276 +0,0 @@ -package store - -import ( - "context" - "encoding/json" - "fmt" - "strconv" - "time" - - "github.com/pys/pdf-count/internal/domain" - "github.com/redis/go-redis/v9" -) - -type Store struct { - client *redis.Client - ttl time.Duration -} - -func New(redisURL string, ttl time.Duration) (*Store, error) { - opts, err := redis.ParseURL(redisURL) - if err != nil { - return nil, err - } - client := redis.NewClient(opts) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := client.Ping(ctx).Err(); err != nil { - return nil, fmt.Errorf("redis ping: %w", err) - } - return &Store{client: client, ttl: ttl}, nil -} - -func (s *Store) Close() error { - return s.client.Close() -} - -func metaKey(jobID string) string { return fmt.Sprintf("job:%s:meta", jobID) } -func filesKey(jobID string) string { return fmt.Sprintf("job:%s:files", jobID) } -func resultsKey(jobID string) string { - return fmt.Sprintf("job:%s:results", jobID) -} - -func (s *Store) CreateJob(ctx context.Context, jobID string, total int, includeRejected bool) error { - pipe := s.client.Pipeline() - pipe.HSet(ctx, metaKey(jobID), map[string]interface{}{ - "status": domain.JobQueued, - "total": total, - "processed": 0, - "failed": 0, - "include_rejected": boolStr(includeRejected), - }) - pipe.Expire(ctx, metaKey(jobID), s.ttl) - _, err := pipe.Exec(ctx) - return err -} - -func (s *Store) AddJobFiles(ctx context.Context, jobID string, filenames []string) error { - if len(filenames) == 0 { - return nil - } - members := make([]interface{}, len(filenames)) - for i, f := range filenames { - members[i] = f - } - pipe := s.client.Pipeline() - pipe.RPush(ctx, filesKey(jobID), members...) - pipe.Expire(ctx, filesKey(jobID), s.ttl) - _, err := pipe.Exec(ctx) - return err -} - -func (s *Store) MarkRunning(ctx context.Context, jobID string) error { - return s.client.HSet(ctx, metaKey(jobID), "status", domain.JobRunning).Err() -} - -var saveResultScript = redis.NewScript(` -local meta = KEYS[1] -local results = KEYS[2] -redis.call('HSET', results, ARGV[1], ARGV[2]) -local processed = redis.call('HINCRBY', meta, 'processed', 1) -if ARGV[3] == '1' then - redis.call('HINCRBY', meta, 'failed', 1) -end -redis.call('EXPIRE', results, ARGV[4]) -local total = tonumber(redis.call('HGET', meta, 'total') or '0') -local status = redis.call('HGET', meta, 'status') -if status ~= 'cancelled' and status ~= 'completed' and total > 0 and processed >= total then - redis.call('HSET', meta, 'status', 'completed') -end -return processed -`) - -func (s *Store) SaveResult(ctx context.Context, jobID string, result domain.PDFResult) error { - data, err := json.Marshal(result) - if err != nil { - return err - } - failFlag := "0" - if result.Status == domain.StatusError { - failFlag = "1" - } - ttlSec := int64(s.ttl.Seconds()) - if ttlSec <= 0 { - ttlSec = 72 * 3600 - } - _, err = saveResultScript.Run(ctx, s.client, - []string{metaKey(jobID), resultsKey(jobID)}, - result.Filename, string(data), failFlag, ttlSec, - ).Result() - return err -} - -func (s *Store) TryCompleteJob(ctx context.Context, jobID string) error { - meta, err := s.client.HGetAll(ctx, metaKey(jobID)).Result() - if err != nil { - return err - } - _, err = s.maybeCompleteJob(ctx, jobID, meta) - return err -} - -func (s *Store) maybeCompleteJob(ctx context.Context, jobID string, meta map[string]string) (string, error) { - status := meta["status"] - if status == domain.JobCancelled || status == domain.JobCompleted { - return status, nil - } - total, _ := strconv.Atoi(meta["total"]) - processed, _ := strconv.Atoi(meta["processed"]) - if processed >= total && total > 0 { - if err := s.client.HSet(ctx, metaKey(jobID), "status", domain.JobCompleted).Err(); err != nil { - return status, err - } - return domain.JobCompleted, nil - } - return status, nil -} - -func (s *Store) GetJob(ctx context.Context, jobID string) (domain.JobStatus, error) { - meta, err := s.client.HGetAll(ctx, metaKey(jobID)).Result() - if err != nil { - return domain.JobStatus{}, err - } - if len(meta) == 0 { - return domain.JobStatus{}, fmt.Errorf("job not found") - } - status, err := s.maybeCompleteJob(ctx, jobID, meta) - if err != nil { - return domain.JobStatus{}, err - } - if status == domain.JobCompleted { - meta["status"] = domain.JobCompleted - } - total, _ := strconv.Atoi(meta["total"]) - processed, _ := strconv.Atoi(meta["processed"]) - failed, _ := strconv.Atoi(meta["failed"]) - return domain.JobStatus{ - JobID: jobID, - Status: status, - Total: total, - Processed: processed, - Failed: failed, - }, nil -} - -func (s *Store) GetResultsPage(ctx context.Context, jobID, cursor string, limit int) (domain.JobResultsPage, error) { - if limit <= 0 || limit > 1000 { - limit = 500 - } - allFiles, err := s.client.LRange(ctx, filesKey(jobID), 0, -1).Result() - if err != nil { - return domain.JobResultsPage{}, err - } - - start := 0 - if cursor != "" { - for i, f := range allFiles { - if f == cursor { - start = i + 1 - break - } - } - } - end := start + limit - if end > len(allFiles) { - end = len(allFiles) - } - pageFiles := allFiles[start:end] - - results := make([]domain.PDFResult, 0, len(pageFiles)) - if len(pageFiles) > 0 { - raw, err := s.client.HMGet(ctx, resultsKey(jobID), pageFiles...).Result() - if err != nil { - return domain.JobResultsPage{}, err - } - for i, v := range raw { - if v == nil { - results = append(results, domain.PDFResult{ - Filename: pageFiles[i], - Status: "pending", - }) - continue - } - var r domain.PDFResult - if err := json.Unmarshal([]byte(v.(string)), &r); err != nil { - continue - } - results = append(results, r) - } - } - - page := domain.JobResultsPage{ - JobID: jobID, - Results: results, - HasMore: end < len(allFiles), - } - if page.HasMore && len(pageFiles) > 0 { - page.NextCursor = pageFiles[len(pageFiles)-1] - } - return page, nil -} - -func (s *Store) CancelJob(ctx context.Context, jobID string) error { - return s.client.HSet(ctx, metaKey(jobID), "status", domain.JobCancelled).Err() -} - -func (s *Store) IsCancelled(ctx context.Context, jobID string) (bool, error) { - status, err := s.client.HGet(ctx, metaKey(jobID), "status").Result() - if err != nil { - return false, err - } - return status == domain.JobCancelled, nil -} - -func (s *Store) PendingCount(ctx context.Context) (int64, error) { - // Approximate: sum of (total - processed) for running jobs - var cursor uint64 - var pending int64 - for { - keys, next, err := s.client.Scan(ctx, cursor, "job:*:meta", 100).Result() - if err != nil { - return pending, err - } - for _, key := range keys { - meta, err := s.client.HGetAll(ctx, key).Result() - if err != nil { - continue - } - if meta["status"] != domain.JobQueued && meta["status"] != domain.JobRunning { - continue - } - total, _ := strconv.Atoi(meta["total"]) - processed, _ := strconv.Atoi(meta["processed"]) - pending += int64(total - processed) - } - cursor = next - if cursor == 0 { - break - } - } - return pending, nil -} - -func (s *Store) IncrementTotal(ctx context.Context, jobID string, delta int) error { - pipe := s.client.Pipeline() - pipe.HIncrBy(ctx, metaKey(jobID), "total", int64(delta)) - pipe.Expire(ctx, metaKey(jobID), s.ttl) - _, err := pipe.Exec(ctx) - return err -} - -func boolStr(v bool) string { - if v { - return "true" - } - return "false" -} diff --git a/services/services-count-facturas-medicas-serving/internal/worker/runner.go b/services/services-count-facturas-medicas-serving/internal/worker/runner.go deleted file mode 100644 index 6879f03..0000000 --- a/services/services-count-facturas-medicas-serving/internal/worker/runner.go +++ /dev/null @@ -1,212 +0,0 @@ -package worker - -import ( - "context" - "log" - "os/signal" - "sync" - "syscall" - "time" - - "github.com/nats-io/nats.go/jetstream" - "github.com/pys/pdf-count/internal/config" - "github.com/pys/pdf-count/internal/domain" - "github.com/pys/pdf-count/internal/model" - "github.com/pys/pdf-count/internal/ocr" - "github.com/pys/pdf-count/internal/pipeline" - "github.com/pys/pdf-count/internal/queue" - "github.com/pys/pdf-count/internal/storage" - "github.com/pys/pdf-count/internal/store" -) - -type Runner struct { - cfg config.Config - predictor *model.Predictor - ocrEngine *ocr.Engine - store *store.Store - storage *storage.Client - consumer jetstream.Consumer - ncClose func() -} - -func NewRunner(cfg config.Config) (*Runner, error) { - predictor, err := model.NewPredictor(cfg.ModelPath, cfg.InferencePath) - if err != nil { - return nil, err - } - poolSize := cfg.OCRPoolSize - if poolSize <= 0 { - poolSize = cfg.WorkerConcurrency - } - ocrEngine, err := ocr.NewEngine(cfg.OCRLang, poolSize) - if err != nil { - predictor.Close() - return nil, err - } - st, err := store.New(cfg.RedisURL, cfg.JobResultTTL) - if err != nil { - ocrEngine.Close() - predictor.Close() - return nil, err - } - stg, err := storage.New(cfg.MinIOEndpoint, cfg.MinIOAccessKey, cfg.MinIOSecretKey, cfg.MinIOBucket, cfg.MinIOUseSSL) - if err != nil { - ocrEngine.Close() - st.Close() - predictor.Close() - return nil, err - } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if err := stg.EnsureBucket(ctx); err != nil { - ocrEngine.Close() - st.Close() - predictor.Close() - return nil, err - } - - _, cons, nc, err := queue.NewConsumer(cfg.NATSURL, cfg.StreamName, cfg.SubjectPrefix, "pdf-workers") - if err != nil { - ocrEngine.Close() - st.Close() - predictor.Close() - return nil, err - } - - return &Runner{ - cfg: cfg, - predictor: predictor, - ocrEngine: ocrEngine, - store: st, - storage: stg, - consumer: cons, - ncClose: func() { nc.Close() }, - }, nil -} - -func (r *Runner) Close() { - r.ncClose() - _ = r.ocrEngine.Close() - _ = r.store.Close() - _ = r.predictor.Close() -} - -func (r *Runner) processSlots() int { - c := r.cfg.WorkerConcurrency - pool := r.cfg.OCRPoolSize - if pool <= 0 { - pool = r.cfg.WorkerConcurrency - } - if pool <= 0 { - pool = 4 - } - if c > pool { - c = pool - } - if c <= 0 { - c = 4 - } - return c -} - -func (r *Runner) Run(ctx context.Context) error { - slots := r.processSlots() - sem := make(chan struct{}, slots) - var wg sync.WaitGroup - - go func() { - <-ctx.Done() - wg.Wait() - }() - - for { - select { - case <-ctx.Done(): - return nil - default: - } - - msgs, err := r.consumer.Fetch(slots, jetstream.FetchMaxWait(r.cfg.NATSFetchMaxWait)) - if err != nil { - continue - } - for msg := range msgs.Messages() { - sem <- struct{}{} - wg.Add(1) - m := msg - go func() { - defer func() { - <-sem - wg.Done() - }() - r.handleMessage(ctx, m) - }() - } - _ = msgs.Error() - } -} - -func (r *Runner) handleMessage(ctx context.Context, msg jetstream.Msg) { - processCtx, cancel := context.WithTimeout(ctx, r.cfg.PDFProcessTimeout) - defer cancel() - - qmsg, err := queue.DecodeMessage(msg.Data()) - if err != nil { - _ = msg.Term() - return - } - - cancelled, err := r.store.IsCancelled(processCtx, qmsg.JobID) - if err != nil || cancelled { - _ = msg.Ack() - return - } - - data, err := r.storage.Get(processCtx, qmsg.ObjectKey) - if err != nil { - _ = r.store.SaveResult(processCtx, qmsg.JobID, domain.PDFResult{ - Filename: qmsg.Filename, - Status: domain.StatusError, - Error: err.Error(), - }) - _ = r.store.TryCompleteJob(processCtx, qmsg.JobID) - _ = msg.Ack() - return - } - - proc := pipeline.NewProcessor(r.predictor, r.ocrEngine, pipeline.Options{ - DPI: r.cfg.OCRDPI, - Lang: r.cfg.OCRLang, - DBSCANEps: r.cfg.DBSCANEps, - DBSCANMinSamples: r.cfg.DBSCANMinSamples, - IncludeRejected: qmsg.IncludeRejected, - LogPipelineTiming: r.cfg.LogPipelineTiming, - }) - result := proc.ProcessPDF(qmsg.Filename, data) - data = nil - - if err := r.store.SaveResult(processCtx, qmsg.JobID, result); err != nil { - _ = msg.NakWithDelay(5 * time.Second) - return - } - _ = r.store.TryCompleteJob(processCtx, qmsg.JobID) - _ = msg.Ack() -} - -func RunMain() { - cfg := config.Load() - runner, err := NewRunner(cfg) - if err != nil { - log.Fatalf("worker init: %v", err) - } - defer runner.Close() - - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer cancel() - - log.Printf("worker started slots=%d", runner.processSlots()) - if err := runner.Run(ctx); err != nil { - log.Fatalf("worker: %v", err) - } - log.Println("worker stopped") -}