feat: Add database migration service with Alembic for PostgreSQL, integrating job management and counting functionalities. Update Docker Compose to include db-migrations service, enhancing the microservices architecture. Revise documentation for clarity on database schema and migration processes.

This commit is contained in:
erik
2026-07-06 00:50:50 -05:00
parent d95f8f6ad8
commit 081da999d4
43 changed files with 2911 additions and 2271 deletions
+56 -19
View File
@@ -1,29 +1,66 @@
# IPSExtractor API Gateway
# api_gateway
Gateway HTTP opcional. Esta carpeta puede vivir como repo propio: solo depende
de HTTP para hablar con `extraction` y `cleaning`; no importa codigo interno de
esos servicios.
Puerta de enlace HTTP (FastAPI) que actúa como punto de entrada único para el sistema IPSExtractor.
Enruta las peticiones del cliente hacia los microservicios internos sin lógica de negocio propia.
## Estructura
---
```text
api/ Endpoints FastAPI y proxy HTTP
requirements.txt
## Puerto
**8000**
---
## Enrutamiento
| Ruta cliente | Servicio destino | Descripción |
|--------------|------------------|-------------|
| `GET /health` | Agrega `/health` de los tres servicios | Estado del sistema |
| `GET /models` | `extraction:8001/models` | Modelos LLM disponibles |
| `GET /config` | `extraction:8001/config` + `cleaning:8002/rules` | Configuración |
| `/extraction/{path}` | `extraction:8001/{path}` | Extracción (quita prefijo) |
| `/tasks/{path}` | `extraction:8001/tasks/{path}` | Tareas de extracción |
| `/jobs/{path}` | `extraction:8001/jobs/{path}` | Jobs |
| `/count-jobs/{path}` | `extraction:8001/count-jobs/{path}` | Jobs de conteo |
| `/cleaning/{path}` | `cleaning:8002/{path}` | Limpieza |
| `/count/{path}` | `count-api:8080/{path}` | Conteo OCR+ML (quita prefijo) |
El proxy reenvía método, query string, headers y body sin modificaciones.
Timeout: ilimitado (para uploads grandes).
---
## Variables de entorno
| Variable | Default | Descripción |
|----------|---------|-------------|
| `EXTRACTION_URL` | `http://extraction:8001` | URL interna del servicio extraction |
| `CLEANING_URL` | `http://cleaning:8002` | URL interna del servicio cleaning |
| `COUNT_URL` | `http://count-api:8080` | URL interna del servicio count-api |
---
## CORS
Solo permite orígenes `localhost` y `127.0.0.1`. Sin autenticación.
---
## Estructura del código
```
api/
main.py — FastAPI app con proxy HTTP (httpx)
Dockerfile
requirements.txt — fastapi, httpx, uvicorn
```
## Desarrollo local
---
## Arrancar localmente
```bash
pip install -r requirements.txt
uvicorn api.main:app --host 0.0.0.0 --port 8000
docker compose up --build api_gateway
```
## Endpoints principales
- `GET /health`
- `GET /models`
- `GET /config`
- `ANY /extraction/{path}`
- `ANY /cleaning/{path}`
API disponible en `http://localhost:8000`.
+28
View File
@@ -11,6 +11,7 @@ from fastapi.middleware.cors import CORSMiddleware
EXTRACTION_URL = os.getenv("EXTRACTION_URL", "http://extraction:8001").rstrip("/")
CLEANING_URL = os.getenv("CLEANING_URL", "http://cleaning:8002").rstrip("/")
COUNT_URL = os.getenv("COUNT_URL", "http://count-api:8080").rstrip("/")
app = FastAPI(title="IPSExtractor API Gateway", version="1.0.0")
@@ -28,11 +29,13 @@ async def health() -> dict[str, Any]:
async with httpx.AsyncClient(timeout=10.0) as client:
extraction = await _safe_health(client, EXTRACTION_URL)
cleaning = await _safe_health(client, CLEANING_URL)
counting = await _safe_health(client, COUNT_URL)
return {
"status": "ok" if extraction["ok"] and cleaning["ok"] else "degraded",
"services": {
"extraction": extraction,
"cleaning": cleaning,
"counting": counting,
},
}
@@ -73,6 +76,31 @@ async def cleaning_proxy(path: str, request: Request) -> Response:
return await _proxy_request(request, f"{CLEANING_URL}/{path}")
@app.api_route("/count/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
async def count_proxy(path: str, request: Request) -> Response:
return await _proxy_request(request, f"{COUNT_URL}/{path}")
@app.api_route("/count-jobs", methods=["GET", "POST"])
async def count_jobs_root_proxy(request: Request) -> Response:
return await _proxy_request(request, f"{EXTRACTION_URL}/count-jobs")
@app.api_route("/count-jobs/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
async def count_jobs_proxy(path: str, request: Request) -> Response:
return await _proxy_request(request, f"{EXTRACTION_URL}/count-jobs/{path}")
@app.api_route("/jobs", methods=["GET"])
async def jobs_root_proxy(request: Request) -> Response:
return await _proxy_request(request, f"{EXTRACTION_URL}/jobs")
@app.api_route("/jobs/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
async def jobs_proxy(path: str, request: Request) -> Response:
return await _proxy_request(request, f"{EXTRACTION_URL}/jobs/{path}")
async def _safe_health(client: httpx.AsyncClient, base_url: str) -> dict[str, Any]:
try:
response = await client.get(f"{base_url}/health")
+87 -23
View File
@@ -1,32 +1,96 @@
# IPSExtractor Cleaning Service
# cleaning
Microservicio independiente de limpieza. Esta carpeta puede vivir como repo
propio: contiene su API REST, reglas de limpieza, infraestructura, dependencias
y Dockerfile sin importar codigo de otros servicios.
Microservicio Python (FastAPI) de limpieza y normalización de datos extraídos.
Recibe lotes de filas extraídas por el servicio `extraction` y aplica reglas configurables de transformación de columnas.
## Estructura
---
```text
api/ Endpoints FastAPI
domain/ Reglas y analisis de limpieza
infrastructure/ Persistencia SQLAlchemy del servicio
requirements.txt
Dockerfile
```
## Puerto
## Desarrollo local
**8002**
---
## Endpoints
| Método | Ruta | Descripción |
|--------|------|-------------|
| `GET` | `/health` | Liveness |
| `GET` | `/rules` | Lista de reglas de limpieza disponibles |
| `GET` | `/batches/{batch_id}/analysis` | Análisis de columnas de un lote de extracción |
| `POST` | `/jobs` | Crear job de limpieza |
| `GET` | `/jobs/{job_id}` | Estado y detalle del job |
| `GET` | `/jobs/{job_id}/rows` | Filas limpias del job |
---
## Crear un job de limpieza
```bash
POST /jobs
{
"extraction_batch_id": "...",
"rules_per_column": {
"valor": ["to_integer"],
"fecha": ["date_to_ddmmyyyy"],
"codigo": ["strip_spaces"]
}
}
```
Las reglas disponibles se listan en `GET /rules`.
---
## Reglas de limpieza
Definidas en `domain/csv_cleaning.py`. Cada regla es una función que transforma los valores de una columna. Se pueden encadenar múltiples reglas por columna.
---
## Variables de entorno
| Variable | Descripción |
|----------|-------------|
| `DATABASE_URL` | PostgreSQL (`postgresql+psycopg://...`) |
---
## Modelo de datos
```
extraction_batches — lote de filas de extracción (leído, no modificado)
extraction_rows — filas originales
cleaning_jobs — job de limpieza (FK → extraction_batches)
cleaned_rows — filas transformadas
```
---
## Estructura del código
```
api/
main.py — FastAPI app con todos los endpoints
domain/
csv_cleaning.py — reglas de limpieza y función de análisis
infrastructure/
database.py — modelos SQLAlchemy y sesión
Dockerfile
requirements.txt
```
---
## Arrancar localmente
```bash
# Con Docker Compose (desde la raíz del repositorio):
docker compose up --build cleaning
# Sin Docker:
cd services/cleaning
pip install -r requirements.txt
uvicorn api.main:app --host 0.0.0.0 --port 8002
```
## Endpoints principales
- `GET /health`
- `GET /rules`
- `GET /batches/{batch_id}/analysis`
- `POST /jobs`
- `GET /jobs/{job_id}`
- `GET /jobs/{job_id}/rows`
+10
View File
@@ -0,0 +1,10 @@
FROM python:3.12-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY . .
CMD ["python", "-m", "run_migrations"]
+66
View File
@@ -0,0 +1,66 @@
# db (migraciones)
Contenedor de un solo uso que aplica las migraciones de base de datos con Alembic.
Se ejecuta al arrancar el stack y termina con código 0 al completar. El orquestador Docker reinicia en caso de error.
---
## Comportamiento
1. Conecta a PostgreSQL.
2. Adquiere un advisory lock para evitar ejecuciones concurrentes.
3. Aplica todas las migraciones pendientes.
4. Termina.
Los demás servicios tienen `depends_on: db-migrations: condition: service_completed_successfully`, por lo que no arrancan hasta que las migraciones terminen.
---
## Variables de entorno
| Variable | Descripción |
|----------|-------------|
| `DATABASE_URL` | PostgreSQL (`postgresql+psycopg://...`) |
---
## Esquema de base de datos
```
jobs — contenedor de un trabajo del usuario (multimódulo)
tasks — ejecución de un módulo dentro de un job
type: counting | extraction | cleaning
count_job_files — resultado de conteo por PDF (job_module_id → tasks.id)
extraction_batches — lote de extracción (job_module_id → tasks.id)
extraction_rows — fila extraída (batch_id → extraction_batches.id)
cleaning_jobs — job de limpieza (task_id → tasks.id)
cleaned_rows — fila limpiada (cleaning_job_id → cleaning_jobs.id)
```
Documentación completa en [`docs/database.dbml`](../../docs/database.dbml).
---
## Estructura del código
```
Dockerfile — imagen Python slim + alembic + psycopg
alembic.ini — configuración de Alembic
alembic/
env.py — entorno de migraciones con advisory lock
versions/
0001_initial_schema.py
0002_add_jobs_and_count.py
run_migrations.py — script principal con manejo de errores
requirements.txt
```
---
## Añadir una migración
```bash
# Generar una nueva revisión
docker run --rm -e DATABASE_URL=... ipsextractor-db-migrations \
alembic revision --autogenerate -m "descripcion"
```
+39
View File
@@ -0,0 +1,39 @@
[alembic]
script_location = alembic
prepend_sys_path = .
version_path_separator = os
sqlalchemy.url = postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
import os
from logging.config import fileConfig
from alembic import context
from sqlalchemy import engine_from_config, pool
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
database_url = os.environ.get("DATABASE_URL") or config.get_main_option("sqlalchemy.url")
if database_url:
config.set_main_option("sqlalchemy.url", database_url)
target_metadata = None
def run_migrations_offline() -> None:
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online() -> None:
connectable = engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
+27
View File
@@ -0,0 +1,27 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from __future__ import annotations
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
@@ -0,0 +1,124 @@
"""Schema inicial: extraction_batches, tasks, extraction_rows, cleaning_jobs, cleaned_rows.
Revision ID: 0001
Revises:
Create Date: 2026-07-05
"""
from __future__ import annotations
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0001"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
existing = set(inspector.get_table_names())
if "extraction_batches" not in existing:
op.create_table(
"extraction_batches",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
sa.Column("root_path", sa.Text, nullable=True),
sa.Column("request", sa.JSON, nullable=False, server_default="{}"),
sa.Column("error", sa.Text, nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_extraction_batches_status", "extraction_batches", ["status"])
if "tasks" not in existing:
op.create_table(
"tasks",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("type", sa.String(64), nullable=False),
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
sa.Column("title", sa.Text, nullable=True),
sa.Column("request", sa.JSON, nullable=False, server_default="{}"),
sa.Column("result", sa.JSON, nullable=False, server_default="{}"),
sa.Column("error", sa.Text, nullable=True),
sa.Column("owner_id", sa.String(36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
)
op.create_index("ix_tasks_type", "tasks", ["type"])
op.create_index("ix_tasks_status", "tasks", ["status"])
op.create_index("ix_tasks_owner_id", "tasks", ["owner_id"])
if "extraction_rows" not in existing:
op.create_table(
"extraction_rows",
sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
sa.Column(
"batch_id", sa.String(36),
sa.ForeignKey("extraction_batches.id"), nullable=False,
),
sa.Column("radicado", sa.String(255), nullable=True),
sa.Column("source_pdf", sa.Text, nullable=True),
sa.Column("row_index", sa.Integer, nullable=False, server_default="0"),
sa.Column("row_json", sa.JSON, nullable=False, server_default="{}"),
sa.Column("raw_json", sa.JSON, nullable=False, server_default="{}"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_extraction_rows_batch_id", "extraction_rows", ["batch_id"])
op.create_index("ix_extraction_rows_radicado", "extraction_rows", ["radicado"])
if "cleaning_jobs" not in existing:
op.create_table(
"cleaning_jobs",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column(
"extraction_batch_id", sa.String(36),
sa.ForeignKey("extraction_batches.id"), nullable=False,
),
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
sa.Column("request", sa.JSON, nullable=False, server_default="{}"),
sa.Column("log", sa.JSON, nullable=False, server_default="[]"),
sa.Column("error", sa.Text, nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index(
"ix_cleaning_jobs_extraction_batch_id", "cleaning_jobs", ["extraction_batch_id"]
)
op.create_index("ix_cleaning_jobs_status", "cleaning_jobs", ["status"])
if "cleaned_rows" not in existing:
op.create_table(
"cleaned_rows",
sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
sa.Column(
"cleaning_job_id", sa.String(36),
sa.ForeignKey("cleaning_jobs.id"), nullable=False,
),
sa.Column(
"extraction_batch_id", sa.String(36),
sa.ForeignKey("extraction_batches.id"), nullable=False,
),
sa.Column("row_index", sa.Integer, nullable=False, server_default="0"),
sa.Column("row_json", sa.JSON, nullable=False, server_default="{}"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_cleaned_rows_cleaning_job_id", "cleaned_rows", ["cleaning_job_id"])
op.create_index(
"ix_cleaned_rows_extraction_batch_id", "cleaned_rows", ["extraction_batch_id"]
)
def downgrade() -> None:
op.drop_table("cleaned_rows")
op.drop_table("cleaning_jobs")
op.drop_table("extraction_rows")
op.drop_table("tasks")
op.drop_table("extraction_batches")
@@ -0,0 +1,104 @@
"""Agregar tabla jobs, count_job_files, job_id en tasks, job_module_id en extraction_batches.
Revision ID: 0002
Revises: 0001
Create Date: 2026-07-05
"""
from __future__ import annotations
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0002"
down_revision: Union[str, None] = "0001"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _col_exists(conn: sa.engine.Connection, table: str, column: str) -> bool:
inspector = sa.inspect(conn)
return any(c["name"] == column for c in inspector.get_columns(table))
def _table_exists(conn: sa.engine.Connection, table: str) -> bool:
inspector = sa.inspect(conn)
return table in inspector.get_table_names()
def upgrade() -> None:
conn = op.get_bind()
# ── tabla jobs ──────────────────────────────────────────────────────────
if not _table_exists(conn, "jobs"):
op.create_table(
"jobs",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
sa.Column("title", sa.Text, nullable=True),
sa.Column("metadata", sa.JSON, nullable=False, server_default="{}"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_jobs_status", "jobs", ["status"])
# ── columna job_id en tasks ──────────────────────────────────────────────
if not _col_exists(conn, "tasks", "job_id"):
op.add_column(
"tasks",
sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True),
)
op.create_index("ix_tasks_job_id", "tasks", ["job_id"])
# ── tabla count_job_files ────────────────────────────────────────────────
if not _table_exists(conn, "count_job_files"):
op.create_table(
"count_job_files",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column(
"job_module_id", sa.String(36),
sa.ForeignKey("tasks.id"), nullable=False,
),
sa.Column("radicado", sa.String(255), nullable=True),
sa.Column("filename", sa.String(512), nullable=False),
sa.Column("file_path", sa.Text, nullable=True),
sa.Column("doc_type", sa.String(128), nullable=True),
sa.Column("service_count", sa.Integer, nullable=True),
sa.Column("services", sa.JSON, nullable=False, server_default="[]"),
sa.Column("status", sa.String(32), nullable=False, server_default="ok"),
sa.Column("is_valid", sa.Boolean, nullable=True),
sa.Column("error", sa.Text, nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_count_job_files_job_module_id", "count_job_files", ["job_module_id"])
op.create_index("ix_count_job_files_radicado", "count_job_files", ["radicado"])
op.create_index("ix_count_job_files_status", "count_job_files", ["status"])
# ── columna job_module_id en extraction_batches ──────────────────────────
if not _col_exists(conn, "extraction_batches", "job_module_id"):
op.add_column(
"extraction_batches",
sa.Column(
"job_module_id", sa.String(36),
sa.ForeignKey("tasks.id"), nullable=True,
),
)
op.create_index(
"ix_extraction_batches_job_module_id", "extraction_batches", ["job_module_id"]
)
def downgrade() -> None:
conn = op.get_bind()
if _col_exists(conn, "extraction_batches", "job_module_id"):
op.drop_index("ix_extraction_batches_job_module_id", "extraction_batches")
op.drop_column("extraction_batches", "job_module_id")
if _table_exists(conn, "count_job_files"):
op.drop_table("count_job_files")
if _col_exists(conn, "tasks", "job_id"):
op.drop_index("ix_tasks_job_id", "tasks")
op.drop_column("tasks", "job_id")
if _table_exists(conn, "jobs"):
op.drop_table("jobs")
+3
View File
@@ -0,0 +1,3 @@
alembic==1.16.2
psycopg[binary]==3.2.3
SQLAlchemy==2.0.36
+55
View File
@@ -0,0 +1,55 @@
"""Punto de entrada del servicio de migraciones.
Se ejecuta como init container antes de que arranquen los demás servicios.
Aplica todas las migraciones pendientes hasta 'head' con bloqueo advisory
para evitar condiciones de carrera en despliegues con múltiples réplicas.
"""
from __future__ import annotations
import os
import sys
import logging
from sqlalchemy import create_engine, text
from alembic.config import Config as AlembicConfig
from alembic import command as alembic_command
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
log = logging.getLogger(__name__)
DATABASE_URL = os.environ.get(
"DATABASE_URL",
"postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor",
)
def main() -> None:
log.info("Conectando a la base de datos…")
engine = create_engine(DATABASE_URL, pool_pre_ping=True)
alembic_cfg = AlembicConfig()
alembic_cfg.set_main_option("script_location", "alembic")
alembic_cfg.set_main_option("sqlalchemy.url", DATABASE_URL)
if DATABASE_URL.startswith("postgresql"):
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
conn.execute(text("SELECT pg_advisory_lock(74512001)"))
log.info("Bloqueo advisory adquirido — ejecutando migraciones…")
try:
alembic_command.upgrade(alembic_cfg, "head")
log.info("Migraciones completadas.")
finally:
conn.execute(text("SELECT pg_advisory_unlock(74512001)"))
else:
alembic_command.upgrade(alembic_cfg, "head")
log.info("Migraciones completadas.")
engine.dispose()
if __name__ == "__main__":
try:
main()
except Exception as exc:
log.error("Migraciones fallidas: %s", exc)
sys.exit(1)
+92 -34
View File
@@ -1,48 +1,106 @@
# IPSExtractor Desktop App
# desktop_app
Aplicacion desktop Windows/Mac para ejecutar el flujo de extraccion usando el
API Gateway de IPSExtractor. Esta carpeta funciona como repo independiente.
Aplicación de escritorio Electron + React para interactuar con el stack IPSExtractor.
Permite seleccionar carpetas locales, contar servicios en facturas médicas, revisar resultados y lanzar extracciones LLM.
## Requisitos
---
- Node.js 20+
- Backend levantado con `docker compose up --build`
- Electron (incluido en `npm run dev`)
## Flujos principales
No necesitas montar la carpeta del usuario en Docker. La app indexa los archivos
localmente y los sube uno a uno al servicio de extraccion.
### Conteo de facturas
## Desarrollo
1. Seleccionar carpeta con PDFs.
2. Los PDFs se envían en lotes de 10 a `POST /count/v1/count` (count-api).
3. Los resultados se guardan en PostgreSQL vía `POST /count-jobs` (extraction).
4. El usuario revisa y marca archivos como válidos/inválidos.
### Extracción desde conteo
1. Desde el wizard de conteo, clic en "Extraer archivos válidos".
2. Se llama `POST /tasks/extraction/from-count` con el `counting_task_id`.
3. El servicio extraction lee los PDFs desde el volumen `/data` y los procesa con LLM.
### Extracción directa
1. Seleccionar carpeta → discovery de campos → seleccionar campos → estimar costo.
2. Subir PDFs en lotes a `/tasks/{id}/ingest-many`.
3. Polling de progreso vía `GET /tasks/{id}`.
4. Exportar resultados como CSV.
---
## Arquitectura técnica
| Capa | Tecnología |
|------|-----------|
| Shell | Electron 35+ |
| UI | React 19 + TypeScript |
| Bundler | Vite |
| Estilos | CSS custom (sin framework UI) |
| IPC | `contextBridge` (preload.cjs) |
### Comunicación con el backend
Toda la comunicación pasa por el API Gateway (`http://localhost:8000`).
El cliente HTTP es [`src/renderer/lib/apiClient.ts`](src/renderer/lib/apiClient.ts).
### Indexación de carpetas
El proceso principal (Electron) indexa carpetas locales con [`electron/folderIndex.cjs`](electron/folderIndex.cjs):
- Modo `general`: lista todos los PDFs/TIFFs de la carpeta.
- Modo `radicados`: busca un PDF por tipo de documento dentro de subdirectorios con nombre de radicado.
---
## Estructura del código
```
electron/
main.cjs — proceso principal Electron
preload.cjs — bridge seguro main ↔ renderer (contextBridge)
folderIndex.cjs — indexación de carpetas locales
src/
renderer/
App.tsx — root: navegación y estado global
components/
CountingWizard.tsx — wizard de conteo en 3 pasos
CountResultsTable.tsx — tabla de resultados con validación
CountingStepper.tsx — indicador de progreso
ExtractionWizard.tsx — wizard de extracción en 4 pasos
ExtractionStepper.tsx — indicador de progreso
lib/
apiClient.ts — cliente HTTP hacia el API Gateway
payloads.ts — builders de FormData y JSON
pathMapping.ts — mapeo rutas locales ↔ contenedor
csv.ts — exportación CSV
styles.css — estilos globales
shared/
types.ts — tipos compartidos main/renderer
```
---
## Variables de configuración (runtime)
| Variable | Default | Descripción |
|----------|---------|-------------|
| `API Gateway URL` | `http://localhost:8000` | Configurable en la pantalla de Configuración |
| Ruta local de documentos | — | Seleccionada con el explorador de archivos |
| Ruta en contenedor | `/data` | Prefijo donde Docker monta los documentos |
---
## Arrancar en desarrollo
```bash
cd services/desktop_app
npm install
npm run dev
```
El gateway por defecto es `http://localhost:8000`.
## Tests y build
## Build para producción
```bash
npm test
npm run build
npm run dist
npm run build # compila el renderer
npm run electron:build # empaqueta la app
```
## Flujo desktop
1. Verificar conexion al gateway.
2. Seleccionar una carpeta local (Electron indexa PDF/TIFF).
3. Configurar modo `radicados` o `general`.
4. Descubrir campos (`POST /extraction/schemas/upload` con un archivo de muestra).
5. Seleccionar campos y alias.
6. Ejecutar extraccion (`POST /tasks/extraction` con `source=desktop` + ingestas).
7. Consultar el historial de tareas.
8. Ver filas extraidas desde la base de datos.
9. Exportar CSV local de forma opcional.
Los PDFs permanecen en el disco del usuario. El servidor solo usa archivos
temporales durante cada ingest y guarda JSON en Postgres.
La app no incluye limpieza, preclasificacion, postproceso ni dashboard completo
en esta version.
+181 -40
View File
@@ -1,6 +1,6 @@
import { Activity, AlertCircle, CheckCircle2, Clock, Database, Download, FileSearch, Loader2, RefreshCw, Server, Settings, XCircle } from "lucide-react";
import { Activity, AlertCircle, CheckCircle2, Clock, Database, Download, FileSearch, Hash, Loader2, RefreshCw, Server, Settings, XCircle } from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import type { DocumentType, ExtractionFormState, FieldSelection, ManifestEntry, RowsResponse, SchemaResponse, Task } from "../shared/types";
import type { DocumentType, ExtractionFormState, ExtractionFromCountPayload, FieldSelection, ManifestEntry, RowsResponse, SchemaResponse, Task } from "../shared/types";
import { ApiClient } from "./lib/apiClient";
import { rowsToCsv } from "./lib/csv";
import {
@@ -16,6 +16,7 @@ import { defaultAlias } from "./lib/schemaDisplay";
import { splitRadicados } from "./lib/pathMapping";
import { ExtractionWizard } from "./components/ExtractionWizard";
import type { ExtractionStep } from "./components/ExtractionStepper";
import { CountingWizard } from "./components/CountingWizard";
const defaultForm: ExtractionFormState = {
apiBaseUrl: localStorage.getItem("apiBaseUrl") || "http://localhost:8000",
@@ -36,7 +37,7 @@ const defaultForm: ExtractionFormState = {
};
type Status = "idle" | "ok" | "error" | "busy";
type View = "extraction" | "tasks" | "settings";
type View = "extraction" | "counting" | "tasks" | "settings";
function App() {
const [view, setView] = useState<View>("extraction");
@@ -59,6 +60,8 @@ function App() {
const [error, setError] = useState<string | null>(null);
const [extractionStep, setExtractionStep] = useState<ExtractionStep>(1);
const [maxReachedStep, setMaxReachedStep] = useState<ExtractionStep>(1);
const [pendingCountTaskId, setPendingCountTaskId] = useState<string | null>(null);
const [pendingValidCount, setPendingValidCount] = useState<number>(0);
const client = useMemo(() => new ApiClient(form.apiBaseUrl), [form.apiBaseUrl]);
const flattened = useMemo(() => (schemaResponse ? flattenSchema(schemaResponse) : { schema: {}, sample: null }), [schemaResponse]);
@@ -393,12 +396,19 @@ function App() {
});
}
const currentTitle = view === "extraction" ? "Nueva extraccion" : view === "tasks" ? "Tareas y resultados" : "Configuracion";
const currentDescription = view === "extraction"
? "Asistente paso a paso para extraer datos de tus documentos."
: view === "tasks"
? "Consulta el historial de ejecuciones y las filas guardadas en la base de datos."
: "Administra la conexion al API Gateway local.";
const currentTitle =
view === "extraction" ? "Nueva extraccion"
: view === "counting" ? "Contar servicios"
: view === "tasks" ? "Tareas y resultados"
: "Configuracion";
const currentDescription =
view === "extraction"
? "Asistente paso a paso para extraer datos de tus documentos."
: view === "counting"
? "Cuenta servicios en facturas medicas y selecciona archivos validos para extraer."
: view === "tasks"
? "Consulta el historial de ejecuciones y las filas guardadas en la base de datos."
: "Administra la conexion al API Gateway local.";
return (
<main className="app-frame">
@@ -416,6 +426,10 @@ function App() {
<FileSearch size={18} />
<span>Extraccion</span>
</button>
<button type="button" className={view === "counting" ? "active" : ""} onClick={() => setView("counting")}>
<Hash size={18} />
<span>Conteo</span>
</button>
<button type="button" className={view === "tasks" ? "active" : ""} onClick={() => setView("tasks")}>
<Activity size={18} />
<span>Tareas</span>
@@ -436,8 +450,8 @@ function App() {
</div>
</aside>
<section className={`workspace ${view === "extraction" ? "workspace--wiz" : ""} ${view === "tasks" ? "workspace--tasks" : ""} ${view === "settings" ? "workspace--settings" : ""}`}>
{view === "extraction" ? (
<section className={`workspace ${view === "extraction" ? "workspace--wiz" : ""} ${view === "counting" ? "workspace--wiz" : ""} ${view === "tasks" ? "workspace--tasks" : ""} ${view === "settings" ? "workspace--settings" : ""}`}>
{(view === "extraction" || view === "counting") ? (
<header className="topbar-inline">
<h1>{currentTitle}</h1>
<button type="button" className="btn btn-sm" onClick={verifyConnection} disabled={busy}>
@@ -459,35 +473,66 @@ function App() {
)}
{view === "extraction" && (
<ExtractionWizard
step={extractionStep}
maxReached={maxReachedStep}
onStepChange={goToStep}
form={form}
updateForm={updateForm}
onModeChange={handleModeChange}
busy={busy}
error={error}
statusText={statusText}
selectRoot={selectRoot}
discoverSchema={discoverSchema}
estimate={estimate}
runExtraction={runExtraction}
selection={selection}
setSelection={setSelection}
schemaSearch={schemaSearch}
setSchemaSearch={setSchemaSearch}
flattened={flattened}
schemaResponse={schemaResponse}
estimateJson={estimateJson}
manifest={manifest}
ingestProgress={ingestProgress}
setDocType={setDocType}
toggleField={toggleField}
selectAllFields={selectAllFields}
canGoNext={canGoNext}
onBack={handleBack}
onNext={handleNext}
<>
{pendingCountTaskId && (
<FromCountBanner
countTaskId={pendingCountTaskId}
validCount={pendingValidCount}
onDismiss={() => setPendingCountTaskId(null)}
client={client}
form={form}
selection={selection}
onTaskCreated={(task) => {
setPendingCountTaskId(null);
setTasks((prev) => [task, ...prev]);
setActiveTask(task);
setView("tasks");
}}
/>
)}
<ExtractionWizard
step={extractionStep}
maxReached={maxReachedStep}
onStepChange={goToStep}
form={form}
updateForm={updateForm}
onModeChange={handleModeChange}
busy={busy}
error={error}
statusText={statusText}
selectRoot={selectRoot}
discoverSchema={discoverSchema}
estimate={estimate}
runExtraction={runExtraction}
selection={selection}
setSelection={setSelection}
schemaSearch={schemaSearch}
setSchemaSearch={setSchemaSearch}
flattened={flattened}
schemaResponse={schemaResponse}
estimateJson={estimateJson}
manifest={manifest}
ingestProgress={ingestProgress}
setDocType={setDocType}
toggleField={toggleField}
selectAllFields={selectAllFields}
canGoNext={canGoNext}
onBack={handleBack}
onNext={handleNext}
/>
</>
)}
{view === "counting" && (
<CountingWizard
client={client}
localRoot={form.localRoot}
containerRoot={form.containerRoot}
onStartExtraction={(countTaskId, validCount) => {
setPendingCountTaskId(countTaskId);
setPendingValidCount(validCount);
setView("extraction");
}}
/>
)}
@@ -849,4 +894,100 @@ function formatCell(value: unknown): string {
return String(value);
}
function FromCountBanner({
countTaskId,
validCount,
onDismiss,
client,
form,
selection,
onTaskCreated,
}: {
countTaskId: string;
validCount: number;
onDismiss: () => void;
client: ApiClient;
form: ExtractionFormState;
selection: FieldSelection;
onTaskCreated: (task: Task) => void;
}) {
const [minServiceCount, setMinServiceCount] = useState(1);
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
async function run() {
if (!selection.fields.length) {
setError("Descubre y selecciona campos en el paso 2 y 3 antes de extraer.");
return;
}
setBusy(true);
setError(null);
try {
const payload: ExtractionFromCountPayload = {
counting_task_id: countTaskId,
min_service_count: minServiceCount,
fields: selection.fields,
alias_map: selection.aliasMap,
doc_types: form.docTypes,
structure_mode: "general",
model: form.model || null,
};
const task = await client.createExtractionFromCount(payload);
onTaskCreated(task);
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setBusy(false);
}
}
return (
<div className="mb-4 rounded-lg border border-indigo-300 dark:border-indigo-700 bg-indigo-50 dark:bg-indigo-900/30 p-4 space-y-3">
<div className="flex items-start justify-between">
<div>
<p className="text-sm font-semibold text-indigo-800 dark:text-indigo-200">
Extraccion desde conteo — {validCount} archivo(s) valido(s)
</p>
<p className="text-xs text-indigo-600 dark:text-indigo-400 mt-0.5">
Configura los campos (pasos 2 y 3) y luego haz clic en Extraer.
</p>
</div>
<button onClick={onDismiss} className="text-indigo-400 hover:text-indigo-600 text-xs">
Descartar
</button>
</div>
<div className="flex items-center gap-4">
<div className="flex items-center gap-2">
<label className="text-xs text-indigo-700 dark:text-indigo-300 font-medium whitespace-nowrap">
Min. servicios
</label>
<input
type="number"
min={0}
value={minServiceCount}
onChange={(e) => setMinServiceCount(Number(e.target.value))}
className="w-16 px-2 py-1 text-xs rounded border border-indigo-300 dark:border-indigo-600 bg-white dark:bg-slate-800"
/>
</div>
<button
onClick={run}
disabled={busy || !selection.fields.length}
className="flex items-center gap-1.5 px-4 py-1.5 rounded-lg bg-indigo-600 hover:bg-indigo-700 disabled:opacity-50 text-white text-sm font-medium transition-colors"
>
{busy ? <Loader2 className="w-4 h-4 animate-spin" /> : <Hash className="w-4 h-4" />}
{busy ? "Iniciando..." : "Extraer desde conteo"}
</button>
</div>
{error && (
<p className="text-xs text-red-600 dark:text-red-400">
<AlertCircle className="inline w-3 h-3 mr-1" />
{error}
</p>
)}
</div>
);
}
export default App;
@@ -0,0 +1,201 @@
import { AlertCircle, CheckCircle2, Filter, XCircle } from "lucide-react";
import { useMemo, useState } from "react";
import type { CountJobFile } from "../../shared/types";
interface Props {
files: CountJobFile[];
minServiceCount: number;
onMinServiceCountChange: (v: number) => void;
onToggleValid: (file: CountJobFile, isValid: boolean | null) => void;
loading?: boolean;
}
export function CountResultsTable({
files,
minServiceCount,
onMinServiceCountChange,
onToggleValid,
loading,
}: Props) {
const [filter, setFilter] = useState<"all" | "valid" | "invalid" | "unreviewed">("all");
const maxCount = useMemo(() => Math.max(0, ...files.map((f) => f.service_count ?? 0)), [files]);
const filtered = useMemo(() => {
return files.filter((f) => {
if (filter === "valid") return f.is_valid === true || (f.is_valid === null && (f.service_count ?? 0) >= minServiceCount);
if (filter === "invalid") return f.is_valid === false || (f.is_valid === null && (f.service_count ?? 0) < minServiceCount);
if (filter === "unreviewed") return f.is_valid === null;
return true;
});
}, [files, filter, minServiceCount]);
const summary = useMemo(() => {
const valid = files.filter(
(f) => f.is_valid === true || (f.is_valid === null && (f.service_count ?? 0) >= minServiceCount && f.status === "ok"),
).length;
const errors = files.filter((f) => f.status === "error").length;
return { valid, errors, total: files.length };
}, [files, minServiceCount]);
function validityClass(f: CountJobFile): string {
if (f.status === "error") return "bg-red-50 dark:bg-red-900/20";
if (f.is_valid === true) return "bg-green-50 dark:bg-green-900/20";
if (f.is_valid === false) return "bg-red-50 dark:bg-red-900/20";
if ((f.service_count ?? 0) >= minServiceCount) return "bg-green-50 dark:bg-green-900/20";
return "bg-yellow-50 dark:bg-yellow-900/20";
}
function validityIcon(f: CountJobFile) {
if (f.status === "error") return <XCircle className="w-4 h-4 text-red-500" />;
if (f.is_valid === true) return <CheckCircle2 className="w-4 h-4 text-green-600" />;
if (f.is_valid === false) return <XCircle className="w-4 h-4 text-red-500" />;
if ((f.service_count ?? 0) >= minServiceCount) return <CheckCircle2 className="w-4 h-4 text-green-600" />;
return <AlertCircle className="w-4 h-4 text-yellow-500" />;
}
return (
<div className="space-y-4">
{/* Summary */}
<div className="grid grid-cols-3 gap-3">
<div className="rounded-lg border border-slate-200 dark:border-slate-700 bg-white dark:bg-slate-800 p-3 text-center">
<p className="text-2xl font-bold text-slate-900 dark:text-white">{summary.total}</p>
<p className="text-xs text-slate-500">Total archivos</p>
</div>
<div className="rounded-lg border border-green-200 dark:border-green-800 bg-green-50 dark:bg-green-900/20 p-3 text-center">
<p className="text-2xl font-bold text-green-700 dark:text-green-400">{summary.valid}</p>
<p className="text-xs text-green-600 dark:text-green-500">Válidos para extracción</p>
</div>
<div className="rounded-lg border border-red-200 dark:border-red-800 bg-red-50 dark:bg-red-900/20 p-3 text-center">
<p className="text-2xl font-bold text-red-700 dark:text-red-400">{summary.errors}</p>
<p className="text-xs text-red-600 dark:text-red-500">Errores de procesamiento</p>
</div>
</div>
{/* Threshold slider */}
<div className="rounded-lg border border-slate-200 dark:border-slate-700 bg-white dark:bg-slate-800 p-4 space-y-2">
<div className="flex items-center justify-between">
<label className="text-sm font-medium text-slate-700 dark:text-slate-300 flex items-center gap-2">
<Filter className="w-4 h-4" />
Umbral mínimo de servicios
</label>
<span className="text-sm font-bold text-indigo-600 dark:text-indigo-400 tabular-nums">
≥ {minServiceCount}
</span>
</div>
<input
type="range"
min={0}
max={maxCount > 0 ? maxCount : 20}
value={minServiceCount}
onChange={(e) => onMinServiceCountChange(Number(e.target.value))}
className="w-full accent-indigo-600"
disabled={loading}
/>
<p className="text-xs text-slate-500">
Solo se extraerán facturas con {minServiceCount === 0 ? "cualquier número" : `≥ ${minServiceCount}`} de servicio(s).
{minServiceCount > 0 && ` Se excluyen ${files.filter((f) => (f.service_count ?? 0) < minServiceCount && f.is_valid !== true).length} archivo(s).`}
</p>
</div>
{/* Filter tabs */}
<div className="flex gap-1 text-xs">
{(["all", "valid", "invalid", "unreviewed"] as const).map((f) => (
<button
key={f}
onClick={() => setFilter(f)}
className={`px-3 py-1 rounded-full border transition-colors ${
filter === f
? "bg-indigo-600 border-indigo-600 text-white"
: "border-slate-300 dark:border-slate-600 text-slate-600 dark:text-slate-400 hover:bg-slate-100 dark:hover:bg-slate-700"
}`}
>
{f === "all" ? `Todos (${files.length})` : f === "valid" ? "Válidos" : f === "invalid" ? "Rechazados" : "Sin revisar"}
</button>
))}
</div>
{/* Table */}
<div className="rounded-lg border border-slate-200 dark:border-slate-700 overflow-hidden">
<div className="max-h-96 overflow-y-auto">
<table className="w-full text-sm">
<thead className="sticky top-0 bg-slate-50 dark:bg-slate-800 border-b border-slate-200 dark:border-slate-700">
<tr>
<th className="px-3 py-2 text-left text-xs font-medium text-slate-500 uppercase tracking-wider">Archivo</th>
<th className="px-3 py-2 text-center text-xs font-medium text-slate-500 uppercase tracking-wider w-24">Servicios</th>
<th className="px-3 py-2 text-center text-xs font-medium text-slate-500 uppercase tracking-wider w-24">Estado</th>
<th className="px-3 py-2 text-center text-xs font-medium text-slate-500 uppercase tracking-wider w-28">Validez</th>
</tr>
</thead>
<tbody className="divide-y divide-slate-100 dark:divide-slate-700">
{filtered.length === 0 && (
<tr>
<td colSpan={4} className="px-3 py-6 text-center text-slate-400 text-sm">
No hay archivos que mostrar
</td>
</tr>
)}
{filtered.map((f) => (
<tr key={f.id} className={`${validityClass(f)} transition-colors`}>
<td className="px-3 py-2">
<div className="font-medium text-slate-900 dark:text-white truncate max-w-xs" title={f.filename}>
{f.filename}
</div>
{f.radicado && f.radicado !== f.filename && (
<div className="text-xs text-slate-500">{f.radicado}</div>
)}
{f.status === "error" && (
<div className="text-xs text-red-500 mt-0.5">{f.error}</div>
)}
</td>
<td className="px-3 py-2 text-center">
{f.status === "error" ? (
<span className="text-red-400 text-xs">—</span>
) : (
<span className="font-bold tabular-nums text-slate-900 dark:text-white">
{f.service_count ?? "?"}
</span>
)}
</td>
<td className="px-3 py-2">
<div className="flex justify-center">{validityIcon(f)}</div>
</td>
<td className="px-3 py-2">
{f.status !== "error" && (
<div className="flex gap-1 justify-center">
<button
onClick={() => onToggleValid(f, f.is_valid === true ? null : true)}
disabled={loading}
title="Marcar como válido"
className={`px-2 py-0.5 rounded text-xs font-medium transition-colors ${
f.is_valid === true
? "bg-green-600 text-white"
: "bg-slate-100 dark:bg-slate-700 text-slate-600 dark:text-slate-400 hover:bg-green-100 dark:hover:bg-green-900/30"
}`}
>
✓
</button>
<button
onClick={() => onToggleValid(f, f.is_valid === false ? null : false)}
disabled={loading}
title="Rechazar"
className={`px-2 py-0.5 rounded text-xs font-medium transition-colors ${
f.is_valid === false
? "bg-red-600 text-white"
: "bg-slate-100 dark:bg-slate-700 text-slate-600 dark:text-slate-400 hover:bg-red-100 dark:hover:bg-red-900/30"
}`}
>
✗
</button>
</div>
)}
</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
</div>
);
}
@@ -0,0 +1,44 @@
import { Check } from "lucide-react";
export type CountingStep = 1 | 2 | 3;
const STEPS: Array<{ id: CountingStep; label: string }> = [
{ id: 1, label: "Carpeta" },
{ id: 2, label: "Contar" },
{ id: 3, label: "Resultados" },
];
interface CountingStepperProps {
current: CountingStep;
maxReached: CountingStep;
onStepClick?: (step: CountingStep) => void;
}
export function CountingStepper({ current, maxReached, onStepClick }: CountingStepperProps) {
return (
<nav className="stepper" aria-label="Pasos de conteo">
{STEPS.map((step, index) => {
const isDone = step.id < current;
const isActive = step.id === current;
const isClickable = step.id <= maxReached && !isActive;
const state = isDone ? "done" : isActive ? "active" : step.id <= maxReached ? "available" : "todo";
return (
<div className="stepper-item" key={step.id}>
<button
type="button"
className={`stepper-dot ${state}`}
disabled={!isClickable}
onClick={() => isClickable && onStepClick?.(step.id)}
aria-current={isActive ? "step" : undefined}
>
{isDone ? <Check size={14} strokeWidth={3} /> : step.id}
</button>
<span className={`stepper-label ${state}`}>{step.label}</span>
{index < STEPS.length - 1 && <div className={`stepper-connector ${isDone ? "done" : ""}`} />}
</div>
);
})}
</nav>
);
}
@@ -0,0 +1,477 @@
import {
AlertCircle,
ArrowRight,
CheckCircle2,
FileText,
FolderOpen,
Hash,
Loader2,
Play,
RotateCcw,
} from "lucide-react";
import { useCallback, useEffect, useRef, useState } from "react";
import type {
CountJobFile,
CountSyncResult,
CountingFormState,
CreateCountJobPayload,
DocumentType,
ManifestEntry,
Task,
} from "../../shared/types";
import { ApiClient } from "../lib/apiClient";
import { CountResultsTable } from "./CountResultsTable";
import { CountingStepper, type CountingStep } from "./CountingStepper";
declare const desktopApi: import("../../shared/types").DesktopBridge;
interface Props {
client: ApiClient;
localRoot: string;
containerRoot: string;
onStartExtraction: (countingTaskId: string, validCount: number) => void;
}
interface CountProgress {
processed: number;
total: number;
failed: number;
}
const BATCH_SIZE = 10;
export function CountingWizard({ client, localRoot, containerRoot, onStartExtraction }: Props) {
const [step, setStep] = useState<CountingStep>(1);
const [maxReached, setMaxReached] = useState<CountingStep>(1);
const [form, setForm] = useState<CountingFormState>({
localRoot,
containerRoot,
includeRejected: false,
minServiceCount: 1,
});
const [docTypes] = useState<DocumentType[]>([
{ name: "Factura", keyword: "Factura_O_Documento_Equivalente" },
]);
const [manifest, setManifest] = useState<ManifestEntry[]>([]);
const [progress, setProgress] = useState<CountProgress>({ processed: 0, total: 0, failed: 0 });
const [savedTask, setSavedTask] = useState<Task | null>(null);
const [savedFiles, setSavedFiles] = useState<CountJobFile[]>([]);
const [error, setError] = useState<string | null>(null);
const [running, setRunning] = useState(false);
const abortRef = useRef(false);
const indexLocalFolder = useCallback(
async (dir: string) => {
const entries = await desktopApi.indexFolder(dir, "general", docTypes, []);
setManifest(entries);
if (entries.length === 0) {
setError("No se encontraron PDF en la carpeta seleccionada.");
} else {
setError(null);
}
return entries;
},
[docTypes],
);
useEffect(() => {
setForm((f) => ({
...f,
localRoot: localRoot || f.localRoot,
containerRoot: containerRoot || f.containerRoot,
}));
}, [localRoot, containerRoot]);
useEffect(() => {
if (!localRoot) return;
let cancelled = false;
indexLocalFolder(localRoot).catch((e) => {
if (!cancelled) setError(String(e));
});
return () => {
cancelled = true;
};
}, [localRoot, indexLocalFolder]);
function goToStep(target: CountingStep) {
setStep(target);
setMaxReached((prev) => (target > prev ? target : prev));
}
const selectFolder = useCallback(async () => {
const dir = await desktopApi.selectDirectory();
if (!dir) return;
setForm((f) => ({ ...f, localRoot: dir }));
try {
await indexLocalFolder(dir);
} catch (e) {
setError(String(e));
}
}, [indexLocalFolder]);
function localToContainer(localPath: string): string {
const root = form.localRoot.replace(/\\/g, "/");
const container = form.containerRoot.replace(/\\/g, "/");
const norm = localPath.replace(/\\/g, "/");
return norm.startsWith(root) ? container + norm.slice(root.length) : norm;
}
async function runCounting() {
if (!manifest.length) {
setError("No hay archivos en la carpeta seleccionada.");
return;
}
abortRef.current = false;
setRunning(true);
setError(null);
setSavedTask(null);
setSavedFiles([]);
setProgress({ processed: 0, total: manifest.length, failed: 0 });
goToStep(2);
const allResults: CountSyncResult[] = [];
for (let i = 0; i < manifest.length; i += BATCH_SIZE) {
if (abortRef.current) break;
const batch = manifest.slice(i, i + BATCH_SIZE);
try {
const formData = new FormData();
for (const entry of batch) {
const fileData = await desktopApi.readFile(entry.localPath);
const blob = new Blob([fileData.data], { type: "application/pdf" });
formData.append("files", blob, fileData.filename);
}
const response = await client.countFiles(formData, false);
allResults.push(...response.results);
setProgress((p) => ({
...p,
processed: Math.min(p.processed + batch.length, manifest.length),
failed: p.failed + response.failed,
}));
} catch (e) {
for (const entry of batch) {
allResults.push({
filename: entry.filename,
service_count: 0,
status: "error",
services: [],
error: String(e),
});
}
setProgress((p) => ({
...p,
processed: Math.min(p.processed + batch.length, manifest.length),
failed: p.failed + batch.length,
}));
}
}
try {
const payload: CreateCountJobPayload = {
title: `Conteo — ${form.localRoot.split(/[\\/]/).pop()} (${allResults.length} archivos)`,
results: allResults.map((r) => {
const entry = manifest.find((m) => m.filename === r.filename);
return {
radicado: entry?.radicado ?? null,
filename: r.filename,
file_path: entry ? localToContainer(entry.localPath) : null,
doc_type: entry?.docType ?? null,
service_count: r.status === "ok" ? r.service_count : null,
services: r.services ?? [],
status: r.status === "ok" ? "ok" : "error",
error: r.error ?? null,
};
}),
};
const task = await client.createCountJob(payload);
const files = await client.countJobFiles(task.id);
setSavedTask(task);
setSavedFiles(files);
} catch (e) {
setError(`Resultados procesados pero no se pudieron guardar en BD: ${e}`);
}
setRunning(false);
goToStep(3);
}
async function handleToggleValid(file: CountJobFile, isValid: boolean | null) {
if (!savedTask) return;
try {
const updated = await client.markCountFileValid(savedTask.id, file.id, isValid);
setSavedFiles((prev) => prev.map((f) => (f.id === updated.id ? updated : f)));
} catch (e) {
setError(String(e));
}
}
function handleStartExtraction() {
if (!savedTask) return;
const validCount = savedFiles.filter(
(f) =>
f.status === "ok" &&
f.is_valid !== false &&
(f.service_count ?? 0) >= form.minServiceCount,
).length;
onStartExtraction(savedTask.id, validCount);
}
function resetWizard() {
setStep(1);
setMaxReached(1);
setManifest([]);
setSavedTask(null);
setSavedFiles([]);
setError(null);
setProgress({ processed: 0, total: 0, failed: 0 });
}
const canExtract =
savedTask !== null &&
savedFiles.some(
(f) =>
f.status === "ok" &&
f.is_valid !== false &&
(f.service_count ?? 0) >= form.minServiceCount,
);
const progressPercent =
progress.total > 0 ? Math.round((progress.processed / progress.total) * 100) : 0;
return (
<div className="wiz">
<CountingStepper current={step} maxReached={maxReached} onStepClick={goToStep} />
<div className="wiz-body">
{/* ── PASO 1: Carpeta ── */}
{step === 1 && (
<div className="wiz-step">
<div className="wiz-section">
<h3 className="wiz-section-title">Origen de las facturas</h3>
<p className="wiz-desc">
Selecciona la carpeta local que contiene los PDF de facturas medicas a analizar.
El servicio contara cuantos servicios vienen en cada documento usando OCR y ML.
</p>
</div>
<div className="wiz-section">
<label className="wiz-field-label">Carpeta a procesar</label>
<div className="folder-picker">
<div className="folder-picker-display">
{form.localRoot ? (
<>
<FolderOpen size={16} className="folder-picker-icon" />
<span className="folder-picker-path">{form.localRoot}</span>
</>
) : (
<span className="folder-picker-placeholder">Ninguna carpeta seleccionada</span>
)}
</div>
<button type="button" className="btn btn-primary" onClick={selectFolder}>
<FolderOpen size={15} /> Explorar
</button>
</div>
{manifest.length > 0 && (
<div className="wiz-found-badge">
<FileText size={15} />
<span>
<strong>{manifest.length}</strong> documento(s) encontrados y listos para contar
</span>
</div>
)}
{form.localRoot && manifest.length === 0 && !error && (
<p className="wiz-hint">Buscando PDF en la carpeta…</p>
)}
</div>
<div className="wiz-section">
<label className="wiz-field-label">Ruta en contenedor</label>
<input
type="text"
value={form.containerRoot}
onChange={(e) => setForm((f) => ({ ...f, containerRoot: e.target.value }))}
placeholder="/data"
/>
<p className="wiz-hint">
Prefijo de ruta dentro del contenedor Docker donde estan montados los archivos.
</p>
</div>
</div>
)}
{/* ── PASO 2: Contando ── */}
{step === 2 && (
<div className="wiz-step wiz-step-centered">
<div className="wiz-hero">
<div className="wiz-hero-icon">
{running ? <Loader2 size={28} className="spin" /> : <CheckCircle2 size={28} />}
</div>
<h3 className="wiz-hero-title">
{running ? "Analizando facturas…" : "Analisis completado"}
</h3>
<p className="wiz-desc">
{running
? "Enviando documentos al servicio de conteo. Se analiza cada PDF con OCR y ML para detectar servicios medicos."
: "Todos los documentos fueron procesados. Revisa los resultados en el siguiente paso."}
</p>
</div>
<div className="wiz-progress">
<div className="wiz-progress-header">
<span>
{running ? "Procesando documentos" : "Procesados"}
</span>
<span>
{progress.processed} / {progress.total}
{progress.failed > 0 && (
<span style={{ color: "var(--color-error)", marginLeft: 8 }}>
{progress.failed} error(es)
</span>
)}
</span>
</div>
<div className="wiz-progress-track">
<div className="wiz-progress-fill" style={{ width: `${progressPercent}%` }} />
</div>
</div>
<div className="wiz-stats-row">
<div className="wiz-stat">
<span className="wiz-stat-value">{progress.total}</span>
<span className="wiz-stat-label">Total</span>
</div>
<div className="wiz-stat">
<span className="wiz-stat-value">{progress.processed - progress.failed}</span>
<span className="wiz-stat-label">Exitosos</span>
</div>
<div className="wiz-stat">
<span className="wiz-stat-value">{progress.failed}</span>
<span className="wiz-stat-label">Errores</span>
</div>
</div>
</div>
)}
{/* ── PASO 3: Resultados ── */}
{step === 3 && (
<div className="wiz-step">
<div className="wiz-section">
<div className="wiz-section-header">
<h3 className="wiz-section-title">
Resultados del conteo
{savedFiles.length > 0 && (
<span className="wiz-count-badge">{savedFiles.length} archivos</span>
)}
</h3>
<button
type="button"
className="btn btn-sm btn-ghost"
onClick={resetWizard}
>
<RotateCcw size={13} /> Nuevo conteo
</button>
</div>
{savedTask && (
<CountResultsTable
files={savedFiles}
minServiceCount={form.minServiceCount}
onMinServiceCountChange={(v) => setForm((f) => ({ ...f, minServiceCount: v }))}
onToggleValid={handleToggleValid}
loading={running}
/>
)}
</div>
<div className="wiz-action-group">
<button
type="button"
className="btn btn-primary btn-lg"
onClick={handleStartExtraction}
disabled={!canExtract}
>
<ArrowRight size={18} />
Extraer archivos validos
</button>
{canExtract && (
<p className="wiz-hint" style={{ marginTop: 6 }}>
Se procesaran los archivos con &ge; {form.minServiceCount} servicio(s)
</p>
)}
{!canExtract && savedFiles.length > 0 && (
<p className="wiz-hint" style={{ marginTop: 6 }}>
Ajusta el umbral de servicios minimos para habilitar la extraccion.
</p>
)}
</div>
</div>
)}
</div>
{/* ── Status / Errores ── */}
{error && (
<div className="wiz-toast wiz-toast--error">
<AlertCircle size={15} />
<span>{error}</span>
</div>
)}
{/* ── Footer de navegacion ── */}
<footer className="wiz-nav">
<button
type="button"
className="btn"
onClick={() => goToStep((step - 1) as CountingStep)}
disabled={running || step === 1}
>
Atras
</button>
<div className="wiz-nav-dots">
{([1, 2, 3] as CountingStep[]).map((id) => (
<button
key={id}
type="button"
className={`wiz-nav-dot ${step === id ? "active" : ""}`}
onClick={() => id <= maxReached && !running && goToStep(id)}
disabled={id > maxReached || running}
aria-label={`Ir al paso ${id}`}
/>
))}
</div>
{step === 1 && (
<button
type="button"
className="btn btn-primary"
onClick={runCounting}
disabled={running || !form.localRoot || manifest.length === 0}
>
<Play size={15} /> Iniciar conteo
</button>
)}
{step === 2 && (
<button
type="button"
className="btn"
onClick={() => { abortRef.current = true; }}
disabled={!running}
>
Cancelar
</button>
)}
{step === 3 && (
<button
type="button"
className="btn btn-primary"
onClick={handleStartExtraction}
disabled={!canExtract}
>
<Hash size={15} /> Extraer validos
</button>
)}
</footer>
</div>
);
}
@@ -1,4 +1,14 @@
import type { ExtractionBatch, RowsResponse, SchemaResponse, Task } from "../../shared/types";
import type {
CountJobFile,
CountSyncResponse,
CreateCountJobPayload,
ExtractionBatch,
ExtractionFromCountPayload,
Job,
RowsResponse,
SchemaResponse,
Task,
} from "../../shared/types";
export class ApiClient {
constructor(private readonly baseUrl: string) {}
@@ -89,6 +99,59 @@ export class ApiClient {
return this.getJson<RowsResponse>(`/extraction/batches/${encodeURIComponent(batchId)}/rows?limit=${limit}&offset=${offset}`);
}
// ── Jobs (contenedores multimodulo) ────────────────────────────────────────
async jobs(): Promise<Job[]> {
return this.getJson<Job[]>("/jobs");
}
async job(jobId: string): Promise<Job> {
return this.getJson<Job>(`/jobs/${encodeURIComponent(jobId)}`);
}
async jobTasks(jobId: string): Promise<Task[]> {
return this.getJson<Task[]>(`/jobs/${encodeURIComponent(jobId)}/tasks`);
}
// ── Count Jobs ─────────────────────────────────────────────────────────────
/** Envía PDFs al servicio de conteo (síncrono). */
async countFiles(form: FormData, includeRejected = false): Promise<CountSyncResponse> {
const url = includeRejected ? "/count/v1/count?include_rejected=true" : "/count/v1/count";
return this.postForm<CountSyncResponse>(url, form);
}
/** Persiste los resultados del conteo en la BD compartida. */
async createCountJob(payload: CreateCountJobPayload): Promise<Task> {
return this.postJson<Task>("/count-jobs", payload);
}
async countJobs(): Promise<Task[]> {
return this.getJson<Task[]>("/count-jobs");
}
async countJob(taskId: string): Promise<Task> {
return this.getJson<Task>(`/count-jobs/${encodeURIComponent(taskId)}`);
}
async countJobFiles(taskId: string): Promise<CountJobFile[]> {
return this.getJson<CountJobFile[]>(`/count-jobs/${encodeURIComponent(taskId)}/files`);
}
async markCountFileValid(taskId: string, fileId: string, isValid: boolean | null): Promise<CountJobFile> {
const response = await fetch(this.url(`/count-jobs/${encodeURIComponent(taskId)}/files/${encodeURIComponent(fileId)}`), {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ is_valid: isValid }),
});
return this.handle<CountJobFile>(response);
}
/** Crea una tarea de extracción usando los archivos válidos de un job de conteo. */
async createExtractionFromCount(payload: ExtractionFromCountPayload): Promise<Task> {
return this.postJson<Task>("/tasks/extraction/from-count", payload);
}
private async handle<T>(response: Response): Promise<T> {
const text = await response.text();
let data: unknown = null;
+73
View File
@@ -83,6 +83,79 @@ export interface RowsResponse {
offset: number;
}
export interface Job {
id: string;
status: "pending" | "running" | "completed" | "failed" | string;
title: string | null;
metadata: Record<string, unknown>;
created_at: string;
updated_at: string;
}
export interface CountJobFile {
id: string;
job_module_id: string;
radicado: string | null;
filename: string;
file_path: string | null;
doc_type: string | null;
service_count: number | null;
services: Array<{ text: string; probability: number; is_service?: boolean | null }>;
status: "ok" | "error" | string;
is_valid: boolean | null;
error: string | null;
created_at: string;
}
export interface CountJobFileCreateEntry {
radicado?: string | null;
filename: string;
file_path?: string | null;
doc_type?: string | null;
service_count?: number | null;
services?: Array<{ text: string; probability: number; is_service?: boolean | null }>;
status?: "ok" | "error";
error?: string | null;
}
export interface CreateCountJobPayload {
title?: string;
results: CountJobFileCreateEntry[];
}
export interface ExtractionFromCountPayload {
counting_task_id: string;
min_service_count: number;
fields: string[];
alias_map: Record<string, string>;
doc_types?: Array<{ name: string; keyword: string }>;
structure_mode?: "general" | "radicados";
instruction?: string | null;
tiff_page_range?: [number, number] | null;
model?: string | null;
}
export interface CountSyncResult {
filename: string;
service_count: number;
status: "ok" | "error";
services: Array<{ text: string; probability: number; is_service?: boolean | null }>;
error?: string;
}
export interface CountSyncResponse {
results: CountSyncResult[];
processed: number;
failed: number;
}
export interface CountingFormState {
localRoot: string;
containerRoot: string;
includeRejected: boolean;
minServiceCount: number;
}
export interface RuntimeConfig {
documentRootLocal: string;
documentRootContainer: string;
+133 -39
View File
@@ -1,45 +1,139 @@
# IPSExtractor Extraction Service
# extraction
Microservicio independiente de extraccion. Esta carpeta puede vivir como repo
propio: contiene su API REST, dominio, infraestructura, dependencias y
Dockerfile sin importar codigo de otros servicios.
Servicio Python (FastAPI) de extracción de datos estructurados desde documentos médicos PDF.
Usa LLMs (compatible con OpenAI API) para descubrir campos, estimar costos y extraer datos por lotes.
## Estructura
También actúa como capa de persistencia para el flujo de conteo: guarda los resultados del `count-api` en PostgreSQL y orquesta la extracción de documentos validados.
```text
api/ Endpoints FastAPI
domain/ Logica de extraccion, prompts, estimacion y normalizacion
core/ Utilidades propias de PDF, texto y CSV
infrastructure/ Persistencia SQLAlchemy del servicio
requirements.txt
Dockerfile
```
## Desarrollo local
```bash
pip install -r requirements.txt
uvicorn api.main:app --host 0.0.0.0 --port 8001
```
## Poppler
El contenedor instala `poppler-utils` para que `pdf2image` encuentre
`pdfinfo` y `pdftoppm` desde el PATH del sistema, sin configurar variables de
entorno.
En desarrollo local Windows, el servicio usa automaticamente la carpeta incluida
en `dependencies/poppler-24.08.0/Library/bin`.
---
## Endpoints principales
- `GET /health`
- `GET /config`
- `GET /models`
- `POST /schemas`
- `POST /schemas/upload`
- `POST /estimate`
- `POST /batches`
- `POST /tasks/extraction`
- `POST /tasks/{task_id}/ingest`
- `GET /tasks`
Puerto: **8001**
### Salud y configuración
| Método | Ruta | Descripción |
|--------|------|-------------|
| `GET` | `/health` | Liveness |
| `GET` | `/config` | Configuración LLM activa |
| `GET` | `/models` | Modelos disponibles en el backend LLM |
### Descubrimiento de esquema
| Método | Ruta | Descripción |
|--------|------|-------------|
| `POST` | `/schemas` | Descubrir campos desde ruta en disco |
| `POST` | `/schemas/upload` | Descubrir campos desde PDF subido |
### Estimación de costo
| Método | Ruta | Descripción |
|--------|------|-------------|
| `POST` | `/estimate/upload` | Estimar tokens/costo desde PDF subido |
### Tareas de extracción
| Método | Ruta | Descripción |
|--------|------|-------------|
| `POST` | `/tasks/extraction` | Crear tarea de extracción |
| `POST` | `/tasks/{id}/ingest` | Subir un PDF a la tarea |
| `POST` | `/tasks/{id}/ingest-many` | Subir varios PDFs en bulk |
| `GET` | `/tasks` | Listar tareas |
| `GET` | `/tasks/{id}` | Detalle de tarea |
| `GET` | `/tasks/{id}/rows` | Filas extraídas de la tarea |
| `POST` | `/tasks/extraction/from-count` | Crear extracción desde conteo previo |
### Jobs (contenedor de tareas)
| Método | Ruta | Descripción |
|--------|------|-------------|
| `GET` | `/jobs` | Listar jobs |
| `GET` | `/jobs/{id}` | Detalle de job |
| `GET` | `/jobs/{id}/tasks` | Tareas del job |
### Conteo (persistencia desde count-api)
| Método | Ruta | Descripción |
|--------|------|-------------|
| `POST` | `/count-jobs` | Guardar resultados de conteo en BD |
| `GET` | `/count-jobs` | Listar jobs de conteo |
| `GET` | `/count-jobs/{task_id}` | Detalle de job de conteo |
| `GET` | `/count-jobs/{task_id}/files` | Archivos del conteo |
| `PATCH` | `/count-jobs/{task_id}/files/{file_id}` | Marcar archivo como válido/inválido |
---
## Flujo de datos
```
PDF local (desktop) → POST /tasks/{id}/ingest-many → procesamiento LLM
↓
filas en extraction_rows (PG)
count-api results → POST /count-jobs → count_job_files (PG)
→ POST /tasks/extraction/from-count → lee file_path desde /data
```
---
## Variables de entorno
| Variable | Descripción |
|----------|-------------|
| `DATABASE_URL` | PostgreSQL (`postgresql+psycopg://...`) |
| `REDIS_URL` | Redis para cola ARQ (`redis://...`) |
| `EXTRACTOR_BASE_URL` | URL base del LLM (compatible OpenAI) |
| `EXTRACTOR_API_KEY` | API key del LLM extractor |
| `EXTRACTOR_MODEL` | Modelo de extracción (ej. `gpt-5.4-mini`) |
| `NORMALIZER_BASE_URL` | URL base del LLM normalizador |
| `NORMALIZER_API_KEY` | API key del LLM normalizador |
| `NORMALIZER_MODEL` | Modelo normalizador (ej. `gpt-5.4-mini`) |
| `EXTRACTOR_TIMEOUT_S` | Timeout por documento (default `120`) |
| `PDF_DPI` | DPI para renderizado PDF (default `150`) |
| `PDF_THREAD_COUNT` | Hilos de renderizado PDF (default `4`) |
| `CONCURRENCY_LIMIT` | Máximo de extracciones paralelas (default `50`) |
| `ALLOWED_DOCUMENT_TYPES` | Tipos de documento permitidos |
| `ASYNC_QUEUE` | `true` para encolar en ARQ/Redis (default: sync) |
---
## Modelo de datos (PostgreSQL)
```
jobs — contenedor de tareas del usuario
tasks — una tarea por módulo (counting / extraction / cleaning)
count_job_files — resultados de conteo por PDF
extraction_batches — lotes de extracción
extraction_rows — filas extraídas por documento
```
---
## Estructura del código
```
api/
main.py — FastAPI app (~1400 líneas), todos los endpoints
desktop_ingest.py — lógica de ingesta desde desktop Electron
domain/ — lógica de negocio: LLM, prompts, extractor, batch
core/ — utilidades PDF, CSV, texto
infrastructure/ — SQLAlchemy, migraciones, uploads temporales
worker/ — workers ARQ (procesamiento async)
main.py — punto de entrada del worker
tasks.py — tareas ARQ
settings.py — configuración del worker
Dockerfile
requirements.txt
```
---
## Arrancar localmente
```bash
# Desde la raíz del repositorio:
docker compose up --build extraction worker
```
API disponible en `http://localhost:8001`.
+419
View File
@@ -28,8 +28,10 @@ from domain.estimator import Estimator
from domain.service import InvoiceProcessingService
from domain.strategies import OpenAICompatibleExtractor
from infrastructure.database import (
CountJobFile,
ExtractionBatch,
ExtractionRow,
Job,
Task,
init_db,
session_scope,
@@ -938,3 +940,420 @@ def get_rows(batch_id: str, limit: int = 500, offset: int = 0) -> dict[str, Any]
"limit": limit,
"offset": offset,
}
# ═══════════════════════════════════════════════════════════════════════════════
# Jobs (contenedores multimodulo)
# ═══════════════════════════════════════════════════════════════════════════════
class JobRead(BaseModel):
id: str
status: str
title: str | None
metadata: dict[str, Any]
created_at: str
updated_at: str
def serialize_job(job: Job) -> dict[str, Any]:
return {
"id": job.id,
"status": job.status,
"title": job.title,
"metadata": job.job_metadata or {},
"created_at": job.created_at.isoformat(),
"updated_at": job.updated_at.isoformat(),
}
@app.get("/jobs", response_model=list[JobRead])
def list_jobs() -> list[dict[str, Any]]:
with session_scope() as session:
jobs = session.scalars(select(Job).order_by(Job.created_at.desc())).all()
return [serialize_job(j) for j in jobs]
@app.get("/jobs/{job_id}", response_model=JobRead)
def get_job(job_id: str) -> dict[str, Any]:
with session_scope() as session:
job = session.get(Job, job_id)
if job is None:
raise HTTPException(status_code=404, detail="Job no encontrado")
return serialize_job(job)
@app.get("/jobs/{job_id}/tasks", response_model=list[TaskRead])
def get_job_tasks(job_id: str) -> list[dict[str, Any]]:
with session_scope() as session:
job = session.get(Job, job_id)
if job is None:
raise HTTPException(status_code=404, detail="Job no encontrado")
tasks = session.scalars(
select(Task).where(Task.job_id == job_id).order_by(Task.created_at)
).all()
return [serialize_task(t) for t in tasks]
# ═══════════════════════════════════════════════════════════════════════════════
# Count Jobs — persistir y gestionar resultados del servicio de conteo
# ═══════════════════════════════════════════════════════════════════════════════
class CountJobFileCreate(BaseModel):
radicado: str | None = None
filename: str
file_path: str | None = None
doc_type: str | None = None
service_count: int | None = None
services: list[dict[str, Any]] = Field(default_factory=list)
status: str = "ok"
error: str | None = None
class CreateCountJobRequest(BaseModel):
title: str | None = None
results: list[CountJobFileCreate]
class CountJobFileRead(BaseModel):
id: str
job_module_id: str
radicado: str | None
filename: str
file_path: str | None
doc_type: str | None
service_count: int | None
services: list[dict[str, Any]]
status: str
is_valid: bool | None
error: str | None
created_at: str
class UpdateCountJobFileRequest(BaseModel):
is_valid: bool | None
class ExtractionFromCountRequest(BaseModel):
counting_task_id: str
min_service_count: int = 0
fields: list[str] = Field(default_factory=list)
alias_map: dict[str, str] = Field(default_factory=dict)
doc_types: list[DocumentType] | None = None
structure_mode: str = "general"
instruction: str | None = None
tiff_page_range: tuple[int, int] | None = None
retry_on_empty: bool = False
model: str | None = None
def serialize_count_file(f: CountJobFile) -> dict[str, Any]:
return {
"id": f.id,
"job_module_id": f.job_module_id,
"radicado": f.radicado,
"filename": f.filename,
"file_path": f.file_path,
"doc_type": f.doc_type,
"service_count": f.service_count,
"services": f.services or [],
"status": f.status,
"is_valid": f.is_valid,
"error": f.error,
"created_at": f.created_at.isoformat(),
}
@app.post("/count-jobs", response_model=TaskRead)
def create_count_job(request: CreateCountJobRequest) -> dict[str, Any]:
"""Persiste los resultados del servicio de conteo de facturas en la BD compartida."""
total = len(request.results)
with session_scope() as session:
job = Job(
status="completed",
title=request.title or f"Conteo ({total} archivos)",
updated_at=utcnow(),
)
session.add(job)
session.flush()
task = Task(
job_id=job.id,
type="counting",
status="completed",
title=job.title,
request={},
result={
"total": total,
"processed": total,
"ok": sum(1 for r in request.results if r.status == "ok"),
"failed": sum(1 for r in request.results if r.status != "ok"),
},
started_at=utcnow(),
finished_at=utcnow(),
updated_at=utcnow(),
)
session.add(task)
session.flush()
for r in request.results:
session.add(
CountJobFile(
job_module_id=task.id,
radicado=r.radicado,
filename=r.filename,
file_path=r.file_path,
doc_type=r.doc_type,
service_count=r.service_count,
services=r.services,
status=r.status,
error=r.error,
)
)
session.flush()
return serialize_task(task)
@app.get("/count-jobs", response_model=list[TaskRead])
def list_count_jobs() -> list[dict[str, Any]]:
with session_scope() as session:
tasks = session.scalars(
select(Task)
.where(Task.type == "counting")
.order_by(Task.created_at.desc())
).all()
return [serialize_task(t) for t in tasks]
@app.get("/count-jobs/{task_id}", response_model=TaskRead)
def get_count_job(task_id: str) -> dict[str, Any]:
with session_scope() as session:
task = session.get(Task, task_id)
if task is None or task.type != "counting":
raise HTTPException(status_code=404, detail="Tarea de conteo no encontrada")
return serialize_task(task)
@app.get("/count-jobs/{task_id}/files", response_model=list[CountJobFileRead])
def list_count_job_files(task_id: str) -> list[dict[str, Any]]:
with session_scope() as session:
task = session.get(Task, task_id)
if task is None or task.type != "counting":
raise HTTPException(status_code=404, detail="Tarea de conteo no encontrada")
files = session.scalars(
select(CountJobFile)
.where(CountJobFile.job_module_id == task_id)
.order_by(CountJobFile.created_at)
).all()
return [serialize_count_file(f) for f in files]
@app.patch("/count-jobs/{task_id}/files/{file_id}", response_model=CountJobFileRead)
def update_count_job_file(
task_id: str, file_id: str, request: UpdateCountJobFileRequest
) -> dict[str, Any]:
with session_scope() as session:
f = session.get(CountJobFile, file_id)
if f is None or f.job_module_id != task_id:
raise HTTPException(status_code=404, detail="Archivo no encontrado")
f.is_valid = request.is_valid
session.flush()
return serialize_count_file(f)
# ═══════════════════════════════════════════════════════════════════════════════
# Extraccion desde resultados de conteo (sin re-subir archivos)
# ═══════════════════════════════════════════════════════════════════════════════
def run_from_count_task(task_id: str) -> None:
"""Procesa archivos validos de un job de conteo en background (lee desde el volumen /data)."""
import logging as _logging
_log = _logging.getLogger("extraction.from_count")
from api.desktop_ingest import _service_for_model, mark_task_completed_if_done
with session_scope() as session:
task = session.get(Task, task_id)
if task is None:
return
files_data: list[dict[str, Any]] = list((task.result or {}).get("files", []))
req = dict(task.request or {})
batch_id = str((task.result or {}).get("extraction_batch_id") or "")
model_name: str | None = req.get("model")
instruction: str | None = req.get("instruction")
tiff_page_range = req.get("tiff_page_range")
for file_data in files_data:
file_path: str | None = file_data.get("file_path")
filename = str(file_data.get("filename") or "documento.pdf")
radicado = str(file_data.get("radicado") or file_data.get("filename") or "")
doc_type: str | None = file_data.get("doc_type")
if not file_path or not __import__("os").path.exists(file_path):
_log.warning("Archivo no disponible: %s", file_path or filename)
with session_scope() as session:
task_obj = session.get(Task, task_id)
if task_obj is None:
return
result = dict(task_obj.result or {})
errors = list(result.get("errors") or [])
errors.append(f"Archivo no encontrado: {filename}")
result["errors"] = errors
result["processed"] = int(result.get("processed") or 0) + 1
task_obj.result = result
task_obj.updated_at = utcnow()
mark_task_completed_if_done(session, task_obj, batch_id)
continue
try:
svc = _service_for_model(processing_service, model_name)
with session_scope() as session:
task_obj = session.get(Task, task_id, with_for_update=True)
if task_obj is None:
return
from api.desktop_ingest import process_desktop_ingest
process_desktop_ingest(
svc, session,
task=task_obj,
batch_id=batch_id,
temp_path=file_path,
radicado=radicado,
filename=filename,
doc_type=doc_type,
instruction=instruction,
tiff_page_range=tiff_page_range,
model=model_name,
)
mark_task_completed_if_done(session, task_obj, batch_id)
except Exception as exc:
_log.exception("Error procesando %s: %s", filename, exc)
with session_scope() as session:
task_obj = session.get(Task, task_id)
if task_obj is None:
return
result = dict(task_obj.result or {})
errors = list(result.get("errors") or [])
errors.append(f"{filename}: {exc}")
result["errors"] = errors
result["processed"] = int(result.get("processed") or 0) + 1
task_obj.result = result
task_obj.updated_at = utcnow()
mark_task_completed_if_done(session, task_obj, batch_id)
@app.post("/tasks/extraction/from-count", response_model=TaskRead)
def create_extraction_from_count(
request: ExtractionFromCountRequest, background_tasks: BackgroundTasks
) -> dict[str, Any]:
"""Crea una tarea de extraccion usando archivos validos de un job de conteo previo.
No requiere subir archivos: usa las rutas almacenadas en count_job_files.
"""
if not request.fields:
raise HTTPException(
status_code=400,
detail="Envia fields. Usa POST /extraction/schemas/upload para descubrir campos.",
)
with session_scope() as session:
counting_task = session.get(Task, request.counting_task_id)
if counting_task is None or counting_task.type != "counting":
raise HTTPException(status_code=404, detail="Tarea de conteo no encontrada")
all_files = session.scalars(
select(CountJobFile).where(
CountJobFile.job_module_id == request.counting_task_id
)
).all()
valid_files = [
f for f in all_files
if f.status == "ok"
and f.is_valid is not False
and (f.service_count is None or f.service_count >= request.min_service_count)
]
if not valid_files:
raise HTTPException(
status_code=400,
detail="No hay archivos validos para extraer con los criterios indicados.",
)
files_data = [
{
"id": f.id,
"radicado": f.radicado or f.filename,
"filename": f.filename,
"file_path": f.file_path,
"doc_type": f.doc_type,
"service_count": f.service_count,
}
for f in valid_files
]
total = len(files_data)
job = Job(
status="running",
title=f"Extraccion desde conteo ({total} archivos)",
updated_at=utcnow(),
)
session.add(job)
session.flush()
batch = ExtractionBatch(
status="running",
root_path="count://",
request={
"source": "from-count",
"counting_task_id": request.counting_task_id,
"fields": request.fields,
"alias_map": request.alias_map,
"min_service_count": request.min_service_count,
"structure_mode": request.structure_mode,
},
updated_at=utcnow(),
)
session.add(batch)
session.flush()
task = Task(
job_id=job.id,
type="extraction",
status="running",
title=job.title,
request={
"source": "from-count",
"counting_task_id": request.counting_task_id,
"fields": request.fields,
"alias_map": request.alias_map,
"doc_types": [dt.model_dump() for dt in request.doc_types or []],
"structure_mode": request.structure_mode,
"instruction": request.instruction,
"tiff_page_range": list(request.tiff_page_range) if request.tiff_page_range else None,
"min_service_count": request.min_service_count,
"model": request.model,
},
result={
"source": "from-count",
"extraction_batch_id": batch.id,
"processed": 0,
"total": total,
"row_count": 0,
"errors": [],
"files": files_data,
"partials": {},
},
started_at=utcnow(),
updated_at=utcnow(),
)
session.add(task)
session.flush()
batch.job_module_id = task.id
session.flush()
serialized = serialize_task(task)
task_id = task.id
background_tasks.add_task(run_from_count_task, task_id)
return serialized
+48 -9
View File
@@ -9,6 +9,7 @@ from typing import Any, Iterator
from sqlalchemy import (
JSON,
Boolean,
DateTime,
ForeignKey,
Integer,
@@ -42,6 +43,20 @@ def new_id() -> str:
return str(uuid.uuid4())
class Job(Base):
"""Contenedor de trabajo del usuario; puede abarcar múltiples módulos/servicios."""
__tablename__ = "jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
title: Mapped[str | None] = mapped_column(Text, nullable=True)
# "metadata" es reservado por DeclarativeBase; el atributo Python usa job_metadata
# pero la columna en la BD sigue llamándose "metadata".
job_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
class ExtractionBatch(Base):
__tablename__ = "extraction_batches"
@@ -50,14 +65,21 @@ class ExtractionBatch(Base):
root_path: Mapped[str | None] = mapped_column(Text, nullable=True)
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
job_module_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("tasks.id"), nullable=True, index=True
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
class Task(Base):
"""Módulo/resultado dentro de un Job; un job puede tener N tasks (una por servicio)."""
__tablename__ = "tasks"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
job_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("jobs.id"), nullable=True, index=True
)
type: Mapped[str] = mapped_column(String(64), index=True)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
title: Mapped[str | None] = mapped_column(Text, nullable=True)
@@ -71,6 +93,26 @@ class Task(Base):
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class CountJobFile(Base):
"""Resultado del servicio de conteo para un PDF individual dentro de una tarea de conteo."""
__tablename__ = "count_job_files"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
job_module_id: Mapped[str] = mapped_column(
String(36), ForeignKey("tasks.id"), index=True
)
radicado: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
filename: Mapped[str] = mapped_column(String(512))
file_path: Mapped[str | None] = mapped_column(Text, nullable=True)
doc_type: Mapped[str | None] = mapped_column(String(128), nullable=True)
service_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
services: Mapped[list[Any]] = mapped_column(JSON, default=list)
status: Mapped[str] = mapped_column(String(32), default="ok", index=True)
is_valid: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
class ExtractionRow(Base):
__tablename__ = "extraction_rows"
@@ -109,16 +151,13 @@ class CleanedRow(Base):
def init_db() -> None:
if DATABASE_URL.startswith("postgresql"):
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:
connection.execute(text("SELECT pg_advisory_lock(74512001)"))
try:
Base.metadata.create_all(bind=connection)
finally:
connection.execute(text("SELECT pg_advisory_unlock(74512001)"))
return
"""Verifica conectividad con la BD.
Base.metadata.create_all(bind=engine)
Las migraciones son responsabilidad exclusiva de services/db (init container).
Este servicio asume que el esquema ya está actualizado al arrancar.
"""
with engine.connect() as conn:
conn.execute(text("SELECT 1"))
@contextmanager
@@ -1,51 +1,59 @@
# syntax=docker/dockerfile:1
FROM golang:1.23-bookworm AS builder
# ── Etapa 1: Descargar ONNX Runtime ───────────────────────────────────────────
FROM debian:bookworm-slim AS onnx-download
ARG ONNX_VERSION=1.22.0
RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates \
&& rm -rf /var/lib/apt/lists/*
RUN wget -q https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-x64-${ONNX_VERSION}.tgz \
&& tar -xzf onnxruntime-linux-x64-${ONNX_VERSION}.tgz \
&& mkdir -p /onnx/lib \
&& cp onnxruntime-linux-x64-${ONNX_VERSION}/lib/libonnxruntime.so* /onnx/lib/ \
&& rm -rf onnxruntime-linux-x64-${ONNX_VERSION}*
# ── Etapa 2: Build ─────────────────────────────────────────────────────────────
FROM golang:1.25-bookworm AS builder
RUN apt-get update && apt-get install -y --no-install-recommends \
gcc g++ pkg-config \
libtesseract-dev libleptonica-dev \
tesseract-ocr tesseract-ocr-spa \
wget ca-certificates \
&& rm -rf /var/lib/apt/lists/*
# ONNX Runtime
ARG ONNX_VERSION=1.22.0
RUN wget -q https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-x64-${ONNX_VERSION}.tgz \
&& tar -xzf onnxruntime-linux-x64-${ONNX_VERSION}.tgz \
&& cp onnxruntime-linux-x64-${ONNX_VERSION}/lib/libonnxruntime.so* /usr/local/lib/ \
&& ldconfig
COPY --from=onnx-download /onnx/lib/ /usr/local/lib/
RUN ldconfig
ENV CGO_ENABLED=1
ENV ONNXRUNTIME_SHARED_LIBRARY=/usr/local/lib/libonnxruntime.so
# Limitar paralelismo CGO para evitar OOM en Docker Desktop
ENV GOMAXPROCS=2
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN go build -o /out/api ./cmd/api && go build -o /out/worker ./cmd/worker
# -p 2 limita los paquetes compilados en paralelo (menor uso de RAM)
RUN go build -p 2 -o /out/api ./cmd/api
# ── Etapa 3: Runtime ───────────────────────────────────────────────────────────
FROM debian:bookworm-slim AS runtime
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl wget \
ca-certificates curl \
tesseract-ocr tesseract-ocr-spa \
&& rm -rf /var/lib/apt/lists/*
ARG ONNX_VERSION=1.22.0
RUN wget -q https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-x64-${ONNX_VERSION}.tgz \
&& tar -xzf onnxruntime-linux-x64-${ONNX_VERSION}.tgz \
&& cp onnxruntime-linux-x64-${ONNX_VERSION}/lib/libonnxruntime.so* /usr/local/lib/ \
&& rm -rf onnxruntime-linux-x64-${ONNX_VERSION}* \
&& ldconfig
COPY --from=onnx-download /onnx/lib/ /usr/local/lib/
RUN ldconfig
ENV ONNXRUNTIME_SHARED_LIBRARY=/usr/local/lib/libonnxruntime.so
WORKDIR /app
COPY --from=builder /out/api /app/api
COPY --from=builder /out/worker /app/worker
COPY api/openapi.yaml /app/openapi.yaml
COPY --from=builder /out/api /app/api
COPY api/openapi.yaml /app/openapi.yaml
ENV OPENAPI_PATH=/app/openapi.yaml
ENV MODEL_PATH=/app/models/model.onnx
@@ -1,14 +0,0 @@
FROM golang:1.23-bookworm AS builder
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN go build -o /out/autoscaler ./cmd/autoscaler
FROM docker:27-cli
COPY --from=builder /out/autoscaler /usr/local/bin/autoscaler
WORKDIR /project
ENTRYPOINT ["/usr/local/bin/autoscaler"]
@@ -1,339 +1,175 @@
# Services Count — API de Facturas Médicas
# count-api
![Go](https://img.shields.io/badge/Go-1.23-blue)
Microservicio Go que cuenta servicios facturados en facturas médicas PDF.
Aplica OCR (Tesseract) + clustering espacial (DBSCAN) + clasificación ML (Random Forest vía ONNX Runtime).
Microservicio de inferencia en Go que cuenta servicios facturados en facturas médicas PDF. El entrenamiento del modelo vive en el repositorio hermano [services-count-facturas-medicas-ml](https://github.com/builker-col/services-count-facturas-medicas-ml); este repositorio despliega la API HTTP y los workers de procesamiento.
Es un servicio **sin estado**: recibe PDFs por HTTP, procesa en memoria y devuelve el resultado. No persiste datos ni depende de bases de datos, colas ni almacenamiento de objetos.
## Características
La persistencia de resultados la maneja el servicio `extraction` (Python) a través de los endpoints `/count-jobs`.
- Conteo **síncrono** (1–50 PDFs por petición) y **asíncrono** (hasta 50 000 PDFs por job).
- Pipeline de inferencia: OCR → clustering DBSCAN → Random Forest vía ONNX.
- Escalado horizontal de workers con Docker Compose.
- Especificación OpenAPI y descriptor de herramientas para agentes AI.
---
## Arquitectura
El sistema se compone de una API HTTP, workers consumidores y tres servicios de infraestructura (Redis, NATS JetStream y MinIO), definidos en [`docker-compose.yml`](docker-compose.yml).
```mermaid
flowchart TB
Client[Cliente_o_Agente]
API[API_Go_8080]
Worker[Worker_Go_xN]
Redis[(Redis)]
NATS[NATS_JetStream]
MinIO[(MinIO)]
Pipeline[OCR_Tesseract_ONNX]
Client -->|sync_POST_v1_count| API
Client -->|async_POST_v1_jobs| API
API --> Pipeline
API --> Redis
API --> MinIO
API -->|publish| NATS
NATS -->|consume| Worker
Worker --> MinIO
Worker --> Pipeline
Worker --> Redis
```
### Pipeline de inferencia
Por cada PDF se procesa la primera página siguiendo este flujo (`internal/pipeline`):
```mermaid
flowchart LR
PDF[PDF_primera_pagina] --> Render[Render_MuPDF]
Render --> OCR[Tesseract_spa]
OCR --> Cluster[DBSCAN_filas]
Cluster --> Features[8_features]
Features --> Infer[ONNX_RandomForest]
Infer --> Result[service_count_y_services]
```
1. **Render** — La primera página se rasteriza con MuPDF (go-fitz) a 300 DPI.
2. **OCR** — Tesseract extrae cajas de texto en español.
3. **Clustering** — DBSCAN agrupa palabras en filas candidatas.
4. **Features** — Se calculan 8 atributos por fila (códigos, moneda, posición, etc.).
5. **Inferencia** — Un Random Forest exportado a ONNX clasifica cada fila como servicio o no.
## Componentes
| Componente | Binario / servicio | Paquete | Responsabilidad |
|------------|-------------------|---------|-----------------|
| API HTTP | `cmd/api` | `internal/api` | Endpoints REST, gestión de jobs, health probes |
| Worker | `cmd/worker` | `internal/worker` | Consume NATS, descarga PDFs de MinIO, persiste resultados |
| Pipeline | — | `internal/pipeline` | Orquestación OCR → clustering → features → inferencia |
| Modelo ONNX | — | `internal/model` | Carga `model.onnx` e `inference.json` |
| OCR | — | `internal/ocr` | Render PDF (go-fitz) + Tesseract (gosseract) |
| Estado de jobs | Redis 7 | `internal/store` | Metadatos, resultados y progreso (TTL 72 h) |
| Cola | NATS JetStream | `internal/queue` | Stream `PDF_JOBS`, subject `pdf.process.*` |
| Object storage | MinIO | `internal/storage` | Almacenamiento de PDFs en jobs asíncronos |
### Estructura del repositorio
```
├── cmd/
│ ├── api/ # Punto de entrada HTTP
│ └── worker/ # Punto de entrada worker
├── internal/
│ ├── api/ # Servidor HTTP (chi)
│ ├── config/ # Variables de entorno
│ ├── domain/ # Tipos de dominio
│ ├── model/ # Inferencia ONNX
│ ├── ocr/ # Extracción OCR
│ ├── pipeline/ # Procesamiento por PDF
│ ├── queue/ # NATS JetStream
│ ├── storage/ # Cliente MinIO
│ ├── store/ # Estado en Redis
│ └── worker/ # Runner del worker
├── api/openapi.yaml # Especificación OpenAPI 3.1
├── mcp/tool.json # Descriptor de herramientas para agentes
├── web/ # Cliente demo (embebido en la API en /demo/)
├── scripts/ # Export ONNX, paridad con Python
└── models/latest/ # model.onnx + inference.json (montado en Docker)
```
## Modos de operación
| Modo | Endpoint principal | Límite | Cuándo usarlo |
|------|-------------------|--------|---------------|
| Síncrono | `POST /v1/count` | 50 PDFs | Pruebas, integraciones en tiempo real, lotes pequeños |
| Asíncrono | `POST /v1/jobs` + poll | 50 000 PDFs/job | MinIO + NATS + workers (path producción) |
**Flujo asíncrono (producción):**
1. Crear job (`POST /v1/jobs`) — shell vacío, prefijo MinIO o lista de claves.
2. Subir PDFs por chunks (`POST /v1/jobs/{jobID}/upload`, máx. 500 por chunk) → **MinIO Put** → **NATS Publish**.
3. Workers consumen la cola, descargan de MinIO, OCR, guardan en Redis.
4. Consultar progreso (`GET /v1/jobs/{jobID}?wait_ms=30000` para long-polling).
5. Obtener resultados paginados (`GET /v1/jobs/{jobID}/results?cursor=&limit=`).
## API REST
Especificación completa: `GET /openapi.json` o [`api/openapi.yaml`](api/openapi.yaml).
## Endpoints
| Método | Ruta | Descripción |
|--------|------|-------------|
| `GET` | `/health` | Liveness probe |
| `GET` | `/ready` | Readiness probe |
| `POST` | `/v1/count` | Conteo síncrono (multipart `files`) |
| `POST` | `/v1/jobs` | Crear job asíncrono |
| `POST` | `/v1/jobs/{jobID}/upload` | Subir chunk de PDFs |
| `GET` | `/v1/jobs/{jobID}` | Estado y progreso del job |
| `GET` | `/v1/jobs/{jobID}/results` | Resultados paginados |
| `DELETE` | `/v1/jobs/{jobID}` | Cancelar job |
| `GET` | `/health` | Liveness — `{"status":"ok"}` |
| `GET` | `/ready` | Readiness — `{"status":"ready"}` |
| `GET` | `/openapi.json` | Especificación OpenAPI |
| `POST` | `/v1/count` | Conteo síncrono de PDFs |
**Parámetros relevantes:**
### POST /v1/count
- `include_rejected` (query) — Si es `true`, incluye todas las filas OCR con el campo `is_service`.
- Jobs asíncronos devuelven `202 Accepted`; la cola rechaza con `429` si supera `QUEUE_MAX_PENDING`.
**Request**
**Respuesta por PDF:**
```
Content-Type: multipart/form-data
Query: include_rejected=true|false (default: false)
Field: files — uno o más archivos PDF (máx. SYNC_MAX_FILES)
```
**Response 200**
```json
{
"filename": "factura.pdf",
"service_count": 2,
"status": "ok",
"services": [
{ "text": "890201 Consulta ...", "probability": 0.94 }
]
"results": [
{
"filename": "factura_001.pdf",
"service_count": 3,
"status": "ok",
"services": [
{ "text": "CONSULTA MEDICA", "probability": 0.91 },
{ "text": "LABORATORIO HEMOGRAMA", "probability": 0.87 },
{ "text": "RADIOGRAFIA TORAX", "probability": 0.83 }
]
}
],
"processed": 1,
"failed": 0
}
```
## Stack tecnológico
**Response 4xx/5xx**
| Área | Tecnología |
|------|------------|
| Lenguaje | Go 1.23 |
| HTTP router | [go-chi/chi](https://github.com/go-chi/chi) v5 |
| OCR | go-fitz (MuPDF) + gosseract (Tesseract) |
| ML | ONNX Runtime + Random Forest exportado |
| Cola | NATS JetStream |
| Estado | Redis 7 |
| Storage | MinIO |
| Contenedores | Docker multi-stage ([`Dockerfile`](Dockerfile)) |
## Inicio rápido
### 1. Exportar el modelo ONNX
Desde el repositorio de entrenamiento, generar los artefactos de inferencia:
```bash
pip install -r requirements-export.txt
python scripts/export_onnx.py \
--joblib ../services-count/models/latest/model.joblib \
--metadata ../services-count/models/v1.0.0/model_metadata.json
```json
{ "error": "descripción del error" }
```
Esto produce `models/latest/model.onnx` e `inference.json`.
---
### 2. Levantar con Docker Compose
## Pipeline de procesamiento
```bash
docker compose up --build
```
PDF bytes (memoria)
│
▼ gen2brain/go-fitz (MuPDF)
Render de la primera página a imagen (OCR_DPI, default 300)
│
▼ otiai10/gosseract (Tesseract)
OCR palabra a palabra con coordenadas (X, Y, W, H)
│
▼ DBSCAN 1D sobre coordenada Y
Agrupación de palabras en filas candidatas
│
▼ 8 features por fila
first_token_is_code · currency_count · integer_count · x_span_norm
y_norm · token_count · special_char_ratio · digit_ratio
│
▼ ONNX Runtime (Random Forest)
P(servicio) por fila → filtrar por threshold (inference.json)
│ Si el modelo no está disponible → heurística: todas las filas = servicio
▼
PDFResult { service_count, services[], status }
```
| Servicio | Puerto / acceso |
|----------|-----------------|
| API | `http://localhost:8080` |
| MinIO Console | `http://localhost:9002` |
| Workers | 1 réplica inicial; autoscaler sube hasta 8 según cola |
| Autoscaler | Escala `worker` vía Docker socket según `PendingCount` en Redis |
---
### 3. Probar
## Modelo ONNX
```bash
# Síncrono
curl -F "files=@factura.pdf" http://localhost:8080/v1/count
El modelo es opcional. Si no está disponible, el servicio arranca con una advertencia y usa heurística de conteo.
# Asíncrono (PDFs ya en MinIO bajo un prefijo)
curl -X POST http://localhost:8080/v1/jobs \
-H "Content-Type: application/json" \
-d '{"prefix":"jobs/batch-1/"}'
Los archivos deben montarse en `/app/models/`:
```
/app/models/model.onnx — modelo Random Forest exportado con skl2onnx
/app/models/inference.json — threshold y feature_columns
```
## Cliente web de ejemplo
Con la API en ejecución, abre **[http://localhost:8080/demo/](http://localhost:8080/demo/)** para probar la integración desde el navegador. Los archivos estáticos se sirven desde la propia API (mismo origen, sin CORS).
| Pestaña | Demuestra |
|---------|-----------|
| **Síncrono** | `POST /v1/count` con subida multipart de hasta 50 PDFs |
| **Asíncrono** | `POST /v1/jobs` → upload → polling de estado → resultados paginados |
El modo asíncrono requiere que workers, Redis, NATS y MinIO estén levantados (igual que con `docker compose up`). Código fuente en [`web/`](web/).
## Desarrollo local
Requisitos adicionales a Go 1.23+:
- Tesseract con datos de idioma `spa`
- ONNX Runtime (la variable `ONNXRUNTIME_SHARED_LIBRARY` apunta a la librería compartida)
- Redis, NATS y MinIO en ejecución
Para generar el modelo desde un `model.joblib` existente:
```bash
go build -o api ./cmd/api
go build -o worker ./cmd/worker
cd scripts
pip install -r ../requirements-export.txt
python export_onnx.py --joblib /ruta/model.joblib --out ../models/latest
```
Configurar las variables de entorno para apuntar a los servicios locales (ver tabla siguiente).
---
## Configuración
Variables definidas en [`internal/config/config.go`](internal/config/config.go):
## Variables de entorno
| Variable | Default | Descripción |
|----------|---------|-------------|
| `PORT` | `8080` | Puerto de la API |
| `REDIS_URL` | `redis://redis:6379/0` | Conexión Redis |
| `NATS_URL` | `nats://nats:4222` | Conexión NATS |
| `NATS_STREAM` | `PDF_JOBS` | Nombre del stream JetStream |
| `NATS_SUBJECT_PREFIX` | `pdf.process` | Prefijo de subjects |
| `MINIO_ENDPOINT` | `minio:9000` | Endpoint MinIO |
| `MINIO_ACCESS_KEY` | `minioadmin` | Access key |
| `MINIO_SECRET_KEY` | `minioadmin` | Secret key |
| `MINIO_BUCKET` | `pdf-count` | Bucket de PDFs |
| `MINIO_USE_SSL` | `false` | Usar SSL con MinIO |
| `PORT` | `8080` | Puerto HTTP |
| `MODEL_PATH` | `/app/models/model.onnx` | Ruta al modelo ONNX |
| `INFERENCE_CONFIG` | `/app/models/inference.json` | Metadata de inferencia |
| `OCR_DPI` | `300` | DPI de renderizado PDF (150 reduce latencia ~2–4×) |
| `INFERENCE_CONFIG` | `/app/models/inference.json` | Threshold y metadatos del modelo |
| `OPENAPI_PATH` | `api/openapi.yaml` | Ruta a la especificación OpenAPI |
| `OCR_DPI` | `300` | DPI de renderizado PDF para OCR |
| `OCR_LANG` | `spa` | Idioma Tesseract |
| `DBSCAN_EPS` | `10` | Parámetro epsilon de DBSCAN |
| `DBSCAN_MIN_SAMPLES` | `1` | Mínimo de muestras DBSCAN |
| `SYNC_MAX_FILES` | `50` | Máximo de PDFs en modo síncrono |
| `JOB_MAX_FILES` | `50000` | Máximo de PDFs por job |
| `QUEUE_MAX_PENDING` | `100000` | Límite de mensajes pendientes en cola |
| `WORKER_CONCURRENCY` | `4` | Goroutines paralelas por worker |
| `MAX_PDF_SIZE_MB` | `20` | Tamaño máximo por PDF |
| `PDF_PROCESS_TIMEOUT_SEC` | `120` | Timeout de procesamiento por PDF |
| `JOB_RESULT_TTL_HOURS` | `72` | TTL de resultados en Redis |
| `OCR_POOL_SIZE` | `4` (API) / `WORKER_CONCURRENCY` (worker) | Clientes Tesseract reutilizables en pool |
| `SYNC_PROCESS_CONCURRENCY` | `4` (o `OCR_POOL_SIZE`) | PDFs procesados en paralelo en `/v1/count` |
| `JOB_INLINE_PROCESSING` | `false` | `true` procesa uploads en la API sin MinIO (solo dev local) |
| `LOG_PIPELINE_TIMING` | `false` | Logs de duración por etapa del pipeline |
| `NATS_FETCH_MAX_WAIT_MS` | `300` | Tiempo máximo de espera del worker al leer NATS |
| `OCR_POOL_SIZE` | `0` (→ 4) | Tamaño del pool de clientes Tesseract |
| `DBSCAN_EPS` | `10` | Épsilon de clustering DBSCAN (píxeles en Y) |
| `DBSCAN_MIN_SAMPLES` | `1` | Mínimo de puntos por cluster DBSCAN |
| `SYNC_MAX_FILES` | `50` | Máximo de PDFs por request a `/v1/count` |
| `SYNC_PROCESS_CONCURRENCY` | `0` (→ OCR_POOL_SIZE) | Goroutines paralelas por request |
| `MAX_PDF_SIZE_MB` | `20` | Tamaño máximo permitido por PDF |
| `PDF_PROCESS_TIMEOUT_SEC` | `120` | Timeout por PDF |
| `LOG_PIPELINE_TIMING` | `false` | Loguear tiempos de cada etapa |
## Rendimiento
---
El cuello de botella de `/v1/count` es el **OCR con Tesseract**, no la inferencia ONNX. Los clientes Tesseract se crean una vez al arranque y se reutilizan desde un pool (`internal/ocr/pool.go`).
## Dependencias en producción (Docker)
| Etapa | Tiempo típico (1 PDF) |
|-------|------------------------|
| Render PDF (MuPDF) | ~100–300 ms |
| OCR Tesseract | ~3–8 s |
| Clustering + ONNX | ~10–50 ms |
- `libonnxruntime.so` — ONNX Runtime (copiado en build multi-etapa)
- `tesseract-ocr tesseract-ocr-spa` — OCR en español
- `libmupdf` — via go-fitz, renderizado de PDFs
**Latencia esperada (síncrono):** ~3–10 s por PDF. Varios PDFs se procesan en paralelo (`SYNC_PROCESS_CONCURRENCY`, default 4). Ejemplo: 20 PDFs ≈ `ceil(20/4) × 2.85s ≈ 15s`.
Sin Redis, MinIO, NATS ni bases de datos.
**Latencia esperada (asíncrono por upload):** path producción MinIO → NATS → worker. Con `WORKER_CONCURRENCY=4` y autoscaler activo, 16 PDFs ≈ `ceil(16/4) × 2.85s` más overhead Put/Get (~15–25 s). 100 PDFs escala hasta 8 workers (32 slots OCR).
---
**Asíncrono por prefix MinIO** (`POST /v1/jobs {"prefix":"..."}`): mismos workers; los PDFs ya están en el bucket.
**Ajustes para reducir latencia:**
- `OCR_DPI=150` — reduce tiempo de render y OCR (~2–4× más rápido) con posible ligera pérdida de precisión. Validar con `scripts/compare_parity.py` antes de usar en producción.
- `OCR_POOL_SIZE=4` y `WORKER_CONCURRENCY=4` — paralelismo por réplica worker.
- Upload pipeline publica en NATS tras cada Put (no al final del lote).
- `LOG_PIPELINE_TIMING=true` — logs por etapa: `ocr_render`, `ocr_tesseract`, `cluster`, `infer`, `total`.
- `NATS_FETCH_MAX_WAIT_MS=300` — el worker recoge mensajes con menor espera.
- `GET /v1/jobs/{id}?wait_ms=30000` — long-polling en lugar de poll cada 500 ms.
## Escalado
### Autoscaler (docker compose)
El servicio `autoscaler` lee `PendingCount` en Redis cada 5 s y ajusta réplicas `worker`:
## Estructura del código
```
desired = clamp(ceil(pending / WORKER_SLOTS), MIN_WORKERS, MAX_WORKERS)
cmd/api/ — punto de entrada del servidor HTTP
internal/
api/ — router y handler countSync
config/ — carga de variables de entorno
domain/ — tipos PDFResult, CountResponse, ServiceRow
model/ — sesión ONNX Runtime (Predictor)
ocr/ — pool de clientes Tesseract + renderizado MuPDF
pipeline/ — orquestación OCR → DBSCAN → features → ML
processor.go — ProcessPDF por documento
clustering.go — DBSCAN 1D en Y
features.go — 8 features por fila
api/openapi.yaml — especificación OpenAPI 3.1
models/latest/ — model.onnx + inference.json (gitignored, montar en Docker)
scripts/ — export_onnx.py para generar el modelo
Dockerfile — build multi-etapa: ONNX download + Go builder + runtime slim
```
| Variable | Default | Descripción |
|----------|---------|-------------|
| `MIN_WORKERS` | `1` | Réplicas mínimas |
| `MAX_WORKERS` | `8` | Réplicas máximas |
| `WORKER_SLOTS` | `4` | PDFs en paralelo por réplica (`WORKER_CONCURRENCY`) |
| `SCALE_DOWN_IDLE_SEC` | `60` | Segundos sin cola antes de bajar 1 réplica |
---
Requiere montar `/var/run/docker.sock` (solo entorno local). En Kubernetes usar HPA sobre lag de NATS o métricas custom.
### Escalado manual
## Arrancar localmente (Docker)
```bash
docker compose up --scale worker=8 -d
# Desde la raíz del repositorio:
docker compose up --build count-api
```
Cada worker procesa hasta `WORKER_CONCURRENCY` PDFs en paralelo. NATS JetStream distribuye los mensajes entre réplicas con política work-queue.
El servicio queda disponible en `http://localhost:8081`.
## Integración con agentes (MCP)
[`mcp/tool.json`](mcp/tool.json) es un **descriptor declarativo** de herramientas para agentes AI (Cursor, etc.). No incluye un servidor MCP embebido.
| Herramienta | Modo | Flujo |
|-------------|------|-------|
| `count_pdf_services` | Síncrono | `POST /v1/count` con multipart |
| `count_pdf_services_batch` | Asíncrono | Crear job → poll estado → resultados paginados |
## Paridad con Python
Scripts para validar que la inferencia Go reproduce los resultados del pipeline Python original:
Para probar el endpoint:
```bash
set SERVICES_COUNT_ROOT=../services-count
python scripts/parity_baseline.py
python scripts/compare_parity.py
curl -X POST http://localhost:8081/v1/count \
-F "files=@factura.pdf" | jq .
```
## Troubleshooting
| Síntoma | Causa probable | Acción |
|---------|------------------|--------|
| API no arranca | Modelo ONNX ausente | Ejecutar `scripts/export_onnx.py` y verificar `models/latest/` |
| `/ready` falla | Redis o NATS no disponibles | Comprobar healthchecks en `docker compose ps` |
| Jobs en `pending` sin avance | Workers no consumen cola | Verificar logs del worker y conectividad a NATS/MinIO |
| Job atascado en `running` con N/N procesados | Race en metadata del job (versiones antiguas) | Reconstruir con `docker compose up --build`; `GetJob` ahora finaliza jobs completados |
| `429` al crear job | Cola saturada | Escalar workers o esperar a que baje `QUEUE_MAX_PENDING` |
| Resultados vacíos o erróneos | OCR o modelo desactualizado | Revisar `OCR_LANG=spa`, reexportar ONNX desde el repo ML |
@@ -6,15 +6,11 @@ import (
"os"
"os/signal"
"syscall"
"time"
"github.com/pys/pdf-count/internal/api"
"github.com/pys/pdf-count/internal/config"
"github.com/pys/pdf-count/internal/model"
"github.com/pys/pdf-count/internal/ocr"
"github.com/pys/pdf-count/internal/queue"
"github.com/pys/pdf-count/internal/storage"
"github.com/pys/pdf-count/internal/store"
)
func main() {
@@ -24,9 +20,12 @@ func main() {
predictor, err := model.NewPredictor(cfg.ModelPath, cfg.InferencePath)
if err != nil {
log.Fatalf("model: %v", err)
log.Printf("WARN: modelo ML no disponible (%v) — usando heuristica de conteo", err)
predictor = nil
} else {
defer predictor.Close()
log.Printf("modelo ONNX cargado: %s", cfg.ModelPath)
}
defer predictor.Close()
poolSize := cfg.OCRPoolSize
if poolSize <= 0 {
@@ -38,37 +37,16 @@ func main() {
}
defer ocrEngine.Close()
st, err := store.New(cfg.RedisURL, cfg.JobResultTTL)
if err != nil {
log.Fatalf("redis: %v", err)
}
defer st.Close()
stg, err := storage.New(cfg.MinIOEndpoint, cfg.MinIOAccessKey, cfg.MinIOSecretKey, cfg.MinIOBucket, cfg.MinIOUseSSL)
if err != nil {
log.Fatalf("minio: %v", err)
}
initCtx, initCancel := context.WithTimeout(ctx, 10*time.Second)
if err := stg.EnsureBucket(initCtx); err != nil {
initCancel()
log.Fatalf("minio bucket: %v", err)
}
initCancel()
pub, err := queue.NewPublisher(cfg.NATSURL, cfg.StreamName, cfg.SubjectPrefix)
if err != nil {
log.Fatalf("nats: %v", err)
}
openAPIPath := os.Getenv("OPENAPI_PATH")
if openAPIPath == "" {
openAPIPath = "api/openapi.json"
openAPIPath = "api/openapi.yaml"
}
srv := api.NewServer(cfg, predictor, ocrEngine, st, stg, pub, api.LoadOpenAPI(openAPIPath))
srv := api.NewServer(cfg, predictor, ocrEngine, api.LoadOpenAPI(openAPIPath))
addr := ":" + cfg.Port
log.Printf("api listening on %s", addr)
log.Printf("count-api escuchando en %s", addr)
if err := api.ListenAndServe(ctx, addr, srv.Router()); err != nil {
log.Fatalf("api: %v", err)
log.Fatalf("server: %v", err)
}
}
@@ -1,157 +0,0 @@
package main
import (
"context"
"log"
"math"
"os"
"os/exec"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/pys/pdf-count/internal/store"
)
func main() {
cfg := loadConfig()
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
st, err := store.New(cfg.redisURL, 72*time.Hour)
if err != nil {
log.Fatalf("redis: %v", err)
}
defer st.Close()
log.Printf("autoscaler started min=%d max=%d slots=%d service=%s",
cfg.minWorkers, cfg.maxWorkers, cfg.workerSlots, cfg.composeService)
var idleSince time.Time
for {
select {
case <-ctx.Done():
log.Println("autoscaler stopped")
return
default:
}
pending, err := st.PendingCount(ctx)
if err != nil {
log.Printf("autoscaler pending count: %v", err)
time.Sleep(cfg.pollInterval)
continue
}
current, err := countWorkers(cfg)
if err != nil {
log.Printf("autoscaler count workers: %v", err)
time.Sleep(cfg.pollInterval)
continue
}
desired := cfg.minWorkers
if pending > 0 {
desired = int(math.Ceil(float64(pending) / float64(cfg.workerSlots)))
if desired < cfg.minWorkers {
desired = cfg.minWorkers
}
if desired > cfg.maxWorkers {
desired = cfg.maxWorkers
}
idleSince = time.Time{}
} else if current > cfg.minWorkers {
if idleSince.IsZero() {
idleSince = time.Now()
} else if time.Since(idleSince) >= cfg.scaleDownIdle {
desired = current - 1
idleSince = time.Time{}
} else {
desired = current
}
}
if desired != current {
log.Printf("autoscaler pending=%d current=%d desired=%d scaling", pending, current, desired)
if err := scaleWorkers(cfg, desired); err != nil {
log.Printf("autoscaler scale: %v", err)
}
}
time.Sleep(cfg.pollInterval)
}
}
type autoscalerConfig struct {
redisURL string
composeFile string
composeService string
minWorkers int
maxWorkers int
workerSlots int
pollInterval time.Duration
scaleDownIdle time.Duration
}
func loadConfig() autoscalerConfig {
return autoscalerConfig{
redisURL: getenv("REDIS_URL", "redis://redis:6379/0"),
composeFile: getenv("COMPOSE_FILE", "/project/docker-compose.yml"),
composeService: getenv("COMPOSE_SERVICE", "worker"),
minWorkers: getenvInt("MIN_WORKERS", 1),
maxWorkers: getenvInt("MAX_WORKERS", 8),
workerSlots: getenvInt("WORKER_SLOTS", 4),
pollInterval: time.Duration(getenvInt("POLL_INTERVAL_SEC", 5)) * time.Second,
scaleDownIdle: time.Duration(getenvInt("SCALE_DOWN_IDLE_SEC", 60)) * time.Second,
}
}
func countWorkers(cfg autoscalerConfig) (int, error) {
cmd := exec.Command("docker", "compose",
"-f", cfg.composeFile,
"ps", cfg.composeService,
"--status", "running",
"-q",
)
out, err := cmd.Output()
if err != nil {
return 0, err
}
trimmed := strings.TrimSpace(string(out))
if trimmed == "" {
return 0, nil
}
return len(strings.Split(trimmed, "\n")), nil
}
func scaleWorkers(cfg autoscalerConfig, n int) error {
scaleArg := cfg.composeService + "=" + strconv.Itoa(n)
cmd := exec.Command("docker", "compose",
"-f", cfg.composeFile,
"up", "-d", "--no-recreate",
"--scale", scaleArg,
cfg.composeService,
)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd.Run()
}
func getenv(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
func getenvInt(key string, fallback int) int {
if v := os.Getenv(key); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return fallback
}
@@ -1,7 +0,0 @@
package main
import "github.com/pys/pdf-count/internal/worker"
func main() {
worker.RunMain()
}
@@ -1,40 +1,20 @@
module github.com/pys/pdf-count
go 1.23.0
toolchain go1.23.4
go 1.25.0
require (
github.com/gen2brain/go-fitz v1.24.14
github.com/go-chi/chi/v5 v5.2.1
github.com/google/uuid v1.6.0
github.com/minio/minio-go/v7 v7.0.90
github.com/nats-io/nats.go v1.39.1
github.com/otiai10/gosseract/v2 v2.4.1
github.com/redis/go-redis/v9 v9.7.3
github.com/yalue/onnxruntime_go v1.21.0
gopkg.in/yaml.v3 v3.0.1
)
require github.com/rogpeppe/go-internal v1.15.0 // indirect
require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/ebitengine/purego v0.8.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/jupiterrider/ffi v0.2.0 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/minio/crc64nvme v1.0.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/nats-io/nkeys v0.4.9 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rs/xid v1.6.0 // indirect
golang.org/x/crypto v0.36.0 // indirect
golang.org/x/net v0.38.0 // indirect
golang.org/x/sys v0.31.0 // indirect
golang.org/x/text v0.23.0 // indirect
github.com/kr/pretty v0.3.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)
@@ -1,69 +1,33 @@
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/ebitengine/purego v0.8.0 h1:JbqvnEzRvPpxhCJzJJ2y0RbiZ8nyjccVUrSM3q+GvvE=
github.com/ebitengine/purego v0.8.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/gen2brain/go-fitz v1.24.14 h1:09weRkjVtLYNGo7l0J7DyOwBExbwi8SJ9h8YPhw9WEo=
github.com/gen2brain/go-fitz v1.24.14/go.mod h1:0KaZeQgASc20Yp5R/pFzyy7SmP01XcoHKNF842U2/S4=
github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8=
github.com/go-chi/chi/v5 v5.2.1/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jupiterrider/ffi v0.2.0 h1:tMM70PexgYNmV+WyaYhJgCvQAvtTCs3wXeILPutihnA=
github.com/jupiterrider/ffi v0.2.0/go.mod h1:yqYqX5DdEccAsHeMn+6owkoI2llBLySVAF8dwCDZPVs=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY=
github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.90 h1:TmSj1083wtAD0kEYTx7a5pFsv3iRYMsOJ6A4crjA1lE=
github.com/minio/minio-go/v7 v7.0.90/go.mod h1:uvMUcGrpgeSAAI6+sD3818508nUyMULw94j2Nxku/Go=
github.com/nats-io/nats.go v1.39.1 h1:oTkfKBmz7W047vRxV762M67ZdXeOtUgvbBaNoQ+3PPk=
github.com/nats-io/nats.go v1.39.1/go.mod h1:MgRb8oOdigA6cYpEPhXJuRVH6UE/V4jblJ2jQ27IXYM=
github.com/nats-io/nkeys v0.4.9 h1:qe9Faq2Gxwi6RZnZMXfmGMZkg3afLLOtrU+gDZJ35b0=
github.com/nats-io/nkeys v0.4.9/go.mod h1:jcMqs+FLG+W5YO36OX6wFIFcmpdAns+w1Wm6D3I/evE=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/otiai10/gosseract/v2 v2.4.1 h1:G8AyBpXEeSlcq8TI85LH/pM5SXk8Djy2GEXisgyblRw=
github.com/otiai10/gosseract/v2 v2.4.1/go.mod h1:1gNWP4Hgr2o7yqWfs6r5bZxAatjOIdqWxJLWsTsembk=
github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs=
github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM=
github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
github.com/yalue/onnxruntime_go v1.21.0 h1:DdtvfY7OP5gR8mwPDqAOAQckf+KcI30hPNJL8hQaYWI=
github.com/yalue/onnxruntime_go v1.21.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -1,53 +0,0 @@
package api
import (
"context"
"github.com/pys/pdf-count/internal/pipeline"
"github.com/pys/pdf-count/internal/store"
)
// FilePayload holds a PDF ready for inline job processing.
type FilePayload struct {
Filename string
Data []byte
}
// JobRunner processes uploaded job files in-process using the API OCR pool.
type JobRunner struct {
store *store.Store
newProc func(includeRejected bool) *pipeline.Processor
sem chan struct{}
}
func NewJobRunner(st *store.Store, newProc func(includeRejected bool) *pipeline.Processor, concurrency int) *JobRunner {
if concurrency < 1 {
concurrency = 1
}
return &JobRunner{
store: st,
newProc: newProc,
sem: make(chan struct{}, concurrency),
}
}
// ProcessOne schedules a single PDF for inline OCR (non-blocking).
func (jr *JobRunner) ProcessOne(jobID string, file FilePayload, includeRejected bool) {
go func() {
jr.sem <- struct{}{}
defer func() { <-jr.sem }()
ctx := context.Background()
proc := jr.newProc(includeRejected)
result := proc.ProcessPDF(file.Filename, file.Data)
_ = jr.store.SaveResult(ctx, jobID, result)
_ = jr.store.TryCompleteJob(ctx, jobID)
}()
}
// ProcessFiles runs OCR in background goroutines and updates job progress in Redis.
func (jr *JobRunner) ProcessFiles(jobID string, files []FilePayload, includeRejected bool) {
for _, file := range files {
jr.ProcessOne(jobID, file, includeRejected)
}
}
@@ -4,87 +4,47 @@ import (
"context"
"encoding/json"
"io"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/google/uuid"
"github.com/pys/pdf-count/internal/config"
"github.com/pys/pdf-count/internal/domain"
"github.com/pys/pdf-count/internal/model"
"github.com/pys/pdf-count/internal/ocr"
"github.com/pys/pdf-count/internal/pipeline"
"github.com/pys/pdf-count/internal/queue"
"github.com/pys/pdf-count/internal/storage"
"github.com/pys/pdf-count/internal/store"
"github.com/pys/pdf-count/web"
"gopkg.in/yaml.v3"
)
// Server holds the shared, stateless dependencies of the count-api.
type Server struct {
cfg config.Config
predictor *model.Predictor
predictor *model.Predictor // nil → heuristic fallback
ocrEngine *ocr.Engine
store *store.Store
storage *storage.Client
publisher *queue.Publisher
jobRunner *JobRunner
openAPI []byte
ready bool
readyMu sync.RWMutex
}
func NewServer(cfg config.Config, predictor *model.Predictor, ocrEngine *ocr.Engine, st *store.Store, stg *storage.Client, pub *queue.Publisher, openAPI []byte) *Server {
srv := &Server{
// NewServer constructs a Server. predictor may be nil when the ONNX model is
// unavailable; the pipeline will fall back to a counting heuristic.
func NewServer(
cfg config.Config,
predictor *model.Predictor,
ocrEngine *ocr.Engine,
openAPI []byte,
) *Server {
return &Server{
cfg: cfg,
predictor: predictor,
ocrEngine: ocrEngine,
store: st,
storage: stg,
publisher: pub,
openAPI: openAPI,
ready: true,
}
srv.jobRunner = NewJobRunner(st, srv.newProcessor, srv.ocrConcurrency())
return srv
}
func (s *Server) processConcurrency() int {
c := s.cfg.SyncProcessConcurrency
if c <= 0 {
c = s.cfg.OCRPoolSize
}
if c <= 0 {
c = 4
}
return c
}
func (s *Server) ocrConcurrency() int {
c := s.processConcurrency()
pool := s.cfg.OCRPoolSize
if pool <= 0 {
pool = 4
}
if c > pool {
c = pool
}
return c
}
func (s *Server) SetReady(v bool) {
s.readyMu.Lock()
s.ready = v
s.readyMu.Unlock()
}
// Router registers all HTTP routes.
func (s *Server) Router() http.Handler {
r := chi.NewRouter()
r.Use(middleware.RequestID)
@@ -92,46 +52,41 @@ func (s *Server) Router() http.Handler {
r.Use(middleware.Recoverer)
r.Get("/health", s.health)
r.Get("/ready", s.readyHandler)
r.Get("/ready", s.ready)
r.Get("/openapi.json", s.openAPIHandler)
r.Get("/demo", func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/demo/", http.StatusMovedPermanently)
})
r.Handle("/demo/*", http.StripPrefix("/demo/", http.FileServer(http.FS(web.FS))))
r.Route("/v1", func(r chi.Router) {
r.Post("/count", s.countSync)
r.Post("/jobs", s.createJob)
r.Post("/jobs/{jobID}/upload", s.uploadChunk)
r.Get("/jobs/{jobID}", s.getJob)
r.Get("/jobs/{jobID}/results", s.getJobResults)
r.Delete("/jobs/{jobID}", s.cancelJob)
})
return r
}
func (s *Server) health(w http.ResponseWriter, r *http.Request) {
// health returns a simple liveness response.
func (s *Server) health(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
}
func (s *Server) readyHandler(w http.ResponseWriter, r *http.Request) {
s.readyMu.RLock()
ok := s.ready
s.readyMu.RUnlock()
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"status": "not_ready"})
return
}
// ready returns 200 once the server is accepting traffic (always ready after init).
func (s *Server) ready(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, map[string]string{"status": "ready"})
}
func (s *Server) openAPIHandler(w http.ResponseWriter, r *http.Request) {
// openAPIHandler serves the OpenAPI specification as JSON.
func (s *Server) openAPIHandler(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(s.openAPI)
}
// countSync processes a multipart batch of PDFs synchronously and returns the
// counting results in a single JSON response. No state is persisted.
//
// POST /v1/count
//
// Query: include_rejected=true|false
// Body: multipart/form-data, field "files" (one or more PDFs)
// Limits: SYNC_MAX_FILES (default 50), MAX_PDF_SIZE_MB (default 20)
func (s *Server) countSync(w http.ResponseWriter, r *http.Request) {
includeRejected := r.URL.Query().Get("include_rejected") == "true"
maxBytes := int64(s.cfg.MaxPDFSizeMB) * 1024 * 1024
@@ -151,61 +106,51 @@ func (s *Server) countSync(w http.ResponseWriter, r *http.Request) {
return
}
proc := s.newProcessor(includeRejected)
type fileInput struct {
filename string
data []byte
errRes *domain.PDFResult
}
inputs := make([]fileInput, 0, len(files))
inputs := make([]fileInput, 0, len(files))
for _, fh := range files {
name := filepath.Base(fh.Filename)
f, err := fh.Open()
if err != nil {
errRes := domain.PDFResult{
Filename: filepath.Base(fh.Filename),
Status: domain.StatusError,
Error: err.Error(),
}
errRes := domain.PDFResult{Filename: name, Status: domain.StatusError, Error: err.Error()}
inputs = append(inputs, fileInput{errRes: &errRes})
continue
}
data, err := io.ReadAll(io.LimitReader(f, maxBytes+1))
_ = f.Close()
if err != nil {
errRes := domain.PDFResult{
Filename: filepath.Base(fh.Filename),
Status: domain.StatusError,
Error: err.Error(),
}
errRes := domain.PDFResult{Filename: name, Status: domain.StatusError, Error: err.Error()}
inputs = append(inputs, fileInput{errRes: &errRes})
continue
}
if int64(len(data)) > maxBytes {
errRes := domain.PDFResult{
Filename: filepath.Base(fh.Filename),
Status: domain.StatusError,
Error: "file too large",
}
errRes := domain.PDFResult{Filename: name, Status: domain.StatusError, Error: "file too large"}
inputs = append(inputs, fileInput{errRes: &errRes})
continue
}
inputs = append(inputs, fileInput{
filename: filepath.Base(fh.Filename),
data: data,
})
inputs = append(inputs, fileInput{filename: name, data: data})
}
proc := s.newProcessor(includeRejected)
results := make([]domain.PDFResult, len(inputs))
failed := 0
sem := make(chan struct{}, s.ocrConcurrency())
concurrency := s.syncConcurrency()
sem := make(chan struct{}, concurrency)
var wg sync.WaitGroup
var mu sync.Mutex
for i, in := range inputs {
if in.errRes != nil {
results[i] = *in.errRes
mu.Lock()
failed++
mu.Unlock()
continue
}
wg.Add(1)
@@ -231,314 +176,23 @@ func (s *Server) countSync(w http.ResponseWriter, r *http.Request) {
})
}
type createJobRequest struct {
Files []string `json:"files"`
Prefix string `json:"prefix"`
IncludeRejected bool `json:"include_rejected"`
CallbackURL string `json:"callback_url"`
}
func (s *Server) createJob(w http.ResponseWriter, r *http.Request) {
var req createJobRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
httpError(w, http.StatusBadRequest, "invalid json body")
return
// syncConcurrency returns the effective parallelism for /v1/count requests.
func (s *Server) syncConcurrency() int {
c := s.cfg.SyncProcessConcurrency
if c <= 0 {
c = s.cfg.OCRPoolSize
}
ctx := r.Context()
pending, err := s.store.PendingCount(ctx)
if err == nil && int(pending) > s.cfg.QueueMaxPending {
httpError(w, http.StatusTooManyRequests, "queue full")
return
if c <= 0 {
c = 4
}
jobID := uuid.NewString()
var objectKeys []string
var filenames []string
if req.Prefix != "" {
prefix := strings.TrimSuffix(req.Prefix, "/")
keys, err := s.storage.ListByPrefix(ctx, prefix)
if err != nil {
httpError(w, http.StatusBadRequest, err.Error())
return
}
objectKeys = keys
for _, k := range keys {
filenames = append(filenames, filepath.Base(k))
}
} else if len(req.Files) > 0 {
for _, f := range req.Files {
name := filepath.Base(f)
objectKeys = append(objectKeys, storage.JobObjectKey(jobID, name))
filenames = append(filenames, name)
}
} else {
// Empty job shell for chunked upload flow.
if err := s.store.CreateJob(ctx, jobID, 0, req.IncludeRejected); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusAccepted, domain.JobStatus{
JobID: jobID,
Status: domain.JobQueued,
Total: 0,
})
return
pool := s.cfg.OCRPoolSize
if pool <= 0 {
pool = 4
}
if len(objectKeys) > s.cfg.JobMaxFiles {
httpError(w, http.StatusBadRequest, "too many files in job")
return
if c > pool {
c = pool
}
if err := s.store.CreateJob(ctx, jobID, len(objectKeys), req.IncludeRejected); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
if err := s.store.AddJobFiles(ctx, jobID, filenames); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
_ = s.store.MarkRunning(ctx, jobID)
for i, key := range objectKeys {
msg := domain.QueueMessage{
JobID: jobID,
ObjectKey: key,
Filename: filenames[i],
IncludeRejected: req.IncludeRejected,
}
if err := s.publisher.Publish(ctx, msg); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
}
writeJSON(w, http.StatusAccepted, domain.JobStatus{
JobID: jobID,
Status: domain.JobRunning,
Total: len(objectKeys),
})
}
func (s *Server) uploadChunk(w http.ResponseWriter, r *http.Request) {
jobID := chi.URLParam(r, "jobID")
includeRejected := r.URL.Query().Get("include_rejected") == "true"
maxBytes := int64(s.cfg.MaxPDFSizeMB) * 1024 * 1024
if err := r.ParseMultipartForm(maxBytes * 500); err != nil {
httpError(w, http.StatusBadRequest, "invalid multipart form")
return
}
files := r.MultipartForm.File["files"]
if len(files) == 0 {
httpError(w, http.StatusBadRequest, "field 'files' required")
return
}
if len(files) > 500 {
httpError(w, http.StatusBadRequest, "max 500 files per chunk")
return
}
ctx := r.Context()
status, err := s.store.GetJob(ctx, jobID)
if err != nil {
_ = s.store.CreateJob(ctx, jobID, 0, includeRejected)
status = domain.JobStatus{JobID: jobID, Status: domain.JobQueued}
}
if status.Status == domain.JobCancelled {
httpError(w, http.StatusConflict, "job cancelled")
return
}
var filenames []string
if s.cfg.JobInlineProcessing {
_ = s.store.MarkRunning(ctx, jobID)
for _, fh := range files {
f, err := fh.Open()
if err != nil {
continue
}
data, err := io.ReadAll(io.LimitReader(f, maxBytes+1))
_ = f.Close()
if err != nil || int64(len(data)) > maxBytes {
continue
}
name := filepath.Base(fh.Filename)
if err := s.store.AddJobFiles(ctx, jobID, []string{name}); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
if err := s.store.IncrementTotal(ctx, jobID, 1); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
filenames = append(filenames, name)
s.jobRunner.ProcessOne(jobID, FilePayload{Filename: name, Data: data}, includeRejected)
}
} else {
var err error
filenames, err = s.uploadChunkMinIO(ctx, jobID, files, includeRejected, maxBytes)
if err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
}
if len(filenames) == 0 {
httpError(w, http.StatusBadRequest, "no valid files uploaded")
return
}
writeJSON(w, http.StatusAccepted, map[string]interface{}{
"job_id": jobID,
"uploaded": len(filenames),
})
}
func (s *Server) uploadChunkMinIO(ctx context.Context, jobID string, files []*multipart.FileHeader, includeRejected bool, maxBytes int64) ([]string, error) {
filenames := make([]string, 0, len(files))
for _, fh := range files {
filenames = append(filenames, filepath.Base(fh.Filename))
}
if len(filenames) == 0 {
return nil, nil
}
if err := s.store.AddJobFiles(ctx, jobID, filenames); err != nil {
return nil, err
}
if err := s.store.IncrementTotal(ctx, jobID, len(filenames)); err != nil {
return nil, err
}
if err := s.store.MarkRunning(ctx, jobID); err != nil {
return nil, err
}
ioLimit := 8
if len(files) < ioLimit {
ioLimit = len(files)
}
ioSem := make(chan struct{}, ioLimit)
var wg sync.WaitGroup
var mu sync.Mutex
uploadedNames := make([]string, 0, len(files))
for _, fh := range files {
wg.Add(1)
go func(fh *multipart.FileHeader) {
defer wg.Done()
ioSem <- struct{}{}
defer func() { <-ioSem }()
name := filepath.Base(fh.Filename)
f, err := fh.Open()
if err != nil {
s.recordUploadError(ctx, jobID, name, err.Error())
return
}
data, err := io.ReadAll(io.LimitReader(f, maxBytes+1))
_ = f.Close()
if err != nil {
s.recordUploadError(ctx, jobID, name, err.Error())
return
}
if int64(len(data)) > maxBytes {
s.recordUploadError(ctx, jobID, name, "file too large")
return
}
key := storage.JobObjectKey(jobID, name)
if err := s.storage.Put(ctx, key, data, "application/pdf"); err != nil {
s.recordUploadError(ctx, jobID, name, err.Error())
return
}
msg := domain.QueueMessage{
JobID: jobID,
ObjectKey: key,
Filename: name,
IncludeRejected: includeRejected,
}
if err := s.publisher.Publish(ctx, msg); err != nil {
s.recordUploadError(ctx, jobID, name, err.Error())
return
}
mu.Lock()
uploadedNames = append(uploadedNames, name)
mu.Unlock()
}(fh)
}
wg.Wait()
if len(uploadedNames) == 0 {
return nil, nil
}
return uploadedNames, nil
}
func (s *Server) recordUploadError(ctx context.Context, jobID, filename, errMsg string) {
_ = s.store.SaveResult(ctx, jobID, domain.PDFResult{
Filename: filename,
Status: domain.StatusError,
Error: errMsg,
})
_ = s.store.TryCompleteJob(ctx, jobID)
}
func (s *Server) getJob(w http.ResponseWriter, r *http.Request) {
jobID := chi.URLParam(r, "jobID")
waitMs, _ := strconv.Atoi(r.URL.Query().Get("wait_ms"))
if waitMs > 30000 {
waitMs = 30000
}
deadline := time.Now()
if waitMs > 0 {
deadline = deadline.Add(time.Duration(waitMs) * time.Millisecond)
}
for {
status, err := s.store.GetJob(r.Context(), jobID)
if err != nil {
httpError(w, http.StatusNotFound, "job not found")
return
}
if isTerminalJobStatus(status.Status) || waitMs <= 0 || !time.Now().Before(deadline) {
writeJSON(w, http.StatusOK, status)
return
}
time.Sleep(100 * time.Millisecond)
}
}
func isTerminalJobStatus(status string) bool {
return status == domain.JobCompleted || status == domain.JobCancelled
}
func (s *Server) getJobResults(w http.ResponseWriter, r *http.Request) {
jobID := chi.URLParam(r, "jobID")
cursor := r.URL.Query().Get("cursor")
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
page, err := s.store.GetResultsPage(r.Context(), jobID, cursor, limit)
if err != nil {
httpError(w, http.StatusNotFound, err.Error())
return
}
writeJSON(w, http.StatusOK, page)
}
func (s *Server) cancelJob(w http.ResponseWriter, r *http.Request) {
jobID := chi.URLParam(r, "jobID")
if err := s.store.CancelJob(r.Context(), jobID); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"job_id": jobID, "status": domain.JobCancelled})
return c
}
func (s *Server) newProcessor(includeRejected bool) *pipeline.Processor {
@@ -552,20 +206,24 @@ func (s *Server) newProcessor(includeRejected bool) *pipeline.Processor {
})
}
// writeJSON encodes v as JSON and writes it with the given HTTP status.
func writeJSON(w http.ResponseWriter, status int, v interface{}) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(v)
}
// httpError writes a JSON error response.
func httpError(w http.ResponseWriter, status int, msg string) {
writeJSON(w, status, map[string]string{"error": msg})
}
// LoadOpenAPI reads an OpenAPI spec from path.
// YAML files are converted to JSON on the fly. Returns a minimal stub on error.
func LoadOpenAPI(path string) []byte {
data, err := os.ReadFile(path)
if err != nil {
return []byte(`{"openapi":"3.1.0","info":{"title":"pdf-count","version":"1.0.0"}}`)
return []byte(`{"openapi":"3.1.0","info":{"title":"count-api","version":"1.0.0"}}`)
}
if strings.HasSuffix(strings.ToLower(path), ".yaml") || strings.HasSuffix(strings.ToLower(path), ".yml") {
var doc interface{}
@@ -578,6 +236,7 @@ func LoadOpenAPI(path string) []byte {
return data
}
// ListenAndServe starts the HTTP server and blocks until ctx is cancelled.
func ListenAndServe(ctx context.Context, addr string, handler http.Handler) error {
srv := &http.Server{Addr: addr, Handler: handler}
go func() {
@@ -6,67 +6,39 @@ import (
"time"
)
// Config holds all runtime parameters for the count-api service.
// All values are loaded from environment variables with safe defaults.
type Config struct {
Port string
RedisURL string
NATSURL string
MinIOEndpoint string
MinIOAccessKey string
MinIOSecretKey string
MinIOBucket string
MinIOUseSSL bool
ModelPath string
InferencePath string
OCRDPI int
OCRLang string
DBSCANEps float64
DBSCANMinSamples int
SyncMaxFiles int
JobMaxFiles int
QueueMaxPending int
WorkerConcurrency int
MaxPDFSizeMB int
PDFProcessTimeout time.Duration
JobResultTTL time.Duration
StreamName string
SubjectPrefix string
Port string
ModelPath string
InferencePath string
OCRDPI int
OCRLang string
OCRPoolSize int
LogPipelineTiming bool
NATSFetchMaxWait time.Duration
JobInlineProcessing bool
DBSCANEps float64
DBSCANMinSamples int
SyncMaxFiles int
MaxPDFSizeMB int
PDFProcessTimeout time.Duration
SyncProcessConcurrency int
LogPipelineTiming bool
}
func Load() Config {
return Config{
Port: getenv("PORT", "8080"),
RedisURL: getenv("REDIS_URL", "redis://redis:6379/0"),
NATSURL: getenv("NATS_URL", "nats://nats:4222"),
MinIOEndpoint: getenv("MINIO_ENDPOINT", "minio:9000"),
MinIOAccessKey: getenv("MINIO_ACCESS_KEY", "minioadmin"),
MinIOSecretKey: getenv("MINIO_SECRET_KEY", "minioadmin"),
MinIOBucket: getenv("MINIO_BUCKET", "pdf-count"),
MinIOUseSSL: getenvBool("MINIO_USE_SSL", false),
ModelPath: getenv("MODEL_PATH", "/app/models/model.onnx"),
InferencePath: getenv("INFERENCE_CONFIG", "/app/models/inference.json"),
OCRDPI: getenvInt("OCR_DPI", 300),
OCRLang: getenv("OCR_LANG", "spa"),
DBSCANEps: getenvFloat("DBSCAN_EPS", 10),
DBSCANMinSamples: getenvInt("DBSCAN_MIN_SAMPLES", 1),
SyncMaxFiles: getenvInt("SYNC_MAX_FILES", 50),
JobMaxFiles: getenvInt("JOB_MAX_FILES", 50000),
QueueMaxPending: getenvInt("QUEUE_MAX_PENDING", 100000),
WorkerConcurrency: getenvInt("WORKER_CONCURRENCY", 4),
MaxPDFSizeMB: getenvInt("MAX_PDF_SIZE_MB", 20),
PDFProcessTimeout: time.Duration(getenvInt("PDF_PROCESS_TIMEOUT_SEC", 120)) * time.Second,
JobResultTTL: time.Duration(getenvInt("JOB_RESULT_TTL_HOURS", 72)) * time.Hour,
StreamName: getenv("NATS_STREAM", "PDF_JOBS"),
SubjectPrefix: getenv("NATS_SUBJECT_PREFIX", "pdf.process"),
Port: getenv("PORT", "8080"),
ModelPath: getenv("MODEL_PATH", "/app/models/model.onnx"),
InferencePath: getenv("INFERENCE_CONFIG", "/app/models/inference.json"),
OCRDPI: getenvInt("OCR_DPI", 300),
OCRLang: getenv("OCR_LANG", "spa"),
OCRPoolSize: getenvInt("OCR_POOL_SIZE", 0),
LogPipelineTiming: getenvBool("LOG_PIPELINE_TIMING", false),
NATSFetchMaxWait: time.Duration(getenvInt("NATS_FETCH_MAX_WAIT_MS", 300)) * time.Millisecond,
JobInlineProcessing: getenvBool("JOB_INLINE_PROCESSING", false),
DBSCANEps: getenvFloat("DBSCAN_EPS", 10),
DBSCANMinSamples: getenvInt("DBSCAN_MIN_SAMPLES", 1),
SyncMaxFiles: getenvInt("SYNC_MAX_FILES", 50),
MaxPDFSizeMB: getenvInt("MAX_PDF_SIZE_MB", 20),
PDFProcessTimeout: time.Duration(getenvInt("PDF_PROCESS_TIMEOUT_SEC", 120)) * time.Second,
SyncProcessConcurrency: getenvInt("SYNC_PROCESS_CONCURRENCY", 0),
LogPipelineTiming: getenvBool("LOG_PIPELINE_TIMING", false),
}
}
@@ -1,13 +1,13 @@
package domain
// ServiceRow is one detected invoice line.
// ServiceRow represents a single detected invoice line in a PDF page.
type ServiceRow struct {
Text string `json:"text"`
Probability float64 `json:"probability"`
IsService *bool `json:"is_service,omitempty"`
}
// PDFResult is the processing outcome for a single PDF.
// PDFResult is the counting outcome for a single PDF document.
type PDFResult struct {
Filename string `json:"filename"`
ServiceCount int `json:"service_count"`
@@ -16,44 +16,14 @@ type PDFResult struct {
Error string `json:"error,omitempty"`
}
// CountResponse wraps sync batch results.
// CountResponse wraps the results of a synchronous batch request to POST /v1/count.
type CountResponse struct {
Results []PDFResult `json:"results"`
Processed int `json:"processed"`
Failed int `json:"failed"`
}
// JobStatus tracks async job progress.
type JobStatus struct {
JobID string `json:"job_id"`
Status string `json:"status"`
Total int `json:"total"`
Processed int `json:"processed"`
Failed int `json:"failed"`
}
// JobResultsPage is a paginated slice of PDF results.
type JobResultsPage struct {
JobID string `json:"job_id"`
Results []PDFResult `json:"results"`
NextCursor string `json:"next_cursor,omitempty"`
HasMore bool `json:"has_more"`
}
// QueueMessage is published per PDF for workers.
type QueueMessage struct {
JobID string `json:"job_id"`
ObjectKey string `json:"object_key"`
Filename string `json:"filename"`
IncludeRejected bool `json:"include_rejected"`
}
const (
StatusOK = "ok"
StatusError = "error"
JobQueued = "queued"
JobRunning = "running"
JobCompleted = "completed"
JobCancelled = "cancelled"
)
@@ -73,6 +73,21 @@ func (p *Processor) ProcessPDF(filename string, pdfData []byte) domain.PDFResult
return result
}
// Sin modelo ML: contar todas las filas agrupadas como servicios (heurística).
if p.predictor == nil {
services := make([]domain.ServiceRow, 0, len(rows))
for _, row := range rows {
services = append(services, domain.ServiceRow{
Text: row.TextoFila,
Probability: 1.0,
})
}
result.Services = services
result.ServiceCount = len(rows)
p.logTiming(filename, renderDur, ocrDur, clusterDur, 0, time.Since(start))
return result
}
features := make([][8]float32, len(rows))
for i, r := range rows {
features[i] = r.Features
@@ -1,109 +0,0 @@
package queue
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/pys/pdf-count/internal/domain"
)
type Publisher struct {
js jetstream.JetStream
stream string
subject string
}
func NewPublisher(natsURL, streamName, subjectPrefix string) (*Publisher, error) {
nc, err := nats.Connect(natsURL)
if err != nil {
return nil, fmt.Errorf("nats connect: %w", err)
}
js, err := jetstream.New(nc)
if err != nil {
nc.Close()
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, err = js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{
Name: streamName,
Subjects: []string{subjectPrefix + ".>"},
Retention: jetstream.WorkQueuePolicy,
MaxMsgs: 200000,
})
if err != nil {
nc.Close()
return nil, fmt.Errorf("create stream: %w", err)
}
return &Publisher{js: js, stream: streamName, subject: subjectPrefix}, nil
}
func (p *Publisher) Publish(ctx context.Context, msg domain.QueueMessage) error {
data, err := json.Marshal(msg)
if err != nil {
return err
}
subject := fmt.Sprintf("%s.%s", p.subject, msg.JobID)
_, err = p.js.Publish(ctx, subject, data)
return err
}
type Consumer struct {
js jetstream.JetStream
subject string
consumerName string
}
func NewConsumer(natsURL, streamName, subjectPrefix, consumerName string) (*Consumer, jetstream.Consumer, *nats.Conn, error) {
nc, err := nats.Connect(natsURL)
if err != nil {
return nil, nil, nil, err
}
js, err := jetstream.New(nc)
if err != nil {
nc.Close()
return nil, nil, nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, err = js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{
Name: streamName,
Subjects: []string{subjectPrefix + ".>"},
Retention: jetstream.WorkQueuePolicy,
MaxMsgs: 200000,
})
if err != nil {
nc.Close()
return nil, nil, nil, err
}
cons, err := js.CreateOrUpdateConsumer(ctx, streamName, jetstream.ConsumerConfig{
Name: consumerName,
FilterSubject: subjectPrefix + ".>",
AckPolicy: jetstream.AckExplicitPolicy,
MaxDeliver: 3,
AckWait: 2 * time.Minute,
MaxAckPending: 256,
})
if err != nil {
nc.Close()
return nil, nil, nil, err
}
return &Consumer{js: js, subject: subjectPrefix, consumerName: consumerName}, cons, nc, nil
}
func DecodeMessage(data []byte) (domain.QueueMessage, error) {
var msg domain.QueueMessage
err := json.Unmarshal(data, &msg)
return msg, err
}
@@ -1,86 +0,0 @@
package storage
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
type Client struct {
client *minio.Client
bucket string
}
func New(endpoint, accessKey, secretKey, bucket string, useSSL bool) (*Client, error) {
client, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
Secure: useSSL,
})
if err != nil {
return nil, err
}
return &Client{client: client, bucket: bucket}, nil
}
func (c *Client) EnsureBucket(ctx context.Context) error {
exists, err := c.client.BucketExists(ctx, c.bucket)
if err != nil {
return err
}
if exists {
return nil
}
err = c.client.MakeBucket(ctx, c.bucket, minio.MakeBucketOptions{})
if err == nil {
return nil
}
resp := minio.ToErrorResponse(err)
if resp.Code == "BucketAlreadyOwnedByYou" || resp.Code == "BucketAlreadyExists" {
return nil
}
return err
}
func (c *Client) Put(ctx context.Context, objectKey string, data []byte, contentType string) error {
if contentType == "" {
contentType = "application/pdf"
}
_, err := c.client.PutObject(ctx, c.bucket, objectKey, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{
ContentType: contentType,
})
return err
}
func (c *Client) Get(ctx context.Context, objectKey string) ([]byte, error) {
obj, err := c.client.GetObject(ctx, c.bucket, objectKey, minio.GetObjectOptions{})
if err != nil {
return nil, err
}
defer obj.Close()
return io.ReadAll(obj)
}
func (c *Client) ListByPrefix(ctx context.Context, prefix string) ([]string, error) {
var keys []string
for obj := range c.client.ListObjects(ctx, c.bucket, minio.ListObjectsOptions{
Prefix: prefix,
Recursive: true,
}) {
if obj.Err != nil {
return nil, obj.Err
}
if strings.HasSuffix(strings.ToLower(obj.Key), ".pdf") {
keys = append(keys, obj.Key)
}
}
return keys, nil
}
func JobObjectKey(jobID, filename string) string {
return fmt.Sprintf("jobs/%s/%s", jobID, filename)
}
@@ -1,276 +0,0 @@
package store
import (
"context"
"encoding/json"
"fmt"
"strconv"
"time"
"github.com/pys/pdf-count/internal/domain"
"github.com/redis/go-redis/v9"
)
type Store struct {
client *redis.Client
ttl time.Duration
}
func New(redisURL string, ttl time.Duration) (*Store, error) {
opts, err := redis.ParseURL(redisURL)
if err != nil {
return nil, err
}
client := redis.NewClient(opts)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := client.Ping(ctx).Err(); err != nil {
return nil, fmt.Errorf("redis ping: %w", err)
}
return &Store{client: client, ttl: ttl}, nil
}
func (s *Store) Close() error {
return s.client.Close()
}
func metaKey(jobID string) string { return fmt.Sprintf("job:%s:meta", jobID) }
func filesKey(jobID string) string { return fmt.Sprintf("job:%s:files", jobID) }
func resultsKey(jobID string) string {
return fmt.Sprintf("job:%s:results", jobID)
}
func (s *Store) CreateJob(ctx context.Context, jobID string, total int, includeRejected bool) error {
pipe := s.client.Pipeline()
pipe.HSet(ctx, metaKey(jobID), map[string]interface{}{
"status": domain.JobQueued,
"total": total,
"processed": 0,
"failed": 0,
"include_rejected": boolStr(includeRejected),
})
pipe.Expire(ctx, metaKey(jobID), s.ttl)
_, err := pipe.Exec(ctx)
return err
}
func (s *Store) AddJobFiles(ctx context.Context, jobID string, filenames []string) error {
if len(filenames) == 0 {
return nil
}
members := make([]interface{}, len(filenames))
for i, f := range filenames {
members[i] = f
}
pipe := s.client.Pipeline()
pipe.RPush(ctx, filesKey(jobID), members...)
pipe.Expire(ctx, filesKey(jobID), s.ttl)
_, err := pipe.Exec(ctx)
return err
}
func (s *Store) MarkRunning(ctx context.Context, jobID string) error {
return s.client.HSet(ctx, metaKey(jobID), "status", domain.JobRunning).Err()
}
var saveResultScript = redis.NewScript(`
local meta = KEYS[1]
local results = KEYS[2]
redis.call('HSET', results, ARGV[1], ARGV[2])
local processed = redis.call('HINCRBY', meta, 'processed', 1)
if ARGV[3] == '1' then
redis.call('HINCRBY', meta, 'failed', 1)
end
redis.call('EXPIRE', results, ARGV[4])
local total = tonumber(redis.call('HGET', meta, 'total') or '0')
local status = redis.call('HGET', meta, 'status')
if status ~= 'cancelled' and status ~= 'completed' and total > 0 and processed >= total then
redis.call('HSET', meta, 'status', 'completed')
end
return processed
`)
func (s *Store) SaveResult(ctx context.Context, jobID string, result domain.PDFResult) error {
data, err := json.Marshal(result)
if err != nil {
return err
}
failFlag := "0"
if result.Status == domain.StatusError {
failFlag = "1"
}
ttlSec := int64(s.ttl.Seconds())
if ttlSec <= 0 {
ttlSec = 72 * 3600
}
_, err = saveResultScript.Run(ctx, s.client,
[]string{metaKey(jobID), resultsKey(jobID)},
result.Filename, string(data), failFlag, ttlSec,
).Result()
return err
}
func (s *Store) TryCompleteJob(ctx context.Context, jobID string) error {
meta, err := s.client.HGetAll(ctx, metaKey(jobID)).Result()
if err != nil {
return err
}
_, err = s.maybeCompleteJob(ctx, jobID, meta)
return err
}
func (s *Store) maybeCompleteJob(ctx context.Context, jobID string, meta map[string]string) (string, error) {
status := meta["status"]
if status == domain.JobCancelled || status == domain.JobCompleted {
return status, nil
}
total, _ := strconv.Atoi(meta["total"])
processed, _ := strconv.Atoi(meta["processed"])
if processed >= total && total > 0 {
if err := s.client.HSet(ctx, metaKey(jobID), "status", domain.JobCompleted).Err(); err != nil {
return status, err
}
return domain.JobCompleted, nil
}
return status, nil
}
func (s *Store) GetJob(ctx context.Context, jobID string) (domain.JobStatus, error) {
meta, err := s.client.HGetAll(ctx, metaKey(jobID)).Result()
if err != nil {
return domain.JobStatus{}, err
}
if len(meta) == 0 {
return domain.JobStatus{}, fmt.Errorf("job not found")
}
status, err := s.maybeCompleteJob(ctx, jobID, meta)
if err != nil {
return domain.JobStatus{}, err
}
if status == domain.JobCompleted {
meta["status"] = domain.JobCompleted
}
total, _ := strconv.Atoi(meta["total"])
processed, _ := strconv.Atoi(meta["processed"])
failed, _ := strconv.Atoi(meta["failed"])
return domain.JobStatus{
JobID: jobID,
Status: status,
Total: total,
Processed: processed,
Failed: failed,
}, nil
}
func (s *Store) GetResultsPage(ctx context.Context, jobID, cursor string, limit int) (domain.JobResultsPage, error) {
if limit <= 0 || limit > 1000 {
limit = 500
}
allFiles, err := s.client.LRange(ctx, filesKey(jobID), 0, -1).Result()
if err != nil {
return domain.JobResultsPage{}, err
}
start := 0
if cursor != "" {
for i, f := range allFiles {
if f == cursor {
start = i + 1
break
}
}
}
end := start + limit
if end > len(allFiles) {
end = len(allFiles)
}
pageFiles := allFiles[start:end]
results := make([]domain.PDFResult, 0, len(pageFiles))
if len(pageFiles) > 0 {
raw, err := s.client.HMGet(ctx, resultsKey(jobID), pageFiles...).Result()
if err != nil {
return domain.JobResultsPage{}, err
}
for i, v := range raw {
if v == nil {
results = append(results, domain.PDFResult{
Filename: pageFiles[i],
Status: "pending",
})
continue
}
var r domain.PDFResult
if err := json.Unmarshal([]byte(v.(string)), &r); err != nil {
continue
}
results = append(results, r)
}
}
page := domain.JobResultsPage{
JobID: jobID,
Results: results,
HasMore: end < len(allFiles),
}
if page.HasMore && len(pageFiles) > 0 {
page.NextCursor = pageFiles[len(pageFiles)-1]
}
return page, nil
}
func (s *Store) CancelJob(ctx context.Context, jobID string) error {
return s.client.HSet(ctx, metaKey(jobID), "status", domain.JobCancelled).Err()
}
func (s *Store) IsCancelled(ctx context.Context, jobID string) (bool, error) {
status, err := s.client.HGet(ctx, metaKey(jobID), "status").Result()
if err != nil {
return false, err
}
return status == domain.JobCancelled, nil
}
func (s *Store) PendingCount(ctx context.Context) (int64, error) {
// Approximate: sum of (total - processed) for running jobs
var cursor uint64
var pending int64
for {
keys, next, err := s.client.Scan(ctx, cursor, "job:*:meta", 100).Result()
if err != nil {
return pending, err
}
for _, key := range keys {
meta, err := s.client.HGetAll(ctx, key).Result()
if err != nil {
continue
}
if meta["status"] != domain.JobQueued && meta["status"] != domain.JobRunning {
continue
}
total, _ := strconv.Atoi(meta["total"])
processed, _ := strconv.Atoi(meta["processed"])
pending += int64(total - processed)
}
cursor = next
if cursor == 0 {
break
}
}
return pending, nil
}
func (s *Store) IncrementTotal(ctx context.Context, jobID string, delta int) error {
pipe := s.client.Pipeline()
pipe.HIncrBy(ctx, metaKey(jobID), "total", int64(delta))
pipe.Expire(ctx, metaKey(jobID), s.ttl)
_, err := pipe.Exec(ctx)
return err
}
func boolStr(v bool) string {
if v {
return "true"
}
return "false"
}
@@ -1,212 +0,0 @@
package worker
import (
"context"
"log"
"os/signal"
"sync"
"syscall"
"time"
"github.com/nats-io/nats.go/jetstream"
"github.com/pys/pdf-count/internal/config"
"github.com/pys/pdf-count/internal/domain"
"github.com/pys/pdf-count/internal/model"
"github.com/pys/pdf-count/internal/ocr"
"github.com/pys/pdf-count/internal/pipeline"
"github.com/pys/pdf-count/internal/queue"
"github.com/pys/pdf-count/internal/storage"
"github.com/pys/pdf-count/internal/store"
)
type Runner struct {
cfg config.Config
predictor *model.Predictor
ocrEngine *ocr.Engine
store *store.Store
storage *storage.Client
consumer jetstream.Consumer
ncClose func()
}
func NewRunner(cfg config.Config) (*Runner, error) {
predictor, err := model.NewPredictor(cfg.ModelPath, cfg.InferencePath)
if err != nil {
return nil, err
}
poolSize := cfg.OCRPoolSize
if poolSize <= 0 {
poolSize = cfg.WorkerConcurrency
}
ocrEngine, err := ocr.NewEngine(cfg.OCRLang, poolSize)
if err != nil {
predictor.Close()
return nil, err
}
st, err := store.New(cfg.RedisURL, cfg.JobResultTTL)
if err != nil {
ocrEngine.Close()
predictor.Close()
return nil, err
}
stg, err := storage.New(cfg.MinIOEndpoint, cfg.MinIOAccessKey, cfg.MinIOSecretKey, cfg.MinIOBucket, cfg.MinIOUseSSL)
if err != nil {
ocrEngine.Close()
st.Close()
predictor.Close()
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := stg.EnsureBucket(ctx); err != nil {
ocrEngine.Close()
st.Close()
predictor.Close()
return nil, err
}
_, cons, nc, err := queue.NewConsumer(cfg.NATSURL, cfg.StreamName, cfg.SubjectPrefix, "pdf-workers")
if err != nil {
ocrEngine.Close()
st.Close()
predictor.Close()
return nil, err
}
return &Runner{
cfg: cfg,
predictor: predictor,
ocrEngine: ocrEngine,
store: st,
storage: stg,
consumer: cons,
ncClose: func() { nc.Close() },
}, nil
}
func (r *Runner) Close() {
r.ncClose()
_ = r.ocrEngine.Close()
_ = r.store.Close()
_ = r.predictor.Close()
}
func (r *Runner) processSlots() int {
c := r.cfg.WorkerConcurrency
pool := r.cfg.OCRPoolSize
if pool <= 0 {
pool = r.cfg.WorkerConcurrency
}
if pool <= 0 {
pool = 4
}
if c > pool {
c = pool
}
if c <= 0 {
c = 4
}
return c
}
func (r *Runner) Run(ctx context.Context) error {
slots := r.processSlots()
sem := make(chan struct{}, slots)
var wg sync.WaitGroup
go func() {
<-ctx.Done()
wg.Wait()
}()
for {
select {
case <-ctx.Done():
return nil
default:
}
msgs, err := r.consumer.Fetch(slots, jetstream.FetchMaxWait(r.cfg.NATSFetchMaxWait))
if err != nil {
continue
}
for msg := range msgs.Messages() {
sem <- struct{}{}
wg.Add(1)
m := msg
go func() {
defer func() {
<-sem
wg.Done()
}()
r.handleMessage(ctx, m)
}()
}
_ = msgs.Error()
}
}
func (r *Runner) handleMessage(ctx context.Context, msg jetstream.Msg) {
processCtx, cancel := context.WithTimeout(ctx, r.cfg.PDFProcessTimeout)
defer cancel()
qmsg, err := queue.DecodeMessage(msg.Data())
if err != nil {
_ = msg.Term()
return
}
cancelled, err := r.store.IsCancelled(processCtx, qmsg.JobID)
if err != nil || cancelled {
_ = msg.Ack()
return
}
data, err := r.storage.Get(processCtx, qmsg.ObjectKey)
if err != nil {
_ = r.store.SaveResult(processCtx, qmsg.JobID, domain.PDFResult{
Filename: qmsg.Filename,
Status: domain.StatusError,
Error: err.Error(),
})
_ = r.store.TryCompleteJob(processCtx, qmsg.JobID)
_ = msg.Ack()
return
}
proc := pipeline.NewProcessor(r.predictor, r.ocrEngine, pipeline.Options{
DPI: r.cfg.OCRDPI,
Lang: r.cfg.OCRLang,
DBSCANEps: r.cfg.DBSCANEps,
DBSCANMinSamples: r.cfg.DBSCANMinSamples,
IncludeRejected: qmsg.IncludeRejected,
LogPipelineTiming: r.cfg.LogPipelineTiming,
})
result := proc.ProcessPDF(qmsg.Filename, data)
data = nil
if err := r.store.SaveResult(processCtx, qmsg.JobID, result); err != nil {
_ = msg.NakWithDelay(5 * time.Second)
return
}
_ = r.store.TryCompleteJob(processCtx, qmsg.JobID)
_ = msg.Ack()
}
func RunMain() {
cfg := config.Load()
runner, err := NewRunner(cfg)
if err != nil {
log.Fatalf("worker init: %v", err)
}
defer runner.Close()
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
log.Printf("worker started slots=%d", runner.processSlots())
if err := runner.Run(ctx); err != nil {
log.Fatalf("worker: %v", err)
}
log.Println("worker stopped")
}