From d95f8f6ad8aecf13e36eedc569f084e2a11995f0 Mon Sep 17 00:00:00 2001 From: erik Date: Sun, 5 Jul 2026 22:39:07 -0500 Subject: [PATCH] feat: Add .env.example configuration file and integrate Redis service into Docker Compose for improved task management. Update desktop application to support document indexing and extraction workflows, enhancing user experience with a new extraction wizard and stepper components. Revise README and documentation for clarity on setup and usage. --- .env.example | 36 + README.md | 43 +- docker-compose.yml | 68 +- docs/database.dbdiagram | 14 + docs/database.dbml | 140 ++ services/desktop_app/README.md | 31 +- services/desktop_app/electron/folderIndex.cjs | 125 ++ services/desktop_app/electron/main.cjs | 15 + services/desktop_app/electron/preload.cjs | 3 + .../desktop_app/src/renderer/App.test.tsx | 98 +- services/desktop_app/src/renderer/App.tsx | 779 ++++--- .../renderer/components/ExtractionStepper.tsx | 45 + .../renderer/components/ExtractionWizard.tsx | 631 ++++++ .../renderer/components/SampleTreeView.tsx | 163 ++ .../renderer/components/SchemaFieldPicker.tsx | 146 ++ .../src/renderer/lib/apiClient.test.ts | 22 + .../desktop_app/src/renderer/lib/apiClient.ts | 38 +- .../src/renderer/lib/payloads.test.ts | 61 +- .../desktop_app/src/renderer/lib/payloads.ts | 156 +- .../src/renderer/lib/schemaDisplay.test.ts | 40 + .../src/renderer/lib/schemaDisplay.ts | 137 ++ services/desktop_app/src/renderer/styles.css | 1976 ++++++++++++++++- .../desktop_app/src/renderer/test/setup.ts | 4 + services/desktop_app/src/shared/types.ts | 22 + services/extraction/Dockerfile | 1 + services/extraction/README.md | 6 +- services/extraction/api/desktop_ingest.py | 235 ++ services/extraction/api/main.py | 525 ++++- services/extraction/domain/config.py | 29 +- services/extraction/domain/estimator.py | 54 + services/extraction/domain/rate_limiter.py | 156 ++ services/extraction/infrastructure/uploads.py | 36 + services/extraction/requirements.txt | 4 + services/extraction/worker/__init__.py | 1 + services/extraction/worker/main.py | 18 + services/extraction/worker/settings.py | 84 + services/extraction/worker/tasks.py | 236 ++ .../.gitignore | 15 + .../Dockerfile | 56 + .../Dockerfile.autoscaler | 14 + .../README.md | 339 +++ .../api/openapi.yaml | 230 ++ .../cmd/api/main.go | 74 + .../cmd/autoscaler/main.go | 157 ++ .../cmd/worker/main.go | 7 + .../docker-compose.yml | 125 ++ .../go.mod | 40 + .../go.sum | 69 + .../internal/api/jobrunner.go | 53 + .../internal/api/server.go | 591 +++++ .../internal/config/config.go | 105 + .../internal/domain/types.go | 59 + .../internal/model/onnx.go | 110 + .../internal/ocr/ocr.go | 99 + .../internal/ocr/pool.go | 69 + .../internal/pipeline/clustering.go | 112 + .../internal/pipeline/features.go | 122 + .../internal/pipeline/processor.go | 128 ++ .../internal/queue/nats.go | 109 + .../internal/storage/minio.go | 86 + .../internal/store/redis.go | 276 +++ .../internal/worker/runner.go | 212 ++ .../models/latest/.gitkeep | 0 .../requirements-export.txt | 4 + .../scripts/compare_parity.py | 59 + .../scripts/export_onnx.py | 99 + .../scripts/parity_baseline.py | 49 + .../web/app.js | 347 +++ .../web/embed.go | 6 + .../web/index.html | 307 +++ 70 files changed, 9807 insertions(+), 499 deletions(-) create mode 100644 .env.example create mode 100644 docs/database.dbdiagram create mode 100644 docs/database.dbml create mode 100644 services/desktop_app/electron/folderIndex.cjs create mode 100644 services/desktop_app/src/renderer/components/ExtractionStepper.tsx create mode 100644 services/desktop_app/src/renderer/components/ExtractionWizard.tsx create mode 100644 services/desktop_app/src/renderer/components/SampleTreeView.tsx create mode 100644 services/desktop_app/src/renderer/components/SchemaFieldPicker.tsx create mode 100644 services/desktop_app/src/renderer/lib/apiClient.test.ts create mode 100644 services/desktop_app/src/renderer/lib/schemaDisplay.test.ts create mode 100644 services/desktop_app/src/renderer/lib/schemaDisplay.ts create mode 100644 services/extraction/api/desktop_ingest.py create mode 100644 services/extraction/domain/rate_limiter.py create mode 100644 services/extraction/infrastructure/uploads.py create mode 100644 services/extraction/worker/__init__.py create mode 100644 services/extraction/worker/main.py create mode 100644 services/extraction/worker/settings.py create mode 100644 services/extraction/worker/tasks.py create mode 100644 services/services-count-facturas-medicas-serving/.gitignore create mode 100644 services/services-count-facturas-medicas-serving/Dockerfile create mode 100644 services/services-count-facturas-medicas-serving/Dockerfile.autoscaler create mode 100644 services/services-count-facturas-medicas-serving/README.md create mode 100644 services/services-count-facturas-medicas-serving/api/openapi.yaml create mode 100644 services/services-count-facturas-medicas-serving/cmd/api/main.go create mode 100644 services/services-count-facturas-medicas-serving/cmd/autoscaler/main.go create mode 100644 services/services-count-facturas-medicas-serving/cmd/worker/main.go create mode 100644 services/services-count-facturas-medicas-serving/docker-compose.yml create mode 100644 services/services-count-facturas-medicas-serving/go.mod create mode 100644 services/services-count-facturas-medicas-serving/go.sum create mode 100644 services/services-count-facturas-medicas-serving/internal/api/jobrunner.go create mode 100644 services/services-count-facturas-medicas-serving/internal/api/server.go create mode 100644 services/services-count-facturas-medicas-serving/internal/config/config.go create mode 100644 services/services-count-facturas-medicas-serving/internal/domain/types.go create mode 100644 services/services-count-facturas-medicas-serving/internal/model/onnx.go create mode 100644 services/services-count-facturas-medicas-serving/internal/ocr/ocr.go create mode 100644 services/services-count-facturas-medicas-serving/internal/ocr/pool.go create mode 100644 services/services-count-facturas-medicas-serving/internal/pipeline/clustering.go create mode 100644 services/services-count-facturas-medicas-serving/internal/pipeline/features.go create mode 100644 services/services-count-facturas-medicas-serving/internal/pipeline/processor.go create mode 100644 services/services-count-facturas-medicas-serving/internal/queue/nats.go create mode 100644 services/services-count-facturas-medicas-serving/internal/storage/minio.go create mode 100644 services/services-count-facturas-medicas-serving/internal/store/redis.go create mode 100644 services/services-count-facturas-medicas-serving/internal/worker/runner.go create mode 100644 services/services-count-facturas-medicas-serving/models/latest/.gitkeep create mode 100644 services/services-count-facturas-medicas-serving/requirements-export.txt create mode 100644 services/services-count-facturas-medicas-serving/scripts/compare_parity.py create mode 100644 services/services-count-facturas-medicas-serving/scripts/export_onnx.py create mode 100644 services/services-count-facturas-medicas-serving/scripts/parity_baseline.py create mode 100644 services/services-count-facturas-medicas-serving/web/app.js create mode 100644 services/services-count-facturas-medicas-serving/web/embed.go create mode 100644 services/services-count-facturas-medicas-serving/web/index.html diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..acf84aa --- /dev/null +++ b/.env.example @@ -0,0 +1,36 @@ +# Copia este archivo a `.env` en la raiz del repositorio. + +# ── OpenAI ──────────────────────────────────────────────── +# API key usada para extracción y normalización. +# El modelo del extractor lo elige el usuario en la app (ej. gpt-4o, gpt-5.4). +# El normalizador siempre usa gpt-4o-mini (fijado en docker-compose). +EXTRACTOR_API_KEY=sk-... +EXTRACTOR_MODEL=gpt-4o +EXTRACTOR_TIMEOUT_S=120 + +# ── Procesamiento de imagenes ───────────────────────────── +# DPI mas bajo = mas rapido; 150 es suficiente para la mayoria de documentos. +PDF_DPI=150 +PDF_THREAD_COUNT=4 + +# ── Concurrencia ────────────────────────────────────────── +# Cuantas llamadas LLM simultáneas por worker ARQ +WORKER_CONCURRENCY=25 +# Cuántas réplicas del worker levantar +WORKER_REPLICAS=2 + +# ── Rate limiting OpenAI ────────────────────────────────── +# Ajusta según tu tier de OpenAI (estos son los valores tier-1 de gpt-4o) +OPENAI_RPM_LIMIT=500 +OPENAI_TPM_LIMIT=800000 + +# ── Cola de tareas (Redis + ARQ) ───────────────────────── +# ASYNC_QUEUE=false (default) → procesa documentos sincronamente, sin Redis. +# ASYNC_QUEUE=true → encola en Redis/ARQ para procesamiento distribuido +# (requiere el servicio `worker` y `redis` corriendo via docker-compose). +ASYNC_QUEUE=false +# Solo necesario cuando ASYNC_QUEUE=true. Se levanta via docker-compose. +REDIS_URL=redis://redis:6379/0 + +# ── Misc ────────────────────────────────────────────────── +ALLOWED_DOCUMENT_TYPES=CC,TI,CE,RC,PA,PPT,PEP,EPT,NUIP diff --git a/README.md b/README.md index fe0f9a1..fce1a94 100644 --- a/README.md +++ b/README.md @@ -222,10 +222,45 @@ npm install npm run dev ``` -La app usa `http://localhost:8000` por defecto y traduce rutas locales dentro de -`IPS_DOCUMENT_ROOT` hacia `/data` para llamar al API. La pantalla principal solo -pide seleccionar la carpeta a procesar; los resultados se guardan en base de -datos y la exportacion CSV es opcional. +La app desktop usa `http://localhost:8000` por defecto. **No monta la carpeta +en Docker**: indexa los documentos localmente y los sube uno a uno al API +(`POST /extraction/schemas/upload` y `POST /tasks/{id}/ingest`). Solo se +persisten las filas extraidas en Postgres; los PDFs permanecen en el equipo del +usuario. + +```bash +cd services/desktop_app +npm install +npm run dev +``` + +#### Flujo desktop (sin volumen `/data`) + +1. Seleccionar carpeta local en la app. +2. Descubrir campos (sube un PDF de muestra por tipo). +3. Ejecutar extraccion (crea tarea `source=desktop` e ingesta cada archivo). +4. Consultar tareas y exportar CSV opcional. + +#### Modo legacy API / curl + +Si llamas al API con `root_path` y rutas bajo `/data`, monta documentos al +levantar Docker: + +```bash +IPS_DOCUMENT_ROOT=C:/ruta/a/documentos docker compose up --build +``` + +### Error: No se encontraron documentos en /data + +Ese mensaje aparece en clientes legacy que envian `root_path=/data` sin montar +documentos. La app desktop no deberia usar ese modo. Verifica: + +```bash +docker exec ipsextractor-extraction ls -la /data +``` + +Para la app desktop, asegurate de usar Electron (`npm run dev`) y de que el +gateway responda en `http://localhost:8000`. ### Contrato De Base De Datos diff --git a/docker-compose.yml b/docker-compose.yml index 58e8623..f965d6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,18 @@ services: timeout: 5s retries: 10 + redis: + image: redis:7-alpine + container_name: ipsextractor-redis + command: redis-server --maxmemory 512mb --maxmemory-policy allkeys-lru --save "" + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + extraction: build: context: ./services/extraction @@ -24,26 +36,57 @@ services: depends_on: database: condition: service_healthy + redis: + condition: service_healthy environment: DATABASE_URL: postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor - EXTRACTOR_PROVIDER: ${EXTRACTOR_PROVIDER:-} - EXTRACTOR_BASE_URL: ${EXTRACTOR_BASE_URL:-} - EXTRACTOR_API_KEY: ${EXTRACTOR_API_KEY:-} - EXTRACTOR_MODEL: ${EXTRACTOR_MODEL:-} - NORMALIZER_PROVIDER: ${NORMALIZER_PROVIDER:-} - NORMALIZER_BASE_URL: ${NORMALIZER_BASE_URL:-} - NORMALIZER_API_KEY: ${NORMALIZER_API_KEY:-} - NORMALIZER_MODEL: ${NORMALIZER_MODEL:-} - NORMALIZER_TIMEOUT_S: ${NORMALIZER_TIMEOUT_S:-0} + REDIS_URL: redis://redis:6379/0 + EXTRACTOR_BASE_URL: https://api.openai.com/v1 + EXTRACTOR_API_KEY: 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 + EXTRACTOR_MODEL: ${EXTRACTOR_MODEL:-gpt-5.4-mini} + NORMALIZER_BASE_URL: https://api.openai.com/v1 + NORMALIZER_API_KEY: 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 + NORMALIZER_MODEL: gpt-5.4-mini EXTRACTOR_TIMEOUT_S: ${EXTRACTOR_TIMEOUT_S:-120} - PDF_DPI: ${PDF_DPI:-600} - PDF_THREAD_COUNT: ${PDF_THREAD_COUNT:-10} - CONCURRENCY_LIMIT: ${CONCURRENCY_LIMIT:-4} + PDF_DPI: ${PDF_DPI:-150} + PDF_THREAD_COUNT: ${PDF_THREAD_COUNT:-4} + CONCURRENCY_LIMIT: ${CONCURRENCY_LIMIT:-50} ALLOWED_DOCUMENT_TYPES: ${ALLOWED_DOCUMENT_TYPES:-CC,TI,CE,RC,PA,PPT,PEP,EPT,NUIP} ports: - "8001:8001" volumes: - ${IPS_DOCUMENT_ROOT:-./data}:/data + - ips_queue:/tmp/ips_queue + + # Escala horizontalmente: docker compose up --scale worker=4 + worker: + build: + context: ./services/extraction + dockerfile: Dockerfile + command: python -m worker.main + depends_on: + database: + condition: service_healthy + redis: + condition: service_healthy + deploy: + replicas: ${WORKER_REPLICAS:-2} + environment: + DATABASE_URL: postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor + REDIS_URL: redis://redis:6379/0 + EXTRACTOR_BASE_URL: https://api.openai.com/v1 + EXTRACTOR_API_KEY: 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 + EXTRACTOR_MODEL: ${EXTRACTOR_MODEL:-gpt-5.4-mini} + NORMALIZER_BASE_URL: https://api.openai.com/v1 + NORMALIZER_API_KEY: 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 + NORMALIZER_MODEL: gpt-5.4-mini + EXTRACTOR_TIMEOUT_S: ${EXTRACTOR_TIMEOUT_S:-120} + PDF_DPI: ${PDF_DPI:-150} + PDF_THREAD_COUNT: ${PDF_THREAD_COUNT:-4} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-25} + ALLOWED_DOCUMENT_TYPES: ${ALLOWED_DOCUMENT_TYPES:-CC,TI,CE,RC,PA,PPT,PEP,EPT,NUIP} + volumes: + - ips_queue:/tmp/ips_queue cleaning: build: @@ -74,3 +117,4 @@ services: volumes: ipsextractor_db: + ips_queue: diff --git a/docs/database.dbdiagram b/docs/database.dbdiagram new file mode 100644 index 0000000..b23a930 --- /dev/null +++ b/docs/database.dbdiagram @@ -0,0 +1,14 @@ +{ + "version": "2.0.0", + "darkMode": false, + "gridEnabling": false, + "currentViewName": null, + "defaultView": { + "detailLevel": "All", + "tablePositions": [], + "tableGroupCollapseStates": [], + "stickyNoteLayouts": [], + "referencePaths": [] + }, + "views": {} +} \ No newline at end of file diff --git a/docs/database.dbml b/docs/database.dbml new file mode 100644 index 0000000..09dd1bb --- /dev/null +++ b/docs/database.dbml @@ -0,0 +1,140 @@ +// IPSExtractor — esquema de integración PostgreSQL +// Fuente: services/extraction/infrastructure/database.py +// services/cleaning/infrastructure/database.py +// +// Visualizar en https://dbdiagram.io (importar este archivo) +// o con la CLI de dbml: npx @dbml/cli dbml2sql docs/database.dbml + +Project ipsextractor { + database_type: 'PostgreSQL' + Note: ''' + Base de datos compartida entre los microservicios de extracción y limpieza. + Cada servicio define su propia copia del modelo SQLAlchemy y ejecuta + `init_db()` con bloqueo advisory (pg_advisory_lock) para crear tablas + sin condiciones de carrera. + ''' +} + +TableGroup pipeline { + extraction_batches + extraction_rows + cleaning_jobs + cleaned_rows +} + +TableGroup async_jobs { + tasks +} + +// --------------------------------------------------------------------------- +// Extracción +// --------------------------------------------------------------------------- + +Table extraction_batches { + id varchar(36) [pk, note: 'UUID v4'] + status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed'] + root_path text [note: 'Ruta raíz de documentos procesados (IPS_DOCUMENT_ROOT)'] + request jsonb [not null, default: '{}', note: 'ExtractionRequest serializado (keywords, fields, doc_types, etc.)'] + error text + created_at timestamptz [not null] + updated_at timestamptz [not null] + + indexes { + status + } + + Note: 'Lote creado al iniciar una extracción; agrupa todas las filas extraídas de un procesamiento.' +} + +Table extraction_rows { + id integer [pk, increment] + batch_id varchar(36) [not null, ref: > extraction_batches.id] + radicado varchar(255) [note: 'Identificador del radicado IPS'] + source_pdf text [note: 'Ruta del PDF origen dentro del volumen montado'] + row_index integer [not null, default: 0, note: 'Orden estable dentro del lote'] + row_json jsonb [not null, default: '{}', note: 'Fila aplanada con campos solicitados'] + raw_json jsonb [not null, default: '{}', note: 'Resultado bruto del procesador por radicado'] + created_at timestamptz [not null] + + indexes { + batch_id + radicado + } + + Note: 'Una o más filas por radicado según flatten_extraction_result().' +} + +// --------------------------------------------------------------------------- +// Tareas asíncronas (API de extracción) +// --------------------------------------------------------------------------- + +Table tasks { + id varchar(36) [pk, note: 'UUID v4'] + type varchar(64) [not null, note: 'p. ej. extraction'] + status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed'] + title text + request jsonb [not null, default: '{}', note: 'Payload de la petición que originó la tarea'] + result jsonb [not null, default: '{}', note: 'Al completar incluye extraction_batch_id'] + error text + owner_id varchar(36) [note: 'Reservado para multi-usuario'] + created_at timestamptz [not null] + updated_at timestamptz [not null] + started_at timestamptz + finished_at timestamptz + + indexes { + type + status + owner_id + } + + Note: ''' + Cola de trabajos en segundo plano del servicio de extracción. + No hay FK a extraction_batches; el vínculo se guarda en + result.extraction_batch_id al finalizar la tarea. + ''' +} + +// --------------------------------------------------------------------------- +// Limpieza +// --------------------------------------------------------------------------- + +Table cleaning_jobs { + id varchar(36) [pk, note: 'UUID v4'] + extraction_batch_id varchar(36) [not null, ref: > extraction_batches.id] + status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed'] + request jsonb [not null, default: '{}', note: 'CleaningRequest (rules_per_column, cop_mod_*, etc.)'] + log jsonb [not null, default: '[]', note: 'Lista de mensajes de auditoría del job'] + error text + created_at timestamptz [not null] + updated_at timestamptz [not null] + + indexes { + extraction_batch_id + status + } + + Note: 'Ejecución de reglas de limpieza sobre las filas de un lote de extracción.' +} + +Table cleaned_rows { + id integer [pk, increment] + cleaning_job_id varchar(36) [not null, ref: > cleaning_jobs.id] + extraction_batch_id varchar(36) [not null, ref: > extraction_batches.id] + row_index integer [not null, default: 0, note: 'Conserva el orden del lote original'] + row_json jsonb [not null, default: '{}', note: 'Fila tras aplicar reglas de limpieza'] + created_at timestamptz [not null] + + indexes { + cleaning_job_id + extraction_batch_id + } + + Note: 'Salida del servicio de limpieza; reemplaza las filas previas del mismo job en cada ejecución.' +} + +// --------------------------------------------------------------------------- +// Relaciones lógicas (sin FK en BD) +// --------------------------------------------------------------------------- + +// tasks.result.extraction_batch_id -> extraction_batches.id (referencia en JSON) diff --git a/services/desktop_app/README.md b/services/desktop_app/README.md index e6694b2..97e4376 100644 --- a/services/desktop_app/README.md +++ b/services/desktop_app/README.md @@ -7,24 +7,10 @@ API Gateway de IPSExtractor. Esta carpeta funciona como repo independiente. - Node.js 20+ - Backend levantado con `docker compose up --build` -- Documentos montados en Docker como `/data` +- Electron (incluido en `npm run dev`) -## Configuracion de rutas - -La app no sube archivos en v1. El usuario solo selecciona la carpeta a procesar. -Internamente, la app traduce esa carpeta al path montado en Docker. - -Si `IPS_DOCUMENT_ROOT` existe en el entorno de la app, la carpeta seleccionada -debe estar dentro de ese root. Si no existe, la app toma la carpeta elegida como -root local para traducirla a `/data`. - -Para procesar documentos fuera de `data`, levanta Docker y la app con el mismo -root local: - -```bash -IPS_DOCUMENT_ROOT=/ruta/a/documentos docker compose up --build -IPS_DOCUMENT_ROOT=/ruta/a/documentos npm run dev -``` +No necesitas montar la carpeta del usuario en Docker. La app indexa los archivos +localmente y los sube uno a uno al servicio de extraccion. ## Desarrollo @@ -43,17 +29,20 @@ npm run build npm run dist ``` -## Flujo v1 +## Flujo desktop 1. Verificar conexion al gateway. -2. Seleccionar una carpeta local que este montada para los servicios Docker. +2. Seleccionar una carpeta local (Electron indexa PDF/TIFF). 3. Configurar modo `radicados` o `general`. -4. Descubrir campos con `/extraction/schemas`. +4. Descubrir campos (`POST /extraction/schemas/upload` con un archivo de muestra). 5. Seleccionar campos y alias. -6. Estimar o crear tarea de extraccion. +6. Ejecutar extraccion (`POST /tasks/extraction` con `source=desktop` + ingestas). 7. Consultar el historial de tareas. 8. Ver filas extraidas desde la base de datos. 9. Exportar CSV local de forma opcional. +Los PDFs permanecen en el disco del usuario. El servidor solo usa archivos +temporales durante cada ingest y guarda JSON en Postgres. + La app no incluye limpieza, preclasificacion, postproceso ni dashboard completo en esta version. diff --git a/services/desktop_app/electron/folderIndex.cjs b/services/desktop_app/electron/folderIndex.cjs new file mode 100644 index 0000000..2ed7fbd --- /dev/null +++ b/services/desktop_app/electron/folderIndex.cjs @@ -0,0 +1,125 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); + +const ALLOWED_DOC_EXTS = new Set([".pdf", ".tif", ".tiff"]); + +function matchesPattern(filename, pattern) { + if (!filename || !pattern) return false; + const fname = filename.toLowerCase(); + const pat = pattern.toLowerCase().trim(); + const ext = path.extname(fname); + if (!ALLOWED_DOC_EXTS.has(ext)) return false; + + const hasWildcards = pat.includes("*") || pat.includes("?"); + if (hasWildcards) { + const pExt = path.extname(pat); + if (ALLOWED_DOC_EXTS.has(pExt)) { + return fnmatch(fname, pat); + } + const stem = path.basename(fname, ext); + return fnmatch(stem, pat); + } + return fname.includes(pat); +} + +function fnmatch(name, pattern) { + const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, "."); + return new RegExp(`^${escaped}$`, "i").test(name); +} + +async function listDirSafe(dirPath) { + try { + return await fs.readdir(dirPath, { withFileTypes: true }); + } catch { + return []; + } +} + +async function statSize(filePath) { + try { + const stat = await fs.stat(filePath); + return stat.size; + } catch { + return 0; + } +} + +async function findFirstMatchingFile(files, keyword) { + const sorted = [...files].sort((a, b) => a.localeCompare(b, undefined, { numeric: true })); + if (!keyword) { + return sorted.find((name) => ALLOWED_DOC_EXTS.has(path.extname(name).toLowerCase())) || null; + } + return sorted.find((name) => matchesPattern(name, keyword)) || null; +} + +async function indexRadicados(rootPath, docTypes) { + const entries = []; + const radicadoDirs = (await listDirSafe(rootPath)) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort((a, b) => a.localeCompare(b, undefined, { numeric: true })); + + for (const radicado of radicadoDirs) { + const radPath = path.join(rootPath, radicado); + const files = (await listDirSafe(radPath)) + .filter((entry) => entry.isFile()) + .map((entry) => entry.name); + + for (const docType of docTypes) { + const name = String(docType.name || "").trim(); + const keyword = String(docType.keyword || "").trim(); + if (!name || !keyword) continue; + const match = await findFirstMatchingFile(files, keyword); + if (!match) continue; + const localPath = path.join(radPath, match); + entries.push({ + radicado, + filename: match, + docType: name, + localPath, + size: await statSize(localPath), + }); + } + } + return entries; +} + +async function indexGeneral(rootPath, keywords) { + const entries = []; + const normalizedKeywords = (keywords || []) + .map((item) => String(item || "").trim()) + .filter(Boolean); + const files = (await listDirSafe(rootPath)) + .filter((entry) => entry.isFile()) + .map((entry) => entry.name) + .filter((name) => ALLOWED_DOC_EXTS.has(path.extname(name).toLowerCase())) + .sort((a, b) => a.localeCompare(b, undefined, { numeric: true })); + + for (const filename of files) { + if (normalizedKeywords.length > 0 && !normalizedKeywords.some((kw) => matchesPattern(filename, kw))) { + continue; + } + const localPath = path.join(rootPath, filename); + entries.push({ + radicado: path.basename(filename, path.extname(filename)), + filename, + localPath, + size: await statSize(localPath), + }); + } + return entries; +} + +async function indexFolder(rootPath, structureMode, docTypes, keywords) { + const mode = structureMode === "general" ? "general" : "radicados"; + if (mode === "general") { + return indexGeneral(rootPath, keywords); + } + return indexRadicados(rootPath, docTypes || []); +} + +module.exports = { + indexFolder, + matchesPattern, + ALLOWED_DOC_EXTS, +}; diff --git a/services/desktop_app/electron/main.cjs b/services/desktop_app/electron/main.cjs index 5f52ea9..ffb864c 100644 --- a/services/desktop_app/electron/main.cjs +++ b/services/desktop_app/electron/main.cjs @@ -1,6 +1,7 @@ const { app, BrowserWindow, dialog, ipcMain } = require("electron"); const fs = require("node:fs/promises"); const path = require("node:path"); +const { indexFolder } = require("./folderIndex.cjs"); const DOCUMENT_EXTENSIONS = new Set([".pdf", ".tif", ".tiff", ".png", ".jpg", ".jpeg"]); @@ -72,6 +73,20 @@ ipcMain.handle("fs:first-document", async (_event, rootPath, keywords) => { return matches[0] || null; }); +ipcMain.handle("fs:index-folder", async (_event, rootPath, structureMode, docTypes, keywords) => { + if (!rootPath) return []; + return indexFolder(rootPath, structureMode, Array.isArray(docTypes) ? docTypes : [], Array.isArray(keywords) ? keywords : []); +}); + +ipcMain.handle("fs:read-file", async (_event, filePath) => { + const buffer = await fs.readFile(filePath); + return { + filename: path.basename(filePath), + size: buffer.byteLength, + data: buffer, + }; +}); + async function findDocuments(rootPath, keywords) { const normalizedKeywords = keywords .map((item) => String(item || "").trim().toLowerCase()) diff --git a/services/desktop_app/electron/preload.cjs b/services/desktop_app/electron/preload.cjs index 044b981..747e7a4 100644 --- a/services/desktop_app/electron/preload.cjs +++ b/services/desktop_app/electron/preload.cjs @@ -6,4 +6,7 @@ contextBridge.exposeInMainWorld("desktopApi", { saveCsvDialog: (defaultPath) => ipcRenderer.invoke("dialog:save-csv", defaultPath), writeTextFile: (filePath, content) => ipcRenderer.invoke("file:write-text", filePath, content), findFirstDocument: (rootPath, keywords) => ipcRenderer.invoke("fs:first-document", rootPath, keywords), + indexFolder: (rootPath, structureMode, docTypes, keywords) => + ipcRenderer.invoke("fs:index-folder", rootPath, structureMode, docTypes, keywords), + readFile: (filePath) => ipcRenderer.invoke("fs:read-file", filePath), }); diff --git a/services/desktop_app/src/renderer/App.test.tsx b/services/desktop_app/src/renderer/App.test.tsx index 8e62e43..cecd6f7 100644 --- a/services/desktop_app/src/renderer/App.test.tsx +++ b/services/desktop_app/src/renderer/App.test.tsx @@ -3,18 +3,28 @@ import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; import App from "./App"; +async function advanceToDiscover(user: ReturnType) { + await user.click(screen.getByRole("button", { name: /explorar/i })); + await user.click(screen.getByRole("button", { name: /siguiente/i })); +} + +async function discoverFields(user: ReturnType) { + await advanceToDiscover(user); + await user.click(screen.getByRole("button", { name: /descubrir campos/i })); +} + describe("App", () => { - it("renders the extraction form", () => { + it("renders the extraction wizard", () => { render(); expect(screen.getByText("IPSExtractor")).toBeInTheDocument(); expect(screen.getByRole("button", { name: /^extraccion$/i })).toBeInTheDocument(); expect(screen.getByRole("button", { name: /tareas/i })).toBeInTheDocument(); expect(screen.getByRole("button", { name: /configuracion/i })).toBeInTheDocument(); - expect(screen.getByText("Carpeta y estructura")).toBeInTheDocument(); + expect(screen.getByText("Origen de los documentos")).toBeInTheDocument(); expect(screen.queryByText("IPS_DOCUMENT_ROOT_LOCAL")).not.toBeInTheDocument(); expect(screen.queryByText("IPS_DOCUMENT_ROOT_CONTAINER")).not.toBeInTheDocument(); expect(screen.queryByText("Ruta enviada a Docker/API")).not.toBeInTheDocument(); - expect(screen.getByRole("button", { name: /descubrir campos/i })).toBeDisabled(); + expect(screen.getByRole("button", { name: /siguiente/i })).toBeDisabled(); }); it("shows API URL settings in the configuration page", async () => { @@ -31,13 +41,22 @@ describe("App", () => { it("adds and removes document types", async () => { const user = userEvent.setup(); render(); - await user.click(screen.getByRole("button", { name: /añadir tipo/i })); + await user.click(screen.getByRole("button", { name: /agregar tipo/i })); expect(screen.getByLabelText("Nombre tipo 2")).toBeInTheDocument(); const removeButtons = screen.getAllByLabelText("Eliminar tipo"); await user.click(removeButtons[1]); expect(screen.queryByLabelText("Nombre tipo 2")).not.toBeInTheDocument(); }); + it("does not show keywords in general folder mode", async () => { + const user = userEvent.setup(); + render(); + await user.click(screen.getByRole("button", { name: /carpeta general/i })); + expect(screen.queryByLabelText(/keywords/i)).not.toBeInTheDocument(); + expect(screen.queryByText(/^Keywords$/)).not.toBeInTheDocument(); + expect(screen.getByText(/se procesaran todos los pdf y tiff/i)).toBeInTheDocument(); + }); + it("shows schema fields from a mocked API response", async () => { const user = userEvent.setup(); vi.spyOn(window.desktopApi!, "selectDirectory").mockResolvedValue("C:\\docs\\lote"); @@ -46,14 +65,22 @@ describe("App", () => { text: async () => JSON.stringify({ schema_by_type: { Factura: { numero: "str" } }, sample_by_type: { Factura: { numero: "A1" } } }), } as Response); render(); - await user.click(screen.getByRole("button", { name: /explorar/i })); - await user.click(screen.getByRole("button", { name: /descubrir campos/i })); - expect(await screen.findByText("Factura/numero")).toBeInTheDocument(); + await discoverFields(user); + await user.click(screen.getByRole("button", { name: /siguiente/i })); + expect(await screen.findByRole("checkbox", { name: "Seleccionar numero" })).toBeInTheDocument(); }); - it("blocks extraction while no fields are selected", () => { + it("blocks advancing while no fields are selected", async () => { + const user = userEvent.setup(); + vi.spyOn(window.desktopApi!, "selectDirectory").mockResolvedValue("C:\\docs\\lote"); + vi.spyOn(globalThis, "fetch").mockResolvedValue({ + ok: true, + text: async () => JSON.stringify({ schema_by_type: { Factura: { numero: "str" } }, sample_by_type: { Factura: { numero: "A1" } } }), + } as Response); render(); - expect(screen.getByRole("button", { name: /ejecutar extraccion/i })).toBeDisabled(); + await discoverFields(user); + await user.click(screen.getByRole("button", { name: /siguiente/i })); + expect(screen.getByRole("button", { name: /siguiente/i })).toBeDisabled(); }); it("creates an extraction task with mocked API responses", async () => { @@ -61,7 +88,7 @@ describe("App", () => { vi.spyOn(window.desktopApi!, "selectDirectory").mockResolvedValue("C:\\docs\\lote"); vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { const url = String(input); - if (url.includes("/extraction/schemas")) { + if (url.includes("/extraction/schemas/upload")) { return { ok: true, text: async () => JSON.stringify({ schema_by_type: { Factura: { numero: "str" } }, sample_by_type: { Factura: { numero: "A1" } } }), @@ -70,13 +97,51 @@ describe("App", () => { if (url.includes("/tasks/extraction")) { return { ok: true, - text: async () => JSON.stringify({ id: "task-1", type: "extraction", status: "pending", title: "Extraccion /data", request: {}, result: {}, error: null, owner_id: null, created_at: "now", updated_at: "now", started_at: null, finished_at: null }), + text: async () => JSON.stringify({ + id: "task-1", + type: "extraction", + status: "running", + title: "Extraccion desktop", + request: { source: "desktop" }, + result: { extraction_batch_id: "batch-1", processed: 0, total: 1 }, + error: null, + owner_id: null, + created_at: "now", + updated_at: "now", + started_at: "now", + finished_at: null, + }), + } as Response; + } + if (url.includes("/tasks/task-1/ingest")) { + return { + ok: true, + text: async () => JSON.stringify({ processed: 1, total: 1, task: { id: "task-1", status: "completed", result: { extraction_batch_id: "batch-1" } } }), + } as Response; + } + if (url.includes("/tasks/task-1") && !url.includes("/rows") && !url.includes("/ingest")) { + return { + ok: true, + text: async () => JSON.stringify({ + id: "task-1", + type: "extraction", + status: "completed", + title: "Extraccion desktop", + request: { source: "desktop" }, + result: { extraction_batch_id: "batch-1", processed: 1, total: 1 }, + error: null, + owner_id: null, + created_at: "now", + updated_at: "now", + started_at: "now", + finished_at: "now", + }), } as Response; } if (url.includes("/tasks?")) { return { ok: true, - text: async () => JSON.stringify([{ id: "task-1", type: "extraction", status: "completed", title: "Extraccion /data", request: {}, result: { extraction_batch_id: "batch-1" }, error: null, owner_id: null, created_at: "now", updated_at: "now", started_at: "now", finished_at: "now" }]), + text: async () => JSON.stringify([{ id: "task-1", type: "extraction", status: "completed", title: "Extraccion desktop", request: {}, result: { extraction_batch_id: "batch-1" }, error: null, owner_id: null, created_at: "now", updated_at: "now", started_at: "now", finished_at: "now" }]), } as Response; } if (url.includes("/tasks/task-1/rows")) { @@ -89,12 +154,13 @@ describe("App", () => { }); render(); - await user.click(screen.getByRole("button", { name: /explorar/i })); - await user.click(screen.getByRole("button", { name: /descubrir campos/i })); - await user.click(await screen.findByText("Factura/numero")); + await discoverFields(user); + await user.click(screen.getByRole("button", { name: /siguiente/i })); + await user.click(await screen.findByRole("checkbox", { name: "Seleccionar numero" })); + await user.click(screen.getByRole("button", { name: /siguiente/i })); await user.click(screen.getByRole("button", { name: /ejecutar extraccion/i })); expect(await screen.findByText("Tareas y resultados")).toBeInTheDocument(); - expect(await screen.findByText("completed")).toBeInTheDocument(); + expect((await screen.findAllByText("completed")).length).toBeGreaterThan(0); expect(await screen.findByText("A1")).toBeInTheDocument(); }); }); diff --git a/services/desktop_app/src/renderer/App.tsx b/services/desktop_app/src/renderer/App.tsx index 82d331b..a8587ac 100644 --- a/services/desktop_app/src/renderer/App.tsx +++ b/services/desktop_app/src/renderer/App.tsx @@ -1,10 +1,21 @@ -import { Activity, AlertCircle, Check, Database, Download, FileSearch, FolderOpen, Loader2, Play, Plus, RefreshCw, Search, Server, Settings, Trash2 } from "lucide-react"; +import { Activity, AlertCircle, CheckCircle2, Clock, Database, Download, FileSearch, Loader2, RefreshCw, Server, Settings, XCircle } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; -import type { DocumentType, ExtractionFormState, FieldSelection, RowsResponse, SchemaResponse, Task } from "../shared/types"; +import type { DocumentType, ExtractionFormState, FieldSelection, ManifestEntry, RowsResponse, SchemaResponse, Task } from "../shared/types"; import { ApiClient } from "./lib/apiClient"; import { rowsToCsv } from "./lib/csv"; -import { buildBatchPayload, buildEstimatePayload, buildSchemaPayload, flattenSchema } from "./lib/payloads"; -import { defaultContainerRoot, mapLocalPathToContainer, splitKeywords } from "./lib/pathMapping"; +import { + buildDesktopBatchPayload, + buildEstimateUploadForm, + buildIngestFormBulk, + buildSchemaUploadForm, + flattenSchema, + mergeSchemaResponses, + validDocTypes, +} from "./lib/payloads"; +import { defaultAlias } from "./lib/schemaDisplay"; +import { splitRadicados } from "./lib/pathMapping"; +import { ExtractionWizard } from "./components/ExtractionWizard"; +import type { ExtractionStep } from "./components/ExtractionStepper"; const defaultForm: ExtractionFormState = { apiBaseUrl: localStorage.getItem("apiBaseUrl") || "http://localhost:8000", @@ -21,6 +32,7 @@ const defaultForm: ExtractionFormState = { tiffEnabled: false, tiffStart: "0", tiffEnd: "0", + model: localStorage.getItem("extractorModel") || "gpt-5.4", }; type Status = "idle" | "ok" | "error" | "busy"; @@ -40,16 +52,22 @@ function App() { const [tasks, setTasks] = useState([]); const [activeTask, setActiveTask] = useState(null); const [rows, setRows] = useState(null); + const [manifest, setManifest] = useState([]); + const [ingestProgress, setIngestProgress] = useState<{ processed: number; total: number } | null>(null); const [statusText, setStatusText] = useState("Listo"); const [busy, setBusy] = useState(false); const [error, setError] = useState(null); + const [extractionStep, setExtractionStep] = useState(1); + const [maxReachedStep, setMaxReachedStep] = useState(1); const client = useMemo(() => new ApiClient(form.apiBaseUrl), [form.apiBaseUrl]); const flattened = useMemo(() => (schemaResponse ? flattenSchema(schemaResponse) : { schema: {}, sample: null }), [schemaResponse]); - const filteredSchemaEntries = useMemo( - () => Object.entries(flattened.schema).filter(([path]) => path.toLowerCase().includes(schemaSearch.toLowerCase())), - [flattened.schema, schemaSearch], - ); + const fieldCount = Object.keys(flattened.schema).length; + + const canGoNext = + (extractionStep === 1 && manifest.length > 0) || + (extractionStep === 2 && fieldCount > 0) || + (extractionStep === 3 && selection.fields.length > 0); useEffect(() => { if (!window.desktopApi?.getRuntimeConfig) return; @@ -76,6 +94,7 @@ function App() { localStorage.setItem("apiBaseUrl", next.apiBaseUrl); if (next.localRoot) localStorage.setItem("localRoot", next.localRoot); if (next.containerRoot) localStorage.setItem("containerRoot", next.containerRoot); + if (next.model) localStorage.setItem("extractorModel", next.model); return next; }); } @@ -97,6 +116,23 @@ function App() { } } + async function indexFolderAt(path: string, mode: ExtractionFormState["mode"], docTypes: DocumentType[]) { + if (!window.desktopApi) throw new Error("La seleccion de carpetas requiere Electron."); + const indexed = await window.desktopApi.indexFolder(path, mode, validDocTypes(docTypes), []); + if (indexed.length === 0) { + throw new Error("No se encontraron documentos compatibles en la carpeta seleccionada."); + } + return indexed; + } + + function resetSchemaState() { + setSchemaResponse(null); + setSelection({ fields: [], aliasMap: {} }); + setEstimateJson(null); + setMaxReachedStep(1); + if (extractionStep > 1) setExtractionStep(1); + } + async function selectRoot() { if (!window.desktopApi) { setError("La seleccion de carpetas requiere Electron."); @@ -106,57 +142,100 @@ function App() { const selected = await window.desktopApi.selectDirectory(); if (!selected) return; try { - const localRoot = form.localRoot || selected; - const selectedContainerPath = mapLocalPathToContainer({ - localRoot, - containerRoot: defaultContainerRoot(form.containerRoot), - selectedPath: selected, - }); - updateForm({ localRoot, selectedLocalPath: selected, selectedContainerPath }); - setStatusText("Carpeta lista para procesar"); + const indexed = await indexFolderAt(selected, form.mode, form.docTypes); + setManifest(indexed); + updateForm({ localRoot: selected, selectedLocalPath: selected, selectedContainerPath: selected }); + resetSchemaState(); + setMaxReachedStep(1); + setStatusText(`${indexed.length} documento(s) listos para subir desde tu equipo`); } catch (err) { - const selectedContainerPath = mapLocalPathToContainer({ - localRoot: selected, - containerRoot: defaultContainerRoot(form.containerRoot), - selectedPath: selected, - }); - updateForm({ localRoot: selected, selectedLocalPath: selected, selectedContainerPath }); - setStatusText("Carpeta seleccionada como nueva raiz local"); + setManifest([]); + setError(err instanceof Error ? err.message : "No se pudo indexar la carpeta"); + setStatusText("Error al indexar carpeta"); + } + } + + async function handleModeChange(mode: ExtractionFormState["mode"]) { + if (mode === form.mode) return; + updateForm({ mode }); + if (!form.selectedLocalPath) return; + try { + const indexed = await indexFolderAt(form.selectedLocalPath, mode, form.docTypes); + setManifest(indexed); + resetSchemaState(); + setStatusText(`${indexed.length} documento(s) indexados en modo ${mode === "general" ? "carpeta general" : "radicados"}`); + } catch (err) { + setManifest([]); + setError(err instanceof Error ? err.message : "No se pudo reindexar la carpeta"); } } async function discoverSchema() { await runTask("Descubriendo campos...", async () => { - let documentPath: string | undefined; + if (!window.desktopApi) throw new Error("La subida de documentos requiere Electron."); + if (manifest.length === 0) throw new Error("Selecciona una carpeta con documentos antes de continuar."); + + const responses: SchemaResponse[] = []; if (form.mode === "general") { - if (!window.desktopApi) throw new Error("La busqueda de documentos requiere Electron."); - const firstLocal = await window.desktopApi.findFirstDocument(form.selectedLocalPath, splitKeywords(form.keywordsText)); - if (!firstLocal) throw new Error("No se encontro un documento de muestra en la carpeta seleccionada."); - documentPath = mapLocalPathToContainer({ - localRoot: form.localRoot, - containerRoot: defaultContainerRoot(form.containerRoot), - selectedPath: firstLocal, - }); + const entry = manifest[0]; + const fileData = await window.desktopApi.readFile(entry.localPath); + const blob = new Blob([fileData.data]); + responses.push(await client.discoverSchemaUpload(buildSchemaUploadForm(form, blob, fileData.filename))); + } else { + for (const docType of validDocTypes(form.docTypes)) { + const entry = manifest.find((item) => item.docType === docType.name); + if (!entry) throw new Error(`No se encontro un documento de muestra para el tipo ${docType.name}.`); + const fileData = await window.desktopApi.readFile(entry.localPath); + const blob = new Blob([fileData.data]); + responses.push( + await client.discoverSchemaUpload(buildSchemaUploadForm(form, blob, fileData.filename, docType.name)), + ); + } } - const payload = buildSchemaPayload(form, documentPath); - const schema = await client.discoverSchema(payload); + + const schema = mergeSchemaResponses(responses); setSchemaResponse(schema); const flat = flattenSchema(schema); setSelection({ fields: [], - aliasMap: Object.fromEntries(Object.keys(flat.schema).map((path) => [path, path])), + aliasMap: Object.fromEntries(Object.keys(flat.schema).map((path) => [path, defaultAlias(path)])), }); setEstimateJson(null); setActiveTask(null); setRows(null); setStatusText(`Campos descubiertos: ${Object.keys(flat.schema).length}`); + setMaxReachedStep((current) => (current < 3 ? 3 : current)); + setExtractionStep(3); }); } + function goToStep(step: ExtractionStep) { + setExtractionStep(step); + setMaxReachedStep((current) => (step > current ? step : current)); + } + + function handleBack() { + if (extractionStep > 1) goToStep((extractionStep - 1) as ExtractionStep); + } + + function handleNext() { + if (!canGoNext) return; + if (extractionStep < 4) goToStep((extractionStep + 1) as ExtractionStep); + } + async function estimate() { await runTask("Estimando...", async () => { requireFields(); - const result = await client.estimate(buildEstimatePayload(form, schemaResponse, selection)); + if (!window.desktopApi) throw new Error("La estimacion requiere Electron."); + if (manifest.length === 0) throw new Error("Selecciona una carpeta con documentos antes de estimar."); + + const sampleEntry = manifest[0]; + const fileData = await window.desktopApi.readFile(sampleEntry.localPath); + const blob = new Blob([fileData.data]); + const uniqueRadicados = new Set(manifest.map((e) => e.radicado)).size; + + const formData = buildEstimateUploadForm(form, blob, fileData.filename, uniqueRadicados, selection.fields, form.model); + const result = await client.estimateUpload(formData); setEstimateJson(result); setStatusText("Estimacion completada"); }); @@ -165,13 +244,64 @@ function App() { async function runExtraction() { await runTask("Creando tarea de extraccion...", async () => { requireFields(); - const task = await client.createExtractionTask(buildBatchPayload(form, selection)); + if (!window.desktopApi) throw new Error("La subida de documentos requiere Electron."); + if (manifest.length === 0) throw new Error("Selecciona una carpeta con documentos antes de continuar."); + + const payload = buildDesktopBatchPayload(form, selection, manifest, form.model); + const task = await client.createExtractionTask(payload); setActiveTask(task); setTasks((current) => [task, ...current.filter((item) => item.id !== task.id)]); setRows(null); - setStatusText(`Tarea creada: ${task.id}`); - await refreshTasks(task.id); setView("tasks"); + + let workManifest = [...manifest]; + if (form.filterEnabled) { + const allowed = new Set(splitRadicados(form.radicadoFilterText)); + workManifest = workManifest.filter((entry) => allowed.has(entry.radicado)); + } + if (workManifest.length === 0) throw new Error("El filtro de radicados no dejo documentos para procesar."); + + const total = workManifest.length; + setIngestProgress({ processed: 0, total }); + + // El backend procesa en paralelo con Redis/ARQ; aqui solo subimos y + // encolamos por lotes (mas rapido que 1 peticion HTTP por archivo). + // La extraccion real ocurre en segundo plano y se refleja via polling + // de la tarea (ver useEffect de refreshTasks). + const BATCH_SIZE = 20; + const BATCH_CONCURRENCY = 3; + const batches: ManifestEntry[][] = []; + for (let i = 0; i < workManifest.length; i += BATCH_SIZE) { + batches.push(workManifest.slice(i, i + BATCH_SIZE)); + } + + let queued = 0; + let nextBatchIndex = 0; + + async function uploadBatch(batch: ManifestEntry[]) { + const files = await Promise.all( + batch.map((entry) => window.desktopApi!.readFile(entry.localPath)), + ); + const blobs = files.map((fileData) => new Blob([fileData.data])); + await client.ingestTaskDocumentsBulk(task.id, buildIngestFormBulk(batch, blobs)); + queued += batch.length; + setIngestProgress({ processed: queued, total }); + setStatusText(`Encolados ${queued} de ${total} documento(s) para procesamiento...`); + } + + async function worker() { + while (nextBatchIndex < batches.length) { + const batch = batches[nextBatchIndex]; + nextBatchIndex += 1; + await uploadBatch(batch); + } + } + + await Promise.all(Array.from({ length: Math.min(BATCH_CONCURRENCY, batches.length) }, () => worker())); + + setIngestProgress(null); + setStatusText(`${total} documento(s) encolados. Procesando en segundo plano...`); + await refreshTasks(task.id); }); } @@ -233,7 +363,9 @@ function App() { } function requireFields() { - if (!form.selectedContainerPath) throw new Error("Selecciona una carpeta raiz antes de continuar."); + if (!form.selectedLocalPath || manifest.length === 0) { + throw new Error("Selecciona una carpeta con documentos antes de continuar."); + } if (selection.fields.length === 0) throw new Error("Selecciona al menos un campo."); } @@ -246,16 +378,24 @@ function App() { function toggleField(path: string, checked: boolean) { setSelection((current) => { const fields = checked ? Array.from(new Set([...current.fields, path])) : current.fields.filter((field) => field !== path); - return { - fields, - aliasMap: { ...current.aliasMap, [path]: current.aliasMap[path] || path }, - }; + const aliasMap = { ...current.aliasMap }; + if (checked && !aliasMap[path]) aliasMap[path] = defaultAlias(path); + return { fields, aliasMap }; }); } - const currentTitle = view === "extraction" ? "Extraccion de documentos" : view === "tasks" ? "Tareas y resultados" : "Configuracion"; + function selectAllFields() { + setSelection({ + fields: Object.keys(flattened.schema), + aliasMap: Object.fromEntries( + Object.keys(flattened.schema).map((path) => [path, selection.aliasMap[path] || defaultAlias(path)]), + ), + }); + } + + const currentTitle = view === "extraction" ? "Nueva extraccion" : view === "tasks" ? "Tareas y resultados" : "Configuracion"; const currentDescription = view === "extraction" - ? "Selecciona una carpeta, descubre campos y crea una tarea de extraccion." + ? "Asistente paso a paso para extraer datos de tus documentos." : view === "tasks" ? "Consulta el historial de ejecuciones y las filas guardadas en la base de datos." : "Administra la conexion al API Gateway local."; @@ -265,45 +405,67 @@ function App() { -
-
-
+
+ {view === "extraction" ? ( +

{currentTitle}

-

{currentDescription}

-
-
- -
-
+ + ) : ( +
+
+

{currentTitle}

+

{currentDescription}

+
+
+ +
+
+ )} {view === "extraction" && ( - )} @@ -357,178 +520,41 @@ function App() { ); } -interface ExtractionViewProps { - form: ExtractionFormState; - updateForm: (patch: Partial) => void; - busy: boolean; - error: string | null; - statusText: string; - selectRoot: () => Promise; - discoverSchema: () => Promise; - estimate: () => Promise; - runExtraction: () => Promise; - exportCsv: () => Promise; - rows: RowsResponse | null; - selection: FieldSelection; - setSelection: React.Dispatch>; - schemaSearch: string; - setSchemaSearch: React.Dispatch>; - filteredSchemaEntries: [string, string][]; - flattened: { schema: Record; sample: unknown }; - estimateJson: unknown; - activeTask: Task | null; - configJson: unknown; - modelsJson: unknown; - setDocType: (index: number, patch: Partial) => void; - toggleField: (path: string, checked: boolean) => void; +function TaskProgress({ result }: { result: Record }) { + const processed = Number(result.processed || 0); + const total = Number(result.total || 0); + if (!total || total <= 0) return null; + const percent = Math.min(100, Math.round((processed / total) * 100)); + return ( +
+
+ Procesando documentos + {processed} / {total} +
+
+
+
+
+ ); } -function ExtractionView({ - form, - updateForm, - busy, - error, - statusText, - selectRoot, - discoverSchema, - estimate, - runExtraction, - exportCsv, - rows, - selection, - setSelection, - schemaSearch, - setSchemaSearch, - filteredSchemaEntries, - flattened, - estimateJson, - activeTask, - configJson, - modelsJson, - setDocType, - toggleField, -}: ExtractionViewProps) { - return ( - <> -
- -
- - -
-
- - -
- +function TaskStatusIcon({ status }: { status: string }) { + if (status === "completed") return ; + if (status === "failed") return ; + if (status === "running") return ; + return ; +} - -
- - -
- {form.tiffEnabled && ( -
- - -
- )} - - {form.filterEnabled &&