feat: Add .env.example configuration file and integrate Redis service into Docker Compose for improved task management. Update desktop application to support document indexing and extraction workflows, enhancing user experience with a new extraction wizard and stepper components. Revise README and documentation for clarity on setup and usage.

This commit is contained in:
erik
2026-07-05 22:39:07 -05:00
parent 9aca926988
commit d95f8f6ad8
70 changed files with 9807 additions and 499 deletions
+10 -21
View File
@@ -7,24 +7,10 @@ API Gateway de IPSExtractor. Esta carpeta funciona como repo independiente.
- Node.js 20+
- Backend levantado con `docker compose up --build`
- Documentos montados en Docker como `/data`
- Electron (incluido en `npm run dev`)
## Configuracion de rutas
La app no sube archivos en v1. El usuario solo selecciona la carpeta a procesar.
Internamente, la app traduce esa carpeta al path montado en Docker.
Si `IPS_DOCUMENT_ROOT` existe en el entorno de la app, la carpeta seleccionada
debe estar dentro de ese root. Si no existe, la app toma la carpeta elegida como
root local para traducirla a `/data`.
Para procesar documentos fuera de `data`, levanta Docker y la app con el mismo
root local:
```bash
IPS_DOCUMENT_ROOT=/ruta/a/documentos docker compose up --build
IPS_DOCUMENT_ROOT=/ruta/a/documentos npm run dev
```
No necesitas montar la carpeta del usuario en Docker. La app indexa los archivos
localmente y los sube uno a uno al servicio de extraccion.
## Desarrollo
@@ -43,17 +29,20 @@ npm run build
npm run dist
```
## Flujo v1
## Flujo desktop
1. Verificar conexion al gateway.
2. Seleccionar una carpeta local que este montada para los servicios Docker.
2. Seleccionar una carpeta local (Electron indexa PDF/TIFF).
3. Configurar modo `radicados` o `general`.
4. Descubrir campos con `/extraction/schemas`.
4. Descubrir campos (`POST /extraction/schemas/upload` con un archivo de muestra).
5. Seleccionar campos y alias.
6. Estimar o crear tarea de extraccion.
6. Ejecutar extraccion (`POST /tasks/extraction` con `source=desktop` + ingestas).
7. Consultar el historial de tareas.
8. Ver filas extraidas desde la base de datos.
9. Exportar CSV local de forma opcional.
Los PDFs permanecen en el disco del usuario. El servidor solo usa archivos
temporales durante cada ingest y guarda JSON en Postgres.
La app no incluye limpieza, preclasificacion, postproceso ni dashboard completo
en esta version.
@@ -0,0 +1,125 @@
const fs = require("node:fs/promises");
const path = require("node:path");
const ALLOWED_DOC_EXTS = new Set([".pdf", ".tif", ".tiff"]);
function matchesPattern(filename, pattern) {
if (!filename || !pattern) return false;
const fname = filename.toLowerCase();
const pat = pattern.toLowerCase().trim();
const ext = path.extname(fname);
if (!ALLOWED_DOC_EXTS.has(ext)) return false;
const hasWildcards = pat.includes("*") || pat.includes("?");
if (hasWildcards) {
const pExt = path.extname(pat);
if (ALLOWED_DOC_EXTS.has(pExt)) {
return fnmatch(fname, pat);
}
const stem = path.basename(fname, ext);
return fnmatch(stem, pat);
}
return fname.includes(pat);
}
function fnmatch(name, pattern) {
const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".");
return new RegExp(`^${escaped}$`, "i").test(name);
}
async function listDirSafe(dirPath) {
try {
return await fs.readdir(dirPath, { withFileTypes: true });
} catch {
return [];
}
}
async function statSize(filePath) {
try {
const stat = await fs.stat(filePath);
return stat.size;
} catch {
return 0;
}
}
async function findFirstMatchingFile(files, keyword) {
const sorted = [...files].sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));
if (!keyword) {
return sorted.find((name) => ALLOWED_DOC_EXTS.has(path.extname(name).toLowerCase())) || null;
}
return sorted.find((name) => matchesPattern(name, keyword)) || null;
}
async function indexRadicados(rootPath, docTypes) {
const entries = [];
const radicadoDirs = (await listDirSafe(rootPath))
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));
for (const radicado of radicadoDirs) {
const radPath = path.join(rootPath, radicado);
const files = (await listDirSafe(radPath))
.filter((entry) => entry.isFile())
.map((entry) => entry.name);
for (const docType of docTypes) {
const name = String(docType.name || "").trim();
const keyword = String(docType.keyword || "").trim();
if (!name || !keyword) continue;
const match = await findFirstMatchingFile(files, keyword);
if (!match) continue;
const localPath = path.join(radPath, match);
entries.push({
radicado,
filename: match,
docType: name,
localPath,
size: await statSize(localPath),
});
}
}
return entries;
}
async function indexGeneral(rootPath, keywords) {
const entries = [];
const normalizedKeywords = (keywords || [])
.map((item) => String(item || "").trim())
.filter(Boolean);
const files = (await listDirSafe(rootPath))
.filter((entry) => entry.isFile())
.map((entry) => entry.name)
.filter((name) => ALLOWED_DOC_EXTS.has(path.extname(name).toLowerCase()))
.sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));
for (const filename of files) {
if (normalizedKeywords.length > 0 && !normalizedKeywords.some((kw) => matchesPattern(filename, kw))) {
continue;
}
const localPath = path.join(rootPath, filename);
entries.push({
radicado: path.basename(filename, path.extname(filename)),
filename,
localPath,
size: await statSize(localPath),
});
}
return entries;
}
async function indexFolder(rootPath, structureMode, docTypes, keywords) {
const mode = structureMode === "general" ? "general" : "radicados";
if (mode === "general") {
return indexGeneral(rootPath, keywords);
}
return indexRadicados(rootPath, docTypes || []);
}
module.exports = {
indexFolder,
matchesPattern,
ALLOWED_DOC_EXTS,
};
+15
View File
@@ -1,6 +1,7 @@
const { app, BrowserWindow, dialog, ipcMain } = require("electron");
const fs = require("node:fs/promises");
const path = require("node:path");
const { indexFolder } = require("./folderIndex.cjs");
const DOCUMENT_EXTENSIONS = new Set([".pdf", ".tif", ".tiff", ".png", ".jpg", ".jpeg"]);
@@ -72,6 +73,20 @@ ipcMain.handle("fs:first-document", async (_event, rootPath, keywords) => {
return matches[0] || null;
});
ipcMain.handle("fs:index-folder", async (_event, rootPath, structureMode, docTypes, keywords) => {
if (!rootPath) return [];
return indexFolder(rootPath, structureMode, Array.isArray(docTypes) ? docTypes : [], Array.isArray(keywords) ? keywords : []);
});
ipcMain.handle("fs:read-file", async (_event, filePath) => {
const buffer = await fs.readFile(filePath);
return {
filename: path.basename(filePath),
size: buffer.byteLength,
data: buffer,
};
});
async function findDocuments(rootPath, keywords) {
const normalizedKeywords = keywords
.map((item) => String(item || "").trim().toLowerCase())
@@ -6,4 +6,7 @@ contextBridge.exposeInMainWorld("desktopApi", {
saveCsvDialog: (defaultPath) => ipcRenderer.invoke("dialog:save-csv", defaultPath),
writeTextFile: (filePath, content) => ipcRenderer.invoke("file:write-text", filePath, content),
findFirstDocument: (rootPath, keywords) => ipcRenderer.invoke("fs:first-document", rootPath, keywords),
indexFolder: (rootPath, structureMode, docTypes, keywords) =>
ipcRenderer.invoke("fs:index-folder", rootPath, structureMode, docTypes, keywords),
readFile: (filePath) => ipcRenderer.invoke("fs:read-file", filePath),
});
+82 -16
View File
@@ -3,18 +3,28 @@ import userEvent from "@testing-library/user-event";
import { describe, expect, it, vi } from "vitest";
import App from "./App";
async function advanceToDiscover(user: ReturnType<typeof userEvent.setup>) {
await user.click(screen.getByRole("button", { name: /explorar/i }));
await user.click(screen.getByRole("button", { name: /siguiente/i }));
}
async function discoverFields(user: ReturnType<typeof userEvent.setup>) {
await advanceToDiscover(user);
await user.click(screen.getByRole("button", { name: /descubrir campos/i }));
}
describe("App", () => {
it("renders the extraction form", () => {
it("renders the extraction wizard", () => {
render(<App />);
expect(screen.getByText("IPSExtractor")).toBeInTheDocument();
expect(screen.getByRole("button", { name: /^extraccion$/i })).toBeInTheDocument();
expect(screen.getByRole("button", { name: /tareas/i })).toBeInTheDocument();
expect(screen.getByRole("button", { name: /configuracion/i })).toBeInTheDocument();
expect(screen.getByText("Carpeta y estructura")).toBeInTheDocument();
expect(screen.getByText("Origen de los documentos")).toBeInTheDocument();
expect(screen.queryByText("IPS_DOCUMENT_ROOT_LOCAL")).not.toBeInTheDocument();
expect(screen.queryByText("IPS_DOCUMENT_ROOT_CONTAINER")).not.toBeInTheDocument();
expect(screen.queryByText("Ruta enviada a Docker/API")).not.toBeInTheDocument();
expect(screen.getByRole("button", { name: /descubrir campos/i })).toBeDisabled();
expect(screen.getByRole("button", { name: /siguiente/i })).toBeDisabled();
});
it("shows API URL settings in the configuration page", async () => {
@@ -31,13 +41,22 @@ describe("App", () => {
it("adds and removes document types", async () => {
const user = userEvent.setup();
render(<App />);
await user.click(screen.getByRole("button", { name: /añadir tipo/i }));
await user.click(screen.getByRole("button", { name: /agregar tipo/i }));
expect(screen.getByLabelText("Nombre tipo 2")).toBeInTheDocument();
const removeButtons = screen.getAllByLabelText("Eliminar tipo");
await user.click(removeButtons[1]);
expect(screen.queryByLabelText("Nombre tipo 2")).not.toBeInTheDocument();
});
it("does not show keywords in general folder mode", async () => {
const user = userEvent.setup();
render(<App />);
await user.click(screen.getByRole("button", { name: /carpeta general/i }));
expect(screen.queryByLabelText(/keywords/i)).not.toBeInTheDocument();
expect(screen.queryByText(/^Keywords$/)).not.toBeInTheDocument();
expect(screen.getByText(/se procesaran todos los pdf y tiff/i)).toBeInTheDocument();
});
it("shows schema fields from a mocked API response", async () => {
const user = userEvent.setup();
vi.spyOn(window.desktopApi!, "selectDirectory").mockResolvedValue("C:\\docs\\lote");
@@ -46,14 +65,22 @@ describe("App", () => {
text: async () => JSON.stringify({ schema_by_type: { Factura: { numero: "str" } }, sample_by_type: { Factura: { numero: "A1" } } }),
} as Response);
render(<App />);
await user.click(screen.getByRole("button", { name: /explorar/i }));
await user.click(screen.getByRole("button", { name: /descubrir campos/i }));
expect(await screen.findByText("Factura/numero")).toBeInTheDocument();
await discoverFields(user);
await user.click(screen.getByRole("button", { name: /siguiente/i }));
expect(await screen.findByRole("checkbox", { name: "Seleccionar numero" })).toBeInTheDocument();
});
it("blocks extraction while no fields are selected", () => {
it("blocks advancing while no fields are selected", async () => {
const user = userEvent.setup();
vi.spyOn(window.desktopApi!, "selectDirectory").mockResolvedValue("C:\\docs\\lote");
vi.spyOn(globalThis, "fetch").mockResolvedValue({
ok: true,
text: async () => JSON.stringify({ schema_by_type: { Factura: { numero: "str" } }, sample_by_type: { Factura: { numero: "A1" } } }),
} as Response);
render(<App />);
expect(screen.getByRole("button", { name: /ejecutar extraccion/i })).toBeDisabled();
await discoverFields(user);
await user.click(screen.getByRole("button", { name: /siguiente/i }));
expect(screen.getByRole("button", { name: /siguiente/i })).toBeDisabled();
});
it("creates an extraction task with mocked API responses", async () => {
@@ -61,7 +88,7 @@ describe("App", () => {
vi.spyOn(window.desktopApi!, "selectDirectory").mockResolvedValue("C:\\docs\\lote");
vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
const url = String(input);
if (url.includes("/extraction/schemas")) {
if (url.includes("/extraction/schemas/upload")) {
return {
ok: true,
text: async () => JSON.stringify({ schema_by_type: { Factura: { numero: "str" } }, sample_by_type: { Factura: { numero: "A1" } } }),
@@ -70,13 +97,51 @@ describe("App", () => {
if (url.includes("/tasks/extraction")) {
return {
ok: true,
text: async () => JSON.stringify({ id: "task-1", type: "extraction", status: "pending", title: "Extraccion /data", request: {}, result: {}, error: null, owner_id: null, created_at: "now", updated_at: "now", started_at: null, finished_at: null }),
text: async () => JSON.stringify({
id: "task-1",
type: "extraction",
status: "running",
title: "Extraccion desktop",
request: { source: "desktop" },
result: { extraction_batch_id: "batch-1", processed: 0, total: 1 },
error: null,
owner_id: null,
created_at: "now",
updated_at: "now",
started_at: "now",
finished_at: null,
}),
} as Response;
}
if (url.includes("/tasks/task-1/ingest")) {
return {
ok: true,
text: async () => JSON.stringify({ processed: 1, total: 1, task: { id: "task-1", status: "completed", result: { extraction_batch_id: "batch-1" } } }),
} as Response;
}
if (url.includes("/tasks/task-1") && !url.includes("/rows") && !url.includes("/ingest")) {
return {
ok: true,
text: async () => JSON.stringify({
id: "task-1",
type: "extraction",
status: "completed",
title: "Extraccion desktop",
request: { source: "desktop" },
result: { extraction_batch_id: "batch-1", processed: 1, total: 1 },
error: null,
owner_id: null,
created_at: "now",
updated_at: "now",
started_at: "now",
finished_at: "now",
}),
} as Response;
}
if (url.includes("/tasks?")) {
return {
ok: true,
text: async () => JSON.stringify([{ id: "task-1", type: "extraction", status: "completed", title: "Extraccion /data", request: {}, result: { extraction_batch_id: "batch-1" }, error: null, owner_id: null, created_at: "now", updated_at: "now", started_at: "now", finished_at: "now" }]),
text: async () => JSON.stringify([{ id: "task-1", type: "extraction", status: "completed", title: "Extraccion desktop", request: {}, result: { extraction_batch_id: "batch-1" }, error: null, owner_id: null, created_at: "now", updated_at: "now", started_at: "now", finished_at: "now" }]),
} as Response;
}
if (url.includes("/tasks/task-1/rows")) {
@@ -89,12 +154,13 @@ describe("App", () => {
});
render(<App />);
await user.click(screen.getByRole("button", { name: /explorar/i }));
await user.click(screen.getByRole("button", { name: /descubrir campos/i }));
await user.click(await screen.findByText("Factura/numero"));
await discoverFields(user);
await user.click(screen.getByRole("button", { name: /siguiente/i }));
await user.click(await screen.findByRole("checkbox", { name: "Seleccionar numero" }));
await user.click(screen.getByRole("button", { name: /siguiente/i }));
await user.click(screen.getByRole("button", { name: /ejecutar extraccion/i }));
expect(await screen.findByText("Tareas y resultados")).toBeInTheDocument();
expect(await screen.findByText("completed")).toBeInTheDocument();
expect((await screen.findAllByText("completed")).length).toBeGreaterThan(0);
expect(await screen.findByText("A1")).toBeInTheDocument();
});
});
+482 -297
View File
@@ -1,10 +1,21 @@
import { Activity, AlertCircle, Check, Database, Download, FileSearch, FolderOpen, Loader2, Play, Plus, RefreshCw, Search, Server, Settings, Trash2 } from "lucide-react";
import { Activity, AlertCircle, CheckCircle2, Clock, Database, Download, FileSearch, Loader2, RefreshCw, Server, Settings, XCircle } from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import type { DocumentType, ExtractionFormState, FieldSelection, RowsResponse, SchemaResponse, Task } from "../shared/types";
import type { DocumentType, ExtractionFormState, FieldSelection, ManifestEntry, RowsResponse, SchemaResponse, Task } from "../shared/types";
import { ApiClient } from "./lib/apiClient";
import { rowsToCsv } from "./lib/csv";
import { buildBatchPayload, buildEstimatePayload, buildSchemaPayload, flattenSchema } from "./lib/payloads";
import { defaultContainerRoot, mapLocalPathToContainer, splitKeywords } from "./lib/pathMapping";
import {
buildDesktopBatchPayload,
buildEstimateUploadForm,
buildIngestFormBulk,
buildSchemaUploadForm,
flattenSchema,
mergeSchemaResponses,
validDocTypes,
} from "./lib/payloads";
import { defaultAlias } from "./lib/schemaDisplay";
import { splitRadicados } from "./lib/pathMapping";
import { ExtractionWizard } from "./components/ExtractionWizard";
import type { ExtractionStep } from "./components/ExtractionStepper";
const defaultForm: ExtractionFormState = {
apiBaseUrl: localStorage.getItem("apiBaseUrl") || "http://localhost:8000",
@@ -21,6 +32,7 @@ const defaultForm: ExtractionFormState = {
tiffEnabled: false,
tiffStart: "0",
tiffEnd: "0",
model: localStorage.getItem("extractorModel") || "gpt-5.4",
};
type Status = "idle" | "ok" | "error" | "busy";
@@ -40,16 +52,22 @@ function App() {
const [tasks, setTasks] = useState<Task[]>([]);
const [activeTask, setActiveTask] = useState<Task | null>(null);
const [rows, setRows] = useState<RowsResponse | null>(null);
const [manifest, setManifest] = useState<ManifestEntry[]>([]);
const [ingestProgress, setIngestProgress] = useState<{ processed: number; total: number } | null>(null);
const [statusText, setStatusText] = useState("Listo");
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
const [extractionStep, setExtractionStep] = useState<ExtractionStep>(1);
const [maxReachedStep, setMaxReachedStep] = useState<ExtractionStep>(1);
const client = useMemo(() => new ApiClient(form.apiBaseUrl), [form.apiBaseUrl]);
const flattened = useMemo(() => (schemaResponse ? flattenSchema(schemaResponse) : { schema: {}, sample: null }), [schemaResponse]);
const filteredSchemaEntries = useMemo(
() => Object.entries(flattened.schema).filter(([path]) => path.toLowerCase().includes(schemaSearch.toLowerCase())),
[flattened.schema, schemaSearch],
);
const fieldCount = Object.keys(flattened.schema).length;
const canGoNext =
(extractionStep === 1 && manifest.length > 0) ||
(extractionStep === 2 && fieldCount > 0) ||
(extractionStep === 3 && selection.fields.length > 0);
useEffect(() => {
if (!window.desktopApi?.getRuntimeConfig) return;
@@ -76,6 +94,7 @@ function App() {
localStorage.setItem("apiBaseUrl", next.apiBaseUrl);
if (next.localRoot) localStorage.setItem("localRoot", next.localRoot);
if (next.containerRoot) localStorage.setItem("containerRoot", next.containerRoot);
if (next.model) localStorage.setItem("extractorModel", next.model);
return next;
});
}
@@ -97,6 +116,23 @@ function App() {
}
}
async function indexFolderAt(path: string, mode: ExtractionFormState["mode"], docTypes: DocumentType[]) {
if (!window.desktopApi) throw new Error("La seleccion de carpetas requiere Electron.");
const indexed = await window.desktopApi.indexFolder(path, mode, validDocTypes(docTypes), []);
if (indexed.length === 0) {
throw new Error("No se encontraron documentos compatibles en la carpeta seleccionada.");
}
return indexed;
}
function resetSchemaState() {
setSchemaResponse(null);
setSelection({ fields: [], aliasMap: {} });
setEstimateJson(null);
setMaxReachedStep(1);
if (extractionStep > 1) setExtractionStep(1);
}
async function selectRoot() {
if (!window.desktopApi) {
setError("La seleccion de carpetas requiere Electron.");
@@ -106,57 +142,100 @@ function App() {
const selected = await window.desktopApi.selectDirectory();
if (!selected) return;
try {
const localRoot = form.localRoot || selected;
const selectedContainerPath = mapLocalPathToContainer({
localRoot,
containerRoot: defaultContainerRoot(form.containerRoot),
selectedPath: selected,
});
updateForm({ localRoot, selectedLocalPath: selected, selectedContainerPath });
setStatusText("Carpeta lista para procesar");
const indexed = await indexFolderAt(selected, form.mode, form.docTypes);
setManifest(indexed);
updateForm({ localRoot: selected, selectedLocalPath: selected, selectedContainerPath: selected });
resetSchemaState();
setMaxReachedStep(1);
setStatusText(`${indexed.length} documento(s) listos para subir desde tu equipo`);
} catch (err) {
const selectedContainerPath = mapLocalPathToContainer({
localRoot: selected,
containerRoot: defaultContainerRoot(form.containerRoot),
selectedPath: selected,
});
updateForm({ localRoot: selected, selectedLocalPath: selected, selectedContainerPath });
setStatusText("Carpeta seleccionada como nueva raiz local");
setManifest([]);
setError(err instanceof Error ? err.message : "No se pudo indexar la carpeta");
setStatusText("Error al indexar carpeta");
}
}
async function handleModeChange(mode: ExtractionFormState["mode"]) {
if (mode === form.mode) return;
updateForm({ mode });
if (!form.selectedLocalPath) return;
try {
const indexed = await indexFolderAt(form.selectedLocalPath, mode, form.docTypes);
setManifest(indexed);
resetSchemaState();
setStatusText(`${indexed.length} documento(s) indexados en modo ${mode === "general" ? "carpeta general" : "radicados"}`);
} catch (err) {
setManifest([]);
setError(err instanceof Error ? err.message : "No se pudo reindexar la carpeta");
}
}
async function discoverSchema() {
await runTask("Descubriendo campos...", async () => {
let documentPath: string | undefined;
if (!window.desktopApi) throw new Error("La subida de documentos requiere Electron.");
if (manifest.length === 0) throw new Error("Selecciona una carpeta con documentos antes de continuar.");
const responses: SchemaResponse[] = [];
if (form.mode === "general") {
if (!window.desktopApi) throw new Error("La busqueda de documentos requiere Electron.");
const firstLocal = await window.desktopApi.findFirstDocument(form.selectedLocalPath, splitKeywords(form.keywordsText));
if (!firstLocal) throw new Error("No se encontro un documento de muestra en la carpeta seleccionada.");
documentPath = mapLocalPathToContainer({
localRoot: form.localRoot,
containerRoot: defaultContainerRoot(form.containerRoot),
selectedPath: firstLocal,
});
const entry = manifest[0];
const fileData = await window.desktopApi.readFile(entry.localPath);
const blob = new Blob([fileData.data]);
responses.push(await client.discoverSchemaUpload(buildSchemaUploadForm(form, blob, fileData.filename)));
} else {
for (const docType of validDocTypes(form.docTypes)) {
const entry = manifest.find((item) => item.docType === docType.name);
if (!entry) throw new Error(`No se encontro un documento de muestra para el tipo ${docType.name}.`);
const fileData = await window.desktopApi.readFile(entry.localPath);
const blob = new Blob([fileData.data]);
responses.push(
await client.discoverSchemaUpload(buildSchemaUploadForm(form, blob, fileData.filename, docType.name)),
);
}
}
const payload = buildSchemaPayload(form, documentPath);
const schema = await client.discoverSchema(payload);
const schema = mergeSchemaResponses(responses);
setSchemaResponse(schema);
const flat = flattenSchema(schema);
setSelection({
fields: [],
aliasMap: Object.fromEntries(Object.keys(flat.schema).map((path) => [path, path])),
aliasMap: Object.fromEntries(Object.keys(flat.schema).map((path) => [path, defaultAlias(path)])),
});
setEstimateJson(null);
setActiveTask(null);
setRows(null);
setStatusText(`Campos descubiertos: ${Object.keys(flat.schema).length}`);
setMaxReachedStep((current) => (current < 3 ? 3 : current));
setExtractionStep(3);
});
}
function goToStep(step: ExtractionStep) {
setExtractionStep(step);
setMaxReachedStep((current) => (step > current ? step : current));
}
function handleBack() {
if (extractionStep > 1) goToStep((extractionStep - 1) as ExtractionStep);
}
function handleNext() {
if (!canGoNext) return;
if (extractionStep < 4) goToStep((extractionStep + 1) as ExtractionStep);
}
async function estimate() {
await runTask("Estimando...", async () => {
requireFields();
const result = await client.estimate(buildEstimatePayload(form, schemaResponse, selection));
if (!window.desktopApi) throw new Error("La estimacion requiere Electron.");
if (manifest.length === 0) throw new Error("Selecciona una carpeta con documentos antes de estimar.");
const sampleEntry = manifest[0];
const fileData = await window.desktopApi.readFile(sampleEntry.localPath);
const blob = new Blob([fileData.data]);
const uniqueRadicados = new Set(manifest.map((e) => e.radicado)).size;
const formData = buildEstimateUploadForm(form, blob, fileData.filename, uniqueRadicados, selection.fields, form.model);
const result = await client.estimateUpload(formData);
setEstimateJson(result);
setStatusText("Estimacion completada");
});
@@ -165,13 +244,64 @@ function App() {
async function runExtraction() {
await runTask("Creando tarea de extraccion...", async () => {
requireFields();
const task = await client.createExtractionTask(buildBatchPayload(form, selection));
if (!window.desktopApi) throw new Error("La subida de documentos requiere Electron.");
if (manifest.length === 0) throw new Error("Selecciona una carpeta con documentos antes de continuar.");
const payload = buildDesktopBatchPayload(form, selection, manifest, form.model);
const task = await client.createExtractionTask(payload);
setActiveTask(task);
setTasks((current) => [task, ...current.filter((item) => item.id !== task.id)]);
setRows(null);
setStatusText(`Tarea creada: ${task.id}`);
await refreshTasks(task.id);
setView("tasks");
let workManifest = [...manifest];
if (form.filterEnabled) {
const allowed = new Set(splitRadicados(form.radicadoFilterText));
workManifest = workManifest.filter((entry) => allowed.has(entry.radicado));
}
if (workManifest.length === 0) throw new Error("El filtro de radicados no dejo documentos para procesar.");
const total = workManifest.length;
setIngestProgress({ processed: 0, total });
// El backend procesa en paralelo con Redis/ARQ; aqui solo subimos y
// encolamos por lotes (mas rapido que 1 peticion HTTP por archivo).
// La extraccion real ocurre en segundo plano y se refleja via polling
// de la tarea (ver useEffect de refreshTasks).
const BATCH_SIZE = 20;
const BATCH_CONCURRENCY = 3;
const batches: ManifestEntry[][] = [];
for (let i = 0; i < workManifest.length; i += BATCH_SIZE) {
batches.push(workManifest.slice(i, i + BATCH_SIZE));
}
let queued = 0;
let nextBatchIndex = 0;
async function uploadBatch(batch: ManifestEntry[]) {
const files = await Promise.all(
batch.map((entry) => window.desktopApi!.readFile(entry.localPath)),
);
const blobs = files.map((fileData) => new Blob([fileData.data]));
await client.ingestTaskDocumentsBulk(task.id, buildIngestFormBulk(batch, blobs));
queued += batch.length;
setIngestProgress({ processed: queued, total });
setStatusText(`Encolados ${queued} de ${total} documento(s) para procesamiento...`);
}
async function worker() {
while (nextBatchIndex < batches.length) {
const batch = batches[nextBatchIndex];
nextBatchIndex += 1;
await uploadBatch(batch);
}
}
await Promise.all(Array.from({ length: Math.min(BATCH_CONCURRENCY, batches.length) }, () => worker()));
setIngestProgress(null);
setStatusText(`${total} documento(s) encolados. Procesando en segundo plano...`);
await refreshTasks(task.id);
});
}
@@ -233,7 +363,9 @@ function App() {
}
function requireFields() {
if (!form.selectedContainerPath) throw new Error("Selecciona una carpeta raiz antes de continuar.");
if (!form.selectedLocalPath || manifest.length === 0) {
throw new Error("Selecciona una carpeta con documentos antes de continuar.");
}
if (selection.fields.length === 0) throw new Error("Selecciona al menos un campo.");
}
@@ -246,16 +378,24 @@ function App() {
function toggleField(path: string, checked: boolean) {
setSelection((current) => {
const fields = checked ? Array.from(new Set([...current.fields, path])) : current.fields.filter((field) => field !== path);
return {
fields,
aliasMap: { ...current.aliasMap, [path]: current.aliasMap[path] || path },
};
const aliasMap = { ...current.aliasMap };
if (checked && !aliasMap[path]) aliasMap[path] = defaultAlias(path);
return { fields, aliasMap };
});
}
const currentTitle = view === "extraction" ? "Extraccion de documentos" : view === "tasks" ? "Tareas y resultados" : "Configuracion";
function selectAllFields() {
setSelection({
fields: Object.keys(flattened.schema),
aliasMap: Object.fromEntries(
Object.keys(flattened.schema).map((path) => [path, selection.aliasMap[path] || defaultAlias(path)]),
),
});
}
const currentTitle = view === "extraction" ? "Nueva extraccion" : view === "tasks" ? "Tareas y resultados" : "Configuracion";
const currentDescription = view === "extraction"
? "Selecciona una carpeta, descubre campos y crea una tarea de extraccion."
? "Asistente paso a paso para extraer datos de tus documentos."
: view === "tasks"
? "Consulta el historial de ejecuciones y las filas guardadas en la base de datos."
: "Administra la conexion al API Gateway local.";
@@ -265,45 +405,67 @@ function App() {
<aside className="sidebar">
<div className="brand">
<div className="brand-mark">IP</div>
<div>
<div className="brand-text">
<strong>IPSExtractor</strong>
<span>Desktop</span>
</div>
</div>
<nav className="side-nav" aria-label="Principal">
<button type="button" className={view === "extraction" ? "active" : ""} onClick={() => setView("extraction")}>
<FileSearch size={18} /> Extraccion
<FileSearch size={18} />
<span>Extraccion</span>
</button>
<button type="button" className={view === "tasks" ? "active" : ""} onClick={() => setView("tasks")}>
<Activity size={18} /> Tareas
<Activity size={18} />
<span>Tareas</span>
</button>
<button type="button" className={view === "settings" ? "active" : ""} onClick={() => setView("settings")}>
<Settings size={18} /> Configuracion
<Settings size={18} />
<span>Configuracion</span>
</button>
</nav>
<div className="sidebar-status">
<span>Gateway</span>
<StatusPill status={connectionStatus} text={connectionStatus === "ok" ? "Conectado" : connectionStatus === "busy" ? "Verificando" : connectionStatus === "error" ? "Error" : "Sin verificar"} />
<div className="sidebar-footer">
<div className={`sidebar-gateway ${connectionStatus}`}>
<div className={`gateway-dot ${connectionStatus}`} />
<span className="gateway-label">
{connectionStatus === "ok" ? "Conectado" : connectionStatus === "busy" ? "Verificando..." : connectionStatus === "error" ? "Sin conexion" : "Sin verificar"}
</span>
</div>
</div>
</aside>
<section className="workspace">
<header className="topbar">
<div>
<section className={`workspace ${view === "extraction" ? "workspace--wiz" : ""} ${view === "tasks" ? "workspace--tasks" : ""} ${view === "settings" ? "workspace--settings" : ""}`}>
{view === "extraction" ? (
<header className="topbar-inline">
<h1>{currentTitle}</h1>
<p>{currentDescription}</p>
</div>
<div className="topbar-actions">
<button type="button" className="icon-button" onClick={verifyConnection} disabled={busy}>
<Server size={16} /> Verificar API
<button type="button" className="btn btn-sm" onClick={verifyConnection} disabled={busy}>
<Server size={14} /> Verificar API
</button>
</div>
</header>
</header>
) : (
<header className="topbar">
<div>
<h1>{currentTitle}</h1>
<p>{currentDescription}</p>
</div>
<div className="topbar-actions">
<button type="button" className="icon-button" onClick={verifyConnection} disabled={busy}>
<Server size={16} /> Verificar API
</button>
</div>
</header>
)}
{view === "extraction" && (
<ExtractionView
<ExtractionWizard
step={extractionStep}
maxReached={maxReachedStep}
onStepChange={goToStep}
form={form}
updateForm={updateForm}
onModeChange={handleModeChange}
busy={busy}
error={error}
statusText={statusText}
@@ -311,20 +473,21 @@ function App() {
discoverSchema={discoverSchema}
estimate={estimate}
runExtraction={runExtraction}
exportCsv={exportCsv}
rows={rows}
selection={selection}
setSelection={setSelection}
schemaSearch={schemaSearch}
setSchemaSearch={setSchemaSearch}
filteredSchemaEntries={filteredSchemaEntries}
flattened={flattened}
schemaResponse={schemaResponse}
estimateJson={estimateJson}
activeTask={activeTask}
configJson={configJson}
modelsJson={modelsJson}
manifest={manifest}
ingestProgress={ingestProgress}
setDocType={setDocType}
toggleField={toggleField}
selectAllFields={selectAllFields}
canGoNext={canGoNext}
onBack={handleBack}
onNext={handleNext}
/>
)}
@@ -357,178 +520,41 @@ function App() {
);
}
interface ExtractionViewProps {
form: ExtractionFormState;
updateForm: (patch: Partial<ExtractionFormState>) => void;
busy: boolean;
error: string | null;
statusText: string;
selectRoot: () => Promise<void>;
discoverSchema: () => Promise<void>;
estimate: () => Promise<void>;
runExtraction: () => Promise<void>;
exportCsv: () => Promise<void>;
rows: RowsResponse | null;
selection: FieldSelection;
setSelection: React.Dispatch<React.SetStateAction<FieldSelection>>;
schemaSearch: string;
setSchemaSearch: React.Dispatch<React.SetStateAction<string>>;
filteredSchemaEntries: [string, string][];
flattened: { schema: Record<string, string>; sample: unknown };
estimateJson: unknown;
activeTask: Task | null;
configJson: unknown;
modelsJson: unknown;
setDocType: (index: number, patch: Partial<DocumentType>) => void;
toggleField: (path: string, checked: boolean) => void;
function TaskProgress({ result }: { result: Record<string, unknown> }) {
const processed = Number(result.processed || 0);
const total = Number(result.total || 0);
if (!total || total <= 0) return null;
const percent = Math.min(100, Math.round((processed / total) * 100));
return (
<div className="wiz-progress tasks-inline-progress">
<div className="wiz-progress-header">
<span>Procesando documentos</span>
<span>{processed} / {total}</span>
</div>
<div className="wiz-progress-track">
<div className="wiz-progress-fill" style={{ width: `${percent}%` }} />
</div>
</div>
);
}
function ExtractionView({
form,
updateForm,
busy,
error,
statusText,
selectRoot,
discoverSchema,
estimate,
runExtraction,
exportCsv,
rows,
selection,
setSelection,
schemaSearch,
setSchemaSearch,
filteredSchemaEntries,
flattened,
estimateJson,
activeTask,
configJson,
modelsJson,
setDocType,
toggleField,
}: ExtractionViewProps) {
return (
<>
<section className="grid two">
<Panel title="Carpeta y estructura">
<div className="path-row">
<label>
Carpeta a procesar
<input value={form.selectedLocalPath} readOnly placeholder="Selecciona la carpeta con los documentos" />
</label>
<button type="button" className="icon-button" onClick={selectRoot} disabled={busy}>
<FolderOpen size={16} /> Explorar
</button>
</div>
<div className="segmented">
<button type="button" className={form.mode === "radicados" ? "active" : ""} onClick={() => updateForm({ mode: "radicados" })}>
Por radicados
</button>
<button type="button" className={form.mode === "general" ? "active" : ""} onClick={() => updateForm({ mode: "general" })}>
Carpeta general
</button>
</div>
</Panel>
function TaskStatusIcon({ status }: { status: string }) {
if (status === "completed") return <CheckCircle2 size={15} className="task-icon--ok" />;
if (status === "failed") return <XCircle size={15} className="task-icon--err" />;
if (status === "running") return <Loader2 size={15} className="spin task-icon--run" />;
return <Clock size={15} className="task-icon--idle" />;
}
<Panel title="Opciones de procesamiento">
<div className="switch-row">
<label className="check"><input type="checkbox" checked={form.retryOnEmpty} onChange={(event) => updateForm({ retryOnEmpty: event.target.checked })} /> Reintentar filas vacias</label>
<label className="check"><input type="checkbox" checked={form.tiffEnabled} onChange={(event) => updateForm({ tiffEnabled: event.target.checked })} /> Rango TIFF</label>
</div>
{form.tiffEnabled && (
<div className="field-grid compact">
<label>Inicio<input value={form.tiffStart} onChange={(event) => updateForm({ tiffStart: event.target.value })} /></label>
<label>Fin<input value={form.tiffEnd} onChange={(event) => updateForm({ tiffEnd: event.target.value })} /></label>
</div>
)}
<label className="check block"><input type="checkbox" checked={form.filterEnabled} onChange={(event) => updateForm({ filterEnabled: event.target.checked })} /> Procesar solo radicados seleccionados</label>
{form.filterEnabled && <textarea rows={4} value={form.radicadoFilterText} onChange={(event) => updateForm({ radicadoFilterText: event.target.value })} placeholder="Uno por linea o separados por coma" />}
</Panel>
</section>
<section className="grid two">
<Panel title={form.mode === "radicados" ? "Tipos de documento" : "Keywords"}>
{form.mode === "radicados" ? (
<>
<div className="doc-list">
{form.docTypes.map((item, index) => (
<div className="doc-row" key={index}>
<input aria-label={`Nombre tipo ${index + 1}`} value={item.name} onChange={(event) => setDocType(index, { name: event.target.value })} placeholder="Nombre logico" />
<input aria-label={`Keyword tipo ${index + 1}`} value={item.keyword} onChange={(event) => setDocType(index, { keyword: event.target.value })} placeholder="Keyword / patron" />
<button type="button" className="square" aria-label="Eliminar tipo" onClick={() => updateForm({ docTypes: form.docTypes.filter((_, idx) => idx !== index) })}>
<Trash2 size={15} />
</button>
</div>
))}
</div>
<button type="button" className="icon-button secondary" onClick={() => updateForm({ docTypes: [...form.docTypes, { name: "", keyword: "" }] })}>
<Plus size={16} /> Añadir tipo
</button>
</>
) : (
<label>
Keywords
<input value={form.keywordsText} onChange={(event) => updateForm({ keywordsText: event.target.value })} placeholder="factura, cuenta" />
</label>
)}
</Panel>
<Panel title="Acciones">
<div className="action-grid">
<button type="button" className="icon-button primary" onClick={discoverSchema} disabled={busy || !form.selectedContainerPath}>
{busy ? <Loader2 className="spin" size={16} /> : <Search size={16} />} Descubrir campos
</button>
<button type="button" className="icon-button" onClick={estimate} disabled={busy || selection.fields.length === 0}>
<Database size={16} /> Estimar
</button>
<button type="button" className="icon-button primary" onClick={runExtraction} disabled={busy || selection.fields.length === 0}>
<Play size={16} /> Ejecutar extraccion
</button>
<button type="button" className="icon-button" onClick={exportCsv} disabled={busy || !rows?.rows.length}>
<Download size={16} /> Exportar CSV
</button>
</div>
<p className="status-line">{statusText}</p>
{error && <p className="error-line"><AlertCircle size={16} /> {error}</p>}
</Panel>
</section>
<section className="grid two wide-left">
<Panel title={`Selector de campos (${selection.fields.length} seleccionados)`}>
<div className="schema-tools">
<input aria-label="Buscar campo" value={schemaSearch} onChange={(event) => setSchemaSearch(event.target.value)} placeholder="Buscar campo..." />
<button type="button" className="icon-button" onClick={() => setSelection({ fields: Object.keys(flattened.schema), aliasMap: Object.fromEntries(Object.keys(flattened.schema).map((path) => [path, selection.aliasMap[path] || path])) })}>
<Check size={16} /> Todos
</button>
<button type="button" className="icon-button" onClick={() => setSelection({ fields: [], aliasMap: selection.aliasMap })}>
<RefreshCw size={16} /> Limpiar
</button>
</div>
<div className="schema-list">
{filteredSchemaEntries.length === 0 ? (
<p className="muted">Descubre campos para ver el schema.</p>
) : (
filteredSchemaEntries.map(([path, kind]) => (
<div className="schema-row" key={path}>
<label className="check">
<input type="checkbox" checked={selection.fields.includes(path)} onChange={(event) => toggleField(path, event.target.checked)} />
<span>{path}</span>
<small>{kind}</small>
</label>
<input value={selection.aliasMap[path] || path} onChange={(event) => setSelection((current) => ({ ...current, aliasMap: { ...current.aliasMap, [path]: event.target.value } }))} />
</div>
))
)}
</div>
</Panel>
<Panel title="Muestra / Respuesta">
<pre className="json-box">{JSON.stringify(flattened.sample || estimateJson || activeTask || configJson || modelsJson || {}, null, 2)}</pre>
</Panel>
</section>
</>
);
function formatRelativeTime(dateStr: string | null): string {
if (!dateStr) return "—";
const diff = Date.now() - new Date(dateStr).getTime();
const s = Math.floor(diff / 1000);
if (s < 60) return "hace un momento";
const m = Math.floor(s / 60);
if (m < 60) return `hace ${m} min`;
const h = Math.floor(m / 60);
if (h < 24) return `hace ${h} h`;
return `hace ${Math.floor(h / 24)} d`;
}
function TasksView({
@@ -548,38 +574,120 @@ function TasksView({
selectTask: (task: Task) => Promise<void>;
exportCsv: () => Promise<void>;
}) {
const [detailOpen, setDetailOpen] = useState(false);
return (
<>
<section className="content-actions">
<button type="button" className="icon-button" onClick={() => refreshTasks()} disabled={busy}>
<RefreshCw size={16} /> Actualizar
</button>
<button type="button" className="icon-button" onClick={exportCsv} disabled={busy || !rows?.rows.length}>
<Download size={16} /> Exportar CSV
</button>
</section>
<section className="grid two">
<Panel title="Historial de tareas">
<div className="task-list large">
{tasks.length === 0 ? <p className="muted">No hay tareas todavia.</p> : tasks.map((task) => (
<button type="button" className={`task-row ${activeTask?.id === task.id ? "active" : ""}`} key={task.id} onClick={() => selectTask(task)}>
<span>{task.title || task.id}</span>
<strong className={`task-status ${task.status}`}>{task.status}</strong>
<div className="tasks-layout">
{/* ── Panel izquierdo: lista ── */}
<aside className="tasks-sidebar">
<div className="tasks-sidebar-header">
<h2>Historial</h2>
<button type="button" className="btn btn-sm" onClick={() => refreshTasks()} disabled={busy}>
<RefreshCw size={13} className={busy ? "spin" : ""} /> Actualizar
</button>
</div>
{tasks.length === 0 ? (
<div className="tasks-empty">
<Activity size={32} strokeWidth={1.2} />
<p>No hay tareas todavia.</p>
<span>Las extracciones que ejecutes apareceran aqui.</span>
</div>
) : (
<div className="tasks-list">
{tasks.map((task) => (
<button
type="button"
key={task.id}
className={`tasks-item ${activeTask?.id === task.id ? "active" : ""}`}
onClick={() => selectTask(task)}
>
<div className="tasks-item-top">
<TaskStatusIcon status={task.status} />
<span className="tasks-item-title">{task.title || task.id.slice(0, 16) + "…"}</span>
</div>
<div className="tasks-item-meta">
<span className={`tasks-badge tasks-badge--${task.status}`}>{task.status}</span>
<span className="tasks-item-time">{formatRelativeTime(task.created_at)}</span>
</div>
{task.status === "running" && Number(task.result?.total || 0) > 0 && (
<div className="tasks-item-progress-track">
<div
className="tasks-item-progress-fill"
style={{
width: `${Math.min(100, Math.round((Number(task.result?.processed || 0) / Number(task.result?.total || 1)) * 100))}%`,
}}
/>
</div>
)}
</button>
))}
</div>
</Panel>
)}
</aside>
<Panel title={`Resultados${rows ? ` (${rows.rows.length})` : ""}`}>
<ResultsTable rows={rows?.rows || []} />
</Panel>
</section>
<section className="grid one">
<Panel title="Detalle de tarea">
<pre className="json-box compact">{JSON.stringify(activeTask || {}, null, 2)}</pre>
</Panel>
</section>
</>
{/* ── Panel derecho: detalle + resultados ── */}
<div className="tasks-main">
{!activeTask ? (
<div className="tasks-empty tasks-empty--main">
<Activity size={40} strokeWidth={1} />
<p>Selecciona una tarea para ver sus resultados</p>
</div>
) : (
<>
{/* Cabecera de tarea activa */}
<div className="tasks-detail-header">
<div className="tasks-detail-title-row">
<TaskStatusIcon status={activeTask.status} />
<h3>{activeTask.title || activeTask.id}</h3>
<span className={`tasks-badge tasks-badge--${activeTask.status}`}>{activeTask.status}</span>
</div>
<div className="tasks-detail-meta">
{activeTask.created_at && <span>Creada: {new Date(activeTask.created_at).toLocaleString("es-CO")}</span>}
{activeTask.finished_at && <span>Finalizada: {new Date(activeTask.finished_at).toLocaleString("es-CO")}</span>}
{activeTask.error && (
<span className="tasks-detail-error">
<AlertCircle size={13} /> {activeTask.error}
</span>
)}
</div>
{activeTask.status === "running" && <TaskProgress result={activeTask.result || {}} />}
<div className="tasks-detail-actions">
<button
type="button"
className="btn btn-sm btn-outline"
onClick={() => setDetailOpen((v) => !v)}
>
{detailOpen ? "Ocultar detalle" : "Ver detalle JSON"}
</button>
<button
type="button"
className="btn btn-primary btn-sm"
onClick={exportCsv}
disabled={busy || !rows?.rows.length}
>
<Download size={14} /> Exportar CSV
</button>
</div>
{detailOpen && (
<pre className="tasks-json-box">{JSON.stringify(activeTask, null, 2)}</pre>
)}
</div>
{/* Tabla de resultados */}
<div className="tasks-results">
<div className="tasks-results-header">
<span className="tasks-results-title">
Resultados
{rows && <span className="wiz-count-badge">{rows.rows.length} filas</span>}
</span>
</div>
<ResultsTable rows={rows?.rows || []} status={activeTask.status} />
</div>
</>
)}
</div>
</div>
);
}
@@ -602,46 +710,123 @@ function SettingsView({
configJson: unknown;
modelsJson: unknown;
}) {
const [backendTab, setBackendTab] = useState<"config" | "models">("config");
const connectionLabel =
connectionStatus === "ok"
? "Conectado"
: connectionStatus === "busy"
? "Verificando..."
: connectionStatus === "error"
? "Sin conexion"
: "Sin verificar";
const backendData = backendTab === "config" ? configJson : modelsJson;
const hasBackendData = backendData !== null && backendData !== undefined;
return (
<section className="settings-layout">
<Panel title="Conexion API">
<div className="settings-stack">
<label>
URL del API Gateway
<input value={form.apiBaseUrl} onChange={(event) => updateForm({ apiBaseUrl: event.target.value })} placeholder="http://localhost:8000" />
</label>
<div className="settings-actions">
<button type="button" className="icon-button primary" onClick={verifyConnection} disabled={busy}>
<Server size={16} /> Verificar conexion
<div className="settings-layout">
<aside className="settings-sidebar">
<div className="settings-sidebar-header">
<h2>Conexion API</h2>
</div>
<div className="settings-sidebar-body">
<div className={`settings-status-card settings-status-card--${connectionStatus}`}>
<div className={`settings-status-icon settings-status-icon--${connectionStatus}`}>
{connectionStatus === "ok" ? (
<CheckCircle2 size={20} />
) : connectionStatus === "error" ? (
<XCircle size={20} />
) : connectionStatus === "busy" ? (
<Loader2 size={20} className="spin" />
) : (
<Server size={20} />
)}
</div>
<div className="settings-status-text">
<strong>{connectionLabel}</strong>
<span>{connectionMessage}</span>
</div>
</div>
<div className="settings-field">
<label className="wiz-field-label" htmlFor="api-base-url">
URL del API Gateway
</label>
<input
id="api-base-url"
value={form.apiBaseUrl}
onChange={(event) => updateForm({ apiBaseUrl: event.target.value })}
placeholder="http://localhost:8000"
/>
<p className="wiz-hint">Direccion del API Gateway local. Los cambios se guardan automaticamente.</p>
</div>
<button type="button" className="btn btn-primary" onClick={verifyConnection} disabled={busy}>
<Server size={15} />
{busy ? "Verificando..." : "Verificar conexion"}
</button>
</div>
</aside>
<div className="settings-main">
<div className="settings-main-header">
<div className="settings-main-title">
<Database size={16} />
<h3>Configuracion del backend</h3>
</div>
<div className="settings-tabs" role="tablist" aria-label="Vista del backend">
<button
type="button"
role="tab"
aria-selected={backendTab === "config"}
className={backendTab === "config" ? "active" : ""}
onClick={() => setBackendTab("config")}
>
Configuracion
</button>
<button
type="button"
role="tab"
aria-selected={backendTab === "models"}
className={backendTab === "models" ? "active" : ""}
onClick={() => setBackendTab("models")}
>
Modelos
</button>
<StatusPill status={connectionStatus} text={connectionMessage} />
</div>
</div>
</Panel>
<Panel title="Configuracion del backend">
<pre className="json-box compact">{JSON.stringify({ config: configJson, models: modelsJson }, null, 2)}</pre>
</Panel>
</section>
<div className="settings-json-area">
{hasBackendData ? (
<pre className="settings-json-box">{JSON.stringify(backendData, null, 2)}</pre>
) : (
<div className="settings-empty">
<Server size={36} strokeWidth={1.2} />
<p>Sin datos del backend</p>
<span>Verifica la conexion para cargar la {backendTab === "config" ? "configuracion" : "lista de modelos"}.</span>
</div>
)}
</div>
</div>
</div>
);
}
function Panel({ title, children }: { title: string; children: React.ReactNode }) {
return (
<section className="panel">
<h2>{title}</h2>
{children}
</section>
);
}
function StatusPill({ status, text }: { status: Status; text: string }) {
return <div className={`status-pill ${status}`}>{status === "busy" && <Loader2 className="spin" size={14} />}{text}</div>;
}
function ResultsTable({ rows }: { rows: Array<Record<string, unknown>> }) {
function ResultsTable({ rows, status }: { rows: Array<Record<string, unknown>>; status?: string }) {
const columns = useMemo(() => Array.from(new Set(rows.flatMap((row) => Object.keys(row)))).slice(0, 12), [rows]);
if (rows.length === 0) return <p className="muted">Aun no hay filas extraidas.</p>;
if (rows.length === 0) {
const isRunning = status === "running" || status === "pending";
return (
<div className="tasks-table-empty">
{isRunning
? <><Loader2 size={24} className="spin" /><p>Extraccion en curso...</p></>
: <><Download size={24} strokeWidth={1.2} /><p>No hay filas extraidas aun.</p></>
}
</div>
);
}
return (
<div className="table-wrap">
<table>
@@ -0,0 +1,45 @@
import { Check } from "lucide-react";
export type ExtractionStep = 1 | 2 | 3 | 4;
const STEPS: Array<{ id: ExtractionStep; label: string }> = [
{ id: 1, label: "Carpeta" },
{ id: 2, label: "Descubrir" },
{ id: 3, label: "Seleccionar" },
{ id: 4, label: "Ejecutar" },
];
interface ExtractionStepperProps {
current: ExtractionStep;
maxReached: ExtractionStep;
onStepClick?: (step: ExtractionStep) => void;
}
export function ExtractionStepper({ current, maxReached, onStepClick }: ExtractionStepperProps) {
return (
<nav className="stepper" aria-label="Pasos de extraccion">
{STEPS.map((step, index) => {
const isDone = step.id < current;
const isActive = step.id === current;
const isClickable = step.id <= maxReached && !isActive;
const state = isDone ? "done" : isActive ? "active" : step.id <= maxReached ? "available" : "todo";
return (
<div className="stepper-item" key={step.id}>
<button
type="button"
className={`stepper-dot ${state}`}
disabled={!isClickable}
onClick={() => isClickable && onStepClick?.(step.id)}
aria-current={isActive ? "step" : undefined}
>
{isDone ? <Check size={14} strokeWidth={3} /> : step.id}
</button>
<span className={`stepper-label ${state}`}>{step.label}</span>
{index < STEPS.length - 1 && <div className={`stepper-connector ${isDone ? "done" : ""}`} />}
</div>
);
})}
</nav>
);
}
@@ -0,0 +1,631 @@
import {
AlertCircle,
ArrowRight,
Check,
ChevronDown,
ChevronRight,
Cpu,
Database,
FileText,
FolderOpen,
Layers,
Loader2,
Play,
Plus,
RefreshCw,
Search,
Settings2,
Trash2,
Zap,
} from "lucide-react";
import { useState } from "react";
import type { DocumentType, ExtractionFormState, FieldSelection, ManifestEntry, SchemaResponse } from "../../shared/types";
import { SchemaFieldPicker } from "./SchemaFieldPicker";
import { SampleTreeView } from "./SampleTreeView";
import { ExtractionStepper, type ExtractionStep } from "./ExtractionStepper";
interface ExtractionWizardProps {
step: ExtractionStep;
maxReached: ExtractionStep;
onStepChange: (step: ExtractionStep) => void;
form: ExtractionFormState;
updateForm: (patch: Partial<ExtractionFormState>) => void;
onModeChange: (mode: ExtractionFormState["mode"]) => Promise<void>;
busy: boolean;
error: string | null;
statusText: string;
selectRoot: () => Promise<void>;
discoverSchema: () => Promise<void>;
estimate: () => Promise<void>;
runExtraction: () => Promise<void>;
selection: FieldSelection;
setSelection: React.Dispatch<React.SetStateAction<FieldSelection>>;
schemaSearch: string;
setSchemaSearch: React.Dispatch<React.SetStateAction<string>>;
flattened: { schema: Record<string, string>; sample: unknown };
schemaResponse: SchemaResponse | null;
estimateJson: unknown;
manifest: ManifestEntry[];
ingestProgress: { processed: number; total: number } | null;
setDocType: (index: number, patch: Partial<DocumentType>) => void;
toggleField: (path: string, checked: boolean) => void;
selectAllFields: () => void;
canGoNext: boolean;
onBack: () => void;
onNext: () => void;
}
const MODELS = [
{ id: "gpt-5.4", label: "GPT-5.4", tier: "Máxima precisión", priceIn: 2.50, priceOut: 15.00 },
{ id: "gpt-5.4-mini", label: "GPT-5.4 Mini", tier: "Balance óptimo", priceIn: 0.75, priceOut: 4.50 },
{ id: "gpt-5.4-nano", label: "GPT-5.4 Nano", tier: "Más económico", priceIn: 0.20, priceOut: 0.80 },
{ id: "gpt-4.1-mini", label: "GPT-4.1 Mini", tier: "Legacy rápido", priceIn: 0.40, priceOut: 1.60 },
];
// Tipos del resultado del estimado que devuelve el backend
interface EstimateResult {
total_docs?: number;
extractor_usage?: { prompt_tokens?: number; completion_tokens?: number; prompt_tokens_details?: { cached_tokens?: number } };
normalizer_usage?: { prompt_tokens?: number; completion_tokens?: number };
per_doc_cost?: { input_usd?: number; cached_input_usd?: number; output_usd?: number; total_usd?: number };
estimated_total_cost?: { input_usd?: number; output_usd?: number; total_usd?: number };
extractor_model?: string;
normalizer_model?: string;
pages_per_doc?: number;
}
function fmtUsd(val: number | undefined): string {
if (val === undefined || val === null) return "—";
if (val < 0.001) return `$${(val * 1000).toFixed(3)}m`;
return `$${val.toFixed(4)}`;
}
function fmtTokens(val: number | undefined): string {
if (!val) return "0";
return val >= 1000 ? `${(val / 1000).toFixed(1)}k` : String(val);
}
export function ExtractionWizard({
step,
maxReached,
onStepChange,
form,
updateForm,
onModeChange,
busy,
error,
statusText,
selectRoot,
discoverSchema,
estimate,
runExtraction,
selection,
setSelection,
schemaSearch,
setSchemaSearch,
flattened,
schemaResponse,
estimateJson,
manifest,
ingestProgress,
setDocType,
toggleField,
selectAllFields,
canGoNext,
onBack,
onNext,
}: ExtractionWizardProps) {
const [advancedOpen, setAdvancedOpen] = useState(false);
const fieldCount = Object.keys(flattened.schema).length;
const ingestPercent = ingestProgress ? Math.round((ingestProgress.processed / ingestProgress.total) * 100) : 0;
return (
<div className="wiz">
<ExtractionStepper current={step} maxReached={maxReached} onStepClick={onStepChange} />
<div className="wiz-body">
{/* ── PASO 1: Carpeta ── */}
{step === 1 && (
<div className="wiz-step">
<div className="wiz-section">
<h3 className="wiz-section-title">Origen de los documentos</h3>
<p className="wiz-desc">
Elige como estan organizados tus archivos y selecciona la carpeta que contiene los documentos a procesar.
</p>
<div className="mode-switch">
<button
type="button"
className={form.mode === "radicados" ? "active" : ""}
onClick={() => onModeChange("radicados")}
disabled={busy}
>
<Layers size={15} />
Por radicados
</button>
<button
type="button"
className={form.mode === "general" ? "active" : ""}
onClick={() => onModeChange("general")}
disabled={busy}
>
<FolderOpen size={15} />
Carpeta general
</button>
</div>
</div>
<div className="wiz-section">
<label className="wiz-field-label">Carpeta a procesar</label>
<div className="folder-picker">
<div className="folder-picker-display">
{form.selectedLocalPath ? (
<>
<FolderOpen size={16} className="folder-picker-icon" />
<span className="folder-picker-path">{form.selectedLocalPath}</span>
</>
) : (
<span className="folder-picker-placeholder">Ninguna carpeta seleccionada</span>
)}
</div>
<button type="button" className="btn btn-primary" onClick={selectRoot} disabled={busy}>
<FolderOpen size={15} /> Explorar
</button>
</div>
{manifest.length > 0 && (
<div className="wiz-found-badge">
<FileText size={15} />
<span><strong>{manifest.length}</strong> documento(s) encontrados y listos para procesar</span>
</div>
)}
{form.mode === "general" && (
<p className="wiz-hint">Se procesaran todos los PDF y TIFF encontrados en la carpeta.</p>
)}
</div>
{form.mode === "radicados" && (
<div className="wiz-section">
<label className="wiz-field-label">Tipos de documento</label>
<p className="wiz-hint" style={{ marginTop: 0, marginBottom: 12 }}>
Define los tipos de documento y el patron de archivo dentro de cada radicado.
</p>
<div className="doc-type-list">
{form.docTypes.map((item, index) => (
<div className="doc-type-row" key={index}>
<input
aria-label={`Nombre tipo ${index + 1}`}
value={item.name}
onChange={(event) => setDocType(index, { name: event.target.value })}
placeholder="Nombre logico"
/>
<input
aria-label={`Keyword tipo ${index + 1}`}
value={item.keyword}
onChange={(event) => setDocType(index, { keyword: event.target.value })}
placeholder="Keyword / patron"
/>
<button
type="button"
className="btn-icon-danger"
aria-label="Eliminar tipo"
onClick={() => updateForm({ docTypes: form.docTypes.filter((_, idx) => idx !== index) })}
>
<Trash2 size={14} />
</button>
</div>
))}
</div>
<button
type="button"
className="btn btn-ghost"
onClick={() => updateForm({ docTypes: [...form.docTypes, { name: "", keyword: "" }] })}
>
<Plus size={15} /> Agregar tipo
</button>
</div>
)}
<details className="wiz-advanced" open={advancedOpen} onToggle={(e) => setAdvancedOpen((e.target as HTMLDetailsElement).open)}>
<summary className="wiz-advanced-trigger">
<Settings2 size={15} />
Opciones avanzadas
{advancedOpen ? <ChevronDown size={14} /> : <ChevronRight size={14} />}
</summary>
<div className="wiz-advanced-body">
<div className="wiz-checks-row">
<label className="check">
<input
type="checkbox"
checked={form.retryOnEmpty}
onChange={(event) => updateForm({ retryOnEmpty: event.target.checked })}
/>
Reintentar filas vacias
</label>
<label className="check">
<input
type="checkbox"
checked={form.tiffEnabled}
onChange={(event) => updateForm({ tiffEnabled: event.target.checked })}
/>
Rango TIFF
</label>
</div>
{form.tiffEnabled && (
<div className="wiz-inline-fields">
<label>
Inicio
<input value={form.tiffStart} onChange={(event) => updateForm({ tiffStart: event.target.value })} />
</label>
<label>
Fin
<input value={form.tiffEnd} onChange={(event) => updateForm({ tiffEnd: event.target.value })} />
</label>
</div>
)}
{form.mode === "radicados" && (
<>
<label className="check">
<input
type="checkbox"
checked={form.filterEnabled}
onChange={(event) => updateForm({ filterEnabled: event.target.checked })}
/>
Procesar solo radicados seleccionados
</label>
{form.filterEnabled && (
<textarea
rows={3}
value={form.radicadoFilterText}
onChange={(event) => updateForm({ radicadoFilterText: event.target.value })}
placeholder="Uno por linea o separados por coma"
/>
)}
</>
)}
</div>
</details>
</div>
)}
{/* ── PASO 2: Descubrir ── */}
{step === 2 && (
<div className="wiz-step wiz-step-centered">
<div className="wiz-hero">
<div className="wiz-hero-icon">
<Search size={28} />
</div>
<h3 className="wiz-hero-title">Modelo y descubrimiento</h3>
<p className="wiz-desc">
Elige el modelo de IA que usara la extraccion y descubre los campos disponibles en tus documentos.
</p>
</div>
{/* Selector de modelo */}
<div className="model-picker">
<div className="model-picker-label">
<Cpu size={14} /> Modelo extractor
</div>
<div className="model-picker-grid">
{MODELS.map((m) => (
<button
key={m.id}
type="button"
className={`model-card ${form.model === m.id ? "model-card--active" : ""}`}
onClick={() => updateForm({ model: m.id })}
disabled={busy}
>
<span className="model-card-name">{m.label}</span>
<span className="model-card-tier">{m.tier}</span>
<span className="model-card-price">
${m.priceIn}/<span className="model-card-price-out">${m.priceOut}</span>
<span className="model-card-price-unit"> /1M</span>
</span>
</button>
))}
</div>
<p className="wiz-hint" style={{ marginTop: 6 }}>
El normalizador siempre usa <strong>GPT-5.4 Mini</strong> ($0.75 in / $4.50 out /1M tokens).
</p>
</div>
<div className="wiz-stats-row">
<div className="wiz-stat">
<span className="wiz-stat-value">{manifest.length}</span>
<span className="wiz-stat-label">Documentos</span>
</div>
<div className="wiz-stat">
<span className="wiz-stat-value">{form.mode === "radicados" ? "Radicados" : "General"}</span>
<span className="wiz-stat-label">Modo</span>
</div>
<div className="wiz-stat">
<span className="wiz-stat-value">{fieldCount}</span>
<span className="wiz-stat-label">Campos detectados</span>
</div>
</div>
<button
type="button"
className="btn btn-primary btn-lg"
onClick={discoverSchema}
disabled={busy || manifest.length === 0}
>
{busy ? <Loader2 className="spin" size={18} /> : <Search size={18} />}
{schemaResponse ? "Volver a descubrir" : "Descubrir campos"}
</button>
{fieldCount > 0 && (
<div className="wiz-success-msg">
<Check size={16} />
{fieldCount} campos detectados. Continua al siguiente paso para elegir cuales exportar.
</div>
)}
</div>
)}
{/* ── PASO 3: Seleccionar ── */}
{step === 3 && (
<div className="wiz-step wiz-step-split">
<div className="wiz-split-main">
<div className="wiz-section-header">
<h3 className="wiz-section-title">
Campos a extraer
<span className="wiz-count-badge">{selection.fields.length} / {fieldCount}</span>
</h3>
<div className="wiz-toolbar">
<div className="wiz-search-box">
<Search size={14} />
<input
aria-label="Buscar campo"
value={schemaSearch}
onChange={(event) => setSchemaSearch(event.target.value)}
placeholder="Buscar campo..."
/>
</div>
<button type="button" className="btn btn-sm" onClick={selectAllFields}>
<Check size={14} /> Todos
</button>
<button
type="button"
className="btn btn-sm"
onClick={() => setSelection({ fields: [], aliasMap: selection.aliasMap })}
>
<RefreshCw size={14} /> Limpiar
</button>
</div>
</div>
<SchemaFieldPicker
schema={flattened.schema}
sample={flattened.sample}
search={schemaSearch}
selection={selection}
onToggleField={toggleField}
onAliasChange={(path, alias) =>
setSelection((current) => ({ ...current, aliasMap: { ...current.aliasMap, [path]: alias } }))
}
onSelectAll={selectAllFields}
onClearAll={() => setSelection({ fields: [], aliasMap: selection.aliasMap })}
/>
</div>
<div className="wiz-split-aside">
<h3 className="wiz-section-title">
Vista previa
{selection.fields.length > 0 && (
<span className="wiz-count-badge">{selection.fields.length} seleccionados</span>
)}
</h3>
{flattened.sample ? (
<SampleTreeView sample={flattened.sample} selectedPaths={selection.fields} />
) : (
<p className="muted">Descubre campos para ver la muestra.</p>
)}
</div>
</div>
)}
{/* ── PASO 4: Ejecutar ── */}
{step === 4 && (
<div className="wiz-step wiz-step-centered">
<div className="wiz-hero">
<div className="wiz-hero-icon wiz-hero-icon--go">
<Zap size={28} />
</div>
<h3 className="wiz-hero-title">Listo para extraer</h3>
<p className="wiz-desc">
Revisa el resumen y lanza la extraccion. Los archivos se suben uno a uno sin guardarse en el servidor.
</p>
</div>
<div className="wiz-review-grid">
<div className="wiz-review-item">
<FolderOpen size={18} />
<div>
<span className="wiz-review-label">Carpeta</span>
<span className="wiz-review-value" title={form.selectedLocalPath}>
{form.selectedLocalPath || "—"}
</span>
</div>
</div>
<div className="wiz-review-item">
<FileText size={18} />
<div>
<span className="wiz-review-label">Documentos</span>
<span className="wiz-review-value">{manifest.length}</span>
</div>
</div>
<div className="wiz-review-item">
<Database size={18} />
<div>
<span className="wiz-review-label">Campos seleccionados</span>
<span className="wiz-review-value">{selection.fields.length}</span>
</div>
</div>
<div className="wiz-review-item">
<Layers size={18} />
<div>
<span className="wiz-review-label">Modo</span>
<span className="wiz-review-value">{form.mode === "radicados" ? "Por radicados" : "General"}</span>
</div>
</div>
</div>
<div className="wiz-action-group">
<button type="button" className="btn btn-outline" onClick={estimate} disabled={busy || selection.fields.length === 0}>
{busy ? <Loader2 className="spin" size={15} /> : <Database size={15} />} Estimar costo
</button>
<button type="button" className="btn btn-primary btn-lg" onClick={runExtraction} disabled={busy || selection.fields.length === 0}>
{busy ? <Loader2 className="spin" size={18} /> : <Play size={18} />}
Ejecutar extraccion
</button>
</div>
{estimateJson != null && (() => {
const est = estimateJson as EstimateResult;
const extUsage = est.extractor_usage;
const normUsage = est.normalizer_usage;
const perDoc = est.per_doc_cost;
const total = est.estimated_total_cost;
const docs = est.total_docs ?? manifest.length;
const extModel = est.extractor_model ?? form.model;
const normModel = est.normalizer_model ?? "gpt-5.4-mini";
const extPrompt = extUsage?.prompt_tokens ?? 0;
const extCompl = extUsage?.completion_tokens ?? 0;
const extCached = extUsage?.prompt_tokens_details?.cached_tokens ?? 0;
const normPrompt = normUsage?.prompt_tokens ?? 0;
const normCompl = normUsage?.completion_tokens ?? 0;
const pages = est.pages_per_doc ?? 1;
return (
<div className="est-card">
<div className="est-card-header">
<Database size={15} />
<span>Estimado de costo — muestra de 1 documento × {docs}</span>
</div>
<div className="est-phases">
{/* Extractor */}
<div className="est-phase">
<div className="est-phase-title">
<Cpu size={13} /> {extModel}
<span className="est-phase-role">Extractor</span>
</div>
<div className="est-phase-tokens">
<span className="est-token-row">
<span className="est-token-label">Entrada</span>
<span className="est-token-val">{fmtTokens(extPrompt)} tok</span>
<span className="est-token-hint">{pages} pág(s) imagen + prompt</span>
</span>
{extCached > 0 && (
<span className="est-token-row">
<span className="est-token-label">Cacheado</span>
<span className="est-token-val est-token-cached">{fmtTokens(extCached)} tok</span>
</span>
)}
<span className="est-token-row">
<span className="est-token-label">Salida</span>
<span className="est-token-val">{fmtTokens(extCompl)} tok</span>
<span className="est-token-hint">JSON extraído</span>
</span>
</div>
</div>
<ArrowRight size={14} className="est-arrow" />
{/* Normalizador */}
<div className="est-phase">
<div className="est-phase-title">
<Cpu size={13} /> {normModel}
<span className="est-phase-role">Normalizador</span>
</div>
<div className="est-phase-tokens">
<span className="est-token-row">
<span className="est-token-label">Entrada</span>
<span className="est-token-val">{fmtTokens(normPrompt)} tok</span>
<span className="est-token-hint">JSON + esquema</span>
</span>
<span className="est-token-row">
<span className="est-token-label">Salida</span>
<span className="est-token-val">{fmtTokens(normCompl)} tok</span>
<span className="est-token-hint">Datos normalizados</span>
</span>
</div>
</div>
</div>
<div className="est-totals">
<div className="est-total-row">
<span className="est-total-label">Por documento</span>
<span className="est-total-val">{fmtUsd(perDoc?.total_usd)}</span>
</div>
<div className="est-total-row est-total-row--big">
<span className="est-total-label">Total estimado ({docs} doc{docs !== 1 ? "s" : ""})</span>
<span className="est-total-val est-total-grand">{fmtUsd(total?.total_usd)}</span>
</div>
<p className="est-disclaimer">
Estimado basado en 1 documento de muestra. El costo real puede variar según el numero de páginas y la complejidad del contenido.
</p>
</div>
</div>
);
})()}
{ingestProgress && (
<div className="wiz-progress">
<div className="wiz-progress-header">
<span>Subiendo documentos</span>
<span>{ingestProgress.processed} / {ingestProgress.total}</span>
</div>
<div className="wiz-progress-track">
<div className="wiz-progress-fill" style={{ width: `${ingestPercent}%` }} />
</div>
</div>
)}
</div>
)}
</div>
{/* ── Status / Errores ── */}
{(statusText || error) && (
<div className={`wiz-toast ${error ? "wiz-toast--error" : ""}`}>
{error ? (
<>
<AlertCircle size={15} />
<span>{error}</span>
</>
) : (
<span>{statusText}</span>
)}
</div>
)}
{/* ── Footer de navegacion ── */}
<footer className="wiz-nav">
<button type="button" className="btn" onClick={onBack} disabled={busy || step === 1}>
Atras
</button>
<div className="wiz-nav-dots">
{([1, 2, 3, 4] as ExtractionStep[]).map((id) => (
<button
key={id}
type="button"
className={`wiz-nav-dot ${step === id ? "active" : ""}`}
onClick={() => id <= maxReached && onStepChange(id)}
disabled={id > maxReached || busy}
aria-label={`Ir al paso ${id}`}
/>
))}
</div>
{step < 4 ? (
<button type="button" className="btn btn-primary" onClick={onNext} disabled={busy || !canGoNext}>
Siguiente
</button>
) : (
<button type="button" className="btn btn-primary" onClick={runExtraction} disabled={busy || selection.fields.length === 0}>
{busy ? <Loader2 className="spin" size={16} /> : <Play size={16} />} Ejecutar
</button>
)}
</footer>
</div>
);
}
@@ -0,0 +1,163 @@
import { ChevronDown, ChevronRight } from "lucide-react";
import { useMemo, useState } from "react";
interface JsonNodeProps {
value: unknown;
depth?: number;
pathPrefix?: string;
selectedPaths: Set<string>;
}
function isPlainObject(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}
function ValueBadge({ value }: { value: unknown }) {
if (value === null || value === undefined) {
return <span className="stv-null">vacío</span>;
}
if (typeof value === "boolean") {
return <span className="stv-bool">{value ? "verdadero" : "falso"}</span>;
}
if (typeof value === "number") {
return <span className="stv-number">{String(value)}</span>;
}
return <span className="stv-string">{String(value)}</span>;
}
function CollapsibleNode({
label,
count,
children,
depth,
isArray,
highlighted,
}: {
label: string;
count: number;
children: React.ReactNode;
depth: number;
isArray?: boolean;
highlighted?: boolean;
}) {
const [open, setOpen] = useState(depth < 2);
return (
<div className={`stv-group ${highlighted ? "stv-group--hl" : ""}`}>
<button type="button" className="stv-group-header" onClick={() => setOpen((v) => !v)}>
<span className="stv-chevron">{open ? <ChevronDown size={13} /> : <ChevronRight size={13} />}</span>
<span className={`stv-key ${highlighted ? "stv-key--hl" : ""}`}>{label}</span>
<span className="stv-badge">{isArray ? `[${count}]` : `{${count}}`}</span>
</button>
{open && <div className="stv-group-body">{children}</div>}
</div>
);
}
function JsonNode({ value, depth = 0, pathPrefix = "", selectedPaths }: JsonNodeProps) {
if (Array.isArray(value)) {
if (value.length === 0) return <span className="stv-null">[vacío]</span>;
return (
<>
{value.map((item, index) => {
const itemPath = `${pathPrefix}[${index}]`;
return (
<div className="stv-row" key={index}>
<span className="stv-key stv-index">[{index + 1}]</span>
<div className="stv-val">
{isPlainObject(item) || Array.isArray(item) ? (
<JsonNode value={item} depth={depth + 1} pathPrefix={itemPath} selectedPaths={selectedPaths} />
) : (
<ValueBadge value={item} />
)}
</div>
</div>
);
})}
</>
);
}
if (isPlainObject(value)) {
return (
<>
{Object.entries(value).map(([key, child]) => {
const fullPath = pathPrefix ? `${pathPrefix}.${key}` : key;
const isSelected = selectedPaths.has(fullPath);
// check if any selected path starts with this prefix (child is selected)
const hasSelectedDescendant = !isSelected && Array.from(selectedPaths).some(
(p) => p === fullPath || p.startsWith(`${fullPath}.`) || p.startsWith(`${fullPath}[`)
);
if (isPlainObject(child)) {
return (
<CollapsibleNode
key={key}
label={key}
count={Object.keys(child).length}
depth={depth}
highlighted={hasSelectedDescendant}
>
<JsonNode value={child} depth={depth + 1} pathPrefix={fullPath} selectedPaths={selectedPaths} />
</CollapsibleNode>
);
}
if (Array.isArray(child)) {
return (
<CollapsibleNode
key={key}
label={key}
count={child.length}
depth={depth}
isArray
highlighted={hasSelectedDescendant}
>
<JsonNode value={child} depth={depth + 1} pathPrefix={fullPath} selectedPaths={selectedPaths} />
</CollapsibleNode>
);
}
return (
<div className={`stv-row ${isSelected ? "stv-row--selected" : ""}`} key={key}>
<span className={`stv-key ${isSelected ? "stv-key--hl" : ""}`}>{key}</span>
<div className="stv-val">
<ValueBadge value={child} />
</div>
</div>
);
})}
</>
);
}
return <ValueBadge value={value} />;
}
export function SampleTreeView({ sample, selectedPaths = [] }: { sample: unknown; selectedPaths?: string[] }) {
const selectedSet = useMemo(() => new Set(selectedPaths), [selectedPaths]);
const normalized = useMemo(() => {
if (!sample || typeof sample !== "object") return sample;
const record = sample as Record<string, unknown>;
if ("radicado" in record) {
const { radicado, ...rest } = record;
const keys = Object.keys(rest);
if (keys.length === 1 && isPlainObject(rest[keys[0]])) {
return { radicado, ...(rest[keys[0]] as Record<string, unknown>) };
}
return record;
}
return sample;
}, [sample]);
if (!normalized || (typeof normalized === "object" && Object.keys(normalized as object).length === 0)) {
return <p className="muted">Sin muestra disponible.</p>;
}
return (
<div className="stv-root">
<JsonNode value={normalized} selectedPaths={selectedSet} />
</div>
);
}
@@ -0,0 +1,146 @@
import { ChevronDown, ChevronRight } from "lucide-react";
import { useMemo, useState } from "react";
import type { FieldSelection } from "../../shared/types";
import {
buildSchemaFieldNodes,
defaultAlias,
filterSchemaFieldNodes,
groupSchemaFields,
type SchemaFieldGroup,
} from "../lib/schemaDisplay";
interface SchemaFieldPickerProps {
schema: Record<string, string>;
sample: unknown;
search: string;
selection: FieldSelection;
onToggleField: (path: string, checked: boolean) => void;
onAliasChange: (path: string, alias: string) => void;
onSelectAll: () => void;
onClearAll: () => void;
}
function GroupSection({
group,
selection,
onToggleField,
onAliasChange,
onToggleGroup,
expanded,
}: {
group: SchemaFieldGroup;
selection: FieldSelection;
onToggleField: (path: string, checked: boolean) => void;
onAliasChange: (path: string, alias: string) => void;
onToggleGroup: () => void;
expanded: boolean;
}) {
const selectedInGroup = group.fields.filter((field) => selection.fields.includes(field.path)).length;
function toggleGroupSelection(checked: boolean) {
for (const field of group.fields) {
onToggleField(field.path, checked);
}
}
return (
<section className="schema-group">
<header className="schema-group-header">
<button type="button" className="schema-group-toggle" onClick={onToggleGroup} aria-expanded={expanded}>
{expanded ? <ChevronDown size={16} /> : <ChevronRight size={16} />}
<span className="schema-group-title">{group.label}</span>
<span className="schema-group-count">{selectedInGroup}/{group.fields.length}</span>
</button>
<label className="check compact">
<input
type="checkbox"
checked={selectedInGroup === group.fields.length && group.fields.length > 0}
onChange={(event) => toggleGroupSelection(event.target.checked)}
/>
Grupo
</label>
</header>
{expanded && (
<div className="schema-group-body">
{group.fields.map((field) => (
<div className="schema-field-row" key={field.path}>
<label className="schema-field-main check" title={field.path}>
<input
type="checkbox"
aria-label={`Seleccionar ${field.label}`}
checked={selection.fields.includes(field.path)}
onChange={(event) => onToggleField(field.path, event.target.checked)}
/>
<span className="schema-field-label">{field.label}</span>
<span className="schema-kind">{field.kind}</span>
</label>
<div className="schema-field-preview" title={field.sampleValue ?? "Sin muestra"}>
{field.sampleValue ?? <span className="muted">—</span>}
</div>
<input
className="schema-alias-input"
aria-label={`Alias para ${field.label}`}
placeholder={defaultAlias(field.path)}
value={selection.aliasMap[field.path] ?? ""}
onChange={(event) => onAliasChange(field.path, event.target.value)}
/>
</div>
))}
</div>
)}
</section>
);
}
export function SchemaFieldPicker({
schema,
sample,
search,
selection,
onToggleField,
onAliasChange,
}: SchemaFieldPickerProps) {
const groups = useMemo(() => {
const nodes = filterSchemaFieldNodes(buildSchemaFieldNodes(schema, sample), search);
return groupSchemaFields(nodes);
}, [schema, sample, search]);
const [expanded, setExpanded] = useState<Record<string, boolean>>({});
const allExpanded = groups.length > 0 && groups.every((group) => expanded[group.key] !== false);
function toggleGroup(key: string) {
setExpanded((current) => ({ ...current, [key]: !(current[key] ?? true) }));
}
function setAllGroups(next: boolean) {
setExpanded(Object.fromEntries(groups.map((group) => [group.key, next])));
}
if (groups.length === 0) {
return <p className="muted">Descubre campos para ver el schema.</p>;
}
return (
<div className="schema-picker">
<div className="schema-picker-toolbar">
<button type="button" className="text-button" onClick={() => setAllGroups(true)}>Expandir grupos</button>
<button type="button" className="text-button" onClick={() => setAllGroups(false)}>Contraer grupos</button>
<span className="muted">{allExpanded ? "Vista expandida" : "Vista parcial"}</span>
</div>
<div className="schema-group-list">
{groups.map((group) => (
<GroupSection
key={group.key}
group={group}
selection={selection}
onToggleField={onToggleField}
onAliasChange={onAliasChange}
onToggleGroup={() => toggleGroup(group.key)}
expanded={expanded[group.key] ?? true}
/>
))}
</div>
</div>
);
}
@@ -0,0 +1,22 @@
import { describe, expect, it } from "vitest";
import { ApiClient } from "./apiClient";
describe("ApiClient error handling", () => {
it("parses JSON error bodies", async () => {
const client = new ApiClient("http://localhost:8000");
const fetchMock = vi.fn().mockResolvedValue(
new Response(JSON.stringify({ detail: "No se encontraron documentos" }), { status: 400 }),
);
vi.stubGlobal("fetch", fetchMock);
await expect(client.health()).rejects.toThrow("No se encontraron documentos");
});
it("falls back to plain text error bodies", async () => {
const client = new ApiClient("http://localhost:8000");
const fetchMock = vi.fn().mockResolvedValue(new Response("Internal Server Error", { status: 500 }));
vi.stubGlobal("fetch", fetchMock);
await expect(client.health()).rejects.toThrow("Internal Server Error");
});
});
@@ -21,6 +21,14 @@ export class ApiClient {
return this.handle<T>(response);
}
async postForm<T>(path: string, form: FormData): Promise<T> {
const response = await fetch(this.url(path), {
method: "POST",
body: form,
});
return this.handle<T>(response);
}
async health(): Promise<unknown> {
return this.getJson("/health");
}
@@ -37,10 +45,18 @@ export class ApiClient {
return this.postJson<SchemaResponse>("/extraction/schemas", payload);
}
async discoverSchemaUpload(form: FormData): Promise<SchemaResponse> {
return this.postForm<SchemaResponse>("/extraction/schemas/upload", form);
}
async estimate(payload: Record<string, unknown>): Promise<unknown> {
return this.postJson("/extraction/estimate", payload);
}
async estimateUpload(form: FormData): Promise<unknown> {
return this.postForm("/extraction/estimate/upload", form);
}
async createBatch(payload: Record<string, unknown>): Promise<ExtractionBatch> {
return this.postJson<ExtractionBatch>("/extraction/batches", payload);
}
@@ -49,6 +65,14 @@ export class ApiClient {
return this.postJson<Task>("/tasks/extraction", payload);
}
async ingestTaskDocument(taskId: string, form: FormData): Promise<Record<string, unknown>> {
return this.postForm<Record<string, unknown>>(`/tasks/${encodeURIComponent(taskId)}/ingest`, form);
}
async ingestTaskDocumentsBulk(taskId: string, form: FormData): Promise<Record<string, unknown>> {
return this.postForm<Record<string, unknown>>(`/tasks/${encodeURIComponent(taskId)}/ingest-many`, form);
}
async tasks(type = "extraction"): Promise<Task[]> {
return this.getJson<Task[]>(`/tasks?type=${encodeURIComponent(type)}`);
}
@@ -67,9 +91,19 @@ export class ApiClient {
private async handle<T>(response: Response): Promise<T> {
const text = await response.text();
const data = text ? JSON.parse(text) : null;
let data: unknown = null;
if (text) {
try {
data = JSON.parse(text);
} catch {
data = null;
}
}
if (!response.ok) {
const detail = typeof data?.detail === "string" ? data.detail : response.statusText;
const detail =
typeof (data as { detail?: unknown } | null)?.detail === "string"
? (data as { detail: string }).detail
: text || response.statusText;
throw new Error(detail || `HTTP ${response.status}`);
}
return data as T;
@@ -1,6 +1,6 @@
import { describe, expect, it } from "vitest";
import type { ExtractionFormState } from "../../shared/types";
import { buildBatchPayload, buildEstimatePayload, buildSchemaPayload, flattenSchema } from "./payloads";
import type { ExtractionFormState, ManifestEntry } from "../../shared/types";
import { buildDesktopBatchPayload, buildEstimatePayload, flattenSchema } from "./payloads";
const baseForm: ExtractionFormState = {
apiBaseUrl: "http://localhost:8000",
@@ -19,36 +19,20 @@ const baseForm: ExtractionFormState = {
tiffEnd: "3",
};
const manifest: ManifestEntry[] = [
{ radicado: "R1", filename: "factura.pdf", docType: "Factura", localPath: "C:\\docs\\lote\\R1\\factura.pdf", size: 10 },
];
describe("payload builders", () => {
it("builds schema payload for radicados", () => {
expect(buildSchemaPayload(baseForm)).toEqual({
root_path: "/data/lote",
doc_types: [{ name: "Factura", keyword: "factura" }],
tiff_page_range: [1, 3],
structure_mode: "radicados",
});
});
it("builds schema payload for general mode with a document path", () => {
expect(buildSchemaPayload({ ...baseForm, mode: "general" }, "/data/lote/factura.pdf")).toEqual({
document_path: "/data/lote/factura.pdf",
tiff_page_range: [1, 3],
structure_mode: "general",
});
});
it("builds batch payload with fields, aliases and filters", () => {
it("builds desktop batch payload with manifest", () => {
expect(
buildBatchPayload(baseForm, {
fields: ["Factura/numero"],
aliasMap: { "Factura/numero": "numero" },
}),
buildDesktopBatchPayload(baseForm, { fields: ["Factura/numero"], aliasMap: { "Factura/numero": "numero" } }, manifest),
).toMatchObject({
root_path: "/data/lote",
source: "desktop",
fields: ["Factura/numero"],
alias_map: { "Factura/numero": "numero" },
retry_on_empty: true,
radicado_filter: ["R1", "R2"],
documents: [{ radicado: "R1", filename: "factura.pdf", doc_type: "Factura" }],
});
});
@@ -60,12 +44,35 @@ describe("payload builders", () => {
{ fields: ["Factura/numero"], aliasMap: {} },
),
).toMatchObject({
root_path: "/data/lote",
root_path: "C:\\docs\\lote",
doc_types: [{ name: "Factura", keyword: "factura" }],
selected_paths: ["Factura/numero"],
});
});
it("omits keywords in general mode batch payload", () => {
const generalForm: ExtractionFormState = { ...baseForm, mode: "general" };
const payload = buildDesktopBatchPayload(
generalForm,
{ fields: ["numero"], aliasMap: { numero: "numero" } },
[{ radicado: "doc-1", filename: "factura.pdf", docType: undefined, localPath: "C:\\docs\\factura.pdf", size: 10 }],
);
expect(payload).not.toHaveProperty("keywords");
expect(payload).toMatchObject({ structure_mode: "general", source: "desktop" });
});
it("omits keywords in general mode estimate payload", () => {
const generalForm: ExtractionFormState = { ...baseForm, mode: "general" };
expect(
buildEstimatePayload(generalForm, { schema: { numero: "str" } }, { fields: ["numero"], aliasMap: {} }),
).toEqual({
root_path: "C:\\docs\\lote",
structure_mode: "general",
schema: { numero: "str" },
selected_fields: ["numero"],
});
});
it("flattens multi-document schema", () => {
expect(
flattenSchema({
+118 -38
View File
@@ -1,5 +1,5 @@
import type { DocumentType, ExtractionFormState, FieldSelection, SchemaResponse } from "../../shared/types";
import { splitKeywords, splitRadicados } from "./pathMapping";
import type { DocumentType, ExtractionFormState, FieldSelection, ManifestEntry, SchemaResponse } from "../../shared/types";
import { splitRadicados } from "./pathMapping";
export function parseTiffRange(form: Pick<ExtractionFormState, "tiffEnabled" | "tiffStart" | "tiffEnd">): [number, number] | undefined {
if (!form.tiffEnabled) return undefined;
@@ -21,30 +21,85 @@ export function validDocTypes(docTypes: DocumentType[]): DocumentType[] {
.filter((item) => item.name && item.keyword);
}
export function buildSchemaPayload(form: ExtractionFormState, documentPath?: string): Record<string, unknown> {
export function appendTiffRange(form: ExtractionFormState, formData: FormData): void {
const tiffRange = parseTiffRange(form);
if (form.mode === "general") {
if (!documentPath) throw new Error("No se encontro un documento de muestra en la carpeta general.");
return {
document_path: documentPath,
tiff_page_range: tiffRange,
structure_mode: form.mode,
};
}
if (!tiffRange) return;
formData.set("tiff_start", String(tiffRange[0]));
formData.set("tiff_end", String(tiffRange[1]));
}
const docTypes = validDocTypes(form.docTypes);
if (docTypes.length === 0) throw new Error("Agrega al menos un tipo de documento.");
return {
root_path: form.selectedContainerPath,
doc_types: docTypes,
export function buildSchemaUploadForm(
form: ExtractionFormState,
file: Blob,
filename: string,
docTypeName?: string,
): FormData {
const payload = new FormData();
payload.append("file", file, filename);
payload.append("structure_mode", form.mode);
if (docTypeName) payload.append("doc_type_name", docTypeName);
appendTiffRange(form, payload);
return payload;
}
export function buildDesktopBatchPayload(
form: ExtractionFormState,
selection: FieldSelection,
manifest: ManifestEntry[],
model?: string,
): Record<string, unknown> {
const tiffRange = parseTiffRange(form);
const documents = manifest.map((entry) => ({
radicado: entry.radicado,
filename: entry.filename,
doc_type: entry.docType || null,
}));
const uniqueRadicados = new Set(manifest.map((entry) => entry.radicado));
const payload: Record<string, unknown> = {
source: "desktop",
fields: selection.fields,
alias_map: selection.aliasMap,
tiff_page_range: tiffRange,
structure_mode: form.mode,
retry_on_empty: form.retryOnEmpty,
documents,
document_count: uniqueRadicados.size || documents.length,
model: model || null,
};
if (form.mode === "radicados") {
payload.doc_types = validDocTypes(form.docTypes);
if (form.filterEnabled) {
const allowed = new Set(splitRadicados(form.radicadoFilterText));
payload.documents = documents.filter((item) => allowed.has(String(item.radicado)));
payload.document_count = new Set((payload.documents as Array<{ radicado: string }>).map((item) => item.radicado)).size;
}
}
return payload;
}
export function buildEstimateUploadForm(
form: ExtractionFormState,
file: Blob,
filename: string,
docCount: number,
selectedPaths: string[],
model?: string,
): FormData {
const payload = new FormData();
payload.append("file", file, filename);
payload.append("doc_count", String(Math.max(1, docCount)));
payload.append("structure_mode", form.mode);
payload.append("selected_paths", JSON.stringify(selectedPaths));
if (model) payload.append("model", model);
appendTiffRange(form, payload);
return payload;
}
export function buildEstimatePayload(form: ExtractionFormState, schema: SchemaResponse | null, selection: FieldSelection): Record<string, unknown> {
const base = {
root_path: form.selectedContainerPath,
root_path: form.selectedLocalPath || "desktop://upload",
structure_mode: form.mode,
};
if (form.mode === "radicados") {
@@ -57,32 +112,11 @@ export function buildEstimatePayload(form: ExtractionFormState, schema: SchemaRe
}
return {
...base,
keywords: splitKeywords(form.keywordsText),
schema: schema?.schema,
selected_fields: selection.fields,
};
}
export function buildBatchPayload(form: ExtractionFormState, selection: FieldSelection): Record<string, unknown> {
const tiffRange = parseTiffRange(form);
const payload: Record<string, unknown> = {
root_path: form.selectedContainerPath,
fields: selection.fields,
alias_map: selection.aliasMap,
tiff_page_range: tiffRange,
structure_mode: form.mode,
retry_on_empty: form.retryOnEmpty,
radicado_filter: form.filterEnabled ? splitRadicados(form.radicadoFilterText) : undefined,
};
if (form.mode === "radicados") {
payload.doc_types = validDocTypes(form.docTypes);
} else {
payload.keywords = splitKeywords(form.keywordsText);
}
return payload;
}
export function flattenSchema(response: SchemaResponse): { schema: Record<string, string>; sample: unknown } {
if (response.schema) return { schema: response.schema, sample: response.sample };
const schema: Record<string, string> = {};
@@ -97,3 +131,49 @@ export function flattenSchema(response: SchemaResponse): { schema: Record<string
};
return { schema, sample };
}
export function mergeSchemaResponses(responses: SchemaResponse[]): SchemaResponse {
if (responses.length === 1) return responses[0];
const merged: SchemaResponse = {};
const schemaByType: Record<string, Record<string, string>> = {};
const sampleByType: Record<string, unknown> = {};
for (const response of responses) {
if (response.schema) {
return response;
}
for (const [typeName, fields] of Object.entries(response.schema_by_type || {})) {
schemaByType[typeName] = fields;
sampleByType[typeName] = response.sample_by_type?.[typeName];
}
if (response.first_radicado) merged.first_radicado = response.first_radicado;
}
merged.schema_by_type = schemaByType;
merged.sample_by_type = sampleByType;
return merged;
}
export function buildIngestForm(entry: ManifestEntry, file: Blob): FormData {
const form = new FormData();
form.append("file", file, entry.filename);
form.append("radicado", entry.radicado);
form.append("filename", entry.filename);
if (entry.docType) form.append("doc_type", entry.docType);
return form;
}
/**
* Construye un FormData multipart para /tasks/{id}/ingest-many.
* Envia N archivos en una sola peticion HTTP, con los metadatos
* (radicados, filenames, doc_types) como listas separadas por "|"
* en el mismo orden posicional que los archivos.
*/
export function buildIngestFormBulk(entries: ManifestEntry[], files: Blob[]): FormData {
const form = new FormData();
entries.forEach((entry, index) => {
form.append("files", files[index], entry.filename);
});
form.append("radicados", entries.map((entry) => entry.radicado).join("|"));
form.append("filenames", entries.map((entry) => entry.filename).join("|"));
form.append("doc_types", entries.map((entry) => entry.docType || "").join("|"));
return form;
}
@@ -0,0 +1,40 @@
import { describe, expect, it } from "vitest";
import { buildSchemaFieldNodes, getSampleValueAtPath, groupSchemaFields, splitFieldPath } from "./schemaDisplay";
const sample = {
"CLINICA MARYBAU LA LOMA LTDA.": {
"N.I.T.": "900.277.630-4",
"Ciudad": "El Paso, Cesar",
},
"Factura Electrónica de Venta No.": "C145288",
"Descripción de los Servicios Facturados": {
items: [{ Código: "890701", "Vr. Total": "145,000" }],
},
};
describe("schemaDisplay", () => {
it("reads nested sample values", () => {
expect(getSampleValueAtPath(sample, "CLINICA MARYBAU LA LOMA LTDA./N.I.T.")).toBe("900.277.630-4");
expect(getSampleValueAtPath(sample, "Factura Electrónica de Venta No.")).toBe("C145288");
expect(getSampleValueAtPath(sample, "Descripción de los Servicios Facturados/items/*/Código")).toBe("890701");
});
it("groups fields by parent path", () => {
const nodes = buildSchemaFieldNodes(
{
"CLINICA MARYBAU LA LOMA LTDA./N.I.T.": "str",
"CLINICA MARYBAU LA LOMA LTDA./Ciudad": "str",
"Factura Electrónica de Venta No.": "str",
},
sample,
);
const groups = groupSchemaFields(nodes);
expect(groups).toHaveLength(2);
expect(groups[0].fields).toHaveLength(2);
});
it("uses short labels for nested paths", () => {
expect(splitFieldPath("Datos del Paciente/Nombre:").label).toBe("Nombre:");
expect(splitFieldPath("Datos del Paciente/Nombre:").groupLabel).toBe("Datos del Paciente");
});
});
@@ -0,0 +1,137 @@
export interface SchemaFieldNode {
path: string;
kind: string;
label: string;
groupKey: string;
groupLabel: string;
sampleValue: string | null;
}
export interface SchemaFieldGroup {
key: string;
label: string;
fields: SchemaFieldNode[];
}
function formatPreview(value: unknown): string | null {
if (value === null || value === undefined || value === "") return null;
if (Array.isArray(value)) {
if (value.length === 0) return null;
if (value.every((item) => item && typeof item === "object" && !Array.isArray(item))) {
return `${value.length} fila(s)`;
}
return value.map((item) => formatPreview(item) ?? "").filter(Boolean).join(", ").slice(0, 120) || `${value.length} item(s)`;
}
if (typeof value === "object") return "{...}";
const text = String(value).replace(/\s+/g, " ").trim();
return text || null;
}
function resolveSampleRoot(sample: unknown, path: string): { root: unknown; parts: string[] } {
const parts = path.split("/").filter(Boolean);
if (!sample || typeof sample !== "object" || parts.length === 0) {
return { root: sample, parts };
}
const record = sample as Record<string, unknown>;
const first = parts[0];
if (first in record) {
return { root: record[first], parts: parts.slice(1) };
}
const typeKeys = Object.keys(record).filter((key) => key !== "radicado" && record[key] && typeof record[key] === "object");
if (typeKeys.length === 1 && typeKeys[0] !== first) {
const nested = record[typeKeys[0]];
if (nested && typeof nested === "object" && first in (nested as Record<string, unknown>)) {
return { root: (nested as Record<string, unknown>)[first], parts: parts.slice(1) };
}
if (nested && typeof nested === "object" && parts.length > 0) {
return { root: nested, parts };
}
}
return { root: sample, parts };
}
export function getSampleValueAtPath(sample: unknown, path: string): string | null {
const { root, parts } = resolveSampleRoot(sample, path);
let current: unknown = root;
for (const part of parts) {
if (current === null || current === undefined) return null;
if (part === "*") {
if (Array.isArray(current)) current = current[0];
continue;
}
if (typeof current !== "object") return null;
current = (current as Record<string, unknown>)[part];
}
return formatPreview(current);
}
export function splitFieldPath(path: string): { groupKey: string; groupLabel: string; label: string } {
const parts = path.split("/").filter(Boolean);
if (parts.length <= 1) {
return {
groupKey: "__root__",
groupLabel: "Campos principales",
label: parts[0] ?? path,
};
}
const label = parts[parts.length - 1];
const groupParts = parts.slice(0, -1);
const groupKey = groupParts.join("/");
const groupLabel = groupParts.length > 2 ? groupParts.slice(-2).join(" / ") : groupParts.join(" / ");
return { groupKey, groupLabel, label };
}
export function buildSchemaFieldNodes(schema: Record<string, string>, sample: unknown): SchemaFieldNode[] {
return Object.entries(schema).map(([path, kind]) => {
const { groupKey, groupLabel, label } = splitFieldPath(path);
return {
path,
kind,
label,
groupKey,
groupLabel,
sampleValue: getSampleValueAtPath(sample, path),
};
});
}
export function groupSchemaFields(nodes: SchemaFieldNode[]): SchemaFieldGroup[] {
const groups = new Map<string, SchemaFieldGroup>();
for (const node of nodes) {
const existing = groups.get(node.groupKey);
if (existing) {
existing.fields.push(node);
continue;
}
groups.set(node.groupKey, {
key: node.groupKey,
label: node.groupLabel,
fields: [node],
});
}
return Array.from(groups.values()).map((group) => ({
...group,
fields: group.fields.sort((a, b) => a.label.localeCompare(b.label, undefined, { numeric: true })),
}));
}
export function filterSchemaFieldNodes(nodes: SchemaFieldNode[], query: string): SchemaFieldNode[] {
const normalized = query.trim().toLowerCase();
if (!normalized) return nodes;
return nodes.filter((node) => {
const haystack = `${node.path} ${node.label} ${node.groupLabel} ${node.sampleValue ?? ""}`.toLowerCase();
return haystack.includes(normalized);
});
}
export function defaultAlias(path: string): string {
const { label } = splitFieldPath(path);
return label.replace(/[:.]/g, "").trim() || path;
}
File diff suppressed because it is too large Load Diff
@@ -9,5 +9,9 @@ beforeEach(() => {
saveCsvDialog: vi.fn(),
writeTextFile: vi.fn(),
findFirstDocument: vi.fn(),
indexFolder: vi.fn().mockResolvedValue([
{ radicado: "R1", filename: "factura.pdf", docType: "Factura", localPath: "C:\\docs\\R1\\factura.pdf", size: 10 },
]),
readFile: vi.fn().mockResolvedValue({ filename: "factura.pdf", size: 10, data: new ArrayBuffer(8) }),
};
});
+22
View File
@@ -1,5 +1,19 @@
export type StructureMode = "radicados" | "general";
export interface ManifestEntry {
radicado: string;
filename: string;
docType?: string;
localPath: string;
size: number;
}
export interface FileUploadData {
filename: string;
size: number;
data: ArrayBuffer;
}
export interface DocumentType {
name: string;
keyword: string;
@@ -20,6 +34,7 @@ export interface ExtractionFormState {
tiffEnabled: boolean;
tiffStart: string;
tiffEnd: string;
model: string;
}
export interface FieldSelection {
@@ -79,4 +94,11 @@ export interface DesktopBridge {
saveCsvDialog(defaultPath?: string): Promise<string | null>;
writeTextFile(filePath: string, content: string): Promise<{ ok: true; filePath: string }>;
findFirstDocument(rootPath: string, keywords: string[]): Promise<string | null>;
indexFolder(
rootPath: string,
structureMode: StructureMode,
docTypes: DocumentType[],
keywords: string[],
): Promise<ManifestEntry[]>;
readFile(filePath: string): Promise<FileUploadData>;
}
+1
View File
@@ -17,6 +17,7 @@ COPY domain /app/domain
COPY core /app/core
COPY infrastructure /app/infrastructure
COPY api /app/api
COPY worker /app/worker
EXPOSE 8001
CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8001"]
+4 -2
View File
@@ -37,7 +37,9 @@ en `dependencies/poppler-24.08.0/Library/bin`.
- `GET /config`
- `GET /models`
- `POST /schemas`
- `POST /schemas/upload`
- `POST /estimate`
- `POST /batches`
- `GET /batches/{batch_id}`
- `GET /batches/{batch_id}/rows`
- `POST /tasks/extraction`
- `POST /tasks/{task_id}/ingest`
- `GET /tasks`
+235
View File
@@ -0,0 +1,235 @@
"""Procesamiento de ingestas individuales desde la app desktop."""
from __future__ import annotations
import logging
from typing import Any
from domain.normalizer import normalize_to_schema
from domain.profiles import get_profile
from domain.prompts import PromptBuilder
from domain.service import InvoiceProcessingService
from infrastructure.database import ExtractionRow, Task, utcnow
from infrastructure.serialization import flatten_extraction_result
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
def _paths_for_type(fields: list[str], doc_type: str | None) -> list[str]:
if not doc_type:
return list(fields or [])
prefix = f"{doc_type}/"
return [path.split("/", 1)[1] for path in fields if path.startswith(prefix)]
def _expected_doc_types_for_radicado(request: dict[str, Any], radicado: str) -> set[str]:
expected: set[str] = set()
for item in request.get("documents") or []:
if str(item.get("radicado") or "") != radicado:
continue
doc_type = str(item.get("doc_type") or item.get("docType") or "").strip()
if doc_type:
expected.add(doc_type)
return expected
def _finalize_radicado_if_ready(
session: Session,
*,
task: Task,
batch_id: str,
radicado: str,
request: dict[str, Any],
fields: list[str],
alias_map: dict[str, str],
) -> bool:
partials = dict((task.result or {}).get("partials") or {})
radicado_partials = dict(partials.get(radicado) or {})
expected = _expected_doc_types_for_radicado(request, radicado)
if expected and not expected.issubset(set(radicado_partials.keys())):
return False
fused: dict[str, Any] = {"radicado": radicado, **radicado_partials}
source_pdf = None
for item in request.get("documents") or []:
if str(item.get("radicado") or "") == radicado:
source_pdf = item.get("filename")
break
if source_pdf:
fused["_source_pdf"] = source_pdf
row_index = int((task.result or {}).get("row_count") or 0)
for row in flatten_extraction_result(fused, fields, alias_map):
row_index += 1
session.add(
ExtractionRow(
batch_id=batch_id,
radicado=radicado,
source_pdf=source_pdf,
row_index=row_index,
row_json=row,
raw_json=fused,
)
)
partials.pop(radicado, None)
result = dict(task.result or {})
result["partials"] = partials
result["row_count"] = row_index
result["processed"] = int(result.get("processed") or 0) + 1
task.result = result
task.updated_at = utcnow()
return True
def _service_for_model(base_service: InvoiceProcessingService, model: str | None) -> InvoiceProcessingService:
"""Devuelve un servicio con el modelo solicitado; reutiliza base_service si coincide."""
if not model:
return base_service
current_model = getattr(base_service.extractor, "model", None)
if current_model == model:
return base_service
from domain.strategies import OpenAICompatibleExtractor
from domain.batch_processor import BatchProcessor
from domain.estimator import Estimator
extractor = OpenAICompatibleExtractor(model=model)
return InvoiceProcessingService(
extractor,
batch_processor_factory=lambda s: BatchProcessor(s),
estimator_factory=lambda s: Estimator(s),
)
def process_desktop_ingest(
service: InvoiceProcessingService,
session: Session,
*,
task: Task,
batch_id: str,
temp_path: str,
radicado: str,
filename: str,
doc_type: str | None,
instruction: str | None,
tiff_page_range: tuple[int, int] | None,
model: str | None = None,
) -> dict[str, Any]:
service = _service_for_model(service, model)
request = dict(task.request or {})
fields = list(request.get("fields") or [])
alias_map = dict(request.get("alias_map") or {})
structure_mode = str(request.get("structure_mode") or "radicados")
doc_types = list(request.get("doc_types") or [])
prof = get_profile("invoice")
selected_paths = _paths_for_type(fields, doc_type if structure_mode == "radicados" and doc_types else None)
if doc_type and structure_mode == "radicados" and doc_types:
try:
prompt = PromptBuilder.build_for_type(doc_type, selected_paths, {}, profile_name=prof.get("name", "invoice"))
except Exception:
prompt = PromptBuilder.build(selected_paths, {}, profile_name=prof.get("name", "invoice"))
else:
prompt = instruction or PromptBuilder.build(selected_paths, {}, profile_name=prof.get("name", "invoice"))
data, _usage = service.extract_full_from_pdf(
temp_path,
instruction=prompt,
tiff_page_range=tiff_page_range,
)
raw_data = data if isinstance(data, dict) else {"data": data}
if isinstance(data, dict):
data["radicado"] = radicado
else:
data = {"radicado": radicado, "data": data}
norm = normalize_to_schema(
data,
paths=selected_paths,
profile_name=prof.get("name", "invoice"),
)
normalized_obj = data
if isinstance(norm, dict):
normalized_obj = norm.get("normalized") or data
result = dict(task.result or {})
errors = list(result.get("errors") or [])
if structure_mode == "radicados" and doc_types and doc_type:
partials = dict(result.get("partials") or {})
radicado_partials = dict(partials.get(radicado) or {})
radicado_partials[doc_type] = normalized_obj or {}
partials[radicado] = radicado_partials
result["partials"] = partials
task.result = result
session.flush()
finalized = _finalize_radicado_if_ready(
session,
task=task,
batch_id=batch_id,
radicado=radicado,
request=request,
fields=fields,
alias_map=alias_map,
)
return {
"radicado": radicado,
"filename": filename,
"doc_type": doc_type,
"finalized": finalized,
"processed": int((task.result or {}).get("processed") or 0),
"total": int((task.result or {}).get("total") or 0),
}
fused = {"radicado": radicado, "_source_pdf": filename}
if doc_type:
fused[doc_type] = normalized_obj or {}
else:
fused.update(normalized_obj if isinstance(normalized_obj, dict) else {"data": normalized_obj})
row_index = int(result.get("row_count") or 0)
for row in flatten_extraction_result(fused, fields, alias_map):
row_index += 1
session.add(
ExtractionRow(
batch_id=batch_id,
radicado=radicado,
source_pdf=filename,
row_index=row_index,
row_json=row,
raw_json=fused,
)
)
result["row_count"] = row_index
result["processed"] = int(result.get("processed") or 0) + 1
task.result = result
task.updated_at = utcnow()
session.flush()
return {
"radicado": radicado,
"filename": filename,
"doc_type": doc_type,
"finalized": True,
"processed": int(result.get("processed") or 0),
"total": int(result.get("total") or 0),
}
def mark_task_completed_if_done(session: Session, task: Task, batch_id: str) -> None:
result = dict(task.result or {})
processed = int(result.get("processed") or 0)
total = int(result.get("total") or 0)
partials = dict(result.get("partials") or {})
if processed < total or partials:
return
from infrastructure.database import ExtractionBatch
task.status = "completed"
task.finished_at = utcnow()
task.updated_at = utcnow()
batch = session.get(ExtractionBatch, batch_id)
if batch is not None:
batch.status = "completed"
batch.updated_at = utcnow()
+492 -33
View File
@@ -1,13 +1,26 @@
"""API REST del microservicio de extraccion."""
from __future__ import annotations
import os
import uuid
from contextlib import asynccontextmanager
from typing import Any
from fastapi import BackgroundTasks, FastAPI, HTTPException
from pydantic import BaseModel, Field
try:
from arq import create_pool as _arq_create_pool
from arq.connections import RedisSettings as _RedisSettings
_ARQ_AVAILABLE = True
except ImportError:
_arq_create_pool = None # type: ignore[assignment]
_RedisSettings = None # type: ignore[assignment]
_ARQ_AVAILABLE = False
from fastapi import BackgroundTasks, FastAPI, File, Form, HTTPException, UploadFile
from pydantic import BaseModel, Field, model_validator
from sqlalchemy import select
from api.desktop_ingest import mark_task_completed_if_done, process_desktop_ingest
from domain.batch_processor import BatchProcessor
from domain.config import extractor_llm_config, normalizer_llm_config, settings
from domain.config import get_allowed_document_types
@@ -23,6 +36,7 @@ from infrastructure.database import (
utcnow,
)
from infrastructure.serialization import flatten_extraction_result
from infrastructure.uploads import temp_upload_path
def build_processing_service() -> InvoiceProcessingService:
@@ -42,10 +56,43 @@ def build_processing_service() -> InvoiceProcessingService:
processing_service = build_processing_service()
def _redis_settings_from_url(url: str):
url = url.removeprefix("redis://")
host_port, *db_parts = url.split("/")
host, *port_parts = host_port.split(":")
port = int(port_parts[0]) if port_parts else 6379
db = int(db_parts[0]) if db_parts else 0
return _RedisSettings(host=host, port=port, database=db)
# Pool ARQ global — compartido por todos los requests
_arq_pool = None
async def get_arq_pool():
global _arq_pool
if not _ARQ_AVAILABLE:
raise RuntimeError("arq no instalado; ASYNC_QUEUE requiere 'pip install arq'.")
if _arq_pool is None:
_arq_pool = await _arq_create_pool(_redis_settings_from_url(settings.redis_url))
return _arq_pool
@asynccontextmanager
async def lifespan(app: FastAPI):
init_db()
if settings.processing.async_queue:
os.makedirs(settings.queue_dir, exist_ok=True)
try:
await get_arq_pool()
except Exception as exc:
import logging as _logging
_logging.getLogger(__name__).warning(
"No se pudo conectar a Redis (%s). Usando modo sincrono como fallback.", exc
)
yield
if _arq_pool is not None:
await _arq_pool.aclose()
app = FastAPI(
@@ -69,8 +116,15 @@ class SchemaRequest(BaseModel):
structure_mode: str = "radicados"
class DesktopDocument(BaseModel):
radicado: str
filename: str
doc_type: str | None = None
class ExtractionRequest(BaseModel):
root_path: str
source: str = "legacy"
root_path: str = ""
keywords: list[str] | None = None
doc_types: list[DocumentType] | None = None
fields: list[str] = Field(default_factory=list)
@@ -80,6 +134,18 @@ class ExtractionRequest(BaseModel):
structure_mode: str = "radicados"
retry_on_empty: bool = False
radicado_filter: list[str] | None = None
documents: list[DesktopDocument] | None = None
document_count: int | None = None
@model_validator(mode="after")
def validate_source(self) -> "ExtractionRequest":
if self.source == "desktop":
if not self.documents:
raise ValueError("Envia documents cuando source=desktop.")
return self
if not self.root_path:
raise ValueError("Envia root_path para extraccion legacy.")
return self
class EstimateRequest(BaseModel):
@@ -309,29 +375,34 @@ def list_models() -> dict[str, Any]:
@app.post("/schemas")
def build_schema(request: SchemaRequest) -> dict[str, Any]:
"""Descubre campos disponibles para que el cliente API elija `fields`."""
if request.document_path:
schema, sample = processing_service.build_schema_from_first(
request.document_path,
instruction=request.instruction,
tiff_page_range=request.tiff_page_range,
)
return {"schema": schema, "sample": sample}
if request.root_path and request.doc_types:
schema_by_type, sample_by_type, first_radicado = (
processing_service.build_schema_from_first_for_types(
request.root_path,
[dt.model_dump() for dt in request.doc_types],
try:
if request.document_path:
schema, sample = processing_service.build_schema_from_first(
request.document_path,
instruction=request.instruction,
tiff_page_range=request.tiff_page_range,
structure_mode=request.structure_mode,
)
)
return {
"schema_by_type": schema_by_type,
"sample_by_type": sample_by_type,
"first_radicado": first_radicado,
}
return {"schema": schema, "sample": sample}
if request.root_path and request.doc_types:
schema_by_type, sample_by_type, first_radicado = (
processing_service.build_schema_from_first_for_types(
request.root_path,
[dt.model_dump() for dt in request.doc_types],
instruction=request.instruction,
tiff_page_range=request.tiff_page_range,
structure_mode=request.structure_mode,
)
)
return {
"schema_by_type": schema_by_type,
"sample_by_type": sample_by_type,
"first_radicado": first_radicado,
}
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except FileNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
raise HTTPException(
status_code=400,
@@ -339,18 +410,92 @@ def build_schema(request: SchemaRequest) -> dict[str, Any]:
)
@app.post("/schemas/upload")
def build_schema_upload(
file: UploadFile = File(...),
structure_mode: str = Form("radicados"),
doc_type_name: str | None = Form(None),
instruction: str | None = Form(None),
tiff_start: int | None = Form(None),
tiff_end: int | None = Form(None),
) -> dict[str, Any]:
"""Descubre campos a partir de un documento subido desde la app desktop."""
tiff_page_range = None
if tiff_start is not None and tiff_end is not None and not (tiff_start == 0 and tiff_end == 0):
tiff_page_range = (tiff_start, tiff_end)
try:
with temp_upload_path(file) as temp_path:
schema, sample = processing_service.build_schema_from_first(
temp_path,
instruction=instruction,
tiff_page_range=tiff_page_range,
)
if doc_type_name:
return {
"schema_by_type": {doc_type_name: schema},
"sample_by_type": {doc_type_name: sample},
"structure_mode": structure_mode,
}
return {"schema": schema, "sample": sample, "structure_mode": structure_mode}
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except FileNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.post("/estimate")
def estimate(request: EstimateRequest) -> dict[str, Any]:
return processing_service.estimate_tokens_and_cost(
request.root_path,
request.keywords,
request.schema_map,
request.selected_fields,
doc_types=[dt.model_dump() for dt in request.doc_types or []] or None,
schema_by_type=request.schema_by_type,
selected_paths=request.selected_paths,
structure_mode=request.structure_mode,
)
try:
return processing_service.estimate_tokens_and_cost(
request.root_path,
request.keywords,
request.schema_map,
request.selected_fields,
doc_types=[dt.model_dump() for dt in request.doc_types or []] or None,
schema_by_type=request.schema_by_type,
selected_paths=request.selected_paths,
structure_mode=request.structure_mode,
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.post("/estimate/upload")
def estimate_upload(
file: UploadFile = File(...),
doc_count: int = Form(1),
selected_paths: str | None = Form(None),
structure_mode: str = Form("radicados"),
tiff_start: int | None = Form(None),
tiff_end: int | None = Form(None),
model: str | None = Form(None),
) -> dict[str, Any]:
"""Estima tokens y costos a partir de un documento subido desde la app desktop."""
import json as _json
tiff_page_range = None
if tiff_start is not None and tiff_end is not None and not (tiff_start == 0 and tiff_end == 0):
tiff_page_range = (tiff_start, tiff_end)
paths_list: list[str] = []
if selected_paths:
try:
paths_list = _json.loads(selected_paths)
except Exception:
paths_list = []
try:
with temp_upload_path(file) as temp_path:
return processing_service.estimator.estimate_from_file(
temp_path,
doc_count=max(1, doc_count),
selected_paths=paths_list or None,
tiff_page_range=tiff_page_range,
model=model or None,
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.post("/batches", response_model=ExtractionBatchRead)
@@ -375,6 +520,42 @@ def create_extraction_task(request: ExtractionRequest, background_tasks: Backgro
detail="Envia fields. Usa POST /schemas para descubrir campos antes de crear la tarea.",
)
if request.source == "desktop":
documents = [doc.model_dump() for doc in request.documents or []]
unique_radicados = {str(doc["radicado"]) for doc in documents}
total = len(unique_radicados) if unique_radicados else (request.document_count or len(documents))
title = f"Extraccion desktop ({total} documentos)"
with session_scope() as session:
batch = ExtractionBatch(
status="running",
root_path="desktop://upload",
request=request.model_dump(),
updated_at=utcnow(),
)
session.add(batch)
session.flush()
batch_id = batch.id
task = Task(
type="extraction",
status="running",
title=title,
request=request.model_dump(),
result={
"source": "desktop",
"extraction_batch_id": batch_id,
"processed": 0,
"total": total,
"row_count": 0,
"partials": {},
"errors": [],
},
started_at=utcnow(),
updated_at=utcnow(),
)
session.add(task)
session.flush()
return serialize_task(task)
with session_scope() as session:
task = Task(
type="extraction",
@@ -393,6 +574,284 @@ def create_extraction_task(request: ExtractionRequest, background_tasks: Backgro
return serialized
async def _process_sync(
task_id: str,
batch_id: str,
content: bytes,
effective_name: str,
radicado: str,
doc_type: str | None,
request: dict[str, Any],
) -> dict[str, Any]:
"""Procesa un documento sincronicamente en un thread pool (modo sin Redis).
Estrategia de concurrencia:
- La llamada a la LLM (lenta) se ejecuta sin ningún lock, permitiendo
que todos los documentos llamen a OpenAI en paralelo.
- Solo la escritura del resultado en la BD usa SELECT FOR UPDATE para
evitar la race condition en el contador `processed`.
"""
import asyncio as _asyncio
import logging as _logging
import tempfile as _tempfile
from domain.service import InvoiceProcessingService
from domain.strategies import OpenAICompatibleExtractor
from domain.batch_processor import BatchProcessor
from domain.estimator import Estimator
from api.desktop_ingest import _service_for_model, _paths_for_type
from domain.profiles import get_profile
from domain.prompts import PromptBuilder
from domain.normalizer import normalize_to_schema
_log = _logging.getLogger("extraction.ingest")
ext = os.path.splitext(effective_name)[1] or ".pdf"
with _tempfile.NamedTemporaryFile(delete=False, suffix=ext) as tmp:
tmp.write(content)
temp_path = tmp.name
try:
# ── Fase 1: llamada a la LLM (sin lock, en paralelo) ──────────────
def _call_llm():
model = request.get("model") or None
svc = _service_for_model(processing_service, model)
req = dict(request)
fields = list(req.get("fields") or [])
doc_type_arg = doc_type
structure_mode = str(req.get("structure_mode") or "radicados")
doc_types = list(req.get("doc_types") or [])
prof = get_profile("invoice")
selected_paths = _paths_for_type(
fields,
doc_type_arg if structure_mode == "radicados" and doc_types else None,
)
if doc_type_arg and structure_mode == "radicados" and doc_types:
try:
prompt = PromptBuilder.build_for_type(
doc_type_arg, selected_paths, {}, profile_name=prof.get("name", "invoice")
)
except Exception:
prompt = PromptBuilder.build(selected_paths, {}, profile_name=prof.get("name", "invoice"))
else:
instr = req.get("instruction")
prompt = instr or PromptBuilder.build(selected_paths, {}, profile_name=prof.get("name", "invoice"))
tiff_range = req.get("tiff_page_range")
model_name = getattr(svc.extractor, "model", "?")
_log.info("→ OpenAI [%s] radicado=%s filename=%s", model_name, radicado, effective_name)
data, _usage = svc.extract_full_from_pdf(
temp_path,
instruction=prompt,
tiff_page_range=tiff_range,
)
tok_in = (_usage or {}).get("prompt_tokens", "?")
tok_out = (_usage or {}).get("completion_tokens", "?")
_log.info("← OpenAI [%s] radicado=%s tokens=%s→%s", model_name, radicado, tok_in, tok_out)
if isinstance(data, dict):
data["radicado"] = radicado
else:
data = {"radicado": radicado, "data": data}
norm = normalize_to_schema(data, paths=selected_paths, profile_name=prof.get("name", "invoice"))
normalized_obj = data
if isinstance(norm, dict):
normalized_obj = norm.get("normalized") or data
return normalized_obj, selected_paths
normalized_obj, selected_paths = await _asyncio.to_thread(_call_llm)
# ── Fase 2: persistir resultado (con FOR UPDATE para serializar) ───
def _persist():
from infrastructure.serialization import flatten_extraction_result
from infrastructure.database import ExtractionRow, ExtractionBatch, utcnow
req = dict(request)
fields = list(req.get("fields") or [])
alias_map = dict(req.get("alias_map") or {})
with session_scope() as session:
task = session.get(Task, task_id, with_for_update=True)
if task is None:
return {"error": "task_not_found"}
result = dict(task.result or {})
fused = {"radicado": radicado, "_source_pdf": effective_name}
if doc_type:
fused[doc_type] = normalized_obj or {}
else:
fused.update(
normalized_obj if isinstance(normalized_obj, dict)
else {"data": normalized_obj}
)
row_index = int(result.get("row_count") or 0)
for row in flatten_extraction_result(fused, fields, alias_map):
row_index += 1
session.add(ExtractionRow(
batch_id=batch_id,
radicado=radicado,
source_pdf=effective_name,
row_index=row_index,
row_json=row,
raw_json=fused,
))
result["row_count"] = row_index
processed_now = int(result.get("processed") or 0) + 1
result["processed"] = processed_now
task.result = result
task.updated_at = utcnow()
session.flush()
mark_task_completed_if_done(session, task, batch_id)
session.flush() # persiste status=completed / finished_at si aplica
_log.info(
"✓ guardado radicado=%s processed=%d/%d status=%s",
radicado, processed_now, int(result.get("total") or 0), task.status,
)
session.refresh(task)
return {
"radicado": radicado,
"filename": effective_name,
"processed": processed_now,
"total": int(result.get("total") or 0),
"task": serialize_task(task),
}
return await _asyncio.to_thread(_persist)
finally:
try:
os.unlink(temp_path)
except OSError:
pass
async def _enqueue_async(
task_id: str,
batch_id: str,
content: bytes,
effective_name: str,
radicado: str,
doc_type: str | None,
instruction: str | None,
tiff_page_range: Any,
) -> None:
"""Guarda el archivo en el volumen compartido y encola el job en ARQ."""
ext = os.path.splitext(effective_name)[1] or ".pdf"
job_file = os.path.join(settings.queue_dir, f"{uuid.uuid4().hex}{ext}")
with open(job_file, "wb") as fh:
fh.write(content)
pool = await get_arq_pool()
await pool.enqueue_job(
"process_document",
task_id=task_id,
batch_id=batch_id,
queue_path=job_file,
radicado=radicado,
filename=effective_name,
doc_type=doc_type,
instruction=instruction,
tiff_page_range=list(tiff_page_range) if tiff_page_range else None,
)
def _load_task_info(task_id: str) -> tuple[str, dict[str, Any], dict[str, Any]]:
"""Carga batch_id, request y task serializado desde BD."""
with session_scope() as session:
task = session.get(Task, task_id)
if task is None:
raise HTTPException(status_code=404, detail="Tarea no encontrada")
if task.status not in {"pending", "running"}:
raise HTTPException(status_code=400, detail="La tarea ya no acepta documentos.")
if str((task.request or {}).get("source") or "") != "desktop":
raise HTTPException(status_code=400, detail="La tarea no fue creada en modo desktop.")
batch_id = str((task.result or {}).get("extraction_batch_id") or "")
if not batch_id:
raise HTTPException(status_code=400, detail="La tarea no tiene lote asociado.")
return batch_id, dict(task.request or {}), serialize_task(task)
@app.post("/tasks/{task_id}/ingest")
async def ingest_task_document(
task_id: str,
file: UploadFile = File(...),
radicado: str = Form(...),
filename: str | None = Form(None),
doc_type: str | None = Form(None),
) -> dict[str, Any]:
"""Procesa o encola un documento.
- ASYNC_QUEUE=false (default): procesa sincronicamente, responde con la tarea actualizada.
- ASYNC_QUEUE=true: encola en Redis/ARQ, responde de inmediato (sondea GET /tasks/{id}).
"""
batch_id, request, serialized = _load_task_info(task_id)
effective_name = filename or file.filename or "documento.pdf"
content = await file.read()
if settings.processing.async_queue:
await _enqueue_async(
task_id, batch_id, content, effective_name,
radicado, doc_type, request.get("instruction"), request.get("tiff_page_range"),
)
return {"queued": True, "task": serialized}
# Modo sincrono: procesa ahora y devuelve resultado completo
return await _process_sync(task_id, batch_id, content, effective_name, radicado, doc_type, request)
@app.post("/tasks/{task_id}/ingest-many")
async def ingest_task_documents_bulk(
task_id: str,
files: list[UploadFile] = File(...),
radicados: str = Form(...),
filenames: str | None = Form(None),
doc_types: str | None = Form(None),
) -> dict[str, Any]:
"""Procesa o encola múltiples documentos en una sola petición HTTP.
Los parámetros `radicados`, `filenames` y `doc_types` son listas
separadas por `|` que se corresponden posicionalmente con `files`.
En modo sincrono (ASYNC_QUEUE=false) procesa en paralelo con asyncio.
En modo async (ASYNC_QUEUE=true) encola todos en ARQ.
"""
import asyncio as _asyncio
rad_list = radicados.split("|")
fname_list = (filenames or "").split("|") if filenames else [""] * len(files)
dtype_list = (doc_types or "").split("|") if doc_types else [None] * len(files)
if len(files) != len(rad_list):
raise HTTPException(
status_code=400,
detail=f"Numero de archivos ({len(files)}) no coincide con radicados ({len(rad_list)})",
)
batch_id, request, serialized = _load_task_info(task_id)
contents = await _asyncio.gather(*[f.read() for f in files])
if settings.processing.async_queue:
await _asyncio.gather(*[
_enqueue_async(
task_id, batch_id, contents[i],
fname_list[i] or files[i].filename or f"doc_{i}.pdf",
rad_list[i], dtype_list[i] or None,
request.get("instruction"), request.get("tiff_page_range"),
)
for i in range(len(files))
])
return {"queued": len(files), "task": serialized}
# Modo sincrono: procesar todos en paralelo con semaforo para no saturar la API
sem = _asyncio.Semaphore(settings.processing.concurrency_limit)
async def _bounded(i: int) -> dict[str, Any]:
async with sem:
return await _process_sync(
task_id, batch_id, contents[i],
fname_list[i] or files[i].filename or f"doc_{i}.pdf",
rad_list[i], dtype_list[i] or None,
request,
)
results = await _asyncio.gather(*[_bounded(i) for i in range(len(files))], return_exceptions=True)
errors = [str(r) for r in results if isinstance(r, Exception)]
return {"processed": len(files) - len(errors), "errors": errors, "task": serialized}
@app.get("/tasks", response_model=list[TaskRead])
def list_tasks(type: str | None = None, status: str | None = None) -> list[dict[str, Any]]:
with session_scope() as session:
+28 -1
View File
@@ -89,7 +89,17 @@ class RetrySettings(BaseSettings):
class ProcessingSettings(BaseSettings):
"""Configuración de procesamiento y concurrencia."""
concurrency_limit: int = Field(100, validation_alias=AliasChoices("CONCURRENCY_LIMIT"))
concurrency_limit: int = Field(50, validation_alias=AliasChoices("CONCURRENCY_LIMIT"))
redis_url: str = Field("redis://localhost:6379/0", validation_alias=AliasChoices("REDIS_URL"))
worker_concurrency: int = Field(25, validation_alias=AliasChoices("WORKER_CONCURRENCY"))
# Límites OpenAI para el rate limiter distribuido
openai_rpm_limit: int = Field(500, validation_alias=AliasChoices("OPENAI_RPM_LIMIT"))
openai_tpm_limit: int = Field(800_000, validation_alias=AliasChoices("OPENAI_TPM_LIMIT"))
# Directorio compartido entre API y workers para encolar archivos
queue_dir: str = Field("/tmp/ips_queue", validation_alias=AliasChoices("QUEUE_DIR"))
# False (default) = procesamiento sincrono sin Redis/workers.
# True = cola ARQ distribuida (requiere Redis corriendo y workers activos).
async_queue: bool = Field(False, validation_alias=AliasChoices("ASYNC_QUEUE"))
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
@@ -102,6 +112,23 @@ class AppSettings(BaseSettings):
csv_delimiter: str = Field("|", validation_alias=AliasChoices("CSV_DELIMITER"))
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
# Accesos directos para conveniencia
@property
def redis_url(self) -> str:
return self.processing.redis_url
@property
def openai_rpm_limit(self) -> int:
return self.processing.openai_rpm_limit
@property
def openai_tpm_limit(self) -> int:
return self.processing.openai_tpm_limit
@property
def queue_dir(self) -> str:
return self.processing.queue_dir
settings = AppSettings()
+54
View File
@@ -110,6 +110,60 @@ class Estimator:
def _accumulate_usage(self, agg, new_usage):
accumulate_usage(agg, new_usage)
def estimate_from_file(
self,
file_path: str,
doc_count: int = 1,
selected_paths: Optional[List[str]] = None,
tiff_page_range: Optional[tuple] = None,
model: Optional[str] = None,
) -> Dict[str, Any]:
"""Estima tokens y costos procesando un único archivo subido desde el cliente.
Si `model` se especifica y es distinto al modelo actual del extractor,
crea un extractor temporal con ese modelo para obtener el uso real de tokens.
"""
from api.desktop_ingest import _service_for_model
service = _service_for_model(self.service, model)
prof = get_profile('invoice')
paths = selected_paths or []
prompt = PromptBuilder.build(paths, {}, profile_name=prof.get('name', 'invoice'))
data, usage_ext = service.extract_full_from_pdf(
file_path,
instruction=prompt,
tiff_page_range=tiff_page_range,
)
if isinstance(data, dict):
data['radicado'] = 'sample'
else:
data = {"radicado": 'sample', "data": data}
norm = normalize_to_schema(data, paths=paths, profile_name=prof.get('name', 'invoice'))
usage_norm = None
if isinstance(norm, dict):
usage_norm = norm.get("_usage")
result = self._build_result(usage_ext, usage_norm, doc_count, 'sample')
# Enriquecer con modelo efectivo y conteo de páginas para el frontend
effective_model = model or getattr(service.extractor, 'model', None) or getattr(self.service.extractor, 'model', None)
result['extractor_model'] = effective_model
result['normalizer_model'] = settings.api.normalizer_model or 'gpt-5.4-mini'
# Estimar numero de paginas a partir de prompt_tokens de imagen:
# GPT-4o cobra ~85 tok base + ~170 tok por tile 512x512.
# Una A4 a 150 DPI (~1240x1754) genera ~6 tiles → ~1105 tokens.
# Aproximamos: pages = max(1, round(image_tokens / 1000))
prompt_tokens = int((usage_ext or {}).get('prompt_tokens') or 0)
text_prompt_est = len(prompt) // 4 # 1 token ≈ 4 chars
image_tokens_est = max(0, prompt_tokens - text_prompt_est)
pages_est = max(1, round(image_tokens_est / 1000))
result['pages_per_doc'] = pages_est
return result
def _build_result(self, usage_ext, usage_norm, total_docs, first_radicado):
model_ext = getattr(self.service.extractor, "model", None)
cost_ext = compute_cost_from_usage(usage_ext, "extract", model=model_ext)
+156
View File
@@ -0,0 +1,156 @@
"""Rate limiter distribuido para llamadas a OpenAI.
Usa Redis como backend para compartir el estado entre múltiples workers.
Implementa el algoritmo de token bucket con ventana deslizante para respetar
los límites de RPM (requests per minute) y TPM (tokens per minute) de OpenAI.
"""
from __future__ import annotations
import asyncio
import logging
import time
from typing import Optional
import redis.asyncio as aioredis
logger = logging.getLogger(__name__)
# Límites por defecto para OpenAI gpt-4o (tier 1).
# Sobreescribibles con variables de entorno vía config.
DEFAULT_RPM = 500 # requests per minute
DEFAULT_TPM = 800_000 # tokens per minute
class DistributedRateLimiter:
"""Token bucket distribuido respaldado por Redis.
Un solo objeto se comparte entre todos los workers de un proceso.
Múltiples réplicas del worker comparten el estado en Redis.
Uso:
limiter = DistributedRateLimiter(redis_url="redis://redis:6379/1")
async with limiter.acquire(estimated_tokens=4000):
response = await openai_client.chat.completions.create(...)
"""
def __init__(
self,
redis_url: str,
rpm: int = DEFAULT_RPM,
tpm: int = DEFAULT_TPM,
key_prefix: str = "ips:rl",
) -> None:
self._redis_url = redis_url
self.rpm = rpm
self.tpm = tpm
self._prefix = key_prefix
self._client: Optional[aioredis.Redis] = None
self._lock = asyncio.Lock()
async def _get_client(self) -> aioredis.Redis:
if self._client is None:
async with self._lock:
if self._client is None:
self._client = aioredis.from_url(
self._redis_url,
encoding="utf-8",
decode_responses=True,
socket_connect_timeout=5,
)
return self._client
async def _sliding_window_count(self, key: str, window_s: float = 60.0) -> int:
"""Cuenta eventos en la ventana deslizante usando un Sorted Set de Redis."""
client = await self._get_client()
now = time.monotonic()
# Eliminar eventos expirados
await client.zremrangebyscore(key, "-inf", now - window_s)
return await client.zcard(key)
async def _record_event(self, key: str, tokens: int = 0, window_s: float = 60.0) -> None:
"""Registra un evento en el sorted set. El score es el timestamp."""
client = await self._get_client()
now = time.monotonic()
member = f"{now}:{tokens}"
pipe = client.pipeline()
pipe.zadd(key, {member: now})
pipe.expire(key, int(window_s) + 5)
await pipe.execute()
async def _token_sum_in_window(self, key: str, window_s: float = 60.0) -> int:
"""Suma los tokens de todos los eventos en la ventana."""
client = await self._get_client()
now = time.monotonic()
await client.zremrangebyscore(key, "-inf", now - window_s)
members = await client.zrange(key, 0, -1)
total = 0
for m in members:
parts = m.split(":", 1)
if len(parts) == 2:
try:
total += int(parts[1])
except ValueError:
pass
return total
async def acquire(self, estimated_tokens: int = 2000, max_wait_s: float = 120.0) -> None:
"""Espera hasta que haya capacidad disponible en RPM y TPM.
Args:
estimated_tokens: Tokens que se espera consumir (input + output).
max_wait_s: Tiempo máximo de espera antes de lanzar TimeoutError.
"""
rpm_key = f"{self._prefix}:rpm"
tpm_key = f"{self._prefix}:tpm"
start = time.monotonic()
sleep_s = 0.5
while True:
rpm_count = await self._sliding_window_count(rpm_key)
tpm_used = await self._token_sum_in_window(tpm_key)
rpm_ok = rpm_count < self.rpm
tpm_ok = (tpm_used + estimated_tokens) <= self.tpm
if rpm_ok and tpm_ok:
await self._record_event(rpm_key, tokens=0)
await self._record_event(tpm_key, tokens=estimated_tokens)
return
elapsed = time.monotonic() - start
if elapsed >= max_wait_s:
raise TimeoutError(
f"Rate limiter: espera maxima alcanzada ({max_wait_s}s). "
f"rpm={rpm_count}/{self.rpm} tpm={tpm_used}/{self.tpm}"
)
reason = "RPM" if not rpm_ok else "TPM"
logger.debug(
"Rate limiter: esperando por %s (rpm=%d/%d tpm=%d/%d) — sleep %.1fs",
reason, rpm_count, self.rpm, tpm_used, self.tpm, sleep_s,
)
await asyncio.sleep(sleep_s)
sleep_s = min(sleep_s * 1.5, 10.0)
async def close(self) -> None:
if self._client is not None:
await self._client.aclose()
self._client = None
# Instancia global compartida por todos los workers del proceso.
# Se inicializa de forma lazy en el primer uso.
_limiter: Optional[DistributedRateLimiter] = None
def get_rate_limiter() -> DistributedRateLimiter:
"""Devuelve el rate limiter global, inicializándolo si es necesario."""
global _limiter
if _limiter is None:
from domain.config import settings
_limiter = DistributedRateLimiter(
redis_url=settings.redis_url,
rpm=settings.openai_rpm_limit,
tpm=settings.openai_tpm_limit,
)
return _limiter
@@ -0,0 +1,36 @@
"""Helpers para subidas temporales desde la app desktop."""
from __future__ import annotations
import os
import tempfile
from contextlib import contextmanager
from typing import Iterator
from fastapi import UploadFile
@contextmanager
def temp_upload_path(upload: UploadFile, *, suffix: str | None = None) -> Iterator[str]:
"""Guarda un UploadFile en disco temporal y lo elimina al salir."""
ext = suffix
if ext is None and upload.filename:
_, ext = os.path.splitext(upload.filename)
if not ext:
ext = ".bin"
fd, path = tempfile.mkstemp(suffix=ext)
os.close(fd)
try:
upload.file.seek(0)
with open(path, "wb") as handle:
while True:
chunk = upload.file.read(1024 * 1024)
if not chunk:
break
handle.write(chunk)
yield path
finally:
try:
os.unlink(path)
except OSError:
pass
+4
View File
@@ -1,5 +1,6 @@
coloredlogs==15.0.1
fastapi==0.115.6
python-multipart==0.0.20
openai==2.1.0
pdf2image==1.17.0
pillow==11.3.0
@@ -10,3 +11,6 @@ python-dotenv==1.1.1
SQLAlchemy==2.0.36
tenacity==9.1.2
uvicorn[standard]==0.32.1
# Cola de tareas distribuida
arq==0.26.3
redis[hiredis]==5.2.1
+1
View File
@@ -0,0 +1 @@
"""Worker ARQ para procesamiento paralelo de documentos."""
+18
View File
@@ -0,0 +1,18 @@
"""Punto de entrada del worker ARQ.
Ejecutar con:
python -m worker.main
"""
import asyncio
import logging
from arq import run_worker
from worker.settings import WorkerSettings
from domain.logging_config import configure_logging
configure_logging()
logger = logging.getLogger(__name__)
if __name__ == "__main__":
logger.info("Iniciando worker ARQ...")
asyncio.run(run_worker(WorkerSettings))
+84
View File
@@ -0,0 +1,84 @@
"""Configuración del worker ARQ.
ARQ usa esta función `WorkerSettings` para saber qué funciones
manejar, cuántos jobs en paralelo y cómo conectarse a Redis.
"""
from __future__ import annotations
import logging
import os
from typing import Any
from arq.connections import RedisSettings
from domain.config import settings
from domain.logging_config import configure_logging
from domain.service import InvoiceProcessingService
logger = logging.getLogger(__name__)
async def startup(ctx: dict[str, Any]) -> None:
"""Inicializa el servicio de extracción compartido entre corrutinas del worker."""
configure_logging()
logger.info("Worker ARQ iniciando...")
from infrastructure.database import init_db
from domain.config import extractor_llm_config
from domain.strategies import OpenAICompatibleExtractor
from domain.batch_processor import BatchProcessor
from domain.estimator import Estimator
init_db()
extractor_config = extractor_llm_config()
extractor = OpenAICompatibleExtractor(
api_key=str(extractor_config["api_key"]),
base_url=str(extractor_config["base_url"]),
model=str(extractor_config["model"]),
)
ctx["service"] = InvoiceProcessingService(
extractor,
batch_processor_factory=lambda s: BatchProcessor(s),
estimator_factory=lambda s: Estimator(s),
)
logger.info("Worker ARQ listo. concurrency=%d", int(os.getenv("WORKER_CONCURRENCY", "25")))
async def shutdown(ctx: dict[str, Any]) -> None:
logger.info("Worker ARQ deteniendo...")
from domain.rate_limiter import get_rate_limiter
try:
await get_rate_limiter().close()
except Exception:
pass
def _parse_redis_settings() -> RedisSettings:
url = settings.redis_url
# Extraer host/port/db de la URL redis://host:port/db
url = url.removeprefix("redis://")
host_port, *db_parts = url.split("/")
host, *port_parts = host_port.split(":")
port = int(port_parts[0]) if port_parts else 6379
db = int(db_parts[0]) if db_parts else 0
return RedisSettings(host=host, port=port, database=db)
class WorkerSettings:
"""Configuración declarativa que ARQ espera en el módulo del worker."""
from worker.tasks import process_document
functions = [process_document]
redis_settings = _parse_redis_settings()
# Cuántos jobs corre en paralelo ESTE proceso worker
max_jobs: int = int(os.getenv("WORKER_CONCURRENCY", "25"))
# Tiempo máximo por job antes de cancelarlo (segundos)
job_timeout: int = int(os.getenv("EXTRACTOR_TIMEOUT_S", "120")) * 3
# Retener resultados en Redis 1 hora para diagnóstico
keep_result: int = 3600
on_startup = startup
on_shutdown = shutdown
+236
View File
@@ -0,0 +1,236 @@
"""Tareas ARQ para procesamiento paralelo de documentos.
Cada tarea recibe la ruta de un archivo temporal en el volumen compartido
`/tmp/ips_queue/`, lo procesa y actualiza la tarea en la base de datos.
El archivo se elimina al finalizar (éxito o error).
"""
from __future__ import annotations
import asyncio
import logging
import os
from typing import Any
from infrastructure.database import ExtractionRow, Task, session_scope, utcnow
from infrastructure.serialization import flatten_extraction_result
from api.desktop_ingest import (
_finalize_radicado_if_ready,
_paths_for_type,
mark_task_completed_if_done,
)
from domain.profiles import get_profile
from domain.prompts import PromptBuilder
from domain.normalizer import normalize_to_schema
from domain.rate_limiter import get_rate_limiter
logger = logging.getLogger(__name__)
async def process_document(
ctx: dict[str, Any],
*,
task_id: str,
batch_id: str,
queue_path: str,
radicado: str,
filename: str,
doc_type: str | None,
instruction: str | None,
tiff_page_range: list[int] | None,
) -> dict[str, Any]:
"""Extrae datos de un documento y los persiste en la BD.
Args:
ctx: Contexto ARQ con 'service' (InvoiceProcessingService).
task_id: ID de la tarea parent.
batch_id: ID del lote de extracción.
queue_path: Ruta absoluta al archivo en el volumen compartido.
radicado: Radicado al que pertenece el documento.
filename: Nombre original del archivo.
doc_type: Tipo documental (None en modo carpeta general).
instruction: Instrucción de extracción personalizada.
tiff_page_range: Rango de páginas TIFF [inicio, fin].
"""
service = ctx["service"]
rate_limiter = get_rate_limiter()
tiff_range = tuple(tiff_page_range) if tiff_page_range else None
logger.info("Procesando %s / %s (task=%s)", radicado, filename, task_id)
try:
# Obtener config de la tarea desde BD
def _load_task_config() -> dict[str, Any] | None:
with session_scope() as session:
task = session.get(Task, task_id)
if task is None:
return None
return dict(task.request or {})
request = await asyncio.to_thread(_load_task_config)
if request is None:
logger.warning("Tarea %s no encontrada — saltando %s", task_id, filename)
return {"skipped": True, "reason": "task_not_found"}
fields: list[str] = list(request.get("fields") or [])
alias_map: dict[str, str] = dict(request.get("alias_map") or {})
structure_mode: str = str(request.get("structure_mode") or "radicados")
doc_types: list = list(request.get("doc_types") or [])
prof = get_profile("invoice")
selected_paths = _paths_for_type(
fields,
doc_type if structure_mode == "radicados" and doc_types else None,
)
if doc_type and structure_mode == "radicados" and doc_types:
try:
prompt = PromptBuilder.build_for_type(
doc_type, selected_paths, {}, profile_name=prof.get("name", "invoice")
)
except Exception:
prompt = PromptBuilder.build(
selected_paths, {}, profile_name=prof.get("name", "invoice")
)
else:
prompt = instruction or PromptBuilder.build(
selected_paths, {}, profile_name=prof.get("name", "invoice")
)
# Esperar turno en el rate limiter antes de llamar a OpenAI
estimated_tokens = max(len(selected_paths) * 80, 2000)
await rate_limiter.acquire(estimated_tokens=estimated_tokens)
# Extracción + normalización (CPU + IO bound, en thread pool)
def _extract_and_normalize() -> tuple[dict, dict]:
data, _usage = service.extract_full_from_pdf(
queue_path,
instruction=prompt,
tiff_page_range=tiff_range,
)
raw_data: dict = data if isinstance(data, dict) else {"data": data}
if isinstance(data, dict):
data["radicado"] = radicado
else:
data = {"radicado": radicado, "data": data}
norm = normalize_to_schema(
data,
paths=selected_paths,
profile_name=prof.get("name", "invoice"),
)
normalized_obj = data
if isinstance(norm, dict):
normalized_obj = norm.get("normalized") or data
return raw_data, normalized_obj
_raw, normalized_obj = await asyncio.to_thread(_extract_and_normalize)
# Persistir resultado en BD
def _persist() -> dict[str, Any]:
with session_scope() as session:
task = session.get(Task, task_id)
if task is None:
return {"skipped": True, "reason": "task_not_found"}
result = dict(task.result or {})
if structure_mode == "radicados" and doc_types and doc_type:
partials = dict(result.get("partials") or {})
radicado_partials = dict(partials.get(radicado) or {})
radicado_partials[doc_type] = normalized_obj or {}
partials[radicado] = radicado_partials
result["partials"] = partials
task.result = result
session.flush()
finalized = _finalize_radicado_if_ready(
session,
task=task,
batch_id=batch_id,
radicado=radicado,
request=request,
fields=fields,
alias_map=alias_map,
)
mark_task_completed_if_done(session, task, batch_id)
return {
"radicado": radicado,
"filename": filename,
"doc_type": doc_type,
"finalized": finalized,
"processed": int((task.result or {}).get("processed") or 0),
}
fused = {"radicado": radicado, "_source_pdf": filename}
if doc_type:
fused[doc_type] = normalized_obj or {}
else:
fused.update(
normalized_obj if isinstance(normalized_obj, dict) else {"data": normalized_obj}
)
row_index = int(result.get("row_count") or 0)
for row in flatten_extraction_result(fused, fields, alias_map):
row_index += 1
session.add(
ExtractionRow(
batch_id=batch_id,
radicado=radicado,
source_pdf=filename,
row_index=row_index,
row_json=row,
raw_json=fused,
)
)
result["row_count"] = row_index
result["processed"] = int(result.get("processed") or 0) + 1
task.result = result
task.updated_at = utcnow()
session.flush()
mark_task_completed_if_done(session, task, batch_id)
return {
"radicado": radicado,
"filename": filename,
"doc_type": doc_type,
"finalized": True,
"processed": int(result.get("processed") or 0),
}
payload = await asyncio.to_thread(_persist)
logger.info("Completado %s / %s — payload=%s", radicado, filename, payload)
return payload
except Exception as exc:
logger.exception("Error procesando %s / %s: %s", radicado, filename, exc)
# Registrar error en la tarea sin fallar el worker
def _record_error() -> None:
try:
with session_scope() as session:
task = session.get(Task, task_id)
if task is None:
return
result = dict(task.result or {})
errors = list(result.get("errors") or [])
errors.append({
"radicado": radicado,
"filename": filename,
"error": str(exc),
})
result["errors"] = errors
result["processed"] = int(result.get("processed") or 0) + 1
task.result = result
task.updated_at = utcnow()
mark_task_completed_if_done(session, task, batch_id)
except Exception as inner:
logger.error("No se pudo registrar el error en BD: %s", inner)
await asyncio.to_thread(_record_error)
return {"error": str(exc), "radicado": radicado, "filename": filename}
finally:
# Eliminar archivo temporal del volumen compartido
try:
if os.path.exists(queue_path):
os.unlink(queue_path)
except OSError as e:
logger.warning("No se pudo eliminar %s: %s", queue_path, e)
@@ -0,0 +1,15 @@
# Go
bin/
vendor/
# Modelos ONNX (regenerables con export_onnx.py)
models/latest/*
!models/latest/.gitkeep
# Datos de paridad
data/
# IDE
.idea/
.vscode/
.cursor/
@@ -0,0 +1,56 @@
# syntax=docker/dockerfile:1
FROM golang:1.23-bookworm AS builder
RUN apt-get update && apt-get install -y --no-install-recommends \
gcc g++ pkg-config \
libtesseract-dev libleptonica-dev \
tesseract-ocr tesseract-ocr-spa \
wget ca-certificates \
&& rm -rf /var/lib/apt/lists/*
# ONNX Runtime
ARG ONNX_VERSION=1.22.0
RUN wget -q https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-x64-${ONNX_VERSION}.tgz \
&& tar -xzf onnxruntime-linux-x64-${ONNX_VERSION}.tgz \
&& cp onnxruntime-linux-x64-${ONNX_VERSION}/lib/libonnxruntime.so* /usr/local/lib/ \
&& ldconfig
ENV CGO_ENABLED=1
ENV ONNXRUNTIME_SHARED_LIBRARY=/usr/local/lib/libonnxruntime.so
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN go build -o /out/api ./cmd/api && go build -o /out/worker ./cmd/worker
FROM debian:bookworm-slim AS runtime
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl wget \
tesseract-ocr tesseract-ocr-spa \
&& rm -rf /var/lib/apt/lists/*
ARG ONNX_VERSION=1.22.0
RUN wget -q https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-x64-${ONNX_VERSION}.tgz \
&& tar -xzf onnxruntime-linux-x64-${ONNX_VERSION}.tgz \
&& cp onnxruntime-linux-x64-${ONNX_VERSION}/lib/libonnxruntime.so* /usr/local/lib/ \
&& rm -rf onnxruntime-linux-x64-${ONNX_VERSION}* \
&& ldconfig
ENV ONNXRUNTIME_SHARED_LIBRARY=/usr/local/lib/libonnxruntime.so
WORKDIR /app
COPY --from=builder /out/api /app/api
COPY --from=builder /out/worker /app/worker
COPY api/openapi.yaml /app/openapi.yaml
ENV OPENAPI_PATH=/app/openapi.yaml
ENV MODEL_PATH=/app/models/model.onnx
ENV INFERENCE_CONFIG=/app/models/inference.json
ENV PORT=8080
EXPOSE 8080
CMD ["/app/api"]
@@ -0,0 +1,14 @@
FROM golang:1.23-bookworm AS builder
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN go build -o /out/autoscaler ./cmd/autoscaler
FROM docker:27-cli
COPY --from=builder /out/autoscaler /usr/local/bin/autoscaler
WORKDIR /project
ENTRYPOINT ["/usr/local/bin/autoscaler"]
@@ -0,0 +1,339 @@
# Services Count — API de Facturas Médicas
![Go](https://img.shields.io/badge/Go-1.23-blue)
Microservicio de inferencia en Go que cuenta servicios facturados en facturas médicas PDF. El entrenamiento del modelo vive en el repositorio hermano [services-count-facturas-medicas-ml](https://github.com/builker-col/services-count-facturas-medicas-ml); este repositorio despliega la API HTTP y los workers de procesamiento.
## Características
- Conteo **síncrono** (1–50 PDFs por petición) y **asíncrono** (hasta 50 000 PDFs por job).
- Pipeline de inferencia: OCR → clustering DBSCAN → Random Forest vía ONNX.
- Escalado horizontal de workers con Docker Compose.
- Especificación OpenAPI y descriptor de herramientas para agentes AI.
## Arquitectura
El sistema se compone de una API HTTP, workers consumidores y tres servicios de infraestructura (Redis, NATS JetStream y MinIO), definidos en [`docker-compose.yml`](docker-compose.yml).
```mermaid
flowchart TB
Client[Cliente_o_Agente]
API[API_Go_8080]
Worker[Worker_Go_xN]
Redis[(Redis)]
NATS[NATS_JetStream]
MinIO[(MinIO)]
Pipeline[OCR_Tesseract_ONNX]
Client -->|sync_POST_v1_count| API
Client -->|async_POST_v1_jobs| API
API --> Pipeline
API --> Redis
API --> MinIO
API -->|publish| NATS
NATS -->|consume| Worker
Worker --> MinIO
Worker --> Pipeline
Worker --> Redis
```
### Pipeline de inferencia
Por cada PDF se procesa la primera página siguiendo este flujo (`internal/pipeline`):
```mermaid
flowchart LR
PDF[PDF_primera_pagina] --> Render[Render_MuPDF]
Render --> OCR[Tesseract_spa]
OCR --> Cluster[DBSCAN_filas]
Cluster --> Features[8_features]
Features --> Infer[ONNX_RandomForest]
Infer --> Result[service_count_y_services]
```
1. **Render** — La primera página se rasteriza con MuPDF (go-fitz) a 300 DPI.
2. **OCR** — Tesseract extrae cajas de texto en español.
3. **Clustering** — DBSCAN agrupa palabras en filas candidatas.
4. **Features** — Se calculan 8 atributos por fila (códigos, moneda, posición, etc.).
5. **Inferencia** — Un Random Forest exportado a ONNX clasifica cada fila como servicio o no.
## Componentes
| Componente | Binario / servicio | Paquete | Responsabilidad |
|------------|-------------------|---------|-----------------|
| API HTTP | `cmd/api` | `internal/api` | Endpoints REST, gestión de jobs, health probes |
| Worker | `cmd/worker` | `internal/worker` | Consume NATS, descarga PDFs de MinIO, persiste resultados |
| Pipeline | — | `internal/pipeline` | Orquestación OCR → clustering → features → inferencia |
| Modelo ONNX | — | `internal/model` | Carga `model.onnx` e `inference.json` |
| OCR | — | `internal/ocr` | Render PDF (go-fitz) + Tesseract (gosseract) |
| Estado de jobs | Redis 7 | `internal/store` | Metadatos, resultados y progreso (TTL 72 h) |
| Cola | NATS JetStream | `internal/queue` | Stream `PDF_JOBS`, subject `pdf.process.*` |
| Object storage | MinIO | `internal/storage` | Almacenamiento de PDFs en jobs asíncronos |
### Estructura del repositorio
```
├── cmd/
│ ├── api/ # Punto de entrada HTTP
│ └── worker/ # Punto de entrada worker
├── internal/
│ ├── api/ # Servidor HTTP (chi)
│ ├── config/ # Variables de entorno
│ ├── domain/ # Tipos de dominio
│ ├── model/ # Inferencia ONNX
│ ├── ocr/ # Extracción OCR
│ ├── pipeline/ # Procesamiento por PDF
│ ├── queue/ # NATS JetStream
│ ├── storage/ # Cliente MinIO
│ ├── store/ # Estado en Redis
│ └── worker/ # Runner del worker
├── api/openapi.yaml # Especificación OpenAPI 3.1
├── mcp/tool.json # Descriptor de herramientas para agentes
├── web/ # Cliente demo (embebido en la API en /demo/)
├── scripts/ # Export ONNX, paridad con Python
└── models/latest/ # model.onnx + inference.json (montado en Docker)
```
## Modos de operación
| Modo | Endpoint principal | Límite | Cuándo usarlo |
|------|-------------------|--------|---------------|
| Síncrono | `POST /v1/count` | 50 PDFs | Pruebas, integraciones en tiempo real, lotes pequeños |
| Asíncrono | `POST /v1/jobs` + poll | 50 000 PDFs/job | MinIO + NATS + workers (path producción) |
**Flujo asíncrono (producción):**
1. Crear job (`POST /v1/jobs`) — shell vacío, prefijo MinIO o lista de claves.
2. Subir PDFs por chunks (`POST /v1/jobs/{jobID}/upload`, máx. 500 por chunk) → **MinIO Put** → **NATS Publish**.
3. Workers consumen la cola, descargan de MinIO, OCR, guardan en Redis.
4. Consultar progreso (`GET /v1/jobs/{jobID}?wait_ms=30000` para long-polling).
5. Obtener resultados paginados (`GET /v1/jobs/{jobID}/results?cursor=&limit=`).
## API REST
Especificación completa: `GET /openapi.json` o [`api/openapi.yaml`](api/openapi.yaml).
| Método | Ruta | Descripción |
|--------|------|-------------|
| `GET` | `/health` | Liveness probe |
| `GET` | `/ready` | Readiness probe |
| `POST` | `/v1/count` | Conteo síncrono (multipart `files`) |
| `POST` | `/v1/jobs` | Crear job asíncrono |
| `POST` | `/v1/jobs/{jobID}/upload` | Subir chunk de PDFs |
| `GET` | `/v1/jobs/{jobID}` | Estado y progreso del job |
| `GET` | `/v1/jobs/{jobID}/results` | Resultados paginados |
| `DELETE` | `/v1/jobs/{jobID}` | Cancelar job |
**Parámetros relevantes:**
- `include_rejected` (query) — Si es `true`, incluye todas las filas OCR con el campo `is_service`.
- Jobs asíncronos devuelven `202 Accepted`; la cola rechaza con `429` si supera `QUEUE_MAX_PENDING`.
**Respuesta por PDF:**
```json
{
"filename": "factura.pdf",
"service_count": 2,
"status": "ok",
"services": [
{ "text": "890201 Consulta ...", "probability": 0.94 }
]
}
```
## Stack tecnológico
| Área | Tecnología |
|------|------------|
| Lenguaje | Go 1.23 |
| HTTP router | [go-chi/chi](https://github.com/go-chi/chi) v5 |
| OCR | go-fitz (MuPDF) + gosseract (Tesseract) |
| ML | ONNX Runtime + Random Forest exportado |
| Cola | NATS JetStream |
| Estado | Redis 7 |
| Storage | MinIO |
| Contenedores | Docker multi-stage ([`Dockerfile`](Dockerfile)) |
## Inicio rápido
### 1. Exportar el modelo ONNX
Desde el repositorio de entrenamiento, generar los artefactos de inferencia:
```bash
pip install -r requirements-export.txt
python scripts/export_onnx.py \
--joblib ../services-count/models/latest/model.joblib \
--metadata ../services-count/models/v1.0.0/model_metadata.json
```
Esto produce `models/latest/model.onnx` e `inference.json`.
### 2. Levantar con Docker Compose
```bash
docker compose up --build
```
| Servicio | Puerto / acceso |
|----------|-----------------|
| API | `http://localhost:8080` |
| MinIO Console | `http://localhost:9002` |
| Workers | 1 réplica inicial; autoscaler sube hasta 8 según cola |
| Autoscaler | Escala `worker` vía Docker socket según `PendingCount` en Redis |
### 3. Probar
```bash
# Síncrono
curl -F "files=@factura.pdf" http://localhost:8080/v1/count
# Asíncrono (PDFs ya en MinIO bajo un prefijo)
curl -X POST http://localhost:8080/v1/jobs \
-H "Content-Type: application/json" \
-d '{"prefix":"jobs/batch-1/"}'
```
## Cliente web de ejemplo
Con la API en ejecución, abre **[http://localhost:8080/demo/](http://localhost:8080/demo/)** para probar la integración desde el navegador. Los archivos estáticos se sirven desde la propia API (mismo origen, sin CORS).
| Pestaña | Demuestra |
|---------|-----------|
| **Síncrono** | `POST /v1/count` con subida multipart de hasta 50 PDFs |
| **Asíncrono** | `POST /v1/jobs` → upload → polling de estado → resultados paginados |
El modo asíncrono requiere que workers, Redis, NATS y MinIO estén levantados (igual que con `docker compose up`). Código fuente en [`web/`](web/).
## Desarrollo local
Requisitos adicionales a Go 1.23+:
- Tesseract con datos de idioma `spa`
- ONNX Runtime (la variable `ONNXRUNTIME_SHARED_LIBRARY` apunta a la librería compartida)
- Redis, NATS y MinIO en ejecución
```bash
go build -o api ./cmd/api
go build -o worker ./cmd/worker
```
Configurar las variables de entorno para apuntar a los servicios locales (ver tabla siguiente).
## Configuración
Variables definidas en [`internal/config/config.go`](internal/config/config.go):
| Variable | Default | Descripción |
|----------|---------|-------------|
| `PORT` | `8080` | Puerto de la API |
| `REDIS_URL` | `redis://redis:6379/0` | Conexión Redis |
| `NATS_URL` | `nats://nats:4222` | Conexión NATS |
| `NATS_STREAM` | `PDF_JOBS` | Nombre del stream JetStream |
| `NATS_SUBJECT_PREFIX` | `pdf.process` | Prefijo de subjects |
| `MINIO_ENDPOINT` | `minio:9000` | Endpoint MinIO |
| `MINIO_ACCESS_KEY` | `minioadmin` | Access key |
| `MINIO_SECRET_KEY` | `minioadmin` | Secret key |
| `MINIO_BUCKET` | `pdf-count` | Bucket de PDFs |
| `MINIO_USE_SSL` | `false` | Usar SSL con MinIO |
| `MODEL_PATH` | `/app/models/model.onnx` | Ruta al modelo ONNX |
| `INFERENCE_CONFIG` | `/app/models/inference.json` | Metadata de inferencia |
| `OCR_DPI` | `300` | DPI de renderizado PDF (150 reduce latencia ~2–4×) |
| `OCR_LANG` | `spa` | Idioma Tesseract |
| `DBSCAN_EPS` | `10` | Parámetro epsilon de DBSCAN |
| `DBSCAN_MIN_SAMPLES` | `1` | Mínimo de muestras DBSCAN |
| `SYNC_MAX_FILES` | `50` | Máximo de PDFs en modo síncrono |
| `JOB_MAX_FILES` | `50000` | Máximo de PDFs por job |
| `QUEUE_MAX_PENDING` | `100000` | Límite de mensajes pendientes en cola |
| `WORKER_CONCURRENCY` | `4` | Goroutines paralelas por worker |
| `MAX_PDF_SIZE_MB` | `20` | Tamaño máximo por PDF |
| `PDF_PROCESS_TIMEOUT_SEC` | `120` | Timeout de procesamiento por PDF |
| `JOB_RESULT_TTL_HOURS` | `72` | TTL de resultados en Redis |
| `OCR_POOL_SIZE` | `4` (API) / `WORKER_CONCURRENCY` (worker) | Clientes Tesseract reutilizables en pool |
| `SYNC_PROCESS_CONCURRENCY` | `4` (o `OCR_POOL_SIZE`) | PDFs procesados en paralelo en `/v1/count` |
| `JOB_INLINE_PROCESSING` | `false` | `true` procesa uploads en la API sin MinIO (solo dev local) |
| `LOG_PIPELINE_TIMING` | `false` | Logs de duración por etapa del pipeline |
| `NATS_FETCH_MAX_WAIT_MS` | `300` | Tiempo máximo de espera del worker al leer NATS |
## Rendimiento
El cuello de botella de `/v1/count` es el **OCR con Tesseract**, no la inferencia ONNX. Los clientes Tesseract se crean una vez al arranque y se reutilizan desde un pool (`internal/ocr/pool.go`).
| Etapa | Tiempo típico (1 PDF) |
|-------|------------------------|
| Render PDF (MuPDF) | ~100–300 ms |
| OCR Tesseract | ~3–8 s |
| Clustering + ONNX | ~10–50 ms |
**Latencia esperada (síncrono):** ~3–10 s por PDF. Varios PDFs se procesan en paralelo (`SYNC_PROCESS_CONCURRENCY`, default 4). Ejemplo: 20 PDFs ≈ `ceil(20/4) × 2.85s ≈ 15s`.
**Latencia esperada (asíncrono por upload):** path producción MinIO → NATS → worker. Con `WORKER_CONCURRENCY=4` y autoscaler activo, 16 PDFs ≈ `ceil(16/4) × 2.85s` más overhead Put/Get (~15–25 s). 100 PDFs escala hasta 8 workers (32 slots OCR).
**Asíncrono por prefix MinIO** (`POST /v1/jobs {"prefix":"..."}`): mismos workers; los PDFs ya están en el bucket.
**Ajustes para reducir latencia:**
- `OCR_DPI=150` — reduce tiempo de render y OCR (~2–4× más rápido) con posible ligera pérdida de precisión. Validar con `scripts/compare_parity.py` antes de usar en producción.
- `OCR_POOL_SIZE=4` y `WORKER_CONCURRENCY=4` — paralelismo por réplica worker.
- Upload pipeline publica en NATS tras cada Put (no al final del lote).
- `LOG_PIPELINE_TIMING=true` — logs por etapa: `ocr_render`, `ocr_tesseract`, `cluster`, `infer`, `total`.
- `NATS_FETCH_MAX_WAIT_MS=300` — el worker recoge mensajes con menor espera.
- `GET /v1/jobs/{id}?wait_ms=30000` — long-polling en lugar de poll cada 500 ms.
## Escalado
### Autoscaler (docker compose)
El servicio `autoscaler` lee `PendingCount` en Redis cada 5 s y ajusta réplicas `worker`:
```
desired = clamp(ceil(pending / WORKER_SLOTS), MIN_WORKERS, MAX_WORKERS)
```
| Variable | Default | Descripción |
|----------|---------|-------------|
| `MIN_WORKERS` | `1` | Réplicas mínimas |
| `MAX_WORKERS` | `8` | Réplicas máximas |
| `WORKER_SLOTS` | `4` | PDFs en paralelo por réplica (`WORKER_CONCURRENCY`) |
| `SCALE_DOWN_IDLE_SEC` | `60` | Segundos sin cola antes de bajar 1 réplica |
Requiere montar `/var/run/docker.sock` (solo entorno local). En Kubernetes usar HPA sobre lag de NATS o métricas custom.
### Escalado manual
```bash
docker compose up --scale worker=8 -d
```
Cada worker procesa hasta `WORKER_CONCURRENCY` PDFs en paralelo. NATS JetStream distribuye los mensajes entre réplicas con política work-queue.
## Integración con agentes (MCP)
[`mcp/tool.json`](mcp/tool.json) es un **descriptor declarativo** de herramientas para agentes AI (Cursor, etc.). No incluye un servidor MCP embebido.
| Herramienta | Modo | Flujo |
|-------------|------|-------|
| `count_pdf_services` | Síncrono | `POST /v1/count` con multipart |
| `count_pdf_services_batch` | Asíncrono | Crear job → poll estado → resultados paginados |
## Paridad con Python
Scripts para validar que la inferencia Go reproduce los resultados del pipeline Python original:
```bash
set SERVICES_COUNT_ROOT=../services-count
python scripts/parity_baseline.py
python scripts/compare_parity.py
```
## Troubleshooting
| Síntoma | Causa probable | Acción |
|---------|------------------|--------|
| API no arranca | Modelo ONNX ausente | Ejecutar `scripts/export_onnx.py` y verificar `models/latest/` |
| `/ready` falla | Redis o NATS no disponibles | Comprobar healthchecks en `docker compose ps` |
| Jobs en `pending` sin avance | Workers no consumen cola | Verificar logs del worker y conectividad a NATS/MinIO |
| Job atascado en `running` con N/N procesados | Race en metadata del job (versiones antiguas) | Reconstruir con `docker compose up --build`; `GetJob` ahora finaliza jobs completados |
| `429` al crear job | Cola saturada | Escalar workers o esperar a que baje `QUEUE_MAX_PENDING` |
| Resultados vacíos o erróneos | OCR o modelo desactualizado | Revisar `OCR_LANG=spa`, reexportar ONNX desde el repo ML |
@@ -0,0 +1,230 @@
openapi: 3.1.0
info:
title: PDF Count API
description: Conteo de servicios facturados en facturas médicas PDF.
version: 1.0.0
servers:
- url: http://localhost:8080
paths:
/health:
get:
summary: Liveness probe
responses:
"200":
description: OK
/ready:
get:
summary: Readiness probe
responses:
"200":
description: Ready
/v1/count:
post:
summary: Conteo síncrono (hasta 50 PDFs)
parameters:
- name: include_rejected
in: query
schema:
type: boolean
default: false
requestBody:
required: true
content:
multipart/form-data:
schema:
type: object
properties:
files:
type: array
items:
type: string
format: binary
responses:
"200":
description: Resultados por PDF
content:
application/json:
schema:
$ref: "#/components/schemas/CountResponse"
/v1/jobs:
post:
summary: Crear job asíncrono
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/CreateJobRequest"
responses:
"202":
description: Job aceptado
content:
application/json:
schema:
$ref: "#/components/schemas/JobStatus"
/v1/jobs/{jobID}:
get:
summary: Estado del job
parameters:
- name: jobID
in: path
required: true
schema:
type: string
- name: wait_ms
in: query
description: Long-polling; espera hasta wait_ms (máx. 30000) a que el job cambie de estado
schema:
type: integer
minimum: 0
maximum: 30000
responses:
"200":
description: Progreso
content:
application/json:
schema:
$ref: "#/components/schemas/JobStatus"
delete:
summary: Cancelar job
parameters:
- name: jobID
in: path
required: true
schema:
type: string
responses:
"200":
description: Cancelado
/v1/jobs/{jobID}/upload:
post:
summary: Subir chunk de PDFs a un job
parameters:
- name: jobID
in: path
required: true
schema:
type: string
- name: include_rejected
in: query
schema:
type: boolean
requestBody:
content:
multipart/form-data:
schema:
type: object
properties:
files:
type: array
items:
type: string
format: binary
responses:
"202":
description: Chunk aceptado
/v1/jobs/{jobID}/results:
get:
summary: Resultados paginados
parameters:
- name: jobID
in: path
required: true
schema:
type: string
- name: cursor
in: query
schema:
type: string
- name: limit
in: query
schema:
type: integer
default: 500
responses:
"200":
description: Página de resultados
content:
application/json:
schema:
$ref: "#/components/schemas/JobResultsPage"
components:
schemas:
ServiceRow:
type: object
properties:
text:
type: string
probability:
type: number
format: float
is_service:
type: boolean
PDFResult:
type: object
properties:
filename:
type: string
service_count:
type: integer
status:
type: string
enum: [ok, error, pending]
services:
type: array
items:
$ref: "#/components/schemas/ServiceRow"
error:
type: string
CountResponse:
type: object
properties:
results:
type: array
items:
$ref: "#/components/schemas/PDFResult"
processed:
type: integer
failed:
type: integer
JobStatus:
type: object
properties:
job_id:
type: string
status:
type: string
total:
type: integer
processed:
type: integer
failed:
type: integer
JobResultsPage:
type: object
properties:
job_id:
type: string
results:
type: array
items:
$ref: "#/components/schemas/PDFResult"
next_cursor:
type: string
has_more:
type: boolean
CreateJobRequest:
type: object
properties:
files:
type: array
items:
type: string
prefix:
type: string
description: Prefijo MinIO con PDFs ya subidos
include_rejected:
type: boolean
default: false
callback_url:
type: string
@@ -0,0 +1,74 @@
package main
import (
"context"
"log"
"os"
"os/signal"
"syscall"
"time"
"github.com/pys/pdf-count/internal/api"
"github.com/pys/pdf-count/internal/config"
"github.com/pys/pdf-count/internal/model"
"github.com/pys/pdf-count/internal/ocr"
"github.com/pys/pdf-count/internal/queue"
"github.com/pys/pdf-count/internal/storage"
"github.com/pys/pdf-count/internal/store"
)
func main() {
cfg := config.Load()
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
predictor, err := model.NewPredictor(cfg.ModelPath, cfg.InferencePath)
if err != nil {
log.Fatalf("model: %v", err)
}
defer predictor.Close()
poolSize := cfg.OCRPoolSize
if poolSize <= 0 {
poolSize = 4
}
ocrEngine, err := ocr.NewEngine(cfg.OCRLang, poolSize)
if err != nil {
log.Fatalf("ocr: %v", err)
}
defer ocrEngine.Close()
st, err := store.New(cfg.RedisURL, cfg.JobResultTTL)
if err != nil {
log.Fatalf("redis: %v", err)
}
defer st.Close()
stg, err := storage.New(cfg.MinIOEndpoint, cfg.MinIOAccessKey, cfg.MinIOSecretKey, cfg.MinIOBucket, cfg.MinIOUseSSL)
if err != nil {
log.Fatalf("minio: %v", err)
}
initCtx, initCancel := context.WithTimeout(ctx, 10*time.Second)
if err := stg.EnsureBucket(initCtx); err != nil {
initCancel()
log.Fatalf("minio bucket: %v", err)
}
initCancel()
pub, err := queue.NewPublisher(cfg.NATSURL, cfg.StreamName, cfg.SubjectPrefix)
if err != nil {
log.Fatalf("nats: %v", err)
}
openAPIPath := os.Getenv("OPENAPI_PATH")
if openAPIPath == "" {
openAPIPath = "api/openapi.json"
}
srv := api.NewServer(cfg, predictor, ocrEngine, st, stg, pub, api.LoadOpenAPI(openAPIPath))
addr := ":" + cfg.Port
log.Printf("api listening on %s", addr)
if err := api.ListenAndServe(ctx, addr, srv.Router()); err != nil {
log.Fatalf("api: %v", err)
}
}
@@ -0,0 +1,157 @@
package main
import (
"context"
"log"
"math"
"os"
"os/exec"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/pys/pdf-count/internal/store"
)
func main() {
cfg := loadConfig()
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
st, err := store.New(cfg.redisURL, 72*time.Hour)
if err != nil {
log.Fatalf("redis: %v", err)
}
defer st.Close()
log.Printf("autoscaler started min=%d max=%d slots=%d service=%s",
cfg.minWorkers, cfg.maxWorkers, cfg.workerSlots, cfg.composeService)
var idleSince time.Time
for {
select {
case <-ctx.Done():
log.Println("autoscaler stopped")
return
default:
}
pending, err := st.PendingCount(ctx)
if err != nil {
log.Printf("autoscaler pending count: %v", err)
time.Sleep(cfg.pollInterval)
continue
}
current, err := countWorkers(cfg)
if err != nil {
log.Printf("autoscaler count workers: %v", err)
time.Sleep(cfg.pollInterval)
continue
}
desired := cfg.minWorkers
if pending > 0 {
desired = int(math.Ceil(float64(pending) / float64(cfg.workerSlots)))
if desired < cfg.minWorkers {
desired = cfg.minWorkers
}
if desired > cfg.maxWorkers {
desired = cfg.maxWorkers
}
idleSince = time.Time{}
} else if current > cfg.minWorkers {
if idleSince.IsZero() {
idleSince = time.Now()
} else if time.Since(idleSince) >= cfg.scaleDownIdle {
desired = current - 1
idleSince = time.Time{}
} else {
desired = current
}
}
if desired != current {
log.Printf("autoscaler pending=%d current=%d desired=%d scaling", pending, current, desired)
if err := scaleWorkers(cfg, desired); err != nil {
log.Printf("autoscaler scale: %v", err)
}
}
time.Sleep(cfg.pollInterval)
}
}
type autoscalerConfig struct {
redisURL string
composeFile string
composeService string
minWorkers int
maxWorkers int
workerSlots int
pollInterval time.Duration
scaleDownIdle time.Duration
}
func loadConfig() autoscalerConfig {
return autoscalerConfig{
redisURL: getenv("REDIS_URL", "redis://redis:6379/0"),
composeFile: getenv("COMPOSE_FILE", "/project/docker-compose.yml"),
composeService: getenv("COMPOSE_SERVICE", "worker"),
minWorkers: getenvInt("MIN_WORKERS", 1),
maxWorkers: getenvInt("MAX_WORKERS", 8),
workerSlots: getenvInt("WORKER_SLOTS", 4),
pollInterval: time.Duration(getenvInt("POLL_INTERVAL_SEC", 5)) * time.Second,
scaleDownIdle: time.Duration(getenvInt("SCALE_DOWN_IDLE_SEC", 60)) * time.Second,
}
}
func countWorkers(cfg autoscalerConfig) (int, error) {
cmd := exec.Command("docker", "compose",
"-f", cfg.composeFile,
"ps", cfg.composeService,
"--status", "running",
"-q",
)
out, err := cmd.Output()
if err != nil {
return 0, err
}
trimmed := strings.TrimSpace(string(out))
if trimmed == "" {
return 0, nil
}
return len(strings.Split(trimmed, "\n")), nil
}
func scaleWorkers(cfg autoscalerConfig, n int) error {
scaleArg := cfg.composeService + "=" + strconv.Itoa(n)
cmd := exec.Command("docker", "compose",
"-f", cfg.composeFile,
"up", "-d", "--no-recreate",
"--scale", scaleArg,
cfg.composeService,
)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd.Run()
}
func getenv(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
func getenvInt(key string, fallback int) int {
if v := os.Getenv(key); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return fallback
}
@@ -0,0 +1,7 @@
package main
import "github.com/pys/pdf-count/internal/worker"
func main() {
worker.RunMain()
}
@@ -0,0 +1,125 @@
services:␍
redis:␍
image: redis:7-alpine␍
command: redis-server --maxmemory 1gb --maxmemory-policy allkeys-lru␍
healthcheck:␍
test: ["CMD", "redis-cli", "ping"]␍
interval: 5s␍
timeout: 3s␍
retries: 5␍
␍
nats:␍
image: nats:2-alpine␍
command: ["-js", "-m", "8222"]␍
healthcheck:␍
test: ["CMD", "wget", "-qO-", "http://localhost:8222/healthz"]␍
interval: 5s␍
timeout: 3s␍
retries: 5␍
␍
minio:␍
image: minio/minio:latest␍
command: server /data --console-address ":9001"␍
environment:␍
MINIO_ROOT_USER: minioadmin␍
MINIO_ROOT_PASSWORD: minioadmin␍
volumes:␍
- minio_data:/data␍
ports:␍
- "9002:9001"␍
healthcheck:␍
test: ["CMD-SHELL", "curl -f http://localhost:9000/minio/health/live || exit 1"]␍
interval: 10s␍
timeout: 5s␍
retries: 5␍
start_period: 10s␍
␍
api:␍
build: .␍
command: ["/app/api"]␍
ports:␍
- "8080:8080"␍
depends_on:␍
redis:␍
condition: service_healthy␍
nats:␍
condition: service_healthy␍
minio:␍
condition: service_started␍
environment:␍
REDIS_URL: redis://redis:6379/0␍
NATS_URL: nats://nats:4222␍
MINIO_ENDPOINT: minio:9000␍
MINIO_ACCESS_KEY: minioadmin␍
MINIO_SECRET_KEY: minioadmin␍
MINIO_BUCKET: pdf-count␍
MODEL_PATH: /app/models/model.onnx␍
INFERENCE_CONFIG: /app/models/inference.json␍
SYNC_MAX_FILES: 50␍
JOB_MAX_FILES: 50000␍
QUEUE_MAX_PENDING: 100000␍
OPENAPI_PATH: /app/openapi.yaml␍
OCR_POOL_SIZE: 4␍
SYNC_PROCESS_CONCURRENCY: 4␍
JOB_INLINE_PROCESSING: "false"␍
LOG_PIPELINE_TIMING: "true"␍
volumes:␍
- ./models/latest:/app/models:ro␍
healthcheck:␍
test: ["CMD", "curl", "-f", "http://localhost:8080/health"]␍
interval: 10s␍
timeout: 5s␍
retries: 5␍
␍
worker:␍
build: .␍
command: ["/app/worker"]␍
depends_on:␍
redis:␍
condition: service_healthy␍
nats:␍
condition: service_healthy␍
minio:␍
condition: service_started␍
environment:␍
REDIS_URL: redis://redis:6379/0␍
NATS_URL: nats://nats:4222␍
MINIO_ENDPOINT: minio:9000␍
MINIO_ACCESS_KEY: minioadmin␍
MINIO_SECRET_KEY: minioadmin␍
MINIO_BUCKET: pdf-count␍
MODEL_PATH: /app/models/model.onnx␍
INFERENCE_CONFIG: /app/models/inference.json␍
WORKER_CONCURRENCY: 4␍
PDF_PROCESS_TIMEOUT_SEC: 120␍
OCR_POOL_SIZE: 4␍
NATS_FETCH_MAX_WAIT_MS: 300␍
LOG_PIPELINE_TIMING: "true"␍
JOB_INLINE_PROCESSING: "false"␍
volumes:␍
- ./models/latest:/app/models:ro␍
␍
autoscaler:␍
build:␍
context: .␍
dockerfile: Dockerfile.autoscaler␍
depends_on:␍
redis:␍
condition: service_healthy␍
volumes:␍
- /var/run/docker.sock:/var/run/docker.sock␍
- .:/project:ro␍
working_dir: /project␍
environment:␍
REDIS_URL: redis://redis:6379/0␍
COMPOSE_FILE: /project/docker-compose.yml␍
COMPOSE_SERVICE: worker␍
MIN_WORKERS: 1␍
MAX_WORKERS: 8␍
WORKER_SLOTS: 4␍
POLL_INTERVAL_SEC: 5␍
SCALE_DOWN_IDLE_SEC: 60␍
␍
volumes:␍
minio_data:␍
␍
@@ -0,0 +1,40 @@
module github.com/pys/pdf-count
go 1.23.0
toolchain go1.23.4
require (
github.com/gen2brain/go-fitz v1.24.14
github.com/go-chi/chi/v5 v5.2.1
github.com/google/uuid v1.6.0
github.com/minio/minio-go/v7 v7.0.90
github.com/nats-io/nats.go v1.39.1
github.com/otiai10/gosseract/v2 v2.4.1
github.com/redis/go-redis/v9 v9.7.3
github.com/yalue/onnxruntime_go v1.21.0
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/ebitengine/purego v0.8.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/jupiterrider/ffi v0.2.0 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/minio/crc64nvme v1.0.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/nats-io/nkeys v0.4.9 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rs/xid v1.6.0 // indirect
golang.org/x/crypto v0.36.0 // indirect
golang.org/x/net v0.38.0 // indirect
golang.org/x/sys v0.31.0 // indirect
golang.org/x/text v0.23.0 // indirect
)
@@ -0,0 +1,69 @@
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/ebitengine/purego v0.8.0 h1:JbqvnEzRvPpxhCJzJJ2y0RbiZ8nyjccVUrSM3q+GvvE=
github.com/ebitengine/purego v0.8.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/gen2brain/go-fitz v1.24.14 h1:09weRkjVtLYNGo7l0J7DyOwBExbwi8SJ9h8YPhw9WEo=
github.com/gen2brain/go-fitz v1.24.14/go.mod h1:0KaZeQgASc20Yp5R/pFzyy7SmP01XcoHKNF842U2/S4=
github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8=
github.com/go-chi/chi/v5 v5.2.1/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jupiterrider/ffi v0.2.0 h1:tMM70PexgYNmV+WyaYhJgCvQAvtTCs3wXeILPutihnA=
github.com/jupiterrider/ffi v0.2.0/go.mod h1:yqYqX5DdEccAsHeMn+6owkoI2llBLySVAF8dwCDZPVs=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY=
github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.90 h1:TmSj1083wtAD0kEYTx7a5pFsv3iRYMsOJ6A4crjA1lE=
github.com/minio/minio-go/v7 v7.0.90/go.mod h1:uvMUcGrpgeSAAI6+sD3818508nUyMULw94j2Nxku/Go=
github.com/nats-io/nats.go v1.39.1 h1:oTkfKBmz7W047vRxV762M67ZdXeOtUgvbBaNoQ+3PPk=
github.com/nats-io/nats.go v1.39.1/go.mod h1:MgRb8oOdigA6cYpEPhXJuRVH6UE/V4jblJ2jQ27IXYM=
github.com/nats-io/nkeys v0.4.9 h1:qe9Faq2Gxwi6RZnZMXfmGMZkg3afLLOtrU+gDZJ35b0=
github.com/nats-io/nkeys v0.4.9/go.mod h1:jcMqs+FLG+W5YO36OX6wFIFcmpdAns+w1Wm6D3I/evE=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/otiai10/gosseract/v2 v2.4.1 h1:G8AyBpXEeSlcq8TI85LH/pM5SXk8Djy2GEXisgyblRw=
github.com/otiai10/gosseract/v2 v2.4.1/go.mod h1:1gNWP4Hgr2o7yqWfs6r5bZxAatjOIdqWxJLWsTsembk=
github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs=
github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM=
github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/yalue/onnxruntime_go v1.21.0 h1:DdtvfY7OP5gR8mwPDqAOAQckf+KcI30hPNJL8hQaYWI=
github.com/yalue/onnxruntime_go v1.21.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -0,0 +1,53 @@
package api
import (
"context"
"github.com/pys/pdf-count/internal/pipeline"
"github.com/pys/pdf-count/internal/store"
)
// FilePayload holds a PDF ready for inline job processing.
type FilePayload struct {
Filename string
Data []byte
}
// JobRunner processes uploaded job files in-process using the API OCR pool.
type JobRunner struct {
store *store.Store
newProc func(includeRejected bool) *pipeline.Processor
sem chan struct{}
}
func NewJobRunner(st *store.Store, newProc func(includeRejected bool) *pipeline.Processor, concurrency int) *JobRunner {
if concurrency < 1 {
concurrency = 1
}
return &JobRunner{
store: st,
newProc: newProc,
sem: make(chan struct{}, concurrency),
}
}
// ProcessOne schedules a single PDF for inline OCR (non-blocking).
func (jr *JobRunner) ProcessOne(jobID string, file FilePayload, includeRejected bool) {
go func() {
jr.sem <- struct{}{}
defer func() { <-jr.sem }()
ctx := context.Background()
proc := jr.newProc(includeRejected)
result := proc.ProcessPDF(file.Filename, file.Data)
_ = jr.store.SaveResult(ctx, jobID, result)
_ = jr.store.TryCompleteJob(ctx, jobID)
}()
}
// ProcessFiles runs OCR in background goroutines and updates job progress in Redis.
func (jr *JobRunner) ProcessFiles(jobID string, files []FilePayload, includeRejected bool) {
for _, file := range files {
jr.ProcessOne(jobID, file, includeRejected)
}
}
@@ -0,0 +1,591 @@
package api
import (
"context"
"encoding/json"
"io"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/google/uuid"
"github.com/pys/pdf-count/internal/config"
"github.com/pys/pdf-count/internal/domain"
"github.com/pys/pdf-count/internal/model"
"github.com/pys/pdf-count/internal/ocr"
"github.com/pys/pdf-count/internal/pipeline"
"github.com/pys/pdf-count/internal/queue"
"github.com/pys/pdf-count/internal/storage"
"github.com/pys/pdf-count/internal/store"
"github.com/pys/pdf-count/web"
"gopkg.in/yaml.v3"
)
type Server struct {
cfg config.Config
predictor *model.Predictor
ocrEngine *ocr.Engine
store *store.Store
storage *storage.Client
publisher *queue.Publisher
jobRunner *JobRunner
openAPI []byte
ready bool
readyMu sync.RWMutex
}
func NewServer(cfg config.Config, predictor *model.Predictor, ocrEngine *ocr.Engine, st *store.Store, stg *storage.Client, pub *queue.Publisher, openAPI []byte) *Server {
srv := &Server{
cfg: cfg,
predictor: predictor,
ocrEngine: ocrEngine,
store: st,
storage: stg,
publisher: pub,
openAPI: openAPI,
ready: true,
}
srv.jobRunner = NewJobRunner(st, srv.newProcessor, srv.ocrConcurrency())
return srv
}
func (s *Server) processConcurrency() int {
c := s.cfg.SyncProcessConcurrency
if c <= 0 {
c = s.cfg.OCRPoolSize
}
if c <= 0 {
c = 4
}
return c
}
func (s *Server) ocrConcurrency() int {
c := s.processConcurrency()
pool := s.cfg.OCRPoolSize
if pool <= 0 {
pool = 4
}
if c > pool {
c = pool
}
return c
}
func (s *Server) SetReady(v bool) {
s.readyMu.Lock()
s.ready = v
s.readyMu.Unlock()
}
func (s *Server) Router() http.Handler {
r := chi.NewRouter()
r.Use(middleware.RequestID)
r.Use(middleware.RealIP)
r.Use(middleware.Recoverer)
r.Get("/health", s.health)
r.Get("/ready", s.readyHandler)
r.Get("/openapi.json", s.openAPIHandler)
r.Get("/demo", func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/demo/", http.StatusMovedPermanently)
})
r.Handle("/demo/*", http.StripPrefix("/demo/", http.FileServer(http.FS(web.FS))))
r.Route("/v1", func(r chi.Router) {
r.Post("/count", s.countSync)
r.Post("/jobs", s.createJob)
r.Post("/jobs/{jobID}/upload", s.uploadChunk)
r.Get("/jobs/{jobID}", s.getJob)
r.Get("/jobs/{jobID}/results", s.getJobResults)
r.Delete("/jobs/{jobID}", s.cancelJob)
})
return r
}
func (s *Server) health(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
}
func (s *Server) readyHandler(w http.ResponseWriter, r *http.Request) {
s.readyMu.RLock()
ok := s.ready
s.readyMu.RUnlock()
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"status": "not_ready"})
return
}
writeJSON(w, http.StatusOK, map[string]string{"status": "ready"})
}
func (s *Server) openAPIHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(s.openAPI)
}
func (s *Server) countSync(w http.ResponseWriter, r *http.Request) {
includeRejected := r.URL.Query().Get("include_rejected") == "true"
maxBytes := int64(s.cfg.MaxPDFSizeMB) * 1024 * 1024
if err := r.ParseMultipartForm(maxBytes * int64(s.cfg.SyncMaxFiles)); err != nil {
httpError(w, http.StatusBadRequest, "invalid multipart form")
return
}
files := r.MultipartForm.File["files"]
if len(files) == 0 {
httpError(w, http.StatusBadRequest, "field 'files' required")
return
}
if len(files) > s.cfg.SyncMaxFiles {
httpError(w, http.StatusBadRequest, "too many files")
return
}
proc := s.newProcessor(includeRejected)
type fileInput struct {
filename string
data []byte
errRes *domain.PDFResult
}
inputs := make([]fileInput, 0, len(files))
for _, fh := range files {
f, err := fh.Open()
if err != nil {
errRes := domain.PDFResult{
Filename: filepath.Base(fh.Filename),
Status: domain.StatusError,
Error: err.Error(),
}
inputs = append(inputs, fileInput{errRes: &errRes})
continue
}
data, err := io.ReadAll(io.LimitReader(f, maxBytes+1))
_ = f.Close()
if err != nil {
errRes := domain.PDFResult{
Filename: filepath.Base(fh.Filename),
Status: domain.StatusError,
Error: err.Error(),
}
inputs = append(inputs, fileInput{errRes: &errRes})
continue
}
if int64(len(data)) > maxBytes {
errRes := domain.PDFResult{
Filename: filepath.Base(fh.Filename),
Status: domain.StatusError,
Error: "file too large",
}
inputs = append(inputs, fileInput{errRes: &errRes})
continue
}
inputs = append(inputs, fileInput{
filename: filepath.Base(fh.Filename),
data: data,
})
}
results := make([]domain.PDFResult, len(inputs))
failed := 0
sem := make(chan struct{}, s.ocrConcurrency())
var wg sync.WaitGroup
var mu sync.Mutex
for i, in := range inputs {
if in.errRes != nil {
results[i] = *in.errRes
failed++
continue
}
wg.Add(1)
go func(idx int, in fileInput) {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()
res := proc.ProcessPDF(in.filename, in.data)
mu.Lock()
results[idx] = res
if res.Status == domain.StatusError {
failed++
}
mu.Unlock()
}(i, in)
}
wg.Wait()
writeJSON(w, http.StatusOK, domain.CountResponse{
Results: results,
Processed: len(results) - failed,
Failed: failed,
})
}
type createJobRequest struct {
Files []string `json:"files"`
Prefix string `json:"prefix"`
IncludeRejected bool `json:"include_rejected"`
CallbackURL string `json:"callback_url"`
}
func (s *Server) createJob(w http.ResponseWriter, r *http.Request) {
var req createJobRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
httpError(w, http.StatusBadRequest, "invalid json body")
return
}
ctx := r.Context()
pending, err := s.store.PendingCount(ctx)
if err == nil && int(pending) > s.cfg.QueueMaxPending {
httpError(w, http.StatusTooManyRequests, "queue full")
return
}
jobID := uuid.NewString()
var objectKeys []string
var filenames []string
if req.Prefix != "" {
prefix := strings.TrimSuffix(req.Prefix, "/")
keys, err := s.storage.ListByPrefix(ctx, prefix)
if err != nil {
httpError(w, http.StatusBadRequest, err.Error())
return
}
objectKeys = keys
for _, k := range keys {
filenames = append(filenames, filepath.Base(k))
}
} else if len(req.Files) > 0 {
for _, f := range req.Files {
name := filepath.Base(f)
objectKeys = append(objectKeys, storage.JobObjectKey(jobID, name))
filenames = append(filenames, name)
}
} else {
// Empty job shell for chunked upload flow.
if err := s.store.CreateJob(ctx, jobID, 0, req.IncludeRejected); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusAccepted, domain.JobStatus{
JobID: jobID,
Status: domain.JobQueued,
Total: 0,
})
return
}
if len(objectKeys) > s.cfg.JobMaxFiles {
httpError(w, http.StatusBadRequest, "too many files in job")
return
}
if err := s.store.CreateJob(ctx, jobID, len(objectKeys), req.IncludeRejected); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
if err := s.store.AddJobFiles(ctx, jobID, filenames); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
_ = s.store.MarkRunning(ctx, jobID)
for i, key := range objectKeys {
msg := domain.QueueMessage{
JobID: jobID,
ObjectKey: key,
Filename: filenames[i],
IncludeRejected: req.IncludeRejected,
}
if err := s.publisher.Publish(ctx, msg); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
}
writeJSON(w, http.StatusAccepted, domain.JobStatus{
JobID: jobID,
Status: domain.JobRunning,
Total: len(objectKeys),
})
}
func (s *Server) uploadChunk(w http.ResponseWriter, r *http.Request) {
jobID := chi.URLParam(r, "jobID")
includeRejected := r.URL.Query().Get("include_rejected") == "true"
maxBytes := int64(s.cfg.MaxPDFSizeMB) * 1024 * 1024
if err := r.ParseMultipartForm(maxBytes * 500); err != nil {
httpError(w, http.StatusBadRequest, "invalid multipart form")
return
}
files := r.MultipartForm.File["files"]
if len(files) == 0 {
httpError(w, http.StatusBadRequest, "field 'files' required")
return
}
if len(files) > 500 {
httpError(w, http.StatusBadRequest, "max 500 files per chunk")
return
}
ctx := r.Context()
status, err := s.store.GetJob(ctx, jobID)
if err != nil {
_ = s.store.CreateJob(ctx, jobID, 0, includeRejected)
status = domain.JobStatus{JobID: jobID, Status: domain.JobQueued}
}
if status.Status == domain.JobCancelled {
httpError(w, http.StatusConflict, "job cancelled")
return
}
var filenames []string
if s.cfg.JobInlineProcessing {
_ = s.store.MarkRunning(ctx, jobID)
for _, fh := range files {
f, err := fh.Open()
if err != nil {
continue
}
data, err := io.ReadAll(io.LimitReader(f, maxBytes+1))
_ = f.Close()
if err != nil || int64(len(data)) > maxBytes {
continue
}
name := filepath.Base(fh.Filename)
if err := s.store.AddJobFiles(ctx, jobID, []string{name}); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
if err := s.store.IncrementTotal(ctx, jobID, 1); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
filenames = append(filenames, name)
s.jobRunner.ProcessOne(jobID, FilePayload{Filename: name, Data: data}, includeRejected)
}
} else {
var err error
filenames, err = s.uploadChunkMinIO(ctx, jobID, files, includeRejected, maxBytes)
if err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
}
if len(filenames) == 0 {
httpError(w, http.StatusBadRequest, "no valid files uploaded")
return
}
writeJSON(w, http.StatusAccepted, map[string]interface{}{
"job_id": jobID,
"uploaded": len(filenames),
})
}
func (s *Server) uploadChunkMinIO(ctx context.Context, jobID string, files []*multipart.FileHeader, includeRejected bool, maxBytes int64) ([]string, error) {
filenames := make([]string, 0, len(files))
for _, fh := range files {
filenames = append(filenames, filepath.Base(fh.Filename))
}
if len(filenames) == 0 {
return nil, nil
}
if err := s.store.AddJobFiles(ctx, jobID, filenames); err != nil {
return nil, err
}
if err := s.store.IncrementTotal(ctx, jobID, len(filenames)); err != nil {
return nil, err
}
if err := s.store.MarkRunning(ctx, jobID); err != nil {
return nil, err
}
ioLimit := 8
if len(files) < ioLimit {
ioLimit = len(files)
}
ioSem := make(chan struct{}, ioLimit)
var wg sync.WaitGroup
var mu sync.Mutex
uploadedNames := make([]string, 0, len(files))
for _, fh := range files {
wg.Add(1)
go func(fh *multipart.FileHeader) {
defer wg.Done()
ioSem <- struct{}{}
defer func() { <-ioSem }()
name := filepath.Base(fh.Filename)
f, err := fh.Open()
if err != nil {
s.recordUploadError(ctx, jobID, name, err.Error())
return
}
data, err := io.ReadAll(io.LimitReader(f, maxBytes+1))
_ = f.Close()
if err != nil {
s.recordUploadError(ctx, jobID, name, err.Error())
return
}
if int64(len(data)) > maxBytes {
s.recordUploadError(ctx, jobID, name, "file too large")
return
}
key := storage.JobObjectKey(jobID, name)
if err := s.storage.Put(ctx, key, data, "application/pdf"); err != nil {
s.recordUploadError(ctx, jobID, name, err.Error())
return
}
msg := domain.QueueMessage{
JobID: jobID,
ObjectKey: key,
Filename: name,
IncludeRejected: includeRejected,
}
if err := s.publisher.Publish(ctx, msg); err != nil {
s.recordUploadError(ctx, jobID, name, err.Error())
return
}
mu.Lock()
uploadedNames = append(uploadedNames, name)
mu.Unlock()
}(fh)
}
wg.Wait()
if len(uploadedNames) == 0 {
return nil, nil
}
return uploadedNames, nil
}
func (s *Server) recordUploadError(ctx context.Context, jobID, filename, errMsg string) {
_ = s.store.SaveResult(ctx, jobID, domain.PDFResult{
Filename: filename,
Status: domain.StatusError,
Error: errMsg,
})
_ = s.store.TryCompleteJob(ctx, jobID)
}
func (s *Server) getJob(w http.ResponseWriter, r *http.Request) {
jobID := chi.URLParam(r, "jobID")
waitMs, _ := strconv.Atoi(r.URL.Query().Get("wait_ms"))
if waitMs > 30000 {
waitMs = 30000
}
deadline := time.Now()
if waitMs > 0 {
deadline = deadline.Add(time.Duration(waitMs) * time.Millisecond)
}
for {
status, err := s.store.GetJob(r.Context(), jobID)
if err != nil {
httpError(w, http.StatusNotFound, "job not found")
return
}
if isTerminalJobStatus(status.Status) || waitMs <= 0 || !time.Now().Before(deadline) {
writeJSON(w, http.StatusOK, status)
return
}
time.Sleep(100 * time.Millisecond)
}
}
func isTerminalJobStatus(status string) bool {
return status == domain.JobCompleted || status == domain.JobCancelled
}
func (s *Server) getJobResults(w http.ResponseWriter, r *http.Request) {
jobID := chi.URLParam(r, "jobID")
cursor := r.URL.Query().Get("cursor")
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
page, err := s.store.GetResultsPage(r.Context(), jobID, cursor, limit)
if err != nil {
httpError(w, http.StatusNotFound, err.Error())
return
}
writeJSON(w, http.StatusOK, page)
}
func (s *Server) cancelJob(w http.ResponseWriter, r *http.Request) {
jobID := chi.URLParam(r, "jobID")
if err := s.store.CancelJob(r.Context(), jobID); err != nil {
httpError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"job_id": jobID, "status": domain.JobCancelled})
}
func (s *Server) newProcessor(includeRejected bool) *pipeline.Processor {
return pipeline.NewProcessor(s.predictor, s.ocrEngine, pipeline.Options{
DPI: s.cfg.OCRDPI,
Lang: s.cfg.OCRLang,
DBSCANEps: s.cfg.DBSCANEps,
DBSCANMinSamples: s.cfg.DBSCANMinSamples,
IncludeRejected: includeRejected,
LogPipelineTiming: s.cfg.LogPipelineTiming,
})
}
func writeJSON(w http.ResponseWriter, status int, v interface{}) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(v)
}
func httpError(w http.ResponseWriter, status int, msg string) {
writeJSON(w, status, map[string]string{"error": msg})
}
func LoadOpenAPI(path string) []byte {
data, err := os.ReadFile(path)
if err != nil {
return []byte(`{"openapi":"3.1.0","info":{"title":"pdf-count","version":"1.0.0"}}`)
}
if strings.HasSuffix(strings.ToLower(path), ".yaml") || strings.HasSuffix(strings.ToLower(path), ".yml") {
var doc interface{}
if err := yaml.Unmarshal(data, &doc); err == nil {
if j, err := json.Marshal(doc); err == nil {
return j
}
}
}
return data
}
func ListenAndServe(ctx context.Context, addr string, handler http.Handler) error {
srv := &http.Server{Addr: addr, Handler: handler}
go func() {
<-ctx.Done()
_ = srv.Shutdown(context.Background())
}()
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
return err
}
return nil
}
@@ -0,0 +1,105 @@
package config
import (
"os"
"strconv"
"time"
)
type Config struct {
Port string
RedisURL string
NATSURL string
MinIOEndpoint string
MinIOAccessKey string
MinIOSecretKey string
MinIOBucket string
MinIOUseSSL bool
ModelPath string
InferencePath string
OCRDPI int
OCRLang string
DBSCANEps float64
DBSCANMinSamples int
SyncMaxFiles int
JobMaxFiles int
QueueMaxPending int
WorkerConcurrency int
MaxPDFSizeMB int
PDFProcessTimeout time.Duration
JobResultTTL time.Duration
StreamName string
SubjectPrefix string
OCRPoolSize int
LogPipelineTiming bool
NATSFetchMaxWait time.Duration
JobInlineProcessing bool
SyncProcessConcurrency int
}
func Load() Config {
return Config{
Port: getenv("PORT", "8080"),
RedisURL: getenv("REDIS_URL", "redis://redis:6379/0"),
NATSURL: getenv("NATS_URL", "nats://nats:4222"),
MinIOEndpoint: getenv("MINIO_ENDPOINT", "minio:9000"),
MinIOAccessKey: getenv("MINIO_ACCESS_KEY", "minioadmin"),
MinIOSecretKey: getenv("MINIO_SECRET_KEY", "minioadmin"),
MinIOBucket: getenv("MINIO_BUCKET", "pdf-count"),
MinIOUseSSL: getenvBool("MINIO_USE_SSL", false),
ModelPath: getenv("MODEL_PATH", "/app/models/model.onnx"),
InferencePath: getenv("INFERENCE_CONFIG", "/app/models/inference.json"),
OCRDPI: getenvInt("OCR_DPI", 300),
OCRLang: getenv("OCR_LANG", "spa"),
DBSCANEps: getenvFloat("DBSCAN_EPS", 10),
DBSCANMinSamples: getenvInt("DBSCAN_MIN_SAMPLES", 1),
SyncMaxFiles: getenvInt("SYNC_MAX_FILES", 50),
JobMaxFiles: getenvInt("JOB_MAX_FILES", 50000),
QueueMaxPending: getenvInt("QUEUE_MAX_PENDING", 100000),
WorkerConcurrency: getenvInt("WORKER_CONCURRENCY", 4),
MaxPDFSizeMB: getenvInt("MAX_PDF_SIZE_MB", 20),
PDFProcessTimeout: time.Duration(getenvInt("PDF_PROCESS_TIMEOUT_SEC", 120)) * time.Second,
JobResultTTL: time.Duration(getenvInt("JOB_RESULT_TTL_HOURS", 72)) * time.Hour,
StreamName: getenv("NATS_STREAM", "PDF_JOBS"),
SubjectPrefix: getenv("NATS_SUBJECT_PREFIX", "pdf.process"),
OCRPoolSize: getenvInt("OCR_POOL_SIZE", 0),
LogPipelineTiming: getenvBool("LOG_PIPELINE_TIMING", false),
NATSFetchMaxWait: time.Duration(getenvInt("NATS_FETCH_MAX_WAIT_MS", 300)) * time.Millisecond,
JobInlineProcessing: getenvBool("JOB_INLINE_PROCESSING", false),
SyncProcessConcurrency: getenvInt("SYNC_PROCESS_CONCURRENCY", 0),
}
}
func getenv(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
func getenvInt(key string, fallback int) int {
if v := os.Getenv(key); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return fallback
}
func getenvFloat(key string, fallback float64) float64 {
if v := os.Getenv(key); v != "" {
if n, err := strconv.ParseFloat(v, 64); err == nil {
return n
}
}
return fallback
}
func getenvBool(key string, fallback bool) bool {
if v := os.Getenv(key); v != "" {
if b, err := strconv.ParseBool(v); err == nil {
return b
}
}
return fallback
}
@@ -0,0 +1,59 @@
package domain
// ServiceRow is one detected invoice line.
type ServiceRow struct {
Text string `json:"text"`
Probability float64 `json:"probability"`
IsService *bool `json:"is_service,omitempty"`
}
// PDFResult is the processing outcome for a single PDF.
type PDFResult struct {
Filename string `json:"filename"`
ServiceCount int `json:"service_count"`
Status string `json:"status"`
Services []ServiceRow `json:"services,omitempty"`
Error string `json:"error,omitempty"`
}
// CountResponse wraps sync batch results.
type CountResponse struct {
Results []PDFResult `json:"results"`
Processed int `json:"processed"`
Failed int `json:"failed"`
}
// JobStatus tracks async job progress.
type JobStatus struct {
JobID string `json:"job_id"`
Status string `json:"status"`
Total int `json:"total"`
Processed int `json:"processed"`
Failed int `json:"failed"`
}
// JobResultsPage is a paginated slice of PDF results.
type JobResultsPage struct {
JobID string `json:"job_id"`
Results []PDFResult `json:"results"`
NextCursor string `json:"next_cursor,omitempty"`
HasMore bool `json:"has_more"`
}
// QueueMessage is published per PDF for workers.
type QueueMessage struct {
JobID string `json:"job_id"`
ObjectKey string `json:"object_key"`
Filename string `json:"filename"`
IncludeRejected bool `json:"include_rejected"`
}
const (
StatusOK = "ok"
StatusError = "error"
JobQueued = "queued"
JobRunning = "running"
JobCompleted = "completed"
JobCancelled = "cancelled"
)
@@ -0,0 +1,110 @@
package model
import (
"encoding/json"
"fmt"
"math"
"os"
"sync"
ort "github.com/yalue/onnxruntime_go"
)
type InferenceMeta struct {
Threshold float64 `json:"threshold"`
FeatureColumns []string `json:"feature_columns"`
}
type Predictor struct {
session *ort.DynamicAdvancedSession
threshold float64
mu sync.Mutex
}
func NewPredictor(modelPath, metaPath string) (*Predictor, error) {
if lib := os.Getenv("ONNXRUNTIME_SHARED_LIBRARY"); lib != "" {
ort.SetSharedLibraryPath(lib)
}
if err := ort.InitializeEnvironment(); err != nil {
return nil, fmt.Errorf("onnx init: %w", err)
}
metaBytes, err := os.ReadFile(metaPath)
if err != nil {
return nil, fmt.Errorf("read inference meta: %w", err)
}
var meta InferenceMeta
if err := json.Unmarshal(metaBytes, &meta); err != nil {
return nil, fmt.Errorf("parse inference meta: %w", err)
}
if meta.Threshold == 0 {
meta.Threshold = 0.5
}
session, err := ort.NewDynamicAdvancedSession(
modelPath,
[]string{"float_input"},
[]string{"probabilities"},
nil,
)
if err != nil {
return nil, fmt.Errorf("onnx session: %w", err)
}
return &Predictor{session: session, threshold: meta.Threshold}, nil
}
func (p *Predictor) Threshold() float64 {
return p.threshold
}
func (p *Predictor) Close() error {
if p.session != nil {
p.session.Destroy()
}
return ort.DestroyEnvironment()
}
func (p *Predictor) PredictProba(features [][8]float32) ([]float64, error) {
if len(features) == 0 {
return nil, nil
}
n := len(features)
flat := make([]float32, 0, n*8)
for _, row := range features {
flat = append(flat, row[:]...)
}
inputShape := ort.NewShape(int64(n), 8)
inputTensor, err := ort.NewTensor(inputShape, flat)
if err != nil {
return nil, err
}
defer inputTensor.Destroy()
outputShape := ort.NewShape(int64(n), 2)
outputTensor, err := ort.NewEmptyTensor[float32](outputShape)
if err != nil {
return nil, err
}
defer outputTensor.Destroy()
p.mu.Lock()
err = p.session.Run([]ort.Value{inputTensor}, []ort.Value{outputTensor})
p.mu.Unlock()
if err != nil {
return nil, err
}
data := outputTensor.GetData()
probs := make([]float64, n)
for i := 0; i < n; i++ {
probs[i] = float64(data[i*2+1])
}
return probs, nil
}
func RoundProb(p float64) float64 {
return math.Round(p*10000) / 10000
}
@@ -0,0 +1,99 @@
package ocr
import (
"bytes"
"fmt"
"image"
"image/png"
"github.com/gen2brain/go-fitz"
"github.com/otiai10/gosseract/v2"
)
type Box struct {
Text string
X int
Y int
W int
H int
}
// RenderPage rasterizes the first page of a PDF.
func RenderPage(pdfData []byte, dpi int) (image.Image, int, int, error) {
doc, err := fitz.NewFromMemory(pdfData)
if err != nil {
return nil, 0, 0, fmt.Errorf("open pdf: %w", err)
}
defer doc.Close()
if doc.NumPage() == 0 {
return nil, 0, 0, fmt.Errorf("pdf has no pages")
}
img, err := doc.ImageDPI(0, float64(dpi))
if err != nil {
return nil, 0, 0, fmt.Errorf("render page: %w", err)
}
bounds := img.Bounds()
return img, bounds.Dx(), bounds.Dy(), nil
}
// OCRImage runs Tesseract word-level OCR on a rendered page image.
func (e *Engine) OCRImage(img image.Image) ([]Box, error) {
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
return nil, fmt.Errorf("encode png: %w", err)
}
client := e.Acquire()
defer e.Release(client)
if err := client.SetImageFromBytes(buf.Bytes()); err != nil {
return nil, fmt.Errorf("tesseract image: %w", err)
}
boxes, err := client.GetBoundingBoxes(gosseract.RIL_WORD)
if err != nil {
return nil, fmt.Errorf("tesseract ocr: %w", err)
}
out := make([]Box, 0, len(boxes))
for _, b := range boxes {
text := trim(b.Word)
if text == "" || b.Confidence <= 0 {
continue
}
out = append(out, Box{
Text: text,
X: b.Box.Min.X,
Y: b.Box.Min.Y,
W: b.Box.Dx(),
H: b.Box.Dy(),
})
}
return out, nil
}
// ExtractBoxes renders the first PDF page and runs OCR with a pooled Tesseract client.
func (e *Engine) ExtractBoxes(pdfData []byte, dpi int) ([]Box, int, int, error) {
img, pageW, pageH, err := RenderPage(pdfData, dpi)
if err != nil {
return nil, 0, 0, err
}
boxes, err := e.OCRImage(img)
if err != nil {
return nil, 0, 0, err
}
return boxes, pageW, pageH, nil
}
func trim(s string) string {
start, end := 0, len(s)
for start < end && (s[start] == ' ' || s[start] == '\t' || s[start] == '\n') {
start++
}
for end > start && (s[end-1] == ' ' || s[end-1] == '\t' || s[end-1] == '\n') {
end--
}
return s[start:end]
}
@@ -0,0 +1,69 @@
package ocr
import (
"fmt"
"github.com/otiai10/gosseract/v2"
)
// Engine holds a pool of reusable Tesseract clients.
type Engine struct {
pool chan *gosseract.Client
clients []*gosseract.Client
lang string
}
// NewEngine warms up poolSize Tesseract clients for the given language.
func NewEngine(lang string, poolSize int) (*Engine, error) {
if poolSize < 1 {
poolSize = 1
}
e := &Engine{
pool: make(chan *gosseract.Client, poolSize),
clients: make([]*gosseract.Client, 0, poolSize),
lang: lang,
}
for i := 0; i < poolSize; i++ {
client, err := newPooledClient(lang)
if err != nil {
e.Close()
return nil, fmt.Errorf("tesseract client %d: %w", i, err)
}
e.clients = append(e.clients, client)
e.pool <- client
}
return e, nil
}
func newPooledClient(lang string) (*gosseract.Client, error) {
client := gosseract.NewClient()
if err := client.SetLanguage(lang); err != nil {
if err2 := client.SetLanguage("eng"); err2 != nil {
_ = client.Close()
return nil, fmt.Errorf("tesseract lang: %w", err)
}
}
return client, nil
}
// Acquire returns a Tesseract client from the pool.
func (e *Engine) Acquire() *gosseract.Client {
return <-e.pool
}
// Release returns a client to the pool.
func (e *Engine) Release(client *gosseract.Client) {
e.pool <- client
}
// Close shuts down all pooled clients.
func (e *Engine) Close() error {
for _, client := range e.clients {
_ = client.Close()
}
e.clients = nil
return nil
}
@@ -0,0 +1,112 @@
package pipeline
import (
"math"
"sort"
)
// ClusterRows runs 1D DBSCAN on Y coordinates (parity with sklearn DBSCAN).
func ClusterRows(boxes []indexedBox, eps float64, minSamples int) {
if len(boxes) == 0 {
return
}
ys := make([]float64, len(boxes))
for i, b := range boxes {
ys[i] = float64(b.Y)
}
labels := dbscan1D(ys, eps, minSamples)
for i := range boxes {
boxes[i].Cluster = labels[i]
}
}
func dbscan1D(points []float64, eps float64, minSamples int) []int {
n := len(points)
labels := make([]int, n)
for i := range labels {
labels[i] = -1
}
clusterID := 0
visited := make([]bool, n)
for i := 0; i < n; i++ {
if visited[i] {
continue
}
visited[i] = true
neighbors := regionQuery(points, i, eps)
if len(neighbors) < minSamples {
continue
}
labels[i] = clusterID
seed := append([]int(nil), neighbors...)
for j := 0; j < len(seed); j++ {
q := seed[j]
if !visited[q] {
visited[q] = true
neighborsQ := regionQuery(points, q, eps)
if len(neighborsQ) >= minSamples {
seed = appendUnique(seed, neighborsQ)
}
}
if labels[q] == -1 {
labels[q] = clusterID
}
}
clusterID++
}
return labels
}
func regionQuery(points []float64, idx int, eps float64) []int {
var neighbors []int
for i, p := range points {
if math.Abs(p-points[idx]) <= eps {
neighbors = append(neighbors, i)
}
}
return neighbors
}
func appendUnique(seed, add []int) []int {
inSeed := make(map[int]bool, len(seed))
for _, v := range seed {
inSeed[v] = true
}
for _, v := range add {
if !inSeed[v] {
seed = append(seed, v)
inSeed[v] = true
}
}
return seed
}
type indexedBox struct {
Text string
X int
Y int
W int
H int
Cluster int
}
func groupByCluster(boxes []indexedBox) map[int][]indexedBox {
groups := make(map[int][]indexedBox)
for _, b := range boxes {
if b.Cluster < 0 {
continue
}
groups[b.Cluster] = append(groups[b.Cluster], b)
}
return groups
}
func sortClusters(groups map[int][]indexedBox) []int {
keys := make([]int, 0, len(groups))
for k := range groups {
keys = append(keys, k)
}
sort.Ints(keys)
return keys
}
@@ -0,0 +1,122 @@
package pipeline
import (
"regexp"
"sort"
"strings"
"unicode"
)
var (
codePattern = regexp.MustCompile(`^\d{4,6}$`)
currencyPattern = regexp.MustCompile(`^\d{1,3}(,\d{3})+(\.\d+)?$`)
)
type rowFeatures struct {
Features [8]float32
TextoFila string
Cluster int
}
func extractRowFeatures(group []indexedBox, pageW, pageH int) rowFeatures {
sort.Slice(group, func(i, j int) bool { return group[i].X < group[j].X })
texts := make([]string, len(group))
for i, b := range group {
texts[i] = b.Text
}
textoFila := strings.Join(texts, " ")
firstToken := ""
if len(texts) > 0 {
firstToken = texts[0]
}
currencyCount := 0
integerCount := 0
for _, t := range texts {
if currencyPattern.MatchString(t) {
currencyCount++
}
if isDigits(t) {
integerCount++
}
}
xMin, xMax := group[0].X, group[0].X
ySum := 0
for _, b := range group {
if b.X < xMin {
xMin = b.X
}
if b.X > xMax {
xMax = b.X
}
ySum += b.Y
}
xSpanNorm := 0.0
if pageW > 0 {
xSpanNorm = float64(xMax-xMin) / float64(pageW)
}
yNorm := 0.0
if pageH > 0 {
yNorm = float64(ySum/len(group)) / float64(pageH)
}
totalChars := len(textoFila)
if totalChars == 0 {
totalChars = 1
}
special := 0
digits := 0
for _, c := range textoFila {
if !unicode.IsLetter(c) && !unicode.IsNumber(c) && !unicode.IsSpace(c) {
special++
}
if unicode.IsDigit(c) {
digits++
}
}
firstIsCode := 0
if codePattern.MatchString(firstToken) {
firstIsCode = 1
}
return rowFeatures{
Features: [8]float32{
float32(firstIsCode),
float32(currencyCount),
float32(integerCount),
float32(xSpanNorm),
float32(yNorm),
float32(len(texts)),
float32(special) / float32(totalChars),
float32(digits) / float32(totalChars),
},
TextoFila: textoFila,
Cluster: group[0].Cluster,
}
}
func isDigits(s string) bool {
if s == "" {
return false
}
for _, c := range s {
if c < '0' || c > '9' {
return false
}
}
return true
}
func buildFeatureRows(boxes []indexedBox, pageW, pageH int) []rowFeatures {
groups := groupByCluster(boxes)
order := sortClusters(groups)
rows := make([]rowFeatures, 0, len(groups))
for _, cid := range order {
rows = append(rows, extractRowFeatures(groups[cid], pageW, pageH))
}
return rows
}
@@ -0,0 +1,128 @@
package pipeline
import (
"fmt"
"log"
"time"
"github.com/pys/pdf-count/internal/domain"
"github.com/pys/pdf-count/internal/model"
"github.com/pys/pdf-count/internal/ocr"
)
type Options struct {
DPI int
Lang string
DBSCANEps float64
DBSCANMinSamples int
IncludeRejected bool
LogPipelineTiming bool
}
type Processor struct {
predictor *model.Predictor
ocrEngine *ocr.Engine
opts Options
}
func NewProcessor(predictor *model.Predictor, ocrEngine *ocr.Engine, opts Options) *Processor {
return &Processor{predictor: predictor, ocrEngine: ocrEngine, opts: opts}
}
func (p *Processor) ProcessPDF(filename string, pdfData []byte) domain.PDFResult {
start := time.Now()
result := domain.PDFResult{Filename: filename, Status: domain.StatusOK}
tRender := time.Now()
img, pageW, pageH, err := ocr.RenderPage(pdfData, p.opts.DPI)
renderDur := time.Since(tRender)
if err != nil {
result.Status = domain.StatusError
result.Error = err.Error()
return result
}
tOCR := time.Now()
boxes, err := p.ocrEngine.OCRImage(img)
ocrDur := time.Since(tOCR)
if err != nil {
result.Status = domain.StatusError
result.Error = err.Error()
return result
}
if len(boxes) == 0 {
result.ServiceCount = 0
result.Services = []domain.ServiceRow{}
p.logTiming(filename, renderDur, ocrDur, 0, 0, time.Since(start))
return result
}
tCluster := time.Now()
indexed := make([]indexedBox, len(boxes))
for i, b := range boxes {
indexed[i] = indexedBox{Text: b.Text, X: b.X, Y: b.Y, W: b.W, H: b.H}
}
ClusterRows(indexed, p.opts.DBSCANEps, p.opts.DBSCANMinSamples)
rows := buildFeatureRows(indexed, pageW, pageH)
clusterDur := time.Since(tCluster)
if len(rows) == 0 {
result.ServiceCount = 0
result.Services = []domain.ServiceRow{}
p.logTiming(filename, renderDur, ocrDur, clusterDur, 0, time.Since(start))
return result
}
features := make([][8]float32, len(rows))
for i, r := range rows {
features[i] = r.Features
}
tInfer := time.Now()
probs, err := p.predictor.PredictProba(features)
inferDur := time.Since(tInfer)
if err != nil {
result.Status = domain.StatusError
result.Error = fmt.Sprintf("inference: %v", err)
return result
}
threshold := p.predictor.Threshold()
services := make([]domain.ServiceRow, 0)
serviceCount := 0
for i, row := range rows {
prob := model.RoundProb(probs[i])
isService := prob >= threshold
if isService {
serviceCount++
}
if p.opts.IncludeRejected {
v := isService
services = append(services, domain.ServiceRow{
Text: row.TextoFila,
Probability: prob,
IsService: &v,
})
} else if isService {
services = append(services, domain.ServiceRow{
Text: row.TextoFila,
Probability: prob,
})
}
}
result.Services = services
result.ServiceCount = serviceCount
p.logTiming(filename, renderDur, ocrDur, clusterDur, inferDur, time.Since(start))
return result
}
func (p *Processor) logTiming(filename string, render, ocr, cluster, infer, total time.Duration) {
if !p.opts.LogPipelineTiming {
return
}
log.Printf(
"pipeline filename=%s ocr_render=%s ocr_tesseract=%s cluster=%s infer=%s total=%s",
filename, render, ocr, cluster, infer, total,
)
}
@@ -0,0 +1,109 @@
package queue
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/pys/pdf-count/internal/domain"
)
type Publisher struct {
js jetstream.JetStream
stream string
subject string
}
func NewPublisher(natsURL, streamName, subjectPrefix string) (*Publisher, error) {
nc, err := nats.Connect(natsURL)
if err != nil {
return nil, fmt.Errorf("nats connect: %w", err)
}
js, err := jetstream.New(nc)
if err != nil {
nc.Close()
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, err = js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{
Name: streamName,
Subjects: []string{subjectPrefix + ".>"},
Retention: jetstream.WorkQueuePolicy,
MaxMsgs: 200000,
})
if err != nil {
nc.Close()
return nil, fmt.Errorf("create stream: %w", err)
}
return &Publisher{js: js, stream: streamName, subject: subjectPrefix}, nil
}
func (p *Publisher) Publish(ctx context.Context, msg domain.QueueMessage) error {
data, err := json.Marshal(msg)
if err != nil {
return err
}
subject := fmt.Sprintf("%s.%s", p.subject, msg.JobID)
_, err = p.js.Publish(ctx, subject, data)
return err
}
type Consumer struct {
js jetstream.JetStream
subject string
consumerName string
}
func NewConsumer(natsURL, streamName, subjectPrefix, consumerName string) (*Consumer, jetstream.Consumer, *nats.Conn, error) {
nc, err := nats.Connect(natsURL)
if err != nil {
return nil, nil, nil, err
}
js, err := jetstream.New(nc)
if err != nil {
nc.Close()
return nil, nil, nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, err = js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{
Name: streamName,
Subjects: []string{subjectPrefix + ".>"},
Retention: jetstream.WorkQueuePolicy,
MaxMsgs: 200000,
})
if err != nil {
nc.Close()
return nil, nil, nil, err
}
cons, err := js.CreateOrUpdateConsumer(ctx, streamName, jetstream.ConsumerConfig{
Name: consumerName,
FilterSubject: subjectPrefix + ".>",
AckPolicy: jetstream.AckExplicitPolicy,
MaxDeliver: 3,
AckWait: 2 * time.Minute,
MaxAckPending: 256,
})
if err != nil {
nc.Close()
return nil, nil, nil, err
}
return &Consumer{js: js, subject: subjectPrefix, consumerName: consumerName}, cons, nc, nil
}
func DecodeMessage(data []byte) (domain.QueueMessage, error) {
var msg domain.QueueMessage
err := json.Unmarshal(data, &msg)
return msg, err
}
@@ -0,0 +1,86 @@
package storage
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
type Client struct {
client *minio.Client
bucket string
}
func New(endpoint, accessKey, secretKey, bucket string, useSSL bool) (*Client, error) {
client, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
Secure: useSSL,
})
if err != nil {
return nil, err
}
return &Client{client: client, bucket: bucket}, nil
}
func (c *Client) EnsureBucket(ctx context.Context) error {
exists, err := c.client.BucketExists(ctx, c.bucket)
if err != nil {
return err
}
if exists {
return nil
}
err = c.client.MakeBucket(ctx, c.bucket, minio.MakeBucketOptions{})
if err == nil {
return nil
}
resp := minio.ToErrorResponse(err)
if resp.Code == "BucketAlreadyOwnedByYou" || resp.Code == "BucketAlreadyExists" {
return nil
}
return err
}
func (c *Client) Put(ctx context.Context, objectKey string, data []byte, contentType string) error {
if contentType == "" {
contentType = "application/pdf"
}
_, err := c.client.PutObject(ctx, c.bucket, objectKey, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{
ContentType: contentType,
})
return err
}
func (c *Client) Get(ctx context.Context, objectKey string) ([]byte, error) {
obj, err := c.client.GetObject(ctx, c.bucket, objectKey, minio.GetObjectOptions{})
if err != nil {
return nil, err
}
defer obj.Close()
return io.ReadAll(obj)
}
func (c *Client) ListByPrefix(ctx context.Context, prefix string) ([]string, error) {
var keys []string
for obj := range c.client.ListObjects(ctx, c.bucket, minio.ListObjectsOptions{
Prefix: prefix,
Recursive: true,
}) {
if obj.Err != nil {
return nil, obj.Err
}
if strings.HasSuffix(strings.ToLower(obj.Key), ".pdf") {
keys = append(keys, obj.Key)
}
}
return keys, nil
}
func JobObjectKey(jobID, filename string) string {
return fmt.Sprintf("jobs/%s/%s", jobID, filename)
}
@@ -0,0 +1,276 @@
package store
import (
"context"
"encoding/json"
"fmt"
"strconv"
"time"
"github.com/pys/pdf-count/internal/domain"
"github.com/redis/go-redis/v9"
)
type Store struct {
client *redis.Client
ttl time.Duration
}
func New(redisURL string, ttl time.Duration) (*Store, error) {
opts, err := redis.ParseURL(redisURL)
if err != nil {
return nil, err
}
client := redis.NewClient(opts)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := client.Ping(ctx).Err(); err != nil {
return nil, fmt.Errorf("redis ping: %w", err)
}
return &Store{client: client, ttl: ttl}, nil
}
func (s *Store) Close() error {
return s.client.Close()
}
func metaKey(jobID string) string { return fmt.Sprintf("job:%s:meta", jobID) }
func filesKey(jobID string) string { return fmt.Sprintf("job:%s:files", jobID) }
func resultsKey(jobID string) string {
return fmt.Sprintf("job:%s:results", jobID)
}
func (s *Store) CreateJob(ctx context.Context, jobID string, total int, includeRejected bool) error {
pipe := s.client.Pipeline()
pipe.HSet(ctx, metaKey(jobID), map[string]interface{}{
"status": domain.JobQueued,
"total": total,
"processed": 0,
"failed": 0,
"include_rejected": boolStr(includeRejected),
})
pipe.Expire(ctx, metaKey(jobID), s.ttl)
_, err := pipe.Exec(ctx)
return err
}
func (s *Store) AddJobFiles(ctx context.Context, jobID string, filenames []string) error {
if len(filenames) == 0 {
return nil
}
members := make([]interface{}, len(filenames))
for i, f := range filenames {
members[i] = f
}
pipe := s.client.Pipeline()
pipe.RPush(ctx, filesKey(jobID), members...)
pipe.Expire(ctx, filesKey(jobID), s.ttl)
_, err := pipe.Exec(ctx)
return err
}
func (s *Store) MarkRunning(ctx context.Context, jobID string) error {
return s.client.HSet(ctx, metaKey(jobID), "status", domain.JobRunning).Err()
}
var saveResultScript = redis.NewScript(`
local meta = KEYS[1]
local results = KEYS[2]
redis.call('HSET', results, ARGV[1], ARGV[2])
local processed = redis.call('HINCRBY', meta, 'processed', 1)
if ARGV[3] == '1' then
redis.call('HINCRBY', meta, 'failed', 1)
end
redis.call('EXPIRE', results, ARGV[4])
local total = tonumber(redis.call('HGET', meta, 'total') or '0')
local status = redis.call('HGET', meta, 'status')
if status ~= 'cancelled' and status ~= 'completed' and total > 0 and processed >= total then
redis.call('HSET', meta, 'status', 'completed')
end
return processed
`)
func (s *Store) SaveResult(ctx context.Context, jobID string, result domain.PDFResult) error {
data, err := json.Marshal(result)
if err != nil {
return err
}
failFlag := "0"
if result.Status == domain.StatusError {
failFlag = "1"
}
ttlSec := int64(s.ttl.Seconds())
if ttlSec <= 0 {
ttlSec = 72 * 3600
}
_, err = saveResultScript.Run(ctx, s.client,
[]string{metaKey(jobID), resultsKey(jobID)},
result.Filename, string(data), failFlag, ttlSec,
).Result()
return err
}
func (s *Store) TryCompleteJob(ctx context.Context, jobID string) error {
meta, err := s.client.HGetAll(ctx, metaKey(jobID)).Result()
if err != nil {
return err
}
_, err = s.maybeCompleteJob(ctx, jobID, meta)
return err
}
func (s *Store) maybeCompleteJob(ctx context.Context, jobID string, meta map[string]string) (string, error) {
status := meta["status"]
if status == domain.JobCancelled || status == domain.JobCompleted {
return status, nil
}
total, _ := strconv.Atoi(meta["total"])
processed, _ := strconv.Atoi(meta["processed"])
if processed >= total && total > 0 {
if err := s.client.HSet(ctx, metaKey(jobID), "status", domain.JobCompleted).Err(); err != nil {
return status, err
}
return domain.JobCompleted, nil
}
return status, nil
}
func (s *Store) GetJob(ctx context.Context, jobID string) (domain.JobStatus, error) {
meta, err := s.client.HGetAll(ctx, metaKey(jobID)).Result()
if err != nil {
return domain.JobStatus{}, err
}
if len(meta) == 0 {
return domain.JobStatus{}, fmt.Errorf("job not found")
}
status, err := s.maybeCompleteJob(ctx, jobID, meta)
if err != nil {
return domain.JobStatus{}, err
}
if status == domain.JobCompleted {
meta["status"] = domain.JobCompleted
}
total, _ := strconv.Atoi(meta["total"])
processed, _ := strconv.Atoi(meta["processed"])
failed, _ := strconv.Atoi(meta["failed"])
return domain.JobStatus{
JobID: jobID,
Status: status,
Total: total,
Processed: processed,
Failed: failed,
}, nil
}
func (s *Store) GetResultsPage(ctx context.Context, jobID, cursor string, limit int) (domain.JobResultsPage, error) {
if limit <= 0 || limit > 1000 {
limit = 500
}
allFiles, err := s.client.LRange(ctx, filesKey(jobID), 0, -1).Result()
if err != nil {
return domain.JobResultsPage{}, err
}
start := 0
if cursor != "" {
for i, f := range allFiles {
if f == cursor {
start = i + 1
break
}
}
}
end := start + limit
if end > len(allFiles) {
end = len(allFiles)
}
pageFiles := allFiles[start:end]
results := make([]domain.PDFResult, 0, len(pageFiles))
if len(pageFiles) > 0 {
raw, err := s.client.HMGet(ctx, resultsKey(jobID), pageFiles...).Result()
if err != nil {
return domain.JobResultsPage{}, err
}
for i, v := range raw {
if v == nil {
results = append(results, domain.PDFResult{
Filename: pageFiles[i],
Status: "pending",
})
continue
}
var r domain.PDFResult
if err := json.Unmarshal([]byte(v.(string)), &r); err != nil {
continue
}
results = append(results, r)
}
}
page := domain.JobResultsPage{
JobID: jobID,
Results: results,
HasMore: end < len(allFiles),
}
if page.HasMore && len(pageFiles) > 0 {
page.NextCursor = pageFiles[len(pageFiles)-1]
}
return page, nil
}
func (s *Store) CancelJob(ctx context.Context, jobID string) error {
return s.client.HSet(ctx, metaKey(jobID), "status", domain.JobCancelled).Err()
}
func (s *Store) IsCancelled(ctx context.Context, jobID string) (bool, error) {
status, err := s.client.HGet(ctx, metaKey(jobID), "status").Result()
if err != nil {
return false, err
}
return status == domain.JobCancelled, nil
}
func (s *Store) PendingCount(ctx context.Context) (int64, error) {
// Approximate: sum of (total - processed) for running jobs
var cursor uint64
var pending int64
for {
keys, next, err := s.client.Scan(ctx, cursor, "job:*:meta", 100).Result()
if err != nil {
return pending, err
}
for _, key := range keys {
meta, err := s.client.HGetAll(ctx, key).Result()
if err != nil {
continue
}
if meta["status"] != domain.JobQueued && meta["status"] != domain.JobRunning {
continue
}
total, _ := strconv.Atoi(meta["total"])
processed, _ := strconv.Atoi(meta["processed"])
pending += int64(total - processed)
}
cursor = next
if cursor == 0 {
break
}
}
return pending, nil
}
func (s *Store) IncrementTotal(ctx context.Context, jobID string, delta int) error {
pipe := s.client.Pipeline()
pipe.HIncrBy(ctx, metaKey(jobID), "total", int64(delta))
pipe.Expire(ctx, metaKey(jobID), s.ttl)
_, err := pipe.Exec(ctx)
return err
}
func boolStr(v bool) string {
if v {
return "true"
}
return "false"
}
@@ -0,0 +1,212 @@
package worker
import (
"context"
"log"
"os/signal"
"sync"
"syscall"
"time"
"github.com/nats-io/nats.go/jetstream"
"github.com/pys/pdf-count/internal/config"
"github.com/pys/pdf-count/internal/domain"
"github.com/pys/pdf-count/internal/model"
"github.com/pys/pdf-count/internal/ocr"
"github.com/pys/pdf-count/internal/pipeline"
"github.com/pys/pdf-count/internal/queue"
"github.com/pys/pdf-count/internal/storage"
"github.com/pys/pdf-count/internal/store"
)
type Runner struct {
cfg config.Config
predictor *model.Predictor
ocrEngine *ocr.Engine
store *store.Store
storage *storage.Client
consumer jetstream.Consumer
ncClose func()
}
func NewRunner(cfg config.Config) (*Runner, error) {
predictor, err := model.NewPredictor(cfg.ModelPath, cfg.InferencePath)
if err != nil {
return nil, err
}
poolSize := cfg.OCRPoolSize
if poolSize <= 0 {
poolSize = cfg.WorkerConcurrency
}
ocrEngine, err := ocr.NewEngine(cfg.OCRLang, poolSize)
if err != nil {
predictor.Close()
return nil, err
}
st, err := store.New(cfg.RedisURL, cfg.JobResultTTL)
if err != nil {
ocrEngine.Close()
predictor.Close()
return nil, err
}
stg, err := storage.New(cfg.MinIOEndpoint, cfg.MinIOAccessKey, cfg.MinIOSecretKey, cfg.MinIOBucket, cfg.MinIOUseSSL)
if err != nil {
ocrEngine.Close()
st.Close()
predictor.Close()
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := stg.EnsureBucket(ctx); err != nil {
ocrEngine.Close()
st.Close()
predictor.Close()
return nil, err
}
_, cons, nc, err := queue.NewConsumer(cfg.NATSURL, cfg.StreamName, cfg.SubjectPrefix, "pdf-workers")
if err != nil {
ocrEngine.Close()
st.Close()
predictor.Close()
return nil, err
}
return &Runner{
cfg: cfg,
predictor: predictor,
ocrEngine: ocrEngine,
store: st,
storage: stg,
consumer: cons,
ncClose: func() { nc.Close() },
}, nil
}
func (r *Runner) Close() {
r.ncClose()
_ = r.ocrEngine.Close()
_ = r.store.Close()
_ = r.predictor.Close()
}
func (r *Runner) processSlots() int {
c := r.cfg.WorkerConcurrency
pool := r.cfg.OCRPoolSize
if pool <= 0 {
pool = r.cfg.WorkerConcurrency
}
if pool <= 0 {
pool = 4
}
if c > pool {
c = pool
}
if c <= 0 {
c = 4
}
return c
}
func (r *Runner) Run(ctx context.Context) error {
slots := r.processSlots()
sem := make(chan struct{}, slots)
var wg sync.WaitGroup
go func() {
<-ctx.Done()
wg.Wait()
}()
for {
select {
case <-ctx.Done():
return nil
default:
}
msgs, err := r.consumer.Fetch(slots, jetstream.FetchMaxWait(r.cfg.NATSFetchMaxWait))
if err != nil {
continue
}
for msg := range msgs.Messages() {
sem <- struct{}{}
wg.Add(1)
m := msg
go func() {
defer func() {
<-sem
wg.Done()
}()
r.handleMessage(ctx, m)
}()
}
_ = msgs.Error()
}
}
func (r *Runner) handleMessage(ctx context.Context, msg jetstream.Msg) {
processCtx, cancel := context.WithTimeout(ctx, r.cfg.PDFProcessTimeout)
defer cancel()
qmsg, err := queue.DecodeMessage(msg.Data())
if err != nil {
_ = msg.Term()
return
}
cancelled, err := r.store.IsCancelled(processCtx, qmsg.JobID)
if err != nil || cancelled {
_ = msg.Ack()
return
}
data, err := r.storage.Get(processCtx, qmsg.ObjectKey)
if err != nil {
_ = r.store.SaveResult(processCtx, qmsg.JobID, domain.PDFResult{
Filename: qmsg.Filename,
Status: domain.StatusError,
Error: err.Error(),
})
_ = r.store.TryCompleteJob(processCtx, qmsg.JobID)
_ = msg.Ack()
return
}
proc := pipeline.NewProcessor(r.predictor, r.ocrEngine, pipeline.Options{
DPI: r.cfg.OCRDPI,
Lang: r.cfg.OCRLang,
DBSCANEps: r.cfg.DBSCANEps,
DBSCANMinSamples: r.cfg.DBSCANMinSamples,
IncludeRejected: qmsg.IncludeRejected,
LogPipelineTiming: r.cfg.LogPipelineTiming,
})
result := proc.ProcessPDF(qmsg.Filename, data)
data = nil
if err := r.store.SaveResult(processCtx, qmsg.JobID, result); err != nil {
_ = msg.NakWithDelay(5 * time.Second)
return
}
_ = r.store.TryCompleteJob(processCtx, qmsg.JobID)
_ = msg.Ack()
}
func RunMain() {
cfg := config.Load()
runner, err := NewRunner(cfg)
if err != nil {
log.Fatalf("worker init: %v", err)
}
defer runner.Close()
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
log.Printf("worker started slots=%d", runner.processSlots())
if err := runner.Run(ctx); err != nil {
log.Fatalf("worker: %v", err)
}
log.Println("worker stopped")
}
@@ -0,0 +1,4 @@
joblib
scikit-learn
skl2onnx
onnx
@@ -0,0 +1,59 @@
"""Compara baseline Python vs respuesta del microservicio Go."""
from __future__ import annotations
import json
import os
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SC_ROOT = Path(os.environ.get("SERVICES_COUNT_ROOT", ROOT.parent / "services-count"))
BASELINE = ROOT / "data" / "parity_baseline.json"
API_URL = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080/v1/count"
def call_api(pdf_path: Path) -> dict:
result = subprocess.run(
["curl", "-s", "-F", f"files=@{pdf_path}", API_URL],
capture_output=True,
text=True,
check=True,
)
payload = json.loads(result.stdout)
for item in payload.get("results", []):
if item.get("filename") == pdf_path.name:
return item
raise RuntimeError(f"sin resultado para {pdf_path.name}")
def main() -> None:
if not BASELINE.exists():
raise SystemExit("Ejecuta primero: python scripts/parity_baseline.py")
import yaml
config_path = SC_ROOT / "config" / "default.yaml"
pdfs_dir = SC_ROOT / yaml.safe_load(config_path.read_text())["paths"]["pdfs_dir"]
baseline = json.loads(BASELINE.read_text(encoding="utf-8"))
mismatches = 0
for expected in baseline:
pdf = pdfs_dir / expected["filename"]
if not pdf.exists():
print(f"SKIP {expected['filename']} (no existe)")
continue
got = call_api(pdf)
if got.get("service_count") != expected.get("service_count"):
mismatches += 1
print(f"MISMATCH {expected['filename']}: py={expected['service_count']} go={got.get('service_count')}")
else:
print(f"OK {expected['filename']}: count={got.get('service_count')}")
if mismatches:
raise SystemExit(f"{mismatches} diferencias encontradas")
print("Paridad OK")
if __name__ == "__main__":
main()
@@ -0,0 +1,99 @@
"""Exporta model.joblib a ONNX + inference.json (sin depender de services-count)."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import joblib
from skl2onnx import convert_sklearn
from skl2onnx.common.data_types import FloatTensorType
FEATURE_COLUMNS = [
"first_token_is_code",
"currency_count",
"integer_count",
"x_span_norm",
"y_norm",
"token_count",
"special_char_ratio",
"digit_ratio",
]
DEFAULT_OCR = {"dpi": 300, "lang": "spa"}
DEFAULT_DBSCAN = {"eps": 10, "min_samples": 1}
def parse_args() -> argparse.Namespace:
root = Path(__file__).resolve().parents[1]
parser = argparse.ArgumentParser(description="Exportar Random Forest joblib a ONNX")
parser.add_argument(
"--joblib",
type=Path,
required=True,
help="Ruta al model.joblib (ej. ../services-count/models/latest/model.joblib)",
)
parser.add_argument(
"--out",
type=Path,
default=root / "models" / "latest",
help="Directorio de salida para model.onnx e inference.json",
)
parser.add_argument(
"--metadata",
type=Path,
default=None,
help="model_metadata.json opcional para pipeline_version y params OCR/DBSCAN",
)
return parser.parse_args()
def load_metadata(path: Path | None) -> dict:
if path is None or not path.exists():
return {}
return json.loads(path.read_text(encoding="utf-8"))
def main() -> None:
args = parse_args()
if not args.joblib.exists():
raise SystemExit(f"No se encontró: {args.joblib}")
bundle = joblib.load(args.joblib)
model = bundle["model"]
threshold = float(bundle.get("threshold", 0.5))
meta = load_metadata(args.metadata)
initial_type = [("float_input", FloatTensorType([None, len(FEATURE_COLUMNS)]))]
options = {id(model): {"zipmap": False}}
onnx_model = convert_sklearn(
model,
initial_types=initial_type,
options=options,
target_opset=12,
)
out_dir = args.out
out_dir.mkdir(parents=True, exist_ok=True)
onnx_path = out_dir / "model.onnx"
meta_path = out_dir / "inference.json"
with open(onnx_path, "wb") as f:
f.write(onnx_model.SerializeToString())
inference = {
"threshold": threshold,
"feature_columns": FEATURE_COLUMNS,
"pipeline_version": meta.get("pipeline_version", "unknown"),
"ocr": meta.get("ocr", DEFAULT_OCR),
"dbscan": meta.get("dbscan", DEFAULT_DBSCAN),
}
meta_path.write_text(json.dumps(inference, indent=2), encoding="utf-8")
print(f"ONNX: {onnx_path}")
print(f"Meta: {meta_path}")
print(f"Umbral: {threshold}")
if __name__ == "__main__":
main()
@@ -0,0 +1,49 @@
"""Genera baseline Python desde services-count para comparar con el servicio Go."""
from __future__ import annotations
import json
import os
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SC_ROOT = Path(os.environ.get("SERVICES_COUNT_ROOT", ROOT.parent / "services-count"))
sys.path.insert(0, str(SC_ROOT / "src"))
from services_count.config import load_config
from services_count.inference import predict_pdf
def python_result(pdf_path: Path) -> dict:
config = load_config(SC_ROOT / "config" / "default.yaml")
features_df, total = predict_pdf(pdf_path, config)
services = []
for row in features_df.itertuples():
if row.prediction == 1:
services.append({
"text": row.texto_fila,
"probability": round(float(row.probability), 4),
})
return {
"filename": pdf_path.name,
"service_count": total,
"status": "ok",
"services": services,
}
def main() -> None:
config = load_config(SC_ROOT / "config" / "default.yaml")
pdfs = sorted(config.pdfs_dir.glob("*.pdf"))
if not pdfs:
raise SystemExit(f"No hay PDFs en {config.pdfs_dir}")
out = ROOT / "data" / "parity_baseline.json"
out.parent.mkdir(parents=True, exist_ok=True)
results = [python_result(p) for p in pdfs]
out.write_text(json.dumps(results, indent=2, ensure_ascii=False), encoding="utf-8")
print(f"Baseline guardado: {out} ({len(results)} PDFs)")
if __name__ == "__main__":
main()
@@ -0,0 +1,347 @@
const MAX_SYNC_FILES = 50;␍
const MAX_ASYNC_FILES = 500;␍
const JOB_CHUNK_SIZE = 500;␍
const LONG_POLL_MS = 30000;␍
const TERMINAL_STATUSES = new Set(["completed", "failed", "cancelled"]);␍
␍
function $(id) {␍
return document.getElementById(id);␍
}␍
␍
function getBaseUrl() {␍
const input = $("baseUrl").value.trim();␍
return (input || window.location.origin).replace(/\/$/, "");␍
}␍
␍
function getIncludeRejected() {␍
return $("includeRejected").checked;␍
}␍
␍
function getSelectedFiles(inputId) {␍
const input = $(inputId);␍
return Array.from(input.files || []);␍
}␍
␍
function validateFiles(files, maxFiles) {␍
if (files.length === 0) {␍
throw new Error("Selecciona al menos un archivo PDF.");␍
}␍
if (files.length > maxFiles) {␍
throw new Error(`Máximo ${maxFiles} archivos por petición.`);␍
}␍
for (const file of files) {␍
if (!file.name.toLowerCase().endsWith(".pdf")) {␍
throw new Error(`"${file.name}" no es un PDF.`);␍
}␍
}␍
}␍
␍
async function parseError(res) {␍
try {␍
const data = await res.json();␍
if (data && data.error) return data.error;␍
} catch (_) {␍
/* ignore */␍
}␍
return `Error HTTP ${res.status}`;␍
}␍
␍
function buildMultipart(files) {␍
const form = new FormData();␍
for (const file of files) {␍
form.append("files", file, file.name);␍
}␍
return form;␍
}␍
␍
function setLoading(button, loading, label) {␍
button.disabled = loading;␍
button.textContent = loading ? "Procesando…" : label;␍
}␍
␍
function showError(containerId, message) {␍
const el = $(containerId);␍
el.hidden = false;␍
el.textContent = message;␍
}␍
␍
function hideError(containerId) {␍
const el = $(containerId);␍
el.hidden = true;␍
el.textContent = "";␍
}␍
␍
function formatProbability(value) {␍
if (typeof value !== "number") return "—";␍
return (value * 100).toFixed(1) + "%";␍
}␍
␍
function renderResults(container, results) {␍
container.innerHTML = "";␍
␍
if (!results || results.length === 0) {␍
container.innerHTML = '<p class="empty">Sin resultados.</p>';␍
return;␍
}␍
␍
for (const item of results) {␍
const card = document.createElement("article");␍
card.className = "result-card";␍
␍
const header = document.createElement("div");␍
header.className = "result-header";␍
header.innerHTML = `␍
<h3>${escapeHtml(item.filename)}</h3>␍
<span class="badge badge-${item.status}">${escapeHtml(item.status)}</span>␍
`;␍
card.appendChild(header);␍
␍
if (item.status === "error" && item.error) {␍
const err = document.createElement("p");␍
err.className = "result-error";␍
err.textContent = item.error;␍
card.appendChild(err);␍
} else {␍
const meta = document.createElement("p");␍
meta.className = "result-meta";␍
meta.textContent = `Servicios detectados: ${item.service_count ?? 0}`;␍
card.appendChild(meta);␍
␍
if (item.services && item.services.length > 0) {␍
const table = document.createElement("table");␍
table.innerHTML = `␍
<thead>␍
<tr>␍
<th>Texto</th>␍
<th>Probabilidad</th>␍
<th>Es servicio</th>␍
</tr>␍
</thead>␍
<tbody></tbody>␍
`;␍
const tbody = table.querySelector("tbody");␍
for (const row of item.services) {␍
const tr = document.createElement("tr");␍
const isService =␍
row.is_service === undefined || row.is_service === null␍
? "—"␍
: row.is_service␍
? "Sí"␍
: "No";␍
tr.innerHTML = `␍
<td>${escapeHtml(row.text)}</td>␍
<td>${formatProbability(row.probability)}</td>␍
<td>${isService}</td>␍
`;␍
tbody.appendChild(tr);␍
}␍
card.appendChild(table);␍
}␍
}␍
␍
container.appendChild(card);␍
}␍
}␍
␍
function escapeHtml(text) {␍
const div = document.createElement("div");␍
div.textContent = text ?? "";␍
return div.innerHTML;␍
}␍
␍
async function countSync(baseUrl, files, includeRejected) {␍
const url = new URL(`${baseUrl}/v1/count`);␍
if (includeRejected) url.searchParams.set("include_rejected", "true");␍
␍
const res = await fetch(url, {␍
method: "POST",␍
body: buildMultipart(files),␍
});␍
␍
if (!res.ok) throw new Error(await parseError(res));␍
const data = await res.json();␍
return data.results || [];␍
}␍
␍
async function createEmptyJob(baseUrl, includeRejected) {␍
const res = await fetch(`${baseUrl}/v1/jobs`, {␍
method: "POST",␍
headers: { "Content-Type": "application/json" },␍
body: JSON.stringify({ include_rejected: includeRejected }),␍
});␍
if (!res.ok) throw new Error(await parseError(res));␍
const data = await res.json();␍
return data.job_id;␍
}␍
␍
async function uploadChunk(baseUrl, jobId, files, includeRejected) {␍
const url = new URL(`${baseUrl}/v1/jobs/${jobId}/upload`);␍
if (includeRejected) url.searchParams.set("include_rejected", "true");␍
␍
const res = await fetch(url, {␍
method: "POST",␍
body: buildMultipart(files),␍
});␍
if (!res.ok) throw new Error(await parseError(res));␍
return res.json();␍
}␍
␍
async function getJobStatus(baseUrl, jobId, waitMs = 0) {␍
const url = new URL(`${baseUrl}/v1/jobs/${jobId}`);␍
if (waitMs > 0) url.searchParams.set("wait_ms", String(waitMs));␍
␍
const res = await fetch(url);␍
if (!res.ok) throw new Error(await parseError(res));␍
return res.json();␍
}␍
␍
async function fetchAllResults(baseUrl, jobId) {␍
const results = [];␍
let cursor = "";␍
let hasMore = true;␍
␍
while (hasMore) {␍
const url = new URL(`${baseUrl}/v1/jobs/${jobId}/results`);␍
url.searchParams.set("limit", "500");␍
if (cursor) url.searchParams.set("cursor", cursor);␍
␍
const res = await fetch(url);␍
if (!res.ok) throw new Error(await parseError(res));␍
const page = await res.json();␍
results.push(...(page.results || []));␍
cursor = page.next_cursor || "";␍
hasMore = Boolean(page.has_more);␍
}␍
␍
return results;␍
}␍
␍
function chunkArray(items, size) {␍
const chunks = [];␍
for (let i = 0; i < items.length; i += size) {␍
chunks.push(items.slice(i, i + size));␍
}␍
return chunks;␍
}␍
␍
async function runAsyncJob(baseUrl, files, includeRejected, onProgress) {␍
const jobId = await createEmptyJob(baseUrl, includeRejected);␍
const chunks = chunkArray(files, JOB_CHUNK_SIZE);␍
␍
for (let i = 0; i < chunks.length; i++) {␍
onProgress({␍
status: "uploading",␍
total: files.length,␍
processed: 0,␍
failed: 0,␍
uploadChunk: i + 1,␍
uploadChunks: chunks.length,␍
});␍
await uploadChunk(baseUrl, jobId, chunks[i], includeRejected);␍
}␍
␍
while (true) {␍
const status = await getJobStatus(baseUrl, jobId, LONG_POLL_MS);␍
onProgress(status);␍
if (TERMINAL_STATUSES.has(status.status)) {␍
if (status.status === "cancelled") {␍
throw new Error("El job fue cancelado.");␍
}␍
break;␍
}␍
}␍
␍
return fetchAllResults(baseUrl, jobId);␍
}␍
␍
function updateProgress(status) {␍
const bar = $("asyncProgress");␍
const label = $("asyncProgressLabel");␍
const total = status.total || 0;␍
const done = (status.processed || 0) + (status.failed || 0);␍
const pct = total > 0 ? Math.min(100, Math.round((done / total) * 100)) : 0;␍
␍
bar.hidden = false;␍
label.hidden = false;␍
␍
if (status.status === "uploading") {␍
const chunkPct =␍
status.uploadChunks > 0␍
? Math.round((status.uploadChunk / status.uploadChunks) * 100)␍
: 0;␍
bar.value = chunkPct;␍
label.textContent = `Subiendo lote ${status.uploadChunk}/${status.uploadChunks} (${total} archivos)…`;␍
return;␍
}␍
␍
bar.value = pct;␍
label.textContent = `Estado: ${status.status} — ${done}/${total} procesados (${status.failed || 0} fallidos)`;␍
}␍
␍
function initTabs() {␍
const tabs = document.querySelectorAll(".tab");␍
const panels = document.querySelectorAll(".panel");␍
␍
tabs.forEach((tab) => {␍
tab.addEventListener("click", () => {␍
tabs.forEach((t) => t.classList.remove("active"));␍
panels.forEach((p) => p.classList.remove("active"));␍
tab.classList.add("active");␍
document.getElementById(tab.dataset.panel).classList.add("active");␍
});␍
});␍
}␍
␍
function init() {␍
$("baseUrl").value = window.location.origin;␍
␍
initTabs();␍
␍
$("syncForm").addEventListener("submit", async (e) => {␍
e.preventDefault();␍
const btn = $("syncSubmit");␍
hideError("syncError");␍
$("syncResults").innerHTML = "";␍
␍
try {␍
const files = getSelectedFiles("syncFiles");␍
validateFiles(files, MAX_SYNC_FILES);␍
setLoading(btn, true, "Contar servicios");␍
const results = await countSync(getBaseUrl(), files, getIncludeRejected());␍
renderResults($("syncResults"), results);␍
} catch (err) {␍
showError("syncError", err.message);␍
} finally {␍
setLoading(btn, false, "Contar servicios");␍
}␍
});␍
␍
$("asyncForm").addEventListener("submit", async (e) => {␍
e.preventDefault();␍
const btn = $("asyncSubmit");␍
hideError("asyncError");␍
$("asyncResults").innerHTML = "";␍
$("asyncProgress").hidden = true;␍
$("asyncProgressLabel").hidden = true;␍
␍
try {␍
const files = getSelectedFiles("asyncFiles");␍
validateFiles(files, MAX_ASYNC_FILES);␍
setLoading(btn, true, "Iniciar job");␍
const results = await runAsyncJob(␍
getBaseUrl(),␍
files,␍
getIncludeRejected(),␍
updateProgress,␍
);␍
renderResults($("asyncResults"), results);␍
} catch (err) {␍
showError("asyncError", err.message);␍
} finally {␍
setLoading(btn, false, "Iniciar job");␍
}␍
});␍
}␍
␍
document.addEventListener("DOMContentLoaded", init);␍
␍
@@ -0,0 +1,6 @@
package web
import "embed"
//go:embed index.html app.js
var FS embed.FS
@@ -0,0 +1,307 @@
<!DOCTYPE html>
<html lang="es">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Services Count — Demo</title>
<style>
:root {
--bg: #f4f6f8;
--surface: #ffffff;
--text: #1a2332;
--muted: #5c6b7a;
--border: #d8dee6;
--primary: #2563eb;
--primary-hover: #1d4ed8;
--error-bg: #fef2f2;
--error-text: #b91c1c;
--ok: #15803d;
--warn: #b45309;
--radius: 8px;
}
* { box-sizing: border-box; }
body {
margin: 0;
font-family: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
background: var(--bg);
color: var(--text);
line-height: 1.5;
}
.container {
max-width: 900px;
margin: 0 auto;
padding: 2rem 1.25rem 3rem;
}
header {
margin-bottom: 1.5rem;
}
header h1 {
margin: 0 0 0.25rem;
font-size: 1.75rem;
font-weight: 700;
}
header p {
margin: 0;
color: var(--muted);
}
header a {
color: var(--primary);
text-decoration: none;
}
header a:hover { text-decoration: underline; }
.card {
background: var(--surface);
border: 1px solid var(--border);
border-radius: var(--radius);
padding: 1.25rem;
margin-bottom: 1rem;
}
.field {
margin-bottom: 1rem;
}
label {
display: block;
font-weight: 600;
margin-bottom: 0.35rem;
font-size: 0.9rem;
}
input[type="text"],
input[type="file"] {
width: 100%;
padding: 0.55rem 0.65rem;
border: 1px solid var(--border);
border-radius: var(--radius);
font: inherit;
}
.checkbox {
display: flex;
align-items: center;
gap: 0.5rem;
font-weight: normal;
}
.checkbox input { width: auto; }
.tabs {
display: flex;
gap: 0.5rem;
margin-bottom: 1rem;
}
.tab {
flex: 1;
padding: 0.65rem 1rem;
border: 1px solid var(--border);
background: var(--surface);
border-radius: var(--radius);
cursor: pointer;
font: inherit;
font-weight: 600;
color: var(--muted);
}
.tab.active {
background: var(--primary);
border-color: var(--primary);
color: #fff;
}
.panel { display: none; }
.panel.active { display: block; }
.btn {
display: inline-block;
padding: 0.6rem 1.25rem;
background: var(--primary);
color: #fff;
border: none;
border-radius: var(--radius);
font: inherit;
font-weight: 600;
cursor: pointer;
}
.btn:hover:not(:disabled) { background: var(--primary-hover); }
.btn:disabled { opacity: 0.6; cursor: not-allowed; }
.error {
background: var(--error-bg);
color: var(--error-text);
padding: 0.75rem 1rem;
border-radius: var(--radius);
margin-bottom: 1rem;
font-size: 0.9rem;
}
.hint {
font-size: 0.85rem;
color: var(--muted);
margin-top: 0.35rem;
}
progress {
width: 100%;
height: 0.65rem;
margin: 0.75rem 0 0.25rem;
}
.progress-label {
font-size: 0.85rem;
color: var(--muted);
}
.results { margin-top: 1.25rem; }
.result-card {
border: 1px solid var(--border);
border-radius: var(--radius);
padding: 1rem;
margin-bottom: 0.75rem;
background: #fafbfc;
}
.result-header {
display: flex;
justify-content: space-between;
align-items: center;
gap: 1rem;
margin-bottom: 0.5rem;
}
.result-header h3 {
margin: 0;
font-size: 1rem;
word-break: break-all;
}
.badge {
font-size: 0.75rem;
font-weight: 700;
text-transform: uppercase;
padding: 0.2rem 0.5rem;
border-radius: 4px;
background: #e2e8f0;
flex-shrink: 0;
}
.badge-ok { background: #dcfce7; color: var(--ok); }
.badge-error { background: #fee2e2; color: var(--error-text); }
.badge-pending { background: #fef3c7; color: var(--warn); }
.result-meta {
margin: 0 0 0.75rem;
font-weight: 600;
}
.result-error {
margin: 0;
color: var(--error-text);
}
table {
width: 100%;
border-collapse: collapse;
font-size: 0.875rem;
}
th, td {
text-align: left;
padding: 0.45rem 0.5rem;
border-bottom: 1px solid var(--border);
vertical-align: top;
}
th {
font-size: 0.75rem;
text-transform: uppercase;
color: var(--muted);
}
.empty {
color: var(--muted);
font-style: italic;
}
</style>
</head>
<body>
<div class="container">
<header>
<h1>Services Count — Demo</h1>
<p>
Cliente de ejemplo para la API de conteo de servicios en facturas médicas.
<a href="/openapi.json" target="_blank" rel="noopener">OpenAPI</a>
</p>
</header>
<div class="card">
<div class="field">
<label for="baseUrl">URL base de la API</label>
<input type="text" id="baseUrl" placeholder="http://localhost:8080">
<p class="hint">Por defecto usa el mismo origen desde el que se sirve esta página.</p>
</div>
<div class="field">
<label class="checkbox">
<input type="checkbox" id="includeRejected">
Incluir filas rechazadas (<code>include_rejected</code>)
</label>
</div>
</div>
<div class="tabs">
<button type="button" class="tab active" data-panel="syncPanel">Síncrono</button>
<button type="button" class="tab" data-panel="asyncPanel">Asíncrono</button>
</div>
<div id="syncPanel" class="panel active">
<div class="card">
<form id="syncForm">
<div class="field">
<label for="syncFiles">Archivos PDF (máx. 50)</label>
<input type="file" id="syncFiles" accept=".pdf,application/pdf" multiple required>
<p class="hint">Envía una petición <code>POST /v1/count</code> y muestra la respuesta al instante.</p>
</div>
<div id="syncError" class="error" hidden></div>
<button type="submit" class="btn" id="syncSubmit">Contar servicios</button>
</form>
<div id="syncResults" class="results"></div>
</div>
</div>
<div id="asyncPanel" class="panel">
<div class="card">
<form id="asyncForm">
<div class="field">
<label for="asyncFiles">Archivos PDF (hasta 500)</label>
<input type="file" id="asyncFiles" accept=".pdf,application/pdf" multiple required>
<p class="hint">
Flujo producción: MinIO → cola NATS → workers (autoscaler).
Sube por lotes de hasta 500; el progreso usa long-polling.
</p>
</div>
<div id="asyncError" class="error" hidden></div>
<progress id="asyncProgress" value="0" max="100" hidden></progress>
<p id="asyncProgressLabel" class="progress-label" hidden></p>
<button type="submit" class="btn" id="asyncSubmit">Iniciar job</button>
</form>
<div id="asyncResults" class="results"></div>
</div>
</div>
</div>
<script src="app.js"></script>
</body>
</html>