PYS/IPSExtractor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Update docker-compose configuration to use environment variables for sensitive data, enhance testing documentation with new LaTeX files for capacity planning, and refine README for clarity on compilation and testing processes.

Browse Source
This commit is contained in:
erik
2026-07-17 08:32:00 -05:00
parent e47ef4f8c6
commit dc3fcc2280
89 changed files with 5331 additions and 1210 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/bot_descarga/api/main.py
+37
View File
@@ -182,6 +182,43 @@ def list_job_files(download_job_id: str, _: str = Depends(require_bearer)) -> di
raise HTTPException(status_code=502, detail=f"Error listando MinIO: {e}") from e
@app.get("/jobs/{download_job_id}/object")
def get_job_object(
download_job_id: str,
key: str = Query(..., min_length=1, description="Object key bajo downloads_prefix"),
_: str = Depends(require_bearer),
):
"""Stream de un PDF del resultado MinIO (solo keys del prefijo del job)."""
job = store.get_job(download_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
prefix = (job.downloads_prefix or f"downloads/{download_job_id}/").lstrip("/")
if not prefix.endswith("/"):
prefix += "/"
normalized = key.lstrip("/")
if not normalized.startswith(prefix):
raise HTTPException(status_code=403, detail="Key fuera del prefijo del job")
if not normalized.lower().endswith(".pdf"):
raise HTTPException(status_code=400, detail="Solo se permiten objetos PDF")
if ".." in normalized.split("/"):
raise HTTPException(status_code=400, detail="Key no permitida")
if normalized.endswith("_manifest.json"):
raise HTTPException(status_code=400, detail="Key no permitida")
try:
data = minio.get_bytes(normalized)
except Exception as e:
raise HTTPException(status_code=502, detail=f"Error leyendo MinIO: {e}") from e
filename = Path(normalized).name
return Response(
content=data,
media_type="application/pdf",
headers={"Content-Disposition": f'inline; filename="{filename}"'},
)
@app.get("/jobs/{download_job_id}/events")
def get_events(
download_job_id: str,
services/bot_descarga/infrastructure/storage_minio.py
+25 -7
View File
@@ -190,24 +190,42 @@ class MinioStorage:
return key
def list_download_files(self, job_id: str, prefix: str | None = None) -> dict:
"""Lista PDFs de una corrida (manifest o list_keys)."""
"""Lista PDFs de una corrida (manifest o list_keys).
Si el manifest falta, falla o queda vacío, hace fallback a list_keys
bajo el prefijo (cubre varias carpetas NIT planas).
"""
prefix = prefix or downloads_prefix(job_id)
files: List[dict] = []
try:
raw = self.get_bytes(manifest_key(job_id))
payload = json.loads(raw.decode("utf-8"))
for key in payload.get("files") or []:
files.append(self._file_entry(str(key)))
entry = self._file_entry(str(key))
if entry is not None:
files.append(entry)
except Exception:
for key in self.list_keys(prefix):
if key.endswith("/_manifest.json") or key.endswith("_manifest.json"):
continue
files.append(self._file_entry(key))
files = []
if not files:
files = self._list_pdf_entries(prefix)
return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)}
def _list_pdf_entries(self, prefix: str) -> List[dict]:
files: List[dict] = []
for key in self.list_keys(prefix):
entry = self._file_entry(key)
if entry is not None:
files.append(entry)
return files
@staticmethod
def _file_entry(key: str) -> dict:
def _file_entry(key: str) -> dict | None:
# downloads/{job_id}/{nit}/{radicado}.pdf
if key.endswith("/_manifest.json") or key.endswith("_manifest.json"):
return None
if not key.lower().endswith(".pdf"):
return None
parts = key.strip("/").split("/")
nit = parts[-2] if len(parts) >= 3 else None
name = parts[-1] if parts else key
services/bot_descarga/tests/test_list_download_files.py
+38
View File
@@ -0,0 +1,38 @@
"""Tests for MinIO list_download_files fallback on empty manifest."""
from __future__ import annotations
from unittest.mock import MagicMock
from infrastructure.storage_minio import MinioStorage
def test_list_download_files_falls_back_when_manifest_empty(monkeypatch):
storage = MinioStorage()
storage.get_bytes = MagicMock(
return_value=b'{"job_id": "j1", "files": [], "count": 0}'
)
storage.list_keys = MagicMock(
return_value=[
"downloads/j1/_manifest.json",
"downloads/j1/900808423/a.pdf",
"downloads/j1/900884937/b.pdf",
"downloads/j1/901679492/c.pdf",
]
)
result = storage.list_download_files("j1")
assert result["count"] == 3
nits = {f["nit"] for f in result["files"]}
assert nits == {"900808423", "900884937", "901679492"}
def test_list_download_files_falls_back_when_manifest_missing(monkeypatch):
storage = MinioStorage()
storage.get_bytes = MagicMock(side_effect=Exception("NoSuchKey"))
storage.list_keys = MagicMock(
return_value=[
"downloads/j1/900808423/a.pdf",
]
)
result = storage.list_download_files("j1")
assert result["count"] == 1
assert result["files"][0]["radicado"] == "a"
services/db/alembic/versions/0005_count_job_files_nit_object_key.py
+45
View File
@@ -0,0 +1,45 @@
"""Add nit and object_key to count_job_files for MinIO download bridge.
Revision ID: 0005
Revises: 0004
Create Date: 2026-07-17
"""
from __future__ import annotations
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0005"
down_revision: Union[str, None] = "0004"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _column_exists(conn: sa.engine.Connection, table: str, column: str) -> bool:
cols = {c["name"] for c in sa.inspect(conn).get_columns(table)}
return column in cols
def upgrade() -> None:
conn = op.get_bind()
if not sa.inspect(conn).has_table("count_job_files"):
return
if not _column_exists(conn, "count_job_files", "nit"):
op.add_column("count_job_files", sa.Column("nit", sa.String(255), nullable=True))
op.create_index("ix_count_job_files_nit", "count_job_files", ["nit"])
if not _column_exists(conn, "count_job_files", "object_key"):
op.add_column("count_job_files", sa.Column("object_key", sa.Text(), nullable=True))
def downgrade() -> None:
conn = op.get_bind()
if not sa.inspect(conn).has_table("count_job_files"):
return
if _column_exists(conn, "count_job_files", "object_key"):
op.drop_column("count_job_files", "object_key")
if _column_exists(conn, "count_job_files", "nit"):
op.drop_index("ix_count_job_files_nit", table_name="count_job_files")
op.drop_column("count_job_files", "nit")
services/desktop_app/src/renderer/components/CountResultsTable.tsx
+269 -98
View File
@@ -8,7 +8,7 @@ import {
Search,
XCircle,
} from "lucide-react";
import { useMemo, useRef, useState } from "react";
import { Fragment, useMemo, useRef, useState } from "react";
import type { CountJobFile } from "../../shared/types";
interface Props {
@@ -41,6 +41,7 @@ export function CountResultsTable({
const [search, setSearch] = useState("");
const [expanded, setExpanded] = useState<Set<string>>(new Set());
const [toggling, setToggling] = useState<Set<string>>(new Set());
const [collapsedNits, setCollapsedNits] = useState<Set<string>>(new Set());
const tagInputRef = useRef<HTMLInputElement>(null);
const summary = useMemo(() => {
@@ -74,11 +75,47 @@ export function CountResultsTable({
else if (filter === "errors") list = list.filter((f) => f.status === "error");
const q = search.trim().toLowerCase();
if (q) list = list.filter((f) => f.filename.toLowerCase().includes(q) || (f.radicado ?? "").toLowerCase().includes(q));
if (q) {
list = list.filter(
(f) =>
f.filename.toLowerCase().includes(q) ||
(f.radicado ?? "").toLowerCase().includes(q) ||
(f.nit ?? "").toLowerCase().includes(q),
);
}
return list;
}, [files, filter, search, validCounts]);
const nitGroups = useMemo(() => {
const hasNit = files.some((f) => f.nit);
if (!hasNit) return null;
const map = new Map<string, CountJobFile[]>();
for (const f of filtered) {
const nit = (f.nit || "").trim() || "Sin NIT";
const list = map.get(nit) ?? [];
list.push(f);
map.set(nit, list);
}
return [...map.entries()]
.sort(([a], [b]) => a.localeCompare(b, undefined, { numeric: true }))
.map(([nit, group]) => ({ nit, files: group }));
}, [files, filtered]);
const folderCount = useMemo(() => {
const nits = new Set(files.map((f) => (f.nit || "").trim()).filter(Boolean));
return nits.size;
}, [files]);
function toggleNitGroup(nit: string) {
setCollapsedNits((prev) => {
const next = new Set(prev);
if (next.has(nit)) next.delete(nit);
else next.add(nit);
return next;
});
}
function toggleExpand(id: string) {
setExpanded((prev) => {
const next = new Set(prev);
@@ -142,6 +179,12 @@ export function CountResultsTable({
<span className="wiz-stat-value">{summary.total}</span>
<span className="wiz-stat-label">Total</span>
</div>
{folderCount > 0 && (
<div className="wiz-stat">
<span className="wiz-stat-value">{folderCount}</span>
<span className="wiz-stat-label">Carpetas</span>
</div>
)}
<div className="wiz-stat count-stat--valid">
<span className="wiz-stat-value">{summary.valid}</span>
<span className="wiz-stat-label">Válidos</span>
@@ -157,6 +200,12 @@ export function CountResultsTable({
</div>
)}
</div>
{folderCount > 0 && (
<p className="wiz-hint" style={{ marginTop: -8, marginBottom: 12 }}>
{folderCount} carpeta{folderCount === 1 ? "" : "s"} · {summary.total} PDF
{summary.total === 1 ? "" : "s"}
</p>
)}
{/* ── Filtro por conteo exacto ── */}
<div className="count-threshold">
@@ -241,7 +290,7 @@ export function CountResultsTable({
<div className="wiz-search-box">
<Search size={14} />
<input
placeholder="Buscar archivo…"
placeholder={folderCount > 0 ? "Buscar archivo o NIT…" : "Buscar archivo…"}
value={search}
onChange={(e) => setSearch(e.target.value)}
/>
@@ -275,109 +324,231 @@ export function CountResultsTable({
</td>
</tr>
)}
{filtered.map((f) => {
const st = rowStatus(f);
const isExpanded = expanded.has(f.id);
const hasServices = f.services && f.services.length > 0;
const isToggling = toggling.has(f.id);
const wasAutoAdjusted = autoAdjustedFilenames?.has(f.filename);
return (
<>
<tr key={f.id} className={`count-row count-row--${st}`}>
<td className="count-td">
<div className="count-file-cell">
{hasServices ? (
{nitGroups
? nitGroups.map(({ nit, files: groupFiles }) => {
const collapsed = collapsedNits.has(nit);
const servicesSum = groupFiles.reduce(
(acc, f) => acc + (f.status === "ok" ? f.service_count ?? 0 : 0),
0,
);
return (
<Fragment key={`nit-${nit}`}>
<tr className="count-row count-row--group">
<td colSpan={4} className="count-td">
<button
type="button"
className="count-expand-btn"
onClick={() => toggleExpand(f.id)}
title={isExpanded ? "Ocultar servicios" : "Ver servicios detectados"}
onClick={() => toggleNitGroup(nit)}
style={{ display: "inline-flex", alignItems: "center", gap: 8, width: "100%" }}
>
{isExpanded ? <ChevronDown size={13} /> : <ChevronRight size={13} />}
{collapsed ? <ChevronRight size={13} /> : <ChevronDown size={13} />}
<strong>NIT {nit}</strong>
<span className="count-radicado">
{groupFiles.length} PDF · {servicesSum} servicios
</span>
</button>
) : (
<span className="count-expand-placeholder" />
)}
<div className="count-file-info">
<div className="count-filename-row">
<span className="count-filename" title={f.filename}>{f.filename}</span>
{wasAutoAdjusted && (
<span
className="count-auto-badge"
title="Conteo ajustado de 0 a 1 automáticamente por regla de negocio"
</td>
</tr>
{!collapsed &&
groupFiles.map((f) => {
const st = rowStatus(f);
const isExpanded = expanded.has(f.id);
const hasServices = f.services && f.services.length > 0;
const isToggling = toggling.has(f.id);
const wasAutoAdjusted = autoAdjustedFilenames?.has(f.filename);
return (
<Fragment key={f.id}>
<tr className={`count-row count-row--${st}`}>
<td className="count-td">
<div className="count-file-cell">
{hasServices ? (
<button
type="button"
className="count-expand-btn"
onClick={() => toggleExpand(f.id)}
title={isExpanded ? "Ocultar servicios" : "Ver servicios detectados"}
>
{isExpanded ? <ChevronDown size={13} /> : <ChevronRight size={13} />}
</button>
) : (
<span className="count-expand-placeholder" />
)}
<div className="count-file-info">
<div className="count-filename-row">
<span className="count-filename" title={f.filename}>{f.filename}</span>
{wasAutoAdjusted && (
<span
className="count-auto-badge"
title="Conteo ajustado de 0 a 1 automáticamente por regla de negocio"
>
auto→1
</span>
)}
</div>
{f.radicado && f.radicado !== f.filename && (
<span className="count-radicado">{f.radicado}</span>
)}
{f.status === "error" && f.error && (
<span className="count-error-msg">{f.error}</span>
)}
</div>
</div>
</td>
<td className="count-td count-td-center">
{f.status === "error" ? (
<span className="count-dash">—</span>
) : (
<span className="count-service-num">{f.service_count ?? "?"}</span>
)}
</td>
<td className="count-td count-td-center">
{st === "error" && <XCircle size={15} className="count-icon count-icon--error" />}
{st === "valid" && <CheckCircle2 size={15} className="count-icon count-icon--valid" />}
{st === "invalid" && <AlertCircle size={15} className="count-icon count-icon--warn" />}
</td>
<td className="count-td count-td-center">
{f.status !== "error" && (
<div className="count-validity-btns">
<button
type="button"
className={`count-validity-btn ${f.is_valid === true ? "count-validity-btn--accept" : ""}`}
onClick={() => handleToggle(f, f.is_valid === true ? null : true)}
disabled={loading || isToggling}
title="Marcar como válido manualmente"
>
✓
</button>
<button
type="button"
className={`count-validity-btn ${f.is_valid === false ? "count-validity-btn--reject" : ""}`}
onClick={() => handleToggle(f, f.is_valid === false ? null : false)}
disabled={loading || isToggling}
title="Rechazar manualmente"
>
✗
</button>
</div>
)}
</td>
</tr>
{isExpanded && hasServices && (
<tr className="count-row count-row-expanded">
<td colSpan={4} className="count-services-cell">
<div className="count-services-list">
{f.services.map((svc, i) => (
<div key={i} className="count-service-item">
<span className="count-service-text">{svc.text}</span>
<span className="count-service-prob">{(svc.probability * 100).toFixed(0)}%</span>
</div>
))}
</div>
</td>
</tr>
)}
</Fragment>
);
})}
</Fragment>
);
})
: filtered.map((f) => {
const st = rowStatus(f);
const isExpanded = expanded.has(f.id);
const hasServices = f.services && f.services.length > 0;
const isToggling = toggling.has(f.id);
const wasAutoAdjusted = autoAdjustedFilenames?.has(f.filename);
return (
<Fragment key={f.id}>
<tr className={`count-row count-row--${st}`}>
<td className="count-td">
<div className="count-file-cell">
{hasServices ? (
<button
type="button"
className="count-expand-btn"
onClick={() => toggleExpand(f.id)}
title={isExpanded ? "Ocultar servicios" : "Ver servicios detectados"}
>
auto→1
</span>
{isExpanded ? <ChevronDown size={13} /> : <ChevronRight size={13} />}
</button>
) : (
<span className="count-expand-placeholder" />
)}
</div>
{f.radicado && f.radicado !== f.filename && (
<span className="count-radicado">{f.radicado}</span>
)}
{f.status === "error" && f.error && (
<span className="count-error-msg">{f.error}</span>
)}
</div>
</div>
</td>
<td className="count-td count-td-center">
{f.status === "error" ? (
<span className="count-dash">—</span>
) : (
<span className="count-service-num">{f.service_count ?? "?"}</span>
)}
</td>
<td className="count-td count-td-center">
{st === "error" && <XCircle size={15} className="count-icon count-icon--error" />}
{st === "valid" && <CheckCircle2 size={15} className="count-icon count-icon--valid" />}
{st === "invalid" && <AlertCircle size={15} className="count-icon count-icon--warn" />}
</td>
<td className="count-td count-td-center">
{f.status !== "error" && (
<div className="count-validity-btns">
<button
type="button"
className={`count-validity-btn ${f.is_valid === true ? "count-validity-btn--accept" : ""}`}
onClick={() => handleToggle(f, f.is_valid === true ? null : true)}
disabled={loading || isToggling}
title="Marcar como válido manualmente"
>
✓
</button>
<button
type="button"
className={`count-validity-btn ${f.is_valid === false ? "count-validity-btn--reject" : ""}`}
onClick={() => handleToggle(f, f.is_valid === false ? null : false)}
disabled={loading || isToggling}
title="Rechazar manualmente"
>
✗
</button>
</div>
)}
</td>
</tr>
{isExpanded && hasServices && (
<tr key={`${f.id}-services`} className="count-row count-row-expanded">
<td colSpan={4} className="count-services-cell">
<div className="count-services-list">
{f.services.map((svc, i) => (
<div key={i} className="count-service-item">
<span className="count-service-text">{svc.text}</span>
<span className="count-service-prob">{(svc.probability * 100).toFixed(0)}%</span>
<div className="count-file-info">
<div className="count-filename-row">
<span className="count-filename" title={f.filename}>{f.filename}</span>
{wasAutoAdjusted && (
<span
className="count-auto-badge"
title="Conteo ajustado de 0 a 1 automáticamente por regla de negocio"
>
auto→1
</span>
)}
</div>
{f.radicado && f.radicado !== f.filename && (
<span className="count-radicado">{f.radicado}</span>
)}
{f.status === "error" && f.error && (
<span className="count-error-msg">{f.error}</span>
)}
</div>
))}
</div>
</td>
</tr>
)}
</>
);
})}
</div>
</td>
<td className="count-td count-td-center">
{f.status === "error" ? (
<span className="count-dash">—</span>
) : (
<span className="count-service-num">{f.service_count ?? "?"}</span>
)}
</td>
<td className="count-td count-td-center">
{st === "error" && <XCircle size={15} className="count-icon count-icon--error" />}
{st === "valid" && <CheckCircle2 size={15} className="count-icon count-icon--valid" />}
{st === "invalid" && <AlertCircle size={15} className="count-icon count-icon--warn" />}
</td>
<td className="count-td count-td-center">
{f.status !== "error" && (
<div className="count-validity-btns">
<button
type="button"
className={`count-validity-btn ${f.is_valid === true ? "count-validity-btn--accept" : ""}`}
onClick={() => handleToggle(f, f.is_valid === true ? null : true)}
disabled={loading || isToggling}
title="Marcar como válido manualmente"
>
✓
</button>
<button
type="button"
className={`count-validity-btn ${f.is_valid === false ? "count-validity-btn--reject" : ""}`}
onClick={() => handleToggle(f, f.is_valid === false ? null : false)}
disabled={loading || isToggling}
title="Rechazar manualmente"
>
✗
</button>
</div>
)}
</td>
</tr>
{isExpanded && hasServices && (
<tr className="count-row count-row-expanded">
<td colSpan={4} className="count-services-cell">
<div className="count-services-list">
{f.services.map((svc, i) => (
<div key={i} className="count-service-item">
<span className="count-service-text">{svc.text}</span>
<span className="count-service-prob">{(svc.probability * 100).toFixed(0)}%</span>
</div>
))}
</div>
</td>
</tr>
)}
</Fragment>
);
})}
</tbody>
</table>
</div>
services/desktop_app/src/renderer/components/CountingWizard.tsx
+204 -62
View File
@@ -41,6 +41,9 @@ interface Props {
onTaskCreated?: (task: Task) => void;
/** Callback de progreso en segundo plano, para mostrarlo desde otras secciones. */
onProgress?: (progress: BackgroundProgress) => void;
/** Origen MinIO: corrida de bot de descarga. */
downloadJobId?: string;
downloadFiles?: Array<{ key: string; nit?: string | null; radicado?: string | null }>;
}
interface CountProgress {
@@ -56,6 +59,32 @@ const BATCH_SIZE = 10;
/** Peticiones HTTP simultáneas al count-api. */
const BATCH_CONCURRENCY = 2;
function downloadFilesToManifest(
files: Array<{ key: string; nit?: string | null; radicado?: string | null }>,
): ManifestEntry[] {
return files
.filter((f) => f.key.toLowerCase().endsWith(".pdf"))
.map((f) => {
const filename = f.key.split("/").pop() || f.key;
const radicado =
(f.radicado || "").trim() ||
filename.replace(/\.pdf$/i, "");
return {
radicado,
filename,
localPath: f.key,
size: 0,
nit: f.nit ?? null,
objectKey: f.key,
};
})
.sort((a, b) => {
const nitCmp = String(a.nit ?? "").localeCompare(String(b.nit ?? ""), undefined, { numeric: true });
if (nitCmp !== 0) return nitCmp;
return a.radicado.localeCompare(b.radicado, undefined, { numeric: true });
});
}
function formatEta(progress: CountProgress): string | null {
if (!progress.startedAt || progress.processed === 0) return null;
const elapsed = (Date.now() - progress.startedAt) / 1000;
@@ -68,7 +97,18 @@ function formatEta(progress: CountProgress): string | null {
return `~${Math.ceil(secs / 60)} min · ${rateStr}`;
}
export function CountingWizard({ client, localRoot, containerRoot, job, onStartExtraction, onTaskCreated, onProgress }: Props) {
export function CountingWizard({
client,
localRoot,
containerRoot,
job,
onStartExtraction,
onTaskCreated,
onProgress,
downloadJobId,
downloadFiles,
}: Props) {
const fromDownload = Boolean(downloadJobId);
const [step, setStep] = useState<CountingStep>(1);
const [maxReached, setMaxReached] = useState<CountingStep>(1);
const [form, setForm] = useState<CountingFormState>({
@@ -95,6 +135,7 @@ export function CountingWizard({ client, localRoot, containerRoot, job, onStartE
const [error, setError] = useState<string | null>(null);
const [running, setRunning] = useState(false);
const abortRef = useRef(false);
const downloadJobIdRef = useRef(downloadJobId);
const indexLocalFolder = useCallback(
async (dir: string) => {
@@ -118,6 +159,33 @@ export function CountingWizard({ client, localRoot, containerRoot, job, onStartE
[docTypes],
);
const indexFromDownload = useCallback(
async (jobId: string, seedFiles?: Array<{ key: string; nit?: string | null; radicado?: string | null }>) => {
setIndexing(true);
setError(null);
try {
let files = seedFiles ?? [];
if (files.length === 0) {
const resp = await client.listDownloadJobFiles(jobId);
files = resp.files;
}
const entries = downloadFilesToManifest(files);
setManifest(entries);
if (entries.length === 0) {
setError("No se encontraron PDF en MinIO para esta corrida de descarga.");
}
return entries;
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
setManifest([]);
return [];
} finally {
setIndexing(false);
}
},
[client],
);
useEffect(() => {
setForm((f) => ({
...f,
@@ -127,13 +195,22 @@ export function CountingWizard({ client, localRoot, containerRoot, job, onStartE
}, [localRoot, containerRoot]);
useEffect(() => {
downloadJobIdRef.current = downloadJobId;
if (downloadJobId) {
void indexFromDownload(downloadJobId, downloadFiles);
return;
}
if (!localRoot) return;
let cancelled = false;
indexLocalFolder(localRoot).catch(() => {
if (!cancelled) setError("Error al indexar la carpeta.");
});
return () => { cancelled = true; };
}, [localRoot, indexLocalFolder]);
return () => {
cancelled = true;
};
// Solo reindexar al cambiar de corrida/carpeta; seed files es opcional.
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [localRoot, downloadJobId, indexLocalFolder, indexFromDownload]);
function goToStep(target: CountingStep) {
setStep(target);
@@ -154,6 +231,15 @@ export function CountingWizard({ client, localRoot, containerRoot, job, onStartE
return norm.startsWith(root) ? container + norm.slice(root.length) : norm;
}
async function readEntryBytes(entry: ManifestEntry): Promise<ArrayBuffer> {
const jobId = downloadJobIdRef.current;
if (jobId && entry.objectKey) {
return client.fetchDownloadObject(jobId, entry.objectKey);
}
const fileData = await desktopApi.readFile(entry.localPath);
return fileData.data;
}
async function runCounting() {
if (!manifest.length) {
setError("No hay archivos en la carpeta seleccionada.");
@@ -215,7 +301,8 @@ export function CountingWizard({ client, localRoot, containerRoot, job, onStartE
running: true,
label: `Preparando archivos (${Math.min(batchStart + fi + 1, manifest.length)}/${manifest.length})…`,
});
return desktopApi.readFile(entry.localPath);
const data = await readEntryBytes(entry);
return { filename: entry.filename, data };
}),
);
@@ -317,16 +404,27 @@ export function CountingWizard({ client, localRoot, containerRoot, job, onStartE
});
try {
const nitCount = new Set(manifest.map((m) => m.nit).filter(Boolean)).size;
const titleBase = fromDownload
? `Conteo desde descarga${nitCount ? ` · ${nitCount} carpeta${nitCount === 1 ? "" : "s"}` : ""}`
: `Conteo — ${form.localRoot.split(/[\\/]/).pop()}`;
const payload: CreateCountJobPayload = {
title: `Conteo — ${form.localRoot.split(/[\\/]/).pop()} (${finalResults.length} archivos)`,
title: `${titleBase} (${finalResults.length} archivos)`,
job_id: job?.id,
results: finalResults.map((r) => {
const entry = manifest.find((m) => m.filename === r.filename);
download_job_id: downloadJobIdRef.current ?? null,
source: fromDownload ? "from-download" : "desktop",
results: finalResults.map((r, i) => {
const entry = manifest[i];
return {
radicado: entry?.radicado ?? null,
filename: r.filename,
file_path: entry ? localToContainer(entry.localPath) : null,
file_path:
fromDownload || !entry
? entry?.objectKey ?? null
: localToContainer(entry.localPath),
doc_type: entry?.docType ?? null,
nit: entry?.nit ?? null,
object_key: entry?.objectKey ?? null,
service_count: r.status === "ok" ? r.service_count : null,
services: r.services ?? [],
status: r.status === "ok" ? "ok" : "error",
@@ -377,16 +475,22 @@ export function CountingWizard({ client, localRoot, containerRoot, job, onStartE
function handleExportCsv() {
if (!savedFiles.length) return;
const headers = ["filename", "radicado", "service_count", "auto_ajustado", "status", "is_valid", "error"];
const rows = savedFiles.map((f) => [
f.filename,
f.radicado ?? "",
f.service_count ?? "",
autoAdjustedFilenames.has(f.filename) ? "si" : "",
f.status,
f.is_valid === null ? "" : String(f.is_valid),
f.error ?? "",
]);
const hasNit = savedFiles.some((f) => f.nit);
const headers = hasNit
? ["nit", "filename", "radicado", "service_count", "auto_ajustado", "status", "is_valid", "error"]
: ["filename", "radicado", "service_count", "auto_ajustado", "status", "is_valid", "error"];
const rows = savedFiles.map((f) => {
const base = [
f.filename,
f.radicado ?? "",
f.service_count ?? "",
autoAdjustedFilenames.has(f.filename) ? "si" : "",
f.status,
f.is_valid === null ? "" : String(f.is_valid),
f.error ?? "",
];
return hasNit ? [f.nit ?? "", ...base] : base;
});
downloadCsv(buildExportCsvFilename("conteo", job?.title), toCsv(headers, rows));
}
@@ -399,8 +503,10 @@ export function CountingWizard({ client, localRoot, containerRoot, job, onStartE
setAutoAdjustedFilenames(new Set());
setError(null);
setProgress({ processed: 0, total: 0, failed: 0, currentBatch: "", startedAt: 0 });
if (form.localRoot) {
indexLocalFolder(form.localRoot);
if (downloadJobIdRef.current) {
void indexFromDownload(downloadJobIdRef.current, downloadFiles);
} else if (form.localRoot) {
void indexLocalFolder(form.localRoot);
}
}
@@ -423,58 +529,94 @@ export function CountingWizard({ client, localRoot, containerRoot, job, onStartE
<div className="wiz-section">
<h3 className="wiz-section-title">Origen de las facturas</h3>
<p className="wiz-desc">
Selecciona la carpeta local con los PDF de facturas médicas.
El servicio contará cuántos servicios contiene cada documento usando OCR.
{fromDownload
? "Los PDF se leen directamente desde MinIO (todas las carpetas NIT de la corrida de descarga)."
: "Selecciona la carpeta local con los PDF de facturas médicas. El servicio contará cuántos servicios contiene cada documento usando OCR."}
</p>
</div>
<div className="wiz-section">
<label className="wiz-field-label">Carpeta a procesar</label>
<div className="folder-picker">
<div className="folder-picker-display">
{form.localRoot ? (
<>
<FolderOpen size={16} className="folder-picker-icon" />
<span className="folder-picker-path">{form.localRoot}</span>
</>
) : (
<span className="folder-picker-placeholder">Ninguna carpeta seleccionada</span>
)}
{fromDownload ? (
<div className="wiz-section">
<label className="wiz-field-label">Corrida MinIO</label>
<div className="folder-picker">
<div className="folder-picker-display">
<FolderOpen size={16} className="folder-picker-icon" />
<span className="folder-picker-path">
downloads/{downloadJobId}
</span>
</div>
<button
type="button"
className="btn btn-ghost"
onClick={() => downloadJobId && void indexFromDownload(downloadJobId)}
disabled={indexing}
>
{indexing ? <Loader2 size={14} className="spin" /> : <RotateCcw size={14} />}
{indexing ? "Indexando…" : "Recargar"}
</button>
</div>
<button type="button" className="btn btn-primary" onClick={selectFolder} disabled={indexing}>
{indexing ? <Loader2 size={14} className="spin" /> : <FolderOpen size={15} />}
{indexing ? "Indexando…" : "Explorar"}
</button>
{manifest.length > 0 && !indexing && (
<div className="wiz-found-badge">
<FileText size={15} />
<span>
<strong>{manifest.length}</strong> PDF(s) en{" "}
<strong>{new Set(manifest.map((m) => m.nit).filter(Boolean)).size || 1}</strong>{" "}
carpeta(s) planas — listos para contar
</span>
</div>
)}
</div>
{manifest.length > 0 && !indexing && (
<div className="wiz-found-badge">
<FileText size={15} />
<span>
<strong>{manifest.length}</strong> documento(s) encontrados y listos para contar
</span>
) : (
<div className="wiz-section">
<label className="wiz-field-label">Carpeta a procesar</label>
<div className="folder-picker">
<div className="folder-picker-display">
{form.localRoot ? (
<>
<FolderOpen size={16} className="folder-picker-icon" />
<span className="folder-picker-path">{form.localRoot}</span>
</>
) : (
<span className="folder-picker-placeholder">Ninguna carpeta seleccionada</span>
)}
</div>
<button type="button" className="btn btn-primary" onClick={selectFolder} disabled={indexing}>
{indexing ? <Loader2 size={14} className="spin" /> : <FolderOpen size={15} />}
{indexing ? "Indexando…" : "Explorar"}
</button>
</div>
)}
{form.localRoot && manifest.length === 0 && !indexing && !error && (
<p className="wiz-hint">No se encontraron PDF en esta carpeta.</p>
)}
</div>
{manifest.length > 0 && !indexing && (
<div className="wiz-found-badge">
<FileText size={15} />
<span>
<strong>{manifest.length}</strong> documento(s) encontrados y listos para contar
</span>
</div>
)}
{form.localRoot && manifest.length === 0 && !indexing && !error && (
<p className="wiz-hint">No se encontraron PDF en esta carpeta.</p>
)}
</div>
)}
<details className="wiz-advanced">
<summary className="wiz-advanced-trigger">Opciones avanzadas</summary>
<div className="wiz-advanced-body">
<div className="wiz-section">
<label className="wiz-field-label">Ruta en contenedor Docker</label>
<input
type="text"
value={form.containerRoot}
onChange={(e) => setForm((f) => ({ ...f, containerRoot: e.target.value }))}
placeholder="/data"
/>
<p className="wiz-hint">
Prefijo donde están montados los documentos dentro de Docker (para guardar rutas en BD).
</p>
</div>
{!fromDownload && (
<div className="wiz-section">
<label className="wiz-field-label">Ruta en contenedor Docker</label>
<input
type="text"
value={form.containerRoot}
onChange={(e) => setForm((f) => ({ ...f, containerRoot: e.target.value }))}
placeholder="/data"
/>
<p className="wiz-hint">
Prefijo donde están montados los documentos dentro de Docker (para guardar rutas en BD).
</p>
</div>
)}
<div className="wiz-checks-row">
<label className="wiz-check-label">
services/desktop_app/src/renderer/lib/apiClient.ts
+15
View File
@@ -262,6 +262,21 @@ export class ApiClient {
return this.handle<DownloadJobFilesResponse>(response);
}
/** Descarga bytes de un PDF del resultado MinIO (proxy bot-descarga). */
async fetchDownloadObject(jobId: string, key: string): Promise<ArrayBuffer> {
const qs = new URLSearchParams({ key });
const response = await fetchWithTimeout(
this.url(`/bot-descarga/jobs/${encodeURIComponent(jobId)}/object?${qs.toString()}`),
{ headers: this.botHeaders() },
120_000,
);
if (!response.ok) {
const text = await response.text().catch(() => "");
throw new Error(text || `Error ${response.status} al leer objeto MinIO`);
}
return response.arrayBuffer();
}
async getDownloadEvents(jobId: string, since = 0): Promise<{ events: DownloadJobEvent[] }> {
const response = await fetchWithTimeout(
this.url(`/bot-descarga/jobs/${encodeURIComponent(jobId)}/events?since=${since}`),
services/desktop_app/src/renderer/modules/counting/CountingWizardModule.tsx
+24 -4
View File
@@ -1,5 +1,5 @@
import { CountingWizard } from "../../components/CountingWizard";
import type { BackgroundProgress } from "../../../shared/types";
import type { BackgroundProgress, DownloadJobFile } from "../../../shared/types";
import type { ModuleWizardProps } from "../registry";
interface Props extends ModuleWizardProps {
@@ -9,21 +9,41 @@ interface Props extends ModuleWizardProps {
/**
* Wrapper del CountingWizard para el sistema de módulos.
* Pasa el job al wizard para que los resultados queden vinculados.
* Si el contexto trae downloadJobId (desde Resultados → Continuar a conteo),
* indexa PDFs desde MinIO en todas las carpetas NIT planas.
*/
export function CountingWizardModule({ client, job, onTaskCreated, onActivateModule, onProgress }: Props) {
export function CountingWizardModule({
client,
job,
context,
onTaskCreated,
onActivateModule,
onProgress,
}: Props) {
const folderPath = String(job.metadata?.folder_path ?? "");
const containerRoot = "/data";
const downloadJobId = context?.downloadJobId as string | undefined;
const downloadFiles = context?.files as DownloadJobFile[] | undefined;
return (
<CountingWizard
key={downloadJobId ? `dl-${downloadJobId}` : `local-${folderPath || "default"}`}
client={client}
job={job}
localRoot={folderPath}
localRoot={downloadJobId ? "" : folderPath}
containerRoot={containerRoot}
downloadJobId={downloadJobId}
downloadFiles={downloadFiles}
onTaskCreated={onTaskCreated}
onProgress={onProgress}
onStartExtraction={(countingTaskId, validCount, validCounts) => {
onActivateModule?.("extraction", { countingTaskId, validCount, validCounts, jobId: job.id });
onActivateModule?.("extraction", {
countingTaskId,
validCount,
validCounts,
jobId: job.id,
downloadJobId,
});
}}
/>
);
services/desktop_app/src/renderer/modules/extraction/JobExtractionWizard.tsx
+103 -19
View File
@@ -82,6 +82,9 @@ export function JobExtractionWizard({
const countingTaskId = countingTaskIdProp ?? (context?.countingTaskId as string | undefined);
const validCounts = validCountsProp ?? (context?.validCounts as number[] | undefined);
const validCount = validCountProp ?? (context?.validCount as number | undefined);
const downloadJobId =
(context?.downloadJobId as string | undefined) ??
undefined;
const [form, setForm] = useState<ExtractionFormState>(() => makeDefaultForm(job, !!countingTaskId));
const [schemaResponse, setSchemaResponse] = useState<SchemaResponse | null>(null);
@@ -100,6 +103,7 @@ export function JobExtractionWizard({
/** Rutas de muestras ya usadas en descubrimiento (para no repetir al reintentar). */
const [usedDiscoveryPaths, setUsedDiscoveryPaths] = useState<string[]>([]);
const [discoverySampleLabels, setDiscoverySampleLabels] = useState<string[]>([]);
const [resolvedDownloadJobId, setResolvedDownloadJobId] = useState<string | undefined>(downloadJobId);
const flattened = useMemo(
() => (schemaResponse ? flattenSchema(schemaResponse) : { schema: {}, sample: null }),
@@ -115,19 +119,69 @@ export function JobExtractionWizard({
// Paso 3: necesita al menos un campo seleccionado
(extractionStep === 3 && selection.fields.length > 0);
// Auto-indexar la carpeta del job al montar (usada como muestra para descubrir campos)
// Auto-indexar: carpeta local, o archivos del conteo desde MinIO
useEffect(() => {
if (countingTaskId) {
void (async () => {
try {
const [countFiles, countTask] = await Promise.all([
client.countJobFiles(countingTaskId),
client.countJob(countingTaskId).catch(() => null),
]);
const req = (countTask?.request ?? {}) as Record<string, unknown>;
const dlId =
downloadJobId ||
(typeof req.download_job_id === "string" ? req.download_job_id : undefined);
if (dlId) setResolvedDownloadJobId(dlId);
const fromMinio = countFiles.some((f) => f.object_key);
if (fromMinio) {
const entries: ManifestEntry[] = countFiles
.filter((f) => f.object_key)
.map((f) => ({
radicado: f.radicado || f.filename.replace(/\.pdf$/i, ""),
filename: f.filename,
localPath: f.object_key!,
size: 0,
nit: f.nit ?? null,
objectKey: f.object_key!,
}));
setManifest(entries);
setForm((cur) => ({ ...cur, mode: "general" }));
setStatusText(
`${entries.length} documento(s) desde MinIO` +
(new Set(entries.map((e) => e.nit).filter(Boolean)).size
? ` · ${new Set(entries.map((e) => e.nit).filter(Boolean)).size} carpetas`
: ""),
);
return;
}
} catch {
/* fallback a carpeta local abajo */
}
})();
}
const folder = form.localRoot || form.selectedLocalPath;
if (!folder || manifest.length > 0) return;
if (!folder || manifest.length > 0 || countingTaskId) return;
indexFolderAt(folder, form.mode, form.docTypes)
.then((entries) => {
setManifest(entries);
// Asegurar que selectedLocalPath esté sincronizado
setForm((cur) => ({ ...cur, selectedLocalPath: folder, selectedContainerPath: folder }));
})
.catch(() => undefined);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
}, [countingTaskId]);
async function readEntryBytes(entry: ManifestEntry): Promise<{ filename: string; data: ArrayBuffer }> {
if (entry.objectKey && resolvedDownloadJobId) {
const data = await client.fetchDownloadObject(resolvedDownloadJobId, entry.objectKey);
return { filename: entry.filename, data };
}
if (!window.desktopApi) throw new Error("Requiere Electron.");
const fileData = await window.desktopApi.readFile(entry.localPath);
return { filename: fileData.filename, data: fileData.data };
}
function updateForm(patch: Partial<ExtractionFormState>) {
setForm((cur) => {
@@ -204,7 +258,8 @@ export function JobExtractionWizard({
if (candidates.length === 0) {
throw new Error("No hay documentos candidatos para la muestra.");
}
const unused = candidates.filter((e) => !exclude.has(e.localPath));
const keyOf = (e: ManifestEntry) => e.objectKey || e.localPath;
const unused = candidates.filter((e) => !exclude.has(keyOf(e)));
const pool = unused.length > 0 ? unused : candidates;
if (!preferRandom && unused.length === candidates.length) {
return candidates[0];
@@ -217,17 +272,35 @@ export function JobExtractionWizard({
await runTask(
randomOther ? "Probando con otro archivo…" : "Descubriendo campos...",
async () => {
if (!window.desktopApi) throw new Error("Requiere Electron.");
if (manifest.length === 0) throw new Error("Selecciona una carpeta con documentos.");
const exclude = new Set(randomOther ? usedDiscoveryPaths : []);
const responses: SchemaResponse[] = [];
const picked: ManifestEntry[] = [];
if (form.mode === "general") {
const fromMinio = manifest.some((m) => m.objectKey);
const nits = [
...new Set(manifest.map((m) => (m.nit || "").trim()).filter(Boolean)),
].sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));
// Desde descarga multi-NIT: 1 muestra por carpeta plana, luego merge.
if (fromMinio && form.mode === "general" && nits.length > 0) {
for (const nit of nits) {
const pool = manifest.filter((m) => (m.nit || "").trim() === nit);
const entry = pickSampleEntry(pool, exclude, randomOther);
picked.push(entry);
const fileData = await readEntryBytes(entry);
const blob = new Blob([fileData.data]);
responses.push(
await client.discoverSchemaUpload(
buildSchemaUploadForm(form, blob, fileData.filename),
),
);
}
} else if (form.mode === "general") {
const entry = pickSampleEntry(manifest, exclude, randomOther);
picked.push(entry);
const fileData = await window.desktopApi.readFile(entry.localPath);
const fileData = await readEntryBytes(entry);
const blob = new Blob([fileData.data]);
responses.push(
await client.discoverSchemaUpload(
@@ -242,7 +315,7 @@ export function JobExtractionWizard({
}
const entry = pickSampleEntry(typed, exclude, randomOther);
picked.push(entry);
const fileData = await window.desktopApi.readFile(entry.localPath);
const fileData = await readEntryBytes(entry);
const blob = new Blob([fileData.data]);
responses.push(
await client.discoverSchemaUpload(
@@ -260,11 +333,13 @@ export function JobExtractionWizard({
aliasMap: Object.fromEntries(Object.keys(flat.schema).map((p) => [p, defaultAlias(p)])),
});
setEstimateJson(null);
const labels = picked.map((e) => e.filename);
const labels = picked.map((e) =>
e.nit ? `${e.nit}/${e.filename}` : e.filename,
);
setDiscoverySampleLabels(labels);
setUsedDiscoveryPaths((prev) => {
const next = new Set(randomOther ? prev : []);
for (const e of picked) next.add(e.localPath);
for (const e of picked) next.add(e.objectKey || e.localPath);
return [...next];
});
setStatusText(
@@ -285,11 +360,10 @@ export function JobExtractionWizard({
async function estimate() {
await runTask("Estimando costo...", async () => {
if (!window.desktopApi) throw new Error("Requiere Electron.");
if (!selection.fields.length) throw new Error("Selecciona al menos un campo.");
if (manifest.length === 0) throw new Error("Selecciona una carpeta.");
const sampleEntry = manifest[0];
const fileData = await window.desktopApi.readFile(sampleEntry.localPath);
const fileData = await readEntryBytes(sampleEntry);
const blob = new Blob([fileData.data]);
const uniqueRadicados = new Set(manifest.map((e) => e.radicado)).size;
const formData = buildEstimateUploadForm(form, blob, fileData.filename, uniqueRadicados, selection.fields, form.model);
@@ -310,22 +384,32 @@ export function JobExtractionWizard({
async function runExtraction() {
await runTask("Iniciando extracción...", async () => {
if (selection.fields.length === 0) throw new Error("Selecciona al menos un campo.");
if (!window.desktopApi) throw new Error("Requiere Electron.");
if (manifest.length === 0) throw new Error("Selecciona una carpeta.");
let workManifest: ManifestEntry[];
// ── Caso 1: desde conteo previo — filtrar manifest a los archivos válidos ──
// Caso 1: desde conteo previo — filtrar manifest a los archivos válidos
if (countingTaskId) {
const countFiles = await client.countJobFiles(countingTaskId);
const validKeys = new Set(
countFiles
.filter(isCountFileValid)
.map((f) => f.object_key || `${f.nit ?? ""}|${f.filename}`),
);
const validFilenames = new Set(
countFiles.filter(isCountFileValid).map((f) => f.filename),
);
workManifest = manifest.filter((e) => validFilenames.has(e.filename));
workManifest = manifest.filter((e) => {
if (e.objectKey) return validKeys.has(e.objectKey);
if (e.nit) return validKeys.has(`${e.nit}|${e.filename}`);
return validFilenames.has(e.filename);
});
if (workManifest.length === 0) {
throw new Error(
"No se encontraron archivos válidos del conteo en la carpeta. " +
"Asegúrate de que los PDFs válidos están en la carpeta seleccionada.",
"No se encontraron archivos válidos del conteo." +
(manifest.some((m) => m.objectKey)
? " Verifica el filtro de conteos válidos."
: " Asegúrate de que los PDFs válidos estén en la carpeta seleccionada."),
);
}
} else {
@@ -378,7 +462,7 @@ export function JobExtractionWizard({
running: true,
label: `Subiendo archivos (${queued + fi}/${total})…`,
});
files.push(await window.desktopApi!.readFile(batch[fi].localPath));
files.push(await readEntryBytes(batch[fi]));
}
const blobs = files.map((f) => new Blob([f.data]));
onProgress?.({
services/desktop_app/src/renderer/views/JobResultsView.tsx
+23 -10
View File
@@ -11,7 +11,7 @@ import {
Sparkles,
} from "lucide-react";
import { useEffect, useState } from "react";
import type { BackgroundProgress, CleaningJob, CountJobFile, Job, Task } from "../../shared/types";
import type { BackgroundProgress, CleaningJob, CountJobFile, DownloadJobFile, Job, Task } from "../../shared/types";
import { CountResultsTable } from "../components/CountResultsTable";
import { ExtractionResultsTable } from "../components/ExtractionResultsTable";
import { BackgroundProgressBanner } from "../components/BackgroundProgressBanner";
@@ -33,6 +33,11 @@ interface Props {
onStartExtraction: (ctx?: Record<string, unknown>) => void;
onStartCleaning: (ctx?: Record<string, unknown>) => void;
onStartDownload: () => void;
onContinueToCounting?: (ctx: {
downloadJobId: string;
downloadsPrefix?: string;
files?: DownloadJobFile[];
}) => void;
}
export function JobResultsView({
@@ -47,6 +52,7 @@ export function JobResultsView({
onStartExtraction,
onStartCleaning,
onStartDownload,
onContinueToCounting,
}: Props) {
const [activeTab, setActiveTab] = useState<ResultTab>("counting");
const [refreshing, setRefreshing] = useState(false);
@@ -177,6 +183,7 @@ export function JobResultsView({
job={job}
tasks={downloadTasks}
onStartDownload={onStartDownload}
onContinueToCounting={onContinueToCounting}
/>
)}
</div>
@@ -236,15 +243,21 @@ function CountingResultsSection({
function exportCsv(subset: "valid" | "invalid") {
const subset_files = subset === "valid" ? files.filter(isFileValid) : files.filter((f) => !isFileValid(f));
const headers = ["filename", "radicado", "service_count", "status", "is_valid", "error"];
const rows = subset_files.map((f) => [
f.filename,
f.radicado ?? "",
f.service_count ?? "",
f.status,
f.is_valid === null ? "" : String(f.is_valid),
f.error ?? "",
]);
const hasNit = subset_files.some((f) => f.nit);
const headers = hasNit
? ["nit", "filename", "radicado", "service_count", "status", "is_valid", "error"]
: ["filename", "radicado", "service_count", "status", "is_valid", "error"];
const rows = subset_files.map((f) => {
const base = [
f.filename,
f.radicado ?? "",
f.service_count ?? "",
f.status,
f.is_valid === null ? "" : String(f.is_valid),
f.error ?? "",
];
return hasNit ? [f.nit ?? "", ...base] : base;
});
downloadCsv(buildExportCsvFilename("conteo", job.title, subset), toCsv(headers, rows));
}
services/desktop_app/src/renderer/views/JobView.tsx
+2
View File
@@ -149,6 +149,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
<CountingWizardModule
client={client}
job={job}
context={sectionContext}
onTaskCreated={handleTaskCreated}
onProgress={(p) => setCountingProgress(p.running ? p : null)}
onActivateModule={(type, ctx) => onActivateSection(type as JobSection, ctx)}
@@ -198,6 +199,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
onStartExtraction={(ctx?: Record<string, unknown>) => onActivateSection("extraction", ctx)}
onStartCleaning={(ctx?: Record<string, unknown>) => onActivateSection("cleaning", ctx)}
onStartDownload={() => onActivateSection("downloads")}
onContinueToCounting={(ctx) => onActivateSection("counting", ctx)}
/>
</div>
</div>
services/desktop_app/src/shared/types.ts
+10
View File
@@ -6,6 +6,10 @@ export interface ManifestEntry {
docType?: string;
localPath: string;
size: number;
/** NIT / carpeta plana (resultados bot descarga MinIO). */
nit?: string | null;
/** Object key en MinIO cuando el origen es descarga remota. */
objectKey?: string | null;
}
export interface FileUploadData {
@@ -117,6 +121,8 @@ export interface CountJobFile {
is_valid: boolean | null;
error: string | null;
created_at: string;
nit?: string | null;
object_key?: string | null;
}
export interface CountJobFileCreateEntry {
@@ -128,12 +134,16 @@ export interface CountJobFileCreateEntry {
services?: Array<{ text: string; probability: number; is_service?: boolean | null }>;
status?: "ok" | "error";
error?: string | null;
nit?: string | null;
object_key?: string | null;
}
export interface CreateCountJobPayload {
title?: string;
job_id?: string;
results: CountJobFileCreateEntry[];
download_job_id?: string | null;
source?: string | null;
}
export interface ExtractionFromCountPayload {
services/extraction/api/main.py
+15 -1
View File
@@ -1107,12 +1107,16 @@ class CountJobFileCreate(BaseModel):
services: list[dict[str, Any]] = Field(default_factory=list)
status: str = "ok"
error: str | None = None
nit: str | None = None
object_key: str | None = None
class CreateCountJobRequest(BaseModel):
title: str | None = None
job_id: str | None = None # si se pasa, se usa el job existente
results: list[CountJobFileCreate]
download_job_id: str | None = None
source: str | None = None
class CountJobFileRead(BaseModel):
@@ -1128,6 +1132,8 @@ class CountJobFileRead(BaseModel):
is_valid: bool | None
error: str | None
created_at: str
nit: str | None = None
object_key: str | None = None
class UpdateCountJobFileRequest(BaseModel):
@@ -1163,6 +1169,8 @@ def serialize_count_file(f: CountJobFile) -> dict[str, Any]:
"is_valid": f.is_valid,
"error": f.error,
"created_at": f.created_at.isoformat(),
"nit": getattr(f, "nit", None),
"object_key": getattr(f, "object_key", None),
}
@@ -1193,12 +1201,16 @@ def create_count_job(request: CreateCountJobRequest) -> dict[str, Any]:
type="counting",
status="completed",
title=default_title,
request={},
request={
"source": request.source or ("from-download" if request.download_job_id else "desktop"),
"download_job_id": request.download_job_id,
},
result={
"total": total,
"processed": total,
"ok": sum(1 for r in request.results if r.status == "ok"),
"failed": sum(1 for r in request.results if r.status != "ok"),
"nits": sorted({r.nit for r in request.results if r.nit}),
},
started_at=utcnow(),
finished_at=utcnow(),
@@ -1215,6 +1227,8 @@ def create_count_job(request: CreateCountJobRequest) -> dict[str, Any]:
filename=r.filename,
file_path=r.file_path,
doc_type=r.doc_type,
nit=r.nit,
object_key=r.object_key,
service_count=r.service_count,
services=r.services,
status=r.status,
services/extraction/infrastructure/database.py
+2
View File
@@ -111,6 +111,8 @@ class CountJobFile(Base):
filename: Mapped[str] = mapped_column(String(512))
file_path: Mapped[str | None] = mapped_column(Text, nullable=True)
doc_type: Mapped[str | None] = mapped_column(String(128), nullable=True)
nit: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
object_key: Mapped[str | None] = mapped_column(Text, nullable=True)
service_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
services: Mapped[list[Any]] = mapped_column(JSON, default=list)
status: Mapped[str] = mapped_column(String(32), default="ok", index=True)
services/extraction/worker/tasks.py
+6 -2
View File
@@ -11,6 +11,7 @@ import logging
import os
from typing import Any
from sqlalchemy.orm.attributes import flag_modified
from infrastructure.database import ExtractionRow, Task, session_scope, utcnow
from infrastructure.serialization import flatten_extraction_result
from api.desktop_ingest import (
@@ -128,7 +129,8 @@ async def process_document(
# Persistir resultado en BD
def _persist() -> dict[str, Any]:
with session_scope() as session:
task = session.get(Task, task_id)
# Lock evita lost-update en result.processed con N workers concurrentes.
task = session.get(Task, task_id, with_for_update=True)
if task is None:
return {"skipped": True, "reason": "task_not_found"}
@@ -185,6 +187,7 @@ async def process_document(
result["row_count"] = row_index
result["processed"] = int(result.get("processed") or 0) + 1
task.result = result
flag_modified(task, "result")
task.updated_at = utcnow()
session.flush()
mark_task_completed_if_done(session, task, batch_id)
@@ -206,7 +209,7 @@ async def process_document(
def _record_error() -> None:
try:
with session_scope() as session:
task = session.get(Task, task_id)
task = session.get(Task, task_id, with_for_update=True)
if task is None:
return
result = dict(task.result or {})
@@ -219,6 +222,7 @@ async def process_document(
result["errors"] = errors
result["processed"] = int(result.get("processed") or 0) + 1
task.result = result
flag_modified(task, "result")
task.updated_at = utcnow()
mark_task_completed_if_done(session, task, batch_id)
except Exception as inner: