feat: Introduce bot-digitador service for automated data entry in Integración Total. Add CLI and GUI interfaces for CSV processing, implement auto-start functionality on Windows, and enhance README with setup instructions and usage details. Include necessary configurations in docker-compose for service integration.

This commit is contained in:
erik
2026-08-09 21:26:56 -05:00
parent a3b4c7fc47
commit e9b766281a
74 changed files with 11177 additions and 115 deletions
+11 -1
View File
@@ -20,6 +20,7 @@ CLEANING_URL = os.getenv("CLEANING_URL", "http://cleaning:8002").rstrip("/")
POBLACION_APTA_URL = os.getenv("POBLACION_APTA_URL", "http://poblacion_apta:8003").rstrip("/")
COUNT_URL = os.getenv("COUNT_URL", "http://count-api:8080").rstrip("/")
BOT_DESCARGA_URL = os.getenv("BOT_DESCARGA_URL", "http://bot-descarga:8200").rstrip("/")
BOT_DIGITADOR_URL = os.getenv("BOT_DIGITADOR_URL", "http://bot-digitador:8210").rstrip("/")
# Timeouts: JSON ligero vs uploads OCR/conteo
_TIMEOUT_JSON = httpx.Timeout(connect=5.0, read=30.0, write=30.0, pool=5.0)
@@ -36,12 +37,13 @@ async def lifespan(app: FastAPI):
limits=httpx.Limits(max_connections=100, max_keepalive_connections=20),
)
_log.info(
"Gateway iniciado — extraction=%s cleaning=%s poblacion_apta=%s count=%s bot_descarga=%s",
"Gateway iniciado — extraction=%s cleaning=%s poblacion_apta=%s count=%s bot_descarga=%s bot_digitador=%s",
EXTRACTION_URL,
CLEANING_URL,
POBLACION_APTA_URL,
COUNT_URL,
BOT_DESCARGA_URL,
BOT_DIGITADOR_URL,
)
yield
await _http_client.aclose()
@@ -84,6 +86,7 @@ def _is_upload_path(path: str) -> bool:
or path.startswith("/poblacion-apta/jobs/from-csv")
or path.startswith("/poblacion-apta/jobs/from-cleaning")
or path.startswith("/bot-descarga/jobs")
or path.startswith("/bot-digitador/jobs")
)
@@ -105,6 +108,7 @@ async def health() -> dict[str, Any]:
poblacion_apta = await _safe_health(client, POBLACION_APTA_URL)
counting = await _safe_health(client, COUNT_URL)
bot_descarga = await _safe_health(client, BOT_DESCARGA_URL)
bot_digitador = await _safe_health(client, BOT_DIGITADOR_URL)
return {
"status": "ok" if extraction["ok"] and cleaning["ok"] and poblacion_apta["ok"] else "degraded",
"services": {
@@ -113,6 +117,7 @@ async def health() -> dict[str, Any]:
"poblacion_apta": poblacion_apta,
"counting": counting,
"bot_descarga": bot_descarga,
"bot_digitador": bot_digitador,
},
}
@@ -183,6 +188,11 @@ async def bot_descarga_proxy(path: str, request: Request) -> Response:
return await _proxy_request(request, f"{BOT_DESCARGA_URL}/{path}")
@app.api_route("/bot-digitador/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
async def bot_digitador_proxy(path: str, request: Request) -> Response:
return await _proxy_request(request, f"{BOT_DIGITADOR_URL}/{path}")
@app.api_route("/jobs", methods=["GET", "POST"])
async def jobs_root_proxy(request: Request) -> Response:
return await _proxy_request(request, f"{EXTRACTION_URL}/jobs")
+239 -45
View File
@@ -87,26 +87,30 @@ def claim_pending(
@app.post("/jobs")
async def create_job(
worker_id: str = Form(...),
worker_id: str = Form(""),
worker_ids: str = Form(""),
job_id: str = Form(...),
title: str = Form(""),
file: UploadFile = File(...),
_: str = Depends(require_bearer),
) -> dict:
if not worker_id.strip():
raise HTTPException(status_code=400, detail="worker_id obligatorio")
"""Crea una corrida de descarga.
- 1 worker → flujo clásico (un solo job claimable).
- N workers (worker_ids JSON o todos online) → padre + N shards,
mismo downloads_prefix del padre.
"""
import json as _json
from domain.excel_split import parse_radicado_rows, rows_to_xlsx_bytes, split_rows
from infrastructure.storage_minio import downloads_prefix as mk_prefix
parent_job_id = job_id.strip()
if not parent_job_id:
raise HTTPException(status_code=400, detail="job_id (Job Desktop) obligatorio")
if not store.parent_job_exists(parent_job_id):
raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado")
if not store.is_worker_online(worker_id):
raise HTTPException(
status_code=409,
detail=f"Worker '{worker_id}' no está online. Enciende el agent primero.",
)
data = await file.read()
if not data:
raise HTTPException(status_code=400, detail="Archivo vacío")
@@ -115,45 +119,224 @@ async def create_job(
content_type = file.content_type or "application/octet-stream"
title_clean = title.strip()
job = store.create_job(
worker_id=worker_id.strip(),
# Resolver workers
selected: List[str] = []
raw_ids = (worker_ids or "").strip()
if raw_ids:
try:
parsed = _json.loads(raw_ids)
if isinstance(parsed, list):
selected = [str(x).strip() for x in parsed if str(x).strip()]
elif isinstance(parsed, str) and parsed.strip():
selected = [parsed.strip()]
except _json.JSONDecodeError:
selected = [x.strip() for x in raw_ids.split(",") if x.strip()]
if not selected and worker_id.strip():
selected = [worker_id.strip()]
online = store.list_online_worker_ids()
online_set = set(online)
if not selected:
selected = list(online)
else:
missing = [w for w in selected if w not in online_set]
if missing:
raise HTTPException(
status_code=409,
detail=f"Workers no online: {', '.join(missing)}. Enciende el agent primero.",
)
if not selected:
raise HTTPException(
status_code=409,
detail="No hay bots online. Enciende al menos un agent.",
)
# Dedup preservando orden
seen: set[str] = set()
workers_unique: List[str] = []
for w in selected:
if w not in seen:
seen.add(w)
workers_unique.append(w)
# Parse filas
try:
rows = parse_radicado_rows(data, original)
except Exception as e:
raise HTTPException(status_code=400, detail=f"No se pudo leer el Excel/CSV: {e}") from e
if not rows:
raise HTTPException(status_code=400, detail="El archivo no tiene filas de radicado/NIT")
# Un solo bot (o una sola fila) → flujo actual
n_workers = min(len(workers_unique), len(rows))
if n_workers <= 1:
only = workers_unique[0]
job = store.create_job(
worker_id=only,
excel_key="",
original_filename=original,
title=title_clean,
job_id=parent_job_id,
)
task_id = store.create_downloads_task(
parent_job_id=parent_job_id,
download_job_id=job.id,
worker_id=only,
title=title_clean or f"Bot descarga ({original})",
original_filename=original,
worker_ids=[only],
mode="single",
)
try:
key = minio.upload_excel(job.id, data, original, content_type=content_type)
except Exception as e:
store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
job = store.update_excel_key(job.id, key)
assert job is not None
store.add_event(
job.id,
"run_started",
f"Job creado y asignado a {only}",
metadata={
"excel_key": key,
"worker_id": only,
"job_id": parent_job_id,
"task_id": task_id,
"mode": "single",
"total_rows": len(rows),
},
)
return job.to_dict()
# Distribuido: padre + shards
workers_unique = workers_unique[:n_workers]
chunks = split_rows(rows, n_workers)
n = len(chunks)
workers_unique = workers_unique[:n]
parent = store.create_job(
worker_id=None,
excel_key="",
original_filename=original,
title=title_clean,
title=title_clean or f"Descarga distribuida ({original})",
job_id=parent_job_id,
)
task_id = store.create_downloads_task(
parent_job_id=parent_job_id,
download_job_id=job.id,
worker_id=worker_id.strip(),
title=title_clean or f"Bot descarga ({original})",
original_filename=original,
)
job = store.get_job(job.id)
assert job is not None
try:
key = minio.upload_excel(job.id, data, original, content_type=content_type)
except Exception as e:
store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
job = store.update_excel_key(job.id, key)
assert job is not None
store.add_event(
job.id,
"run_started",
f"Job creado y asignado a {worker_id}",
metadata={
"excel_key": key,
"worker_id": worker_id,
"job_id": parent_job_id,
"task_id": task_id,
shard_count=n,
status=JobStatus.PENDING.value,
result={
"mode": "distributed",
"shard_ids": [],
"total_rows": len(rows),
"total": len(rows),
"processed": 0,
"exitosos": 0,
"fallidos": 0,
},
)
prefix = parent.downloads_prefix or mk_prefix(parent.id)
task_id = store.create_downloads_task(
parent_job_id=parent_job_id,
download_job_id=parent.id,
worker_id=None,
title=title_clean or f"Bot descarga ×{n} ({original})",
original_filename=original,
worker_ids=workers_unique,
mode="distributed",
)
try:
full_key = minio.upload_excel(parent.id, data, original, content_type=content_type)
except Exception as e:
store.patch_job(parent.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
store.update_excel_key(parent.id, full_key)
shard_ids: List[str] = []
for i, (chunk, wid) in enumerate(zip(chunks, workers_unique)):
shard_bytes = rows_to_xlsx_bytes(chunk)
shard = store.create_job(
worker_id=wid,
excel_key="",
original_filename=f"shard_{i + 1}_of_{n}.xlsx",
title=f"{title_clean or 'Descarga'} · shard {i + 1}/{n}",
job_id=parent_job_id,
task_id=task_id,
downloads_prefix_override=prefix,
parent_download_job_id=parent.id,
shard_index=i,
shard_count=n,
result={"total": len(chunk), "processed": 0},
)
try:
shard_key = minio.upload_excel(
shard.id,
shard_bytes,
f"shard_{i + 1}_of_{n}.xlsx",
content_type=(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
),
)
except Exception as e:
store.patch_job(
parent.id,
status=JobStatus.FAILED.value,
error=f"MinIO shard upload failed: {e}",
)
raise HTTPException(
status_code=502, detail=f"Error subiendo shard a MinIO: {e}"
) from e
store.update_excel_key(shard.id, shard_key)
shard_ids.append(shard.id)
store.add_event(
parent.id,
"info",
f"Shard {i + 1}/{n} → {wid} ({len(chunk)} filas)",
metadata={"shard_id": shard.id, "worker_id": wid, "rows": len(chunk)},
)
store.patch_job(
parent.id,
result={
"mode": "distributed",
"shard_ids": shard_ids,
"total_rows": len(rows),
"total": len(rows),
"processed": 0,
"exitosos": 0,
"fallidos": 0,
"downloads_prefix": prefix,
"worker_ids": workers_unique,
},
)
store.add_event(
parent.id,
"run_started",
f"Corrida distribuida: {n} bots, {len(rows)} filas",
metadata={
"excel_key": full_key,
"worker_ids": workers_unique,
"job_id": parent_job_id,
"task_id": task_id,
"mode": "distributed",
"shard_ids": shard_ids,
"total_rows": len(rows),
},
)
job = store.get_job(parent.id)
assert job is not None
return job.to_dict()
@app.post("/internal/reclaim")
def reclaim_jobs(_: str = Depends(require_bearer)) -> dict:
n = store.reclaim_stale_running_jobs()
return {"reclaimed": n}
@app.get("/jobs")
def list_jobs(
limit: int = Query(100, ge=1, le=500),
@@ -168,7 +351,15 @@ def get_job(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
job = store.get_job(download_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
return job.to_dict()
# Refrescar consolidación si es padre distribuido
if job.is_distributed_parent:
refreshed = store.refresh_parent_from_shards(download_job_id)
if refreshed:
job = refreshed
payload = job.to_dict()
if job.is_distributed_parent:
payload["shards"] = [s.to_dict() for s in store.list_shards(job.id)]
return payload
@app.get("/jobs/{download_job_id}/files")
@@ -279,7 +470,8 @@ def post_event(
body: EventBody,
_: str = Depends(require_bearer),
) -> dict:
if not store.get_job(download_job_id):
job = store.get_job(download_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
event = store.add_event(
download_job_id,
@@ -292,8 +484,7 @@ def post_event(
metadata=body.metadata,
timestamp=body.timestamp,
)
job = store.get_job(download_job_id)
if job and body.index is not None and body.total is not None:
if body.index is not None and body.total is not None:
result = dict(job.result or {})
result["processed"] = body.index
result["total"] = body.total
@@ -327,4 +518,7 @@ def cancel_flag(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
job = store.get_job(download_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
return {"cancel_requested": job.cancel_requested, "status": job.status}
return {
"cancel_requested": store.is_cancel_requested(download_job_id),
"status": job.status,
}
@@ -0,0 +1,99 @@
"""Parseo y partición de Excel/CSV de radicados para descarga distribuida."""
from __future__ import annotations
import csv
import io
import re
from typing import Any
from openpyxl import Workbook, load_workbook
_HEADER_NAMES = ("radicado", "proceso", "numero", "número")
def _clean_nit(valor: Any) -> str:
valor = str(valor).strip()
valor = re.sub(r'[\\/:*?"<>|]', "_", valor)
return valor if valor else "SIN_NIT"
def parse_radicado_rows(data: bytes, filename: str = "input.xlsx") -> list[tuple[str, str]]:
"""Devuelve lista de (radicado, nit)."""
name = (filename or "input.xlsx").lower()
if name.endswith(".csv"):
return _parse_csv(data)
return _parse_xlsx(data)
def _parse_csv(data: bytes) -> list[tuple[str, str]]:
text = None
for enc in ("utf-8-sig", "utf-8", "latin-1", "cp1252"):
try:
text = data.decode(enc)
break
except UnicodeDecodeError:
continue
if text is None:
text = data.decode("utf-8", errors="replace")
sample = text[:4096]
try:
dialect = csv.Sniffer().sniff(sample, delimiters="|,;\t")
delim = dialect.delimiter
except csv.Error:
delim = "|" if sample.count("|") >= sample.count(",") else ","
reader = csv.reader(io.StringIO(text), delimiter=delim)
out: list[tuple[str, str]] = []
for fila in reader:
if not fila:
continue
radicado = fila[0] if len(fila) > 0 else None
nit = fila[1] if len(fila) > 1 else None
if not radicado or str(radicado).strip() in ("", "None"):
continue
radicado_s = str(radicado).strip()
if radicado_s.lower() in _HEADER_NAMES:
continue
out.append((radicado_s, _clean_nit(nit)))
return out
def _parse_xlsx(data: bytes) -> list[tuple[str, str]]:
wb = load_workbook(io.BytesIO(data), data_only=True)
ws = wb.active
out: list[tuple[str, str]] = []
for fila in ws.iter_rows(values_only=True):
radicado = fila[0] if len(fila) > 0 else None
nit = fila[1] if len(fila) > 1 else None
if not radicado or str(radicado).strip() in ("", "None"):
continue
radicado_s = str(radicado).strip()
if radicado_s.lower() in _HEADER_NAMES:
continue
out.append((radicado_s, _clean_nit(nit)))
return out
def split_rows(rows: list[tuple[str, str]], n: int) -> list[list[tuple[str, str]]]:
"""Parte filas en n trozos lo más equilibrados posible (chunks contiguos)."""
if n <= 1 or not rows:
return [rows]
n = min(n, len(rows))
size, rem = divmod(len(rows), n)
chunks: list[list[tuple[str, str]]] = []
idx = 0
for i in range(n):
take = size + (1 if i < rem else 0)
chunks.append(rows[idx : idx + take])
idx += take
return [c for c in chunks if c]
def rows_to_xlsx_bytes(rows: list[tuple[str, str]]) -> bytes:
wb = Workbook()
ws = wb.active
ws.append(["radicado", "nit"])
for radicado, nit in rows:
ws.append([radicado, nit])
buf = io.BytesIO()
wb.save(buf)
return buf.getvalue()
+16 -1
View File
@@ -40,7 +40,7 @@ class Worker:
@dataclass
class Job:
id: str
worker_id: str
worker_id: Optional[str]
status: str = JobStatus.PENDING.value
title: str = ""
excel_key: str = ""
@@ -54,10 +54,25 @@ class Job:
finished_at: Optional[str] = None
job_id: Optional[str] = None
task_id: Optional[str] = None
parent_download_job_id: Optional[str] = None
shard_index: Optional[int] = None
shard_count: Optional[int] = None
def to_dict(self) -> dict:
return asdict(self)
@property
def is_distributed_parent(self) -> bool:
return (
self.parent_download_job_id is None
and self.shard_count is not None
and int(self.shard_count) > 1
)
@property
def is_shard(self) -> bool:
return bool(self.parent_download_job_id)
@dataclass
class JobEventRecord:
@@ -103,7 +103,12 @@ class DownloadJobRow(Base):
task_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("tasks.id"), nullable=True, index=True
)
worker_id: Mapped[str] = mapped_column(String(36), index=True)
worker_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
parent_download_job_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("download_jobs.id"), nullable=True, index=True
)
shard_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
shard_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
title: Mapped[str | None] = mapped_column(Text, nullable=True)
excel_key: Mapped[str] = mapped_column(Text, default="")
@@ -194,21 +194,25 @@ class MinioStorage:
Si el manifest falta, falla o queda vacío, hace fallback a list_keys
bajo el prefijo (cubre varias carpetas NIT planas).
En corridas distribuidas varios shards sobrescriben el manifest parcial:
siempre se fusiona con list_keys del prefijo.
"""
prefix = prefix or downloads_prefix(job_id)
files: List[dict] = []
by_key: dict[str, dict] = {}
try:
raw = self.get_bytes(manifest_key(job_id))
payload = json.loads(raw.decode("utf-8"))
for key in payload.get("files") or []:
entry = self._file_entry(str(key))
if entry is not None:
files.append(entry)
by_key[entry["key"]] = entry
except Exception:
files = []
pass
if not files:
files = self._list_pdf_entries(prefix)
for entry in self._list_pdf_entries(prefix):
by_key[entry["key"]] = entry
files = list(by_key.values())
return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)}
def _list_pdf_entries(self, prefix: str) -> List[dict]:
+403 -34
View File
@@ -73,6 +73,9 @@ class JobStore:
finished_at=_iso(row.finished_at),
job_id=row.job_id,
task_id=row.task_id,
parent_download_job_id=row.parent_download_job_id,
shard_index=row.shard_index,
shard_count=row.shard_count,
)
def _refresh_worker_status(self, worker: Worker) -> Worker:
@@ -129,7 +132,13 @@ class JobStore:
WorkerStatus.BUSY.value if row.current_job_id else WorkerStatus.ONLINE.value
)
session.flush()
return self._row_worker(row)
worker = self._row_worker(row)
# Reclaim fuera de la sesión del heartbeat para no bloquear
try:
self.reclaim_stale_running_jobs()
except Exception:
pass
return worker
def get_worker(self, worker_id: str) -> Optional[Worker]:
with session_scope() as session:
@@ -139,12 +148,23 @@ class JobStore:
return self._row_worker(row)
def list_workers(self) -> List[Worker]:
try:
self.reclaim_stale_running_jobs()
except Exception:
pass
with session_scope() as session:
rows = session.scalars(
select(DownloadWorkerRow).order_by(DownloadWorkerRow.id)
).all()
return [self._row_worker(r) for r in rows]
def list_online_worker_ids(self) -> List[str]:
return [
w.id
for w in self.list_workers()
if w.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value)
]
def is_worker_online(self, worker_id: str) -> bool:
worker = self.get_worker(worker_id)
if not worker:
@@ -162,31 +182,41 @@ class JobStore:
def create_job(
self,
worker_id: str,
worker_id: Optional[str],
excel_key: str,
original_filename: str = "",
title: str = "",
*,
job_id: Optional[str] = None,
task_id: Optional[str] = None,
downloads_prefix_override: Optional[str] = None,
parent_download_job_id: Optional[str] = None,
shard_index: Optional[int] = None,
shard_count: Optional[int] = None,
result: Optional[Dict[str, Any]] = None,
status: Optional[str] = None,
download_id: Optional[str] = None,
) -> Job:
import uuid as uuid_mod
download_id = str(uuid_mod.uuid4())
prefix = downloads_prefix(download_id)
download_id = download_id or str(uuid_mod.uuid4())
prefix = downloads_prefix_override or downloads_prefix(download_id)
with session_scope() as session:
row = DownloadJobRow(
id=download_id,
job_id=job_id,
task_id=task_id,
worker_id=worker_id,
status=JobStatus.PENDING.value,
parent_download_job_id=parent_download_job_id,
shard_index=shard_index,
shard_count=shard_count,
status=status or JobStatus.PENDING.value,
title=title or f"Descarga {original_filename}",
excel_key=excel_key,
downloads_prefix=prefix,
original_filename=original_filename,
cancel_requested=False,
result={},
result=result or {},
created_at=utcnow(),
)
session.add(row)
@@ -205,34 +235,57 @@ class JobStore:
limit: int = 100,
*,
job_id: Optional[str] = None,
parents_only: bool = True,
) -> List[Job]:
with session_scope() as session:
stmt = select(DownloadJobRow).order_by(DownloadJobRow.created_at.desc()).limit(limit)
stmt = select(DownloadJobRow).order_by(DownloadJobRow.created_at.desc())
if job_id:
stmt = (
select(DownloadJobRow)
.where(DownloadJobRow.job_id == job_id)
.order_by(DownloadJobRow.created_at.desc())
.limit(limit)
)
stmt = stmt.where(DownloadJobRow.job_id == job_id)
if parents_only:
stmt = stmt.where(DownloadJobRow.parent_download_job_id.is_(None))
stmt = stmt.limit(limit)
rows = session.scalars(stmt).all()
return [self._row_job(r) for r in rows]
def list_shards(self, parent_id: str) -> List[Job]:
with session_scope() as session:
rows = session.scalars(
select(DownloadJobRow)
.where(DownloadJobRow.parent_download_job_id == parent_id)
.order_by(DownloadJobRow.shard_index.asc())
).all()
return [self._row_job(r) for r in rows]
def claim_pending(self, worker_id: str) -> Optional[Job]:
try:
self.reclaim_stale_running_jobs()
except Exception:
pass
with session_scope() as session:
worker = session.get(DownloadWorkerRow, worker_id)
if not worker or worker.current_job_id:
return None
row = session.scalars(
# No claimar padres orquestadores (shard_count > 1 sin parent)
rows = session.scalars(
select(DownloadJobRow)
.where(
DownloadJobRow.worker_id == worker_id,
DownloadJobRow.status == JobStatus.PENDING.value,
)
.order_by(DownloadJobRow.created_at.asc())
.limit(1)
).first()
).all()
row = None
for candidate in rows:
is_parent = (
candidate.parent_download_job_id is None
and candidate.shard_count is not None
and int(candidate.shard_count) > 1
)
if is_parent:
continue
row = candidate
break
if not row:
return None
@@ -243,9 +296,19 @@ class JobStore:
worker.status = WorkerStatus.BUSY.value
worker.last_heartbeat = now
if row.task_id:
task = session.get(Task, row.task_id)
if task:
# Task del padre (shards heredan task_id del padre)
task_id = row.task_id
if row.parent_download_job_id:
parent = session.get(DownloadJobRow, row.parent_download_job_id)
if parent:
task_id = parent.task_id
if parent.status == JobStatus.PENDING.value:
parent.status = JobStatus.RUNNING.value
parent.started_at = parent.started_at or now
if task_id:
task = session.get(Task, task_id)
if task and task.status == JobStatus.PENDING.value:
task.status = JobStatus.RUNNING.value
task.started_at = now
task.updated_at = now
@@ -271,12 +334,44 @@ class JobStore:
row.status = JobStatus.CANCELLED.value
row.finished_at = now
self._sync_task(session, row, status=JobStatus.CANCELLED.value)
# Cancelar shards hijos si es padre distribuido
if row.parent_download_job_id is None and row.shard_count and int(row.shard_count) > 1:
children = session.scalars(
select(DownloadJobRow).where(
DownloadJobRow.parent_download_job_id == job_id
)
).all()
for child in children:
if child.status in (
JobStatus.COMPLETED.value,
JobStatus.FAILED.value,
JobStatus.CANCELLED.value,
):
continue
child.cancel_requested = True
if child.status == JobStatus.PENDING.value:
child.status = JobStatus.CANCELLED.value
child.finished_at = now
worker = session.get(DownloadWorkerRow, child.worker_id) if child.worker_id else None
if worker and worker.current_job_id == child.id:
worker.current_job_id = None
worker.status = WorkerStatus.ONLINE.value
session.flush()
return self._row_job(row)
def is_cancel_requested(self, job_id: str) -> bool:
job = self.get_job(job_id)
return bool(job and job.cancel_requested)
if not job:
return False
if job.cancel_requested:
return True
if job.parent_download_job_id:
parent = self.get_job(job.parent_download_job_id)
if parent and parent.cancel_requested:
return True
return False
def _sync_task(
self,
@@ -287,9 +382,15 @@ class JobStore:
result: Optional[Dict[str, Any]] = None,
error: Optional[str] = None,
) -> None:
if not row.task_id:
# Sync siempre contra el padre si es shard
target = row
if row.parent_download_job_id:
parent = session.get(DownloadJobRow, row.parent_download_job_id)
if parent:
target = parent
if not target.task_id:
return
task = session.get(Task, row.task_id)
task = session.get(Task, target.task_id)
if not task:
return
now = utcnow()
@@ -298,7 +399,7 @@ class JobStore:
if result is not None:
task.result = {
**dict(task.result or {}),
"download_job_id": row.id,
"download_job_id": target.id,
**result,
}
if error is not None:
@@ -319,6 +420,7 @@ class JobStore:
result: Optional[Dict[str, Any]] = None,
error: Optional[str] = None,
) -> Optional[Job]:
parent_id: Optional[str] = None
with session_scope() as session:
row = session.get(DownloadJobRow, job_id)
if not row:
@@ -340,22 +442,263 @@ class JobStore:
row.finished_at = row.finished_at or utcnow()
finished = True
self._sync_task(
session,
row,
status=new_status if status else None,
result=result,
error=error,
)
# No sync task directo en shards; se consolida en el padre
if not row.parent_download_job_id:
self._sync_task(
session,
row,
status=new_status if status else None,
result=result,
error=error,
)
if finished:
worker = session.get(DownloadWorkerRow, row.worker_id)
worker = session.get(DownloadWorkerRow, row.worker_id) if row.worker_id else None
if worker and worker.current_job_id == job_id:
worker.current_job_id = None
worker.status = WorkerStatus.ONLINE.value
parent_id = row.parent_download_job_id
session.flush()
return self._row_job(row)
job = self._row_job(row)
if parent_id:
self.refresh_parent_from_shards(parent_id)
refreshed = self.get_job(job_id)
return refreshed or job
return job
def refresh_parent_from_shards(self, parent_id: str) -> Optional[Job]:
with session_scope() as session:
parent = session.get(DownloadJobRow, parent_id)
if not parent:
return None
children = list(
session.scalars(
select(DownloadJobRow).where(
DownloadJobRow.parent_download_job_id == parent_id
)
).all()
)
if not children:
return self._row_job(parent)
statuses = [c.status for c in children]
terminal = {
JobStatus.COMPLETED.value,
JobStatus.FAILED.value,
JobStatus.CANCELLED.value,
}
any_active = any(
s in (JobStatus.PENDING.value, JobStatus.RUNNING.value) for s in statuses
)
all_terminal = all(s in terminal for s in statuses)
total = 0
exitosos = 0
fallidos = 0
uploaded = 0
for c in children:
r = dict(c.result or {})
total += int(r.get("total") or 0)
exitosos += int(r.get("exitosos") or 0)
fallidos += int(r.get("fallidos") or 0)
uploaded += int(r.get("uploaded_count") or 0)
# Progreso parcial mientras corre
if c.status == JobStatus.RUNNING.value:
total = max(total, int(r.get("total") or 0))
exitosos = max(exitosos, int(r.get("processed") or r.get("exitosos") or 0))
# Agregar progreso en vuelo (processed) de shards running
processed = 0
for c in children:
r = dict(c.result or {})
if c.status in terminal:
processed += int(r.get("exitosos") or 0) + int(r.get("fallidos") or 0)
else:
processed += int(r.get("processed") or 0)
shard_ids = [c.id for c in children]
parent_result = dict(parent.result or {})
parent_result.update(
{
"mode": "distributed",
"shard_ids": shard_ids,
"total": total or int(parent_result.get("total_rows") or 0),
"total_rows": int(parent_result.get("total_rows") or total or 0),
"exitosos": exitosos,
"fallidos": fallidos,
"processed": processed,
"uploaded_count": uploaded,
"downloads_prefix": parent.downloads_prefix,
"shards": [
{
"id": c.id,
"worker_id": c.worker_id,
"status": c.status,
"shard_index": c.shard_index,
"result": dict(c.result or {}),
"error": c.error,
}
for c in children
],
}
)
parent.result = parent_result
now = utcnow()
if any_active:
parent.status = JobStatus.RUNNING.value
parent.started_at = parent.started_at or now
parent.finished_at = None
parent.error = None
self._sync_task(
session,
parent,
status=JobStatus.RUNNING.value,
result=parent_result,
)
elif all_terminal:
any_failed = any(s == JobStatus.FAILED.value for s in statuses)
any_cancelled = any(s == JobStatus.CANCELLED.value for s in statuses)
all_cancelled = all(s == JobStatus.CANCELLED.value for s in statuses)
if all_cancelled:
parent.status = JobStatus.CANCELLED.value
elif any_failed:
parent.status = JobStatus.FAILED.value
parent.error = next(
(c.error for c in children if c.error),
"Uno o más shards fallaron",
)
elif any_cancelled and not any_failed:
# mezclado completed+cancelled → completed parcial
parent.status = JobStatus.COMPLETED.value
else:
parent.status = JobStatus.COMPLETED.value
parent.finished_at = parent.finished_at or now
self._sync_task(
session,
parent,
status=parent.status,
result=parent_result,
error=parent.error,
)
session.flush()
return self._row_job(parent)
def reclaim_stale_running_jobs(self) -> int:
"""Jobs running cuyo worker está offline → pending + reasignar a otro online."""
reclaimed = 0
with session_scope() as session:
now = utcnow()
cutoff = now - timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS)
workers = list(session.scalars(select(DownloadWorkerRow)).all())
offline_ids: set[str] = set()
online_free: List[DownloadWorkerRow] = []
for w in workers:
hb = w.last_heartbeat
if hb is None:
offline_ids.add(w.id)
continue
if hb.tzinfo is None:
hb = hb.replace(tzinfo=timezone.utc)
if hb < cutoff:
offline_ids.add(w.id)
if w.current_job_id:
w.current_job_id = None
w.status = WorkerStatus.OFFLINE.value
elif not w.current_job_id:
online_free.append(w)
if not offline_ids:
session.flush()
return 0
running = session.scalars(
select(DownloadJobRow).where(
DownloadJobRow.status == JobStatus.RUNNING.value,
DownloadJobRow.worker_id.in_(list(offline_ids)),
)
).all()
rr = 0
for row in running:
# Solo shards o jobs simples (no padres orquestadores)
is_parent = (
row.parent_download_job_id is None
and row.shard_count is not None
and int(row.shard_count) > 1
)
if is_parent:
continue
row.status = JobStatus.PENDING.value
row.started_at = None
row.cancel_requested = False
if online_free:
target = online_free[rr % len(online_free)]
rr += 1
row.worker_id = target.id
# Si no hay online free, queda pending con el mismo worker_id
# (se reasignará cuando haya free en el próximo reclaim)
reclaimed += 1
session.add(
DownloadJobEventRow(
download_job_id=row.id,
event_type="warning",
message=(
f"Worker offline; job reclaimed"
+ (f" → {row.worker_id}" if row.worker_id else "")
),
timestamp=now,
metadata_json={"reclaimed": True, "worker_id": row.worker_id},
)
)
if row.parent_download_job_id:
session.add(
DownloadJobEventRow(
download_job_id=row.parent_download_job_id,
event_type="warning",
message=f"Shard {row.shard_index} reclamado (worker offline)",
timestamp=now,
metadata_json={
"shard_id": row.id,
"worker_id": row.worker_id,
},
)
)
# Reasignar pending de workers offline a online free
if online_free:
pending_orphan = session.scalars(
select(DownloadJobRow).where(
DownloadJobRow.status == JobStatus.PENDING.value,
DownloadJobRow.worker_id.in_(list(offline_ids)),
)
).all()
for row in pending_orphan:
is_parent = (
row.parent_download_job_id is None
and row.shard_count is not None
and int(row.shard_count) > 1
)
if is_parent:
continue
target = online_free[rr % len(online_free)]
rr += 1
if row.worker_id != target.id:
row.worker_id = target.id
reclaimed += 1
session.flush()
# Refrescar padres afectados
# (se hace en siguiente patch; aquí no hace falta)
return reclaimed
def add_event(
self,
@@ -385,6 +728,28 @@ class JobStore:
)
session.add(ev)
session.flush()
# Mirror a padre si es shard
row = session.get(DownloadJobRow, job_id)
if row and row.parent_download_job_id:
session.add(
DownloadJobEventRow(
download_job_id=row.parent_download_job_id,
event_type=event_type,
message=message,
timestamp=ts,
radicado=radicado,
nit=nit,
idx=index,
total=total,
metadata_json={
**(metadata or {}),
"shard_id": job_id,
"shard_index": row.shard_index,
"worker_id": row.worker_id,
},
)
)
session.flush()
return JobEventRecord(
id=int(ev.id),
job_id=job_id,
@@ -429,9 +794,11 @@ class JobStore:
*,
parent_job_id: str,
download_job_id: str,
worker_id: str,
worker_id: Optional[str],
title: str,
original_filename: str,
worker_ids: Optional[List[str]] = None,
mode: str = "single",
) -> str:
"""Crea Task(type=downloads) y la asocia al download_job. Devuelve task_id."""
with session_scope() as session:
@@ -443,10 +810,12 @@ class JobStore:
title=title or f"Bot descarga ({original_filename})",
request={
"worker_id": worker_id,
"worker_ids": worker_ids or ([worker_id] if worker_id else []),
"original_filename": original_filename,
"download_job_id": download_job_id,
"mode": mode,
},
result={"download_job_id": download_job_id},
result={"download_job_id": download_job_id, "mode": mode},
created_at=now,
updated_at=now,
)
+1
View File
@@ -3,6 +3,7 @@ uvicorn[standard]>=0.27.0
python-multipart>=0.0.9
boto3>=1.34.0
httpx>=0.27.0
openpyxl>=3.1.0
pydantic>=2.0.0
sqlalchemy>=2.0.0
psycopg[binary]>=3.1.0
@@ -0,0 +1,33 @@
"""Tests de partición Excel para descarga distribuida."""
from __future__ import annotations
from domain.excel_split import parse_radicado_rows, rows_to_xlsx_bytes, split_rows
def test_split_rows_balanced():
rows = [(str(i), "NIT") for i in range(100)]
chunks = split_rows(rows, 2)
assert len(chunks) == 2
assert len(chunks[0]) == 50
assert len(chunks[1]) == 50
def test_split_rows_uneven():
rows = [(str(i), "N") for i in range(10)]
chunks = split_rows(rows, 3)
assert len(chunks) == 3
assert sum(len(c) for c in chunks) == 10
assert [len(c) for c in chunks] == [4, 3, 3]
def test_roundtrip_xlsx():
rows = [("RAD-1", "900"), ("RAD-2", "901")]
data = rows_to_xlsx_bytes(rows)
parsed = parse_radicado_rows(data, "shard.xlsx")
assert parsed == rows
def test_split_more_workers_than_rows():
rows = [("A", "1"), ("B", "2")]
chunks = split_rows(rows, 5)
assert len(chunks) == 2
+28
View File
@@ -0,0 +1,28 @@
FROM python:3.12-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PYTHONPATH=/app
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends curl \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt
COPY domain /app/domain
COPY infrastructure /app/infrastructure
COPY api /app/api
ENV BOT_API_HOST=0.0.0.0
ENV BOT_API_PORT=8210
EXPOSE 8210
HEALTHCHECK --interval=10s --timeout=5s --retries=5 --start-period=15s \
CMD curl -f http://127.0.0.1:8210/health || exit 1
CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8210"]
+600
View File
@@ -0,0 +1,600 @@
from __future__ import annotations
import os
from pathlib import Path
from typing import Any, Dict, List, Optional
from fastapi import Depends, FastAPI, File, Form, HTTPException, Query, UploadFile
from fastapi.responses import Response
from pydantic import BaseModel, Field
from domain.models import JobStatus
from infrastructure.auth import require_bearer
from infrastructure.storage_minio import MinioSettings, MinioStorage
from infrastructure.store import JobStore
store = JobStore()
minio = MinioStorage(MinioSettings.from_env())
POBLACION_APTA_URL = os.getenv("POBLACION_APTA_URL", "http://poblacion_apta:8003").rstrip("/")
app = FastAPI(title="BotDigitador Remote API", version="1.0.0")
class RegisterWorkerBody(BaseModel):
worker_id: str
hostname: str = ""
max_concurrent: int = 1
class HeartbeatBody(BaseModel):
worker_id: str
class EventBody(BaseModel):
event_type: str
message: str
radicado: Optional[str] = None
nit: Optional[str] = None
index: Optional[int] = None
total: Optional[int] = None
metadata: Dict[str, Any] = Field(default_factory=dict)
timestamp: Optional[str] = None
class PatchJobBody(BaseModel):
status: Optional[str] = None
result: Optional[Dict[str, Any]] = None
error: Optional[str] = None
def _media_type_for_key(key: str) -> str:
lower = key.lower()
if lower.endswith(".csv"):
return "text/csv; charset=utf-8"
if lower.endswith(".log") or lower.endswith(".txt"):
return "text/plain; charset=utf-8"
if lower.endswith(".json"):
return "application/json"
return "application/octet-stream"
@app.get("/health")
def health() -> dict:
return {"status": "ok", "service": "bot-digitador"}
@app.post("/workers/register")
def register_worker(body: RegisterWorkerBody, _: str = Depends(require_bearer)) -> dict:
worker = store.register_worker(
body.worker_id,
hostname=body.hostname,
max_concurrent=body.max_concurrent,
)
return worker.to_dict()
@app.post("/workers/heartbeat")
def heartbeat(body: HeartbeatBody, _: str = Depends(require_bearer)) -> dict:
worker = store.heartbeat(body.worker_id)
if not worker:
raise HTTPException(status_code=404, detail="Worker no registrado")
return worker.to_dict()
@app.get("/workers")
def list_workers(_: str = Depends(require_bearer)) -> List[dict]:
return [w.to_dict() for w in store.list_workers()]
@app.get("/workers/me/pending")
def claim_pending(
worker_id: str = Query(...),
_: str = Depends(require_bearer),
) -> dict:
job = store.claim_pending(worker_id)
if not job:
return {"job": None}
return {"job": job.to_dict()}
@app.post("/jobs")
async def create_job(
job_id: str = Form(...),
worker_id: str = Form(""),
worker_ids: str = Form(""),
title: str = Form(""),
file: Optional[UploadFile] = File(None),
poblacion_apta_job_id: str = Form(""),
subset: str = Form("aptos"),
_: str = Depends(require_bearer),
) -> dict:
"""Crea una corrida de digitación.
Exactamente uno de: file (CSV) o poblacion_apta_job_id.
- 1 worker → flujo clásico.
- N workers → padre + N shards CSV, mismo reports_prefix del padre.
"""
import json as _json
import httpx
from domain.csv_split import parse_digitador_rows, rows_to_csv_bytes, split_rows
from infrastructure.storage_minio import reports_prefix as mk_prefix
parent_job_id = job_id.strip()
if not parent_job_id:
raise HTTPException(status_code=400, detail="job_id (Job Desktop) obligatorio")
if not store.parent_job_exists(parent_job_id):
raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado")
pa_id = (poblacion_apta_job_id or "").strip()
data: bytes = b""
original = "input.csv"
content_type = "text/csv"
source_type = "upload"
file_bytes: bytes = b""
if file is not None:
file_bytes = await file.read()
if pa_id and file_bytes:
raise HTTPException(
status_code=400,
detail="Proporcione exactamente uno de: file o poblacion_apta_job_id",
)
if pa_id:
source_type = "poblacion_apta"
subset_norm = (subset or "aptos").strip().lower()
if subset_norm != "aptos":
raise HTTPException(
status_code=400,
detail="Solo se digitan aptos (PASO_BOT). Use subset=aptos.",
)
url = f"{POBLACION_APTA_URL}/jobs/{pa_id}/export.csv"
try:
async with httpx.AsyncClient(timeout=120.0) as client:
resp = await client.get(url, params={"subset": subset_norm})
except Exception as e:
raise HTTPException(
status_code=502,
detail=f"No se pudo contactar poblacion_apta: {e}",
) from e
if resp.status_code == 404:
raise HTTPException(status_code=404, detail="Job de población apta no encontrado")
if not resp.is_success:
raise HTTPException(
status_code=502,
detail=f"Error exportando CSV de población apta: HTTP {resp.status_code}",
)
data = resp.content
original = f"poblacion_apta_{subset_norm}_{pa_id[:8]}.csv"
content_type = "text/csv; charset=utf-8"
elif file_bytes:
data = file_bytes
original = (file.filename if file else None) or "input.csv"
content_type = (file.content_type if file else None) or "text/csv"
source_type = "upload"
else:
raise HTTPException(
status_code=400,
detail="Proporcione exactamente uno de: file o poblacion_apta_job_id",
)
if not data:
raise HTTPException(status_code=400, detail="CSV vacío")
title_clean = title.strip()
# Resolver workers
selected: List[str] = []
raw_ids = (worker_ids or "").strip()
if raw_ids:
try:
parsed = _json.loads(raw_ids)
if isinstance(parsed, list):
selected = [str(x).strip() for x in parsed if str(x).strip()]
elif isinstance(parsed, str) and parsed.strip():
selected = [parsed.strip()]
except _json.JSONDecodeError:
selected = [x.strip() for x in raw_ids.split(",") if x.strip()]
if not selected and worker_id.strip():
selected = [worker_id.strip()]
online = store.list_online_worker_ids()
online_set = set(online)
if not selected:
selected = list(online)
else:
missing = [w for w in selected if w not in online_set]
if missing:
raise HTTPException(
status_code=409,
detail=f"Workers no online: {', '.join(missing)}. Enciende el agent primero.",
)
if not selected:
raise HTTPException(
status_code=409,
detail="No hay bots online. Enciende al menos un agent.",
)
seen: set[str] = set()
workers_unique: List[str] = []
for w in selected:
if w not in seen:
seen.add(w)
workers_unique.append(w)
try:
rows = parse_digitador_rows(data, original)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
except Exception as e:
raise HTTPException(status_code=400, detail=f"No se pudo leer el CSV: {e}") from e
if not rows:
raise HTTPException(status_code=400, detail="El CSV no tiene filas para digitar")
# Normalizar a CSV canónico del bot (siempre pipe + headers snake_case)
canonical_bytes = rows_to_csv_bytes(rows)
n_workers = min(len(workers_unique), len(rows))
if n_workers <= 1:
only = workers_unique[0]
job = store.create_job(
worker_id=only,
csv_key="",
original_filename=original,
title=title_clean,
job_id=parent_job_id,
result={"source_type": source_type, "total_rows": len(rows), "total": len(rows)},
)
task_id = store.create_digitador_task(
parent_job_id=parent_job_id,
digitador_job_id=job.id,
worker_id=only,
title=title_clean or f"Bot digitador ({original})",
original_filename=original,
worker_ids=[only],
mode="single",
source_type=source_type,
)
try:
key = minio.upload_csv(job.id, canonical_bytes, original, content_type=content_type)
except Exception as e:
store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
job = store.update_csv_key(job.id, key)
assert job is not None
store.add_event(
job.id,
"run_started",
f"Job creado y asignado a {only}",
metadata={
"csv_key": key,
"worker_id": only,
"job_id": parent_job_id,
"task_id": task_id,
"mode": "single",
"total_rows": len(rows),
"source_type": source_type,
},
)
return job.to_dict()
workers_unique = workers_unique[:n_workers]
chunks = split_rows(rows, n_workers)
n = len(chunks)
workers_unique = workers_unique[:n]
parent = store.create_job(
worker_id=None,
csv_key="",
original_filename=original,
title=title_clean or f"Digitación distribuida ({original})",
job_id=parent_job_id,
shard_count=n,
status=JobStatus.PENDING.value,
result={
"mode": "distributed",
"shard_ids": [],
"total_rows": len(rows),
"total": len(rows),
"processed": 0,
"ok": 0,
"error": 0,
"timeout": 0,
"existentes": 0,
"source_type": source_type,
},
)
prefix = parent.reports_prefix or mk_prefix(parent.id)
task_id = store.create_digitador_task(
parent_job_id=parent_job_id,
digitador_job_id=parent.id,
worker_id=None,
title=title_clean or f"Bot digitador ×{n} ({original})",
original_filename=original,
worker_ids=workers_unique,
mode="distributed",
source_type=source_type,
)
try:
full_key = minio.upload_csv(parent.id, canonical_bytes, original, content_type=content_type)
except Exception as e:
store.patch_job(parent.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
store.update_csv_key(parent.id, full_key)
shard_ids: List[str] = []
for i, (chunk, wid) in enumerate(zip(chunks, workers_unique)):
shard_bytes = rows_to_csv_bytes(chunk)
shard = store.create_job(
worker_id=wid,
csv_key="",
original_filename=f"shard_{i + 1}_of_{n}.csv",
title=f"{title_clean or 'Digitador'} · shard {i + 1}/{n}",
job_id=parent_job_id,
task_id=task_id,
reports_prefix_override=prefix,
parent_digitador_job_id=parent.id,
shard_index=i,
shard_count=n,
result={"total": len(chunk), "processed": 0, "source_type": source_type},
)
try:
shard_key = minio.upload_csv(
shard.id,
shard_bytes,
f"shard_{i + 1}_of_{n}.csv",
content_type="text/csv; charset=utf-8",
)
except Exception as e:
store.patch_job(
parent.id,
status=JobStatus.FAILED.value,
error=f"MinIO shard upload failed: {e}",
)
raise HTTPException(
status_code=502, detail=f"Error subiendo shard a MinIO: {e}"
) from e
store.update_csv_key(shard.id, shard_key)
shard_ids.append(shard.id)
store.add_event(
parent.id,
"info",
f"Shard {i + 1}/{n} → {wid} ({len(chunk)} filas)",
metadata={"shard_id": shard.id, "worker_id": wid, "rows": len(chunk)},
)
store.patch_job(
parent.id,
result={
"mode": "distributed",
"shard_ids": shard_ids,
"total_rows": len(rows),
"total": len(rows),
"processed": 0,
"ok": 0,
"error": 0,
"timeout": 0,
"existentes": 0,
"reports_prefix": prefix,
"worker_ids": workers_unique,
"source_type": source_type,
},
)
store.add_event(
parent.id,
"run_started",
f"Corrida distribuida: {n} bots, {len(rows)} filas",
metadata={
"csv_key": full_key,
"worker_ids": workers_unique,
"job_id": parent_job_id,
"task_id": task_id,
"mode": "distributed",
"shard_ids": shard_ids,
"total_rows": len(rows),
"source_type": source_type,
},
)
job = store.get_job(parent.id)
assert job is not None
return job.to_dict()
@app.post("/internal/reclaim")
def reclaim_jobs(_: str = Depends(require_bearer)) -> dict:
n = store.reclaim_stale_running_jobs()
return {"reclaimed": n}
@app.get("/jobs")
def list_jobs(
limit: int = Query(100, ge=1, le=500),
job_id: Optional[str] = Query(None, description="Filtrar por Job Desktop"),
_: str = Depends(require_bearer),
) -> List[dict]:
return [j.to_dict() for j in store.list_jobs(limit=limit, job_id=job_id)]
@app.get("/jobs/{digitador_job_id}")
def get_job(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
job = store.get_job(digitador_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
if job.is_distributed_parent:
refreshed = store.refresh_parent_from_shards(digitador_job_id)
if refreshed:
job = refreshed
payload = job.to_dict()
if job.is_distributed_parent:
payload["shards"] = [s.to_dict() for s in store.list_shards(job.id)]
return payload
@app.get("/jobs/{digitador_job_id}/files")
def list_job_files(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
job = store.get_job(digitador_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
try:
return minio.list_report_files(digitador_job_id, prefix=job.reports_prefix)
except Exception as e:
raise HTTPException(status_code=502, detail=f"Error listando MinIO: {e}") from e
@app.get("/jobs/{digitador_job_id}/object")
def get_job_object(
digitador_job_id: str,
key: str = Query(..., min_length=1, description="Object key bajo reports_prefix"),
_: str = Depends(require_bearer),
):
"""Stream de un reporte MinIO (csv/log/json bajo el prefijo del job)."""
job = store.get_job(digitador_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
prefix = (job.reports_prefix or f"digitados/{digitador_job_id}/").lstrip("/")
if not prefix.endswith("/"):
prefix += "/"
normalized = key.lstrip("/")
if not normalized.startswith(prefix):
raise HTTPException(status_code=403, detail="Key fuera del prefijo del job")
lower = normalized.lower()
if not (lower.endswith(".csv") or lower.endswith(".log") or lower.endswith(".json")):
raise HTTPException(status_code=400, detail="Solo se permiten objetos csv/log/json")
if ".." in normalized.split("/"):
raise HTTPException(status_code=400, detail="Key no permitida")
try:
data = minio.get_bytes(normalized)
except Exception as e:
raise HTTPException(status_code=502, detail=f"Error leyendo MinIO: {e}") from e
filename = Path(normalized).name
return Response(
content=data,
media_type=_media_type_for_key(normalized),
headers={"Content-Disposition": f'inline; filename="{filename}"'},
)
@app.get("/jobs/{digitador_job_id}/events")
def get_events(
digitador_job_id: str,
since: int = Query(0, ge=0),
_: str = Depends(require_bearer),
) -> dict:
if not store.get_job(digitador_job_id):
raise HTTPException(status_code=404, detail="Job no encontrado")
events = store.list_events(digitador_job_id, since_id=since)
return {"events": [e.to_dict() for e in events]}
@app.post("/jobs/{digitador_job_id}/cancel")
def cancel_job(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
job = store.request_cancel(digitador_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
store.add_event(digitador_job_id, "warning", "Cancelación solicitada por el usuario")
return job.to_dict()
@app.get("/jobs/{digitador_job_id}/file")
def get_job_file(digitador_job_id: str, _: str = Depends(require_bearer)):
job = store.get_job(digitador_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
if not job.csv_key:
raise HTTPException(status_code=404, detail="CSV no disponible")
try:
data = minio.get_bytes(job.csv_key)
except Exception as e:
raise HTTPException(status_code=502, detail=f"Error leyendo MinIO: {e}") from e
filename = Path(job.csv_key).name
return Response(
content=data,
media_type="text/csv; charset=utf-8",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
@app.get("/jobs/{digitador_job_id}/file-url")
def get_job_file_url(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
job = store.get_job(digitador_job_id)
if not job or not job.csv_key:
raise HTTPException(status_code=404, detail="Job/archivo no encontrado")
try:
url = minio.presigned_get(job.csv_key)
except Exception as e:
raise HTTPException(status_code=502, detail=str(e)) from e
return {"url": url, "csv_key": job.csv_key}
@app.post("/jobs/{digitador_job_id}/events")
def post_event(
digitador_job_id: str,
body: EventBody,
_: str = Depends(require_bearer),
) -> dict:
job = store.get_job(digitador_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
event = store.add_event(
digitador_job_id,
body.event_type,
body.message,
radicado=body.radicado,
nit=body.nit,
index=body.index,
total=body.total,
metadata=body.metadata,
timestamp=body.timestamp,
)
if body.index is not None and body.total is not None:
result = dict(job.result or {})
result["processed"] = body.index
result["total"] = body.total
if body.nit:
result["nit_actual"] = body.nit
store.patch_job(digitador_job_id, result=result)
return event.to_dict()
@app.patch("/jobs/{digitador_job_id}")
def patch_job(
digitador_job_id: str,
body: PatchJobBody,
_: str = Depends(require_bearer),
) -> dict:
if body.status and body.status not in {s.value for s in JobStatus}:
raise HTTPException(status_code=400, detail="status inválido")
job = store.patch_job(
digitador_job_id,
status=body.status,
result=body.result,
error=body.error,
)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
return job.to_dict()
@app.get("/jobs/{digitador_job_id}/cancel-flag")
def cancel_flag(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
job = store.get_job(digitador_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
return {
"cancel_requested": store.is_cancel_requested(digitador_job_id),
"status": job.status,
}
+157
View File
@@ -0,0 +1,157 @@
"""Parseo y partición de CSV de filas a digitar (bot digitador)."""
from __future__ import annotations
import csv
import io
from typing import Any
REQUIRED_BOT_COLUMNS = (
"radicado",
"documento",
"ingreso",
"total_cargos",
"autorizacion",
"tipo_documento",
"tipo_servicio",
"copago",
"cuota_moderadora",
)
IT_TO_BOT = {
"RADICADO": "radicado",
"NUMERO": "documento",
"FECHA": "ingreso",
"BRUTO": "total_cargos",
"AUTORIZACION": "autorizacion",
"TIPO_ID": "tipo_documento",
"SERVICIO": "tipo_servicio",
"COPAGO": "copago",
"MODERADORA": "cuota_moderadora",
}
def _normalize_header(name: str) -> str:
raw = str(name or "").strip()
if not raw:
return ""
upper = raw.upper()
if upper in IT_TO_BOT:
return IT_TO_BOT[upper]
lower = raw.lower().strip()
# Aceptar snake_case ya normalizado
if lower in REQUIRED_BOT_COLUMNS:
return lower
# Alias comunes
aliases = {
"número": "documento",
"numero": "documento",
"fecha": "ingreso",
"bruto": "total_cargos",
"tipo_id": "tipo_documento",
"servicio": "tipo_servicio",
"moderadora": "cuota_moderadora",
}
return aliases.get(lower, lower)
def _decode_text(data: bytes) -> str:
for enc in ("utf-8-sig", "utf-8", "latin-1", "cp1252"):
try:
return data.decode(enc)
except UnicodeDecodeError:
continue
return data.decode("utf-8", errors="replace")
def _detect_delimiter(sample: str) -> str:
try:
dialect = csv.Sniffer().sniff(sample, delimiters="|,;\t")
return dialect.delimiter
except csv.Error:
return "|" if sample.count("|") >= sample.count(",") else ","
def _validate_columns(headers: list[str]) -> None:
present = set(headers)
missing = [c for c in REQUIRED_BOT_COLUMNS if c not in present]
if missing:
raise ValueError(f"Columnas faltantes: {', '.join(missing)}")
def parse_digitador_rows(data: bytes, filename: str = "input.csv") -> list[dict[str, str]]:
"""Devuelve filas normalizadas con headers del bot (snake_case)."""
_ = filename # reservado por firma / diagnóstico futuro
text = _decode_text(data)
if not text.strip():
return []
delim = _detect_delimiter(text[:4096])
reader = csv.DictReader(io.StringIO(text), delimiter=delim)
if not reader.fieldnames:
raise ValueError("CSV sin encabezados")
rename_map: dict[str, str] = {}
normalized_headers: list[str] = []
for raw in reader.fieldnames:
norm = _normalize_header(raw)
rename_map[raw] = norm
if norm and norm not in normalized_headers:
normalized_headers.append(norm)
_validate_columns(normalized_headers)
out: list[dict[str, str]] = []
for row in reader:
if not row:
continue
mapped: dict[str, str] = {c: "" for c in REQUIRED_BOT_COLUMNS}
empty = True
for raw_key, value in row.items():
if raw_key is None:
continue
col = rename_map.get(raw_key) or _normalize_header(raw_key)
if col not in REQUIRED_BOT_COLUMNS:
continue
cell = "" if value is None else str(value).strip()
mapped[col] = cell
if cell:
empty = False
if empty:
continue
# Filtrar filas que parecen ser un segundo header
if mapped.get("radicado", "").lower() in ("radicado", "radicado"):
if mapped.get("documento", "").lower() in ("documento", "numero", "número", ""):
continue
out.append(mapped)
return out
def split_rows(rows: list[dict[str, Any]], n: int) -> list[list[dict[str, Any]]]:
"""Parte filas en n trozos lo más equilibrados posible (chunks contiguos)."""
if n <= 1 or not rows:
return [rows]
n = min(n, len(rows))
size, rem = divmod(len(rows), n)
chunks: list[list[dict[str, Any]]] = []
idx = 0
for i in range(n):
take = size + (1 if i < rem else 0)
chunks.append(rows[idx : idx + take])
idx += take
return [c for c in chunks if c]
def rows_to_csv_bytes(rows: list[dict[str, Any]]) -> bytes:
"""Serializa filas a CSV pipe-delimited con headers del bot (utf-8-sig)."""
buf = io.StringIO()
writer = csv.DictWriter(
buf,
fieldnames=list(REQUIRED_BOT_COLUMNS),
delimiter="|",
lineterminator="\n",
extrasaction="ignore",
)
writer.writeheader()
for row in rows:
writer.writerow({c: str(row.get(c, "") or "") for c in REQUIRED_BOT_COLUMNS})
return buf.getvalue().encode("utf-8-sig")
+91
View File
@@ -0,0 +1,91 @@
from __future__ import annotations
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from enum import Enum
from typing import Any, Dict, Optional
def utc_now() -> str:
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
class WorkerStatus(str, Enum):
ONLINE = "online"
OFFLINE = "offline"
BUSY = "busy"
class JobStatus(str, Enum):
PENDING = "pending"
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
CANCELLED = "cancelled"
@dataclass
class Worker:
id: str
hostname: str = ""
max_concurrent: int = 1
status: str = WorkerStatus.ONLINE.value
last_heartbeat: str = field(default_factory=utc_now)
current_job_id: Optional[str] = None
def to_dict(self) -> dict:
return asdict(self)
@dataclass
class Job:
id: str
worker_id: Optional[str]
status: str = JobStatus.PENDING.value
title: str = ""
csv_key: str = ""
reports_prefix: str = ""
original_filename: str = ""
cancel_requested: bool = False
result: Dict[str, Any] = field(default_factory=dict)
error: Optional[str] = None
created_at: str = field(default_factory=utc_now)
started_at: Optional[str] = None
finished_at: Optional[str] = None
job_id: Optional[str] = None
task_id: Optional[str] = None
parent_digitador_job_id: Optional[str] = None
shard_index: Optional[int] = None
shard_count: Optional[int] = None
def to_dict(self) -> dict:
return asdict(self)
@property
def is_distributed_parent(self) -> bool:
return (
self.parent_digitador_job_id is None
and self.shard_count is not None
and int(self.shard_count) > 1
)
@property
def is_shard(self) -> bool:
return bool(self.parent_digitador_job_id)
@dataclass
class JobEventRecord:
id: int
job_id: str
event_type: str
message: str
timestamp: str
radicado: Optional[str] = None
nit: Optional[str] = None
index: Optional[int] = None
total: Optional[int] = None
metadata: Dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict:
return asdict(self)
@@ -0,0 +1,20 @@
from __future__ import annotations
import os
from typing import Optional
from fastapi import Header, HTTPException
def get_api_token() -> str:
return os.getenv("BOT_API_TOKEN", "dev-token-change-me")
def require_bearer(authorization: Optional[str] = Header(default=None)) -> str:
expected = get_api_token()
if not authorization or not authorization.lower().startswith("bearer "):
raise HTTPException(status_code=401, detail="Authorization Bearer requerido")
token = authorization.split(" ", 1)[1].strip()
if token != expected:
raise HTTPException(status_code=401, detail="Token inválido")
return token
@@ -0,0 +1,164 @@
"""Persistencia Postgres del servicio bot-digitador (DB compartida IPSExtractor)."""
from __future__ import annotations
import os
import uuid
from contextlib import contextmanager
from datetime import datetime, timezone
from typing import Any, Iterator
from sqlalchemy import (
JSON,
Boolean,
DateTime,
ForeignKey,
Integer,
String,
Text,
create_engine,
text,
)
from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, sessionmaker
DATABASE_URL = os.getenv(
"DATABASE_URL",
"postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor",
)
engine = create_engine(DATABASE_URL, pool_pre_ping=True)
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
class Base(DeclarativeBase):
pass
def utcnow() -> datetime:
return datetime.now(timezone.utc)
def new_id() -> str:
return str(uuid.uuid4())
def _iso(dt: datetime | None) -> str | None:
if dt is None:
return None
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt.astimezone().isoformat(timespec="seconds")
class Job(Base):
"""Contenedor Desktop (lectura / FK)."""
__tablename__ = "jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
title: Mapped[str | None] = mapped_column(Text, nullable=True)
job_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
class Task(Base):
__tablename__ = "tasks"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
job_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("jobs.id"), nullable=True, index=True
)
type: Mapped[str] = mapped_column(String(64), index=True)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
title: Mapped[str | None] = mapped_column(Text, nullable=True)
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
owner_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class DigitadorWorkerRow(Base):
__tablename__ = "digitador_workers"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
hostname: Mapped[str] = mapped_column(Text, default="")
max_concurrent: Mapped[int] = mapped_column(Integer, default=1)
status: Mapped[str] = mapped_column(String(32), index=True)
last_heartbeat: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
current_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
class DigitadorJobRow(Base):
__tablename__ = "digitador_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
job_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("jobs.id"), nullable=True, index=True
)
task_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("tasks.id"), nullable=True, index=True
)
worker_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
parent_digitador_job_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("digitador_jobs.id"), nullable=True, index=True
)
shard_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
shard_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
title: Mapped[str | None] = mapped_column(Text, nullable=True)
csv_key: Mapped[str] = mapped_column(Text, default="")
reports_prefix: Mapped[str] = mapped_column(Text, default="")
original_filename: Mapped[str] = mapped_column(Text, default="")
cancel_requested: Mapped[bool] = mapped_column(Boolean, default=False)
result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class DigitadorJobEventRow(Base):
__tablename__ = "digitador_job_events"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
digitador_job_id: Mapped[str] = mapped_column(
String(36), ForeignKey("digitador_jobs.id"), index=True
)
event_type: Mapped[str] = mapped_column(String(64))
message: Mapped[str] = mapped_column(Text)
timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
radicado: Mapped[str | None] = mapped_column(String(255), nullable=True)
nit: Mapped[str | None] = mapped_column(String(64), nullable=True)
idx: Mapped[int | None] = mapped_column(Integer, nullable=True)
total: Mapped[int | None] = mapped_column(Integer, nullable=True)
metadata_json: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
def init_db() -> None:
if DATABASE_URL.startswith("postgresql"):
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:
connection.execute(text("SELECT pg_advisory_lock(74512005)"))
try:
Base.metadata.create_all(bind=connection)
finally:
connection.execute(text("SELECT pg_advisory_unlock(74512005)"))
return
Base.metadata.create_all(bind=engine)
@contextmanager
def session_scope() -> Iterator[Session]:
session = SessionLocal()
try:
yield session
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()
@@ -0,0 +1,230 @@
from __future__ import annotations
import json
import os
from dataclasses import dataclass
from pathlib import Path
from typing import BinaryIO, List, Optional
@dataclass(frozen=True)
class MinioSettings:
endpoint: str = "https://s3.builker.xyz"
bucket: str = "pys-cuentas-medicas"
access_key: str = ""
secret_key: str = ""
region: str = "us-east-1"
anonymous: bool = True
@classmethod
def from_env(cls) -> "MinioSettings":
access_key = (os.getenv("MINIO_ACCESS_KEY") or "").strip()
secret_key = (os.getenv("MINIO_SECRET_KEY") or "").strip()
anon_flag = (os.getenv("MINIO_ANONYMOUS") or "").strip().lower()
if anon_flag in ("0", "false", "no"):
anonymous = False
elif anon_flag in ("1", "true", "yes"):
anonymous = True
else:
anonymous = not (access_key and secret_key)
return cls(
endpoint=os.getenv("MINIO_ENDPOINT", "https://s3.builker.xyz").rstrip("/"),
bucket=os.getenv("MINIO_BUCKET", "pys-cuentas-medicas"),
access_key=access_key,
secret_key=secret_key,
region=os.getenv("MINIO_REGION", "us-east-1"),
anonymous=anonymous,
)
def csv_key(job_id: str, filename: str = "input.csv") -> str:
return f"uploads/{job_id}/{filename}"
def reports_prefix(job_id: str) -> str:
return f"digitados/{job_id}/"
def meta_key(job_id: str) -> str:
return f"uploads/{job_id}/meta.json"
def manifest_key(job_id: str) -> str:
return f"digitados/{job_id}/_manifest.json"
_REPORT_NAMES = {
"resultados.csv",
"omitidos.csv",
"ejecucion_bot.log",
}
def _s3_client(settings: MinioSettings):
import boto3
from botocore import UNSIGNED
from botocore.client import Config
s3_cfg = {
"addressing_style": "path",
"request_checksum_calculation": "when_required",
"response_checksum_validation": "when_required",
}
if settings.anonymous:
return boto3.client(
"s3",
endpoint_url=settings.endpoint,
region_name=settings.region,
config=Config(signature_version=UNSIGNED, s3=s3_cfg),
)
return boto3.client(
"s3",
endpoint_url=settings.endpoint,
aws_access_key_id=settings.access_key,
aws_secret_access_key=settings.secret_key,
region_name=settings.region,
config=Config(signature_version="s3v4", s3=s3_cfg),
)
class MinioStorage:
"""Cliente S3-compatible para MinIO (uploads/digitados por job_id)."""
def __init__(self, settings: Optional[MinioSettings] = None):
self.settings = settings or MinioSettings.from_env()
self._client = None
@property
def client(self):
if self._client is None:
self._client = _s3_client(self.settings)
return self._client
@property
def bucket(self) -> str:
return self.settings.bucket
def put_bytes(self, key: str, data: bytes, content_type: str = "application/octet-stream") -> str:
self.client.put_object(
Bucket=self.bucket,
Key=key,
Body=data,
ContentType=content_type,
)
return key
def put_file(self, key: str, path: str | Path, content_type: str = "application/octet-stream") -> str:
path = Path(path)
return self.put_bytes(key, path.read_bytes(), content_type=content_type)
def put_stream(self, key: str, stream: BinaryIO, content_type: str = "application/octet-stream") -> str:
data = stream.read()
return self.put_bytes(key, data, content_type=content_type)
def get_bytes(self, key: str) -> bytes:
obj = self.client.get_object(Bucket=self.bucket, Key=key)
return obj["Body"].read()
def download_file(self, key: str, dest: str | Path) -> Path:
dest = Path(dest)
dest.parent.mkdir(parents=True, exist_ok=True)
self.client.download_file(self.bucket, key, str(dest))
return dest
def exists(self, key: str) -> bool:
try:
self.client.head_object(Bucket=self.bucket, Key=key)
return True
except Exception:
return False
def list_keys(self, prefix: str) -> List[str]:
keys: List[str] = []
token = None
while True:
kwargs = {"Bucket": self.bucket, "Prefix": prefix}
if token:
kwargs["ContinuationToken"] = token
resp = self.client.list_objects_v2(**kwargs)
for item in resp.get("Contents") or []:
keys.append(item["Key"])
if not resp.get("IsTruncated"):
break
token = resp.get("NextContinuationToken")
return keys
def presigned_get(self, key: str, expires_in: int = 3600) -> str:
if self.settings.anonymous:
return f"{self.settings.endpoint}/{self.bucket}/{key}"
return self.client.generate_presigned_url(
"get_object",
Params={"Bucket": self.bucket, "Key": key},
ExpiresIn=expires_in,
)
def upload_csv(
self,
job_id: str,
data: bytes,
original_filename: str,
content_type: str = "text/csv",
) -> str:
ext = Path(original_filename).suffix.lower() or ".csv"
canonical = "input.csv" if ext in (".csv", ".txt") else "input.csv"
key = csv_key(job_id, canonical)
self.put_bytes(key, data, content_type=content_type)
meta = {
"original_filename": original_filename,
"size": len(data),
"canonical": canonical,
}
self.put_bytes(meta_key(job_id), json.dumps(meta).encode("utf-8"), "application/json")
return key
def write_manifest(self, job_id: str, keys: List[str]) -> str:
key = manifest_key(job_id)
payload = {"job_id": job_id, "files": keys, "count": len(keys)}
self.put_bytes(key, json.dumps(payload, ensure_ascii=False).encode("utf-8"), "application/json")
return key
def list_report_files(self, job_id: str, prefix: str | None = None) -> dict:
"""Lista reportes de digitación (RESULTADOS/OMITIDOS/log + list_keys).
Fusiona manifest con listado del prefijo (cubriendo shards distribuidos).
"""
prefix = prefix or reports_prefix(job_id)
by_key: dict[str, dict] = {}
try:
raw = self.get_bytes(manifest_key(job_id))
payload = json.loads(raw.decode("utf-8"))
for key in payload.get("files") or []:
entry = self._file_entry(str(key))
if entry is not None:
by_key[entry["key"]] = entry
except Exception:
pass
for entry in self._list_report_entries(prefix):
by_key[entry["key"]] = entry
files = list(by_key.values())
return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)}
def _list_report_entries(self, prefix: str) -> List[dict]:
files: List[dict] = []
for key in self.list_keys(prefix):
entry = self._file_entry(key)
if entry is not None:
files.append(entry)
return files
@staticmethod
def _file_entry(key: str) -> dict | None:
name = Path(key).name
lower = name.lower()
if lower.startswith("_manifest") and lower.endswith(".json"):
return None
# RESULTADOS.csv, OMITIDOS.csv, ejecucion_bot.log (+ variantes _shard_N)
if lower in _REPORT_NAMES or lower.endswith(".csv") or lower.endswith(".log"):
return {"key": key, "name": name}
return None
@@ -0,0 +1,833 @@
"""Repositorio sobre PostgreSQL (tablas digitador_* + sync de tasks)."""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional
from sqlalchemy import select
from domain.models import (
Job,
JobEventRecord,
JobStatus,
Worker,
WorkerStatus,
utc_now,
)
from infrastructure.database import (
DigitadorJobEventRow,
DigitadorJobRow,
DigitadorWorkerRow,
Task,
_iso,
init_db,
session_scope,
utcnow,
)
from infrastructure.storage_minio import reports_prefix
HEARTBEAT_OFFLINE_SECONDS = 45
def _parse_iso(value: str | None) -> datetime | None:
if not value:
return None
dt = datetime.fromisoformat(value)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt
class JobStore:
"""API de persistencia compatible con el patrón bot_descarga."""
def __init__(self, db_path: str | None = None):
# db_path ignorado (compat); usa DATABASE_URL.
init_db()
def _row_worker(self, row: DigitadorWorkerRow) -> Worker:
worker = Worker(
id=row.id,
hostname=row.hostname or "",
max_concurrent=int(row.max_concurrent or 1),
status=row.status,
last_heartbeat=_iso(row.last_heartbeat) or utc_now(),
current_job_id=row.current_job_id,
)
return self._refresh_worker_status(worker)
def _row_job(self, row: DigitadorJobRow) -> Job:
return Job(
id=row.id,
worker_id=row.worker_id,
status=row.status,
title=row.title or "",
csv_key=row.csv_key or "",
reports_prefix=row.reports_prefix or "",
original_filename=row.original_filename or "",
cancel_requested=bool(row.cancel_requested),
result=dict(row.result or {}),
error=row.error,
created_at=_iso(row.created_at) or utc_now(),
started_at=_iso(row.started_at),
finished_at=_iso(row.finished_at),
job_id=row.job_id,
task_id=row.task_id,
parent_digitador_job_id=row.parent_digitador_job_id,
shard_index=row.shard_index,
shard_count=row.shard_count,
)
def _refresh_worker_status(self, worker: Worker) -> Worker:
try:
hb = datetime.fromisoformat(worker.last_heartbeat)
if hb.tzinfo is None:
hb = hb.replace(tzinfo=timezone.utc)
age = datetime.now(timezone.utc) - hb.astimezone(timezone.utc)
if age > timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS):
worker.status = WorkerStatus.OFFLINE.value
elif worker.current_job_id:
worker.status = WorkerStatus.BUSY.value
else:
worker.status = WorkerStatus.ONLINE.value
except Exception:
worker.status = WorkerStatus.OFFLINE.value
return worker
def register_worker(
self,
worker_id: str,
hostname: str = "",
max_concurrent: int = 1,
) -> Worker:
with session_scope() as session:
row = session.get(DigitadorWorkerRow, worker_id)
now = utcnow()
if row:
row.hostname = hostname
row.max_concurrent = max_concurrent
row.status = WorkerStatus.ONLINE.value
row.last_heartbeat = now
row.current_job_id = None
else:
row = DigitadorWorkerRow(
id=worker_id,
hostname=hostname,
max_concurrent=max_concurrent,
status=WorkerStatus.ONLINE.value,
last_heartbeat=now,
current_job_id=None,
)
session.add(row)
session.flush()
return self._row_worker(row)
def heartbeat(self, worker_id: str) -> Optional[Worker]:
with session_scope() as session:
row = session.get(DigitadorWorkerRow, worker_id)
if not row:
return None
row.last_heartbeat = utcnow()
row.status = (
WorkerStatus.BUSY.value if row.current_job_id else WorkerStatus.ONLINE.value
)
session.flush()
worker = self._row_worker(row)
try:
self.reclaim_stale_running_jobs()
except Exception:
pass
return worker
def get_worker(self, worker_id: str) -> Optional[Worker]:
with session_scope() as session:
row = session.get(DigitadorWorkerRow, worker_id)
if not row:
return None
return self._row_worker(row)
def list_workers(self) -> List[Worker]:
try:
self.reclaim_stale_running_jobs()
except Exception:
pass
with session_scope() as session:
rows = session.scalars(
select(DigitadorWorkerRow).order_by(DigitadorWorkerRow.id)
).all()
return [self._row_worker(r) for r in rows]
def list_online_worker_ids(self) -> List[str]:
return [
w.id
for w in self.list_workers()
if w.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value)
]
def is_worker_online(self, worker_id: str) -> bool:
worker = self.get_worker(worker_id)
if not worker:
return False
return worker.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value)
def update_csv_key(self, job_id: str, csv_key: str) -> Optional[Job]:
with session_scope() as session:
row = session.get(DigitadorJobRow, job_id)
if not row:
return None
row.csv_key = csv_key
session.flush()
return self._row_job(row)
def create_job(
self,
worker_id: Optional[str],
csv_key: str,
original_filename: str = "",
title: str = "",
*,
job_id: Optional[str] = None,
task_id: Optional[str] = None,
reports_prefix_override: Optional[str] = None,
parent_digitador_job_id: Optional[str] = None,
shard_index: Optional[int] = None,
shard_count: Optional[int] = None,
result: Optional[Dict[str, Any]] = None,
status: Optional[str] = None,
digitador_id: Optional[str] = None,
) -> Job:
import uuid as uuid_mod
digitador_id = digitador_id or str(uuid_mod.uuid4())
prefix = reports_prefix_override or reports_prefix(digitador_id)
with session_scope() as session:
row = DigitadorJobRow(
id=digitador_id,
job_id=job_id,
task_id=task_id,
worker_id=worker_id,
parent_digitador_job_id=parent_digitador_job_id,
shard_index=shard_index,
shard_count=shard_count,
status=status or JobStatus.PENDING.value,
title=title or f"Digitador {original_filename}",
csv_key=csv_key,
reports_prefix=prefix,
original_filename=original_filename,
cancel_requested=False,
result=result or {},
created_at=utcnow(),
)
session.add(row)
session.flush()
return self._row_job(row)
def get_job(self, job_id: str) -> Optional[Job]:
with session_scope() as session:
row = session.get(DigitadorJobRow, job_id)
if not row:
return None
return self._row_job(row)
def list_jobs(
self,
limit: int = 100,
*,
job_id: Optional[str] = None,
parents_only: bool = True,
) -> List[Job]:
with session_scope() as session:
stmt = select(DigitadorJobRow).order_by(DigitadorJobRow.created_at.desc())
if job_id:
stmt = stmt.where(DigitadorJobRow.job_id == job_id)
if parents_only:
stmt = stmt.where(DigitadorJobRow.parent_digitador_job_id.is_(None))
stmt = stmt.limit(limit)
rows = session.scalars(stmt).all()
return [self._row_job(r) for r in rows]
def list_shards(self, parent_id: str) -> List[Job]:
with session_scope() as session:
rows = session.scalars(
select(DigitadorJobRow)
.where(DigitadorJobRow.parent_digitador_job_id == parent_id)
.order_by(DigitadorJobRow.shard_index.asc())
).all()
return [self._row_job(r) for r in rows]
def claim_pending(self, worker_id: str) -> Optional[Job]:
try:
self.reclaim_stale_running_jobs()
except Exception:
pass
with session_scope() as session:
worker = session.get(DigitadorWorkerRow, worker_id)
if not worker or worker.current_job_id:
return None
rows = session.scalars(
select(DigitadorJobRow)
.where(
DigitadorJobRow.worker_id == worker_id,
DigitadorJobRow.status == JobStatus.PENDING.value,
)
.order_by(DigitadorJobRow.created_at.asc())
).all()
row = None
for candidate in rows:
is_parent = (
candidate.parent_digitador_job_id is None
and candidate.shard_count is not None
and int(candidate.shard_count) > 1
)
if is_parent:
continue
row = candidate
break
if not row:
return None
now = utcnow()
row.status = JobStatus.RUNNING.value
row.started_at = now
worker.current_job_id = row.id
worker.status = WorkerStatus.BUSY.value
worker.last_heartbeat = now
task_id = row.task_id
if row.parent_digitador_job_id:
parent = session.get(DigitadorJobRow, row.parent_digitador_job_id)
if parent:
task_id = parent.task_id
if parent.status == JobStatus.PENDING.value:
parent.status = JobStatus.RUNNING.value
parent.started_at = parent.started_at or now
if task_id:
task = session.get(Task, task_id)
if task and task.status == JobStatus.PENDING.value:
task.status = JobStatus.RUNNING.value
task.started_at = now
task.updated_at = now
session.flush()
return self._row_job(row)
def request_cancel(self, job_id: str) -> Optional[Job]:
with session_scope() as session:
row = session.get(DigitadorJobRow, job_id)
if not row:
return None
if row.status in (
JobStatus.COMPLETED.value,
JobStatus.FAILED.value,
JobStatus.CANCELLED.value,
):
return self._row_job(row)
now = utcnow()
row.cancel_requested = True
if row.status == JobStatus.PENDING.value:
row.status = JobStatus.CANCELLED.value
row.finished_at = now
self._sync_task(session, row, status=JobStatus.CANCELLED.value)
if row.parent_digitador_job_id is None and row.shard_count and int(row.shard_count) > 1:
children = session.scalars(
select(DigitadorJobRow).where(
DigitadorJobRow.parent_digitador_job_id == job_id
)
).all()
for child in children:
if child.status in (
JobStatus.COMPLETED.value,
JobStatus.FAILED.value,
JobStatus.CANCELLED.value,
):
continue
child.cancel_requested = True
if child.status == JobStatus.PENDING.value:
child.status = JobStatus.CANCELLED.value
child.finished_at = now
worker = (
session.get(DigitadorWorkerRow, child.worker_id)
if child.worker_id
else None
)
if worker and worker.current_job_id == child.id:
worker.current_job_id = None
worker.status = WorkerStatus.ONLINE.value
session.flush()
return self._row_job(row)
def is_cancel_requested(self, job_id: str) -> bool:
job = self.get_job(job_id)
if not job:
return False
if job.cancel_requested:
return True
if job.parent_digitador_job_id:
parent = self.get_job(job.parent_digitador_job_id)
if parent and parent.cancel_requested:
return True
return False
def _sync_task(
self,
session,
row: DigitadorJobRow,
*,
status: Optional[str] = None,
result: Optional[Dict[str, Any]] = None,
error: Optional[str] = None,
) -> None:
target = row
if row.parent_digitador_job_id:
parent = session.get(DigitadorJobRow, row.parent_digitador_job_id)
if parent:
target = parent
if not target.task_id:
return
task = session.get(Task, target.task_id)
if not task:
return
now = utcnow()
if status:
task.status = status
if result is not None:
task.result = {
**dict(task.result or {}),
"digitador_job_id": target.id,
**result,
}
if error is not None:
task.error = error
task.updated_at = now
if status in (
JobStatus.COMPLETED.value,
JobStatus.FAILED.value,
JobStatus.CANCELLED.value,
):
task.finished_at = now
def patch_job(
self,
job_id: str,
*,
status: Optional[str] = None,
result: Optional[Dict[str, Any]] = None,
error: Optional[str] = None,
) -> Optional[Job]:
parent_id: Optional[str] = None
with session_scope() as session:
row = session.get(DigitadorJobRow, job_id)
if not row:
return None
new_status = status or row.status
if result is not None:
row.result = result
if error is not None:
row.error = error
row.status = new_status
finished = False
if new_status in (
JobStatus.COMPLETED.value,
JobStatus.FAILED.value,
JobStatus.CANCELLED.value,
):
row.finished_at = row.finished_at or utcnow()
finished = True
if not row.parent_digitador_job_id:
self._sync_task(
session,
row,
status=new_status if status else None,
result=result,
error=error,
)
if finished:
worker = session.get(DigitadorWorkerRow, row.worker_id) if row.worker_id else None
if worker and worker.current_job_id == job_id:
worker.current_job_id = None
worker.status = WorkerStatus.ONLINE.value
parent_id = row.parent_digitador_job_id
session.flush()
job = self._row_job(row)
if parent_id:
self.refresh_parent_from_shards(parent_id)
refreshed = self.get_job(job_id)
return refreshed or job
return job
def refresh_parent_from_shards(self, parent_id: str) -> Optional[Job]:
with session_scope() as session:
parent = session.get(DigitadorJobRow, parent_id)
if not parent:
return None
children = list(
session.scalars(
select(DigitadorJobRow).where(
DigitadorJobRow.parent_digitador_job_id == parent_id
)
).all()
)
if not children:
return self._row_job(parent)
statuses = [c.status for c in children]
terminal = {
JobStatus.COMPLETED.value,
JobStatus.FAILED.value,
JobStatus.CANCELLED.value,
}
any_active = any(
s in (JobStatus.PENDING.value, JobStatus.RUNNING.value) for s in statuses
)
all_terminal = all(s in terminal for s in statuses)
total = 0
ok = 0
error_count = 0
timeout = 0
existentes = 0
for c in children:
r = dict(c.result or {})
total += int(r.get("total") or 0)
ok += int(r.get("ok") or 0)
error_count += int(r.get("error") or 0)
timeout += int(r.get("timeout") or 0)
existentes += int(r.get("existentes") or 0)
processed = 0
for c in children:
r = dict(c.result or {})
if c.status in terminal:
processed += (
int(r.get("ok") or 0)
+ int(r.get("error") or 0)
+ int(r.get("timeout") or 0)
+ int(r.get("existentes") or 0)
)
else:
processed += int(r.get("processed") or 0)
shard_ids = [c.id for c in children]
parent_result = dict(parent.result or {})
parent_result.update(
{
"mode": "distributed",
"shard_ids": shard_ids,
"total": total or int(parent_result.get("total_rows") or 0),
"total_rows": int(parent_result.get("total_rows") or total or 0),
"ok": ok,
"error": error_count,
"timeout": timeout,
"existentes": existentes,
"processed": processed,
"reports_prefix": parent.reports_prefix,
"shards": [
{
"id": c.id,
"worker_id": c.worker_id,
"status": c.status,
"shard_index": c.shard_index,
"result": dict(c.result or {}),
"error": c.error,
}
for c in children
],
}
)
parent.result = parent_result
now = utcnow()
if any_active:
parent.status = JobStatus.RUNNING.value
parent.started_at = parent.started_at or now
parent.finished_at = None
parent.error = None
self._sync_task(
session,
parent,
status=JobStatus.RUNNING.value,
result=parent_result,
)
elif all_terminal:
any_failed = any(s == JobStatus.FAILED.value for s in statuses)
any_cancelled = any(s == JobStatus.CANCELLED.value for s in statuses)
all_cancelled = all(s == JobStatus.CANCELLED.value for s in statuses)
if all_cancelled:
parent.status = JobStatus.CANCELLED.value
elif any_failed:
parent.status = JobStatus.FAILED.value
parent.error = next(
(c.error for c in children if c.error),
"Uno o más shards fallaron",
)
elif any_cancelled and not any_failed:
parent.status = JobStatus.COMPLETED.value
else:
parent.status = JobStatus.COMPLETED.value
parent.finished_at = parent.finished_at or now
self._sync_task(
session,
parent,
status=parent.status,
result=parent_result,
error=parent.error,
)
session.flush()
return self._row_job(parent)
def reclaim_stale_running_jobs(self) -> int:
"""Jobs running cuyo worker está offline → pending + reasignar a otro online."""
reclaimed = 0
with session_scope() as session:
now = utcnow()
cutoff = now - timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS)
workers = list(session.scalars(select(DigitadorWorkerRow)).all())
offline_ids: set[str] = set()
online_free: List[DigitadorWorkerRow] = []
for w in workers:
hb = w.last_heartbeat
if hb is None:
offline_ids.add(w.id)
continue
if hb.tzinfo is None:
hb = hb.replace(tzinfo=timezone.utc)
if hb < cutoff:
offline_ids.add(w.id)
if w.current_job_id:
w.current_job_id = None
w.status = WorkerStatus.OFFLINE.value
elif not w.current_job_id:
online_free.append(w)
if not offline_ids:
session.flush()
return 0
running = session.scalars(
select(DigitadorJobRow).where(
DigitadorJobRow.status == JobStatus.RUNNING.value,
DigitadorJobRow.worker_id.in_(list(offline_ids)),
)
).all()
rr = 0
for row in running:
is_parent = (
row.parent_digitador_job_id is None
and row.shard_count is not None
and int(row.shard_count) > 1
)
if is_parent:
continue
row.status = JobStatus.PENDING.value
row.started_at = None
row.cancel_requested = False
if online_free:
target = online_free[rr % len(online_free)]
rr += 1
row.worker_id = target.id
reclaimed += 1
session.add(
DigitadorJobEventRow(
digitador_job_id=row.id,
event_type="warning",
message=(
"Worker offline; job reclaimed"
+ (f" → {row.worker_id}" if row.worker_id else "")
),
timestamp=now,
metadata_json={"reclaimed": True, "worker_id": row.worker_id},
)
)
if row.parent_digitador_job_id:
session.add(
DigitadorJobEventRow(
digitador_job_id=row.parent_digitador_job_id,
event_type="warning",
message=f"Shard {row.shard_index} reclamado (worker offline)",
timestamp=now,
metadata_json={
"shard_id": row.id,
"worker_id": row.worker_id,
},
)
)
if online_free:
pending_orphan = session.scalars(
select(DigitadorJobRow).where(
DigitadorJobRow.status == JobStatus.PENDING.value,
DigitadorJobRow.worker_id.in_(list(offline_ids)),
)
).all()
for row in pending_orphan:
is_parent = (
row.parent_digitador_job_id is None
and row.shard_count is not None
and int(row.shard_count) > 1
)
if is_parent:
continue
target = online_free[rr % len(online_free)]
rr += 1
if row.worker_id != target.id:
row.worker_id = target.id
reclaimed += 1
session.flush()
return reclaimed
def add_event(
self,
job_id: str,
event_type: str,
message: str,
*,
radicado: Optional[str] = None,
nit: Optional[str] = None,
index: Optional[int] = None,
total: Optional[int] = None,
metadata: Optional[Dict[str, Any]] = None,
timestamp: Optional[str] = None,
) -> JobEventRecord:
with session_scope() as session:
ts = _parse_iso(timestamp) or utcnow()
ev = DigitadorJobEventRow(
digitador_job_id=job_id,
event_type=event_type,
message=message,
timestamp=ts,
radicado=radicado,
nit=nit,
idx=index,
total=total,
metadata_json=metadata or {},
)
session.add(ev)
session.flush()
row = session.get(DigitadorJobRow, job_id)
if row and row.parent_digitador_job_id:
session.add(
DigitadorJobEventRow(
digitador_job_id=row.parent_digitador_job_id,
event_type=event_type,
message=message,
timestamp=ts,
radicado=radicado,
nit=nit,
idx=index,
total=total,
metadata_json={
**(metadata or {}),
"shard_id": job_id,
"shard_index": row.shard_index,
"worker_id": row.worker_id,
},
)
)
session.flush()
return JobEventRecord(
id=int(ev.id),
job_id=job_id,
event_type=event_type,
message=message,
timestamp=_iso(ts) or utc_now(),
radicado=radicado,
nit=nit,
index=index,
total=total,
metadata=metadata or {},
)
def list_events(self, job_id: str, since_id: int = 0) -> List[JobEventRecord]:
with session_scope() as session:
rows = session.scalars(
select(DigitadorJobEventRow)
.where(
DigitadorJobEventRow.digitador_job_id == job_id,
DigitadorJobEventRow.id > since_id,
)
.order_by(DigitadorJobEventRow.id.asc())
).all()
return [
JobEventRecord(
id=int(r.id),
job_id=r.digitador_job_id,
event_type=r.event_type,
message=r.message,
timestamp=_iso(r.timestamp) or utc_now(),
radicado=r.radicado,
nit=r.nit,
index=r.idx,
total=r.total,
metadata=dict(r.metadata_json or {}),
)
for r in rows
]
def create_digitador_task(
self,
*,
parent_job_id: str,
digitador_job_id: str,
worker_id: Optional[str],
title: str,
original_filename: str,
worker_ids: Optional[List[str]] = None,
mode: str = "single",
source_type: Optional[str] = None,
) -> str:
"""Crea Task(type=digitador) y la asocia al digitador_job. Devuelve task_id."""
with session_scope() as session:
now = utcnow()
request: Dict[str, Any] = {
"worker_id": worker_id,
"worker_ids": worker_ids or ([worker_id] if worker_id else []),
"original_filename": original_filename,
"digitador_job_id": digitador_job_id,
"mode": mode,
}
result: Dict[str, Any] = {"digitador_job_id": digitador_job_id, "mode": mode}
if source_type:
request["source_type"] = source_type
result["source_type"] = source_type
task = Task(
job_id=parent_job_id,
type="digitador",
status=JobStatus.PENDING.value,
title=title or f"Bot digitador ({original_filename})",
request=request,
result=result,
created_at=now,
updated_at=now,
)
session.add(task)
session.flush()
row = session.get(DigitadorJobRow, digitador_job_id)
if row:
row.task_id = task.id
row.job_id = parent_job_id
session.flush()
return task.id
def parent_job_exists(self, parent_job_id: str) -> bool:
from infrastructure.database import Job as IpsJob
with session_scope() as session:
return session.get(IpsJob, parent_job_id) is not None
+10
View File
@@ -0,0 +1,10 @@
fastapi>=0.110.0
uvicorn[standard]>=0.27.0
python-multipart>=0.0.9
boto3>=1.34.0
httpx>=0.27.0
openpyxl>=3.1.0
pydantic>=2.0.0
sqlalchemy>=2.0.0
psycopg[binary]>=3.1.0
pytest>=7.0.0
@@ -0,0 +1,96 @@
"""Tests de parseo/partición CSV para digitador."""
from __future__ import annotations
import pytest
from domain.csv_split import (
REQUIRED_BOT_COLUMNS,
parse_digitador_rows,
rows_to_csv_bytes,
split_rows,
)
def _it_csv_bytes() -> bytes:
header = "|".join(
[
"RADICADO",
"NUMERO",
"FECHA",
"BRUTO",
"AUTORIZACION",
"TIPO_ID",
"SERVICIO",
"COPAGO",
"MODERADORA",
]
)
row = "|".join(
["R-1", "123", "2024-01-01", "1000", "A-9", "CC", "AMB", "0", "4500"]
)
return f"{header}\n{row}\n".encode("utf-8-sig")
def _snake_csv_bytes() -> bytes:
header = "|".join(REQUIRED_BOT_COLUMNS)
row = "|".join(
["R-2", "999", "2024-02-02", "2000", "A-1", "TI", "HOS", "100", "0"]
)
return f"{header}\n{row}\n".encode("utf-8")
def test_it_to_bot_mapping():
rows = parse_digitador_rows(_it_csv_bytes(), "aptos.csv")
assert len(rows) == 1
assert rows[0]["radicado"] == "R-1"
assert rows[0]["documento"] == "123"
assert rows[0]["ingreso"] == "2024-01-01"
assert rows[0]["total_cargos"] == "1000"
assert rows[0]["autorizacion"] == "A-9"
assert rows[0]["tipo_documento"] == "CC"
assert rows[0]["tipo_servicio"] == "AMB"
assert rows[0]["copago"] == "0"
assert rows[0]["cuota_moderadora"] == "4500"
def test_snake_case_headers():
rows = parse_digitador_rows(_snake_csv_bytes(), "input.csv")
assert len(rows) == 1
assert rows[0]["radicado"] == "R-2"
assert rows[0]["documento"] == "999"
def test_missing_columns_raises():
bad = b"RADICADO|NUMERO\nR1|1\n"
with pytest.raises(ValueError) as exc:
parse_digitador_rows(bad, "bad.csv")
assert "Columnas faltantes" in str(exc.value)
def test_split_rows_balanced():
rows = [{"radicado": str(i)} for i in range(100)]
chunks = split_rows(rows, 2)
assert len(chunks) == 2
assert len(chunks[0]) == 50
assert len(chunks[1]) == 50
def test_split_rows_uneven():
rows = [{"radicado": str(i)} for i in range(10)]
chunks = split_rows(rows, 3)
assert len(chunks) == 3
assert sum(len(c) for c in chunks) == 10
assert [len(c) for c in chunks] == [4, 3, 3]
def test_roundtrip_csv():
rows = parse_digitador_rows(_it_csv_bytes(), "aptos.csv")
data = rows_to_csv_bytes(rows)
parsed = parse_digitador_rows(data, "shard.csv")
assert parsed == rows
def test_split_more_workers_than_rows():
rows = [{"radicado": "A"}, {"radicado": "B"}]
chunks = split_rows(rows, 5)
assert len(chunks) == 2
@@ -49,6 +49,11 @@ def apply_rules(
if col not in result.columns:
log.append(f"[WARN] Columna no encontrada: {col}")
return
# Eliminar columna: no tiene sentido transformar y luego borrar.
if "drop_column" in rule_ids:
result.drop(columns=[col], inplace=True)
log.append(f"[OK] '{col}' <- {get_rule_name('drop_column')}")
return
split_has_homologation = (
"doc_id_split_tipo_numero" in rule_ids and "doc_type_homologate" in rule_ids
)
@@ -140,5 +140,14 @@ register_rule(
),
fn=special._rule_passthrough_series,
)
register_rule(
"drop_column",
name="Eliminar columna",
description=(
"Quita por completo esta columna del resultado limpio. "
"Útil para campos que no se necesitan en el export o en pasos posteriores."
),
fn=special._rule_passthrough_series,
)
rebuild_cleaning_rules()
@@ -99,7 +99,12 @@ def _infer_column_type(
def _applicable_rules_for_type(inferred_type: str) -> list[str]:
"""Identificadores de reglas a mostrar según el tipo inferido (sin redundancias)."""
base = ["strip_special_spaces", "strip_special_keep_date_seps", "strip_spaces"]
base = [
"strip_special_spaces",
"strip_special_keep_date_seps",
"strip_spaces",
"drop_column",
]
if inferred_type == TYPE_INTEGER:
return base + ["cop_mod_split", "cop_mod_bruto_neto"]
if inferred_type == TYPE_DECIMAL:
@@ -0,0 +1,83 @@
"""Download jobs: parent/shard fields for distributed bots.
Revision ID: 0007
Revises: 0006
Create Date: 2026-08-09
"""
from __future__ import annotations
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0007"
down_revision: Union[str, None] = "0006"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
if not inspector.has_table("download_jobs"):
return
cols = {c["name"] for c in inspector.get_columns("download_jobs")}
with op.batch_alter_table("download_jobs") as batch:
if "parent_download_job_id" not in cols:
batch.add_column(
sa.Column(
"parent_download_job_id",
sa.String(36),
sa.ForeignKey("download_jobs.id"),
nullable=True,
)
)
if "shard_index" not in cols:
batch.add_column(sa.Column("shard_index", sa.Integer(), nullable=True))
if "shard_count" not in cols:
batch.add_column(sa.Column("shard_count", sa.Integer(), nullable=True))
# Padre orquestador no tiene worker asignado
op.alter_column(
"download_jobs",
"worker_id",
existing_type=sa.String(36),
nullable=True,
)
existing_indexes = {ix["name"] for ix in inspector.get_indexes("download_jobs")}
if "ix_download_jobs_parent" not in existing_indexes:
op.create_index(
"ix_download_jobs_parent",
"download_jobs",
["parent_download_job_id"],
)
def downgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
if not inspector.has_table("download_jobs"):
return
existing_indexes = {ix["name"] for ix in inspector.get_indexes("download_jobs")}
if "ix_download_jobs_parent" in existing_indexes:
op.drop_index("ix_download_jobs_parent", table_name="download_jobs")
cols = {c["name"] for c in inspector.get_columns("download_jobs")}
with op.batch_alter_table("download_jobs") as batch:
if "shard_count" in cols:
batch.drop_column("shard_count")
if "shard_index" in cols:
batch.drop_column("shard_index")
if "parent_download_job_id" in cols:
batch.drop_column("parent_download_job_id")
op.alter_column(
"download_jobs",
"worker_id",
existing_type=sa.String(36),
nullable=False,
)
@@ -0,0 +1,152 @@
"""Bot digitador: digitador_workers, digitador_jobs, digitador_job_events.
Revision ID: 0008
Revises: 0007
Create Date: 2026-08-09
"""
from __future__ import annotations
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision: str = "0008"
down_revision: Union[str, None] = "0007"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _table_exists(conn: sa.engine.Connection, table: str) -> bool:
return sa.inspect(conn).has_table(table)
def upgrade() -> None:
conn = op.get_bind()
json_type = (
postgresql.JSONB()
if conn.dialect.name == "postgresql"
else sa.JSON()
)
if not _table_exists(conn, "digitador_workers"):
op.create_table(
"digitador_workers",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("hostname", sa.Text(), nullable=False, server_default=""),
sa.Column("max_concurrent", sa.Integer(), nullable=False, server_default="1"),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("last_heartbeat", sa.DateTime(timezone=True), nullable=False),
sa.Column("current_job_id", sa.String(36), nullable=True),
)
op.create_index("ix_digitador_workers_status", "digitador_workers", ["status"])
if not _table_exists(conn, "digitador_jobs"):
op.create_table(
"digitador_jobs",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True),
sa.Column("task_id", sa.String(36), sa.ForeignKey("tasks.id"), nullable=True),
sa.Column("worker_id", sa.String(36), nullable=True),
sa.Column(
"parent_digitador_job_id",
sa.String(36),
sa.ForeignKey("digitador_jobs.id"),
nullable=True,
),
sa.Column("shard_index", sa.Integer(), nullable=True),
sa.Column("shard_count", sa.Integer(), nullable=True),
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
sa.Column("title", sa.Text(), nullable=True),
sa.Column("csv_key", sa.Text(), nullable=False, server_default=""),
sa.Column("reports_prefix", sa.Text(), nullable=False, server_default=""),
sa.Column("original_filename", sa.Text(), nullable=False, server_default=""),
sa.Column(
"cancel_requested",
sa.Boolean(),
nullable=False,
server_default=sa.false(),
),
sa.Column(
"result",
json_type,
nullable=False,
server_default=(
sa.text("'{}'::jsonb")
if conn.dialect.name == "postgresql"
else sa.text("'{}'")
),
),
sa.Column("error", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
)
op.create_index("ix_digitador_jobs_job_id", "digitador_jobs", ["job_id"])
op.create_index("ix_digitador_jobs_task_id", "digitador_jobs", ["task_id"])
op.create_index("ix_digitador_jobs_worker_id", "digitador_jobs", ["worker_id"])
op.create_index("ix_digitador_jobs_status", "digitador_jobs", ["status"])
op.create_index(
"ix_digitador_jobs_parent",
"digitador_jobs",
["parent_digitador_job_id"],
)
op.create_index(
"ix_digitador_jobs_worker_status",
"digitador_jobs",
["worker_id", "status"],
)
if not _table_exists(conn, "digitador_job_events"):
op.create_table(
"digitador_job_events",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column(
"digitador_job_id",
sa.String(36),
sa.ForeignKey("digitador_jobs.id"),
nullable=False,
),
sa.Column("event_type", sa.String(64), nullable=False),
sa.Column("message", sa.Text(), nullable=False),
sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False),
sa.Column("radicado", sa.String(255), nullable=True),
sa.Column("nit", sa.String(64), nullable=True),
sa.Column("idx", sa.Integer(), nullable=True),
sa.Column("total", sa.Integer(), nullable=True),
sa.Column(
"metadata",
json_type,
nullable=False,
server_default=(
sa.text("'{}'::jsonb")
if conn.dialect.name == "postgresql"
else sa.text("'{}'")
),
),
)
op.create_index(
"ix_digitador_job_events_job_id",
"digitador_job_events",
["digitador_job_id", "id"],
)
def downgrade() -> None:
conn = op.get_bind()
if _table_exists(conn, "digitador_job_events"):
op.drop_index("ix_digitador_job_events_job_id", table_name="digitador_job_events")
op.drop_table("digitador_job_events")
if _table_exists(conn, "digitador_jobs"):
op.drop_index("ix_digitador_jobs_worker_status", table_name="digitador_jobs")
op.drop_index("ix_digitador_jobs_parent", table_name="digitador_jobs")
op.drop_index("ix_digitador_jobs_status", table_name="digitador_jobs")
op.drop_index("ix_digitador_jobs_worker_id", table_name="digitador_jobs")
op.drop_index("ix_digitador_jobs_task_id", table_name="digitador_jobs")
op.drop_index("ix_digitador_jobs_job_id", table_name="digitador_jobs")
op.drop_table("digitador_jobs")
if _table_exists(conn, "digitador_workers"):
op.drop_index("ix_digitador_workers_status", table_name="digitador_workers")
op.drop_table("digitador_workers")
+10
View File
@@ -5,6 +5,7 @@ import {
Cpu,
Download,
Hash,
Keyboard,
Link2,
Package,
Settings,
@@ -35,6 +36,7 @@ import "./modules/extraction/ExtractionModule";
import "./modules/cleaning/CleaningModule";
import "./modules/poblacion_apta/PoblacionAptaModule";
import "./modules/downloads/DownloadsModule";
import "./modules/digitador/DigitadorModule";
const BOT_PACKAGE_URL =
"http://s3.builker.xyz/apps-versions/BotDescargaPYSCuentasMedicas/latest/bot_descarga.zip";
@@ -403,6 +405,14 @@ function App() {
<Users size={17} />
<span>Población apta</span>
</button>
<button
type="button"
className={jobSection === "digitador" ? "active" : ""}
onClick={() => handleActivateSection("digitador")}
>
<Keyboard size={17} />
<span>Bot digitador</span>
</button>
<button
type="button"
className={jobSection === "results" ? "active" : ""}
@@ -28,7 +28,11 @@ interface Props {
function jobLabel(r: DownloadJob): string {
const title = r.title || r.original_filename || r.id.slice(0, 8);
return `${title} · ${r.status} · ${r.worker_id}`;
const bots =
r.shard_count && Number(r.shard_count) > 1
? `${r.shard_count} bots`
: r.worker_id || "—";
return `${title} · ${r.status} · ${bots}`;
}
export function MinioDownloadSourcePicker({
@@ -9,6 +9,10 @@ import type {
CountSyncResponse,
CreateCountJobPayload,
CreateJobPayload,
DigitadorJob,
DigitadorJobEvent,
DigitadorJobFilesResponse,
DigitadorWorker,
DownloadJob,
DownloadJobEvent,
DownloadJobFilesResponse,
@@ -87,7 +91,8 @@ export class ApiClient {
path.startsWith("/cleaning/jobs/from-csv") ||
path.startsWith("/poblacion-apta/jobs/from-csv") ||
path.startsWith("/poblacion-apta/jobs/from-cleaning") ||
path === "/bot-descarga/jobs"
path === "/bot-descarga/jobs" ||
path === "/bot-digitador/jobs"
);
}
@@ -310,6 +315,83 @@ export class ApiClient {
return this.handle<DownloadJob>(response);
}
// ── Bot Digitador (control remoto) ─────────────────────────────────────────
async listDigitadorWorkers(): Promise<DigitadorWorker[]> {
const response = await fetchWithTimeout(
this.url("/bot-digitador/workers"),
{ headers: this.botHeaders() },
this.timeoutFor("/bot-digitador/workers"),
);
return this.handle<DigitadorWorker[]>(response);
}
async createDigitadorJob(form: FormData): Promise<DigitadorJob> {
const response = await fetchWithTimeout(
this.url("/bot-digitador/jobs"),
{ method: "POST", headers: this.botHeaders(), body: form },
this.timeoutFor("/bot-digitador/jobs"),
);
return this.handle<DigitadorJob>(response);
}
async listDigitadorJobs(limit = 50, parentJobId?: string): Promise<DigitadorJob[]> {
const qs = new URLSearchParams({ limit: String(limit) });
if (parentJobId) qs.set("job_id", parentJobId);
const response = await fetchWithTimeout(
this.url(`/bot-digitador/jobs?${qs.toString()}`),
{ headers: this.botHeaders() },
);
return this.handle<DigitadorJob[]>(response);
}
async getDigitadorJob(jobId: string): Promise<DigitadorJob> {
const response = await fetchWithTimeout(
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}`),
{ headers: this.botHeaders() },
);
return this.handle<DigitadorJob>(response);
}
async listDigitadorJobFiles(jobId: string): Promise<DigitadorJobFilesResponse> {
const response = await fetchWithTimeout(
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}/files`),
{ headers: this.botHeaders() },
);
return this.handle<DigitadorJobFilesResponse>(response);
}
/** Descarga bytes de un reporte CSV/log/json del resultado MinIO (proxy bot-digitador). */
async fetchDigitadorObject(jobId: string, key: string): Promise<ArrayBuffer> {
const qs = new URLSearchParams({ key });
const response = await fetchWithTimeout(
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}/object?${qs.toString()}`),
{ headers: this.botHeaders() },
120_000,
);
if (!response.ok) {
const text = await response.text().catch(() => "");
throw new Error(text || `Error ${response.status} al leer objeto MinIO`);
}
return response.arrayBuffer();
}
async getDigitadorEvents(jobId: string, since = 0): Promise<{ events: DigitadorJobEvent[] }> {
const response = await fetchWithTimeout(
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}/events?since=${since}`),
{ headers: this.botHeaders() },
);
return this.handle<{ events: DigitadorJobEvent[] }>(response);
}
async cancelDigitadorJob(jobId: string): Promise<DigitadorJob> {
const response = await fetchWithTimeout(
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}/cancel`),
{ method: "POST", headers: this.botHeaders() },
);
return this.handle<DigitadorJob>(response);
}
// ── Count Jobs ─────────────────────────────────────────────────────────────
/** Envía PDFs al servicio de conteo (síncrono). */
@@ -456,7 +538,7 @@ export class ApiClient {
async poblacionAptaExportCsv(
jobId: string,
subset: "aptos" | "no_aptos" = "aptos",
subset: "aptos" | "no_aptos" | "todos" = "aptos",
): Promise<Blob> {
const params = new URLSearchParams({ subset });
const path = `/poblacion-apta/jobs/${encodeURIComponent(jobId)}/export.csv?${params}`;
@@ -0,0 +1,18 @@
import type { ModuleDescriptor } from "../registry";
import { registerModule } from "../registry";
import type { Task } from "../../../shared/types";
import { DigitadorWizardModule } from "./DigitadorWizardModule";
import { DigitadorResultsPanel } from "./DigitadorResultsPanel";
const DigitadorModule: ModuleDescriptor = {
type: "digitador",
label: "Bot digitador",
iconName: "Keyboard",
hasTask: (tasks: Task[]) => tasks.some((t) => t.type === "digitador"),
Wizard: DigitadorWizardModule as unknown as ModuleDescriptor["Wizard"],
Results: DigitadorResultsPanel as unknown as ModuleDescriptor["Results"],
};
registerModule(DigitadorModule);
export { DigitadorModule };
@@ -0,0 +1,400 @@
import {
AlertCircle,
CheckCircle2,
Download,
Keyboard,
Loader2,
Plus,
RefreshCw,
} from "lucide-react";
import { useCallback, useEffect, useState } from "react";
import type {
DigitadorJob,
DigitadorJobEvent,
DigitadorJobFile,
Job,
Task,
} from "../../../shared/types";
import type { ApiClient } from "../../lib/apiClient";
declare const desktopApi: import("../../../shared/types").DesktopBridge | undefined;
interface Props {
client: ApiClient;
job: Job;
tasks: Task[];
onStartDigitador: () => void;
}
function reportBasename(file: DigitadorJobFile): string {
const named = (file.name || "").trim();
if (named) return named;
return file.key.split("/").pop() || "reporte.csv";
}
function extensionFilters(filename: string): Array<{ name: string; extensions: string[] }> {
const lower = filename.toLowerCase();
if (lower.endsWith(".csv")) return [{ name: "CSV", extensions: ["csv"] }];
if (lower.endsWith(".json")) return [{ name: "JSON", extensions: ["json"] }];
if (lower.endsWith(".log")) return [{ name: "Log", extensions: ["log"] }];
return [{ name: "Archivo", extensions: ["*"] }];
}
async function saveReportAs(filename: string, data: ArrayBuffer): Promise<boolean> {
const safeName = reportBasename({ key: filename, name: filename });
if (typeof desktopApi?.saveFileDialog === "function" && typeof desktopApi?.writeBytesFile === "function") {
const dest = await desktopApi.saveFileDialog(safeName, extensionFilters(safeName));
if (!dest) return false;
await desktopApi.writeBytesFile(dest, data);
return true;
}
const blob = new Blob([data]);
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = safeName;
a.click();
URL.revokeObjectURL(url);
return true;
}
export function DigitadorResultsPanel({ client, job, tasks, onStartDigitador }: Props) {
const [runs, setRuns] = useState<DigitadorJob[]>([]);
const [selectedId, setSelectedId] = useState<string>("");
const [events, setEvents] = useState<DigitadorJobEvent[]>([]);
const [files, setFiles] = useState<DigitadorJobFile[]>([]);
const [filesError, setFilesError] = useState<string | null>(null);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [dlBusyKey, setDlBusyKey] = useState<string | null>(null);
const [dlStatus, setDlStatus] = useState<string | null>(null);
const digitadorTasks = tasks
.filter((t) => t.type === "digitador")
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
const loadRuns = useCallback(async () => {
setLoading(true);
setError(null);
try {
const list = await client.listDigitadorJobs(100, job.id);
setRuns(list);
setSelectedId((prev) => {
if (prev && list.some((r) => r.id === prev)) return prev;
return list[0]?.id ?? "";
});
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setLoading(false);
}
}, [client, job.id]);
useEffect(() => {
void loadRuns();
}, [loadRuns, digitadorTasks.length]);
const selected = runs.find((r) => r.id === selectedId) ?? null;
const running = Boolean(
selected && !["completed", "failed", "cancelled"].includes(selected.status),
);
useEffect(() => {
if (!selectedId) {
setEvents([]);
setFiles([]);
setFilesError(null);
return;
}
let cancelled = false;
async function loadDetail() {
try {
const [{ events: ev }, fresh] = await Promise.all([
client.getDigitadorEvents(selectedId, 0),
client.getDigitadorJob(selectedId),
]);
if (cancelled) return;
setEvents(ev);
setRuns((prev) => prev.map((r) => (r.id === fresh.id ? fresh : r)));
try {
const fileResp = await client.listDigitadorJobFiles(selectedId);
if (cancelled) return;
setFiles(fileResp.files);
setFilesError(null);
} catch (e) {
if (cancelled) return;
setFiles([]);
setFilesError(e instanceof Error ? e.message : String(e));
}
} catch (e) {
if (!cancelled) setError(e instanceof Error ? e.message : String(e));
}
}
void loadDetail();
if (!running) return;
const t = setInterval(() => void loadDetail(), 3000);
return () => {
cancelled = true;
clearInterval(t);
};
}, [client, selectedId, running]);
const result = selected?.result ?? {};
const exitosos = Number(result.exitosos ?? 0);
const fallidos = Number(result.fallidos ?? 0);
const existentes = Number(result.existentes ?? 0);
const timeouts = Number(result.timeouts ?? 0);
const total = Number(result.total ?? 0);
const bulkBusy = Boolean(dlBusyKey);
async function downloadOne(file: DigitadorJobFile) {
if (!selectedId || bulkBusy) return;
const busyId = `file:${file.key}`;
setDlBusyKey(busyId);
setDlStatus(null);
setError(null);
try {
const data = await client.fetchDigitadorObject(selectedId, file.key);
const name = reportBasename(file);
const ok = await saveReportAs(name, data);
if (ok) setDlStatus(`Guardado: ${name}`);
} catch (e) {
setError(e instanceof Error ? e.message : "No se pudo descargar el reporte.");
} finally {
setDlBusyKey(null);
}
}
if (runs.length === 0 && !loading) {
return (
<div className="results-empty">
<Keyboard size={28} />
<p>Aún no hay corridas de bot digitador en este trabajo.</p>
<button type="button" className="btn btn-primary" onClick={onStartDigitador}>
<Plus size={15} /> Nueva digitación
</button>
</div>
);
}
return (
<div className="results-section">
<div className="results-section-header">
<div className="results-section-header-left">
<label className="wiz-field-label">Corrida</label>
<select
className="form-input"
value={selectedId}
onChange={(e) => {
setSelectedId(e.target.value);
setDlStatus(null);
}}
style={{ minWidth: 280 }}
>
{runs.map((r) => (
<option key={r.id} value={r.id}>
{r.title || r.original_filename || r.id.slice(0, 8)} · {r.status}
{r.shard_count && Number(r.shard_count) > 1
? ` · ${r.shard_count} bots`
: r.worker_id
? ` · ${r.worker_id}`
: ""}
</option>
))}
</select>
{selected && (
<span className={`status-badge status-badge--${selected.status}`}>
{selected.status}
</span>
)}
</div>
<div className="results-section-header-right">
<button type="button" className="btn btn-sm btn-ghost" onClick={() => void loadRuns()} disabled={loading}>
{loading ? <Loader2 size={13} className="spin" /> : <RefreshCw size={13} />}
Actualizar
</button>
<button type="button" className="btn btn-sm btn-primary" onClick={onStartDigitador}>
<Plus size={13} /> Nueva digitación
</button>
</div>
</div>
{error && (
<div className="wiz-toast wiz-toast--error" style={{ position: "static", marginBottom: 12 }}>
<AlertCircle size={15} />
<span>{error}</span>
</div>
)}
{selected?.error && (
<div className="wiz-toast wiz-toast--error" style={{ position: "static", marginBottom: 12 }}>
<AlertCircle size={15} />
<span>{selected.error}</span>
</div>
)}
{dlStatus && (
<div className="wiz-toast" style={{ position: "static", marginBottom: 12 }}>
<CheckCircle2 size={15} />
<span>{dlStatus}</span>
</div>
)}
{selected && (
<>
<div className="wiz-stats-row" style={{ marginBottom: 16, flexShrink: 0 }}>
<div className="wiz-stat">
<span className="wiz-stat-value">{exitosos}</span>
<span className="wiz-stat-label">OK</span>
</div>
<div className="wiz-stat">
<span className="wiz-stat-value">{fallidos}</span>
<span className="wiz-stat-label">Fallidos</span>
</div>
<div className="wiz-stat">
<span className="wiz-stat-value">{existentes}</span>
<span className="wiz-stat-label">Existentes</span>
</div>
<div className="wiz-stat">
<span className="wiz-stat-value">{timeouts}</span>
<span className="wiz-stat-label">Timeouts</span>
</div>
<div className="wiz-stat">
<span className="wiz-stat-value">{total || "—"}</span>
<span className="wiz-stat-label">Total</span>
</div>
</div>
<p className="wiz-hint" style={{ marginBottom: 12, flexShrink: 0 }}>
{selected.shard_count && Number(selected.shard_count) > 1
? `${selected.shard_count} bots (distribuido)`
: (
<>
Worker <strong>{selected.worker_id ?? "—"}</strong>
</>
)}
{selected.original_filename ? ` · ${selected.original_filename}` : ""}
{selected.reports_prefix ? (
<>
{" "}
· Prefijo <code className="dl-prefix-code">{selected.reports_prefix}</code>
</>
) : null}
</p>
{(() => {
const shardList = Array.isArray(selected.shards)
? selected.shards
: Array.isArray(result.shards)
? (result.shards as Array<{
id?: string;
worker_id?: string | null;
status?: string;
shard_index?: number | null;
}>)
: [];
if (shardList.length === 0) return null;
return (
<div className="wiz-section" style={{ marginBottom: 12, flexShrink: 0 }}>
<h3 className="wiz-section-title">Estado por bot</h3>
<div className="wiz-review-grid" style={{ marginTop: 8 }}>
{shardList.map((s, i) => (
<div key={s.id || String(i)} className="wiz-review-item">
<div>
<span className="wiz-review-label">
Shard {(s.shard_index ?? i) + 1}
</span>
<span className="wiz-review-value">{s.worker_id || "—"}</span>
</div>
<div>
<span className="wiz-review-label">Estado</span>
<span className="wiz-review-value">{s.status || "—"}</span>
</div>
</div>
))}
</div>
</div>
);
})()}
<div className="wiz-section dl-files-section">
<div className="dl-files-header">
<h3 className="wiz-section-title" style={{ margin: 0 }}>
Reportes en MinIO
{files.length > 0 ? ` · ${files.length} archivo${files.length === 1 ? "" : "s"}` : ""}
</h3>
</div>
{filesError && (
<div className="wiz-toast wiz-toast--error" style={{ position: "static", marginBottom: 12 }}>
<AlertCircle size={15} />
<span>No se pudo listar MinIO: {filesError}</span>
</div>
)}
{!filesError && files.length === 0 ? (
<p className="wiz-hint">Sin reportes listados aún (RESULTADOS / OMITIDOS).</p>
) : (
<table className="dl-files-table">
<thead>
<tr>
<th>Archivo</th>
<th className="dl-files-table-actions"> </th>
</tr>
</thead>
<tbody>
{files.map((f) => {
const fileBusy = dlBusyKey === `file:${f.key}`;
const name = reportBasename(f);
return (
<tr key={f.key}>
<td className="dl-col-file" title={f.key}>
{name}
</td>
<td className="dl-files-table-actions">
<button
type="button"
className="btn btn-sm btn-ghost"
disabled={bulkBusy}
onClick={() => void downloadOne(f)}
title="Descargar reporte"
>
{fileBusy ? (
<Loader2 size={13} className="spin" />
) : (
<Download size={13} />
)}
Descargar
</button>
</td>
</tr>
);
})}
</tbody>
</table>
)}
</div>
<div className="wiz-section">
<h3 className="wiz-section-title">Log</h3>
<div className="wiz-estimate-box dl-log-box">
{events.length === 0 ? (
<span className="wiz-hint">Sin eventos.</span>
) : (
events.map((ev) => (
<div key={ev.id}>
[{ev.timestamp}] {ev.message}
</div>
))
)}
</div>
</div>
</>
)}
</div>
);
}
@@ -0,0 +1,760 @@
import {
AlertCircle,
Bot,
CheckCircle2,
FileSpreadsheet,
Keyboard,
Loader2,
Play,
RefreshCw,
Square,
Upload,
Users,
} from "lucide-react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import type { ApiClient } from "../../lib/apiClient";
import type {
DigitadorJob,
DigitadorJobEvent,
DigitadorWorker,
Job,
PoblacionAptaJob,
} from "../../../shared/types";
export type DigitadorSourceMode = "aptos" | "archivo";
export type DigitadorInitialSource = {
poblacionAptaJobId: string;
};
interface Props {
client: ApiClient;
job: Job;
initialSource?: DigitadorInitialSource | null;
}
type DigitadorStep = 1 | 2;
type ShardInfo = {
id?: string;
worker_id?: string | null;
status?: string;
shard_index?: number | null;
result?: Record<string, unknown>;
error?: string | null;
};
export function DigitadorWizardModule({ client, job, initialSource = null }: Props) {
const [step, setStep] = useState<DigitadorStep>(1);
const [workers, setWorkers] = useState<DigitadorWorker[]>([]);
const [selectedIds, setSelectedIds] = useState<string[]>([]);
const [useAllOnline, setUseAllOnline] = useState(true);
const [sourceMode, setSourceMode] = useState<DigitadorSourceMode>(
initialSource?.poblacionAptaJobId ? "aptos" : "aptos",
);
const [paJobs, setPaJobs] = useState<PoblacionAptaJob[]>([]);
const [selectedPaId, setSelectedPaId] = useState(initialSource?.poblacionAptaJobId ?? "");
const [loadingPa, setLoadingPa] = useState(false);
const [file, setFile] = useState<File | null>(null);
const [title, setTitle] = useState("");
const [busy, setBusy] = useState(false);
const [refreshing, setRefreshing] = useState(false);
const [error, setError] = useState<string | null>(null);
const [activeJob, setActiveJob] = useState<DigitadorJob | null>(null);
const [events, setEvents] = useState<DigitadorJobEvent[]>([]);
const sinceRef = useRef(0);
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
const fileInputRef = useRef<HTMLInputElement | null>(null);
const initializedSelectRef = useRef(false);
const appliedInitialRef = useRef<string | null>(null);
const onlineWorkers = useMemo(
() => workers.filter((w) => w.status === "online" || w.status === "busy"),
[workers],
);
useEffect(() => {
if (!initialSource?.poblacionAptaJobId) return;
const key = initialSource.poblacionAptaJobId;
if (appliedInitialRef.current === key) return;
appliedInitialRef.current = key;
setSourceMode("aptos");
setSelectedPaId(initialSource.poblacionAptaJobId);
setStep(1);
}, [initialSource?.poblacionAptaJobId]);
const refreshWorkers = useCallback(async () => {
setRefreshing(true);
try {
const list = await client.listDigitadorWorkers();
setWorkers(list);
setError(null);
const online = list.filter((w) => w.status === "online" || w.status === "busy");
if (!initializedSelectRef.current && online.length > 0) {
initializedSelectRef.current = true;
setUseAllOnline(online.length >= 2);
setSelectedIds(online.map((w) => w.id));
} else if (useAllOnline) {
setSelectedIds(online.map((w) => w.id));
} else {
setSelectedIds((prev) => prev.filter((id) => online.some((w) => w.id === id)));
}
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setRefreshing(false);
}
}, [client, useAllOnline]);
useEffect(() => {
void refreshWorkers();
const t = setInterval(() => void refreshWorkers(), 5000);
return () => clearInterval(t);
}, [refreshWorkers]);
const loadPaJobs = useCallback(async () => {
setLoadingPa(true);
try {
const list = await client.poblacionAptaJobs(job.id);
const completed = list
.filter((j) => j.status === "completed")
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
setPaJobs(completed);
setSelectedPaId((prev) => {
if (prev && completed.some((j) => j.id === prev)) return prev;
return completed[0]?.id ?? "";
});
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setLoadingPa(false);
}
}, [client, job.id]);
useEffect(() => {
if (sourceMode !== "aptos") return;
void loadPaJobs();
}, [sourceMode, loadPaJobs]);
useEffect(() => {
if (!activeJob || ["completed", "failed", "cancelled"].includes(activeJob.status)) {
if (pollRef.current) {
clearInterval(pollRef.current);
pollRef.current = null;
}
return;
}
async function poll() {
if (!activeJob) return;
try {
const nextJob = await client.getDigitadorJob(activeJob.id);
setActiveJob(nextJob);
const { events: next } = await client.getDigitadorEvents(activeJob.id, sinceRef.current);
if (next.length) {
setEvents((prev) => [...prev, ...next]);
sinceRef.current = next[next.length - 1].id;
}
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
}
}
void poll();
pollRef.current = setInterval(() => void poll(), 3000);
return () => {
if (pollRef.current) clearInterval(pollRef.current);
};
}, [activeJob?.id, activeJob?.status, client]);
function toggleWorker(id: string) {
if (useAllOnline) return;
setSelectedIds((prev) =>
prev.includes(id) ? prev.filter((x) => x !== id) : [...prev, id],
);
}
const selectedPa = paJobs.find((j) => j.id === selectedPaId) ?? null;
const aptosCount = Number(selectedPa?.summary?.paso_bot ?? 0);
async function startJob() {
const targets = useAllOnline ? onlineWorkers.map((w) => w.id) : selectedIds;
if (targets.length === 0) {
setError("Necesitas al menos un bot online.");
return;
}
if (sourceMode === "archivo") {
if (!file) {
setError("Selecciona un CSV.");
return;
}
} else {
if (!selectedPaId) {
setError("Selecciona una corrida de población apta.");
return;
}
if (aptosCount <= 0) {
setError("Esa corrida no tiene aptos (PASO_BOT) para digitar.");
return;
}
}
setBusy(true);
setError(null);
try {
const form = new FormData();
form.append("job_id", job.id);
if (sourceMode === "archivo" && file) {
form.append("file", file);
} else {
form.append("poblacion_apta_job_id", selectedPaId);
form.append("subset", "aptos");
}
if (title.trim()) form.append("title", title.trim());
if (targets.length === 1) {
form.append("worker_id", targets[0]);
} else {
form.append("worker_ids", JSON.stringify(targets));
}
const created = await client.createDigitadorJob(form);
sinceRef.current = 0;
setEvents([]);
setActiveJob(created);
setStep(2);
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setBusy(false);
}
}
async function cancelJob() {
if (!activeJob) return;
setBusy(true);
try {
const updated = await client.cancelDigitadorJob(activeJob.id);
setActiveJob(updated);
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setBusy(false);
}
}
function resetWizard() {
setStep(1);
setActiveJob(null);
setEvents([]);
setFile(null);
setTitle("");
setError(null);
sinceRef.current = 0;
if (fileInputRef.current) fileInputRef.current.value = "";
}
const result = activeJob?.result ?? {};
const processed = Number(result.processed ?? result.exitosos ?? 0);
const total = Number(result.total ?? result.total_rows ?? 0);
const pct = total > 0 ? Math.min(100, Math.round((processed / total) * 100)) : 0;
const running = Boolean(
activeJob && !["completed", "failed", "cancelled"].includes(activeJob.status),
);
const isDistributed =
Boolean(activeJob?.shard_count && Number(activeJob.shard_count) > 1) ||
result.mode === "distributed";
const shards: ShardInfo[] = Array.isArray(activeJob?.shards)
? (activeJob!.shards as ShardInfo[])
: Array.isArray(result.shards)
? (result.shards as ShardInfo[])
: [];
const effectiveTargets = useAllOnline ? onlineWorkers.map((w) => w.id) : selectedIds;
const sourceReady =
sourceMode === "archivo" ? Boolean(file) : Boolean(selectedPaId) && aptosCount > 0;
const canStart = effectiveTargets.length > 0 && sourceReady && !busy;
return (
<div className="wiz">
<nav className="stepper" aria-label="Pasos del bot digitador">
<div className="stepper-item">
<button
type="button"
className={`stepper-dot ${step === 1 ? "active" : step > 1 ? "done" : "todo"}`}
onClick={() => !running && setStep(1)}
disabled={running}
>
1
</button>
<span className={`stepper-label ${step === 1 ? "active" : step > 1 ? "done" : "todo"}`}>
Configurar
</span>
<div className={`stepper-connector ${step > 1 ? "done" : ""}`} />
</div>
<div className="stepper-item">
<button
type="button"
className={`stepper-dot ${step === 2 ? "active" : "todo"}`}
onClick={() => activeJob && setStep(2)}
disabled={!activeJob}
>
2
</button>
<span className={`stepper-label ${step === 2 ? "active" : "todo"}`}>Ejecución</span>
</div>
</nav>
<div className="wiz-body">
{step === 1 && (
<div className="wiz-step">
<div className="wiz-header" style={{ marginBottom: 4 }}>
<h2 style={{ display: "flex", alignItems: "center", gap: 8, margin: 0 }}>
<Keyboard size={18} />
Bot digitador
</h2>
<p className="wiz-desc" style={{ marginTop: 6 }}>
Digita en Integración Total solo los <strong>aptos (PASO_BOT)</strong>, o un CSV
propio con el mismo formato.
</p>
</div>
{/* 1. Origen */}
<div className="wiz-section">
<h3 className="wiz-section-title">1. Origen</h3>
<div style={{ display: "flex", flexWrap: "wrap", gap: 8, marginBottom: 14 }}>
<button
type="button"
className={`btn${sourceMode === "aptos" ? " btn-primary" : ""}`}
onClick={() => setSourceMode("aptos")}
>
<Users size={14} /> Aptos de población apta
</button>
<button
type="button"
className={`btn${sourceMode === "archivo" ? " btn-primary" : ""}`}
onClick={() => setSourceMode("archivo")}
>
<Upload size={14} /> Subir CSV
</button>
</div>
<label className="wiz-field-label">Título (opcional)</label>
<input
className="form-input"
value={title}
onChange={(e) => setTitle(e.target.value)}
placeholder="Ej. Digitación lote mayo 2026"
style={{ width: "100%", marginBottom: 14 }}
/>
{sourceMode === "aptos" ? (
<>
<div className="wiz-section-header" style={{ marginBottom: 8 }}>
<label className="wiz-field-label" style={{ margin: 0 }}>
Corrida de población apta
</label>
<button
type="button"
className="btn btn-sm btn-ghost"
onClick={() => void loadPaJobs()}
disabled={loadingPa}
>
{loadingPa ? <Loader2 size={13} className="spin" /> : <RefreshCw size={13} />}
Actualizar
</button>
</div>
{loadingPa && paJobs.length === 0 ? (
<p className="wiz-hint">
<Loader2 size={13} className="spin" /> Cargando corridas…
</p>
) : paJobs.length === 0 ? (
<div className="wiz-estimate-box" style={{ textAlign: "left" }}>
<p className="wiz-desc" style={{ margin: 0 }}>
No hay corridas de población apta completadas en este trabajo.
</p>
<p className="wiz-hint" style={{ marginTop: 6, marginBottom: 0 }}>
Ejecuta primero Población apta, o cambia a «Subir CSV».
</p>
</div>
) : (
<>
<select
className="form-input"
value={selectedPaId}
onChange={(e) => setSelectedPaId(e.target.value)}
style={{ width: "100%" }}
>
{paJobs.map((pa) => {
const n = Number(pa.summary?.paso_bot ?? 0);
return (
<option key={pa.id} value={pa.id}>
{pa.id.slice(0, 8)} · {n} aptos · {pa.summary?.total ?? 0} total
</option>
);
})}
</select>
{selectedPa && (
<div
className="wiz-estimate-box"
style={{
marginTop: 12,
display: "flex",
alignItems: "center",
gap: 12,
textAlign: "left",
}}
>
{aptosCount > 0 ? (
<CheckCircle2 size={20} style={{ color: "var(--success, #16a34a)", flexShrink: 0 }} />
) : (
<AlertCircle size={20} style={{ color: "var(--danger, #dc2626)", flexShrink: 0 }} />
)}
<div>
<div style={{ fontWeight: 600 }}>
{aptosCount > 0
? `${aptosCount} registros aptos listos para digitar`
: "Sin aptos en esta corrida"}
</div>
<div className="wiz-hint" style={{ margin: 0 }}>
Solo se envían filas PASO_BOT. Los no aptos no se digitan.
</div>
</div>
</div>
)}
</>
)}
</>
) : (
<>
<label className="wiz-field-label">Archivo CSV</label>
<div className="folder-picker">
<div className="folder-picker-display">
{file ? (
<>
<FileSpreadsheet size={16} className="folder-picker-icon" />
<span className="folder-picker-path">{file.name}</span>
</>
) : (
<span className="folder-picker-placeholder">Ningún archivo seleccionado</span>
)}
</div>
<button
type="button"
className="btn btn-primary"
onClick={() => fileInputRef.current?.click()}
>
<FileSpreadsheet size={15} />
Seleccionar CSV
</button>
<input
ref={fileInputRef}
type="file"
accept=".csv,text/csv"
hidden
onChange={(e) => setFile(e.target.files?.[0] ?? null)}
/>
</div>
<p className="wiz-hint" style={{ marginTop: 8 }}>
Columnas: radicado, documento, ingreso, total_cargos, autorizacion,
tipo_documento, tipo_servicio, copago, cuota_moderadora (también acepta
headers IT).
</p>
</>
)}
</div>
{/* 2. Bots */}
<div className="wiz-section">
<div className="wiz-section-header">
<h3 className="wiz-section-title">
2. Bots
{onlineWorkers.length > 0 && (
<span className="wiz-count-badge">{onlineWorkers.length} online</span>
)}
</h3>
<button
type="button"
className="btn btn-sm btn-ghost"
onClick={() => void refreshWorkers()}
disabled={refreshing}
>
{refreshing ? <Loader2 size={13} className="spin" /> : <RefreshCw size={13} />}
Actualizar
</button>
</div>
<p className="wiz-desc">
El agent debe estar encendido en la PC con Integración Total. Con varios bots, las
filas se reparte automáticamente.
</p>
{onlineWorkers.length >= 2 && (
<label
style={{
display: "flex",
alignItems: "center",
gap: 8,
marginTop: 12,
marginBottom: 8,
cursor: "pointer",
}}
>
<input
type="checkbox"
checked={useAllOnline}
onChange={(e) => {
const on = e.target.checked;
setUseAllOnline(on);
if (on) setSelectedIds(onlineWorkers.map((w) => w.id));
}}
/>
<span>Usar todos los bots online ({onlineWorkers.length})</span>
</label>
)}
{workers.length === 0 ? (
<div className="wiz-estimate-box" style={{ marginTop: 12, textAlign: "left" }}>
<div style={{ display: "flex", alignItems: "flex-start", gap: 10 }}>
<Bot size={18} style={{ flexShrink: 0, marginTop: 2 }} />
<div>
<div style={{ fontWeight: 600 }}>Ningún bot conectado</div>
<p className="wiz-hint" style={{ margin: "4px 0 0" }}>
En la PC del digitador ejecuta <code>run_gui.py</code> del agent y conéctalo
al gateway.
</p>
</div>
</div>
</div>
) : (
<div className="wiz-review-grid" style={{ marginTop: 12 }}>
{workers.map((w) => {
const online = w.status === "online" || w.status === "busy";
const selected = effectiveTargets.includes(w.id);
return (
<button
key={w.id}
type="button"
className={`wiz-review-item${selected ? " wiz-review-item--selected" : ""}`}
disabled={!online || useAllOnline}
onClick={() => online && toggleWorker(w.id)}
style={{
textAlign: "left",
cursor: online && !useAllOnline ? "pointer" : "not-allowed",
opacity: online ? 1 : 0.45,
borderColor: selected ? "var(--accent, #3b8ed0)" : undefined,
}}
>
<div>
<span className="wiz-review-label">Worker</span>
<span className="wiz-review-value">{w.id}</span>
</div>
<div>
<span className="wiz-review-label">Estado</span>
<span className="wiz-review-value">
{w.status}
{w.hostname ? ` · ${w.hostname}` : ""}
{selected ? " · seleccionado" : ""}
</span>
</div>
</button>
);
})}
</div>
)}
{onlineWorkers.length === 0 && workers.length > 0 && (
<p className="wiz-hint" style={{ marginTop: 8 }}>
Hay workers registrados pero ninguno online. Revisa el agent.
</p>
)}
{effectiveTargets.length > 1 && (
<p className="wiz-hint" style={{ marginTop: 8 }}>
Se repartirá entre {effectiveTargets.length} bots.
</p>
)}
</div>
</div>
)}
{step === 2 && activeJob && (
<div className="wiz-step wiz-step-centered">
<div className="wiz-hero">
<div className={`wiz-hero-icon ${running ? "wiz-hero-icon--go" : ""}`}>
{running ? (
<Loader2 size={28} className="spin" />
) : activeJob.status === "completed" ? (
<Keyboard size={28} />
) : activeJob.status === "failed" ? (
<AlertCircle size={28} />
) : (
<Bot size={28} />
)}
</div>
<h3 className="wiz-hero-title">
{running
? isDistributed
? "Digitación distribuida en curso"
: "Digitación en curso"
: activeJob.status === "completed"
? "Digitación completada"
: activeJob.status === "cancelled"
? "Digitación cancelada"
: activeJob.status === "failed"
? "Digitación fallida"
: "Digitación finalizada"}
</h3>
<p className="wiz-desc">
{isDistributed
? `${Number(activeJob.shard_count) || shards.length || "?"} bots`
: `Bot ${activeJob.worker_id ?? "—"}`}
{activeJob.original_filename ? ` · ${activeJob.original_filename}` : ""}
</p>
</div>
{activeJob.error && (
<div className="wiz-toast wiz-toast--error" style={{ position: "static", marginBottom: 12 }}>
<AlertCircle size={15} />
<span>{activeJob.error}</span>
</div>
)}
<div className="wiz-progress">
<div className="wiz-progress-header">
<span>
{Number(result.exitosos ?? processed)}/{total || "?"} filas
</span>
<span>{pct}%</span>
</div>
<div className="wiz-progress-track">
<div className="wiz-progress-fill" style={{ width: `${pct}%` }} />
</div>
</div>
<div className="wiz-stats-row">
<div className="wiz-stat">
<span className="wiz-stat-value">{String(result.exitosos ?? processed)}</span>
<span className="wiz-stat-label">OK</span>
</div>
<div className="wiz-stat">
<span className="wiz-stat-value">{String(result.fallidos ?? 0)}</span>
<span className="wiz-stat-label">Fallidos</span>
</div>
<div className="wiz-stat">
<span className="wiz-stat-value">{String(result.existentes ?? 0)}</span>
<span className="wiz-stat-label">Existentes</span>
</div>
<div className="wiz-stat">
<span className="wiz-stat-value">{String(result.timeouts ?? 0)}</span>
<span className="wiz-stat-label">Timeouts</span>
</div>
</div>
{shards.length > 0 && (
<div className="wiz-section" style={{ marginTop: 16, width: "100%", maxWidth: 640 }}>
<h3 className="wiz-section-title">Shards por bot</h3>
<div className="wiz-review-grid" style={{ marginTop: 8 }}>
{shards.map((s, i) => (
<div key={s.id || String(i)} className="wiz-review-item">
<div>
<span className="wiz-review-label">
Shard {(s.shard_index ?? i) + 1}
</span>
<span className="wiz-review-value">{s.worker_id || "—"}</span>
</div>
<div>
<span className="wiz-review-label">Estado</span>
<span className="wiz-review-value">{s.status || "—"}</span>
</div>
</div>
))}
</div>
</div>
)}
<div className="wiz-section" style={{ marginTop: 16, width: "100%", maxWidth: 640 }}>
<h3 className="wiz-section-title">Log del agent</h3>
<div
className="wiz-estimate-box"
style={{
maxHeight: 220,
overflow: "auto",
fontFamily: "ui-monospace, SFMono-Regular, Menlo, monospace",
fontSize: 12,
textAlign: "left",
}}
>
{events.length === 0 ? (
<span className="wiz-hint">Esperando eventos…</span>
) : (
events.map((ev) => (
<div key={ev.id}>
[{ev.timestamp}] {ev.message}
</div>
))
)}
</div>
</div>
</div>
)}
</div>
{error && (
<div className="wiz-toast wiz-toast--error">
<AlertCircle size={15} />
<span>{error}</span>
</div>
)}
<footer className="wiz-nav">
{step === 2 ? (
<button type="button" className="btn" onClick={resetWizard} disabled={running}>
Nueva digitación
</button>
) : (
<span />
)}
<div className="wiz-nav-dots">
<button
type="button"
className={`wiz-nav-dot ${step === 1 ? "active" : ""}`}
onClick={() => !running && setStep(1)}
disabled={running}
aria-label="Paso 1"
/>
<button
type="button"
className={`wiz-nav-dot ${step === 2 ? "active" : ""}`}
onClick={() => activeJob && setStep(2)}
disabled={!activeJob}
aria-label="Paso 2"
/>
</div>
{step === 1 && (
<button
type="button"
className="btn btn-primary"
disabled={!canStart}
onClick={() => void startJob()}
>
{busy ? <Loader2 size={15} className="spin" /> : <Play size={15} />}
{effectiveTargets.length > 1
? `Digitar con ${effectiveTargets.length} bots`
: sourceMode === "aptos" && aptosCount > 0
? `Digitar ${aptosCount} aptos`
: "Iniciar digitación"}
</button>
)}
{step === 2 && running && (
<button type="button" className="btn" disabled={busy} onClick={() => void cancelJob()}>
<Square size={14} /> Cancelar
</button>
)}
{step === 2 && !running && (
<button type="button" className="btn btn-primary" onClick={resetWizard}>
<Play size={15} /> Otra digitación
</button>
)}
</footer>
</div>
);
}
@@ -1,4 +1,4 @@
import { Download, Loader2, RefreshCw, Users } from "lucide-react";
import { Download, Keyboard, Loader2, RefreshCw, Users } from "lucide-react";
import { useEffect, useState } from "react";
import type { PoblacionAptaJob } from "../../../shared/types";
import { PoblacionAptaResultsTable } from "../../components/PoblacionAptaResultsTable";
@@ -7,7 +7,7 @@ import type { ModuleResultsProps } from "../registry";
type Row = Record<string, unknown>;
export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps) {
export function PoblacionAptaResults({ client, task, job, onActivateModule }: ModuleResultsProps) {
const [paJob, setPaJob] = useState<PoblacionAptaJob | null>(null);
const [rows, setRows] = useState<Row[]>([]);
const [loading, setLoading] = useState(true);
@@ -60,6 +60,16 @@ export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps)
downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text);
}
async function exportTodos() {
const paJobId = String(
(task.result as Record<string, unknown>)?.poblacion_apta_job_id ?? "",
);
if (!paJobId) return;
const blob = await client.poblacionAptaExportCsv(paJobId, "todos");
const text = await blob.text();
downloadCsv(buildExportCsvFilename("poblacion_apta_todos", job.title), text);
}
if (loading) {
return (
<div className="module-loading">
@@ -88,6 +98,10 @@ export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps)
}
const summary = paJob?.summary ?? {};
const paJobId = String(
(task.result as Record<string, unknown>)?.poblacion_apta_job_id ?? paJob?.id ?? "",
);
const canDigitador = paJob?.status === "completed" && Boolean(paJobId) && Boolean(onActivateModule);
return (
<div className="module-results">
@@ -108,6 +122,23 @@ export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps)
<button type="button" className="btn btn-sm" onClick={() => void exportNoAptos()}>
<Download size={13} /> CSV no aptos ({summary.con_error ?? 0})
</button>
<button type="button" className="btn btn-sm" onClick={() => void exportTodos()}>
<Download size={13} /> CSV todos ({summary.total ?? 0})
</button>
{canDigitador && (
<button
type="button"
className="btn btn-sm btn-primary"
disabled={Number(summary.paso_bot ?? 0) === 0}
onClick={() =>
onActivateModule?.("digitador", {
poblacionAptaJobId: paJobId,
})
}
>
<Keyboard size={13} /> Digitar aptos
</button>
)}
</div>
</div>
@@ -3,6 +3,7 @@ import {
CheckSquare,
Download,
FileSpreadsheet,
Keyboard,
Loader2,
Square,
Upload,
@@ -235,6 +236,13 @@ export function PoblacionAptaWizard({
downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text);
}
async function exportTodos() {
if (!resultJob) return;
const blob = await client.poblacionAptaExportCsv(resultJob.id, "todos");
const text = await blob.text();
downloadCsv(buildExportCsvFilename("poblacion_apta_todos", job.title), text);
}
function reset() {
setDone(false);
setResultJob(null);
@@ -271,6 +279,23 @@ export function PoblacionAptaWizard({
<button type="button" className="btn btn-sm" onClick={() => void exportNoAptos()}>
<Download size={13} /> CSV no aptos ({summary.con_error ?? 0})
</button>
<button type="button" className="btn btn-sm" onClick={() => void exportTodos()}>
<Download size={13} /> CSV todos ({summary.total ?? 0})
</button>
{onActivateModule && (
<button
type="button"
className="btn btn-sm btn-primary"
disabled={Number(summary.paso_bot ?? 0) === 0}
onClick={() =>
onActivateModule("digitador", {
poblacionAptaJobId: resultJob.id,
})
}
>
<Keyboard size={13} /> Digitar aptos
</button>
)}
<button type="button" className="btn btn-sm" onClick={reset}>
Nueva corrida
</button>
@@ -4,4 +4,5 @@ export type JobSection =
| "cleaning"
| "poblacion_apta"
| "downloads"
| "digitador"
| "results";
@@ -5,6 +5,7 @@ import {
Cpu,
Download,
Hash,
Keyboard,
Loader2,
Plus,
RefreshCw,
@@ -29,8 +30,9 @@ import type { ApiClient } from "../lib/apiClient";
import { downloadCsv, toCsv, buildExportCsvFilename } from "../lib/csv";
import { taskTimingLabel, timingDuration } from "../lib/time";
import { DownloadsResultsPanel } from "../modules/downloads/DownloadsResultsPanel";
import { DigitadorResultsPanel } from "../modules/digitador/DigitadorResultsPanel";
type ResultTab = "counting" | "extraction" | "cleaning" | "poblacion_apta" | "downloads";
type ResultTab = "counting" | "extraction" | "cleaning" | "poblacion_apta" | "downloads" | "digitador";
interface Props {
client: ApiClient;
@@ -46,11 +48,15 @@ interface Props {
onStartCleaning: (ctx?: Record<string, unknown>) => void;
onStartPoblacionApta: () => void;
onStartDownload: () => void;
onStartDigitador: () => void;
onContinueToCounting?: (ctx: {
downloadJobId: string;
downloadsPrefix?: string;
files?: DownloadJobFile[];
}) => void;
onContinueToDigitador?: (ctx: {
poblacionAptaJobId: string;
}) => void;
}
export function JobResultsView({
@@ -67,7 +73,9 @@ export function JobResultsView({
onStartCleaning,
onStartPoblacionApta,
onStartDownload,
onStartDigitador,
onContinueToCounting,
onContinueToDigitador,
}: Props) {
const [activeTab, setActiveTab] = useState<ResultTab>("counting");
const [refreshing, setRefreshing] = useState(false);
@@ -82,6 +90,8 @@ export function JobResultsView({
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
const downloadTasks = tasks.filter((t) => t.type === "downloads")
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
const digitadorTasks = tasks.filter((t) => t.type === "digitador")
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
async function refresh() {
setRefreshing(true);
@@ -164,6 +174,17 @@ export function JobResultsView({
<span className="job-results-tab-count">{downloadTasks.length}</span>
)}
</button>
<button
type="button"
className={`job-results-tab${activeTab === "digitador" ? " job-results-tab--active" : ""}`}
onClick={() => setActiveTab("digitador")}
>
<Keyboard size={15} />
Bot digitador
{digitadorTasks.length > 0 && (
<span className="job-results-tab-count">{digitadorTasks.length}</span>
)}
</button>
<div style={{ flex: 1 }} />
<button
type="button"
@@ -216,6 +237,7 @@ export function JobResultsView({
tasks={poblacionAptaTasks}
liveProgress={poblacionAptaProgress}
onStartPoblacionApta={onStartPoblacionApta}
onContinueToDigitador={onContinueToDigitador}
/>
)}
{activeTab === "downloads" && (
@@ -227,6 +249,14 @@ export function JobResultsView({
onContinueToCounting={onContinueToCounting}
/>
)}
{activeTab === "digitador" && (
<DigitadorResultsPanel
client={client}
job={job}
tasks={digitadorTasks}
onStartDigitador={onStartDigitador}
/>
)}
</div>
</div>
);
@@ -780,12 +810,16 @@ function PoblacionAptaResultsSection({
tasks,
liveProgress,
onStartPoblacionApta,
onContinueToDigitador,
}: {
client: ApiClient;
job: Job;
tasks: Task[];
liveProgress?: BackgroundProgress | null;
onStartPoblacionApta: () => void;
onContinueToDigitador?: (ctx: {
poblacionAptaJobId: string;
}) => void;
}) {
const [selectedTask, setSelectedTask] = useState<Task | null>(tasks[0] ?? null);
const [paJob, setPaJob] = useState<PoblacionAptaJob | null>(null);
@@ -853,6 +887,16 @@ function PoblacionAptaResultsSection({
downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text);
}
async function exportTodos() {
const paJobId = String(
(selectedTask?.result as Record<string, unknown>)?.poblacion_apta_job_id ?? "",
);
if (!paJobId) return;
const blob = await client.poblacionAptaExportCsv(paJobId, "todos");
const text = await blob.text();
downloadCsv(buildExportCsvFilename("poblacion_apta_todos", job.title), text);
}
if (tasks.length === 0 && !liveProgress?.running) {
return (
<div className="job-results-empty">
@@ -874,6 +918,13 @@ function PoblacionAptaResultsSection({
}
const summary = paJob?.summary ?? {};
const paJobId = String(
(selectedTask?.result as Record<string, unknown>)?.poblacion_apta_job_id ?? paJob?.id ?? "",
);
const canDigitador =
Boolean(onContinueToDigitador) &&
paJob?.status === "completed" &&
Boolean(paJobId);
return (
<div className="results-section">
@@ -917,6 +968,21 @@ function PoblacionAptaResultsSection({
<button type="button" className="btn btn-sm" onClick={() => void exportNoAptos()}>
<Download size={13} /> CSV no aptos ({summary.con_error ?? 0})
</button>
<button type="button" className="btn btn-sm" onClick={() => void exportTodos()}>
<Download size={13} /> CSV todos ({summary.total ?? 0})
</button>
{canDigitador && (
<button
type="button"
className="btn btn-sm btn-primary"
onClick={() =>
onContinueToDigitador?.({ poblacionAptaJobId: paJobId })
}
disabled={Number(summary.paso_bot ?? 0) === 0}
>
<Keyboard size={13} /> Digitar aptos
</button>
)}
<button type="button" className="btn btn-sm btn-ghost" onClick={onStartPoblacionApta}>
<Plus size={13} /> Nueva corrida
</button>
@@ -9,6 +9,7 @@ import { CleaningWizardModule } from "../modules/cleaning/CleaningWizardModule";
import { PoblacionAptaWizardModule } from "../modules/poblacion_apta/PoblacionAptaWizardModule";
import { JobExtractionWizard } from "../modules/extraction/JobExtractionWizard";
import { DownloadsWizardModule } from "../modules/downloads/DownloadsWizardModule";
import { DigitadorWizardModule } from "../modules/digitador/DigitadorWizardModule";
import { JobResultsView } from "./JobResultsView";
interface Props {
@@ -127,6 +128,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
{section === "cleaning" && "Limpieza de datos"}
{section === "poblacion_apta" && "Población apta"}
{section === "downloads" && "Bot de descarga"}
{section === "digitador" && "Bot digitador"}
{section === "results" && "Resultados del trabajo"}
</div>
</header>
@@ -201,6 +203,19 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
<DownloadsWizardModule client={client} job={job} />
</div>
<div className={`job-section-pane${section === "digitador" ? "" : " job-section-pane--hidden"}`}>
<DigitadorWizardModule
key={`digitador-${String(sectionContext.poblacionAptaJobId ?? "default")}`}
client={client}
job={job}
initialSource={
typeof sectionContext.poblacionAptaJobId === "string"
? { poblacionAptaJobId: sectionContext.poblacionAptaJobId }
: null
}
/>
</div>
<div className={`job-section-pane${section === "results" ? "" : " job-section-pane--hidden"}`}>
<JobResultsView
client={client}
@@ -216,7 +231,9 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
onStartCleaning={(ctx?: Record<string, unknown>) => onActivateSection("cleaning", ctx)}
onStartPoblacionApta={() => onActivateSection("poblacion_apta")}
onStartDownload={() => onActivateSection("downloads")}
onStartDigitador={() => onActivateSection("digitador")}
onContinueToCounting={(ctx) => onActivateSection("counting", ctx)}
onContinueToDigitador={(ctx) => onActivateSection("digitador", ctx)}
/>
</div>
</div>
+63 -1
View File
@@ -373,7 +373,7 @@ export interface DownloadWorker {
export interface DownloadJob {
id: string;
worker_id: string;
worker_id: string | null;
status: "pending" | "running" | "completed" | "failed" | "cancelled" | string;
title: string;
excel_key: string;
@@ -389,6 +389,11 @@ export interface DownloadJob {
job_id?: string | null;
/** Task type=downloads */
task_id?: string | null;
parent_download_job_id?: string | null;
shard_index?: number | null;
shard_count?: number | null;
/** Presente en GET de padre distribuido */
shards?: DownloadJob[];
}
export interface DownloadJobFile {
@@ -417,6 +422,63 @@ export interface DownloadJobEvent {
metadata?: Record<string, unknown>;
}
/** Worker del bot digitador (control remoto). */
export interface DigitadorWorker {
id: string;
hostname: string;
max_concurrent: number;
status: "online" | "offline" | "busy" | string;
last_heartbeat: string;
current_job_id: string | null;
}
export interface DigitadorJob {
id: string;
worker_id: string | null;
status: "pending" | "running" | "completed" | "failed" | "cancelled" | string;
title: string;
csv_key: string;
reports_prefix: string;
original_filename: string;
cancel_requested: boolean;
result: Record<string, unknown>;
error: string | null;
created_at: string;
started_at: string | null;
finished_at: string | null;
job_id?: string | null;
task_id?: string | null;
parent_digitador_job_id?: string | null;
shard_index?: number | null;
shard_count?: number | null;
shards?: DigitadorJob[];
}
export interface DigitadorJobFile {
key: string;
name?: string | null;
}
export interface DigitadorJobFilesResponse {
job_id: string;
prefix: string;
files: DigitadorJobFile[];
count: number;
}
export interface DigitadorJobEvent {
id: number;
job_id: string;
event_type: string;
message: string;
timestamp: string;
radicado?: string | null;
nit?: string | null;
index?: number | null;
total?: number | null;
metadata?: Record<string, unknown>;
}
export interface DesktopBridge {
getRuntimeConfig(): Promise<RuntimeConfig>;
selectDirectory(): Promise<string | null>;
+9 -3
View File
@@ -15,6 +15,7 @@ from domain.verificador_st import (
consolidado_to_records,
export_no_aptos_csv,
export_paso_bot_csv,
export_todos_csv,
process_cleaning_sources,
)
from domain.verificador_st.core import cargar_origen_servicio_desde_bytes
@@ -483,11 +484,13 @@ def get_rows(
@app.get("/jobs/{job_id}/export.csv")
def export_csv(
job_id: str,
subset: str = Query("aptos", description="aptos | no_aptos"),
subset: str = Query("aptos", description="aptos | no_aptos | todos"),
) -> Response:
subset_norm = (subset or "aptos").strip().lower()
if subset_norm not in {"aptos", "no_aptos", "errores"}:
raise HTTPException(status_code=400, detail="subset debe ser aptos o no_aptos")
if subset_norm in {"all", "completo", "consolidado"}:
subset_norm = "todos"
if subset_norm not in {"aptos", "no_aptos", "errores", "todos"}:
raise HTTPException(status_code=400, detail="subset debe ser aptos, no_aptos o todos")
if subset_norm == "errores":
subset_norm = "no_aptos"
@@ -504,6 +507,9 @@ def export_csv(
if subset_norm == "aptos":
csv_text = export_paso_bot_csv(payload)
filename = f"poblacion_apta_aptos_{job_id[:8]}.csv"
elif subset_norm == "todos":
csv_text = export_todos_csv(payload)
filename = f"poblacion_apta_todos_{job_id[:8]}.csv"
else:
csv_text = export_no_aptos_csv(payload)
filename = f"poblacion_apta_no_aptos_{job_id[:8]}.csv"
@@ -3,6 +3,7 @@ from domain.verificador_st.pipeline import (
consolidado_to_records,
export_no_aptos_csv,
export_paso_bot_csv,
export_todos_csv,
process_cleaning_sources,
)
@@ -10,5 +11,6 @@ __all__ = [
"consolidado_to_records",
"export_no_aptos_csv",
"export_paso_bot_csv",
"export_todos_csv",
"process_cleaning_sources",
]
@@ -51,6 +51,8 @@ TABLA_MODERADORA = {
2025: {4700, 19200, 50300},
2026: {5000, 20100, 52800},
}
# Todos los montos de la tabla; la clasificación no filtra por año.
VALORES_MODERADORA = {v for vals in TABLA_MODERADORA.values() for v in vals}
# Longitudes según tabla entregada por el usuario.
# Los tipos marcados como alfanuméricos permiten letras y números.
@@ -554,14 +556,15 @@ def normalizar_autorizacion(valor):
def separar_copago_moderadora(valor, anio):
"""
Si el valor coincide con tabla moderadora del año => cuota_moderadora.
Si no coincide => copago.
Si el valor coincide con cualquier cuota de la tabla (sin importar el año)
=> cuota_moderadora. Si no coincide => copago.
El parámetro anio se conserva por compatibilidad de firma.
"""
monto = to_int(valor, default=0)
if monto == 0:
return 0, 0
if anio in TABLA_MODERADORA and monto in TABLA_MODERADORA[anio]:
if monto in VALORES_MODERADORA:
return 0, monto
return monto, 0
@@ -223,3 +223,21 @@ def export_no_aptos_csv(rows: list[dict[str, Any]]) -> str:
df = df.rename(columns=CSV_FINAL_IT_RENAME)
df = df.reindex(columns=headers)
return df.to_csv(sep="|", index=False, encoding="utf-8-sig")
def export_todos_csv(rows: list[dict[str, Any]]) -> str:
"""CSV pipe-separated con todas las filas (aptos y no aptos) e OBSERVACION."""
headers = [*CSV_FINAL_IT_HEADERS, "OBSERVACION"]
if not rows:
df = pd.DataFrame(columns=headers)
else:
df = pd.DataFrame(rows)
for c in COLUMNAS_FINAL_IT:
if c not in df.columns:
df[c] = ""
if "OBSERVACION" not in df.columns:
df["OBSERVACION"] = ""
df = df.reindex(columns=[*COLUMNAS_FINAL_IT, "OBSERVACION"]).fillna("")
df = df.rename(columns=CSV_FINAL_IT_RENAME)
df = df.reindex(columns=headers)
return df.to_csv(sep="|", index=False, encoding="utf-8-sig")