feat: Introduce bot-digitador service for automated data entry in Integración Total. Add CLI and GUI interfaces for CSV processing, implement auto-start functionality on Windows, and enhance README with setup instructions and usage details. Include necessary configurations in docker-compose for service integration.
This commit is contained in:
@@ -20,6 +20,7 @@ CLEANING_URL = os.getenv("CLEANING_URL", "http://cleaning:8002").rstrip("/")
|
||||
POBLACION_APTA_URL = os.getenv("POBLACION_APTA_URL", "http://poblacion_apta:8003").rstrip("/")
|
||||
COUNT_URL = os.getenv("COUNT_URL", "http://count-api:8080").rstrip("/")
|
||||
BOT_DESCARGA_URL = os.getenv("BOT_DESCARGA_URL", "http://bot-descarga:8200").rstrip("/")
|
||||
BOT_DIGITADOR_URL = os.getenv("BOT_DIGITADOR_URL", "http://bot-digitador:8210").rstrip("/")
|
||||
|
||||
# Timeouts: JSON ligero vs uploads OCR/conteo
|
||||
_TIMEOUT_JSON = httpx.Timeout(connect=5.0, read=30.0, write=30.0, pool=5.0)
|
||||
@@ -36,12 +37,13 @@ async def lifespan(app: FastAPI):
|
||||
limits=httpx.Limits(max_connections=100, max_keepalive_connections=20),
|
||||
)
|
||||
_log.info(
|
||||
"Gateway iniciado — extraction=%s cleaning=%s poblacion_apta=%s count=%s bot_descarga=%s",
|
||||
"Gateway iniciado — extraction=%s cleaning=%s poblacion_apta=%s count=%s bot_descarga=%s bot_digitador=%s",
|
||||
EXTRACTION_URL,
|
||||
CLEANING_URL,
|
||||
POBLACION_APTA_URL,
|
||||
COUNT_URL,
|
||||
BOT_DESCARGA_URL,
|
||||
BOT_DIGITADOR_URL,
|
||||
)
|
||||
yield
|
||||
await _http_client.aclose()
|
||||
@@ -84,6 +86,7 @@ def _is_upload_path(path: str) -> bool:
|
||||
or path.startswith("/poblacion-apta/jobs/from-csv")
|
||||
or path.startswith("/poblacion-apta/jobs/from-cleaning")
|
||||
or path.startswith("/bot-descarga/jobs")
|
||||
or path.startswith("/bot-digitador/jobs")
|
||||
)
|
||||
|
||||
|
||||
@@ -105,6 +108,7 @@ async def health() -> dict[str, Any]:
|
||||
poblacion_apta = await _safe_health(client, POBLACION_APTA_URL)
|
||||
counting = await _safe_health(client, COUNT_URL)
|
||||
bot_descarga = await _safe_health(client, BOT_DESCARGA_URL)
|
||||
bot_digitador = await _safe_health(client, BOT_DIGITADOR_URL)
|
||||
return {
|
||||
"status": "ok" if extraction["ok"] and cleaning["ok"] and poblacion_apta["ok"] else "degraded",
|
||||
"services": {
|
||||
@@ -113,6 +117,7 @@ async def health() -> dict[str, Any]:
|
||||
"poblacion_apta": poblacion_apta,
|
||||
"counting": counting,
|
||||
"bot_descarga": bot_descarga,
|
||||
"bot_digitador": bot_digitador,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -183,6 +188,11 @@ async def bot_descarga_proxy(path: str, request: Request) -> Response:
|
||||
return await _proxy_request(request, f"{BOT_DESCARGA_URL}/{path}")
|
||||
|
||||
|
||||
@app.api_route("/bot-digitador/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
|
||||
async def bot_digitador_proxy(path: str, request: Request) -> Response:
|
||||
return await _proxy_request(request, f"{BOT_DIGITADOR_URL}/{path}")
|
||||
|
||||
|
||||
@app.api_route("/jobs", methods=["GET", "POST"])
|
||||
async def jobs_root_proxy(request: Request) -> Response:
|
||||
return await _proxy_request(request, f"{EXTRACTION_URL}/jobs")
|
||||
|
||||
@@ -87,26 +87,30 @@ def claim_pending(
|
||||
|
||||
@app.post("/jobs")
|
||||
async def create_job(
|
||||
worker_id: str = Form(...),
|
||||
worker_id: str = Form(""),
|
||||
worker_ids: str = Form(""),
|
||||
job_id: str = Form(...),
|
||||
title: str = Form(""),
|
||||
file: UploadFile = File(...),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
if not worker_id.strip():
|
||||
raise HTTPException(status_code=400, detail="worker_id obligatorio")
|
||||
"""Crea una corrida de descarga.
|
||||
|
||||
- 1 worker → flujo clásico (un solo job claimable).
|
||||
- N workers (worker_ids JSON o todos online) → padre + N shards,
|
||||
mismo downloads_prefix del padre.
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
from domain.excel_split import parse_radicado_rows, rows_to_xlsx_bytes, split_rows
|
||||
from infrastructure.storage_minio import downloads_prefix as mk_prefix
|
||||
|
||||
parent_job_id = job_id.strip()
|
||||
if not parent_job_id:
|
||||
raise HTTPException(status_code=400, detail="job_id (Job Desktop) obligatorio")
|
||||
if not store.parent_job_exists(parent_job_id):
|
||||
raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado")
|
||||
|
||||
if not store.is_worker_online(worker_id):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"Worker '{worker_id}' no está online. Enciende el agent primero.",
|
||||
)
|
||||
|
||||
data = await file.read()
|
||||
if not data:
|
||||
raise HTTPException(status_code=400, detail="Archivo vacío")
|
||||
@@ -115,45 +119,224 @@ async def create_job(
|
||||
content_type = file.content_type or "application/octet-stream"
|
||||
title_clean = title.strip()
|
||||
|
||||
job = store.create_job(
|
||||
worker_id=worker_id.strip(),
|
||||
# Resolver workers
|
||||
selected: List[str] = []
|
||||
raw_ids = (worker_ids or "").strip()
|
||||
if raw_ids:
|
||||
try:
|
||||
parsed = _json.loads(raw_ids)
|
||||
if isinstance(parsed, list):
|
||||
selected = [str(x).strip() for x in parsed if str(x).strip()]
|
||||
elif isinstance(parsed, str) and parsed.strip():
|
||||
selected = [parsed.strip()]
|
||||
except _json.JSONDecodeError:
|
||||
selected = [x.strip() for x in raw_ids.split(",") if x.strip()]
|
||||
if not selected and worker_id.strip():
|
||||
selected = [worker_id.strip()]
|
||||
|
||||
online = store.list_online_worker_ids()
|
||||
online_set = set(online)
|
||||
if not selected:
|
||||
selected = list(online)
|
||||
else:
|
||||
missing = [w for w in selected if w not in online_set]
|
||||
if missing:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"Workers no online: {', '.join(missing)}. Enciende el agent primero.",
|
||||
)
|
||||
|
||||
if not selected:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="No hay bots online. Enciende al menos un agent.",
|
||||
)
|
||||
|
||||
# Dedup preservando orden
|
||||
seen: set[str] = set()
|
||||
workers_unique: List[str] = []
|
||||
for w in selected:
|
||||
if w not in seen:
|
||||
seen.add(w)
|
||||
workers_unique.append(w)
|
||||
|
||||
# Parse filas
|
||||
try:
|
||||
rows = parse_radicado_rows(data, original)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=f"No se pudo leer el Excel/CSV: {e}") from e
|
||||
if not rows:
|
||||
raise HTTPException(status_code=400, detail="El archivo no tiene filas de radicado/NIT")
|
||||
|
||||
# Un solo bot (o una sola fila) → flujo actual
|
||||
n_workers = min(len(workers_unique), len(rows))
|
||||
if n_workers <= 1:
|
||||
only = workers_unique[0]
|
||||
job = store.create_job(
|
||||
worker_id=only,
|
||||
excel_key="",
|
||||
original_filename=original,
|
||||
title=title_clean,
|
||||
job_id=parent_job_id,
|
||||
)
|
||||
task_id = store.create_downloads_task(
|
||||
parent_job_id=parent_job_id,
|
||||
download_job_id=job.id,
|
||||
worker_id=only,
|
||||
title=title_clean or f"Bot descarga ({original})",
|
||||
original_filename=original,
|
||||
worker_ids=[only],
|
||||
mode="single",
|
||||
)
|
||||
try:
|
||||
key = minio.upload_excel(job.id, data, original, content_type=content_type)
|
||||
except Exception as e:
|
||||
store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
|
||||
|
||||
job = store.update_excel_key(job.id, key)
|
||||
assert job is not None
|
||||
store.add_event(
|
||||
job.id,
|
||||
"run_started",
|
||||
f"Job creado y asignado a {only}",
|
||||
metadata={
|
||||
"excel_key": key,
|
||||
"worker_id": only,
|
||||
"job_id": parent_job_id,
|
||||
"task_id": task_id,
|
||||
"mode": "single",
|
||||
"total_rows": len(rows),
|
||||
},
|
||||
)
|
||||
return job.to_dict()
|
||||
|
||||
# Distribuido: padre + shards
|
||||
workers_unique = workers_unique[:n_workers]
|
||||
chunks = split_rows(rows, n_workers)
|
||||
n = len(chunks)
|
||||
workers_unique = workers_unique[:n]
|
||||
|
||||
parent = store.create_job(
|
||||
worker_id=None,
|
||||
excel_key="",
|
||||
original_filename=original,
|
||||
title=title_clean,
|
||||
title=title_clean or f"Descarga distribuida ({original})",
|
||||
job_id=parent_job_id,
|
||||
)
|
||||
task_id = store.create_downloads_task(
|
||||
parent_job_id=parent_job_id,
|
||||
download_job_id=job.id,
|
||||
worker_id=worker_id.strip(),
|
||||
title=title_clean or f"Bot descarga ({original})",
|
||||
original_filename=original,
|
||||
)
|
||||
job = store.get_job(job.id)
|
||||
assert job is not None
|
||||
|
||||
try:
|
||||
key = minio.upload_excel(job.id, data, original, content_type=content_type)
|
||||
except Exception as e:
|
||||
store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
|
||||
|
||||
job = store.update_excel_key(job.id, key)
|
||||
assert job is not None
|
||||
store.add_event(
|
||||
job.id,
|
||||
"run_started",
|
||||
f"Job creado y asignado a {worker_id}",
|
||||
metadata={
|
||||
"excel_key": key,
|
||||
"worker_id": worker_id,
|
||||
"job_id": parent_job_id,
|
||||
"task_id": task_id,
|
||||
shard_count=n,
|
||||
status=JobStatus.PENDING.value,
|
||||
result={
|
||||
"mode": "distributed",
|
||||
"shard_ids": [],
|
||||
"total_rows": len(rows),
|
||||
"total": len(rows),
|
||||
"processed": 0,
|
||||
"exitosos": 0,
|
||||
"fallidos": 0,
|
||||
},
|
||||
)
|
||||
prefix = parent.downloads_prefix or mk_prefix(parent.id)
|
||||
|
||||
task_id = store.create_downloads_task(
|
||||
parent_job_id=parent_job_id,
|
||||
download_job_id=parent.id,
|
||||
worker_id=None,
|
||||
title=title_clean or f"Bot descarga ×{n} ({original})",
|
||||
original_filename=original,
|
||||
worker_ids=workers_unique,
|
||||
mode="distributed",
|
||||
)
|
||||
|
||||
try:
|
||||
full_key = minio.upload_excel(parent.id, data, original, content_type=content_type)
|
||||
except Exception as e:
|
||||
store.patch_job(parent.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
|
||||
|
||||
store.update_excel_key(parent.id, full_key)
|
||||
|
||||
shard_ids: List[str] = []
|
||||
for i, (chunk, wid) in enumerate(zip(chunks, workers_unique)):
|
||||
shard_bytes = rows_to_xlsx_bytes(chunk)
|
||||
shard = store.create_job(
|
||||
worker_id=wid,
|
||||
excel_key="",
|
||||
original_filename=f"shard_{i + 1}_of_{n}.xlsx",
|
||||
title=f"{title_clean or 'Descarga'} · shard {i + 1}/{n}",
|
||||
job_id=parent_job_id,
|
||||
task_id=task_id,
|
||||
downloads_prefix_override=prefix,
|
||||
parent_download_job_id=parent.id,
|
||||
shard_index=i,
|
||||
shard_count=n,
|
||||
result={"total": len(chunk), "processed": 0},
|
||||
)
|
||||
try:
|
||||
shard_key = minio.upload_excel(
|
||||
shard.id,
|
||||
shard_bytes,
|
||||
f"shard_{i + 1}_of_{n}.xlsx",
|
||||
content_type=(
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
store.patch_job(
|
||||
parent.id,
|
||||
status=JobStatus.FAILED.value,
|
||||
error=f"MinIO shard upload failed: {e}",
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=502, detail=f"Error subiendo shard a MinIO: {e}"
|
||||
) from e
|
||||
store.update_excel_key(shard.id, shard_key)
|
||||
shard_ids.append(shard.id)
|
||||
store.add_event(
|
||||
parent.id,
|
||||
"info",
|
||||
f"Shard {i + 1}/{n} → {wid} ({len(chunk)} filas)",
|
||||
metadata={"shard_id": shard.id, "worker_id": wid, "rows": len(chunk)},
|
||||
)
|
||||
|
||||
store.patch_job(
|
||||
parent.id,
|
||||
result={
|
||||
"mode": "distributed",
|
||||
"shard_ids": shard_ids,
|
||||
"total_rows": len(rows),
|
||||
"total": len(rows),
|
||||
"processed": 0,
|
||||
"exitosos": 0,
|
||||
"fallidos": 0,
|
||||
"downloads_prefix": prefix,
|
||||
"worker_ids": workers_unique,
|
||||
},
|
||||
)
|
||||
store.add_event(
|
||||
parent.id,
|
||||
"run_started",
|
||||
f"Corrida distribuida: {n} bots, {len(rows)} filas",
|
||||
metadata={
|
||||
"excel_key": full_key,
|
||||
"worker_ids": workers_unique,
|
||||
"job_id": parent_job_id,
|
||||
"task_id": task_id,
|
||||
"mode": "distributed",
|
||||
"shard_ids": shard_ids,
|
||||
"total_rows": len(rows),
|
||||
},
|
||||
)
|
||||
job = store.get_job(parent.id)
|
||||
assert job is not None
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.post("/internal/reclaim")
|
||||
def reclaim_jobs(_: str = Depends(require_bearer)) -> dict:
|
||||
n = store.reclaim_stale_running_jobs()
|
||||
return {"reclaimed": n}
|
||||
|
||||
|
||||
@app.get("/jobs")
|
||||
def list_jobs(
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
@@ -168,7 +351,15 @@ def get_job(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(download_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
return job.to_dict()
|
||||
# Refrescar consolidación si es padre distribuido
|
||||
if job.is_distributed_parent:
|
||||
refreshed = store.refresh_parent_from_shards(download_job_id)
|
||||
if refreshed:
|
||||
job = refreshed
|
||||
payload = job.to_dict()
|
||||
if job.is_distributed_parent:
|
||||
payload["shards"] = [s.to_dict() for s in store.list_shards(job.id)]
|
||||
return payload
|
||||
|
||||
|
||||
@app.get("/jobs/{download_job_id}/files")
|
||||
@@ -279,7 +470,8 @@ def post_event(
|
||||
body: EventBody,
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
if not store.get_job(download_job_id):
|
||||
job = store.get_job(download_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
event = store.add_event(
|
||||
download_job_id,
|
||||
@@ -292,8 +484,7 @@ def post_event(
|
||||
metadata=body.metadata,
|
||||
timestamp=body.timestamp,
|
||||
)
|
||||
job = store.get_job(download_job_id)
|
||||
if job and body.index is not None and body.total is not None:
|
||||
if body.index is not None and body.total is not None:
|
||||
result = dict(job.result or {})
|
||||
result["processed"] = body.index
|
||||
result["total"] = body.total
|
||||
@@ -327,4 +518,7 @@ def cancel_flag(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(download_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
return {"cancel_requested": job.cancel_requested, "status": job.status}
|
||||
return {
|
||||
"cancel_requested": store.is_cancel_requested(download_job_id),
|
||||
"status": job.status,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Parseo y partición de Excel/CSV de radicados para descarga distribuida."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from openpyxl import Workbook, load_workbook
|
||||
|
||||
_HEADER_NAMES = ("radicado", "proceso", "numero", "número")
|
||||
|
||||
|
||||
def _clean_nit(valor: Any) -> str:
|
||||
valor = str(valor).strip()
|
||||
valor = re.sub(r'[\\/:*?"<>|]', "_", valor)
|
||||
return valor if valor else "SIN_NIT"
|
||||
|
||||
|
||||
def parse_radicado_rows(data: bytes, filename: str = "input.xlsx") -> list[tuple[str, str]]:
|
||||
"""Devuelve lista de (radicado, nit)."""
|
||||
name = (filename or "input.xlsx").lower()
|
||||
if name.endswith(".csv"):
|
||||
return _parse_csv(data)
|
||||
return _parse_xlsx(data)
|
||||
|
||||
|
||||
def _parse_csv(data: bytes) -> list[tuple[str, str]]:
|
||||
text = None
|
||||
for enc in ("utf-8-sig", "utf-8", "latin-1", "cp1252"):
|
||||
try:
|
||||
text = data.decode(enc)
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
if text is None:
|
||||
text = data.decode("utf-8", errors="replace")
|
||||
sample = text[:4096]
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters="|,;\t")
|
||||
delim = dialect.delimiter
|
||||
except csv.Error:
|
||||
delim = "|" if sample.count("|") >= sample.count(",") else ","
|
||||
reader = csv.reader(io.StringIO(text), delimiter=delim)
|
||||
out: list[tuple[str, str]] = []
|
||||
for fila in reader:
|
||||
if not fila:
|
||||
continue
|
||||
radicado = fila[0] if len(fila) > 0 else None
|
||||
nit = fila[1] if len(fila) > 1 else None
|
||||
if not radicado or str(radicado).strip() in ("", "None"):
|
||||
continue
|
||||
radicado_s = str(radicado).strip()
|
||||
if radicado_s.lower() in _HEADER_NAMES:
|
||||
continue
|
||||
out.append((radicado_s, _clean_nit(nit)))
|
||||
return out
|
||||
|
||||
|
||||
def _parse_xlsx(data: bytes) -> list[tuple[str, str]]:
|
||||
wb = load_workbook(io.BytesIO(data), data_only=True)
|
||||
ws = wb.active
|
||||
out: list[tuple[str, str]] = []
|
||||
for fila in ws.iter_rows(values_only=True):
|
||||
radicado = fila[0] if len(fila) > 0 else None
|
||||
nit = fila[1] if len(fila) > 1 else None
|
||||
if not radicado or str(radicado).strip() in ("", "None"):
|
||||
continue
|
||||
radicado_s = str(radicado).strip()
|
||||
if radicado_s.lower() in _HEADER_NAMES:
|
||||
continue
|
||||
out.append((radicado_s, _clean_nit(nit)))
|
||||
return out
|
||||
|
||||
|
||||
def split_rows(rows: list[tuple[str, str]], n: int) -> list[list[tuple[str, str]]]:
|
||||
"""Parte filas en n trozos lo más equilibrados posible (chunks contiguos)."""
|
||||
if n <= 1 or not rows:
|
||||
return [rows]
|
||||
n = min(n, len(rows))
|
||||
size, rem = divmod(len(rows), n)
|
||||
chunks: list[list[tuple[str, str]]] = []
|
||||
idx = 0
|
||||
for i in range(n):
|
||||
take = size + (1 if i < rem else 0)
|
||||
chunks.append(rows[idx : idx + take])
|
||||
idx += take
|
||||
return [c for c in chunks if c]
|
||||
|
||||
|
||||
def rows_to_xlsx_bytes(rows: list[tuple[str, str]]) -> bytes:
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.append(["radicado", "nit"])
|
||||
for radicado, nit in rows:
|
||||
ws.append([radicado, nit])
|
||||
buf = io.BytesIO()
|
||||
wb.save(buf)
|
||||
return buf.getvalue()
|
||||
@@ -40,7 +40,7 @@ class Worker:
|
||||
@dataclass
|
||||
class Job:
|
||||
id: str
|
||||
worker_id: str
|
||||
worker_id: Optional[str]
|
||||
status: str = JobStatus.PENDING.value
|
||||
title: str = ""
|
||||
excel_key: str = ""
|
||||
@@ -54,10 +54,25 @@ class Job:
|
||||
finished_at: Optional[str] = None
|
||||
job_id: Optional[str] = None
|
||||
task_id: Optional[str] = None
|
||||
parent_download_job_id: Optional[str] = None
|
||||
shard_index: Optional[int] = None
|
||||
shard_count: Optional[int] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
@property
|
||||
def is_distributed_parent(self) -> bool:
|
||||
return (
|
||||
self.parent_download_job_id is None
|
||||
and self.shard_count is not None
|
||||
and int(self.shard_count) > 1
|
||||
)
|
||||
|
||||
@property
|
||||
def is_shard(self) -> bool:
|
||||
return bool(self.parent_download_job_id)
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobEventRecord:
|
||||
|
||||
@@ -103,7 +103,12 @@ class DownloadJobRow(Base):
|
||||
task_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("tasks.id"), nullable=True, index=True
|
||||
)
|
||||
worker_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
worker_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
parent_download_job_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("download_jobs.id"), nullable=True, index=True
|
||||
)
|
||||
shard_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
shard_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
excel_key: Mapped[str] = mapped_column(Text, default="")
|
||||
|
||||
@@ -194,21 +194,25 @@ class MinioStorage:
|
||||
|
||||
Si el manifest falta, falla o queda vacío, hace fallback a list_keys
|
||||
bajo el prefijo (cubre varias carpetas NIT planas).
|
||||
En corridas distribuidas varios shards sobrescriben el manifest parcial:
|
||||
siempre se fusiona con list_keys del prefijo.
|
||||
"""
|
||||
prefix = prefix or downloads_prefix(job_id)
|
||||
files: List[dict] = []
|
||||
by_key: dict[str, dict] = {}
|
||||
try:
|
||||
raw = self.get_bytes(manifest_key(job_id))
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
for key in payload.get("files") or []:
|
||||
entry = self._file_entry(str(key))
|
||||
if entry is not None:
|
||||
files.append(entry)
|
||||
by_key[entry["key"]] = entry
|
||||
except Exception:
|
||||
files = []
|
||||
pass
|
||||
|
||||
if not files:
|
||||
files = self._list_pdf_entries(prefix)
|
||||
for entry in self._list_pdf_entries(prefix):
|
||||
by_key[entry["key"]] = entry
|
||||
|
||||
files = list(by_key.values())
|
||||
return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)}
|
||||
|
||||
def _list_pdf_entries(self, prefix: str) -> List[dict]:
|
||||
|
||||
@@ -73,6 +73,9 @@ class JobStore:
|
||||
finished_at=_iso(row.finished_at),
|
||||
job_id=row.job_id,
|
||||
task_id=row.task_id,
|
||||
parent_download_job_id=row.parent_download_job_id,
|
||||
shard_index=row.shard_index,
|
||||
shard_count=row.shard_count,
|
||||
)
|
||||
|
||||
def _refresh_worker_status(self, worker: Worker) -> Worker:
|
||||
@@ -129,7 +132,13 @@ class JobStore:
|
||||
WorkerStatus.BUSY.value if row.current_job_id else WorkerStatus.ONLINE.value
|
||||
)
|
||||
session.flush()
|
||||
return self._row_worker(row)
|
||||
worker = self._row_worker(row)
|
||||
# Reclaim fuera de la sesión del heartbeat para no bloquear
|
||||
try:
|
||||
self.reclaim_stale_running_jobs()
|
||||
except Exception:
|
||||
pass
|
||||
return worker
|
||||
|
||||
def get_worker(self, worker_id: str) -> Optional[Worker]:
|
||||
with session_scope() as session:
|
||||
@@ -139,12 +148,23 @@ class JobStore:
|
||||
return self._row_worker(row)
|
||||
|
||||
def list_workers(self) -> List[Worker]:
|
||||
try:
|
||||
self.reclaim_stale_running_jobs()
|
||||
except Exception:
|
||||
pass
|
||||
with session_scope() as session:
|
||||
rows = session.scalars(
|
||||
select(DownloadWorkerRow).order_by(DownloadWorkerRow.id)
|
||||
).all()
|
||||
return [self._row_worker(r) for r in rows]
|
||||
|
||||
def list_online_worker_ids(self) -> List[str]:
|
||||
return [
|
||||
w.id
|
||||
for w in self.list_workers()
|
||||
if w.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value)
|
||||
]
|
||||
|
||||
def is_worker_online(self, worker_id: str) -> bool:
|
||||
worker = self.get_worker(worker_id)
|
||||
if not worker:
|
||||
@@ -162,31 +182,41 @@ class JobStore:
|
||||
|
||||
def create_job(
|
||||
self,
|
||||
worker_id: str,
|
||||
worker_id: Optional[str],
|
||||
excel_key: str,
|
||||
original_filename: str = "",
|
||||
title: str = "",
|
||||
*,
|
||||
job_id: Optional[str] = None,
|
||||
task_id: Optional[str] = None,
|
||||
downloads_prefix_override: Optional[str] = None,
|
||||
parent_download_job_id: Optional[str] = None,
|
||||
shard_index: Optional[int] = None,
|
||||
shard_count: Optional[int] = None,
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
status: Optional[str] = None,
|
||||
download_id: Optional[str] = None,
|
||||
) -> Job:
|
||||
import uuid as uuid_mod
|
||||
|
||||
download_id = str(uuid_mod.uuid4())
|
||||
prefix = downloads_prefix(download_id)
|
||||
download_id = download_id or str(uuid_mod.uuid4())
|
||||
prefix = downloads_prefix_override or downloads_prefix(download_id)
|
||||
with session_scope() as session:
|
||||
row = DownloadJobRow(
|
||||
id=download_id,
|
||||
job_id=job_id,
|
||||
task_id=task_id,
|
||||
worker_id=worker_id,
|
||||
status=JobStatus.PENDING.value,
|
||||
parent_download_job_id=parent_download_job_id,
|
||||
shard_index=shard_index,
|
||||
shard_count=shard_count,
|
||||
status=status or JobStatus.PENDING.value,
|
||||
title=title or f"Descarga {original_filename}",
|
||||
excel_key=excel_key,
|
||||
downloads_prefix=prefix,
|
||||
original_filename=original_filename,
|
||||
cancel_requested=False,
|
||||
result={},
|
||||
result=result or {},
|
||||
created_at=utcnow(),
|
||||
)
|
||||
session.add(row)
|
||||
@@ -205,34 +235,57 @@ class JobStore:
|
||||
limit: int = 100,
|
||||
*,
|
||||
job_id: Optional[str] = None,
|
||||
parents_only: bool = True,
|
||||
) -> List[Job]:
|
||||
with session_scope() as session:
|
||||
stmt = select(DownloadJobRow).order_by(DownloadJobRow.created_at.desc()).limit(limit)
|
||||
stmt = select(DownloadJobRow).order_by(DownloadJobRow.created_at.desc())
|
||||
if job_id:
|
||||
stmt = (
|
||||
select(DownloadJobRow)
|
||||
.where(DownloadJobRow.job_id == job_id)
|
||||
.order_by(DownloadJobRow.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
stmt = stmt.where(DownloadJobRow.job_id == job_id)
|
||||
if parents_only:
|
||||
stmt = stmt.where(DownloadJobRow.parent_download_job_id.is_(None))
|
||||
stmt = stmt.limit(limit)
|
||||
rows = session.scalars(stmt).all()
|
||||
return [self._row_job(r) for r in rows]
|
||||
|
||||
def list_shards(self, parent_id: str) -> List[Job]:
|
||||
with session_scope() as session:
|
||||
rows = session.scalars(
|
||||
select(DownloadJobRow)
|
||||
.where(DownloadJobRow.parent_download_job_id == parent_id)
|
||||
.order_by(DownloadJobRow.shard_index.asc())
|
||||
).all()
|
||||
return [self._row_job(r) for r in rows]
|
||||
|
||||
def claim_pending(self, worker_id: str) -> Optional[Job]:
|
||||
try:
|
||||
self.reclaim_stale_running_jobs()
|
||||
except Exception:
|
||||
pass
|
||||
with session_scope() as session:
|
||||
worker = session.get(DownloadWorkerRow, worker_id)
|
||||
if not worker or worker.current_job_id:
|
||||
return None
|
||||
|
||||
row = session.scalars(
|
||||
# No claimar padres orquestadores (shard_count > 1 sin parent)
|
||||
rows = session.scalars(
|
||||
select(DownloadJobRow)
|
||||
.where(
|
||||
DownloadJobRow.worker_id == worker_id,
|
||||
DownloadJobRow.status == JobStatus.PENDING.value,
|
||||
)
|
||||
.order_by(DownloadJobRow.created_at.asc())
|
||||
.limit(1)
|
||||
).first()
|
||||
).all()
|
||||
row = None
|
||||
for candidate in rows:
|
||||
is_parent = (
|
||||
candidate.parent_download_job_id is None
|
||||
and candidate.shard_count is not None
|
||||
and int(candidate.shard_count) > 1
|
||||
)
|
||||
if is_parent:
|
||||
continue
|
||||
row = candidate
|
||||
break
|
||||
if not row:
|
||||
return None
|
||||
|
||||
@@ -243,9 +296,19 @@ class JobStore:
|
||||
worker.status = WorkerStatus.BUSY.value
|
||||
worker.last_heartbeat = now
|
||||
|
||||
if row.task_id:
|
||||
task = session.get(Task, row.task_id)
|
||||
if task:
|
||||
# Task del padre (shards heredan task_id del padre)
|
||||
task_id = row.task_id
|
||||
if row.parent_download_job_id:
|
||||
parent = session.get(DownloadJobRow, row.parent_download_job_id)
|
||||
if parent:
|
||||
task_id = parent.task_id
|
||||
if parent.status == JobStatus.PENDING.value:
|
||||
parent.status = JobStatus.RUNNING.value
|
||||
parent.started_at = parent.started_at or now
|
||||
|
||||
if task_id:
|
||||
task = session.get(Task, task_id)
|
||||
if task and task.status == JobStatus.PENDING.value:
|
||||
task.status = JobStatus.RUNNING.value
|
||||
task.started_at = now
|
||||
task.updated_at = now
|
||||
@@ -271,12 +334,44 @@ class JobStore:
|
||||
row.status = JobStatus.CANCELLED.value
|
||||
row.finished_at = now
|
||||
self._sync_task(session, row, status=JobStatus.CANCELLED.value)
|
||||
|
||||
# Cancelar shards hijos si es padre distribuido
|
||||
if row.parent_download_job_id is None and row.shard_count and int(row.shard_count) > 1:
|
||||
children = session.scalars(
|
||||
select(DownloadJobRow).where(
|
||||
DownloadJobRow.parent_download_job_id == job_id
|
||||
)
|
||||
).all()
|
||||
for child in children:
|
||||
if child.status in (
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
):
|
||||
continue
|
||||
child.cancel_requested = True
|
||||
if child.status == JobStatus.PENDING.value:
|
||||
child.status = JobStatus.CANCELLED.value
|
||||
child.finished_at = now
|
||||
worker = session.get(DownloadWorkerRow, child.worker_id) if child.worker_id else None
|
||||
if worker and worker.current_job_id == child.id:
|
||||
worker.current_job_id = None
|
||||
worker.status = WorkerStatus.ONLINE.value
|
||||
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def is_cancel_requested(self, job_id: str) -> bool:
|
||||
job = self.get_job(job_id)
|
||||
return bool(job and job.cancel_requested)
|
||||
if not job:
|
||||
return False
|
||||
if job.cancel_requested:
|
||||
return True
|
||||
if job.parent_download_job_id:
|
||||
parent = self.get_job(job.parent_download_job_id)
|
||||
if parent and parent.cancel_requested:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _sync_task(
|
||||
self,
|
||||
@@ -287,9 +382,15 @@ class JobStore:
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> None:
|
||||
if not row.task_id:
|
||||
# Sync siempre contra el padre si es shard
|
||||
target = row
|
||||
if row.parent_download_job_id:
|
||||
parent = session.get(DownloadJobRow, row.parent_download_job_id)
|
||||
if parent:
|
||||
target = parent
|
||||
if not target.task_id:
|
||||
return
|
||||
task = session.get(Task, row.task_id)
|
||||
task = session.get(Task, target.task_id)
|
||||
if not task:
|
||||
return
|
||||
now = utcnow()
|
||||
@@ -298,7 +399,7 @@ class JobStore:
|
||||
if result is not None:
|
||||
task.result = {
|
||||
**dict(task.result or {}),
|
||||
"download_job_id": row.id,
|
||||
"download_job_id": target.id,
|
||||
**result,
|
||||
}
|
||||
if error is not None:
|
||||
@@ -319,6 +420,7 @@ class JobStore:
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> Optional[Job]:
|
||||
parent_id: Optional[str] = None
|
||||
with session_scope() as session:
|
||||
row = session.get(DownloadJobRow, job_id)
|
||||
if not row:
|
||||
@@ -340,22 +442,263 @@ class JobStore:
|
||||
row.finished_at = row.finished_at or utcnow()
|
||||
finished = True
|
||||
|
||||
self._sync_task(
|
||||
session,
|
||||
row,
|
||||
status=new_status if status else None,
|
||||
result=result,
|
||||
error=error,
|
||||
)
|
||||
# No sync task directo en shards; se consolida en el padre
|
||||
if not row.parent_download_job_id:
|
||||
self._sync_task(
|
||||
session,
|
||||
row,
|
||||
status=new_status if status else None,
|
||||
result=result,
|
||||
error=error,
|
||||
)
|
||||
|
||||
if finished:
|
||||
worker = session.get(DownloadWorkerRow, row.worker_id)
|
||||
worker = session.get(DownloadWorkerRow, row.worker_id) if row.worker_id else None
|
||||
if worker and worker.current_job_id == job_id:
|
||||
worker.current_job_id = None
|
||||
worker.status = WorkerStatus.ONLINE.value
|
||||
|
||||
parent_id = row.parent_download_job_id
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
job = self._row_job(row)
|
||||
|
||||
if parent_id:
|
||||
self.refresh_parent_from_shards(parent_id)
|
||||
refreshed = self.get_job(job_id)
|
||||
return refreshed or job
|
||||
return job
|
||||
|
||||
def refresh_parent_from_shards(self, parent_id: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
parent = session.get(DownloadJobRow, parent_id)
|
||||
if not parent:
|
||||
return None
|
||||
children = list(
|
||||
session.scalars(
|
||||
select(DownloadJobRow).where(
|
||||
DownloadJobRow.parent_download_job_id == parent_id
|
||||
)
|
||||
).all()
|
||||
)
|
||||
if not children:
|
||||
return self._row_job(parent)
|
||||
|
||||
statuses = [c.status for c in children]
|
||||
terminal = {
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
}
|
||||
any_active = any(
|
||||
s in (JobStatus.PENDING.value, JobStatus.RUNNING.value) for s in statuses
|
||||
)
|
||||
all_terminal = all(s in terminal for s in statuses)
|
||||
|
||||
total = 0
|
||||
exitosos = 0
|
||||
fallidos = 0
|
||||
uploaded = 0
|
||||
for c in children:
|
||||
r = dict(c.result or {})
|
||||
total += int(r.get("total") or 0)
|
||||
exitosos += int(r.get("exitosos") or 0)
|
||||
fallidos += int(r.get("fallidos") or 0)
|
||||
uploaded += int(r.get("uploaded_count") or 0)
|
||||
# Progreso parcial mientras corre
|
||||
if c.status == JobStatus.RUNNING.value:
|
||||
total = max(total, int(r.get("total") or 0))
|
||||
exitosos = max(exitosos, int(r.get("processed") or r.get("exitosos") or 0))
|
||||
|
||||
# Agregar progreso en vuelo (processed) de shards running
|
||||
processed = 0
|
||||
for c in children:
|
||||
r = dict(c.result or {})
|
||||
if c.status in terminal:
|
||||
processed += int(r.get("exitosos") or 0) + int(r.get("fallidos") or 0)
|
||||
else:
|
||||
processed += int(r.get("processed") or 0)
|
||||
|
||||
shard_ids = [c.id for c in children]
|
||||
parent_result = dict(parent.result or {})
|
||||
parent_result.update(
|
||||
{
|
||||
"mode": "distributed",
|
||||
"shard_ids": shard_ids,
|
||||
"total": total or int(parent_result.get("total_rows") or 0),
|
||||
"total_rows": int(parent_result.get("total_rows") or total or 0),
|
||||
"exitosos": exitosos,
|
||||
"fallidos": fallidos,
|
||||
"processed": processed,
|
||||
"uploaded_count": uploaded,
|
||||
"downloads_prefix": parent.downloads_prefix,
|
||||
"shards": [
|
||||
{
|
||||
"id": c.id,
|
||||
"worker_id": c.worker_id,
|
||||
"status": c.status,
|
||||
"shard_index": c.shard_index,
|
||||
"result": dict(c.result or {}),
|
||||
"error": c.error,
|
||||
}
|
||||
for c in children
|
||||
],
|
||||
}
|
||||
)
|
||||
parent.result = parent_result
|
||||
|
||||
now = utcnow()
|
||||
if any_active:
|
||||
parent.status = JobStatus.RUNNING.value
|
||||
parent.started_at = parent.started_at or now
|
||||
parent.finished_at = None
|
||||
parent.error = None
|
||||
self._sync_task(
|
||||
session,
|
||||
parent,
|
||||
status=JobStatus.RUNNING.value,
|
||||
result=parent_result,
|
||||
)
|
||||
elif all_terminal:
|
||||
any_failed = any(s == JobStatus.FAILED.value for s in statuses)
|
||||
any_cancelled = any(s == JobStatus.CANCELLED.value for s in statuses)
|
||||
all_cancelled = all(s == JobStatus.CANCELLED.value for s in statuses)
|
||||
if all_cancelled:
|
||||
parent.status = JobStatus.CANCELLED.value
|
||||
elif any_failed:
|
||||
parent.status = JobStatus.FAILED.value
|
||||
parent.error = next(
|
||||
(c.error for c in children if c.error),
|
||||
"Uno o más shards fallaron",
|
||||
)
|
||||
elif any_cancelled and not any_failed:
|
||||
# mezclado completed+cancelled → completed parcial
|
||||
parent.status = JobStatus.COMPLETED.value
|
||||
else:
|
||||
parent.status = JobStatus.COMPLETED.value
|
||||
parent.finished_at = parent.finished_at or now
|
||||
self._sync_task(
|
||||
session,
|
||||
parent,
|
||||
status=parent.status,
|
||||
result=parent_result,
|
||||
error=parent.error,
|
||||
)
|
||||
|
||||
session.flush()
|
||||
return self._row_job(parent)
|
||||
|
||||
def reclaim_stale_running_jobs(self) -> int:
|
||||
"""Jobs running cuyo worker está offline → pending + reasignar a otro online."""
|
||||
reclaimed = 0
|
||||
with session_scope() as session:
|
||||
now = utcnow()
|
||||
cutoff = now - timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS)
|
||||
workers = list(session.scalars(select(DownloadWorkerRow)).all())
|
||||
offline_ids: set[str] = set()
|
||||
online_free: List[DownloadWorkerRow] = []
|
||||
for w in workers:
|
||||
hb = w.last_heartbeat
|
||||
if hb is None:
|
||||
offline_ids.add(w.id)
|
||||
continue
|
||||
if hb.tzinfo is None:
|
||||
hb = hb.replace(tzinfo=timezone.utc)
|
||||
if hb < cutoff:
|
||||
offline_ids.add(w.id)
|
||||
if w.current_job_id:
|
||||
w.current_job_id = None
|
||||
w.status = WorkerStatus.OFFLINE.value
|
||||
elif not w.current_job_id:
|
||||
online_free.append(w)
|
||||
|
||||
if not offline_ids:
|
||||
session.flush()
|
||||
return 0
|
||||
|
||||
running = session.scalars(
|
||||
select(DownloadJobRow).where(
|
||||
DownloadJobRow.status == JobStatus.RUNNING.value,
|
||||
DownloadJobRow.worker_id.in_(list(offline_ids)),
|
||||
)
|
||||
).all()
|
||||
|
||||
rr = 0
|
||||
for row in running:
|
||||
# Solo shards o jobs simples (no padres orquestadores)
|
||||
is_parent = (
|
||||
row.parent_download_job_id is None
|
||||
and row.shard_count is not None
|
||||
and int(row.shard_count) > 1
|
||||
)
|
||||
if is_parent:
|
||||
continue
|
||||
|
||||
row.status = JobStatus.PENDING.value
|
||||
row.started_at = None
|
||||
row.cancel_requested = False
|
||||
|
||||
if online_free:
|
||||
target = online_free[rr % len(online_free)]
|
||||
rr += 1
|
||||
row.worker_id = target.id
|
||||
# Si no hay online free, queda pending con el mismo worker_id
|
||||
# (se reasignará cuando haya free en el próximo reclaim)
|
||||
|
||||
reclaimed += 1
|
||||
session.add(
|
||||
DownloadJobEventRow(
|
||||
download_job_id=row.id,
|
||||
event_type="warning",
|
||||
message=(
|
||||
f"Worker offline; job reclaimed"
|
||||
+ (f" → {row.worker_id}" if row.worker_id else "")
|
||||
),
|
||||
timestamp=now,
|
||||
metadata_json={"reclaimed": True, "worker_id": row.worker_id},
|
||||
)
|
||||
)
|
||||
if row.parent_download_job_id:
|
||||
session.add(
|
||||
DownloadJobEventRow(
|
||||
download_job_id=row.parent_download_job_id,
|
||||
event_type="warning",
|
||||
message=f"Shard {row.shard_index} reclamado (worker offline)",
|
||||
timestamp=now,
|
||||
metadata_json={
|
||||
"shard_id": row.id,
|
||||
"worker_id": row.worker_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
# Reasignar pending de workers offline a online free
|
||||
if online_free:
|
||||
pending_orphan = session.scalars(
|
||||
select(DownloadJobRow).where(
|
||||
DownloadJobRow.status == JobStatus.PENDING.value,
|
||||
DownloadJobRow.worker_id.in_(list(offline_ids)),
|
||||
)
|
||||
).all()
|
||||
for row in pending_orphan:
|
||||
is_parent = (
|
||||
row.parent_download_job_id is None
|
||||
and row.shard_count is not None
|
||||
and int(row.shard_count) > 1
|
||||
)
|
||||
if is_parent:
|
||||
continue
|
||||
target = online_free[rr % len(online_free)]
|
||||
rr += 1
|
||||
if row.worker_id != target.id:
|
||||
row.worker_id = target.id
|
||||
reclaimed += 1
|
||||
|
||||
session.flush()
|
||||
|
||||
# Refrescar padres afectados
|
||||
# (se hace en siguiente patch; aquí no hace falta)
|
||||
|
||||
return reclaimed
|
||||
|
||||
def add_event(
|
||||
self,
|
||||
@@ -385,6 +728,28 @@ class JobStore:
|
||||
)
|
||||
session.add(ev)
|
||||
session.flush()
|
||||
# Mirror a padre si es shard
|
||||
row = session.get(DownloadJobRow, job_id)
|
||||
if row and row.parent_download_job_id:
|
||||
session.add(
|
||||
DownloadJobEventRow(
|
||||
download_job_id=row.parent_download_job_id,
|
||||
event_type=event_type,
|
||||
message=message,
|
||||
timestamp=ts,
|
||||
radicado=radicado,
|
||||
nit=nit,
|
||||
idx=index,
|
||||
total=total,
|
||||
metadata_json={
|
||||
**(metadata or {}),
|
||||
"shard_id": job_id,
|
||||
"shard_index": row.shard_index,
|
||||
"worker_id": row.worker_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
session.flush()
|
||||
return JobEventRecord(
|
||||
id=int(ev.id),
|
||||
job_id=job_id,
|
||||
@@ -429,9 +794,11 @@ class JobStore:
|
||||
*,
|
||||
parent_job_id: str,
|
||||
download_job_id: str,
|
||||
worker_id: str,
|
||||
worker_id: Optional[str],
|
||||
title: str,
|
||||
original_filename: str,
|
||||
worker_ids: Optional[List[str]] = None,
|
||||
mode: str = "single",
|
||||
) -> str:
|
||||
"""Crea Task(type=downloads) y la asocia al download_job. Devuelve task_id."""
|
||||
with session_scope() as session:
|
||||
@@ -443,10 +810,12 @@ class JobStore:
|
||||
title=title or f"Bot descarga ({original_filename})",
|
||||
request={
|
||||
"worker_id": worker_id,
|
||||
"worker_ids": worker_ids or ([worker_id] if worker_id else []),
|
||||
"original_filename": original_filename,
|
||||
"download_job_id": download_job_id,
|
||||
"mode": mode,
|
||||
},
|
||||
result={"download_job_id": download_job_id},
|
||||
result={"download_job_id": download_job_id, "mode": mode},
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
@@ -3,6 +3,7 @@ uvicorn[standard]>=0.27.0
|
||||
python-multipart>=0.0.9
|
||||
boto3>=1.34.0
|
||||
httpx>=0.27.0
|
||||
openpyxl>=3.1.0
|
||||
pydantic>=2.0.0
|
||||
sqlalchemy>=2.0.0
|
||||
psycopg[binary]>=3.1.0
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Tests de partición Excel para descarga distribuida."""
|
||||
from __future__ import annotations
|
||||
|
||||
from domain.excel_split import parse_radicado_rows, rows_to_xlsx_bytes, split_rows
|
||||
|
||||
|
||||
def test_split_rows_balanced():
|
||||
rows = [(str(i), "NIT") for i in range(100)]
|
||||
chunks = split_rows(rows, 2)
|
||||
assert len(chunks) == 2
|
||||
assert len(chunks[0]) == 50
|
||||
assert len(chunks[1]) == 50
|
||||
|
||||
|
||||
def test_split_rows_uneven():
|
||||
rows = [(str(i), "N") for i in range(10)]
|
||||
chunks = split_rows(rows, 3)
|
||||
assert len(chunks) == 3
|
||||
assert sum(len(c) for c in chunks) == 10
|
||||
assert [len(c) for c in chunks] == [4, 3, 3]
|
||||
|
||||
|
||||
def test_roundtrip_xlsx():
|
||||
rows = [("RAD-1", "900"), ("RAD-2", "901")]
|
||||
data = rows_to_xlsx_bytes(rows)
|
||||
parsed = parse_radicado_rows(data, "shard.xlsx")
|
||||
assert parsed == rows
|
||||
|
||||
|
||||
def test_split_more_workers_than_rows():
|
||||
rows = [("A", "1"), ("B", "2")]
|
||||
chunks = split_rows(rows, 5)
|
||||
assert len(chunks) == 2
|
||||
@@ -0,0 +1,28 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONPATH=/app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY domain /app/domain
|
||||
COPY infrastructure /app/infrastructure
|
||||
COPY api /app/api
|
||||
|
||||
ENV BOT_API_HOST=0.0.0.0
|
||||
ENV BOT_API_PORT=8210
|
||||
|
||||
EXPOSE 8210
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=5s --retries=5 --start-period=15s \
|
||||
CMD curl -f http://127.0.0.1:8210/health || exit 1
|
||||
|
||||
CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8210"]
|
||||
@@ -0,0 +1,600 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import Depends, FastAPI, File, Form, HTTPException, Query, UploadFile
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from domain.models import JobStatus
|
||||
from infrastructure.auth import require_bearer
|
||||
from infrastructure.storage_minio import MinioSettings, MinioStorage
|
||||
from infrastructure.store import JobStore
|
||||
|
||||
store = JobStore()
|
||||
minio = MinioStorage(MinioSettings.from_env())
|
||||
|
||||
POBLACION_APTA_URL = os.getenv("POBLACION_APTA_URL", "http://poblacion_apta:8003").rstrip("/")
|
||||
|
||||
app = FastAPI(title="BotDigitador Remote API", version="1.0.0")
|
||||
|
||||
|
||||
class RegisterWorkerBody(BaseModel):
|
||||
worker_id: str
|
||||
hostname: str = ""
|
||||
max_concurrent: int = 1
|
||||
|
||||
|
||||
class HeartbeatBody(BaseModel):
|
||||
worker_id: str
|
||||
|
||||
|
||||
class EventBody(BaseModel):
|
||||
event_type: str
|
||||
message: str
|
||||
radicado: Optional[str] = None
|
||||
nit: Optional[str] = None
|
||||
index: Optional[int] = None
|
||||
total: Optional[int] = None
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
timestamp: Optional[str] = None
|
||||
|
||||
|
||||
class PatchJobBody(BaseModel):
|
||||
status: Optional[str] = None
|
||||
result: Optional[Dict[str, Any]] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
def _media_type_for_key(key: str) -> str:
|
||||
lower = key.lower()
|
||||
if lower.endswith(".csv"):
|
||||
return "text/csv; charset=utf-8"
|
||||
if lower.endswith(".log") or lower.endswith(".txt"):
|
||||
return "text/plain; charset=utf-8"
|
||||
if lower.endswith(".json"):
|
||||
return "application/json"
|
||||
return "application/octet-stream"
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict:
|
||||
return {"status": "ok", "service": "bot-digitador"}
|
||||
|
||||
|
||||
@app.post("/workers/register")
|
||||
def register_worker(body: RegisterWorkerBody, _: str = Depends(require_bearer)) -> dict:
|
||||
worker = store.register_worker(
|
||||
body.worker_id,
|
||||
hostname=body.hostname,
|
||||
max_concurrent=body.max_concurrent,
|
||||
)
|
||||
return worker.to_dict()
|
||||
|
||||
|
||||
@app.post("/workers/heartbeat")
|
||||
def heartbeat(body: HeartbeatBody, _: str = Depends(require_bearer)) -> dict:
|
||||
worker = store.heartbeat(body.worker_id)
|
||||
if not worker:
|
||||
raise HTTPException(status_code=404, detail="Worker no registrado")
|
||||
return worker.to_dict()
|
||||
|
||||
|
||||
@app.get("/workers")
|
||||
def list_workers(_: str = Depends(require_bearer)) -> List[dict]:
|
||||
return [w.to_dict() for w in store.list_workers()]
|
||||
|
||||
|
||||
@app.get("/workers/me/pending")
|
||||
def claim_pending(
|
||||
worker_id: str = Query(...),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
job = store.claim_pending(worker_id)
|
||||
if not job:
|
||||
return {"job": None}
|
||||
return {"job": job.to_dict()}
|
||||
|
||||
|
||||
@app.post("/jobs")
|
||||
async def create_job(
|
||||
job_id: str = Form(...),
|
||||
worker_id: str = Form(""),
|
||||
worker_ids: str = Form(""),
|
||||
title: str = Form(""),
|
||||
file: Optional[UploadFile] = File(None),
|
||||
poblacion_apta_job_id: str = Form(""),
|
||||
subset: str = Form("aptos"),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
"""Crea una corrida de digitación.
|
||||
|
||||
Exactamente uno de: file (CSV) o poblacion_apta_job_id.
|
||||
- 1 worker → flujo clásico.
|
||||
- N workers → padre + N shards CSV, mismo reports_prefix del padre.
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
import httpx
|
||||
|
||||
from domain.csv_split import parse_digitador_rows, rows_to_csv_bytes, split_rows
|
||||
from infrastructure.storage_minio import reports_prefix as mk_prefix
|
||||
|
||||
parent_job_id = job_id.strip()
|
||||
if not parent_job_id:
|
||||
raise HTTPException(status_code=400, detail="job_id (Job Desktop) obligatorio")
|
||||
if not store.parent_job_exists(parent_job_id):
|
||||
raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado")
|
||||
|
||||
pa_id = (poblacion_apta_job_id or "").strip()
|
||||
data: bytes = b""
|
||||
original = "input.csv"
|
||||
content_type = "text/csv"
|
||||
source_type = "upload"
|
||||
|
||||
file_bytes: bytes = b""
|
||||
if file is not None:
|
||||
file_bytes = await file.read()
|
||||
|
||||
if pa_id and file_bytes:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Proporcione exactamente uno de: file o poblacion_apta_job_id",
|
||||
)
|
||||
|
||||
if pa_id:
|
||||
source_type = "poblacion_apta"
|
||||
subset_norm = (subset or "aptos").strip().lower()
|
||||
if subset_norm != "aptos":
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Solo se digitan aptos (PASO_BOT). Use subset=aptos.",
|
||||
)
|
||||
url = f"{POBLACION_APTA_URL}/jobs/{pa_id}/export.csv"
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
resp = await client.get(url, params={"subset": subset_norm})
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"No se pudo contactar poblacion_apta: {e}",
|
||||
) from e
|
||||
if resp.status_code == 404:
|
||||
raise HTTPException(status_code=404, detail="Job de población apta no encontrado")
|
||||
if not resp.is_success:
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"Error exportando CSV de población apta: HTTP {resp.status_code}",
|
||||
)
|
||||
data = resp.content
|
||||
original = f"poblacion_apta_{subset_norm}_{pa_id[:8]}.csv"
|
||||
content_type = "text/csv; charset=utf-8"
|
||||
elif file_bytes:
|
||||
data = file_bytes
|
||||
original = (file.filename if file else None) or "input.csv"
|
||||
content_type = (file.content_type if file else None) or "text/csv"
|
||||
source_type = "upload"
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Proporcione exactamente uno de: file o poblacion_apta_job_id",
|
||||
)
|
||||
|
||||
if not data:
|
||||
raise HTTPException(status_code=400, detail="CSV vacío")
|
||||
|
||||
title_clean = title.strip()
|
||||
|
||||
# Resolver workers
|
||||
selected: List[str] = []
|
||||
raw_ids = (worker_ids or "").strip()
|
||||
if raw_ids:
|
||||
try:
|
||||
parsed = _json.loads(raw_ids)
|
||||
if isinstance(parsed, list):
|
||||
selected = [str(x).strip() for x in parsed if str(x).strip()]
|
||||
elif isinstance(parsed, str) and parsed.strip():
|
||||
selected = [parsed.strip()]
|
||||
except _json.JSONDecodeError:
|
||||
selected = [x.strip() for x in raw_ids.split(",") if x.strip()]
|
||||
if not selected and worker_id.strip():
|
||||
selected = [worker_id.strip()]
|
||||
|
||||
online = store.list_online_worker_ids()
|
||||
online_set = set(online)
|
||||
if not selected:
|
||||
selected = list(online)
|
||||
else:
|
||||
missing = [w for w in selected if w not in online_set]
|
||||
if missing:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"Workers no online: {', '.join(missing)}. Enciende el agent primero.",
|
||||
)
|
||||
|
||||
if not selected:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="No hay bots online. Enciende al menos un agent.",
|
||||
)
|
||||
|
||||
seen: set[str] = set()
|
||||
workers_unique: List[str] = []
|
||||
for w in selected:
|
||||
if w not in seen:
|
||||
seen.add(w)
|
||||
workers_unique.append(w)
|
||||
|
||||
try:
|
||||
rows = parse_digitador_rows(data, original)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=f"No se pudo leer el CSV: {e}") from e
|
||||
if not rows:
|
||||
raise HTTPException(status_code=400, detail="El CSV no tiene filas para digitar")
|
||||
|
||||
# Normalizar a CSV canónico del bot (siempre pipe + headers snake_case)
|
||||
canonical_bytes = rows_to_csv_bytes(rows)
|
||||
|
||||
n_workers = min(len(workers_unique), len(rows))
|
||||
if n_workers <= 1:
|
||||
only = workers_unique[0]
|
||||
job = store.create_job(
|
||||
worker_id=only,
|
||||
csv_key="",
|
||||
original_filename=original,
|
||||
title=title_clean,
|
||||
job_id=parent_job_id,
|
||||
result={"source_type": source_type, "total_rows": len(rows), "total": len(rows)},
|
||||
)
|
||||
task_id = store.create_digitador_task(
|
||||
parent_job_id=parent_job_id,
|
||||
digitador_job_id=job.id,
|
||||
worker_id=only,
|
||||
title=title_clean or f"Bot digitador ({original})",
|
||||
original_filename=original,
|
||||
worker_ids=[only],
|
||||
mode="single",
|
||||
source_type=source_type,
|
||||
)
|
||||
try:
|
||||
key = minio.upload_csv(job.id, canonical_bytes, original, content_type=content_type)
|
||||
except Exception as e:
|
||||
store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
|
||||
|
||||
job = store.update_csv_key(job.id, key)
|
||||
assert job is not None
|
||||
store.add_event(
|
||||
job.id,
|
||||
"run_started",
|
||||
f"Job creado y asignado a {only}",
|
||||
metadata={
|
||||
"csv_key": key,
|
||||
"worker_id": only,
|
||||
"job_id": parent_job_id,
|
||||
"task_id": task_id,
|
||||
"mode": "single",
|
||||
"total_rows": len(rows),
|
||||
"source_type": source_type,
|
||||
},
|
||||
)
|
||||
return job.to_dict()
|
||||
|
||||
workers_unique = workers_unique[:n_workers]
|
||||
chunks = split_rows(rows, n_workers)
|
||||
n = len(chunks)
|
||||
workers_unique = workers_unique[:n]
|
||||
|
||||
parent = store.create_job(
|
||||
worker_id=None,
|
||||
csv_key="",
|
||||
original_filename=original,
|
||||
title=title_clean or f"Digitación distribuida ({original})",
|
||||
job_id=parent_job_id,
|
||||
shard_count=n,
|
||||
status=JobStatus.PENDING.value,
|
||||
result={
|
||||
"mode": "distributed",
|
||||
"shard_ids": [],
|
||||
"total_rows": len(rows),
|
||||
"total": len(rows),
|
||||
"processed": 0,
|
||||
"ok": 0,
|
||||
"error": 0,
|
||||
"timeout": 0,
|
||||
"existentes": 0,
|
||||
"source_type": source_type,
|
||||
},
|
||||
)
|
||||
prefix = parent.reports_prefix or mk_prefix(parent.id)
|
||||
|
||||
task_id = store.create_digitador_task(
|
||||
parent_job_id=parent_job_id,
|
||||
digitador_job_id=parent.id,
|
||||
worker_id=None,
|
||||
title=title_clean or f"Bot digitador ×{n} ({original})",
|
||||
original_filename=original,
|
||||
worker_ids=workers_unique,
|
||||
mode="distributed",
|
||||
source_type=source_type,
|
||||
)
|
||||
|
||||
try:
|
||||
full_key = minio.upload_csv(parent.id, canonical_bytes, original, content_type=content_type)
|
||||
except Exception as e:
|
||||
store.patch_job(parent.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
|
||||
|
||||
store.update_csv_key(parent.id, full_key)
|
||||
|
||||
shard_ids: List[str] = []
|
||||
for i, (chunk, wid) in enumerate(zip(chunks, workers_unique)):
|
||||
shard_bytes = rows_to_csv_bytes(chunk)
|
||||
shard = store.create_job(
|
||||
worker_id=wid,
|
||||
csv_key="",
|
||||
original_filename=f"shard_{i + 1}_of_{n}.csv",
|
||||
title=f"{title_clean or 'Digitador'} · shard {i + 1}/{n}",
|
||||
job_id=parent_job_id,
|
||||
task_id=task_id,
|
||||
reports_prefix_override=prefix,
|
||||
parent_digitador_job_id=parent.id,
|
||||
shard_index=i,
|
||||
shard_count=n,
|
||||
result={"total": len(chunk), "processed": 0, "source_type": source_type},
|
||||
)
|
||||
try:
|
||||
shard_key = minio.upload_csv(
|
||||
shard.id,
|
||||
shard_bytes,
|
||||
f"shard_{i + 1}_of_{n}.csv",
|
||||
content_type="text/csv; charset=utf-8",
|
||||
)
|
||||
except Exception as e:
|
||||
store.patch_job(
|
||||
parent.id,
|
||||
status=JobStatus.FAILED.value,
|
||||
error=f"MinIO shard upload failed: {e}",
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=502, detail=f"Error subiendo shard a MinIO: {e}"
|
||||
) from e
|
||||
store.update_csv_key(shard.id, shard_key)
|
||||
shard_ids.append(shard.id)
|
||||
store.add_event(
|
||||
parent.id,
|
||||
"info",
|
||||
f"Shard {i + 1}/{n} → {wid} ({len(chunk)} filas)",
|
||||
metadata={"shard_id": shard.id, "worker_id": wid, "rows": len(chunk)},
|
||||
)
|
||||
|
||||
store.patch_job(
|
||||
parent.id,
|
||||
result={
|
||||
"mode": "distributed",
|
||||
"shard_ids": shard_ids,
|
||||
"total_rows": len(rows),
|
||||
"total": len(rows),
|
||||
"processed": 0,
|
||||
"ok": 0,
|
||||
"error": 0,
|
||||
"timeout": 0,
|
||||
"existentes": 0,
|
||||
"reports_prefix": prefix,
|
||||
"worker_ids": workers_unique,
|
||||
"source_type": source_type,
|
||||
},
|
||||
)
|
||||
store.add_event(
|
||||
parent.id,
|
||||
"run_started",
|
||||
f"Corrida distribuida: {n} bots, {len(rows)} filas",
|
||||
metadata={
|
||||
"csv_key": full_key,
|
||||
"worker_ids": workers_unique,
|
||||
"job_id": parent_job_id,
|
||||
"task_id": task_id,
|
||||
"mode": "distributed",
|
||||
"shard_ids": shard_ids,
|
||||
"total_rows": len(rows),
|
||||
"source_type": source_type,
|
||||
},
|
||||
)
|
||||
job = store.get_job(parent.id)
|
||||
assert job is not None
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.post("/internal/reclaim")
|
||||
def reclaim_jobs(_: str = Depends(require_bearer)) -> dict:
|
||||
n = store.reclaim_stale_running_jobs()
|
||||
return {"reclaimed": n}
|
||||
|
||||
|
||||
@app.get("/jobs")
|
||||
def list_jobs(
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
job_id: Optional[str] = Query(None, description="Filtrar por Job Desktop"),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> List[dict]:
|
||||
return [j.to_dict() for j in store.list_jobs(limit=limit, job_id=job_id)]
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}")
|
||||
def get_job(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
if job.is_distributed_parent:
|
||||
refreshed = store.refresh_parent_from_shards(digitador_job_id)
|
||||
if refreshed:
|
||||
job = refreshed
|
||||
payload = job.to_dict()
|
||||
if job.is_distributed_parent:
|
||||
payload["shards"] = [s.to_dict() for s in store.list_shards(job.id)]
|
||||
return payload
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}/files")
|
||||
def list_job_files(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
try:
|
||||
return minio.list_report_files(digitador_job_id, prefix=job.reports_prefix)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=f"Error listando MinIO: {e}") from e
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}/object")
|
||||
def get_job_object(
|
||||
digitador_job_id: str,
|
||||
key: str = Query(..., min_length=1, description="Object key bajo reports_prefix"),
|
||||
_: str = Depends(require_bearer),
|
||||
):
|
||||
"""Stream de un reporte MinIO (csv/log/json bajo el prefijo del job)."""
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
|
||||
prefix = (job.reports_prefix or f"digitados/{digitador_job_id}/").lstrip("/")
|
||||
if not prefix.endswith("/"):
|
||||
prefix += "/"
|
||||
normalized = key.lstrip("/")
|
||||
if not normalized.startswith(prefix):
|
||||
raise HTTPException(status_code=403, detail="Key fuera del prefijo del job")
|
||||
lower = normalized.lower()
|
||||
if not (lower.endswith(".csv") or lower.endswith(".log") or lower.endswith(".json")):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten objetos csv/log/json")
|
||||
if ".." in normalized.split("/"):
|
||||
raise HTTPException(status_code=400, detail="Key no permitida")
|
||||
|
||||
try:
|
||||
data = minio.get_bytes(normalized)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=f"Error leyendo MinIO: {e}") from e
|
||||
|
||||
filename = Path(normalized).name
|
||||
return Response(
|
||||
content=data,
|
||||
media_type=_media_type_for_key(normalized),
|
||||
headers={"Content-Disposition": f'inline; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}/events")
|
||||
def get_events(
|
||||
digitador_job_id: str,
|
||||
since: int = Query(0, ge=0),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
if not store.get_job(digitador_job_id):
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
events = store.list_events(digitador_job_id, since_id=since)
|
||||
return {"events": [e.to_dict() for e in events]}
|
||||
|
||||
|
||||
@app.post("/jobs/{digitador_job_id}/cancel")
|
||||
def cancel_job(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.request_cancel(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
store.add_event(digitador_job_id, "warning", "Cancelación solicitada por el usuario")
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}/file")
|
||||
def get_job_file(digitador_job_id: str, _: str = Depends(require_bearer)):
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
if not job.csv_key:
|
||||
raise HTTPException(status_code=404, detail="CSV no disponible")
|
||||
|
||||
try:
|
||||
data = minio.get_bytes(job.csv_key)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=f"Error leyendo MinIO: {e}") from e
|
||||
|
||||
filename = Path(job.csv_key).name
|
||||
return Response(
|
||||
content=data,
|
||||
media_type="text/csv; charset=utf-8",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}/file-url")
|
||||
def get_job_file_url(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job or not job.csv_key:
|
||||
raise HTTPException(status_code=404, detail="Job/archivo no encontrado")
|
||||
try:
|
||||
url = minio.presigned_get(job.csv_key)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=str(e)) from e
|
||||
return {"url": url, "csv_key": job.csv_key}
|
||||
|
||||
|
||||
@app.post("/jobs/{digitador_job_id}/events")
|
||||
def post_event(
|
||||
digitador_job_id: str,
|
||||
body: EventBody,
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
event = store.add_event(
|
||||
digitador_job_id,
|
||||
body.event_type,
|
||||
body.message,
|
||||
radicado=body.radicado,
|
||||
nit=body.nit,
|
||||
index=body.index,
|
||||
total=body.total,
|
||||
metadata=body.metadata,
|
||||
timestamp=body.timestamp,
|
||||
)
|
||||
if body.index is not None and body.total is not None:
|
||||
result = dict(job.result or {})
|
||||
result["processed"] = body.index
|
||||
result["total"] = body.total
|
||||
if body.nit:
|
||||
result["nit_actual"] = body.nit
|
||||
store.patch_job(digitador_job_id, result=result)
|
||||
return event.to_dict()
|
||||
|
||||
|
||||
@app.patch("/jobs/{digitador_job_id}")
|
||||
def patch_job(
|
||||
digitador_job_id: str,
|
||||
body: PatchJobBody,
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
if body.status and body.status not in {s.value for s in JobStatus}:
|
||||
raise HTTPException(status_code=400, detail="status inválido")
|
||||
job = store.patch_job(
|
||||
digitador_job_id,
|
||||
status=body.status,
|
||||
result=body.result,
|
||||
error=body.error,
|
||||
)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}/cancel-flag")
|
||||
def cancel_flag(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
return {
|
||||
"cancel_requested": store.is_cancel_requested(digitador_job_id),
|
||||
"status": job.status,
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Parseo y partición de CSV de filas a digitar (bot digitador)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
from typing import Any
|
||||
|
||||
REQUIRED_BOT_COLUMNS = (
|
||||
"radicado",
|
||||
"documento",
|
||||
"ingreso",
|
||||
"total_cargos",
|
||||
"autorizacion",
|
||||
"tipo_documento",
|
||||
"tipo_servicio",
|
||||
"copago",
|
||||
"cuota_moderadora",
|
||||
)
|
||||
|
||||
IT_TO_BOT = {
|
||||
"RADICADO": "radicado",
|
||||
"NUMERO": "documento",
|
||||
"FECHA": "ingreso",
|
||||
"BRUTO": "total_cargos",
|
||||
"AUTORIZACION": "autorizacion",
|
||||
"TIPO_ID": "tipo_documento",
|
||||
"SERVICIO": "tipo_servicio",
|
||||
"COPAGO": "copago",
|
||||
"MODERADORA": "cuota_moderadora",
|
||||
}
|
||||
|
||||
|
||||
def _normalize_header(name: str) -> str:
|
||||
raw = str(name or "").strip()
|
||||
if not raw:
|
||||
return ""
|
||||
upper = raw.upper()
|
||||
if upper in IT_TO_BOT:
|
||||
return IT_TO_BOT[upper]
|
||||
lower = raw.lower().strip()
|
||||
# Aceptar snake_case ya normalizado
|
||||
if lower in REQUIRED_BOT_COLUMNS:
|
||||
return lower
|
||||
# Alias comunes
|
||||
aliases = {
|
||||
"número": "documento",
|
||||
"numero": "documento",
|
||||
"fecha": "ingreso",
|
||||
"bruto": "total_cargos",
|
||||
"tipo_id": "tipo_documento",
|
||||
"servicio": "tipo_servicio",
|
||||
"moderadora": "cuota_moderadora",
|
||||
}
|
||||
return aliases.get(lower, lower)
|
||||
|
||||
|
||||
def _decode_text(data: bytes) -> str:
|
||||
for enc in ("utf-8-sig", "utf-8", "latin-1", "cp1252"):
|
||||
try:
|
||||
return data.decode(enc)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
return data.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _detect_delimiter(sample: str) -> str:
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters="|,;\t")
|
||||
return dialect.delimiter
|
||||
except csv.Error:
|
||||
return "|" if sample.count("|") >= sample.count(",") else ","
|
||||
|
||||
|
||||
def _validate_columns(headers: list[str]) -> None:
|
||||
present = set(headers)
|
||||
missing = [c for c in REQUIRED_BOT_COLUMNS if c not in present]
|
||||
if missing:
|
||||
raise ValueError(f"Columnas faltantes: {', '.join(missing)}")
|
||||
|
||||
|
||||
def parse_digitador_rows(data: bytes, filename: str = "input.csv") -> list[dict[str, str]]:
|
||||
"""Devuelve filas normalizadas con headers del bot (snake_case)."""
|
||||
_ = filename # reservado por firma / diagnóstico futuro
|
||||
text = _decode_text(data)
|
||||
if not text.strip():
|
||||
return []
|
||||
|
||||
delim = _detect_delimiter(text[:4096])
|
||||
reader = csv.DictReader(io.StringIO(text), delimiter=delim)
|
||||
if not reader.fieldnames:
|
||||
raise ValueError("CSV sin encabezados")
|
||||
|
||||
rename_map: dict[str, str] = {}
|
||||
normalized_headers: list[str] = []
|
||||
for raw in reader.fieldnames:
|
||||
norm = _normalize_header(raw)
|
||||
rename_map[raw] = norm
|
||||
if norm and norm not in normalized_headers:
|
||||
normalized_headers.append(norm)
|
||||
|
||||
_validate_columns(normalized_headers)
|
||||
|
||||
out: list[dict[str, str]] = []
|
||||
for row in reader:
|
||||
if not row:
|
||||
continue
|
||||
mapped: dict[str, str] = {c: "" for c in REQUIRED_BOT_COLUMNS}
|
||||
empty = True
|
||||
for raw_key, value in row.items():
|
||||
if raw_key is None:
|
||||
continue
|
||||
col = rename_map.get(raw_key) or _normalize_header(raw_key)
|
||||
if col not in REQUIRED_BOT_COLUMNS:
|
||||
continue
|
||||
cell = "" if value is None else str(value).strip()
|
||||
mapped[col] = cell
|
||||
if cell:
|
||||
empty = False
|
||||
if empty:
|
||||
continue
|
||||
# Filtrar filas que parecen ser un segundo header
|
||||
if mapped.get("radicado", "").lower() in ("radicado", "radicado"):
|
||||
if mapped.get("documento", "").lower() in ("documento", "numero", "número", ""):
|
||||
continue
|
||||
out.append(mapped)
|
||||
return out
|
||||
|
||||
|
||||
def split_rows(rows: list[dict[str, Any]], n: int) -> list[list[dict[str, Any]]]:
|
||||
"""Parte filas en n trozos lo más equilibrados posible (chunks contiguos)."""
|
||||
if n <= 1 or not rows:
|
||||
return [rows]
|
||||
n = min(n, len(rows))
|
||||
size, rem = divmod(len(rows), n)
|
||||
chunks: list[list[dict[str, Any]]] = []
|
||||
idx = 0
|
||||
for i in range(n):
|
||||
take = size + (1 if i < rem else 0)
|
||||
chunks.append(rows[idx : idx + take])
|
||||
idx += take
|
||||
return [c for c in chunks if c]
|
||||
|
||||
|
||||
def rows_to_csv_bytes(rows: list[dict[str, Any]]) -> bytes:
|
||||
"""Serializa filas a CSV pipe-delimited con headers del bot (utf-8-sig)."""
|
||||
buf = io.StringIO()
|
||||
writer = csv.DictWriter(
|
||||
buf,
|
||||
fieldnames=list(REQUIRED_BOT_COLUMNS),
|
||||
delimiter="|",
|
||||
lineterminator="\n",
|
||||
extrasaction="ignore",
|
||||
)
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
writer.writerow({c: str(row.get(c, "") or "") for c in REQUIRED_BOT_COLUMNS})
|
||||
return buf.getvalue().encode("utf-8-sig")
|
||||
@@ -0,0 +1,91 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
class WorkerStatus(str, Enum):
|
||||
ONLINE = "online"
|
||||
OFFLINE = "offline"
|
||||
BUSY = "busy"
|
||||
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Worker:
|
||||
id: str
|
||||
hostname: str = ""
|
||||
max_concurrent: int = 1
|
||||
status: str = WorkerStatus.ONLINE.value
|
||||
last_heartbeat: str = field(default_factory=utc_now)
|
||||
current_job_id: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Job:
|
||||
id: str
|
||||
worker_id: Optional[str]
|
||||
status: str = JobStatus.PENDING.value
|
||||
title: str = ""
|
||||
csv_key: str = ""
|
||||
reports_prefix: str = ""
|
||||
original_filename: str = ""
|
||||
cancel_requested: bool = False
|
||||
result: Dict[str, Any] = field(default_factory=dict)
|
||||
error: Optional[str] = None
|
||||
created_at: str = field(default_factory=utc_now)
|
||||
started_at: Optional[str] = None
|
||||
finished_at: Optional[str] = None
|
||||
job_id: Optional[str] = None
|
||||
task_id: Optional[str] = None
|
||||
parent_digitador_job_id: Optional[str] = None
|
||||
shard_index: Optional[int] = None
|
||||
shard_count: Optional[int] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
@property
|
||||
def is_distributed_parent(self) -> bool:
|
||||
return (
|
||||
self.parent_digitador_job_id is None
|
||||
and self.shard_count is not None
|
||||
and int(self.shard_count) > 1
|
||||
)
|
||||
|
||||
@property
|
||||
def is_shard(self) -> bool:
|
||||
return bool(self.parent_digitador_job_id)
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobEventRecord:
|
||||
id: int
|
||||
job_id: str
|
||||
event_type: str
|
||||
message: str
|
||||
timestamp: str
|
||||
radicado: Optional[str] = None
|
||||
nit: Optional[str] = None
|
||||
index: Optional[int] = None
|
||||
total: Optional[int] = None
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Header, HTTPException
|
||||
|
||||
|
||||
def get_api_token() -> str:
|
||||
return os.getenv("BOT_API_TOKEN", "dev-token-change-me")
|
||||
|
||||
|
||||
def require_bearer(authorization: Optional[str] = Header(default=None)) -> str:
|
||||
expected = get_api_token()
|
||||
if not authorization or not authorization.lower().startswith("bearer "):
|
||||
raise HTTPException(status_code=401, detail="Authorization Bearer requerido")
|
||||
token = authorization.split(" ", 1)[1].strip()
|
||||
if token != expected:
|
||||
raise HTTPException(status_code=401, detail="Token inválido")
|
||||
return token
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Persistencia Postgres del servicio bot-digitador (DB compartida IPSExtractor)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Iterator
|
||||
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
Boolean,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
create_engine,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, sessionmaker
|
||||
|
||||
DATABASE_URL = os.getenv(
|
||||
"DATABASE_URL",
|
||||
"postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor",
|
||||
)
|
||||
|
||||
engine = create_engine(DATABASE_URL, pool_pre_ping=True)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def new_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def _iso(dt: datetime | None) -> str | None:
|
||||
if dt is None:
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
class Job(Base):
|
||||
"""Contenedor Desktop (lectura / FK)."""
|
||||
|
||||
__tablename__ = "jobs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
job_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
|
||||
class Task(Base):
|
||||
__tablename__ = "tasks"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
||||
job_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("jobs.id"), nullable=True, index=True
|
||||
)
|
||||
type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
owner_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class DigitadorWorkerRow(Base):
|
||||
__tablename__ = "digitador_workers"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
hostname: Mapped[str] = mapped_column(Text, default="")
|
||||
max_concurrent: Mapped[int] = mapped_column(Integer, default=1)
|
||||
status: Mapped[str] = mapped_column(String(32), index=True)
|
||||
last_heartbeat: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
current_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
|
||||
|
||||
class DigitadorJobRow(Base):
|
||||
__tablename__ = "digitador_jobs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
||||
job_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("jobs.id"), nullable=True, index=True
|
||||
)
|
||||
task_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("tasks.id"), nullable=True, index=True
|
||||
)
|
||||
worker_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
parent_digitador_job_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("digitador_jobs.id"), nullable=True, index=True
|
||||
)
|
||||
shard_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
shard_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
csv_key: Mapped[str] = mapped_column(Text, default="")
|
||||
reports_prefix: Mapped[str] = mapped_column(Text, default="")
|
||||
original_filename: Mapped[str] = mapped_column(Text, default="")
|
||||
cancel_requested: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class DigitadorJobEventRow(Base):
|
||||
__tablename__ = "digitador_job_events"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
digitador_job_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("digitador_jobs.id"), index=True
|
||||
)
|
||||
event_type: Mapped[str] = mapped_column(String(64))
|
||||
message: Mapped[str] = mapped_column(Text)
|
||||
timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
radicado: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
nit: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
idx: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
total: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
metadata_json: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
if DATABASE_URL.startswith("postgresql"):
|
||||
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:
|
||||
connection.execute(text("SELECT pg_advisory_lock(74512005)"))
|
||||
try:
|
||||
Base.metadata.create_all(bind=connection)
|
||||
finally:
|
||||
connection.execute(text("SELECT pg_advisory_unlock(74512005)"))
|
||||
return
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def session_scope() -> Iterator[Session]:
|
||||
session = SessionLocal()
|
||||
try:
|
||||
yield session
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
@@ -0,0 +1,230 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO, List, Optional
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MinioSettings:
|
||||
endpoint: str = "https://s3.builker.xyz"
|
||||
bucket: str = "pys-cuentas-medicas"
|
||||
access_key: str = ""
|
||||
secret_key: str = ""
|
||||
region: str = "us-east-1"
|
||||
anonymous: bool = True
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "MinioSettings":
|
||||
access_key = (os.getenv("MINIO_ACCESS_KEY") or "").strip()
|
||||
secret_key = (os.getenv("MINIO_SECRET_KEY") or "").strip()
|
||||
anon_flag = (os.getenv("MINIO_ANONYMOUS") or "").strip().lower()
|
||||
if anon_flag in ("0", "false", "no"):
|
||||
anonymous = False
|
||||
elif anon_flag in ("1", "true", "yes"):
|
||||
anonymous = True
|
||||
else:
|
||||
anonymous = not (access_key and secret_key)
|
||||
return cls(
|
||||
endpoint=os.getenv("MINIO_ENDPOINT", "https://s3.builker.xyz").rstrip("/"),
|
||||
bucket=os.getenv("MINIO_BUCKET", "pys-cuentas-medicas"),
|
||||
access_key=access_key,
|
||||
secret_key=secret_key,
|
||||
region=os.getenv("MINIO_REGION", "us-east-1"),
|
||||
anonymous=anonymous,
|
||||
)
|
||||
|
||||
|
||||
def csv_key(job_id: str, filename: str = "input.csv") -> str:
|
||||
return f"uploads/{job_id}/{filename}"
|
||||
|
||||
|
||||
def reports_prefix(job_id: str) -> str:
|
||||
return f"digitados/{job_id}/"
|
||||
|
||||
|
||||
def meta_key(job_id: str) -> str:
|
||||
return f"uploads/{job_id}/meta.json"
|
||||
|
||||
|
||||
def manifest_key(job_id: str) -> str:
|
||||
return f"digitados/{job_id}/_manifest.json"
|
||||
|
||||
|
||||
_REPORT_NAMES = {
|
||||
"resultados.csv",
|
||||
"omitidos.csv",
|
||||
"ejecucion_bot.log",
|
||||
}
|
||||
|
||||
|
||||
def _s3_client(settings: MinioSettings):
|
||||
import boto3
|
||||
from botocore import UNSIGNED
|
||||
from botocore.client import Config
|
||||
|
||||
s3_cfg = {
|
||||
"addressing_style": "path",
|
||||
"request_checksum_calculation": "when_required",
|
||||
"response_checksum_validation": "when_required",
|
||||
}
|
||||
if settings.anonymous:
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.endpoint,
|
||||
region_name=settings.region,
|
||||
config=Config(signature_version=UNSIGNED, s3=s3_cfg),
|
||||
)
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.endpoint,
|
||||
aws_access_key_id=settings.access_key,
|
||||
aws_secret_access_key=settings.secret_key,
|
||||
region_name=settings.region,
|
||||
config=Config(signature_version="s3v4", s3=s3_cfg),
|
||||
)
|
||||
|
||||
|
||||
class MinioStorage:
|
||||
"""Cliente S3-compatible para MinIO (uploads/digitados por job_id)."""
|
||||
|
||||
def __init__(self, settings: Optional[MinioSettings] = None):
|
||||
self.settings = settings or MinioSettings.from_env()
|
||||
self._client = None
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
if self._client is None:
|
||||
self._client = _s3_client(self.settings)
|
||||
return self._client
|
||||
|
||||
@property
|
||||
def bucket(self) -> str:
|
||||
return self.settings.bucket
|
||||
|
||||
def put_bytes(self, key: str, data: bytes, content_type: str = "application/octet-stream") -> str:
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=data,
|
||||
ContentType=content_type,
|
||||
)
|
||||
return key
|
||||
|
||||
def put_file(self, key: str, path: str | Path, content_type: str = "application/octet-stream") -> str:
|
||||
path = Path(path)
|
||||
return self.put_bytes(key, path.read_bytes(), content_type=content_type)
|
||||
|
||||
def put_stream(self, key: str, stream: BinaryIO, content_type: str = "application/octet-stream") -> str:
|
||||
data = stream.read()
|
||||
return self.put_bytes(key, data, content_type=content_type)
|
||||
|
||||
def get_bytes(self, key: str) -> bytes:
|
||||
obj = self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
return obj["Body"].read()
|
||||
|
||||
def download_file(self, key: str, dest: str | Path) -> Path:
|
||||
dest = Path(dest)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.client.download_file(self.bucket, key, str(dest))
|
||||
return dest
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
try:
|
||||
self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def list_keys(self, prefix: str) -> List[str]:
|
||||
keys: List[str] = []
|
||||
token = None
|
||||
while True:
|
||||
kwargs = {"Bucket": self.bucket, "Prefix": prefix}
|
||||
if token:
|
||||
kwargs["ContinuationToken"] = token
|
||||
resp = self.client.list_objects_v2(**kwargs)
|
||||
for item in resp.get("Contents") or []:
|
||||
keys.append(item["Key"])
|
||||
if not resp.get("IsTruncated"):
|
||||
break
|
||||
token = resp.get("NextContinuationToken")
|
||||
return keys
|
||||
|
||||
def presigned_get(self, key: str, expires_in: int = 3600) -> str:
|
||||
if self.settings.anonymous:
|
||||
return f"{self.settings.endpoint}/{self.bucket}/{key}"
|
||||
return self.client.generate_presigned_url(
|
||||
"get_object",
|
||||
Params={"Bucket": self.bucket, "Key": key},
|
||||
ExpiresIn=expires_in,
|
||||
)
|
||||
|
||||
def upload_csv(
|
||||
self,
|
||||
job_id: str,
|
||||
data: bytes,
|
||||
original_filename: str,
|
||||
content_type: str = "text/csv",
|
||||
) -> str:
|
||||
ext = Path(original_filename).suffix.lower() or ".csv"
|
||||
canonical = "input.csv" if ext in (".csv", ".txt") else "input.csv"
|
||||
key = csv_key(job_id, canonical)
|
||||
self.put_bytes(key, data, content_type=content_type)
|
||||
meta = {
|
||||
"original_filename": original_filename,
|
||||
"size": len(data),
|
||||
"canonical": canonical,
|
||||
}
|
||||
self.put_bytes(meta_key(job_id), json.dumps(meta).encode("utf-8"), "application/json")
|
||||
return key
|
||||
|
||||
def write_manifest(self, job_id: str, keys: List[str]) -> str:
|
||||
key = manifest_key(job_id)
|
||||
payload = {"job_id": job_id, "files": keys, "count": len(keys)}
|
||||
self.put_bytes(key, json.dumps(payload, ensure_ascii=False).encode("utf-8"), "application/json")
|
||||
return key
|
||||
|
||||
def list_report_files(self, job_id: str, prefix: str | None = None) -> dict:
|
||||
"""Lista reportes de digitación (RESULTADOS/OMITIDOS/log + list_keys).
|
||||
|
||||
Fusiona manifest con listado del prefijo (cubriendo shards distribuidos).
|
||||
"""
|
||||
prefix = prefix or reports_prefix(job_id)
|
||||
by_key: dict[str, dict] = {}
|
||||
try:
|
||||
raw = self.get_bytes(manifest_key(job_id))
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
for key in payload.get("files") or []:
|
||||
entry = self._file_entry(str(key))
|
||||
if entry is not None:
|
||||
by_key[entry["key"]] = entry
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for entry in self._list_report_entries(prefix):
|
||||
by_key[entry["key"]] = entry
|
||||
|
||||
files = list(by_key.values())
|
||||
return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)}
|
||||
|
||||
def _list_report_entries(self, prefix: str) -> List[dict]:
|
||||
files: List[dict] = []
|
||||
for key in self.list_keys(prefix):
|
||||
entry = self._file_entry(key)
|
||||
if entry is not None:
|
||||
files.append(entry)
|
||||
return files
|
||||
|
||||
@staticmethod
|
||||
def _file_entry(key: str) -> dict | None:
|
||||
name = Path(key).name
|
||||
lower = name.lower()
|
||||
if lower.startswith("_manifest") and lower.endswith(".json"):
|
||||
return None
|
||||
# RESULTADOS.csv, OMITIDOS.csv, ejecucion_bot.log (+ variantes _shard_N)
|
||||
if lower in _REPORT_NAMES or lower.endswith(".csv") or lower.endswith(".log"):
|
||||
return {"key": key, "name": name}
|
||||
return None
|
||||
@@ -0,0 +1,833 @@
|
||||
"""Repositorio sobre PostgreSQL (tablas digitador_* + sync de tasks)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from domain.models import (
|
||||
Job,
|
||||
JobEventRecord,
|
||||
JobStatus,
|
||||
Worker,
|
||||
WorkerStatus,
|
||||
utc_now,
|
||||
)
|
||||
from infrastructure.database import (
|
||||
DigitadorJobEventRow,
|
||||
DigitadorJobRow,
|
||||
DigitadorWorkerRow,
|
||||
Task,
|
||||
_iso,
|
||||
init_db,
|
||||
session_scope,
|
||||
utcnow,
|
||||
)
|
||||
from infrastructure.storage_minio import reports_prefix
|
||||
|
||||
HEARTBEAT_OFFLINE_SECONDS = 45
|
||||
|
||||
|
||||
def _parse_iso(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
dt = datetime.fromisoformat(value)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
|
||||
|
||||
class JobStore:
|
||||
"""API de persistencia compatible con el patrón bot_descarga."""
|
||||
|
||||
def __init__(self, db_path: str | None = None):
|
||||
# db_path ignorado (compat); usa DATABASE_URL.
|
||||
init_db()
|
||||
|
||||
def _row_worker(self, row: DigitadorWorkerRow) -> Worker:
|
||||
worker = Worker(
|
||||
id=row.id,
|
||||
hostname=row.hostname or "",
|
||||
max_concurrent=int(row.max_concurrent or 1),
|
||||
status=row.status,
|
||||
last_heartbeat=_iso(row.last_heartbeat) or utc_now(),
|
||||
current_job_id=row.current_job_id,
|
||||
)
|
||||
return self._refresh_worker_status(worker)
|
||||
|
||||
def _row_job(self, row: DigitadorJobRow) -> Job:
|
||||
return Job(
|
||||
id=row.id,
|
||||
worker_id=row.worker_id,
|
||||
status=row.status,
|
||||
title=row.title or "",
|
||||
csv_key=row.csv_key or "",
|
||||
reports_prefix=row.reports_prefix or "",
|
||||
original_filename=row.original_filename or "",
|
||||
cancel_requested=bool(row.cancel_requested),
|
||||
result=dict(row.result or {}),
|
||||
error=row.error,
|
||||
created_at=_iso(row.created_at) or utc_now(),
|
||||
started_at=_iso(row.started_at),
|
||||
finished_at=_iso(row.finished_at),
|
||||
job_id=row.job_id,
|
||||
task_id=row.task_id,
|
||||
parent_digitador_job_id=row.parent_digitador_job_id,
|
||||
shard_index=row.shard_index,
|
||||
shard_count=row.shard_count,
|
||||
)
|
||||
|
||||
def _refresh_worker_status(self, worker: Worker) -> Worker:
|
||||
try:
|
||||
hb = datetime.fromisoformat(worker.last_heartbeat)
|
||||
if hb.tzinfo is None:
|
||||
hb = hb.replace(tzinfo=timezone.utc)
|
||||
age = datetime.now(timezone.utc) - hb.astimezone(timezone.utc)
|
||||
if age > timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS):
|
||||
worker.status = WorkerStatus.OFFLINE.value
|
||||
elif worker.current_job_id:
|
||||
worker.status = WorkerStatus.BUSY.value
|
||||
else:
|
||||
worker.status = WorkerStatus.ONLINE.value
|
||||
except Exception:
|
||||
worker.status = WorkerStatus.OFFLINE.value
|
||||
return worker
|
||||
|
||||
def register_worker(
|
||||
self,
|
||||
worker_id: str,
|
||||
hostname: str = "",
|
||||
max_concurrent: int = 1,
|
||||
) -> Worker:
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorWorkerRow, worker_id)
|
||||
now = utcnow()
|
||||
if row:
|
||||
row.hostname = hostname
|
||||
row.max_concurrent = max_concurrent
|
||||
row.status = WorkerStatus.ONLINE.value
|
||||
row.last_heartbeat = now
|
||||
row.current_job_id = None
|
||||
else:
|
||||
row = DigitadorWorkerRow(
|
||||
id=worker_id,
|
||||
hostname=hostname,
|
||||
max_concurrent=max_concurrent,
|
||||
status=WorkerStatus.ONLINE.value,
|
||||
last_heartbeat=now,
|
||||
current_job_id=None,
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return self._row_worker(row)
|
||||
|
||||
def heartbeat(self, worker_id: str) -> Optional[Worker]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorWorkerRow, worker_id)
|
||||
if not row:
|
||||
return None
|
||||
row.last_heartbeat = utcnow()
|
||||
row.status = (
|
||||
WorkerStatus.BUSY.value if row.current_job_id else WorkerStatus.ONLINE.value
|
||||
)
|
||||
session.flush()
|
||||
worker = self._row_worker(row)
|
||||
try:
|
||||
self.reclaim_stale_running_jobs()
|
||||
except Exception:
|
||||
pass
|
||||
return worker
|
||||
|
||||
def get_worker(self, worker_id: str) -> Optional[Worker]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorWorkerRow, worker_id)
|
||||
if not row:
|
||||
return None
|
||||
return self._row_worker(row)
|
||||
|
||||
def list_workers(self) -> List[Worker]:
|
||||
try:
|
||||
self.reclaim_stale_running_jobs()
|
||||
except Exception:
|
||||
pass
|
||||
with session_scope() as session:
|
||||
rows = session.scalars(
|
||||
select(DigitadorWorkerRow).order_by(DigitadorWorkerRow.id)
|
||||
).all()
|
||||
return [self._row_worker(r) for r in rows]
|
||||
|
||||
def list_online_worker_ids(self) -> List[str]:
|
||||
return [
|
||||
w.id
|
||||
for w in self.list_workers()
|
||||
if w.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value)
|
||||
]
|
||||
|
||||
def is_worker_online(self, worker_id: str) -> bool:
|
||||
worker = self.get_worker(worker_id)
|
||||
if not worker:
|
||||
return False
|
||||
return worker.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value)
|
||||
|
||||
def update_csv_key(self, job_id: str, csv_key: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorJobRow, job_id)
|
||||
if not row:
|
||||
return None
|
||||
row.csv_key = csv_key
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def create_job(
|
||||
self,
|
||||
worker_id: Optional[str],
|
||||
csv_key: str,
|
||||
original_filename: str = "",
|
||||
title: str = "",
|
||||
*,
|
||||
job_id: Optional[str] = None,
|
||||
task_id: Optional[str] = None,
|
||||
reports_prefix_override: Optional[str] = None,
|
||||
parent_digitador_job_id: Optional[str] = None,
|
||||
shard_index: Optional[int] = None,
|
||||
shard_count: Optional[int] = None,
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
status: Optional[str] = None,
|
||||
digitador_id: Optional[str] = None,
|
||||
) -> Job:
|
||||
import uuid as uuid_mod
|
||||
|
||||
digitador_id = digitador_id or str(uuid_mod.uuid4())
|
||||
prefix = reports_prefix_override or reports_prefix(digitador_id)
|
||||
with session_scope() as session:
|
||||
row = DigitadorJobRow(
|
||||
id=digitador_id,
|
||||
job_id=job_id,
|
||||
task_id=task_id,
|
||||
worker_id=worker_id,
|
||||
parent_digitador_job_id=parent_digitador_job_id,
|
||||
shard_index=shard_index,
|
||||
shard_count=shard_count,
|
||||
status=status or JobStatus.PENDING.value,
|
||||
title=title or f"Digitador {original_filename}",
|
||||
csv_key=csv_key,
|
||||
reports_prefix=prefix,
|
||||
original_filename=original_filename,
|
||||
cancel_requested=False,
|
||||
result=result or {},
|
||||
created_at=utcnow(),
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def get_job(self, job_id: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorJobRow, job_id)
|
||||
if not row:
|
||||
return None
|
||||
return self._row_job(row)
|
||||
|
||||
def list_jobs(
|
||||
self,
|
||||
limit: int = 100,
|
||||
*,
|
||||
job_id: Optional[str] = None,
|
||||
parents_only: bool = True,
|
||||
) -> List[Job]:
|
||||
with session_scope() as session:
|
||||
stmt = select(DigitadorJobRow).order_by(DigitadorJobRow.created_at.desc())
|
||||
if job_id:
|
||||
stmt = stmt.where(DigitadorJobRow.job_id == job_id)
|
||||
if parents_only:
|
||||
stmt = stmt.where(DigitadorJobRow.parent_digitador_job_id.is_(None))
|
||||
stmt = stmt.limit(limit)
|
||||
rows = session.scalars(stmt).all()
|
||||
return [self._row_job(r) for r in rows]
|
||||
|
||||
def list_shards(self, parent_id: str) -> List[Job]:
|
||||
with session_scope() as session:
|
||||
rows = session.scalars(
|
||||
select(DigitadorJobRow)
|
||||
.where(DigitadorJobRow.parent_digitador_job_id == parent_id)
|
||||
.order_by(DigitadorJobRow.shard_index.asc())
|
||||
).all()
|
||||
return [self._row_job(r) for r in rows]
|
||||
|
||||
def claim_pending(self, worker_id: str) -> Optional[Job]:
|
||||
try:
|
||||
self.reclaim_stale_running_jobs()
|
||||
except Exception:
|
||||
pass
|
||||
with session_scope() as session:
|
||||
worker = session.get(DigitadorWorkerRow, worker_id)
|
||||
if not worker or worker.current_job_id:
|
||||
return None
|
||||
|
||||
rows = session.scalars(
|
||||
select(DigitadorJobRow)
|
||||
.where(
|
||||
DigitadorJobRow.worker_id == worker_id,
|
||||
DigitadorJobRow.status == JobStatus.PENDING.value,
|
||||
)
|
||||
.order_by(DigitadorJobRow.created_at.asc())
|
||||
).all()
|
||||
row = None
|
||||
for candidate in rows:
|
||||
is_parent = (
|
||||
candidate.parent_digitador_job_id is None
|
||||
and candidate.shard_count is not None
|
||||
and int(candidate.shard_count) > 1
|
||||
)
|
||||
if is_parent:
|
||||
continue
|
||||
row = candidate
|
||||
break
|
||||
if not row:
|
||||
return None
|
||||
|
||||
now = utcnow()
|
||||
row.status = JobStatus.RUNNING.value
|
||||
row.started_at = now
|
||||
worker.current_job_id = row.id
|
||||
worker.status = WorkerStatus.BUSY.value
|
||||
worker.last_heartbeat = now
|
||||
|
||||
task_id = row.task_id
|
||||
if row.parent_digitador_job_id:
|
||||
parent = session.get(DigitadorJobRow, row.parent_digitador_job_id)
|
||||
if parent:
|
||||
task_id = parent.task_id
|
||||
if parent.status == JobStatus.PENDING.value:
|
||||
parent.status = JobStatus.RUNNING.value
|
||||
parent.started_at = parent.started_at or now
|
||||
|
||||
if task_id:
|
||||
task = session.get(Task, task_id)
|
||||
if task and task.status == JobStatus.PENDING.value:
|
||||
task.status = JobStatus.RUNNING.value
|
||||
task.started_at = now
|
||||
task.updated_at = now
|
||||
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def request_cancel(self, job_id: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorJobRow, job_id)
|
||||
if not row:
|
||||
return None
|
||||
if row.status in (
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
):
|
||||
return self._row_job(row)
|
||||
|
||||
now = utcnow()
|
||||
row.cancel_requested = True
|
||||
if row.status == JobStatus.PENDING.value:
|
||||
row.status = JobStatus.CANCELLED.value
|
||||
row.finished_at = now
|
||||
self._sync_task(session, row, status=JobStatus.CANCELLED.value)
|
||||
|
||||
if row.parent_digitador_job_id is None and row.shard_count and int(row.shard_count) > 1:
|
||||
children = session.scalars(
|
||||
select(DigitadorJobRow).where(
|
||||
DigitadorJobRow.parent_digitador_job_id == job_id
|
||||
)
|
||||
).all()
|
||||
for child in children:
|
||||
if child.status in (
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
):
|
||||
continue
|
||||
child.cancel_requested = True
|
||||
if child.status == JobStatus.PENDING.value:
|
||||
child.status = JobStatus.CANCELLED.value
|
||||
child.finished_at = now
|
||||
worker = (
|
||||
session.get(DigitadorWorkerRow, child.worker_id)
|
||||
if child.worker_id
|
||||
else None
|
||||
)
|
||||
if worker and worker.current_job_id == child.id:
|
||||
worker.current_job_id = None
|
||||
worker.status = WorkerStatus.ONLINE.value
|
||||
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def is_cancel_requested(self, job_id: str) -> bool:
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
return False
|
||||
if job.cancel_requested:
|
||||
return True
|
||||
if job.parent_digitador_job_id:
|
||||
parent = self.get_job(job.parent_digitador_job_id)
|
||||
if parent and parent.cancel_requested:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _sync_task(
|
||||
self,
|
||||
session,
|
||||
row: DigitadorJobRow,
|
||||
*,
|
||||
status: Optional[str] = None,
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> None:
|
||||
target = row
|
||||
if row.parent_digitador_job_id:
|
||||
parent = session.get(DigitadorJobRow, row.parent_digitador_job_id)
|
||||
if parent:
|
||||
target = parent
|
||||
if not target.task_id:
|
||||
return
|
||||
task = session.get(Task, target.task_id)
|
||||
if not task:
|
||||
return
|
||||
now = utcnow()
|
||||
if status:
|
||||
task.status = status
|
||||
if result is not None:
|
||||
task.result = {
|
||||
**dict(task.result or {}),
|
||||
"digitador_job_id": target.id,
|
||||
**result,
|
||||
}
|
||||
if error is not None:
|
||||
task.error = error
|
||||
task.updated_at = now
|
||||
if status in (
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
):
|
||||
task.finished_at = now
|
||||
|
||||
def patch_job(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
status: Optional[str] = None,
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> Optional[Job]:
|
||||
parent_id: Optional[str] = None
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorJobRow, job_id)
|
||||
if not row:
|
||||
return None
|
||||
|
||||
new_status = status or row.status
|
||||
if result is not None:
|
||||
row.result = result
|
||||
if error is not None:
|
||||
row.error = error
|
||||
row.status = new_status
|
||||
|
||||
finished = False
|
||||
if new_status in (
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
):
|
||||
row.finished_at = row.finished_at or utcnow()
|
||||
finished = True
|
||||
|
||||
if not row.parent_digitador_job_id:
|
||||
self._sync_task(
|
||||
session,
|
||||
row,
|
||||
status=new_status if status else None,
|
||||
result=result,
|
||||
error=error,
|
||||
)
|
||||
|
||||
if finished:
|
||||
worker = session.get(DigitadorWorkerRow, row.worker_id) if row.worker_id else None
|
||||
if worker and worker.current_job_id == job_id:
|
||||
worker.current_job_id = None
|
||||
worker.status = WorkerStatus.ONLINE.value
|
||||
|
||||
parent_id = row.parent_digitador_job_id
|
||||
session.flush()
|
||||
job = self._row_job(row)
|
||||
|
||||
if parent_id:
|
||||
self.refresh_parent_from_shards(parent_id)
|
||||
refreshed = self.get_job(job_id)
|
||||
return refreshed or job
|
||||
return job
|
||||
|
||||
def refresh_parent_from_shards(self, parent_id: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
parent = session.get(DigitadorJobRow, parent_id)
|
||||
if not parent:
|
||||
return None
|
||||
children = list(
|
||||
session.scalars(
|
||||
select(DigitadorJobRow).where(
|
||||
DigitadorJobRow.parent_digitador_job_id == parent_id
|
||||
)
|
||||
).all()
|
||||
)
|
||||
if not children:
|
||||
return self._row_job(parent)
|
||||
|
||||
statuses = [c.status for c in children]
|
||||
terminal = {
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
}
|
||||
any_active = any(
|
||||
s in (JobStatus.PENDING.value, JobStatus.RUNNING.value) for s in statuses
|
||||
)
|
||||
all_terminal = all(s in terminal for s in statuses)
|
||||
|
||||
total = 0
|
||||
ok = 0
|
||||
error_count = 0
|
||||
timeout = 0
|
||||
existentes = 0
|
||||
for c in children:
|
||||
r = dict(c.result or {})
|
||||
total += int(r.get("total") or 0)
|
||||
ok += int(r.get("ok") or 0)
|
||||
error_count += int(r.get("error") or 0)
|
||||
timeout += int(r.get("timeout") or 0)
|
||||
existentes += int(r.get("existentes") or 0)
|
||||
|
||||
processed = 0
|
||||
for c in children:
|
||||
r = dict(c.result or {})
|
||||
if c.status in terminal:
|
||||
processed += (
|
||||
int(r.get("ok") or 0)
|
||||
+ int(r.get("error") or 0)
|
||||
+ int(r.get("timeout") or 0)
|
||||
+ int(r.get("existentes") or 0)
|
||||
)
|
||||
else:
|
||||
processed += int(r.get("processed") or 0)
|
||||
|
||||
shard_ids = [c.id for c in children]
|
||||
parent_result = dict(parent.result or {})
|
||||
parent_result.update(
|
||||
{
|
||||
"mode": "distributed",
|
||||
"shard_ids": shard_ids,
|
||||
"total": total or int(parent_result.get("total_rows") or 0),
|
||||
"total_rows": int(parent_result.get("total_rows") or total or 0),
|
||||
"ok": ok,
|
||||
"error": error_count,
|
||||
"timeout": timeout,
|
||||
"existentes": existentes,
|
||||
"processed": processed,
|
||||
"reports_prefix": parent.reports_prefix,
|
||||
"shards": [
|
||||
{
|
||||
"id": c.id,
|
||||
"worker_id": c.worker_id,
|
||||
"status": c.status,
|
||||
"shard_index": c.shard_index,
|
||||
"result": dict(c.result or {}),
|
||||
"error": c.error,
|
||||
}
|
||||
for c in children
|
||||
],
|
||||
}
|
||||
)
|
||||
parent.result = parent_result
|
||||
|
||||
now = utcnow()
|
||||
if any_active:
|
||||
parent.status = JobStatus.RUNNING.value
|
||||
parent.started_at = parent.started_at or now
|
||||
parent.finished_at = None
|
||||
parent.error = None
|
||||
self._sync_task(
|
||||
session,
|
||||
parent,
|
||||
status=JobStatus.RUNNING.value,
|
||||
result=parent_result,
|
||||
)
|
||||
elif all_terminal:
|
||||
any_failed = any(s == JobStatus.FAILED.value for s in statuses)
|
||||
any_cancelled = any(s == JobStatus.CANCELLED.value for s in statuses)
|
||||
all_cancelled = all(s == JobStatus.CANCELLED.value for s in statuses)
|
||||
if all_cancelled:
|
||||
parent.status = JobStatus.CANCELLED.value
|
||||
elif any_failed:
|
||||
parent.status = JobStatus.FAILED.value
|
||||
parent.error = next(
|
||||
(c.error for c in children if c.error),
|
||||
"Uno o más shards fallaron",
|
||||
)
|
||||
elif any_cancelled and not any_failed:
|
||||
parent.status = JobStatus.COMPLETED.value
|
||||
else:
|
||||
parent.status = JobStatus.COMPLETED.value
|
||||
parent.finished_at = parent.finished_at or now
|
||||
self._sync_task(
|
||||
session,
|
||||
parent,
|
||||
status=parent.status,
|
||||
result=parent_result,
|
||||
error=parent.error,
|
||||
)
|
||||
|
||||
session.flush()
|
||||
return self._row_job(parent)
|
||||
|
||||
def reclaim_stale_running_jobs(self) -> int:
|
||||
"""Jobs running cuyo worker está offline → pending + reasignar a otro online."""
|
||||
reclaimed = 0
|
||||
with session_scope() as session:
|
||||
now = utcnow()
|
||||
cutoff = now - timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS)
|
||||
workers = list(session.scalars(select(DigitadorWorkerRow)).all())
|
||||
offline_ids: set[str] = set()
|
||||
online_free: List[DigitadorWorkerRow] = []
|
||||
for w in workers:
|
||||
hb = w.last_heartbeat
|
||||
if hb is None:
|
||||
offline_ids.add(w.id)
|
||||
continue
|
||||
if hb.tzinfo is None:
|
||||
hb = hb.replace(tzinfo=timezone.utc)
|
||||
if hb < cutoff:
|
||||
offline_ids.add(w.id)
|
||||
if w.current_job_id:
|
||||
w.current_job_id = None
|
||||
w.status = WorkerStatus.OFFLINE.value
|
||||
elif not w.current_job_id:
|
||||
online_free.append(w)
|
||||
|
||||
if not offline_ids:
|
||||
session.flush()
|
||||
return 0
|
||||
|
||||
running = session.scalars(
|
||||
select(DigitadorJobRow).where(
|
||||
DigitadorJobRow.status == JobStatus.RUNNING.value,
|
||||
DigitadorJobRow.worker_id.in_(list(offline_ids)),
|
||||
)
|
||||
).all()
|
||||
|
||||
rr = 0
|
||||
for row in running:
|
||||
is_parent = (
|
||||
row.parent_digitador_job_id is None
|
||||
and row.shard_count is not None
|
||||
and int(row.shard_count) > 1
|
||||
)
|
||||
if is_parent:
|
||||
continue
|
||||
|
||||
row.status = JobStatus.PENDING.value
|
||||
row.started_at = None
|
||||
row.cancel_requested = False
|
||||
|
||||
if online_free:
|
||||
target = online_free[rr % len(online_free)]
|
||||
rr += 1
|
||||
row.worker_id = target.id
|
||||
|
||||
reclaimed += 1
|
||||
session.add(
|
||||
DigitadorJobEventRow(
|
||||
digitador_job_id=row.id,
|
||||
event_type="warning",
|
||||
message=(
|
||||
"Worker offline; job reclaimed"
|
||||
+ (f" → {row.worker_id}" if row.worker_id else "")
|
||||
),
|
||||
timestamp=now,
|
||||
metadata_json={"reclaimed": True, "worker_id": row.worker_id},
|
||||
)
|
||||
)
|
||||
if row.parent_digitador_job_id:
|
||||
session.add(
|
||||
DigitadorJobEventRow(
|
||||
digitador_job_id=row.parent_digitador_job_id,
|
||||
event_type="warning",
|
||||
message=f"Shard {row.shard_index} reclamado (worker offline)",
|
||||
timestamp=now,
|
||||
metadata_json={
|
||||
"shard_id": row.id,
|
||||
"worker_id": row.worker_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
if online_free:
|
||||
pending_orphan = session.scalars(
|
||||
select(DigitadorJobRow).where(
|
||||
DigitadorJobRow.status == JobStatus.PENDING.value,
|
||||
DigitadorJobRow.worker_id.in_(list(offline_ids)),
|
||||
)
|
||||
).all()
|
||||
for row in pending_orphan:
|
||||
is_parent = (
|
||||
row.parent_digitador_job_id is None
|
||||
and row.shard_count is not None
|
||||
and int(row.shard_count) > 1
|
||||
)
|
||||
if is_parent:
|
||||
continue
|
||||
target = online_free[rr % len(online_free)]
|
||||
rr += 1
|
||||
if row.worker_id != target.id:
|
||||
row.worker_id = target.id
|
||||
reclaimed += 1
|
||||
|
||||
session.flush()
|
||||
|
||||
return reclaimed
|
||||
|
||||
def add_event(
|
||||
self,
|
||||
job_id: str,
|
||||
event_type: str,
|
||||
message: str,
|
||||
*,
|
||||
radicado: Optional[str] = None,
|
||||
nit: Optional[str] = None,
|
||||
index: Optional[int] = None,
|
||||
total: Optional[int] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
timestamp: Optional[str] = None,
|
||||
) -> JobEventRecord:
|
||||
with session_scope() as session:
|
||||
ts = _parse_iso(timestamp) or utcnow()
|
||||
ev = DigitadorJobEventRow(
|
||||
digitador_job_id=job_id,
|
||||
event_type=event_type,
|
||||
message=message,
|
||||
timestamp=ts,
|
||||
radicado=radicado,
|
||||
nit=nit,
|
||||
idx=index,
|
||||
total=total,
|
||||
metadata_json=metadata or {},
|
||||
)
|
||||
session.add(ev)
|
||||
session.flush()
|
||||
row = session.get(DigitadorJobRow, job_id)
|
||||
if row and row.parent_digitador_job_id:
|
||||
session.add(
|
||||
DigitadorJobEventRow(
|
||||
digitador_job_id=row.parent_digitador_job_id,
|
||||
event_type=event_type,
|
||||
message=message,
|
||||
timestamp=ts,
|
||||
radicado=radicado,
|
||||
nit=nit,
|
||||
idx=index,
|
||||
total=total,
|
||||
metadata_json={
|
||||
**(metadata or {}),
|
||||
"shard_id": job_id,
|
||||
"shard_index": row.shard_index,
|
||||
"worker_id": row.worker_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
session.flush()
|
||||
return JobEventRecord(
|
||||
id=int(ev.id),
|
||||
job_id=job_id,
|
||||
event_type=event_type,
|
||||
message=message,
|
||||
timestamp=_iso(ts) or utc_now(),
|
||||
radicado=radicado,
|
||||
nit=nit,
|
||||
index=index,
|
||||
total=total,
|
||||
metadata=metadata or {},
|
||||
)
|
||||
|
||||
def list_events(self, job_id: str, since_id: int = 0) -> List[JobEventRecord]:
|
||||
with session_scope() as session:
|
||||
rows = session.scalars(
|
||||
select(DigitadorJobEventRow)
|
||||
.where(
|
||||
DigitadorJobEventRow.digitador_job_id == job_id,
|
||||
DigitadorJobEventRow.id > since_id,
|
||||
)
|
||||
.order_by(DigitadorJobEventRow.id.asc())
|
||||
).all()
|
||||
return [
|
||||
JobEventRecord(
|
||||
id=int(r.id),
|
||||
job_id=r.digitador_job_id,
|
||||
event_type=r.event_type,
|
||||
message=r.message,
|
||||
timestamp=_iso(r.timestamp) or utc_now(),
|
||||
radicado=r.radicado,
|
||||
nit=r.nit,
|
||||
index=r.idx,
|
||||
total=r.total,
|
||||
metadata=dict(r.metadata_json or {}),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
def create_digitador_task(
|
||||
self,
|
||||
*,
|
||||
parent_job_id: str,
|
||||
digitador_job_id: str,
|
||||
worker_id: Optional[str],
|
||||
title: str,
|
||||
original_filename: str,
|
||||
worker_ids: Optional[List[str]] = None,
|
||||
mode: str = "single",
|
||||
source_type: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Crea Task(type=digitador) y la asocia al digitador_job. Devuelve task_id."""
|
||||
with session_scope() as session:
|
||||
now = utcnow()
|
||||
request: Dict[str, Any] = {
|
||||
"worker_id": worker_id,
|
||||
"worker_ids": worker_ids or ([worker_id] if worker_id else []),
|
||||
"original_filename": original_filename,
|
||||
"digitador_job_id": digitador_job_id,
|
||||
"mode": mode,
|
||||
}
|
||||
result: Dict[str, Any] = {"digitador_job_id": digitador_job_id, "mode": mode}
|
||||
if source_type:
|
||||
request["source_type"] = source_type
|
||||
result["source_type"] = source_type
|
||||
task = Task(
|
||||
job_id=parent_job_id,
|
||||
type="digitador",
|
||||
status=JobStatus.PENDING.value,
|
||||
title=title or f"Bot digitador ({original_filename})",
|
||||
request=request,
|
||||
result=result,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(task)
|
||||
session.flush()
|
||||
row = session.get(DigitadorJobRow, digitador_job_id)
|
||||
if row:
|
||||
row.task_id = task.id
|
||||
row.job_id = parent_job_id
|
||||
session.flush()
|
||||
return task.id
|
||||
|
||||
def parent_job_exists(self, parent_job_id: str) -> bool:
|
||||
from infrastructure.database import Job as IpsJob
|
||||
|
||||
with session_scope() as session:
|
||||
return session.get(IpsJob, parent_job_id) is not None
|
||||
@@ -0,0 +1,10 @@
|
||||
fastapi>=0.110.0
|
||||
uvicorn[standard]>=0.27.0
|
||||
python-multipart>=0.0.9
|
||||
boto3>=1.34.0
|
||||
httpx>=0.27.0
|
||||
openpyxl>=3.1.0
|
||||
pydantic>=2.0.0
|
||||
sqlalchemy>=2.0.0
|
||||
psycopg[binary]>=3.1.0
|
||||
pytest>=7.0.0
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Tests de parseo/partición CSV para digitador."""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from domain.csv_split import (
|
||||
REQUIRED_BOT_COLUMNS,
|
||||
parse_digitador_rows,
|
||||
rows_to_csv_bytes,
|
||||
split_rows,
|
||||
)
|
||||
|
||||
|
||||
def _it_csv_bytes() -> bytes:
|
||||
header = "|".join(
|
||||
[
|
||||
"RADICADO",
|
||||
"NUMERO",
|
||||
"FECHA",
|
||||
"BRUTO",
|
||||
"AUTORIZACION",
|
||||
"TIPO_ID",
|
||||
"SERVICIO",
|
||||
"COPAGO",
|
||||
"MODERADORA",
|
||||
]
|
||||
)
|
||||
row = "|".join(
|
||||
["R-1", "123", "2024-01-01", "1000", "A-9", "CC", "AMB", "0", "4500"]
|
||||
)
|
||||
return f"{header}\n{row}\n".encode("utf-8-sig")
|
||||
|
||||
|
||||
def _snake_csv_bytes() -> bytes:
|
||||
header = "|".join(REQUIRED_BOT_COLUMNS)
|
||||
row = "|".join(
|
||||
["R-2", "999", "2024-02-02", "2000", "A-1", "TI", "HOS", "100", "0"]
|
||||
)
|
||||
return f"{header}\n{row}\n".encode("utf-8")
|
||||
|
||||
|
||||
def test_it_to_bot_mapping():
|
||||
rows = parse_digitador_rows(_it_csv_bytes(), "aptos.csv")
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["radicado"] == "R-1"
|
||||
assert rows[0]["documento"] == "123"
|
||||
assert rows[0]["ingreso"] == "2024-01-01"
|
||||
assert rows[0]["total_cargos"] == "1000"
|
||||
assert rows[0]["autorizacion"] == "A-9"
|
||||
assert rows[0]["tipo_documento"] == "CC"
|
||||
assert rows[0]["tipo_servicio"] == "AMB"
|
||||
assert rows[0]["copago"] == "0"
|
||||
assert rows[0]["cuota_moderadora"] == "4500"
|
||||
|
||||
|
||||
def test_snake_case_headers():
|
||||
rows = parse_digitador_rows(_snake_csv_bytes(), "input.csv")
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["radicado"] == "R-2"
|
||||
assert rows[0]["documento"] == "999"
|
||||
|
||||
|
||||
def test_missing_columns_raises():
|
||||
bad = b"RADICADO|NUMERO\nR1|1\n"
|
||||
with pytest.raises(ValueError) as exc:
|
||||
parse_digitador_rows(bad, "bad.csv")
|
||||
assert "Columnas faltantes" in str(exc.value)
|
||||
|
||||
|
||||
def test_split_rows_balanced():
|
||||
rows = [{"radicado": str(i)} for i in range(100)]
|
||||
chunks = split_rows(rows, 2)
|
||||
assert len(chunks) == 2
|
||||
assert len(chunks[0]) == 50
|
||||
assert len(chunks[1]) == 50
|
||||
|
||||
|
||||
def test_split_rows_uneven():
|
||||
rows = [{"radicado": str(i)} for i in range(10)]
|
||||
chunks = split_rows(rows, 3)
|
||||
assert len(chunks) == 3
|
||||
assert sum(len(c) for c in chunks) == 10
|
||||
assert [len(c) for c in chunks] == [4, 3, 3]
|
||||
|
||||
|
||||
def test_roundtrip_csv():
|
||||
rows = parse_digitador_rows(_it_csv_bytes(), "aptos.csv")
|
||||
data = rows_to_csv_bytes(rows)
|
||||
parsed = parse_digitador_rows(data, "shard.csv")
|
||||
assert parsed == rows
|
||||
|
||||
|
||||
def test_split_more_workers_than_rows():
|
||||
rows = [{"radicado": "A"}, {"radicado": "B"}]
|
||||
chunks = split_rows(rows, 5)
|
||||
assert len(chunks) == 2
|
||||
@@ -49,6 +49,11 @@ def apply_rules(
|
||||
if col not in result.columns:
|
||||
log.append(f"[WARN] Columna no encontrada: {col}")
|
||||
return
|
||||
# Eliminar columna: no tiene sentido transformar y luego borrar.
|
||||
if "drop_column" in rule_ids:
|
||||
result.drop(columns=[col], inplace=True)
|
||||
log.append(f"[OK] '{col}' <- {get_rule_name('drop_column')}")
|
||||
return
|
||||
split_has_homologation = (
|
||||
"doc_id_split_tipo_numero" in rule_ids and "doc_type_homologate" in rule_ids
|
||||
)
|
||||
|
||||
@@ -140,5 +140,14 @@ register_rule(
|
||||
),
|
||||
fn=special._rule_passthrough_series,
|
||||
)
|
||||
register_rule(
|
||||
"drop_column",
|
||||
name="Eliminar columna",
|
||||
description=(
|
||||
"Quita por completo esta columna del resultado limpio. "
|
||||
"Útil para campos que no se necesitan en el export o en pasos posteriores."
|
||||
),
|
||||
fn=special._rule_passthrough_series,
|
||||
)
|
||||
|
||||
rebuild_cleaning_rules()
|
||||
|
||||
@@ -99,7 +99,12 @@ def _infer_column_type(
|
||||
|
||||
def _applicable_rules_for_type(inferred_type: str) -> list[str]:
|
||||
"""Identificadores de reglas a mostrar según el tipo inferido (sin redundancias)."""
|
||||
base = ["strip_special_spaces", "strip_special_keep_date_seps", "strip_spaces"]
|
||||
base = [
|
||||
"strip_special_spaces",
|
||||
"strip_special_keep_date_seps",
|
||||
"strip_spaces",
|
||||
"drop_column",
|
||||
]
|
||||
if inferred_type == TYPE_INTEGER:
|
||||
return base + ["cop_mod_split", "cop_mod_bruto_neto"]
|
||||
if inferred_type == TYPE_DECIMAL:
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Download jobs: parent/shard fields for distributed bots.
|
||||
|
||||
Revision ID: 0007
|
||||
Revises: 0006
|
||||
Create Date: 2026-08-09
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0007"
|
||||
down_revision: Union[str, None] = "0006"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
inspector = sa.inspect(conn)
|
||||
if not inspector.has_table("download_jobs"):
|
||||
return
|
||||
cols = {c["name"] for c in inspector.get_columns("download_jobs")}
|
||||
|
||||
with op.batch_alter_table("download_jobs") as batch:
|
||||
if "parent_download_job_id" not in cols:
|
||||
batch.add_column(
|
||||
sa.Column(
|
||||
"parent_download_job_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("download_jobs.id"),
|
||||
nullable=True,
|
||||
)
|
||||
)
|
||||
if "shard_index" not in cols:
|
||||
batch.add_column(sa.Column("shard_index", sa.Integer(), nullable=True))
|
||||
if "shard_count" not in cols:
|
||||
batch.add_column(sa.Column("shard_count", sa.Integer(), nullable=True))
|
||||
|
||||
# Padre orquestador no tiene worker asignado
|
||||
op.alter_column(
|
||||
"download_jobs",
|
||||
"worker_id",
|
||||
existing_type=sa.String(36),
|
||||
nullable=True,
|
||||
)
|
||||
|
||||
existing_indexes = {ix["name"] for ix in inspector.get_indexes("download_jobs")}
|
||||
if "ix_download_jobs_parent" not in existing_indexes:
|
||||
op.create_index(
|
||||
"ix_download_jobs_parent",
|
||||
"download_jobs",
|
||||
["parent_download_job_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
inspector = sa.inspect(conn)
|
||||
if not inspector.has_table("download_jobs"):
|
||||
return
|
||||
existing_indexes = {ix["name"] for ix in inspector.get_indexes("download_jobs")}
|
||||
if "ix_download_jobs_parent" in existing_indexes:
|
||||
op.drop_index("ix_download_jobs_parent", table_name="download_jobs")
|
||||
|
||||
cols = {c["name"] for c in inspector.get_columns("download_jobs")}
|
||||
with op.batch_alter_table("download_jobs") as batch:
|
||||
if "shard_count" in cols:
|
||||
batch.drop_column("shard_count")
|
||||
if "shard_index" in cols:
|
||||
batch.drop_column("shard_index")
|
||||
if "parent_download_job_id" in cols:
|
||||
batch.drop_column("parent_download_job_id")
|
||||
|
||||
op.alter_column(
|
||||
"download_jobs",
|
||||
"worker_id",
|
||||
existing_type=sa.String(36),
|
||||
nullable=False,
|
||||
)
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Bot digitador: digitador_workers, digitador_jobs, digitador_job_events.
|
||||
|
||||
Revision ID: 0008
|
||||
Revises: 0007
|
||||
Create Date: 2026-08-09
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0008"
|
||||
down_revision: Union[str, None] = "0007"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def _table_exists(conn: sa.engine.Connection, table: str) -> bool:
|
||||
return sa.inspect(conn).has_table(table)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
json_type = (
|
||||
postgresql.JSONB()
|
||||
if conn.dialect.name == "postgresql"
|
||||
else sa.JSON()
|
||||
)
|
||||
|
||||
if not _table_exists(conn, "digitador_workers"):
|
||||
op.create_table(
|
||||
"digitador_workers",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("hostname", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column("max_concurrent", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("last_heartbeat", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("current_job_id", sa.String(36), nullable=True),
|
||||
)
|
||||
op.create_index("ix_digitador_workers_status", "digitador_workers", ["status"])
|
||||
|
||||
if not _table_exists(conn, "digitador_jobs"):
|
||||
op.create_table(
|
||||
"digitador_jobs",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True),
|
||||
sa.Column("task_id", sa.String(36), sa.ForeignKey("tasks.id"), nullable=True),
|
||||
sa.Column("worker_id", sa.String(36), nullable=True),
|
||||
sa.Column(
|
||||
"parent_digitador_job_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("digitador_jobs.id"),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column("shard_index", sa.Integer(), nullable=True),
|
||||
sa.Column("shard_count", sa.Integer(), nullable=True),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("title", sa.Text(), nullable=True),
|
||||
sa.Column("csv_key", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column("reports_prefix", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column("original_filename", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column(
|
||||
"cancel_requested",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
sa.Column(
|
||||
"result",
|
||||
json_type,
|
||||
nullable=False,
|
||||
server_default=(
|
||||
sa.text("'{}'::jsonb")
|
||||
if conn.dialect.name == "postgresql"
|
||||
else sa.text("'{}'")
|
||||
),
|
||||
),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
op.create_index("ix_digitador_jobs_job_id", "digitador_jobs", ["job_id"])
|
||||
op.create_index("ix_digitador_jobs_task_id", "digitador_jobs", ["task_id"])
|
||||
op.create_index("ix_digitador_jobs_worker_id", "digitador_jobs", ["worker_id"])
|
||||
op.create_index("ix_digitador_jobs_status", "digitador_jobs", ["status"])
|
||||
op.create_index(
|
||||
"ix_digitador_jobs_parent",
|
||||
"digitador_jobs",
|
||||
["parent_digitador_job_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_digitador_jobs_worker_status",
|
||||
"digitador_jobs",
|
||||
["worker_id", "status"],
|
||||
)
|
||||
|
||||
if not _table_exists(conn, "digitador_job_events"):
|
||||
op.create_table(
|
||||
"digitador_job_events",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column(
|
||||
"digitador_job_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("digitador_jobs.id"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("event_type", sa.String(64), nullable=False),
|
||||
sa.Column("message", sa.Text(), nullable=False),
|
||||
sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("radicado", sa.String(255), nullable=True),
|
||||
sa.Column("nit", sa.String(64), nullable=True),
|
||||
sa.Column("idx", sa.Integer(), nullable=True),
|
||||
sa.Column("total", sa.Integer(), nullable=True),
|
||||
sa.Column(
|
||||
"metadata",
|
||||
json_type,
|
||||
nullable=False,
|
||||
server_default=(
|
||||
sa.text("'{}'::jsonb")
|
||||
if conn.dialect.name == "postgresql"
|
||||
else sa.text("'{}'")
|
||||
),
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_digitador_job_events_job_id",
|
||||
"digitador_job_events",
|
||||
["digitador_job_id", "id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if _table_exists(conn, "digitador_job_events"):
|
||||
op.drop_index("ix_digitador_job_events_job_id", table_name="digitador_job_events")
|
||||
op.drop_table("digitador_job_events")
|
||||
if _table_exists(conn, "digitador_jobs"):
|
||||
op.drop_index("ix_digitador_jobs_worker_status", table_name="digitador_jobs")
|
||||
op.drop_index("ix_digitador_jobs_parent", table_name="digitador_jobs")
|
||||
op.drop_index("ix_digitador_jobs_status", table_name="digitador_jobs")
|
||||
op.drop_index("ix_digitador_jobs_worker_id", table_name="digitador_jobs")
|
||||
op.drop_index("ix_digitador_jobs_task_id", table_name="digitador_jobs")
|
||||
op.drop_index("ix_digitador_jobs_job_id", table_name="digitador_jobs")
|
||||
op.drop_table("digitador_jobs")
|
||||
if _table_exists(conn, "digitador_workers"):
|
||||
op.drop_index("ix_digitador_workers_status", table_name="digitador_workers")
|
||||
op.drop_table("digitador_workers")
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
Cpu,
|
||||
Download,
|
||||
Hash,
|
||||
Keyboard,
|
||||
Link2,
|
||||
Package,
|
||||
Settings,
|
||||
@@ -35,6 +36,7 @@ import "./modules/extraction/ExtractionModule";
|
||||
import "./modules/cleaning/CleaningModule";
|
||||
import "./modules/poblacion_apta/PoblacionAptaModule";
|
||||
import "./modules/downloads/DownloadsModule";
|
||||
import "./modules/digitador/DigitadorModule";
|
||||
|
||||
const BOT_PACKAGE_URL =
|
||||
"http://s3.builker.xyz/apps-versions/BotDescargaPYSCuentasMedicas/latest/bot_descarga.zip";
|
||||
@@ -403,6 +405,14 @@ function App() {
|
||||
<Users size={17} />
|
||||
<span>Población apta</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={jobSection === "digitador" ? "active" : ""}
|
||||
onClick={() => handleActivateSection("digitador")}
|
||||
>
|
||||
<Keyboard size={17} />
|
||||
<span>Bot digitador</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={jobSection === "results" ? "active" : ""}
|
||||
|
||||
@@ -28,7 +28,11 @@ interface Props {
|
||||
|
||||
function jobLabel(r: DownloadJob): string {
|
||||
const title = r.title || r.original_filename || r.id.slice(0, 8);
|
||||
return `${title} · ${r.status} · ${r.worker_id}`;
|
||||
const bots =
|
||||
r.shard_count && Number(r.shard_count) > 1
|
||||
? `${r.shard_count} bots`
|
||||
: r.worker_id || "—";
|
||||
return `${title} · ${r.status} · ${bots}`;
|
||||
}
|
||||
|
||||
export function MinioDownloadSourcePicker({
|
||||
|
||||
@@ -9,6 +9,10 @@ import type {
|
||||
CountSyncResponse,
|
||||
CreateCountJobPayload,
|
||||
CreateJobPayload,
|
||||
DigitadorJob,
|
||||
DigitadorJobEvent,
|
||||
DigitadorJobFilesResponse,
|
||||
DigitadorWorker,
|
||||
DownloadJob,
|
||||
DownloadJobEvent,
|
||||
DownloadJobFilesResponse,
|
||||
@@ -87,7 +91,8 @@ export class ApiClient {
|
||||
path.startsWith("/cleaning/jobs/from-csv") ||
|
||||
path.startsWith("/poblacion-apta/jobs/from-csv") ||
|
||||
path.startsWith("/poblacion-apta/jobs/from-cleaning") ||
|
||||
path === "/bot-descarga/jobs"
|
||||
path === "/bot-descarga/jobs" ||
|
||||
path === "/bot-digitador/jobs"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -310,6 +315,83 @@ export class ApiClient {
|
||||
return this.handle<DownloadJob>(response);
|
||||
}
|
||||
|
||||
// ── Bot Digitador (control remoto) ─────────────────────────────────────────
|
||||
|
||||
async listDigitadorWorkers(): Promise<DigitadorWorker[]> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url("/bot-digitador/workers"),
|
||||
{ headers: this.botHeaders() },
|
||||
this.timeoutFor("/bot-digitador/workers"),
|
||||
);
|
||||
return this.handle<DigitadorWorker[]>(response);
|
||||
}
|
||||
|
||||
async createDigitadorJob(form: FormData): Promise<DigitadorJob> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url("/bot-digitador/jobs"),
|
||||
{ method: "POST", headers: this.botHeaders(), body: form },
|
||||
this.timeoutFor("/bot-digitador/jobs"),
|
||||
);
|
||||
return this.handle<DigitadorJob>(response);
|
||||
}
|
||||
|
||||
async listDigitadorJobs(limit = 50, parentJobId?: string): Promise<DigitadorJob[]> {
|
||||
const qs = new URLSearchParams({ limit: String(limit) });
|
||||
if (parentJobId) qs.set("job_id", parentJobId);
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-digitador/jobs?${qs.toString()}`),
|
||||
{ headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<DigitadorJob[]>(response);
|
||||
}
|
||||
|
||||
async getDigitadorJob(jobId: string): Promise<DigitadorJob> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}`),
|
||||
{ headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<DigitadorJob>(response);
|
||||
}
|
||||
|
||||
async listDigitadorJobFiles(jobId: string): Promise<DigitadorJobFilesResponse> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}/files`),
|
||||
{ headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<DigitadorJobFilesResponse>(response);
|
||||
}
|
||||
|
||||
/** Descarga bytes de un reporte CSV/log/json del resultado MinIO (proxy bot-digitador). */
|
||||
async fetchDigitadorObject(jobId: string, key: string): Promise<ArrayBuffer> {
|
||||
const qs = new URLSearchParams({ key });
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}/object?${qs.toString()}`),
|
||||
{ headers: this.botHeaders() },
|
||||
120_000,
|
||||
);
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => "");
|
||||
throw new Error(text || `Error ${response.status} al leer objeto MinIO`);
|
||||
}
|
||||
return response.arrayBuffer();
|
||||
}
|
||||
|
||||
async getDigitadorEvents(jobId: string, since = 0): Promise<{ events: DigitadorJobEvent[] }> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}/events?since=${since}`),
|
||||
{ headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<{ events: DigitadorJobEvent[] }>(response);
|
||||
}
|
||||
|
||||
async cancelDigitadorJob(jobId: string): Promise<DigitadorJob> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}/cancel`),
|
||||
{ method: "POST", headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<DigitadorJob>(response);
|
||||
}
|
||||
|
||||
// ── Count Jobs ─────────────────────────────────────────────────────────────
|
||||
|
||||
/** Envía PDFs al servicio de conteo (síncrono). */
|
||||
@@ -456,7 +538,7 @@ export class ApiClient {
|
||||
|
||||
async poblacionAptaExportCsv(
|
||||
jobId: string,
|
||||
subset: "aptos" | "no_aptos" = "aptos",
|
||||
subset: "aptos" | "no_aptos" | "todos" = "aptos",
|
||||
): Promise<Blob> {
|
||||
const params = new URLSearchParams({ subset });
|
||||
const path = `/poblacion-apta/jobs/${encodeURIComponent(jobId)}/export.csv?${params}`;
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { ModuleDescriptor } from "../registry";
|
||||
import { registerModule } from "../registry";
|
||||
import type { Task } from "../../../shared/types";
|
||||
import { DigitadorWizardModule } from "./DigitadorWizardModule";
|
||||
import { DigitadorResultsPanel } from "./DigitadorResultsPanel";
|
||||
|
||||
const DigitadorModule: ModuleDescriptor = {
|
||||
type: "digitador",
|
||||
label: "Bot digitador",
|
||||
iconName: "Keyboard",
|
||||
hasTask: (tasks: Task[]) => tasks.some((t) => t.type === "digitador"),
|
||||
Wizard: DigitadorWizardModule as unknown as ModuleDescriptor["Wizard"],
|
||||
Results: DigitadorResultsPanel as unknown as ModuleDescriptor["Results"],
|
||||
};
|
||||
|
||||
registerModule(DigitadorModule);
|
||||
|
||||
export { DigitadorModule };
|
||||
@@ -0,0 +1,400 @@
|
||||
import {
|
||||
AlertCircle,
|
||||
CheckCircle2,
|
||||
Download,
|
||||
Keyboard,
|
||||
Loader2,
|
||||
Plus,
|
||||
RefreshCw,
|
||||
} from "lucide-react";
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import type {
|
||||
DigitadorJob,
|
||||
DigitadorJobEvent,
|
||||
DigitadorJobFile,
|
||||
Job,
|
||||
Task,
|
||||
} from "../../../shared/types";
|
||||
import type { ApiClient } from "../../lib/apiClient";
|
||||
|
||||
declare const desktopApi: import("../../../shared/types").DesktopBridge | undefined;
|
||||
|
||||
interface Props {
|
||||
client: ApiClient;
|
||||
job: Job;
|
||||
tasks: Task[];
|
||||
onStartDigitador: () => void;
|
||||
}
|
||||
|
||||
function reportBasename(file: DigitadorJobFile): string {
|
||||
const named = (file.name || "").trim();
|
||||
if (named) return named;
|
||||
return file.key.split("/").pop() || "reporte.csv";
|
||||
}
|
||||
|
||||
function extensionFilters(filename: string): Array<{ name: string; extensions: string[] }> {
|
||||
const lower = filename.toLowerCase();
|
||||
if (lower.endsWith(".csv")) return [{ name: "CSV", extensions: ["csv"] }];
|
||||
if (lower.endsWith(".json")) return [{ name: "JSON", extensions: ["json"] }];
|
||||
if (lower.endsWith(".log")) return [{ name: "Log", extensions: ["log"] }];
|
||||
return [{ name: "Archivo", extensions: ["*"] }];
|
||||
}
|
||||
|
||||
async function saveReportAs(filename: string, data: ArrayBuffer): Promise<boolean> {
|
||||
const safeName = reportBasename({ key: filename, name: filename });
|
||||
if (typeof desktopApi?.saveFileDialog === "function" && typeof desktopApi?.writeBytesFile === "function") {
|
||||
const dest = await desktopApi.saveFileDialog(safeName, extensionFilters(safeName));
|
||||
if (!dest) return false;
|
||||
await desktopApi.writeBytesFile(dest, data);
|
||||
return true;
|
||||
}
|
||||
const blob = new Blob([data]);
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = safeName;
|
||||
a.click();
|
||||
URL.revokeObjectURL(url);
|
||||
return true;
|
||||
}
|
||||
|
||||
export function DigitadorResultsPanel({ client, job, tasks, onStartDigitador }: Props) {
|
||||
const [runs, setRuns] = useState<DigitadorJob[]>([]);
|
||||
const [selectedId, setSelectedId] = useState<string>("");
|
||||
const [events, setEvents] = useState<DigitadorJobEvent[]>([]);
|
||||
const [files, setFiles] = useState<DigitadorJobFile[]>([]);
|
||||
const [filesError, setFilesError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [dlBusyKey, setDlBusyKey] = useState<string | null>(null);
|
||||
const [dlStatus, setDlStatus] = useState<string | null>(null);
|
||||
|
||||
const digitadorTasks = tasks
|
||||
.filter((t) => t.type === "digitador")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
|
||||
const loadRuns = useCallback(async () => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const list = await client.listDigitadorJobs(100, job.id);
|
||||
setRuns(list);
|
||||
setSelectedId((prev) => {
|
||||
if (prev && list.some((r) => r.id === prev)) return prev;
|
||||
return list[0]?.id ?? "";
|
||||
});
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [client, job.id]);
|
||||
|
||||
useEffect(() => {
|
||||
void loadRuns();
|
||||
}, [loadRuns, digitadorTasks.length]);
|
||||
|
||||
const selected = runs.find((r) => r.id === selectedId) ?? null;
|
||||
const running = Boolean(
|
||||
selected && !["completed", "failed", "cancelled"].includes(selected.status),
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedId) {
|
||||
setEvents([]);
|
||||
setFiles([]);
|
||||
setFilesError(null);
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
async function loadDetail() {
|
||||
try {
|
||||
const [{ events: ev }, fresh] = await Promise.all([
|
||||
client.getDigitadorEvents(selectedId, 0),
|
||||
client.getDigitadorJob(selectedId),
|
||||
]);
|
||||
if (cancelled) return;
|
||||
setEvents(ev);
|
||||
setRuns((prev) => prev.map((r) => (r.id === fresh.id ? fresh : r)));
|
||||
|
||||
try {
|
||||
const fileResp = await client.listDigitadorJobFiles(selectedId);
|
||||
if (cancelled) return;
|
||||
setFiles(fileResp.files);
|
||||
setFilesError(null);
|
||||
} catch (e) {
|
||||
if (cancelled) return;
|
||||
setFiles([]);
|
||||
setFilesError(e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
} catch (e) {
|
||||
if (!cancelled) setError(e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}
|
||||
|
||||
void loadDetail();
|
||||
if (!running) return;
|
||||
|
||||
const t = setInterval(() => void loadDetail(), 3000);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(t);
|
||||
};
|
||||
}, [client, selectedId, running]);
|
||||
|
||||
const result = selected?.result ?? {};
|
||||
const exitosos = Number(result.exitosos ?? 0);
|
||||
const fallidos = Number(result.fallidos ?? 0);
|
||||
const existentes = Number(result.existentes ?? 0);
|
||||
const timeouts = Number(result.timeouts ?? 0);
|
||||
const total = Number(result.total ?? 0);
|
||||
const bulkBusy = Boolean(dlBusyKey);
|
||||
|
||||
async function downloadOne(file: DigitadorJobFile) {
|
||||
if (!selectedId || bulkBusy) return;
|
||||
const busyId = `file:${file.key}`;
|
||||
setDlBusyKey(busyId);
|
||||
setDlStatus(null);
|
||||
setError(null);
|
||||
try {
|
||||
const data = await client.fetchDigitadorObject(selectedId, file.key);
|
||||
const name = reportBasename(file);
|
||||
const ok = await saveReportAs(name, data);
|
||||
if (ok) setDlStatus(`Guardado: ${name}`);
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : "No se pudo descargar el reporte.");
|
||||
} finally {
|
||||
setDlBusyKey(null);
|
||||
}
|
||||
}
|
||||
|
||||
if (runs.length === 0 && !loading) {
|
||||
return (
|
||||
<div className="results-empty">
|
||||
<Keyboard size={28} />
|
||||
<p>Aún no hay corridas de bot digitador en este trabajo.</p>
|
||||
<button type="button" className="btn btn-primary" onClick={onStartDigitador}>
|
||||
<Plus size={15} /> Nueva digitación
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="results-section">
|
||||
<div className="results-section-header">
|
||||
<div className="results-section-header-left">
|
||||
<label className="wiz-field-label">Corrida</label>
|
||||
<select
|
||||
className="form-input"
|
||||
value={selectedId}
|
||||
onChange={(e) => {
|
||||
setSelectedId(e.target.value);
|
||||
setDlStatus(null);
|
||||
}}
|
||||
style={{ minWidth: 280 }}
|
||||
>
|
||||
{runs.map((r) => (
|
||||
<option key={r.id} value={r.id}>
|
||||
{r.title || r.original_filename || r.id.slice(0, 8)} · {r.status}
|
||||
{r.shard_count && Number(r.shard_count) > 1
|
||||
? ` · ${r.shard_count} bots`
|
||||
: r.worker_id
|
||||
? ` · ${r.worker_id}`
|
||||
: ""}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
{selected && (
|
||||
<span className={`status-badge status-badge--${selected.status}`}>
|
||||
{selected.status}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="results-section-header-right">
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={() => void loadRuns()} disabled={loading}>
|
||||
{loading ? <Loader2 size={13} className="spin" /> : <RefreshCw size={13} />}
|
||||
Actualizar
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm btn-primary" onClick={onStartDigitador}>
|
||||
<Plus size={13} /> Nueva digitación
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="wiz-toast wiz-toast--error" style={{ position: "static", marginBottom: 12 }}>
|
||||
<AlertCircle size={15} />
|
||||
<span>{error}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{selected?.error && (
|
||||
<div className="wiz-toast wiz-toast--error" style={{ position: "static", marginBottom: 12 }}>
|
||||
<AlertCircle size={15} />
|
||||
<span>{selected.error}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{dlStatus && (
|
||||
<div className="wiz-toast" style={{ position: "static", marginBottom: 12 }}>
|
||||
<CheckCircle2 size={15} />
|
||||
<span>{dlStatus}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{selected && (
|
||||
<>
|
||||
<div className="wiz-stats-row" style={{ marginBottom: 16, flexShrink: 0 }}>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{exitosos}</span>
|
||||
<span className="wiz-stat-label">OK</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{fallidos}</span>
|
||||
<span className="wiz-stat-label">Fallidos</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{existentes}</span>
|
||||
<span className="wiz-stat-label">Existentes</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{timeouts}</span>
|
||||
<span className="wiz-stat-label">Timeouts</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{total || "—"}</span>
|
||||
<span className="wiz-stat-label">Total</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p className="wiz-hint" style={{ marginBottom: 12, flexShrink: 0 }}>
|
||||
{selected.shard_count && Number(selected.shard_count) > 1
|
||||
? `${selected.shard_count} bots (distribuido)`
|
||||
: (
|
||||
<>
|
||||
Worker <strong>{selected.worker_id ?? "—"}</strong>
|
||||
</>
|
||||
)}
|
||||
{selected.original_filename ? ` · ${selected.original_filename}` : ""}
|
||||
{selected.reports_prefix ? (
|
||||
<>
|
||||
{" "}
|
||||
· Prefijo <code className="dl-prefix-code">{selected.reports_prefix}</code>
|
||||
</>
|
||||
) : null}
|
||||
</p>
|
||||
|
||||
{(() => {
|
||||
const shardList = Array.isArray(selected.shards)
|
||||
? selected.shards
|
||||
: Array.isArray(result.shards)
|
||||
? (result.shards as Array<{
|
||||
id?: string;
|
||||
worker_id?: string | null;
|
||||
status?: string;
|
||||
shard_index?: number | null;
|
||||
}>)
|
||||
: [];
|
||||
if (shardList.length === 0) return null;
|
||||
return (
|
||||
<div className="wiz-section" style={{ marginBottom: 12, flexShrink: 0 }}>
|
||||
<h3 className="wiz-section-title">Estado por bot</h3>
|
||||
<div className="wiz-review-grid" style={{ marginTop: 8 }}>
|
||||
{shardList.map((s, i) => (
|
||||
<div key={s.id || String(i)} className="wiz-review-item">
|
||||
<div>
|
||||
<span className="wiz-review-label">
|
||||
Shard {(s.shard_index ?? i) + 1}
|
||||
</span>
|
||||
<span className="wiz-review-value">{s.worker_id || "—"}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span className="wiz-review-label">Estado</span>
|
||||
<span className="wiz-review-value">{s.status || "—"}</span>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
|
||||
<div className="wiz-section dl-files-section">
|
||||
<div className="dl-files-header">
|
||||
<h3 className="wiz-section-title" style={{ margin: 0 }}>
|
||||
Reportes en MinIO
|
||||
{files.length > 0 ? ` · ${files.length} archivo${files.length === 1 ? "" : "s"}` : ""}
|
||||
</h3>
|
||||
</div>
|
||||
|
||||
{filesError && (
|
||||
<div className="wiz-toast wiz-toast--error" style={{ position: "static", marginBottom: 12 }}>
|
||||
<AlertCircle size={15} />
|
||||
<span>No se pudo listar MinIO: {filesError}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!filesError && files.length === 0 ? (
|
||||
<p className="wiz-hint">Sin reportes listados aún (RESULTADOS / OMITIDOS).</p>
|
||||
) : (
|
||||
<table className="dl-files-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Archivo</th>
|
||||
<th className="dl-files-table-actions"> </th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{files.map((f) => {
|
||||
const fileBusy = dlBusyKey === `file:${f.key}`;
|
||||
const name = reportBasename(f);
|
||||
return (
|
||||
<tr key={f.key}>
|
||||
<td className="dl-col-file" title={f.key}>
|
||||
{name}
|
||||
</td>
|
||||
<td className="dl-files-table-actions">
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-ghost"
|
||||
disabled={bulkBusy}
|
||||
onClick={() => void downloadOne(f)}
|
||||
title="Descargar reporte"
|
||||
>
|
||||
{fileBusy ? (
|
||||
<Loader2 size={13} className="spin" />
|
||||
) : (
|
||||
<Download size={13} />
|
||||
)}
|
||||
Descargar
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="wiz-section">
|
||||
<h3 className="wiz-section-title">Log</h3>
|
||||
<div className="wiz-estimate-box dl-log-box">
|
||||
{events.length === 0 ? (
|
||||
<span className="wiz-hint">Sin eventos.</span>
|
||||
) : (
|
||||
events.map((ev) => (
|
||||
<div key={ev.id}>
|
||||
[{ev.timestamp}] {ev.message}
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,760 @@
|
||||
import {
|
||||
AlertCircle,
|
||||
Bot,
|
||||
CheckCircle2,
|
||||
FileSpreadsheet,
|
||||
Keyboard,
|
||||
Loader2,
|
||||
Play,
|
||||
RefreshCw,
|
||||
Square,
|
||||
Upload,
|
||||
Users,
|
||||
} from "lucide-react";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import type { ApiClient } from "../../lib/apiClient";
|
||||
import type {
|
||||
DigitadorJob,
|
||||
DigitadorJobEvent,
|
||||
DigitadorWorker,
|
||||
Job,
|
||||
PoblacionAptaJob,
|
||||
} from "../../../shared/types";
|
||||
|
||||
export type DigitadorSourceMode = "aptos" | "archivo";
|
||||
|
||||
export type DigitadorInitialSource = {
|
||||
poblacionAptaJobId: string;
|
||||
};
|
||||
|
||||
interface Props {
|
||||
client: ApiClient;
|
||||
job: Job;
|
||||
initialSource?: DigitadorInitialSource | null;
|
||||
}
|
||||
|
||||
type DigitadorStep = 1 | 2;
|
||||
|
||||
type ShardInfo = {
|
||||
id?: string;
|
||||
worker_id?: string | null;
|
||||
status?: string;
|
||||
shard_index?: number | null;
|
||||
result?: Record<string, unknown>;
|
||||
error?: string | null;
|
||||
};
|
||||
|
||||
export function DigitadorWizardModule({ client, job, initialSource = null }: Props) {
|
||||
const [step, setStep] = useState<DigitadorStep>(1);
|
||||
const [workers, setWorkers] = useState<DigitadorWorker[]>([]);
|
||||
const [selectedIds, setSelectedIds] = useState<string[]>([]);
|
||||
const [useAllOnline, setUseAllOnline] = useState(true);
|
||||
const [sourceMode, setSourceMode] = useState<DigitadorSourceMode>(
|
||||
initialSource?.poblacionAptaJobId ? "aptos" : "aptos",
|
||||
);
|
||||
const [paJobs, setPaJobs] = useState<PoblacionAptaJob[]>([]);
|
||||
const [selectedPaId, setSelectedPaId] = useState(initialSource?.poblacionAptaJobId ?? "");
|
||||
const [loadingPa, setLoadingPa] = useState(false);
|
||||
const [file, setFile] = useState<File | null>(null);
|
||||
const [title, setTitle] = useState("");
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [activeJob, setActiveJob] = useState<DigitadorJob | null>(null);
|
||||
const [events, setEvents] = useState<DigitadorJobEvent[]>([]);
|
||||
const sinceRef = useRef(0);
|
||||
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const fileInputRef = useRef<HTMLInputElement | null>(null);
|
||||
const initializedSelectRef = useRef(false);
|
||||
const appliedInitialRef = useRef<string | null>(null);
|
||||
|
||||
const onlineWorkers = useMemo(
|
||||
() => workers.filter((w) => w.status === "online" || w.status === "busy"),
|
||||
[workers],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!initialSource?.poblacionAptaJobId) return;
|
||||
const key = initialSource.poblacionAptaJobId;
|
||||
if (appliedInitialRef.current === key) return;
|
||||
appliedInitialRef.current = key;
|
||||
setSourceMode("aptos");
|
||||
setSelectedPaId(initialSource.poblacionAptaJobId);
|
||||
setStep(1);
|
||||
}, [initialSource?.poblacionAptaJobId]);
|
||||
|
||||
const refreshWorkers = useCallback(async () => {
|
||||
setRefreshing(true);
|
||||
try {
|
||||
const list = await client.listDigitadorWorkers();
|
||||
setWorkers(list);
|
||||
setError(null);
|
||||
const online = list.filter((w) => w.status === "online" || w.status === "busy");
|
||||
if (!initializedSelectRef.current && online.length > 0) {
|
||||
initializedSelectRef.current = true;
|
||||
setUseAllOnline(online.length >= 2);
|
||||
setSelectedIds(online.map((w) => w.id));
|
||||
} else if (useAllOnline) {
|
||||
setSelectedIds(online.map((w) => w.id));
|
||||
} else {
|
||||
setSelectedIds((prev) => prev.filter((id) => online.some((w) => w.id === id)));
|
||||
}
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setRefreshing(false);
|
||||
}
|
||||
}, [client, useAllOnline]);
|
||||
|
||||
useEffect(() => {
|
||||
void refreshWorkers();
|
||||
const t = setInterval(() => void refreshWorkers(), 5000);
|
||||
return () => clearInterval(t);
|
||||
}, [refreshWorkers]);
|
||||
|
||||
const loadPaJobs = useCallback(async () => {
|
||||
setLoadingPa(true);
|
||||
try {
|
||||
const list = await client.poblacionAptaJobs(job.id);
|
||||
const completed = list
|
||||
.filter((j) => j.status === "completed")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
setPaJobs(completed);
|
||||
setSelectedPaId((prev) => {
|
||||
if (prev && completed.some((j) => j.id === prev)) return prev;
|
||||
return completed[0]?.id ?? "";
|
||||
});
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setLoadingPa(false);
|
||||
}
|
||||
}, [client, job.id]);
|
||||
|
||||
useEffect(() => {
|
||||
if (sourceMode !== "aptos") return;
|
||||
void loadPaJobs();
|
||||
}, [sourceMode, loadPaJobs]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!activeJob || ["completed", "failed", "cancelled"].includes(activeJob.status)) {
|
||||
if (pollRef.current) {
|
||||
clearInterval(pollRef.current);
|
||||
pollRef.current = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
async function poll() {
|
||||
if (!activeJob) return;
|
||||
try {
|
||||
const nextJob = await client.getDigitadorJob(activeJob.id);
|
||||
setActiveJob(nextJob);
|
||||
const { events: next } = await client.getDigitadorEvents(activeJob.id, sinceRef.current);
|
||||
if (next.length) {
|
||||
setEvents((prev) => [...prev, ...next]);
|
||||
sinceRef.current = next[next.length - 1].id;
|
||||
}
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}
|
||||
|
||||
void poll();
|
||||
pollRef.current = setInterval(() => void poll(), 3000);
|
||||
return () => {
|
||||
if (pollRef.current) clearInterval(pollRef.current);
|
||||
};
|
||||
}, [activeJob?.id, activeJob?.status, client]);
|
||||
|
||||
function toggleWorker(id: string) {
|
||||
if (useAllOnline) return;
|
||||
setSelectedIds((prev) =>
|
||||
prev.includes(id) ? prev.filter((x) => x !== id) : [...prev, id],
|
||||
);
|
||||
}
|
||||
|
||||
const selectedPa = paJobs.find((j) => j.id === selectedPaId) ?? null;
|
||||
const aptosCount = Number(selectedPa?.summary?.paso_bot ?? 0);
|
||||
|
||||
async function startJob() {
|
||||
const targets = useAllOnline ? onlineWorkers.map((w) => w.id) : selectedIds;
|
||||
if (targets.length === 0) {
|
||||
setError("Necesitas al menos un bot online.");
|
||||
return;
|
||||
}
|
||||
if (sourceMode === "archivo") {
|
||||
if (!file) {
|
||||
setError("Selecciona un CSV.");
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
if (!selectedPaId) {
|
||||
setError("Selecciona una corrida de población apta.");
|
||||
return;
|
||||
}
|
||||
if (aptosCount <= 0) {
|
||||
setError("Esa corrida no tiene aptos (PASO_BOT) para digitar.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
setBusy(true);
|
||||
setError(null);
|
||||
try {
|
||||
const form = new FormData();
|
||||
form.append("job_id", job.id);
|
||||
if (sourceMode === "archivo" && file) {
|
||||
form.append("file", file);
|
||||
} else {
|
||||
form.append("poblacion_apta_job_id", selectedPaId);
|
||||
form.append("subset", "aptos");
|
||||
}
|
||||
if (title.trim()) form.append("title", title.trim());
|
||||
if (targets.length === 1) {
|
||||
form.append("worker_id", targets[0]);
|
||||
} else {
|
||||
form.append("worker_ids", JSON.stringify(targets));
|
||||
}
|
||||
const created = await client.createDigitadorJob(form);
|
||||
sinceRef.current = 0;
|
||||
setEvents([]);
|
||||
setActiveJob(created);
|
||||
setStep(2);
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function cancelJob() {
|
||||
if (!activeJob) return;
|
||||
setBusy(true);
|
||||
try {
|
||||
const updated = await client.cancelDigitadorJob(activeJob.id);
|
||||
setActiveJob(updated);
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
function resetWizard() {
|
||||
setStep(1);
|
||||
setActiveJob(null);
|
||||
setEvents([]);
|
||||
setFile(null);
|
||||
setTitle("");
|
||||
setError(null);
|
||||
sinceRef.current = 0;
|
||||
if (fileInputRef.current) fileInputRef.current.value = "";
|
||||
}
|
||||
|
||||
const result = activeJob?.result ?? {};
|
||||
const processed = Number(result.processed ?? result.exitosos ?? 0);
|
||||
const total = Number(result.total ?? result.total_rows ?? 0);
|
||||
const pct = total > 0 ? Math.min(100, Math.round((processed / total) * 100)) : 0;
|
||||
const running = Boolean(
|
||||
activeJob && !["completed", "failed", "cancelled"].includes(activeJob.status),
|
||||
);
|
||||
const isDistributed =
|
||||
Boolean(activeJob?.shard_count && Number(activeJob.shard_count) > 1) ||
|
||||
result.mode === "distributed";
|
||||
const shards: ShardInfo[] = Array.isArray(activeJob?.shards)
|
||||
? (activeJob!.shards as ShardInfo[])
|
||||
: Array.isArray(result.shards)
|
||||
? (result.shards as ShardInfo[])
|
||||
: [];
|
||||
const effectiveTargets = useAllOnline ? onlineWorkers.map((w) => w.id) : selectedIds;
|
||||
const sourceReady =
|
||||
sourceMode === "archivo" ? Boolean(file) : Boolean(selectedPaId) && aptosCount > 0;
|
||||
const canStart = effectiveTargets.length > 0 && sourceReady && !busy;
|
||||
|
||||
return (
|
||||
<div className="wiz">
|
||||
<nav className="stepper" aria-label="Pasos del bot digitador">
|
||||
<div className="stepper-item">
|
||||
<button
|
||||
type="button"
|
||||
className={`stepper-dot ${step === 1 ? "active" : step > 1 ? "done" : "todo"}`}
|
||||
onClick={() => !running && setStep(1)}
|
||||
disabled={running}
|
||||
>
|
||||
1
|
||||
</button>
|
||||
<span className={`stepper-label ${step === 1 ? "active" : step > 1 ? "done" : "todo"}`}>
|
||||
Configurar
|
||||
</span>
|
||||
<div className={`stepper-connector ${step > 1 ? "done" : ""}`} />
|
||||
</div>
|
||||
<div className="stepper-item">
|
||||
<button
|
||||
type="button"
|
||||
className={`stepper-dot ${step === 2 ? "active" : "todo"}`}
|
||||
onClick={() => activeJob && setStep(2)}
|
||||
disabled={!activeJob}
|
||||
>
|
||||
2
|
||||
</button>
|
||||
<span className={`stepper-label ${step === 2 ? "active" : "todo"}`}>Ejecución</span>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<div className="wiz-body">
|
||||
{step === 1 && (
|
||||
<div className="wiz-step">
|
||||
<div className="wiz-header" style={{ marginBottom: 4 }}>
|
||||
<h2 style={{ display: "flex", alignItems: "center", gap: 8, margin: 0 }}>
|
||||
<Keyboard size={18} />
|
||||
Bot digitador
|
||||
</h2>
|
||||
<p className="wiz-desc" style={{ marginTop: 6 }}>
|
||||
Digita en Integración Total solo los <strong>aptos (PASO_BOT)</strong>, o un CSV
|
||||
propio con el mismo formato.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* 1. Origen */}
|
||||
<div className="wiz-section">
|
||||
<h3 className="wiz-section-title">1. Origen</h3>
|
||||
<div style={{ display: "flex", flexWrap: "wrap", gap: 8, marginBottom: 14 }}>
|
||||
<button
|
||||
type="button"
|
||||
className={`btn${sourceMode === "aptos" ? " btn-primary" : ""}`}
|
||||
onClick={() => setSourceMode("aptos")}
|
||||
>
|
||||
<Users size={14} /> Aptos de población apta
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`btn${sourceMode === "archivo" ? " btn-primary" : ""}`}
|
||||
onClick={() => setSourceMode("archivo")}
|
||||
>
|
||||
<Upload size={14} /> Subir CSV
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<label className="wiz-field-label">Título (opcional)</label>
|
||||
<input
|
||||
className="form-input"
|
||||
value={title}
|
||||
onChange={(e) => setTitle(e.target.value)}
|
||||
placeholder="Ej. Digitación lote mayo 2026"
|
||||
style={{ width: "100%", marginBottom: 14 }}
|
||||
/>
|
||||
|
||||
{sourceMode === "aptos" ? (
|
||||
<>
|
||||
<div className="wiz-section-header" style={{ marginBottom: 8 }}>
|
||||
<label className="wiz-field-label" style={{ margin: 0 }}>
|
||||
Corrida de población apta
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-ghost"
|
||||
onClick={() => void loadPaJobs()}
|
||||
disabled={loadingPa}
|
||||
>
|
||||
{loadingPa ? <Loader2 size={13} className="spin" /> : <RefreshCw size={13} />}
|
||||
Actualizar
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{loadingPa && paJobs.length === 0 ? (
|
||||
<p className="wiz-hint">
|
||||
<Loader2 size={13} className="spin" /> Cargando corridas…
|
||||
</p>
|
||||
) : paJobs.length === 0 ? (
|
||||
<div className="wiz-estimate-box" style={{ textAlign: "left" }}>
|
||||
<p className="wiz-desc" style={{ margin: 0 }}>
|
||||
No hay corridas de población apta completadas en este trabajo.
|
||||
</p>
|
||||
<p className="wiz-hint" style={{ marginTop: 6, marginBottom: 0 }}>
|
||||
Ejecuta primero Población apta, o cambia a «Subir CSV».
|
||||
</p>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<select
|
||||
className="form-input"
|
||||
value={selectedPaId}
|
||||
onChange={(e) => setSelectedPaId(e.target.value)}
|
||||
style={{ width: "100%" }}
|
||||
>
|
||||
{paJobs.map((pa) => {
|
||||
const n = Number(pa.summary?.paso_bot ?? 0);
|
||||
return (
|
||||
<option key={pa.id} value={pa.id}>
|
||||
{pa.id.slice(0, 8)} · {n} aptos · {pa.summary?.total ?? 0} total
|
||||
</option>
|
||||
);
|
||||
})}
|
||||
</select>
|
||||
|
||||
{selectedPa && (
|
||||
<div
|
||||
className="wiz-estimate-box"
|
||||
style={{
|
||||
marginTop: 12,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 12,
|
||||
textAlign: "left",
|
||||
}}
|
||||
>
|
||||
{aptosCount > 0 ? (
|
||||
<CheckCircle2 size={20} style={{ color: "var(--success, #16a34a)", flexShrink: 0 }} />
|
||||
) : (
|
||||
<AlertCircle size={20} style={{ color: "var(--danger, #dc2626)", flexShrink: 0 }} />
|
||||
)}
|
||||
<div>
|
||||
<div style={{ fontWeight: 600 }}>
|
||||
{aptosCount > 0
|
||||
? `${aptosCount} registros aptos listos para digitar`
|
||||
: "Sin aptos en esta corrida"}
|
||||
</div>
|
||||
<div className="wiz-hint" style={{ margin: 0 }}>
|
||||
Solo se envían filas PASO_BOT. Los no aptos no se digitan.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<label className="wiz-field-label">Archivo CSV</label>
|
||||
<div className="folder-picker">
|
||||
<div className="folder-picker-display">
|
||||
{file ? (
|
||||
<>
|
||||
<FileSpreadsheet size={16} className="folder-picker-icon" />
|
||||
<span className="folder-picker-path">{file.name}</span>
|
||||
</>
|
||||
) : (
|
||||
<span className="folder-picker-placeholder">Ningún archivo seleccionado</span>
|
||||
)}
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-primary"
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
>
|
||||
<FileSpreadsheet size={15} />
|
||||
Seleccionar CSV
|
||||
</button>
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept=".csv,text/csv"
|
||||
hidden
|
||||
onChange={(e) => setFile(e.target.files?.[0] ?? null)}
|
||||
/>
|
||||
</div>
|
||||
<p className="wiz-hint" style={{ marginTop: 8 }}>
|
||||
Columnas: radicado, documento, ingreso, total_cargos, autorizacion,
|
||||
tipo_documento, tipo_servicio, copago, cuota_moderadora (también acepta
|
||||
headers IT).
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* 2. Bots */}
|
||||
<div className="wiz-section">
|
||||
<div className="wiz-section-header">
|
||||
<h3 className="wiz-section-title">
|
||||
2. Bots
|
||||
{onlineWorkers.length > 0 && (
|
||||
<span className="wiz-count-badge">{onlineWorkers.length} online</span>
|
||||
)}
|
||||
</h3>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-ghost"
|
||||
onClick={() => void refreshWorkers()}
|
||||
disabled={refreshing}
|
||||
>
|
||||
{refreshing ? <Loader2 size={13} className="spin" /> : <RefreshCw size={13} />}
|
||||
Actualizar
|
||||
</button>
|
||||
</div>
|
||||
<p className="wiz-desc">
|
||||
El agent debe estar encendido en la PC con Integración Total. Con varios bots, las
|
||||
filas se reparte automáticamente.
|
||||
</p>
|
||||
|
||||
{onlineWorkers.length >= 2 && (
|
||||
<label
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 8,
|
||||
marginTop: 12,
|
||||
marginBottom: 8,
|
||||
cursor: "pointer",
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={useAllOnline}
|
||||
onChange={(e) => {
|
||||
const on = e.target.checked;
|
||||
setUseAllOnline(on);
|
||||
if (on) setSelectedIds(onlineWorkers.map((w) => w.id));
|
||||
}}
|
||||
/>
|
||||
<span>Usar todos los bots online ({onlineWorkers.length})</span>
|
||||
</label>
|
||||
)}
|
||||
|
||||
{workers.length === 0 ? (
|
||||
<div className="wiz-estimate-box" style={{ marginTop: 12, textAlign: "left" }}>
|
||||
<div style={{ display: "flex", alignItems: "flex-start", gap: 10 }}>
|
||||
<Bot size={18} style={{ flexShrink: 0, marginTop: 2 }} />
|
||||
<div>
|
||||
<div style={{ fontWeight: 600 }}>Ningún bot conectado</div>
|
||||
<p className="wiz-hint" style={{ margin: "4px 0 0" }}>
|
||||
En la PC del digitador ejecuta <code>run_gui.py</code> del agent y conéctalo
|
||||
al gateway.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="wiz-review-grid" style={{ marginTop: 12 }}>
|
||||
{workers.map((w) => {
|
||||
const online = w.status === "online" || w.status === "busy";
|
||||
const selected = effectiveTargets.includes(w.id);
|
||||
return (
|
||||
<button
|
||||
key={w.id}
|
||||
type="button"
|
||||
className={`wiz-review-item${selected ? " wiz-review-item--selected" : ""}`}
|
||||
disabled={!online || useAllOnline}
|
||||
onClick={() => online && toggleWorker(w.id)}
|
||||
style={{
|
||||
textAlign: "left",
|
||||
cursor: online && !useAllOnline ? "pointer" : "not-allowed",
|
||||
opacity: online ? 1 : 0.45,
|
||||
borderColor: selected ? "var(--accent, #3b8ed0)" : undefined,
|
||||
}}
|
||||
>
|
||||
<div>
|
||||
<span className="wiz-review-label">Worker</span>
|
||||
<span className="wiz-review-value">{w.id}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span className="wiz-review-label">Estado</span>
|
||||
<span className="wiz-review-value">
|
||||
{w.status}
|
||||
{w.hostname ? ` · ${w.hostname}` : ""}
|
||||
{selected ? " · seleccionado" : ""}
|
||||
</span>
|
||||
</div>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
{onlineWorkers.length === 0 && workers.length > 0 && (
|
||||
<p className="wiz-hint" style={{ marginTop: 8 }}>
|
||||
Hay workers registrados pero ninguno online. Revisa el agent.
|
||||
</p>
|
||||
)}
|
||||
{effectiveTargets.length > 1 && (
|
||||
<p className="wiz-hint" style={{ marginTop: 8 }}>
|
||||
Se repartirá entre {effectiveTargets.length} bots.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{step === 2 && activeJob && (
|
||||
<div className="wiz-step wiz-step-centered">
|
||||
<div className="wiz-hero">
|
||||
<div className={`wiz-hero-icon ${running ? "wiz-hero-icon--go" : ""}`}>
|
||||
{running ? (
|
||||
<Loader2 size={28} className="spin" />
|
||||
) : activeJob.status === "completed" ? (
|
||||
<Keyboard size={28} />
|
||||
) : activeJob.status === "failed" ? (
|
||||
<AlertCircle size={28} />
|
||||
) : (
|
||||
<Bot size={28} />
|
||||
)}
|
||||
</div>
|
||||
<h3 className="wiz-hero-title">
|
||||
{running
|
||||
? isDistributed
|
||||
? "Digitación distribuida en curso"
|
||||
: "Digitación en curso"
|
||||
: activeJob.status === "completed"
|
||||
? "Digitación completada"
|
||||
: activeJob.status === "cancelled"
|
||||
? "Digitación cancelada"
|
||||
: activeJob.status === "failed"
|
||||
? "Digitación fallida"
|
||||
: "Digitación finalizada"}
|
||||
</h3>
|
||||
<p className="wiz-desc">
|
||||
{isDistributed
|
||||
? `${Number(activeJob.shard_count) || shards.length || "?"} bots`
|
||||
: `Bot ${activeJob.worker_id ?? "—"}`}
|
||||
{activeJob.original_filename ? ` · ${activeJob.original_filename}` : ""}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{activeJob.error && (
|
||||
<div className="wiz-toast wiz-toast--error" style={{ position: "static", marginBottom: 12 }}>
|
||||
<AlertCircle size={15} />
|
||||
<span>{activeJob.error}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="wiz-progress">
|
||||
<div className="wiz-progress-header">
|
||||
<span>
|
||||
{Number(result.exitosos ?? processed)}/{total || "?"} filas
|
||||
</span>
|
||||
<span>{pct}%</span>
|
||||
</div>
|
||||
<div className="wiz-progress-track">
|
||||
<div className="wiz-progress-fill" style={{ width: `${pct}%` }} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="wiz-stats-row">
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{String(result.exitosos ?? processed)}</span>
|
||||
<span className="wiz-stat-label">OK</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{String(result.fallidos ?? 0)}</span>
|
||||
<span className="wiz-stat-label">Fallidos</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{String(result.existentes ?? 0)}</span>
|
||||
<span className="wiz-stat-label">Existentes</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{String(result.timeouts ?? 0)}</span>
|
||||
<span className="wiz-stat-label">Timeouts</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{shards.length > 0 && (
|
||||
<div className="wiz-section" style={{ marginTop: 16, width: "100%", maxWidth: 640 }}>
|
||||
<h3 className="wiz-section-title">Shards por bot</h3>
|
||||
<div className="wiz-review-grid" style={{ marginTop: 8 }}>
|
||||
{shards.map((s, i) => (
|
||||
<div key={s.id || String(i)} className="wiz-review-item">
|
||||
<div>
|
||||
<span className="wiz-review-label">
|
||||
Shard {(s.shard_index ?? i) + 1}
|
||||
</span>
|
||||
<span className="wiz-review-value">{s.worker_id || "—"}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span className="wiz-review-label">Estado</span>
|
||||
<span className="wiz-review-value">{s.status || "—"}</span>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="wiz-section" style={{ marginTop: 16, width: "100%", maxWidth: 640 }}>
|
||||
<h3 className="wiz-section-title">Log del agent</h3>
|
||||
<div
|
||||
className="wiz-estimate-box"
|
||||
style={{
|
||||
maxHeight: 220,
|
||||
overflow: "auto",
|
||||
fontFamily: "ui-monospace, SFMono-Regular, Menlo, monospace",
|
||||
fontSize: 12,
|
||||
textAlign: "left",
|
||||
}}
|
||||
>
|
||||
{events.length === 0 ? (
|
||||
<span className="wiz-hint">Esperando eventos…</span>
|
||||
) : (
|
||||
events.map((ev) => (
|
||||
<div key={ev.id}>
|
||||
[{ev.timestamp}] {ev.message}
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="wiz-toast wiz-toast--error">
|
||||
<AlertCircle size={15} />
|
||||
<span>{error}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<footer className="wiz-nav">
|
||||
{step === 2 ? (
|
||||
<button type="button" className="btn" onClick={resetWizard} disabled={running}>
|
||||
Nueva digitación
|
||||
</button>
|
||||
) : (
|
||||
<span />
|
||||
)}
|
||||
|
||||
<div className="wiz-nav-dots">
|
||||
<button
|
||||
type="button"
|
||||
className={`wiz-nav-dot ${step === 1 ? "active" : ""}`}
|
||||
onClick={() => !running && setStep(1)}
|
||||
disabled={running}
|
||||
aria-label="Paso 1"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
className={`wiz-nav-dot ${step === 2 ? "active" : ""}`}
|
||||
onClick={() => activeJob && setStep(2)}
|
||||
disabled={!activeJob}
|
||||
aria-label="Paso 2"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{step === 1 && (
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-primary"
|
||||
disabled={!canStart}
|
||||
onClick={() => void startJob()}
|
||||
>
|
||||
{busy ? <Loader2 size={15} className="spin" /> : <Play size={15} />}
|
||||
{effectiveTargets.length > 1
|
||||
? `Digitar con ${effectiveTargets.length} bots`
|
||||
: sourceMode === "aptos" && aptosCount > 0
|
||||
? `Digitar ${aptosCount} aptos`
|
||||
: "Iniciar digitación"}
|
||||
</button>
|
||||
)}
|
||||
{step === 2 && running && (
|
||||
<button type="button" className="btn" disabled={busy} onClick={() => void cancelJob()}>
|
||||
<Square size={14} /> Cancelar
|
||||
</button>
|
||||
)}
|
||||
{step === 2 && !running && (
|
||||
<button type="button" className="btn btn-primary" onClick={resetWizard}>
|
||||
<Play size={15} /> Otra digitación
|
||||
</button>
|
||||
)}
|
||||
</footer>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Download, Loader2, RefreshCw, Users } from "lucide-react";
|
||||
import { Download, Keyboard, Loader2, RefreshCw, Users } from "lucide-react";
|
||||
import { useEffect, useState } from "react";
|
||||
import type { PoblacionAptaJob } from "../../../shared/types";
|
||||
import { PoblacionAptaResultsTable } from "../../components/PoblacionAptaResultsTable";
|
||||
@@ -7,7 +7,7 @@ import type { ModuleResultsProps } from "../registry";
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps) {
|
||||
export function PoblacionAptaResults({ client, task, job, onActivateModule }: ModuleResultsProps) {
|
||||
const [paJob, setPaJob] = useState<PoblacionAptaJob | null>(null);
|
||||
const [rows, setRows] = useState<Row[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
@@ -60,6 +60,16 @@ export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps)
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text);
|
||||
}
|
||||
|
||||
async function exportTodos() {
|
||||
const paJobId = String(
|
||||
(task.result as Record<string, unknown>)?.poblacion_apta_job_id ?? "",
|
||||
);
|
||||
if (!paJobId) return;
|
||||
const blob = await client.poblacionAptaExportCsv(paJobId, "todos");
|
||||
const text = await blob.text();
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_todos", job.title), text);
|
||||
}
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="module-loading">
|
||||
@@ -88,6 +98,10 @@ export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps)
|
||||
}
|
||||
|
||||
const summary = paJob?.summary ?? {};
|
||||
const paJobId = String(
|
||||
(task.result as Record<string, unknown>)?.poblacion_apta_job_id ?? paJob?.id ?? "",
|
||||
);
|
||||
const canDigitador = paJob?.status === "completed" && Boolean(paJobId) && Boolean(onActivateModule);
|
||||
|
||||
return (
|
||||
<div className="module-results">
|
||||
@@ -108,6 +122,23 @@ export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps)
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportNoAptos()}>
|
||||
<Download size={13} /> CSV no aptos ({summary.con_error ?? 0})
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportTodos()}>
|
||||
<Download size={13} /> CSV todos ({summary.total ?? 0})
|
||||
</button>
|
||||
{canDigitador && (
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-primary"
|
||||
disabled={Number(summary.paso_bot ?? 0) === 0}
|
||||
onClick={() =>
|
||||
onActivateModule?.("digitador", {
|
||||
poblacionAptaJobId: paJobId,
|
||||
})
|
||||
}
|
||||
>
|
||||
<Keyboard size={13} /> Digitar aptos
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
CheckSquare,
|
||||
Download,
|
||||
FileSpreadsheet,
|
||||
Keyboard,
|
||||
Loader2,
|
||||
Square,
|
||||
Upload,
|
||||
@@ -235,6 +236,13 @@ export function PoblacionAptaWizard({
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text);
|
||||
}
|
||||
|
||||
async function exportTodos() {
|
||||
if (!resultJob) return;
|
||||
const blob = await client.poblacionAptaExportCsv(resultJob.id, "todos");
|
||||
const text = await blob.text();
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_todos", job.title), text);
|
||||
}
|
||||
|
||||
function reset() {
|
||||
setDone(false);
|
||||
setResultJob(null);
|
||||
@@ -271,6 +279,23 @@ export function PoblacionAptaWizard({
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportNoAptos()}>
|
||||
<Download size={13} /> CSV no aptos ({summary.con_error ?? 0})
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportTodos()}>
|
||||
<Download size={13} /> CSV todos ({summary.total ?? 0})
|
||||
</button>
|
||||
{onActivateModule && (
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-primary"
|
||||
disabled={Number(summary.paso_bot ?? 0) === 0}
|
||||
onClick={() =>
|
||||
onActivateModule("digitador", {
|
||||
poblacionAptaJobId: resultJob.id,
|
||||
})
|
||||
}
|
||||
>
|
||||
<Keyboard size={13} /> Digitar aptos
|
||||
</button>
|
||||
)}
|
||||
<button type="button" className="btn btn-sm" onClick={reset}>
|
||||
Nueva corrida
|
||||
</button>
|
||||
|
||||
@@ -4,4 +4,5 @@ export type JobSection =
|
||||
| "cleaning"
|
||||
| "poblacion_apta"
|
||||
| "downloads"
|
||||
| "digitador"
|
||||
| "results";
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
Cpu,
|
||||
Download,
|
||||
Hash,
|
||||
Keyboard,
|
||||
Loader2,
|
||||
Plus,
|
||||
RefreshCw,
|
||||
@@ -29,8 +30,9 @@ import type { ApiClient } from "../lib/apiClient";
|
||||
import { downloadCsv, toCsv, buildExportCsvFilename } from "../lib/csv";
|
||||
import { taskTimingLabel, timingDuration } from "../lib/time";
|
||||
import { DownloadsResultsPanel } from "../modules/downloads/DownloadsResultsPanel";
|
||||
import { DigitadorResultsPanel } from "../modules/digitador/DigitadorResultsPanel";
|
||||
|
||||
type ResultTab = "counting" | "extraction" | "cleaning" | "poblacion_apta" | "downloads";
|
||||
type ResultTab = "counting" | "extraction" | "cleaning" | "poblacion_apta" | "downloads" | "digitador";
|
||||
|
||||
interface Props {
|
||||
client: ApiClient;
|
||||
@@ -46,11 +48,15 @@ interface Props {
|
||||
onStartCleaning: (ctx?: Record<string, unknown>) => void;
|
||||
onStartPoblacionApta: () => void;
|
||||
onStartDownload: () => void;
|
||||
onStartDigitador: () => void;
|
||||
onContinueToCounting?: (ctx: {
|
||||
downloadJobId: string;
|
||||
downloadsPrefix?: string;
|
||||
files?: DownloadJobFile[];
|
||||
}) => void;
|
||||
onContinueToDigitador?: (ctx: {
|
||||
poblacionAptaJobId: string;
|
||||
}) => void;
|
||||
}
|
||||
|
||||
export function JobResultsView({
|
||||
@@ -67,7 +73,9 @@ export function JobResultsView({
|
||||
onStartCleaning,
|
||||
onStartPoblacionApta,
|
||||
onStartDownload,
|
||||
onStartDigitador,
|
||||
onContinueToCounting,
|
||||
onContinueToDigitador,
|
||||
}: Props) {
|
||||
const [activeTab, setActiveTab] = useState<ResultTab>("counting");
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
@@ -82,6 +90,8 @@ export function JobResultsView({
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
const downloadTasks = tasks.filter((t) => t.type === "downloads")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
const digitadorTasks = tasks.filter((t) => t.type === "digitador")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
|
||||
async function refresh() {
|
||||
setRefreshing(true);
|
||||
@@ -164,6 +174,17 @@ export function JobResultsView({
|
||||
<span className="job-results-tab-count">{downloadTasks.length}</span>
|
||||
)}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`job-results-tab${activeTab === "digitador" ? " job-results-tab--active" : ""}`}
|
||||
onClick={() => setActiveTab("digitador")}
|
||||
>
|
||||
<Keyboard size={15} />
|
||||
Bot digitador
|
||||
{digitadorTasks.length > 0 && (
|
||||
<span className="job-results-tab-count">{digitadorTasks.length}</span>
|
||||
)}
|
||||
</button>
|
||||
<div style={{ flex: 1 }} />
|
||||
<button
|
||||
type="button"
|
||||
@@ -216,6 +237,7 @@ export function JobResultsView({
|
||||
tasks={poblacionAptaTasks}
|
||||
liveProgress={poblacionAptaProgress}
|
||||
onStartPoblacionApta={onStartPoblacionApta}
|
||||
onContinueToDigitador={onContinueToDigitador}
|
||||
/>
|
||||
)}
|
||||
{activeTab === "downloads" && (
|
||||
@@ -227,6 +249,14 @@ export function JobResultsView({
|
||||
onContinueToCounting={onContinueToCounting}
|
||||
/>
|
||||
)}
|
||||
{activeTab === "digitador" && (
|
||||
<DigitadorResultsPanel
|
||||
client={client}
|
||||
job={job}
|
||||
tasks={digitadorTasks}
|
||||
onStartDigitador={onStartDigitador}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
@@ -780,12 +810,16 @@ function PoblacionAptaResultsSection({
|
||||
tasks,
|
||||
liveProgress,
|
||||
onStartPoblacionApta,
|
||||
onContinueToDigitador,
|
||||
}: {
|
||||
client: ApiClient;
|
||||
job: Job;
|
||||
tasks: Task[];
|
||||
liveProgress?: BackgroundProgress | null;
|
||||
onStartPoblacionApta: () => void;
|
||||
onContinueToDigitador?: (ctx: {
|
||||
poblacionAptaJobId: string;
|
||||
}) => void;
|
||||
}) {
|
||||
const [selectedTask, setSelectedTask] = useState<Task | null>(tasks[0] ?? null);
|
||||
const [paJob, setPaJob] = useState<PoblacionAptaJob | null>(null);
|
||||
@@ -853,6 +887,16 @@ function PoblacionAptaResultsSection({
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text);
|
||||
}
|
||||
|
||||
async function exportTodos() {
|
||||
const paJobId = String(
|
||||
(selectedTask?.result as Record<string, unknown>)?.poblacion_apta_job_id ?? "",
|
||||
);
|
||||
if (!paJobId) return;
|
||||
const blob = await client.poblacionAptaExportCsv(paJobId, "todos");
|
||||
const text = await blob.text();
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_todos", job.title), text);
|
||||
}
|
||||
|
||||
if (tasks.length === 0 && !liveProgress?.running) {
|
||||
return (
|
||||
<div className="job-results-empty">
|
||||
@@ -874,6 +918,13 @@ function PoblacionAptaResultsSection({
|
||||
}
|
||||
|
||||
const summary = paJob?.summary ?? {};
|
||||
const paJobId = String(
|
||||
(selectedTask?.result as Record<string, unknown>)?.poblacion_apta_job_id ?? paJob?.id ?? "",
|
||||
);
|
||||
const canDigitador =
|
||||
Boolean(onContinueToDigitador) &&
|
||||
paJob?.status === "completed" &&
|
||||
Boolean(paJobId);
|
||||
|
||||
return (
|
||||
<div className="results-section">
|
||||
@@ -917,6 +968,21 @@ function PoblacionAptaResultsSection({
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportNoAptos()}>
|
||||
<Download size={13} /> CSV no aptos ({summary.con_error ?? 0})
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportTodos()}>
|
||||
<Download size={13} /> CSV todos ({summary.total ?? 0})
|
||||
</button>
|
||||
{canDigitador && (
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-primary"
|
||||
onClick={() =>
|
||||
onContinueToDigitador?.({ poblacionAptaJobId: paJobId })
|
||||
}
|
||||
disabled={Number(summary.paso_bot ?? 0) === 0}
|
||||
>
|
||||
<Keyboard size={13} /> Digitar aptos
|
||||
</button>
|
||||
)}
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={onStartPoblacionApta}>
|
||||
<Plus size={13} /> Nueva corrida
|
||||
</button>
|
||||
|
||||
@@ -9,6 +9,7 @@ import { CleaningWizardModule } from "../modules/cleaning/CleaningWizardModule";
|
||||
import { PoblacionAptaWizardModule } from "../modules/poblacion_apta/PoblacionAptaWizardModule";
|
||||
import { JobExtractionWizard } from "../modules/extraction/JobExtractionWizard";
|
||||
import { DownloadsWizardModule } from "../modules/downloads/DownloadsWizardModule";
|
||||
import { DigitadorWizardModule } from "../modules/digitador/DigitadorWizardModule";
|
||||
import { JobResultsView } from "./JobResultsView";
|
||||
|
||||
interface Props {
|
||||
@@ -127,6 +128,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
{section === "cleaning" && "Limpieza de datos"}
|
||||
{section === "poblacion_apta" && "Población apta"}
|
||||
{section === "downloads" && "Bot de descarga"}
|
||||
{section === "digitador" && "Bot digitador"}
|
||||
{section === "results" && "Resultados del trabajo"}
|
||||
</div>
|
||||
</header>
|
||||
@@ -201,6 +203,19 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
<DownloadsWizardModule client={client} job={job} />
|
||||
</div>
|
||||
|
||||
<div className={`job-section-pane${section === "digitador" ? "" : " job-section-pane--hidden"}`}>
|
||||
<DigitadorWizardModule
|
||||
key={`digitador-${String(sectionContext.poblacionAptaJobId ?? "default")}`}
|
||||
client={client}
|
||||
job={job}
|
||||
initialSource={
|
||||
typeof sectionContext.poblacionAptaJobId === "string"
|
||||
? { poblacionAptaJobId: sectionContext.poblacionAptaJobId }
|
||||
: null
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className={`job-section-pane${section === "results" ? "" : " job-section-pane--hidden"}`}>
|
||||
<JobResultsView
|
||||
client={client}
|
||||
@@ -216,7 +231,9 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
onStartCleaning={(ctx?: Record<string, unknown>) => onActivateSection("cleaning", ctx)}
|
||||
onStartPoblacionApta={() => onActivateSection("poblacion_apta")}
|
||||
onStartDownload={() => onActivateSection("downloads")}
|
||||
onStartDigitador={() => onActivateSection("digitador")}
|
||||
onContinueToCounting={(ctx) => onActivateSection("counting", ctx)}
|
||||
onContinueToDigitador={(ctx) => onActivateSection("digitador", ctx)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -373,7 +373,7 @@ export interface DownloadWorker {
|
||||
|
||||
export interface DownloadJob {
|
||||
id: string;
|
||||
worker_id: string;
|
||||
worker_id: string | null;
|
||||
status: "pending" | "running" | "completed" | "failed" | "cancelled" | string;
|
||||
title: string;
|
||||
excel_key: string;
|
||||
@@ -389,6 +389,11 @@ export interface DownloadJob {
|
||||
job_id?: string | null;
|
||||
/** Task type=downloads */
|
||||
task_id?: string | null;
|
||||
parent_download_job_id?: string | null;
|
||||
shard_index?: number | null;
|
||||
shard_count?: number | null;
|
||||
/** Presente en GET de padre distribuido */
|
||||
shards?: DownloadJob[];
|
||||
}
|
||||
|
||||
export interface DownloadJobFile {
|
||||
@@ -417,6 +422,63 @@ export interface DownloadJobEvent {
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** Worker del bot digitador (control remoto). */
|
||||
export interface DigitadorWorker {
|
||||
id: string;
|
||||
hostname: string;
|
||||
max_concurrent: number;
|
||||
status: "online" | "offline" | "busy" | string;
|
||||
last_heartbeat: string;
|
||||
current_job_id: string | null;
|
||||
}
|
||||
|
||||
export interface DigitadorJob {
|
||||
id: string;
|
||||
worker_id: string | null;
|
||||
status: "pending" | "running" | "completed" | "failed" | "cancelled" | string;
|
||||
title: string;
|
||||
csv_key: string;
|
||||
reports_prefix: string;
|
||||
original_filename: string;
|
||||
cancel_requested: boolean;
|
||||
result: Record<string, unknown>;
|
||||
error: string | null;
|
||||
created_at: string;
|
||||
started_at: string | null;
|
||||
finished_at: string | null;
|
||||
job_id?: string | null;
|
||||
task_id?: string | null;
|
||||
parent_digitador_job_id?: string | null;
|
||||
shard_index?: number | null;
|
||||
shard_count?: number | null;
|
||||
shards?: DigitadorJob[];
|
||||
}
|
||||
|
||||
export interface DigitadorJobFile {
|
||||
key: string;
|
||||
name?: string | null;
|
||||
}
|
||||
|
||||
export interface DigitadorJobFilesResponse {
|
||||
job_id: string;
|
||||
prefix: string;
|
||||
files: DigitadorJobFile[];
|
||||
count: number;
|
||||
}
|
||||
|
||||
export interface DigitadorJobEvent {
|
||||
id: number;
|
||||
job_id: string;
|
||||
event_type: string;
|
||||
message: string;
|
||||
timestamp: string;
|
||||
radicado?: string | null;
|
||||
nit?: string | null;
|
||||
index?: number | null;
|
||||
total?: number | null;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface DesktopBridge {
|
||||
getRuntimeConfig(): Promise<RuntimeConfig>;
|
||||
selectDirectory(): Promise<string | null>;
|
||||
|
||||
@@ -15,6 +15,7 @@ from domain.verificador_st import (
|
||||
consolidado_to_records,
|
||||
export_no_aptos_csv,
|
||||
export_paso_bot_csv,
|
||||
export_todos_csv,
|
||||
process_cleaning_sources,
|
||||
)
|
||||
from domain.verificador_st.core import cargar_origen_servicio_desde_bytes
|
||||
@@ -483,11 +484,13 @@ def get_rows(
|
||||
@app.get("/jobs/{job_id}/export.csv")
|
||||
def export_csv(
|
||||
job_id: str,
|
||||
subset: str = Query("aptos", description="aptos | no_aptos"),
|
||||
subset: str = Query("aptos", description="aptos | no_aptos | todos"),
|
||||
) -> Response:
|
||||
subset_norm = (subset or "aptos").strip().lower()
|
||||
if subset_norm not in {"aptos", "no_aptos", "errores"}:
|
||||
raise HTTPException(status_code=400, detail="subset debe ser aptos o no_aptos")
|
||||
if subset_norm in {"all", "completo", "consolidado"}:
|
||||
subset_norm = "todos"
|
||||
if subset_norm not in {"aptos", "no_aptos", "errores", "todos"}:
|
||||
raise HTTPException(status_code=400, detail="subset debe ser aptos, no_aptos o todos")
|
||||
if subset_norm == "errores":
|
||||
subset_norm = "no_aptos"
|
||||
|
||||
@@ -504,6 +507,9 @@ def export_csv(
|
||||
if subset_norm == "aptos":
|
||||
csv_text = export_paso_bot_csv(payload)
|
||||
filename = f"poblacion_apta_aptos_{job_id[:8]}.csv"
|
||||
elif subset_norm == "todos":
|
||||
csv_text = export_todos_csv(payload)
|
||||
filename = f"poblacion_apta_todos_{job_id[:8]}.csv"
|
||||
else:
|
||||
csv_text = export_no_aptos_csv(payload)
|
||||
filename = f"poblacion_apta_no_aptos_{job_id[:8]}.csv"
|
||||
|
||||
@@ -3,6 +3,7 @@ from domain.verificador_st.pipeline import (
|
||||
consolidado_to_records,
|
||||
export_no_aptos_csv,
|
||||
export_paso_bot_csv,
|
||||
export_todos_csv,
|
||||
process_cleaning_sources,
|
||||
)
|
||||
|
||||
@@ -10,5 +11,6 @@ __all__ = [
|
||||
"consolidado_to_records",
|
||||
"export_no_aptos_csv",
|
||||
"export_paso_bot_csv",
|
||||
"export_todos_csv",
|
||||
"process_cleaning_sources",
|
||||
]
|
||||
|
||||
@@ -51,6 +51,8 @@ TABLA_MODERADORA = {
|
||||
2025: {4700, 19200, 50300},
|
||||
2026: {5000, 20100, 52800},
|
||||
}
|
||||
# Todos los montos de la tabla; la clasificación no filtra por año.
|
||||
VALORES_MODERADORA = {v for vals in TABLA_MODERADORA.values() for v in vals}
|
||||
|
||||
# Longitudes según tabla entregada por el usuario.
|
||||
# Los tipos marcados como alfanuméricos permiten letras y números.
|
||||
@@ -554,14 +556,15 @@ def normalizar_autorizacion(valor):
|
||||
|
||||
def separar_copago_moderadora(valor, anio):
|
||||
"""
|
||||
Si el valor coincide con tabla moderadora del año => cuota_moderadora.
|
||||
Si no coincide => copago.
|
||||
Si el valor coincide con cualquier cuota de la tabla (sin importar el año)
|
||||
=> cuota_moderadora. Si no coincide => copago.
|
||||
El parámetro anio se conserva por compatibilidad de firma.
|
||||
"""
|
||||
monto = to_int(valor, default=0)
|
||||
if monto == 0:
|
||||
return 0, 0
|
||||
|
||||
if anio in TABLA_MODERADORA and monto in TABLA_MODERADORA[anio]:
|
||||
if monto in VALORES_MODERADORA:
|
||||
return 0, monto
|
||||
|
||||
return monto, 0
|
||||
|
||||
@@ -223,3 +223,21 @@ def export_no_aptos_csv(rows: list[dict[str, Any]]) -> str:
|
||||
df = df.rename(columns=CSV_FINAL_IT_RENAME)
|
||||
df = df.reindex(columns=headers)
|
||||
return df.to_csv(sep="|", index=False, encoding="utf-8-sig")
|
||||
|
||||
|
||||
def export_todos_csv(rows: list[dict[str, Any]]) -> str:
|
||||
"""CSV pipe-separated con todas las filas (aptos y no aptos) e OBSERVACION."""
|
||||
headers = [*CSV_FINAL_IT_HEADERS, "OBSERVACION"]
|
||||
if not rows:
|
||||
df = pd.DataFrame(columns=headers)
|
||||
else:
|
||||
df = pd.DataFrame(rows)
|
||||
for c in COLUMNAS_FINAL_IT:
|
||||
if c not in df.columns:
|
||||
df[c] = ""
|
||||
if "OBSERVACION" not in df.columns:
|
||||
df["OBSERVACION"] = ""
|
||||
df = df.reindex(columns=[*COLUMNAS_FINAL_IT, "OBSERVACION"]).fillna("")
|
||||
df = df.rename(columns=CSV_FINAL_IT_RENAME)
|
||||
df = df.reindex(columns=headers)
|
||||
return df.to_csv(sep="|", index=False, encoding="utf-8-sig")
|
||||
|
||||
Reference in New Issue
Block a user