feat: Introduce bot-digitador service for automated data entry in Integración Total. Add CLI and GUI interfaces for CSV processing, implement auto-start functionality on Windows, and enhance README with setup instructions and usage details. Include necessary configurations in docker-compose for service integration.

2026-08-09 21:26:56 -05:00
parent a3b4c7fc47
commit e9b766281a
74 changed files with 11177 additions and 115 deletions
@@ -87,26 +87,30 @@ def claim_pending(

@app.post("/jobs")
 async def create_job(
-    worker_id: str = Form(...),
+    worker_id: str = Form(""),
+    worker_ids: str = Form(""),
    job_id: str = Form(...),
    title: str = Form(""),
    file: UploadFile = File(...),
    _: str = Depends(require_bearer),
 ) -> dict:
-    if not worker_id.strip():
-        raise HTTPException(status_code=400, detail="worker_id obligatorio")
+    """Crea una corrida de descarga.
+
+    - 1 worker → flujo clásico (un solo job claimable).
+    - N workers (worker_ids JSON o todos online) → padre + N shards,
+      mismo downloads_prefix del padre.
+    """
+    import json as _json
+
+    from domain.excel_split import parse_radicado_rows, rows_to_xlsx_bytes, split_rows
+    from infrastructure.storage_minio import downloads_prefix as mk_prefix
+
    parent_job_id = job_id.strip()
    if not parent_job_id:
        raise HTTPException(status_code=400, detail="job_id (Job Desktop) obligatorio")
    if not store.parent_job_exists(parent_job_id):
        raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado")

-    if not store.is_worker_online(worker_id):
-        raise HTTPException(
-            status_code=409,
-            detail=f"Worker '{worker_id}' no está online. Enciende el agent primero.",
-        )
-
    data = await file.read()
    if not data:
        raise HTTPException(status_code=400, detail="Archivo vacío")
@@ -115,45 +119,224 @@ async def create_job(
    content_type = file.content_type or "application/octet-stream"
    title_clean = title.strip()

-    job = store.create_job(
-        worker_id=worker_id.strip(),
+    # Resolver workers
+    selected: List[str] = []
+    raw_ids = (worker_ids or "").strip()
+    if raw_ids:
+        try:
+            parsed = _json.loads(raw_ids)
+            if isinstance(parsed, list):
+                selected = [str(x).strip() for x in parsed if str(x).strip()]
+            elif isinstance(parsed, str) and parsed.strip():
+                selected = [parsed.strip()]
+        except _json.JSONDecodeError:
+            selected = [x.strip() for x in raw_ids.split(",") if x.strip()]
+    if not selected and worker_id.strip():
+        selected = [worker_id.strip()]
+
+    online = store.list_online_worker_ids()
+    online_set = set(online)
+    if not selected:
+        selected = list(online)
+    else:
+        missing = [w for w in selected if w not in online_set]
+        if missing:
+            raise HTTPException(
+                status_code=409,
+                detail=f"Workers no online: {', '.join(missing)}. Enciende el agent primero.",
+            )
+
+    if not selected:
+        raise HTTPException(
+            status_code=409,
+            detail="No hay bots online. Enciende al menos un agent.",
+        )
+
+    # Dedup preservando orden
+    seen: set[str] = set()
+    workers_unique: List[str] = []
+    for w in selected:
+        if w not in seen:
+            seen.add(w)
+            workers_unique.append(w)
+
+    # Parse filas
+    try:
+        rows = parse_radicado_rows(data, original)
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"No se pudo leer el Excel/CSV: {e}") from e
+    if not rows:
+        raise HTTPException(status_code=400, detail="El archivo no tiene filas de radicado/NIT")
+
+    # Un solo bot (o una sola fila) → flujo actual
+    n_workers = min(len(workers_unique), len(rows))
+    if n_workers <= 1:
+        only = workers_unique[0]
+        job = store.create_job(
+            worker_id=only,
+            excel_key="",
+            original_filename=original,
+            title=title_clean,
+            job_id=parent_job_id,
+        )
+        task_id = store.create_downloads_task(
+            parent_job_id=parent_job_id,
+            download_job_id=job.id,
+            worker_id=only,
+            title=title_clean or f"Bot descarga ({original})",
+            original_filename=original,
+            worker_ids=[only],
+            mode="single",
+        )
+        try:
+            key = minio.upload_excel(job.id, data, original, content_type=content_type)
+        except Exception as e:
+            store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
+            raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
+
+        job = store.update_excel_key(job.id, key)
+        assert job is not None
+        store.add_event(
+            job.id,
+            "run_started",
+            f"Job creado y asignado a {only}",
+            metadata={
+                "excel_key": key,
+                "worker_id": only,
+                "job_id": parent_job_id,
+                "task_id": task_id,
+                "mode": "single",
+                "total_rows": len(rows),
+            },
+        )
+        return job.to_dict()
+
+    # Distribuido: padre + shards
+    workers_unique = workers_unique[:n_workers]
+    chunks = split_rows(rows, n_workers)
+    n = len(chunks)
+    workers_unique = workers_unique[:n]
+
+    parent = store.create_job(
+        worker_id=None,
        excel_key="",
        original_filename=original,
-        title=title_clean,
+        title=title_clean or f"Descarga distribuida ({original})",
        job_id=parent_job_id,
-    )
-    task_id = store.create_downloads_task(
-        parent_job_id=parent_job_id,
-        download_job_id=job.id,
-        worker_id=worker_id.strip(),
-        title=title_clean or f"Bot descarga ({original})",
-        original_filename=original,
-    )
-    job = store.get_job(job.id)
-    assert job is not None
-
-    try:
-        key = minio.upload_excel(job.id, data, original, content_type=content_type)
-    except Exception as e:
-        store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
-        raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
-
-    job = store.update_excel_key(job.id, key)
-    assert job is not None
-    store.add_event(
-        job.id,
-        "run_started",
-        f"Job creado y asignado a {worker_id}",
-        metadata={
-            "excel_key": key,
-            "worker_id": worker_id,
-            "job_id": parent_job_id,
-            "task_id": task_id,
+        shard_count=n,
+        status=JobStatus.PENDING.value,
+        result={
+            "mode": "distributed",
+            "shard_ids": [],
+            "total_rows": len(rows),
+            "total": len(rows),
+            "processed": 0,
+            "exitosos": 0,
+            "fallidos": 0,
        },
    )
+    prefix = parent.downloads_prefix or mk_prefix(parent.id)
+
+    task_id = store.create_downloads_task(
+        parent_job_id=parent_job_id,
+        download_job_id=parent.id,
+        worker_id=None,
+        title=title_clean or f"Bot descarga ×{n} ({original})",
+        original_filename=original,
+        worker_ids=workers_unique,
+        mode="distributed",
+    )
+
+    try:
+        full_key = minio.upload_excel(parent.id, data, original, content_type=content_type)
+    except Exception as e:
+        store.patch_job(parent.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
+        raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
+
+    store.update_excel_key(parent.id, full_key)
+
+    shard_ids: List[str] = []
+    for i, (chunk, wid) in enumerate(zip(chunks, workers_unique)):
+        shard_bytes = rows_to_xlsx_bytes(chunk)
+        shard = store.create_job(
+            worker_id=wid,
+            excel_key="",
+            original_filename=f"shard_{i + 1}_of_{n}.xlsx",
+            title=f"{title_clean or 'Descarga'} · shard {i + 1}/{n}",
+            job_id=parent_job_id,
+            task_id=task_id,
+            downloads_prefix_override=prefix,
+            parent_download_job_id=parent.id,
+            shard_index=i,
+            shard_count=n,
+            result={"total": len(chunk), "processed": 0},
+        )
+        try:
+            shard_key = minio.upload_excel(
+                shard.id,
+                shard_bytes,
+                f"shard_{i + 1}_of_{n}.xlsx",
+                content_type=(
+                    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+                ),
+            )
+        except Exception as e:
+            store.patch_job(
+                parent.id,
+                status=JobStatus.FAILED.value,
+                error=f"MinIO shard upload failed: {e}",
+            )
+            raise HTTPException(
+                status_code=502, detail=f"Error subiendo shard a MinIO: {e}"
+            ) from e
+        store.update_excel_key(shard.id, shard_key)
+        shard_ids.append(shard.id)
+        store.add_event(
+            parent.id,
+            "info",
+            f"Shard {i + 1}/{n} → {wid} ({len(chunk)} filas)",
+            metadata={"shard_id": shard.id, "worker_id": wid, "rows": len(chunk)},
+        )
+
+    store.patch_job(
+        parent.id,
+        result={
+            "mode": "distributed",
+            "shard_ids": shard_ids,
+            "total_rows": len(rows),
+            "total": len(rows),
+            "processed": 0,
+            "exitosos": 0,
+            "fallidos": 0,
+            "downloads_prefix": prefix,
+            "worker_ids": workers_unique,
+        },
+    )
+    store.add_event(
+        parent.id,
+        "run_started",
+        f"Corrida distribuida: {n} bots, {len(rows)} filas",
+        metadata={
+            "excel_key": full_key,
+            "worker_ids": workers_unique,
+            "job_id": parent_job_id,
+            "task_id": task_id,
+            "mode": "distributed",
+            "shard_ids": shard_ids,
+            "total_rows": len(rows),
+        },
+    )
+    job = store.get_job(parent.id)
+    assert job is not None
    return job.to_dict()


+@app.post("/internal/reclaim")
+def reclaim_jobs(_: str = Depends(require_bearer)) -> dict:
+    n = store.reclaim_stale_running_jobs()
+    return {"reclaimed": n}
+
+
@app.get("/jobs")
 def list_jobs(
    limit: int = Query(100, ge=1, le=500),
@@ -168,7 +351,15 @@ def get_job(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
    job = store.get_job(download_job_id)
    if not job:
        raise HTTPException(status_code=404, detail="Job no encontrado")
-    return job.to_dict()
+    # Refrescar consolidación si es padre distribuido
+    if job.is_distributed_parent:
+        refreshed = store.refresh_parent_from_shards(download_job_id)
+        if refreshed:
+            job = refreshed
+    payload = job.to_dict()
+    if job.is_distributed_parent:
+        payload["shards"] = [s.to_dict() for s in store.list_shards(job.id)]
+    return payload


@app.get("/jobs/{download_job_id}/files")
@@ -279,7 +470,8 @@ def post_event(
    body: EventBody,
    _: str = Depends(require_bearer),
 ) -> dict:
-    if not store.get_job(download_job_id):
+    job = store.get_job(download_job_id)
+    if not job:
        raise HTTPException(status_code=404, detail="Job no encontrado")
    event = store.add_event(
        download_job_id,
@@ -292,8 +484,7 @@ def post_event(
        metadata=body.metadata,
        timestamp=body.timestamp,
    )
-    job = store.get_job(download_job_id)
-    if job and body.index is not None and body.total is not None:
+    if body.index is not None and body.total is not None:
        result = dict(job.result or {})
        result["processed"] = body.index
        result["total"] = body.total
@@ -327,4 +518,7 @@ def cancel_flag(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
    job = store.get_job(download_job_id)
    if not job:
        raise HTTPException(status_code=404, detail="Job no encontrado")
-    return {"cancel_requested": job.cancel_requested, "status": job.status}
+    return {
+        "cancel_requested": store.is_cancel_requested(download_job_id),
+        "status": job.status,
+    }