feat: Introduce bot-digitador service for automated data entry in Integración Total. Add CLI and GUI interfaces for CSV processing, implement auto-start functionality on Windows, and enhance README with setup instructions and usage details. Include necessary configurations in docker-compose for service integration.
This commit is contained in:
+1395
File diff suppressed because it is too large
Load Diff
@@ -37,6 +37,15 @@ python run_gui.py
|
||||
|
||||
Configura URL API (ej. `https://pys1.builker.xyz/bot-descarga` o `http://127.0.0.1:8189/bot-descarga`), Worker ID y token. Pulsa **Conectar agent**. Queda idle hasta que el desktop asigne un job.
|
||||
|
||||
### Autoarranque en Windows
|
||||
|
||||
En la misma GUI:
|
||||
|
||||
1. Marca **Iniciar con Windows** (crea un `.bat` en la carpeta Startup del usuario).
|
||||
2. Deja **Conectar al abrir** activo para que tras el login el agent quede online solo.
|
||||
|
||||
Tras reiniciar el PC e iniciar sesión, se abre la GUI y se registra el worker automáticamente.
|
||||
|
||||
## Arranque solo agent (CLI)
|
||||
|
||||
```bash
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
|
||||
import customtkinter as ctk
|
||||
@@ -11,6 +12,11 @@ from bot_descarga.ui.settings_store import (
|
||||
normalize_api_url,
|
||||
save_gui_settings,
|
||||
)
|
||||
from bot_descarga.ui.windows_autostart import (
|
||||
disable_autostart,
|
||||
enable_autostart,
|
||||
is_autostart_enabled,
|
||||
)
|
||||
from bot_descarga.worker.agent import WorkerAgent
|
||||
|
||||
ctk.set_appearance_mode("dark")
|
||||
@@ -24,7 +30,7 @@ class BotDashboard(ctk.CTk):
|
||||
super().__init__()
|
||||
|
||||
self.title("Bot de Descargas PDF — Agent")
|
||||
self.geometry("920x680")
|
||||
self.geometry("920x720")
|
||||
self.resizable(False, False)
|
||||
|
||||
self._agent: WorkerAgent | None = None
|
||||
@@ -36,6 +42,10 @@ class BotDashboard(ctk.CTk):
|
||||
|
||||
self.protocol("WM_DELETE_WINDOW", self._on_close)
|
||||
|
||||
# Tras login de Windows: si está marcado, conectar solo
|
||||
if bool(load_gui_settings().get("auto_connect")):
|
||||
self.after(800, self._maybe_auto_connect)
|
||||
|
||||
def _construir_ui(self):
|
||||
root = ctk.CTkFrame(self, corner_radius=18)
|
||||
root.pack(fill="both", expand=True, padx=18, pady=18)
|
||||
@@ -81,7 +91,7 @@ class BotDashboard(ctk.CTk):
|
||||
self.entry_worker.pack(side="left", fill="x", expand=True)
|
||||
|
||||
row_token = ctk.CTkFrame(config, fg_color="transparent")
|
||||
row_token.pack(fill="x", padx=18, pady=(0, 14))
|
||||
row_token.pack(fill="x", padx=18, pady=(0, 8))
|
||||
ctk.CTkLabel(row_token, text="Token", width=90, anchor="w").pack(side="left")
|
||||
self.entry_token = ctk.CTkEntry(
|
||||
row_token,
|
||||
@@ -97,6 +107,23 @@ class BotDashboard(ctk.CTk):
|
||||
command=self._toggle_token,
|
||||
).pack(side="left")
|
||||
|
||||
row_auto = ctk.CTkFrame(config, fg_color="transparent")
|
||||
row_auto.pack(fill="x", padx=18, pady=(0, 14))
|
||||
self.chk_autostart = ctk.CTkCheckBox(
|
||||
row_auto,
|
||||
text="Iniciar con Windows",
|
||||
command=self._on_autostart_toggle,
|
||||
)
|
||||
self.chk_autostart.pack(side="left", padx=(0, 18))
|
||||
self.chk_auto_connect = ctk.CTkCheckBox(
|
||||
row_auto,
|
||||
text="Conectar al abrir",
|
||||
command=self._on_auto_connect_toggle,
|
||||
)
|
||||
self.chk_auto_connect.pack(side="left")
|
||||
if sys.platform != "win32":
|
||||
self.chk_autostart.configure(state="disabled")
|
||||
|
||||
status = ctk.CTkFrame(root, corner_radius=14)
|
||||
status.pack(fill="x", padx=24, pady=(0, 14))
|
||||
|
||||
@@ -158,19 +185,77 @@ class BotDashboard(ctk.CTk):
|
||||
)
|
||||
self.text_logs.pack(fill="both", expand=True, padx=18, pady=(0, 16))
|
||||
|
||||
def _persist_flags(self):
|
||||
save_gui_settings(
|
||||
normalize_api_url(self.entry_api.get()),
|
||||
self.entry_worker.get(),
|
||||
self.entry_token.get(),
|
||||
autostart_windows=bool(self.chk_autostart.get()),
|
||||
auto_connect=bool(self.chk_auto_connect.get()),
|
||||
)
|
||||
|
||||
def _cargar_settings(self):
|
||||
s = load_gui_settings()
|
||||
self.entry_api.insert(0, s.get("api_url") or DEFAULT_API_URL)
|
||||
self.entry_worker.insert(0, s.get("worker_id") or socket.gethostname())
|
||||
self.entry_token.insert(0, s.get("token") or "")
|
||||
|
||||
want_autostart = bool(s.get("autostart_windows"))
|
||||
if want_autostart and sys.platform == "win32":
|
||||
try:
|
||||
if not is_autostart_enabled():
|
||||
enable_autostart()
|
||||
self.chk_autostart.select()
|
||||
except Exception as e:
|
||||
self._log(f"No se pudo restaurar autoarranque: {e}")
|
||||
elif is_autostart_enabled():
|
||||
# Script presente aunque el flag esté off → marcar checkbox
|
||||
self.chk_autostart.select()
|
||||
|
||||
if bool(s.get("auto_connect")):
|
||||
self.chk_auto_connect.select()
|
||||
|
||||
def _guardar_settings(self):
|
||||
api = normalize_api_url(self.entry_api.get())
|
||||
self.entry_api.delete(0, "end")
|
||||
self.entry_api.insert(0, api)
|
||||
save_gui_settings(api, self.entry_worker.get(), self.entry_token.get())
|
||||
self._persist_flags()
|
||||
self._log("Configuración guardada en data/gui_settings.json")
|
||||
|
||||
def _on_autostart_toggle(self):
|
||||
if sys.platform != "win32":
|
||||
return
|
||||
enabled = bool(self.chk_autostart.get())
|
||||
try:
|
||||
if enabled:
|
||||
path = enable_autostart()
|
||||
self._log(f"Autoarranque activado: {path}")
|
||||
# Tiene sentido conectar al abrir si arranca con Windows
|
||||
if not self.chk_auto_connect.get():
|
||||
self.chk_auto_connect.select()
|
||||
else:
|
||||
disable_autostart()
|
||||
self._log("Autoarranque desactivado.")
|
||||
self._persist_flags()
|
||||
except Exception as e:
|
||||
self._log(f"Error configurando autoarranque: {e}")
|
||||
if enabled:
|
||||
self.chk_autostart.deselect()
|
||||
else:
|
||||
self.chk_autostart.select()
|
||||
|
||||
def _on_auto_connect_toggle(self):
|
||||
self._persist_flags()
|
||||
|
||||
def _maybe_auto_connect(self):
|
||||
if self._connected:
|
||||
return
|
||||
if not self.entry_worker.get().strip() or not self.entry_token.get().strip():
|
||||
self._log("Auto-conectar omitido: falta Worker ID o token.")
|
||||
return
|
||||
self._log("Conectando automáticamente…")
|
||||
self._conectar()
|
||||
|
||||
def _toggle_token(self):
|
||||
self._token_visible = not self._token_visible
|
||||
self.entry_token.configure(show="" if self._token_visible else "•")
|
||||
@@ -195,7 +280,7 @@ class BotDashboard(ctk.CTk):
|
||||
|
||||
self.entry_api.delete(0, "end")
|
||||
self.entry_api.insert(0, api)
|
||||
save_gui_settings(api, worker_id, token)
|
||||
self._persist_flags()
|
||||
|
||||
self._agent = WorkerAgent(
|
||||
api_url=api,
|
||||
@@ -251,6 +336,9 @@ class BotDashboard(ctk.CTk):
|
||||
self.entry_api.configure(state=state)
|
||||
self.entry_worker.configure(state=state)
|
||||
self.entry_token.configure(state=state)
|
||||
if sys.platform == "win32":
|
||||
self.chk_autostart.configure(state="normal")
|
||||
self.chk_auto_connect.configure(state="normal")
|
||||
|
||||
def _limpiar_logs(self):
|
||||
self.text_logs.delete("1.0", "end")
|
||||
|
||||
@@ -21,6 +21,8 @@ def load_gui_settings() -> Dict[str, Any]:
|
||||
"api_url": DEFAULT_API_URL,
|
||||
"worker_id": "",
|
||||
"token": DEFAULT_TOKEN,
|
||||
"autostart_windows": False,
|
||||
"auto_connect": False,
|
||||
}
|
||||
if not _SETTINGS_PATH.exists():
|
||||
return defaults
|
||||
@@ -33,13 +35,26 @@ def load_gui_settings() -> Dict[str, Any]:
|
||||
return defaults
|
||||
|
||||
|
||||
def save_gui_settings(api_url: str, worker_id: str, token: str) -> None:
|
||||
def save_gui_settings(
|
||||
api_url: str,
|
||||
worker_id: str,
|
||||
token: str,
|
||||
*,
|
||||
autostart_windows: bool | None = None,
|
||||
auto_connect: bool | None = None,
|
||||
) -> None:
|
||||
_SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
current = load_gui_settings()
|
||||
payload = {
|
||||
**current,
|
||||
"api_url": api_url.strip(),
|
||||
"worker_id": worker_id.strip(),
|
||||
"token": token.strip(),
|
||||
}
|
||||
if autostart_windows is not None:
|
||||
payload["autostart_windows"] = bool(autostart_windows)
|
||||
if auto_connect is not None:
|
||||
payload["auto_connect"] = bool(auto_connect)
|
||||
_SETTINGS_PATH.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Autoarranque con Windows (carpeta Startup del usuario)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
TASK_BASENAME = "IPSExtractor-BotDescarga"
|
||||
|
||||
|
||||
def bot_root() -> Path:
|
||||
"""Raíz del paquete bots/bot_descarga (donde está run_gui.py)."""
|
||||
return Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def startup_dir() -> Path:
|
||||
appdata = os.environ.get("APPDATA") or str(Path.home() / "AppData" / "Roaming")
|
||||
return Path(appdata) / "Microsoft" / "Windows" / "Start Menu" / "Programs" / "Startup"
|
||||
|
||||
|
||||
def startup_script_path() -> Path:
|
||||
return startup_dir() / f"{TASK_BASENAME}.bat"
|
||||
|
||||
|
||||
def resolve_python(root: Path | None = None) -> Path:
|
||||
root = root or bot_root()
|
||||
for candidate in (
|
||||
root / ".venv" / "Scripts" / "pythonw.exe",
|
||||
root / ".venv" / "Scripts" / "python.exe",
|
||||
root / ".venv" / "bin" / "python",
|
||||
):
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
return Path(sys.executable)
|
||||
|
||||
|
||||
def is_autostart_enabled() -> bool:
|
||||
if sys.platform != "win32":
|
||||
return False
|
||||
return startup_script_path().exists()
|
||||
|
||||
|
||||
def enable_autostart(root: Path | None = None) -> Path:
|
||||
"""Crea un .bat en Startup que abre la GUI del bot tras el login."""
|
||||
if sys.platform != "win32":
|
||||
raise RuntimeError("El autoarranque solo está disponible en Windows.")
|
||||
|
||||
root = (root or bot_root()).resolve()
|
||||
gui = root / "run_gui.py"
|
||||
if not gui.exists():
|
||||
raise FileNotFoundError(f"No se encontró run_gui.py en {root}")
|
||||
|
||||
py = resolve_python(root)
|
||||
dest = startup_script_path()
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# start "" deja que el .bat termine; la GUI queda como proceso independiente.
|
||||
content = (
|
||||
"@echo off\r\n"
|
||||
f'cd /d "{root}"\r\n'
|
||||
f'start "" "{py}" "{gui}"\r\n'
|
||||
)
|
||||
dest.write_text(content, encoding="utf-8")
|
||||
return dest
|
||||
|
||||
|
||||
def disable_autostart() -> bool:
|
||||
"""Elimina el .bat de Startup. True si había algo que borrar."""
|
||||
path = startup_script_path()
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
return True
|
||||
return False
|
||||
@@ -131,13 +131,14 @@ class WorkerAgent:
|
||||
|
||||
def _run_job(self, job: dict) -> None:
|
||||
job_id = job["id"]
|
||||
downloads_prefix = (job.get("downloads_prefix") or "").strip() or f"downloads/{job_id}/"
|
||||
self._uploaded_keys = []
|
||||
work_dir = self.staging_root / job_id
|
||||
out_dir = work_dir / "out"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
excel_path = work_dir / (Path(job.get("excel_key") or "input.xlsx").name)
|
||||
|
||||
self._log(f"[worker {self.worker_id}] job {job_id} iniciado")
|
||||
self._log(f"[worker {self.worker_id}] job {job_id} iniciado (prefix={downloads_prefix})")
|
||||
|
||||
try:
|
||||
self.api.download_excel(job_id, excel_path)
|
||||
@@ -168,7 +169,14 @@ class WorkerAgent:
|
||||
self._log(f"[worker] no se pudo enviar evento: {e}")
|
||||
|
||||
def on_pdf_saved(nit: str, radicado: str, local_path: str) -> None:
|
||||
key = self.minio.upload_pdf(job_id, nit, radicado, local_path)
|
||||
key = self.minio.upload_pdf(
|
||||
job_id,
|
||||
nit,
|
||||
radicado,
|
||||
local_path,
|
||||
downloads_prefix=downloads_prefix,
|
||||
skip_if_exists=True,
|
||||
)
|
||||
self._uploaded_keys.append(key)
|
||||
|
||||
service = DownloadService(
|
||||
@@ -181,7 +189,11 @@ class WorkerAgent:
|
||||
summary = service.run()
|
||||
|
||||
try:
|
||||
self.minio.write_manifest(job_id, self._uploaded_keys)
|
||||
self.minio.write_manifest(
|
||||
job_id,
|
||||
self._uploaded_keys,
|
||||
downloads_prefix=downloads_prefix,
|
||||
)
|
||||
except Exception as e:
|
||||
self._log(f"[worker] manifest no escrito: {e}")
|
||||
|
||||
@@ -194,7 +206,7 @@ class WorkerAgent:
|
||||
"exitosos": summary.exitosos,
|
||||
"fallidos": fallidos_count,
|
||||
"cancelled": summary.cancelled,
|
||||
"downloads_prefix": job.get("downloads_prefix"),
|
||||
"downloads_prefix": downloads_prefix,
|
||||
"uploaded_count": len(self._uploaded_keys),
|
||||
}
|
||||
|
||||
|
||||
@@ -40,10 +40,22 @@ class MinioSettings:
|
||||
)
|
||||
|
||||
|
||||
def pdf_key_from_prefix(downloads_prefix: str, nit: str, radicado: str) -> str:
|
||||
prefix = (downloads_prefix or "").strip().rstrip("/")
|
||||
if not prefix:
|
||||
raise ValueError("downloads_prefix vacío")
|
||||
return f"{prefix}/{nit}/{radicado}.pdf"
|
||||
|
||||
|
||||
def pdf_key(job_id: str, nit: str, radicado: str) -> str:
|
||||
return f"downloads/{job_id}/{nit}/{radicado}.pdf"
|
||||
|
||||
|
||||
def manifest_key_from_prefix(downloads_prefix: str) -> str:
|
||||
prefix = (downloads_prefix or "").strip().rstrip("/")
|
||||
return f"{prefix}/_manifest.json"
|
||||
|
||||
|
||||
def manifest_key(job_id: str) -> str:
|
||||
return f"downloads/{job_id}/_manifest.json"
|
||||
|
||||
@@ -89,8 +101,30 @@ class WorkerMinioUploader:
|
||||
self._client = _s3_client(self.settings)
|
||||
return self._client
|
||||
|
||||
def upload_pdf(self, job_id: str, nit: str, radicado: str, local_path: str | Path) -> str:
|
||||
key = pdf_key(job_id, nit, radicado)
|
||||
def object_exists(self, key: str) -> bool:
|
||||
try:
|
||||
self.client.head_object(Bucket=self.settings.bucket, Key=key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def upload_pdf(
|
||||
self,
|
||||
job_id: str,
|
||||
nit: str,
|
||||
radicado: str,
|
||||
local_path: str | Path,
|
||||
*,
|
||||
downloads_prefix: str | None = None,
|
||||
skip_if_exists: bool = True,
|
||||
) -> str:
|
||||
"""Sube PDF bajo downloads_prefix (padre) o downloads/{job_id}/."""
|
||||
if downloads_prefix:
|
||||
key = pdf_key_from_prefix(downloads_prefix, nit, radicado)
|
||||
else:
|
||||
key = pdf_key(job_id, nit, radicado)
|
||||
if skip_if_exists and self.object_exists(key):
|
||||
return key
|
||||
path = Path(local_path)
|
||||
# put_object (no multipart) es más fiable en buckets anónimos.
|
||||
self.client.put_object(
|
||||
@@ -101,9 +135,23 @@ class WorkerMinioUploader:
|
||||
)
|
||||
return key
|
||||
|
||||
def write_manifest(self, job_id: str, keys: List[str]) -> str:
|
||||
key = manifest_key(job_id)
|
||||
payload = {"job_id": job_id, "files": keys, "count": len(keys)}
|
||||
def write_manifest(
|
||||
self,
|
||||
job_id: str,
|
||||
keys: List[str],
|
||||
*,
|
||||
downloads_prefix: str | None = None,
|
||||
) -> str:
|
||||
if downloads_prefix:
|
||||
key = manifest_key_from_prefix(downloads_prefix)
|
||||
else:
|
||||
key = manifest_key(job_id)
|
||||
payload = {
|
||||
"job_id": job_id,
|
||||
"downloads_prefix": downloads_prefix or f"downloads/{job_id}/",
|
||||
"files": keys,
|
||||
"count": len(keys),
|
||||
}
|
||||
self.client.put_object(
|
||||
Bucket=self.settings.bucket,
|
||||
Key=key,
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
from pathlib import Path
|
||||
|
||||
from bot_descarga.ui.windows_autostart import TASK_BASENAME, bot_root, resolve_python
|
||||
|
||||
|
||||
def test_bot_root_has_run_gui():
|
||||
root = bot_root()
|
||||
assert (root / "run_gui.py").exists()
|
||||
|
||||
|
||||
def test_resolve_python_returns_path():
|
||||
py = resolve_python()
|
||||
assert isinstance(py, Path)
|
||||
assert str(py)
|
||||
|
||||
|
||||
def test_task_basename():
|
||||
assert "BotDescarga" in TASK_BASENAME
|
||||
@@ -0,0 +1,7 @@
|
||||
BOT_API_URL=http://localhost:8189/bot-digitador
|
||||
BOT_API_TOKEN=dev-token-change-me
|
||||
BOT_WORKER_ID=
|
||||
DIGITADOR_APP_PATH=C:\Users\Pysctas185\Documents\Integracion Total.lnk
|
||||
MINIO_ENDPOINT=https://s3.builker.xyz
|
||||
MINIO_BUCKET=pys-cuentas-medicas
|
||||
MINIO_ANONYMOUS=1
|
||||
@@ -0,0 +1,92 @@
|
||||
# Bot Digitador (agent Windows)
|
||||
|
||||
Agent que **digita** radicados en **Integración Total** (pywinauto + OpenCV).
|
||||
Vive en este monorepo bajo `bots/` (no es un microservicio de `services/`).
|
||||
|
||||
- **No descarga PDFs.** Solo digitación UI + reportes CSV/log.
|
||||
- **No expone puertos.** Solo HTTP saliente al gateway IPSExtractor y PutObject a MinIO.
|
||||
- La API de control se espera en `…/bot-digitador` (mismo patrón que `bot_descarga`).
|
||||
|
||||
## Requisitos en la PC
|
||||
|
||||
- Windows con **Integración Total** instalado e iniciado sesión
|
||||
- Python 3.11+
|
||||
- Plantillas OpenCV en `template/` (ver [template/README.md](template/README.md))
|
||||
- Acceso de red al gateway (`BOT_API_URL`) y a MinIO
|
||||
- Token `BOT_API_TOKEN`
|
||||
|
||||
## Instalación
|
||||
|
||||
```bash
|
||||
cd bots/bot_digitador
|
||||
python -m venv .venv
|
||||
.venv\Scripts\activate
|
||||
pip install -r requirements.txt
|
||||
copy .env.example .env
|
||||
# Editar .env (especialmente DIGITADOR_APP_PATH)
|
||||
```
|
||||
|
||||
Copia los PNG de plantillas a `template/`.
|
||||
|
||||
## Arranque con GUI
|
||||
|
||||
```bash
|
||||
python run_gui.py
|
||||
```
|
||||
|
||||
Configura URL API (ej. `http://localhost:8189/bot-digitador`), Worker ID, token y
|
||||
ruta de Integración Total (`DIGITADOR_APP_PATH`). Pulsa **Conectar agent**.
|
||||
Queda idle hasta que el desktop asigne un job CSV.
|
||||
|
||||
### Autoarranque en Windows
|
||||
|
||||
1. Marca **Iniciar con Windows**.
|
||||
2. Deja **Conectar al abrir** activo.
|
||||
|
||||
## Arranque solo agent (CLI)
|
||||
|
||||
```bash
|
||||
python -m bot_digitador.worker \
|
||||
--api-url http://localhost:8189/bot-digitador \
|
||||
--worker-id PC-DIGITADOR-01 \
|
||||
--token %BOT_API_TOKEN%
|
||||
```
|
||||
|
||||
## Modo local (sin API)
|
||||
|
||||
Requiere Integración Total abierto y visible:
|
||||
|
||||
```bash
|
||||
python main.py --csv datos.csv
|
||||
python main.py --csv datos.csv --delimiter "|" --date-format dd/mm/yyyy --app-path "C:\ruta\Integracion Total.lnk"
|
||||
```
|
||||
|
||||
Reportes en la carpeta de salida: `RESULTADOS.csv`, `OMITIDOS.csv`, `ejecucion_bot.log`.
|
||||
|
||||
## Arquitectura
|
||||
|
||||
```
|
||||
Desktop IPSExtractor → Gateway :8189 → API bot-digitador
|
||||
↑
|
||||
PC bot (esta carpeta) ────────┘ HTTP outbound
|
||||
└─ PutObject reportes → MinIO …/digitados/{job_id}/
|
||||
└─ pywinauto → Integración Total
|
||||
```
|
||||
|
||||
## Variables
|
||||
|
||||
Ver [`.env.example`](.env.example):
|
||||
|
||||
| Variable | Uso |
|
||||
|----------|-----|
|
||||
| `BOT_API_URL` | Base del API vía gateway (`…/bot-digitador`) |
|
||||
| `BOT_API_TOKEN` | Bearer compartido con el servicio |
|
||||
| `DIGITADOR_APP_PATH` | Acceso directo / ejecutable de Integración Total |
|
||||
| `MINIO_*` | Bucket de reportes; por defecto anónimo |
|
||||
|
||||
## CSV de entrada
|
||||
|
||||
Columnas requeridas: `radicado`, `documento`, `ingreso`, `total_cargos`,
|
||||
`autorizacion`, `tipo_documento`, `tipo_servicio`, `copago`, `cuota_moderadora`.
|
||||
|
||||
Acepta delimitador `,` o `|` (auto-detección).
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Bot Digitador — automatización de digitación en Integración Total."""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
@@ -0,0 +1,3 @@
|
||||
from bot_digitador.core.digitador_service import DigitadorRunSummary, DigitadorService
|
||||
|
||||
__all__ = ["DigitadorService", "DigitadorRunSummary"]
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Coordenadas, estrategia de documentos y constantes de Integración Total."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
DOC_COORDS = {
|
||||
"CC": (796, 343),
|
||||
"TI": (796, 549),
|
||||
"CE": (796, 329),
|
||||
"AS": (796, 296),
|
||||
"CD": (796, 314),
|
||||
"CN": (796, 371),
|
||||
"ME": (796, 378),
|
||||
"NI": (796, 412),
|
||||
"NU": (796, 453),
|
||||
"PS": (796, 450),
|
||||
"PE": (796, 467),
|
||||
"PT": (796, 483),
|
||||
"PJ": (796, 503),
|
||||
"RC": (796, 520),
|
||||
"SC": (796, 536),
|
||||
}
|
||||
|
||||
DOC_STRATEGY = [
|
||||
"CC", "TI", "CE", "RC", "PS", "PT", "PE",
|
||||
"AS", "ME", "CN", "SC", "CD", "NI", "NU", "PJ",
|
||||
]
|
||||
|
||||
SERVICES_COORDS = {
|
||||
"ambulancias y elem.": (1033, 367),
|
||||
"ambulatorio": (1033, 387),
|
||||
"c. externa": (1033, 405),
|
||||
"capacitacion": (1033, 417),
|
||||
"cirugia ambulatoria": (1033, 432),
|
||||
"fomento y prevencion": (1033, 449),
|
||||
"honorarios": (1033, 467),
|
||||
}
|
||||
|
||||
COORD_CLOSE_ERROR = (1223, 704)
|
||||
|
||||
DEFAULT_APP_PATH = os.getenv(
|
||||
"DIGITADOR_APP_PATH",
|
||||
r"C:\Users\Pysctas185\Documents\Integracion Total.lnk",
|
||||
)
|
||||
PROCESS_NAME = "SaludTotal.Health.WinClient.TotalIntegration.exe"
|
||||
REINICIO_CADA = 6
|
||||
TIMEOUT_REGISTRO = 120
|
||||
|
||||
TEMPLATE_ERROR = "error.png"
|
||||
TEMPLATE_SUCCESS = "success.png"
|
||||
TEMPLATE_BIEN = "bien.png"
|
||||
TEMPLATE_DIFERENCIAS = "flecha_diferencias_abierto.png"
|
||||
TEMPLATE_DIFERENCIAS_CERRADO = "flecha_diferencias_cerrado.png"
|
||||
TEMPLATE_FECHA_VACIA_SLASH = "fecha_vacia_slash.png"
|
||||
TEMPLATE_FECHA_VACIA_BLANCO = "fecha_vacia_blanco.png"
|
||||
TEMPLATE_POPUP_NOMBRES = "popup_nombres.png"
|
||||
TEMPLATE_RADICADO_EXISTENTE = "radicado_existente.png"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
from bot_digitador.models.events import DigitadorEvent, create_event
|
||||
|
||||
__all__ = ["DigitadorEvent", "create_event"]
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Eventos simples para callbacks del DigitadorService."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class DigitadorEvent:
|
||||
event_type: str
|
||||
message: str
|
||||
radicado: Optional[str] = None
|
||||
index: Optional[int] = None
|
||||
total: Optional[int] = None
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
timestamp: str = ""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.timestamp:
|
||||
self.timestamp = datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
def display_message(self) -> str:
|
||||
prefix = ""
|
||||
if self.index is not None and self.total is not None:
|
||||
prefix = f"[{self.index}/{self.total}] "
|
||||
return f"{prefix}{self.message}"
|
||||
|
||||
|
||||
def create_event(
|
||||
event_type: str,
|
||||
message: str,
|
||||
*,
|
||||
radicado: Optional[str] = None,
|
||||
index: Optional[int] = None,
|
||||
total: Optional[int] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> DigitadorEvent:
|
||||
return DigitadorEvent(
|
||||
event_type=event_type,
|
||||
message=message,
|
||||
radicado=radicado,
|
||||
index=index,
|
||||
total=total,
|
||||
metadata=metadata or {},
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
"""UI del agent digitador."""
|
||||
@@ -0,0 +1,368 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
|
||||
import customtkinter as ctk
|
||||
|
||||
from bot_digitador.core.coords import DEFAULT_APP_PATH
|
||||
from bot_digitador.ui.settings_store import (
|
||||
DEFAULT_API_URL,
|
||||
load_gui_settings,
|
||||
normalize_api_url,
|
||||
save_gui_settings,
|
||||
)
|
||||
from bot_digitador.ui.windows_autostart import (
|
||||
disable_autostart,
|
||||
enable_autostart,
|
||||
is_autostart_enabled,
|
||||
)
|
||||
from bot_digitador.worker.agent import WorkerAgent
|
||||
|
||||
ctk.set_appearance_mode("dark")
|
||||
ctk.set_default_color_theme("blue")
|
||||
|
||||
|
||||
class BotDashboard(ctk.CTk):
|
||||
"""GUI del agent digitador: configura API remota y conecta como worker idle."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.title("Bot Digitador — Integración Total")
|
||||
self.geometry("960x780")
|
||||
self.resizable(False, False)
|
||||
|
||||
self._agent: WorkerAgent | None = None
|
||||
self._agent_thread: threading.Thread | None = None
|
||||
self._connected = False
|
||||
|
||||
self._construir_ui()
|
||||
self._cargar_settings()
|
||||
|
||||
self.protocol("WM_DELETE_WINDOW", self._on_close)
|
||||
|
||||
if bool(load_gui_settings().get("auto_connect")):
|
||||
self.after(800, self._maybe_auto_connect)
|
||||
|
||||
def _construir_ui(self):
|
||||
root = ctk.CTkFrame(self, corner_radius=18)
|
||||
root.pack(fill="both", expand=True, padx=18, pady=18)
|
||||
|
||||
ctk.CTkLabel(
|
||||
root,
|
||||
text="Bot Digitador",
|
||||
font=ctk.CTkFont(size=26, weight="bold"),
|
||||
).pack(pady=(18, 4))
|
||||
|
||||
ctk.CTkLabel(
|
||||
root,
|
||||
text="Agent remoto — digita en Integración Total (pywinauto + OpenCV)",
|
||||
text_color="gray",
|
||||
font=ctk.CTkFont(size=13),
|
||||
).pack(pady=(0, 18))
|
||||
|
||||
config = ctk.CTkFrame(root, corner_radius=14)
|
||||
config.pack(fill="x", padx=24, pady=(0, 14))
|
||||
|
||||
ctk.CTkLabel(
|
||||
config,
|
||||
text="API remota (IPSExtractor)",
|
||||
font=ctk.CTkFont(size=16, weight="bold"),
|
||||
).pack(anchor="w", padx=18, pady=(14, 8))
|
||||
|
||||
row_api = ctk.CTkFrame(config, fg_color="transparent")
|
||||
row_api.pack(fill="x", padx=18, pady=(0, 8))
|
||||
ctk.CTkLabel(row_api, text="URL API", width=100, anchor="w").pack(side="left")
|
||||
self.entry_api = ctk.CTkEntry(
|
||||
row_api,
|
||||
placeholder_text=DEFAULT_API_URL,
|
||||
)
|
||||
self.entry_api.pack(side="left", fill="x", expand=True)
|
||||
|
||||
row_id = ctk.CTkFrame(config, fg_color="transparent")
|
||||
row_id.pack(fill="x", padx=18, pady=(0, 8))
|
||||
ctk.CTkLabel(row_id, text="Worker ID", width=100, anchor="w").pack(side="left")
|
||||
self.entry_worker = ctk.CTkEntry(
|
||||
row_id,
|
||||
placeholder_text=socket.gethostname() or "PC-01",
|
||||
)
|
||||
self.entry_worker.pack(side="left", fill="x", expand=True)
|
||||
|
||||
row_token = ctk.CTkFrame(config, fg_color="transparent")
|
||||
row_token.pack(fill="x", padx=18, pady=(0, 8))
|
||||
ctk.CTkLabel(row_token, text="Token", width=100, anchor="w").pack(side="left")
|
||||
self.entry_token = ctk.CTkEntry(
|
||||
row_token,
|
||||
placeholder_text="BOT_API_TOKEN",
|
||||
show="•",
|
||||
)
|
||||
self.entry_token.pack(side="left", fill="x", expand=True, padx=(0, 8))
|
||||
self._token_visible = False
|
||||
ctk.CTkButton(
|
||||
row_token,
|
||||
text="👁",
|
||||
width=40,
|
||||
command=self._toggle_token,
|
||||
).pack(side="left")
|
||||
|
||||
row_app = ctk.CTkFrame(config, fg_color="transparent")
|
||||
row_app.pack(fill="x", padx=18, pady=(0, 8))
|
||||
ctk.CTkLabel(row_app, text="APP_PATH", width=100, anchor="w").pack(side="left")
|
||||
self.entry_app = ctk.CTkEntry(
|
||||
row_app,
|
||||
placeholder_text=DEFAULT_APP_PATH,
|
||||
)
|
||||
self.entry_app.pack(side="left", fill="x", expand=True)
|
||||
|
||||
row_auto = ctk.CTkFrame(config, fg_color="transparent")
|
||||
row_auto.pack(fill="x", padx=18, pady=(0, 14))
|
||||
self.chk_autostart = ctk.CTkCheckBox(
|
||||
row_auto,
|
||||
text="Iniciar con Windows",
|
||||
command=self._on_autostart_toggle,
|
||||
)
|
||||
self.chk_autostart.pack(side="left", padx=(0, 18))
|
||||
self.chk_auto_connect = ctk.CTkCheckBox(
|
||||
row_auto,
|
||||
text="Conectar al abrir",
|
||||
command=self._on_auto_connect_toggle,
|
||||
)
|
||||
self.chk_auto_connect.pack(side="left")
|
||||
if sys.platform != "win32":
|
||||
self.chk_autostart.configure(state="disabled")
|
||||
|
||||
status = ctk.CTkFrame(root, corner_radius=14)
|
||||
status.pack(fill="x", padx=24, pady=(0, 14))
|
||||
|
||||
self.lbl_estado = ctk.CTkLabel(
|
||||
status,
|
||||
text="Estado: desconectado",
|
||||
font=ctk.CTkFont(size=15, weight="bold"),
|
||||
)
|
||||
self.lbl_estado.pack(anchor="w", padx=18, pady=(14, 2))
|
||||
|
||||
self.lbl_hint = ctk.CTkLabel(
|
||||
status,
|
||||
text="Configura la API y pulsa Conectar. Este PC no expone puertos.",
|
||||
text_color="gray",
|
||||
font=ctk.CTkFont(size=13),
|
||||
)
|
||||
self.lbl_hint.pack(anchor="w", padx=18, pady=(0, 14))
|
||||
|
||||
botones = ctk.CTkFrame(root, fg_color="transparent")
|
||||
botones.pack(fill="x", padx=24, pady=(0, 14))
|
||||
|
||||
self.btn_connect = ctk.CTkButton(
|
||||
botones,
|
||||
text="▶ Conectar agent",
|
||||
height=42,
|
||||
width=180,
|
||||
font=ctk.CTkFont(size=14, weight="bold"),
|
||||
command=self._toggle_connect,
|
||||
)
|
||||
self.btn_connect.pack(side="left", padx=(0, 10))
|
||||
|
||||
ctk.CTkButton(
|
||||
botones,
|
||||
text="Guardar config",
|
||||
height=42,
|
||||
width=140,
|
||||
command=self._guardar_settings,
|
||||
).pack(side="left", padx=(0, 10))
|
||||
|
||||
ctk.CTkButton(
|
||||
botones,
|
||||
text="Limpiar logs",
|
||||
height=42,
|
||||
width=130,
|
||||
command=self._limpiar_logs,
|
||||
).pack(side="left")
|
||||
|
||||
logs_frame = ctk.CTkFrame(root, corner_radius=14)
|
||||
logs_frame.pack(fill="both", expand=True, padx=24, pady=(0, 18))
|
||||
|
||||
ctk.CTkLabel(
|
||||
logs_frame,
|
||||
text="Detalle del agent",
|
||||
font=ctk.CTkFont(size=15, weight="bold"),
|
||||
).pack(anchor="w", padx=18, pady=(12, 6))
|
||||
|
||||
self.text_logs = ctk.CTkTextbox(
|
||||
logs_frame, font=ctk.CTkFont(family="Courier", size=12)
|
||||
)
|
||||
self.text_logs.pack(fill="both", expand=True, padx=18, pady=(0, 16))
|
||||
|
||||
def _persist_flags(self):
|
||||
save_gui_settings(
|
||||
normalize_api_url(self.entry_api.get()),
|
||||
self.entry_worker.get(),
|
||||
self.entry_token.get(),
|
||||
app_path=self.entry_app.get(),
|
||||
autostart_windows=bool(self.chk_autostart.get()),
|
||||
auto_connect=bool(self.chk_auto_connect.get()),
|
||||
)
|
||||
|
||||
def _cargar_settings(self):
|
||||
s = load_gui_settings()
|
||||
self.entry_api.insert(0, s.get("api_url") or DEFAULT_API_URL)
|
||||
self.entry_worker.insert(0, s.get("worker_id") or socket.gethostname())
|
||||
self.entry_token.insert(0, s.get("token") or "")
|
||||
self.entry_app.insert(0, s.get("app_path") or DEFAULT_APP_PATH)
|
||||
|
||||
want_autostart = bool(s.get("autostart_windows"))
|
||||
if want_autostart and sys.platform == "win32":
|
||||
try:
|
||||
if not is_autostart_enabled():
|
||||
enable_autostart()
|
||||
self.chk_autostart.select()
|
||||
except Exception as e:
|
||||
self._log(f"No se pudo restaurar autoarranque: {e}")
|
||||
elif is_autostart_enabled():
|
||||
self.chk_autostart.select()
|
||||
|
||||
if bool(s.get("auto_connect")):
|
||||
self.chk_auto_connect.select()
|
||||
|
||||
def _guardar_settings(self):
|
||||
api = normalize_api_url(self.entry_api.get())
|
||||
self.entry_api.delete(0, "end")
|
||||
self.entry_api.insert(0, api)
|
||||
self._persist_flags()
|
||||
self._log("Configuración guardada en data/gui_settings.json")
|
||||
|
||||
def _on_autostart_toggle(self):
|
||||
if sys.platform != "win32":
|
||||
return
|
||||
enabled = bool(self.chk_autostart.get())
|
||||
try:
|
||||
if enabled:
|
||||
path = enable_autostart()
|
||||
self._log(f"Autoarranque activado: {path}")
|
||||
if not self.chk_auto_connect.get():
|
||||
self.chk_auto_connect.select()
|
||||
else:
|
||||
disable_autostart()
|
||||
self._log("Autoarranque desactivado.")
|
||||
self._persist_flags()
|
||||
except Exception as e:
|
||||
self._log(f"Error configurando autoarranque: {e}")
|
||||
if enabled:
|
||||
self.chk_autostart.deselect()
|
||||
else:
|
||||
self.chk_autostart.select()
|
||||
|
||||
def _on_auto_connect_toggle(self):
|
||||
self._persist_flags()
|
||||
|
||||
def _maybe_auto_connect(self):
|
||||
if self._connected:
|
||||
return
|
||||
if not self.entry_worker.get().strip() or not self.entry_token.get().strip():
|
||||
self._log("Auto-conectar omitido: falta Worker ID o token.")
|
||||
return
|
||||
self._log("Conectando automáticamente…")
|
||||
self._conectar()
|
||||
|
||||
def _toggle_token(self):
|
||||
self._token_visible = not self._token_visible
|
||||
self.entry_token.configure(show="" if self._token_visible else "•")
|
||||
|
||||
def _toggle_connect(self):
|
||||
if self._connected:
|
||||
self._desconectar()
|
||||
else:
|
||||
self._conectar()
|
||||
|
||||
def _conectar(self):
|
||||
api = normalize_api_url(self.entry_api.get())
|
||||
worker_id = self.entry_worker.get().strip()
|
||||
token = self.entry_token.get().strip()
|
||||
app_path = self.entry_app.get().strip() or DEFAULT_APP_PATH
|
||||
|
||||
if not worker_id:
|
||||
self._log("Debes indicar un Worker ID.")
|
||||
return
|
||||
if not token:
|
||||
self._log("Debes indicar el token (BOT_API_TOKEN).")
|
||||
return
|
||||
|
||||
self.entry_api.delete(0, "end")
|
||||
self.entry_api.insert(0, api)
|
||||
self._persist_flags()
|
||||
|
||||
self._agent = WorkerAgent(
|
||||
api_url=api,
|
||||
worker_id=worker_id,
|
||||
token=token,
|
||||
app_path=app_path,
|
||||
log_func=self._log,
|
||||
)
|
||||
self._connected = True
|
||||
self.btn_connect.configure(text="⏹ Desconectar", fg_color="#c0392b", hover_color="#a93226")
|
||||
self.lbl_estado.configure(text=f"Estado: conectado como {worker_id}")
|
||||
self.lbl_hint.configure(text=f"API: {api} — esperando jobs del desktop…")
|
||||
self._set_config_enabled(False)
|
||||
|
||||
self._agent_thread = threading.Thread(
|
||||
target=self._run_agent,
|
||||
daemon=True,
|
||||
)
|
||||
self._agent_thread.start()
|
||||
self._log(f"Conectando a {api}…")
|
||||
|
||||
def _run_agent(self):
|
||||
try:
|
||||
assert self._agent is not None
|
||||
self._agent.run_forever()
|
||||
except Exception as e:
|
||||
self._log(f"Error del agent: {e}")
|
||||
finally:
|
||||
self.after(0, self._on_agent_stopped)
|
||||
|
||||
def _desconectar(self):
|
||||
if self._agent:
|
||||
self._agent.stop()
|
||||
self._log("Solicitando desconexión…")
|
||||
self.btn_connect.configure(state="disabled", text="Desconectando…")
|
||||
|
||||
def _on_agent_stopped(self):
|
||||
self._connected = False
|
||||
self._agent = None
|
||||
self.btn_connect.configure(
|
||||
state="normal",
|
||||
text="▶ Conectar agent",
|
||||
fg_color=["#3B8ED0", "#1F6AA5"],
|
||||
hover_color=["#36719F", "#144870"],
|
||||
)
|
||||
self.lbl_estado.configure(text="Estado: desconectado")
|
||||
self.lbl_hint.configure(
|
||||
text="Configura la API y pulsa Conectar. Este PC no expone puertos."
|
||||
)
|
||||
self._set_config_enabled(True)
|
||||
|
||||
def _set_config_enabled(self, enabled: bool):
|
||||
state = "normal" if enabled else "disabled"
|
||||
self.entry_api.configure(state=state)
|
||||
self.entry_worker.configure(state=state)
|
||||
self.entry_token.configure(state=state)
|
||||
self.entry_app.configure(state=state)
|
||||
if sys.platform == "win32":
|
||||
self.chk_autostart.configure(state="normal")
|
||||
self.chk_auto_connect.configure(state="normal")
|
||||
|
||||
def _limpiar_logs(self):
|
||||
self.text_logs.delete("1.0", "end")
|
||||
|
||||
def _log(self, mensaje: str):
|
||||
def _escribir():
|
||||
self.text_logs.insert("end", mensaje + "\n")
|
||||
self.text_logs.see("end")
|
||||
|
||||
self.after(0, _escribir)
|
||||
|
||||
def _on_close(self):
|
||||
if self._agent:
|
||||
self._agent.stop()
|
||||
self.destroy()
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Persistencia simple de configuración de la GUI del worker digitador."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
from bot_digitador.core.coords import DEFAULT_APP_PATH
|
||||
|
||||
DEFAULT_API_URL = os.getenv(
|
||||
"BOT_API_URL",
|
||||
"http://localhost:8189/bot-digitador",
|
||||
)
|
||||
DEFAULT_TOKEN = os.getenv("BOT_API_TOKEN", "dev-token-change-me")
|
||||
|
||||
_SETTINGS_PATH = Path("data") / "gui_settings.json"
|
||||
|
||||
|
||||
def load_gui_settings() -> Dict[str, Any]:
|
||||
defaults = {
|
||||
"api_url": DEFAULT_API_URL,
|
||||
"worker_id": "",
|
||||
"token": DEFAULT_TOKEN,
|
||||
"app_path": os.getenv("DIGITADOR_APP_PATH", DEFAULT_APP_PATH),
|
||||
"autostart_windows": False,
|
||||
"auto_connect": False,
|
||||
}
|
||||
if not _SETTINGS_PATH.exists():
|
||||
return defaults
|
||||
try:
|
||||
data = json.loads(_SETTINGS_PATH.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
return defaults
|
||||
return {**defaults, **data}
|
||||
except Exception:
|
||||
return defaults
|
||||
|
||||
|
||||
def save_gui_settings(
|
||||
api_url: str,
|
||||
worker_id: str,
|
||||
token: str,
|
||||
*,
|
||||
app_path: str | None = None,
|
||||
autostart_windows: bool | None = None,
|
||||
auto_connect: bool | None = None,
|
||||
) -> None:
|
||||
_SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
current = load_gui_settings()
|
||||
payload = {
|
||||
**current,
|
||||
"api_url": api_url.strip(),
|
||||
"worker_id": worker_id.strip(),
|
||||
"token": token.strip(),
|
||||
}
|
||||
if app_path is not None:
|
||||
payload["app_path"] = app_path.strip()
|
||||
if autostart_windows is not None:
|
||||
payload["autostart_windows"] = bool(autostart_windows)
|
||||
if auto_connect is not None:
|
||||
payload["auto_connect"] = bool(auto_connect)
|
||||
_SETTINGS_PATH.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def normalize_api_url(url: str) -> str:
|
||||
"""Normaliza la URL base del API bot-digitador."""
|
||||
u = (url or "").strip().rstrip("/")
|
||||
if not u:
|
||||
return DEFAULT_API_URL.rstrip("/")
|
||||
if u.endswith("/bot-digitador"):
|
||||
return u
|
||||
if "builker.xyz" in u or u.endswith(":8189"):
|
||||
return f"{u}/bot-digitador"
|
||||
return u
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Autoarranque con Windows (carpeta Startup del usuario)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
TASK_BASENAME = "IPSExtractor-BotDigitador"
|
||||
|
||||
|
||||
def bot_root() -> Path:
|
||||
"""Raíz del paquete bots/bot_digitador (donde está run_gui.py)."""
|
||||
return Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def startup_dir() -> Path:
|
||||
appdata = os.environ.get("APPDATA") or str(Path.home() / "AppData" / "Roaming")
|
||||
return Path(appdata) / "Microsoft" / "Windows" / "Start Menu" / "Programs" / "Startup"
|
||||
|
||||
|
||||
def startup_script_path() -> Path:
|
||||
return startup_dir() / f"{TASK_BASENAME}.bat"
|
||||
|
||||
|
||||
def resolve_python(root: Path | None = None) -> Path:
|
||||
root = root or bot_root()
|
||||
for candidate in (
|
||||
root / ".venv" / "Scripts" / "pythonw.exe",
|
||||
root / ".venv" / "Scripts" / "python.exe",
|
||||
root / ".venv" / "bin" / "python",
|
||||
):
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
return Path(sys.executable)
|
||||
|
||||
|
||||
def is_autostart_enabled() -> bool:
|
||||
if sys.platform != "win32":
|
||||
return False
|
||||
return startup_script_path().exists()
|
||||
|
||||
|
||||
def enable_autostart(root: Path | None = None) -> Path:
|
||||
"""Crea un .bat en Startup que abre la GUI del bot tras el login."""
|
||||
if sys.platform != "win32":
|
||||
raise RuntimeError("El autoarranque solo está disponible en Windows.")
|
||||
|
||||
root = (root or bot_root()).resolve()
|
||||
gui = root / "run_gui.py"
|
||||
if not gui.exists():
|
||||
raise FileNotFoundError(f"No se encontró run_gui.py en {root}")
|
||||
|
||||
py = resolve_python(root)
|
||||
dest = startup_script_path()
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
content = (
|
||||
"@echo off\r\n"
|
||||
f'cd /d "{root}"\r\n'
|
||||
f'start "" "{py}" "{gui}"\r\n'
|
||||
)
|
||||
dest.write_text(content, encoding="utf-8")
|
||||
return dest
|
||||
|
||||
|
||||
def disable_autostart() -> bool:
|
||||
"""Elimina el .bat de Startup. True si había algo que borrar."""
|
||||
path = startup_script_path()
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
return True
|
||||
return False
|
||||
@@ -0,0 +1 @@
|
||||
"""Worker remoto del bot digitador."""
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Permite: python -m bot_digitador.worker"""
|
||||
|
||||
from bot_digitador.worker.run import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,315 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import socket
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from bot_digitador.core.coords import DEFAULT_APP_PATH
|
||||
from bot_digitador.core.digitador_service import DigitadorService
|
||||
from bot_digitador.worker.minio_upload import MinioSettings, WorkerMinioUploader
|
||||
|
||||
JOB_COMPLETED = "completed"
|
||||
JOB_FAILED = "failed"
|
||||
JOB_CANCELLED = "cancelled"
|
||||
|
||||
REPORT_FILES = ("RESULTADOS.csv", "OMITIDOS.csv", "ejecucion_bot.log")
|
||||
|
||||
|
||||
class RemoteApiClient:
|
||||
"""Cliente HTTP hacia la API bot-digitador (vía gateway). Sin puertos locales."""
|
||||
|
||||
def __init__(self, base_url: str, token: str):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.token = token
|
||||
self._client = httpx.Client(
|
||||
timeout=httpx.Timeout(120.0, connect=10.0),
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
self._client.close()
|
||||
|
||||
def _url(self, path: str) -> str:
|
||||
return f"{self.base_url}{path}"
|
||||
|
||||
def register(self, worker_id: str, hostname: str = "", max_concurrent: int = 1) -> dict:
|
||||
r = self._client.post(
|
||||
self._url("/workers/register"),
|
||||
json={
|
||||
"worker_id": worker_id,
|
||||
"hostname": hostname or socket.gethostname(),
|
||||
"max_concurrent": max_concurrent,
|
||||
},
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def heartbeat(self, worker_id: str) -> dict:
|
||||
r = self._client.post(
|
||||
self._url("/workers/heartbeat"),
|
||||
json={"worker_id": worker_id},
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def claim_pending(self, worker_id: str) -> Optional[dict]:
|
||||
r = self._client.get(
|
||||
self._url("/workers/me/pending"),
|
||||
params={"worker_id": worker_id},
|
||||
)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
return data.get("job")
|
||||
|
||||
def download_file(self, job_id: str, dest: Path) -> Path:
|
||||
r = self._client.get(self._url(f"/jobs/{job_id}/file"))
|
||||
r.raise_for_status()
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(r.content)
|
||||
return dest
|
||||
|
||||
def post_event(self, job_id: str, event: Dict[str, Any]) -> None:
|
||||
r = self._client.post(self._url(f"/jobs/{job_id}/events"), json=event)
|
||||
r.raise_for_status()
|
||||
|
||||
def cancel_flag(self, job_id: str) -> bool:
|
||||
r = self._client.get(self._url(f"/jobs/{job_id}/cancel-flag"))
|
||||
r.raise_for_status()
|
||||
return bool(r.json().get("cancel_requested"))
|
||||
|
||||
def patch_job(self, job_id: str, **body) -> dict:
|
||||
r = self._client.patch(self._url(f"/jobs/{job_id}"), json=body)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
class WorkerAgent:
|
||||
"""Agent idle: heartbeat + claim jobs de digitación asignados."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_url: str,
|
||||
worker_id: str,
|
||||
token: str,
|
||||
poll_seconds: float = 8.0,
|
||||
staging_root: Optional[Path] = None,
|
||||
app_path: Optional[str] = None,
|
||||
date_format: str = "dd/mm/yyyy",
|
||||
delimiter: str = "|",
|
||||
log_func=None,
|
||||
):
|
||||
self.api = RemoteApiClient(api_url, token)
|
||||
self.worker_id = worker_id
|
||||
self.poll_seconds = poll_seconds
|
||||
self.staging_root = staging_root or Path(tempfile.gettempdir()) / "bot_digitador"
|
||||
self.app_path = app_path or os.getenv("DIGITADOR_APP_PATH") or DEFAULT_APP_PATH
|
||||
self.date_format = date_format
|
||||
self.delimiter = delimiter
|
||||
self.minio = WorkerMinioUploader(MinioSettings.from_env())
|
||||
self._running = True
|
||||
self._uploaded_keys: List[str] = []
|
||||
self._log = log_func or print
|
||||
|
||||
def stop(self) -> None:
|
||||
self._running = False
|
||||
|
||||
def run_forever(self) -> None:
|
||||
self.api.register(self.worker_id)
|
||||
self._log(f"[worker {self.worker_id}] registrado — idle (esperando jobs de digitación)")
|
||||
try:
|
||||
while self._running:
|
||||
try:
|
||||
self.api.heartbeat(self.worker_id)
|
||||
job = self.api.claim_pending(self.worker_id)
|
||||
if job:
|
||||
self._run_job(job)
|
||||
else:
|
||||
time.sleep(self.poll_seconds)
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except Exception as e:
|
||||
self._log(f"[worker {self.worker_id}] error en loop: {e}")
|
||||
time.sleep(self.poll_seconds)
|
||||
finally:
|
||||
self.api.close()
|
||||
self._log(f"[worker {self.worker_id}] detenido")
|
||||
|
||||
def _run_job(self, job: dict) -> None:
|
||||
job_id = job["id"]
|
||||
reports_prefix = (job.get("reports_prefix") or "").strip() or f"digitados/{job_id}/"
|
||||
self._uploaded_keys = []
|
||||
work_dir = self.staging_root / job_id
|
||||
out_dir = work_dir / "out"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
csv_name = Path(job.get("csv_key") or job.get("file_key") or "input.csv").name
|
||||
if not csv_name.lower().endswith(".csv"):
|
||||
csv_name = "input.csv"
|
||||
csv_path = work_dir / csv_name
|
||||
|
||||
self._log(f"[worker {self.worker_id}] job {job_id} iniciado (prefix={reports_prefix})")
|
||||
|
||||
try:
|
||||
self.api.download_file(job_id, csv_path)
|
||||
|
||||
def should_cancel() -> bool:
|
||||
try:
|
||||
return self.api.cancel_flag(job_id)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def on_event(event) -> None:
|
||||
self._log(event.display_message())
|
||||
payload = {
|
||||
"event_type": event.event_type,
|
||||
"message": event.message,
|
||||
"radicado": event.radicado,
|
||||
"index": event.index,
|
||||
"total": event.total,
|
||||
"metadata": event.metadata,
|
||||
"timestamp": event.timestamp,
|
||||
}
|
||||
try:
|
||||
self.api.post_event(job_id, payload)
|
||||
except Exception as e:
|
||||
self._log(f"[worker] no se pudo enviar evento: {e}")
|
||||
|
||||
date_format = job.get("date_format") or self.date_format
|
||||
delimiter = job.get("delimiter") or self.delimiter
|
||||
|
||||
service = DigitadorService(
|
||||
csv_path=str(csv_path),
|
||||
output_dir=str(out_dir),
|
||||
date_format=date_format,
|
||||
delimiter=delimiter,
|
||||
app_path=self.app_path,
|
||||
event_callback=on_event,
|
||||
should_cancel=should_cancel,
|
||||
)
|
||||
summary = service.run()
|
||||
|
||||
# En shards, nombres únicos bajo el prefijo compartido del padre
|
||||
# para no pisar RESULTADOS.csv / OMITIDOS.csv entre bots.
|
||||
shard_index = job.get("shard_index")
|
||||
shard_suffix = ""
|
||||
if shard_index is not None:
|
||||
try:
|
||||
shard_suffix = f"_shard_{int(shard_index) + 1}"
|
||||
except (TypeError, ValueError):
|
||||
shard_suffix = f"_shard_{shard_index}"
|
||||
|
||||
def remote_name(local_name: str) -> str:
|
||||
if not shard_suffix:
|
||||
return local_name
|
||||
stem = Path(local_name).stem
|
||||
suffix = Path(local_name).suffix
|
||||
return f"{stem}{shard_suffix}{suffix}"
|
||||
|
||||
for name in REPORT_FILES:
|
||||
local = out_dir / name
|
||||
if not local.exists():
|
||||
continue
|
||||
try:
|
||||
key = self.minio.upload_report(
|
||||
job_id,
|
||||
local,
|
||||
reports_prefix=reports_prefix,
|
||||
object_name=remote_name(name),
|
||||
)
|
||||
self._uploaded_keys.append(key)
|
||||
self._log(f"[worker] subido {key}")
|
||||
except Exception as e:
|
||||
self._log(f"[worker] error subiendo {name}: {e}")
|
||||
|
||||
try:
|
||||
self.minio.write_manifest(
|
||||
job_id,
|
||||
self._uploaded_keys,
|
||||
reports_prefix=reports_prefix,
|
||||
manifest_name=(
|
||||
f"_manifest{shard_suffix}.json" if shard_suffix else "_manifest.json"
|
||||
),
|
||||
extra={
|
||||
"exitosos": summary.exitosos,
|
||||
"fallidos": summary.fallidos,
|
||||
"existentes": summary.existentes,
|
||||
"omitidos": summary.omitidos,
|
||||
"timeouts": summary.timeouts,
|
||||
"shard_index": shard_index,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
self._log(f"[worker] manifest no escrito: {e}")
|
||||
|
||||
result = {
|
||||
"total": summary.total,
|
||||
"exitosos": summary.exitosos,
|
||||
"fallidos": summary.fallidos,
|
||||
"existentes": summary.existentes,
|
||||
"omitidos": summary.omitidos,
|
||||
"timeouts": summary.timeouts,
|
||||
"cancelled": summary.cancelled,
|
||||
"reports_prefix": reports_prefix,
|
||||
"uploaded_count": len(self._uploaded_keys),
|
||||
}
|
||||
|
||||
if summary.cancelled:
|
||||
status = JOB_CANCELLED
|
||||
error_msg = None
|
||||
elif summary.fatal_error:
|
||||
status = JOB_FAILED
|
||||
error_msg = summary.fatal_error
|
||||
elif summary.fallidos and summary.exitosos == 0 and summary.existentes == 0:
|
||||
status = JOB_FAILED
|
||||
error_msg = "Todos los radicados fallaron"
|
||||
else:
|
||||
status = JOB_COMPLETED
|
||||
error_msg = None
|
||||
|
||||
if error_msg:
|
||||
try:
|
||||
self.api.post_event(
|
||||
job_id,
|
||||
{
|
||||
"event_type": "warning",
|
||||
"message": error_msg,
|
||||
"metadata": {"fatal": bool(summary.fatal_error)},
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self.api.patch_job(
|
||||
job_id,
|
||||
status=status,
|
||||
result=result,
|
||||
error=error_msg,
|
||||
)
|
||||
self._log(f"[worker {self.worker_id}] job {job_id} → {status}")
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"[worker {self.worker_id}] job {job_id} error: {e}")
|
||||
try:
|
||||
self.api.post_event(
|
||||
job_id,
|
||||
{
|
||||
"event_type": "warning",
|
||||
"message": f"Error fatal: {e}",
|
||||
"metadata": {"fatal": True},
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.api.patch_job(
|
||||
job_id,
|
||||
status=JOB_FAILED,
|
||||
error=str(e),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Cliente MinIO para subir reportes del digitador (solo salida)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MinioSettings:
|
||||
endpoint: str = "https://s3.builker.xyz"
|
||||
bucket: str = "pys-cuentas-medicas"
|
||||
access_key: str = ""
|
||||
secret_key: str = ""
|
||||
region: str = "us-east-1"
|
||||
anonymous: bool = True
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "MinioSettings":
|
||||
access_key = (os.getenv("MINIO_ACCESS_KEY") or "").strip()
|
||||
secret_key = (os.getenv("MINIO_SECRET_KEY") or "").strip()
|
||||
anon_flag = (os.getenv("MINIO_ANONYMOUS") or "").strip().lower()
|
||||
if anon_flag in ("0", "false", "no"):
|
||||
anonymous = False
|
||||
elif anon_flag in ("1", "true", "yes"):
|
||||
anonymous = True
|
||||
else:
|
||||
anonymous = not (access_key and secret_key)
|
||||
return cls(
|
||||
endpoint=os.getenv("MINIO_ENDPOINT", "https://s3.builker.xyz").rstrip("/"),
|
||||
bucket=os.getenv("MINIO_BUCKET", "pys-cuentas-medicas"),
|
||||
access_key=access_key,
|
||||
secret_key=secret_key,
|
||||
region=os.getenv("MINIO_REGION", "us-east-1"),
|
||||
anonymous=anonymous,
|
||||
)
|
||||
|
||||
|
||||
def report_key_from_prefix(reports_prefix: str, filename: str) -> str:
|
||||
prefix = (reports_prefix or "").strip().rstrip("/")
|
||||
if not prefix:
|
||||
raise ValueError("reports_prefix vacío")
|
||||
name = Path(filename).name
|
||||
return f"{prefix}/{name}"
|
||||
|
||||
|
||||
def report_key(job_id: str, filename: str) -> str:
|
||||
return f"digitados/{job_id}/{Path(filename).name}"
|
||||
|
||||
|
||||
def manifest_key_from_prefix(reports_prefix: str) -> str:
|
||||
prefix = (reports_prefix or "").strip().rstrip("/")
|
||||
return f"{prefix}/_manifest.json"
|
||||
|
||||
|
||||
def manifest_key(job_id: str) -> str:
|
||||
return f"digitados/{job_id}/_manifest.json"
|
||||
|
||||
|
||||
def _s3_client(settings: MinioSettings):
|
||||
import boto3
|
||||
from botocore import UNSIGNED
|
||||
from botocore.client import Config
|
||||
|
||||
s3_cfg = {
|
||||
"addressing_style": "path",
|
||||
"request_checksum_calculation": "when_required",
|
||||
"response_checksum_validation": "when_required",
|
||||
}
|
||||
if settings.anonymous:
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.endpoint,
|
||||
region_name=settings.region,
|
||||
config=Config(signature_version=UNSIGNED, s3=s3_cfg),
|
||||
)
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.endpoint,
|
||||
aws_access_key_id=settings.access_key,
|
||||
aws_secret_access_key=settings.secret_key,
|
||||
region_name=settings.region,
|
||||
config=Config(signature_version="s3v4", s3=s3_cfg),
|
||||
)
|
||||
|
||||
|
||||
class WorkerMinioUploader:
|
||||
"""PutObject de reportes CSV/log y manifest bajo digitados/."""
|
||||
|
||||
def __init__(self, settings: Optional[MinioSettings] = None):
|
||||
self.settings = settings or MinioSettings.from_env()
|
||||
self._client = None
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
if self._client is None:
|
||||
self._client = _s3_client(self.settings)
|
||||
return self._client
|
||||
|
||||
def object_exists(self, key: str) -> bool:
|
||||
try:
|
||||
self.client.head_object(Bucket=self.settings.bucket, Key=key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def upload_report(
|
||||
self,
|
||||
job_id: str,
|
||||
local_path: str | Path,
|
||||
*,
|
||||
reports_prefix: str | None = None,
|
||||
object_name: str | None = None,
|
||||
content_type: str | None = None,
|
||||
) -> str:
|
||||
path = Path(local_path)
|
||||
filename = object_name or path.name
|
||||
if reports_prefix:
|
||||
key = report_key_from_prefix(reports_prefix, filename)
|
||||
else:
|
||||
key = report_key(job_id, filename)
|
||||
|
||||
ctype = content_type or mimetypes.guess_type(filename)[0] or "application/octet-stream"
|
||||
self.client.put_object(
|
||||
Bucket=self.settings.bucket,
|
||||
Key=key,
|
||||
Body=path.read_bytes(),
|
||||
ContentType=ctype,
|
||||
)
|
||||
return key
|
||||
|
||||
def write_manifest(
|
||||
self,
|
||||
job_id: str,
|
||||
keys: List[str],
|
||||
*,
|
||||
reports_prefix: str | None = None,
|
||||
manifest_name: str = "_manifest.json",
|
||||
extra: Optional[dict] = None,
|
||||
) -> str:
|
||||
if reports_prefix:
|
||||
key = report_key_from_prefix(reports_prefix, manifest_name)
|
||||
else:
|
||||
key = report_key(job_id, manifest_name)
|
||||
payload = {
|
||||
"job_id": job_id,
|
||||
"reports_prefix": reports_prefix or f"digitados/{job_id}/",
|
||||
"files": keys,
|
||||
"count": len(keys),
|
||||
}
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
self.client.put_object(
|
||||
Bucket=self.settings.bucket,
|
||||
Key=key,
|
||||
Body=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||||
ContentType="application/json",
|
||||
)
|
||||
return key
|
||||
@@ -0,0 +1,64 @@
|
||||
"""CLI del agent worker digitador: idle hasta que el usuario asigne un job."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
from bot_digitador.core.coords import DEFAULT_APP_PATH
|
||||
from bot_digitador.worker.agent import WorkerAgent
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(
|
||||
description="Agent BotDigitador: registra el PC y espera jobs de digitación."
|
||||
)
|
||||
p.add_argument(
|
||||
"--api-url",
|
||||
default=os.getenv("BOT_API_URL", "http://localhost:8189/bot-digitador"),
|
||||
help="URL del API vía gateway (default: …/bot-digitador)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--worker-id",
|
||||
required=True,
|
||||
help="Identificador único de este PC (ej. PC-DIGITADOR-01)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--token",
|
||||
default=os.getenv("BOT_API_TOKEN", "dev-token-change-me"),
|
||||
help="Bearer token (BOT_API_TOKEN)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--app-path",
|
||||
default=os.getenv("DIGITADOR_APP_PATH", DEFAULT_APP_PATH),
|
||||
help="Ruta al acceso directo / ejecutable de Integración Total",
|
||||
)
|
||||
p.add_argument(
|
||||
"--poll",
|
||||
type=float,
|
||||
default=8.0,
|
||||
help="Segundos entre heartbeat/claim",
|
||||
)
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
agent = WorkerAgent(
|
||||
api_url=args.api_url,
|
||||
worker_id=args.worker_id,
|
||||
token=args.token,
|
||||
poll_seconds=args.poll,
|
||||
app_path=args.app_path,
|
||||
)
|
||||
try:
|
||||
agent.run_forever()
|
||||
except KeyboardInterrupt:
|
||||
print("\nDetenido por usuario")
|
||||
agent.stop()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,106 @@
|
||||
"""CLI local del Bot Digitador (sin API remota)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from bot_digitador.core.coords import DEFAULT_APP_PATH
|
||||
from bot_digitador.core.digitador_service import DigitadorService
|
||||
from bot_digitador.models.events import DigitadorEvent
|
||||
|
||||
|
||||
def _print_event(event: DigitadorEvent) -> None:
|
||||
print(event.display_message(), flush=True)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Digita radicados en Integración Total desde un CSV local."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--csv",
|
||||
required=True,
|
||||
help="Ruta al CSV de entrada (coma o pipe)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default=None,
|
||||
help="Carpeta de reportes (default: <csv_dir>/reportes_bot/<timestamp>)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--delimiter",
|
||||
choices=[",", "|"],
|
||||
default="|",
|
||||
help="Delimitador CSV (se auto-detecta si no coincide con columnas)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--date-format",
|
||||
choices=["dd/mm/yyyy", "mm/dd/yyyy"],
|
||||
default="dd/mm/yyyy",
|
||||
help="Formato preferido de fechas en el CSV",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--app-path",
|
||||
default=os.getenv("DIGITADOR_APP_PATH", DEFAULT_APP_PATH),
|
||||
help="Ruta al acceso directo / ejecutable de Integración Total",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reinicio-cada",
|
||||
type=int,
|
||||
default=6,
|
||||
help="Reinicio completo de la app cada N registros",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
|
||||
csv_path = Path(args.csv).resolve()
|
||||
if not csv_path.exists():
|
||||
print(f"CSV no encontrado: {csv_path}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
if args.output_dir:
|
||||
output_dir = Path(args.output_dir).resolve()
|
||||
else:
|
||||
from datetime import datetime
|
||||
|
||||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
output_dir = csv_path.parent / "reportes_bot" / stamp
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
service = DigitadorService(
|
||||
csv_path=str(csv_path),
|
||||
output_dir=str(output_dir),
|
||||
date_format=args.date_format,
|
||||
delimiter=args.delimiter,
|
||||
app_path=args.app_path,
|
||||
reinicio_cada=args.reinicio_cada,
|
||||
event_callback=_print_event,
|
||||
)
|
||||
summary = service.run()
|
||||
|
||||
print(
|
||||
f"Resumen: OK={summary.exitosos}/{summary.total}, "
|
||||
f"fallidos={summary.fallidos}, existentes={summary.existentes}, "
|
||||
f"omitidos={summary.omitidos}, timeouts={summary.timeouts}, "
|
||||
f"cancelled={summary.cancelled}",
|
||||
flush=True,
|
||||
)
|
||||
print(f"Reportes en: {summary.output_dir}", flush=True)
|
||||
|
||||
if summary.fatal_error:
|
||||
print(f"Error fatal: {summary.fatal_error}", file=sys.stderr)
|
||||
return 1
|
||||
if summary.cancelled:
|
||||
return 130
|
||||
return 0 if summary.fallidos == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,8 @@
|
||||
httpx>=0.27.0
|
||||
boto3>=1.34.0
|
||||
pywinauto>=0.6.8
|
||||
opencv-python-headless>=4.8.0
|
||||
numpy>=1.24.0
|
||||
customtkinter>=5.2.0
|
||||
python-dotenv>=1.0.0
|
||||
Pillow>=10.0.0
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Punto de entrada de la interfaz gráfica del bot digitador."""
|
||||
|
||||
from bot_digitador.ui.dashboard import BotDashboard
|
||||
|
||||
|
||||
def main():
|
||||
app = BotDashboard()
|
||||
app.mainloop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,16 @@
|
||||
# Plantillas OpenCV
|
||||
|
||||
Copia aquí los PNG necesarios desde la carpeta `template/` del script original
|
||||
(`bot_saludtotal.py` / proyecto de digitación):
|
||||
|
||||
- `error.png`
|
||||
- `success.png`
|
||||
- `bien.png`
|
||||
- `flecha_diferencias_abierto.png`
|
||||
- `flecha_diferencias_cerrado.png`
|
||||
- `fecha_vacia_slash.png`
|
||||
- `fecha_vacia_blanco.png`
|
||||
- `popup_nombres.png`
|
||||
- `radicado_existente.png`
|
||||
|
||||
Sin estos archivos el bot no puede validar pantallas de Integración Total.
|
||||
+1847
File diff suppressed because it is too large
Load Diff
@@ -199,6 +199,36 @@ services:
|
||||
retries: 5
|
||||
start_period: 20s
|
||||
|
||||
bot-digitador:
|
||||
build:
|
||||
context: ./services/bot_digitador
|
||||
dockerfile: Dockerfile
|
||||
container_name: ipsextractor-bot-digitador
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
database:
|
||||
condition: service_healthy
|
||||
poblacion_apta:
|
||||
condition: service_started
|
||||
environment:
|
||||
BOT_API_TOKEN: ${BOT_API_TOKEN:-dev-token-change-me}
|
||||
BOT_API_HOST: 0.0.0.0
|
||||
BOT_API_PORT: "8210"
|
||||
DATABASE_URL: postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor
|
||||
POBLACION_APTA_URL: http://poblacion_apta:8003
|
||||
MINIO_ENDPOINT: ${MINIO_ENDPOINT:-https://s3.builker.xyz}
|
||||
MINIO_BUCKET: ${MINIO_BUCKET:-pys-cuentas-medicas}
|
||||
MINIO_ANONYMOUS: ${MINIO_ANONYMOUS:-1}
|
||||
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-}
|
||||
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-}
|
||||
MINIO_REGION: ${MINIO_REGION:-us-east-1}
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://127.0.0.1:8210/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 20s
|
||||
|
||||
api_gateway:
|
||||
build:
|
||||
context: ./services/api_gateway
|
||||
@@ -215,12 +245,15 @@ services:
|
||||
condition: service_healthy
|
||||
bot-descarga:
|
||||
condition: service_healthy
|
||||
bot-digitador:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
EXTRACTION_URL: http://extraction:8001
|
||||
CLEANING_URL: http://cleaning:8002
|
||||
POBLACION_APTA_URL: http://poblacion_apta:8003
|
||||
COUNT_URL: http://count-api:8080
|
||||
BOT_DESCARGA_URL: http://bot-descarga:8200
|
||||
BOT_DIGITADOR_URL: http://bot-digitador:8210
|
||||
ports:
|
||||
# Único puerto expuesto al host. Los demás servicios solo son accesibles en la red Docker.
|
||||
- "0.0.0.0:8189:8000"
|
||||
|
||||
@@ -20,6 +20,7 @@ CLEANING_URL = os.getenv("CLEANING_URL", "http://cleaning:8002").rstrip("/")
|
||||
POBLACION_APTA_URL = os.getenv("POBLACION_APTA_URL", "http://poblacion_apta:8003").rstrip("/")
|
||||
COUNT_URL = os.getenv("COUNT_URL", "http://count-api:8080").rstrip("/")
|
||||
BOT_DESCARGA_URL = os.getenv("BOT_DESCARGA_URL", "http://bot-descarga:8200").rstrip("/")
|
||||
BOT_DIGITADOR_URL = os.getenv("BOT_DIGITADOR_URL", "http://bot-digitador:8210").rstrip("/")
|
||||
|
||||
# Timeouts: JSON ligero vs uploads OCR/conteo
|
||||
_TIMEOUT_JSON = httpx.Timeout(connect=5.0, read=30.0, write=30.0, pool=5.0)
|
||||
@@ -36,12 +37,13 @@ async def lifespan(app: FastAPI):
|
||||
limits=httpx.Limits(max_connections=100, max_keepalive_connections=20),
|
||||
)
|
||||
_log.info(
|
||||
"Gateway iniciado — extraction=%s cleaning=%s poblacion_apta=%s count=%s bot_descarga=%s",
|
||||
"Gateway iniciado — extraction=%s cleaning=%s poblacion_apta=%s count=%s bot_descarga=%s bot_digitador=%s",
|
||||
EXTRACTION_URL,
|
||||
CLEANING_URL,
|
||||
POBLACION_APTA_URL,
|
||||
COUNT_URL,
|
||||
BOT_DESCARGA_URL,
|
||||
BOT_DIGITADOR_URL,
|
||||
)
|
||||
yield
|
||||
await _http_client.aclose()
|
||||
@@ -84,6 +86,7 @@ def _is_upload_path(path: str) -> bool:
|
||||
or path.startswith("/poblacion-apta/jobs/from-csv")
|
||||
or path.startswith("/poblacion-apta/jobs/from-cleaning")
|
||||
or path.startswith("/bot-descarga/jobs")
|
||||
or path.startswith("/bot-digitador/jobs")
|
||||
)
|
||||
|
||||
|
||||
@@ -105,6 +108,7 @@ async def health() -> dict[str, Any]:
|
||||
poblacion_apta = await _safe_health(client, POBLACION_APTA_URL)
|
||||
counting = await _safe_health(client, COUNT_URL)
|
||||
bot_descarga = await _safe_health(client, BOT_DESCARGA_URL)
|
||||
bot_digitador = await _safe_health(client, BOT_DIGITADOR_URL)
|
||||
return {
|
||||
"status": "ok" if extraction["ok"] and cleaning["ok"] and poblacion_apta["ok"] else "degraded",
|
||||
"services": {
|
||||
@@ -113,6 +117,7 @@ async def health() -> dict[str, Any]:
|
||||
"poblacion_apta": poblacion_apta,
|
||||
"counting": counting,
|
||||
"bot_descarga": bot_descarga,
|
||||
"bot_digitador": bot_digitador,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -183,6 +188,11 @@ async def bot_descarga_proxy(path: str, request: Request) -> Response:
|
||||
return await _proxy_request(request, f"{BOT_DESCARGA_URL}/{path}")
|
||||
|
||||
|
||||
@app.api_route("/bot-digitador/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
|
||||
async def bot_digitador_proxy(path: str, request: Request) -> Response:
|
||||
return await _proxy_request(request, f"{BOT_DIGITADOR_URL}/{path}")
|
||||
|
||||
|
||||
@app.api_route("/jobs", methods=["GET", "POST"])
|
||||
async def jobs_root_proxy(request: Request) -> Response:
|
||||
return await _proxy_request(request, f"{EXTRACTION_URL}/jobs")
|
||||
|
||||
@@ -87,26 +87,30 @@ def claim_pending(
|
||||
|
||||
@app.post("/jobs")
|
||||
async def create_job(
|
||||
worker_id: str = Form(...),
|
||||
worker_id: str = Form(""),
|
||||
worker_ids: str = Form(""),
|
||||
job_id: str = Form(...),
|
||||
title: str = Form(""),
|
||||
file: UploadFile = File(...),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
if not worker_id.strip():
|
||||
raise HTTPException(status_code=400, detail="worker_id obligatorio")
|
||||
"""Crea una corrida de descarga.
|
||||
|
||||
- 1 worker → flujo clásico (un solo job claimable).
|
||||
- N workers (worker_ids JSON o todos online) → padre + N shards,
|
||||
mismo downloads_prefix del padre.
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
from domain.excel_split import parse_radicado_rows, rows_to_xlsx_bytes, split_rows
|
||||
from infrastructure.storage_minio import downloads_prefix as mk_prefix
|
||||
|
||||
parent_job_id = job_id.strip()
|
||||
if not parent_job_id:
|
||||
raise HTTPException(status_code=400, detail="job_id (Job Desktop) obligatorio")
|
||||
if not store.parent_job_exists(parent_job_id):
|
||||
raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado")
|
||||
|
||||
if not store.is_worker_online(worker_id):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"Worker '{worker_id}' no está online. Enciende el agent primero.",
|
||||
)
|
||||
|
||||
data = await file.read()
|
||||
if not data:
|
||||
raise HTTPException(status_code=400, detail="Archivo vacío")
|
||||
@@ -115,45 +119,224 @@ async def create_job(
|
||||
content_type = file.content_type or "application/octet-stream"
|
||||
title_clean = title.strip()
|
||||
|
||||
job = store.create_job(
|
||||
worker_id=worker_id.strip(),
|
||||
# Resolver workers
|
||||
selected: List[str] = []
|
||||
raw_ids = (worker_ids or "").strip()
|
||||
if raw_ids:
|
||||
try:
|
||||
parsed = _json.loads(raw_ids)
|
||||
if isinstance(parsed, list):
|
||||
selected = [str(x).strip() for x in parsed if str(x).strip()]
|
||||
elif isinstance(parsed, str) and parsed.strip():
|
||||
selected = [parsed.strip()]
|
||||
except _json.JSONDecodeError:
|
||||
selected = [x.strip() for x in raw_ids.split(",") if x.strip()]
|
||||
if not selected and worker_id.strip():
|
||||
selected = [worker_id.strip()]
|
||||
|
||||
online = store.list_online_worker_ids()
|
||||
online_set = set(online)
|
||||
if not selected:
|
||||
selected = list(online)
|
||||
else:
|
||||
missing = [w for w in selected if w not in online_set]
|
||||
if missing:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"Workers no online: {', '.join(missing)}. Enciende el agent primero.",
|
||||
)
|
||||
|
||||
if not selected:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="No hay bots online. Enciende al menos un agent.",
|
||||
)
|
||||
|
||||
# Dedup preservando orden
|
||||
seen: set[str] = set()
|
||||
workers_unique: List[str] = []
|
||||
for w in selected:
|
||||
if w not in seen:
|
||||
seen.add(w)
|
||||
workers_unique.append(w)
|
||||
|
||||
# Parse filas
|
||||
try:
|
||||
rows = parse_radicado_rows(data, original)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=f"No se pudo leer el Excel/CSV: {e}") from e
|
||||
if not rows:
|
||||
raise HTTPException(status_code=400, detail="El archivo no tiene filas de radicado/NIT")
|
||||
|
||||
# Un solo bot (o una sola fila) → flujo actual
|
||||
n_workers = min(len(workers_unique), len(rows))
|
||||
if n_workers <= 1:
|
||||
only = workers_unique[0]
|
||||
job = store.create_job(
|
||||
worker_id=only,
|
||||
excel_key="",
|
||||
original_filename=original,
|
||||
title=title_clean,
|
||||
job_id=parent_job_id,
|
||||
)
|
||||
task_id = store.create_downloads_task(
|
||||
parent_job_id=parent_job_id,
|
||||
download_job_id=job.id,
|
||||
worker_id=only,
|
||||
title=title_clean or f"Bot descarga ({original})",
|
||||
original_filename=original,
|
||||
worker_ids=[only],
|
||||
mode="single",
|
||||
)
|
||||
try:
|
||||
key = minio.upload_excel(job.id, data, original, content_type=content_type)
|
||||
except Exception as e:
|
||||
store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
|
||||
|
||||
job = store.update_excel_key(job.id, key)
|
||||
assert job is not None
|
||||
store.add_event(
|
||||
job.id,
|
||||
"run_started",
|
||||
f"Job creado y asignado a {only}",
|
||||
metadata={
|
||||
"excel_key": key,
|
||||
"worker_id": only,
|
||||
"job_id": parent_job_id,
|
||||
"task_id": task_id,
|
||||
"mode": "single",
|
||||
"total_rows": len(rows),
|
||||
},
|
||||
)
|
||||
return job.to_dict()
|
||||
|
||||
# Distribuido: padre + shards
|
||||
workers_unique = workers_unique[:n_workers]
|
||||
chunks = split_rows(rows, n_workers)
|
||||
n = len(chunks)
|
||||
workers_unique = workers_unique[:n]
|
||||
|
||||
parent = store.create_job(
|
||||
worker_id=None,
|
||||
excel_key="",
|
||||
original_filename=original,
|
||||
title=title_clean,
|
||||
title=title_clean or f"Descarga distribuida ({original})",
|
||||
job_id=parent_job_id,
|
||||
)
|
||||
task_id = store.create_downloads_task(
|
||||
parent_job_id=parent_job_id,
|
||||
download_job_id=job.id,
|
||||
worker_id=worker_id.strip(),
|
||||
title=title_clean or f"Bot descarga ({original})",
|
||||
original_filename=original,
|
||||
)
|
||||
job = store.get_job(job.id)
|
||||
assert job is not None
|
||||
|
||||
try:
|
||||
key = minio.upload_excel(job.id, data, original, content_type=content_type)
|
||||
except Exception as e:
|
||||
store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
|
||||
|
||||
job = store.update_excel_key(job.id, key)
|
||||
assert job is not None
|
||||
store.add_event(
|
||||
job.id,
|
||||
"run_started",
|
||||
f"Job creado y asignado a {worker_id}",
|
||||
metadata={
|
||||
"excel_key": key,
|
||||
"worker_id": worker_id,
|
||||
"job_id": parent_job_id,
|
||||
"task_id": task_id,
|
||||
shard_count=n,
|
||||
status=JobStatus.PENDING.value,
|
||||
result={
|
||||
"mode": "distributed",
|
||||
"shard_ids": [],
|
||||
"total_rows": len(rows),
|
||||
"total": len(rows),
|
||||
"processed": 0,
|
||||
"exitosos": 0,
|
||||
"fallidos": 0,
|
||||
},
|
||||
)
|
||||
prefix = parent.downloads_prefix or mk_prefix(parent.id)
|
||||
|
||||
task_id = store.create_downloads_task(
|
||||
parent_job_id=parent_job_id,
|
||||
download_job_id=parent.id,
|
||||
worker_id=None,
|
||||
title=title_clean or f"Bot descarga ×{n} ({original})",
|
||||
original_filename=original,
|
||||
worker_ids=workers_unique,
|
||||
mode="distributed",
|
||||
)
|
||||
|
||||
try:
|
||||
full_key = minio.upload_excel(parent.id, data, original, content_type=content_type)
|
||||
except Exception as e:
|
||||
store.patch_job(parent.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
|
||||
|
||||
store.update_excel_key(parent.id, full_key)
|
||||
|
||||
shard_ids: List[str] = []
|
||||
for i, (chunk, wid) in enumerate(zip(chunks, workers_unique)):
|
||||
shard_bytes = rows_to_xlsx_bytes(chunk)
|
||||
shard = store.create_job(
|
||||
worker_id=wid,
|
||||
excel_key="",
|
||||
original_filename=f"shard_{i + 1}_of_{n}.xlsx",
|
||||
title=f"{title_clean or 'Descarga'} · shard {i + 1}/{n}",
|
||||
job_id=parent_job_id,
|
||||
task_id=task_id,
|
||||
downloads_prefix_override=prefix,
|
||||
parent_download_job_id=parent.id,
|
||||
shard_index=i,
|
||||
shard_count=n,
|
||||
result={"total": len(chunk), "processed": 0},
|
||||
)
|
||||
try:
|
||||
shard_key = minio.upload_excel(
|
||||
shard.id,
|
||||
shard_bytes,
|
||||
f"shard_{i + 1}_of_{n}.xlsx",
|
||||
content_type=(
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
store.patch_job(
|
||||
parent.id,
|
||||
status=JobStatus.FAILED.value,
|
||||
error=f"MinIO shard upload failed: {e}",
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=502, detail=f"Error subiendo shard a MinIO: {e}"
|
||||
) from e
|
||||
store.update_excel_key(shard.id, shard_key)
|
||||
shard_ids.append(shard.id)
|
||||
store.add_event(
|
||||
parent.id,
|
||||
"info",
|
||||
f"Shard {i + 1}/{n} → {wid} ({len(chunk)} filas)",
|
||||
metadata={"shard_id": shard.id, "worker_id": wid, "rows": len(chunk)},
|
||||
)
|
||||
|
||||
store.patch_job(
|
||||
parent.id,
|
||||
result={
|
||||
"mode": "distributed",
|
||||
"shard_ids": shard_ids,
|
||||
"total_rows": len(rows),
|
||||
"total": len(rows),
|
||||
"processed": 0,
|
||||
"exitosos": 0,
|
||||
"fallidos": 0,
|
||||
"downloads_prefix": prefix,
|
||||
"worker_ids": workers_unique,
|
||||
},
|
||||
)
|
||||
store.add_event(
|
||||
parent.id,
|
||||
"run_started",
|
||||
f"Corrida distribuida: {n} bots, {len(rows)} filas",
|
||||
metadata={
|
||||
"excel_key": full_key,
|
||||
"worker_ids": workers_unique,
|
||||
"job_id": parent_job_id,
|
||||
"task_id": task_id,
|
||||
"mode": "distributed",
|
||||
"shard_ids": shard_ids,
|
||||
"total_rows": len(rows),
|
||||
},
|
||||
)
|
||||
job = store.get_job(parent.id)
|
||||
assert job is not None
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.post("/internal/reclaim")
|
||||
def reclaim_jobs(_: str = Depends(require_bearer)) -> dict:
|
||||
n = store.reclaim_stale_running_jobs()
|
||||
return {"reclaimed": n}
|
||||
|
||||
|
||||
@app.get("/jobs")
|
||||
def list_jobs(
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
@@ -168,7 +351,15 @@ def get_job(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(download_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
return job.to_dict()
|
||||
# Refrescar consolidación si es padre distribuido
|
||||
if job.is_distributed_parent:
|
||||
refreshed = store.refresh_parent_from_shards(download_job_id)
|
||||
if refreshed:
|
||||
job = refreshed
|
||||
payload = job.to_dict()
|
||||
if job.is_distributed_parent:
|
||||
payload["shards"] = [s.to_dict() for s in store.list_shards(job.id)]
|
||||
return payload
|
||||
|
||||
|
||||
@app.get("/jobs/{download_job_id}/files")
|
||||
@@ -279,7 +470,8 @@ def post_event(
|
||||
body: EventBody,
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
if not store.get_job(download_job_id):
|
||||
job = store.get_job(download_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
event = store.add_event(
|
||||
download_job_id,
|
||||
@@ -292,8 +484,7 @@ def post_event(
|
||||
metadata=body.metadata,
|
||||
timestamp=body.timestamp,
|
||||
)
|
||||
job = store.get_job(download_job_id)
|
||||
if job and body.index is not None and body.total is not None:
|
||||
if body.index is not None and body.total is not None:
|
||||
result = dict(job.result or {})
|
||||
result["processed"] = body.index
|
||||
result["total"] = body.total
|
||||
@@ -327,4 +518,7 @@ def cancel_flag(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(download_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
return {"cancel_requested": job.cancel_requested, "status": job.status}
|
||||
return {
|
||||
"cancel_requested": store.is_cancel_requested(download_job_id),
|
||||
"status": job.status,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Parseo y partición de Excel/CSV de radicados para descarga distribuida."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from openpyxl import Workbook, load_workbook
|
||||
|
||||
_HEADER_NAMES = ("radicado", "proceso", "numero", "número")
|
||||
|
||||
|
||||
def _clean_nit(valor: Any) -> str:
|
||||
valor = str(valor).strip()
|
||||
valor = re.sub(r'[\\/:*?"<>|]', "_", valor)
|
||||
return valor if valor else "SIN_NIT"
|
||||
|
||||
|
||||
def parse_radicado_rows(data: bytes, filename: str = "input.xlsx") -> list[tuple[str, str]]:
|
||||
"""Devuelve lista de (radicado, nit)."""
|
||||
name = (filename or "input.xlsx").lower()
|
||||
if name.endswith(".csv"):
|
||||
return _parse_csv(data)
|
||||
return _parse_xlsx(data)
|
||||
|
||||
|
||||
def _parse_csv(data: bytes) -> list[tuple[str, str]]:
|
||||
text = None
|
||||
for enc in ("utf-8-sig", "utf-8", "latin-1", "cp1252"):
|
||||
try:
|
||||
text = data.decode(enc)
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
if text is None:
|
||||
text = data.decode("utf-8", errors="replace")
|
||||
sample = text[:4096]
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters="|,;\t")
|
||||
delim = dialect.delimiter
|
||||
except csv.Error:
|
||||
delim = "|" if sample.count("|") >= sample.count(",") else ","
|
||||
reader = csv.reader(io.StringIO(text), delimiter=delim)
|
||||
out: list[tuple[str, str]] = []
|
||||
for fila in reader:
|
||||
if not fila:
|
||||
continue
|
||||
radicado = fila[0] if len(fila) > 0 else None
|
||||
nit = fila[1] if len(fila) > 1 else None
|
||||
if not radicado or str(radicado).strip() in ("", "None"):
|
||||
continue
|
||||
radicado_s = str(radicado).strip()
|
||||
if radicado_s.lower() in _HEADER_NAMES:
|
||||
continue
|
||||
out.append((radicado_s, _clean_nit(nit)))
|
||||
return out
|
||||
|
||||
|
||||
def _parse_xlsx(data: bytes) -> list[tuple[str, str]]:
|
||||
wb = load_workbook(io.BytesIO(data), data_only=True)
|
||||
ws = wb.active
|
||||
out: list[tuple[str, str]] = []
|
||||
for fila in ws.iter_rows(values_only=True):
|
||||
radicado = fila[0] if len(fila) > 0 else None
|
||||
nit = fila[1] if len(fila) > 1 else None
|
||||
if not radicado or str(radicado).strip() in ("", "None"):
|
||||
continue
|
||||
radicado_s = str(radicado).strip()
|
||||
if radicado_s.lower() in _HEADER_NAMES:
|
||||
continue
|
||||
out.append((radicado_s, _clean_nit(nit)))
|
||||
return out
|
||||
|
||||
|
||||
def split_rows(rows: list[tuple[str, str]], n: int) -> list[list[tuple[str, str]]]:
|
||||
"""Parte filas en n trozos lo más equilibrados posible (chunks contiguos)."""
|
||||
if n <= 1 or not rows:
|
||||
return [rows]
|
||||
n = min(n, len(rows))
|
||||
size, rem = divmod(len(rows), n)
|
||||
chunks: list[list[tuple[str, str]]] = []
|
||||
idx = 0
|
||||
for i in range(n):
|
||||
take = size + (1 if i < rem else 0)
|
||||
chunks.append(rows[idx : idx + take])
|
||||
idx += take
|
||||
return [c for c in chunks if c]
|
||||
|
||||
|
||||
def rows_to_xlsx_bytes(rows: list[tuple[str, str]]) -> bytes:
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.append(["radicado", "nit"])
|
||||
for radicado, nit in rows:
|
||||
ws.append([radicado, nit])
|
||||
buf = io.BytesIO()
|
||||
wb.save(buf)
|
||||
return buf.getvalue()
|
||||
@@ -40,7 +40,7 @@ class Worker:
|
||||
@dataclass
|
||||
class Job:
|
||||
id: str
|
||||
worker_id: str
|
||||
worker_id: Optional[str]
|
||||
status: str = JobStatus.PENDING.value
|
||||
title: str = ""
|
||||
excel_key: str = ""
|
||||
@@ -54,10 +54,25 @@ class Job:
|
||||
finished_at: Optional[str] = None
|
||||
job_id: Optional[str] = None
|
||||
task_id: Optional[str] = None
|
||||
parent_download_job_id: Optional[str] = None
|
||||
shard_index: Optional[int] = None
|
||||
shard_count: Optional[int] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
@property
|
||||
def is_distributed_parent(self) -> bool:
|
||||
return (
|
||||
self.parent_download_job_id is None
|
||||
and self.shard_count is not None
|
||||
and int(self.shard_count) > 1
|
||||
)
|
||||
|
||||
@property
|
||||
def is_shard(self) -> bool:
|
||||
return bool(self.parent_download_job_id)
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobEventRecord:
|
||||
|
||||
@@ -103,7 +103,12 @@ class DownloadJobRow(Base):
|
||||
task_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("tasks.id"), nullable=True, index=True
|
||||
)
|
||||
worker_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
worker_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
parent_download_job_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("download_jobs.id"), nullable=True, index=True
|
||||
)
|
||||
shard_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
shard_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
excel_key: Mapped[str] = mapped_column(Text, default="")
|
||||
|
||||
@@ -194,21 +194,25 @@ class MinioStorage:
|
||||
|
||||
Si el manifest falta, falla o queda vacío, hace fallback a list_keys
|
||||
bajo el prefijo (cubre varias carpetas NIT planas).
|
||||
En corridas distribuidas varios shards sobrescriben el manifest parcial:
|
||||
siempre se fusiona con list_keys del prefijo.
|
||||
"""
|
||||
prefix = prefix or downloads_prefix(job_id)
|
||||
files: List[dict] = []
|
||||
by_key: dict[str, dict] = {}
|
||||
try:
|
||||
raw = self.get_bytes(manifest_key(job_id))
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
for key in payload.get("files") or []:
|
||||
entry = self._file_entry(str(key))
|
||||
if entry is not None:
|
||||
files.append(entry)
|
||||
by_key[entry["key"]] = entry
|
||||
except Exception:
|
||||
files = []
|
||||
pass
|
||||
|
||||
if not files:
|
||||
files = self._list_pdf_entries(prefix)
|
||||
for entry in self._list_pdf_entries(prefix):
|
||||
by_key[entry["key"]] = entry
|
||||
|
||||
files = list(by_key.values())
|
||||
return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)}
|
||||
|
||||
def _list_pdf_entries(self, prefix: str) -> List[dict]:
|
||||
|
||||
@@ -73,6 +73,9 @@ class JobStore:
|
||||
finished_at=_iso(row.finished_at),
|
||||
job_id=row.job_id,
|
||||
task_id=row.task_id,
|
||||
parent_download_job_id=row.parent_download_job_id,
|
||||
shard_index=row.shard_index,
|
||||
shard_count=row.shard_count,
|
||||
)
|
||||
|
||||
def _refresh_worker_status(self, worker: Worker) -> Worker:
|
||||
@@ -129,7 +132,13 @@ class JobStore:
|
||||
WorkerStatus.BUSY.value if row.current_job_id else WorkerStatus.ONLINE.value
|
||||
)
|
||||
session.flush()
|
||||
return self._row_worker(row)
|
||||
worker = self._row_worker(row)
|
||||
# Reclaim fuera de la sesión del heartbeat para no bloquear
|
||||
try:
|
||||
self.reclaim_stale_running_jobs()
|
||||
except Exception:
|
||||
pass
|
||||
return worker
|
||||
|
||||
def get_worker(self, worker_id: str) -> Optional[Worker]:
|
||||
with session_scope() as session:
|
||||
@@ -139,12 +148,23 @@ class JobStore:
|
||||
return self._row_worker(row)
|
||||
|
||||
def list_workers(self) -> List[Worker]:
|
||||
try:
|
||||
self.reclaim_stale_running_jobs()
|
||||
except Exception:
|
||||
pass
|
||||
with session_scope() as session:
|
||||
rows = session.scalars(
|
||||
select(DownloadWorkerRow).order_by(DownloadWorkerRow.id)
|
||||
).all()
|
||||
return [self._row_worker(r) for r in rows]
|
||||
|
||||
def list_online_worker_ids(self) -> List[str]:
|
||||
return [
|
||||
w.id
|
||||
for w in self.list_workers()
|
||||
if w.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value)
|
||||
]
|
||||
|
||||
def is_worker_online(self, worker_id: str) -> bool:
|
||||
worker = self.get_worker(worker_id)
|
||||
if not worker:
|
||||
@@ -162,31 +182,41 @@ class JobStore:
|
||||
|
||||
def create_job(
|
||||
self,
|
||||
worker_id: str,
|
||||
worker_id: Optional[str],
|
||||
excel_key: str,
|
||||
original_filename: str = "",
|
||||
title: str = "",
|
||||
*,
|
||||
job_id: Optional[str] = None,
|
||||
task_id: Optional[str] = None,
|
||||
downloads_prefix_override: Optional[str] = None,
|
||||
parent_download_job_id: Optional[str] = None,
|
||||
shard_index: Optional[int] = None,
|
||||
shard_count: Optional[int] = None,
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
status: Optional[str] = None,
|
||||
download_id: Optional[str] = None,
|
||||
) -> Job:
|
||||
import uuid as uuid_mod
|
||||
|
||||
download_id = str(uuid_mod.uuid4())
|
||||
prefix = downloads_prefix(download_id)
|
||||
download_id = download_id or str(uuid_mod.uuid4())
|
||||
prefix = downloads_prefix_override or downloads_prefix(download_id)
|
||||
with session_scope() as session:
|
||||
row = DownloadJobRow(
|
||||
id=download_id,
|
||||
job_id=job_id,
|
||||
task_id=task_id,
|
||||
worker_id=worker_id,
|
||||
status=JobStatus.PENDING.value,
|
||||
parent_download_job_id=parent_download_job_id,
|
||||
shard_index=shard_index,
|
||||
shard_count=shard_count,
|
||||
status=status or JobStatus.PENDING.value,
|
||||
title=title or f"Descarga {original_filename}",
|
||||
excel_key=excel_key,
|
||||
downloads_prefix=prefix,
|
||||
original_filename=original_filename,
|
||||
cancel_requested=False,
|
||||
result={},
|
||||
result=result or {},
|
||||
created_at=utcnow(),
|
||||
)
|
||||
session.add(row)
|
||||
@@ -205,34 +235,57 @@ class JobStore:
|
||||
limit: int = 100,
|
||||
*,
|
||||
job_id: Optional[str] = None,
|
||||
parents_only: bool = True,
|
||||
) -> List[Job]:
|
||||
with session_scope() as session:
|
||||
stmt = select(DownloadJobRow).order_by(DownloadJobRow.created_at.desc()).limit(limit)
|
||||
stmt = select(DownloadJobRow).order_by(DownloadJobRow.created_at.desc())
|
||||
if job_id:
|
||||
stmt = (
|
||||
select(DownloadJobRow)
|
||||
.where(DownloadJobRow.job_id == job_id)
|
||||
.order_by(DownloadJobRow.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
stmt = stmt.where(DownloadJobRow.job_id == job_id)
|
||||
if parents_only:
|
||||
stmt = stmt.where(DownloadJobRow.parent_download_job_id.is_(None))
|
||||
stmt = stmt.limit(limit)
|
||||
rows = session.scalars(stmt).all()
|
||||
return [self._row_job(r) for r in rows]
|
||||
|
||||
def list_shards(self, parent_id: str) -> List[Job]:
|
||||
with session_scope() as session:
|
||||
rows = session.scalars(
|
||||
select(DownloadJobRow)
|
||||
.where(DownloadJobRow.parent_download_job_id == parent_id)
|
||||
.order_by(DownloadJobRow.shard_index.asc())
|
||||
).all()
|
||||
return [self._row_job(r) for r in rows]
|
||||
|
||||
def claim_pending(self, worker_id: str) -> Optional[Job]:
|
||||
try:
|
||||
self.reclaim_stale_running_jobs()
|
||||
except Exception:
|
||||
pass
|
||||
with session_scope() as session:
|
||||
worker = session.get(DownloadWorkerRow, worker_id)
|
||||
if not worker or worker.current_job_id:
|
||||
return None
|
||||
|
||||
row = session.scalars(
|
||||
# No claimar padres orquestadores (shard_count > 1 sin parent)
|
||||
rows = session.scalars(
|
||||
select(DownloadJobRow)
|
||||
.where(
|
||||
DownloadJobRow.worker_id == worker_id,
|
||||
DownloadJobRow.status == JobStatus.PENDING.value,
|
||||
)
|
||||
.order_by(DownloadJobRow.created_at.asc())
|
||||
.limit(1)
|
||||
).first()
|
||||
).all()
|
||||
row = None
|
||||
for candidate in rows:
|
||||
is_parent = (
|
||||
candidate.parent_download_job_id is None
|
||||
and candidate.shard_count is not None
|
||||
and int(candidate.shard_count) > 1
|
||||
)
|
||||
if is_parent:
|
||||
continue
|
||||
row = candidate
|
||||
break
|
||||
if not row:
|
||||
return None
|
||||
|
||||
@@ -243,9 +296,19 @@ class JobStore:
|
||||
worker.status = WorkerStatus.BUSY.value
|
||||
worker.last_heartbeat = now
|
||||
|
||||
if row.task_id:
|
||||
task = session.get(Task, row.task_id)
|
||||
if task:
|
||||
# Task del padre (shards heredan task_id del padre)
|
||||
task_id = row.task_id
|
||||
if row.parent_download_job_id:
|
||||
parent = session.get(DownloadJobRow, row.parent_download_job_id)
|
||||
if parent:
|
||||
task_id = parent.task_id
|
||||
if parent.status == JobStatus.PENDING.value:
|
||||
parent.status = JobStatus.RUNNING.value
|
||||
parent.started_at = parent.started_at or now
|
||||
|
||||
if task_id:
|
||||
task = session.get(Task, task_id)
|
||||
if task and task.status == JobStatus.PENDING.value:
|
||||
task.status = JobStatus.RUNNING.value
|
||||
task.started_at = now
|
||||
task.updated_at = now
|
||||
@@ -271,12 +334,44 @@ class JobStore:
|
||||
row.status = JobStatus.CANCELLED.value
|
||||
row.finished_at = now
|
||||
self._sync_task(session, row, status=JobStatus.CANCELLED.value)
|
||||
|
||||
# Cancelar shards hijos si es padre distribuido
|
||||
if row.parent_download_job_id is None and row.shard_count and int(row.shard_count) > 1:
|
||||
children = session.scalars(
|
||||
select(DownloadJobRow).where(
|
||||
DownloadJobRow.parent_download_job_id == job_id
|
||||
)
|
||||
).all()
|
||||
for child in children:
|
||||
if child.status in (
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
):
|
||||
continue
|
||||
child.cancel_requested = True
|
||||
if child.status == JobStatus.PENDING.value:
|
||||
child.status = JobStatus.CANCELLED.value
|
||||
child.finished_at = now
|
||||
worker = session.get(DownloadWorkerRow, child.worker_id) if child.worker_id else None
|
||||
if worker and worker.current_job_id == child.id:
|
||||
worker.current_job_id = None
|
||||
worker.status = WorkerStatus.ONLINE.value
|
||||
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def is_cancel_requested(self, job_id: str) -> bool:
|
||||
job = self.get_job(job_id)
|
||||
return bool(job and job.cancel_requested)
|
||||
if not job:
|
||||
return False
|
||||
if job.cancel_requested:
|
||||
return True
|
||||
if job.parent_download_job_id:
|
||||
parent = self.get_job(job.parent_download_job_id)
|
||||
if parent and parent.cancel_requested:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _sync_task(
|
||||
self,
|
||||
@@ -287,9 +382,15 @@ class JobStore:
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> None:
|
||||
if not row.task_id:
|
||||
# Sync siempre contra el padre si es shard
|
||||
target = row
|
||||
if row.parent_download_job_id:
|
||||
parent = session.get(DownloadJobRow, row.parent_download_job_id)
|
||||
if parent:
|
||||
target = parent
|
||||
if not target.task_id:
|
||||
return
|
||||
task = session.get(Task, row.task_id)
|
||||
task = session.get(Task, target.task_id)
|
||||
if not task:
|
||||
return
|
||||
now = utcnow()
|
||||
@@ -298,7 +399,7 @@ class JobStore:
|
||||
if result is not None:
|
||||
task.result = {
|
||||
**dict(task.result or {}),
|
||||
"download_job_id": row.id,
|
||||
"download_job_id": target.id,
|
||||
**result,
|
||||
}
|
||||
if error is not None:
|
||||
@@ -319,6 +420,7 @@ class JobStore:
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> Optional[Job]:
|
||||
parent_id: Optional[str] = None
|
||||
with session_scope() as session:
|
||||
row = session.get(DownloadJobRow, job_id)
|
||||
if not row:
|
||||
@@ -340,22 +442,263 @@ class JobStore:
|
||||
row.finished_at = row.finished_at or utcnow()
|
||||
finished = True
|
||||
|
||||
self._sync_task(
|
||||
session,
|
||||
row,
|
||||
status=new_status if status else None,
|
||||
result=result,
|
||||
error=error,
|
||||
)
|
||||
# No sync task directo en shards; se consolida en el padre
|
||||
if not row.parent_download_job_id:
|
||||
self._sync_task(
|
||||
session,
|
||||
row,
|
||||
status=new_status if status else None,
|
||||
result=result,
|
||||
error=error,
|
||||
)
|
||||
|
||||
if finished:
|
||||
worker = session.get(DownloadWorkerRow, row.worker_id)
|
||||
worker = session.get(DownloadWorkerRow, row.worker_id) if row.worker_id else None
|
||||
if worker and worker.current_job_id == job_id:
|
||||
worker.current_job_id = None
|
||||
worker.status = WorkerStatus.ONLINE.value
|
||||
|
||||
parent_id = row.parent_download_job_id
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
job = self._row_job(row)
|
||||
|
||||
if parent_id:
|
||||
self.refresh_parent_from_shards(parent_id)
|
||||
refreshed = self.get_job(job_id)
|
||||
return refreshed or job
|
||||
return job
|
||||
|
||||
def refresh_parent_from_shards(self, parent_id: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
parent = session.get(DownloadJobRow, parent_id)
|
||||
if not parent:
|
||||
return None
|
||||
children = list(
|
||||
session.scalars(
|
||||
select(DownloadJobRow).where(
|
||||
DownloadJobRow.parent_download_job_id == parent_id
|
||||
)
|
||||
).all()
|
||||
)
|
||||
if not children:
|
||||
return self._row_job(parent)
|
||||
|
||||
statuses = [c.status for c in children]
|
||||
terminal = {
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
}
|
||||
any_active = any(
|
||||
s in (JobStatus.PENDING.value, JobStatus.RUNNING.value) for s in statuses
|
||||
)
|
||||
all_terminal = all(s in terminal for s in statuses)
|
||||
|
||||
total = 0
|
||||
exitosos = 0
|
||||
fallidos = 0
|
||||
uploaded = 0
|
||||
for c in children:
|
||||
r = dict(c.result or {})
|
||||
total += int(r.get("total") or 0)
|
||||
exitosos += int(r.get("exitosos") or 0)
|
||||
fallidos += int(r.get("fallidos") or 0)
|
||||
uploaded += int(r.get("uploaded_count") or 0)
|
||||
# Progreso parcial mientras corre
|
||||
if c.status == JobStatus.RUNNING.value:
|
||||
total = max(total, int(r.get("total") or 0))
|
||||
exitosos = max(exitosos, int(r.get("processed") or r.get("exitosos") or 0))
|
||||
|
||||
# Agregar progreso en vuelo (processed) de shards running
|
||||
processed = 0
|
||||
for c in children:
|
||||
r = dict(c.result or {})
|
||||
if c.status in terminal:
|
||||
processed += int(r.get("exitosos") or 0) + int(r.get("fallidos") or 0)
|
||||
else:
|
||||
processed += int(r.get("processed") or 0)
|
||||
|
||||
shard_ids = [c.id for c in children]
|
||||
parent_result = dict(parent.result or {})
|
||||
parent_result.update(
|
||||
{
|
||||
"mode": "distributed",
|
||||
"shard_ids": shard_ids,
|
||||
"total": total or int(parent_result.get("total_rows") or 0),
|
||||
"total_rows": int(parent_result.get("total_rows") or total or 0),
|
||||
"exitosos": exitosos,
|
||||
"fallidos": fallidos,
|
||||
"processed": processed,
|
||||
"uploaded_count": uploaded,
|
||||
"downloads_prefix": parent.downloads_prefix,
|
||||
"shards": [
|
||||
{
|
||||
"id": c.id,
|
||||
"worker_id": c.worker_id,
|
||||
"status": c.status,
|
||||
"shard_index": c.shard_index,
|
||||
"result": dict(c.result or {}),
|
||||
"error": c.error,
|
||||
}
|
||||
for c in children
|
||||
],
|
||||
}
|
||||
)
|
||||
parent.result = parent_result
|
||||
|
||||
now = utcnow()
|
||||
if any_active:
|
||||
parent.status = JobStatus.RUNNING.value
|
||||
parent.started_at = parent.started_at or now
|
||||
parent.finished_at = None
|
||||
parent.error = None
|
||||
self._sync_task(
|
||||
session,
|
||||
parent,
|
||||
status=JobStatus.RUNNING.value,
|
||||
result=parent_result,
|
||||
)
|
||||
elif all_terminal:
|
||||
any_failed = any(s == JobStatus.FAILED.value for s in statuses)
|
||||
any_cancelled = any(s == JobStatus.CANCELLED.value for s in statuses)
|
||||
all_cancelled = all(s == JobStatus.CANCELLED.value for s in statuses)
|
||||
if all_cancelled:
|
||||
parent.status = JobStatus.CANCELLED.value
|
||||
elif any_failed:
|
||||
parent.status = JobStatus.FAILED.value
|
||||
parent.error = next(
|
||||
(c.error for c in children if c.error),
|
||||
"Uno o más shards fallaron",
|
||||
)
|
||||
elif any_cancelled and not any_failed:
|
||||
# mezclado completed+cancelled → completed parcial
|
||||
parent.status = JobStatus.COMPLETED.value
|
||||
else:
|
||||
parent.status = JobStatus.COMPLETED.value
|
||||
parent.finished_at = parent.finished_at or now
|
||||
self._sync_task(
|
||||
session,
|
||||
parent,
|
||||
status=parent.status,
|
||||
result=parent_result,
|
||||
error=parent.error,
|
||||
)
|
||||
|
||||
session.flush()
|
||||
return self._row_job(parent)
|
||||
|
||||
def reclaim_stale_running_jobs(self) -> int:
|
||||
"""Jobs running cuyo worker está offline → pending + reasignar a otro online."""
|
||||
reclaimed = 0
|
||||
with session_scope() as session:
|
||||
now = utcnow()
|
||||
cutoff = now - timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS)
|
||||
workers = list(session.scalars(select(DownloadWorkerRow)).all())
|
||||
offline_ids: set[str] = set()
|
||||
online_free: List[DownloadWorkerRow] = []
|
||||
for w in workers:
|
||||
hb = w.last_heartbeat
|
||||
if hb is None:
|
||||
offline_ids.add(w.id)
|
||||
continue
|
||||
if hb.tzinfo is None:
|
||||
hb = hb.replace(tzinfo=timezone.utc)
|
||||
if hb < cutoff:
|
||||
offline_ids.add(w.id)
|
||||
if w.current_job_id:
|
||||
w.current_job_id = None
|
||||
w.status = WorkerStatus.OFFLINE.value
|
||||
elif not w.current_job_id:
|
||||
online_free.append(w)
|
||||
|
||||
if not offline_ids:
|
||||
session.flush()
|
||||
return 0
|
||||
|
||||
running = session.scalars(
|
||||
select(DownloadJobRow).where(
|
||||
DownloadJobRow.status == JobStatus.RUNNING.value,
|
||||
DownloadJobRow.worker_id.in_(list(offline_ids)),
|
||||
)
|
||||
).all()
|
||||
|
||||
rr = 0
|
||||
for row in running:
|
||||
# Solo shards o jobs simples (no padres orquestadores)
|
||||
is_parent = (
|
||||
row.parent_download_job_id is None
|
||||
and row.shard_count is not None
|
||||
and int(row.shard_count) > 1
|
||||
)
|
||||
if is_parent:
|
||||
continue
|
||||
|
||||
row.status = JobStatus.PENDING.value
|
||||
row.started_at = None
|
||||
row.cancel_requested = False
|
||||
|
||||
if online_free:
|
||||
target = online_free[rr % len(online_free)]
|
||||
rr += 1
|
||||
row.worker_id = target.id
|
||||
# Si no hay online free, queda pending con el mismo worker_id
|
||||
# (se reasignará cuando haya free en el próximo reclaim)
|
||||
|
||||
reclaimed += 1
|
||||
session.add(
|
||||
DownloadJobEventRow(
|
||||
download_job_id=row.id,
|
||||
event_type="warning",
|
||||
message=(
|
||||
f"Worker offline; job reclaimed"
|
||||
+ (f" → {row.worker_id}" if row.worker_id else "")
|
||||
),
|
||||
timestamp=now,
|
||||
metadata_json={"reclaimed": True, "worker_id": row.worker_id},
|
||||
)
|
||||
)
|
||||
if row.parent_download_job_id:
|
||||
session.add(
|
||||
DownloadJobEventRow(
|
||||
download_job_id=row.parent_download_job_id,
|
||||
event_type="warning",
|
||||
message=f"Shard {row.shard_index} reclamado (worker offline)",
|
||||
timestamp=now,
|
||||
metadata_json={
|
||||
"shard_id": row.id,
|
||||
"worker_id": row.worker_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
# Reasignar pending de workers offline a online free
|
||||
if online_free:
|
||||
pending_orphan = session.scalars(
|
||||
select(DownloadJobRow).where(
|
||||
DownloadJobRow.status == JobStatus.PENDING.value,
|
||||
DownloadJobRow.worker_id.in_(list(offline_ids)),
|
||||
)
|
||||
).all()
|
||||
for row in pending_orphan:
|
||||
is_parent = (
|
||||
row.parent_download_job_id is None
|
||||
and row.shard_count is not None
|
||||
and int(row.shard_count) > 1
|
||||
)
|
||||
if is_parent:
|
||||
continue
|
||||
target = online_free[rr % len(online_free)]
|
||||
rr += 1
|
||||
if row.worker_id != target.id:
|
||||
row.worker_id = target.id
|
||||
reclaimed += 1
|
||||
|
||||
session.flush()
|
||||
|
||||
# Refrescar padres afectados
|
||||
# (se hace en siguiente patch; aquí no hace falta)
|
||||
|
||||
return reclaimed
|
||||
|
||||
def add_event(
|
||||
self,
|
||||
@@ -385,6 +728,28 @@ class JobStore:
|
||||
)
|
||||
session.add(ev)
|
||||
session.flush()
|
||||
# Mirror a padre si es shard
|
||||
row = session.get(DownloadJobRow, job_id)
|
||||
if row and row.parent_download_job_id:
|
||||
session.add(
|
||||
DownloadJobEventRow(
|
||||
download_job_id=row.parent_download_job_id,
|
||||
event_type=event_type,
|
||||
message=message,
|
||||
timestamp=ts,
|
||||
radicado=radicado,
|
||||
nit=nit,
|
||||
idx=index,
|
||||
total=total,
|
||||
metadata_json={
|
||||
**(metadata or {}),
|
||||
"shard_id": job_id,
|
||||
"shard_index": row.shard_index,
|
||||
"worker_id": row.worker_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
session.flush()
|
||||
return JobEventRecord(
|
||||
id=int(ev.id),
|
||||
job_id=job_id,
|
||||
@@ -429,9 +794,11 @@ class JobStore:
|
||||
*,
|
||||
parent_job_id: str,
|
||||
download_job_id: str,
|
||||
worker_id: str,
|
||||
worker_id: Optional[str],
|
||||
title: str,
|
||||
original_filename: str,
|
||||
worker_ids: Optional[List[str]] = None,
|
||||
mode: str = "single",
|
||||
) -> str:
|
||||
"""Crea Task(type=downloads) y la asocia al download_job. Devuelve task_id."""
|
||||
with session_scope() as session:
|
||||
@@ -443,10 +810,12 @@ class JobStore:
|
||||
title=title or f"Bot descarga ({original_filename})",
|
||||
request={
|
||||
"worker_id": worker_id,
|
||||
"worker_ids": worker_ids or ([worker_id] if worker_id else []),
|
||||
"original_filename": original_filename,
|
||||
"download_job_id": download_job_id,
|
||||
"mode": mode,
|
||||
},
|
||||
result={"download_job_id": download_job_id},
|
||||
result={"download_job_id": download_job_id, "mode": mode},
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
@@ -3,6 +3,7 @@ uvicorn[standard]>=0.27.0
|
||||
python-multipart>=0.0.9
|
||||
boto3>=1.34.0
|
||||
httpx>=0.27.0
|
||||
openpyxl>=3.1.0
|
||||
pydantic>=2.0.0
|
||||
sqlalchemy>=2.0.0
|
||||
psycopg[binary]>=3.1.0
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Tests de partición Excel para descarga distribuida."""
|
||||
from __future__ import annotations
|
||||
|
||||
from domain.excel_split import parse_radicado_rows, rows_to_xlsx_bytes, split_rows
|
||||
|
||||
|
||||
def test_split_rows_balanced():
|
||||
rows = [(str(i), "NIT") for i in range(100)]
|
||||
chunks = split_rows(rows, 2)
|
||||
assert len(chunks) == 2
|
||||
assert len(chunks[0]) == 50
|
||||
assert len(chunks[1]) == 50
|
||||
|
||||
|
||||
def test_split_rows_uneven():
|
||||
rows = [(str(i), "N") for i in range(10)]
|
||||
chunks = split_rows(rows, 3)
|
||||
assert len(chunks) == 3
|
||||
assert sum(len(c) for c in chunks) == 10
|
||||
assert [len(c) for c in chunks] == [4, 3, 3]
|
||||
|
||||
|
||||
def test_roundtrip_xlsx():
|
||||
rows = [("RAD-1", "900"), ("RAD-2", "901")]
|
||||
data = rows_to_xlsx_bytes(rows)
|
||||
parsed = parse_radicado_rows(data, "shard.xlsx")
|
||||
assert parsed == rows
|
||||
|
||||
|
||||
def test_split_more_workers_than_rows():
|
||||
rows = [("A", "1"), ("B", "2")]
|
||||
chunks = split_rows(rows, 5)
|
||||
assert len(chunks) == 2
|
||||
@@ -0,0 +1,28 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONPATH=/app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY domain /app/domain
|
||||
COPY infrastructure /app/infrastructure
|
||||
COPY api /app/api
|
||||
|
||||
ENV BOT_API_HOST=0.0.0.0
|
||||
ENV BOT_API_PORT=8210
|
||||
|
||||
EXPOSE 8210
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=5s --retries=5 --start-period=15s \
|
||||
CMD curl -f http://127.0.0.1:8210/health || exit 1
|
||||
|
||||
CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8210"]
|
||||
@@ -0,0 +1,600 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import Depends, FastAPI, File, Form, HTTPException, Query, UploadFile
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from domain.models import JobStatus
|
||||
from infrastructure.auth import require_bearer
|
||||
from infrastructure.storage_minio import MinioSettings, MinioStorage
|
||||
from infrastructure.store import JobStore
|
||||
|
||||
store = JobStore()
|
||||
minio = MinioStorage(MinioSettings.from_env())
|
||||
|
||||
POBLACION_APTA_URL = os.getenv("POBLACION_APTA_URL", "http://poblacion_apta:8003").rstrip("/")
|
||||
|
||||
app = FastAPI(title="BotDigitador Remote API", version="1.0.0")
|
||||
|
||||
|
||||
class RegisterWorkerBody(BaseModel):
|
||||
worker_id: str
|
||||
hostname: str = ""
|
||||
max_concurrent: int = 1
|
||||
|
||||
|
||||
class HeartbeatBody(BaseModel):
|
||||
worker_id: str
|
||||
|
||||
|
||||
class EventBody(BaseModel):
|
||||
event_type: str
|
||||
message: str
|
||||
radicado: Optional[str] = None
|
||||
nit: Optional[str] = None
|
||||
index: Optional[int] = None
|
||||
total: Optional[int] = None
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
timestamp: Optional[str] = None
|
||||
|
||||
|
||||
class PatchJobBody(BaseModel):
|
||||
status: Optional[str] = None
|
||||
result: Optional[Dict[str, Any]] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
def _media_type_for_key(key: str) -> str:
|
||||
lower = key.lower()
|
||||
if lower.endswith(".csv"):
|
||||
return "text/csv; charset=utf-8"
|
||||
if lower.endswith(".log") or lower.endswith(".txt"):
|
||||
return "text/plain; charset=utf-8"
|
||||
if lower.endswith(".json"):
|
||||
return "application/json"
|
||||
return "application/octet-stream"
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict:
|
||||
return {"status": "ok", "service": "bot-digitador"}
|
||||
|
||||
|
||||
@app.post("/workers/register")
|
||||
def register_worker(body: RegisterWorkerBody, _: str = Depends(require_bearer)) -> dict:
|
||||
worker = store.register_worker(
|
||||
body.worker_id,
|
||||
hostname=body.hostname,
|
||||
max_concurrent=body.max_concurrent,
|
||||
)
|
||||
return worker.to_dict()
|
||||
|
||||
|
||||
@app.post("/workers/heartbeat")
|
||||
def heartbeat(body: HeartbeatBody, _: str = Depends(require_bearer)) -> dict:
|
||||
worker = store.heartbeat(body.worker_id)
|
||||
if not worker:
|
||||
raise HTTPException(status_code=404, detail="Worker no registrado")
|
||||
return worker.to_dict()
|
||||
|
||||
|
||||
@app.get("/workers")
|
||||
def list_workers(_: str = Depends(require_bearer)) -> List[dict]:
|
||||
return [w.to_dict() for w in store.list_workers()]
|
||||
|
||||
|
||||
@app.get("/workers/me/pending")
|
||||
def claim_pending(
|
||||
worker_id: str = Query(...),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
job = store.claim_pending(worker_id)
|
||||
if not job:
|
||||
return {"job": None}
|
||||
return {"job": job.to_dict()}
|
||||
|
||||
|
||||
@app.post("/jobs")
|
||||
async def create_job(
|
||||
job_id: str = Form(...),
|
||||
worker_id: str = Form(""),
|
||||
worker_ids: str = Form(""),
|
||||
title: str = Form(""),
|
||||
file: Optional[UploadFile] = File(None),
|
||||
poblacion_apta_job_id: str = Form(""),
|
||||
subset: str = Form("aptos"),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
"""Crea una corrida de digitación.
|
||||
|
||||
Exactamente uno de: file (CSV) o poblacion_apta_job_id.
|
||||
- 1 worker → flujo clásico.
|
||||
- N workers → padre + N shards CSV, mismo reports_prefix del padre.
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
import httpx
|
||||
|
||||
from domain.csv_split import parse_digitador_rows, rows_to_csv_bytes, split_rows
|
||||
from infrastructure.storage_minio import reports_prefix as mk_prefix
|
||||
|
||||
parent_job_id = job_id.strip()
|
||||
if not parent_job_id:
|
||||
raise HTTPException(status_code=400, detail="job_id (Job Desktop) obligatorio")
|
||||
if not store.parent_job_exists(parent_job_id):
|
||||
raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado")
|
||||
|
||||
pa_id = (poblacion_apta_job_id or "").strip()
|
||||
data: bytes = b""
|
||||
original = "input.csv"
|
||||
content_type = "text/csv"
|
||||
source_type = "upload"
|
||||
|
||||
file_bytes: bytes = b""
|
||||
if file is not None:
|
||||
file_bytes = await file.read()
|
||||
|
||||
if pa_id and file_bytes:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Proporcione exactamente uno de: file o poblacion_apta_job_id",
|
||||
)
|
||||
|
||||
if pa_id:
|
||||
source_type = "poblacion_apta"
|
||||
subset_norm = (subset or "aptos").strip().lower()
|
||||
if subset_norm != "aptos":
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Solo se digitan aptos (PASO_BOT). Use subset=aptos.",
|
||||
)
|
||||
url = f"{POBLACION_APTA_URL}/jobs/{pa_id}/export.csv"
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
resp = await client.get(url, params={"subset": subset_norm})
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"No se pudo contactar poblacion_apta: {e}",
|
||||
) from e
|
||||
if resp.status_code == 404:
|
||||
raise HTTPException(status_code=404, detail="Job de población apta no encontrado")
|
||||
if not resp.is_success:
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"Error exportando CSV de población apta: HTTP {resp.status_code}",
|
||||
)
|
||||
data = resp.content
|
||||
original = f"poblacion_apta_{subset_norm}_{pa_id[:8]}.csv"
|
||||
content_type = "text/csv; charset=utf-8"
|
||||
elif file_bytes:
|
||||
data = file_bytes
|
||||
original = (file.filename if file else None) or "input.csv"
|
||||
content_type = (file.content_type if file else None) or "text/csv"
|
||||
source_type = "upload"
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Proporcione exactamente uno de: file o poblacion_apta_job_id",
|
||||
)
|
||||
|
||||
if not data:
|
||||
raise HTTPException(status_code=400, detail="CSV vacío")
|
||||
|
||||
title_clean = title.strip()
|
||||
|
||||
# Resolver workers
|
||||
selected: List[str] = []
|
||||
raw_ids = (worker_ids or "").strip()
|
||||
if raw_ids:
|
||||
try:
|
||||
parsed = _json.loads(raw_ids)
|
||||
if isinstance(parsed, list):
|
||||
selected = [str(x).strip() for x in parsed if str(x).strip()]
|
||||
elif isinstance(parsed, str) and parsed.strip():
|
||||
selected = [parsed.strip()]
|
||||
except _json.JSONDecodeError:
|
||||
selected = [x.strip() for x in raw_ids.split(",") if x.strip()]
|
||||
if not selected and worker_id.strip():
|
||||
selected = [worker_id.strip()]
|
||||
|
||||
online = store.list_online_worker_ids()
|
||||
online_set = set(online)
|
||||
if not selected:
|
||||
selected = list(online)
|
||||
else:
|
||||
missing = [w for w in selected if w not in online_set]
|
||||
if missing:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"Workers no online: {', '.join(missing)}. Enciende el agent primero.",
|
||||
)
|
||||
|
||||
if not selected:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="No hay bots online. Enciende al menos un agent.",
|
||||
)
|
||||
|
||||
seen: set[str] = set()
|
||||
workers_unique: List[str] = []
|
||||
for w in selected:
|
||||
if w not in seen:
|
||||
seen.add(w)
|
||||
workers_unique.append(w)
|
||||
|
||||
try:
|
||||
rows = parse_digitador_rows(data, original)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=f"No se pudo leer el CSV: {e}") from e
|
||||
if not rows:
|
||||
raise HTTPException(status_code=400, detail="El CSV no tiene filas para digitar")
|
||||
|
||||
# Normalizar a CSV canónico del bot (siempre pipe + headers snake_case)
|
||||
canonical_bytes = rows_to_csv_bytes(rows)
|
||||
|
||||
n_workers = min(len(workers_unique), len(rows))
|
||||
if n_workers <= 1:
|
||||
only = workers_unique[0]
|
||||
job = store.create_job(
|
||||
worker_id=only,
|
||||
csv_key="",
|
||||
original_filename=original,
|
||||
title=title_clean,
|
||||
job_id=parent_job_id,
|
||||
result={"source_type": source_type, "total_rows": len(rows), "total": len(rows)},
|
||||
)
|
||||
task_id = store.create_digitador_task(
|
||||
parent_job_id=parent_job_id,
|
||||
digitador_job_id=job.id,
|
||||
worker_id=only,
|
||||
title=title_clean or f"Bot digitador ({original})",
|
||||
original_filename=original,
|
||||
worker_ids=[only],
|
||||
mode="single",
|
||||
source_type=source_type,
|
||||
)
|
||||
try:
|
||||
key = minio.upload_csv(job.id, canonical_bytes, original, content_type=content_type)
|
||||
except Exception as e:
|
||||
store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
|
||||
|
||||
job = store.update_csv_key(job.id, key)
|
||||
assert job is not None
|
||||
store.add_event(
|
||||
job.id,
|
||||
"run_started",
|
||||
f"Job creado y asignado a {only}",
|
||||
metadata={
|
||||
"csv_key": key,
|
||||
"worker_id": only,
|
||||
"job_id": parent_job_id,
|
||||
"task_id": task_id,
|
||||
"mode": "single",
|
||||
"total_rows": len(rows),
|
||||
"source_type": source_type,
|
||||
},
|
||||
)
|
||||
return job.to_dict()
|
||||
|
||||
workers_unique = workers_unique[:n_workers]
|
||||
chunks = split_rows(rows, n_workers)
|
||||
n = len(chunks)
|
||||
workers_unique = workers_unique[:n]
|
||||
|
||||
parent = store.create_job(
|
||||
worker_id=None,
|
||||
csv_key="",
|
||||
original_filename=original,
|
||||
title=title_clean or f"Digitación distribuida ({original})",
|
||||
job_id=parent_job_id,
|
||||
shard_count=n,
|
||||
status=JobStatus.PENDING.value,
|
||||
result={
|
||||
"mode": "distributed",
|
||||
"shard_ids": [],
|
||||
"total_rows": len(rows),
|
||||
"total": len(rows),
|
||||
"processed": 0,
|
||||
"ok": 0,
|
||||
"error": 0,
|
||||
"timeout": 0,
|
||||
"existentes": 0,
|
||||
"source_type": source_type,
|
||||
},
|
||||
)
|
||||
prefix = parent.reports_prefix or mk_prefix(parent.id)
|
||||
|
||||
task_id = store.create_digitador_task(
|
||||
parent_job_id=parent_job_id,
|
||||
digitador_job_id=parent.id,
|
||||
worker_id=None,
|
||||
title=title_clean or f"Bot digitador ×{n} ({original})",
|
||||
original_filename=original,
|
||||
worker_ids=workers_unique,
|
||||
mode="distributed",
|
||||
source_type=source_type,
|
||||
)
|
||||
|
||||
try:
|
||||
full_key = minio.upload_csv(parent.id, canonical_bytes, original, content_type=content_type)
|
||||
except Exception as e:
|
||||
store.patch_job(parent.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
|
||||
|
||||
store.update_csv_key(parent.id, full_key)
|
||||
|
||||
shard_ids: List[str] = []
|
||||
for i, (chunk, wid) in enumerate(zip(chunks, workers_unique)):
|
||||
shard_bytes = rows_to_csv_bytes(chunk)
|
||||
shard = store.create_job(
|
||||
worker_id=wid,
|
||||
csv_key="",
|
||||
original_filename=f"shard_{i + 1}_of_{n}.csv",
|
||||
title=f"{title_clean or 'Digitador'} · shard {i + 1}/{n}",
|
||||
job_id=parent_job_id,
|
||||
task_id=task_id,
|
||||
reports_prefix_override=prefix,
|
||||
parent_digitador_job_id=parent.id,
|
||||
shard_index=i,
|
||||
shard_count=n,
|
||||
result={"total": len(chunk), "processed": 0, "source_type": source_type},
|
||||
)
|
||||
try:
|
||||
shard_key = minio.upload_csv(
|
||||
shard.id,
|
||||
shard_bytes,
|
||||
f"shard_{i + 1}_of_{n}.csv",
|
||||
content_type="text/csv; charset=utf-8",
|
||||
)
|
||||
except Exception as e:
|
||||
store.patch_job(
|
||||
parent.id,
|
||||
status=JobStatus.FAILED.value,
|
||||
error=f"MinIO shard upload failed: {e}",
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=502, detail=f"Error subiendo shard a MinIO: {e}"
|
||||
) from e
|
||||
store.update_csv_key(shard.id, shard_key)
|
||||
shard_ids.append(shard.id)
|
||||
store.add_event(
|
||||
parent.id,
|
||||
"info",
|
||||
f"Shard {i + 1}/{n} → {wid} ({len(chunk)} filas)",
|
||||
metadata={"shard_id": shard.id, "worker_id": wid, "rows": len(chunk)},
|
||||
)
|
||||
|
||||
store.patch_job(
|
||||
parent.id,
|
||||
result={
|
||||
"mode": "distributed",
|
||||
"shard_ids": shard_ids,
|
||||
"total_rows": len(rows),
|
||||
"total": len(rows),
|
||||
"processed": 0,
|
||||
"ok": 0,
|
||||
"error": 0,
|
||||
"timeout": 0,
|
||||
"existentes": 0,
|
||||
"reports_prefix": prefix,
|
||||
"worker_ids": workers_unique,
|
||||
"source_type": source_type,
|
||||
},
|
||||
)
|
||||
store.add_event(
|
||||
parent.id,
|
||||
"run_started",
|
||||
f"Corrida distribuida: {n} bots, {len(rows)} filas",
|
||||
metadata={
|
||||
"csv_key": full_key,
|
||||
"worker_ids": workers_unique,
|
||||
"job_id": parent_job_id,
|
||||
"task_id": task_id,
|
||||
"mode": "distributed",
|
||||
"shard_ids": shard_ids,
|
||||
"total_rows": len(rows),
|
||||
"source_type": source_type,
|
||||
},
|
||||
)
|
||||
job = store.get_job(parent.id)
|
||||
assert job is not None
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.post("/internal/reclaim")
|
||||
def reclaim_jobs(_: str = Depends(require_bearer)) -> dict:
|
||||
n = store.reclaim_stale_running_jobs()
|
||||
return {"reclaimed": n}
|
||||
|
||||
|
||||
@app.get("/jobs")
|
||||
def list_jobs(
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
job_id: Optional[str] = Query(None, description="Filtrar por Job Desktop"),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> List[dict]:
|
||||
return [j.to_dict() for j in store.list_jobs(limit=limit, job_id=job_id)]
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}")
|
||||
def get_job(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
if job.is_distributed_parent:
|
||||
refreshed = store.refresh_parent_from_shards(digitador_job_id)
|
||||
if refreshed:
|
||||
job = refreshed
|
||||
payload = job.to_dict()
|
||||
if job.is_distributed_parent:
|
||||
payload["shards"] = [s.to_dict() for s in store.list_shards(job.id)]
|
||||
return payload
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}/files")
|
||||
def list_job_files(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
try:
|
||||
return minio.list_report_files(digitador_job_id, prefix=job.reports_prefix)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=f"Error listando MinIO: {e}") from e
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}/object")
|
||||
def get_job_object(
|
||||
digitador_job_id: str,
|
||||
key: str = Query(..., min_length=1, description="Object key bajo reports_prefix"),
|
||||
_: str = Depends(require_bearer),
|
||||
):
|
||||
"""Stream de un reporte MinIO (csv/log/json bajo el prefijo del job)."""
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
|
||||
prefix = (job.reports_prefix or f"digitados/{digitador_job_id}/").lstrip("/")
|
||||
if not prefix.endswith("/"):
|
||||
prefix += "/"
|
||||
normalized = key.lstrip("/")
|
||||
if not normalized.startswith(prefix):
|
||||
raise HTTPException(status_code=403, detail="Key fuera del prefijo del job")
|
||||
lower = normalized.lower()
|
||||
if not (lower.endswith(".csv") or lower.endswith(".log") or lower.endswith(".json")):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten objetos csv/log/json")
|
||||
if ".." in normalized.split("/"):
|
||||
raise HTTPException(status_code=400, detail="Key no permitida")
|
||||
|
||||
try:
|
||||
data = minio.get_bytes(normalized)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=f"Error leyendo MinIO: {e}") from e
|
||||
|
||||
filename = Path(normalized).name
|
||||
return Response(
|
||||
content=data,
|
||||
media_type=_media_type_for_key(normalized),
|
||||
headers={"Content-Disposition": f'inline; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}/events")
|
||||
def get_events(
|
||||
digitador_job_id: str,
|
||||
since: int = Query(0, ge=0),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
if not store.get_job(digitador_job_id):
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
events = store.list_events(digitador_job_id, since_id=since)
|
||||
return {"events": [e.to_dict() for e in events]}
|
||||
|
||||
|
||||
@app.post("/jobs/{digitador_job_id}/cancel")
|
||||
def cancel_job(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.request_cancel(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
store.add_event(digitador_job_id, "warning", "Cancelación solicitada por el usuario")
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}/file")
|
||||
def get_job_file(digitador_job_id: str, _: str = Depends(require_bearer)):
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
if not job.csv_key:
|
||||
raise HTTPException(status_code=404, detail="CSV no disponible")
|
||||
|
||||
try:
|
||||
data = minio.get_bytes(job.csv_key)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=f"Error leyendo MinIO: {e}") from e
|
||||
|
||||
filename = Path(job.csv_key).name
|
||||
return Response(
|
||||
content=data,
|
||||
media_type="text/csv; charset=utf-8",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}/file-url")
|
||||
def get_job_file_url(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job or not job.csv_key:
|
||||
raise HTTPException(status_code=404, detail="Job/archivo no encontrado")
|
||||
try:
|
||||
url = minio.presigned_get(job.csv_key)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=str(e)) from e
|
||||
return {"url": url, "csv_key": job.csv_key}
|
||||
|
||||
|
||||
@app.post("/jobs/{digitador_job_id}/events")
|
||||
def post_event(
|
||||
digitador_job_id: str,
|
||||
body: EventBody,
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
event = store.add_event(
|
||||
digitador_job_id,
|
||||
body.event_type,
|
||||
body.message,
|
||||
radicado=body.radicado,
|
||||
nit=body.nit,
|
||||
index=body.index,
|
||||
total=body.total,
|
||||
metadata=body.metadata,
|
||||
timestamp=body.timestamp,
|
||||
)
|
||||
if body.index is not None and body.total is not None:
|
||||
result = dict(job.result or {})
|
||||
result["processed"] = body.index
|
||||
result["total"] = body.total
|
||||
if body.nit:
|
||||
result["nit_actual"] = body.nit
|
||||
store.patch_job(digitador_job_id, result=result)
|
||||
return event.to_dict()
|
||||
|
||||
|
||||
@app.patch("/jobs/{digitador_job_id}")
|
||||
def patch_job(
|
||||
digitador_job_id: str,
|
||||
body: PatchJobBody,
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
if body.status and body.status not in {s.value for s in JobStatus}:
|
||||
raise HTTPException(status_code=400, detail="status inválido")
|
||||
job = store.patch_job(
|
||||
digitador_job_id,
|
||||
status=body.status,
|
||||
result=body.result,
|
||||
error=body.error,
|
||||
)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.get("/jobs/{digitador_job_id}/cancel-flag")
|
||||
def cancel_flag(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(digitador_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
return {
|
||||
"cancel_requested": store.is_cancel_requested(digitador_job_id),
|
||||
"status": job.status,
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Parseo y partición de CSV de filas a digitar (bot digitador)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
from typing import Any
|
||||
|
||||
REQUIRED_BOT_COLUMNS = (
|
||||
"radicado",
|
||||
"documento",
|
||||
"ingreso",
|
||||
"total_cargos",
|
||||
"autorizacion",
|
||||
"tipo_documento",
|
||||
"tipo_servicio",
|
||||
"copago",
|
||||
"cuota_moderadora",
|
||||
)
|
||||
|
||||
IT_TO_BOT = {
|
||||
"RADICADO": "radicado",
|
||||
"NUMERO": "documento",
|
||||
"FECHA": "ingreso",
|
||||
"BRUTO": "total_cargos",
|
||||
"AUTORIZACION": "autorizacion",
|
||||
"TIPO_ID": "tipo_documento",
|
||||
"SERVICIO": "tipo_servicio",
|
||||
"COPAGO": "copago",
|
||||
"MODERADORA": "cuota_moderadora",
|
||||
}
|
||||
|
||||
|
||||
def _normalize_header(name: str) -> str:
|
||||
raw = str(name or "").strip()
|
||||
if not raw:
|
||||
return ""
|
||||
upper = raw.upper()
|
||||
if upper in IT_TO_BOT:
|
||||
return IT_TO_BOT[upper]
|
||||
lower = raw.lower().strip()
|
||||
# Aceptar snake_case ya normalizado
|
||||
if lower in REQUIRED_BOT_COLUMNS:
|
||||
return lower
|
||||
# Alias comunes
|
||||
aliases = {
|
||||
"número": "documento",
|
||||
"numero": "documento",
|
||||
"fecha": "ingreso",
|
||||
"bruto": "total_cargos",
|
||||
"tipo_id": "tipo_documento",
|
||||
"servicio": "tipo_servicio",
|
||||
"moderadora": "cuota_moderadora",
|
||||
}
|
||||
return aliases.get(lower, lower)
|
||||
|
||||
|
||||
def _decode_text(data: bytes) -> str:
|
||||
for enc in ("utf-8-sig", "utf-8", "latin-1", "cp1252"):
|
||||
try:
|
||||
return data.decode(enc)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
return data.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _detect_delimiter(sample: str) -> str:
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters="|,;\t")
|
||||
return dialect.delimiter
|
||||
except csv.Error:
|
||||
return "|" if sample.count("|") >= sample.count(",") else ","
|
||||
|
||||
|
||||
def _validate_columns(headers: list[str]) -> None:
|
||||
present = set(headers)
|
||||
missing = [c for c in REQUIRED_BOT_COLUMNS if c not in present]
|
||||
if missing:
|
||||
raise ValueError(f"Columnas faltantes: {', '.join(missing)}")
|
||||
|
||||
|
||||
def parse_digitador_rows(data: bytes, filename: str = "input.csv") -> list[dict[str, str]]:
|
||||
"""Devuelve filas normalizadas con headers del bot (snake_case)."""
|
||||
_ = filename # reservado por firma / diagnóstico futuro
|
||||
text = _decode_text(data)
|
||||
if not text.strip():
|
||||
return []
|
||||
|
||||
delim = _detect_delimiter(text[:4096])
|
||||
reader = csv.DictReader(io.StringIO(text), delimiter=delim)
|
||||
if not reader.fieldnames:
|
||||
raise ValueError("CSV sin encabezados")
|
||||
|
||||
rename_map: dict[str, str] = {}
|
||||
normalized_headers: list[str] = []
|
||||
for raw in reader.fieldnames:
|
||||
norm = _normalize_header(raw)
|
||||
rename_map[raw] = norm
|
||||
if norm and norm not in normalized_headers:
|
||||
normalized_headers.append(norm)
|
||||
|
||||
_validate_columns(normalized_headers)
|
||||
|
||||
out: list[dict[str, str]] = []
|
||||
for row in reader:
|
||||
if not row:
|
||||
continue
|
||||
mapped: dict[str, str] = {c: "" for c in REQUIRED_BOT_COLUMNS}
|
||||
empty = True
|
||||
for raw_key, value in row.items():
|
||||
if raw_key is None:
|
||||
continue
|
||||
col = rename_map.get(raw_key) or _normalize_header(raw_key)
|
||||
if col not in REQUIRED_BOT_COLUMNS:
|
||||
continue
|
||||
cell = "" if value is None else str(value).strip()
|
||||
mapped[col] = cell
|
||||
if cell:
|
||||
empty = False
|
||||
if empty:
|
||||
continue
|
||||
# Filtrar filas que parecen ser un segundo header
|
||||
if mapped.get("radicado", "").lower() in ("radicado", "radicado"):
|
||||
if mapped.get("documento", "").lower() in ("documento", "numero", "número", ""):
|
||||
continue
|
||||
out.append(mapped)
|
||||
return out
|
||||
|
||||
|
||||
def split_rows(rows: list[dict[str, Any]], n: int) -> list[list[dict[str, Any]]]:
|
||||
"""Parte filas en n trozos lo más equilibrados posible (chunks contiguos)."""
|
||||
if n <= 1 or not rows:
|
||||
return [rows]
|
||||
n = min(n, len(rows))
|
||||
size, rem = divmod(len(rows), n)
|
||||
chunks: list[list[dict[str, Any]]] = []
|
||||
idx = 0
|
||||
for i in range(n):
|
||||
take = size + (1 if i < rem else 0)
|
||||
chunks.append(rows[idx : idx + take])
|
||||
idx += take
|
||||
return [c for c in chunks if c]
|
||||
|
||||
|
||||
def rows_to_csv_bytes(rows: list[dict[str, Any]]) -> bytes:
|
||||
"""Serializa filas a CSV pipe-delimited con headers del bot (utf-8-sig)."""
|
||||
buf = io.StringIO()
|
||||
writer = csv.DictWriter(
|
||||
buf,
|
||||
fieldnames=list(REQUIRED_BOT_COLUMNS),
|
||||
delimiter="|",
|
||||
lineterminator="\n",
|
||||
extrasaction="ignore",
|
||||
)
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
writer.writerow({c: str(row.get(c, "") or "") for c in REQUIRED_BOT_COLUMNS})
|
||||
return buf.getvalue().encode("utf-8-sig")
|
||||
@@ -0,0 +1,91 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
class WorkerStatus(str, Enum):
|
||||
ONLINE = "online"
|
||||
OFFLINE = "offline"
|
||||
BUSY = "busy"
|
||||
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Worker:
|
||||
id: str
|
||||
hostname: str = ""
|
||||
max_concurrent: int = 1
|
||||
status: str = WorkerStatus.ONLINE.value
|
||||
last_heartbeat: str = field(default_factory=utc_now)
|
||||
current_job_id: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Job:
|
||||
id: str
|
||||
worker_id: Optional[str]
|
||||
status: str = JobStatus.PENDING.value
|
||||
title: str = ""
|
||||
csv_key: str = ""
|
||||
reports_prefix: str = ""
|
||||
original_filename: str = ""
|
||||
cancel_requested: bool = False
|
||||
result: Dict[str, Any] = field(default_factory=dict)
|
||||
error: Optional[str] = None
|
||||
created_at: str = field(default_factory=utc_now)
|
||||
started_at: Optional[str] = None
|
||||
finished_at: Optional[str] = None
|
||||
job_id: Optional[str] = None
|
||||
task_id: Optional[str] = None
|
||||
parent_digitador_job_id: Optional[str] = None
|
||||
shard_index: Optional[int] = None
|
||||
shard_count: Optional[int] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
@property
|
||||
def is_distributed_parent(self) -> bool:
|
||||
return (
|
||||
self.parent_digitador_job_id is None
|
||||
and self.shard_count is not None
|
||||
and int(self.shard_count) > 1
|
||||
)
|
||||
|
||||
@property
|
||||
def is_shard(self) -> bool:
|
||||
return bool(self.parent_digitador_job_id)
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobEventRecord:
|
||||
id: int
|
||||
job_id: str
|
||||
event_type: str
|
||||
message: str
|
||||
timestamp: str
|
||||
radicado: Optional[str] = None
|
||||
nit: Optional[str] = None
|
||||
index: Optional[int] = None
|
||||
total: Optional[int] = None
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Header, HTTPException
|
||||
|
||||
|
||||
def get_api_token() -> str:
|
||||
return os.getenv("BOT_API_TOKEN", "dev-token-change-me")
|
||||
|
||||
|
||||
def require_bearer(authorization: Optional[str] = Header(default=None)) -> str:
|
||||
expected = get_api_token()
|
||||
if not authorization or not authorization.lower().startswith("bearer "):
|
||||
raise HTTPException(status_code=401, detail="Authorization Bearer requerido")
|
||||
token = authorization.split(" ", 1)[1].strip()
|
||||
if token != expected:
|
||||
raise HTTPException(status_code=401, detail="Token inválido")
|
||||
return token
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Persistencia Postgres del servicio bot-digitador (DB compartida IPSExtractor)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Iterator
|
||||
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
Boolean,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
create_engine,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, sessionmaker
|
||||
|
||||
DATABASE_URL = os.getenv(
|
||||
"DATABASE_URL",
|
||||
"postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor",
|
||||
)
|
||||
|
||||
engine = create_engine(DATABASE_URL, pool_pre_ping=True)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def new_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def _iso(dt: datetime | None) -> str | None:
|
||||
if dt is None:
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
class Job(Base):
|
||||
"""Contenedor Desktop (lectura / FK)."""
|
||||
|
||||
__tablename__ = "jobs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
job_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
|
||||
class Task(Base):
|
||||
__tablename__ = "tasks"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
||||
job_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("jobs.id"), nullable=True, index=True
|
||||
)
|
||||
type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
owner_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class DigitadorWorkerRow(Base):
|
||||
__tablename__ = "digitador_workers"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
hostname: Mapped[str] = mapped_column(Text, default="")
|
||||
max_concurrent: Mapped[int] = mapped_column(Integer, default=1)
|
||||
status: Mapped[str] = mapped_column(String(32), index=True)
|
||||
last_heartbeat: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
current_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
|
||||
|
||||
class DigitadorJobRow(Base):
|
||||
__tablename__ = "digitador_jobs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
||||
job_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("jobs.id"), nullable=True, index=True
|
||||
)
|
||||
task_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("tasks.id"), nullable=True, index=True
|
||||
)
|
||||
worker_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
parent_digitador_job_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("digitador_jobs.id"), nullable=True, index=True
|
||||
)
|
||||
shard_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
shard_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
csv_key: Mapped[str] = mapped_column(Text, default="")
|
||||
reports_prefix: Mapped[str] = mapped_column(Text, default="")
|
||||
original_filename: Mapped[str] = mapped_column(Text, default="")
|
||||
cancel_requested: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class DigitadorJobEventRow(Base):
|
||||
__tablename__ = "digitador_job_events"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
digitador_job_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("digitador_jobs.id"), index=True
|
||||
)
|
||||
event_type: Mapped[str] = mapped_column(String(64))
|
||||
message: Mapped[str] = mapped_column(Text)
|
||||
timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
radicado: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
nit: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
idx: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
total: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
metadata_json: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
if DATABASE_URL.startswith("postgresql"):
|
||||
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:
|
||||
connection.execute(text("SELECT pg_advisory_lock(74512005)"))
|
||||
try:
|
||||
Base.metadata.create_all(bind=connection)
|
||||
finally:
|
||||
connection.execute(text("SELECT pg_advisory_unlock(74512005)"))
|
||||
return
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def session_scope() -> Iterator[Session]:
|
||||
session = SessionLocal()
|
||||
try:
|
||||
yield session
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
@@ -0,0 +1,230 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO, List, Optional
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MinioSettings:
|
||||
endpoint: str = "https://s3.builker.xyz"
|
||||
bucket: str = "pys-cuentas-medicas"
|
||||
access_key: str = ""
|
||||
secret_key: str = ""
|
||||
region: str = "us-east-1"
|
||||
anonymous: bool = True
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "MinioSettings":
|
||||
access_key = (os.getenv("MINIO_ACCESS_KEY") or "").strip()
|
||||
secret_key = (os.getenv("MINIO_SECRET_KEY") or "").strip()
|
||||
anon_flag = (os.getenv("MINIO_ANONYMOUS") or "").strip().lower()
|
||||
if anon_flag in ("0", "false", "no"):
|
||||
anonymous = False
|
||||
elif anon_flag in ("1", "true", "yes"):
|
||||
anonymous = True
|
||||
else:
|
||||
anonymous = not (access_key and secret_key)
|
||||
return cls(
|
||||
endpoint=os.getenv("MINIO_ENDPOINT", "https://s3.builker.xyz").rstrip("/"),
|
||||
bucket=os.getenv("MINIO_BUCKET", "pys-cuentas-medicas"),
|
||||
access_key=access_key,
|
||||
secret_key=secret_key,
|
||||
region=os.getenv("MINIO_REGION", "us-east-1"),
|
||||
anonymous=anonymous,
|
||||
)
|
||||
|
||||
|
||||
def csv_key(job_id: str, filename: str = "input.csv") -> str:
|
||||
return f"uploads/{job_id}/{filename}"
|
||||
|
||||
|
||||
def reports_prefix(job_id: str) -> str:
|
||||
return f"digitados/{job_id}/"
|
||||
|
||||
|
||||
def meta_key(job_id: str) -> str:
|
||||
return f"uploads/{job_id}/meta.json"
|
||||
|
||||
|
||||
def manifest_key(job_id: str) -> str:
|
||||
return f"digitados/{job_id}/_manifest.json"
|
||||
|
||||
|
||||
_REPORT_NAMES = {
|
||||
"resultados.csv",
|
||||
"omitidos.csv",
|
||||
"ejecucion_bot.log",
|
||||
}
|
||||
|
||||
|
||||
def _s3_client(settings: MinioSettings):
|
||||
import boto3
|
||||
from botocore import UNSIGNED
|
||||
from botocore.client import Config
|
||||
|
||||
s3_cfg = {
|
||||
"addressing_style": "path",
|
||||
"request_checksum_calculation": "when_required",
|
||||
"response_checksum_validation": "when_required",
|
||||
}
|
||||
if settings.anonymous:
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.endpoint,
|
||||
region_name=settings.region,
|
||||
config=Config(signature_version=UNSIGNED, s3=s3_cfg),
|
||||
)
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.endpoint,
|
||||
aws_access_key_id=settings.access_key,
|
||||
aws_secret_access_key=settings.secret_key,
|
||||
region_name=settings.region,
|
||||
config=Config(signature_version="s3v4", s3=s3_cfg),
|
||||
)
|
||||
|
||||
|
||||
class MinioStorage:
|
||||
"""Cliente S3-compatible para MinIO (uploads/digitados por job_id)."""
|
||||
|
||||
def __init__(self, settings: Optional[MinioSettings] = None):
|
||||
self.settings = settings or MinioSettings.from_env()
|
||||
self._client = None
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
if self._client is None:
|
||||
self._client = _s3_client(self.settings)
|
||||
return self._client
|
||||
|
||||
@property
|
||||
def bucket(self) -> str:
|
||||
return self.settings.bucket
|
||||
|
||||
def put_bytes(self, key: str, data: bytes, content_type: str = "application/octet-stream") -> str:
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=data,
|
||||
ContentType=content_type,
|
||||
)
|
||||
return key
|
||||
|
||||
def put_file(self, key: str, path: str | Path, content_type: str = "application/octet-stream") -> str:
|
||||
path = Path(path)
|
||||
return self.put_bytes(key, path.read_bytes(), content_type=content_type)
|
||||
|
||||
def put_stream(self, key: str, stream: BinaryIO, content_type: str = "application/octet-stream") -> str:
|
||||
data = stream.read()
|
||||
return self.put_bytes(key, data, content_type=content_type)
|
||||
|
||||
def get_bytes(self, key: str) -> bytes:
|
||||
obj = self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
return obj["Body"].read()
|
||||
|
||||
def download_file(self, key: str, dest: str | Path) -> Path:
|
||||
dest = Path(dest)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.client.download_file(self.bucket, key, str(dest))
|
||||
return dest
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
try:
|
||||
self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def list_keys(self, prefix: str) -> List[str]:
|
||||
keys: List[str] = []
|
||||
token = None
|
||||
while True:
|
||||
kwargs = {"Bucket": self.bucket, "Prefix": prefix}
|
||||
if token:
|
||||
kwargs["ContinuationToken"] = token
|
||||
resp = self.client.list_objects_v2(**kwargs)
|
||||
for item in resp.get("Contents") or []:
|
||||
keys.append(item["Key"])
|
||||
if not resp.get("IsTruncated"):
|
||||
break
|
||||
token = resp.get("NextContinuationToken")
|
||||
return keys
|
||||
|
||||
def presigned_get(self, key: str, expires_in: int = 3600) -> str:
|
||||
if self.settings.anonymous:
|
||||
return f"{self.settings.endpoint}/{self.bucket}/{key}"
|
||||
return self.client.generate_presigned_url(
|
||||
"get_object",
|
||||
Params={"Bucket": self.bucket, "Key": key},
|
||||
ExpiresIn=expires_in,
|
||||
)
|
||||
|
||||
def upload_csv(
|
||||
self,
|
||||
job_id: str,
|
||||
data: bytes,
|
||||
original_filename: str,
|
||||
content_type: str = "text/csv",
|
||||
) -> str:
|
||||
ext = Path(original_filename).suffix.lower() or ".csv"
|
||||
canonical = "input.csv" if ext in (".csv", ".txt") else "input.csv"
|
||||
key = csv_key(job_id, canonical)
|
||||
self.put_bytes(key, data, content_type=content_type)
|
||||
meta = {
|
||||
"original_filename": original_filename,
|
||||
"size": len(data),
|
||||
"canonical": canonical,
|
||||
}
|
||||
self.put_bytes(meta_key(job_id), json.dumps(meta).encode("utf-8"), "application/json")
|
||||
return key
|
||||
|
||||
def write_manifest(self, job_id: str, keys: List[str]) -> str:
|
||||
key = manifest_key(job_id)
|
||||
payload = {"job_id": job_id, "files": keys, "count": len(keys)}
|
||||
self.put_bytes(key, json.dumps(payload, ensure_ascii=False).encode("utf-8"), "application/json")
|
||||
return key
|
||||
|
||||
def list_report_files(self, job_id: str, prefix: str | None = None) -> dict:
|
||||
"""Lista reportes de digitación (RESULTADOS/OMITIDOS/log + list_keys).
|
||||
|
||||
Fusiona manifest con listado del prefijo (cubriendo shards distribuidos).
|
||||
"""
|
||||
prefix = prefix or reports_prefix(job_id)
|
||||
by_key: dict[str, dict] = {}
|
||||
try:
|
||||
raw = self.get_bytes(manifest_key(job_id))
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
for key in payload.get("files") or []:
|
||||
entry = self._file_entry(str(key))
|
||||
if entry is not None:
|
||||
by_key[entry["key"]] = entry
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for entry in self._list_report_entries(prefix):
|
||||
by_key[entry["key"]] = entry
|
||||
|
||||
files = list(by_key.values())
|
||||
return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)}
|
||||
|
||||
def _list_report_entries(self, prefix: str) -> List[dict]:
|
||||
files: List[dict] = []
|
||||
for key in self.list_keys(prefix):
|
||||
entry = self._file_entry(key)
|
||||
if entry is not None:
|
||||
files.append(entry)
|
||||
return files
|
||||
|
||||
@staticmethod
|
||||
def _file_entry(key: str) -> dict | None:
|
||||
name = Path(key).name
|
||||
lower = name.lower()
|
||||
if lower.startswith("_manifest") and lower.endswith(".json"):
|
||||
return None
|
||||
# RESULTADOS.csv, OMITIDOS.csv, ejecucion_bot.log (+ variantes _shard_N)
|
||||
if lower in _REPORT_NAMES or lower.endswith(".csv") or lower.endswith(".log"):
|
||||
return {"key": key, "name": name}
|
||||
return None
|
||||
@@ -0,0 +1,833 @@
|
||||
"""Repositorio sobre PostgreSQL (tablas digitador_* + sync de tasks)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from domain.models import (
|
||||
Job,
|
||||
JobEventRecord,
|
||||
JobStatus,
|
||||
Worker,
|
||||
WorkerStatus,
|
||||
utc_now,
|
||||
)
|
||||
from infrastructure.database import (
|
||||
DigitadorJobEventRow,
|
||||
DigitadorJobRow,
|
||||
DigitadorWorkerRow,
|
||||
Task,
|
||||
_iso,
|
||||
init_db,
|
||||
session_scope,
|
||||
utcnow,
|
||||
)
|
||||
from infrastructure.storage_minio import reports_prefix
|
||||
|
||||
HEARTBEAT_OFFLINE_SECONDS = 45
|
||||
|
||||
|
||||
def _parse_iso(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
dt = datetime.fromisoformat(value)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
|
||||
|
||||
class JobStore:
|
||||
"""API de persistencia compatible con el patrón bot_descarga."""
|
||||
|
||||
def __init__(self, db_path: str | None = None):
|
||||
# db_path ignorado (compat); usa DATABASE_URL.
|
||||
init_db()
|
||||
|
||||
def _row_worker(self, row: DigitadorWorkerRow) -> Worker:
|
||||
worker = Worker(
|
||||
id=row.id,
|
||||
hostname=row.hostname or "",
|
||||
max_concurrent=int(row.max_concurrent or 1),
|
||||
status=row.status,
|
||||
last_heartbeat=_iso(row.last_heartbeat) or utc_now(),
|
||||
current_job_id=row.current_job_id,
|
||||
)
|
||||
return self._refresh_worker_status(worker)
|
||||
|
||||
def _row_job(self, row: DigitadorJobRow) -> Job:
|
||||
return Job(
|
||||
id=row.id,
|
||||
worker_id=row.worker_id,
|
||||
status=row.status,
|
||||
title=row.title or "",
|
||||
csv_key=row.csv_key or "",
|
||||
reports_prefix=row.reports_prefix or "",
|
||||
original_filename=row.original_filename or "",
|
||||
cancel_requested=bool(row.cancel_requested),
|
||||
result=dict(row.result or {}),
|
||||
error=row.error,
|
||||
created_at=_iso(row.created_at) or utc_now(),
|
||||
started_at=_iso(row.started_at),
|
||||
finished_at=_iso(row.finished_at),
|
||||
job_id=row.job_id,
|
||||
task_id=row.task_id,
|
||||
parent_digitador_job_id=row.parent_digitador_job_id,
|
||||
shard_index=row.shard_index,
|
||||
shard_count=row.shard_count,
|
||||
)
|
||||
|
||||
def _refresh_worker_status(self, worker: Worker) -> Worker:
|
||||
try:
|
||||
hb = datetime.fromisoformat(worker.last_heartbeat)
|
||||
if hb.tzinfo is None:
|
||||
hb = hb.replace(tzinfo=timezone.utc)
|
||||
age = datetime.now(timezone.utc) - hb.astimezone(timezone.utc)
|
||||
if age > timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS):
|
||||
worker.status = WorkerStatus.OFFLINE.value
|
||||
elif worker.current_job_id:
|
||||
worker.status = WorkerStatus.BUSY.value
|
||||
else:
|
||||
worker.status = WorkerStatus.ONLINE.value
|
||||
except Exception:
|
||||
worker.status = WorkerStatus.OFFLINE.value
|
||||
return worker
|
||||
|
||||
def register_worker(
|
||||
self,
|
||||
worker_id: str,
|
||||
hostname: str = "",
|
||||
max_concurrent: int = 1,
|
||||
) -> Worker:
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorWorkerRow, worker_id)
|
||||
now = utcnow()
|
||||
if row:
|
||||
row.hostname = hostname
|
||||
row.max_concurrent = max_concurrent
|
||||
row.status = WorkerStatus.ONLINE.value
|
||||
row.last_heartbeat = now
|
||||
row.current_job_id = None
|
||||
else:
|
||||
row = DigitadorWorkerRow(
|
||||
id=worker_id,
|
||||
hostname=hostname,
|
||||
max_concurrent=max_concurrent,
|
||||
status=WorkerStatus.ONLINE.value,
|
||||
last_heartbeat=now,
|
||||
current_job_id=None,
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return self._row_worker(row)
|
||||
|
||||
def heartbeat(self, worker_id: str) -> Optional[Worker]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorWorkerRow, worker_id)
|
||||
if not row:
|
||||
return None
|
||||
row.last_heartbeat = utcnow()
|
||||
row.status = (
|
||||
WorkerStatus.BUSY.value if row.current_job_id else WorkerStatus.ONLINE.value
|
||||
)
|
||||
session.flush()
|
||||
worker = self._row_worker(row)
|
||||
try:
|
||||
self.reclaim_stale_running_jobs()
|
||||
except Exception:
|
||||
pass
|
||||
return worker
|
||||
|
||||
def get_worker(self, worker_id: str) -> Optional[Worker]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorWorkerRow, worker_id)
|
||||
if not row:
|
||||
return None
|
||||
return self._row_worker(row)
|
||||
|
||||
def list_workers(self) -> List[Worker]:
|
||||
try:
|
||||
self.reclaim_stale_running_jobs()
|
||||
except Exception:
|
||||
pass
|
||||
with session_scope() as session:
|
||||
rows = session.scalars(
|
||||
select(DigitadorWorkerRow).order_by(DigitadorWorkerRow.id)
|
||||
).all()
|
||||
return [self._row_worker(r) for r in rows]
|
||||
|
||||
def list_online_worker_ids(self) -> List[str]:
|
||||
return [
|
||||
w.id
|
||||
for w in self.list_workers()
|
||||
if w.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value)
|
||||
]
|
||||
|
||||
def is_worker_online(self, worker_id: str) -> bool:
|
||||
worker = self.get_worker(worker_id)
|
||||
if not worker:
|
||||
return False
|
||||
return worker.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value)
|
||||
|
||||
def update_csv_key(self, job_id: str, csv_key: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorJobRow, job_id)
|
||||
if not row:
|
||||
return None
|
||||
row.csv_key = csv_key
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def create_job(
|
||||
self,
|
||||
worker_id: Optional[str],
|
||||
csv_key: str,
|
||||
original_filename: str = "",
|
||||
title: str = "",
|
||||
*,
|
||||
job_id: Optional[str] = None,
|
||||
task_id: Optional[str] = None,
|
||||
reports_prefix_override: Optional[str] = None,
|
||||
parent_digitador_job_id: Optional[str] = None,
|
||||
shard_index: Optional[int] = None,
|
||||
shard_count: Optional[int] = None,
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
status: Optional[str] = None,
|
||||
digitador_id: Optional[str] = None,
|
||||
) -> Job:
|
||||
import uuid as uuid_mod
|
||||
|
||||
digitador_id = digitador_id or str(uuid_mod.uuid4())
|
||||
prefix = reports_prefix_override or reports_prefix(digitador_id)
|
||||
with session_scope() as session:
|
||||
row = DigitadorJobRow(
|
||||
id=digitador_id,
|
||||
job_id=job_id,
|
||||
task_id=task_id,
|
||||
worker_id=worker_id,
|
||||
parent_digitador_job_id=parent_digitador_job_id,
|
||||
shard_index=shard_index,
|
||||
shard_count=shard_count,
|
||||
status=status or JobStatus.PENDING.value,
|
||||
title=title or f"Digitador {original_filename}",
|
||||
csv_key=csv_key,
|
||||
reports_prefix=prefix,
|
||||
original_filename=original_filename,
|
||||
cancel_requested=False,
|
||||
result=result or {},
|
||||
created_at=utcnow(),
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def get_job(self, job_id: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorJobRow, job_id)
|
||||
if not row:
|
||||
return None
|
||||
return self._row_job(row)
|
||||
|
||||
def list_jobs(
|
||||
self,
|
||||
limit: int = 100,
|
||||
*,
|
||||
job_id: Optional[str] = None,
|
||||
parents_only: bool = True,
|
||||
) -> List[Job]:
|
||||
with session_scope() as session:
|
||||
stmt = select(DigitadorJobRow).order_by(DigitadorJobRow.created_at.desc())
|
||||
if job_id:
|
||||
stmt = stmt.where(DigitadorJobRow.job_id == job_id)
|
||||
if parents_only:
|
||||
stmt = stmt.where(DigitadorJobRow.parent_digitador_job_id.is_(None))
|
||||
stmt = stmt.limit(limit)
|
||||
rows = session.scalars(stmt).all()
|
||||
return [self._row_job(r) for r in rows]
|
||||
|
||||
def list_shards(self, parent_id: str) -> List[Job]:
|
||||
with session_scope() as session:
|
||||
rows = session.scalars(
|
||||
select(DigitadorJobRow)
|
||||
.where(DigitadorJobRow.parent_digitador_job_id == parent_id)
|
||||
.order_by(DigitadorJobRow.shard_index.asc())
|
||||
).all()
|
||||
return [self._row_job(r) for r in rows]
|
||||
|
||||
def claim_pending(self, worker_id: str) -> Optional[Job]:
|
||||
try:
|
||||
self.reclaim_stale_running_jobs()
|
||||
except Exception:
|
||||
pass
|
||||
with session_scope() as session:
|
||||
worker = session.get(DigitadorWorkerRow, worker_id)
|
||||
if not worker or worker.current_job_id:
|
||||
return None
|
||||
|
||||
rows = session.scalars(
|
||||
select(DigitadorJobRow)
|
||||
.where(
|
||||
DigitadorJobRow.worker_id == worker_id,
|
||||
DigitadorJobRow.status == JobStatus.PENDING.value,
|
||||
)
|
||||
.order_by(DigitadorJobRow.created_at.asc())
|
||||
).all()
|
||||
row = None
|
||||
for candidate in rows:
|
||||
is_parent = (
|
||||
candidate.parent_digitador_job_id is None
|
||||
and candidate.shard_count is not None
|
||||
and int(candidate.shard_count) > 1
|
||||
)
|
||||
if is_parent:
|
||||
continue
|
||||
row = candidate
|
||||
break
|
||||
if not row:
|
||||
return None
|
||||
|
||||
now = utcnow()
|
||||
row.status = JobStatus.RUNNING.value
|
||||
row.started_at = now
|
||||
worker.current_job_id = row.id
|
||||
worker.status = WorkerStatus.BUSY.value
|
||||
worker.last_heartbeat = now
|
||||
|
||||
task_id = row.task_id
|
||||
if row.parent_digitador_job_id:
|
||||
parent = session.get(DigitadorJobRow, row.parent_digitador_job_id)
|
||||
if parent:
|
||||
task_id = parent.task_id
|
||||
if parent.status == JobStatus.PENDING.value:
|
||||
parent.status = JobStatus.RUNNING.value
|
||||
parent.started_at = parent.started_at or now
|
||||
|
||||
if task_id:
|
||||
task = session.get(Task, task_id)
|
||||
if task and task.status == JobStatus.PENDING.value:
|
||||
task.status = JobStatus.RUNNING.value
|
||||
task.started_at = now
|
||||
task.updated_at = now
|
||||
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def request_cancel(self, job_id: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorJobRow, job_id)
|
||||
if not row:
|
||||
return None
|
||||
if row.status in (
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
):
|
||||
return self._row_job(row)
|
||||
|
||||
now = utcnow()
|
||||
row.cancel_requested = True
|
||||
if row.status == JobStatus.PENDING.value:
|
||||
row.status = JobStatus.CANCELLED.value
|
||||
row.finished_at = now
|
||||
self._sync_task(session, row, status=JobStatus.CANCELLED.value)
|
||||
|
||||
if row.parent_digitador_job_id is None and row.shard_count and int(row.shard_count) > 1:
|
||||
children = session.scalars(
|
||||
select(DigitadorJobRow).where(
|
||||
DigitadorJobRow.parent_digitador_job_id == job_id
|
||||
)
|
||||
).all()
|
||||
for child in children:
|
||||
if child.status in (
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
):
|
||||
continue
|
||||
child.cancel_requested = True
|
||||
if child.status == JobStatus.PENDING.value:
|
||||
child.status = JobStatus.CANCELLED.value
|
||||
child.finished_at = now
|
||||
worker = (
|
||||
session.get(DigitadorWorkerRow, child.worker_id)
|
||||
if child.worker_id
|
||||
else None
|
||||
)
|
||||
if worker and worker.current_job_id == child.id:
|
||||
worker.current_job_id = None
|
||||
worker.status = WorkerStatus.ONLINE.value
|
||||
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def is_cancel_requested(self, job_id: str) -> bool:
|
||||
job = self.get_job(job_id)
|
||||
if not job:
|
||||
return False
|
||||
if job.cancel_requested:
|
||||
return True
|
||||
if job.parent_digitador_job_id:
|
||||
parent = self.get_job(job.parent_digitador_job_id)
|
||||
if parent and parent.cancel_requested:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _sync_task(
|
||||
self,
|
||||
session,
|
||||
row: DigitadorJobRow,
|
||||
*,
|
||||
status: Optional[str] = None,
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> None:
|
||||
target = row
|
||||
if row.parent_digitador_job_id:
|
||||
parent = session.get(DigitadorJobRow, row.parent_digitador_job_id)
|
||||
if parent:
|
||||
target = parent
|
||||
if not target.task_id:
|
||||
return
|
||||
task = session.get(Task, target.task_id)
|
||||
if not task:
|
||||
return
|
||||
now = utcnow()
|
||||
if status:
|
||||
task.status = status
|
||||
if result is not None:
|
||||
task.result = {
|
||||
**dict(task.result or {}),
|
||||
"digitador_job_id": target.id,
|
||||
**result,
|
||||
}
|
||||
if error is not None:
|
||||
task.error = error
|
||||
task.updated_at = now
|
||||
if status in (
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
):
|
||||
task.finished_at = now
|
||||
|
||||
def patch_job(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
status: Optional[str] = None,
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> Optional[Job]:
|
||||
parent_id: Optional[str] = None
|
||||
with session_scope() as session:
|
||||
row = session.get(DigitadorJobRow, job_id)
|
||||
if not row:
|
||||
return None
|
||||
|
||||
new_status = status or row.status
|
||||
if result is not None:
|
||||
row.result = result
|
||||
if error is not None:
|
||||
row.error = error
|
||||
row.status = new_status
|
||||
|
||||
finished = False
|
||||
if new_status in (
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
):
|
||||
row.finished_at = row.finished_at or utcnow()
|
||||
finished = True
|
||||
|
||||
if not row.parent_digitador_job_id:
|
||||
self._sync_task(
|
||||
session,
|
||||
row,
|
||||
status=new_status if status else None,
|
||||
result=result,
|
||||
error=error,
|
||||
)
|
||||
|
||||
if finished:
|
||||
worker = session.get(DigitadorWorkerRow, row.worker_id) if row.worker_id else None
|
||||
if worker and worker.current_job_id == job_id:
|
||||
worker.current_job_id = None
|
||||
worker.status = WorkerStatus.ONLINE.value
|
||||
|
||||
parent_id = row.parent_digitador_job_id
|
||||
session.flush()
|
||||
job = self._row_job(row)
|
||||
|
||||
if parent_id:
|
||||
self.refresh_parent_from_shards(parent_id)
|
||||
refreshed = self.get_job(job_id)
|
||||
return refreshed or job
|
||||
return job
|
||||
|
||||
def refresh_parent_from_shards(self, parent_id: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
parent = session.get(DigitadorJobRow, parent_id)
|
||||
if not parent:
|
||||
return None
|
||||
children = list(
|
||||
session.scalars(
|
||||
select(DigitadorJobRow).where(
|
||||
DigitadorJobRow.parent_digitador_job_id == parent_id
|
||||
)
|
||||
).all()
|
||||
)
|
||||
if not children:
|
||||
return self._row_job(parent)
|
||||
|
||||
statuses = [c.status for c in children]
|
||||
terminal = {
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
}
|
||||
any_active = any(
|
||||
s in (JobStatus.PENDING.value, JobStatus.RUNNING.value) for s in statuses
|
||||
)
|
||||
all_terminal = all(s in terminal for s in statuses)
|
||||
|
||||
total = 0
|
||||
ok = 0
|
||||
error_count = 0
|
||||
timeout = 0
|
||||
existentes = 0
|
||||
for c in children:
|
||||
r = dict(c.result or {})
|
||||
total += int(r.get("total") or 0)
|
||||
ok += int(r.get("ok") or 0)
|
||||
error_count += int(r.get("error") or 0)
|
||||
timeout += int(r.get("timeout") or 0)
|
||||
existentes += int(r.get("existentes") or 0)
|
||||
|
||||
processed = 0
|
||||
for c in children:
|
||||
r = dict(c.result or {})
|
||||
if c.status in terminal:
|
||||
processed += (
|
||||
int(r.get("ok") or 0)
|
||||
+ int(r.get("error") or 0)
|
||||
+ int(r.get("timeout") or 0)
|
||||
+ int(r.get("existentes") or 0)
|
||||
)
|
||||
else:
|
||||
processed += int(r.get("processed") or 0)
|
||||
|
||||
shard_ids = [c.id for c in children]
|
||||
parent_result = dict(parent.result or {})
|
||||
parent_result.update(
|
||||
{
|
||||
"mode": "distributed",
|
||||
"shard_ids": shard_ids,
|
||||
"total": total or int(parent_result.get("total_rows") or 0),
|
||||
"total_rows": int(parent_result.get("total_rows") or total or 0),
|
||||
"ok": ok,
|
||||
"error": error_count,
|
||||
"timeout": timeout,
|
||||
"existentes": existentes,
|
||||
"processed": processed,
|
||||
"reports_prefix": parent.reports_prefix,
|
||||
"shards": [
|
||||
{
|
||||
"id": c.id,
|
||||
"worker_id": c.worker_id,
|
||||
"status": c.status,
|
||||
"shard_index": c.shard_index,
|
||||
"result": dict(c.result or {}),
|
||||
"error": c.error,
|
||||
}
|
||||
for c in children
|
||||
],
|
||||
}
|
||||
)
|
||||
parent.result = parent_result
|
||||
|
||||
now = utcnow()
|
||||
if any_active:
|
||||
parent.status = JobStatus.RUNNING.value
|
||||
parent.started_at = parent.started_at or now
|
||||
parent.finished_at = None
|
||||
parent.error = None
|
||||
self._sync_task(
|
||||
session,
|
||||
parent,
|
||||
status=JobStatus.RUNNING.value,
|
||||
result=parent_result,
|
||||
)
|
||||
elif all_terminal:
|
||||
any_failed = any(s == JobStatus.FAILED.value for s in statuses)
|
||||
any_cancelled = any(s == JobStatus.CANCELLED.value for s in statuses)
|
||||
all_cancelled = all(s == JobStatus.CANCELLED.value for s in statuses)
|
||||
if all_cancelled:
|
||||
parent.status = JobStatus.CANCELLED.value
|
||||
elif any_failed:
|
||||
parent.status = JobStatus.FAILED.value
|
||||
parent.error = next(
|
||||
(c.error for c in children if c.error),
|
||||
"Uno o más shards fallaron",
|
||||
)
|
||||
elif any_cancelled and not any_failed:
|
||||
parent.status = JobStatus.COMPLETED.value
|
||||
else:
|
||||
parent.status = JobStatus.COMPLETED.value
|
||||
parent.finished_at = parent.finished_at or now
|
||||
self._sync_task(
|
||||
session,
|
||||
parent,
|
||||
status=parent.status,
|
||||
result=parent_result,
|
||||
error=parent.error,
|
||||
)
|
||||
|
||||
session.flush()
|
||||
return self._row_job(parent)
|
||||
|
||||
def reclaim_stale_running_jobs(self) -> int:
|
||||
"""Jobs running cuyo worker está offline → pending + reasignar a otro online."""
|
||||
reclaimed = 0
|
||||
with session_scope() as session:
|
||||
now = utcnow()
|
||||
cutoff = now - timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS)
|
||||
workers = list(session.scalars(select(DigitadorWorkerRow)).all())
|
||||
offline_ids: set[str] = set()
|
||||
online_free: List[DigitadorWorkerRow] = []
|
||||
for w in workers:
|
||||
hb = w.last_heartbeat
|
||||
if hb is None:
|
||||
offline_ids.add(w.id)
|
||||
continue
|
||||
if hb.tzinfo is None:
|
||||
hb = hb.replace(tzinfo=timezone.utc)
|
||||
if hb < cutoff:
|
||||
offline_ids.add(w.id)
|
||||
if w.current_job_id:
|
||||
w.current_job_id = None
|
||||
w.status = WorkerStatus.OFFLINE.value
|
||||
elif not w.current_job_id:
|
||||
online_free.append(w)
|
||||
|
||||
if not offline_ids:
|
||||
session.flush()
|
||||
return 0
|
||||
|
||||
running = session.scalars(
|
||||
select(DigitadorJobRow).where(
|
||||
DigitadorJobRow.status == JobStatus.RUNNING.value,
|
||||
DigitadorJobRow.worker_id.in_(list(offline_ids)),
|
||||
)
|
||||
).all()
|
||||
|
||||
rr = 0
|
||||
for row in running:
|
||||
is_parent = (
|
||||
row.parent_digitador_job_id is None
|
||||
and row.shard_count is not None
|
||||
and int(row.shard_count) > 1
|
||||
)
|
||||
if is_parent:
|
||||
continue
|
||||
|
||||
row.status = JobStatus.PENDING.value
|
||||
row.started_at = None
|
||||
row.cancel_requested = False
|
||||
|
||||
if online_free:
|
||||
target = online_free[rr % len(online_free)]
|
||||
rr += 1
|
||||
row.worker_id = target.id
|
||||
|
||||
reclaimed += 1
|
||||
session.add(
|
||||
DigitadorJobEventRow(
|
||||
digitador_job_id=row.id,
|
||||
event_type="warning",
|
||||
message=(
|
||||
"Worker offline; job reclaimed"
|
||||
+ (f" → {row.worker_id}" if row.worker_id else "")
|
||||
),
|
||||
timestamp=now,
|
||||
metadata_json={"reclaimed": True, "worker_id": row.worker_id},
|
||||
)
|
||||
)
|
||||
if row.parent_digitador_job_id:
|
||||
session.add(
|
||||
DigitadorJobEventRow(
|
||||
digitador_job_id=row.parent_digitador_job_id,
|
||||
event_type="warning",
|
||||
message=f"Shard {row.shard_index} reclamado (worker offline)",
|
||||
timestamp=now,
|
||||
metadata_json={
|
||||
"shard_id": row.id,
|
||||
"worker_id": row.worker_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
if online_free:
|
||||
pending_orphan = session.scalars(
|
||||
select(DigitadorJobRow).where(
|
||||
DigitadorJobRow.status == JobStatus.PENDING.value,
|
||||
DigitadorJobRow.worker_id.in_(list(offline_ids)),
|
||||
)
|
||||
).all()
|
||||
for row in pending_orphan:
|
||||
is_parent = (
|
||||
row.parent_digitador_job_id is None
|
||||
and row.shard_count is not None
|
||||
and int(row.shard_count) > 1
|
||||
)
|
||||
if is_parent:
|
||||
continue
|
||||
target = online_free[rr % len(online_free)]
|
||||
rr += 1
|
||||
if row.worker_id != target.id:
|
||||
row.worker_id = target.id
|
||||
reclaimed += 1
|
||||
|
||||
session.flush()
|
||||
|
||||
return reclaimed
|
||||
|
||||
def add_event(
|
||||
self,
|
||||
job_id: str,
|
||||
event_type: str,
|
||||
message: str,
|
||||
*,
|
||||
radicado: Optional[str] = None,
|
||||
nit: Optional[str] = None,
|
||||
index: Optional[int] = None,
|
||||
total: Optional[int] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
timestamp: Optional[str] = None,
|
||||
) -> JobEventRecord:
|
||||
with session_scope() as session:
|
||||
ts = _parse_iso(timestamp) or utcnow()
|
||||
ev = DigitadorJobEventRow(
|
||||
digitador_job_id=job_id,
|
||||
event_type=event_type,
|
||||
message=message,
|
||||
timestamp=ts,
|
||||
radicado=radicado,
|
||||
nit=nit,
|
||||
idx=index,
|
||||
total=total,
|
||||
metadata_json=metadata or {},
|
||||
)
|
||||
session.add(ev)
|
||||
session.flush()
|
||||
row = session.get(DigitadorJobRow, job_id)
|
||||
if row and row.parent_digitador_job_id:
|
||||
session.add(
|
||||
DigitadorJobEventRow(
|
||||
digitador_job_id=row.parent_digitador_job_id,
|
||||
event_type=event_type,
|
||||
message=message,
|
||||
timestamp=ts,
|
||||
radicado=radicado,
|
||||
nit=nit,
|
||||
idx=index,
|
||||
total=total,
|
||||
metadata_json={
|
||||
**(metadata or {}),
|
||||
"shard_id": job_id,
|
||||
"shard_index": row.shard_index,
|
||||
"worker_id": row.worker_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
session.flush()
|
||||
return JobEventRecord(
|
||||
id=int(ev.id),
|
||||
job_id=job_id,
|
||||
event_type=event_type,
|
||||
message=message,
|
||||
timestamp=_iso(ts) or utc_now(),
|
||||
radicado=radicado,
|
||||
nit=nit,
|
||||
index=index,
|
||||
total=total,
|
||||
metadata=metadata or {},
|
||||
)
|
||||
|
||||
def list_events(self, job_id: str, since_id: int = 0) -> List[JobEventRecord]:
|
||||
with session_scope() as session:
|
||||
rows = session.scalars(
|
||||
select(DigitadorJobEventRow)
|
||||
.where(
|
||||
DigitadorJobEventRow.digitador_job_id == job_id,
|
||||
DigitadorJobEventRow.id > since_id,
|
||||
)
|
||||
.order_by(DigitadorJobEventRow.id.asc())
|
||||
).all()
|
||||
return [
|
||||
JobEventRecord(
|
||||
id=int(r.id),
|
||||
job_id=r.digitador_job_id,
|
||||
event_type=r.event_type,
|
||||
message=r.message,
|
||||
timestamp=_iso(r.timestamp) or utc_now(),
|
||||
radicado=r.radicado,
|
||||
nit=r.nit,
|
||||
index=r.idx,
|
||||
total=r.total,
|
||||
metadata=dict(r.metadata_json or {}),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
def create_digitador_task(
|
||||
self,
|
||||
*,
|
||||
parent_job_id: str,
|
||||
digitador_job_id: str,
|
||||
worker_id: Optional[str],
|
||||
title: str,
|
||||
original_filename: str,
|
||||
worker_ids: Optional[List[str]] = None,
|
||||
mode: str = "single",
|
||||
source_type: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Crea Task(type=digitador) y la asocia al digitador_job. Devuelve task_id."""
|
||||
with session_scope() as session:
|
||||
now = utcnow()
|
||||
request: Dict[str, Any] = {
|
||||
"worker_id": worker_id,
|
||||
"worker_ids": worker_ids or ([worker_id] if worker_id else []),
|
||||
"original_filename": original_filename,
|
||||
"digitador_job_id": digitador_job_id,
|
||||
"mode": mode,
|
||||
}
|
||||
result: Dict[str, Any] = {"digitador_job_id": digitador_job_id, "mode": mode}
|
||||
if source_type:
|
||||
request["source_type"] = source_type
|
||||
result["source_type"] = source_type
|
||||
task = Task(
|
||||
job_id=parent_job_id,
|
||||
type="digitador",
|
||||
status=JobStatus.PENDING.value,
|
||||
title=title or f"Bot digitador ({original_filename})",
|
||||
request=request,
|
||||
result=result,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(task)
|
||||
session.flush()
|
||||
row = session.get(DigitadorJobRow, digitador_job_id)
|
||||
if row:
|
||||
row.task_id = task.id
|
||||
row.job_id = parent_job_id
|
||||
session.flush()
|
||||
return task.id
|
||||
|
||||
def parent_job_exists(self, parent_job_id: str) -> bool:
|
||||
from infrastructure.database import Job as IpsJob
|
||||
|
||||
with session_scope() as session:
|
||||
return session.get(IpsJob, parent_job_id) is not None
|
||||
@@ -0,0 +1,10 @@
|
||||
fastapi>=0.110.0
|
||||
uvicorn[standard]>=0.27.0
|
||||
python-multipart>=0.0.9
|
||||
boto3>=1.34.0
|
||||
httpx>=0.27.0
|
||||
openpyxl>=3.1.0
|
||||
pydantic>=2.0.0
|
||||
sqlalchemy>=2.0.0
|
||||
psycopg[binary]>=3.1.0
|
||||
pytest>=7.0.0
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Tests de parseo/partición CSV para digitador."""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from domain.csv_split import (
|
||||
REQUIRED_BOT_COLUMNS,
|
||||
parse_digitador_rows,
|
||||
rows_to_csv_bytes,
|
||||
split_rows,
|
||||
)
|
||||
|
||||
|
||||
def _it_csv_bytes() -> bytes:
|
||||
header = "|".join(
|
||||
[
|
||||
"RADICADO",
|
||||
"NUMERO",
|
||||
"FECHA",
|
||||
"BRUTO",
|
||||
"AUTORIZACION",
|
||||
"TIPO_ID",
|
||||
"SERVICIO",
|
||||
"COPAGO",
|
||||
"MODERADORA",
|
||||
]
|
||||
)
|
||||
row = "|".join(
|
||||
["R-1", "123", "2024-01-01", "1000", "A-9", "CC", "AMB", "0", "4500"]
|
||||
)
|
||||
return f"{header}\n{row}\n".encode("utf-8-sig")
|
||||
|
||||
|
||||
def _snake_csv_bytes() -> bytes:
|
||||
header = "|".join(REQUIRED_BOT_COLUMNS)
|
||||
row = "|".join(
|
||||
["R-2", "999", "2024-02-02", "2000", "A-1", "TI", "HOS", "100", "0"]
|
||||
)
|
||||
return f"{header}\n{row}\n".encode("utf-8")
|
||||
|
||||
|
||||
def test_it_to_bot_mapping():
|
||||
rows = parse_digitador_rows(_it_csv_bytes(), "aptos.csv")
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["radicado"] == "R-1"
|
||||
assert rows[0]["documento"] == "123"
|
||||
assert rows[0]["ingreso"] == "2024-01-01"
|
||||
assert rows[0]["total_cargos"] == "1000"
|
||||
assert rows[0]["autorizacion"] == "A-9"
|
||||
assert rows[0]["tipo_documento"] == "CC"
|
||||
assert rows[0]["tipo_servicio"] == "AMB"
|
||||
assert rows[0]["copago"] == "0"
|
||||
assert rows[0]["cuota_moderadora"] == "4500"
|
||||
|
||||
|
||||
def test_snake_case_headers():
|
||||
rows = parse_digitador_rows(_snake_csv_bytes(), "input.csv")
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["radicado"] == "R-2"
|
||||
assert rows[0]["documento"] == "999"
|
||||
|
||||
|
||||
def test_missing_columns_raises():
|
||||
bad = b"RADICADO|NUMERO\nR1|1\n"
|
||||
with pytest.raises(ValueError) as exc:
|
||||
parse_digitador_rows(bad, "bad.csv")
|
||||
assert "Columnas faltantes" in str(exc.value)
|
||||
|
||||
|
||||
def test_split_rows_balanced():
|
||||
rows = [{"radicado": str(i)} for i in range(100)]
|
||||
chunks = split_rows(rows, 2)
|
||||
assert len(chunks) == 2
|
||||
assert len(chunks[0]) == 50
|
||||
assert len(chunks[1]) == 50
|
||||
|
||||
|
||||
def test_split_rows_uneven():
|
||||
rows = [{"radicado": str(i)} for i in range(10)]
|
||||
chunks = split_rows(rows, 3)
|
||||
assert len(chunks) == 3
|
||||
assert sum(len(c) for c in chunks) == 10
|
||||
assert [len(c) for c in chunks] == [4, 3, 3]
|
||||
|
||||
|
||||
def test_roundtrip_csv():
|
||||
rows = parse_digitador_rows(_it_csv_bytes(), "aptos.csv")
|
||||
data = rows_to_csv_bytes(rows)
|
||||
parsed = parse_digitador_rows(data, "shard.csv")
|
||||
assert parsed == rows
|
||||
|
||||
|
||||
def test_split_more_workers_than_rows():
|
||||
rows = [{"radicado": "A"}, {"radicado": "B"}]
|
||||
chunks = split_rows(rows, 5)
|
||||
assert len(chunks) == 2
|
||||
@@ -49,6 +49,11 @@ def apply_rules(
|
||||
if col not in result.columns:
|
||||
log.append(f"[WARN] Columna no encontrada: {col}")
|
||||
return
|
||||
# Eliminar columna: no tiene sentido transformar y luego borrar.
|
||||
if "drop_column" in rule_ids:
|
||||
result.drop(columns=[col], inplace=True)
|
||||
log.append(f"[OK] '{col}' <- {get_rule_name('drop_column')}")
|
||||
return
|
||||
split_has_homologation = (
|
||||
"doc_id_split_tipo_numero" in rule_ids and "doc_type_homologate" in rule_ids
|
||||
)
|
||||
|
||||
@@ -140,5 +140,14 @@ register_rule(
|
||||
),
|
||||
fn=special._rule_passthrough_series,
|
||||
)
|
||||
register_rule(
|
||||
"drop_column",
|
||||
name="Eliminar columna",
|
||||
description=(
|
||||
"Quita por completo esta columna del resultado limpio. "
|
||||
"Útil para campos que no se necesitan en el export o en pasos posteriores."
|
||||
),
|
||||
fn=special._rule_passthrough_series,
|
||||
)
|
||||
|
||||
rebuild_cleaning_rules()
|
||||
|
||||
@@ -99,7 +99,12 @@ def _infer_column_type(
|
||||
|
||||
def _applicable_rules_for_type(inferred_type: str) -> list[str]:
|
||||
"""Identificadores de reglas a mostrar según el tipo inferido (sin redundancias)."""
|
||||
base = ["strip_special_spaces", "strip_special_keep_date_seps", "strip_spaces"]
|
||||
base = [
|
||||
"strip_special_spaces",
|
||||
"strip_special_keep_date_seps",
|
||||
"strip_spaces",
|
||||
"drop_column",
|
||||
]
|
||||
if inferred_type == TYPE_INTEGER:
|
||||
return base + ["cop_mod_split", "cop_mod_bruto_neto"]
|
||||
if inferred_type == TYPE_DECIMAL:
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Download jobs: parent/shard fields for distributed bots.
|
||||
|
||||
Revision ID: 0007
|
||||
Revises: 0006
|
||||
Create Date: 2026-08-09
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0007"
|
||||
down_revision: Union[str, None] = "0006"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
inspector = sa.inspect(conn)
|
||||
if not inspector.has_table("download_jobs"):
|
||||
return
|
||||
cols = {c["name"] for c in inspector.get_columns("download_jobs")}
|
||||
|
||||
with op.batch_alter_table("download_jobs") as batch:
|
||||
if "parent_download_job_id" not in cols:
|
||||
batch.add_column(
|
||||
sa.Column(
|
||||
"parent_download_job_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("download_jobs.id"),
|
||||
nullable=True,
|
||||
)
|
||||
)
|
||||
if "shard_index" not in cols:
|
||||
batch.add_column(sa.Column("shard_index", sa.Integer(), nullable=True))
|
||||
if "shard_count" not in cols:
|
||||
batch.add_column(sa.Column("shard_count", sa.Integer(), nullable=True))
|
||||
|
||||
# Padre orquestador no tiene worker asignado
|
||||
op.alter_column(
|
||||
"download_jobs",
|
||||
"worker_id",
|
||||
existing_type=sa.String(36),
|
||||
nullable=True,
|
||||
)
|
||||
|
||||
existing_indexes = {ix["name"] for ix in inspector.get_indexes("download_jobs")}
|
||||
if "ix_download_jobs_parent" not in existing_indexes:
|
||||
op.create_index(
|
||||
"ix_download_jobs_parent",
|
||||
"download_jobs",
|
||||
["parent_download_job_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
inspector = sa.inspect(conn)
|
||||
if not inspector.has_table("download_jobs"):
|
||||
return
|
||||
existing_indexes = {ix["name"] for ix in inspector.get_indexes("download_jobs")}
|
||||
if "ix_download_jobs_parent" in existing_indexes:
|
||||
op.drop_index("ix_download_jobs_parent", table_name="download_jobs")
|
||||
|
||||
cols = {c["name"] for c in inspector.get_columns("download_jobs")}
|
||||
with op.batch_alter_table("download_jobs") as batch:
|
||||
if "shard_count" in cols:
|
||||
batch.drop_column("shard_count")
|
||||
if "shard_index" in cols:
|
||||
batch.drop_column("shard_index")
|
||||
if "parent_download_job_id" in cols:
|
||||
batch.drop_column("parent_download_job_id")
|
||||
|
||||
op.alter_column(
|
||||
"download_jobs",
|
||||
"worker_id",
|
||||
existing_type=sa.String(36),
|
||||
nullable=False,
|
||||
)
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Bot digitador: digitador_workers, digitador_jobs, digitador_job_events.
|
||||
|
||||
Revision ID: 0008
|
||||
Revises: 0007
|
||||
Create Date: 2026-08-09
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0008"
|
||||
down_revision: Union[str, None] = "0007"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def _table_exists(conn: sa.engine.Connection, table: str) -> bool:
|
||||
return sa.inspect(conn).has_table(table)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
json_type = (
|
||||
postgresql.JSONB()
|
||||
if conn.dialect.name == "postgresql"
|
||||
else sa.JSON()
|
||||
)
|
||||
|
||||
if not _table_exists(conn, "digitador_workers"):
|
||||
op.create_table(
|
||||
"digitador_workers",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("hostname", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column("max_concurrent", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("last_heartbeat", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("current_job_id", sa.String(36), nullable=True),
|
||||
)
|
||||
op.create_index("ix_digitador_workers_status", "digitador_workers", ["status"])
|
||||
|
||||
if not _table_exists(conn, "digitador_jobs"):
|
||||
op.create_table(
|
||||
"digitador_jobs",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True),
|
||||
sa.Column("task_id", sa.String(36), sa.ForeignKey("tasks.id"), nullable=True),
|
||||
sa.Column("worker_id", sa.String(36), nullable=True),
|
||||
sa.Column(
|
||||
"parent_digitador_job_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("digitador_jobs.id"),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column("shard_index", sa.Integer(), nullable=True),
|
||||
sa.Column("shard_count", sa.Integer(), nullable=True),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("title", sa.Text(), nullable=True),
|
||||
sa.Column("csv_key", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column("reports_prefix", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column("original_filename", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column(
|
||||
"cancel_requested",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
sa.Column(
|
||||
"result",
|
||||
json_type,
|
||||
nullable=False,
|
||||
server_default=(
|
||||
sa.text("'{}'::jsonb")
|
||||
if conn.dialect.name == "postgresql"
|
||||
else sa.text("'{}'")
|
||||
),
|
||||
),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
op.create_index("ix_digitador_jobs_job_id", "digitador_jobs", ["job_id"])
|
||||
op.create_index("ix_digitador_jobs_task_id", "digitador_jobs", ["task_id"])
|
||||
op.create_index("ix_digitador_jobs_worker_id", "digitador_jobs", ["worker_id"])
|
||||
op.create_index("ix_digitador_jobs_status", "digitador_jobs", ["status"])
|
||||
op.create_index(
|
||||
"ix_digitador_jobs_parent",
|
||||
"digitador_jobs",
|
||||
["parent_digitador_job_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_digitador_jobs_worker_status",
|
||||
"digitador_jobs",
|
||||
["worker_id", "status"],
|
||||
)
|
||||
|
||||
if not _table_exists(conn, "digitador_job_events"):
|
||||
op.create_table(
|
||||
"digitador_job_events",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column(
|
||||
"digitador_job_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("digitador_jobs.id"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("event_type", sa.String(64), nullable=False),
|
||||
sa.Column("message", sa.Text(), nullable=False),
|
||||
sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("radicado", sa.String(255), nullable=True),
|
||||
sa.Column("nit", sa.String(64), nullable=True),
|
||||
sa.Column("idx", sa.Integer(), nullable=True),
|
||||
sa.Column("total", sa.Integer(), nullable=True),
|
||||
sa.Column(
|
||||
"metadata",
|
||||
json_type,
|
||||
nullable=False,
|
||||
server_default=(
|
||||
sa.text("'{}'::jsonb")
|
||||
if conn.dialect.name == "postgresql"
|
||||
else sa.text("'{}'")
|
||||
),
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_digitador_job_events_job_id",
|
||||
"digitador_job_events",
|
||||
["digitador_job_id", "id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if _table_exists(conn, "digitador_job_events"):
|
||||
op.drop_index("ix_digitador_job_events_job_id", table_name="digitador_job_events")
|
||||
op.drop_table("digitador_job_events")
|
||||
if _table_exists(conn, "digitador_jobs"):
|
||||
op.drop_index("ix_digitador_jobs_worker_status", table_name="digitador_jobs")
|
||||
op.drop_index("ix_digitador_jobs_parent", table_name="digitador_jobs")
|
||||
op.drop_index("ix_digitador_jobs_status", table_name="digitador_jobs")
|
||||
op.drop_index("ix_digitador_jobs_worker_id", table_name="digitador_jobs")
|
||||
op.drop_index("ix_digitador_jobs_task_id", table_name="digitador_jobs")
|
||||
op.drop_index("ix_digitador_jobs_job_id", table_name="digitador_jobs")
|
||||
op.drop_table("digitador_jobs")
|
||||
if _table_exists(conn, "digitador_workers"):
|
||||
op.drop_index("ix_digitador_workers_status", table_name="digitador_workers")
|
||||
op.drop_table("digitador_workers")
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
Cpu,
|
||||
Download,
|
||||
Hash,
|
||||
Keyboard,
|
||||
Link2,
|
||||
Package,
|
||||
Settings,
|
||||
@@ -35,6 +36,7 @@ import "./modules/extraction/ExtractionModule";
|
||||
import "./modules/cleaning/CleaningModule";
|
||||
import "./modules/poblacion_apta/PoblacionAptaModule";
|
||||
import "./modules/downloads/DownloadsModule";
|
||||
import "./modules/digitador/DigitadorModule";
|
||||
|
||||
const BOT_PACKAGE_URL =
|
||||
"http://s3.builker.xyz/apps-versions/BotDescargaPYSCuentasMedicas/latest/bot_descarga.zip";
|
||||
@@ -403,6 +405,14 @@ function App() {
|
||||
<Users size={17} />
|
||||
<span>Población apta</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={jobSection === "digitador" ? "active" : ""}
|
||||
onClick={() => handleActivateSection("digitador")}
|
||||
>
|
||||
<Keyboard size={17} />
|
||||
<span>Bot digitador</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={jobSection === "results" ? "active" : ""}
|
||||
|
||||
@@ -28,7 +28,11 @@ interface Props {
|
||||
|
||||
function jobLabel(r: DownloadJob): string {
|
||||
const title = r.title || r.original_filename || r.id.slice(0, 8);
|
||||
return `${title} · ${r.status} · ${r.worker_id}`;
|
||||
const bots =
|
||||
r.shard_count && Number(r.shard_count) > 1
|
||||
? `${r.shard_count} bots`
|
||||
: r.worker_id || "—";
|
||||
return `${title} · ${r.status} · ${bots}`;
|
||||
}
|
||||
|
||||
export function MinioDownloadSourcePicker({
|
||||
|
||||
@@ -9,6 +9,10 @@ import type {
|
||||
CountSyncResponse,
|
||||
CreateCountJobPayload,
|
||||
CreateJobPayload,
|
||||
DigitadorJob,
|
||||
DigitadorJobEvent,
|
||||
DigitadorJobFilesResponse,
|
||||
DigitadorWorker,
|
||||
DownloadJob,
|
||||
DownloadJobEvent,
|
||||
DownloadJobFilesResponse,
|
||||
@@ -87,7 +91,8 @@ export class ApiClient {
|
||||
path.startsWith("/cleaning/jobs/from-csv") ||
|
||||
path.startsWith("/poblacion-apta/jobs/from-csv") ||
|
||||
path.startsWith("/poblacion-apta/jobs/from-cleaning") ||
|
||||
path === "/bot-descarga/jobs"
|
||||
path === "/bot-descarga/jobs" ||
|
||||
path === "/bot-digitador/jobs"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -310,6 +315,83 @@ export class ApiClient {
|
||||
return this.handle<DownloadJob>(response);
|
||||
}
|
||||
|
||||
// ── Bot Digitador (control remoto) ─────────────────────────────────────────
|
||||
|
||||
async listDigitadorWorkers(): Promise<DigitadorWorker[]> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url("/bot-digitador/workers"),
|
||||
{ headers: this.botHeaders() },
|
||||
this.timeoutFor("/bot-digitador/workers"),
|
||||
);
|
||||
return this.handle<DigitadorWorker[]>(response);
|
||||
}
|
||||
|
||||
async createDigitadorJob(form: FormData): Promise<DigitadorJob> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url("/bot-digitador/jobs"),
|
||||
{ method: "POST", headers: this.botHeaders(), body: form },
|
||||
this.timeoutFor("/bot-digitador/jobs"),
|
||||
);
|
||||
return this.handle<DigitadorJob>(response);
|
||||
}
|
||||
|
||||
async listDigitadorJobs(limit = 50, parentJobId?: string): Promise<DigitadorJob[]> {
|
||||
const qs = new URLSearchParams({ limit: String(limit) });
|
||||
if (parentJobId) qs.set("job_id", parentJobId);
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-digitador/jobs?${qs.toString()}`),
|
||||
{ headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<DigitadorJob[]>(response);
|
||||
}
|
||||
|
||||
async getDigitadorJob(jobId: string): Promise<DigitadorJob> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}`),
|
||||
{ headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<DigitadorJob>(response);
|
||||
}
|
||||
|
||||
async listDigitadorJobFiles(jobId: string): Promise<DigitadorJobFilesResponse> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}/files`),
|
||||
{ headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<DigitadorJobFilesResponse>(response);
|
||||
}
|
||||
|
||||
/** Descarga bytes de un reporte CSV/log/json del resultado MinIO (proxy bot-digitador). */
|
||||
async fetchDigitadorObject(jobId: string, key: string): Promise<ArrayBuffer> {
|
||||
const qs = new URLSearchParams({ key });
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}/object?${qs.toString()}`),
|
||||
{ headers: this.botHeaders() },
|
||||
120_000,
|
||||
);
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => "");
|
||||
throw new Error(text || `Error ${response.status} al leer objeto MinIO`);
|
||||
}
|
||||
return response.arrayBuffer();
|
||||
}
|
||||
|
||||
async getDigitadorEvents(jobId: string, since = 0): Promise<{ events: DigitadorJobEvent[] }> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}/events?since=${since}`),
|
||||
{ headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<{ events: DigitadorJobEvent[] }>(response);
|
||||
}
|
||||
|
||||
async cancelDigitadorJob(jobId: string): Promise<DigitadorJob> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-digitador/jobs/${encodeURIComponent(jobId)}/cancel`),
|
||||
{ method: "POST", headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<DigitadorJob>(response);
|
||||
}
|
||||
|
||||
// ── Count Jobs ─────────────────────────────────────────────────────────────
|
||||
|
||||
/** Envía PDFs al servicio de conteo (síncrono). */
|
||||
@@ -456,7 +538,7 @@ export class ApiClient {
|
||||
|
||||
async poblacionAptaExportCsv(
|
||||
jobId: string,
|
||||
subset: "aptos" | "no_aptos" = "aptos",
|
||||
subset: "aptos" | "no_aptos" | "todos" = "aptos",
|
||||
): Promise<Blob> {
|
||||
const params = new URLSearchParams({ subset });
|
||||
const path = `/poblacion-apta/jobs/${encodeURIComponent(jobId)}/export.csv?${params}`;
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { ModuleDescriptor } from "../registry";
|
||||
import { registerModule } from "../registry";
|
||||
import type { Task } from "../../../shared/types";
|
||||
import { DigitadorWizardModule } from "./DigitadorWizardModule";
|
||||
import { DigitadorResultsPanel } from "./DigitadorResultsPanel";
|
||||
|
||||
const DigitadorModule: ModuleDescriptor = {
|
||||
type: "digitador",
|
||||
label: "Bot digitador",
|
||||
iconName: "Keyboard",
|
||||
hasTask: (tasks: Task[]) => tasks.some((t) => t.type === "digitador"),
|
||||
Wizard: DigitadorWizardModule as unknown as ModuleDescriptor["Wizard"],
|
||||
Results: DigitadorResultsPanel as unknown as ModuleDescriptor["Results"],
|
||||
};
|
||||
|
||||
registerModule(DigitadorModule);
|
||||
|
||||
export { DigitadorModule };
|
||||
@@ -0,0 +1,400 @@
|
||||
import {
|
||||
AlertCircle,
|
||||
CheckCircle2,
|
||||
Download,
|
||||
Keyboard,
|
||||
Loader2,
|
||||
Plus,
|
||||
RefreshCw,
|
||||
} from "lucide-react";
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import type {
|
||||
DigitadorJob,
|
||||
DigitadorJobEvent,
|
||||
DigitadorJobFile,
|
||||
Job,
|
||||
Task,
|
||||
} from "../../../shared/types";
|
||||
import type { ApiClient } from "../../lib/apiClient";
|
||||
|
||||
declare const desktopApi: import("../../../shared/types").DesktopBridge | undefined;
|
||||
|
||||
interface Props {
|
||||
client: ApiClient;
|
||||
job: Job;
|
||||
tasks: Task[];
|
||||
onStartDigitador: () => void;
|
||||
}
|
||||
|
||||
function reportBasename(file: DigitadorJobFile): string {
|
||||
const named = (file.name || "").trim();
|
||||
if (named) return named;
|
||||
return file.key.split("/").pop() || "reporte.csv";
|
||||
}
|
||||
|
||||
function extensionFilters(filename: string): Array<{ name: string; extensions: string[] }> {
|
||||
const lower = filename.toLowerCase();
|
||||
if (lower.endsWith(".csv")) return [{ name: "CSV", extensions: ["csv"] }];
|
||||
if (lower.endsWith(".json")) return [{ name: "JSON", extensions: ["json"] }];
|
||||
if (lower.endsWith(".log")) return [{ name: "Log", extensions: ["log"] }];
|
||||
return [{ name: "Archivo", extensions: ["*"] }];
|
||||
}
|
||||
|
||||
async function saveReportAs(filename: string, data: ArrayBuffer): Promise<boolean> {
|
||||
const safeName = reportBasename({ key: filename, name: filename });
|
||||
if (typeof desktopApi?.saveFileDialog === "function" && typeof desktopApi?.writeBytesFile === "function") {
|
||||
const dest = await desktopApi.saveFileDialog(safeName, extensionFilters(safeName));
|
||||
if (!dest) return false;
|
||||
await desktopApi.writeBytesFile(dest, data);
|
||||
return true;
|
||||
}
|
||||
const blob = new Blob([data]);
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = safeName;
|
||||
a.click();
|
||||
URL.revokeObjectURL(url);
|
||||
return true;
|
||||
}
|
||||
|
||||
export function DigitadorResultsPanel({ client, job, tasks, onStartDigitador }: Props) {
|
||||
const [runs, setRuns] = useState<DigitadorJob[]>([]);
|
||||
const [selectedId, setSelectedId] = useState<string>("");
|
||||
const [events, setEvents] = useState<DigitadorJobEvent[]>([]);
|
||||
const [files, setFiles] = useState<DigitadorJobFile[]>([]);
|
||||
const [filesError, setFilesError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [dlBusyKey, setDlBusyKey] = useState<string | null>(null);
|
||||
const [dlStatus, setDlStatus] = useState<string | null>(null);
|
||||
|
||||
const digitadorTasks = tasks
|
||||
.filter((t) => t.type === "digitador")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
|
||||
const loadRuns = useCallback(async () => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const list = await client.listDigitadorJobs(100, job.id);
|
||||
setRuns(list);
|
||||
setSelectedId((prev) => {
|
||||
if (prev && list.some((r) => r.id === prev)) return prev;
|
||||
return list[0]?.id ?? "";
|
||||
});
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [client, job.id]);
|
||||
|
||||
useEffect(() => {
|
||||
void loadRuns();
|
||||
}, [loadRuns, digitadorTasks.length]);
|
||||
|
||||
const selected = runs.find((r) => r.id === selectedId) ?? null;
|
||||
const running = Boolean(
|
||||
selected && !["completed", "failed", "cancelled"].includes(selected.status),
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedId) {
|
||||
setEvents([]);
|
||||
setFiles([]);
|
||||
setFilesError(null);
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
async function loadDetail() {
|
||||
try {
|
||||
const [{ events: ev }, fresh] = await Promise.all([
|
||||
client.getDigitadorEvents(selectedId, 0),
|
||||
client.getDigitadorJob(selectedId),
|
||||
]);
|
||||
if (cancelled) return;
|
||||
setEvents(ev);
|
||||
setRuns((prev) => prev.map((r) => (r.id === fresh.id ? fresh : r)));
|
||||
|
||||
try {
|
||||
const fileResp = await client.listDigitadorJobFiles(selectedId);
|
||||
if (cancelled) return;
|
||||
setFiles(fileResp.files);
|
||||
setFilesError(null);
|
||||
} catch (e) {
|
||||
if (cancelled) return;
|
||||
setFiles([]);
|
||||
setFilesError(e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
} catch (e) {
|
||||
if (!cancelled) setError(e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}
|
||||
|
||||
void loadDetail();
|
||||
if (!running) return;
|
||||
|
||||
const t = setInterval(() => void loadDetail(), 3000);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(t);
|
||||
};
|
||||
}, [client, selectedId, running]);
|
||||
|
||||
const result = selected?.result ?? {};
|
||||
const exitosos = Number(result.exitosos ?? 0);
|
||||
const fallidos = Number(result.fallidos ?? 0);
|
||||
const existentes = Number(result.existentes ?? 0);
|
||||
const timeouts = Number(result.timeouts ?? 0);
|
||||
const total = Number(result.total ?? 0);
|
||||
const bulkBusy = Boolean(dlBusyKey);
|
||||
|
||||
async function downloadOne(file: DigitadorJobFile) {
|
||||
if (!selectedId || bulkBusy) return;
|
||||
const busyId = `file:${file.key}`;
|
||||
setDlBusyKey(busyId);
|
||||
setDlStatus(null);
|
||||
setError(null);
|
||||
try {
|
||||
const data = await client.fetchDigitadorObject(selectedId, file.key);
|
||||
const name = reportBasename(file);
|
||||
const ok = await saveReportAs(name, data);
|
||||
if (ok) setDlStatus(`Guardado: ${name}`);
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : "No se pudo descargar el reporte.");
|
||||
} finally {
|
||||
setDlBusyKey(null);
|
||||
}
|
||||
}
|
||||
|
||||
if (runs.length === 0 && !loading) {
|
||||
return (
|
||||
<div className="results-empty">
|
||||
<Keyboard size={28} />
|
||||
<p>Aún no hay corridas de bot digitador en este trabajo.</p>
|
||||
<button type="button" className="btn btn-primary" onClick={onStartDigitador}>
|
||||
<Plus size={15} /> Nueva digitación
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="results-section">
|
||||
<div className="results-section-header">
|
||||
<div className="results-section-header-left">
|
||||
<label className="wiz-field-label">Corrida</label>
|
||||
<select
|
||||
className="form-input"
|
||||
value={selectedId}
|
||||
onChange={(e) => {
|
||||
setSelectedId(e.target.value);
|
||||
setDlStatus(null);
|
||||
}}
|
||||
style={{ minWidth: 280 }}
|
||||
>
|
||||
{runs.map((r) => (
|
||||
<option key={r.id} value={r.id}>
|
||||
{r.title || r.original_filename || r.id.slice(0, 8)} · {r.status}
|
||||
{r.shard_count && Number(r.shard_count) > 1
|
||||
? ` · ${r.shard_count} bots`
|
||||
: r.worker_id
|
||||
? ` · ${r.worker_id}`
|
||||
: ""}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
{selected && (
|
||||
<span className={`status-badge status-badge--${selected.status}`}>
|
||||
{selected.status}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="results-section-header-right">
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={() => void loadRuns()} disabled={loading}>
|
||||
{loading ? <Loader2 size={13} className="spin" /> : <RefreshCw size={13} />}
|
||||
Actualizar
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm btn-primary" onClick={onStartDigitador}>
|
||||
<Plus size={13} /> Nueva digitación
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="wiz-toast wiz-toast--error" style={{ position: "static", marginBottom: 12 }}>
|
||||
<AlertCircle size={15} />
|
||||
<span>{error}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{selected?.error && (
|
||||
<div className="wiz-toast wiz-toast--error" style={{ position: "static", marginBottom: 12 }}>
|
||||
<AlertCircle size={15} />
|
||||
<span>{selected.error}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{dlStatus && (
|
||||
<div className="wiz-toast" style={{ position: "static", marginBottom: 12 }}>
|
||||
<CheckCircle2 size={15} />
|
||||
<span>{dlStatus}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{selected && (
|
||||
<>
|
||||
<div className="wiz-stats-row" style={{ marginBottom: 16, flexShrink: 0 }}>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{exitosos}</span>
|
||||
<span className="wiz-stat-label">OK</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{fallidos}</span>
|
||||
<span className="wiz-stat-label">Fallidos</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{existentes}</span>
|
||||
<span className="wiz-stat-label">Existentes</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{timeouts}</span>
|
||||
<span className="wiz-stat-label">Timeouts</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{total || "—"}</span>
|
||||
<span className="wiz-stat-label">Total</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p className="wiz-hint" style={{ marginBottom: 12, flexShrink: 0 }}>
|
||||
{selected.shard_count && Number(selected.shard_count) > 1
|
||||
? `${selected.shard_count} bots (distribuido)`
|
||||
: (
|
||||
<>
|
||||
Worker <strong>{selected.worker_id ?? "—"}</strong>
|
||||
</>
|
||||
)}
|
||||
{selected.original_filename ? ` · ${selected.original_filename}` : ""}
|
||||
{selected.reports_prefix ? (
|
||||
<>
|
||||
{" "}
|
||||
· Prefijo <code className="dl-prefix-code">{selected.reports_prefix}</code>
|
||||
</>
|
||||
) : null}
|
||||
</p>
|
||||
|
||||
{(() => {
|
||||
const shardList = Array.isArray(selected.shards)
|
||||
? selected.shards
|
||||
: Array.isArray(result.shards)
|
||||
? (result.shards as Array<{
|
||||
id?: string;
|
||||
worker_id?: string | null;
|
||||
status?: string;
|
||||
shard_index?: number | null;
|
||||
}>)
|
||||
: [];
|
||||
if (shardList.length === 0) return null;
|
||||
return (
|
||||
<div className="wiz-section" style={{ marginBottom: 12, flexShrink: 0 }}>
|
||||
<h3 className="wiz-section-title">Estado por bot</h3>
|
||||
<div className="wiz-review-grid" style={{ marginTop: 8 }}>
|
||||
{shardList.map((s, i) => (
|
||||
<div key={s.id || String(i)} className="wiz-review-item">
|
||||
<div>
|
||||
<span className="wiz-review-label">
|
||||
Shard {(s.shard_index ?? i) + 1}
|
||||
</span>
|
||||
<span className="wiz-review-value">{s.worker_id || "—"}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span className="wiz-review-label">Estado</span>
|
||||
<span className="wiz-review-value">{s.status || "—"}</span>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
|
||||
<div className="wiz-section dl-files-section">
|
||||
<div className="dl-files-header">
|
||||
<h3 className="wiz-section-title" style={{ margin: 0 }}>
|
||||
Reportes en MinIO
|
||||
{files.length > 0 ? ` · ${files.length} archivo${files.length === 1 ? "" : "s"}` : ""}
|
||||
</h3>
|
||||
</div>
|
||||
|
||||
{filesError && (
|
||||
<div className="wiz-toast wiz-toast--error" style={{ position: "static", marginBottom: 12 }}>
|
||||
<AlertCircle size={15} />
|
||||
<span>No se pudo listar MinIO: {filesError}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!filesError && files.length === 0 ? (
|
||||
<p className="wiz-hint">Sin reportes listados aún (RESULTADOS / OMITIDOS).</p>
|
||||
) : (
|
||||
<table className="dl-files-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Archivo</th>
|
||||
<th className="dl-files-table-actions"> </th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{files.map((f) => {
|
||||
const fileBusy = dlBusyKey === `file:${f.key}`;
|
||||
const name = reportBasename(f);
|
||||
return (
|
||||
<tr key={f.key}>
|
||||
<td className="dl-col-file" title={f.key}>
|
||||
{name}
|
||||
</td>
|
||||
<td className="dl-files-table-actions">
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-ghost"
|
||||
disabled={bulkBusy}
|
||||
onClick={() => void downloadOne(f)}
|
||||
title="Descargar reporte"
|
||||
>
|
||||
{fileBusy ? (
|
||||
<Loader2 size={13} className="spin" />
|
||||
) : (
|
||||
<Download size={13} />
|
||||
)}
|
||||
Descargar
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="wiz-section">
|
||||
<h3 className="wiz-section-title">Log</h3>
|
||||
<div className="wiz-estimate-box dl-log-box">
|
||||
{events.length === 0 ? (
|
||||
<span className="wiz-hint">Sin eventos.</span>
|
||||
) : (
|
||||
events.map((ev) => (
|
||||
<div key={ev.id}>
|
||||
[{ev.timestamp}] {ev.message}
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,760 @@
|
||||
import {
|
||||
AlertCircle,
|
||||
Bot,
|
||||
CheckCircle2,
|
||||
FileSpreadsheet,
|
||||
Keyboard,
|
||||
Loader2,
|
||||
Play,
|
||||
RefreshCw,
|
||||
Square,
|
||||
Upload,
|
||||
Users,
|
||||
} from "lucide-react";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import type { ApiClient } from "../../lib/apiClient";
|
||||
import type {
|
||||
DigitadorJob,
|
||||
DigitadorJobEvent,
|
||||
DigitadorWorker,
|
||||
Job,
|
||||
PoblacionAptaJob,
|
||||
} from "../../../shared/types";
|
||||
|
||||
export type DigitadorSourceMode = "aptos" | "archivo";
|
||||
|
||||
export type DigitadorInitialSource = {
|
||||
poblacionAptaJobId: string;
|
||||
};
|
||||
|
||||
interface Props {
|
||||
client: ApiClient;
|
||||
job: Job;
|
||||
initialSource?: DigitadorInitialSource | null;
|
||||
}
|
||||
|
||||
type DigitadorStep = 1 | 2;
|
||||
|
||||
type ShardInfo = {
|
||||
id?: string;
|
||||
worker_id?: string | null;
|
||||
status?: string;
|
||||
shard_index?: number | null;
|
||||
result?: Record<string, unknown>;
|
||||
error?: string | null;
|
||||
};
|
||||
|
||||
export function DigitadorWizardModule({ client, job, initialSource = null }: Props) {
|
||||
const [step, setStep] = useState<DigitadorStep>(1);
|
||||
const [workers, setWorkers] = useState<DigitadorWorker[]>([]);
|
||||
const [selectedIds, setSelectedIds] = useState<string[]>([]);
|
||||
const [useAllOnline, setUseAllOnline] = useState(true);
|
||||
const [sourceMode, setSourceMode] = useState<DigitadorSourceMode>(
|
||||
initialSource?.poblacionAptaJobId ? "aptos" : "aptos",
|
||||
);
|
||||
const [paJobs, setPaJobs] = useState<PoblacionAptaJob[]>([]);
|
||||
const [selectedPaId, setSelectedPaId] = useState(initialSource?.poblacionAptaJobId ?? "");
|
||||
const [loadingPa, setLoadingPa] = useState(false);
|
||||
const [file, setFile] = useState<File | null>(null);
|
||||
const [title, setTitle] = useState("");
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [activeJob, setActiveJob] = useState<DigitadorJob | null>(null);
|
||||
const [events, setEvents] = useState<DigitadorJobEvent[]>([]);
|
||||
const sinceRef = useRef(0);
|
||||
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const fileInputRef = useRef<HTMLInputElement | null>(null);
|
||||
const initializedSelectRef = useRef(false);
|
||||
const appliedInitialRef = useRef<string | null>(null);
|
||||
|
||||
const onlineWorkers = useMemo(
|
||||
() => workers.filter((w) => w.status === "online" || w.status === "busy"),
|
||||
[workers],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!initialSource?.poblacionAptaJobId) return;
|
||||
const key = initialSource.poblacionAptaJobId;
|
||||
if (appliedInitialRef.current === key) return;
|
||||
appliedInitialRef.current = key;
|
||||
setSourceMode("aptos");
|
||||
setSelectedPaId(initialSource.poblacionAptaJobId);
|
||||
setStep(1);
|
||||
}, [initialSource?.poblacionAptaJobId]);
|
||||
|
||||
const refreshWorkers = useCallback(async () => {
|
||||
setRefreshing(true);
|
||||
try {
|
||||
const list = await client.listDigitadorWorkers();
|
||||
setWorkers(list);
|
||||
setError(null);
|
||||
const online = list.filter((w) => w.status === "online" || w.status === "busy");
|
||||
if (!initializedSelectRef.current && online.length > 0) {
|
||||
initializedSelectRef.current = true;
|
||||
setUseAllOnline(online.length >= 2);
|
||||
setSelectedIds(online.map((w) => w.id));
|
||||
} else if (useAllOnline) {
|
||||
setSelectedIds(online.map((w) => w.id));
|
||||
} else {
|
||||
setSelectedIds((prev) => prev.filter((id) => online.some((w) => w.id === id)));
|
||||
}
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setRefreshing(false);
|
||||
}
|
||||
}, [client, useAllOnline]);
|
||||
|
||||
useEffect(() => {
|
||||
void refreshWorkers();
|
||||
const t = setInterval(() => void refreshWorkers(), 5000);
|
||||
return () => clearInterval(t);
|
||||
}, [refreshWorkers]);
|
||||
|
||||
const loadPaJobs = useCallback(async () => {
|
||||
setLoadingPa(true);
|
||||
try {
|
||||
const list = await client.poblacionAptaJobs(job.id);
|
||||
const completed = list
|
||||
.filter((j) => j.status === "completed")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
setPaJobs(completed);
|
||||
setSelectedPaId((prev) => {
|
||||
if (prev && completed.some((j) => j.id === prev)) return prev;
|
||||
return completed[0]?.id ?? "";
|
||||
});
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setLoadingPa(false);
|
||||
}
|
||||
}, [client, job.id]);
|
||||
|
||||
useEffect(() => {
|
||||
if (sourceMode !== "aptos") return;
|
||||
void loadPaJobs();
|
||||
}, [sourceMode, loadPaJobs]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!activeJob || ["completed", "failed", "cancelled"].includes(activeJob.status)) {
|
||||
if (pollRef.current) {
|
||||
clearInterval(pollRef.current);
|
||||
pollRef.current = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
async function poll() {
|
||||
if (!activeJob) return;
|
||||
try {
|
||||
const nextJob = await client.getDigitadorJob(activeJob.id);
|
||||
setActiveJob(nextJob);
|
||||
const { events: next } = await client.getDigitadorEvents(activeJob.id, sinceRef.current);
|
||||
if (next.length) {
|
||||
setEvents((prev) => [...prev, ...next]);
|
||||
sinceRef.current = next[next.length - 1].id;
|
||||
}
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}
|
||||
|
||||
void poll();
|
||||
pollRef.current = setInterval(() => void poll(), 3000);
|
||||
return () => {
|
||||
if (pollRef.current) clearInterval(pollRef.current);
|
||||
};
|
||||
}, [activeJob?.id, activeJob?.status, client]);
|
||||
|
||||
function toggleWorker(id: string) {
|
||||
if (useAllOnline) return;
|
||||
setSelectedIds((prev) =>
|
||||
prev.includes(id) ? prev.filter((x) => x !== id) : [...prev, id],
|
||||
);
|
||||
}
|
||||
|
||||
const selectedPa = paJobs.find((j) => j.id === selectedPaId) ?? null;
|
||||
const aptosCount = Number(selectedPa?.summary?.paso_bot ?? 0);
|
||||
|
||||
async function startJob() {
|
||||
const targets = useAllOnline ? onlineWorkers.map((w) => w.id) : selectedIds;
|
||||
if (targets.length === 0) {
|
||||
setError("Necesitas al menos un bot online.");
|
||||
return;
|
||||
}
|
||||
if (sourceMode === "archivo") {
|
||||
if (!file) {
|
||||
setError("Selecciona un CSV.");
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
if (!selectedPaId) {
|
||||
setError("Selecciona una corrida de población apta.");
|
||||
return;
|
||||
}
|
||||
if (aptosCount <= 0) {
|
||||
setError("Esa corrida no tiene aptos (PASO_BOT) para digitar.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
setBusy(true);
|
||||
setError(null);
|
||||
try {
|
||||
const form = new FormData();
|
||||
form.append("job_id", job.id);
|
||||
if (sourceMode === "archivo" && file) {
|
||||
form.append("file", file);
|
||||
} else {
|
||||
form.append("poblacion_apta_job_id", selectedPaId);
|
||||
form.append("subset", "aptos");
|
||||
}
|
||||
if (title.trim()) form.append("title", title.trim());
|
||||
if (targets.length === 1) {
|
||||
form.append("worker_id", targets[0]);
|
||||
} else {
|
||||
form.append("worker_ids", JSON.stringify(targets));
|
||||
}
|
||||
const created = await client.createDigitadorJob(form);
|
||||
sinceRef.current = 0;
|
||||
setEvents([]);
|
||||
setActiveJob(created);
|
||||
setStep(2);
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function cancelJob() {
|
||||
if (!activeJob) return;
|
||||
setBusy(true);
|
||||
try {
|
||||
const updated = await client.cancelDigitadorJob(activeJob.id);
|
||||
setActiveJob(updated);
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
function resetWizard() {
|
||||
setStep(1);
|
||||
setActiveJob(null);
|
||||
setEvents([]);
|
||||
setFile(null);
|
||||
setTitle("");
|
||||
setError(null);
|
||||
sinceRef.current = 0;
|
||||
if (fileInputRef.current) fileInputRef.current.value = "";
|
||||
}
|
||||
|
||||
const result = activeJob?.result ?? {};
|
||||
const processed = Number(result.processed ?? result.exitosos ?? 0);
|
||||
const total = Number(result.total ?? result.total_rows ?? 0);
|
||||
const pct = total > 0 ? Math.min(100, Math.round((processed / total) * 100)) : 0;
|
||||
const running = Boolean(
|
||||
activeJob && !["completed", "failed", "cancelled"].includes(activeJob.status),
|
||||
);
|
||||
const isDistributed =
|
||||
Boolean(activeJob?.shard_count && Number(activeJob.shard_count) > 1) ||
|
||||
result.mode === "distributed";
|
||||
const shards: ShardInfo[] = Array.isArray(activeJob?.shards)
|
||||
? (activeJob!.shards as ShardInfo[])
|
||||
: Array.isArray(result.shards)
|
||||
? (result.shards as ShardInfo[])
|
||||
: [];
|
||||
const effectiveTargets = useAllOnline ? onlineWorkers.map((w) => w.id) : selectedIds;
|
||||
const sourceReady =
|
||||
sourceMode === "archivo" ? Boolean(file) : Boolean(selectedPaId) && aptosCount > 0;
|
||||
const canStart = effectiveTargets.length > 0 && sourceReady && !busy;
|
||||
|
||||
return (
|
||||
<div className="wiz">
|
||||
<nav className="stepper" aria-label="Pasos del bot digitador">
|
||||
<div className="stepper-item">
|
||||
<button
|
||||
type="button"
|
||||
className={`stepper-dot ${step === 1 ? "active" : step > 1 ? "done" : "todo"}`}
|
||||
onClick={() => !running && setStep(1)}
|
||||
disabled={running}
|
||||
>
|
||||
1
|
||||
</button>
|
||||
<span className={`stepper-label ${step === 1 ? "active" : step > 1 ? "done" : "todo"}`}>
|
||||
Configurar
|
||||
</span>
|
||||
<div className={`stepper-connector ${step > 1 ? "done" : ""}`} />
|
||||
</div>
|
||||
<div className="stepper-item">
|
||||
<button
|
||||
type="button"
|
||||
className={`stepper-dot ${step === 2 ? "active" : "todo"}`}
|
||||
onClick={() => activeJob && setStep(2)}
|
||||
disabled={!activeJob}
|
||||
>
|
||||
2
|
||||
</button>
|
||||
<span className={`stepper-label ${step === 2 ? "active" : "todo"}`}>Ejecución</span>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<div className="wiz-body">
|
||||
{step === 1 && (
|
||||
<div className="wiz-step">
|
||||
<div className="wiz-header" style={{ marginBottom: 4 }}>
|
||||
<h2 style={{ display: "flex", alignItems: "center", gap: 8, margin: 0 }}>
|
||||
<Keyboard size={18} />
|
||||
Bot digitador
|
||||
</h2>
|
||||
<p className="wiz-desc" style={{ marginTop: 6 }}>
|
||||
Digita en Integración Total solo los <strong>aptos (PASO_BOT)</strong>, o un CSV
|
||||
propio con el mismo formato.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* 1. Origen */}
|
||||
<div className="wiz-section">
|
||||
<h3 className="wiz-section-title">1. Origen</h3>
|
||||
<div style={{ display: "flex", flexWrap: "wrap", gap: 8, marginBottom: 14 }}>
|
||||
<button
|
||||
type="button"
|
||||
className={`btn${sourceMode === "aptos" ? " btn-primary" : ""}`}
|
||||
onClick={() => setSourceMode("aptos")}
|
||||
>
|
||||
<Users size={14} /> Aptos de población apta
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`btn${sourceMode === "archivo" ? " btn-primary" : ""}`}
|
||||
onClick={() => setSourceMode("archivo")}
|
||||
>
|
||||
<Upload size={14} /> Subir CSV
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<label className="wiz-field-label">Título (opcional)</label>
|
||||
<input
|
||||
className="form-input"
|
||||
value={title}
|
||||
onChange={(e) => setTitle(e.target.value)}
|
||||
placeholder="Ej. Digitación lote mayo 2026"
|
||||
style={{ width: "100%", marginBottom: 14 }}
|
||||
/>
|
||||
|
||||
{sourceMode === "aptos" ? (
|
||||
<>
|
||||
<div className="wiz-section-header" style={{ marginBottom: 8 }}>
|
||||
<label className="wiz-field-label" style={{ margin: 0 }}>
|
||||
Corrida de población apta
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-ghost"
|
||||
onClick={() => void loadPaJobs()}
|
||||
disabled={loadingPa}
|
||||
>
|
||||
{loadingPa ? <Loader2 size={13} className="spin" /> : <RefreshCw size={13} />}
|
||||
Actualizar
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{loadingPa && paJobs.length === 0 ? (
|
||||
<p className="wiz-hint">
|
||||
<Loader2 size={13} className="spin" /> Cargando corridas…
|
||||
</p>
|
||||
) : paJobs.length === 0 ? (
|
||||
<div className="wiz-estimate-box" style={{ textAlign: "left" }}>
|
||||
<p className="wiz-desc" style={{ margin: 0 }}>
|
||||
No hay corridas de población apta completadas en este trabajo.
|
||||
</p>
|
||||
<p className="wiz-hint" style={{ marginTop: 6, marginBottom: 0 }}>
|
||||
Ejecuta primero Población apta, o cambia a «Subir CSV».
|
||||
</p>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<select
|
||||
className="form-input"
|
||||
value={selectedPaId}
|
||||
onChange={(e) => setSelectedPaId(e.target.value)}
|
||||
style={{ width: "100%" }}
|
||||
>
|
||||
{paJobs.map((pa) => {
|
||||
const n = Number(pa.summary?.paso_bot ?? 0);
|
||||
return (
|
||||
<option key={pa.id} value={pa.id}>
|
||||
{pa.id.slice(0, 8)} · {n} aptos · {pa.summary?.total ?? 0} total
|
||||
</option>
|
||||
);
|
||||
})}
|
||||
</select>
|
||||
|
||||
{selectedPa && (
|
||||
<div
|
||||
className="wiz-estimate-box"
|
||||
style={{
|
||||
marginTop: 12,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 12,
|
||||
textAlign: "left",
|
||||
}}
|
||||
>
|
||||
{aptosCount > 0 ? (
|
||||
<CheckCircle2 size={20} style={{ color: "var(--success, #16a34a)", flexShrink: 0 }} />
|
||||
) : (
|
||||
<AlertCircle size={20} style={{ color: "var(--danger, #dc2626)", flexShrink: 0 }} />
|
||||
)}
|
||||
<div>
|
||||
<div style={{ fontWeight: 600 }}>
|
||||
{aptosCount > 0
|
||||
? `${aptosCount} registros aptos listos para digitar`
|
||||
: "Sin aptos en esta corrida"}
|
||||
</div>
|
||||
<div className="wiz-hint" style={{ margin: 0 }}>
|
||||
Solo se envían filas PASO_BOT. Los no aptos no se digitan.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<label className="wiz-field-label">Archivo CSV</label>
|
||||
<div className="folder-picker">
|
||||
<div className="folder-picker-display">
|
||||
{file ? (
|
||||
<>
|
||||
<FileSpreadsheet size={16} className="folder-picker-icon" />
|
||||
<span className="folder-picker-path">{file.name}</span>
|
||||
</>
|
||||
) : (
|
||||
<span className="folder-picker-placeholder">Ningún archivo seleccionado</span>
|
||||
)}
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-primary"
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
>
|
||||
<FileSpreadsheet size={15} />
|
||||
Seleccionar CSV
|
||||
</button>
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept=".csv,text/csv"
|
||||
hidden
|
||||
onChange={(e) => setFile(e.target.files?.[0] ?? null)}
|
||||
/>
|
||||
</div>
|
||||
<p className="wiz-hint" style={{ marginTop: 8 }}>
|
||||
Columnas: radicado, documento, ingreso, total_cargos, autorizacion,
|
||||
tipo_documento, tipo_servicio, copago, cuota_moderadora (también acepta
|
||||
headers IT).
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* 2. Bots */}
|
||||
<div className="wiz-section">
|
||||
<div className="wiz-section-header">
|
||||
<h3 className="wiz-section-title">
|
||||
2. Bots
|
||||
{onlineWorkers.length > 0 && (
|
||||
<span className="wiz-count-badge">{onlineWorkers.length} online</span>
|
||||
)}
|
||||
</h3>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-ghost"
|
||||
onClick={() => void refreshWorkers()}
|
||||
disabled={refreshing}
|
||||
>
|
||||
{refreshing ? <Loader2 size={13} className="spin" /> : <RefreshCw size={13} />}
|
||||
Actualizar
|
||||
</button>
|
||||
</div>
|
||||
<p className="wiz-desc">
|
||||
El agent debe estar encendido en la PC con Integración Total. Con varios bots, las
|
||||
filas se reparte automáticamente.
|
||||
</p>
|
||||
|
||||
{onlineWorkers.length >= 2 && (
|
||||
<label
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 8,
|
||||
marginTop: 12,
|
||||
marginBottom: 8,
|
||||
cursor: "pointer",
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={useAllOnline}
|
||||
onChange={(e) => {
|
||||
const on = e.target.checked;
|
||||
setUseAllOnline(on);
|
||||
if (on) setSelectedIds(onlineWorkers.map((w) => w.id));
|
||||
}}
|
||||
/>
|
||||
<span>Usar todos los bots online ({onlineWorkers.length})</span>
|
||||
</label>
|
||||
)}
|
||||
|
||||
{workers.length === 0 ? (
|
||||
<div className="wiz-estimate-box" style={{ marginTop: 12, textAlign: "left" }}>
|
||||
<div style={{ display: "flex", alignItems: "flex-start", gap: 10 }}>
|
||||
<Bot size={18} style={{ flexShrink: 0, marginTop: 2 }} />
|
||||
<div>
|
||||
<div style={{ fontWeight: 600 }}>Ningún bot conectado</div>
|
||||
<p className="wiz-hint" style={{ margin: "4px 0 0" }}>
|
||||
En la PC del digitador ejecuta <code>run_gui.py</code> del agent y conéctalo
|
||||
al gateway.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="wiz-review-grid" style={{ marginTop: 12 }}>
|
||||
{workers.map((w) => {
|
||||
const online = w.status === "online" || w.status === "busy";
|
||||
const selected = effectiveTargets.includes(w.id);
|
||||
return (
|
||||
<button
|
||||
key={w.id}
|
||||
type="button"
|
||||
className={`wiz-review-item${selected ? " wiz-review-item--selected" : ""}`}
|
||||
disabled={!online || useAllOnline}
|
||||
onClick={() => online && toggleWorker(w.id)}
|
||||
style={{
|
||||
textAlign: "left",
|
||||
cursor: online && !useAllOnline ? "pointer" : "not-allowed",
|
||||
opacity: online ? 1 : 0.45,
|
||||
borderColor: selected ? "var(--accent, #3b8ed0)" : undefined,
|
||||
}}
|
||||
>
|
||||
<div>
|
||||
<span className="wiz-review-label">Worker</span>
|
||||
<span className="wiz-review-value">{w.id}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span className="wiz-review-label">Estado</span>
|
||||
<span className="wiz-review-value">
|
||||
{w.status}
|
||||
{w.hostname ? ` · ${w.hostname}` : ""}
|
||||
{selected ? " · seleccionado" : ""}
|
||||
</span>
|
||||
</div>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
{onlineWorkers.length === 0 && workers.length > 0 && (
|
||||
<p className="wiz-hint" style={{ marginTop: 8 }}>
|
||||
Hay workers registrados pero ninguno online. Revisa el agent.
|
||||
</p>
|
||||
)}
|
||||
{effectiveTargets.length > 1 && (
|
||||
<p className="wiz-hint" style={{ marginTop: 8 }}>
|
||||
Se repartirá entre {effectiveTargets.length} bots.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{step === 2 && activeJob && (
|
||||
<div className="wiz-step wiz-step-centered">
|
||||
<div className="wiz-hero">
|
||||
<div className={`wiz-hero-icon ${running ? "wiz-hero-icon--go" : ""}`}>
|
||||
{running ? (
|
||||
<Loader2 size={28} className="spin" />
|
||||
) : activeJob.status === "completed" ? (
|
||||
<Keyboard size={28} />
|
||||
) : activeJob.status === "failed" ? (
|
||||
<AlertCircle size={28} />
|
||||
) : (
|
||||
<Bot size={28} />
|
||||
)}
|
||||
</div>
|
||||
<h3 className="wiz-hero-title">
|
||||
{running
|
||||
? isDistributed
|
||||
? "Digitación distribuida en curso"
|
||||
: "Digitación en curso"
|
||||
: activeJob.status === "completed"
|
||||
? "Digitación completada"
|
||||
: activeJob.status === "cancelled"
|
||||
? "Digitación cancelada"
|
||||
: activeJob.status === "failed"
|
||||
? "Digitación fallida"
|
||||
: "Digitación finalizada"}
|
||||
</h3>
|
||||
<p className="wiz-desc">
|
||||
{isDistributed
|
||||
? `${Number(activeJob.shard_count) || shards.length || "?"} bots`
|
||||
: `Bot ${activeJob.worker_id ?? "—"}`}
|
||||
{activeJob.original_filename ? ` · ${activeJob.original_filename}` : ""}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{activeJob.error && (
|
||||
<div className="wiz-toast wiz-toast--error" style={{ position: "static", marginBottom: 12 }}>
|
||||
<AlertCircle size={15} />
|
||||
<span>{activeJob.error}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="wiz-progress">
|
||||
<div className="wiz-progress-header">
|
||||
<span>
|
||||
{Number(result.exitosos ?? processed)}/{total || "?"} filas
|
||||
</span>
|
||||
<span>{pct}%</span>
|
||||
</div>
|
||||
<div className="wiz-progress-track">
|
||||
<div className="wiz-progress-fill" style={{ width: `${pct}%` }} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="wiz-stats-row">
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{String(result.exitosos ?? processed)}</span>
|
||||
<span className="wiz-stat-label">OK</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{String(result.fallidos ?? 0)}</span>
|
||||
<span className="wiz-stat-label">Fallidos</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{String(result.existentes ?? 0)}</span>
|
||||
<span className="wiz-stat-label">Existentes</span>
|
||||
</div>
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{String(result.timeouts ?? 0)}</span>
|
||||
<span className="wiz-stat-label">Timeouts</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{shards.length > 0 && (
|
||||
<div className="wiz-section" style={{ marginTop: 16, width: "100%", maxWidth: 640 }}>
|
||||
<h3 className="wiz-section-title">Shards por bot</h3>
|
||||
<div className="wiz-review-grid" style={{ marginTop: 8 }}>
|
||||
{shards.map((s, i) => (
|
||||
<div key={s.id || String(i)} className="wiz-review-item">
|
||||
<div>
|
||||
<span className="wiz-review-label">
|
||||
Shard {(s.shard_index ?? i) + 1}
|
||||
</span>
|
||||
<span className="wiz-review-value">{s.worker_id || "—"}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span className="wiz-review-label">Estado</span>
|
||||
<span className="wiz-review-value">{s.status || "—"}</span>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="wiz-section" style={{ marginTop: 16, width: "100%", maxWidth: 640 }}>
|
||||
<h3 className="wiz-section-title">Log del agent</h3>
|
||||
<div
|
||||
className="wiz-estimate-box"
|
||||
style={{
|
||||
maxHeight: 220,
|
||||
overflow: "auto",
|
||||
fontFamily: "ui-monospace, SFMono-Regular, Menlo, monospace",
|
||||
fontSize: 12,
|
||||
textAlign: "left",
|
||||
}}
|
||||
>
|
||||
{events.length === 0 ? (
|
||||
<span className="wiz-hint">Esperando eventos…</span>
|
||||
) : (
|
||||
events.map((ev) => (
|
||||
<div key={ev.id}>
|
||||
[{ev.timestamp}] {ev.message}
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="wiz-toast wiz-toast--error">
|
||||
<AlertCircle size={15} />
|
||||
<span>{error}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<footer className="wiz-nav">
|
||||
{step === 2 ? (
|
||||
<button type="button" className="btn" onClick={resetWizard} disabled={running}>
|
||||
Nueva digitación
|
||||
</button>
|
||||
) : (
|
||||
<span />
|
||||
)}
|
||||
|
||||
<div className="wiz-nav-dots">
|
||||
<button
|
||||
type="button"
|
||||
className={`wiz-nav-dot ${step === 1 ? "active" : ""}`}
|
||||
onClick={() => !running && setStep(1)}
|
||||
disabled={running}
|
||||
aria-label="Paso 1"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
className={`wiz-nav-dot ${step === 2 ? "active" : ""}`}
|
||||
onClick={() => activeJob && setStep(2)}
|
||||
disabled={!activeJob}
|
||||
aria-label="Paso 2"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{step === 1 && (
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-primary"
|
||||
disabled={!canStart}
|
||||
onClick={() => void startJob()}
|
||||
>
|
||||
{busy ? <Loader2 size={15} className="spin" /> : <Play size={15} />}
|
||||
{effectiveTargets.length > 1
|
||||
? `Digitar con ${effectiveTargets.length} bots`
|
||||
: sourceMode === "aptos" && aptosCount > 0
|
||||
? `Digitar ${aptosCount} aptos`
|
||||
: "Iniciar digitación"}
|
||||
</button>
|
||||
)}
|
||||
{step === 2 && running && (
|
||||
<button type="button" className="btn" disabled={busy} onClick={() => void cancelJob()}>
|
||||
<Square size={14} /> Cancelar
|
||||
</button>
|
||||
)}
|
||||
{step === 2 && !running && (
|
||||
<button type="button" className="btn btn-primary" onClick={resetWizard}>
|
||||
<Play size={15} /> Otra digitación
|
||||
</button>
|
||||
)}
|
||||
</footer>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Download, Loader2, RefreshCw, Users } from "lucide-react";
|
||||
import { Download, Keyboard, Loader2, RefreshCw, Users } from "lucide-react";
|
||||
import { useEffect, useState } from "react";
|
||||
import type { PoblacionAptaJob } from "../../../shared/types";
|
||||
import { PoblacionAptaResultsTable } from "../../components/PoblacionAptaResultsTable";
|
||||
@@ -7,7 +7,7 @@ import type { ModuleResultsProps } from "../registry";
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps) {
|
||||
export function PoblacionAptaResults({ client, task, job, onActivateModule }: ModuleResultsProps) {
|
||||
const [paJob, setPaJob] = useState<PoblacionAptaJob | null>(null);
|
||||
const [rows, setRows] = useState<Row[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
@@ -60,6 +60,16 @@ export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps)
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text);
|
||||
}
|
||||
|
||||
async function exportTodos() {
|
||||
const paJobId = String(
|
||||
(task.result as Record<string, unknown>)?.poblacion_apta_job_id ?? "",
|
||||
);
|
||||
if (!paJobId) return;
|
||||
const blob = await client.poblacionAptaExportCsv(paJobId, "todos");
|
||||
const text = await blob.text();
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_todos", job.title), text);
|
||||
}
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="module-loading">
|
||||
@@ -88,6 +98,10 @@ export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps)
|
||||
}
|
||||
|
||||
const summary = paJob?.summary ?? {};
|
||||
const paJobId = String(
|
||||
(task.result as Record<string, unknown>)?.poblacion_apta_job_id ?? paJob?.id ?? "",
|
||||
);
|
||||
const canDigitador = paJob?.status === "completed" && Boolean(paJobId) && Boolean(onActivateModule);
|
||||
|
||||
return (
|
||||
<div className="module-results">
|
||||
@@ -108,6 +122,23 @@ export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps)
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportNoAptos()}>
|
||||
<Download size={13} /> CSV no aptos ({summary.con_error ?? 0})
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportTodos()}>
|
||||
<Download size={13} /> CSV todos ({summary.total ?? 0})
|
||||
</button>
|
||||
{canDigitador && (
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-primary"
|
||||
disabled={Number(summary.paso_bot ?? 0) === 0}
|
||||
onClick={() =>
|
||||
onActivateModule?.("digitador", {
|
||||
poblacionAptaJobId: paJobId,
|
||||
})
|
||||
}
|
||||
>
|
||||
<Keyboard size={13} /> Digitar aptos
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
CheckSquare,
|
||||
Download,
|
||||
FileSpreadsheet,
|
||||
Keyboard,
|
||||
Loader2,
|
||||
Square,
|
||||
Upload,
|
||||
@@ -235,6 +236,13 @@ export function PoblacionAptaWizard({
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text);
|
||||
}
|
||||
|
||||
async function exportTodos() {
|
||||
if (!resultJob) return;
|
||||
const blob = await client.poblacionAptaExportCsv(resultJob.id, "todos");
|
||||
const text = await blob.text();
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_todos", job.title), text);
|
||||
}
|
||||
|
||||
function reset() {
|
||||
setDone(false);
|
||||
setResultJob(null);
|
||||
@@ -271,6 +279,23 @@ export function PoblacionAptaWizard({
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportNoAptos()}>
|
||||
<Download size={13} /> CSV no aptos ({summary.con_error ?? 0})
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportTodos()}>
|
||||
<Download size={13} /> CSV todos ({summary.total ?? 0})
|
||||
</button>
|
||||
{onActivateModule && (
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-primary"
|
||||
disabled={Number(summary.paso_bot ?? 0) === 0}
|
||||
onClick={() =>
|
||||
onActivateModule("digitador", {
|
||||
poblacionAptaJobId: resultJob.id,
|
||||
})
|
||||
}
|
||||
>
|
||||
<Keyboard size={13} /> Digitar aptos
|
||||
</button>
|
||||
)}
|
||||
<button type="button" className="btn btn-sm" onClick={reset}>
|
||||
Nueva corrida
|
||||
</button>
|
||||
|
||||
@@ -4,4 +4,5 @@ export type JobSection =
|
||||
| "cleaning"
|
||||
| "poblacion_apta"
|
||||
| "downloads"
|
||||
| "digitador"
|
||||
| "results";
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
Cpu,
|
||||
Download,
|
||||
Hash,
|
||||
Keyboard,
|
||||
Loader2,
|
||||
Plus,
|
||||
RefreshCw,
|
||||
@@ -29,8 +30,9 @@ import type { ApiClient } from "../lib/apiClient";
|
||||
import { downloadCsv, toCsv, buildExportCsvFilename } from "../lib/csv";
|
||||
import { taskTimingLabel, timingDuration } from "../lib/time";
|
||||
import { DownloadsResultsPanel } from "../modules/downloads/DownloadsResultsPanel";
|
||||
import { DigitadorResultsPanel } from "../modules/digitador/DigitadorResultsPanel";
|
||||
|
||||
type ResultTab = "counting" | "extraction" | "cleaning" | "poblacion_apta" | "downloads";
|
||||
type ResultTab = "counting" | "extraction" | "cleaning" | "poblacion_apta" | "downloads" | "digitador";
|
||||
|
||||
interface Props {
|
||||
client: ApiClient;
|
||||
@@ -46,11 +48,15 @@ interface Props {
|
||||
onStartCleaning: (ctx?: Record<string, unknown>) => void;
|
||||
onStartPoblacionApta: () => void;
|
||||
onStartDownload: () => void;
|
||||
onStartDigitador: () => void;
|
||||
onContinueToCounting?: (ctx: {
|
||||
downloadJobId: string;
|
||||
downloadsPrefix?: string;
|
||||
files?: DownloadJobFile[];
|
||||
}) => void;
|
||||
onContinueToDigitador?: (ctx: {
|
||||
poblacionAptaJobId: string;
|
||||
}) => void;
|
||||
}
|
||||
|
||||
export function JobResultsView({
|
||||
@@ -67,7 +73,9 @@ export function JobResultsView({
|
||||
onStartCleaning,
|
||||
onStartPoblacionApta,
|
||||
onStartDownload,
|
||||
onStartDigitador,
|
||||
onContinueToCounting,
|
||||
onContinueToDigitador,
|
||||
}: Props) {
|
||||
const [activeTab, setActiveTab] = useState<ResultTab>("counting");
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
@@ -82,6 +90,8 @@ export function JobResultsView({
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
const downloadTasks = tasks.filter((t) => t.type === "downloads")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
const digitadorTasks = tasks.filter((t) => t.type === "digitador")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
|
||||
async function refresh() {
|
||||
setRefreshing(true);
|
||||
@@ -164,6 +174,17 @@ export function JobResultsView({
|
||||
<span className="job-results-tab-count">{downloadTasks.length}</span>
|
||||
)}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`job-results-tab${activeTab === "digitador" ? " job-results-tab--active" : ""}`}
|
||||
onClick={() => setActiveTab("digitador")}
|
||||
>
|
||||
<Keyboard size={15} />
|
||||
Bot digitador
|
||||
{digitadorTasks.length > 0 && (
|
||||
<span className="job-results-tab-count">{digitadorTasks.length}</span>
|
||||
)}
|
||||
</button>
|
||||
<div style={{ flex: 1 }} />
|
||||
<button
|
||||
type="button"
|
||||
@@ -216,6 +237,7 @@ export function JobResultsView({
|
||||
tasks={poblacionAptaTasks}
|
||||
liveProgress={poblacionAptaProgress}
|
||||
onStartPoblacionApta={onStartPoblacionApta}
|
||||
onContinueToDigitador={onContinueToDigitador}
|
||||
/>
|
||||
)}
|
||||
{activeTab === "downloads" && (
|
||||
@@ -227,6 +249,14 @@ export function JobResultsView({
|
||||
onContinueToCounting={onContinueToCounting}
|
||||
/>
|
||||
)}
|
||||
{activeTab === "digitador" && (
|
||||
<DigitadorResultsPanel
|
||||
client={client}
|
||||
job={job}
|
||||
tasks={digitadorTasks}
|
||||
onStartDigitador={onStartDigitador}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
@@ -780,12 +810,16 @@ function PoblacionAptaResultsSection({
|
||||
tasks,
|
||||
liveProgress,
|
||||
onStartPoblacionApta,
|
||||
onContinueToDigitador,
|
||||
}: {
|
||||
client: ApiClient;
|
||||
job: Job;
|
||||
tasks: Task[];
|
||||
liveProgress?: BackgroundProgress | null;
|
||||
onStartPoblacionApta: () => void;
|
||||
onContinueToDigitador?: (ctx: {
|
||||
poblacionAptaJobId: string;
|
||||
}) => void;
|
||||
}) {
|
||||
const [selectedTask, setSelectedTask] = useState<Task | null>(tasks[0] ?? null);
|
||||
const [paJob, setPaJob] = useState<PoblacionAptaJob | null>(null);
|
||||
@@ -853,6 +887,16 @@ function PoblacionAptaResultsSection({
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text);
|
||||
}
|
||||
|
||||
async function exportTodos() {
|
||||
const paJobId = String(
|
||||
(selectedTask?.result as Record<string, unknown>)?.poblacion_apta_job_id ?? "",
|
||||
);
|
||||
if (!paJobId) return;
|
||||
const blob = await client.poblacionAptaExportCsv(paJobId, "todos");
|
||||
const text = await blob.text();
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_todos", job.title), text);
|
||||
}
|
||||
|
||||
if (tasks.length === 0 && !liveProgress?.running) {
|
||||
return (
|
||||
<div className="job-results-empty">
|
||||
@@ -874,6 +918,13 @@ function PoblacionAptaResultsSection({
|
||||
}
|
||||
|
||||
const summary = paJob?.summary ?? {};
|
||||
const paJobId = String(
|
||||
(selectedTask?.result as Record<string, unknown>)?.poblacion_apta_job_id ?? paJob?.id ?? "",
|
||||
);
|
||||
const canDigitador =
|
||||
Boolean(onContinueToDigitador) &&
|
||||
paJob?.status === "completed" &&
|
||||
Boolean(paJobId);
|
||||
|
||||
return (
|
||||
<div className="results-section">
|
||||
@@ -917,6 +968,21 @@ function PoblacionAptaResultsSection({
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportNoAptos()}>
|
||||
<Download size={13} /> CSV no aptos ({summary.con_error ?? 0})
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportTodos()}>
|
||||
<Download size={13} /> CSV todos ({summary.total ?? 0})
|
||||
</button>
|
||||
{canDigitador && (
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-primary"
|
||||
onClick={() =>
|
||||
onContinueToDigitador?.({ poblacionAptaJobId: paJobId })
|
||||
}
|
||||
disabled={Number(summary.paso_bot ?? 0) === 0}
|
||||
>
|
||||
<Keyboard size={13} /> Digitar aptos
|
||||
</button>
|
||||
)}
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={onStartPoblacionApta}>
|
||||
<Plus size={13} /> Nueva corrida
|
||||
</button>
|
||||
|
||||
@@ -9,6 +9,7 @@ import { CleaningWizardModule } from "../modules/cleaning/CleaningWizardModule";
|
||||
import { PoblacionAptaWizardModule } from "../modules/poblacion_apta/PoblacionAptaWizardModule";
|
||||
import { JobExtractionWizard } from "../modules/extraction/JobExtractionWizard";
|
||||
import { DownloadsWizardModule } from "../modules/downloads/DownloadsWizardModule";
|
||||
import { DigitadorWizardModule } from "../modules/digitador/DigitadorWizardModule";
|
||||
import { JobResultsView } from "./JobResultsView";
|
||||
|
||||
interface Props {
|
||||
@@ -127,6 +128,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
{section === "cleaning" && "Limpieza de datos"}
|
||||
{section === "poblacion_apta" && "Población apta"}
|
||||
{section === "downloads" && "Bot de descarga"}
|
||||
{section === "digitador" && "Bot digitador"}
|
||||
{section === "results" && "Resultados del trabajo"}
|
||||
</div>
|
||||
</header>
|
||||
@@ -201,6 +203,19 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
<DownloadsWizardModule client={client} job={job} />
|
||||
</div>
|
||||
|
||||
<div className={`job-section-pane${section === "digitador" ? "" : " job-section-pane--hidden"}`}>
|
||||
<DigitadorWizardModule
|
||||
key={`digitador-${String(sectionContext.poblacionAptaJobId ?? "default")}`}
|
||||
client={client}
|
||||
job={job}
|
||||
initialSource={
|
||||
typeof sectionContext.poblacionAptaJobId === "string"
|
||||
? { poblacionAptaJobId: sectionContext.poblacionAptaJobId }
|
||||
: null
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className={`job-section-pane${section === "results" ? "" : " job-section-pane--hidden"}`}>
|
||||
<JobResultsView
|
||||
client={client}
|
||||
@@ -216,7 +231,9 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
onStartCleaning={(ctx?: Record<string, unknown>) => onActivateSection("cleaning", ctx)}
|
||||
onStartPoblacionApta={() => onActivateSection("poblacion_apta")}
|
||||
onStartDownload={() => onActivateSection("downloads")}
|
||||
onStartDigitador={() => onActivateSection("digitador")}
|
||||
onContinueToCounting={(ctx) => onActivateSection("counting", ctx)}
|
||||
onContinueToDigitador={(ctx) => onActivateSection("digitador", ctx)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -373,7 +373,7 @@ export interface DownloadWorker {
|
||||
|
||||
export interface DownloadJob {
|
||||
id: string;
|
||||
worker_id: string;
|
||||
worker_id: string | null;
|
||||
status: "pending" | "running" | "completed" | "failed" | "cancelled" | string;
|
||||
title: string;
|
||||
excel_key: string;
|
||||
@@ -389,6 +389,11 @@ export interface DownloadJob {
|
||||
job_id?: string | null;
|
||||
/** Task type=downloads */
|
||||
task_id?: string | null;
|
||||
parent_download_job_id?: string | null;
|
||||
shard_index?: number | null;
|
||||
shard_count?: number | null;
|
||||
/** Presente en GET de padre distribuido */
|
||||
shards?: DownloadJob[];
|
||||
}
|
||||
|
||||
export interface DownloadJobFile {
|
||||
@@ -417,6 +422,63 @@ export interface DownloadJobEvent {
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** Worker del bot digitador (control remoto). */
|
||||
export interface DigitadorWorker {
|
||||
id: string;
|
||||
hostname: string;
|
||||
max_concurrent: number;
|
||||
status: "online" | "offline" | "busy" | string;
|
||||
last_heartbeat: string;
|
||||
current_job_id: string | null;
|
||||
}
|
||||
|
||||
export interface DigitadorJob {
|
||||
id: string;
|
||||
worker_id: string | null;
|
||||
status: "pending" | "running" | "completed" | "failed" | "cancelled" | string;
|
||||
title: string;
|
||||
csv_key: string;
|
||||
reports_prefix: string;
|
||||
original_filename: string;
|
||||
cancel_requested: boolean;
|
||||
result: Record<string, unknown>;
|
||||
error: string | null;
|
||||
created_at: string;
|
||||
started_at: string | null;
|
||||
finished_at: string | null;
|
||||
job_id?: string | null;
|
||||
task_id?: string | null;
|
||||
parent_digitador_job_id?: string | null;
|
||||
shard_index?: number | null;
|
||||
shard_count?: number | null;
|
||||
shards?: DigitadorJob[];
|
||||
}
|
||||
|
||||
export interface DigitadorJobFile {
|
||||
key: string;
|
||||
name?: string | null;
|
||||
}
|
||||
|
||||
export interface DigitadorJobFilesResponse {
|
||||
job_id: string;
|
||||
prefix: string;
|
||||
files: DigitadorJobFile[];
|
||||
count: number;
|
||||
}
|
||||
|
||||
export interface DigitadorJobEvent {
|
||||
id: number;
|
||||
job_id: string;
|
||||
event_type: string;
|
||||
message: string;
|
||||
timestamp: string;
|
||||
radicado?: string | null;
|
||||
nit?: string | null;
|
||||
index?: number | null;
|
||||
total?: number | null;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface DesktopBridge {
|
||||
getRuntimeConfig(): Promise<RuntimeConfig>;
|
||||
selectDirectory(): Promise<string | null>;
|
||||
|
||||
@@ -15,6 +15,7 @@ from domain.verificador_st import (
|
||||
consolidado_to_records,
|
||||
export_no_aptos_csv,
|
||||
export_paso_bot_csv,
|
||||
export_todos_csv,
|
||||
process_cleaning_sources,
|
||||
)
|
||||
from domain.verificador_st.core import cargar_origen_servicio_desde_bytes
|
||||
@@ -483,11 +484,13 @@ def get_rows(
|
||||
@app.get("/jobs/{job_id}/export.csv")
|
||||
def export_csv(
|
||||
job_id: str,
|
||||
subset: str = Query("aptos", description="aptos | no_aptos"),
|
||||
subset: str = Query("aptos", description="aptos | no_aptos | todos"),
|
||||
) -> Response:
|
||||
subset_norm = (subset or "aptos").strip().lower()
|
||||
if subset_norm not in {"aptos", "no_aptos", "errores"}:
|
||||
raise HTTPException(status_code=400, detail="subset debe ser aptos o no_aptos")
|
||||
if subset_norm in {"all", "completo", "consolidado"}:
|
||||
subset_norm = "todos"
|
||||
if subset_norm not in {"aptos", "no_aptos", "errores", "todos"}:
|
||||
raise HTTPException(status_code=400, detail="subset debe ser aptos, no_aptos o todos")
|
||||
if subset_norm == "errores":
|
||||
subset_norm = "no_aptos"
|
||||
|
||||
@@ -504,6 +507,9 @@ def export_csv(
|
||||
if subset_norm == "aptos":
|
||||
csv_text = export_paso_bot_csv(payload)
|
||||
filename = f"poblacion_apta_aptos_{job_id[:8]}.csv"
|
||||
elif subset_norm == "todos":
|
||||
csv_text = export_todos_csv(payload)
|
||||
filename = f"poblacion_apta_todos_{job_id[:8]}.csv"
|
||||
else:
|
||||
csv_text = export_no_aptos_csv(payload)
|
||||
filename = f"poblacion_apta_no_aptos_{job_id[:8]}.csv"
|
||||
|
||||
@@ -3,6 +3,7 @@ from domain.verificador_st.pipeline import (
|
||||
consolidado_to_records,
|
||||
export_no_aptos_csv,
|
||||
export_paso_bot_csv,
|
||||
export_todos_csv,
|
||||
process_cleaning_sources,
|
||||
)
|
||||
|
||||
@@ -10,5 +11,6 @@ __all__ = [
|
||||
"consolidado_to_records",
|
||||
"export_no_aptos_csv",
|
||||
"export_paso_bot_csv",
|
||||
"export_todos_csv",
|
||||
"process_cleaning_sources",
|
||||
]
|
||||
|
||||
@@ -51,6 +51,8 @@ TABLA_MODERADORA = {
|
||||
2025: {4700, 19200, 50300},
|
||||
2026: {5000, 20100, 52800},
|
||||
}
|
||||
# Todos los montos de la tabla; la clasificación no filtra por año.
|
||||
VALORES_MODERADORA = {v for vals in TABLA_MODERADORA.values() for v in vals}
|
||||
|
||||
# Longitudes según tabla entregada por el usuario.
|
||||
# Los tipos marcados como alfanuméricos permiten letras y números.
|
||||
@@ -554,14 +556,15 @@ def normalizar_autorizacion(valor):
|
||||
|
||||
def separar_copago_moderadora(valor, anio):
|
||||
"""
|
||||
Si el valor coincide con tabla moderadora del año => cuota_moderadora.
|
||||
Si no coincide => copago.
|
||||
Si el valor coincide con cualquier cuota de la tabla (sin importar el año)
|
||||
=> cuota_moderadora. Si no coincide => copago.
|
||||
El parámetro anio se conserva por compatibilidad de firma.
|
||||
"""
|
||||
monto = to_int(valor, default=0)
|
||||
if monto == 0:
|
||||
return 0, 0
|
||||
|
||||
if anio in TABLA_MODERADORA and monto in TABLA_MODERADORA[anio]:
|
||||
if monto in VALORES_MODERADORA:
|
||||
return 0, monto
|
||||
|
||||
return monto, 0
|
||||
|
||||
@@ -223,3 +223,21 @@ def export_no_aptos_csv(rows: list[dict[str, Any]]) -> str:
|
||||
df = df.rename(columns=CSV_FINAL_IT_RENAME)
|
||||
df = df.reindex(columns=headers)
|
||||
return df.to_csv(sep="|", index=False, encoding="utf-8-sig")
|
||||
|
||||
|
||||
def export_todos_csv(rows: list[dict[str, Any]]) -> str:
|
||||
"""CSV pipe-separated con todas las filas (aptos y no aptos) e OBSERVACION."""
|
||||
headers = [*CSV_FINAL_IT_HEADERS, "OBSERVACION"]
|
||||
if not rows:
|
||||
df = pd.DataFrame(columns=headers)
|
||||
else:
|
||||
df = pd.DataFrame(rows)
|
||||
for c in COLUMNAS_FINAL_IT:
|
||||
if c not in df.columns:
|
||||
df[c] = ""
|
||||
if "OBSERVACION" not in df.columns:
|
||||
df["OBSERVACION"] = ""
|
||||
df = df.reindex(columns=[*COLUMNAS_FINAL_IT, "OBSERVACION"]).fillna("")
|
||||
df = df.rename(columns=CSV_FINAL_IT_RENAME)
|
||||
df = df.reindex(columns=headers)
|
||||
return df.to_csv(sep="|", index=False, encoding="utf-8-sig")
|
||||
|
||||
Reference in New Issue
Block a user