feat: Introduce bot-descarga service for PDF downloads and remote bot control. Add Docker configuration, environment variables, and a GUI for agent management. Enhance README with bot details and installation instructions.
This commit is contained in:
@@ -18,6 +18,7 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name
|
||||
EXTRACTION_URL = os.getenv("EXTRACTION_URL", "http://extraction:8001").rstrip("/")
|
||||
CLEANING_URL = os.getenv("CLEANING_URL", "http://cleaning:8002").rstrip("/")
|
||||
COUNT_URL = os.getenv("COUNT_URL", "http://count-api:8080").rstrip("/")
|
||||
BOT_DESCARGA_URL = os.getenv("BOT_DESCARGA_URL", "http://bot-descarga:8200").rstrip("/")
|
||||
|
||||
# Timeouts: JSON ligero vs uploads OCR/conteo
|
||||
_TIMEOUT_JSON = httpx.Timeout(connect=5.0, read=30.0, write=30.0, pool=5.0)
|
||||
@@ -33,7 +34,13 @@ async def lifespan(app: FastAPI):
|
||||
timeout=_TIMEOUT_JSON,
|
||||
limits=httpx.Limits(max_connections=100, max_keepalive_connections=20),
|
||||
)
|
||||
_log.info("Gateway iniciado — extraction=%s cleaning=%s count=%s", EXTRACTION_URL, CLEANING_URL, COUNT_URL)
|
||||
_log.info(
|
||||
"Gateway iniciado — extraction=%s cleaning=%s count=%s bot_descarga=%s",
|
||||
EXTRACTION_URL,
|
||||
CLEANING_URL,
|
||||
COUNT_URL,
|
||||
BOT_DESCARGA_URL,
|
||||
)
|
||||
yield
|
||||
await _http_client.aclose()
|
||||
_http_client = None
|
||||
@@ -72,6 +79,7 @@ def _is_upload_path(path: str) -> bool:
|
||||
or "/upload" in path
|
||||
or path.startswith("/cleaning/analyze-csv")
|
||||
or path.startswith("/cleaning/jobs/from-csv")
|
||||
or path.startswith("/bot-descarga/jobs")
|
||||
)
|
||||
|
||||
|
||||
@@ -91,12 +99,14 @@ async def health() -> dict[str, Any]:
|
||||
extraction = await _safe_health(client, EXTRACTION_URL)
|
||||
cleaning = await _safe_health(client, CLEANING_URL)
|
||||
counting = await _safe_health(client, COUNT_URL)
|
||||
bot_descarga = await _safe_health(client, BOT_DESCARGA_URL)
|
||||
return {
|
||||
"status": "ok" if extraction["ok"] and cleaning["ok"] else "degraded",
|
||||
"services": {
|
||||
"extraction": extraction,
|
||||
"cleaning": cleaning,
|
||||
"counting": counting,
|
||||
"bot_descarga": bot_descarga,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -152,6 +162,11 @@ async def count_jobs_proxy(path: str, request: Request) -> Response:
|
||||
return await _proxy_request(request, f"{EXTRACTION_URL}/count-jobs/{path}")
|
||||
|
||||
|
||||
@app.api_route("/bot-descarga/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
|
||||
async def bot_descarga_proxy(path: str, request: Request) -> Response:
|
||||
return await _proxy_request(request, f"{BOT_DESCARGA_URL}/{path}")
|
||||
|
||||
|
||||
@app.api_route("/jobs", methods=["GET", "POST"])
|
||||
async def jobs_root_proxy(request: Request) -> Response:
|
||||
return await _proxy_request(request, f"{EXTRACTION_URL}/jobs")
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONPATH=/app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY domain /app/domain
|
||||
COPY infrastructure /app/infrastructure
|
||||
COPY api /app/api
|
||||
|
||||
ENV BOT_API_HOST=0.0.0.0
|
||||
ENV BOT_API_PORT=8200
|
||||
|
||||
EXPOSE 8200
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=5s --retries=5 --start-period=15s \
|
||||
CMD curl -f http://127.0.0.1:8200/health || exit 1
|
||||
|
||||
CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8200"]
|
||||
@@ -0,0 +1,51 @@
|
||||
# Bot Descarga API (infraestructura IPSExtractor)
|
||||
|
||||
Microservicio FastAPI de control remoto de bots. Corre en Docker;
|
||||
las PCs de bot **no** exponen puertos — solo el agent en [`bots/bot_descarga`](../../bots/bot_descarga).
|
||||
|
||||
Persistencia: **PostgreSQL compartido** (`DATABASE_URL`), tablas
|
||||
`download_workers`, `download_jobs`, `download_job_events` + `tasks` (`type=downloads`).
|
||||
|
||||
## Layout MinIO (`pys-cuentas-medicas`)
|
||||
|
||||
```
|
||||
uploads/{download_job_id}/input.xlsx
|
||||
downloads/{download_job_id}/{nit}/{radicado}.pdf
|
||||
```
|
||||
|
||||
## Arranque (compose)
|
||||
|
||||
Servicio `bot-descarga` en el `docker-compose.yml` raíz.
|
||||
Gateway: `/bot-descarga/*` → este servicio `:8200`.
|
||||
|
||||
## Variables
|
||||
|
||||
```
|
||||
BOT_API_TOKEN=dev-token-change-me
|
||||
DATABASE_URL=postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor
|
||||
MINIO_ENDPOINT=https://s3.builker.xyz
|
||||
MINIO_BUCKET=pys-cuentas-medicas
|
||||
MINIO_ANONYMOUS=1
|
||||
```
|
||||
|
||||
## Worker en PCs (`bots/bot_descarga`)
|
||||
|
||||
```bash
|
||||
cd bots/bot_descarga
|
||||
python -m bot_descarga.worker.run \
|
||||
--api-url http://GATEWAY:8189/bot-descarga \
|
||||
--worker-id PC-01 \
|
||||
--token $BOT_API_TOKEN
|
||||
```
|
||||
|
||||
O con GUI: `python run_gui.py`. Ver [README del agent](../../bots/bot_descarga/README.md).
|
||||
|
||||
El agent queda idle hasta que el usuario cree un job desde el desktop
|
||||
(enviando `job_id` del Job IPSExtractor).
|
||||
|
||||
## Tests (API)
|
||||
|
||||
```bash
|
||||
cd services/bot_descarga
|
||||
pytest -q
|
||||
```
|
||||
@@ -0,0 +1,293 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import Depends, FastAPI, File, Form, HTTPException, Query, UploadFile
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from domain.models import JobStatus
|
||||
from infrastructure.auth import require_bearer
|
||||
from infrastructure.storage_minio import MinioSettings, MinioStorage
|
||||
from infrastructure.store import JobStore
|
||||
|
||||
store = JobStore()
|
||||
minio = MinioStorage(MinioSettings.from_env())
|
||||
|
||||
app = FastAPI(title="BotDescarga Remote API", version="1.0.0")
|
||||
|
||||
|
||||
class RegisterWorkerBody(BaseModel):
|
||||
worker_id: str
|
||||
hostname: str = ""
|
||||
max_concurrent: int = 1
|
||||
|
||||
|
||||
class HeartbeatBody(BaseModel):
|
||||
worker_id: str
|
||||
|
||||
|
||||
class EventBody(BaseModel):
|
||||
event_type: str
|
||||
message: str
|
||||
radicado: Optional[str] = None
|
||||
nit: Optional[str] = None
|
||||
index: Optional[int] = None
|
||||
total: Optional[int] = None
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
timestamp: Optional[str] = None
|
||||
|
||||
|
||||
class PatchJobBody(BaseModel):
|
||||
status: Optional[str] = None
|
||||
result: Optional[Dict[str, Any]] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict:
|
||||
return {"status": "ok", "service": "bot-descarga"}
|
||||
|
||||
|
||||
@app.post("/workers/register")
|
||||
def register_worker(body: RegisterWorkerBody, _: str = Depends(require_bearer)) -> dict:
|
||||
worker = store.register_worker(
|
||||
body.worker_id,
|
||||
hostname=body.hostname,
|
||||
max_concurrent=body.max_concurrent,
|
||||
)
|
||||
return worker.to_dict()
|
||||
|
||||
|
||||
@app.post("/workers/heartbeat")
|
||||
def heartbeat(body: HeartbeatBody, _: str = Depends(require_bearer)) -> dict:
|
||||
worker = store.heartbeat(body.worker_id)
|
||||
if not worker:
|
||||
raise HTTPException(status_code=404, detail="Worker no registrado")
|
||||
return worker.to_dict()
|
||||
|
||||
|
||||
@app.get("/workers")
|
||||
def list_workers(_: str = Depends(require_bearer)) -> List[dict]:
|
||||
return [w.to_dict() for w in store.list_workers()]
|
||||
|
||||
|
||||
@app.get("/workers/me/pending")
|
||||
def claim_pending(
|
||||
worker_id: str = Query(...),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
job = store.claim_pending(worker_id)
|
||||
if not job:
|
||||
return {"job": None}
|
||||
return {"job": job.to_dict()}
|
||||
|
||||
|
||||
@app.post("/jobs")
|
||||
async def create_job(
|
||||
worker_id: str = Form(...),
|
||||
job_id: str = Form(...),
|
||||
title: str = Form(""),
|
||||
file: UploadFile = File(...),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
if not worker_id.strip():
|
||||
raise HTTPException(status_code=400, detail="worker_id obligatorio")
|
||||
parent_job_id = job_id.strip()
|
||||
if not parent_job_id:
|
||||
raise HTTPException(status_code=400, detail="job_id (Job Desktop) obligatorio")
|
||||
if not store.parent_job_exists(parent_job_id):
|
||||
raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado")
|
||||
|
||||
if not store.is_worker_online(worker_id):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"Worker '{worker_id}' no está online. Enciende el agent primero.",
|
||||
)
|
||||
|
||||
data = await file.read()
|
||||
if not data:
|
||||
raise HTTPException(status_code=400, detail="Archivo vacío")
|
||||
|
||||
original = file.filename or "input.xlsx"
|
||||
content_type = file.content_type or "application/octet-stream"
|
||||
title_clean = title.strip()
|
||||
|
||||
job = store.create_job(
|
||||
worker_id=worker_id.strip(),
|
||||
excel_key="",
|
||||
original_filename=original,
|
||||
title=title_clean,
|
||||
job_id=parent_job_id,
|
||||
)
|
||||
task_id = store.create_downloads_task(
|
||||
parent_job_id=parent_job_id,
|
||||
download_job_id=job.id,
|
||||
worker_id=worker_id.strip(),
|
||||
title=title_clean or f"Bot descarga ({original})",
|
||||
original_filename=original,
|
||||
)
|
||||
job = store.get_job(job.id)
|
||||
assert job is not None
|
||||
|
||||
try:
|
||||
key = minio.upload_excel(job.id, data, original, content_type=content_type)
|
||||
except Exception as e:
|
||||
store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
|
||||
|
||||
job = store.update_excel_key(job.id, key)
|
||||
assert job is not None
|
||||
store.add_event(
|
||||
job.id,
|
||||
"run_started",
|
||||
f"Job creado y asignado a {worker_id}",
|
||||
metadata={
|
||||
"excel_key": key,
|
||||
"worker_id": worker_id,
|
||||
"job_id": parent_job_id,
|
||||
"task_id": task_id,
|
||||
},
|
||||
)
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.get("/jobs")
|
||||
def list_jobs(
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
job_id: Optional[str] = Query(None, description="Filtrar por Job Desktop"),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> List[dict]:
|
||||
return [j.to_dict() for j in store.list_jobs(limit=limit, job_id=job_id)]
|
||||
|
||||
|
||||
@app.get("/jobs/{download_job_id}")
|
||||
def get_job(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(download_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.get("/jobs/{download_job_id}/files")
|
||||
def list_job_files(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(download_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
try:
|
||||
return minio.list_download_files(download_job_id, prefix=job.downloads_prefix)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=f"Error listando MinIO: {e}") from e
|
||||
|
||||
|
||||
@app.get("/jobs/{download_job_id}/events")
|
||||
def get_events(
|
||||
download_job_id: str,
|
||||
since: int = Query(0, ge=0),
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
if not store.get_job(download_job_id):
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
events = store.list_events(download_job_id, since_id=since)
|
||||
return {"events": [e.to_dict() for e in events]}
|
||||
|
||||
|
||||
@app.post("/jobs/{download_job_id}/cancel")
|
||||
def cancel_job(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.request_cancel(download_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
store.add_event(download_job_id, "warning", "Cancelación solicitada por el usuario")
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.get("/jobs/{download_job_id}/file")
|
||||
def get_job_file(download_job_id: str, _: str = Depends(require_bearer)):
|
||||
job = store.get_job(download_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
if not job.excel_key:
|
||||
raise HTTPException(status_code=404, detail="Excel no disponible")
|
||||
|
||||
try:
|
||||
data = minio.get_bytes(job.excel_key)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=f"Error leyendo MinIO: {e}") from e
|
||||
|
||||
filename = Path(job.excel_key).name
|
||||
return Response(
|
||||
content=data,
|
||||
media_type="application/octet-stream",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/jobs/{download_job_id}/file-url")
|
||||
def get_job_file_url(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(download_job_id)
|
||||
if not job or not job.excel_key:
|
||||
raise HTTPException(status_code=404, detail="Job/archivo no encontrado")
|
||||
try:
|
||||
url = minio.presigned_get(job.excel_key)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=str(e)) from e
|
||||
return {"url": url, "excel_key": job.excel_key}
|
||||
|
||||
|
||||
@app.post("/jobs/{download_job_id}/events")
|
||||
def post_event(
|
||||
download_job_id: str,
|
||||
body: EventBody,
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
if not store.get_job(download_job_id):
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
event = store.add_event(
|
||||
download_job_id,
|
||||
body.event_type,
|
||||
body.message,
|
||||
radicado=body.radicado,
|
||||
nit=body.nit,
|
||||
index=body.index,
|
||||
total=body.total,
|
||||
metadata=body.metadata,
|
||||
timestamp=body.timestamp,
|
||||
)
|
||||
job = store.get_job(download_job_id)
|
||||
if job and body.index is not None and body.total is not None:
|
||||
result = dict(job.result or {})
|
||||
result["processed"] = body.index
|
||||
result["total"] = body.total
|
||||
if body.nit:
|
||||
result["nit_actual"] = body.nit
|
||||
store.patch_job(download_job_id, result=result)
|
||||
return event.to_dict()
|
||||
|
||||
|
||||
@app.patch("/jobs/{download_job_id}")
|
||||
def patch_job(
|
||||
download_job_id: str,
|
||||
body: PatchJobBody,
|
||||
_: str = Depends(require_bearer),
|
||||
) -> dict:
|
||||
if body.status and body.status not in {s.value for s in JobStatus}:
|
||||
raise HTTPException(status_code=400, detail="status inválido")
|
||||
job = store.patch_job(
|
||||
download_job_id,
|
||||
status=body.status,
|
||||
result=body.result,
|
||||
error=body.error,
|
||||
)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.get("/jobs/{download_job_id}/cancel-flag")
|
||||
def cancel_flag(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
|
||||
job = store.get_job(download_job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job no encontrado")
|
||||
return {"cancel_requested": job.cancel_requested, "status": job.status}
|
||||
@@ -0,0 +1,76 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
class WorkerStatus(str, Enum):
|
||||
ONLINE = "online"
|
||||
OFFLINE = "offline"
|
||||
BUSY = "busy"
|
||||
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Worker:
|
||||
id: str
|
||||
hostname: str = ""
|
||||
max_concurrent: int = 1
|
||||
status: str = WorkerStatus.ONLINE.value
|
||||
last_heartbeat: str = field(default_factory=utc_now)
|
||||
current_job_id: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Job:
|
||||
id: str
|
||||
worker_id: str
|
||||
status: str = JobStatus.PENDING.value
|
||||
title: str = ""
|
||||
excel_key: str = ""
|
||||
downloads_prefix: str = ""
|
||||
original_filename: str = ""
|
||||
cancel_requested: bool = False
|
||||
result: Dict[str, Any] = field(default_factory=dict)
|
||||
error: Optional[str] = None
|
||||
created_at: str = field(default_factory=utc_now)
|
||||
started_at: Optional[str] = None
|
||||
finished_at: Optional[str] = None
|
||||
job_id: Optional[str] = None
|
||||
task_id: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobEventRecord:
|
||||
id: int
|
||||
job_id: str
|
||||
event_type: str
|
||||
message: str
|
||||
timestamp: str
|
||||
radicado: Optional[str] = None
|
||||
nit: Optional[str] = None
|
||||
index: Optional[int] = None
|
||||
total: Optional[int] = None
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Header, HTTPException
|
||||
|
||||
|
||||
def get_api_token() -> str:
|
||||
return os.getenv("BOT_API_TOKEN", "dev-token-change-me")
|
||||
|
||||
|
||||
def require_bearer(authorization: Optional[str] = Header(default=None)) -> str:
|
||||
expected = get_api_token()
|
||||
if not authorization or not authorization.lower().startswith("bearer "):
|
||||
raise HTTPException(status_code=401, detail="Authorization Bearer requerido")
|
||||
token = authorization.split(" ", 1)[1].strip()
|
||||
if token != expected:
|
||||
raise HTTPException(status_code=401, detail="Token inválido")
|
||||
return token
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Persistencia Postgres del servicio bot-descarga (DB compartida IPSExtractor)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Iterator
|
||||
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
Boolean,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
create_engine,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, sessionmaker
|
||||
|
||||
DATABASE_URL = os.getenv(
|
||||
"DATABASE_URL",
|
||||
"postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor",
|
||||
)
|
||||
|
||||
engine = create_engine(DATABASE_URL, pool_pre_ping=True)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def new_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def _iso(dt: datetime | None) -> str | None:
|
||||
if dt is None:
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
class Job(Base):
|
||||
"""Contenedor Desktop (lectura / FK)."""
|
||||
|
||||
__tablename__ = "jobs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
job_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
|
||||
class Task(Base):
|
||||
__tablename__ = "tasks"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
||||
job_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("jobs.id"), nullable=True, index=True
|
||||
)
|
||||
type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
owner_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class DownloadWorkerRow(Base):
|
||||
__tablename__ = "download_workers"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
hostname: Mapped[str] = mapped_column(Text, default="")
|
||||
max_concurrent: Mapped[int] = mapped_column(Integer, default=1)
|
||||
status: Mapped[str] = mapped_column(String(32), index=True)
|
||||
last_heartbeat: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
current_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
|
||||
|
||||
class DownloadJobRow(Base):
|
||||
__tablename__ = "download_jobs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
||||
job_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("jobs.id"), nullable=True, index=True
|
||||
)
|
||||
task_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("tasks.id"), nullable=True, index=True
|
||||
)
|
||||
worker_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
excel_key: Mapped[str] = mapped_column(Text, default="")
|
||||
downloads_prefix: Mapped[str] = mapped_column(Text, default="")
|
||||
original_filename: Mapped[str] = mapped_column(Text, default="")
|
||||
cancel_requested: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class DownloadJobEventRow(Base):
|
||||
__tablename__ = "download_job_events"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
download_job_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("download_jobs.id"), index=True
|
||||
)
|
||||
event_type: Mapped[str] = mapped_column(String(64))
|
||||
message: Mapped[str] = mapped_column(Text)
|
||||
timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
radicado: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
nit: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
idx: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
total: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
metadata_json: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
if DATABASE_URL.startswith("postgresql"):
|
||||
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:
|
||||
connection.execute(text("SELECT pg_advisory_lock(74512004)"))
|
||||
try:
|
||||
Base.metadata.create_all(bind=connection)
|
||||
finally:
|
||||
connection.execute(text("SELECT pg_advisory_unlock(74512004)"))
|
||||
return
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def session_scope() -> Iterator[Session]:
|
||||
session = SessionLocal()
|
||||
try:
|
||||
yield session
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
@@ -0,0 +1,215 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO, List, Optional
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MinioSettings:
|
||||
endpoint: str = "https://s3.builker.xyz"
|
||||
bucket: str = "pys-cuentas-medicas"
|
||||
access_key: str = ""
|
||||
secret_key: str = ""
|
||||
region: str = "us-east-1"
|
||||
anonymous: bool = True
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "MinioSettings":
|
||||
access_key = (os.getenv("MINIO_ACCESS_KEY") or "").strip()
|
||||
secret_key = (os.getenv("MINIO_SECRET_KEY") or "").strip()
|
||||
anon_flag = (os.getenv("MINIO_ANONYMOUS") or "").strip().lower()
|
||||
if anon_flag in ("0", "false", "no"):
|
||||
anonymous = False
|
||||
elif anon_flag in ("1", "true", "yes"):
|
||||
anonymous = True
|
||||
else:
|
||||
anonymous = not (access_key and secret_key)
|
||||
return cls(
|
||||
endpoint=os.getenv("MINIO_ENDPOINT", "https://s3.builker.xyz").rstrip("/"),
|
||||
bucket=os.getenv("MINIO_BUCKET", "pys-cuentas-medicas"),
|
||||
access_key=access_key,
|
||||
secret_key=secret_key,
|
||||
region=os.getenv("MINIO_REGION", "us-east-1"),
|
||||
anonymous=anonymous,
|
||||
)
|
||||
|
||||
|
||||
def excel_key(job_id: str, filename: str = "input.xlsx") -> str:
|
||||
return f"uploads/{job_id}/{filename}"
|
||||
|
||||
|
||||
def downloads_prefix(job_id: str) -> str:
|
||||
return f"downloads/{job_id}/"
|
||||
|
||||
|
||||
def pdf_key(job_id: str, nit: str, radicado: str) -> str:
|
||||
return f"downloads/{job_id}/{nit}/{radicado}.pdf"
|
||||
|
||||
|
||||
def meta_key(job_id: str) -> str:
|
||||
return f"uploads/{job_id}/meta.json"
|
||||
|
||||
|
||||
def manifest_key(job_id: str) -> str:
|
||||
return f"downloads/{job_id}/_manifest.json"
|
||||
|
||||
|
||||
def _s3_client(settings: MinioSettings):
|
||||
import boto3
|
||||
from botocore import UNSIGNED
|
||||
from botocore.client import Config
|
||||
|
||||
s3_cfg = {
|
||||
"addressing_style": "path",
|
||||
"request_checksum_calculation": "when_required",
|
||||
"response_checksum_validation": "when_required",
|
||||
}
|
||||
if settings.anonymous:
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.endpoint,
|
||||
region_name=settings.region,
|
||||
config=Config(signature_version=UNSIGNED, s3=s3_cfg),
|
||||
)
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.endpoint,
|
||||
aws_access_key_id=settings.access_key,
|
||||
aws_secret_access_key=settings.secret_key,
|
||||
region_name=settings.region,
|
||||
config=Config(signature_version="s3v4", s3=s3_cfg),
|
||||
)
|
||||
|
||||
|
||||
class MinioStorage:
|
||||
"""Cliente S3-compatible para MinIO (uploads/downloads por job_id)."""
|
||||
|
||||
def __init__(self, settings: Optional[MinioSettings] = None):
|
||||
self.settings = settings or MinioSettings.from_env()
|
||||
self._client = None
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
if self._client is None:
|
||||
self._client = _s3_client(self.settings)
|
||||
return self._client
|
||||
|
||||
@property
|
||||
def bucket(self) -> str:
|
||||
return self.settings.bucket
|
||||
|
||||
def put_bytes(self, key: str, data: bytes, content_type: str = "application/octet-stream") -> str:
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=data,
|
||||
ContentType=content_type,
|
||||
)
|
||||
return key
|
||||
|
||||
def put_file(self, key: str, path: str | Path, content_type: str = "application/octet-stream") -> str:
|
||||
path = Path(path)
|
||||
return self.put_bytes(key, path.read_bytes(), content_type=content_type)
|
||||
|
||||
def put_stream(self, key: str, stream: BinaryIO, content_type: str = "application/octet-stream") -> str:
|
||||
data = stream.read()
|
||||
return self.put_bytes(key, data, content_type=content_type)
|
||||
|
||||
def get_bytes(self, key: str) -> bytes:
|
||||
obj = self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
return obj["Body"].read()
|
||||
|
||||
def download_file(self, key: str, dest: str | Path) -> Path:
|
||||
dest = Path(dest)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.client.download_file(self.bucket, key, str(dest))
|
||||
return dest
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
try:
|
||||
self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def list_keys(self, prefix: str) -> List[str]:
|
||||
keys: List[str] = []
|
||||
token = None
|
||||
while True:
|
||||
kwargs = {"Bucket": self.bucket, "Prefix": prefix}
|
||||
if token:
|
||||
kwargs["ContinuationToken"] = token
|
||||
resp = self.client.list_objects_v2(**kwargs)
|
||||
for item in resp.get("Contents") or []:
|
||||
keys.append(item["Key"])
|
||||
if not resp.get("IsTruncated"):
|
||||
break
|
||||
token = resp.get("NextContinuationToken")
|
||||
return keys
|
||||
|
||||
def presigned_get(self, key: str, expires_in: int = 3600) -> str:
|
||||
if self.settings.anonymous:
|
||||
return f"{self.settings.endpoint}/{self.bucket}/{key}"
|
||||
return self.client.generate_presigned_url(
|
||||
"get_object",
|
||||
Params={"Bucket": self.bucket, "Key": key},
|
||||
ExpiresIn=expires_in,
|
||||
)
|
||||
|
||||
def upload_excel(
|
||||
self,
|
||||
job_id: str,
|
||||
data: bytes,
|
||||
original_filename: str,
|
||||
content_type: str = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
) -> str:
|
||||
ext = Path(original_filename).suffix.lower() or ".xlsx"
|
||||
canonical = f"input{ext}" if ext in (".xlsx", ".xls", ".csv") else "input.xlsx"
|
||||
key = excel_key(job_id, canonical)
|
||||
self.put_bytes(key, data, content_type=content_type)
|
||||
meta = {
|
||||
"original_filename": original_filename,
|
||||
"size": len(data),
|
||||
"canonical": canonical,
|
||||
}
|
||||
self.put_bytes(meta_key(job_id), json.dumps(meta).encode("utf-8"), "application/json")
|
||||
return key
|
||||
|
||||
def upload_pdf(self, job_id: str, nit: str, radicado: str, local_path: str | Path) -> str:
|
||||
key = pdf_key(job_id, nit, radicado)
|
||||
self.put_file(key, local_path, content_type="application/pdf")
|
||||
return key
|
||||
|
||||
def write_manifest(self, job_id: str, keys: List[str]) -> str:
|
||||
key = manifest_key(job_id)
|
||||
payload = {"job_id": job_id, "files": keys, "count": len(keys)}
|
||||
self.put_bytes(key, json.dumps(payload, ensure_ascii=False).encode("utf-8"), "application/json")
|
||||
return key
|
||||
|
||||
def list_download_files(self, job_id: str, prefix: str | None = None) -> dict:
|
||||
"""Lista PDFs de una corrida (manifest o list_keys)."""
|
||||
prefix = prefix or downloads_prefix(job_id)
|
||||
files: List[dict] = []
|
||||
try:
|
||||
raw = self.get_bytes(manifest_key(job_id))
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
for key in payload.get("files") or []:
|
||||
files.append(self._file_entry(str(key)))
|
||||
except Exception:
|
||||
for key in self.list_keys(prefix):
|
||||
if key.endswith("/_manifest.json") or key.endswith("_manifest.json"):
|
||||
continue
|
||||
files.append(self._file_entry(key))
|
||||
return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)}
|
||||
|
||||
@staticmethod
|
||||
def _file_entry(key: str) -> dict:
|
||||
# downloads/{job_id}/{nit}/{radicado}.pdf
|
||||
parts = key.strip("/").split("/")
|
||||
nit = parts[-2] if len(parts) >= 3 else None
|
||||
name = parts[-1] if parts else key
|
||||
radicado = name[:-4] if name.lower().endswith(".pdf") else name
|
||||
return {"key": key, "nit": nit, "radicado": radicado}
|
||||
@@ -0,0 +1,466 @@
|
||||
"""Repositorio sobre PostgreSQL (tablas download_* + sync de tasks)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from domain.models import (
|
||||
Job,
|
||||
JobEventRecord,
|
||||
JobStatus,
|
||||
Worker,
|
||||
WorkerStatus,
|
||||
utc_now,
|
||||
)
|
||||
from infrastructure.database import (
|
||||
DownloadJobEventRow,
|
||||
DownloadJobRow,
|
||||
DownloadWorkerRow,
|
||||
Task,
|
||||
_iso,
|
||||
init_db,
|
||||
session_scope,
|
||||
utcnow,
|
||||
)
|
||||
from infrastructure.storage_minio import downloads_prefix
|
||||
|
||||
HEARTBEAT_OFFLINE_SECONDS = 45
|
||||
|
||||
|
||||
def _parse_iso(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
dt = datetime.fromisoformat(value)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
|
||||
|
||||
class JobStore:
|
||||
"""API de persistencia compatible con el store SQLite anterior."""
|
||||
|
||||
def __init__(self, db_path: str | None = None):
|
||||
# db_path ignorado (compat); usa DATABASE_URL.
|
||||
init_db()
|
||||
|
||||
def _row_worker(self, row: DownloadWorkerRow) -> Worker:
|
||||
worker = Worker(
|
||||
id=row.id,
|
||||
hostname=row.hostname or "",
|
||||
max_concurrent=int(row.max_concurrent or 1),
|
||||
status=row.status,
|
||||
last_heartbeat=_iso(row.last_heartbeat) or utc_now(),
|
||||
current_job_id=row.current_job_id,
|
||||
)
|
||||
return self._refresh_worker_status(worker)
|
||||
|
||||
def _row_job(self, row: DownloadJobRow) -> Job:
|
||||
return Job(
|
||||
id=row.id,
|
||||
worker_id=row.worker_id,
|
||||
status=row.status,
|
||||
title=row.title or "",
|
||||
excel_key=row.excel_key or "",
|
||||
downloads_prefix=row.downloads_prefix or "",
|
||||
original_filename=row.original_filename or "",
|
||||
cancel_requested=bool(row.cancel_requested),
|
||||
result=dict(row.result or {}),
|
||||
error=row.error,
|
||||
created_at=_iso(row.created_at) or utc_now(),
|
||||
started_at=_iso(row.started_at),
|
||||
finished_at=_iso(row.finished_at),
|
||||
job_id=row.job_id,
|
||||
task_id=row.task_id,
|
||||
)
|
||||
|
||||
def _refresh_worker_status(self, worker: Worker) -> Worker:
|
||||
try:
|
||||
hb = datetime.fromisoformat(worker.last_heartbeat)
|
||||
if hb.tzinfo is None:
|
||||
hb = hb.replace(tzinfo=timezone.utc)
|
||||
age = datetime.now(timezone.utc) - hb.astimezone(timezone.utc)
|
||||
if age > timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS):
|
||||
worker.status = WorkerStatus.OFFLINE.value
|
||||
elif worker.current_job_id:
|
||||
worker.status = WorkerStatus.BUSY.value
|
||||
else:
|
||||
worker.status = WorkerStatus.ONLINE.value
|
||||
except Exception:
|
||||
worker.status = WorkerStatus.OFFLINE.value
|
||||
return worker
|
||||
|
||||
def register_worker(
|
||||
self,
|
||||
worker_id: str,
|
||||
hostname: str = "",
|
||||
max_concurrent: int = 1,
|
||||
) -> Worker:
|
||||
with session_scope() as session:
|
||||
row = session.get(DownloadWorkerRow, worker_id)
|
||||
now = utcnow()
|
||||
if row:
|
||||
row.hostname = hostname
|
||||
row.max_concurrent = max_concurrent
|
||||
row.status = WorkerStatus.ONLINE.value
|
||||
row.last_heartbeat = now
|
||||
row.current_job_id = None
|
||||
else:
|
||||
row = DownloadWorkerRow(
|
||||
id=worker_id,
|
||||
hostname=hostname,
|
||||
max_concurrent=max_concurrent,
|
||||
status=WorkerStatus.ONLINE.value,
|
||||
last_heartbeat=now,
|
||||
current_job_id=None,
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return self._row_worker(row)
|
||||
|
||||
def heartbeat(self, worker_id: str) -> Optional[Worker]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DownloadWorkerRow, worker_id)
|
||||
if not row:
|
||||
return None
|
||||
row.last_heartbeat = utcnow()
|
||||
row.status = (
|
||||
WorkerStatus.BUSY.value if row.current_job_id else WorkerStatus.ONLINE.value
|
||||
)
|
||||
session.flush()
|
||||
return self._row_worker(row)
|
||||
|
||||
def get_worker(self, worker_id: str) -> Optional[Worker]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DownloadWorkerRow, worker_id)
|
||||
if not row:
|
||||
return None
|
||||
return self._row_worker(row)
|
||||
|
||||
def list_workers(self) -> List[Worker]:
|
||||
with session_scope() as session:
|
||||
rows = session.scalars(
|
||||
select(DownloadWorkerRow).order_by(DownloadWorkerRow.id)
|
||||
).all()
|
||||
return [self._row_worker(r) for r in rows]
|
||||
|
||||
def is_worker_online(self, worker_id: str) -> bool:
|
||||
worker = self.get_worker(worker_id)
|
||||
if not worker:
|
||||
return False
|
||||
return worker.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value)
|
||||
|
||||
def update_excel_key(self, job_id: str, excel_key: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DownloadJobRow, job_id)
|
||||
if not row:
|
||||
return None
|
||||
row.excel_key = excel_key
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def create_job(
|
||||
self,
|
||||
worker_id: str,
|
||||
excel_key: str,
|
||||
original_filename: str = "",
|
||||
title: str = "",
|
||||
*,
|
||||
job_id: Optional[str] = None,
|
||||
task_id: Optional[str] = None,
|
||||
) -> Job:
|
||||
import uuid as uuid_mod
|
||||
|
||||
download_id = str(uuid_mod.uuid4())
|
||||
prefix = downloads_prefix(download_id)
|
||||
with session_scope() as session:
|
||||
row = DownloadJobRow(
|
||||
id=download_id,
|
||||
job_id=job_id,
|
||||
task_id=task_id,
|
||||
worker_id=worker_id,
|
||||
status=JobStatus.PENDING.value,
|
||||
title=title or f"Descarga {original_filename}",
|
||||
excel_key=excel_key,
|
||||
downloads_prefix=prefix,
|
||||
original_filename=original_filename,
|
||||
cancel_requested=False,
|
||||
result={},
|
||||
created_at=utcnow(),
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def get_job(self, job_id: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DownloadJobRow, job_id)
|
||||
if not row:
|
||||
return None
|
||||
return self._row_job(row)
|
||||
|
||||
def list_jobs(
|
||||
self,
|
||||
limit: int = 100,
|
||||
*,
|
||||
job_id: Optional[str] = None,
|
||||
) -> List[Job]:
|
||||
with session_scope() as session:
|
||||
stmt = select(DownloadJobRow).order_by(DownloadJobRow.created_at.desc()).limit(limit)
|
||||
if job_id:
|
||||
stmt = (
|
||||
select(DownloadJobRow)
|
||||
.where(DownloadJobRow.job_id == job_id)
|
||||
.order_by(DownloadJobRow.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
rows = session.scalars(stmt).all()
|
||||
return [self._row_job(r) for r in rows]
|
||||
|
||||
def claim_pending(self, worker_id: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
worker = session.get(DownloadWorkerRow, worker_id)
|
||||
if not worker or worker.current_job_id:
|
||||
return None
|
||||
|
||||
row = session.scalars(
|
||||
select(DownloadJobRow)
|
||||
.where(
|
||||
DownloadJobRow.worker_id == worker_id,
|
||||
DownloadJobRow.status == JobStatus.PENDING.value,
|
||||
)
|
||||
.order_by(DownloadJobRow.created_at.asc())
|
||||
.limit(1)
|
||||
).first()
|
||||
if not row:
|
||||
return None
|
||||
|
||||
now = utcnow()
|
||||
row.status = JobStatus.RUNNING.value
|
||||
row.started_at = now
|
||||
worker.current_job_id = row.id
|
||||
worker.status = WorkerStatus.BUSY.value
|
||||
worker.last_heartbeat = now
|
||||
|
||||
if row.task_id:
|
||||
task = session.get(Task, row.task_id)
|
||||
if task:
|
||||
task.status = JobStatus.RUNNING.value
|
||||
task.started_at = now
|
||||
task.updated_at = now
|
||||
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def request_cancel(self, job_id: str) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DownloadJobRow, job_id)
|
||||
if not row:
|
||||
return None
|
||||
if row.status in (
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
):
|
||||
return self._row_job(row)
|
||||
|
||||
now = utcnow()
|
||||
row.cancel_requested = True
|
||||
if row.status == JobStatus.PENDING.value:
|
||||
row.status = JobStatus.CANCELLED.value
|
||||
row.finished_at = now
|
||||
self._sync_task(session, row, status=JobStatus.CANCELLED.value)
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def is_cancel_requested(self, job_id: str) -> bool:
|
||||
job = self.get_job(job_id)
|
||||
return bool(job and job.cancel_requested)
|
||||
|
||||
def _sync_task(
|
||||
self,
|
||||
session,
|
||||
row: DownloadJobRow,
|
||||
*,
|
||||
status: Optional[str] = None,
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> None:
|
||||
if not row.task_id:
|
||||
return
|
||||
task = session.get(Task, row.task_id)
|
||||
if not task:
|
||||
return
|
||||
now = utcnow()
|
||||
if status:
|
||||
task.status = status
|
||||
if result is not None:
|
||||
task.result = {
|
||||
**dict(task.result or {}),
|
||||
"download_job_id": row.id,
|
||||
**result,
|
||||
}
|
||||
if error is not None:
|
||||
task.error = error
|
||||
task.updated_at = now
|
||||
if status in (
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
):
|
||||
task.finished_at = now
|
||||
|
||||
def patch_job(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
status: Optional[str] = None,
|
||||
result: Optional[Dict[str, Any]] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> Optional[Job]:
|
||||
with session_scope() as session:
|
||||
row = session.get(DownloadJobRow, job_id)
|
||||
if not row:
|
||||
return None
|
||||
|
||||
new_status = status or row.status
|
||||
if result is not None:
|
||||
row.result = result
|
||||
if error is not None:
|
||||
row.error = error
|
||||
row.status = new_status
|
||||
|
||||
finished = False
|
||||
if new_status in (
|
||||
JobStatus.COMPLETED.value,
|
||||
JobStatus.FAILED.value,
|
||||
JobStatus.CANCELLED.value,
|
||||
):
|
||||
row.finished_at = row.finished_at or utcnow()
|
||||
finished = True
|
||||
|
||||
self._sync_task(
|
||||
session,
|
||||
row,
|
||||
status=new_status if status else None,
|
||||
result=result,
|
||||
error=error,
|
||||
)
|
||||
|
||||
if finished:
|
||||
worker = session.get(DownloadWorkerRow, row.worker_id)
|
||||
if worker and worker.current_job_id == job_id:
|
||||
worker.current_job_id = None
|
||||
worker.status = WorkerStatus.ONLINE.value
|
||||
|
||||
session.flush()
|
||||
return self._row_job(row)
|
||||
|
||||
def add_event(
|
||||
self,
|
||||
job_id: str,
|
||||
event_type: str,
|
||||
message: str,
|
||||
*,
|
||||
radicado: Optional[str] = None,
|
||||
nit: Optional[str] = None,
|
||||
index: Optional[int] = None,
|
||||
total: Optional[int] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
timestamp: Optional[str] = None,
|
||||
) -> JobEventRecord:
|
||||
with session_scope() as session:
|
||||
ts = _parse_iso(timestamp) or utcnow()
|
||||
ev = DownloadJobEventRow(
|
||||
download_job_id=job_id,
|
||||
event_type=event_type,
|
||||
message=message,
|
||||
timestamp=ts,
|
||||
radicado=radicado,
|
||||
nit=nit,
|
||||
idx=index,
|
||||
total=total,
|
||||
metadata_json=metadata or {},
|
||||
)
|
||||
session.add(ev)
|
||||
session.flush()
|
||||
return JobEventRecord(
|
||||
id=int(ev.id),
|
||||
job_id=job_id,
|
||||
event_type=event_type,
|
||||
message=message,
|
||||
timestamp=_iso(ts) or utc_now(),
|
||||
radicado=radicado,
|
||||
nit=nit,
|
||||
index=index,
|
||||
total=total,
|
||||
metadata=metadata or {},
|
||||
)
|
||||
|
||||
def list_events(self, job_id: str, since_id: int = 0) -> List[JobEventRecord]:
|
||||
with session_scope() as session:
|
||||
rows = session.scalars(
|
||||
select(DownloadJobEventRow)
|
||||
.where(
|
||||
DownloadJobEventRow.download_job_id == job_id,
|
||||
DownloadJobEventRow.id > since_id,
|
||||
)
|
||||
.order_by(DownloadJobEventRow.id.asc())
|
||||
).all()
|
||||
return [
|
||||
JobEventRecord(
|
||||
id=int(r.id),
|
||||
job_id=r.download_job_id,
|
||||
event_type=r.event_type,
|
||||
message=r.message,
|
||||
timestamp=_iso(r.timestamp) or utc_now(),
|
||||
radicado=r.radicado,
|
||||
nit=r.nit,
|
||||
index=r.idx,
|
||||
total=r.total,
|
||||
metadata=dict(r.metadata_json or {}),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
def create_downloads_task(
|
||||
self,
|
||||
*,
|
||||
parent_job_id: str,
|
||||
download_job_id: str,
|
||||
worker_id: str,
|
||||
title: str,
|
||||
original_filename: str,
|
||||
) -> str:
|
||||
"""Crea Task(type=downloads) y la asocia al download_job. Devuelve task_id."""
|
||||
with session_scope() as session:
|
||||
now = utcnow()
|
||||
task = Task(
|
||||
job_id=parent_job_id,
|
||||
type="downloads",
|
||||
status=JobStatus.PENDING.value,
|
||||
title=title or f"Bot descarga ({original_filename})",
|
||||
request={
|
||||
"worker_id": worker_id,
|
||||
"original_filename": original_filename,
|
||||
"download_job_id": download_job_id,
|
||||
},
|
||||
result={"download_job_id": download_job_id},
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(task)
|
||||
session.flush()
|
||||
row = session.get(DownloadJobRow, download_job_id)
|
||||
if row:
|
||||
row.task_id = task.id
|
||||
row.job_id = parent_job_id
|
||||
session.flush()
|
||||
return task.id
|
||||
|
||||
def parent_job_exists(self, parent_job_id: str) -> bool:
|
||||
from infrastructure.database import Job as IpsJob
|
||||
|
||||
with session_scope() as session:
|
||||
return session.get(IpsJob, parent_job_id) is not None
|
||||
@@ -0,0 +1,9 @@
|
||||
fastapi>=0.110.0
|
||||
uvicorn[standard]>=0.27.0
|
||||
python-multipart>=0.0.9
|
||||
boto3>=1.34.0
|
||||
httpx>=0.27.0
|
||||
pydantic>=2.0.0
|
||||
sqlalchemy>=2.0.0
|
||||
psycopg[binary]>=3.1.0
|
||||
pytest>=7.0.0
|
||||
@@ -0,0 +1,151 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from infrastructure.storage_minio import downloads_prefix, excel_key, pdf_key
|
||||
|
||||
|
||||
def test_minio_key_helpers():
|
||||
assert excel_key("abc", "input.xlsx") == "uploads/abc/input.xlsx"
|
||||
assert downloads_prefix("abc") == "downloads/abc/"
|
||||
assert pdf_key("abc", "900", "123") == "downloads/abc/900/123.pdf"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def store(tmp_path, monkeypatch):
|
||||
db_url = f"sqlite:///{(tmp_path / 'bot.db').as_posix()}"
|
||||
monkeypatch.setenv("DATABASE_URL", db_url)
|
||||
|
||||
# Recargar módulos para que el engine use SQLITE
|
||||
for name in [
|
||||
"infrastructure.database",
|
||||
"infrastructure.store",
|
||||
]:
|
||||
if name in sys.modules:
|
||||
importlib.reload(sys.modules[name])
|
||||
else:
|
||||
importlib.import_module(name)
|
||||
|
||||
from domain.models import JobStatus, WorkerStatus
|
||||
from infrastructure import database as db
|
||||
from infrastructure.store import JobStore
|
||||
|
||||
importlib.reload(db)
|
||||
import infrastructure.store as store_mod
|
||||
|
||||
importlib.reload(store_mod)
|
||||
return store_mod.JobStore(), JobStatus, WorkerStatus
|
||||
|
||||
|
||||
def test_register_heartbeat_and_offline(store):
|
||||
job_store, _JobStatus, WorkerStatus = store
|
||||
w = job_store.register_worker("PC-01", hostname="host1")
|
||||
assert w.status == WorkerStatus.ONLINE.value
|
||||
assert job_store.is_worker_online("PC-01")
|
||||
|
||||
job_store.heartbeat("PC-01")
|
||||
workers = job_store.list_workers()
|
||||
assert len(workers) == 1
|
||||
assert workers[0].id == "PC-01"
|
||||
|
||||
|
||||
def test_create_job_requires_worker_and_claim_is_atomic(store):
|
||||
job_store, JobStatus, _WorkerStatus = store
|
||||
job_store.register_worker("PC-01")
|
||||
|
||||
job = job_store.create_job(
|
||||
worker_id="PC-01",
|
||||
excel_key="uploads/x/input.xlsx",
|
||||
original_filename="datos.xlsx",
|
||||
title="Lote 1",
|
||||
)
|
||||
assert job.status == JobStatus.PENDING.value
|
||||
assert job.worker_id == "PC-01"
|
||||
assert job.downloads_prefix == downloads_prefix(job.id)
|
||||
|
||||
claimed = job_store.claim_pending("PC-01")
|
||||
assert claimed is not None
|
||||
assert claimed.id == job.id
|
||||
assert claimed.status == JobStatus.RUNNING.value
|
||||
|
||||
again = job_store.claim_pending("PC-01")
|
||||
assert again is None
|
||||
|
||||
job_store.register_worker("PC-02")
|
||||
other = job_store.claim_pending("PC-02")
|
||||
assert other is None
|
||||
|
||||
|
||||
def test_cancel_pending_marks_cancelled(store):
|
||||
job_store, JobStatus, _WorkerStatus = store
|
||||
job_store.register_worker("PC-01")
|
||||
job = job_store.create_job("PC-01", "uploads/j/input.xlsx", "a.xlsx")
|
||||
cancelled = job_store.request_cancel(job.id)
|
||||
assert cancelled is not None
|
||||
assert cancelled.status == JobStatus.CANCELLED.value
|
||||
assert cancelled.cancel_requested is True
|
||||
|
||||
|
||||
def test_cancel_running_sets_flag(store):
|
||||
job_store, JobStatus, _WorkerStatus = store
|
||||
job_store.register_worker("PC-01")
|
||||
job = job_store.create_job("PC-01", "uploads/j/input.xlsx", "a.xlsx")
|
||||
job_store.claim_pending("PC-01")
|
||||
flagged = job_store.request_cancel(job.id)
|
||||
assert flagged is not None
|
||||
assert flagged.cancel_requested is True
|
||||
assert flagged.status == JobStatus.RUNNING.value
|
||||
assert job_store.is_cancel_requested(job.id) is True
|
||||
|
||||
|
||||
def test_events_and_patch_complete(store):
|
||||
job_store, JobStatus, _WorkerStatus = store
|
||||
job_store.register_worker("PC-01")
|
||||
job = job_store.create_job("PC-01", "uploads/j/input.xlsx", "a.xlsx")
|
||||
job_store.claim_pending("PC-01")
|
||||
e1 = job_store.add_event(
|
||||
job.id, "radicado_saved", "ok", radicado="1", nit="9", index=1, total=2
|
||||
)
|
||||
assert e1.id >= 1
|
||||
events = job_store.list_events(job.id, since_id=0)
|
||||
assert len(events) == 1
|
||||
|
||||
done = job_store.patch_job(
|
||||
job.id,
|
||||
status=JobStatus.COMPLETED.value,
|
||||
result={"exitosos": 1},
|
||||
)
|
||||
assert done is not None
|
||||
assert done.status == JobStatus.COMPLETED.value
|
||||
worker = job_store.get_worker("PC-01")
|
||||
assert worker is not None
|
||||
assert worker.current_job_id is None
|
||||
|
||||
|
||||
def test_list_jobs_by_parent(store):
|
||||
job_store, _JobStatus, _WorkerStatus = store
|
||||
from infrastructure.database import Job as IpsJob
|
||||
from infrastructure.database import session_scope, utcnow
|
||||
|
||||
job_store.register_worker("PC-01")
|
||||
with session_scope() as session:
|
||||
session.add(
|
||||
IpsJob(
|
||||
id="parent-1",
|
||||
status="pending",
|
||||
title="Trabajo",
|
||||
job_metadata={},
|
||||
created_at=utcnow(),
|
||||
updated_at=utcnow(),
|
||||
)
|
||||
)
|
||||
|
||||
a = job_store.create_job("PC-01", "uploads/a/input.xlsx", "a.xlsx", job_id="parent-1")
|
||||
job_store.create_job("PC-01", "uploads/b/input.xlsx", "b.xlsx", job_id="other")
|
||||
listed = job_store.list_jobs(limit=50, job_id="parent-1")
|
||||
assert len(listed) == 1
|
||||
assert listed[0].id == a.id
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Bot descarga: download_workers, download_jobs, download_job_events.
|
||||
|
||||
Revision ID: 0004
|
||||
Revises: 0003
|
||||
Create Date: 2026-07-12
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0004"
|
||||
down_revision: Union[str, None] = "0003"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def _table_exists(conn: sa.engine.Connection, table: str) -> bool:
|
||||
return sa.inspect(conn).has_table(table)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
json_type = (
|
||||
postgresql.JSONB()
|
||||
if conn.dialect.name == "postgresql"
|
||||
else sa.JSON()
|
||||
)
|
||||
|
||||
if not _table_exists(conn, "download_workers"):
|
||||
op.create_table(
|
||||
"download_workers",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("hostname", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column("max_concurrent", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("last_heartbeat", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("current_job_id", sa.String(36), nullable=True),
|
||||
)
|
||||
op.create_index("ix_download_workers_status", "download_workers", ["status"])
|
||||
|
||||
if not _table_exists(conn, "download_jobs"):
|
||||
op.create_table(
|
||||
"download_jobs",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True),
|
||||
sa.Column("task_id", sa.String(36), sa.ForeignKey("tasks.id"), nullable=True),
|
||||
sa.Column("worker_id", sa.String(36), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("title", sa.Text(), nullable=True),
|
||||
sa.Column("excel_key", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column("downloads_prefix", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column("original_filename", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column("cancel_requested", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("result", json_type, nullable=False, server_default=sa.text("'{}'::jsonb") if conn.dialect.name == "postgresql" else sa.text("'{}'")),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
op.create_index("ix_download_jobs_job_id", "download_jobs", ["job_id"])
|
||||
op.create_index("ix_download_jobs_task_id", "download_jobs", ["task_id"])
|
||||
op.create_index("ix_download_jobs_worker_id", "download_jobs", ["worker_id"])
|
||||
op.create_index("ix_download_jobs_status", "download_jobs", ["status"])
|
||||
op.create_index(
|
||||
"ix_download_jobs_worker_status",
|
||||
"download_jobs",
|
||||
["worker_id", "status"],
|
||||
)
|
||||
|
||||
if not _table_exists(conn, "download_job_events"):
|
||||
op.create_table(
|
||||
"download_job_events",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column(
|
||||
"download_job_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("download_jobs.id"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("event_type", sa.String(64), nullable=False),
|
||||
sa.Column("message", sa.Text(), nullable=False),
|
||||
sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("radicado", sa.String(255), nullable=True),
|
||||
sa.Column("nit", sa.String(64), nullable=True),
|
||||
sa.Column("idx", sa.Integer(), nullable=True),
|
||||
sa.Column("total", sa.Integer(), nullable=True),
|
||||
sa.Column(
|
||||
"metadata",
|
||||
json_type,
|
||||
nullable=False,
|
||||
server_default=sa.text("'{}'::jsonb") if conn.dialect.name == "postgresql" else sa.text("'{}'"),
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_download_job_events_job_id",
|
||||
"download_job_events",
|
||||
["download_job_id", "id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if _table_exists(conn, "download_job_events"):
|
||||
op.drop_index("ix_download_job_events_job_id", table_name="download_job_events")
|
||||
op.drop_table("download_job_events")
|
||||
if _table_exists(conn, "download_jobs"):
|
||||
op.drop_index("ix_download_jobs_worker_status", table_name="download_jobs")
|
||||
op.drop_index("ix_download_jobs_status", table_name="download_jobs")
|
||||
op.drop_index("ix_download_jobs_worker_id", table_name="download_jobs")
|
||||
op.drop_index("ix_download_jobs_task_id", table_name="download_jobs")
|
||||
op.drop_index("ix_download_jobs_job_id", table_name="download_jobs")
|
||||
op.drop_table("download_jobs")
|
||||
if _table_exists(conn, "download_workers"):
|
||||
op.drop_index("ix_download_workers_status", table_name="download_workers")
|
||||
op.drop_table("download_workers")
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "ipsextractor-desktop",
|
||||
"version": "1.0.3",
|
||||
"version": "1.1.0",
|
||||
"private": true,
|
||||
"description": "Desktop extraction client for IPSExtractor microservices.",
|
||||
"main": "electron/main.cjs",
|
||||
|
||||
@@ -29,6 +29,7 @@ import { UpdatePrompt, CheckUpdatesButton, useAutoUpdateCheck } from "./componen
|
||||
import "./modules/counting/CountingModule";
|
||||
import "./modules/extraction/ExtractionModule";
|
||||
import "./modules/cleaning/CleaningModule";
|
||||
import "./modules/downloads/DownloadsModule";
|
||||
|
||||
type Status = "idle" | "ok" | "error" | "busy";
|
||||
type View = "home" | "job" | "settings";
|
||||
@@ -39,7 +40,7 @@ function App() {
|
||||
const [settingsReturnView, setSettingsReturnView] = useState<"home" | "job">("home");
|
||||
const [settingsSection, setSettingsSection] = useState<SettingsSection>("connection");
|
||||
const [activeJob, setActiveJob] = useState<Job | null>(null);
|
||||
const [jobSection, setJobSection] = useState<JobSection>("counting");
|
||||
const [jobSection, setJobSection] = useState<JobSection>("downloads");
|
||||
const [jobSectionContext, setJobSectionContext] = useState<Record<string, unknown>>({});
|
||||
const [apiBaseUrl, setApiBaseUrl] = useState(loadApiBaseUrl);
|
||||
const [connectionStatus, setConnectionStatus] = useState<Status>("idle");
|
||||
@@ -109,7 +110,7 @@ function App() {
|
||||
|
||||
function openJob(job: Job) {
|
||||
setActiveJob(job);
|
||||
setJobSection("counting");
|
||||
setJobSection("downloads");
|
||||
setJobSectionContext({});
|
||||
setView("job");
|
||||
}
|
||||
@@ -201,6 +202,14 @@ function App() {
|
||||
</div>
|
||||
<div className="side-section-divider" />
|
||||
<nav className="side-nav" aria-label="Módulos del trabajo">
|
||||
<button
|
||||
type="button"
|
||||
className={jobSection === "downloads" ? "active" : ""}
|
||||
onClick={() => handleActivateSection("downloads")}
|
||||
>
|
||||
<Download size={17} />
|
||||
<span>Bot de descarga</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={jobSection === "counting" ? "active" : ""}
|
||||
|
||||
@@ -9,6 +9,10 @@ import type {
|
||||
CountSyncResponse,
|
||||
CreateCountJobPayload,
|
||||
CreateJobPayload,
|
||||
DownloadJob,
|
||||
DownloadJobEvent,
|
||||
DownloadJobFilesResponse,
|
||||
DownloadWorker,
|
||||
ExtractionBatch,
|
||||
ExtractionFromCountPayload,
|
||||
Job,
|
||||
@@ -22,6 +26,16 @@ import { DEFAULT_API_BASE_URL } from "./apiDefaults";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const UPLOAD_TIMEOUT_MS = 600_000;
|
||||
const BOT_TOKEN_KEY = "botApiToken";
|
||||
const DEFAULT_BOT_TOKEN = "dev-token-change-me";
|
||||
|
||||
export function loadBotApiToken(): string {
|
||||
return (localStorage.getItem(BOT_TOKEN_KEY) || DEFAULT_BOT_TOKEN).trim();
|
||||
}
|
||||
|
||||
export function saveBotApiToken(token: string): void {
|
||||
localStorage.setItem(BOT_TOKEN_KEY, token.trim() || DEFAULT_BOT_TOKEN);
|
||||
}
|
||||
|
||||
async function fetchWithTimeout(
|
||||
url: string,
|
||||
@@ -48,19 +62,27 @@ async function fetchWithTimeout(
|
||||
}
|
||||
|
||||
export class ApiClient {
|
||||
constructor(private readonly baseUrl: string) {}
|
||||
constructor(
|
||||
private readonly baseUrl: string,
|
||||
private readonly botToken: string = loadBotApiToken(),
|
||||
) {}
|
||||
|
||||
private url(path: string): string {
|
||||
return `${this.baseUrl.replace(/\/+$/, "")}${path}`;
|
||||
}
|
||||
|
||||
private botHeaders(): HeadersInit {
|
||||
return { Authorization: `Bearer ${this.botToken}` };
|
||||
}
|
||||
|
||||
private isUploadPath(path: string): boolean {
|
||||
return (
|
||||
path.startsWith("/count/") ||
|
||||
path.includes("/ingest") ||
|
||||
path.includes("/upload") ||
|
||||
path.startsWith("/cleaning/analyze-csv") ||
|
||||
path.startsWith("/cleaning/jobs/from-csv")
|
||||
path.startsWith("/cleaning/jobs/from-csv") ||
|
||||
path === "/bot-descarga/jobs"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -194,6 +216,68 @@ export class ApiClient {
|
||||
await this.handle(response);
|
||||
}
|
||||
|
||||
// ── Bot Descarga (control remoto) ──────────────────────────────────────────
|
||||
|
||||
async listDownloadWorkers(): Promise<DownloadWorker[]> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url("/bot-descarga/workers"),
|
||||
{ headers: this.botHeaders() },
|
||||
this.timeoutFor("/bot-descarga/workers"),
|
||||
);
|
||||
return this.handle<DownloadWorker[]>(response);
|
||||
}
|
||||
|
||||
async createDownloadJob(form: FormData): Promise<DownloadJob> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url("/bot-descarga/jobs"),
|
||||
{ method: "POST", headers: this.botHeaders(), body: form },
|
||||
this.timeoutFor("/bot-descarga/jobs"),
|
||||
);
|
||||
return this.handle<DownloadJob>(response);
|
||||
}
|
||||
|
||||
async listDownloadJobs(limit = 50, parentJobId?: string): Promise<DownloadJob[]> {
|
||||
const qs = new URLSearchParams({ limit: String(limit) });
|
||||
if (parentJobId) qs.set("job_id", parentJobId);
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-descarga/jobs?${qs.toString()}`),
|
||||
{ headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<DownloadJob[]>(response);
|
||||
}
|
||||
|
||||
async getDownloadJob(jobId: string): Promise<DownloadJob> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-descarga/jobs/${encodeURIComponent(jobId)}`),
|
||||
{ headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<DownloadJob>(response);
|
||||
}
|
||||
|
||||
async listDownloadJobFiles(jobId: string): Promise<DownloadJobFilesResponse> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-descarga/jobs/${encodeURIComponent(jobId)}/files`),
|
||||
{ headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<DownloadJobFilesResponse>(response);
|
||||
}
|
||||
|
||||
async getDownloadEvents(jobId: string, since = 0): Promise<{ events: DownloadJobEvent[] }> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-descarga/jobs/${encodeURIComponent(jobId)}/events?since=${since}`),
|
||||
{ headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<{ events: DownloadJobEvent[] }>(response);
|
||||
}
|
||||
|
||||
async cancelDownloadJob(jobId: string): Promise<DownloadJob> {
|
||||
const response = await fetchWithTimeout(
|
||||
this.url(`/bot-descarga/jobs/${encodeURIComponent(jobId)}/cancel`),
|
||||
{ method: "POST", headers: this.botHeaders() },
|
||||
);
|
||||
return this.handle<DownloadJob>(response);
|
||||
}
|
||||
|
||||
// ── Count Jobs ─────────────────────────────────────────────────────────────
|
||||
|
||||
/** Envía PDFs al servicio de conteo (síncrono). */
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
export type JobSection = "counting" | "extraction" | "cleaning" | "results";
|
||||
export type JobSection = "counting" | "extraction" | "cleaning" | "downloads" | "results";
|
||||
|
||||
|
||||
@@ -17,8 +17,9 @@ import { ExtractionResultsTable } from "../components/ExtractionResultsTable";
|
||||
import { BackgroundProgressBanner } from "../components/BackgroundProgressBanner";
|
||||
import type { ApiClient } from "../lib/apiClient";
|
||||
import { downloadCsv, toCsv, buildExportCsvFilename } from "../lib/csv";
|
||||
import { DownloadsResultsPanel } from "../modules/downloads/DownloadsResultsPanel";
|
||||
|
||||
type ResultTab = "counting" | "extraction" | "cleaning";
|
||||
type ResultTab = "counting" | "extraction" | "cleaning" | "downloads";
|
||||
|
||||
interface Props {
|
||||
client: ApiClient;
|
||||
@@ -31,6 +32,7 @@ interface Props {
|
||||
onStartCounting: () => void;
|
||||
onStartExtraction: (ctx?: Record<string, unknown>) => void;
|
||||
onStartCleaning: (ctx?: Record<string, unknown>) => void;
|
||||
onStartDownload: () => void;
|
||||
}
|
||||
|
||||
export function JobResultsView({
|
||||
@@ -44,6 +46,7 @@ export function JobResultsView({
|
||||
onStartCounting,
|
||||
onStartExtraction,
|
||||
onStartCleaning,
|
||||
onStartDownload,
|
||||
}: Props) {
|
||||
const [activeTab, setActiveTab] = useState<ResultTab>("counting");
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
@@ -54,6 +57,8 @@ export function JobResultsView({
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
const cleaningTasks = tasks.filter((t) => t.type === "cleaning")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
const downloadTasks = tasks.filter((t) => t.type === "downloads")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
|
||||
async function refresh() {
|
||||
setRefreshing(true);
|
||||
@@ -63,7 +68,7 @@ export function JobResultsView({
|
||||
|
||||
return (
|
||||
<div className="job-results-view">
|
||||
{/* ── Sub-tabs: Conteo | Extracción | Limpieza ── */}
|
||||
{/* ── Sub-tabs: Conteo | Extracción | Limpieza | Bot de descarga ── */}
|
||||
<div className="job-results-tabs">
|
||||
<button
|
||||
type="button"
|
||||
@@ -110,6 +115,17 @@ export function JobResultsView({
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`job-results-tab${activeTab === "downloads" ? " job-results-tab--active" : ""}`}
|
||||
onClick={() => setActiveTab("downloads")}
|
||||
>
|
||||
<Download size={15} />
|
||||
Bot de descarga
|
||||
{downloadTasks.length > 0 && (
|
||||
<span className="job-results-tab-count">{downloadTasks.length}</span>
|
||||
)}
|
||||
</button>
|
||||
<div style={{ flex: 1 }} />
|
||||
<button
|
||||
type="button"
|
||||
@@ -155,6 +171,14 @@ export function JobResultsView({
|
||||
onStartCleaning={() => onStartCleaning()}
|
||||
/>
|
||||
)}
|
||||
{activeTab === "downloads" && (
|
||||
<DownloadsResultsPanel
|
||||
client={client}
|
||||
job={job}
|
||||
tasks={downloadTasks}
|
||||
onStartDownload={onStartDownload}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -7,6 +7,7 @@ import { BackgroundProgressBanner } from "../components/BackgroundProgressBanner
|
||||
import { CountingWizardModule } from "../modules/counting/CountingWizardModule";
|
||||
import { CleaningWizardModule } from "../modules/cleaning/CleaningWizardModule";
|
||||
import { JobExtractionWizard } from "../modules/extraction/JobExtractionWizard";
|
||||
import { DownloadsWizardModule } from "../modules/downloads/DownloadsWizardModule";
|
||||
import { JobResultsView } from "./JobResultsView";
|
||||
|
||||
interface Props {
|
||||
@@ -120,6 +121,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
{section === "counting" && "Conteo de servicios"}
|
||||
{section === "extraction" && "Extracción de datos"}
|
||||
{section === "cleaning" && "Limpieza de datos"}
|
||||
{section === "downloads" && "Bot de descarga"}
|
||||
{section === "results" && "Resultados del trabajo"}
|
||||
</div>
|
||||
</header>
|
||||
@@ -179,6 +181,10 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className={`job-section-pane${section === "downloads" ? "" : " job-section-pane--hidden"}`}>
|
||||
<DownloadsWizardModule client={client} job={job} />
|
||||
</div>
|
||||
|
||||
<div className={`job-section-pane${section === "results" ? "" : " job-section-pane--hidden"}`}>
|
||||
<JobResultsView
|
||||
client={client}
|
||||
@@ -191,6 +197,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
onStartCounting={() => onActivateSection("counting")}
|
||||
onStartExtraction={(ctx?: Record<string, unknown>) => onActivateSection("extraction", ctx)}
|
||||
onStartCleaning={(ctx?: Record<string, unknown>) => onActivateSection("cleaning", ctx)}
|
||||
onStartDownload={() => onActivateSection("downloads")}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -303,6 +303,62 @@ export interface UpdateProgress {
|
||||
percent?: number;
|
||||
}
|
||||
|
||||
/** Worker del bot de descargas PDF (control remoto). */
|
||||
export interface DownloadWorker {
|
||||
id: string;
|
||||
hostname: string;
|
||||
max_concurrent: number;
|
||||
status: "online" | "offline" | "busy" | string;
|
||||
last_heartbeat: string;
|
||||
current_job_id: string | null;
|
||||
}
|
||||
|
||||
export interface DownloadJob {
|
||||
id: string;
|
||||
worker_id: string;
|
||||
status: "pending" | "running" | "completed" | "failed" | "cancelled" | string;
|
||||
title: string;
|
||||
excel_key: string;
|
||||
downloads_prefix: string;
|
||||
original_filename: string;
|
||||
cancel_requested: boolean;
|
||||
result: Record<string, unknown>;
|
||||
error: string | null;
|
||||
created_at: string;
|
||||
started_at: string | null;
|
||||
finished_at: string | null;
|
||||
/** Job Desktop contenedor (IPSExtractor jobs.id) */
|
||||
job_id?: string | null;
|
||||
/** Task type=downloads */
|
||||
task_id?: string | null;
|
||||
}
|
||||
|
||||
export interface DownloadJobFile {
|
||||
key: string;
|
||||
nit?: string | null;
|
||||
radicado?: string | null;
|
||||
}
|
||||
|
||||
export interface DownloadJobFilesResponse {
|
||||
job_id: string;
|
||||
prefix: string;
|
||||
files: DownloadJobFile[];
|
||||
count: number;
|
||||
}
|
||||
|
||||
export interface DownloadJobEvent {
|
||||
id: number;
|
||||
job_id: string;
|
||||
event_type: string;
|
||||
message: string;
|
||||
timestamp: string;
|
||||
radicado?: string | null;
|
||||
nit?: string | null;
|
||||
index?: number | null;
|
||||
total?: number | null;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface DesktopBridge {
|
||||
getRuntimeConfig(): Promise<RuntimeConfig>;
|
||||
selectDirectory(): Promise<string | null>;
|
||||
|
||||
Reference in New Issue
Block a user