feat: Introduce bot-descarga service for PDF downloads and remote bot control. Add Docker configuration, environment variables, and a GUI for agent management. Enhance README with bot details and installation instructions.

This commit is contained in:
erik
2026-07-12 21:20:29 -05:00
parent a75ccce13f
commit 103bcea15b
66 changed files with 4469 additions and 13 deletions
+16 -1
View File
@@ -18,6 +18,7 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name
EXTRACTION_URL = os.getenv("EXTRACTION_URL", "http://extraction:8001").rstrip("/")
CLEANING_URL = os.getenv("CLEANING_URL", "http://cleaning:8002").rstrip("/")
COUNT_URL = os.getenv("COUNT_URL", "http://count-api:8080").rstrip("/")
BOT_DESCARGA_URL = os.getenv("BOT_DESCARGA_URL", "http://bot-descarga:8200").rstrip("/")
# Timeouts: JSON ligero vs uploads OCR/conteo
_TIMEOUT_JSON = httpx.Timeout(connect=5.0, read=30.0, write=30.0, pool=5.0)
@@ -33,7 +34,13 @@ async def lifespan(app: FastAPI):
timeout=_TIMEOUT_JSON,
limits=httpx.Limits(max_connections=100, max_keepalive_connections=20),
)
_log.info("Gateway iniciado — extraction=%s cleaning=%s count=%s", EXTRACTION_URL, CLEANING_URL, COUNT_URL)
_log.info(
"Gateway iniciado — extraction=%s cleaning=%s count=%s bot_descarga=%s",
EXTRACTION_URL,
CLEANING_URL,
COUNT_URL,
BOT_DESCARGA_URL,
)
yield
await _http_client.aclose()
_http_client = None
@@ -72,6 +79,7 @@ def _is_upload_path(path: str) -> bool:
or "/upload" in path
or path.startswith("/cleaning/analyze-csv")
or path.startswith("/cleaning/jobs/from-csv")
or path.startswith("/bot-descarga/jobs")
)
@@ -91,12 +99,14 @@ async def health() -> dict[str, Any]:
extraction = await _safe_health(client, EXTRACTION_URL)
cleaning = await _safe_health(client, CLEANING_URL)
counting = await _safe_health(client, COUNT_URL)
bot_descarga = await _safe_health(client, BOT_DESCARGA_URL)
return {
"status": "ok" if extraction["ok"] and cleaning["ok"] else "degraded",
"services": {
"extraction": extraction,
"cleaning": cleaning,
"counting": counting,
"bot_descarga": bot_descarga,
},
}
@@ -152,6 +162,11 @@ async def count_jobs_proxy(path: str, request: Request) -> Response:
return await _proxy_request(request, f"{EXTRACTION_URL}/count-jobs/{path}")
@app.api_route("/bot-descarga/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
async def bot_descarga_proxy(path: str, request: Request) -> Response:
return await _proxy_request(request, f"{BOT_DESCARGA_URL}/{path}")
@app.api_route("/jobs", methods=["GET", "POST"])
async def jobs_root_proxy(request: Request) -> Response:
return await _proxy_request(request, f"{EXTRACTION_URL}/jobs")
+28
View File
@@ -0,0 +1,28 @@
FROM python:3.12-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PYTHONPATH=/app
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends curl \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt
COPY domain /app/domain
COPY infrastructure /app/infrastructure
COPY api /app/api
ENV BOT_API_HOST=0.0.0.0
ENV BOT_API_PORT=8200
EXPOSE 8200
HEALTHCHECK --interval=10s --timeout=5s --retries=5 --start-period=15s \
CMD curl -f http://127.0.0.1:8200/health || exit 1
CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8200"]
+51
View File
@@ -0,0 +1,51 @@
# Bot Descarga API (infraestructura IPSExtractor)
Microservicio FastAPI de control remoto de bots. Corre en Docker;
las PCs de bot **no** exponen puertos — solo el agent en [`bots/bot_descarga`](../../bots/bot_descarga).
Persistencia: **PostgreSQL compartido** (`DATABASE_URL`), tablas
`download_workers`, `download_jobs`, `download_job_events` + `tasks` (`type=downloads`).
## Layout MinIO (`pys-cuentas-medicas`)
```
uploads/{download_job_id}/input.xlsx
downloads/{download_job_id}/{nit}/{radicado}.pdf
```
## Arranque (compose)
Servicio `bot-descarga` en el `docker-compose.yml` raíz.
Gateway: `/bot-descarga/*` → este servicio `:8200`.
## Variables
```
BOT_API_TOKEN=dev-token-change-me
DATABASE_URL=postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor
MINIO_ENDPOINT=https://s3.builker.xyz
MINIO_BUCKET=pys-cuentas-medicas
MINIO_ANONYMOUS=1
```
## Worker en PCs (`bots/bot_descarga`)
```bash
cd bots/bot_descarga
python -m bot_descarga.worker.run \
--api-url http://GATEWAY:8189/bot-descarga \
--worker-id PC-01 \
--token $BOT_API_TOKEN
```
O con GUI: `python run_gui.py`. Ver [README del agent](../../bots/bot_descarga/README.md).
El agent queda idle hasta que el usuario cree un job desde el desktop
(enviando `job_id` del Job IPSExtractor).
## Tests (API)
```bash
cd services/bot_descarga
pytest -q
```
+293
View File
@@ -0,0 +1,293 @@
from __future__ import annotations
import os
from pathlib import Path
from typing import Any, Dict, List, Optional
from fastapi import Depends, FastAPI, File, Form, HTTPException, Query, UploadFile
from fastapi.responses import Response
from pydantic import BaseModel, Field
from domain.models import JobStatus
from infrastructure.auth import require_bearer
from infrastructure.storage_minio import MinioSettings, MinioStorage
from infrastructure.store import JobStore
store = JobStore()
minio = MinioStorage(MinioSettings.from_env())
app = FastAPI(title="BotDescarga Remote API", version="1.0.0")
class RegisterWorkerBody(BaseModel):
worker_id: str
hostname: str = ""
max_concurrent: int = 1
class HeartbeatBody(BaseModel):
worker_id: str
class EventBody(BaseModel):
event_type: str
message: str
radicado: Optional[str] = None
nit: Optional[str] = None
index: Optional[int] = None
total: Optional[int] = None
metadata: Dict[str, Any] = Field(default_factory=dict)
timestamp: Optional[str] = None
class PatchJobBody(BaseModel):
status: Optional[str] = None
result: Optional[Dict[str, Any]] = None
error: Optional[str] = None
@app.get("/health")
def health() -> dict:
return {"status": "ok", "service": "bot-descarga"}
@app.post("/workers/register")
def register_worker(body: RegisterWorkerBody, _: str = Depends(require_bearer)) -> dict:
worker = store.register_worker(
body.worker_id,
hostname=body.hostname,
max_concurrent=body.max_concurrent,
)
return worker.to_dict()
@app.post("/workers/heartbeat")
def heartbeat(body: HeartbeatBody, _: str = Depends(require_bearer)) -> dict:
worker = store.heartbeat(body.worker_id)
if not worker:
raise HTTPException(status_code=404, detail="Worker no registrado")
return worker.to_dict()
@app.get("/workers")
def list_workers(_: str = Depends(require_bearer)) -> List[dict]:
return [w.to_dict() for w in store.list_workers()]
@app.get("/workers/me/pending")
def claim_pending(
worker_id: str = Query(...),
_: str = Depends(require_bearer),
) -> dict:
job = store.claim_pending(worker_id)
if not job:
return {"job": None}
return {"job": job.to_dict()}
@app.post("/jobs")
async def create_job(
worker_id: str = Form(...),
job_id: str = Form(...),
title: str = Form(""),
file: UploadFile = File(...),
_: str = Depends(require_bearer),
) -> dict:
if not worker_id.strip():
raise HTTPException(status_code=400, detail="worker_id obligatorio")
parent_job_id = job_id.strip()
if not parent_job_id:
raise HTTPException(status_code=400, detail="job_id (Job Desktop) obligatorio")
if not store.parent_job_exists(parent_job_id):
raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado")
if not store.is_worker_online(worker_id):
raise HTTPException(
status_code=409,
detail=f"Worker '{worker_id}' no está online. Enciende el agent primero.",
)
data = await file.read()
if not data:
raise HTTPException(status_code=400, detail="Archivo vacío")
original = file.filename or "input.xlsx"
content_type = file.content_type or "application/octet-stream"
title_clean = title.strip()
job = store.create_job(
worker_id=worker_id.strip(),
excel_key="",
original_filename=original,
title=title_clean,
job_id=parent_job_id,
)
task_id = store.create_downloads_task(
parent_job_id=parent_job_id,
download_job_id=job.id,
worker_id=worker_id.strip(),
title=title_clean or f"Bot descarga ({original})",
original_filename=original,
)
job = store.get_job(job.id)
assert job is not None
try:
key = minio.upload_excel(job.id, data, original, content_type=content_type)
except Exception as e:
store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}")
raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e
job = store.update_excel_key(job.id, key)
assert job is not None
store.add_event(
job.id,
"run_started",
f"Job creado y asignado a {worker_id}",
metadata={
"excel_key": key,
"worker_id": worker_id,
"job_id": parent_job_id,
"task_id": task_id,
},
)
return job.to_dict()
@app.get("/jobs")
def list_jobs(
limit: int = Query(100, ge=1, le=500),
job_id: Optional[str] = Query(None, description="Filtrar por Job Desktop"),
_: str = Depends(require_bearer),
) -> List[dict]:
return [j.to_dict() for j in store.list_jobs(limit=limit, job_id=job_id)]
@app.get("/jobs/{download_job_id}")
def get_job(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
job = store.get_job(download_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
return job.to_dict()
@app.get("/jobs/{download_job_id}/files")
def list_job_files(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
job = store.get_job(download_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
try:
return minio.list_download_files(download_job_id, prefix=job.downloads_prefix)
except Exception as e:
raise HTTPException(status_code=502, detail=f"Error listando MinIO: {e}") from e
@app.get("/jobs/{download_job_id}/events")
def get_events(
download_job_id: str,
since: int = Query(0, ge=0),
_: str = Depends(require_bearer),
) -> dict:
if not store.get_job(download_job_id):
raise HTTPException(status_code=404, detail="Job no encontrado")
events = store.list_events(download_job_id, since_id=since)
return {"events": [e.to_dict() for e in events]}
@app.post("/jobs/{download_job_id}/cancel")
def cancel_job(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
job = store.request_cancel(download_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
store.add_event(download_job_id, "warning", "Cancelación solicitada por el usuario")
return job.to_dict()
@app.get("/jobs/{download_job_id}/file")
def get_job_file(download_job_id: str, _: str = Depends(require_bearer)):
job = store.get_job(download_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
if not job.excel_key:
raise HTTPException(status_code=404, detail="Excel no disponible")
try:
data = minio.get_bytes(job.excel_key)
except Exception as e:
raise HTTPException(status_code=502, detail=f"Error leyendo MinIO: {e}") from e
filename = Path(job.excel_key).name
return Response(
content=data,
media_type="application/octet-stream",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
@app.get("/jobs/{download_job_id}/file-url")
def get_job_file_url(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
job = store.get_job(download_job_id)
if not job or not job.excel_key:
raise HTTPException(status_code=404, detail="Job/archivo no encontrado")
try:
url = minio.presigned_get(job.excel_key)
except Exception as e:
raise HTTPException(status_code=502, detail=str(e)) from e
return {"url": url, "excel_key": job.excel_key}
@app.post("/jobs/{download_job_id}/events")
def post_event(
download_job_id: str,
body: EventBody,
_: str = Depends(require_bearer),
) -> dict:
if not store.get_job(download_job_id):
raise HTTPException(status_code=404, detail="Job no encontrado")
event = store.add_event(
download_job_id,
body.event_type,
body.message,
radicado=body.radicado,
nit=body.nit,
index=body.index,
total=body.total,
metadata=body.metadata,
timestamp=body.timestamp,
)
job = store.get_job(download_job_id)
if job and body.index is not None and body.total is not None:
result = dict(job.result or {})
result["processed"] = body.index
result["total"] = body.total
if body.nit:
result["nit_actual"] = body.nit
store.patch_job(download_job_id, result=result)
return event.to_dict()
@app.patch("/jobs/{download_job_id}")
def patch_job(
download_job_id: str,
body: PatchJobBody,
_: str = Depends(require_bearer),
) -> dict:
if body.status and body.status not in {s.value for s in JobStatus}:
raise HTTPException(status_code=400, detail="status inválido")
job = store.patch_job(
download_job_id,
status=body.status,
result=body.result,
error=body.error,
)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
return job.to_dict()
@app.get("/jobs/{download_job_id}/cancel-flag")
def cancel_flag(download_job_id: str, _: str = Depends(require_bearer)) -> dict:
job = store.get_job(download_job_id)
if not job:
raise HTTPException(status_code=404, detail="Job no encontrado")
return {"cancel_requested": job.cancel_requested, "status": job.status}
+76
View File
@@ -0,0 +1,76 @@
from __future__ import annotations
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from enum import Enum
from typing import Any, Dict, List, Optional
def utc_now() -> str:
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
class WorkerStatus(str, Enum):
ONLINE = "online"
OFFLINE = "offline"
BUSY = "busy"
class JobStatus(str, Enum):
PENDING = "pending"
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
CANCELLED = "cancelled"
@dataclass
class Worker:
id: str
hostname: str = ""
max_concurrent: int = 1
status: str = WorkerStatus.ONLINE.value
last_heartbeat: str = field(default_factory=utc_now)
current_job_id: Optional[str] = None
def to_dict(self) -> dict:
return asdict(self)
@dataclass
class Job:
id: str
worker_id: str
status: str = JobStatus.PENDING.value
title: str = ""
excel_key: str = ""
downloads_prefix: str = ""
original_filename: str = ""
cancel_requested: bool = False
result: Dict[str, Any] = field(default_factory=dict)
error: Optional[str] = None
created_at: str = field(default_factory=utc_now)
started_at: Optional[str] = None
finished_at: Optional[str] = None
job_id: Optional[str] = None
task_id: Optional[str] = None
def to_dict(self) -> dict:
return asdict(self)
@dataclass
class JobEventRecord:
id: int
job_id: str
event_type: str
message: str
timestamp: str
radicado: Optional[str] = None
nit: Optional[str] = None
index: Optional[int] = None
total: Optional[int] = None
metadata: Dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict:
return asdict(self)
@@ -0,0 +1,20 @@
from __future__ import annotations
import os
from typing import Optional
from fastapi import Header, HTTPException
def get_api_token() -> str:
return os.getenv("BOT_API_TOKEN", "dev-token-change-me")
def require_bearer(authorization: Optional[str] = Header(default=None)) -> str:
expected = get_api_token()
if not authorization or not authorization.lower().startswith("bearer "):
raise HTTPException(status_code=401, detail="Authorization Bearer requerido")
token = authorization.split(" ", 1)[1].strip()
if token != expected:
raise HTTPException(status_code=401, detail="Token inválido")
return token
@@ -0,0 +1,159 @@
"""Persistencia Postgres del servicio bot-descarga (DB compartida IPSExtractor)."""
from __future__ import annotations
import os
import uuid
from contextlib import contextmanager
from datetime import datetime, timezone
from typing import Any, Iterator
from sqlalchemy import (
JSON,
Boolean,
DateTime,
ForeignKey,
Integer,
String,
Text,
create_engine,
text,
)
from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, sessionmaker
DATABASE_URL = os.getenv(
"DATABASE_URL",
"postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor",
)
engine = create_engine(DATABASE_URL, pool_pre_ping=True)
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
class Base(DeclarativeBase):
pass
def utcnow() -> datetime:
return datetime.now(timezone.utc)
def new_id() -> str:
return str(uuid.uuid4())
def _iso(dt: datetime | None) -> str | None:
if dt is None:
return None
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt.astimezone().isoformat(timespec="seconds")
class Job(Base):
"""Contenedor Desktop (lectura / FK)."""
__tablename__ = "jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
title: Mapped[str | None] = mapped_column(Text, nullable=True)
job_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
class Task(Base):
__tablename__ = "tasks"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
job_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("jobs.id"), nullable=True, index=True
)
type: Mapped[str] = mapped_column(String(64), index=True)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
title: Mapped[str | None] = mapped_column(Text, nullable=True)
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
owner_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class DownloadWorkerRow(Base):
__tablename__ = "download_workers"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
hostname: Mapped[str] = mapped_column(Text, default="")
max_concurrent: Mapped[int] = mapped_column(Integer, default=1)
status: Mapped[str] = mapped_column(String(32), index=True)
last_heartbeat: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
current_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
class DownloadJobRow(Base):
__tablename__ = "download_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
job_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("jobs.id"), nullable=True, index=True
)
task_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("tasks.id"), nullable=True, index=True
)
worker_id: Mapped[str] = mapped_column(String(36), index=True)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
title: Mapped[str | None] = mapped_column(Text, nullable=True)
excel_key: Mapped[str] = mapped_column(Text, default="")
downloads_prefix: Mapped[str] = mapped_column(Text, default="")
original_filename: Mapped[str] = mapped_column(Text, default="")
cancel_requested: Mapped[bool] = mapped_column(Boolean, default=False)
result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class DownloadJobEventRow(Base):
__tablename__ = "download_job_events"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
download_job_id: Mapped[str] = mapped_column(
String(36), ForeignKey("download_jobs.id"), index=True
)
event_type: Mapped[str] = mapped_column(String(64))
message: Mapped[str] = mapped_column(Text)
timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
radicado: Mapped[str | None] = mapped_column(String(255), nullable=True)
nit: Mapped[str | None] = mapped_column(String(64), nullable=True)
idx: Mapped[int | None] = mapped_column(Integer, nullable=True)
total: Mapped[int | None] = mapped_column(Integer, nullable=True)
metadata_json: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
def init_db() -> None:
if DATABASE_URL.startswith("postgresql"):
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:
connection.execute(text("SELECT pg_advisory_lock(74512004)"))
try:
Base.metadata.create_all(bind=connection)
finally:
connection.execute(text("SELECT pg_advisory_unlock(74512004)"))
return
Base.metadata.create_all(bind=engine)
@contextmanager
def session_scope() -> Iterator[Session]:
session = SessionLocal()
try:
yield session
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()
@@ -0,0 +1,215 @@
from __future__ import annotations
import json
import os
from dataclasses import dataclass
from pathlib import Path
from typing import BinaryIO, List, Optional
@dataclass(frozen=True)
class MinioSettings:
endpoint: str = "https://s3.builker.xyz"
bucket: str = "pys-cuentas-medicas"
access_key: str = ""
secret_key: str = ""
region: str = "us-east-1"
anonymous: bool = True
@classmethod
def from_env(cls) -> "MinioSettings":
access_key = (os.getenv("MINIO_ACCESS_KEY") or "").strip()
secret_key = (os.getenv("MINIO_SECRET_KEY") or "").strip()
anon_flag = (os.getenv("MINIO_ANONYMOUS") or "").strip().lower()
if anon_flag in ("0", "false", "no"):
anonymous = False
elif anon_flag in ("1", "true", "yes"):
anonymous = True
else:
anonymous = not (access_key and secret_key)
return cls(
endpoint=os.getenv("MINIO_ENDPOINT", "https://s3.builker.xyz").rstrip("/"),
bucket=os.getenv("MINIO_BUCKET", "pys-cuentas-medicas"),
access_key=access_key,
secret_key=secret_key,
region=os.getenv("MINIO_REGION", "us-east-1"),
anonymous=anonymous,
)
def excel_key(job_id: str, filename: str = "input.xlsx") -> str:
return f"uploads/{job_id}/{filename}"
def downloads_prefix(job_id: str) -> str:
return f"downloads/{job_id}/"
def pdf_key(job_id: str, nit: str, radicado: str) -> str:
return f"downloads/{job_id}/{nit}/{radicado}.pdf"
def meta_key(job_id: str) -> str:
return f"uploads/{job_id}/meta.json"
def manifest_key(job_id: str) -> str:
return f"downloads/{job_id}/_manifest.json"
def _s3_client(settings: MinioSettings):
import boto3
from botocore import UNSIGNED
from botocore.client import Config
s3_cfg = {
"addressing_style": "path",
"request_checksum_calculation": "when_required",
"response_checksum_validation": "when_required",
}
if settings.anonymous:
return boto3.client(
"s3",
endpoint_url=settings.endpoint,
region_name=settings.region,
config=Config(signature_version=UNSIGNED, s3=s3_cfg),
)
return boto3.client(
"s3",
endpoint_url=settings.endpoint,
aws_access_key_id=settings.access_key,
aws_secret_access_key=settings.secret_key,
region_name=settings.region,
config=Config(signature_version="s3v4", s3=s3_cfg),
)
class MinioStorage:
"""Cliente S3-compatible para MinIO (uploads/downloads por job_id)."""
def __init__(self, settings: Optional[MinioSettings] = None):
self.settings = settings or MinioSettings.from_env()
self._client = None
@property
def client(self):
if self._client is None:
self._client = _s3_client(self.settings)
return self._client
@property
def bucket(self) -> str:
return self.settings.bucket
def put_bytes(self, key: str, data: bytes, content_type: str = "application/octet-stream") -> str:
self.client.put_object(
Bucket=self.bucket,
Key=key,
Body=data,
ContentType=content_type,
)
return key
def put_file(self, key: str, path: str | Path, content_type: str = "application/octet-stream") -> str:
path = Path(path)
return self.put_bytes(key, path.read_bytes(), content_type=content_type)
def put_stream(self, key: str, stream: BinaryIO, content_type: str = "application/octet-stream") -> str:
data = stream.read()
return self.put_bytes(key, data, content_type=content_type)
def get_bytes(self, key: str) -> bytes:
obj = self.client.get_object(Bucket=self.bucket, Key=key)
return obj["Body"].read()
def download_file(self, key: str, dest: str | Path) -> Path:
dest = Path(dest)
dest.parent.mkdir(parents=True, exist_ok=True)
self.client.download_file(self.bucket, key, str(dest))
return dest
def exists(self, key: str) -> bool:
try:
self.client.head_object(Bucket=self.bucket, Key=key)
return True
except Exception:
return False
def list_keys(self, prefix: str) -> List[str]:
keys: List[str] = []
token = None
while True:
kwargs = {"Bucket": self.bucket, "Prefix": prefix}
if token:
kwargs["ContinuationToken"] = token
resp = self.client.list_objects_v2(**kwargs)
for item in resp.get("Contents") or []:
keys.append(item["Key"])
if not resp.get("IsTruncated"):
break
token = resp.get("NextContinuationToken")
return keys
def presigned_get(self, key: str, expires_in: int = 3600) -> str:
if self.settings.anonymous:
return f"{self.settings.endpoint}/{self.bucket}/{key}"
return self.client.generate_presigned_url(
"get_object",
Params={"Bucket": self.bucket, "Key": key},
ExpiresIn=expires_in,
)
def upload_excel(
self,
job_id: str,
data: bytes,
original_filename: str,
content_type: str = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
) -> str:
ext = Path(original_filename).suffix.lower() or ".xlsx"
canonical = f"input{ext}" if ext in (".xlsx", ".xls", ".csv") else "input.xlsx"
key = excel_key(job_id, canonical)
self.put_bytes(key, data, content_type=content_type)
meta = {
"original_filename": original_filename,
"size": len(data),
"canonical": canonical,
}
self.put_bytes(meta_key(job_id), json.dumps(meta).encode("utf-8"), "application/json")
return key
def upload_pdf(self, job_id: str, nit: str, radicado: str, local_path: str | Path) -> str:
key = pdf_key(job_id, nit, radicado)
self.put_file(key, local_path, content_type="application/pdf")
return key
def write_manifest(self, job_id: str, keys: List[str]) -> str:
key = manifest_key(job_id)
payload = {"job_id": job_id, "files": keys, "count": len(keys)}
self.put_bytes(key, json.dumps(payload, ensure_ascii=False).encode("utf-8"), "application/json")
return key
def list_download_files(self, job_id: str, prefix: str | None = None) -> dict:
"""Lista PDFs de una corrida (manifest o list_keys)."""
prefix = prefix or downloads_prefix(job_id)
files: List[dict] = []
try:
raw = self.get_bytes(manifest_key(job_id))
payload = json.loads(raw.decode("utf-8"))
for key in payload.get("files") or []:
files.append(self._file_entry(str(key)))
except Exception:
for key in self.list_keys(prefix):
if key.endswith("/_manifest.json") or key.endswith("_manifest.json"):
continue
files.append(self._file_entry(key))
return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)}
@staticmethod
def _file_entry(key: str) -> dict:
# downloads/{job_id}/{nit}/{radicado}.pdf
parts = key.strip("/").split("/")
nit = parts[-2] if len(parts) >= 3 else None
name = parts[-1] if parts else key
radicado = name[:-4] if name.lower().endswith(".pdf") else name
return {"key": key, "nit": nit, "radicado": radicado}
@@ -0,0 +1,466 @@
"""Repositorio sobre PostgreSQL (tablas download_* + sync de tasks)."""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional
from sqlalchemy import select
from domain.models import (
Job,
JobEventRecord,
JobStatus,
Worker,
WorkerStatus,
utc_now,
)
from infrastructure.database import (
DownloadJobEventRow,
DownloadJobRow,
DownloadWorkerRow,
Task,
_iso,
init_db,
session_scope,
utcnow,
)
from infrastructure.storage_minio import downloads_prefix
HEARTBEAT_OFFLINE_SECONDS = 45
def _parse_iso(value: str | None) -> datetime | None:
if not value:
return None
dt = datetime.fromisoformat(value)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt
class JobStore:
"""API de persistencia compatible con el store SQLite anterior."""
def __init__(self, db_path: str | None = None):
# db_path ignorado (compat); usa DATABASE_URL.
init_db()
def _row_worker(self, row: DownloadWorkerRow) -> Worker:
worker = Worker(
id=row.id,
hostname=row.hostname or "",
max_concurrent=int(row.max_concurrent or 1),
status=row.status,
last_heartbeat=_iso(row.last_heartbeat) or utc_now(),
current_job_id=row.current_job_id,
)
return self._refresh_worker_status(worker)
def _row_job(self, row: DownloadJobRow) -> Job:
return Job(
id=row.id,
worker_id=row.worker_id,
status=row.status,
title=row.title or "",
excel_key=row.excel_key or "",
downloads_prefix=row.downloads_prefix or "",
original_filename=row.original_filename or "",
cancel_requested=bool(row.cancel_requested),
result=dict(row.result or {}),
error=row.error,
created_at=_iso(row.created_at) or utc_now(),
started_at=_iso(row.started_at),
finished_at=_iso(row.finished_at),
job_id=row.job_id,
task_id=row.task_id,
)
def _refresh_worker_status(self, worker: Worker) -> Worker:
try:
hb = datetime.fromisoformat(worker.last_heartbeat)
if hb.tzinfo is None:
hb = hb.replace(tzinfo=timezone.utc)
age = datetime.now(timezone.utc) - hb.astimezone(timezone.utc)
if age > timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS):
worker.status = WorkerStatus.OFFLINE.value
elif worker.current_job_id:
worker.status = WorkerStatus.BUSY.value
else:
worker.status = WorkerStatus.ONLINE.value
except Exception:
worker.status = WorkerStatus.OFFLINE.value
return worker
def register_worker(
self,
worker_id: str,
hostname: str = "",
max_concurrent: int = 1,
) -> Worker:
with session_scope() as session:
row = session.get(DownloadWorkerRow, worker_id)
now = utcnow()
if row:
row.hostname = hostname
row.max_concurrent = max_concurrent
row.status = WorkerStatus.ONLINE.value
row.last_heartbeat = now
row.current_job_id = None
else:
row = DownloadWorkerRow(
id=worker_id,
hostname=hostname,
max_concurrent=max_concurrent,
status=WorkerStatus.ONLINE.value,
last_heartbeat=now,
current_job_id=None,
)
session.add(row)
session.flush()
return self._row_worker(row)
def heartbeat(self, worker_id: str) -> Optional[Worker]:
with session_scope() as session:
row = session.get(DownloadWorkerRow, worker_id)
if not row:
return None
row.last_heartbeat = utcnow()
row.status = (
WorkerStatus.BUSY.value if row.current_job_id else WorkerStatus.ONLINE.value
)
session.flush()
return self._row_worker(row)
def get_worker(self, worker_id: str) -> Optional[Worker]:
with session_scope() as session:
row = session.get(DownloadWorkerRow, worker_id)
if not row:
return None
return self._row_worker(row)
def list_workers(self) -> List[Worker]:
with session_scope() as session:
rows = session.scalars(
select(DownloadWorkerRow).order_by(DownloadWorkerRow.id)
).all()
return [self._row_worker(r) for r in rows]
def is_worker_online(self, worker_id: str) -> bool:
worker = self.get_worker(worker_id)
if not worker:
return False
return worker.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value)
def update_excel_key(self, job_id: str, excel_key: str) -> Optional[Job]:
with session_scope() as session:
row = session.get(DownloadJobRow, job_id)
if not row:
return None
row.excel_key = excel_key
session.flush()
return self._row_job(row)
def create_job(
self,
worker_id: str,
excel_key: str,
original_filename: str = "",
title: str = "",
*,
job_id: Optional[str] = None,
task_id: Optional[str] = None,
) -> Job:
import uuid as uuid_mod
download_id = str(uuid_mod.uuid4())
prefix = downloads_prefix(download_id)
with session_scope() as session:
row = DownloadJobRow(
id=download_id,
job_id=job_id,
task_id=task_id,
worker_id=worker_id,
status=JobStatus.PENDING.value,
title=title or f"Descarga {original_filename}",
excel_key=excel_key,
downloads_prefix=prefix,
original_filename=original_filename,
cancel_requested=False,
result={},
created_at=utcnow(),
)
session.add(row)
session.flush()
return self._row_job(row)
def get_job(self, job_id: str) -> Optional[Job]:
with session_scope() as session:
row = session.get(DownloadJobRow, job_id)
if not row:
return None
return self._row_job(row)
def list_jobs(
self,
limit: int = 100,
*,
job_id: Optional[str] = None,
) -> List[Job]:
with session_scope() as session:
stmt = select(DownloadJobRow).order_by(DownloadJobRow.created_at.desc()).limit(limit)
if job_id:
stmt = (
select(DownloadJobRow)
.where(DownloadJobRow.job_id == job_id)
.order_by(DownloadJobRow.created_at.desc())
.limit(limit)
)
rows = session.scalars(stmt).all()
return [self._row_job(r) for r in rows]
def claim_pending(self, worker_id: str) -> Optional[Job]:
with session_scope() as session:
worker = session.get(DownloadWorkerRow, worker_id)
if not worker or worker.current_job_id:
return None
row = session.scalars(
select(DownloadJobRow)
.where(
DownloadJobRow.worker_id == worker_id,
DownloadJobRow.status == JobStatus.PENDING.value,
)
.order_by(DownloadJobRow.created_at.asc())
.limit(1)
).first()
if not row:
return None
now = utcnow()
row.status = JobStatus.RUNNING.value
row.started_at = now
worker.current_job_id = row.id
worker.status = WorkerStatus.BUSY.value
worker.last_heartbeat = now
if row.task_id:
task = session.get(Task, row.task_id)
if task:
task.status = JobStatus.RUNNING.value
task.started_at = now
task.updated_at = now
session.flush()
return self._row_job(row)
def request_cancel(self, job_id: str) -> Optional[Job]:
with session_scope() as session:
row = session.get(DownloadJobRow, job_id)
if not row:
return None
if row.status in (
JobStatus.COMPLETED.value,
JobStatus.FAILED.value,
JobStatus.CANCELLED.value,
):
return self._row_job(row)
now = utcnow()
row.cancel_requested = True
if row.status == JobStatus.PENDING.value:
row.status = JobStatus.CANCELLED.value
row.finished_at = now
self._sync_task(session, row, status=JobStatus.CANCELLED.value)
session.flush()
return self._row_job(row)
def is_cancel_requested(self, job_id: str) -> bool:
job = self.get_job(job_id)
return bool(job and job.cancel_requested)
def _sync_task(
self,
session,
row: DownloadJobRow,
*,
status: Optional[str] = None,
result: Optional[Dict[str, Any]] = None,
error: Optional[str] = None,
) -> None:
if not row.task_id:
return
task = session.get(Task, row.task_id)
if not task:
return
now = utcnow()
if status:
task.status = status
if result is not None:
task.result = {
**dict(task.result or {}),
"download_job_id": row.id,
**result,
}
if error is not None:
task.error = error
task.updated_at = now
if status in (
JobStatus.COMPLETED.value,
JobStatus.FAILED.value,
JobStatus.CANCELLED.value,
):
task.finished_at = now
def patch_job(
self,
job_id: str,
*,
status: Optional[str] = None,
result: Optional[Dict[str, Any]] = None,
error: Optional[str] = None,
) -> Optional[Job]:
with session_scope() as session:
row = session.get(DownloadJobRow, job_id)
if not row:
return None
new_status = status or row.status
if result is not None:
row.result = result
if error is not None:
row.error = error
row.status = new_status
finished = False
if new_status in (
JobStatus.COMPLETED.value,
JobStatus.FAILED.value,
JobStatus.CANCELLED.value,
):
row.finished_at = row.finished_at or utcnow()
finished = True
self._sync_task(
session,
row,
status=new_status if status else None,
result=result,
error=error,
)
if finished:
worker = session.get(DownloadWorkerRow, row.worker_id)
if worker and worker.current_job_id == job_id:
worker.current_job_id = None
worker.status = WorkerStatus.ONLINE.value
session.flush()
return self._row_job(row)
def add_event(
self,
job_id: str,
event_type: str,
message: str,
*,
radicado: Optional[str] = None,
nit: Optional[str] = None,
index: Optional[int] = None,
total: Optional[int] = None,
metadata: Optional[Dict[str, Any]] = None,
timestamp: Optional[str] = None,
) -> JobEventRecord:
with session_scope() as session:
ts = _parse_iso(timestamp) or utcnow()
ev = DownloadJobEventRow(
download_job_id=job_id,
event_type=event_type,
message=message,
timestamp=ts,
radicado=radicado,
nit=nit,
idx=index,
total=total,
metadata_json=metadata or {},
)
session.add(ev)
session.flush()
return JobEventRecord(
id=int(ev.id),
job_id=job_id,
event_type=event_type,
message=message,
timestamp=_iso(ts) or utc_now(),
radicado=radicado,
nit=nit,
index=index,
total=total,
metadata=metadata or {},
)
def list_events(self, job_id: str, since_id: int = 0) -> List[JobEventRecord]:
with session_scope() as session:
rows = session.scalars(
select(DownloadJobEventRow)
.where(
DownloadJobEventRow.download_job_id == job_id,
DownloadJobEventRow.id > since_id,
)
.order_by(DownloadJobEventRow.id.asc())
).all()
return [
JobEventRecord(
id=int(r.id),
job_id=r.download_job_id,
event_type=r.event_type,
message=r.message,
timestamp=_iso(r.timestamp) or utc_now(),
radicado=r.radicado,
nit=r.nit,
index=r.idx,
total=r.total,
metadata=dict(r.metadata_json or {}),
)
for r in rows
]
def create_downloads_task(
self,
*,
parent_job_id: str,
download_job_id: str,
worker_id: str,
title: str,
original_filename: str,
) -> str:
"""Crea Task(type=downloads) y la asocia al download_job. Devuelve task_id."""
with session_scope() as session:
now = utcnow()
task = Task(
job_id=parent_job_id,
type="downloads",
status=JobStatus.PENDING.value,
title=title or f"Bot descarga ({original_filename})",
request={
"worker_id": worker_id,
"original_filename": original_filename,
"download_job_id": download_job_id,
},
result={"download_job_id": download_job_id},
created_at=now,
updated_at=now,
)
session.add(task)
session.flush()
row = session.get(DownloadJobRow, download_job_id)
if row:
row.task_id = task.id
row.job_id = parent_job_id
session.flush()
return task.id
def parent_job_exists(self, parent_job_id: str) -> bool:
from infrastructure.database import Job as IpsJob
with session_scope() as session:
return session.get(IpsJob, parent_job_id) is not None
+9
View File
@@ -0,0 +1,9 @@
fastapi>=0.110.0
uvicorn[standard]>=0.27.0
python-multipart>=0.0.9
boto3>=1.34.0
httpx>=0.27.0
pydantic>=2.0.0
sqlalchemy>=2.0.0
psycopg[binary]>=3.1.0
pytest>=7.0.0
+151
View File
@@ -0,0 +1,151 @@
from __future__ import annotations
import importlib
import sys
from pathlib import Path
import pytest
from infrastructure.storage_minio import downloads_prefix, excel_key, pdf_key
def test_minio_key_helpers():
assert excel_key("abc", "input.xlsx") == "uploads/abc/input.xlsx"
assert downloads_prefix("abc") == "downloads/abc/"
assert pdf_key("abc", "900", "123") == "downloads/abc/900/123.pdf"
@pytest.fixture()
def store(tmp_path, monkeypatch):
db_url = f"sqlite:///{(tmp_path / 'bot.db').as_posix()}"
monkeypatch.setenv("DATABASE_URL", db_url)
# Recargar módulos para que el engine use SQLITE
for name in [
"infrastructure.database",
"infrastructure.store",
]:
if name in sys.modules:
importlib.reload(sys.modules[name])
else:
importlib.import_module(name)
from domain.models import JobStatus, WorkerStatus
from infrastructure import database as db
from infrastructure.store import JobStore
importlib.reload(db)
import infrastructure.store as store_mod
importlib.reload(store_mod)
return store_mod.JobStore(), JobStatus, WorkerStatus
def test_register_heartbeat_and_offline(store):
job_store, _JobStatus, WorkerStatus = store
w = job_store.register_worker("PC-01", hostname="host1")
assert w.status == WorkerStatus.ONLINE.value
assert job_store.is_worker_online("PC-01")
job_store.heartbeat("PC-01")
workers = job_store.list_workers()
assert len(workers) == 1
assert workers[0].id == "PC-01"
def test_create_job_requires_worker_and_claim_is_atomic(store):
job_store, JobStatus, _WorkerStatus = store
job_store.register_worker("PC-01")
job = job_store.create_job(
worker_id="PC-01",
excel_key="uploads/x/input.xlsx",
original_filename="datos.xlsx",
title="Lote 1",
)
assert job.status == JobStatus.PENDING.value
assert job.worker_id == "PC-01"
assert job.downloads_prefix == downloads_prefix(job.id)
claimed = job_store.claim_pending("PC-01")
assert claimed is not None
assert claimed.id == job.id
assert claimed.status == JobStatus.RUNNING.value
again = job_store.claim_pending("PC-01")
assert again is None
job_store.register_worker("PC-02")
other = job_store.claim_pending("PC-02")
assert other is None
def test_cancel_pending_marks_cancelled(store):
job_store, JobStatus, _WorkerStatus = store
job_store.register_worker("PC-01")
job = job_store.create_job("PC-01", "uploads/j/input.xlsx", "a.xlsx")
cancelled = job_store.request_cancel(job.id)
assert cancelled is not None
assert cancelled.status == JobStatus.CANCELLED.value
assert cancelled.cancel_requested is True
def test_cancel_running_sets_flag(store):
job_store, JobStatus, _WorkerStatus = store
job_store.register_worker("PC-01")
job = job_store.create_job("PC-01", "uploads/j/input.xlsx", "a.xlsx")
job_store.claim_pending("PC-01")
flagged = job_store.request_cancel(job.id)
assert flagged is not None
assert flagged.cancel_requested is True
assert flagged.status == JobStatus.RUNNING.value
assert job_store.is_cancel_requested(job.id) is True
def test_events_and_patch_complete(store):
job_store, JobStatus, _WorkerStatus = store
job_store.register_worker("PC-01")
job = job_store.create_job("PC-01", "uploads/j/input.xlsx", "a.xlsx")
job_store.claim_pending("PC-01")
e1 = job_store.add_event(
job.id, "radicado_saved", "ok", radicado="1", nit="9", index=1, total=2
)
assert e1.id >= 1
events = job_store.list_events(job.id, since_id=0)
assert len(events) == 1
done = job_store.patch_job(
job.id,
status=JobStatus.COMPLETED.value,
result={"exitosos": 1},
)
assert done is not None
assert done.status == JobStatus.COMPLETED.value
worker = job_store.get_worker("PC-01")
assert worker is not None
assert worker.current_job_id is None
def test_list_jobs_by_parent(store):
job_store, _JobStatus, _WorkerStatus = store
from infrastructure.database import Job as IpsJob
from infrastructure.database import session_scope, utcnow
job_store.register_worker("PC-01")
with session_scope() as session:
session.add(
IpsJob(
id="parent-1",
status="pending",
title="Trabajo",
job_metadata={},
created_at=utcnow(),
updated_at=utcnow(),
)
)
a = job_store.create_job("PC-01", "uploads/a/input.xlsx", "a.xlsx", job_id="parent-1")
job_store.create_job("PC-01", "uploads/b/input.xlsx", "b.xlsx", job_id="other")
listed = job_store.list_jobs(limit=50, job_id="parent-1")
assert len(listed) == 1
assert listed[0].id == a.id
@@ -0,0 +1,120 @@
"""Bot descarga: download_workers, download_jobs, download_job_events.
Revision ID: 0004
Revises: 0003
Create Date: 2026-07-12
"""
from __future__ import annotations
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision: str = "0004"
down_revision: Union[str, None] = "0003"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _table_exists(conn: sa.engine.Connection, table: str) -> bool:
return sa.inspect(conn).has_table(table)
def upgrade() -> None:
conn = op.get_bind()
json_type = (
postgresql.JSONB()
if conn.dialect.name == "postgresql"
else sa.JSON()
)
if not _table_exists(conn, "download_workers"):
op.create_table(
"download_workers",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("hostname", sa.Text(), nullable=False, server_default=""),
sa.Column("max_concurrent", sa.Integer(), nullable=False, server_default="1"),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("last_heartbeat", sa.DateTime(timezone=True), nullable=False),
sa.Column("current_job_id", sa.String(36), nullable=True),
)
op.create_index("ix_download_workers_status", "download_workers", ["status"])
if not _table_exists(conn, "download_jobs"):
op.create_table(
"download_jobs",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True),
sa.Column("task_id", sa.String(36), sa.ForeignKey("tasks.id"), nullable=True),
sa.Column("worker_id", sa.String(36), nullable=False),
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
sa.Column("title", sa.Text(), nullable=True),
sa.Column("excel_key", sa.Text(), nullable=False, server_default=""),
sa.Column("downloads_prefix", sa.Text(), nullable=False, server_default=""),
sa.Column("original_filename", sa.Text(), nullable=False, server_default=""),
sa.Column("cancel_requested", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("result", json_type, nullable=False, server_default=sa.text("'{}'::jsonb") if conn.dialect.name == "postgresql" else sa.text("'{}'")),
sa.Column("error", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
)
op.create_index("ix_download_jobs_job_id", "download_jobs", ["job_id"])
op.create_index("ix_download_jobs_task_id", "download_jobs", ["task_id"])
op.create_index("ix_download_jobs_worker_id", "download_jobs", ["worker_id"])
op.create_index("ix_download_jobs_status", "download_jobs", ["status"])
op.create_index(
"ix_download_jobs_worker_status",
"download_jobs",
["worker_id", "status"],
)
if not _table_exists(conn, "download_job_events"):
op.create_table(
"download_job_events",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column(
"download_job_id",
sa.String(36),
sa.ForeignKey("download_jobs.id"),
nullable=False,
),
sa.Column("event_type", sa.String(64), nullable=False),
sa.Column("message", sa.Text(), nullable=False),
sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False),
sa.Column("radicado", sa.String(255), nullable=True),
sa.Column("nit", sa.String(64), nullable=True),
sa.Column("idx", sa.Integer(), nullable=True),
sa.Column("total", sa.Integer(), nullable=True),
sa.Column(
"metadata",
json_type,
nullable=False,
server_default=sa.text("'{}'::jsonb") if conn.dialect.name == "postgresql" else sa.text("'{}'"),
),
)
op.create_index(
"ix_download_job_events_job_id",
"download_job_events",
["download_job_id", "id"],
)
def downgrade() -> None:
conn = op.get_bind()
if _table_exists(conn, "download_job_events"):
op.drop_index("ix_download_job_events_job_id", table_name="download_job_events")
op.drop_table("download_job_events")
if _table_exists(conn, "download_jobs"):
op.drop_index("ix_download_jobs_worker_status", table_name="download_jobs")
op.drop_index("ix_download_jobs_status", table_name="download_jobs")
op.drop_index("ix_download_jobs_worker_id", table_name="download_jobs")
op.drop_index("ix_download_jobs_task_id", table_name="download_jobs")
op.drop_index("ix_download_jobs_job_id", table_name="download_jobs")
op.drop_table("download_jobs")
if _table_exists(conn, "download_workers"):
op.drop_index("ix_download_workers_status", table_name="download_workers")
op.drop_table("download_workers")
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "ipsextractor-desktop",
"version": "1.0.3",
"version": "1.1.0",
"private": true,
"description": "Desktop extraction client for IPSExtractor microservices.",
"main": "electron/main.cjs",
+11 -2
View File
@@ -29,6 +29,7 @@ import { UpdatePrompt, CheckUpdatesButton, useAutoUpdateCheck } from "./componen
import "./modules/counting/CountingModule";
import "./modules/extraction/ExtractionModule";
import "./modules/cleaning/CleaningModule";
import "./modules/downloads/DownloadsModule";
type Status = "idle" | "ok" | "error" | "busy";
type View = "home" | "job" | "settings";
@@ -39,7 +40,7 @@ function App() {
const [settingsReturnView, setSettingsReturnView] = useState<"home" | "job">("home");
const [settingsSection, setSettingsSection] = useState<SettingsSection>("connection");
const [activeJob, setActiveJob] = useState<Job | null>(null);
const [jobSection, setJobSection] = useState<JobSection>("counting");
const [jobSection, setJobSection] = useState<JobSection>("downloads");
const [jobSectionContext, setJobSectionContext] = useState<Record<string, unknown>>({});
const [apiBaseUrl, setApiBaseUrl] = useState(loadApiBaseUrl);
const [connectionStatus, setConnectionStatus] = useState<Status>("idle");
@@ -109,7 +110,7 @@ function App() {
function openJob(job: Job) {
setActiveJob(job);
setJobSection("counting");
setJobSection("downloads");
setJobSectionContext({});
setView("job");
}
@@ -201,6 +202,14 @@ function App() {
</div>
<div className="side-section-divider" />
<nav className="side-nav" aria-label="Módulos del trabajo">
<button
type="button"
className={jobSection === "downloads" ? "active" : ""}
onClick={() => handleActivateSection("downloads")}
>
<Download size={17} />
<span>Bot de descarga</span>
</button>
<button
type="button"
className={jobSection === "counting" ? "active" : ""}
@@ -9,6 +9,10 @@ import type {
CountSyncResponse,
CreateCountJobPayload,
CreateJobPayload,
DownloadJob,
DownloadJobEvent,
DownloadJobFilesResponse,
DownloadWorker,
ExtractionBatch,
ExtractionFromCountPayload,
Job,
@@ -22,6 +26,16 @@ import { DEFAULT_API_BASE_URL } from "./apiDefaults";
const DEFAULT_TIMEOUT_MS = 30_000;
const UPLOAD_TIMEOUT_MS = 600_000;
const BOT_TOKEN_KEY = "botApiToken";
const DEFAULT_BOT_TOKEN = "dev-token-change-me";
export function loadBotApiToken(): string {
return (localStorage.getItem(BOT_TOKEN_KEY) || DEFAULT_BOT_TOKEN).trim();
}
export function saveBotApiToken(token: string): void {
localStorage.setItem(BOT_TOKEN_KEY, token.trim() || DEFAULT_BOT_TOKEN);
}
async function fetchWithTimeout(
url: string,
@@ -48,19 +62,27 @@ async function fetchWithTimeout(
}
export class ApiClient {
constructor(private readonly baseUrl: string) {}
constructor(
private readonly baseUrl: string,
private readonly botToken: string = loadBotApiToken(),
) {}
private url(path: string): string {
return `${this.baseUrl.replace(/\/+$/, "")}${path}`;
}
private botHeaders(): HeadersInit {
return { Authorization: `Bearer ${this.botToken}` };
}
private isUploadPath(path: string): boolean {
return (
path.startsWith("/count/") ||
path.includes("/ingest") ||
path.includes("/upload") ||
path.startsWith("/cleaning/analyze-csv") ||
path.startsWith("/cleaning/jobs/from-csv")
path.startsWith("/cleaning/jobs/from-csv") ||
path === "/bot-descarga/jobs"
);
}
@@ -194,6 +216,68 @@ export class ApiClient {
await this.handle(response);
}
// ── Bot Descarga (control remoto) ──────────────────────────────────────────
async listDownloadWorkers(): Promise<DownloadWorker[]> {
const response = await fetchWithTimeout(
this.url("/bot-descarga/workers"),
{ headers: this.botHeaders() },
this.timeoutFor("/bot-descarga/workers"),
);
return this.handle<DownloadWorker[]>(response);
}
async createDownloadJob(form: FormData): Promise<DownloadJob> {
const response = await fetchWithTimeout(
this.url("/bot-descarga/jobs"),
{ method: "POST", headers: this.botHeaders(), body: form },
this.timeoutFor("/bot-descarga/jobs"),
);
return this.handle<DownloadJob>(response);
}
async listDownloadJobs(limit = 50, parentJobId?: string): Promise<DownloadJob[]> {
const qs = new URLSearchParams({ limit: String(limit) });
if (parentJobId) qs.set("job_id", parentJobId);
const response = await fetchWithTimeout(
this.url(`/bot-descarga/jobs?${qs.toString()}`),
{ headers: this.botHeaders() },
);
return this.handle<DownloadJob[]>(response);
}
async getDownloadJob(jobId: string): Promise<DownloadJob> {
const response = await fetchWithTimeout(
this.url(`/bot-descarga/jobs/${encodeURIComponent(jobId)}`),
{ headers: this.botHeaders() },
);
return this.handle<DownloadJob>(response);
}
async listDownloadJobFiles(jobId: string): Promise<DownloadJobFilesResponse> {
const response = await fetchWithTimeout(
this.url(`/bot-descarga/jobs/${encodeURIComponent(jobId)}/files`),
{ headers: this.botHeaders() },
);
return this.handle<DownloadJobFilesResponse>(response);
}
async getDownloadEvents(jobId: string, since = 0): Promise<{ events: DownloadJobEvent[] }> {
const response = await fetchWithTimeout(
this.url(`/bot-descarga/jobs/${encodeURIComponent(jobId)}/events?since=${since}`),
{ headers: this.botHeaders() },
);
return this.handle<{ events: DownloadJobEvent[] }>(response);
}
async cancelDownloadJob(jobId: string): Promise<DownloadJob> {
const response = await fetchWithTimeout(
this.url(`/bot-descarga/jobs/${encodeURIComponent(jobId)}/cancel`),
{ method: "POST", headers: this.botHeaders() },
);
return this.handle<DownloadJob>(response);
}
// ── Count Jobs ─────────────────────────────────────────────────────────────
/** Envía PDFs al servicio de conteo (síncrono). */
@@ -1 +1,2 @@
export type JobSection = "counting" | "extraction" | "cleaning" | "results";
export type JobSection = "counting" | "extraction" | "cleaning" | "downloads" | "results";
@@ -17,8 +17,9 @@ import { ExtractionResultsTable } from "../components/ExtractionResultsTable";
import { BackgroundProgressBanner } from "../components/BackgroundProgressBanner";
import type { ApiClient } from "../lib/apiClient";
import { downloadCsv, toCsv, buildExportCsvFilename } from "../lib/csv";
import { DownloadsResultsPanel } from "../modules/downloads/DownloadsResultsPanel";
type ResultTab = "counting" | "extraction" | "cleaning";
type ResultTab = "counting" | "extraction" | "cleaning" | "downloads";
interface Props {
client: ApiClient;
@@ -31,6 +32,7 @@ interface Props {
onStartCounting: () => void;
onStartExtraction: (ctx?: Record<string, unknown>) => void;
onStartCleaning: (ctx?: Record<string, unknown>) => void;
onStartDownload: () => void;
}
export function JobResultsView({
@@ -44,6 +46,7 @@ export function JobResultsView({
onStartCounting,
onStartExtraction,
onStartCleaning,
onStartDownload,
}: Props) {
const [activeTab, setActiveTab] = useState<ResultTab>("counting");
const [refreshing, setRefreshing] = useState(false);
@@ -54,6 +57,8 @@ export function JobResultsView({
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
const cleaningTasks = tasks.filter((t) => t.type === "cleaning")
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
const downloadTasks = tasks.filter((t) => t.type === "downloads")
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
async function refresh() {
setRefreshing(true);
@@ -63,7 +68,7 @@ export function JobResultsView({
return (
<div className="job-results-view">
{/* ── Sub-tabs: Conteo | Extracción | Limpieza ── */}
{/* ── Sub-tabs: Conteo | Extracción | Limpieza | Bot de descarga ── */}
<div className="job-results-tabs">
<button
type="button"
@@ -110,6 +115,17 @@ export function JobResultsView({
</span>
)}
</button>
<button
type="button"
className={`job-results-tab${activeTab === "downloads" ? " job-results-tab--active" : ""}`}
onClick={() => setActiveTab("downloads")}
>
<Download size={15} />
Bot de descarga
{downloadTasks.length > 0 && (
<span className="job-results-tab-count">{downloadTasks.length}</span>
)}
</button>
<div style={{ flex: 1 }} />
<button
type="button"
@@ -155,6 +171,14 @@ export function JobResultsView({
onStartCleaning={() => onStartCleaning()}
/>
)}
{activeTab === "downloads" && (
<DownloadsResultsPanel
client={client}
job={job}
tasks={downloadTasks}
onStartDownload={onStartDownload}
/>
)}
</div>
</div>
);
@@ -7,6 +7,7 @@ import { BackgroundProgressBanner } from "../components/BackgroundProgressBanner
import { CountingWizardModule } from "../modules/counting/CountingWizardModule";
import { CleaningWizardModule } from "../modules/cleaning/CleaningWizardModule";
import { JobExtractionWizard } from "../modules/extraction/JobExtractionWizard";
import { DownloadsWizardModule } from "../modules/downloads/DownloadsWizardModule";
import { JobResultsView } from "./JobResultsView";
interface Props {
@@ -120,6 +121,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
{section === "counting" && "Conteo de servicios"}
{section === "extraction" && "Extracción de datos"}
{section === "cleaning" && "Limpieza de datos"}
{section === "downloads" && "Bot de descarga"}
{section === "results" && "Resultados del trabajo"}
</div>
</header>
@@ -179,6 +181,10 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
/>
</div>
<div className={`job-section-pane${section === "downloads" ? "" : " job-section-pane--hidden"}`}>
<DownloadsWizardModule client={client} job={job} />
</div>
<div className={`job-section-pane${section === "results" ? "" : " job-section-pane--hidden"}`}>
<JobResultsView
client={client}
@@ -191,6 +197,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
onStartCounting={() => onActivateSection("counting")}
onStartExtraction={(ctx?: Record<string, unknown>) => onActivateSection("extraction", ctx)}
onStartCleaning={(ctx?: Record<string, unknown>) => onActivateSection("cleaning", ctx)}
onStartDownload={() => onActivateSection("downloads")}
/>
</div>
</div>
+56
View File
@@ -303,6 +303,62 @@ export interface UpdateProgress {
percent?: number;
}
/** Worker del bot de descargas PDF (control remoto). */
export interface DownloadWorker {
id: string;
hostname: string;
max_concurrent: number;
status: "online" | "offline" | "busy" | string;
last_heartbeat: string;
current_job_id: string | null;
}
export interface DownloadJob {
id: string;
worker_id: string;
status: "pending" | "running" | "completed" | "failed" | "cancelled" | string;
title: string;
excel_key: string;
downloads_prefix: string;
original_filename: string;
cancel_requested: boolean;
result: Record<string, unknown>;
error: string | null;
created_at: string;
started_at: string | null;
finished_at: string | null;
/** Job Desktop contenedor (IPSExtractor jobs.id) */
job_id?: string | null;
/** Task type=downloads */
task_id?: string | null;
}
export interface DownloadJobFile {
key: string;
nit?: string | null;
radicado?: string | null;
}
export interface DownloadJobFilesResponse {
job_id: string;
prefix: string;
files: DownloadJobFile[];
count: number;
}
export interface DownloadJobEvent {
id: number;
job_id: string;
event_type: string;
message: string;
timestamp: string;
radicado?: string | null;
nit?: string | null;
index?: number | null;
total?: number | null;
metadata?: Record<string, unknown>;
}
export interface DesktopBridge {
getRuntimeConfig(): Promise<RuntimeConfig>;
selectDirectory(): Promise<string | null>;