PYS/IPSExtractor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Introduce bot-digitador service for automated data entry in Integración Total. Add CLI and GUI interfaces for CSV processing, implement auto-start functionality on Windows, and enhance README with setup instructions and usage details. Include necessary configurations in docker-compose for service integration.

Browse Source
This commit is contained in:
erik
2026-08-09 21:26:56 -05:00
parent a3b4c7fc47
commit e9b766281a
74 changed files with 11177 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/bot_descarga/infrastructure/database.py
+6 -1
View File
@@ -103,7 +103,12 @@ class DownloadJobRow(Base):
task_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("tasks.id"), nullable=True, index=True
)
worker_id: Mapped[str] = mapped_column(String(36), index=True)
worker_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
parent_download_job_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("download_jobs.id"), nullable=True, index=True
)
shard_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
shard_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
title: Mapped[str | None] = mapped_column(Text, nullable=True)
excel_key: Mapped[str] = mapped_column(Text, default="")
services/bot_descarga/infrastructure/storage_minio.py
+9 -5
View File
@@ -194,21 +194,25 @@ class MinioStorage:
Si el manifest falta, falla o queda vacío, hace fallback a list_keys
bajo el prefijo (cubre varias carpetas NIT planas).
En corridas distribuidas varios shards sobrescriben el manifest parcial:
siempre se fusiona con list_keys del prefijo.
"""
prefix = prefix or downloads_prefix(job_id)
files: List[dict] = []
by_key: dict[str, dict] = {}
try:
raw = self.get_bytes(manifest_key(job_id))
payload = json.loads(raw.decode("utf-8"))
for key in payload.get("files") or []:
entry = self._file_entry(str(key))
if entry is not None:
files.append(entry)
by_key[entry["key"]] = entry
except Exception:
files = []
pass
if not files:
files = self._list_pdf_entries(prefix)
for entry in self._list_pdf_entries(prefix):
by_key[entry["key"]] = entry
files = list(by_key.values())
return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)}
def _list_pdf_entries(self, prefix: str) -> List[dict]:
services/bot_descarga/infrastructure/store.py
+403 -34
View File
@@ -73,6 +73,9 @@ class JobStore:
finished_at=_iso(row.finished_at),
job_id=row.job_id,
task_id=row.task_id,
parent_download_job_id=row.parent_download_job_id,
shard_index=row.shard_index,
shard_count=row.shard_count,
)
def _refresh_worker_status(self, worker: Worker) -> Worker:
@@ -129,7 +132,13 @@ class JobStore:
WorkerStatus.BUSY.value if row.current_job_id else WorkerStatus.ONLINE.value
)
session.flush()
return self._row_worker(row)
worker = self._row_worker(row)
# Reclaim fuera de la sesión del heartbeat para no bloquear
try:
self.reclaim_stale_running_jobs()
except Exception:
pass
return worker
def get_worker(self, worker_id: str) -> Optional[Worker]:
with session_scope() as session:
@@ -139,12 +148,23 @@ class JobStore:
return self._row_worker(row)
def list_workers(self) -> List[Worker]:
try:
self.reclaim_stale_running_jobs()
except Exception:
pass
with session_scope() as session:
rows = session.scalars(
select(DownloadWorkerRow).order_by(DownloadWorkerRow.id)
).all()
return [self._row_worker(r) for r in rows]
def list_online_worker_ids(self) -> List[str]:
return [
w.id
for w in self.list_workers()
if w.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value)
]
def is_worker_online(self, worker_id: str) -> bool:
worker = self.get_worker(worker_id)
if not worker:
@@ -162,31 +182,41 @@ class JobStore:
def create_job(
self,
worker_id: str,
worker_id: Optional[str],
excel_key: str,
original_filename: str = "",
title: str = "",
*,
job_id: Optional[str] = None,
task_id: Optional[str] = None,
downloads_prefix_override: Optional[str] = None,
parent_download_job_id: Optional[str] = None,
shard_index: Optional[int] = None,
shard_count: Optional[int] = None,
result: Optional[Dict[str, Any]] = None,
status: Optional[str] = None,
download_id: Optional[str] = None,
) -> Job:
import uuid as uuid_mod
download_id = str(uuid_mod.uuid4())
prefix = downloads_prefix(download_id)
download_id = download_id or str(uuid_mod.uuid4())
prefix = downloads_prefix_override or downloads_prefix(download_id)
with session_scope() as session:
row = DownloadJobRow(
id=download_id,
job_id=job_id,
task_id=task_id,
worker_id=worker_id,
status=JobStatus.PENDING.value,
parent_download_job_id=parent_download_job_id,
shard_index=shard_index,
shard_count=shard_count,
status=status or JobStatus.PENDING.value,
title=title or f"Descarga {original_filename}",
excel_key=excel_key,
downloads_prefix=prefix,
original_filename=original_filename,
cancel_requested=False,
result={},
result=result or {},
created_at=utcnow(),
)
session.add(row)
@@ -205,34 +235,57 @@ class JobStore:
limit: int = 100,
*,
job_id: Optional[str] = None,
parents_only: bool = True,
) -> List[Job]:
with session_scope() as session:
stmt = select(DownloadJobRow).order_by(DownloadJobRow.created_at.desc()).limit(limit)
stmt = select(DownloadJobRow).order_by(DownloadJobRow.created_at.desc())
if job_id:
stmt = (
select(DownloadJobRow)
.where(DownloadJobRow.job_id == job_id)
.order_by(DownloadJobRow.created_at.desc())
.limit(limit)
)
stmt = stmt.where(DownloadJobRow.job_id == job_id)
if parents_only:
stmt = stmt.where(DownloadJobRow.parent_download_job_id.is_(None))
stmt = stmt.limit(limit)
rows = session.scalars(stmt).all()
return [self._row_job(r) for r in rows]
def list_shards(self, parent_id: str) -> List[Job]:
with session_scope() as session:
rows = session.scalars(
select(DownloadJobRow)
.where(DownloadJobRow.parent_download_job_id == parent_id)
.order_by(DownloadJobRow.shard_index.asc())
).all()
return [self._row_job(r) for r in rows]
def claim_pending(self, worker_id: str) -> Optional[Job]:
try:
self.reclaim_stale_running_jobs()
except Exception:
pass
with session_scope() as session:
worker = session.get(DownloadWorkerRow, worker_id)
if not worker or worker.current_job_id:
return None
row = session.scalars(
# No claimar padres orquestadores (shard_count > 1 sin parent)
rows = session.scalars(
select(DownloadJobRow)
.where(
DownloadJobRow.worker_id == worker_id,
DownloadJobRow.status == JobStatus.PENDING.value,
)
.order_by(DownloadJobRow.created_at.asc())
.limit(1)
).first()
).all()
row = None
for candidate in rows:
is_parent = (
candidate.parent_download_job_id is None
and candidate.shard_count is not None
and int(candidate.shard_count) > 1
)
if is_parent:
continue
row = candidate
break
if not row:
return None
@@ -243,9 +296,19 @@ class JobStore:
worker.status = WorkerStatus.BUSY.value
worker.last_heartbeat = now
if row.task_id:
task = session.get(Task, row.task_id)
if task:
# Task del padre (shards heredan task_id del padre)
task_id = row.task_id
if row.parent_download_job_id:
parent = session.get(DownloadJobRow, row.parent_download_job_id)
if parent:
task_id = parent.task_id
if parent.status == JobStatus.PENDING.value:
parent.status = JobStatus.RUNNING.value
parent.started_at = parent.started_at or now
if task_id:
task = session.get(Task, task_id)
if task and task.status == JobStatus.PENDING.value:
task.status = JobStatus.RUNNING.value
task.started_at = now
task.updated_at = now
@@ -271,12 +334,44 @@ class JobStore:
row.status = JobStatus.CANCELLED.value
row.finished_at = now
self._sync_task(session, row, status=JobStatus.CANCELLED.value)
# Cancelar shards hijos si es padre distribuido
if row.parent_download_job_id is None and row.shard_count and int(row.shard_count) > 1:
children = session.scalars(
select(DownloadJobRow).where(
DownloadJobRow.parent_download_job_id == job_id
)
).all()
for child in children:
if child.status in (
JobStatus.COMPLETED.value,
JobStatus.FAILED.value,
JobStatus.CANCELLED.value,
):
continue
child.cancel_requested = True
if child.status == JobStatus.PENDING.value:
child.status = JobStatus.CANCELLED.value
child.finished_at = now
worker = session.get(DownloadWorkerRow, child.worker_id) if child.worker_id else None
if worker and worker.current_job_id == child.id:
worker.current_job_id = None
worker.status = WorkerStatus.ONLINE.value
session.flush()
return self._row_job(row)
def is_cancel_requested(self, job_id: str) -> bool:
job = self.get_job(job_id)
return bool(job and job.cancel_requested)
if not job:
return False
if job.cancel_requested:
return True
if job.parent_download_job_id:
parent = self.get_job(job.parent_download_job_id)
if parent and parent.cancel_requested:
return True
return False
def _sync_task(
self,
@@ -287,9 +382,15 @@ class JobStore:
result: Optional[Dict[str, Any]] = None,
error: Optional[str] = None,
) -> None:
if not row.task_id:
# Sync siempre contra el padre si es shard
target = row
if row.parent_download_job_id:
parent = session.get(DownloadJobRow, row.parent_download_job_id)
if parent:
target = parent
if not target.task_id:
return
task = session.get(Task, row.task_id)
task = session.get(Task, target.task_id)
if not task:
return
now = utcnow()
@@ -298,7 +399,7 @@ class JobStore:
if result is not None:
task.result = {
**dict(task.result or {}),
"download_job_id": row.id,
"download_job_id": target.id,
**result,
}
if error is not None:
@@ -319,6 +420,7 @@ class JobStore:
result: Optional[Dict[str, Any]] = None,
error: Optional[str] = None,
) -> Optional[Job]:
parent_id: Optional[str] = None
with session_scope() as session:
row = session.get(DownloadJobRow, job_id)
if not row:
@@ -340,22 +442,263 @@ class JobStore:
row.finished_at = row.finished_at or utcnow()
finished = True
self._sync_task(
session,
row,
status=new_status if status else None,
result=result,
error=error,
)
# No sync task directo en shards; se consolida en el padre
if not row.parent_download_job_id:
self._sync_task(
session,
row,
status=new_status if status else None,
result=result,
error=error,
)
if finished:
worker = session.get(DownloadWorkerRow, row.worker_id)
worker = session.get(DownloadWorkerRow, row.worker_id) if row.worker_id else None
if worker and worker.current_job_id == job_id:
worker.current_job_id = None
worker.status = WorkerStatus.ONLINE.value
parent_id = row.parent_download_job_id
session.flush()
return self._row_job(row)
job = self._row_job(row)
if parent_id:
self.refresh_parent_from_shards(parent_id)
refreshed = self.get_job(job_id)
return refreshed or job
return job
def refresh_parent_from_shards(self, parent_id: str) -> Optional[Job]:
with session_scope() as session:
parent = session.get(DownloadJobRow, parent_id)
if not parent:
return None
children = list(
session.scalars(
select(DownloadJobRow).where(
DownloadJobRow.parent_download_job_id == parent_id
)
).all()
)
if not children:
return self._row_job(parent)
statuses = [c.status for c in children]
terminal = {
JobStatus.COMPLETED.value,
JobStatus.FAILED.value,
JobStatus.CANCELLED.value,
}
any_active = any(
s in (JobStatus.PENDING.value, JobStatus.RUNNING.value) for s in statuses
)
all_terminal = all(s in terminal for s in statuses)
total = 0
exitosos = 0
fallidos = 0
uploaded = 0
for c in children:
r = dict(c.result or {})
total += int(r.get("total") or 0)
exitosos += int(r.get("exitosos") or 0)
fallidos += int(r.get("fallidos") or 0)
uploaded += int(r.get("uploaded_count") or 0)
# Progreso parcial mientras corre
if c.status == JobStatus.RUNNING.value:
total = max(total, int(r.get("total") or 0))
exitosos = max(exitosos, int(r.get("processed") or r.get("exitosos") or 0))
# Agregar progreso en vuelo (processed) de shards running
processed = 0
for c in children:
r = dict(c.result or {})
if c.status in terminal:
processed += int(r.get("exitosos") or 0) + int(r.get("fallidos") or 0)
else:
processed += int(r.get("processed") or 0)
shard_ids = [c.id for c in children]
parent_result = dict(parent.result or {})
parent_result.update(
{
"mode": "distributed",
"shard_ids": shard_ids,
"total": total or int(parent_result.get("total_rows") or 0),
"total_rows": int(parent_result.get("total_rows") or total or 0),
"exitosos": exitosos,
"fallidos": fallidos,
"processed": processed,
"uploaded_count": uploaded,
"downloads_prefix": parent.downloads_prefix,
"shards": [
{
"id": c.id,
"worker_id": c.worker_id,
"status": c.status,
"shard_index": c.shard_index,
"result": dict(c.result or {}),
"error": c.error,
}
for c in children
],
}
)
parent.result = parent_result
now = utcnow()
if any_active:
parent.status = JobStatus.RUNNING.value
parent.started_at = parent.started_at or now
parent.finished_at = None
parent.error = None
self._sync_task(
session,
parent,
status=JobStatus.RUNNING.value,
result=parent_result,
)
elif all_terminal:
any_failed = any(s == JobStatus.FAILED.value for s in statuses)
any_cancelled = any(s == JobStatus.CANCELLED.value for s in statuses)
all_cancelled = all(s == JobStatus.CANCELLED.value for s in statuses)
if all_cancelled:
parent.status = JobStatus.CANCELLED.value
elif any_failed:
parent.status = JobStatus.FAILED.value
parent.error = next(
(c.error for c in children if c.error),
"Uno o más shards fallaron",
)
elif any_cancelled and not any_failed:
# mezclado completed+cancelled → completed parcial
parent.status = JobStatus.COMPLETED.value
else:
parent.status = JobStatus.COMPLETED.value
parent.finished_at = parent.finished_at or now
self._sync_task(
session,
parent,
status=parent.status,
result=parent_result,
error=parent.error,
)
session.flush()
return self._row_job(parent)
def reclaim_stale_running_jobs(self) -> int:
"""Jobs running cuyo worker está offline → pending + reasignar a otro online."""
reclaimed = 0
with session_scope() as session:
now = utcnow()
cutoff = now - timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS)
workers = list(session.scalars(select(DownloadWorkerRow)).all())
offline_ids: set[str] = set()
online_free: List[DownloadWorkerRow] = []
for w in workers:
hb = w.last_heartbeat
if hb is None:
offline_ids.add(w.id)
continue
if hb.tzinfo is None:
hb = hb.replace(tzinfo=timezone.utc)
if hb < cutoff:
offline_ids.add(w.id)
if w.current_job_id:
w.current_job_id = None
w.status = WorkerStatus.OFFLINE.value
elif not w.current_job_id:
online_free.append(w)
if not offline_ids:
session.flush()
return 0
running = session.scalars(
select(DownloadJobRow).where(
DownloadJobRow.status == JobStatus.RUNNING.value,
DownloadJobRow.worker_id.in_(list(offline_ids)),
)
).all()
rr = 0
for row in running:
# Solo shards o jobs simples (no padres orquestadores)
is_parent = (
row.parent_download_job_id is None
and row.shard_count is not None
and int(row.shard_count) > 1
)
if is_parent:
continue
row.status = JobStatus.PENDING.value
row.started_at = None
row.cancel_requested = False
if online_free:
target = online_free[rr % len(online_free)]
rr += 1
row.worker_id = target.id
# Si no hay online free, queda pending con el mismo worker_id
# (se reasignará cuando haya free en el próximo reclaim)
reclaimed += 1
session.add(
DownloadJobEventRow(
download_job_id=row.id,
event_type="warning",
message=(
f"Worker offline; job reclaimed"
+ (f" → {row.worker_id}" if row.worker_id else "")
),
timestamp=now,
metadata_json={"reclaimed": True, "worker_id": row.worker_id},
)
)
if row.parent_download_job_id:
session.add(
DownloadJobEventRow(
download_job_id=row.parent_download_job_id,
event_type="warning",
message=f"Shard {row.shard_index} reclamado (worker offline)",
timestamp=now,
metadata_json={
"shard_id": row.id,
"worker_id": row.worker_id,
},
)
)
# Reasignar pending de workers offline a online free
if online_free:
pending_orphan = session.scalars(
select(DownloadJobRow).where(
DownloadJobRow.status == JobStatus.PENDING.value,
DownloadJobRow.worker_id.in_(list(offline_ids)),
)
).all()
for row in pending_orphan:
is_parent = (
row.parent_download_job_id is None
and row.shard_count is not None
and int(row.shard_count) > 1
)
if is_parent:
continue
target = online_free[rr % len(online_free)]
rr += 1
if row.worker_id != target.id:
row.worker_id = target.id
reclaimed += 1
session.flush()
# Refrescar padres afectados
# (se hace en siguiente patch; aquí no hace falta)
return reclaimed
def add_event(
self,
@@ -385,6 +728,28 @@ class JobStore:
)
session.add(ev)
session.flush()
# Mirror a padre si es shard
row = session.get(DownloadJobRow, job_id)
if row and row.parent_download_job_id:
session.add(
DownloadJobEventRow(
download_job_id=row.parent_download_job_id,
event_type=event_type,
message=message,
timestamp=ts,
radicado=radicado,
nit=nit,
idx=index,
total=total,
metadata_json={
**(metadata or {}),
"shard_id": job_id,
"shard_index": row.shard_index,
"worker_id": row.worker_id,
},
)
)
session.flush()
return JobEventRecord(
id=int(ev.id),
job_id=job_id,
@@ -429,9 +794,11 @@ class JobStore:
*,
parent_job_id: str,
download_job_id: str,
worker_id: str,
worker_id: Optional[str],
title: str,
original_filename: str,
worker_ids: Optional[List[str]] = None,
mode: str = "single",
) -> str:
"""Crea Task(type=downloads) y la asocia al download_job. Devuelve task_id."""
with session_scope() as session:
@@ -443,10 +810,12 @@ class JobStore:
title=title or f"Bot descarga ({original_filename})",
request={
"worker_id": worker_id,
"worker_ids": worker_ids or ([worker_id] if worker_id else []),
"original_filename": original_filename,
"download_job_id": download_job_id,
"mode": mode,
},
result={"download_job_id": download_job_id},
result={"download_job_id": download_job_id, "mode": mode},
created_at=now,
updated_at=now,
)