feat: Introduce bot-digitador service for automated data entry in Integración Total. Add CLI and GUI interfaces for CSV processing, implement auto-start functionality on Windows, and enhance README with setup instructions and usage details. Include necessary configurations in docker-compose for service integration.
This commit is contained in:
erik
@@ -0,0 +1,157 @@
|
||||
"""Parseo y partición de CSV de filas a digitar (bot digitador)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
from typing import Any
|
||||
|
||||
REQUIRED_BOT_COLUMNS = (
|
||||
"radicado",
|
||||
"documento",
|
||||
"ingreso",
|
||||
"total_cargos",
|
||||
"autorizacion",
|
||||
"tipo_documento",
|
||||
"tipo_servicio",
|
||||
"copago",
|
||||
"cuota_moderadora",
|
||||
)
|
||||
|
||||
IT_TO_BOT = {
|
||||
"RADICADO": "radicado",
|
||||
"NUMERO": "documento",
|
||||
"FECHA": "ingreso",
|
||||
"BRUTO": "total_cargos",
|
||||
"AUTORIZACION": "autorizacion",
|
||||
"TIPO_ID": "tipo_documento",
|
||||
"SERVICIO": "tipo_servicio",
|
||||
"COPAGO": "copago",
|
||||
"MODERADORA": "cuota_moderadora",
|
||||
}
|
||||
|
||||
|
||||
def _normalize_header(name: str) -> str:
|
||||
raw = str(name or "").strip()
|
||||
if not raw:
|
||||
return ""
|
||||
upper = raw.upper()
|
||||
if upper in IT_TO_BOT:
|
||||
return IT_TO_BOT[upper]
|
||||
lower = raw.lower().strip()
|
||||
# Aceptar snake_case ya normalizado
|
||||
if lower in REQUIRED_BOT_COLUMNS:
|
||||
return lower
|
||||
# Alias comunes
|
||||
aliases = {
|
||||
"número": "documento",
|
||||
"numero": "documento",
|
||||
"fecha": "ingreso",
|
||||
"bruto": "total_cargos",
|
||||
"tipo_id": "tipo_documento",
|
||||
"servicio": "tipo_servicio",
|
||||
"moderadora": "cuota_moderadora",
|
||||
}
|
||||
return aliases.get(lower, lower)
|
||||
|
||||
|
||||
def _decode_text(data: bytes) -> str:
|
||||
for enc in ("utf-8-sig", "utf-8", "latin-1", "cp1252"):
|
||||
try:
|
||||
return data.decode(enc)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
return data.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _detect_delimiter(sample: str) -> str:
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters="|,;\t")
|
||||
return dialect.delimiter
|
||||
except csv.Error:
|
||||
return "|" if sample.count("|") >= sample.count(",") else ","
|
||||
|
||||
|
||||
def _validate_columns(headers: list[str]) -> None:
|
||||
present = set(headers)
|
||||
missing = [c for c in REQUIRED_BOT_COLUMNS if c not in present]
|
||||
if missing:
|
||||
raise ValueError(f"Columnas faltantes: {', '.join(missing)}")
|
||||
|
||||
|
||||
def parse_digitador_rows(data: bytes, filename: str = "input.csv") -> list[dict[str, str]]:
|
||||
"""Devuelve filas normalizadas con headers del bot (snake_case)."""
|
||||
_ = filename # reservado por firma / diagnóstico futuro
|
||||
text = _decode_text(data)
|
||||
if not text.strip():
|
||||
return []
|
||||
|
||||
delim = _detect_delimiter(text[:4096])
|
||||
reader = csv.DictReader(io.StringIO(text), delimiter=delim)
|
||||
if not reader.fieldnames:
|
||||
raise ValueError("CSV sin encabezados")
|
||||
|
||||
rename_map: dict[str, str] = {}
|
||||
normalized_headers: list[str] = []
|
||||
for raw in reader.fieldnames:
|
||||
norm = _normalize_header(raw)
|
||||
rename_map[raw] = norm
|
||||
if norm and norm not in normalized_headers:
|
||||
normalized_headers.append(norm)
|
||||
|
||||
_validate_columns(normalized_headers)
|
||||
|
||||
out: list[dict[str, str]] = []
|
||||
for row in reader:
|
||||
if not row:
|
||||
continue
|
||||
mapped: dict[str, str] = {c: "" for c in REQUIRED_BOT_COLUMNS}
|
||||
empty = True
|
||||
for raw_key, value in row.items():
|
||||
if raw_key is None:
|
||||
continue
|
||||
col = rename_map.get(raw_key) or _normalize_header(raw_key)
|
||||
if col not in REQUIRED_BOT_COLUMNS:
|
||||
continue
|
||||
cell = "" if value is None else str(value).strip()
|
||||
mapped[col] = cell
|
||||
if cell:
|
||||
empty = False
|
||||
if empty:
|
||||
continue
|
||||
# Filtrar filas que parecen ser un segundo header
|
||||
if mapped.get("radicado", "").lower() in ("radicado", "radicado"):
|
||||
if mapped.get("documento", "").lower() in ("documento", "numero", "número", ""):
|
||||
continue
|
||||
out.append(mapped)
|
||||
return out
|
||||
|
||||
|
||||
def split_rows(rows: list[dict[str, Any]], n: int) -> list[list[dict[str, Any]]]:
|
||||
"""Parte filas en n trozos lo más equilibrados posible (chunks contiguos)."""
|
||||
if n <= 1 or not rows:
|
||||
return [rows]
|
||||
n = min(n, len(rows))
|
||||
size, rem = divmod(len(rows), n)
|
||||
chunks: list[list[dict[str, Any]]] = []
|
||||
idx = 0
|
||||
for i in range(n):
|
||||
take = size + (1 if i < rem else 0)
|
||||
chunks.append(rows[idx : idx + take])
|
||||
idx += take
|
||||
return [c for c in chunks if c]
|
||||
|
||||
|
||||
def rows_to_csv_bytes(rows: list[dict[str, Any]]) -> bytes:
|
||||
"""Serializa filas a CSV pipe-delimited con headers del bot (utf-8-sig)."""
|
||||
buf = io.StringIO()
|
||||
writer = csv.DictWriter(
|
||||
buf,
|
||||
fieldnames=list(REQUIRED_BOT_COLUMNS),
|
||||
delimiter="|",
|
||||
lineterminator="\n",
|
||||
extrasaction="ignore",
|
||||
)
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
writer.writerow({c: str(row.get(c, "") or "") for c in REQUIRED_BOT_COLUMNS})
|
||||
return buf.getvalue().encode("utf-8-sig")
|
||||
@@ -0,0 +1,91 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
class WorkerStatus(str, Enum):
|
||||
ONLINE = "online"
|
||||
OFFLINE = "offline"
|
||||
BUSY = "busy"
|
||||
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Worker:
|
||||
id: str
|
||||
hostname: str = ""
|
||||
max_concurrent: int = 1
|
||||
status: str = WorkerStatus.ONLINE.value
|
||||
last_heartbeat: str = field(default_factory=utc_now)
|
||||
current_job_id: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Job:
|
||||
id: str
|
||||
worker_id: Optional[str]
|
||||
status: str = JobStatus.PENDING.value
|
||||
title: str = ""
|
||||
csv_key: str = ""
|
||||
reports_prefix: str = ""
|
||||
original_filename: str = ""
|
||||
cancel_requested: bool = False
|
||||
result: Dict[str, Any] = field(default_factory=dict)
|
||||
error: Optional[str] = None
|
||||
created_at: str = field(default_factory=utc_now)
|
||||
started_at: Optional[str] = None
|
||||
finished_at: Optional[str] = None
|
||||
job_id: Optional[str] = None
|
||||
task_id: Optional[str] = None
|
||||
parent_digitador_job_id: Optional[str] = None
|
||||
shard_index: Optional[int] = None
|
||||
shard_count: Optional[int] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
@property
|
||||
def is_distributed_parent(self) -> bool:
|
||||
return (
|
||||
self.parent_digitador_job_id is None
|
||||
and self.shard_count is not None
|
||||
and int(self.shard_count) > 1
|
||||
)
|
||||
|
||||
@property
|
||||
def is_shard(self) -> bool:
|
||||
return bool(self.parent_digitador_job_id)
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobEventRecord:
|
||||
id: int
|
||||
job_id: str
|
||||
event_type: str
|
||||
message: str
|
||||
timestamp: str
|
||||
radicado: Optional[str] = None
|
||||
nit: Optional[str] = None
|
||||
index: Optional[int] = None
|
||||
total: Optional[int] = None
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
Reference in New Issue
Block a user