158 lines
4.6 KiB
Python
158 lines
4.6 KiB
Python
"""Parseo y partición de CSV de filas a digitar (bot digitador)."""
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import io
|
|
from typing import Any
|
|
|
|
REQUIRED_BOT_COLUMNS = (
|
|
"radicado",
|
|
"documento",
|
|
"ingreso",
|
|
"total_cargos",
|
|
"autorizacion",
|
|
"tipo_documento",
|
|
"tipo_servicio",
|
|
"copago",
|
|
"cuota_moderadora",
|
|
)
|
|
|
|
IT_TO_BOT = {
|
|
"RADICADO": "radicado",
|
|
"NUMERO": "documento",
|
|
"FECHA": "ingreso",
|
|
"BRUTO": "total_cargos",
|
|
"AUTORIZACION": "autorizacion",
|
|
"TIPO_ID": "tipo_documento",
|
|
"SERVICIO": "tipo_servicio",
|
|
"COPAGO": "copago",
|
|
"MODERADORA": "cuota_moderadora",
|
|
}
|
|
|
|
|
|
def _normalize_header(name: str) -> str:
|
|
raw = str(name or "").strip()
|
|
if not raw:
|
|
return ""
|
|
upper = raw.upper()
|
|
if upper in IT_TO_BOT:
|
|
return IT_TO_BOT[upper]
|
|
lower = raw.lower().strip()
|
|
# Aceptar snake_case ya normalizado
|
|
if lower in REQUIRED_BOT_COLUMNS:
|
|
return lower
|
|
# Alias comunes
|
|
aliases = {
|
|
"número": "documento",
|
|
"numero": "documento",
|
|
"fecha": "ingreso",
|
|
"bruto": "total_cargos",
|
|
"tipo_id": "tipo_documento",
|
|
"servicio": "tipo_servicio",
|
|
"moderadora": "cuota_moderadora",
|
|
}
|
|
return aliases.get(lower, lower)
|
|
|
|
|
|
def _decode_text(data: bytes) -> str:
|
|
for enc in ("utf-8-sig", "utf-8", "latin-1", "cp1252"):
|
|
try:
|
|
return data.decode(enc)
|
|
except UnicodeDecodeError:
|
|
continue
|
|
return data.decode("utf-8", errors="replace")
|
|
|
|
|
|
def _detect_delimiter(sample: str) -> str:
|
|
try:
|
|
dialect = csv.Sniffer().sniff(sample, delimiters="|,;\t")
|
|
return dialect.delimiter
|
|
except csv.Error:
|
|
return "|" if sample.count("|") >= sample.count(",") else ","
|
|
|
|
|
|
def _validate_columns(headers: list[str]) -> None:
|
|
present = set(headers)
|
|
missing = [c for c in REQUIRED_BOT_COLUMNS if c not in present]
|
|
if missing:
|
|
raise ValueError(f"Columnas faltantes: {', '.join(missing)}")
|
|
|
|
|
|
def parse_digitador_rows(data: bytes, filename: str = "input.csv") -> list[dict[str, str]]:
|
|
"""Devuelve filas normalizadas con headers del bot (snake_case)."""
|
|
_ = filename # reservado por firma / diagnóstico futuro
|
|
text = _decode_text(data)
|
|
if not text.strip():
|
|
return []
|
|
|
|
delim = _detect_delimiter(text[:4096])
|
|
reader = csv.DictReader(io.StringIO(text), delimiter=delim)
|
|
if not reader.fieldnames:
|
|
raise ValueError("CSV sin encabezados")
|
|
|
|
rename_map: dict[str, str] = {}
|
|
normalized_headers: list[str] = []
|
|
for raw in reader.fieldnames:
|
|
norm = _normalize_header(raw)
|
|
rename_map[raw] = norm
|
|
if norm and norm not in normalized_headers:
|
|
normalized_headers.append(norm)
|
|
|
|
_validate_columns(normalized_headers)
|
|
|
|
out: list[dict[str, str]] = []
|
|
for row in reader:
|
|
if not row:
|
|
continue
|
|
mapped: dict[str, str] = {c: "" for c in REQUIRED_BOT_COLUMNS}
|
|
empty = True
|
|
for raw_key, value in row.items():
|
|
if raw_key is None:
|
|
continue
|
|
col = rename_map.get(raw_key) or _normalize_header(raw_key)
|
|
if col not in REQUIRED_BOT_COLUMNS:
|
|
continue
|
|
cell = "" if value is None else str(value).strip()
|
|
mapped[col] = cell
|
|
if cell:
|
|
empty = False
|
|
if empty:
|
|
continue
|
|
# Filtrar filas que parecen ser un segundo header
|
|
if mapped.get("radicado", "").lower() in ("radicado", "radicado"):
|
|
if mapped.get("documento", "").lower() in ("documento", "numero", "número", ""):
|
|
continue
|
|
out.append(mapped)
|
|
return out
|
|
|
|
|
|
def split_rows(rows: list[dict[str, Any]], n: int) -> list[list[dict[str, Any]]]:
|
|
"""Parte filas en n trozos lo más equilibrados posible (chunks contiguos)."""
|
|
if n <= 1 or not rows:
|
|
return [rows]
|
|
n = min(n, len(rows))
|
|
size, rem = divmod(len(rows), n)
|
|
chunks: list[list[dict[str, Any]]] = []
|
|
idx = 0
|
|
for i in range(n):
|
|
take = size + (1 if i < rem else 0)
|
|
chunks.append(rows[idx : idx + take])
|
|
idx += take
|
|
return [c for c in chunks if c]
|
|
|
|
|
|
def rows_to_csv_bytes(rows: list[dict[str, Any]]) -> bytes:
|
|
"""Serializa filas a CSV pipe-delimited con headers del bot (utf-8-sig)."""
|
|
buf = io.StringIO()
|
|
writer = csv.DictWriter(
|
|
buf,
|
|
fieldnames=list(REQUIRED_BOT_COLUMNS),
|
|
delimiter="|",
|
|
lineterminator="\n",
|
|
extrasaction="ignore",
|
|
)
|
|
writer.writeheader()
|
|
for row in rows:
|
|
writer.writerow({c: str(row.get(c, "") or "") for c in REQUIRED_BOT_COLUMNS})
|
|
return buf.getvalue().encode("utf-8-sig")
|