feat: Enhance CSV cleaning service with new endpoints for CSV analysis and preview functionality. Introduce support for job management linked to extraction batches, and update the cleaning rules structure. Add new utility functions for handling CSV data and improve documentation for clarity on usage and available endpoints.

2026-07-11 00:27:05 -05:00
parent cf0ebc72ba
commit b770761832
73 changed files with 7614 additions and 2671 deletions
@@ -0,0 +1,210 @@
+"""Heurísticas por nombre de columna y presentación del contenido."""
+from __future__ import annotations
+
+import re
+from typing import Optional
+
+import pandas as pd
+
+from .nullish import _is_nullish_scalar
+
+
+def _column_name_suggests_date_like(name: str) -> bool:
+  """Columnas cuyo nombre pide tratar el contenido como fecha a priori."""
+  n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip()
+  if not n:
+    return False
+  if re.search(
+    r"\b(fecha|date|año|mes|dia|día|f[_-]?(alta|baja|proc|corte|pago|ini|fin|"
+    r"desde|hasta|vigenc|vence|venc|caduc|f\s?ec)\b|periodo|anio|ano|expir)",
+    n,
+  ) or n.startswith("fec_") or n.startswith("fecha"):
+    return True
+  return False
+
+
+def _column_name_suggests_monetary_value(name: str) -> bool:
+  """importe, valor, neto, IVA, etc. — no interpretar cifras como fechas.
+
+  Encabezados de tipo `fecha_…` no se tratan como moneda salvo mezcla con monto/valor.
+  """
+  n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip()
+  if not n:
+    return False
+  if _column_name_suggests_date_like(name) and not re.search(
+    r"\b(monto|valor|importe|pago|total|bruto|neto|import|unit|precio|copago)\b", n
+  ):
+    return False
+  for key in (
+    "monto", "importe", "import", "bruto", "neto", "subtotal", "sub total",
+    "total", "unitario", "unidad", "valor", "precio", "pago", "copago", "sald", "dcto",
+    "franq", "cuota", "iva", "tarif", "costo", "carg", "reten", "retencion",
+  ):
+    if key in n:
+      if key == "iva" and re.search(r"(priva|giva|diva|viva|cliva)\b", n):
+        continue
+      if key == "carg" and re.search(r"descarg|recarg|abord", n):
+        continue
+      return True
+  return False
+
+
+def _column_name_suggests_code_id(name: str) -> bool:
+  """Código, ref., radicado, tipoynúm de documento — no forzar 6-8 cifras como fechas (DDMMYY, etc.)."""
+  n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip()
+  if not n or _column_name_suggests_date_like(name):
+    return False
+  for key in (
+    "codigo", "código", "codig", "códig", "cód_",
+    "sku", "ean", "upc", "isbn", "upn",
+    "ref", "referen", "radic", "nro_fac", "nro fac", "nit", "cédul", "cedul",
+    "autoriz", "correlat", "secuenc", "línea de", "linea de", "cupo",
+    "tipy", "tipi", "tipynum", "typnum", "type_n", "n_doc", "n_docu",
+    "núm doc", "n_docu", "doc_id", "docum", "id_doc",
+  ):
+    if key in n:
+      return True
+  if re.search(
+    r"tipo[_\s\\-]*(&|/|u|e|y)[_\s\\-]*num|tipo[_\s\\-]+doc|tipo_y_n",
+    n,
+    re.IGNORECASE,
+  ) or n.startswith("tip_"):
+    return True
+  if n.startswith("cod_") or n.startswith("cód_") or n.startswith("ref_"):
+    return True
+  return bool(re.search(r"\b(nro[._]fac|#?\s*autor|#?\s*ref)\b", n, re.IGNORECASE))
+
+
+def _column_name_suggests_quantity_or_number_not_bool(name: str) -> bool:
+  """Nombres típicos de cantidad, importe, fechas: no inferir bool por 0/1."""
+  n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip()
+  if not n:
+    return False
+  if _column_name_suggests_monetary_value(name) or _column_name_suggests_code_id(name):
+    return True
+  for key in (
+    "fecha",
+    "monto",
+    "importe",
+    "bruto",
+    "neto",
+    "subtotal",
+    "unitario",
+    "unidad",
+    "valor",
+    "precio",
+    "pago",
+    "copago",
+    "cantid",
+    "quantity",
+    "qty",
+    "nro",
+    "núm",
+    "n°",
+  ):
+    if key in n:
+      return True
+  return False
+
+
+def _raw_strings_suggest_monetary_presentation(non_null: pd.Series) -> bool:
+  """$ €, miles 1,234,12 o .00: presentación monetaria aunque el float sea "entero"."""
+  if non_null is None or len(non_null) == 0:
+    return False
+  n_hit = 0
+  n_seen = 0
+  for raw in non_null.head(500):
+    if _is_nullish_scalar(raw):
+      continue
+    t = str(raw)
+    s = t.strip()
+    n_seen += 1
+    if re.search(r"[$\u00a3\u20ac\uffe5\uffe6\ufe69\uff04]", t):
+      n_hit += 1
+      continue
+    if re.search(r"\b\d{1,3}(?:[.,\u00a0]\d{3})+\b", s) or re.search(
+      r"(?<!\d)\d+[.,][0-9]{1,2}\b", s
+    ):
+      n_hit += 1
+  if n_seen == 0:
+    return False
+  return n_hit / n_seen >= 0.10
+
+
+def _cell_looks_cifra_con_moneda_iso_al_final(s: str) -> bool:
+  """Solo cifra/símbolo (no prefijo de documento) y código de moneda al final.
+
+  Excluye filas al estilo `CC12…` (típico alfanumérico).
+  """
+  t = str(s).strip()
+  u = t.upper()
+  if not re.search(
+    r"(?i)(USD|COP|EUR|CLP|ARS|GBP|MXN|PEN|DOP|UYU|VES|CRC|PYG|BRL|BOB|GTQ)\b\s*\Z", u
+  ):
+    return False
+  if re.match(
+    r"^(?i)(CC|CE|NIT|NUIP|TI|PA|MS|RC|DNI|NUIP|CN|SC|AS|PE|PT)\D*[\d0]", u
+  ):
+    return False
+  return re.match(
+    r"^[\$€0-9\s,.\-+%/]", t,
+  ) is not None
+
+
+def _raw_strings_suggest_alnum_mixed_presentation(non_null: pd.Series) -> bool:
+  """Letras y dígitos juntos (p. ej. `CC32…`, `A1-2B`); no es puro 'entero/decimal' lógicamente.
+
+  Excluido: columnas con apariencia monetaria, y celdas solo cifra + ISO al final (100 USD).
+  """
+  if _raw_strings_suggest_monetary_presentation(non_null):
+    return False
+  n_m = 0
+  n_s = 0
+  for raw in non_null.head(500):
+    if _is_nullish_scalar(raw):
+      continue
+    t = str(raw).strip()
+    n_s += 1
+    if not re.search(r"\d", t) or not re.search(
+      r"[A-Za-zÑñ\u00c0-\u024f]", t, re.IGNORECASE
+    ):
+      continue
+    if _cell_looks_cifra_con_moneda_iso_al_final(t):
+      continue
+    n_m += 1
+  if n_s == 0:
+    return False
+  return n_m / n_s >= 0.20
+
+
+def _cell_looks_like_documento_id_tipo_nit(s: str) -> bool:
+  """Cadenas estilo `CC32324029`, `CE 1.234.567` — no son fechas (evita 8 cifras como DDMM).
+
+  Al menos una letra al inicio y 5-16 dígitos en la celda (no `Jan`+`2020` = 4 dígitos).
+  """
+  t = str(s).strip()
+  if not t or len(t) < 3:
+    return False
+  if not re.match(r"^[A-Za-z]{1,3}", t):
+    return False
+  dig = re.sub(r"[^\d]", "", t)
+  if len(dig) < 5 or len(dig) > 16:
+    return False
+  return True
+
+
+def _content_looks_like_doc_id_letter_prefix_column(non_null: pd.Series) -> bool:
+  """Muestras con prefijo documental (CC, NIT, …) y ≥5 dígitos → no columna de fecha."""
+  n_ok = 0
+  n_seen = 0
+  for raw in non_null.head(500):
+    if _is_nullish_scalar(raw):
+      continue
+    s = str(raw).strip()
+    n_seen += 1
+    if _cell_looks_like_documento_id_tipo_nit(s):
+      n_ok += 1
+  if n_seen == 0:
+    return False
+  return n_ok / n_seen >= 0.20
+