feat: Enhance CSV cleaning service with new endpoints for CSV analysis and preview functionality. Introduce support for job management linked to extraction batches, and update the cleaning rules structure. Add new utility functions for handling CSV data and improve documentation for clarity on usage and available endpoints.

2026-07-11 00:27:05 -05:00
parent cf0ebc72ba
commit b770761832
73 changed files with 7614 additions and 2671 deletions
@@ -0,0 +1,175 @@
+"""Homologación y división de tipo/número de documento."""
+from __future__ import annotations
+
+import re
+import unicodedata
+from typing import Any, Optional
+
+import pandas as pd
+
+from .nullish import _is_nullish_scalar
+
+
+_DOC_TYPE_ALIASES: dict[str, tuple[str, ...]] = {
+  "CC": ("CC", "Cedula", "CD", "C", "C.C", "C C", "CCC"),
+  "TI": ("TI", "Tarjeta", "Tarjeta Identidad", "T1", "T", "TL"),
+  "CN": ("CN", "Certificado", "Nacido Vivo", "Certificado Nacido Vivo"),
+  "CE": ("CE", "Cedula de Extranjeria", "Extranjera"),
+  "ME": ("ME", "Menor sin Identificacion"),
+  "PS": ("PS", "Pasaporte"),
+  "PT": ("PT", "Permiso por proteccion Temporal"),
+  "RC": ("RC", "Registro Civil", "Registro", "TC"),
+  "SC": ("SC", "Salvo Conducto", "Salvoconducto"),
+  "PE": ("PE", "Permiso especial de permanencia", "Permiso"),
+}
+
+
+def _ascii_upper(value: Any) -> str:
+  text = unicodedata.normalize("NFKD", str(value or ""))
+  text = "".join(c for c in text if not unicodedata.combining(c))
+  return text.upper()
+
+
+def _normalize_doc_type_key(value: Any) -> str:
+  """Clave comparable: sin acentos, espacios, puntos ni símbolos."""
+  return re.sub(r"[^A-Z0-9]", "", _ascii_upper(value))
+
+
+def _clean_doc_type_text(value: Any) -> Optional[str]:
+  cleaned = re.sub(r"[^A-Z0-9]+", " ", _ascii_upper(value)).strip()
+  cleaned = re.sub(r"\s+", " ", cleaned)
+  return cleaned or None
+
+
+_DOC_TYPE_ALIAS_BY_KEY: dict[str, str] = {
+  _normalize_doc_type_key(alias): canonical
+  for canonical, aliases in _DOC_TYPE_ALIASES.items()
+  for alias in aliases
+}
+
+_DOC_TYPE_ALIAS_DISPLAY_BY_KEY: dict[str, str] = {}
+for _canonical, _aliases in _DOC_TYPE_ALIASES.items():
+  for _alias in _aliases:
+    _alias_key = _normalize_doc_type_key(_alias)
+    _alias_display = _clean_doc_type_text(_alias)
+    if _alias_key and _alias_display:
+      _DOC_TYPE_ALIAS_DISPLAY_BY_KEY.setdefault(_alias_key, _alias_display)
+
+_DOC_TYPE_ALIAS_KEYS_SORTED: list[str] = sorted(
+  _DOC_TYPE_ALIAS_BY_KEY,
+  key=lambda k: (len(k), k),
+  reverse=True,
+)
+
+
+def homologate_doc_type(value: Any) -> Optional[str]:
+  """Homologa un tipo documental a sigla; si no existe, conserva texto limpio."""
+  if _is_nullish_scalar(value):
+    return None
+  key = _normalize_doc_type_key(value)
+  if not key or key.lower() in ("nan", "none", "nat"):
+    return None
+  return _DOC_TYPE_ALIAS_BY_KEY.get(key) or _clean_doc_type_text(value)
+
+
+def _compact_doc_text(value: Any) -> str:
+  return _normalize_doc_type_key(value)
+
+
+def _looks_like_short_alias_prefix(compact: str, alias_key: str) -> bool:
+  """Evita que alias de 1-2 chars capturen palabras como CODIGO o TOTAL."""
+  if not compact.startswith(alias_key):
+    return False
+  rest = compact[len(alias_key):]
+  if not rest:
+    return True
+  return bool(rest[0].isdigit())
+
+
+def _match_known_doc_type_prefix(compact: str) -> Optional[tuple[str, str]]:
+  for alias_key in _DOC_TYPE_ALIAS_KEYS_SORTED:
+    if len(alias_key) <= 2:
+      if not _looks_like_short_alias_prefix(compact, alias_key):
+        continue
+    elif not compact.startswith(alias_key):
+      continue
+    return _DOC_TYPE_ALIAS_BY_KEY[alias_key], alias_key
+  return None
+
+
+def split_doc_tipo_y_numero_texto(
+  raw: Any,
+  *,
+  homologate: bool = True,
+) -> tuple[Optional[str], Optional[str]]:
+  """Separa tipo documental y número, tolerando nombres largos y símbolos.
+
+  Si `homologate` es True, tipos conocidos se devuelven como sigla (CC, TI,
+  etc.). Los tipos desconocidos se conservan limpios en mayúsculas.
+  """
+  if _is_nullish_scalar(raw) and not (isinstance(raw, str) and not str(raw).strip()):
+    return (None, None)
+  t = str(raw).strip()
+  if t.lower() in ("nan", "none", "nat", "<na>"):
+    return (None, None)
+  if t and t[0].isdigit():
+    return (None, None)
+
+  compact = _compact_doc_text(t)
+  if not compact:
+    return (None, None)
+
+  match = _match_known_doc_type_prefix(compact)
+  if match is not None:
+    canonical, alias_key = match
+    rest = compact[len(alias_key):]
+    nums = re.sub(r"[^\d]", "", rest)
+    if homologate:
+      return (canonical, nums if nums else None)
+    raw_type = _DOC_TYPE_ALIAS_DISPLAY_BY_KEY.get(alias_key) or canonical
+    return (raw_type, nums if nums else None)
+
+  m = re.match(
+    r"^([^\d]+?)\s*([0-9].*)\s*$",
+    t,
+  )
+  if m:
+    to_t = _clean_doc_type_text(m.group(1))
+    nums = re.sub(r"[^\d]", "", m.group(2))
+    return (homologate_doc_type(to_t) if homologate else to_t, nums if nums else None)
+
+  if not re.search(r"\d", t):
+    to_t = homologate_doc_type(t) if homologate else _clean_doc_type_text(t)
+    return (to_t, None)
+  return (None, None)
+
+
+def _alloc_doc_split_column_names(
+  col_name: str, existing: set[str]
+) -> tuple[str, str]:
+  """Nombres únicos para las dos columnas derivadas."""
+  base = re.sub(r"[^\w]+", "_", str(col_name).strip(), flags=re.ASCII) or "col"
+  for i in range(0, 200):
+    suf = f"_{i}" if i else ""
+    t, n = f"{base}{suf}_doc_tipo", f"{base}{suf}_doc_numeros"
+    if t not in existing and n not in existing:
+      return t, n
+  raise ValueError("No se pudieron asignar nombres de columnas únicos.")
+
+
+def _apply_doc_id_split_columns(
+  result: pd.DataFrame, col: str, log: list[str], *, homologate_type: bool = True
+) -> None:
+  tcol, ncol = _alloc_doc_split_column_names(str(col), set(result.columns))
+  st_tipo: list[Optional[str]] = []
+  st_num: list[Any] = []
+  for v in result[col].tolist():
+    tt, nn = split_doc_tipo_y_numero_texto(v, homologate=homologate_type)
+    st_tipo.append(tt)
+    st_num.append(nn if nn is not None else pd.NA)
+  result[tcol] = st_tipo
+  result[ncol] = st_num
+  log.append(
+    f"[OK] '{str(col)}' -> columnas '{tcol}' y '{ncol}' <- Dividir tipo y número (documento)"
+  )
+