feat: Enhance CSV cleaning service with new endpoints for CSV analysis and preview functionality. Introduce support for job management linked to extraction batches, and update the cleaning rules structure. Add new utility functions for handling CSV data and improve documentation for clarity on usage and available endpoints.

services/cleaning/domain/csv_cleaning/io.py

@@ -0,0 +1,132 @@
+"""Carga robusta de CSV (delimitador, encoding)."""
+from __future__ import annotations
+
+import csv as _csv
+from collections import Counter
+from typing import Optional
+
+import pandas as pd
+
+
+_CANDIDATE_DELIMITERS = ["|", ";", "\t", ",", ":"]
+
+
+def detect_delimiter(filepath: str) -> str:
+  """Detecta el delimitador del CSV de forma robusta.
+
+  Estrategia:
+  1. Intentar `csv.Sniffer` con un sample grande.
+  2. Si falla o el resultado parece incorrecto, contar ocurrencias por línea
+     de cada delimitador candidato y elegir el que produce el mayor número
+     de columnas estable entre líneas.
+  """
+  encodings = ("utf-8", "utf-8-sig", "latin-1", "cp1252")
+  sample = ""
+  for enc in encodings:
+    try:
+      with open(filepath, "r", encoding=enc, errors="replace") as f:
+        sample = f.read(65536)
+      break
+    except Exception:
+      continue
+
+  if not sample:
+    return ","
+
+  try:
+    dialect = _csv.Sniffer().sniff(sample, delimiters="".join(_CANDIDATE_DELIMITERS))
+    sniffed = dialect.delimiter
+  except _csv.Error:
+    sniffed = None
+
+  lines = [ln for ln in sample.splitlines() if ln.strip()]
+  if not lines:
+    return sniffed or ","
+
+  # Para cada candidato, calcular el número promedio de columnas y la
+  # consistencia (cuántas líneas tienen el mismo conteo dominante).
+  scores: dict[str, tuple[int, int]] = {}
+  for delim in _CANDIDATE_DELIMITERS:
+    counts = [ln.count(delim) for ln in lines[:50]]
+    if max(counts) == 0:
+      continue
+    # Modo (valor más frecuente)
+    common = Counter(counts).most_common(1)[0]
+    mode_count, mode_freq = common
+    if mode_count == 0:
+      continue
+    scores[delim] = (mode_count, mode_freq)
+
+  if scores:
+    # Elegir el delimitador con mayor número de columnas (mode_count) y,
+    # como desempate, mayor consistencia (mode_freq).
+    best = max(scores.items(), key=lambda kv: (kv[1][0], kv[1][1]))
+    best_delim = best[0]
+    # Si el sniffer dio algo razonable y coincide, prevalece; si difiere y
+    # nuestro candidato es claramente mejor, usamos el nuestro.
+    if sniffed and sniffed in scores:
+      sniffed_score = scores[sniffed]
+      if sniffed_score[0] >= best[1][0] - 1:
+        return sniffed
+    return best_delim
+
+  return sniffed or ","
+
+
+def load_csv(filepath: str) -> tuple[pd.DataFrame, str]:
+  """Carga un CSV con detección de delimitador y tolerancia a filas malas.
+
+  Devuelve `(df, delimiter)`. Lanza excepción si todos los intentos fallan.
+  """
+  delimiter = detect_delimiter(filepath)
+
+  encodings_to_try = ["utf-8", "utf-8-sig", "latin-1", "cp1252"]
+  last_error: Optional[Exception] = None
+
+  for enc in encodings_to_try:
+    try:
+      df = pd.read_csv(
+        filepath,
+        delimiter=delimiter,
+        dtype=str,
+        encoding=enc,
+        engine="python",
+        on_bad_lines="skip",
+      )
+      return df, delimiter
+    except Exception as exc:  # noqa: BLE001
+      last_error = exc
+      continue
+
+  # Último intento con quoting permisivo y sin separador detectado
+  try:
+    df = pd.read_csv(
+      filepath,
+      sep=None,
+      dtype=str,
+      engine="python",
+      on_bad_lines="skip",
+    )
+    return df, "(auto)"
+  except Exception as exc:  # noqa: BLE001
+    last_error = exc
+
+  raise RuntimeError(f"No se pudo leer el CSV: {last_error}")
+
+
+def load_csv_bytes(content: bytes, filename: str = "upload.csv") -> tuple[pd.DataFrame, str]:
+  """Carga un CSV desde bytes (upload HTTP) vía archivo temporal."""
+  import os
+  import tempfile
+
+  suffix = os.path.splitext(filename)[1] or ".csv"
+  with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
+    tmp.write(content)
+    tmp_path = tmp.name
+  try:
+    return load_csv(tmp_path)
+  finally:
+    try:
+      os.unlink(tmp_path)
+    except OSError:
+      pass