feat: Enhance CSV cleaning service with new endpoints for CSV analysis and preview functionality. Introduce support for job management linked to extraction batches, and update the cleaning rules structure. Add new utility functions for handling CSV data and improve documentation for clarity on usage and available endpoints.

2026-07-11 00:27:05 -05:00
parent cf0ebc72ba
commit b770761832
73 changed files with 7614 additions and 2671 deletions
@@ -0,0 +1,94 @@
+"""Análisis de DataFrames: resumen por columna y muestras."""
+from __future__ import annotations
+
+import json
+import random
+from typing import Any
+
+import pandas as pd
+
+from .models import (
+  TYPE_CATEGORICAL,
+  TYPE_DATE,
+  TYPE_DECIMAL,
+  TYPE_INTEGER,
+  TYPE_TEXT,
+  ColumnSummary,
+)
+from .numbers import _to_numeric_lenient
+from .type_inference import _applicable_rules_for_type, _infer_column_type
+
+
+def analyze_dataframe(df: pd.DataFrame) -> list[ColumnSummary]:
+  """Genera un resumen por columna del DataFrame."""
+  summaries: list[ColumnSummary] = []
+  total_rows = len(df)
+  rng = random.Random(42)
+
+  for col in df.columns:
+    series = df[col]
+    null_count = int(series.isna().sum() + (series.astype(str).str.strip() == "").sum())
+    # Evitar doble conteo: considerar vacío solo si no-NaN
+    null_count = int((series.isna() | (series.astype(str).str.strip() == "")).sum())
+    null_pct = (null_count / total_rows * 100) if total_rows else 0.0
+
+    non_null = series.dropna().astype(str)
+    non_null = non_null[non_null.str.strip() != ""]
+
+    example = ""
+    samples: list[str] = []
+    if not non_null.empty:
+      values = non_null.tolist()
+      k = min(5, len(values))
+      picked = rng.sample(values, k) if len(values) >= k else list(values)
+      for raw in picked:
+        text = str(raw)
+        if len(text) > 80:
+          text = text[:77] + "..."
+        samples.append(text)
+      example = samples[0] if samples else ""
+
+    unique_count = int(non_null.nunique())
+    unique_pct = (unique_count / total_rows * 100) if total_rows else 0.0
+
+    inferred_type, date_fmt = _infer_column_type(series, str(col))
+
+    summary = ColumnSummary(
+      name=str(col),
+      inferred_type=inferred_type,
+      example=example,
+      samples=samples,
+      null_count=null_count,
+      null_pct=null_pct,
+      unique_count=unique_count,
+      unique_pct=unique_pct,
+      detected_date_format=date_fmt,
+    )
+
+    if inferred_type in (TYPE_INTEGER, TYPE_DECIMAL):
+      numeric = _to_numeric_lenient(non_null)
+      if numeric.notna().any():
+        summary.min = float(numeric.min())
+        summary.max = float(numeric.max())
+        summary.mean = float(numeric.mean())
+    elif inferred_type in (TYPE_TEXT, TYPE_CATEGORICAL):
+      if not non_null.empty:
+        summary.avg_length = float(non_null.str.len().mean())
+        counts = non_null.value_counts().head(3)
+        summary.top_values = [(str(v), int(c)) for v, c in counts.items()]
+
+    summary.applicable_rules = _applicable_rules_for_type(inferred_type)
+    summaries.append(summary)
+
+  return summaries
+
+
+def sample_dataframe_rows(df: pd.DataFrame, n: int = 8) -> list[dict[str, Any]]:
+  """Devuelve hasta `n` filas aleatorias como dicts JSON-serializables."""
+  if df.empty or n <= 0:
+    return []
+  take = min(n, len(df))
+  rng = random.Random(42)
+  indices = rng.sample(list(range(len(df))), take)
+  subset = df.iloc[indices]
+  return json.loads(subset.to_json(orient="records", date_format="iso"))