133 lines
3.7 KiB
Python
133 lines
3.7 KiB
Python
"""Carga robusta de CSV (delimitador, encoding)."""
|
|
from __future__ import annotations
|
|
|
|
import csv as _csv
|
|
from collections import Counter
|
|
from typing import Optional
|
|
|
|
import pandas as pd
|
|
|
|
|
|
_CANDIDATE_DELIMITERS = ["|", ";", "\t", ",", ":"]
|
|
|
|
|
|
def detect_delimiter(filepath: str) -> str:
|
|
"""Detecta el delimitador del CSV de forma robusta.
|
|
|
|
Estrategia:
|
|
1. Intentar `csv.Sniffer` con un sample grande.
|
|
2. Si falla o el resultado parece incorrecto, contar ocurrencias por línea
|
|
de cada delimitador candidato y elegir el que produce el mayor número
|
|
de columnas estable entre líneas.
|
|
"""
|
|
encodings = ("utf-8", "utf-8-sig", "latin-1", "cp1252")
|
|
sample = ""
|
|
for enc in encodings:
|
|
try:
|
|
with open(filepath, "r", encoding=enc, errors="replace") as f:
|
|
sample = f.read(65536)
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
if not sample:
|
|
return ","
|
|
|
|
try:
|
|
dialect = _csv.Sniffer().sniff(sample, delimiters="".join(_CANDIDATE_DELIMITERS))
|
|
sniffed = dialect.delimiter
|
|
except _csv.Error:
|
|
sniffed = None
|
|
|
|
lines = [ln for ln in sample.splitlines() if ln.strip()]
|
|
if not lines:
|
|
return sniffed or ","
|
|
|
|
# Para cada candidato, calcular el número promedio de columnas y la
|
|
# consistencia (cuántas líneas tienen el mismo conteo dominante).
|
|
scores: dict[str, tuple[int, int]] = {}
|
|
for delim in _CANDIDATE_DELIMITERS:
|
|
counts = [ln.count(delim) for ln in lines[:50]]
|
|
if max(counts) == 0:
|
|
continue
|
|
# Modo (valor más frecuente)
|
|
common = Counter(counts).most_common(1)[0]
|
|
mode_count, mode_freq = common
|
|
if mode_count == 0:
|
|
continue
|
|
scores[delim] = (mode_count, mode_freq)
|
|
|
|
if scores:
|
|
# Elegir el delimitador con mayor número de columnas (mode_count) y,
|
|
# como desempate, mayor consistencia (mode_freq).
|
|
best = max(scores.items(), key=lambda kv: (kv[1][0], kv[1][1]))
|
|
best_delim = best[0]
|
|
# Si el sniffer dio algo razonable y coincide, prevalece; si difiere y
|
|
# nuestro candidato es claramente mejor, usamos el nuestro.
|
|
if sniffed and sniffed in scores:
|
|
sniffed_score = scores[sniffed]
|
|
if sniffed_score[0] >= best[1][0] - 1:
|
|
return sniffed
|
|
return best_delim
|
|
|
|
return sniffed or ","
|
|
|
|
|
|
def load_csv(filepath: str) -> tuple[pd.DataFrame, str]:
|
|
"""Carga un CSV con detección de delimitador y tolerancia a filas malas.
|
|
|
|
Devuelve `(df, delimiter)`. Lanza excepción si todos los intentos fallan.
|
|
"""
|
|
delimiter = detect_delimiter(filepath)
|
|
|
|
encodings_to_try = ["utf-8", "utf-8-sig", "latin-1", "cp1252"]
|
|
last_error: Optional[Exception] = None
|
|
|
|
for enc in encodings_to_try:
|
|
try:
|
|
df = pd.read_csv(
|
|
filepath,
|
|
delimiter=delimiter,
|
|
dtype=str,
|
|
encoding=enc,
|
|
engine="python",
|
|
on_bad_lines="skip",
|
|
)
|
|
return df, delimiter
|
|
except Exception as exc: # noqa: BLE001
|
|
last_error = exc
|
|
continue
|
|
|
|
# Último intento con quoting permisivo y sin separador detectado
|
|
try:
|
|
df = pd.read_csv(
|
|
filepath,
|
|
sep=None,
|
|
dtype=str,
|
|
engine="python",
|
|
on_bad_lines="skip",
|
|
)
|
|
return df, "(auto)"
|
|
except Exception as exc: # noqa: BLE001
|
|
last_error = exc
|
|
|
|
raise RuntimeError(f"No se pudo leer el CSV: {last_error}")
|
|
|
|
|
|
def load_csv_bytes(content: bytes, filename: str = "upload.csv") -> tuple[pd.DataFrame, str]:
|
|
"""Carga un CSV desde bytes (upload HTTP) vía archivo temporal."""
|
|
import os
|
|
import tempfile
|
|
|
|
suffix = os.path.splitext(filename)[1] or ".csv"
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
|
|
tmp.write(content)
|
|
tmp_path = tmp.name
|
|
try:
|
|
return load_csv(tmp_path)
|
|
finally:
|
|
try:
|
|
os.unlink(tmp_path)
|
|
except OSError:
|
|
pass
|