feat: Enhance CSV cleaning service with new endpoints for CSV analysis and preview functionality. Introduce support for job management linked to extraction batches, and update the cleaning rules structure. Add new utility functions for handling CSV data and improve documentation for clarity on usage and available endpoints.
This commit is contained in:
erik
@@ -0,0 +1,258 @@
|
||||
"""Parsing numérico tolerante (miles, decimales, moneda)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Literal, Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from .nullish import _is_nullish_scalar
|
||||
|
||||
_HINT_MAJORITY = 0.6
|
||||
_MIN_HINT_VOTES = 1
|
||||
|
||||
|
||||
def _strip_currency(s: str) -> str:
|
||||
t = s.replace("\u00a0", " ").replace("\u202f", " ")
|
||||
t = re.sub(r"[\s]+", "", t)
|
||||
t = t.strip()
|
||||
if not t or t.lower() in ("nan", "none", "nat"):
|
||||
return ""
|
||||
neg = False
|
||||
if t[0:1] == "-":
|
||||
neg, t = not neg, t[1:].lstrip()
|
||||
if t[0:1] == "(" and t.endswith(")"):
|
||||
neg, t = not neg, t[1:-1]
|
||||
t = re.sub(
|
||||
r"^[\$€\u00a3\u0e3f\uffe5\uffe6\uff04\ufe69]+",
|
||||
"",
|
||||
t,
|
||||
count=1,
|
||||
)
|
||||
t = re.sub(r"[\$€%]+$", "", t)
|
||||
if not t and neg:
|
||||
return "-0"
|
||||
return ("-" if neg else "") + t
|
||||
|
||||
|
||||
def _extract_numeric_chunk(s: str) -> str:
|
||||
t = s.strip()
|
||||
t = t.replace("$", "").replace("€", "").replace("\u00a3", "")
|
||||
t = t.replace(" ", "")
|
||||
cands1 = re.findall(
|
||||
r"[+\-]?(?:\d{1,3}(?:[.,]\d{3})*|\d+)(?:[.,]\d+)?", t
|
||||
)
|
||||
cands2 = re.findall(
|
||||
r"[+\-]?\d[.,\d ]*\d", t
|
||||
)
|
||||
cands = cands1 + cands2
|
||||
if not cands:
|
||||
m = re.search(r"[\d.,\-]+", t)
|
||||
if m:
|
||||
c = m.group(0).replace("(", "-").replace(")", "")
|
||||
return c.lstrip("+")
|
||||
return ""
|
||||
best = max(cands, key=len) if cands else ""
|
||||
best = best.replace(" ", "").replace("(", "-").replace(")", "")
|
||||
if not re.search(r"\d", best):
|
||||
return ""
|
||||
return best.lstrip("+")
|
||||
|
||||
|
||||
def _as_float_s(s: str) -> Optional[float]:
|
||||
if not s:
|
||||
return None
|
||||
try:
|
||||
f = float(s)
|
||||
if not np.isfinite(f) or abs(f) > 1e100:
|
||||
return None
|
||||
return f
|
||||
except (ValueError, TypeError, OverflowError):
|
||||
return None
|
||||
|
||||
|
||||
def _parse_both_separators(body: str) -> Optional[float]:
|
||||
"""El separador más a la DERECHA entre `,` y `.` es el decimal; el otro, miles."""
|
||||
if "," not in body or "." not in body:
|
||||
return None
|
||||
t = body.strip()
|
||||
if re.search(r"[^0-9,.\-]", t):
|
||||
t2 = re.sub(r"[^0-9,.\-]", "", t)
|
||||
t = t2
|
||||
if not t:
|
||||
return None
|
||||
neg = t[0:1] == "-"
|
||||
b = t[1:].lstrip() if neg else t
|
||||
if not b:
|
||||
return None
|
||||
i_comma, i_dot = b.rfind(","), b.rfind(".")
|
||||
if min(i_comma, i_dot) < 0:
|
||||
return None
|
||||
dpos = max(i_comma, i_dot)
|
||||
th = "," if b[dpos] == "." else "."
|
||||
intp, fr = b[:dpos], b[dpos + 1 :]
|
||||
if not re.fullmatch(r"\d{1,4}$", fr):
|
||||
return None
|
||||
intc = intp.replace(th, "")
|
||||
if not intc.isdigit():
|
||||
return None
|
||||
v = _as_float_s(f"{intc}.{fr}")
|
||||
if v is None:
|
||||
return None
|
||||
if neg and v and v is not None:
|
||||
return -abs(v)
|
||||
return v
|
||||
|
||||
|
||||
def _only_commas(w: str, dec_hint: Optional[Literal[",", "."]]) -> Optional[float]:
|
||||
neg = w[0:1] == "-"
|
||||
wn = w[1:].lstrip() if neg else w
|
||||
if wn and re.search(r"[^0-9,]", wn):
|
||||
wn = re.sub(r"[^0-9,]", "", wn)
|
||||
n = wn.count(",")
|
||||
if n == 0 and wn.isdigit():
|
||||
v = float(wn)
|
||||
return -v if neg else v
|
||||
if n == 0 or "." in wn:
|
||||
return None
|
||||
if n == 1:
|
||||
i = wn.rfind(",")
|
||||
h, t = wn[:i], wn[i + 1 :]
|
||||
if not t.isdigit() or not h.isdigit():
|
||||
return None
|
||||
if len(t) in (1, 2):
|
||||
v = _as_float_s(f"{h}.{t}")
|
||||
if v is None:
|
||||
return None
|
||||
return -v if neg else v
|
||||
if len(t) == 3:
|
||||
if dec_hint is None or dec_hint == ",":
|
||||
v = _as_float_s(f"{h}{t}")
|
||||
if v is None:
|
||||
return None
|
||||
return -v if neg else v
|
||||
if dec_hint == ".":
|
||||
v = _as_float_s(wn.replace(",", ""))
|
||||
if v is None:
|
||||
return None
|
||||
return -v if neg else v
|
||||
v = _as_float_s(wn.replace(",", ""))
|
||||
if v is None:
|
||||
return None
|
||||
return -v if neg else v
|
||||
|
||||
|
||||
def _only_dots(w: str, dec_hint: Optional[Literal[",", "."]]) -> Optional[float]:
|
||||
neg = w[0:1] == "-"
|
||||
wn = w[1:].lstrip() if neg else w
|
||||
if wn and re.search(r"[^0-9.]", wn):
|
||||
wn = re.sub(r"[^0-9.]", "", wn)
|
||||
n = wn.count(".")
|
||||
if n == 0 and wn.isdigit():
|
||||
v = float(wn)
|
||||
return -v if neg else v
|
||||
if n == 0 or "," in wn:
|
||||
return None
|
||||
if n == 1:
|
||||
i = wn.rfind(".")
|
||||
h, t = wn[:i], wn[i + 1 :]
|
||||
if not t.isdigit() or not h.isdigit():
|
||||
return None
|
||||
if len(t) in (1, 2):
|
||||
v = _as_float_s(f"{h}.{t}")
|
||||
if v is None:
|
||||
return None
|
||||
return -v if neg else v
|
||||
if len(t) == 3:
|
||||
if dec_hint is None or dec_hint == ".":
|
||||
v = _as_float_s(f"{h}{t}")
|
||||
if v is None:
|
||||
return None
|
||||
return -v if neg else v
|
||||
if dec_hint == ",":
|
||||
v = _as_float_s(wn.replace(".", ""))
|
||||
if v is None:
|
||||
return None
|
||||
return -v if neg else v
|
||||
v = _as_float_s(wn.replace(".", ""))
|
||||
if v is None:
|
||||
return None
|
||||
return -v if neg else v
|
||||
|
||||
|
||||
def _parse_lenient_one(
|
||||
raw: Any,
|
||||
decimal_hint: Optional[Literal[",", "."]] = None,
|
||||
) -> float:
|
||||
if _is_nullish_scalar(raw):
|
||||
return float("nan")
|
||||
t = _strip_currency(str(raw))
|
||||
if not t or t == "-":
|
||||
return float("nan")
|
||||
c = _extract_numeric_chunk(t) or t
|
||||
c = c.strip()
|
||||
if not c or c == "-":
|
||||
return float("nan")
|
||||
n_comma, n_dot = c.count(","), c.count(".")
|
||||
if n_comma and n_dot:
|
||||
f = _parse_both_separators(c)
|
||||
elif n_comma and not n_dot:
|
||||
f = _only_commas(c, decimal_hint)
|
||||
elif n_dot and not n_comma:
|
||||
f = _only_dots(c, decimal_hint)
|
||||
else:
|
||||
c2 = re.sub(r"[^0-9.\-]", "", c)
|
||||
f = _as_float_s(c2) if c2 and re.fullmatch(r"-?\d+$", c2) else None
|
||||
if f is not None and np.isfinite(f):
|
||||
return f
|
||||
return float("nan")
|
||||
|
||||
|
||||
def _infer_decimal_hint_from_column(
|
||||
series: pd.Series,
|
||||
) -> Optional[Literal[",", "."]]:
|
||||
"""Voto con celdas que traen `,` y `.`: gana el de más a la derecha = decimal."""
|
||||
votes: list[Literal[",", "."]] = []
|
||||
for raw in series.dropna():
|
||||
if _is_nullish_scalar(raw):
|
||||
continue
|
||||
t = _strip_currency(str(raw))
|
||||
c = _extract_numeric_chunk(t) or t
|
||||
c = c.strip()
|
||||
if "," not in c or "." not in c or _parse_both_separators(c) is None:
|
||||
continue
|
||||
if c.rfind(",") > c.rfind("."):
|
||||
votes.append(",")
|
||||
else:
|
||||
votes.append(".")
|
||||
n = len(votes)
|
||||
if n < _MIN_HINT_VOTES:
|
||||
return None
|
||||
a = sum(1 for v in votes if v == ",")
|
||||
b = sum(1 for v in votes if v == ".")
|
||||
if a + b == 0:
|
||||
return None
|
||||
if a / n >= _HINT_MAJORITY and a > b:
|
||||
return ","
|
||||
if b / n >= _HINT_MAJORITY and b > a:
|
||||
return "."
|
||||
return None
|
||||
|
||||
|
||||
def _to_numeric_lenient(
|
||||
series: pd.Series,
|
||||
decimal_hint: Optional[Literal[",", "."]] = None,
|
||||
) -> pd.Series:
|
||||
"""Convierte a flotante; pista de columna por voto (si `decimal_hint` es None)."""
|
||||
hint = decimal_hint if decimal_hint is not None else _infer_decimal_hint_from_column(
|
||||
series
|
||||
)
|
||||
|
||||
def _cell(x: Any) -> float:
|
||||
if _is_nullish_scalar(x):
|
||||
return float("nan")
|
||||
return _parse_lenient_one(x, hint)
|
||||
|
||||
return series.map(_cell)
|
||||
|
||||
Reference in New Issue
Block a user