460 lines
13 KiB
Python
460 lines
13 KiB
Python
"""Detección y normalización de fechas."""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import unicodedata
|
|
from datetime import date
|
|
from typing import Any, Optional
|
|
|
|
import pandas as pd
|
|
|
|
from .nullish import _is_nullish_scalar
|
|
|
|
_DATE_NUM_RE = re.compile(r"^\s*(\d{1,4})[\s\/\-\.](\d{1,2})[\s\/\-\.](\d{1,4})\s*$")
|
|
_DATE_COMPACT_RE = re.compile(r"^\s*(\d{6}|\d{8})\s*$")
|
|
# Hora opcional: HH[:MM[:SS]][ AM|PM] al final de la cadena
|
|
_TIME_TAIL_RE = re.compile(
|
|
r"[\sT]+(\d{1,2})(?::(\d{1,2})(?::(\d{1,2}))?)?\s*([APap]\.?\s*[Mm]\.?)?\s*$"
|
|
)
|
|
|
|
_MONTH_NAME_TO_NUM = {
|
|
"enero": 1,
|
|
"ene": 1,
|
|
"febrero": 2,
|
|
"feb": 2,
|
|
"marzo": 3,
|
|
"mar": 3,
|
|
"abril": 4,
|
|
"abr": 4,
|
|
"mayo": 5,
|
|
"may": 5,
|
|
"junio": 6,
|
|
"jun": 6,
|
|
"julio": 7,
|
|
"jul": 7,
|
|
"agosto": 8,
|
|
"ago": 8,
|
|
"septiembre": 9,
|
|
"setiembre": 9,
|
|
"sep": 9,
|
|
"sept": 9,
|
|
"octubre": 10,
|
|
"oct": 10,
|
|
"noviembre": 11,
|
|
"nov": 11,
|
|
"diciembre": 12,
|
|
"dic": 12,
|
|
}
|
|
|
|
|
|
def _split_date_time(value: str) -> tuple[str, Optional[str]]:
|
|
"""Separa la parte de fecha de la parte de hora si está presente.
|
|
|
|
Devuelve `(date_part, time_part_normalized)`. `time_part_normalized` es
|
|
None si no hay hora. La hora se normaliza a `HH:MM[:SS][ AM|PM]`.
|
|
"""
|
|
if value is None:
|
|
return "", None
|
|
s = str(value).strip()
|
|
if not s:
|
|
return "", None
|
|
|
|
m = _TIME_TAIL_RE.search(s)
|
|
if not m:
|
|
return s, None
|
|
|
|
hh = int(m.group(1))
|
|
mm = int(m.group(2)) if m.group(2) else 0
|
|
ss = m.group(3)
|
|
ampm = m.group(4)
|
|
if not (0 <= hh <= 23 and 0 <= mm <= 59):
|
|
return s, None
|
|
if ss is not None:
|
|
try:
|
|
ssi = int(ss)
|
|
if not (0 <= ssi <= 59):
|
|
return s, None
|
|
except ValueError:
|
|
return s, None
|
|
# Validar contra AM/PM (con AM/PM, hora 1-12)
|
|
if ampm and not (1 <= hh <= 12):
|
|
return s, None
|
|
|
|
# Quitar separadores habituales entre fecha y hora (p. ej. "12/27/24, 12:00 AM").
|
|
date_part = s[: m.start()].rstrip(" \t,;T")
|
|
if not date_part:
|
|
return s, None
|
|
|
|
parts: list[str] = [f"{hh:02d}:{mm:02d}"]
|
|
if ss is not None:
|
|
parts[0] += f":{int(ss):02d}"
|
|
if ampm:
|
|
norm_ampm = ampm.replace(".", "").replace(" ", "").upper()
|
|
if norm_ampm in ("AM", "PM"):
|
|
parts.append(norm_ampm)
|
|
time_norm = " ".join(parts)
|
|
return date_part, time_norm
|
|
|
|
|
|
def _extract_date_parts(value: str) -> Optional[tuple[int, int, int, str]]:
|
|
"""Intenta extraer (a, b, c, layout) de una fecha en cualquier formato común.
|
|
|
|
`layout` es uno de: 'sep' (con separadores), 'compact8' (YYYYMMDD o
|
|
DDMMYYYY), 'compact6' (DDMMYY o YYMMDD). No interpreta todavía qué
|
|
componente es día/mes/año; eso lo decide `detect_date_format` viendo la
|
|
columna completa. Acepta valores con hora al final (ignorada para los
|
|
componentes de fecha).
|
|
"""
|
|
if value is None:
|
|
return None
|
|
s = str(value).strip()
|
|
if not s or s.lower() in ("nan", "none", "nat"):
|
|
return None
|
|
|
|
date_part, _ = _split_date_time(s)
|
|
if date_part:
|
|
s = date_part
|
|
|
|
m = _DATE_NUM_RE.match(s)
|
|
if m:
|
|
try:
|
|
a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
|
return (a, b, c, "sep")
|
|
except ValueError:
|
|
return None
|
|
|
|
m2 = _DATE_COMPACT_RE.match(s)
|
|
if m2:
|
|
digits = m2.group(1)
|
|
if len(digits) == 8:
|
|
return (int(digits[0:2]), int(digits[2:4]), int(digits[4:8]), "compact8")
|
|
if len(digits) == 6:
|
|
return (int(digits[0:2]), int(digits[2:4]), int(digits[4:6]), "compact6")
|
|
|
|
return None
|
|
|
|
|
|
def detect_date_format(series: pd.Series, sample_size: int = 500) -> Optional[str]:
|
|
"""Analiza la serie y determina el formato de fecha dominante.
|
|
|
|
Devuelve un string con el layout detectado (p. ej. "DD/MM/YYYY",
|
|
"MM/DD/YYYY", "YYYYMMDD", "DDMMYYYY") o None si no parece fechas.
|
|
"""
|
|
if series is None or len(series) == 0:
|
|
return None
|
|
|
|
non_null = series.dropna()
|
|
if non_null.empty:
|
|
return None
|
|
|
|
if len(non_null) > sample_size:
|
|
non_null = non_null.sample(sample_size, random_state=42)
|
|
|
|
layouts = {"sep": 0, "compact8": 0, "compact6": 0}
|
|
sep_first_gt12 = 0 # primer componente > 12 → probable día
|
|
sep_second_gt12 = 0 # segundo componente > 12 → probable mes solo posible si MM/DD
|
|
sep_third_4digits = 0 # tercer componente de 4 dígitos → año al final
|
|
sep_first_4digits = 0 # primer componente de 4 dígitos → año al inicio
|
|
compact8_year_first = 0 # primeros 4 dígitos > 1900 → YYYYMMDD
|
|
compact8_year_last = 0 # últimos 4 dígitos > 1900 → DDMMYYYY
|
|
total_parsed = 0
|
|
|
|
for val in non_null:
|
|
parsed = _extract_date_parts(val)
|
|
if parsed is None:
|
|
continue
|
|
total_parsed += 1
|
|
a, b, c, layout = parsed
|
|
layouts[layout] = layouts.get(layout, 0) + 1
|
|
|
|
if layout == "sep":
|
|
# año puede estar al inicio o al final según cantidad de dígitos
|
|
if _digit_count(a) == 4:
|
|
sep_first_4digits += 1
|
|
if _digit_count(c) == 4:
|
|
sep_third_4digits += 1
|
|
if a > 12:
|
|
sep_first_gt12 += 1
|
|
if b > 12:
|
|
sep_second_gt12 += 1
|
|
elif layout == "compact8":
|
|
# a son los primeros 2 dígitos, c son los últimos 4
|
|
if c >= 1900:
|
|
compact8_year_last += 1
|
|
# a concatenado con b son los primeros 4 dígitos
|
|
first4 = a * 100 + b
|
|
if first4 >= 1900:
|
|
compact8_year_first += 1
|
|
|
|
if total_parsed == 0:
|
|
return None
|
|
|
|
# Requerimos que >=70% de los no nulos parseen como fecha para considerarla
|
|
# columna de fechas.
|
|
if total_parsed / len(non_null) < 0.7:
|
|
return None
|
|
|
|
dominant = max(layouts, key=lambda k: layouts[k])
|
|
|
|
if dominant == "compact8":
|
|
if compact8_year_first > compact8_year_last:
|
|
return "YYYYMMDD"
|
|
return "DDMMYYYY"
|
|
if dominant == "compact6":
|
|
return "DDMMYY"
|
|
|
|
# Caso con separadores
|
|
if sep_first_4digits > sep_third_4digits:
|
|
return "YYYY/MM/DD"
|
|
# año al final
|
|
if sep_first_gt12 > 0 and sep_second_gt12 == 0:
|
|
return "DD/MM/YYYY"
|
|
if sep_second_gt12 > 0 and sep_first_gt12 == 0:
|
|
return "MM/DD/YYYY"
|
|
# ambiguo → preferencia hispana
|
|
return "DD/MM/YYYY"
|
|
|
|
|
|
def _digit_count(n: int) -> int:
|
|
return len(str(abs(n)))
|
|
|
|
|
|
def _parse_date_with_format(value: str, fmt: str) -> Optional[tuple[int, int, int]]:
|
|
"""Convierte un valor de fecha a (día, mes, año) según el layout detectado."""
|
|
parsed = _extract_date_parts(value)
|
|
if parsed is None:
|
|
return None
|
|
a, b, c, layout = parsed
|
|
|
|
if fmt == "DD/MM/YYYY":
|
|
d, m, y = a, b, c
|
|
elif fmt == "MM/DD/YYYY":
|
|
m, d, y = a, b, c
|
|
elif fmt == "YYYY/MM/DD":
|
|
y, m, d = a, b, c
|
|
elif fmt == "YYYYMMDD" and layout == "compact8":
|
|
# reconstruir: primer4 = a*100 + b, luego mes y día
|
|
y = a * 100 + b
|
|
m = c // 100
|
|
d = c % 100
|
|
elif fmt == "DDMMYYYY" and layout == "compact8":
|
|
d = a
|
|
m = b
|
|
y = c
|
|
elif fmt == "DDMMYY" and layout == "compact6":
|
|
d = a
|
|
m = b
|
|
y = 2000 + c if c < 50 else 1900 + c
|
|
else:
|
|
# intentar como sep genérico
|
|
d, m, y = a, b, c
|
|
|
|
if not (1 <= m <= 12 and 1 <= d <= 31 and 1000 <= y <= 9999):
|
|
# año en 2 dígitos
|
|
if y < 100:
|
|
y = 2000 + y if y < 50 else 1900 + y
|
|
if not (1 <= m <= 12 and 1 <= d <= 31 and 1000 <= y <= 9999):
|
|
return None
|
|
return (d, m, y)
|
|
|
|
|
|
def _normalize_date_text(value: Any) -> str:
|
|
text = unicodedata.normalize("NFKD", str(value or ""))
|
|
text = "".join(c for c in text if not unicodedata.combining(c))
|
|
return text.lower()
|
|
|
|
|
|
def _valid_date_tuple(d: int, m: int, y: int) -> Optional[tuple[int, int, int]]:
|
|
if y < 100:
|
|
y = 2000 + y if y < 50 else 1900 + y
|
|
if not (1000 <= y <= 9999):
|
|
return None
|
|
try:
|
|
date(y, m, d)
|
|
except ValueError:
|
|
return None
|
|
return (d, m, y)
|
|
|
|
|
|
def _split_glued_year_suffix_hhmm(digits: str) -> Optional[tuple[int, int, int]]:
|
|
"""Detecta `YYYY` + `HHMM` en una sola tira de 8 cifras (p. ej. `20241344`).
|
|
|
|
Tras quitar separadores puede quedar pegado al mes (p. ej. `15may20241344`).
|
|
Devuelve (año, hora, minuto) si la parte horaria es válida; si no, None.
|
|
"""
|
|
if len(digits) != 8 or not digits.isdigit():
|
|
return None
|
|
y = int(digits[:4])
|
|
hh = int(digits[4:6])
|
|
mm = int(digits[6:8])
|
|
if not (1000 <= y <= 9999 and 0 <= hh <= 23 and 0 <= mm <= 59):
|
|
return None
|
|
return (y, hh, mm)
|
|
|
|
|
|
def _parse_text_month_candidate(
|
|
first: tuple[int, str],
|
|
month: int,
|
|
second: tuple[int, str],
|
|
*,
|
|
layout: Literal["around", "after", "before"],
|
|
) -> Optional[tuple[int, int, int]]:
|
|
a, astr = first
|
|
b, bstr = second
|
|
if layout == "around":
|
|
if len(astr) == 4:
|
|
return _valid_date_tuple(b, month, a)
|
|
if len(bstr) == 4:
|
|
return _valid_date_tuple(a, month, b)
|
|
return _valid_date_tuple(a, month, b)
|
|
if layout == "after":
|
|
if len(astr) == 4:
|
|
return _valid_date_tuple(b, month, a)
|
|
return _valid_date_tuple(a, month, b)
|
|
if len(astr) == 4:
|
|
return _valid_date_tuple(b, month, a)
|
|
return _valid_date_tuple(a, month, b)
|
|
|
|
|
|
def _parse_date_with_text_month(raw: Any) -> Optional[tuple[int, int, int]]:
|
|
if _is_nullish_scalar(raw):
|
|
return None
|
|
date_part, _ = _split_date_time(str(raw))
|
|
normalized = _normalize_date_text(date_part or raw)
|
|
# `\d{8}` antes que `\d{1,4}` para no partir año+hora pegados tras strip agresivo.
|
|
tokens = re.findall(r"[a-z]+|\d{8}|\d{1,4}", normalized)
|
|
if not tokens:
|
|
return None
|
|
|
|
for i, token in enumerate(tokens):
|
|
month = _MONTH_NAME_TO_NUM.get(token)
|
|
if month is None:
|
|
continue
|
|
nums_before = [
|
|
(int(t), t) for t in tokens[:i] if t.isdigit()
|
|
]
|
|
nums_after = [
|
|
(int(t), t) for t in tokens[i + 1 :] if t.isdigit()
|
|
]
|
|
candidates: list[Optional[tuple[int, int, int]]] = []
|
|
if nums_before and len(nums_after) == 1:
|
|
glued = _split_glued_year_suffix_hhmm(nums_after[0][1])
|
|
if glued is not None:
|
|
y, _hh, _mm = glued
|
|
candidates.append(
|
|
_parse_text_month_candidate(
|
|
nums_before[-1], month, (y, str(y)), layout="around"
|
|
)
|
|
)
|
|
if nums_before and nums_after:
|
|
candidates.append(
|
|
_parse_text_month_candidate(
|
|
nums_before[-1], month, nums_after[0], layout="around"
|
|
)
|
|
)
|
|
if len(nums_after) >= 2:
|
|
candidates.append(
|
|
_parse_text_month_candidate(
|
|
nums_after[0], month, nums_after[1], layout="after"
|
|
)
|
|
)
|
|
if len(nums_before) >= 2:
|
|
candidates.append(
|
|
_parse_text_month_candidate(
|
|
nums_before[-2], month, nums_before[-1], layout="before"
|
|
)
|
|
)
|
|
for parsed in candidates:
|
|
if parsed is not None:
|
|
return parsed
|
|
return None
|
|
|
|
|
|
|
|
def _detect_compact_date_formats(
|
|
series: pd.Series,
|
|
) -> tuple[Optional[str], Optional[str]]:
|
|
cser = series.map(_normalize_messy_date_to_compact)
|
|
c8s = pd.Series(
|
|
[x for x in cser.tolist() if isinstance(x, str) and len(x) == 8]
|
|
)
|
|
c6s = pd.Series(
|
|
[x for x in cser.tolist() if isinstance(x, str) and len(x) == 6]
|
|
)
|
|
fmt8 = detect_date_format(c8s) if not c8s.empty else None
|
|
fmt6 = detect_date_format(c6s) if not c6s.empty else None
|
|
return fmt8, fmt6
|
|
|
|
|
|
def _parse_any_date_to_tuple(
|
|
raw: Any,
|
|
*,
|
|
sep_fmt: Optional[str],
|
|
compact8_fmt: Optional[str],
|
|
compact6_fmt: Optional[str],
|
|
) -> Optional[tuple[int, int, int]]:
|
|
parsed_text = _parse_date_with_text_month(raw)
|
|
if parsed_text is not None:
|
|
return parsed_text
|
|
|
|
if _is_nullish_scalar(raw):
|
|
return None
|
|
|
|
raw_text = str(raw)
|
|
date_part, _ = _split_date_time(raw_text)
|
|
target = date_part if date_part else raw_text
|
|
if sep_fmt:
|
|
parsed = _parse_date_with_format(target, sep_fmt)
|
|
if parsed is not None:
|
|
return parsed
|
|
for fallback_fmt in ("DD/MM/YYYY", "MM/DD/YYYY", "YYYY/MM/DD"):
|
|
if fallback_fmt == sep_fmt:
|
|
continue
|
|
parsed = _parse_date_with_format(target, fallback_fmt)
|
|
if parsed is not None:
|
|
return parsed
|
|
|
|
compact = _normalize_messy_date_to_compact(raw)
|
|
if not isinstance(compact, str):
|
|
return None
|
|
if len(compact) == 8:
|
|
for fmt in (compact8_fmt, "DDMMYYYY", "YYYYMMDD"):
|
|
if not fmt:
|
|
continue
|
|
parsed = _parse_date_with_format(compact, fmt)
|
|
if parsed is not None:
|
|
return parsed
|
|
if len(compact) == 6:
|
|
parsed = _parse_date_with_format(compact, compact6_fmt or "DDMMYY")
|
|
if parsed is not None:
|
|
return parsed
|
|
return None
|
|
|
|
|
|
def _normalize_messy_date_to_compact(raw: Any) -> Optional[str]:
|
|
"""Quita ruido (* + _ / espacios, etc.); deja 6 o 8 dígitos o None.
|
|
|
|
Soporta separadores entremezclados; hora al final vía `_split_date_time`.
|
|
Con 9+ dígitos, heurística: tomar 8 iniciales (p. ej. 9.º = 0 colado) y
|
|
validar luego con `_parse_date_with_format`; con más de 8 sobrantes,
|
|
8 iniciales.
|
|
"""
|
|
if _is_nullish_scalar(raw):
|
|
return None
|
|
s = str(raw).strip()
|
|
if s.lower() in ("nan", "none", "nat", "<na>"):
|
|
return None
|
|
date_part, _ = _split_date_time(s)
|
|
body = date_part if date_part else s
|
|
d = re.sub(r"[^\d]", "", body)
|
|
if not d:
|
|
return None
|
|
n = len(d)
|
|
if n in (6, 8):
|
|
return d
|
|
if n < 6 or n == 7:
|
|
return None
|
|
# 9+ dígitos: 8 iniciales (p. ej. 9.º 0 sobrante, o ruido al final)
|
|
return d[:8]
|
|
|