582 lines
17 KiB
Python
582 lines
17 KiB
Python
"""API REST del microservicio de limpieza."""
|
|
from __future__ import annotations
|
|
|
|
from contextlib import asynccontextmanager
|
|
import json
|
|
from typing import Any
|
|
|
|
import pandas as pd
|
|
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
|
|
from pydantic import BaseModel, Field
|
|
from sqlalchemy import delete, select
|
|
|
|
from domain.csv_cleaning import (
|
|
CLEANING_RULES,
|
|
analyze_dataframe,
|
|
apply_rules,
|
|
load_csv_bytes,
|
|
sample_dataframe_rows,
|
|
)
|
|
from infrastructure.database import (
|
|
CleanedRow,
|
|
CleaningJob,
|
|
ExtractionBatch,
|
|
ExtractionRow,
|
|
Job,
|
|
Task,
|
|
init_db,
|
|
session_scope,
|
|
utcnow,
|
|
)
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
init_db()
|
|
yield
|
|
|
|
|
|
app = FastAPI(
|
|
title="IPSExtractor Cleaning Service",
|
|
version="1.0.0",
|
|
lifespan=lifespan,
|
|
)
|
|
|
|
|
|
class CleaningRequest(BaseModel):
|
|
extraction_batch_id: str | None = None
|
|
job_id: str | None = None
|
|
rules_per_column: dict[str, list[str]] = Field(default_factory=dict)
|
|
cop_mod_fecha_column: dict[str, str] = Field(default_factory=dict)
|
|
cop_mod_bruto_column: dict[str, str] = Field(default_factory=dict)
|
|
cop_mod_neto_column: dict[str, str] = Field(default_factory=dict)
|
|
cop_mod_config: dict[str, Any] | None = None
|
|
|
|
|
|
class PreviewRequest(BaseModel):
|
|
rows: list[dict[str, Any]] = Field(default_factory=list)
|
|
extraction_batch_id: str | None = None
|
|
rules_per_column: dict[str, list[str]] = Field(default_factory=dict)
|
|
cop_mod_fecha_column: dict[str, str] = Field(default_factory=dict)
|
|
cop_mod_bruto_column: dict[str, str] = Field(default_factory=dict)
|
|
cop_mod_neto_column: dict[str, str] = Field(default_factory=dict)
|
|
cop_mod_config: dict[str, Any] | None = None
|
|
sample_size: int = 8
|
|
|
|
|
|
def dataframe_to_records(df: pd.DataFrame) -> list[dict[str, Any]]:
|
|
return json.loads(df.to_json(orient="records", date_format="iso"))
|
|
|
|
|
|
def serialize_job(job: CleaningJob) -> dict[str, Any]:
|
|
# started/finished derivados: created_at = inicio; updated_at = fin si ya terminó
|
|
finished = (
|
|
job.updated_at.isoformat()
|
|
if job.status in {"completed", "failed"}
|
|
else None
|
|
)
|
|
return {
|
|
"id": job.id,
|
|
"extraction_batch_id": job.extraction_batch_id,
|
|
"job_id": job.job_id,
|
|
"source": job.source,
|
|
"status": job.status,
|
|
"request": job.request,
|
|
"log": job.log,
|
|
"error": job.error,
|
|
"created_at": job.created_at.isoformat(),
|
|
"updated_at": job.updated_at.isoformat(),
|
|
"started_at": job.created_at.isoformat(),
|
|
"finished_at": finished,
|
|
}
|
|
|
|
|
|
def _validate_job_id(session: Any, job_id: str | None) -> None:
|
|
if not job_id:
|
|
return
|
|
if session.get(Job, job_id) is None:
|
|
raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado")
|
|
|
|
|
|
def _create_cleaning_task(
|
|
session: Any,
|
|
*,
|
|
job_id: str | None,
|
|
cleaning_job_id: str,
|
|
source: str,
|
|
status: str,
|
|
request: dict[str, Any],
|
|
error: str | None = None,
|
|
title: str | None = None,
|
|
started_at: Any | None = None,
|
|
) -> str | None:
|
|
"""Crea Task(type=cleaning) vinculada al job Desktop. Devuelve task.id o None."""
|
|
if not job_id:
|
|
return None
|
|
now = utcnow()
|
|
task = Task(
|
|
job_id=job_id,
|
|
type="cleaning",
|
|
status=status,
|
|
title=title or f"Limpieza ({source})",
|
|
request=request,
|
|
result={"cleaning_job_id": cleaning_job_id},
|
|
error=error,
|
|
started_at=started_at or now,
|
|
finished_at=now,
|
|
updated_at=now,
|
|
)
|
|
session.add(task)
|
|
session.flush()
|
|
return task.id
|
|
|
|
|
|
def _persist_cleaned_rows(
|
|
session: Any,
|
|
*,
|
|
job_id: str,
|
|
extraction_batch_id: str | None,
|
|
output_rows: list[dict[str, Any]],
|
|
) -> None:
|
|
session.execute(delete(CleanedRow).where(CleanedRow.cleaning_job_id == job_id))
|
|
for idx, row in enumerate(output_rows, start=1):
|
|
session.add(
|
|
CleanedRow(
|
|
cleaning_job_id=job_id,
|
|
extraction_batch_id=extraction_batch_id,
|
|
row_index=idx,
|
|
row_json=row,
|
|
)
|
|
)
|
|
|
|
|
|
def _run_rules_on_dataframe(
|
|
df: pd.DataFrame,
|
|
rules_per_column: dict[str, list[str]],
|
|
cop_mod_fecha_column: dict[str, str],
|
|
cop_mod_config: dict[str, Any] | None,
|
|
cop_mod_bruto_column: dict[str, str] | None = None,
|
|
cop_mod_neto_column: dict[str, str] | None = None,
|
|
) -> tuple[list[dict[str, Any]], list[str]]:
|
|
result_df, log = apply_rules(
|
|
df,
|
|
rules_per_column,
|
|
cop_mod_fecha_column=cop_mod_fecha_column,
|
|
cop_mod_config=cop_mod_config,
|
|
cop_mod_bruto_column=cop_mod_bruto_column,
|
|
cop_mod_neto_column=cop_mod_neto_column,
|
|
)
|
|
return dataframe_to_records(result_df), log
|
|
|
|
|
|
def _parse_json_form(raw: str | None, default: Any) -> Any:
|
|
if not raw or not raw.strip():
|
|
return default
|
|
try:
|
|
return json.loads(raw)
|
|
except json.JSONDecodeError as exc:
|
|
raise HTTPException(status_code=400, detail=f"JSON inválido: {exc}") from exc
|
|
|
|
|
|
def _analysis_payload(df: pd.DataFrame, **extra: Any) -> dict[str, Any]:
|
|
return {
|
|
**extra,
|
|
"row_count": len(df),
|
|
"columns": [summary.__dict__ for summary in analyze_dataframe(df)],
|
|
"sample_rows": sample_dataframe_rows(df, 8),
|
|
}
|
|
|
|
|
|
def _cell_str(value: Any) -> str:
|
|
if value is None:
|
|
return ""
|
|
text = str(value)
|
|
if text in ("None", "nan", "NaT"):
|
|
return ""
|
|
if len(text) > 80:
|
|
return text[:77] + "..."
|
|
return text
|
|
|
|
|
|
def _preview_from_dataframe(
|
|
df: pd.DataFrame,
|
|
rules_per_column: dict[str, list[str]],
|
|
cop_mod_fecha_column: dict[str, str],
|
|
cop_mod_config: dict[str, Any] | None,
|
|
cop_mod_bruto_column: dict[str, str] | None = None,
|
|
cop_mod_neto_column: dict[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
rows_before = dataframe_to_records(df)
|
|
result_df, _log = apply_rules(
|
|
df,
|
|
rules_per_column,
|
|
cop_mod_fecha_column=cop_mod_fecha_column,
|
|
cop_mod_config=cop_mod_config,
|
|
cop_mod_bruto_column=cop_mod_bruto_column,
|
|
cop_mod_neto_column=cop_mod_neto_column,
|
|
)
|
|
rows_after = dataframe_to_records(result_df)
|
|
|
|
# Emparejar por índice de fila de muestra (misma posición)
|
|
n = min(len(rows_before), len(rows_after))
|
|
all_cols = list(dict.fromkeys([*df.columns.astype(str), *result_df.columns.astype(str)]))
|
|
columns_preview: list[dict[str, Any]] = []
|
|
for col in all_cols:
|
|
before_vals = [_cell_str(rows_before[i].get(col)) for i in range(n)]
|
|
after_vals = [_cell_str(rows_after[i].get(col)) for i in range(n)]
|
|
columns_preview.append({
|
|
"name": col,
|
|
"before": before_vals,
|
|
"after": after_vals,
|
|
"changed": before_vals != after_vals,
|
|
})
|
|
|
|
return {
|
|
"rows_before": rows_before,
|
|
"rows_after": rows_after,
|
|
"columns": columns_preview,
|
|
}
|
|
|
|
|
|
@app.get("/health")
|
|
def health() -> dict[str, str]:
|
|
return {"status": "ok"}
|
|
|
|
|
|
@app.get("/rules")
|
|
def list_rules() -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"id": rule["id"],
|
|
"name": rule["name"],
|
|
"description": rule.get("description", ""),
|
|
}
|
|
for rule in CLEANING_RULES
|
|
]
|
|
|
|
|
|
@app.get("/batches/{batch_id}/analysis")
|
|
def analyze_batch(batch_id: str) -> dict[str, Any]:
|
|
with session_scope() as session:
|
|
batch = session.get(ExtractionBatch, batch_id)
|
|
if batch is None:
|
|
raise HTTPException(status_code=404, detail="Lote de extraccion no encontrado")
|
|
rows = session.execute(
|
|
select(ExtractionRow.row_json)
|
|
.where(ExtractionRow.batch_id == batch_id)
|
|
.order_by(ExtractionRow.row_index, ExtractionRow.id)
|
|
).all()
|
|
records = [row[0] for row in rows]
|
|
|
|
df = pd.DataFrame(records)
|
|
return _analysis_payload(df, batch_id=batch_id)
|
|
|
|
|
|
@app.post("/analyze-csv")
|
|
async def analyze_csv(file: UploadFile = File(...)) -> dict[str, Any]:
|
|
content = await file.read()
|
|
if not content:
|
|
raise HTTPException(status_code=400, detail="El archivo CSV está vacío")
|
|
try:
|
|
df, delimiter = load_csv_bytes(content, file.filename or "upload.csv")
|
|
except Exception as exc: # noqa: BLE001
|
|
raise HTTPException(status_code=400, detail=f"No se pudo leer el CSV: {exc}") from exc
|
|
if df.empty:
|
|
raise HTTPException(status_code=400, detail="El CSV no tiene filas")
|
|
return _analysis_payload(df, filename=file.filename, delimiter=delimiter)
|
|
|
|
|
|
@app.post("/preview")
|
|
def preview_cleaning(request: PreviewRequest) -> dict[str, Any]:
|
|
"""Aplica reglas sobre una muestra pequeña y devuelve Antes/Después."""
|
|
records: list[dict[str, Any]] = list(request.rows or [])
|
|
|
|
if not records and request.extraction_batch_id:
|
|
with session_scope() as session:
|
|
batch = session.get(ExtractionBatch, request.extraction_batch_id)
|
|
if batch is None:
|
|
raise HTTPException(status_code=404, detail="Lote de extraccion no encontrado")
|
|
rows = session.execute(
|
|
select(ExtractionRow.row_json)
|
|
.where(ExtractionRow.batch_id == request.extraction_batch_id)
|
|
.order_by(ExtractionRow.row_index, ExtractionRow.id)
|
|
).all()
|
|
all_records = [row[0] for row in rows]
|
|
if not all_records:
|
|
raise HTTPException(status_code=400, detail="El lote no tiene filas")
|
|
df_full = pd.DataFrame(all_records)
|
|
records = sample_dataframe_rows(df_full, max(1, min(request.sample_size, 20)))
|
|
|
|
if not records:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="Se requieren rows o extraction_batch_id con filas",
|
|
)
|
|
|
|
df = pd.DataFrame(records)
|
|
try:
|
|
return _preview_from_dataframe(
|
|
df,
|
|
request.rules_per_column,
|
|
request.cop_mod_fecha_column,
|
|
request.cop_mod_config,
|
|
request.cop_mod_bruto_column,
|
|
request.cop_mod_neto_column,
|
|
)
|
|
except Exception as exc: # noqa: BLE001
|
|
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
|
|
|
|
|
@app.post("/jobs")
|
|
def run_cleaning(request: CleaningRequest) -> dict[str, Any]:
|
|
if not request.extraction_batch_id:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="extraction_batch_id es requerido (use /jobs/from-csv para CSV)",
|
|
)
|
|
|
|
with session_scope() as session:
|
|
batch = session.get(ExtractionBatch, request.extraction_batch_id)
|
|
if batch is None:
|
|
raise HTTPException(status_code=404, detail="Lote de extraccion no encontrado")
|
|
_validate_job_id(session, request.job_id)
|
|
|
|
extracted_rows = session.execute(
|
|
select(ExtractionRow.row_json)
|
|
.where(ExtractionRow.batch_id == request.extraction_batch_id)
|
|
.order_by(ExtractionRow.row_index, ExtractionRow.id)
|
|
).all()
|
|
records = [row[0] for row in extracted_rows]
|
|
if not records:
|
|
raise HTTPException(status_code=400, detail="El lote no tiene filas extraidas")
|
|
|
|
job = CleaningJob(
|
|
extraction_batch_id=request.extraction_batch_id,
|
|
job_id=request.job_id,
|
|
source="extraction",
|
|
status="running",
|
|
request=request.model_dump(),
|
|
updated_at=utcnow(),
|
|
)
|
|
session.add(job)
|
|
session.flush()
|
|
job_id = job.id
|
|
batch_id = request.extraction_batch_id
|
|
desktop_job_id = request.job_id
|
|
request_dump = request.model_dump()
|
|
|
|
try:
|
|
df = pd.DataFrame(records)
|
|
output_rows, log = _run_rules_on_dataframe(
|
|
df,
|
|
request.rules_per_column,
|
|
request.cop_mod_fecha_column,
|
|
request.cop_mod_config,
|
|
request.cop_mod_bruto_column,
|
|
request.cop_mod_neto_column,
|
|
)
|
|
|
|
with session_scope() as session:
|
|
_persist_cleaned_rows(
|
|
session,
|
|
job_id=job_id,
|
|
extraction_batch_id=batch_id,
|
|
output_rows=output_rows,
|
|
)
|
|
job = session.get(CleaningJob, job_id)
|
|
if job is None:
|
|
raise RuntimeError("No se encontro el job de limpieza recien creado.")
|
|
job.status = "completed"
|
|
job.log = log
|
|
job.updated_at = utcnow()
|
|
_create_cleaning_task(
|
|
session,
|
|
job_id=desktop_job_id,
|
|
cleaning_job_id=job_id,
|
|
source="extraction",
|
|
status="completed",
|
|
request=request_dump,
|
|
title="Limpieza desde extracción",
|
|
started_at=job.created_at,
|
|
)
|
|
session.flush()
|
|
return serialize_job(job)
|
|
except HTTPException:
|
|
raise
|
|
except Exception as exc:
|
|
with session_scope() as session:
|
|
job = session.get(CleaningJob, job_id)
|
|
started = job.created_at if job is not None else None
|
|
if job is not None:
|
|
job.status = "failed"
|
|
job.error = str(exc)
|
|
job.updated_at = utcnow()
|
|
_create_cleaning_task(
|
|
session,
|
|
job_id=desktop_job_id,
|
|
cleaning_job_id=job_id,
|
|
source="extraction",
|
|
status="failed",
|
|
request=request_dump,
|
|
error=str(exc),
|
|
title="Limpieza desde extracción",
|
|
started_at=started,
|
|
)
|
|
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
|
|
|
|
|
@app.post("/jobs/from-csv")
|
|
async def run_cleaning_from_csv(
|
|
file: UploadFile = File(...),
|
|
job_id: str | None = Form(None),
|
|
rules_per_column: str | None = Form(None),
|
|
cop_mod_fecha_column: str | None = Form(None),
|
|
cop_mod_bruto_column: str | None = Form(None),
|
|
cop_mod_neto_column: str | None = Form(None),
|
|
cop_mod_config: str | None = Form(None),
|
|
) -> dict[str, Any]:
|
|
content = await file.read()
|
|
if not content:
|
|
raise HTTPException(status_code=400, detail="El archivo CSV está vacío")
|
|
|
|
rules = _parse_json_form(rules_per_column, {})
|
|
fecha_map = _parse_json_form(cop_mod_fecha_column, {})
|
|
bruto_map = _parse_json_form(cop_mod_bruto_column, {})
|
|
neto_map = _parse_json_form(cop_mod_neto_column, {})
|
|
cop_cfg = _parse_json_form(cop_mod_config, None)
|
|
if not isinstance(rules, dict):
|
|
raise HTTPException(status_code=400, detail="rules_per_column debe ser un objeto JSON")
|
|
if not isinstance(fecha_map, dict):
|
|
raise HTTPException(status_code=400, detail="cop_mod_fecha_column debe ser un objeto JSON")
|
|
if not isinstance(bruto_map, dict):
|
|
raise HTTPException(status_code=400, detail="cop_mod_bruto_column debe ser un objeto JSON")
|
|
if not isinstance(neto_map, dict):
|
|
raise HTTPException(status_code=400, detail="cop_mod_neto_column debe ser un objeto JSON")
|
|
|
|
try:
|
|
df, delimiter = load_csv_bytes(content, file.filename or "upload.csv")
|
|
except Exception as exc: # noqa: BLE001
|
|
raise HTTPException(status_code=400, detail=f"No se pudo leer el CSV: {exc}") from exc
|
|
if df.empty:
|
|
raise HTTPException(status_code=400, detail="El CSV no tiene filas")
|
|
|
|
request_dump: dict[str, Any] = {
|
|
"source": "csv",
|
|
"filename": file.filename,
|
|
"delimiter": delimiter,
|
|
"job_id": job_id,
|
|
"rules_per_column": rules,
|
|
"cop_mod_fecha_column": fecha_map,
|
|
"cop_mod_bruto_column": bruto_map,
|
|
"cop_mod_neto_column": neto_map,
|
|
"cop_mod_config": cop_cfg,
|
|
}
|
|
|
|
with session_scope() as session:
|
|
_validate_job_id(session, job_id)
|
|
job = CleaningJob(
|
|
extraction_batch_id=None,
|
|
job_id=job_id,
|
|
source="csv",
|
|
status="running",
|
|
request=request_dump,
|
|
updated_at=utcnow(),
|
|
)
|
|
session.add(job)
|
|
session.flush()
|
|
cleaning_job_id = job.id
|
|
desktop_job_id = job_id
|
|
|
|
try:
|
|
output_rows, log = _run_rules_on_dataframe(
|
|
df, rules, fecha_map, cop_cfg, bruto_map, neto_map
|
|
)
|
|
|
|
with session_scope() as session:
|
|
_persist_cleaned_rows(
|
|
session,
|
|
job_id=cleaning_job_id,
|
|
extraction_batch_id=None,
|
|
output_rows=output_rows,
|
|
)
|
|
job = session.get(CleaningJob, cleaning_job_id)
|
|
if job is None:
|
|
raise RuntimeError("No se encontro el job de limpieza recien creado.")
|
|
job.status = "completed"
|
|
job.log = log
|
|
job.updated_at = utcnow()
|
|
_create_cleaning_task(
|
|
session,
|
|
job_id=desktop_job_id,
|
|
cleaning_job_id=cleaning_job_id,
|
|
source="csv",
|
|
status="completed",
|
|
request=request_dump,
|
|
title=f"Limpieza CSV — {file.filename or 'upload.csv'}",
|
|
started_at=job.created_at,
|
|
)
|
|
session.flush()
|
|
return serialize_job(job)
|
|
except HTTPException:
|
|
raise
|
|
except Exception as exc:
|
|
with session_scope() as session:
|
|
job = session.get(CleaningJob, cleaning_job_id)
|
|
started = job.created_at if job is not None else None
|
|
if job is not None:
|
|
job.status = "failed"
|
|
job.error = str(exc)
|
|
job.updated_at = utcnow()
|
|
_create_cleaning_task(
|
|
session,
|
|
job_id=desktop_job_id,
|
|
cleaning_job_id=cleaning_job_id,
|
|
source="csv",
|
|
status="failed",
|
|
request=request_dump,
|
|
error=str(exc),
|
|
title=f"Limpieza CSV — {file.filename or 'upload.csv'}",
|
|
started_at=started,
|
|
)
|
|
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
|
|
|
|
|
@app.get("/jobs")
|
|
def list_jobs(job_id: str | None = None) -> list[dict[str, Any]]:
|
|
with session_scope() as session:
|
|
stmt = select(CleaningJob).order_by(CleaningJob.created_at.desc())
|
|
if job_id:
|
|
stmt = stmt.where(CleaningJob.job_id == job_id)
|
|
jobs = session.scalars(stmt).all()
|
|
return [serialize_job(job) for job in jobs]
|
|
|
|
|
|
@app.get("/jobs/{job_id}")
|
|
def get_job(job_id: str) -> dict[str, Any]:
|
|
with session_scope() as session:
|
|
job = session.get(CleaningJob, job_id)
|
|
if job is None:
|
|
raise HTTPException(status_code=404, detail="Job de limpieza no encontrado")
|
|
return serialize_job(job)
|
|
|
|
|
|
@app.get("/jobs/{job_id}/rows")
|
|
def get_cleaned_rows(job_id: str, limit: int = 500, offset: int = 0) -> dict[str, Any]:
|
|
with session_scope() as session:
|
|
job = session.get(CleaningJob, job_id)
|
|
if job is None:
|
|
raise HTTPException(status_code=404, detail="Job de limpieza no encontrado")
|
|
rows = session.scalars(
|
|
select(CleanedRow)
|
|
.where(CleanedRow.cleaning_job_id == job_id)
|
|
.order_by(CleanedRow.row_index, CleanedRow.id)
|
|
.offset(offset)
|
|
.limit(limit)
|
|
).all()
|
|
return {
|
|
"job_id": job_id,
|
|
"rows": [row.row_json for row in rows],
|
|
"limit": limit,
|
|
"offset": offset,
|
|
}
|