"""API REST del microservicio de limpieza.""" from __future__ import annotations from contextlib import asynccontextmanager import json from typing import Any import pandas as pd from fastapi import FastAPI, File, Form, HTTPException, UploadFile from pydantic import BaseModel, Field from sqlalchemy import delete, select from domain.csv_cleaning import ( CLEANING_RULES, analyze_dataframe, apply_rules, load_csv_bytes, sample_dataframe_rows, ) from infrastructure.database import ( CleanedRow, CleaningJob, ExtractionBatch, ExtractionRow, Job, Task, init_db, session_scope, utcnow, ) @asynccontextmanager async def lifespan(app: FastAPI): init_db() yield app = FastAPI( title="IPSExtractor Cleaning Service", version="1.0.0", lifespan=lifespan, ) class CleaningRequest(BaseModel): extraction_batch_id: str | None = None job_id: str | None = None rules_per_column: dict[str, list[str]] = Field(default_factory=dict) cop_mod_fecha_column: dict[str, str] = Field(default_factory=dict) cop_mod_bruto_column: dict[str, str] = Field(default_factory=dict) cop_mod_neto_column: dict[str, str] = Field(default_factory=dict) cop_mod_config: dict[str, Any] | None = None class PreviewRequest(BaseModel): rows: list[dict[str, Any]] = Field(default_factory=list) extraction_batch_id: str | None = None rules_per_column: dict[str, list[str]] = Field(default_factory=dict) cop_mod_fecha_column: dict[str, str] = Field(default_factory=dict) cop_mod_bruto_column: dict[str, str] = Field(default_factory=dict) cop_mod_neto_column: dict[str, str] = Field(default_factory=dict) cop_mod_config: dict[str, Any] | None = None sample_size: int = 8 def dataframe_to_records(df: pd.DataFrame) -> list[dict[str, Any]]: return json.loads(df.to_json(orient="records", date_format="iso")) def serialize_job(job: CleaningJob) -> dict[str, Any]: # started/finished derivados: created_at = inicio; updated_at = fin si ya terminó finished = ( job.updated_at.isoformat() if job.status in {"completed", "failed"} else None ) return { "id": job.id, "extraction_batch_id": job.extraction_batch_id, "job_id": job.job_id, "source": job.source, "status": job.status, "request": job.request, "log": job.log, "error": job.error, "created_at": job.created_at.isoformat(), "updated_at": job.updated_at.isoformat(), "started_at": job.created_at.isoformat(), "finished_at": finished, } def _validate_job_id(session: Any, job_id: str | None) -> None: if not job_id: return if session.get(Job, job_id) is None: raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado") def _create_cleaning_task( session: Any, *, job_id: str | None, cleaning_job_id: str, source: str, status: str, request: dict[str, Any], error: str | None = None, title: str | None = None, started_at: Any | None = None, ) -> str | None: """Crea Task(type=cleaning) vinculada al job Desktop. Devuelve task.id o None.""" if not job_id: return None now = utcnow() task = Task( job_id=job_id, type="cleaning", status=status, title=title or f"Limpieza ({source})", request=request, result={"cleaning_job_id": cleaning_job_id}, error=error, started_at=started_at or now, finished_at=now, updated_at=now, ) session.add(task) session.flush() return task.id def _persist_cleaned_rows( session: Any, *, job_id: str, extraction_batch_id: str | None, output_rows: list[dict[str, Any]], ) -> None: session.execute(delete(CleanedRow).where(CleanedRow.cleaning_job_id == job_id)) for idx, row in enumerate(output_rows, start=1): session.add( CleanedRow( cleaning_job_id=job_id, extraction_batch_id=extraction_batch_id, row_index=idx, row_json=row, ) ) def _run_rules_on_dataframe( df: pd.DataFrame, rules_per_column: dict[str, list[str]], cop_mod_fecha_column: dict[str, str], cop_mod_config: dict[str, Any] | None, cop_mod_bruto_column: dict[str, str] | None = None, cop_mod_neto_column: dict[str, str] | None = None, ) -> tuple[list[dict[str, Any]], list[str]]: result_df, log = apply_rules( df, rules_per_column, cop_mod_fecha_column=cop_mod_fecha_column, cop_mod_config=cop_mod_config, cop_mod_bruto_column=cop_mod_bruto_column, cop_mod_neto_column=cop_mod_neto_column, ) return dataframe_to_records(result_df), log def _parse_json_form(raw: str | None, default: Any) -> Any: if not raw or not raw.strip(): return default try: return json.loads(raw) except json.JSONDecodeError as exc: raise HTTPException(status_code=400, detail=f"JSON inválido: {exc}") from exc def _analysis_payload(df: pd.DataFrame, **extra: Any) -> dict[str, Any]: return { **extra, "row_count": len(df), "columns": [summary.__dict__ for summary in analyze_dataframe(df)], "sample_rows": sample_dataframe_rows(df, 8), } def _cell_str(value: Any) -> str: if value is None: return "" text = str(value) if text in ("None", "nan", "NaT"): return "" if len(text) > 80: return text[:77] + "..." return text def _preview_from_dataframe( df: pd.DataFrame, rules_per_column: dict[str, list[str]], cop_mod_fecha_column: dict[str, str], cop_mod_config: dict[str, Any] | None, cop_mod_bruto_column: dict[str, str] | None = None, cop_mod_neto_column: dict[str, str] | None = None, ) -> dict[str, Any]: rows_before = dataframe_to_records(df) result_df, _log = apply_rules( df, rules_per_column, cop_mod_fecha_column=cop_mod_fecha_column, cop_mod_config=cop_mod_config, cop_mod_bruto_column=cop_mod_bruto_column, cop_mod_neto_column=cop_mod_neto_column, ) rows_after = dataframe_to_records(result_df) # Emparejar por índice de fila de muestra (misma posición) n = min(len(rows_before), len(rows_after)) all_cols = list(dict.fromkeys([*df.columns.astype(str), *result_df.columns.astype(str)])) columns_preview: list[dict[str, Any]] = [] for col in all_cols: before_vals = [_cell_str(rows_before[i].get(col)) for i in range(n)] after_vals = [_cell_str(rows_after[i].get(col)) for i in range(n)] columns_preview.append({ "name": col, "before": before_vals, "after": after_vals, "changed": before_vals != after_vals, }) return { "rows_before": rows_before, "rows_after": rows_after, "columns": columns_preview, } @app.get("/health") def health() -> dict[str, str]: return {"status": "ok"} @app.get("/rules") def list_rules() -> list[dict[str, Any]]: return [ { "id": rule["id"], "name": rule["name"], "description": rule.get("description", ""), } for rule in CLEANING_RULES ] @app.get("/batches/{batch_id}/analysis") def analyze_batch(batch_id: str) -> dict[str, Any]: with session_scope() as session: batch = session.get(ExtractionBatch, batch_id) if batch is None: raise HTTPException(status_code=404, detail="Lote de extraccion no encontrado") rows = session.execute( select(ExtractionRow.row_json) .where(ExtractionRow.batch_id == batch_id) .order_by(ExtractionRow.row_index, ExtractionRow.id) ).all() records = [row[0] for row in rows] df = pd.DataFrame(records) return _analysis_payload(df, batch_id=batch_id) @app.post("/analyze-csv") async def analyze_csv(file: UploadFile = File(...)) -> dict[str, Any]: content = await file.read() if not content: raise HTTPException(status_code=400, detail="El archivo CSV está vacío") try: df, delimiter = load_csv_bytes(content, file.filename or "upload.csv") except Exception as exc: # noqa: BLE001 raise HTTPException(status_code=400, detail=f"No se pudo leer el CSV: {exc}") from exc if df.empty: raise HTTPException(status_code=400, detail="El CSV no tiene filas") return _analysis_payload(df, filename=file.filename, delimiter=delimiter) @app.post("/preview") def preview_cleaning(request: PreviewRequest) -> dict[str, Any]: """Aplica reglas sobre una muestra pequeña y devuelve Antes/Después.""" records: list[dict[str, Any]] = list(request.rows or []) if not records and request.extraction_batch_id: with session_scope() as session: batch = session.get(ExtractionBatch, request.extraction_batch_id) if batch is None: raise HTTPException(status_code=404, detail="Lote de extraccion no encontrado") rows = session.execute( select(ExtractionRow.row_json) .where(ExtractionRow.batch_id == request.extraction_batch_id) .order_by(ExtractionRow.row_index, ExtractionRow.id) ).all() all_records = [row[0] for row in rows] if not all_records: raise HTTPException(status_code=400, detail="El lote no tiene filas") df_full = pd.DataFrame(all_records) records = sample_dataframe_rows(df_full, max(1, min(request.sample_size, 20))) if not records: raise HTTPException( status_code=400, detail="Se requieren rows o extraction_batch_id con filas", ) df = pd.DataFrame(records) try: return _preview_from_dataframe( df, request.rules_per_column, request.cop_mod_fecha_column, request.cop_mod_config, request.cop_mod_bruto_column, request.cop_mod_neto_column, ) except Exception as exc: # noqa: BLE001 raise HTTPException(status_code=500, detail=str(exc)) from exc @app.post("/jobs") def run_cleaning(request: CleaningRequest) -> dict[str, Any]: if not request.extraction_batch_id: raise HTTPException( status_code=400, detail="extraction_batch_id es requerido (use /jobs/from-csv para CSV)", ) with session_scope() as session: batch = session.get(ExtractionBatch, request.extraction_batch_id) if batch is None: raise HTTPException(status_code=404, detail="Lote de extraccion no encontrado") _validate_job_id(session, request.job_id) extracted_rows = session.execute( select(ExtractionRow.row_json) .where(ExtractionRow.batch_id == request.extraction_batch_id) .order_by(ExtractionRow.row_index, ExtractionRow.id) ).all() records = [row[0] for row in extracted_rows] if not records: raise HTTPException(status_code=400, detail="El lote no tiene filas extraidas") job = CleaningJob( extraction_batch_id=request.extraction_batch_id, job_id=request.job_id, source="extraction", status="running", request=request.model_dump(), updated_at=utcnow(), ) session.add(job) session.flush() job_id = job.id batch_id = request.extraction_batch_id desktop_job_id = request.job_id request_dump = request.model_dump() try: df = pd.DataFrame(records) output_rows, log = _run_rules_on_dataframe( df, request.rules_per_column, request.cop_mod_fecha_column, request.cop_mod_config, request.cop_mod_bruto_column, request.cop_mod_neto_column, ) with session_scope() as session: _persist_cleaned_rows( session, job_id=job_id, extraction_batch_id=batch_id, output_rows=output_rows, ) job = session.get(CleaningJob, job_id) if job is None: raise RuntimeError("No se encontro el job de limpieza recien creado.") job.status = "completed" job.log = log job.updated_at = utcnow() _create_cleaning_task( session, job_id=desktop_job_id, cleaning_job_id=job_id, source="extraction", status="completed", request=request_dump, title="Limpieza desde extracción", started_at=job.created_at, ) session.flush() return serialize_job(job) except HTTPException: raise except Exception as exc: with session_scope() as session: job = session.get(CleaningJob, job_id) started = job.created_at if job is not None else None if job is not None: job.status = "failed" job.error = str(exc) job.updated_at = utcnow() _create_cleaning_task( session, job_id=desktop_job_id, cleaning_job_id=job_id, source="extraction", status="failed", request=request_dump, error=str(exc), title="Limpieza desde extracción", started_at=started, ) raise HTTPException(status_code=500, detail=str(exc)) from exc @app.post("/jobs/from-csv") async def run_cleaning_from_csv( file: UploadFile = File(...), job_id: str | None = Form(None), rules_per_column: str | None = Form(None), cop_mod_fecha_column: str | None = Form(None), cop_mod_bruto_column: str | None = Form(None), cop_mod_neto_column: str | None = Form(None), cop_mod_config: str | None = Form(None), ) -> dict[str, Any]: content = await file.read() if not content: raise HTTPException(status_code=400, detail="El archivo CSV está vacío") rules = _parse_json_form(rules_per_column, {}) fecha_map = _parse_json_form(cop_mod_fecha_column, {}) bruto_map = _parse_json_form(cop_mod_bruto_column, {}) neto_map = _parse_json_form(cop_mod_neto_column, {}) cop_cfg = _parse_json_form(cop_mod_config, None) if not isinstance(rules, dict): raise HTTPException(status_code=400, detail="rules_per_column debe ser un objeto JSON") if not isinstance(fecha_map, dict): raise HTTPException(status_code=400, detail="cop_mod_fecha_column debe ser un objeto JSON") if not isinstance(bruto_map, dict): raise HTTPException(status_code=400, detail="cop_mod_bruto_column debe ser un objeto JSON") if not isinstance(neto_map, dict): raise HTTPException(status_code=400, detail="cop_mod_neto_column debe ser un objeto JSON") try: df, delimiter = load_csv_bytes(content, file.filename or "upload.csv") except Exception as exc: # noqa: BLE001 raise HTTPException(status_code=400, detail=f"No se pudo leer el CSV: {exc}") from exc if df.empty: raise HTTPException(status_code=400, detail="El CSV no tiene filas") request_dump: dict[str, Any] = { "source": "csv", "filename": file.filename, "delimiter": delimiter, "job_id": job_id, "rules_per_column": rules, "cop_mod_fecha_column": fecha_map, "cop_mod_bruto_column": bruto_map, "cop_mod_neto_column": neto_map, "cop_mod_config": cop_cfg, } with session_scope() as session: _validate_job_id(session, job_id) job = CleaningJob( extraction_batch_id=None, job_id=job_id, source="csv", status="running", request=request_dump, updated_at=utcnow(), ) session.add(job) session.flush() cleaning_job_id = job.id desktop_job_id = job_id try: output_rows, log = _run_rules_on_dataframe( df, rules, fecha_map, cop_cfg, bruto_map, neto_map ) with session_scope() as session: _persist_cleaned_rows( session, job_id=cleaning_job_id, extraction_batch_id=None, output_rows=output_rows, ) job = session.get(CleaningJob, cleaning_job_id) if job is None: raise RuntimeError("No se encontro el job de limpieza recien creado.") job.status = "completed" job.log = log job.updated_at = utcnow() _create_cleaning_task( session, job_id=desktop_job_id, cleaning_job_id=cleaning_job_id, source="csv", status="completed", request=request_dump, title=f"Limpieza CSV — {file.filename or 'upload.csv'}", started_at=job.created_at, ) session.flush() return serialize_job(job) except HTTPException: raise except Exception as exc: with session_scope() as session: job = session.get(CleaningJob, cleaning_job_id) started = job.created_at if job is not None else None if job is not None: job.status = "failed" job.error = str(exc) job.updated_at = utcnow() _create_cleaning_task( session, job_id=desktop_job_id, cleaning_job_id=cleaning_job_id, source="csv", status="failed", request=request_dump, error=str(exc), title=f"Limpieza CSV — {file.filename or 'upload.csv'}", started_at=started, ) raise HTTPException(status_code=500, detail=str(exc)) from exc @app.get("/jobs") def list_jobs(job_id: str | None = None) -> list[dict[str, Any]]: with session_scope() as session: stmt = select(CleaningJob).order_by(CleaningJob.created_at.desc()) if job_id: stmt = stmt.where(CleaningJob.job_id == job_id) jobs = session.scalars(stmt).all() return [serialize_job(job) for job in jobs] @app.get("/jobs/{job_id}") def get_job(job_id: str) -> dict[str, Any]: with session_scope() as session: job = session.get(CleaningJob, job_id) if job is None: raise HTTPException(status_code=404, detail="Job de limpieza no encontrado") return serialize_job(job) @app.get("/jobs/{job_id}/rows") def get_cleaned_rows(job_id: str, limit: int = 500, offset: int = 0) -> dict[str, Any]: with session_scope() as session: job = session.get(CleaningJob, job_id) if job is None: raise HTTPException(status_code=404, detail="Job de limpieza no encontrado") rows = session.scalars( select(CleanedRow) .where(CleanedRow.cleaning_job_id == job_id) .order_by(CleanedRow.row_index, CleanedRow.id) .offset(offset) .limit(limit) ).all() return { "job_id": job_id, "rows": [row.row_json for row in rows], "limit": limit, "offset": offset, }