feat: Enhance CSV cleaning service with new endpoints for CSV analysis and preview functionality. Introduce support for job management linked to extraction batches, and update the cleaning rules structure. Add new utility functions for handling CSV data and improve documentation for clarity on usage and available endpoints.

2026-07-11 00:27:05 -05:00
parent cf0ebc72ba
commit b770761832
73 changed files with 7614 additions and 2671 deletions
@@ -0,0 +1,90 @@
+"""cleaning_jobs: batch_id nullable, job_id, source; cleaned_rows.batch_id nullable.
+
+Revision ID: 0003
+Revises: 0002
+Create Date: 2026-07-10
+
+"""
+from __future__ import annotations
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0003"
+down_revision: Union[str, None] = "0002"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _col_exists(conn: sa.engine.Connection, table: str, column: str) -> bool:
+    inspector = sa.inspect(conn)
+    return any(c["name"] == column for c in inspector.get_columns(table))
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+
+    if _col_exists(conn, "cleaning_jobs", "extraction_batch_id"):
+        op.alter_column(
+            "cleaning_jobs",
+            "extraction_batch_id",
+            existing_type=sa.String(36),
+            nullable=True,
+        )
+
+    if not _col_exists(conn, "cleaning_jobs", "job_id"):
+        op.add_column(
+            "cleaning_jobs",
+            sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True),
+        )
+        op.create_index("ix_cleaning_jobs_job_id", "cleaning_jobs", ["job_id"])
+
+    if not _col_exists(conn, "cleaning_jobs", "source"):
+        op.add_column(
+            "cleaning_jobs",
+            sa.Column(
+                "source",
+                sa.String(32),
+                nullable=False,
+                server_default="extraction",
+            ),
+        )
+        op.create_index("ix_cleaning_jobs_source", "cleaning_jobs", ["source"])
+
+    if _col_exists(conn, "cleaned_rows", "extraction_batch_id"):
+        op.alter_column(
+            "cleaned_rows",
+            "extraction_batch_id",
+            existing_type=sa.String(36),
+            nullable=True,
+        )
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+
+    if _col_exists(conn, "cleaned_rows", "extraction_batch_id"):
+        op.alter_column(
+            "cleaned_rows",
+            "extraction_batch_id",
+            existing_type=sa.String(36),
+            nullable=False,
+        )
+
+    if _col_exists(conn, "cleaning_jobs", "source"):
+        op.drop_index("ix_cleaning_jobs_source", table_name="cleaning_jobs")
+        op.drop_column("cleaning_jobs", "source")
+
+    if _col_exists(conn, "cleaning_jobs", "job_id"):
+        op.drop_index("ix_cleaning_jobs_job_id", table_name="cleaning_jobs")
+        op.drop_column("cleaning_jobs", "job_id")
+
+    if _col_exists(conn, "cleaning_jobs", "extraction_batch_id"):
+        op.alter_column(
+            "cleaning_jobs",
+            "extraction_batch_id",
+            existing_type=sa.String(36),
+            nullable=False,
+        )