feat: Add database migration service with Alembic for PostgreSQL, integrating job management and counting functionalities. Update Docker Compose to include db-migrations service, enhancing the microservices architecture. Revise documentation for clarity on database schema and migration processes.
This commit is contained in:
erik
@@ -0,0 +1,124 @@
|
||||
"""Schema inicial: extraction_batches, tasks, extraction_rows, cleaning_jobs, cleaned_rows.
|
||||
|
||||
Revision ID: 0001
|
||||
Revises:
|
||||
Create Date: 2026-07-05
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0001"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
inspector = sa.inspect(conn)
|
||||
existing = set(inspector.get_table_names())
|
||||
|
||||
if "extraction_batches" not in existing:
|
||||
op.create_table(
|
||||
"extraction_batches",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("root_path", sa.Text, nullable=True),
|
||||
sa.Column("request", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("error", sa.Text, nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_extraction_batches_status", "extraction_batches", ["status"])
|
||||
|
||||
if "tasks" not in existing:
|
||||
op.create_table(
|
||||
"tasks",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("type", sa.String(64), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("title", sa.Text, nullable=True),
|
||||
sa.Column("request", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("result", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("error", sa.Text, nullable=True),
|
||||
sa.Column("owner_id", sa.String(36), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
op.create_index("ix_tasks_type", "tasks", ["type"])
|
||||
op.create_index("ix_tasks_status", "tasks", ["status"])
|
||||
op.create_index("ix_tasks_owner_id", "tasks", ["owner_id"])
|
||||
|
||||
if "extraction_rows" not in existing:
|
||||
op.create_table(
|
||||
"extraction_rows",
|
||||
sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
|
||||
sa.Column(
|
||||
"batch_id", sa.String(36),
|
||||
sa.ForeignKey("extraction_batches.id"), nullable=False,
|
||||
),
|
||||
sa.Column("radicado", sa.String(255), nullable=True),
|
||||
sa.Column("source_pdf", sa.Text, nullable=True),
|
||||
sa.Column("row_index", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("row_json", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("raw_json", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_extraction_rows_batch_id", "extraction_rows", ["batch_id"])
|
||||
op.create_index("ix_extraction_rows_radicado", "extraction_rows", ["radicado"])
|
||||
|
||||
if "cleaning_jobs" not in existing:
|
||||
op.create_table(
|
||||
"cleaning_jobs",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column(
|
||||
"extraction_batch_id", sa.String(36),
|
||||
sa.ForeignKey("extraction_batches.id"), nullable=False,
|
||||
),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("request", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("log", sa.JSON, nullable=False, server_default="[]"),
|
||||
sa.Column("error", sa.Text, nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_cleaning_jobs_extraction_batch_id", "cleaning_jobs", ["extraction_batch_id"]
|
||||
)
|
||||
op.create_index("ix_cleaning_jobs_status", "cleaning_jobs", ["status"])
|
||||
|
||||
if "cleaned_rows" not in existing:
|
||||
op.create_table(
|
||||
"cleaned_rows",
|
||||
sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
|
||||
sa.Column(
|
||||
"cleaning_job_id", sa.String(36),
|
||||
sa.ForeignKey("cleaning_jobs.id"), nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"extraction_batch_id", sa.String(36),
|
||||
sa.ForeignKey("extraction_batches.id"), nullable=False,
|
||||
),
|
||||
sa.Column("row_index", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("row_json", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_cleaned_rows_cleaning_job_id", "cleaned_rows", ["cleaning_job_id"])
|
||||
op.create_index(
|
||||
"ix_cleaned_rows_extraction_batch_id", "cleaned_rows", ["extraction_batch_id"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("cleaned_rows")
|
||||
op.drop_table("cleaning_jobs")
|
||||
op.drop_table("extraction_rows")
|
||||
op.drop_table("tasks")
|
||||
op.drop_table("extraction_batches")
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Agregar tabla jobs, count_job_files, job_id en tasks, job_module_id en extraction_batches.
|
||||
|
||||
Revision ID: 0002
|
||||
Revises: 0001
|
||||
Create Date: 2026-07-05
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0002"
|
||||
down_revision: Union[str, None] = "0001"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def _col_exists(conn: sa.engine.Connection, table: str, column: str) -> bool:
|
||||
inspector = sa.inspect(conn)
|
||||
return any(c["name"] == column for c in inspector.get_columns(table))
|
||||
|
||||
|
||||
def _table_exists(conn: sa.engine.Connection, table: str) -> bool:
|
||||
inspector = sa.inspect(conn)
|
||||
return table in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# ── tabla jobs ──────────────────────────────────────────────────────────
|
||||
if not _table_exists(conn, "jobs"):
|
||||
op.create_table(
|
||||
"jobs",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("title", sa.Text, nullable=True),
|
||||
sa.Column("metadata", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_jobs_status", "jobs", ["status"])
|
||||
|
||||
# ── columna job_id en tasks ──────────────────────────────────────────────
|
||||
if not _col_exists(conn, "tasks", "job_id"):
|
||||
op.add_column(
|
||||
"tasks",
|
||||
sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True),
|
||||
)
|
||||
op.create_index("ix_tasks_job_id", "tasks", ["job_id"])
|
||||
|
||||
# ── tabla count_job_files ────────────────────────────────────────────────
|
||||
if not _table_exists(conn, "count_job_files"):
|
||||
op.create_table(
|
||||
"count_job_files",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column(
|
||||
"job_module_id", sa.String(36),
|
||||
sa.ForeignKey("tasks.id"), nullable=False,
|
||||
),
|
||||
sa.Column("radicado", sa.String(255), nullable=True),
|
||||
sa.Column("filename", sa.String(512), nullable=False),
|
||||
sa.Column("file_path", sa.Text, nullable=True),
|
||||
sa.Column("doc_type", sa.String(128), nullable=True),
|
||||
sa.Column("service_count", sa.Integer, nullable=True),
|
||||
sa.Column("services", sa.JSON, nullable=False, server_default="[]"),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="ok"),
|
||||
sa.Column("is_valid", sa.Boolean, nullable=True),
|
||||
sa.Column("error", sa.Text, nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_count_job_files_job_module_id", "count_job_files", ["job_module_id"])
|
||||
op.create_index("ix_count_job_files_radicado", "count_job_files", ["radicado"])
|
||||
op.create_index("ix_count_job_files_status", "count_job_files", ["status"])
|
||||
|
||||
# ── columna job_module_id en extraction_batches ──────────────────────────
|
||||
if not _col_exists(conn, "extraction_batches", "job_module_id"):
|
||||
op.add_column(
|
||||
"extraction_batches",
|
||||
sa.Column(
|
||||
"job_module_id", sa.String(36),
|
||||
sa.ForeignKey("tasks.id"), nullable=True,
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_extraction_batches_job_module_id", "extraction_batches", ["job_module_id"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if _col_exists(conn, "extraction_batches", "job_module_id"):
|
||||
op.drop_index("ix_extraction_batches_job_module_id", "extraction_batches")
|
||||
op.drop_column("extraction_batches", "job_module_id")
|
||||
if _table_exists(conn, "count_job_files"):
|
||||
op.drop_table("count_job_files")
|
||||
if _col_exists(conn, "tasks", "job_id"):
|
||||
op.drop_index("ix_tasks_job_id", "tasks")
|
||||
op.drop_column("tasks", "job_id")
|
||||
if _table_exists(conn, "jobs"):
|
||||
op.drop_table("jobs")
|
||||
Reference in New Issue
Block a user