feat: Add database migration service with Alembic for PostgreSQL, integrating job management and counting functionalities. Update Docker Compose to include db-migrations service, enhancing the microservices architecture. Revise documentation for clarity on database schema and migration processes.
This commit is contained in:
erik
@@ -0,0 +1,10 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
CMD ["python", "-m", "run_migrations"]
|
||||
@@ -0,0 +1,66 @@
|
||||
# db (migraciones)
|
||||
|
||||
Contenedor de un solo uso que aplica las migraciones de base de datos con Alembic.
|
||||
Se ejecuta al arrancar el stack y termina con código 0 al completar. El orquestador Docker reinicia en caso de error.
|
||||
|
||||
---
|
||||
|
||||
## Comportamiento
|
||||
|
||||
1. Conecta a PostgreSQL.
|
||||
2. Adquiere un advisory lock para evitar ejecuciones concurrentes.
|
||||
3. Aplica todas las migraciones pendientes.
|
||||
4. Termina.
|
||||
|
||||
Los demás servicios tienen `depends_on: db-migrations: condition: service_completed_successfully`, por lo que no arrancan hasta que las migraciones terminen.
|
||||
|
||||
---
|
||||
|
||||
## Variables de entorno
|
||||
|
||||
| Variable | Descripción |
|
||||
|----------|-------------|
|
||||
| `DATABASE_URL` | PostgreSQL (`postgresql+psycopg://...`) |
|
||||
|
||||
---
|
||||
|
||||
## Esquema de base de datos
|
||||
|
||||
```
|
||||
jobs — contenedor de un trabajo del usuario (multimódulo)
|
||||
tasks — ejecución de un módulo dentro de un job
|
||||
type: counting | extraction | cleaning
|
||||
count_job_files — resultado de conteo por PDF (job_module_id → tasks.id)
|
||||
extraction_batches — lote de extracción (job_module_id → tasks.id)
|
||||
extraction_rows — fila extraída (batch_id → extraction_batches.id)
|
||||
cleaning_jobs — job de limpieza (task_id → tasks.id)
|
||||
cleaned_rows — fila limpiada (cleaning_job_id → cleaning_jobs.id)
|
||||
```
|
||||
|
||||
Documentación completa en [`docs/database.dbml`](../../docs/database.dbml).
|
||||
|
||||
---
|
||||
|
||||
## Estructura del código
|
||||
|
||||
```
|
||||
Dockerfile — imagen Python slim + alembic + psycopg
|
||||
alembic.ini — configuración de Alembic
|
||||
alembic/
|
||||
env.py — entorno de migraciones con advisory lock
|
||||
versions/
|
||||
0001_initial_schema.py
|
||||
0002_add_jobs_and_count.py
|
||||
run_migrations.py — script principal con manejo de errores
|
||||
requirements.txt
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Añadir una migración
|
||||
|
||||
```bash
|
||||
# Generar una nueva revisión
|
||||
docker run --rm -e DATABASE_URL=... ipsextractor-db-migrations \
|
||||
alembic revision --autogenerate -m "descripcion"
|
||||
```
|
||||
@@ -0,0 +1,39 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
version_path_separator = os
|
||||
sqlalchemy.url = postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -0,0 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
config = context.config
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
database_url = os.environ.get("DATABASE_URL") or config.get_main_option("sqlalchemy.url")
|
||||
if database_url:
|
||||
config.set_main_option("sqlalchemy.url", database_url)
|
||||
|
||||
target_metadata = None
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,27 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Schema inicial: extraction_batches, tasks, extraction_rows, cleaning_jobs, cleaned_rows.
|
||||
|
||||
Revision ID: 0001
|
||||
Revises:
|
||||
Create Date: 2026-07-05
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0001"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
inspector = sa.inspect(conn)
|
||||
existing = set(inspector.get_table_names())
|
||||
|
||||
if "extraction_batches" not in existing:
|
||||
op.create_table(
|
||||
"extraction_batches",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("root_path", sa.Text, nullable=True),
|
||||
sa.Column("request", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("error", sa.Text, nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_extraction_batches_status", "extraction_batches", ["status"])
|
||||
|
||||
if "tasks" not in existing:
|
||||
op.create_table(
|
||||
"tasks",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("type", sa.String(64), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("title", sa.Text, nullable=True),
|
||||
sa.Column("request", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("result", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("error", sa.Text, nullable=True),
|
||||
sa.Column("owner_id", sa.String(36), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
op.create_index("ix_tasks_type", "tasks", ["type"])
|
||||
op.create_index("ix_tasks_status", "tasks", ["status"])
|
||||
op.create_index("ix_tasks_owner_id", "tasks", ["owner_id"])
|
||||
|
||||
if "extraction_rows" not in existing:
|
||||
op.create_table(
|
||||
"extraction_rows",
|
||||
sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
|
||||
sa.Column(
|
||||
"batch_id", sa.String(36),
|
||||
sa.ForeignKey("extraction_batches.id"), nullable=False,
|
||||
),
|
||||
sa.Column("radicado", sa.String(255), nullable=True),
|
||||
sa.Column("source_pdf", sa.Text, nullable=True),
|
||||
sa.Column("row_index", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("row_json", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("raw_json", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_extraction_rows_batch_id", "extraction_rows", ["batch_id"])
|
||||
op.create_index("ix_extraction_rows_radicado", "extraction_rows", ["radicado"])
|
||||
|
||||
if "cleaning_jobs" not in existing:
|
||||
op.create_table(
|
||||
"cleaning_jobs",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column(
|
||||
"extraction_batch_id", sa.String(36),
|
||||
sa.ForeignKey("extraction_batches.id"), nullable=False,
|
||||
),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("request", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("log", sa.JSON, nullable=False, server_default="[]"),
|
||||
sa.Column("error", sa.Text, nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_cleaning_jobs_extraction_batch_id", "cleaning_jobs", ["extraction_batch_id"]
|
||||
)
|
||||
op.create_index("ix_cleaning_jobs_status", "cleaning_jobs", ["status"])
|
||||
|
||||
if "cleaned_rows" not in existing:
|
||||
op.create_table(
|
||||
"cleaned_rows",
|
||||
sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
|
||||
sa.Column(
|
||||
"cleaning_job_id", sa.String(36),
|
||||
sa.ForeignKey("cleaning_jobs.id"), nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"extraction_batch_id", sa.String(36),
|
||||
sa.ForeignKey("extraction_batches.id"), nullable=False,
|
||||
),
|
||||
sa.Column("row_index", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("row_json", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_cleaned_rows_cleaning_job_id", "cleaned_rows", ["cleaning_job_id"])
|
||||
op.create_index(
|
||||
"ix_cleaned_rows_extraction_batch_id", "cleaned_rows", ["extraction_batch_id"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("cleaned_rows")
|
||||
op.drop_table("cleaning_jobs")
|
||||
op.drop_table("extraction_rows")
|
||||
op.drop_table("tasks")
|
||||
op.drop_table("extraction_batches")
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Agregar tabla jobs, count_job_files, job_id en tasks, job_module_id en extraction_batches.
|
||||
|
||||
Revision ID: 0002
|
||||
Revises: 0001
|
||||
Create Date: 2026-07-05
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0002"
|
||||
down_revision: Union[str, None] = "0001"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def _col_exists(conn: sa.engine.Connection, table: str, column: str) -> bool:
|
||||
inspector = sa.inspect(conn)
|
||||
return any(c["name"] == column for c in inspector.get_columns(table))
|
||||
|
||||
|
||||
def _table_exists(conn: sa.engine.Connection, table: str) -> bool:
|
||||
inspector = sa.inspect(conn)
|
||||
return table in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# ── tabla jobs ──────────────────────────────────────────────────────────
|
||||
if not _table_exists(conn, "jobs"):
|
||||
op.create_table(
|
||||
"jobs",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("title", sa.Text, nullable=True),
|
||||
sa.Column("metadata", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_jobs_status", "jobs", ["status"])
|
||||
|
||||
# ── columna job_id en tasks ──────────────────────────────────────────────
|
||||
if not _col_exists(conn, "tasks", "job_id"):
|
||||
op.add_column(
|
||||
"tasks",
|
||||
sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True),
|
||||
)
|
||||
op.create_index("ix_tasks_job_id", "tasks", ["job_id"])
|
||||
|
||||
# ── tabla count_job_files ────────────────────────────────────────────────
|
||||
if not _table_exists(conn, "count_job_files"):
|
||||
op.create_table(
|
||||
"count_job_files",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column(
|
||||
"job_module_id", sa.String(36),
|
||||
sa.ForeignKey("tasks.id"), nullable=False,
|
||||
),
|
||||
sa.Column("radicado", sa.String(255), nullable=True),
|
||||
sa.Column("filename", sa.String(512), nullable=False),
|
||||
sa.Column("file_path", sa.Text, nullable=True),
|
||||
sa.Column("doc_type", sa.String(128), nullable=True),
|
||||
sa.Column("service_count", sa.Integer, nullable=True),
|
||||
sa.Column("services", sa.JSON, nullable=False, server_default="[]"),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="ok"),
|
||||
sa.Column("is_valid", sa.Boolean, nullable=True),
|
||||
sa.Column("error", sa.Text, nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_count_job_files_job_module_id", "count_job_files", ["job_module_id"])
|
||||
op.create_index("ix_count_job_files_radicado", "count_job_files", ["radicado"])
|
||||
op.create_index("ix_count_job_files_status", "count_job_files", ["status"])
|
||||
|
||||
# ── columna job_module_id en extraction_batches ──────────────────────────
|
||||
if not _col_exists(conn, "extraction_batches", "job_module_id"):
|
||||
op.add_column(
|
||||
"extraction_batches",
|
||||
sa.Column(
|
||||
"job_module_id", sa.String(36),
|
||||
sa.ForeignKey("tasks.id"), nullable=True,
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_extraction_batches_job_module_id", "extraction_batches", ["job_module_id"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if _col_exists(conn, "extraction_batches", "job_module_id"):
|
||||
op.drop_index("ix_extraction_batches_job_module_id", "extraction_batches")
|
||||
op.drop_column("extraction_batches", "job_module_id")
|
||||
if _table_exists(conn, "count_job_files"):
|
||||
op.drop_table("count_job_files")
|
||||
if _col_exists(conn, "tasks", "job_id"):
|
||||
op.drop_index("ix_tasks_job_id", "tasks")
|
||||
op.drop_column("tasks", "job_id")
|
||||
if _table_exists(conn, "jobs"):
|
||||
op.drop_table("jobs")
|
||||
@@ -0,0 +1,3 @@
|
||||
alembic==1.16.2
|
||||
psycopg[binary]==3.2.3
|
||||
SQLAlchemy==2.0.36
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Punto de entrada del servicio de migraciones.
|
||||
|
||||
Se ejecuta como init container antes de que arranquen los demás servicios.
|
||||
Aplica todas las migraciones pendientes hasta 'head' con bloqueo advisory
|
||||
para evitar condiciones de carrera en despliegues con múltiples réplicas.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
from alembic.config import Config as AlembicConfig
|
||||
from alembic import command as alembic_command
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DATABASE_URL = os.environ.get(
|
||||
"DATABASE_URL",
|
||||
"postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor",
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
log.info("Conectando a la base de datos…")
|
||||
engine = create_engine(DATABASE_URL, pool_pre_ping=True)
|
||||
|
||||
alembic_cfg = AlembicConfig()
|
||||
alembic_cfg.set_main_option("script_location", "alembic")
|
||||
alembic_cfg.set_main_option("sqlalchemy.url", DATABASE_URL)
|
||||
|
||||
if DATABASE_URL.startswith("postgresql"):
|
||||
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
|
||||
conn.execute(text("SELECT pg_advisory_lock(74512001)"))
|
||||
log.info("Bloqueo advisory adquirido — ejecutando migraciones…")
|
||||
try:
|
||||
alembic_command.upgrade(alembic_cfg, "head")
|
||||
log.info("Migraciones completadas.")
|
||||
finally:
|
||||
conn.execute(text("SELECT pg_advisory_unlock(74512001)"))
|
||||
else:
|
||||
alembic_command.upgrade(alembic_cfg, "head")
|
||||
log.info("Migraciones completadas.")
|
||||
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
log.error("Migraciones fallidas: %s", exc)
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user