feat: Introduce bot-digitador service for automated data entry in Integración Total. Add CLI and GUI interfaces for CSV processing, implement auto-start functionality on Windows, and enhance README with setup instructions and usage details. Include necessary configurations in docker-compose for service integration.

2026-08-09 21:26:56 -05:00
parent a3b4c7fc47
commit e9b766281a
74 changed files with 11177 additions and 115 deletions
@@ -0,0 +1,83 @@
+"""Download jobs: parent/shard fields for distributed bots.
+
+Revision ID: 0007
+Revises: 0006
+Create Date: 2026-08-09
+
+"""
+from __future__ import annotations
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0007"
+down_revision: Union[str, None] = "0006"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    if not inspector.has_table("download_jobs"):
+        return
+    cols = {c["name"] for c in inspector.get_columns("download_jobs")}
+
+    with op.batch_alter_table("download_jobs") as batch:
+        if "parent_download_job_id" not in cols:
+            batch.add_column(
+                sa.Column(
+                    "parent_download_job_id",
+                    sa.String(36),
+                    sa.ForeignKey("download_jobs.id"),
+                    nullable=True,
+                )
+            )
+        if "shard_index" not in cols:
+            batch.add_column(sa.Column("shard_index", sa.Integer(), nullable=True))
+        if "shard_count" not in cols:
+            batch.add_column(sa.Column("shard_count", sa.Integer(), nullable=True))
+
+    # Padre orquestador no tiene worker asignado
+    op.alter_column(
+        "download_jobs",
+        "worker_id",
+        existing_type=sa.String(36),
+        nullable=True,
+    )
+
+    existing_indexes = {ix["name"] for ix in inspector.get_indexes("download_jobs")}
+    if "ix_download_jobs_parent" not in existing_indexes:
+        op.create_index(
+            "ix_download_jobs_parent",
+            "download_jobs",
+            ["parent_download_job_id"],
+        )
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    if not inspector.has_table("download_jobs"):
+        return
+    existing_indexes = {ix["name"] for ix in inspector.get_indexes("download_jobs")}
+    if "ix_download_jobs_parent" in existing_indexes:
+        op.drop_index("ix_download_jobs_parent", table_name="download_jobs")
+
+    cols = {c["name"] for c in inspector.get_columns("download_jobs")}
+    with op.batch_alter_table("download_jobs") as batch:
+        if "shard_count" in cols:
+            batch.drop_column("shard_count")
+        if "shard_index" in cols:
+            batch.drop_column("shard_index")
+        if "parent_download_job_id" in cols:
+            batch.drop_column("parent_download_job_id")
+
+    op.alter_column(
+        "download_jobs",
+        "worker_id",
+        existing_type=sa.String(36),
+        nullable=False,
+    )
@@ -0,0 +1,152 @@
+"""Bot digitador: digitador_workers, digitador_jobs, digitador_job_events.
+
+Revision ID: 0008
+Revises: 0007
+Create Date: 2026-08-09
+
+"""
+from __future__ import annotations
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision: str = "0008"
+down_revision: Union[str, None] = "0007"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _table_exists(conn: sa.engine.Connection, table: str) -> bool:
+    return sa.inspect(conn).has_table(table)
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    json_type = (
+        postgresql.JSONB()
+        if conn.dialect.name == "postgresql"
+        else sa.JSON()
+    )
+
+    if not _table_exists(conn, "digitador_workers"):
+        op.create_table(
+            "digitador_workers",
+            sa.Column("id", sa.String(36), primary_key=True),
+            sa.Column("hostname", sa.Text(), nullable=False, server_default=""),
+            sa.Column("max_concurrent", sa.Integer(), nullable=False, server_default="1"),
+            sa.Column("status", sa.String(32), nullable=False),
+            sa.Column("last_heartbeat", sa.DateTime(timezone=True), nullable=False),
+            sa.Column("current_job_id", sa.String(36), nullable=True),
+        )
+        op.create_index("ix_digitador_workers_status", "digitador_workers", ["status"])
+
+    if not _table_exists(conn, "digitador_jobs"):
+        op.create_table(
+            "digitador_jobs",
+            sa.Column("id", sa.String(36), primary_key=True),
+            sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True),
+            sa.Column("task_id", sa.String(36), sa.ForeignKey("tasks.id"), nullable=True),
+            sa.Column("worker_id", sa.String(36), nullable=True),
+            sa.Column(
+                "parent_digitador_job_id",
+                sa.String(36),
+                sa.ForeignKey("digitador_jobs.id"),
+                nullable=True,
+            ),
+            sa.Column("shard_index", sa.Integer(), nullable=True),
+            sa.Column("shard_count", sa.Integer(), nullable=True),
+            sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
+            sa.Column("title", sa.Text(), nullable=True),
+            sa.Column("csv_key", sa.Text(), nullable=False, server_default=""),
+            sa.Column("reports_prefix", sa.Text(), nullable=False, server_default=""),
+            sa.Column("original_filename", sa.Text(), nullable=False, server_default=""),
+            sa.Column(
+                "cancel_requested",
+                sa.Boolean(),
+                nullable=False,
+                server_default=sa.false(),
+            ),
+            sa.Column(
+                "result",
+                json_type,
+                nullable=False,
+                server_default=(
+                    sa.text("'{}'::jsonb")
+                    if conn.dialect.name == "postgresql"
+                    else sa.text("'{}'")
+                ),
+            ),
+            sa.Column("error", sa.Text(), nullable=True),
+            sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+            sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        )
+        op.create_index("ix_digitador_jobs_job_id", "digitador_jobs", ["job_id"])
+        op.create_index("ix_digitador_jobs_task_id", "digitador_jobs", ["task_id"])
+        op.create_index("ix_digitador_jobs_worker_id", "digitador_jobs", ["worker_id"])
+        op.create_index("ix_digitador_jobs_status", "digitador_jobs", ["status"])
+        op.create_index(
+            "ix_digitador_jobs_parent",
+            "digitador_jobs",
+            ["parent_digitador_job_id"],
+        )
+        op.create_index(
+            "ix_digitador_jobs_worker_status",
+            "digitador_jobs",
+            ["worker_id", "status"],
+        )
+
+    if not _table_exists(conn, "digitador_job_events"):
+        op.create_table(
+            "digitador_job_events",
+            sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
+            sa.Column(
+                "digitador_job_id",
+                sa.String(36),
+                sa.ForeignKey("digitador_jobs.id"),
+                nullable=False,
+            ),
+            sa.Column("event_type", sa.String(64), nullable=False),
+            sa.Column("message", sa.Text(), nullable=False),
+            sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False),
+            sa.Column("radicado", sa.String(255), nullable=True),
+            sa.Column("nit", sa.String(64), nullable=True),
+            sa.Column("idx", sa.Integer(), nullable=True),
+            sa.Column("total", sa.Integer(), nullable=True),
+            sa.Column(
+                "metadata",
+                json_type,
+                nullable=False,
+                server_default=(
+                    sa.text("'{}'::jsonb")
+                    if conn.dialect.name == "postgresql"
+                    else sa.text("'{}'")
+                ),
+            ),
+        )
+        op.create_index(
+            "ix_digitador_job_events_job_id",
+            "digitador_job_events",
+            ["digitador_job_id", "id"],
+        )
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    if _table_exists(conn, "digitador_job_events"):
+        op.drop_index("ix_digitador_job_events_job_id", table_name="digitador_job_events")
+        op.drop_table("digitador_job_events")
+    if _table_exists(conn, "digitador_jobs"):
+        op.drop_index("ix_digitador_jobs_worker_status", table_name="digitador_jobs")
+        op.drop_index("ix_digitador_jobs_parent", table_name="digitador_jobs")
+        op.drop_index("ix_digitador_jobs_status", table_name="digitador_jobs")
+        op.drop_index("ix_digitador_jobs_worker_id", table_name="digitador_jobs")
+        op.drop_index("ix_digitador_jobs_task_id", table_name="digitador_jobs")
+        op.drop_index("ix_digitador_jobs_job_id", table_name="digitador_jobs")
+        op.drop_table("digitador_jobs")
+    if _table_exists(conn, "digitador_workers"):
+        op.drop_index("ix_digitador_workers_status", table_name="digitador_workers")
+        op.drop_table("digitador_workers")