feat: Add database migration service with Alembic for PostgreSQL, integrating job management and counting functionalities. Update Docker Compose to include db-migrations service, enhancing the microservices architecture. Revise documentation for clarity on database schema and migration processes.

2026-07-06 00:50:50 -05:00
parent d95f8f6ad8
commit 081da999d4
43 changed files with 2911 additions and 2271 deletions
@@ -6,67 +6,39 @@ import (
 	"time"
 )

+// Config holds all runtime parameters for the count-api service.
+// All values are loaded from environment variables with safe defaults.
 type Config struct {
-	Port              string
-	RedisURL          string
-	NATSURL           string
-	MinIOEndpoint     string
-	MinIOAccessKey    string
-	MinIOSecretKey    string
-	MinIOBucket       string
-	MinIOUseSSL       bool
-	ModelPath         string
-	InferencePath     string
-	OCRDPI            int
-	OCRLang           string
-	DBSCANEps         float64
-	DBSCANMinSamples  int
-	SyncMaxFiles      int
-	JobMaxFiles       int
-	QueueMaxPending   int
-	WorkerConcurrency int
-	MaxPDFSizeMB      int
-	PDFProcessTimeout time.Duration
-	JobResultTTL      time.Duration
-	StreamName        string
-	SubjectPrefix     string
+	Port                   string
+	ModelPath              string
+	InferencePath          string
+	OCRDPI                 int
+	OCRLang                string
 	OCRPoolSize            int
-	LogPipelineTiming      bool
-	NATSFetchMaxWait       time.Duration
-	JobInlineProcessing    bool
+	DBSCANEps              float64
+	DBSCANMinSamples       int
+	SyncMaxFiles           int
+	MaxPDFSizeMB           int
+	PDFProcessTimeout      time.Duration
 	SyncProcessConcurrency int
+	LogPipelineTiming      bool
 }

 func Load() Config {
 	return Config{
-		Port:              getenv("PORT", "8080"),
-		RedisURL:          getenv("REDIS_URL", "redis://redis:6379/0"),
-		NATSURL:           getenv("NATS_URL", "nats://nats:4222"),
-		MinIOEndpoint:     getenv("MINIO_ENDPOINT", "minio:9000"),
-		MinIOAccessKey:    getenv("MINIO_ACCESS_KEY", "minioadmin"),
-		MinIOSecretKey:    getenv("MINIO_SECRET_KEY", "minioadmin"),
-		MinIOBucket:       getenv("MINIO_BUCKET", "pdf-count"),
-		MinIOUseSSL:       getenvBool("MINIO_USE_SSL", false),
-		ModelPath:         getenv("MODEL_PATH", "/app/models/model.onnx"),
-		InferencePath:     getenv("INFERENCE_CONFIG", "/app/models/inference.json"),
-		OCRDPI:            getenvInt("OCR_DPI", 300),
-		OCRLang:           getenv("OCR_LANG", "spa"),
-		DBSCANEps:         getenvFloat("DBSCAN_EPS", 10),
-		DBSCANMinSamples:  getenvInt("DBSCAN_MIN_SAMPLES", 1),
-		SyncMaxFiles:      getenvInt("SYNC_MAX_FILES", 50),
-		JobMaxFiles:       getenvInt("JOB_MAX_FILES", 50000),
-		QueueMaxPending:   getenvInt("QUEUE_MAX_PENDING", 100000),
-		WorkerConcurrency: getenvInt("WORKER_CONCURRENCY", 4),
-		MaxPDFSizeMB:      getenvInt("MAX_PDF_SIZE_MB", 20),
-		PDFProcessTimeout: time.Duration(getenvInt("PDF_PROCESS_TIMEOUT_SEC", 120)) * time.Second,
-		JobResultTTL:      time.Duration(getenvInt("JOB_RESULT_TTL_HOURS", 72)) * time.Hour,
-		StreamName:        getenv("NATS_STREAM", "PDF_JOBS"),
-		SubjectPrefix:     getenv("NATS_SUBJECT_PREFIX", "pdf.process"),
+		Port:                   getenv("PORT", "8080"),
+		ModelPath:              getenv("MODEL_PATH", "/app/models/model.onnx"),
+		InferencePath:          getenv("INFERENCE_CONFIG", "/app/models/inference.json"),
+		OCRDPI:                 getenvInt("OCR_DPI", 300),
+		OCRLang:                getenv("OCR_LANG", "spa"),
 		OCRPoolSize:            getenvInt("OCR_POOL_SIZE", 0),
-		LogPipelineTiming:      getenvBool("LOG_PIPELINE_TIMING", false),
-		NATSFetchMaxWait:       time.Duration(getenvInt("NATS_FETCH_MAX_WAIT_MS", 300)) * time.Millisecond,
-		JobInlineProcessing:    getenvBool("JOB_INLINE_PROCESSING", false),
+		DBSCANEps:              getenvFloat("DBSCAN_EPS", 10),
+		DBSCANMinSamples:       getenvInt("DBSCAN_MIN_SAMPLES", 1),
+		SyncMaxFiles:           getenvInt("SYNC_MAX_FILES", 50),
+		MaxPDFSizeMB:           getenvInt("MAX_PDF_SIZE_MB", 20),
+		PDFProcessTimeout:      time.Duration(getenvInt("PDF_PROCESS_TIMEOUT_SEC", 120)) * time.Second,
 		SyncProcessConcurrency: getenvInt("SYNC_PROCESS_CONCURRENCY", 0),
+		LogPipelineTiming:      getenvBool("LOG_PIPELINE_TIMING", false),
 	}
 }