PYS/IPSExtractor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add .env.example configuration file and integrate Redis service into Docker Compose for improved task management. Update desktop application to support document indexing and extraction workflows, enhancing user experience with a new extraction wizard and stepper components. Revise README and documentation for clarity on setup and usage.

Browse Source
This commit is contained in:
erik
2026-07-05 22:39:07 -05:00
parent 9aca926988
commit d95f8f6ad8
70 changed files with 9807 additions and 499 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker-compose.yml
+56 -12
View File
@@ -16,6 +16,18 @@ services:
timeout: 5s
retries: 10
redis:
image: redis:7-alpine
container_name: ipsextractor-redis
command: redis-server --maxmemory 512mb --maxmemory-policy allkeys-lru --save ""
ports:
- "6379:6379"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 10
extraction:
build:
context: ./services/extraction
@@ -24,26 +36,57 @@ services:
depends_on:
database:
condition: service_healthy
redis:
condition: service_healthy
environment:
DATABASE_URL: postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor
EXTRACTOR_PROVIDER: ${EXTRACTOR_PROVIDER:-}
EXTRACTOR_BASE_URL: ${EXTRACTOR_BASE_URL:-}
EXTRACTOR_API_KEY: ${EXTRACTOR_API_KEY:-}
EXTRACTOR_MODEL: ${EXTRACTOR_MODEL:-}
NORMALIZER_PROVIDER: ${NORMALIZER_PROVIDER:-}
NORMALIZER_BASE_URL: ${NORMALIZER_BASE_URL:-}
NORMALIZER_API_KEY: ${NORMALIZER_API_KEY:-}
NORMALIZER_MODEL: ${NORMALIZER_MODEL:-}
NORMALIZER_TIMEOUT_S: ${NORMALIZER_TIMEOUT_S:-0}
REDIS_URL: redis://redis:6379/0
EXTRACTOR_BASE_URL: https://api.openai.com/v1
EXTRACTOR_API_KEY: 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
EXTRACTOR_MODEL: ${EXTRACTOR_MODEL:-gpt-5.4-mini}
NORMALIZER_BASE_URL: https://api.openai.com/v1
NORMALIZER_API_KEY: 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
NORMALIZER_MODEL: gpt-5.4-mini
EXTRACTOR_TIMEOUT_S: ${EXTRACTOR_TIMEOUT_S:-120}
PDF_DPI: ${PDF_DPI:-600}
PDF_THREAD_COUNT: ${PDF_THREAD_COUNT:-10}
CONCURRENCY_LIMIT: ${CONCURRENCY_LIMIT:-4}
PDF_DPI: ${PDF_DPI:-150}
PDF_THREAD_COUNT: ${PDF_THREAD_COUNT:-4}
CONCURRENCY_LIMIT: ${CONCURRENCY_LIMIT:-50}
ALLOWED_DOCUMENT_TYPES: ${ALLOWED_DOCUMENT_TYPES:-CC,TI,CE,RC,PA,PPT,PEP,EPT,NUIP}
ports:
- "8001:8001"
volumes:
- ${IPS_DOCUMENT_ROOT:-./data}:/data
- ips_queue:/tmp/ips_queue
# Escala horizontalmente: docker compose up --scale worker=4
worker:
build:
context: ./services/extraction
dockerfile: Dockerfile
command: python -m worker.main
depends_on:
database:
condition: service_healthy
redis:
condition: service_healthy
deploy:
replicas: ${WORKER_REPLICAS:-2}
environment:
DATABASE_URL: postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor
REDIS_URL: redis://redis:6379/0
EXTRACTOR_BASE_URL: https://api.openai.com/v1
EXTRACTOR_API_KEY: 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
EXTRACTOR_MODEL: ${EXTRACTOR_MODEL:-gpt-5.4-mini}
NORMALIZER_BASE_URL: https://api.openai.com/v1
NORMALIZER_API_KEY: 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
NORMALIZER_MODEL: gpt-5.4-mini
EXTRACTOR_TIMEOUT_S: ${EXTRACTOR_TIMEOUT_S:-120}
PDF_DPI: ${PDF_DPI:-150}
PDF_THREAD_COUNT: ${PDF_THREAD_COUNT:-4}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-25}
ALLOWED_DOCUMENT_TYPES: ${ALLOWED_DOCUMENT_TYPES:-CC,TI,CE,RC,PA,PPT,PEP,EPT,NUIP}
volumes:
- ips_queue:/tmp/ips_queue
cleaning:
build:
@@ -74,3 +117,4 @@ services:
volumes:
ipsextractor_db:
ips_queue: