227 lines
7.4 KiB
Python
227 lines
7.4 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import List, Optional
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MinioSettings:
|
|
endpoint: str = "https://s3.builker.xyz"
|
|
bucket: str = "pys-cuentas-medicas"
|
|
access_key: str = ""
|
|
secret_key: str = ""
|
|
region: str = "us-east-1"
|
|
anonymous: bool = True
|
|
|
|
@classmethod
|
|
def from_env(cls) -> "MinioSettings":
|
|
access_key = (os.getenv("MINIO_ACCESS_KEY") or "").strip()
|
|
secret_key = (os.getenv("MINIO_SECRET_KEY") or "").strip()
|
|
anon_flag = (os.getenv("MINIO_ANONYMOUS") or "").strip().lower()
|
|
if anon_flag in ("0", "false", "no"):
|
|
anonymous = False
|
|
elif anon_flag in ("1", "true", "yes"):
|
|
anonymous = True
|
|
else:
|
|
anonymous = not (access_key and secret_key)
|
|
return cls(
|
|
endpoint=os.getenv("MINIO_ENDPOINT", "https://s3.builker.xyz").rstrip("/"),
|
|
bucket=os.getenv("MINIO_BUCKET", "pys-cuentas-medicas"),
|
|
access_key=access_key,
|
|
secret_key=secret_key,
|
|
region=os.getenv("MINIO_REGION", "us-east-1"),
|
|
anonymous=anonymous,
|
|
)
|
|
|
|
|
|
def csv_key(job_id: str, filename: str = "input.csv") -> str:
|
|
return f"uploads/{job_id}/{filename}"
|
|
|
|
|
|
def reports_prefix(job_id: str) -> str:
|
|
return f"digitados/{job_id}/"
|
|
|
|
|
|
def meta_key(job_id: str) -> str:
|
|
return f"uploads/{job_id}/meta.json"
|
|
|
|
|
|
def manifest_key(job_id: str) -> str:
|
|
return f"digitados/{job_id}/_manifest.json"
|
|
|
|
|
|
_REPORT_NAMES = {
|
|
"resultados.csv",
|
|
"omitidos.csv",
|
|
"ejecucion_bot.log",
|
|
}
|
|
|
|
|
|
def _s3_client(settings: MinioSettings):
|
|
import boto3
|
|
from botocore import UNSIGNED
|
|
from botocore.client import Config
|
|
|
|
s3_cfg = {
|
|
"addressing_style": "path",
|
|
"request_checksum_calculation": "when_required",
|
|
"response_checksum_validation": "when_required",
|
|
}
|
|
if settings.anonymous:
|
|
return boto3.client(
|
|
"s3",
|
|
endpoint_url=settings.endpoint,
|
|
region_name=settings.region,
|
|
config=Config(signature_version=UNSIGNED, s3=s3_cfg),
|
|
)
|
|
return boto3.client(
|
|
"s3",
|
|
endpoint_url=settings.endpoint,
|
|
aws_access_key_id=settings.access_key,
|
|
aws_secret_access_key=settings.secret_key,
|
|
region_name=settings.region,
|
|
config=Config(signature_version="s3v4", s3=s3_cfg),
|
|
)
|
|
|
|
|
|
class MinioStorage:
|
|
"""Cliente S3-compatible para MinIO (uploads/digitados por job_id)."""
|
|
|
|
def __init__(self, settings: Optional[MinioSettings] = None):
|
|
self.settings = settings or MinioSettings.from_env()
|
|
self._client = None
|
|
|
|
@property
|
|
def client(self):
|
|
if self._client is None:
|
|
self._client = _s3_client(self.settings)
|
|
return self._client
|
|
|
|
@property
|
|
def bucket(self) -> str:
|
|
return self.settings.bucket
|
|
|
|
def put_bytes(self, key: str, data: bytes, content_type: str = "application/octet-stream") -> str:
|
|
self.client.put_object(
|
|
Bucket=self.bucket,
|
|
Key=key,
|
|
Body=data,
|
|
ContentType=content_type,
|
|
)
|
|
return key
|
|
|
|
def put_file(self, key: str, path: str | Path, content_type: str = "application/octet-stream") -> str:
|
|
path = Path(path)
|
|
return self.put_bytes(key, path.read_bytes(), content_type=content_type)
|
|
|
|
def get_bytes(self, key: str) -> bytes:
|
|
obj = self.client.get_object(Bucket=self.bucket, Key=key)
|
|
return obj["Body"].read()
|
|
|
|
def download_file(self, key: str, dest: str | Path) -> Path:
|
|
dest = Path(dest)
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
self.client.download_file(self.bucket, key, str(dest))
|
|
return dest
|
|
|
|
def exists(self, key: str) -> bool:
|
|
try:
|
|
self.client.head_object(Bucket=self.bucket, Key=key)
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
def list_keys(self, prefix: str) -> List[str]:
|
|
keys: List[str] = []
|
|
token = None
|
|
while True:
|
|
kwargs = {"Bucket": self.bucket, "Prefix": prefix}
|
|
if token:
|
|
kwargs["ContinuationToken"] = token
|
|
resp = self.client.list_objects_v2(**kwargs)
|
|
for item in resp.get("Contents") or []:
|
|
keys.append(item["Key"])
|
|
if not resp.get("IsTruncated"):
|
|
break
|
|
token = resp.get("NextContinuationToken")
|
|
return keys
|
|
|
|
def presigned_get(self, key: str, expires_in: int = 3600) -> str:
|
|
if self.settings.anonymous:
|
|
return f"{self.settings.endpoint}/{self.bucket}/{key}"
|
|
return self.client.generate_presigned_url(
|
|
"get_object",
|
|
Params={"Bucket": self.bucket, "Key": key},
|
|
ExpiresIn=expires_in,
|
|
)
|
|
|
|
def upload_csv(
|
|
self,
|
|
job_id: str,
|
|
data: bytes,
|
|
original_filename: str,
|
|
content_type: str = "text/csv",
|
|
) -> str:
|
|
ext = Path(original_filename).suffix.lower() or ".csv"
|
|
canonical = "input.csv" if ext in (".csv", ".txt") else "input.csv"
|
|
key = csv_key(job_id, canonical)
|
|
self.put_bytes(key, data, content_type=content_type)
|
|
meta = {
|
|
"original_filename": original_filename,
|
|
"size": len(data),
|
|
"canonical": canonical,
|
|
}
|
|
self.put_bytes(meta_key(job_id), json.dumps(meta).encode("utf-8"), "application/json")
|
|
return key
|
|
|
|
def write_manifest(self, job_id: str, keys: List[str]) -> str:
|
|
key = manifest_key(job_id)
|
|
payload = {"job_id": job_id, "files": keys, "count": len(keys)}
|
|
self.put_bytes(key, json.dumps(payload, ensure_ascii=False).encode("utf-8"), "application/json")
|
|
return key
|
|
|
|
def list_report_files(self, job_id: str, prefix: str | None = None) -> dict:
|
|
"""Lista reportes de digitación (RESULTADOS/OMITIDOS/log + list_keys).
|
|
|
|
Fusiona manifest con listado del prefijo (cubriendo shards distribuidos).
|
|
"""
|
|
prefix = prefix or reports_prefix(job_id)
|
|
by_key: dict[str, dict] = {}
|
|
try:
|
|
raw = self.get_bytes(manifest_key(job_id))
|
|
payload = json.loads(raw.decode("utf-8"))
|
|
for key in payload.get("files") or []:
|
|
entry = self._file_entry(str(key))
|
|
if entry is not None:
|
|
by_key[entry["key"]] = entry
|
|
except Exception:
|
|
pass
|
|
|
|
for entry in self._list_report_entries(prefix):
|
|
by_key[entry["key"]] = entry
|
|
|
|
files = list(by_key.values())
|
|
return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)}
|
|
|
|
def _list_report_entries(self, prefix: str) -> List[dict]:
|
|
files: List[dict] = []
|
|
for key in self.list_keys(prefix):
|
|
entry = self._file_entry(key)
|
|
if entry is not None:
|
|
files.append(entry)
|
|
return files
|
|
|
|
@staticmethod
|
|
def _file_entry(key: str) -> dict | None:
|
|
name = Path(key).name
|
|
lower = name.lower()
|
|
if lower.startswith("_manifest") and lower.endswith(".json"):
|
|
return None
|
|
# RESULTADOS.csv, OMITIDOS.csv, ejecucion_bot.log (+ variantes _shard_N)
|
|
if lower in _REPORT_NAMES or lower.endswith(".csv") or lower.endswith(".log"):
|
|
return {"key": key, "name": name}
|
|
return None
|