from __future__ import annotations import json import os from dataclasses import dataclass from pathlib import Path from typing import List, Optional @dataclass(frozen=True) class MinioSettings: endpoint: str = "https://s3.builker.xyz" bucket: str = "pys-cuentas-medicas" access_key: str = "" secret_key: str = "" region: str = "us-east-1" anonymous: bool = True @classmethod def from_env(cls) -> "MinioSettings": access_key = (os.getenv("MINIO_ACCESS_KEY") or "").strip() secret_key = (os.getenv("MINIO_SECRET_KEY") or "").strip() anon_flag = (os.getenv("MINIO_ANONYMOUS") or "").strip().lower() if anon_flag in ("0", "false", "no"): anonymous = False elif anon_flag in ("1", "true", "yes"): anonymous = True else: anonymous = not (access_key and secret_key) return cls( endpoint=os.getenv("MINIO_ENDPOINT", "https://s3.builker.xyz").rstrip("/"), bucket=os.getenv("MINIO_BUCKET", "pys-cuentas-medicas"), access_key=access_key, secret_key=secret_key, region=os.getenv("MINIO_REGION", "us-east-1"), anonymous=anonymous, ) def csv_key(job_id: str, filename: str = "input.csv") -> str: return f"uploads/{job_id}/{filename}" def reports_prefix(job_id: str) -> str: return f"digitados/{job_id}/" def meta_key(job_id: str) -> str: return f"uploads/{job_id}/meta.json" def manifest_key(job_id: str) -> str: return f"digitados/{job_id}/_manifest.json" _REPORT_NAMES = { "resultados.csv", "omitidos.csv", "ejecucion_bot.log", } def _s3_client(settings: MinioSettings): import boto3 from botocore import UNSIGNED from botocore.client import Config s3_cfg = { "addressing_style": "path", "request_checksum_calculation": "when_required", "response_checksum_validation": "when_required", } if settings.anonymous: return boto3.client( "s3", endpoint_url=settings.endpoint, region_name=settings.region, config=Config(signature_version=UNSIGNED, s3=s3_cfg), ) return boto3.client( "s3", endpoint_url=settings.endpoint, aws_access_key_id=settings.access_key, aws_secret_access_key=settings.secret_key, region_name=settings.region, config=Config(signature_version="s3v4", s3=s3_cfg), ) class MinioStorage: """Cliente S3-compatible para MinIO (uploads/digitados por job_id).""" def __init__(self, settings: Optional[MinioSettings] = None): self.settings = settings or MinioSettings.from_env() self._client = None @property def client(self): if self._client is None: self._client = _s3_client(self.settings) return self._client @property def bucket(self) -> str: return self.settings.bucket def put_bytes(self, key: str, data: bytes, content_type: str = "application/octet-stream") -> str: self.client.put_object( Bucket=self.bucket, Key=key, Body=data, ContentType=content_type, ) return key def put_file(self, key: str, path: str | Path, content_type: str = "application/octet-stream") -> str: path = Path(path) return self.put_bytes(key, path.read_bytes(), content_type=content_type) def get_bytes(self, key: str) -> bytes: obj = self.client.get_object(Bucket=self.bucket, Key=key) return obj["Body"].read() def download_file(self, key: str, dest: str | Path) -> Path: dest = Path(dest) dest.parent.mkdir(parents=True, exist_ok=True) self.client.download_file(self.bucket, key, str(dest)) return dest def exists(self, key: str) -> bool: try: self.client.head_object(Bucket=self.bucket, Key=key) return True except Exception: return False def list_keys(self, prefix: str) -> List[str]: keys: List[str] = [] token = None while True: kwargs = {"Bucket": self.bucket, "Prefix": prefix} if token: kwargs["ContinuationToken"] = token resp = self.client.list_objects_v2(**kwargs) for item in resp.get("Contents") or []: keys.append(item["Key"]) if not resp.get("IsTruncated"): break token = resp.get("NextContinuationToken") return keys def presigned_get(self, key: str, expires_in: int = 3600) -> str: if self.settings.anonymous: return f"{self.settings.endpoint}/{self.bucket}/{key}" return self.client.generate_presigned_url( "get_object", Params={"Bucket": self.bucket, "Key": key}, ExpiresIn=expires_in, ) def upload_csv( self, job_id: str, data: bytes, original_filename: str, content_type: str = "text/csv", ) -> str: ext = Path(original_filename).suffix.lower() or ".csv" canonical = "input.csv" if ext in (".csv", ".txt") else "input.csv" key = csv_key(job_id, canonical) self.put_bytes(key, data, content_type=content_type) meta = { "original_filename": original_filename, "size": len(data), "canonical": canonical, } self.put_bytes(meta_key(job_id), json.dumps(meta).encode("utf-8"), "application/json") return key def write_manifest(self, job_id: str, keys: List[str]) -> str: key = manifest_key(job_id) payload = {"job_id": job_id, "files": keys, "count": len(keys)} self.put_bytes(key, json.dumps(payload, ensure_ascii=False).encode("utf-8"), "application/json") return key def list_report_files(self, job_id: str, prefix: str | None = None) -> dict: """Lista reportes de digitación (RESULTADOS/OMITIDOS/log + list_keys). Fusiona manifest con listado del prefijo (cubriendo shards distribuidos). """ prefix = prefix or reports_prefix(job_id) by_key: dict[str, dict] = {} try: raw = self.get_bytes(manifest_key(job_id)) payload = json.loads(raw.decode("utf-8")) for key in payload.get("files") or []: entry = self._file_entry(str(key)) if entry is not None: by_key[entry["key"]] = entry except Exception: pass for entry in self._list_report_entries(prefix): by_key[entry["key"]] = entry files = list(by_key.values()) return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)} def _list_report_entries(self, prefix: str) -> List[dict]: files: List[dict] = [] for key in self.list_keys(prefix): entry = self._file_entry(key) if entry is not None: files.append(entry) return files @staticmethod def _file_entry(key: str) -> dict | None: name = Path(key).name lower = name.lower() if lower.startswith("_manifest") and lower.endswith(".json"): return None # RESULTADOS.csv, OMITIDOS.csv, ejecucion_bot.log (+ variantes _shard_N) if lower in _REPORT_NAMES or lower.endswith(".csv") or lower.endswith(".log"): return {"key": key, "name": name} return None