feat: Add .env.example configuration file and integrate Redis service into Docker Compose for improved task management. Update desktop application to support document indexing and extraction workflows, enhancing user experience with a new extraction wizard and stepper components. Revise README and documentation for clarity on setup and usage.

2026-07-05 22:39:07 -05:00
parent 9aca926988
commit d95f8f6ad8
70 changed files with 9807 additions and 499 deletions
@@ -0,0 +1,59 @@
+"""Compara baseline Python vs respuesta del microservicio Go."""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+SC_ROOT = Path(os.environ.get("SERVICES_COUNT_ROOT", ROOT.parent / "services-count"))
+BASELINE = ROOT / "data" / "parity_baseline.json"
+API_URL = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080/v1/count"
+
+
+def call_api(pdf_path: Path) -> dict:
+    result = subprocess.run(
+        ["curl", "-s", "-F", f"files=@{pdf_path}", API_URL],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    payload = json.loads(result.stdout)
+    for item in payload.get("results", []):
+        if item.get("filename") == pdf_path.name:
+            return item
+    raise RuntimeError(f"sin resultado para {pdf_path.name}")
+
+
+def main() -> None:
+    if not BASELINE.exists():
+        raise SystemExit("Ejecuta primero: python scripts/parity_baseline.py")
+
+    import yaml
+
+    config_path = SC_ROOT / "config" / "default.yaml"
+    pdfs_dir = SC_ROOT / yaml.safe_load(config_path.read_text())["paths"]["pdfs_dir"]
+
+    baseline = json.loads(BASELINE.read_text(encoding="utf-8"))
+    mismatches = 0
+    for expected in baseline:
+        pdf = pdfs_dir / expected["filename"]
+        if not pdf.exists():
+            print(f"SKIP {expected['filename']} (no existe)")
+            continue
+        got = call_api(pdf)
+        if got.get("service_count") != expected.get("service_count"):
+            mismatches += 1
+            print(f"MISMATCH {expected['filename']}: py={expected['service_count']} go={got.get('service_count')}")
+        else:
+            print(f"OK {expected['filename']}: count={got.get('service_count')}")
+
+    if mismatches:
+        raise SystemExit(f"{mismatches} diferencias encontradas")
+    print("Paridad OK")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,99 @@
+"""Exporta model.joblib a ONNX + inference.json (sin depender de services-count)."""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import joblib
+from skl2onnx import convert_sklearn
+from skl2onnx.common.data_types import FloatTensorType
+
+FEATURE_COLUMNS = [
+    "first_token_is_code",
+    "currency_count",
+    "integer_count",
+    "x_span_norm",
+    "y_norm",
+    "token_count",
+    "special_char_ratio",
+    "digit_ratio",
+]
+
+DEFAULT_OCR = {"dpi": 300, "lang": "spa"}
+DEFAULT_DBSCAN = {"eps": 10, "min_samples": 1}
+
+
+def parse_args() -> argparse.Namespace:
+    root = Path(__file__).resolve().parents[1]
+    parser = argparse.ArgumentParser(description="Exportar Random Forest joblib a ONNX")
+    parser.add_argument(
+        "--joblib",
+        type=Path,
+        required=True,
+        help="Ruta al model.joblib (ej. ../services-count/models/latest/model.joblib)",
+    )
+    parser.add_argument(
+        "--out",
+        type=Path,
+        default=root / "models" / "latest",
+        help="Directorio de salida para model.onnx e inference.json",
+    )
+    parser.add_argument(
+        "--metadata",
+        type=Path,
+        default=None,
+        help="model_metadata.json opcional para pipeline_version y params OCR/DBSCAN",
+    )
+    return parser.parse_args()
+
+
+def load_metadata(path: Path | None) -> dict:
+    if path is None or not path.exists():
+        return {}
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def main() -> None:
+    args = parse_args()
+    if not args.joblib.exists():
+        raise SystemExit(f"No se encontró: {args.joblib}")
+
+    bundle = joblib.load(args.joblib)
+    model = bundle["model"]
+    threshold = float(bundle.get("threshold", 0.5))
+    meta = load_metadata(args.metadata)
+
+    initial_type = [("float_input", FloatTensorType([None, len(FEATURE_COLUMNS)]))]
+    options = {id(model): {"zipmap": False}}
+    onnx_model = convert_sklearn(
+        model,
+        initial_types=initial_type,
+        options=options,
+        target_opset=12,
+    )
+
+    out_dir = args.out
+    out_dir.mkdir(parents=True, exist_ok=True)
+    onnx_path = out_dir / "model.onnx"
+    meta_path = out_dir / "inference.json"
+
+    with open(onnx_path, "wb") as f:
+        f.write(onnx_model.SerializeToString())
+
+    inference = {
+        "threshold": threshold,
+        "feature_columns": FEATURE_COLUMNS,
+        "pipeline_version": meta.get("pipeline_version", "unknown"),
+        "ocr": meta.get("ocr", DEFAULT_OCR),
+        "dbscan": meta.get("dbscan", DEFAULT_DBSCAN),
+    }
+    meta_path.write_text(json.dumps(inference, indent=2), encoding="utf-8")
+
+    print(f"ONNX: {onnx_path}")
+    print(f"Meta: {meta_path}")
+    print(f"Umbral: {threshold}")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,49 @@
+"""Genera baseline Python desde services-count para comparar con el servicio Go."""
+from __future__ import annotations
+
+import json
+import os
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+SC_ROOT = Path(os.environ.get("SERVICES_COUNT_ROOT", ROOT.parent / "services-count"))
+sys.path.insert(0, str(SC_ROOT / "src"))
+
+from services_count.config import load_config
+from services_count.inference import predict_pdf
+
+
+def python_result(pdf_path: Path) -> dict:
+    config = load_config(SC_ROOT / "config" / "default.yaml")
+    features_df, total = predict_pdf(pdf_path, config)
+    services = []
+    for row in features_df.itertuples():
+        if row.prediction == 1:
+            services.append({
+                "text": row.texto_fila,
+                "probability": round(float(row.probability), 4),
+            })
+    return {
+        "filename": pdf_path.name,
+        "service_count": total,
+        "status": "ok",
+        "services": services,
+    }
+
+
+def main() -> None:
+    config = load_config(SC_ROOT / "config" / "default.yaml")
+    pdfs = sorted(config.pdfs_dir.glob("*.pdf"))
+    if not pdfs:
+        raise SystemExit(f"No hay PDFs en {config.pdfs_dir}")
+
+    out = ROOT / "data" / "parity_baseline.json"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    results = [python_result(p) for p in pdfs]
+    out.write_text(json.dumps(results, indent=2, ensure_ascii=False), encoding="utf-8")
+    print(f"Baseline guardado: {out} ({len(results)} PDFs)")
+
+
+if __name__ == "__main__":
+    main()