diff --git a/bot_saludtotal.py b/bot_saludtotal.py new file mode 100644 index 0000000..a26a746 --- /dev/null +++ b/bot_saludtotal.py @@ -0,0 +1,1395 @@ +import tkinter as tk +from tkinter import filedialog, messagebox, scrolledtext +import threading +import csv +import time +import ctypes +import os +from datetime import datetime +from pywinauto.application import Application +from pywinauto import mouse + +try: + import cv2 + import numpy as np + OPENCV_AVAILABLE = True +except ImportError: + OPENCV_AVAILABLE = False + + +try: + ctypes.windll.shcore.SetProcessDpiAwareness(2) +except Exception: + try: + ctypes.windll.user32.SetProcessDPIAware() + except Exception: + pass + + +DOC_COORDS = { + "CC": (796, 343), + "TI": (796, 549), + "CE": (796, 329), + "AS": (796, 296), + "CD": (796, 314), + "CN": (796, 371), + "ME": (796, 378), + "NI": (796, 412), + "NU": (796, 453), + "PS": (796, 450), + "PE": (796, 467), + "PT": (796, 483), + "PJ": (796, 503), + "RC": (796, 520), + "SC": (796, 536), +} + +DOC_STRATEGY = [ + "CC", "TI", "CE", "RC", "PS", "PT", "PE", + "AS", "ME", "CN", "SC", "CD", "NI", "NU", "PJ" +] + +SERVICES_COORDS = { + "ambulancias y elem.": (1033, 367), + "ambulatorio": (1033, 387), + "c. externa": (1033, 405), + "capacitacion": (1033, 417), + "cirugia ambulatoria": (1033, 432), + "fomento y prevencion": (1033, 449), + "honorarios": (1033, 467), +} + +COORD_CLOSE_ERROR = (1223, 704) +TEMPLATE_ERROR_PATH = os.path.join("template", "error.png") +TEMPLATE_SUCCESS_PATH = os.path.join("template", "success.png") +TEMPLATE_BIEN_PATH = os.path.join("template", "bien.png") +TEMPLATE_DIFERENCIAS_PATH = os.path.join("template", "flecha_diferencias_abierto.png") +TEMPLATE_DIFERENCIAS_CERRADO_PATH = os.path.join("template", "flecha_diferencias_cerrado.png") +TEMPLATE_FECHA_VACIA_SLASH_PATH = os.path.join("template", "fecha_vacia_slash.png") +TEMPLATE_FECHA_VACIA_BLANCO_PATH = os.path.join("template", "fecha_vacia_blanco.png") +TEMPLATE_POPUP_NOMBRES_PATH = os.path.join("template", "popup_nombres.png") +TEMPLATE_RADICADO_EXISTENTE_PATH = os.path.join("template", "radicado_existente.png") + +# ========================= +# CONFIGURACIÓN REINICIO COMPLETO APP +# ========================= +APP_PATH = r"C:\Users\Pysctas185\Documents\Integracion Total.lnk" +PROCESS_NAME = "SaludTotal.Health.WinClient.TotalIntegration.exe" +REINICIO_CADA = 6 +TIMEOUT_REGISTRO = 120 +REPORTS_DIR_NAME = "reportes_bot" + + +def click_abs(window, x: int, y: int): + try: + window.wrapper_object().set_focus() + except Exception: + pass + + ctypes.windll.user32.SetCursorPos(x, y) + time.sleep(0.05) + ctypes.windll.user32.SetCursorPos(x + 1, y) + time.sleep(0.05) + ctypes.windll.user32.SetCursorPos(x, y) + + time.sleep(0.2) + mouse.click(button="left", coords=(x, y)) + + +class BotSaludTotal: + def __init__(self, root): + self.first_time = True + self.root = root + self.root.title("Bot Salud Total - Automatización con OpenCV") + self.root.geometry("600x700") + + self.csv_path = tk.StringVar() + self.date_format = tk.StringVar(value="dd/mm/yyyy") + self.delimiter = tk.StringVar(value=",") + self.is_running = False + + self.omitted_records = [] + self.log_entries = [] + self.resultados = [] + + # ========================= + # CONTROL DE EJECUCIÓN / OVERLAY / REPORTES + # ========================= + self.run_id = datetime.now().strftime("%Y%m%d_%H%M%S") + self.output_dir = os.path.join(os.getcwd(), REPORTS_DIR_NAME, self.run_id) + self.total_registros = 0 + self.procesados_ok = 0 + self.errores = 0 + self.omitidos = 0 + self.existentes = 0 + self.reinicios = 0 + self.registro_actual = "-" + self.estado_actual = "Listo" + self.inicio_proceso = None + self.inicio_registro = None + self.overlay = None + self.overlay_vars = {} + + tk.Label( + root, + text="Seleccione el archivo CSV de entrada:", + font=("Arial", 10, "bold") + ).pack(pady=10) + + frame_file = tk.Frame(root) + frame_file.pack(pady=5, padx=20, fill="x") + + self.entry_file = tk.Entry(frame_file, textvariable=self.csv_path, state="readonly") + self.entry_file.pack(side="left", fill="x", expand=True, padx=(0, 5)) + + tk.Button(frame_file, text="Buscar CSV", command=self.browse_file).pack(side="right") + + frame_delim = tk.LabelFrame(root, text="Delimitador de CSV", padx=10, pady=5) + frame_delim.pack(pady=10, padx=20, fill="x") + + tk.Radiobutton(frame_delim, text="Coma (,)", variable=self.delimiter, value=",").pack(anchor="w") + tk.Radiobutton(frame_delim, text="Pipe (|)", variable=self.delimiter, value="|").pack(anchor="w") + + frame_date = tk.LabelFrame(root, text="Formato de Fecha en CSV", padx=10, pady=5) + frame_date.pack(pady=10, padx=20, fill="x") + + tk.Radiobutton( + frame_date, + text="Día/Mes/Año (Ej: 31/12/2025)", + variable=self.date_format, + value="dd/mm/yyyy" + ).pack(anchor="w") + + tk.Radiobutton( + frame_date, + text="Mes/Día/Año (Ej: 12/31/2025)", + variable=self.date_format, + value="mm/dd/yyyy" + ).pack(anchor="w") + + self.btn_start = tk.Button( + root, + text="INICIAR BOT", + command=self.start_thread, + bg="green", + fg="white", + font=("Arial", 11, "bold") + ) + self.btn_start.pack(pady=20) + + self.log_area = scrolledtext.ScrolledText(root, height=10, state="disabled") + self.log_area.pack(padx=20, pady=10, fill="both", expand=True) + + self.crear_overlay_fantasma() + self.actualizar_overlay() + + def crear_overlay_fantasma(self): + """Crea un mini panel flotante, pequeño y liviano, para monitorear el bot.""" + try: + self.overlay = tk.Toplevel(self.root) + self.overlay.title("BOT ST") + self.overlay.geometry("210x235+15+555") + self.overlay.overrideredirect(True) + self.overlay.attributes("-topmost", True) + self.overlay.attributes("-alpha", 0.88) + self.overlay.configure(bg="#101010") + + def add_label(key, text, font=("Consolas", 8), fg="#00ff88"): + var = tk.StringVar(value=text) + lbl = tk.Label( + self.overlay, + textvariable=var, + bg="#101010", + fg=fg, + font=font, + anchor="w", + justify="left" + ) + lbl.pack(fill="x", padx=8, pady=1) + self.overlay_vars[key] = var + return lbl + + add_label("titulo", "BOT SALUD TOTAL", ("Consolas", 9, "bold"), "#00ff88") + add_label("estado", "ESTADO: Listo", ("Consolas", 8, "bold"), "#ffffff") + add_label("progreso", "0/0 0%", ("Consolas", 8), "#ffffff") + add_label("ok", "OK: 0", ("Consolas", 8), "#00ff88") + add_label("err", "ERR: 0 | OMIT: 0", ("Consolas", 8), "#ff6666") + add_label("rei", "REI: 0", ("Consolas", 8), "#66ccff") + add_label("rad", "RAD: -", ("Consolas", 8), "#ffffff") + add_label("vel", "VEL: 0 rad/min", ("Consolas", 8), "#ffff99") + add_label("tiempo", "TIME: 00:00:00", ("Consolas", 8), "#ffffff") + add_label("ultimo", "ULT: -", ("Consolas", 8), "#cccccc") + + except Exception as e: + print(f"No se pudo crear overlay: {e}") + + def set_estado_overlay(self, estado, ultimo=None): + self.estado_actual = estado + if ultimo: + self.ultimo_evento = ultimo + self.actualizar_overlay() + + def actualizar_overlay(self): + """Actualiza el overlay sin usar Textbox pesado.""" + try: + if not self.overlay or not self.overlay_vars: + return + + total_procesados = self.procesados_ok + self.errores + getattr(self, "existentes", 0) + porcentaje = int((total_procesados / self.total_registros) * 100) if self.total_registros else 0 + + if self.inicio_proceso: + segundos = int(time.time() - self.inicio_proceso) + h = segundos // 3600 + m = (segundos % 3600) // 60 + s = segundos % 60 + tiempo_txt = f"{h:02d}:{m:02d}:{s:02d}" + minutos = max(segundos / 60, 0.01) + velocidad = total_procesados / minutos + else: + tiempo_txt = "00:00:00" + velocidad = 0 + + self.overlay_vars["estado"].set(f"ESTADO: {self.estado_actual[:18]}") + self.overlay_vars["progreso"].set(f"{total_procesados}/{self.total_registros} {porcentaje}%") + self.overlay_vars["ok"].set(f"OK: {self.procesados_ok}") + self.overlay_vars["err"].set(f"ERR: {self.errores} | EX: {getattr(self, 'existentes', 0)}") + self.overlay_vars["rei"].set(f"REI: {self.reinicios}") + self.overlay_vars["rad"].set(f"RAD: {str(self.registro_actual)[:18]}") + self.overlay_vars["vel"].set(f"VEL: {velocidad:.1f} rad/min") + self.overlay_vars["tiempo"].set(f"TIME: {tiempo_txt}") + self.overlay_vars["ultimo"].set(f"ULT: {getattr(self, 'ultimo_evento', '-')[:18]}") + self.overlay.update_idletasks() + except Exception: + pass + + def preparar_reportes_ejecucion(self): + """Crea una carpeta única por ejecución para no sobrescribir reportes anteriores.""" + self.run_id = datetime.now().strftime("%Y%m%d_%H%M%S") + base_dir = os.path.dirname(self.csv_path.get()) if self.csv_path.get() else os.getcwd() + self.output_dir = os.path.join(base_dir, REPORTS_DIR_NAME, self.run_id) + os.makedirs(self.output_dir, exist_ok=True) + + def ruta_reporte(self, nombre_base, extension): + return os.path.join(self.output_dir, f"{nombre_base}_{self.run_id}.{extension}") + + def registrar_omitido(self, row, tipo_error, detalle_error, accion_bot="Omitido y reinicio de flujo"): + """Registra omitidos con estructura estándar y error exacto.""" + radicado = row.get("radicado", "Desconocido") if row else "Desconocido" + duracion = int(time.time() - self.inicio_registro) if self.inicio_registro else 0 + + registro = dict(row) if row else {} + registro.update({ + "fecha_hora_error": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "estado": "OMITIDO", + "tipo_error": tipo_error, + "detalle_error": detalle_error, + "accion_bot": accion_bot, + "duracion_segundos": duracion, + }) + + self.omitted_records.append(registro) + self.omitidos += 1 + self.errores += 1 + self.guardar_resultado( + radicado, + "ERROR", + tipo_error, + detalle_error, + accion_bot, + duracion_segundos=duracion, + tipo_error=tipo_error + ) + self.log(f"OMITIDO: Radicado {radicado} | {tipo_error} | {detalle_error}") + self.set_estado_overlay("ERROR", tipo_error) + + def browse_file(self): + filename = filedialog.askopenfilename( + title="Seleccionar archivo CSV", + filetypes=(("Archivos CSV", "*.csv"), ("Todos los archivos", "*.*")) + ) + if filename: + self.csv_path.set(filename) + + def limpiar_clipboard(self): + """Limpia el portapapeles para evitar leer valores antiguos.""" + try: + self.root.clipboard_clear() + self.root.update() + except Exception: + pass + + def leer_clipboard(self): + """Lee el texto actual del portapapeles de Windows.""" + try: + return self.root.clipboard_get().strip() + except Exception: + return "" + + def fecha_tiene_numeros(self, texto): + """Valida si el campo copiado parece tener una fecha real. + Ejemplos: + - '09/08/2025' => True + - '//' o '' => False + """ + texto = str(texto).strip() + return any(c.isdigit() for c in texto) + + def fecha_esta_vacia_opencv(self, window): + """ + Detecta si el campo Fecha del servicio está vacío usando OpenCV + templates. + + Requiere estos archivos en la carpeta template/: + - fecha_vacia_slash.png -> recorte del campo cuando muestra // o __/__/____ + - fecha_vacia_blanco.png -> recorte del campo amarillo totalmente vacío + + Si detecta alguno de los dos, el bot debe digitar la fecha del CSV. + Si no detecta ninguno, asume que ya hay una fecha precargada y no la toca. + """ + if not OPENCV_AVAILABLE: + self.log("OpenCV no disponible para detectar fecha. Se asume fecha vacía.") + return True + + detecto_slash = False + detecto_blanco = False + + if os.path.exists(TEMPLATE_FECHA_VACIA_SLASH_PATH): + detecto_slash = self.check_template_opencv( + window, + TEMPLATE_FECHA_VACIA_SLASH_PATH, + "fecha_vacia_slash.png" + ) + else: + self.log(f"Template no encontrado: {TEMPLATE_FECHA_VACIA_SLASH_PATH}") + + if os.path.exists(TEMPLATE_FECHA_VACIA_BLANCO_PATH): + detecto_blanco = self.check_template_opencv( + window, + TEMPLATE_FECHA_VACIA_BLANCO_PATH, + "fecha_vacia_blanco.png" + ) + else: + self.log(f"Template no encontrado: {TEMPLATE_FECHA_VACIA_BLANCO_PATH}") + + if detecto_slash: + self.log("Fecha vacía detectada por template slash.") + + if detecto_blanco: + self.log("Fecha vacía detectada por template blanco.") + + return detecto_slash or detecto_blanco + + def check_popup_nombres_opencv(self, window): + """Detecta el popup azul de búsqueda por nombres/apellidos.""" + if not OPENCV_AVAILABLE: + return False + + if not os.path.exists(TEMPLATE_POPUP_NOMBRES_PATH): + return False + + return self.check_template_opencv( + window, + TEMPLATE_POPUP_NOMBRES_PATH, + "popup_nombres.png" + ) + + def check_radicado_existente_opencv(self, window): + """Detecta el popup que indica que el radicado ya fue procesado/validado.""" + if not OPENCV_AVAILABLE: + return False + + if not os.path.exists(TEMPLATE_RADICADO_EXISTENTE_PATH): + return False + + return self.check_template_opencv( + window, + TEMPLATE_RADICADO_EXISTENTE_PATH, + "radicado_existente.png" + ) + + def cerrar_popup_especial(self, window): + """Cierra popups especiales con teclado para evitar depender de coordenadas.""" + try: + window.wrapper_object().type_keys("{ENTER}", set_foreground=True) + time.sleep(0.3) + except Exception: + pass + + try: + window.wrapper_object().type_keys("{ESC}", set_foreground=True) + time.sleep(0.3) + except Exception: + pass + + def manejar_radicado_existente(self, window, row, val_radicado): + """Registra radicado existente y vuelve al menú seguro sin tratarlo como error técnico.""" + duracion = int(time.time() - self.inicio_registro) if self.inicio_registro else 0 + + self.guardar_resultado( + val_radicado, + "RADICADO_EXISTENTE", + "Radicado previamente procesado", + "La aplicación indicó que ya se realizó validación del NIT y valor de la factura.", + "Cerró popup y volvió al menú principal", + duracion_segundos=duracion, + tipo_error="" + ) + + self.existentes += 1 + self.log(f"RADICADO EXISTENTE: {val_radicado}. Se registra y vuelve al menú principal.") + self.set_estado_overlay("EXISTENTE", "Radicado existente") + self.save_reports() + + self.cerrar_popup_especial(window) + + # Volver al menú seguro, sin registrar como omitido técnico. + time.sleep(0.2) + click_abs(window, 366, 172) + time.sleep(0.2) + click_abs(window, 341, 247) + + self.first_time = True + self.reinicios += 1 + self.set_estado_overlay("REINICIO", "Menú principal") + + + def log(self, message): + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + full_msg = f"[{timestamp}] {message}" + self.log_entries.append(full_msg) + + self.log_area.config(state="normal") + self.log_area.insert(tk.END, message + "\n") + self.log_area.see(tk.END) + self.log_area.config(state="disabled") + + def guardar_resultado(self, radicado, estado, motivo, detalle, accion_bot, duracion_segundos=None, tipo_error=""): + """Guarda una trazabilidad del resultado de cada radicado.""" + if duracion_segundos is None: + duracion_segundos = int(time.time() - self.inicio_registro) if self.inicio_registro else 0 + + self.resultados.append({ + "radicado": radicado, + "estado": estado, + "tipo_error": tipo_error, + "motivo": motivo, + "detalle": detalle, + "fecha_hora": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "duracion_segundos": duracion_segundos, + "accion_bot": accion_bot, + }) + + def save_reports(self): + try: + os.makedirs(self.output_dir, exist_ok=True) + + log_filename = self.ruta_reporte("ejecucion_bot", "log") + with open(log_filename, "a", encoding="utf-8") as f: + for entry in self.log_entries: + f.write(entry + "\n") + + self.log_entries = [] + + if self.omitted_records: + omitted_filename = self.ruta_reporte("OMITIDOS", "csv") + fieldnames = [] + for registro in self.omitted_records: + for key in registro.keys(): + if key not in fieldnames: + fieldnames.append(key) + + with open(omitted_filename, "w", newline="", encoding="utf-8-sig") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + writer.writerows(self.omitted_records) + + self.log(f"Archivo de omitidos actualizado: {omitted_filename}") + + if self.resultados: + resultados_filename = self.ruta_reporte("RESULTADOS", "csv") + with open(resultados_filename, "w", newline="", encoding="utf-8-sig") as f: + fieldnames = [ + "radicado", "estado", "tipo_error", "motivo", "detalle", + "fecha_hora", "duracion_segundos", "accion_bot" + ] + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(self.resultados) + + self.log(f"Archivo de resultados actualizado: {resultados_filename}") + + except Exception as e: + print(f"Error al guardar reportes: {e}") + + def restart_bot(self, window, row=None, tipo_error="ERROR_VALIDACION", detalle_error="Se detectó una validación/error en pantalla."): + if row: + self.registrar_omitido( + row=row, + tipo_error=tipo_error, + detalle_error=detalle_error, + accion_bot="Omitido y reinicio de flujo" + ) + self.save_reports() + + time.sleep(0.05) + + try: + click_abs(window, COORD_CLOSE_ERROR[0], COORD_CLOSE_ERROR[1]) + except Exception: + pass + + time.sleep(0.2) + click_abs(window, 366, 172) + time.sleep(0.2) + click_abs(window, 341, 247) + + self.first_time = True + self.reinicios += 1 + self.log("Bot reiniciado exitosamente.") + self.set_estado_overlay("REINICIO", "Flujo reiniciado") + + def navegar_menu_inicial(self, window): + """Navega al menú principal después de abrir Integración Total.""" + self.log("Navegando al menú inicial...") + + # 1. Seleccionar Cuentas médicas + click_abs(window, 67, 167) + time.sleep(0.3) + + # 2. Desplegar Cuentas médicas + window.wrapper_object().type_keys("{RIGHT}", set_foreground=True) + time.sleep(0.3) + + # 3. Bajar a Digitación / Digitalización + window.wrapper_object().type_keys("{DOWN}", set_foreground=True) + time.sleep(0.3) + + # 4. Desplegar Digitación / Digitalización + window.wrapper_object().type_keys("{RIGHT}", set_foreground=True) + time.sleep(1) + + # 5. Clic en Digitar radicación + click_abs(window, 385, 177) + time.sleep(1) + + # 6. Clic en Digitación documentos + click_abs(window, 338, 234) + time.sleep(1) + + # IMPORTANTE: después de reiniciar, el siguiente registro debe comportarse como primera vez + self.first_time = True + self.log("Menú listo para radicar.") + + def reinicio_completo_app(self): + """Cierra completamente la app, la abre nuevamente, navega al menú y reconecta pywinauto.""" + try: + self.reinicios += 1 + self.set_estado_overlay("REINICIO APP", "Reinicio completo") + self.log("Reinicio completo preventivo de la aplicación...") + + # Cerrar Integración Total + os.system(f'taskkill /f /im "{PROCESS_NAME}"') + time.sleep(3) + + # Abrir desde el acceso directo validado + os.startfile(APP_PATH) + + self.log("Esperando carga de la aplicación...") + time.sleep(12) + + # Reconectar con la ventana nueva + app = Application(backend="uia").connect(title_re=".*Integracion Total Fase 2.*") + window = app.window(title_re=".*Integracion Total Fase 2.*") + + window.wait("visible enabled ready", timeout=20) + window.wrapper_object().set_focus() + + # Dejar la app en el menú correcto + self.navegar_menu_inicial(window) + + self.first_time = True + self.log("Aplicación reiniciada correctamente.") + return app, window + + except Exception as e: + self.log(f"ERROR reiniciando aplicación: {e}") + return None, None + + def convert_date(self, date_str): + if not date_str: + return "" + + input_formats = [ + "%d/%m/%Y", + "%m/%d/%Y", + "%Y-%m-%d", + "%Y/%m/%d", + "%d-%m-%Y", + "%m-%d-%Y", + ] + + user_fmt = "%d/%m/%Y" if self.date_format.get() == "dd/mm/yyyy" else "%m/%d/%Y" + + if user_fmt in input_formats: + input_formats.remove(user_fmt) + + input_formats.insert(0, user_fmt) + + date_clean = str(date_str).strip() + + for fmt in input_formats: + try: + dt_obj = datetime.strptime(date_clean, fmt) + return dt_obj.strftime("%m%d%Y") + except ValueError: + continue + + self.log(f"ADVERTENCIA: No se pudo convertir la fecha '{date_str}'. Se usará tal cual.") + return date_str + + def limpiar_valor_monetario(self, valor): + if valor is None: + return 0.0 + + texto = str(valor).strip() + + if texto == "": + return 0.0 + + texto = texto.replace("$", "").replace(" ", "") + + if "." in texto and "," not in texto: + partes = texto.split(".") + if len(partes[-1]) == 3: + texto = texto.replace(".", "") + + texto = texto.replace(",", ".") + + try: + return float(texto) + except Exception: + return 0.0 + + def formatear_valor_para_digitar(self, valor): + numero = self.limpiar_valor_monetario(valor) + + if numero == int(numero): + return str(int(numero)) + + return str(numero).replace(".", ",") + + def valor_mayor_a_cero(self, valor): + return self.limpiar_valor_monetario(valor) > 0 + + def get_valid_doc_types(self, doc_value): + doc_value = str(doc_value).strip() + + if not doc_value: + return [] + + length = len(doc_value) + is_numeric = doc_value.isdigit() + valid_types = [] + + if length in [10, 12]: + valid_types.append("AS") + if 3 <= length <= 11: + valid_types.append("CD") + if 3 <= length <= 7 and is_numeric: + valid_types.append("CE") + if 3 <= length <= 10 and length != 9 and is_numeric: + valid_types.append("CC") + if 9 <= length <= 14 and is_numeric: + valid_types.append("CN") + if length in [10, 12]: + valid_types.append("ME") + if length == 9 and is_numeric: + valid_types.append("NI") + if length == 9 and is_numeric: + valid_types.append("NU") + if 3 <= length <= 16: + valid_types.append("PS") + if length == 15 and is_numeric: + valid_types.append("PE") + if 7 <= length <= 8 and is_numeric: + valid_types.append("PT") + if length == 9 and is_numeric: + valid_types.append("PJ") + if length == 10 and is_numeric: + valid_types.append("RC") + if 1 <= length <= 8 and is_numeric: + valid_types.append("SC") + if length == 10 and is_numeric: + valid_types.append("TI") + + return valid_types + + def check_error_opencv(self, window): + if not OPENCV_AVAILABLE: + self.log("OpenCV no disponible. Asumiendo OK.") + return False + + if not os.path.exists(TEMPLATE_ERROR_PATH): + self.log(f"Template no encontrado en {TEMPLATE_ERROR_PATH}. Asumiendo OK.") + return False + + try: + screenshot_path = "temp_check.png" + img_cap = window.capture_as_image() + img_cap.save(screenshot_path) + + img_rgb = cv2.imread(screenshot_path) + template = cv2.imread(TEMPLATE_ERROR_PATH) + + if img_rgb is None or template is None: + self.log("Error al cargar imágenes para OpenCV.") + return False + + img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY) + template_gray = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY) + + res = cv2.matchTemplate(img_gray, template_gray, cv2.TM_CCOEFF_NORMED) + threshold = 0.7 + loc = np.where(res >= threshold) + + if len(loc[0]) > 0: + self.log("Error detectado mediante Template Matching.") + return True + + return False + + except Exception as e: + self.log(f"Error en chequeo OpenCV: {e}") + return False + + def check_template_opencv(self, window, template_path, nombre="template"): + """ + Detecta una imagen template en pantalla usando OpenCV. + Retorna True si el score supera el umbral. + """ + if not OPENCV_AVAILABLE: + return False + + if not os.path.exists(template_path): + self.log(f"Template no encontrado: {template_path}") + return False + + try: + screenshot_path = "temp_check_visual.png" + img_cap = window.capture_as_image() + img_cap.save(screenshot_path) + + img_rgb = cv2.imread(screenshot_path) + template = cv2.imread(template_path) + + if img_rgb is None or template is None: + self.log(f"No se pudo leer imagen/template: {nombre}") + return False + + img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY) + template_gray = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY) + + res = cv2.matchTemplate(img_gray, template_gray, cv2.TM_CCOEFF_NORMED) + _, max_val, _, _ = cv2.minMaxLoc(res) + + self.log(f"Score {nombre}: {max_val:.2f}") + + return max_val >= 0.60 + + except Exception as e: + self.log(f"Error en chequeo OpenCV de {nombre}: {e}") + return False + + def check_diferencias_abierto_opencv(self, window): + """ + Detecta si está abierto el panel Diferencias carátula-detalle + usando el template flecha_diferencias_abierto.png. + """ + return self.check_template_opencv( + window, + TEMPLATE_DIFERENCIAS_PATH, + "flecha_diferencias_abierto.png" + ) + + def check_diferencias_cerrado_opencv(self, window): + return self.check_template_opencv( + window, + TEMPLATE_DIFERENCIAS_CERRADO_PATH, + "flecha_diferencias_cerrado.png" + ) + + def normalizar_pantalla_radicacion(self, window): + try: + score_abierto = self.get_template_score( + window, + TEMPLATE_DIFERENCIAS_PATH, + "flecha_diferencias_abierto.png" + ) + + score_cerrado = self.get_template_score( + window, + TEMPLATE_DIFERENCIAS_CERRADO_PATH, + "flecha_diferencias_cerrado.png" + ) + + self.log(f"Score abierto: {score_abierto:.2f} | Score cerrado: {score_cerrado:.2f}") + + if score_abierto > score_cerrado and score_abierto >= 0.60: + self.log("Panel confirmado ABIERTO por score. Cerrando...") + click_abs(window, 558, 192) + time.sleep(1) + else: + self.log("Panel confirmado CERRADO o incierto. No se toca.") + + except Exception as e: + self.log(f"Error normalizando: {e}") + + def get_template_score(self, window, template_path, nombre="template"): + if not OPENCV_AVAILABLE or not os.path.exists(template_path): + return 0.0 + + try: + screenshot_path = "temp_score_visual.png" + img_cap = window.capture_as_image() + img_cap.save(screenshot_path) + + img_rgb = cv2.imread(screenshot_path) + template = cv2.imread(template_path) + + if img_rgb is None or template is None: + return 0.0 + + img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY) + template_gray = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY) + + res = cv2.matchTemplate(img_gray, template_gray, cv2.TM_CCOEFF_NORMED) + _, max_val, _, _ = cv2.minMaxLoc(res) + + return max_val + + except Exception: + return 0.0 + + def check_success_opencv(self, window): + return self.check_template_opencv( + window, + TEMPLATE_SUCCESS_PATH, + "success.png" + ) + + def check_bien_opencv(self, window): + return self.check_template_opencv( + window, + TEMPLATE_BIEN_PATH, + "bien.png" + ) + + def esperar_resultado_registro(self, window, timeout=TIMEOUT_REGISTRO): + """ + Espera hasta que la app responda después de registrar. + Retorna: OK, BIEN, ERROR o TIMEOUT. + + IMPORTANTE: + Se deja un tiempo de gracia inicial para evitar falsos positivos, + porque durante la carga la pantalla se oscurece y puede parecerse + visualmente al template de error. + """ + self.log(f"Esperando respuesta de registro hasta {timeout} segundos...") + + if not os.path.exists(TEMPLATE_SUCCESS_PATH): + self.log(f"ADVERTENCIA: No existe {TEMPLATE_SUCCESS_PATH}. No se podrá detectar success.png.") + + if not os.path.exists(TEMPLATE_BIEN_PATH): + self.log(f"ADVERTENCIA: No existe {TEMPLATE_BIEN_PATH}. No se podrá detectar bien.png.") + + # Ventana de gracia: no detectar error mientras la pantalla está cargando. + self.log("Tiempo de gracia inicial para evitar falso positivo de error (4s)...") + time.sleep(4) + + inicio = time.time() + + while time.time() - inicio < timeout: + if self.check_success_opencv(window): + return "OK" + + if self.check_bien_opencv(window): + return "BIEN" + + if self.check_error_opencv(window): + return "ERROR" + + time.sleep(1) + + return "TIMEOUT" + + def start_thread(self): + if not self.csv_path.get(): + messagebox.showwarning("Error", "Por favor seleccione un archivo CSV primero.") + return + + if self.is_running: + return + + self.is_running = True + self.btn_start.config(state="disabled", text="Ejecutando...") + + threading.Thread(target=self.run_process, daemon=True).start() + + def run_process(self): + try: + self.log("Conectando a la aplicación...") + + app = Application(backend="uia").connect(title_re=".*Integracion Total Fase 2.*") + window = app.window(title_re=".*Integracion Total Fase 2.*") + + self.preparar_reportes_ejecucion() + self.omitted_records = [] + self.log_entries = [] + self.resultados = [] + self.procesados_ok = 0 + self.errores = 0 + self.omitidos = 0 + self.existentes = 0 + self.reinicios = 0 + self.registro_actual = "-" + self.estado_actual = "Conectando" + self.ultimo_evento = "Inicio" + self.inicio_proceso = time.time() + self.first_time = True + self.actualizar_overlay() + + self.log(f"Iniciando nuevo proceso. Reportes en: {self.output_dir}") + + try: + window.wait("visible enabled ready", timeout=5) + except Exception as e: + self.log(f"Advertencia en navegación inicial: {e}") + + data = [] + + with open(self.csv_path.get(), newline="", encoding="utf-8") as csvfile: + reader = csv.DictReader(csvfile, delimiter=self.delimiter.get()) + + columnas_requeridas = { + "radicado", + "documento", + "ingreso", + "total_cargos", + "autorizacion", + "tipo_documento", + "tipo_servicio", + "copago", + "cuota_moderadora", + } + + headers = [h.strip() for h in reader.fieldnames] if reader.fieldnames else [] + reader.fieldnames = headers + + if not columnas_requeridas.issubset(headers): + missing = columnas_requeridas - set(headers) + raise ValueError( + "El CSV no tiene las columnas requeridas.\n" + f"Faltan: {', '.join(missing)}\n" + "Verifique si el delimitador es correcto." + ) + + for row in reader: + data.append(row) + + self.total_registros = len(data) + self.actualizar_overlay() + self.log(f"Se encontraron {len(data)} registros para procesar.") + + for i, row in enumerate(data): + self.inicio_registro = time.time() + self.registro_actual = row.get('radicado', 'N/A') + self.set_estado_overlay("PROCESANDO", f"Registro {i + 1}") + self.log(f"--- Procesando registro {i + 1}/{len(data)}: {row.get('radicado', 'N/A')} ---") + + val_radicado = row.get("radicado", "").strip() + val_total_cargos = row.get("total_cargos", "0").strip() + val_identificacion = row.get("documento", "").strip() + val_fecha = row.get("ingreso", "").strip() + val_auth = row.get("autorizacion", "").strip() + val_tipo_doc = row.get("tipo_documento", "").strip().upper() + val_service = row.get("tipo_servicio", "").strip().lower() + val_copago = row.get("copago", "").strip() + val_cuota_mod = row.get("cuota_moderadora", "").strip() + + val_fecha_final = self.convert_date(val_fecha) + + click_abs(window, 926, 309) + + time.sleep(0.2) + window.wrapper_object().type_keys( + val_radicado, + with_spaces=True, + set_foreground=True + ) + + time.sleep(0.2) + click_abs(window, 960, 531) + + # Validación inmediata después del botón de búsqueda del radicado + time.sleep(0.5) + if self.check_radicado_existente_opencv(window): + self.manejar_radicado_existente(window, row, val_radicado) + continue + + if self.check_error_opencv(window): + self.log("Error al procesar el registro.") + self.restart_bot(window, row, "ERROR_RADICADO", "Error visual después de consultar el radicado.") + continue + + time.sleep(0.2) + click_abs(window, 1375, 305) + + time.sleep(0.2) + click_abs(window, 1041, 329) + + if val_service: + if val_service in SERVICES_COORDS: + self.log(f"Tipo de servicio: {val_service}") + time.sleep(0.2) + click_abs( + window, + SERVICES_COORDS[val_service][0], + SERVICES_COORDS[val_service][1] + ) + else: + self.log(f"ERROR: El tipo de servicio '{val_service}' no es válido.") + self.restart_bot(window, row, "ERROR_TIPO_SERVICIO", f"Tipo de servicio no configurado: {val_service}") + continue + else: + self.log("ERROR: El campo 'tipo_servicio' está vacío.") + self.restart_bot(window, row, "ERROR_TIPO_SERVICIO", "El campo tipo_servicio está vacío.") + continue + + # IMPORTANTE: + # Antes aquí se hacía clic en "Cuota a cobrar" y se digitaba un 0. + # Eso dañaba el valor/foco cuando la aplicación ya traía la cuota o cuando + # el campo no debía tocarse. Ahora NO se escribe nada en cuota a cobrar. + # Después de seleccionar el tipo de servicio, se ejecuta la búsqueda directamente. + time.sleep(0.2) + click_abs(window, 1300, 302) + + self.log("Esperando carga (7s)...") + time.sleep(7) + + if self.check_error_opencv(window): + self.log("Error al procesar el registro.") + self.restart_bot(window, row, "ERROR_CARGA_DETALLE", "Error visual después de cargar el detalle del radicado.") + continue + + self.normalizar_pantalla_radicacion(window) + + time.sleep(0.2) + click_abs(window, 730, 321) + + time.sleep(0.2) + click_abs(window, 700, 359) + + candidatos_validos = self.get_valid_doc_types(val_identificacion) + candidatos_ordenados = [t for t in DOC_STRATEGY if t in candidatos_validos] + + tipos_a_probar = [] + + if val_tipo_doc and val_tipo_doc in DOC_COORDS: + tipos_a_probar.append(val_tipo_doc) + + for t in candidatos_ordenados: + if t != val_tipo_doc: + tipos_a_probar.append(t) + else: + if not candidatos_ordenados: + self.log( + f"ADVERTENCIA: El documento {val_identificacion} " + "no cumple reglas conocidas. Se probará estrategia completa." + ) + tipos_a_probar = DOC_STRATEGY + else: + tipos_a_probar = candidatos_ordenados + + self.log(f"Estrategia de documentos a probar: {tipos_a_probar}") + + documento_exitoso = False + error_documento_fatal = False + + for tipo_doc in tipos_a_probar: + self.log(f"Probando tipo de documento: {tipo_doc}") + + time.sleep(1) + click_abs(window, 729, 279) + + coords_doc = DOC_COORDS.get(tipo_doc, (0, 0)) + + if coords_doc == (0, 0): + self.log(f"Advertencia: Coordenadas para {tipo_doc} no configuradas. Saltando.") + continue + + time.sleep(0.5) + click_abs(window, coords_doc[0], coords_doc[1]) + + time.sleep(0.2) + click_abs(window, 1015, 280) + + time.sleep(0.2) + window.wrapper_object().type_keys("^a{DELETE}", set_foreground=True) + time.sleep(0.1) + window.wrapper_object().type_keys( + val_identificacion, + with_spaces=True, + set_foreground=True + ) + + # ========================= + # FECHA SERVICIO INTELIGENTE POR OPENCV + # ========================= + # La fecha puede venir precargada. Para no sobrescribirla: + # - Si detecta template de fecha vacía (slash o blanco), digita la fecha del CSV. + # - Si NO detecta fecha vacía, asume que ya hay fecha cargada y no la toca. + time.sleep(0.05) + click_abs(window, 1172, 276) + time.sleep(0.05) + + if self.fecha_esta_vacia_opencv(window): + self.log(f"Fecha vacía detectada por OpenCV. Digitando fecha CSV: {val_fecha_final}") + + time.sleep(0.1) + window.wrapper_object().type_keys( + val_fecha_final, + pause=0.1, + with_spaces=False, + set_foreground=True + ) + + time.sleep(0.1) + window.wrapper_object().type_keys("{TAB}{ENTER}", set_foreground=True) + + else: + self.log("Fecha precargada detectada por OpenCV. No se modifica.") + + time.sleep(0.1) + window.wrapper_object().type_keys("{TAB}{ENTER}", set_foreground=True) + + time.sleep(1) + + if self.check_popup_nombres_opencv(window): + self.log("Popup de búsqueda por nombres detectado. Se omite el registro.") + self.cerrar_popup_especial(window) + self.restart_bot( + window, + row, + "ERROR_BUSQUEDA_DOCUMENTO", + "La aplicación abrió el popup de búsqueda por nombres/apellidos al validar el documento." + ) + error_documento_fatal = True + break + + if self.check_error_opencv(window): + self.log(f"Error detectado con {tipo_doc}. Intentando siguiente...") + click_abs(window, COORD_CLOSE_ERROR[0], COORD_CLOSE_ERROR[1]) + time.sleep(1) + continue + else: + self.log("Validación OpenCV exitosa.") + documento_exitoso = True + break + + if error_documento_fatal: + continue + + if not documento_exitoso: + self.log( + f"ERROR: No se pudo procesar el documento {val_identificacion} " + "con ningún tipo. Omitiendo y reiniciando bot." + ) + self.restart_bot(window, row, "ERROR_DOCUMENTO", f"No validó documento {val_identificacion} con tipos probados: {', '.join(tipos_a_probar)}") + continue +#posible clic de antes de autorizar (xd) + time.sleep(0.6) + click_abs(window, 1126, 352) + + time.sleep(0.2) + window.wrapper_object().type_keys( + val_auth, + with_spaces=True, + set_foreground=True + ) + + time.sleep(0.2) + window.wrapper_object().type_keys("{TAB}{ENTER}", set_foreground=True) + + if self.check_error_opencv(window): + self.log("Error al procesar el registro.") + self.restart_bot(window, row, "ERROR_AUTORIZACION", "Error visual después de ingresar autorización.") + continue + + time.sleep(0.3) + click_abs(window, 741, 421) + + time.sleep(0.2) + window.wrapper_object().type_keys("{SPACE}", set_foreground=True) + + time.sleep(0.3) + click_abs(window, 740, 535) + + # ========================= + # TOTAL CARGOS + COPAGO/MODERADORA EXCLUYENTES + AGREGAR + # ========================= + time.sleep(0.3) + self.log(f"Digitando total_cargos: {val_total_cargos}") + + click_abs(window, 1105, 658) + time.sleep(0.2) + window.wrapper_object().type_keys( + val_total_cargos, + with_spaces=True, + set_foreground=True + ) + + copago_txt = self.formatear_valor_para_digitar(val_copago) + cuota_mod_txt = self.formatear_valor_para_digitar(val_cuota_mod) + + copago_mayor = self.valor_mayor_a_cero(val_copago) + cuota_mod_mayor = self.valor_mayor_a_cero(val_cuota_mod) + + if copago_mayor: + self.log(f"Digitando copago: {copago_txt}") + + # 2 TAB → Vlr Copago + time.sleep(0.3) + window.wrapper_object().type_keys("{TAB}{TAB}", set_foreground=True) + + time.sleep(0.3) + window.wrapper_object().type_keys( + copago_txt, + with_spaces=True, + set_foreground=True + ) + + # Como cuota moderadora se bloquea, NO se toca. + # 3 TAB → botón Agregar, ENTER + time.sleep(0.3) + window.wrapper_object().type_keys("{TAB}{TAB}{TAB}", set_foreground=True) + + time.sleep(0.3) + window.wrapper_object().type_keys("{ENTER}", set_foreground=True) + + elif cuota_mod_mayor: + self.log(f"Digitando cuota moderadora: {cuota_mod_txt}") + + # 2 TAB → Vlr Copago + time.sleep(0.3) + window.wrapper_object().type_keys("{TAB}{TAB}", set_foreground=True) + + # 1 TAB → Vlr Cuota Mod + time.sleep(0.3) + window.wrapper_object().type_keys("{TAB}", set_foreground=True) + + time.sleep(0.3) + window.wrapper_object().type_keys( + cuota_mod_txt, + with_spaces=True, + set_foreground=True + ) + + # 2 TAB → botón Agregar, ENTER + time.sleep(0.3) + window.wrapper_object().type_keys("{TAB}{TAB}", set_foreground=True) + + time.sleep(0.3) + window.wrapper_object().type_keys("{ENTER}", set_foreground=True) + + else: + self.log("Copago y cuota moderadora en 0. Va directo a agregar.") + + # total_cargos → 2 TAB copago → 1 TAB moderadora → 2 TAB agregar + time.sleep(0.2) + window.wrapper_object().type_keys("{TAB}{TAB}{TAB}{TAB}{TAB}", set_foreground=True) + + time.sleep(0.2) + window.wrapper_object().type_keys("{ENTER}", set_foreground=True) + + self.log("Esperando confirmación visual antes de continuar al siguiente radicado...") + resultado_registro = self.esperar_resultado_registro(window, TIMEOUT_REGISTRO) + + if resultado_registro in ("OK", "BIEN"): + self.log(f"Registro exitoso/confirmación detectada: {resultado_registro}.") + self.guardar_resultado( + val_radicado, + "OK", + "Cuenta registrada", + f"Mensaje visual detectado en pantalla: {resultado_registro}.", + "Continuó" + ) + + # Mantiene los clics originales para cerrar confirmación y volver al menú seguro. + click_abs(window, 1229, 979) + + time.sleep(1) + click_abs(window, 1223, 666) + + time.sleep(0.5) + click_abs(window, 941, 305) + + self.procesados_ok += 1 + self.set_estado_overlay("OK", "Registro OK") + self.log(f"Registro {val_radicado} completado.") + time.sleep(1) + self.first_time = False + + elif resultado_registro == "ERROR": + self.log("Error detectado después de registrar. Se omite el registro.") + self.restart_bot(window, row, "ERROR_REGISTRO", "Error visual después de presionar agregar/registrar.") + continue + + else: + self.log("TIMEOUT: No apareció éxito ni error después de registrar.") + self.errores += 1 + self.guardar_resultado( + val_radicado, + "TIMEOUT", + "ERROR_TIMEOUT", + f"No apareció mensaje de éxito ni error en {TIMEOUT_REGISTRO} segundos.", + "Reinicio completo y continúa", + tipo_error="ERROR_TIMEOUT" + ) + self.set_estado_overlay("TIMEOUT", "Sin respuesta") + self.save_reports() + app, window = self.reinicio_completo_app() + if window is None: + self.log("No se pudo reiniciar después del timeout. Deteniendo proceso.") + break + continue + + # Reinicio completo preventivo para evitar lentitud acumulada de la app. + # No se ejecuta si ya era el último registro. + if (i + 1) % REINICIO_CADA == 0 and (i + 1) < len(data): + app, window = self.reinicio_completo_app() + + if window is None: + self.log("No se pudo reiniciar la aplicación. Deteniendo proceso.") + break + + self.set_estado_overlay("FINALIZADO", "Proceso finalizado") + self.log("--- PROCESO FINALIZADO ---") + self.save_reports() + messagebox.showinfo("Fin", "El proceso ha terminado exitosamente.") + + except Exception as e: + self.log(f"ERROR: {str(e)}") + messagebox.showerror("Error", f"Ocurrió un error:\n{str(e)}") + + finally: + self.is_running = False + try: + self.btn_start.config(state="normal", text="INICIAR BOT") + except Exception: + pass + + +if __name__ == "__main__": + root = tk.Tk() + app = BotSaludTotal(root) + root.mainloop() diff --git a/bots/bot_descarga/README.md b/bots/bot_descarga/README.md index d0893ce..3d63aad 100644 --- a/bots/bot_descarga/README.md +++ b/bots/bot_descarga/README.md @@ -37,6 +37,15 @@ python run_gui.py Configura URL API (ej. `https://pys1.builker.xyz/bot-descarga` o `http://127.0.0.1:8189/bot-descarga`), Worker ID y token. Pulsa **Conectar agent**. Queda idle hasta que el desktop asigne un job. +### Autoarranque en Windows + +En la misma GUI: + +1. Marca **Iniciar con Windows** (crea un `.bat` en la carpeta Startup del usuario). +2. Deja **Conectar al abrir** activo para que tras el login el agent quede online solo. + +Tras reiniciar el PC e iniciar sesión, se abre la GUI y se registra el worker automáticamente. + ## Arranque solo agent (CLI) ```bash diff --git a/bots/bot_descarga/bot_descarga/ui/dashboard.py b/bots/bot_descarga/bot_descarga/ui/dashboard.py index 3e5e21e..880508a 100644 --- a/bots/bot_descarga/bot_descarga/ui/dashboard.py +++ b/bots/bot_descarga/bot_descarga/ui/dashboard.py @@ -1,6 +1,7 @@ from __future__ import annotations import socket +import sys import threading import customtkinter as ctk @@ -11,6 +12,11 @@ from bot_descarga.ui.settings_store import ( normalize_api_url, save_gui_settings, ) +from bot_descarga.ui.windows_autostart import ( + disable_autostart, + enable_autostart, + is_autostart_enabled, +) from bot_descarga.worker.agent import WorkerAgent ctk.set_appearance_mode("dark") @@ -24,7 +30,7 @@ class BotDashboard(ctk.CTk): super().__init__() self.title("Bot de Descargas PDF — Agent") - self.geometry("920x680") + self.geometry("920x720") self.resizable(False, False) self._agent: WorkerAgent | None = None @@ -36,6 +42,10 @@ class BotDashboard(ctk.CTk): self.protocol("WM_DELETE_WINDOW", self._on_close) + # Tras login de Windows: si está marcado, conectar solo + if bool(load_gui_settings().get("auto_connect")): + self.after(800, self._maybe_auto_connect) + def _construir_ui(self): root = ctk.CTkFrame(self, corner_radius=18) root.pack(fill="both", expand=True, padx=18, pady=18) @@ -81,7 +91,7 @@ class BotDashboard(ctk.CTk): self.entry_worker.pack(side="left", fill="x", expand=True) row_token = ctk.CTkFrame(config, fg_color="transparent") - row_token.pack(fill="x", padx=18, pady=(0, 14)) + row_token.pack(fill="x", padx=18, pady=(0, 8)) ctk.CTkLabel(row_token, text="Token", width=90, anchor="w").pack(side="left") self.entry_token = ctk.CTkEntry( row_token, @@ -97,6 +107,23 @@ class BotDashboard(ctk.CTk): command=self._toggle_token, ).pack(side="left") + row_auto = ctk.CTkFrame(config, fg_color="transparent") + row_auto.pack(fill="x", padx=18, pady=(0, 14)) + self.chk_autostart = ctk.CTkCheckBox( + row_auto, + text="Iniciar con Windows", + command=self._on_autostart_toggle, + ) + self.chk_autostart.pack(side="left", padx=(0, 18)) + self.chk_auto_connect = ctk.CTkCheckBox( + row_auto, + text="Conectar al abrir", + command=self._on_auto_connect_toggle, + ) + self.chk_auto_connect.pack(side="left") + if sys.platform != "win32": + self.chk_autostart.configure(state="disabled") + status = ctk.CTkFrame(root, corner_radius=14) status.pack(fill="x", padx=24, pady=(0, 14)) @@ -158,19 +185,77 @@ class BotDashboard(ctk.CTk): ) self.text_logs.pack(fill="both", expand=True, padx=18, pady=(0, 16)) + def _persist_flags(self): + save_gui_settings( + normalize_api_url(self.entry_api.get()), + self.entry_worker.get(), + self.entry_token.get(), + autostart_windows=bool(self.chk_autostart.get()), + auto_connect=bool(self.chk_auto_connect.get()), + ) + def _cargar_settings(self): s = load_gui_settings() self.entry_api.insert(0, s.get("api_url") or DEFAULT_API_URL) self.entry_worker.insert(0, s.get("worker_id") or socket.gethostname()) self.entry_token.insert(0, s.get("token") or "") + want_autostart = bool(s.get("autostart_windows")) + if want_autostart and sys.platform == "win32": + try: + if not is_autostart_enabled(): + enable_autostart() + self.chk_autostart.select() + except Exception as e: + self._log(f"No se pudo restaurar autoarranque: {e}") + elif is_autostart_enabled(): + # Script presente aunque el flag esté off → marcar checkbox + self.chk_autostart.select() + + if bool(s.get("auto_connect")): + self.chk_auto_connect.select() + def _guardar_settings(self): api = normalize_api_url(self.entry_api.get()) self.entry_api.delete(0, "end") self.entry_api.insert(0, api) - save_gui_settings(api, self.entry_worker.get(), self.entry_token.get()) + self._persist_flags() self._log("Configuración guardada en data/gui_settings.json") + def _on_autostart_toggle(self): + if sys.platform != "win32": + return + enabled = bool(self.chk_autostart.get()) + try: + if enabled: + path = enable_autostart() + self._log(f"Autoarranque activado: {path}") + # Tiene sentido conectar al abrir si arranca con Windows + if not self.chk_auto_connect.get(): + self.chk_auto_connect.select() + else: + disable_autostart() + self._log("Autoarranque desactivado.") + self._persist_flags() + except Exception as e: + self._log(f"Error configurando autoarranque: {e}") + if enabled: + self.chk_autostart.deselect() + else: + self.chk_autostart.select() + + def _on_auto_connect_toggle(self): + self._persist_flags() + + def _maybe_auto_connect(self): + if self._connected: + return + if not self.entry_worker.get().strip() or not self.entry_token.get().strip(): + self._log("Auto-conectar omitido: falta Worker ID o token.") + return + self._log("Conectando automáticamente…") + self._conectar() + def _toggle_token(self): self._token_visible = not self._token_visible self.entry_token.configure(show="" if self._token_visible else "•") @@ -195,7 +280,7 @@ class BotDashboard(ctk.CTk): self.entry_api.delete(0, "end") self.entry_api.insert(0, api) - save_gui_settings(api, worker_id, token) + self._persist_flags() self._agent = WorkerAgent( api_url=api, @@ -251,6 +336,9 @@ class BotDashboard(ctk.CTk): self.entry_api.configure(state=state) self.entry_worker.configure(state=state) self.entry_token.configure(state=state) + if sys.platform == "win32": + self.chk_autostart.configure(state="normal") + self.chk_auto_connect.configure(state="normal") def _limpiar_logs(self): self.text_logs.delete("1.0", "end") diff --git a/bots/bot_descarga/bot_descarga/ui/settings_store.py b/bots/bot_descarga/bot_descarga/ui/settings_store.py index 01372db..71c181f 100644 --- a/bots/bot_descarga/bot_descarga/ui/settings_store.py +++ b/bots/bot_descarga/bot_descarga/ui/settings_store.py @@ -21,6 +21,8 @@ def load_gui_settings() -> Dict[str, Any]: "api_url": DEFAULT_API_URL, "worker_id": "", "token": DEFAULT_TOKEN, + "autostart_windows": False, + "auto_connect": False, } if not _SETTINGS_PATH.exists(): return defaults @@ -33,13 +35,26 @@ def load_gui_settings() -> Dict[str, Any]: return defaults -def save_gui_settings(api_url: str, worker_id: str, token: str) -> None: +def save_gui_settings( + api_url: str, + worker_id: str, + token: str, + *, + autostart_windows: bool | None = None, + auto_connect: bool | None = None, +) -> None: _SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True) + current = load_gui_settings() payload = { + **current, "api_url": api_url.strip(), "worker_id": worker_id.strip(), "token": token.strip(), } + if autostart_windows is not None: + payload["autostart_windows"] = bool(autostart_windows) + if auto_connect is not None: + payload["auto_connect"] = bool(auto_connect) _SETTINGS_PATH.write_text( json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8", diff --git a/bots/bot_descarga/bot_descarga/ui/windows_autostart.py b/bots/bot_descarga/bot_descarga/ui/windows_autostart.py new file mode 100644 index 0000000..8f70ca6 --- /dev/null +++ b/bots/bot_descarga/bot_descarga/ui/windows_autostart.py @@ -0,0 +1,74 @@ +"""Autoarranque con Windows (carpeta Startup del usuario).""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +TASK_BASENAME = "IPSExtractor-BotDescarga" + + +def bot_root() -> Path: + """Raíz del paquete bots/bot_descarga (donde está run_gui.py).""" + return Path(__file__).resolve().parents[2] + + +def startup_dir() -> Path: + appdata = os.environ.get("APPDATA") or str(Path.home() / "AppData" / "Roaming") + return Path(appdata) / "Microsoft" / "Windows" / "Start Menu" / "Programs" / "Startup" + + +def startup_script_path() -> Path: + return startup_dir() / f"{TASK_BASENAME}.bat" + + +def resolve_python(root: Path | None = None) -> Path: + root = root or bot_root() + for candidate in ( + root / ".venv" / "Scripts" / "pythonw.exe", + root / ".venv" / "Scripts" / "python.exe", + root / ".venv" / "bin" / "python", + ): + if candidate.exists(): + return candidate + return Path(sys.executable) + + +def is_autostart_enabled() -> bool: + if sys.platform != "win32": + return False + return startup_script_path().exists() + + +def enable_autostart(root: Path | None = None) -> Path: + """Crea un .bat en Startup que abre la GUI del bot tras el login.""" + if sys.platform != "win32": + raise RuntimeError("El autoarranque solo está disponible en Windows.") + + root = (root or bot_root()).resolve() + gui = root / "run_gui.py" + if not gui.exists(): + raise FileNotFoundError(f"No se encontró run_gui.py en {root}") + + py = resolve_python(root) + dest = startup_script_path() + dest.parent.mkdir(parents=True, exist_ok=True) + + # start "" deja que el .bat termine; la GUI queda como proceso independiente. + content = ( + "@echo off\r\n" + f'cd /d "{root}"\r\n' + f'start "" "{py}" "{gui}"\r\n' + ) + dest.write_text(content, encoding="utf-8") + return dest + + +def disable_autostart() -> bool: + """Elimina el .bat de Startup. True si había algo que borrar.""" + path = startup_script_path() + if path.exists(): + path.unlink() + return True + return False diff --git a/bots/bot_descarga/bot_descarga/worker/agent.py b/bots/bot_descarga/bot_descarga/worker/agent.py index 7edc9a0..250f5b6 100644 --- a/bots/bot_descarga/bot_descarga/worker/agent.py +++ b/bots/bot_descarga/bot_descarga/worker/agent.py @@ -131,13 +131,14 @@ class WorkerAgent: def _run_job(self, job: dict) -> None: job_id = job["id"] + downloads_prefix = (job.get("downloads_prefix") or "").strip() or f"downloads/{job_id}/" self._uploaded_keys = [] work_dir = self.staging_root / job_id out_dir = work_dir / "out" out_dir.mkdir(parents=True, exist_ok=True) excel_path = work_dir / (Path(job.get("excel_key") or "input.xlsx").name) - self._log(f"[worker {self.worker_id}] job {job_id} iniciado") + self._log(f"[worker {self.worker_id}] job {job_id} iniciado (prefix={downloads_prefix})") try: self.api.download_excel(job_id, excel_path) @@ -168,7 +169,14 @@ class WorkerAgent: self._log(f"[worker] no se pudo enviar evento: {e}") def on_pdf_saved(nit: str, radicado: str, local_path: str) -> None: - key = self.minio.upload_pdf(job_id, nit, radicado, local_path) + key = self.minio.upload_pdf( + job_id, + nit, + radicado, + local_path, + downloads_prefix=downloads_prefix, + skip_if_exists=True, + ) self._uploaded_keys.append(key) service = DownloadService( @@ -181,7 +189,11 @@ class WorkerAgent: summary = service.run() try: - self.minio.write_manifest(job_id, self._uploaded_keys) + self.minio.write_manifest( + job_id, + self._uploaded_keys, + downloads_prefix=downloads_prefix, + ) except Exception as e: self._log(f"[worker] manifest no escrito: {e}") @@ -194,7 +206,7 @@ class WorkerAgent: "exitosos": summary.exitosos, "fallidos": fallidos_count, "cancelled": summary.cancelled, - "downloads_prefix": job.get("downloads_prefix"), + "downloads_prefix": downloads_prefix, "uploaded_count": len(self._uploaded_keys), } diff --git a/bots/bot_descarga/bot_descarga/worker/minio_upload.py b/bots/bot_descarga/bot_descarga/worker/minio_upload.py index 9dc5fe6..0b2b11c 100644 --- a/bots/bot_descarga/bot_descarga/worker/minio_upload.py +++ b/bots/bot_descarga/bot_descarga/worker/minio_upload.py @@ -40,10 +40,22 @@ class MinioSettings: ) +def pdf_key_from_prefix(downloads_prefix: str, nit: str, radicado: str) -> str: + prefix = (downloads_prefix or "").strip().rstrip("/") + if not prefix: + raise ValueError("downloads_prefix vacío") + return f"{prefix}/{nit}/{radicado}.pdf" + + def pdf_key(job_id: str, nit: str, radicado: str) -> str: return f"downloads/{job_id}/{nit}/{radicado}.pdf" +def manifest_key_from_prefix(downloads_prefix: str) -> str: + prefix = (downloads_prefix or "").strip().rstrip("/") + return f"{prefix}/_manifest.json" + + def manifest_key(job_id: str) -> str: return f"downloads/{job_id}/_manifest.json" @@ -89,8 +101,30 @@ class WorkerMinioUploader: self._client = _s3_client(self.settings) return self._client - def upload_pdf(self, job_id: str, nit: str, radicado: str, local_path: str | Path) -> str: - key = pdf_key(job_id, nit, radicado) + def object_exists(self, key: str) -> bool: + try: + self.client.head_object(Bucket=self.settings.bucket, Key=key) + return True + except Exception: + return False + + def upload_pdf( + self, + job_id: str, + nit: str, + radicado: str, + local_path: str | Path, + *, + downloads_prefix: str | None = None, + skip_if_exists: bool = True, + ) -> str: + """Sube PDF bajo downloads_prefix (padre) o downloads/{job_id}/.""" + if downloads_prefix: + key = pdf_key_from_prefix(downloads_prefix, nit, radicado) + else: + key = pdf_key(job_id, nit, radicado) + if skip_if_exists and self.object_exists(key): + return key path = Path(local_path) # put_object (no multipart) es más fiable en buckets anónimos. self.client.put_object( @@ -101,9 +135,23 @@ class WorkerMinioUploader: ) return key - def write_manifest(self, job_id: str, keys: List[str]) -> str: - key = manifest_key(job_id) - payload = {"job_id": job_id, "files": keys, "count": len(keys)} + def write_manifest( + self, + job_id: str, + keys: List[str], + *, + downloads_prefix: str | None = None, + ) -> str: + if downloads_prefix: + key = manifest_key_from_prefix(downloads_prefix) + else: + key = manifest_key(job_id) + payload = { + "job_id": job_id, + "downloads_prefix": downloads_prefix or f"downloads/{job_id}/", + "files": keys, + "count": len(keys), + } self.client.put_object( Bucket=self.settings.bucket, Key=key, diff --git a/bots/bot_descarga/tests/test_windows_autostart.py b/bots/bot_descarga/tests/test_windows_autostart.py new file mode 100644 index 0000000..18eb474 --- /dev/null +++ b/bots/bot_descarga/tests/test_windows_autostart.py @@ -0,0 +1,18 @@ +from pathlib import Path + +from bot_descarga.ui.windows_autostart import TASK_BASENAME, bot_root, resolve_python + + +def test_bot_root_has_run_gui(): + root = bot_root() + assert (root / "run_gui.py").exists() + + +def test_resolve_python_returns_path(): + py = resolve_python() + assert isinstance(py, Path) + assert str(py) + + +def test_task_basename(): + assert "BotDescarga" in TASK_BASENAME diff --git a/bots/bot_digitador/.env.example b/bots/bot_digitador/.env.example new file mode 100644 index 0000000..2405548 --- /dev/null +++ b/bots/bot_digitador/.env.example @@ -0,0 +1,7 @@ +BOT_API_URL=http://localhost:8189/bot-digitador +BOT_API_TOKEN=dev-token-change-me +BOT_WORKER_ID= +DIGITADOR_APP_PATH=C:\Users\Pysctas185\Documents\Integracion Total.lnk +MINIO_ENDPOINT=https://s3.builker.xyz +MINIO_BUCKET=pys-cuentas-medicas +MINIO_ANONYMOUS=1 diff --git a/bots/bot_digitador/README.md b/bots/bot_digitador/README.md new file mode 100644 index 0000000..14a998e --- /dev/null +++ b/bots/bot_digitador/README.md @@ -0,0 +1,92 @@ +# Bot Digitador (agent Windows) + +Agent que **digita** radicados en **Integración Total** (pywinauto + OpenCV). +Vive en este monorepo bajo `bots/` (no es un microservicio de `services/`). + +- **No descarga PDFs.** Solo digitación UI + reportes CSV/log. +- **No expone puertos.** Solo HTTP saliente al gateway IPSExtractor y PutObject a MinIO. +- La API de control se espera en `…/bot-digitador` (mismo patrón que `bot_descarga`). + +## Requisitos en la PC + +- Windows con **Integración Total** instalado e iniciado sesión +- Python 3.11+ +- Plantillas OpenCV en `template/` (ver [template/README.md](template/README.md)) +- Acceso de red al gateway (`BOT_API_URL`) y a MinIO +- Token `BOT_API_TOKEN` + +## Instalación + +```bash +cd bots/bot_digitador +python -m venv .venv +.venv\Scripts\activate +pip install -r requirements.txt +copy .env.example .env +# Editar .env (especialmente DIGITADOR_APP_PATH) +``` + +Copia los PNG de plantillas a `template/`. + +## Arranque con GUI + +```bash +python run_gui.py +``` + +Configura URL API (ej. `http://localhost:8189/bot-digitador`), Worker ID, token y +ruta de Integración Total (`DIGITADOR_APP_PATH`). Pulsa **Conectar agent**. +Queda idle hasta que el desktop asigne un job CSV. + +### Autoarranque en Windows + +1. Marca **Iniciar con Windows**. +2. Deja **Conectar al abrir** activo. + +## Arranque solo agent (CLI) + +```bash +python -m bot_digitador.worker \ + --api-url http://localhost:8189/bot-digitador \ + --worker-id PC-DIGITADOR-01 \ + --token %BOT_API_TOKEN% +``` + +## Modo local (sin API) + +Requiere Integración Total abierto y visible: + +```bash +python main.py --csv datos.csv +python main.py --csv datos.csv --delimiter "|" --date-format dd/mm/yyyy --app-path "C:\ruta\Integracion Total.lnk" +``` + +Reportes en la carpeta de salida: `RESULTADOS.csv`, `OMITIDOS.csv`, `ejecucion_bot.log`. + +## Arquitectura + +``` +Desktop IPSExtractor → Gateway :8189 → API bot-digitador + ↑ +PC bot (esta carpeta) ────────┘ HTTP outbound + └─ PutObject reportes → MinIO …/digitados/{job_id}/ + └─ pywinauto → Integración Total +``` + +## Variables + +Ver [`.env.example`](.env.example): + +| Variable | Uso | +|----------|-----| +| `BOT_API_URL` | Base del API vía gateway (`…/bot-digitador`) | +| `BOT_API_TOKEN` | Bearer compartido con el servicio | +| `DIGITADOR_APP_PATH` | Acceso directo / ejecutable de Integración Total | +| `MINIO_*` | Bucket de reportes; por defecto anónimo | + +## CSV de entrada + +Columnas requeridas: `radicado`, `documento`, `ingreso`, `total_cargos`, +`autorizacion`, `tipo_documento`, `tipo_servicio`, `copago`, `cuota_moderadora`. + +Acepta delimitador `,` o `|` (auto-detección). diff --git a/bots/bot_digitador/bot_digitador/__init__.py b/bots/bot_digitador/bot_digitador/__init__.py new file mode 100644 index 0000000..18c4249 --- /dev/null +++ b/bots/bot_digitador/bot_digitador/__init__.py @@ -0,0 +1,3 @@ +"""Bot Digitador — automatización de digitación en Integración Total.""" + +__version__ = "1.0.0" diff --git a/bots/bot_digitador/bot_digitador/core/__init__.py b/bots/bot_digitador/bot_digitador/core/__init__.py new file mode 100644 index 0000000..6a47aec --- /dev/null +++ b/bots/bot_digitador/bot_digitador/core/__init__.py @@ -0,0 +1,3 @@ +from bot_digitador.core.digitador_service import DigitadorRunSummary, DigitadorService + +__all__ = ["DigitadorService", "DigitadorRunSummary"] diff --git a/bots/bot_digitador/bot_digitador/core/coords.py b/bots/bot_digitador/bot_digitador/core/coords.py new file mode 100644 index 0000000..49edac2 --- /dev/null +++ b/bots/bot_digitador/bot_digitador/core/coords.py @@ -0,0 +1,58 @@ +"""Coordenadas, estrategia de documentos y constantes de Integración Total.""" + +from __future__ import annotations + +import os + +DOC_COORDS = { + "CC": (796, 343), + "TI": (796, 549), + "CE": (796, 329), + "AS": (796, 296), + "CD": (796, 314), + "CN": (796, 371), + "ME": (796, 378), + "NI": (796, 412), + "NU": (796, 453), + "PS": (796, 450), + "PE": (796, 467), + "PT": (796, 483), + "PJ": (796, 503), + "RC": (796, 520), + "SC": (796, 536), +} + +DOC_STRATEGY = [ + "CC", "TI", "CE", "RC", "PS", "PT", "PE", + "AS", "ME", "CN", "SC", "CD", "NI", "NU", "PJ", +] + +SERVICES_COORDS = { + "ambulancias y elem.": (1033, 367), + "ambulatorio": (1033, 387), + "c. externa": (1033, 405), + "capacitacion": (1033, 417), + "cirugia ambulatoria": (1033, 432), + "fomento y prevencion": (1033, 449), + "honorarios": (1033, 467), +} + +COORD_CLOSE_ERROR = (1223, 704) + +DEFAULT_APP_PATH = os.getenv( + "DIGITADOR_APP_PATH", + r"C:\Users\Pysctas185\Documents\Integracion Total.lnk", +) +PROCESS_NAME = "SaludTotal.Health.WinClient.TotalIntegration.exe" +REINICIO_CADA = 6 +TIMEOUT_REGISTRO = 120 + +TEMPLATE_ERROR = "error.png" +TEMPLATE_SUCCESS = "success.png" +TEMPLATE_BIEN = "bien.png" +TEMPLATE_DIFERENCIAS = "flecha_diferencias_abierto.png" +TEMPLATE_DIFERENCIAS_CERRADO = "flecha_diferencias_cerrado.png" +TEMPLATE_FECHA_VACIA_SLASH = "fecha_vacia_slash.png" +TEMPLATE_FECHA_VACIA_BLANCO = "fecha_vacia_blanco.png" +TEMPLATE_POPUP_NOMBRES = "popup_nombres.png" +TEMPLATE_RADICADO_EXISTENTE = "radicado_existente.png" diff --git a/bots/bot_digitador/bot_digitador/core/digitador_service.py b/bots/bot_digitador/bot_digitador/core/digitador_service.py new file mode 100644 index 0000000..643bf34 --- /dev/null +++ b/bots/bot_digitador/bot_digitador/core/digitador_service.py @@ -0,0 +1,1377 @@ +"""DigitadorService — núcleo headless de digitación (sin tkinter).""" + +from __future__ import annotations + +import csv +import ctypes +import os +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple + +from pywinauto import mouse +from pywinauto.application import Application + +from bot_digitador.core.coords import ( + COORD_CLOSE_ERROR, + DEFAULT_APP_PATH, + DOC_COORDS, + DOC_STRATEGY, + PROCESS_NAME, + REINICIO_CADA, + SERVICES_COORDS, + TEMPLATE_BIEN, + TEMPLATE_DIFERENCIAS, + TEMPLATE_DIFERENCIAS_CERRADO, + TEMPLATE_ERROR, + TEMPLATE_FECHA_VACIA_BLANCO, + TEMPLATE_FECHA_VACIA_SLASH, + TEMPLATE_POPUP_NOMBRES, + TEMPLATE_RADICADO_EXISTENTE, + TEMPLATE_SUCCESS, + TIMEOUT_REGISTRO, +) +from bot_digitador.models.events import DigitadorEvent, create_event + +try: + import cv2 + import numpy as np + + OPENCV_AVAILABLE = True +except ImportError: + OPENCV_AVAILABLE = False + +try: + ctypes.windll.shcore.SetProcessDpiAwareness(2) +except Exception: + try: + ctypes.windll.user32.SetProcessDPIAware() + except Exception: + pass + +REQUIRED_COLUMNS = { + "radicado", + "documento", + "ingreso", + "total_cargos", + "autorizacion", + "tipo_documento", + "tipo_servicio", + "copago", + "cuota_moderadora", +} + +EventCallback = Callable[[DigitadorEvent], None] +CancelCallback = Callable[[], bool] + + +@dataclass +class DigitadorRunSummary: + total: int = 0 + exitosos: int = 0 + fallidos: int = 0 + existentes: int = 0 + omitidos: int = 0 + timeouts: int = 0 + cancelled: bool = False + fatal_error: Optional[str] = None + output_dir: str = "" + resultados: List[Dict[str, Any]] = field(default_factory=list) + + +def click_abs(window, x: int, y: int) -> None: + try: + window.wrapper_object().set_focus() + except Exception: + pass + + ctypes.windll.user32.SetCursorPos(x, y) + time.sleep(0.05) + ctypes.windll.user32.SetCursorPos(x + 1, y) + time.sleep(0.05) + ctypes.windll.user32.SetCursorPos(x, y) + + time.sleep(0.2) + mouse.click(button="left", coords=(x, y)) + + +def _default_template_dir() -> str: + return str(Path(__file__).resolve().parents[2] / "template") + + +def detect_csv_delimiter(csv_path: str, preferred: Optional[str] = None) -> str: + """Detecta delimitador (, o |) mirando cabeceras requeridas.""" + candidates: List[str] = [] + if preferred in (",", "|"): + candidates.append(preferred) + for d in (",", "|"): + if d not in candidates: + candidates.append(d) + + for delim in candidates: + try: + with open(csv_path, newline="", encoding="utf-8-sig") as f: + reader = csv.DictReader(f, delimiter=delim) + headers = [h.strip() for h in (reader.fieldnames or [])] + if REQUIRED_COLUMNS.issubset(set(headers)): + return delim + except Exception: + continue + + if preferred in (",", "|"): + return preferred + return "|" + + +class DigitadorService: + """ + Servicio headless de digitación en Integración Total. + + Extraído de bot_saludtotal.py sin dependencia de tkinter en el loop. + """ + + def __init__( + self, + csv_path: str, + output_dir: str, + *, + date_format: str = "dd/mm/yyyy", + delimiter: str = "|", + app_path: Optional[str] = None, + reinicio_cada: int = REINICIO_CADA, + event_callback: Optional[EventCallback] = None, + should_cancel: Optional[CancelCallback] = None, + template_dir: Optional[str] = None, + timeout_registro: int = TIMEOUT_REGISTRO, + auto_detect_delimiter: bool = True, + ): + self.csv_path = str(csv_path) + self.output_dir = str(output_dir) + self.date_format = date_format + self.delimiter = delimiter + self.auto_detect_delimiter = auto_detect_delimiter + self.app_path = ( + app_path + or os.getenv("DIGITADOR_APP_PATH") + or DEFAULT_APP_PATH + ) + self.reinicio_cada = reinicio_cada + self.timeout_registro = timeout_registro + self.event_callback = event_callback + self.should_cancel = should_cancel + self.template_dir = template_dir or _default_template_dir() + + self.first_time = True + self.omitted_records: List[Dict[str, Any]] = [] + self.log_entries: List[str] = [] + self.resultados: List[Dict[str, Any]] = [] + + self.total_registros = 0 + self.procesados_ok = 0 + self.errores = 0 + self.omitidos = 0 + self.existentes = 0 + self.timeouts = 0 + self.reinicios = 0 + self.registro_actual = "-" + self.inicio_proceso: Optional[float] = None + self.inicio_registro: Optional[float] = None + + self._cancelled = False + self._fatal_error: Optional[str] = None + + os.makedirs(self.output_dir, exist_ok=True) + + # ------------------------------------------------------------------ + # Templates + # ------------------------------------------------------------------ + def _tpl(self, name: str) -> str: + return os.path.join(self.template_dir, name) + + @property + def template_error_path(self) -> str: + return self._tpl(TEMPLATE_ERROR) + + @property + def template_success_path(self) -> str: + return self._tpl(TEMPLATE_SUCCESS) + + @property + def template_bien_path(self) -> str: + return self._tpl(TEMPLATE_BIEN) + + @property + def template_diferencias_path(self) -> str: + return self._tpl(TEMPLATE_DIFERENCIAS) + + @property + def template_diferencias_cerrado_path(self) -> str: + return self._tpl(TEMPLATE_DIFERENCIAS_CERRADO) + + @property + def template_fecha_vacia_slash_path(self) -> str: + return self._tpl(TEMPLATE_FECHA_VACIA_SLASH) + + @property + def template_fecha_vacia_blanco_path(self) -> str: + return self._tpl(TEMPLATE_FECHA_VACIA_BLANCO) + + @property + def template_popup_nombres_path(self) -> str: + return self._tpl(TEMPLATE_POPUP_NOMBRES) + + @property + def template_radicado_existente_path(self) -> str: + return self._tpl(TEMPLATE_RADICADO_EXISTENTE) + + # ------------------------------------------------------------------ + # Eventos / log / reportes + # ------------------------------------------------------------------ + def _emit( + self, + event_type: str, + message: str, + *, + radicado: Optional[str] = None, + index: Optional[int] = None, + total: Optional[int] = None, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: + event = create_event( + event_type, + message, + radicado=radicado, + index=index, + total=total, + metadata=metadata, + ) + if self.event_callback: + try: + self.event_callback(event) + except Exception: + pass + + def log(self, message: str) -> None: + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + full_msg = f"[{timestamp}] {message}" + self.log_entries.append(full_msg) + print(full_msg, flush=True) + + def _cancel_requested(self) -> bool: + if self._cancelled: + return True + if self.should_cancel: + try: + if self.should_cancel(): + self._cancelled = True + return True + except Exception: + return False + return False + + def ruta_reporte(self, nombre: str) -> str: + return os.path.join(self.output_dir, nombre) + + def guardar_resultado( + self, + radicado, + estado, + motivo, + detalle, + accion_bot, + duracion_segundos=None, + tipo_error="", + ) -> None: + if duracion_segundos is None: + duracion_segundos = ( + int(time.time() - self.inicio_registro) if self.inicio_registro else 0 + ) + + self.resultados.append({ + "radicado": radicado, + "estado": estado, + "tipo_error": tipo_error, + "motivo": motivo, + "detalle": detalle, + "fecha_hora": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "duracion_segundos": duracion_segundos, + "accion_bot": accion_bot, + }) + + def registrar_omitido( + self, + row, + tipo_error, + detalle_error, + accion_bot="Omitido y reinicio de flujo", + ) -> None: + radicado = row.get("radicado", "Desconocido") if row else "Desconocido" + duracion = int(time.time() - self.inicio_registro) if self.inicio_registro else 0 + + registro = dict(row) if row else {} + registro.update({ + "fecha_hora_error": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "estado": "OMITIDO", + "tipo_error": tipo_error, + "detalle_error": detalle_error, + "accion_bot": accion_bot, + "duracion_segundos": duracion, + }) + + self.omitted_records.append(registro) + self.omitidos += 1 + self.errores += 1 + self.guardar_resultado( + radicado, + "ERROR", + tipo_error, + detalle_error, + accion_bot, + duracion_segundos=duracion, + tipo_error=tipo_error, + ) + self.log(f"OMITIDO: Radicado {radicado} | {tipo_error} | {detalle_error}") + self._emit( + "radicado_failed", + f"Omitido: {tipo_error}", + radicado=str(radicado), + metadata={"tipo_error": tipo_error, "detalle": detalle_error}, + ) + + def save_reports(self) -> None: + try: + os.makedirs(self.output_dir, exist_ok=True) + + log_filename = self.ruta_reporte("ejecucion_bot.log") + with open(log_filename, "a", encoding="utf-8") as f: + for entry in self.log_entries: + f.write(entry + "\n") + self.log_entries = [] + + if self.omitted_records: + omitted_filename = self.ruta_reporte("OMITIDOS.csv") + fieldnames: List[str] = [] + for registro in self.omitted_records: + for key in registro.keys(): + if key not in fieldnames: + fieldnames.append(key) + + with open(omitted_filename, "w", newline="", encoding="utf-8-sig") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + writer.writerows(self.omitted_records) + + if self.resultados: + resultados_filename = self.ruta_reporte("RESULTADOS.csv") + with open(resultados_filename, "w", newline="", encoding="utf-8-sig") as f: + fieldnames = [ + "radicado", "estado", "tipo_error", "motivo", "detalle", + "fecha_hora", "duracion_segundos", "accion_bot", + ] + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(self.resultados) + + except Exception as e: + print(f"Error al guardar reportes: {e}") + + # ------------------------------------------------------------------ + # Clipboard (sin tkinter) + # ------------------------------------------------------------------ + def limpiar_clipboard(self) -> None: + try: + user32 = ctypes.windll.user32 + if user32.OpenClipboard(0): + user32.EmptyClipboard() + user32.CloseClipboard() + except Exception: + pass + + def leer_clipboard(self) -> str: + try: + CF_UNICODETEXT = 13 + user32 = ctypes.windll.user32 + kernel32 = ctypes.windll.kernel32 + if not user32.OpenClipboard(0): + return "" + try: + handle = user32.GetClipboardData(CF_UNICODETEXT) + if not handle: + return "" + ptr = kernel32.GlobalLock(handle) + if not ptr: + return "" + try: + return ctypes.wstring_at(ptr).strip() + finally: + kernel32.GlobalUnlock(handle) + finally: + user32.CloseClipboard() + except Exception: + return "" + + def fecha_tiene_numeros(self, texto) -> bool: + texto = str(texto).strip() + return any(c.isdigit() for c in texto) + + # ------------------------------------------------------------------ + # Fechas / valores / docs + # ------------------------------------------------------------------ + def convert_date(self, date_str) -> str: + if not date_str: + return "" + + input_formats = [ + "%d/%m/%Y", + "%m/%d/%Y", + "%Y-%m-%d", + "%Y/%m/%d", + "%d-%m-%Y", + "%m-%d-%Y", + ] + + user_fmt = "%d/%m/%Y" if self.date_format == "dd/mm/yyyy" else "%m/%d/%Y" + + if user_fmt in input_formats: + input_formats.remove(user_fmt) + input_formats.insert(0, user_fmt) + + date_clean = str(date_str).strip() + + for fmt in input_formats: + try: + dt_obj = datetime.strptime(date_clean, fmt) + return dt_obj.strftime("%m%d%Y") + except ValueError: + continue + + self.log(f"ADVERTENCIA: No se pudo convertir la fecha '{date_str}'. Se usará tal cual.") + return date_str + + def limpiar_valor_monetario(self, valor) -> float: + if valor is None: + return 0.0 + + texto = str(valor).strip() + if texto == "": + return 0.0 + + texto = texto.replace("$", "").replace(" ", "") + + if "." in texto and "," not in texto: + partes = texto.split(".") + if len(partes[-1]) == 3: + texto = texto.replace(".", "") + + texto = texto.replace(",", ".") + + try: + return float(texto) + except Exception: + return 0.0 + + def formatear_valor_para_digitar(self, valor) -> str: + numero = self.limpiar_valor_monetario(valor) + if numero == int(numero): + return str(int(numero)) + return str(numero).replace(".", ",") + + def valor_mayor_a_cero(self, valor) -> bool: + return self.limpiar_valor_monetario(valor) > 0 + + def get_valid_doc_types(self, doc_value) -> List[str]: + doc_value = str(doc_value).strip() + if not doc_value: + return [] + + length = len(doc_value) + is_numeric = doc_value.isdigit() + valid_types: List[str] = [] + + if length in [10, 12]: + valid_types.append("AS") + if 3 <= length <= 11: + valid_types.append("CD") + if 3 <= length <= 7 and is_numeric: + valid_types.append("CE") + if 3 <= length <= 10 and length != 9 and is_numeric: + valid_types.append("CC") + if 9 <= length <= 14 and is_numeric: + valid_types.append("CN") + if length in [10, 12]: + valid_types.append("ME") + if length == 9 and is_numeric: + valid_types.append("NI") + if length == 9 and is_numeric: + valid_types.append("NU") + if 3 <= length <= 16: + valid_types.append("PS") + if length == 15 and is_numeric: + valid_types.append("PE") + if 7 <= length <= 8 and is_numeric: + valid_types.append("PT") + if length == 9 and is_numeric: + valid_types.append("PJ") + if length == 10 and is_numeric: + valid_types.append("RC") + if 1 <= length <= 8 and is_numeric: + valid_types.append("SC") + if length == 10 and is_numeric: + valid_types.append("TI") + + return valid_types + + # ------------------------------------------------------------------ + # OpenCV + # ------------------------------------------------------------------ + def check_template_opencv(self, window, template_path, nombre="template") -> bool: + if not OPENCV_AVAILABLE: + return False + + if not os.path.exists(template_path): + self.log(f"Template no encontrado: {template_path}") + return False + + try: + screenshot_path = os.path.join(self.output_dir, "temp_check_visual.png") + img_cap = window.capture_as_image() + img_cap.save(screenshot_path) + + img_rgb = cv2.imread(screenshot_path) + template = cv2.imread(template_path) + + if img_rgb is None or template is None: + self.log(f"No se pudo leer imagen/template: {nombre}") + return False + + img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY) + template_gray = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY) + + res = cv2.matchTemplate(img_gray, template_gray, cv2.TM_CCOEFF_NORMED) + _, max_val, _, _ = cv2.minMaxLoc(res) + + self.log(f"Score {nombre}: {max_val:.2f}") + return max_val >= 0.60 + + except Exception as e: + self.log(f"Error en chequeo OpenCV de {nombre}: {e}") + return False + + def get_template_score(self, window, template_path, nombre="template") -> float: + if not OPENCV_AVAILABLE or not os.path.exists(template_path): + return 0.0 + + try: + screenshot_path = os.path.join(self.output_dir, "temp_score_visual.png") + img_cap = window.capture_as_image() + img_cap.save(screenshot_path) + + img_rgb = cv2.imread(screenshot_path) + template = cv2.imread(template_path) + + if img_rgb is None or template is None: + return 0.0 + + img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY) + template_gray = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY) + + res = cv2.matchTemplate(img_gray, template_gray, cv2.TM_CCOEFF_NORMED) + _, max_val, _, _ = cv2.minMaxLoc(res) + return max_val + + except Exception: + return 0.0 + + def check_error_opencv(self, window) -> bool: + if not OPENCV_AVAILABLE: + self.log("OpenCV no disponible. Asumiendo OK.") + return False + + if not os.path.exists(self.template_error_path): + self.log(f"Template no encontrado en {self.template_error_path}. Asumiendo OK.") + return False + + try: + screenshot_path = os.path.join(self.output_dir, "temp_check.png") + img_cap = window.capture_as_image() + img_cap.save(screenshot_path) + + img_rgb = cv2.imread(screenshot_path) + template = cv2.imread(self.template_error_path) + + if img_rgb is None or template is None: + self.log("Error al cargar imágenes para OpenCV.") + return False + + img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY) + template_gray = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY) + + res = cv2.matchTemplate(img_gray, template_gray, cv2.TM_CCOEFF_NORMED) + threshold = 0.7 + loc = np.where(res >= threshold) + + if len(loc[0]) > 0: + self.log("Error detectado mediante Template Matching.") + return True + return False + + except Exception as e: + self.log(f"Error en chequeo OpenCV: {e}") + return False + + def fecha_esta_vacia_opencv(self, window) -> bool: + if not OPENCV_AVAILABLE: + self.log("OpenCV no disponible para detectar fecha. Se asume fecha vacía.") + return True + + detecto_slash = False + detecto_blanco = False + + if os.path.exists(self.template_fecha_vacia_slash_path): + detecto_slash = self.check_template_opencv( + window, + self.template_fecha_vacia_slash_path, + "fecha_vacia_slash.png", + ) + else: + self.log(f"Template no encontrado: {self.template_fecha_vacia_slash_path}") + + if os.path.exists(self.template_fecha_vacia_blanco_path): + detecto_blanco = self.check_template_opencv( + window, + self.template_fecha_vacia_blanco_path, + "fecha_vacia_blanco.png", + ) + else: + self.log(f"Template no encontrado: {self.template_fecha_vacia_blanco_path}") + + if detecto_slash: + self.log("Fecha vacía detectada por template slash.") + if detecto_blanco: + self.log("Fecha vacía detectada por template blanco.") + + return detecto_slash or detecto_blanco + + def check_popup_nombres_opencv(self, window) -> bool: + if not OPENCV_AVAILABLE: + return False + if not os.path.exists(self.template_popup_nombres_path): + return False + return self.check_template_opencv( + window, + self.template_popup_nombres_path, + "popup_nombres.png", + ) + + def check_radicado_existente_opencv(self, window) -> bool: + if not OPENCV_AVAILABLE: + return False + if not os.path.exists(self.template_radicado_existente_path): + return False + return self.check_template_opencv( + window, + self.template_radicado_existente_path, + "radicado_existente.png", + ) + + def check_diferencias_abierto_opencv(self, window) -> bool: + return self.check_template_opencv( + window, + self.template_diferencias_path, + "flecha_diferencias_abierto.png", + ) + + def check_diferencias_cerrado_opencv(self, window) -> bool: + return self.check_template_opencv( + window, + self.template_diferencias_cerrado_path, + "flecha_diferencias_cerrado.png", + ) + + def check_success_opencv(self, window) -> bool: + return self.check_template_opencv( + window, + self.template_success_path, + "success.png", + ) + + def check_bien_opencv(self, window) -> bool: + return self.check_template_opencv( + window, + self.template_bien_path, + "bien.png", + ) + + def normalizar_pantalla_radicacion(self, window) -> None: + try: + score_abierto = self.get_template_score( + window, + self.template_diferencias_path, + "flecha_diferencias_abierto.png", + ) + score_cerrado = self.get_template_score( + window, + self.template_diferencias_cerrado_path, + "flecha_diferencias_cerrado.png", + ) + + self.log(f"Score abierto: {score_abierto:.2f} | Score cerrado: {score_cerrado:.2f}") + + if score_abierto > score_cerrado and score_abierto >= 0.60: + self.log("Panel confirmado ABIERTO por score. Cerrando...") + click_abs(window, 558, 192) + time.sleep(1) + else: + self.log("Panel confirmado CERRADO o incierto. No se toca.") + + except Exception as e: + self.log(f"Error normalizando: {e}") + + def esperar_resultado_registro(self, window, timeout=None) -> str: + timeout = timeout if timeout is not None else self.timeout_registro + self.log(f"Esperando respuesta de registro hasta {timeout} segundos...") + + if not os.path.exists(self.template_success_path): + self.log(f"ADVERTENCIA: No existe {self.template_success_path}.") + if not os.path.exists(self.template_bien_path): + self.log(f"ADVERTENCIA: No existe {self.template_bien_path}.") + + self.log("Tiempo de gracia inicial para evitar falso positivo de error (4s)...") + time.sleep(4) + + inicio = time.time() + while time.time() - inicio < timeout: + if self._cancel_requested(): + return "CANCELLED" + if self.check_success_opencv(window): + return "OK" + if self.check_bien_opencv(window): + return "BIEN" + if self.check_error_opencv(window): + return "ERROR" + time.sleep(1) + + return "TIMEOUT" + + # ------------------------------------------------------------------ + # Navegación / reinicios + # ------------------------------------------------------------------ + def cerrar_popup_especial(self, window) -> None: + try: + window.wrapper_object().type_keys("{ENTER}", set_foreground=True) + time.sleep(0.3) + except Exception: + pass + try: + window.wrapper_object().type_keys("{ESC}", set_foreground=True) + time.sleep(0.3) + except Exception: + pass + + def manejar_radicado_existente(self, window, row, val_radicado) -> None: + duracion = int(time.time() - self.inicio_registro) if self.inicio_registro else 0 + + self.guardar_resultado( + val_radicado, + "RADICADO_EXISTENTE", + "Radicado previamente procesado", + "La aplicación indicó que ya se realizó validación del NIT y valor de la factura.", + "Cerró popup y volvió al menú principal", + duracion_segundos=duracion, + tipo_error="", + ) + + self.existentes += 1 + self.log(f"RADICADO EXISTENTE: {val_radicado}. Se registra y vuelve al menú principal.") + self._emit("radicado_skipped", f"Radicado existente: {val_radicado}", radicado=val_radicado) + self.save_reports() + + self.cerrar_popup_especial(window) + time.sleep(0.2) + click_abs(window, 366, 172) + time.sleep(0.2) + click_abs(window, 341, 247) + + self.first_time = True + self.reinicios += 1 + + def restart_bot( + self, + window, + row=None, + tipo_error="ERROR_VALIDACION", + detalle_error="Se detectó una validación/error en pantalla.", + ) -> None: + if row: + self.registrar_omitido( + row=row, + tipo_error=tipo_error, + detalle_error=detalle_error, + accion_bot="Omitido y reinicio de flujo", + ) + self.save_reports() + + time.sleep(0.05) + try: + click_abs(window, COORD_CLOSE_ERROR[0], COORD_CLOSE_ERROR[1]) + except Exception: + pass + + time.sleep(0.2) + click_abs(window, 366, 172) + time.sleep(0.2) + click_abs(window, 341, 247) + + self.first_time = True + self.reinicios += 1 + self.log("Bot reiniciado exitosamente.") + + def navegar_menu_inicial(self, window) -> None: + self.log("Navegando al menú inicial...") + + click_abs(window, 67, 167) + time.sleep(0.3) + + window.wrapper_object().type_keys("{RIGHT}", set_foreground=True) + time.sleep(0.3) + + window.wrapper_object().type_keys("{DOWN}", set_foreground=True) + time.sleep(0.3) + + window.wrapper_object().type_keys("{RIGHT}", set_foreground=True) + time.sleep(1) + + click_abs(window, 385, 177) + time.sleep(1) + + click_abs(window, 338, 234) + time.sleep(1) + + self.first_time = True + self.log("Menú listo para radicar.") + + def reinicio_completo_app(self) -> Tuple[Any, Any]: + try: + self.reinicios += 1 + self.log("Reinicio completo preventivo de la aplicación...") + self._emit("browser_reconnected", "Reinicio completo de Integración Total") + + os.system(f'taskkill /f /im "{PROCESS_NAME}"') + time.sleep(3) + + os.startfile(self.app_path) + + self.log("Esperando carga de la aplicación...") + time.sleep(12) + + app = Application(backend="uia").connect(title_re=".*Integracion Total Fase 2.*") + window = app.window(title_re=".*Integracion Total Fase 2.*") + + window.wait("visible enabled ready", timeout=20) + window.wrapper_object().set_focus() + + self.navegar_menu_inicial(window) + + self.first_time = True + self.log("Aplicación reiniciada correctamente.") + return app, window + + except Exception as e: + self.log(f"ERROR reiniciando aplicación: {e}") + return None, None + + # ------------------------------------------------------------------ + # CSV + # ------------------------------------------------------------------ + def _load_csv_rows(self) -> List[Dict[str, str]]: + delim = self.delimiter + if self.auto_detect_delimiter or delim not in (",", "|"): + delim = detect_csv_delimiter(self.csv_path, preferred=self.delimiter) + self.delimiter = delim + self.log(f"Delimitador CSV detectado/usado: '{delim}'") + + data: List[Dict[str, str]] = [] + with open(self.csv_path, newline="", encoding="utf-8-sig") as csvfile: + reader = csv.DictReader(csvfile, delimiter=delim) + headers = [h.strip() for h in reader.fieldnames] if reader.fieldnames else [] + reader.fieldnames = headers + + if not REQUIRED_COLUMNS.issubset(headers): + missing = REQUIRED_COLUMNS - set(headers) + raise ValueError( + "El CSV no tiene las columnas requeridas.\n" + f"Faltan: {', '.join(sorted(missing))}\n" + "Verifique si el delimitador es correcto." + ) + + for row in reader: + cleaned = {k.strip(): (v.strip() if isinstance(v, str) else v) for k, v in row.items()} + data.append(cleaned) + + return data + + # ------------------------------------------------------------------ + # Loop principal + # ------------------------------------------------------------------ + def run(self) -> DigitadorRunSummary: + summary = DigitadorRunSummary(output_dir=self.output_dir) + app = None + window = None + + try: + self.log("Conectando a la aplicación...") + self._emit("run_started", f"Iniciando digitación. CSV: {self.csv_path}") + + app = Application(backend="uia").connect(title_re=".*Integracion Total Fase 2.*") + window = app.window(title_re=".*Integracion Total Fase 2.*") + + self.omitted_records = [] + self.log_entries = [] + self.resultados = [] + self.procesados_ok = 0 + self.errores = 0 + self.omitidos = 0 + self.existentes = 0 + self.timeouts = 0 + self.reinicios = 0 + self.registro_actual = "-" + self.inicio_proceso = time.time() + self.first_time = True + self._cancelled = False + self._fatal_error = None + + self.log(f"Iniciando nuevo proceso. Reportes en: {self.output_dir}") + self.log(f"APP_PATH={self.app_path}") + self.log(f"Templates: {self.template_dir}") + + try: + window.wait("visible enabled ready", timeout=5) + except Exception as e: + self.log(f"Advertencia en navegación inicial: {e}") + + data = self._load_csv_rows() + self.total_registros = len(data) + summary.total = self.total_registros + self.log(f"Se encontraron {len(data)} registros para procesar.") + + for i, row in enumerate(data): + if self._cancel_requested(): + self.log("Cancelación solicitada. Deteniendo proceso.") + summary.cancelled = True + self._emit("run_finished", "Proceso cancelado", metadata={"cancelled": True}) + break + + self.inicio_registro = time.time() + self.registro_actual = row.get("radicado", "N/A") + self._emit( + "radicado_processing", + f"Procesando registro {i + 1}/{len(data)}: {self.registro_actual}", + radicado=str(self.registro_actual), + index=i + 1, + total=len(data), + ) + self.log(f"--- Procesando registro {i + 1}/{len(data)}: {row.get('radicado', 'N/A')} ---") + + val_radicado = row.get("radicado", "").strip() + val_total_cargos = row.get("total_cargos", "0").strip() + val_identificacion = row.get("documento", "").strip() + val_fecha = row.get("ingreso", "").strip() + val_auth = row.get("autorizacion", "").strip() + val_tipo_doc = row.get("tipo_documento", "").strip().upper() + val_service = row.get("tipo_servicio", "").strip().lower() + val_copago = row.get("copago", "").strip() + val_cuota_mod = row.get("cuota_moderadora", "").strip() + + val_fecha_final = self.convert_date(val_fecha) + + click_abs(window, 926, 309) + + time.sleep(0.2) + window.wrapper_object().type_keys( + val_radicado, + with_spaces=True, + set_foreground=True, + ) + + time.sleep(0.2) + click_abs(window, 960, 531) + + time.sleep(0.5) + if self.check_radicado_existente_opencv(window): + self.manejar_radicado_existente(window, row, val_radicado) + continue + + if self.check_error_opencv(window): + self.log("Error al procesar el registro.") + self.restart_bot(window, row, "ERROR_RADICADO", "Error visual después de consultar el radicado.") + continue + + time.sleep(0.2) + click_abs(window, 1375, 305) + + time.sleep(0.2) + click_abs(window, 1041, 329) + + if val_service: + if val_service in SERVICES_COORDS: + self.log(f"Tipo de servicio: {val_service}") + time.sleep(0.2) + click_abs( + window, + SERVICES_COORDS[val_service][0], + SERVICES_COORDS[val_service][1], + ) + else: + self.log(f"ERROR: El tipo de servicio '{val_service}' no es válido.") + self.restart_bot( + window, + row, + "ERROR_TIPO_SERVICIO", + f"Tipo de servicio no configurado: {val_service}", + ) + continue + else: + self.log("ERROR: El campo 'tipo_servicio' está vacío.") + self.restart_bot( + window, + row, + "ERROR_TIPO_SERVICIO", + "El campo tipo_servicio está vacío.", + ) + continue + + time.sleep(0.2) + click_abs(window, 1300, 302) + + self.log("Esperando carga (7s)...") + time.sleep(7) + + if self.check_error_opencv(window): + self.log("Error al procesar el registro.") + self.restart_bot( + window, + row, + "ERROR_CARGA_DETALLE", + "Error visual después de cargar el detalle del radicado.", + ) + continue + + self.normalizar_pantalla_radicacion(window) + + time.sleep(0.2) + click_abs(window, 730, 321) + + time.sleep(0.2) + click_abs(window, 700, 359) + + candidatos_validos = self.get_valid_doc_types(val_identificacion) + candidatos_ordenados = [t for t in DOC_STRATEGY if t in candidatos_validos] + + tipos_a_probar: List[str] = [] + + if val_tipo_doc and val_tipo_doc in DOC_COORDS: + tipos_a_probar.append(val_tipo_doc) + for t in candidatos_ordenados: + if t != val_tipo_doc: + tipos_a_probar.append(t) + else: + if not candidatos_ordenados: + self.log( + f"ADVERTENCIA: El documento {val_identificacion} " + "no cumple reglas conocidas. Se probará estrategia completa." + ) + tipos_a_probar = list(DOC_STRATEGY) + else: + tipos_a_probar = candidatos_ordenados + + self.log(f"Estrategia de documentos a probar: {tipos_a_probar}") + + documento_exitoso = False + error_documento_fatal = False + + for tipo_doc in tipos_a_probar: + if self._cancel_requested(): + break + + self.log(f"Probando tipo de documento: {tipo_doc}") + + time.sleep(1) + click_abs(window, 729, 279) + + coords_doc = DOC_COORDS.get(tipo_doc, (0, 0)) + if coords_doc == (0, 0): + self.log(f"Advertencia: Coordenadas para {tipo_doc} no configuradas. Saltando.") + continue + + time.sleep(0.5) + click_abs(window, coords_doc[0], coords_doc[1]) + + time.sleep(0.2) + click_abs(window, 1015, 280) + + time.sleep(0.2) + window.wrapper_object().type_keys("^a{DELETE}", set_foreground=True) + time.sleep(0.1) + window.wrapper_object().type_keys( + val_identificacion, + with_spaces=True, + set_foreground=True, + ) + + time.sleep(0.05) + click_abs(window, 1172, 276) + time.sleep(0.05) + + if self.fecha_esta_vacia_opencv(window): + self.log(f"Fecha vacía detectada por OpenCV. Digitando fecha CSV: {val_fecha_final}") + time.sleep(0.1) + window.wrapper_object().type_keys( + val_fecha_final, + pause=0.1, + with_spaces=False, + set_foreground=True, + ) + time.sleep(0.1) + window.wrapper_object().type_keys("{TAB}{ENTER}", set_foreground=True) + else: + self.log("Fecha precargada detectada por OpenCV. No se modifica.") + time.sleep(0.1) + window.wrapper_object().type_keys("{TAB}{ENTER}", set_foreground=True) + + time.sleep(1) + + if self.check_popup_nombres_opencv(window): + self.log("Popup de búsqueda por nombres detectado. Se omite el registro.") + self.cerrar_popup_especial(window) + self.restart_bot( + window, + row, + "ERROR_BUSQUEDA_DOCUMENTO", + "La aplicación abrió el popup de búsqueda por nombres/apellidos al validar el documento.", + ) + error_documento_fatal = True + break + + if self.check_error_opencv(window): + self.log(f"Error detectado con {tipo_doc}. Intentando siguiente...") + click_abs(window, COORD_CLOSE_ERROR[0], COORD_CLOSE_ERROR[1]) + time.sleep(1) + continue + + self.log("Validación OpenCV exitosa.") + documento_exitoso = True + break + + if self._cancel_requested(): + summary.cancelled = True + break + + if error_documento_fatal: + continue + + if not documento_exitoso: + self.log( + f"ERROR: No se pudo procesar el documento {val_identificacion} " + "con ningún tipo. Omitiendo y reiniciando bot." + ) + self.restart_bot( + window, + row, + "ERROR_DOCUMENTO", + f"No validó documento {val_identificacion} con tipos probados: {', '.join(tipos_a_probar)}", + ) + continue + + time.sleep(0.6) + click_abs(window, 1126, 352) + + time.sleep(0.2) + window.wrapper_object().type_keys( + val_auth, + with_spaces=True, + set_foreground=True, + ) + + time.sleep(0.2) + window.wrapper_object().type_keys("{TAB}{ENTER}", set_foreground=True) + + if self.check_error_opencv(window): + self.log("Error al procesar el registro.") + self.restart_bot( + window, + row, + "ERROR_AUTORIZACION", + "Error visual después de ingresar autorización.", + ) + continue + + time.sleep(0.3) + click_abs(window, 741, 421) + + time.sleep(0.2) + window.wrapper_object().type_keys("{SPACE}", set_foreground=True) + + time.sleep(0.3) + click_abs(window, 740, 535) + + time.sleep(0.3) + self.log(f"Digitando total_cargos: {val_total_cargos}") + + click_abs(window, 1105, 658) + time.sleep(0.2) + window.wrapper_object().type_keys( + val_total_cargos, + with_spaces=True, + set_foreground=True, + ) + + copago_txt = self.formatear_valor_para_digitar(val_copago) + cuota_mod_txt = self.formatear_valor_para_digitar(val_cuota_mod) + + copago_mayor = self.valor_mayor_a_cero(val_copago) + cuota_mod_mayor = self.valor_mayor_a_cero(val_cuota_mod) + + if copago_mayor: + self.log(f"Digitando copago: {copago_txt}") + time.sleep(0.3) + window.wrapper_object().type_keys("{TAB}{TAB}", set_foreground=True) + time.sleep(0.3) + window.wrapper_object().type_keys( + copago_txt, + with_spaces=True, + set_foreground=True, + ) + time.sleep(0.3) + window.wrapper_object().type_keys("{TAB}{TAB}{TAB}", set_foreground=True) + time.sleep(0.3) + window.wrapper_object().type_keys("{ENTER}", set_foreground=True) + + elif cuota_mod_mayor: + self.log(f"Digitando cuota moderadora: {cuota_mod_txt}") + time.sleep(0.3) + window.wrapper_object().type_keys("{TAB}{TAB}", set_foreground=True) + time.sleep(0.3) + window.wrapper_object().type_keys("{TAB}", set_foreground=True) + time.sleep(0.3) + window.wrapper_object().type_keys( + cuota_mod_txt, + with_spaces=True, + set_foreground=True, + ) + time.sleep(0.3) + window.wrapper_object().type_keys("{TAB}{TAB}", set_foreground=True) + time.sleep(0.3) + window.wrapper_object().type_keys("{ENTER}", set_foreground=True) + + else: + self.log("Copago y cuota moderadora en 0. Va directo a agregar.") + time.sleep(0.2) + window.wrapper_object().type_keys("{TAB}{TAB}{TAB}{TAB}{TAB}", set_foreground=True) + time.sleep(0.2) + window.wrapper_object().type_keys("{ENTER}", set_foreground=True) + + self.log("Esperando confirmación visual antes de continuar al siguiente radicado...") + resultado_registro = self.esperar_resultado_registro(window, self.timeout_registro) + + if resultado_registro == "CANCELLED": + summary.cancelled = True + self.log("Cancelación durante espera de registro.") + break + + if resultado_registro in ("OK", "BIEN"): + self.log(f"Registro exitoso/confirmación detectada: {resultado_registro}.") + self.guardar_resultado( + val_radicado, + "OK", + "Cuenta registrada", + f"Mensaje visual detectado en pantalla: {resultado_registro}.", + "Continuó", + ) + + click_abs(window, 1229, 979) + time.sleep(1) + click_abs(window, 1223, 666) + time.sleep(0.5) + click_abs(window, 941, 305) + + self.procesados_ok += 1 + self._emit( + "radicado_saved", + f"Registro OK: {val_radicado}", + radicado=val_radicado, + index=i + 1, + total=len(data), + ) + self.log(f"Registro {val_radicado} completado.") + time.sleep(1) + self.first_time = False + self.save_reports() + + elif resultado_registro == "ERROR": + self.log("Error detectado después de registrar. Se omite el registro.") + self.restart_bot( + window, + row, + "ERROR_REGISTRO", + "Error visual después de presionar agregar/registrar.", + ) + continue + + else: + self.log("TIMEOUT: No apareció éxito ni error después de registrar.") + self.errores += 1 + self.timeouts += 1 + self.guardar_resultado( + val_radicado, + "TIMEOUT", + "ERROR_TIMEOUT", + f"No apareció mensaje de éxito ni error en {self.timeout_registro} segundos.", + "Reinicio completo y continúa", + tipo_error="ERROR_TIMEOUT", + ) + self._emit( + "radicado_failed", + f"TIMEOUT: {val_radicado}", + radicado=val_radicado, + metadata={"tipo_error": "ERROR_TIMEOUT"}, + ) + self.save_reports() + app, window = self.reinicio_completo_app() + if window is None: + self.log("No se pudo reiniciar después del timeout. Deteniendo proceso.") + self._fatal_error = "No se pudo reiniciar la aplicación tras timeout" + break + continue + + if (i + 1) % self.reinicio_cada == 0 and (i + 1) < len(data): + app, window = self.reinicio_completo_app() + if window is None: + self.log("No se pudo reiniciar la aplicación. Deteniendo proceso.") + self._fatal_error = "No se pudo reiniciar la aplicación" + break + + if not summary.cancelled: + self.log("--- PROCESO FINALIZADO ---") + self._emit("run_finished", "Proceso finalizado") + + self.save_reports() + + except Exception as e: + self._fatal_error = str(e) + self.log(f"ERROR: {str(e)}") + self._emit("warning", f"Error fatal: {e}", metadata={"fatal": True}) + self.save_reports() + + summary.total = self.total_registros + summary.exitosos = self.procesados_ok + summary.existentes = self.existentes + summary.omitidos = self.omitidos + summary.timeouts = self.timeouts + summary.fallidos = self.omitidos + self.timeouts + summary.cancelled = summary.cancelled or self._cancelled + summary.fatal_error = self._fatal_error + summary.resultados = list(self.resultados) + summary.output_dir = self.output_dir + return summary diff --git a/bots/bot_digitador/bot_digitador/models/__init__.py b/bots/bot_digitador/bot_digitador/models/__init__.py new file mode 100644 index 0000000..af94035 --- /dev/null +++ b/bots/bot_digitador/bot_digitador/models/__init__.py @@ -0,0 +1,3 @@ +from bot_digitador.models.events import DigitadorEvent, create_event + +__all__ = ["DigitadorEvent", "create_event"] diff --git a/bots/bot_digitador/bot_digitador/models/events.py b/bots/bot_digitador/bot_digitador/models/events.py new file mode 100644 index 0000000..bc18600 --- /dev/null +++ b/bots/bot_digitador/bot_digitador/models/events.py @@ -0,0 +1,50 @@ +"""Eventos simples para callbacks del DigitadorService.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from typing import Any, Dict, Optional + + +@dataclass +class DigitadorEvent: + event_type: str + message: str + radicado: Optional[str] = None + index: Optional[int] = None + total: Optional[int] = None + metadata: Dict[str, Any] = field(default_factory=dict) + timestamp: str = "" + + def __post_init__(self) -> None: + if not self.timestamp: + self.timestamp = datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + + def to_dict(self) -> dict: + return asdict(self) + + def display_message(self) -> str: + prefix = "" + if self.index is not None and self.total is not None: + prefix = f"[{self.index}/{self.total}] " + return f"{prefix}{self.message}" + + +def create_event( + event_type: str, + message: str, + *, + radicado: Optional[str] = None, + index: Optional[int] = None, + total: Optional[int] = None, + metadata: Optional[Dict[str, Any]] = None, +) -> DigitadorEvent: + return DigitadorEvent( + event_type=event_type, + message=message, + radicado=radicado, + index=index, + total=total, + metadata=metadata or {}, + ) diff --git a/bots/bot_digitador/bot_digitador/ui/__init__.py b/bots/bot_digitador/bot_digitador/ui/__init__.py new file mode 100644 index 0000000..f4baf44 --- /dev/null +++ b/bots/bot_digitador/bot_digitador/ui/__init__.py @@ -0,0 +1 @@ +"""UI del agent digitador.""" diff --git a/bots/bot_digitador/bot_digitador/ui/dashboard.py b/bots/bot_digitador/bot_digitador/ui/dashboard.py new file mode 100644 index 0000000..1ce595c --- /dev/null +++ b/bots/bot_digitador/bot_digitador/ui/dashboard.py @@ -0,0 +1,368 @@ +from __future__ import annotations + +import socket +import sys +import threading + +import customtkinter as ctk + +from bot_digitador.core.coords import DEFAULT_APP_PATH +from bot_digitador.ui.settings_store import ( + DEFAULT_API_URL, + load_gui_settings, + normalize_api_url, + save_gui_settings, +) +from bot_digitador.ui.windows_autostart import ( + disable_autostart, + enable_autostart, + is_autostart_enabled, +) +from bot_digitador.worker.agent import WorkerAgent + +ctk.set_appearance_mode("dark") +ctk.set_default_color_theme("blue") + + +class BotDashboard(ctk.CTk): + """GUI del agent digitador: configura API remota y conecta como worker idle.""" + + def __init__(self): + super().__init__() + + self.title("Bot Digitador — Integración Total") + self.geometry("960x780") + self.resizable(False, False) + + self._agent: WorkerAgent | None = None + self._agent_thread: threading.Thread | None = None + self._connected = False + + self._construir_ui() + self._cargar_settings() + + self.protocol("WM_DELETE_WINDOW", self._on_close) + + if bool(load_gui_settings().get("auto_connect")): + self.after(800, self._maybe_auto_connect) + + def _construir_ui(self): + root = ctk.CTkFrame(self, corner_radius=18) + root.pack(fill="both", expand=True, padx=18, pady=18) + + ctk.CTkLabel( + root, + text="Bot Digitador", + font=ctk.CTkFont(size=26, weight="bold"), + ).pack(pady=(18, 4)) + + ctk.CTkLabel( + root, + text="Agent remoto — digita en Integración Total (pywinauto + OpenCV)", + text_color="gray", + font=ctk.CTkFont(size=13), + ).pack(pady=(0, 18)) + + config = ctk.CTkFrame(root, corner_radius=14) + config.pack(fill="x", padx=24, pady=(0, 14)) + + ctk.CTkLabel( + config, + text="API remota (IPSExtractor)", + font=ctk.CTkFont(size=16, weight="bold"), + ).pack(anchor="w", padx=18, pady=(14, 8)) + + row_api = ctk.CTkFrame(config, fg_color="transparent") + row_api.pack(fill="x", padx=18, pady=(0, 8)) + ctk.CTkLabel(row_api, text="URL API", width=100, anchor="w").pack(side="left") + self.entry_api = ctk.CTkEntry( + row_api, + placeholder_text=DEFAULT_API_URL, + ) + self.entry_api.pack(side="left", fill="x", expand=True) + + row_id = ctk.CTkFrame(config, fg_color="transparent") + row_id.pack(fill="x", padx=18, pady=(0, 8)) + ctk.CTkLabel(row_id, text="Worker ID", width=100, anchor="w").pack(side="left") + self.entry_worker = ctk.CTkEntry( + row_id, + placeholder_text=socket.gethostname() or "PC-01", + ) + self.entry_worker.pack(side="left", fill="x", expand=True) + + row_token = ctk.CTkFrame(config, fg_color="transparent") + row_token.pack(fill="x", padx=18, pady=(0, 8)) + ctk.CTkLabel(row_token, text="Token", width=100, anchor="w").pack(side="left") + self.entry_token = ctk.CTkEntry( + row_token, + placeholder_text="BOT_API_TOKEN", + show="•", + ) + self.entry_token.pack(side="left", fill="x", expand=True, padx=(0, 8)) + self._token_visible = False + ctk.CTkButton( + row_token, + text="👁", + width=40, + command=self._toggle_token, + ).pack(side="left") + + row_app = ctk.CTkFrame(config, fg_color="transparent") + row_app.pack(fill="x", padx=18, pady=(0, 8)) + ctk.CTkLabel(row_app, text="APP_PATH", width=100, anchor="w").pack(side="left") + self.entry_app = ctk.CTkEntry( + row_app, + placeholder_text=DEFAULT_APP_PATH, + ) + self.entry_app.pack(side="left", fill="x", expand=True) + + row_auto = ctk.CTkFrame(config, fg_color="transparent") + row_auto.pack(fill="x", padx=18, pady=(0, 14)) + self.chk_autostart = ctk.CTkCheckBox( + row_auto, + text="Iniciar con Windows", + command=self._on_autostart_toggle, + ) + self.chk_autostart.pack(side="left", padx=(0, 18)) + self.chk_auto_connect = ctk.CTkCheckBox( + row_auto, + text="Conectar al abrir", + command=self._on_auto_connect_toggle, + ) + self.chk_auto_connect.pack(side="left") + if sys.platform != "win32": + self.chk_autostart.configure(state="disabled") + + status = ctk.CTkFrame(root, corner_radius=14) + status.pack(fill="x", padx=24, pady=(0, 14)) + + self.lbl_estado = ctk.CTkLabel( + status, + text="Estado: desconectado", + font=ctk.CTkFont(size=15, weight="bold"), + ) + self.lbl_estado.pack(anchor="w", padx=18, pady=(14, 2)) + + self.lbl_hint = ctk.CTkLabel( + status, + text="Configura la API y pulsa Conectar. Este PC no expone puertos.", + text_color="gray", + font=ctk.CTkFont(size=13), + ) + self.lbl_hint.pack(anchor="w", padx=18, pady=(0, 14)) + + botones = ctk.CTkFrame(root, fg_color="transparent") + botones.pack(fill="x", padx=24, pady=(0, 14)) + + self.btn_connect = ctk.CTkButton( + botones, + text="▶ Conectar agent", + height=42, + width=180, + font=ctk.CTkFont(size=14, weight="bold"), + command=self._toggle_connect, + ) + self.btn_connect.pack(side="left", padx=(0, 10)) + + ctk.CTkButton( + botones, + text="Guardar config", + height=42, + width=140, + command=self._guardar_settings, + ).pack(side="left", padx=(0, 10)) + + ctk.CTkButton( + botones, + text="Limpiar logs", + height=42, + width=130, + command=self._limpiar_logs, + ).pack(side="left") + + logs_frame = ctk.CTkFrame(root, corner_radius=14) + logs_frame.pack(fill="both", expand=True, padx=24, pady=(0, 18)) + + ctk.CTkLabel( + logs_frame, + text="Detalle del agent", + font=ctk.CTkFont(size=15, weight="bold"), + ).pack(anchor="w", padx=18, pady=(12, 6)) + + self.text_logs = ctk.CTkTextbox( + logs_frame, font=ctk.CTkFont(family="Courier", size=12) + ) + self.text_logs.pack(fill="both", expand=True, padx=18, pady=(0, 16)) + + def _persist_flags(self): + save_gui_settings( + normalize_api_url(self.entry_api.get()), + self.entry_worker.get(), + self.entry_token.get(), + app_path=self.entry_app.get(), + autostart_windows=bool(self.chk_autostart.get()), + auto_connect=bool(self.chk_auto_connect.get()), + ) + + def _cargar_settings(self): + s = load_gui_settings() + self.entry_api.insert(0, s.get("api_url") or DEFAULT_API_URL) + self.entry_worker.insert(0, s.get("worker_id") or socket.gethostname()) + self.entry_token.insert(0, s.get("token") or "") + self.entry_app.insert(0, s.get("app_path") or DEFAULT_APP_PATH) + + want_autostart = bool(s.get("autostart_windows")) + if want_autostart and sys.platform == "win32": + try: + if not is_autostart_enabled(): + enable_autostart() + self.chk_autostart.select() + except Exception as e: + self._log(f"No se pudo restaurar autoarranque: {e}") + elif is_autostart_enabled(): + self.chk_autostart.select() + + if bool(s.get("auto_connect")): + self.chk_auto_connect.select() + + def _guardar_settings(self): + api = normalize_api_url(self.entry_api.get()) + self.entry_api.delete(0, "end") + self.entry_api.insert(0, api) + self._persist_flags() + self._log("Configuración guardada en data/gui_settings.json") + + def _on_autostart_toggle(self): + if sys.platform != "win32": + return + enabled = bool(self.chk_autostart.get()) + try: + if enabled: + path = enable_autostart() + self._log(f"Autoarranque activado: {path}") + if not self.chk_auto_connect.get(): + self.chk_auto_connect.select() + else: + disable_autostart() + self._log("Autoarranque desactivado.") + self._persist_flags() + except Exception as e: + self._log(f"Error configurando autoarranque: {e}") + if enabled: + self.chk_autostart.deselect() + else: + self.chk_autostart.select() + + def _on_auto_connect_toggle(self): + self._persist_flags() + + def _maybe_auto_connect(self): + if self._connected: + return + if not self.entry_worker.get().strip() or not self.entry_token.get().strip(): + self._log("Auto-conectar omitido: falta Worker ID o token.") + return + self._log("Conectando automáticamente…") + self._conectar() + + def _toggle_token(self): + self._token_visible = not self._token_visible + self.entry_token.configure(show="" if self._token_visible else "•") + + def _toggle_connect(self): + if self._connected: + self._desconectar() + else: + self._conectar() + + def _conectar(self): + api = normalize_api_url(self.entry_api.get()) + worker_id = self.entry_worker.get().strip() + token = self.entry_token.get().strip() + app_path = self.entry_app.get().strip() or DEFAULT_APP_PATH + + if not worker_id: + self._log("Debes indicar un Worker ID.") + return + if not token: + self._log("Debes indicar el token (BOT_API_TOKEN).") + return + + self.entry_api.delete(0, "end") + self.entry_api.insert(0, api) + self._persist_flags() + + self._agent = WorkerAgent( + api_url=api, + worker_id=worker_id, + token=token, + app_path=app_path, + log_func=self._log, + ) + self._connected = True + self.btn_connect.configure(text="⏹ Desconectar", fg_color="#c0392b", hover_color="#a93226") + self.lbl_estado.configure(text=f"Estado: conectado como {worker_id}") + self.lbl_hint.configure(text=f"API: {api} — esperando jobs del desktop…") + self._set_config_enabled(False) + + self._agent_thread = threading.Thread( + target=self._run_agent, + daemon=True, + ) + self._agent_thread.start() + self._log(f"Conectando a {api}…") + + def _run_agent(self): + try: + assert self._agent is not None + self._agent.run_forever() + except Exception as e: + self._log(f"Error del agent: {e}") + finally: + self.after(0, self._on_agent_stopped) + + def _desconectar(self): + if self._agent: + self._agent.stop() + self._log("Solicitando desconexión…") + self.btn_connect.configure(state="disabled", text="Desconectando…") + + def _on_agent_stopped(self): + self._connected = False + self._agent = None + self.btn_connect.configure( + state="normal", + text="▶ Conectar agent", + fg_color=["#3B8ED0", "#1F6AA5"], + hover_color=["#36719F", "#144870"], + ) + self.lbl_estado.configure(text="Estado: desconectado") + self.lbl_hint.configure( + text="Configura la API y pulsa Conectar. Este PC no expone puertos." + ) + self._set_config_enabled(True) + + def _set_config_enabled(self, enabled: bool): + state = "normal" if enabled else "disabled" + self.entry_api.configure(state=state) + self.entry_worker.configure(state=state) + self.entry_token.configure(state=state) + self.entry_app.configure(state=state) + if sys.platform == "win32": + self.chk_autostart.configure(state="normal") + self.chk_auto_connect.configure(state="normal") + + def _limpiar_logs(self): + self.text_logs.delete("1.0", "end") + + def _log(self, mensaje: str): + def _escribir(): + self.text_logs.insert("end", mensaje + "\n") + self.text_logs.see("end") + + self.after(0, _escribir) + + def _on_close(self): + if self._agent: + self._agent.stop() + self.destroy() diff --git a/bots/bot_digitador/bot_digitador/ui/settings_store.py b/bots/bot_digitador/bot_digitador/ui/settings_store.py new file mode 100644 index 0000000..e1d56f4 --- /dev/null +++ b/bots/bot_digitador/bot_digitador/ui/settings_store.py @@ -0,0 +1,79 @@ +"""Persistencia simple de configuración de la GUI del worker digitador.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Dict + +from bot_digitador.core.coords import DEFAULT_APP_PATH + +DEFAULT_API_URL = os.getenv( + "BOT_API_URL", + "http://localhost:8189/bot-digitador", +) +DEFAULT_TOKEN = os.getenv("BOT_API_TOKEN", "dev-token-change-me") + +_SETTINGS_PATH = Path("data") / "gui_settings.json" + + +def load_gui_settings() -> Dict[str, Any]: + defaults = { + "api_url": DEFAULT_API_URL, + "worker_id": "", + "token": DEFAULT_TOKEN, + "app_path": os.getenv("DIGITADOR_APP_PATH", DEFAULT_APP_PATH), + "autostart_windows": False, + "auto_connect": False, + } + if not _SETTINGS_PATH.exists(): + return defaults + try: + data = json.loads(_SETTINGS_PATH.read_text(encoding="utf-8")) + if not isinstance(data, dict): + return defaults + return {**defaults, **data} + except Exception: + return defaults + + +def save_gui_settings( + api_url: str, + worker_id: str, + token: str, + *, + app_path: str | None = None, + autostart_windows: bool | None = None, + auto_connect: bool | None = None, +) -> None: + _SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True) + current = load_gui_settings() + payload = { + **current, + "api_url": api_url.strip(), + "worker_id": worker_id.strip(), + "token": token.strip(), + } + if app_path is not None: + payload["app_path"] = app_path.strip() + if autostart_windows is not None: + payload["autostart_windows"] = bool(autostart_windows) + if auto_connect is not None: + payload["auto_connect"] = bool(auto_connect) + _SETTINGS_PATH.write_text( + json.dumps(payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + +def normalize_api_url(url: str) -> str: + """Normaliza la URL base del API bot-digitador.""" + u = (url or "").strip().rstrip("/") + if not u: + return DEFAULT_API_URL.rstrip("/") + if u.endswith("/bot-digitador"): + return u + if "builker.xyz" in u or u.endswith(":8189"): + return f"{u}/bot-digitador" + return u diff --git a/bots/bot_digitador/bot_digitador/ui/windows_autostart.py b/bots/bot_digitador/bot_digitador/ui/windows_autostart.py new file mode 100644 index 0000000..dae5d88 --- /dev/null +++ b/bots/bot_digitador/bot_digitador/ui/windows_autostart.py @@ -0,0 +1,73 @@ +"""Autoarranque con Windows (carpeta Startup del usuario).""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +TASK_BASENAME = "IPSExtractor-BotDigitador" + + +def bot_root() -> Path: + """Raíz del paquete bots/bot_digitador (donde está run_gui.py).""" + return Path(__file__).resolve().parents[2] + + +def startup_dir() -> Path: + appdata = os.environ.get("APPDATA") or str(Path.home() / "AppData" / "Roaming") + return Path(appdata) / "Microsoft" / "Windows" / "Start Menu" / "Programs" / "Startup" + + +def startup_script_path() -> Path: + return startup_dir() / f"{TASK_BASENAME}.bat" + + +def resolve_python(root: Path | None = None) -> Path: + root = root or bot_root() + for candidate in ( + root / ".venv" / "Scripts" / "pythonw.exe", + root / ".venv" / "Scripts" / "python.exe", + root / ".venv" / "bin" / "python", + ): + if candidate.exists(): + return candidate + return Path(sys.executable) + + +def is_autostart_enabled() -> bool: + if sys.platform != "win32": + return False + return startup_script_path().exists() + + +def enable_autostart(root: Path | None = None) -> Path: + """Crea un .bat en Startup que abre la GUI del bot tras el login.""" + if sys.platform != "win32": + raise RuntimeError("El autoarranque solo está disponible en Windows.") + + root = (root or bot_root()).resolve() + gui = root / "run_gui.py" + if not gui.exists(): + raise FileNotFoundError(f"No se encontró run_gui.py en {root}") + + py = resolve_python(root) + dest = startup_script_path() + dest.parent.mkdir(parents=True, exist_ok=True) + + content = ( + "@echo off\r\n" + f'cd /d "{root}"\r\n' + f'start "" "{py}" "{gui}"\r\n' + ) + dest.write_text(content, encoding="utf-8") + return dest + + +def disable_autostart() -> bool: + """Elimina el .bat de Startup. True si había algo que borrar.""" + path = startup_script_path() + if path.exists(): + path.unlink() + return True + return False diff --git a/bots/bot_digitador/bot_digitador/worker/__init__.py b/bots/bot_digitador/bot_digitador/worker/__init__.py new file mode 100644 index 0000000..98c3f38 --- /dev/null +++ b/bots/bot_digitador/bot_digitador/worker/__init__.py @@ -0,0 +1 @@ +"""Worker remoto del bot digitador.""" diff --git a/bots/bot_digitador/bot_digitador/worker/__main__.py b/bots/bot_digitador/bot_digitador/worker/__main__.py new file mode 100644 index 0000000..7aa208c --- /dev/null +++ b/bots/bot_digitador/bot_digitador/worker/__main__.py @@ -0,0 +1,6 @@ +"""Permite: python -m bot_digitador.worker""" + +from bot_digitador.worker.run import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bots/bot_digitador/bot_digitador/worker/agent.py b/bots/bot_digitador/bot_digitador/worker/agent.py new file mode 100644 index 0000000..435ca14 --- /dev/null +++ b/bots/bot_digitador/bot_digitador/worker/agent.py @@ -0,0 +1,315 @@ +from __future__ import annotations + +import os +import socket +import tempfile +import time +from pathlib import Path +from typing import Any, Dict, List, Optional + +import httpx + +from bot_digitador.core.coords import DEFAULT_APP_PATH +from bot_digitador.core.digitador_service import DigitadorService +from bot_digitador.worker.minio_upload import MinioSettings, WorkerMinioUploader + +JOB_COMPLETED = "completed" +JOB_FAILED = "failed" +JOB_CANCELLED = "cancelled" + +REPORT_FILES = ("RESULTADOS.csv", "OMITIDOS.csv", "ejecucion_bot.log") + + +class RemoteApiClient: + """Cliente HTTP hacia la API bot-digitador (vía gateway). Sin puertos locales.""" + + def __init__(self, base_url: str, token: str): + self.base_url = base_url.rstrip("/") + self.token = token + self._client = httpx.Client( + timeout=httpx.Timeout(120.0, connect=10.0), + headers={"Authorization": f"Bearer {token}"}, + ) + + def close(self) -> None: + self._client.close() + + def _url(self, path: str) -> str: + return f"{self.base_url}{path}" + + def register(self, worker_id: str, hostname: str = "", max_concurrent: int = 1) -> dict: + r = self._client.post( + self._url("/workers/register"), + json={ + "worker_id": worker_id, + "hostname": hostname or socket.gethostname(), + "max_concurrent": max_concurrent, + }, + ) + r.raise_for_status() + return r.json() + + def heartbeat(self, worker_id: str) -> dict: + r = self._client.post( + self._url("/workers/heartbeat"), + json={"worker_id": worker_id}, + ) + r.raise_for_status() + return r.json() + + def claim_pending(self, worker_id: str) -> Optional[dict]: + r = self._client.get( + self._url("/workers/me/pending"), + params={"worker_id": worker_id}, + ) + r.raise_for_status() + data = r.json() + return data.get("job") + + def download_file(self, job_id: str, dest: Path) -> Path: + r = self._client.get(self._url(f"/jobs/{job_id}/file")) + r.raise_for_status() + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(r.content) + return dest + + def post_event(self, job_id: str, event: Dict[str, Any]) -> None: + r = self._client.post(self._url(f"/jobs/{job_id}/events"), json=event) + r.raise_for_status() + + def cancel_flag(self, job_id: str) -> bool: + r = self._client.get(self._url(f"/jobs/{job_id}/cancel-flag")) + r.raise_for_status() + return bool(r.json().get("cancel_requested")) + + def patch_job(self, job_id: str, **body) -> dict: + r = self._client.patch(self._url(f"/jobs/{job_id}"), json=body) + r.raise_for_status() + return r.json() + + +class WorkerAgent: + """Agent idle: heartbeat + claim jobs de digitación asignados.""" + + def __init__( + self, + api_url: str, + worker_id: str, + token: str, + poll_seconds: float = 8.0, + staging_root: Optional[Path] = None, + app_path: Optional[str] = None, + date_format: str = "dd/mm/yyyy", + delimiter: str = "|", + log_func=None, + ): + self.api = RemoteApiClient(api_url, token) + self.worker_id = worker_id + self.poll_seconds = poll_seconds + self.staging_root = staging_root or Path(tempfile.gettempdir()) / "bot_digitador" + self.app_path = app_path or os.getenv("DIGITADOR_APP_PATH") or DEFAULT_APP_PATH + self.date_format = date_format + self.delimiter = delimiter + self.minio = WorkerMinioUploader(MinioSettings.from_env()) + self._running = True + self._uploaded_keys: List[str] = [] + self._log = log_func or print + + def stop(self) -> None: + self._running = False + + def run_forever(self) -> None: + self.api.register(self.worker_id) + self._log(f"[worker {self.worker_id}] registrado — idle (esperando jobs de digitación)") + try: + while self._running: + try: + self.api.heartbeat(self.worker_id) + job = self.api.claim_pending(self.worker_id) + if job: + self._run_job(job) + else: + time.sleep(self.poll_seconds) + except KeyboardInterrupt: + raise + except Exception as e: + self._log(f"[worker {self.worker_id}] error en loop: {e}") + time.sleep(self.poll_seconds) + finally: + self.api.close() + self._log(f"[worker {self.worker_id}] detenido") + + def _run_job(self, job: dict) -> None: + job_id = job["id"] + reports_prefix = (job.get("reports_prefix") or "").strip() or f"digitados/{job_id}/" + self._uploaded_keys = [] + work_dir = self.staging_root / job_id + out_dir = work_dir / "out" + out_dir.mkdir(parents=True, exist_ok=True) + + csv_name = Path(job.get("csv_key") or job.get("file_key") or "input.csv").name + if not csv_name.lower().endswith(".csv"): + csv_name = "input.csv" + csv_path = work_dir / csv_name + + self._log(f"[worker {self.worker_id}] job {job_id} iniciado (prefix={reports_prefix})") + + try: + self.api.download_file(job_id, csv_path) + + def should_cancel() -> bool: + try: + return self.api.cancel_flag(job_id) + except Exception: + return False + + def on_event(event) -> None: + self._log(event.display_message()) + payload = { + "event_type": event.event_type, + "message": event.message, + "radicado": event.radicado, + "index": event.index, + "total": event.total, + "metadata": event.metadata, + "timestamp": event.timestamp, + } + try: + self.api.post_event(job_id, payload) + except Exception as e: + self._log(f"[worker] no se pudo enviar evento: {e}") + + date_format = job.get("date_format") or self.date_format + delimiter = job.get("delimiter") or self.delimiter + + service = DigitadorService( + csv_path=str(csv_path), + output_dir=str(out_dir), + date_format=date_format, + delimiter=delimiter, + app_path=self.app_path, + event_callback=on_event, + should_cancel=should_cancel, + ) + summary = service.run() + + # En shards, nombres únicos bajo el prefijo compartido del padre + # para no pisar RESULTADOS.csv / OMITIDOS.csv entre bots. + shard_index = job.get("shard_index") + shard_suffix = "" + if shard_index is not None: + try: + shard_suffix = f"_shard_{int(shard_index) + 1}" + except (TypeError, ValueError): + shard_suffix = f"_shard_{shard_index}" + + def remote_name(local_name: str) -> str: + if not shard_suffix: + return local_name + stem = Path(local_name).stem + suffix = Path(local_name).suffix + return f"{stem}{shard_suffix}{suffix}" + + for name in REPORT_FILES: + local = out_dir / name + if not local.exists(): + continue + try: + key = self.minio.upload_report( + job_id, + local, + reports_prefix=reports_prefix, + object_name=remote_name(name), + ) + self._uploaded_keys.append(key) + self._log(f"[worker] subido {key}") + except Exception as e: + self._log(f"[worker] error subiendo {name}: {e}") + + try: + self.minio.write_manifest( + job_id, + self._uploaded_keys, + reports_prefix=reports_prefix, + manifest_name=( + f"_manifest{shard_suffix}.json" if shard_suffix else "_manifest.json" + ), + extra={ + "exitosos": summary.exitosos, + "fallidos": summary.fallidos, + "existentes": summary.existentes, + "omitidos": summary.omitidos, + "timeouts": summary.timeouts, + "shard_index": shard_index, + }, + ) + except Exception as e: + self._log(f"[worker] manifest no escrito: {e}") + + result = { + "total": summary.total, + "exitosos": summary.exitosos, + "fallidos": summary.fallidos, + "existentes": summary.existentes, + "omitidos": summary.omitidos, + "timeouts": summary.timeouts, + "cancelled": summary.cancelled, + "reports_prefix": reports_prefix, + "uploaded_count": len(self._uploaded_keys), + } + + if summary.cancelled: + status = JOB_CANCELLED + error_msg = None + elif summary.fatal_error: + status = JOB_FAILED + error_msg = summary.fatal_error + elif summary.fallidos and summary.exitosos == 0 and summary.existentes == 0: + status = JOB_FAILED + error_msg = "Todos los radicados fallaron" + else: + status = JOB_COMPLETED + error_msg = None + + if error_msg: + try: + self.api.post_event( + job_id, + { + "event_type": "warning", + "message": error_msg, + "metadata": {"fatal": bool(summary.fatal_error)}, + }, + ) + except Exception: + pass + + self.api.patch_job( + job_id, + status=status, + result=result, + error=error_msg, + ) + self._log(f"[worker {self.worker_id}] job {job_id} → {status}") + + except Exception as e: + self._log(f"[worker {self.worker_id}] job {job_id} error: {e}") + try: + self.api.post_event( + job_id, + { + "event_type": "warning", + "message": f"Error fatal: {e}", + "metadata": {"fatal": True}, + }, + ) + except Exception: + pass + try: + self.api.patch_job( + job_id, + status=JOB_FAILED, + error=str(e), + ) + except Exception: + pass diff --git a/bots/bot_digitador/bot_digitador/worker/minio_upload.py b/bots/bot_digitador/bot_digitador/worker/minio_upload.py new file mode 100644 index 0000000..90dcf21 --- /dev/null +++ b/bots/bot_digitador/bot_digitador/worker/minio_upload.py @@ -0,0 +1,163 @@ +"""Cliente MinIO para subir reportes del digitador (solo salida).""" + +from __future__ import annotations + +import json +import mimetypes +import os +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional + + +@dataclass(frozen=True) +class MinioSettings: + endpoint: str = "https://s3.builker.xyz" + bucket: str = "pys-cuentas-medicas" + access_key: str = "" + secret_key: str = "" + region: str = "us-east-1" + anonymous: bool = True + + @classmethod + def from_env(cls) -> "MinioSettings": + access_key = (os.getenv("MINIO_ACCESS_KEY") or "").strip() + secret_key = (os.getenv("MINIO_SECRET_KEY") or "").strip() + anon_flag = (os.getenv("MINIO_ANONYMOUS") or "").strip().lower() + if anon_flag in ("0", "false", "no"): + anonymous = False + elif anon_flag in ("1", "true", "yes"): + anonymous = True + else: + anonymous = not (access_key and secret_key) + return cls( + endpoint=os.getenv("MINIO_ENDPOINT", "https://s3.builker.xyz").rstrip("/"), + bucket=os.getenv("MINIO_BUCKET", "pys-cuentas-medicas"), + access_key=access_key, + secret_key=secret_key, + region=os.getenv("MINIO_REGION", "us-east-1"), + anonymous=anonymous, + ) + + +def report_key_from_prefix(reports_prefix: str, filename: str) -> str: + prefix = (reports_prefix or "").strip().rstrip("/") + if not prefix: + raise ValueError("reports_prefix vacío") + name = Path(filename).name + return f"{prefix}/{name}" + + +def report_key(job_id: str, filename: str) -> str: + return f"digitados/{job_id}/{Path(filename).name}" + + +def manifest_key_from_prefix(reports_prefix: str) -> str: + prefix = (reports_prefix or "").strip().rstrip("/") + return f"{prefix}/_manifest.json" + + +def manifest_key(job_id: str) -> str: + return f"digitados/{job_id}/_manifest.json" + + +def _s3_client(settings: MinioSettings): + import boto3 + from botocore import UNSIGNED + from botocore.client import Config + + s3_cfg = { + "addressing_style": "path", + "request_checksum_calculation": "when_required", + "response_checksum_validation": "when_required", + } + if settings.anonymous: + return boto3.client( + "s3", + endpoint_url=settings.endpoint, + region_name=settings.region, + config=Config(signature_version=UNSIGNED, s3=s3_cfg), + ) + return boto3.client( + "s3", + endpoint_url=settings.endpoint, + aws_access_key_id=settings.access_key, + aws_secret_access_key=settings.secret_key, + region_name=settings.region, + config=Config(signature_version="s3v4", s3=s3_cfg), + ) + + +class WorkerMinioUploader: + """PutObject de reportes CSV/log y manifest bajo digitados/.""" + + def __init__(self, settings: Optional[MinioSettings] = None): + self.settings = settings or MinioSettings.from_env() + self._client = None + + @property + def client(self): + if self._client is None: + self._client = _s3_client(self.settings) + return self._client + + def object_exists(self, key: str) -> bool: + try: + self.client.head_object(Bucket=self.settings.bucket, Key=key) + return True + except Exception: + return False + + def upload_report( + self, + job_id: str, + local_path: str | Path, + *, + reports_prefix: str | None = None, + object_name: str | None = None, + content_type: str | None = None, + ) -> str: + path = Path(local_path) + filename = object_name or path.name + if reports_prefix: + key = report_key_from_prefix(reports_prefix, filename) + else: + key = report_key(job_id, filename) + + ctype = content_type or mimetypes.guess_type(filename)[0] or "application/octet-stream" + self.client.put_object( + Bucket=self.settings.bucket, + Key=key, + Body=path.read_bytes(), + ContentType=ctype, + ) + return key + + def write_manifest( + self, + job_id: str, + keys: List[str], + *, + reports_prefix: str | None = None, + manifest_name: str = "_manifest.json", + extra: Optional[dict] = None, + ) -> str: + if reports_prefix: + key = report_key_from_prefix(reports_prefix, manifest_name) + else: + key = report_key(job_id, manifest_name) + payload = { + "job_id": job_id, + "reports_prefix": reports_prefix or f"digitados/{job_id}/", + "files": keys, + "count": len(keys), + } + if extra: + payload.update(extra) + self.client.put_object( + Bucket=self.settings.bucket, + Key=key, + Body=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + ContentType="application/json", + ) + return key diff --git a/bots/bot_digitador/bot_digitador/worker/run.py b/bots/bot_digitador/bot_digitador/worker/run.py new file mode 100644 index 0000000..415b48f --- /dev/null +++ b/bots/bot_digitador/bot_digitador/worker/run.py @@ -0,0 +1,64 @@ +"""CLI del agent worker digitador: idle hasta que el usuario asigne un job.""" + +from __future__ import annotations + +import argparse +import os +import sys + +from bot_digitador.core.coords import DEFAULT_APP_PATH +from bot_digitador.worker.agent import WorkerAgent + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + description="Agent BotDigitador: registra el PC y espera jobs de digitación." + ) + p.add_argument( + "--api-url", + default=os.getenv("BOT_API_URL", "http://localhost:8189/bot-digitador"), + help="URL del API vía gateway (default: …/bot-digitador)", + ) + p.add_argument( + "--worker-id", + required=True, + help="Identificador único de este PC (ej. PC-DIGITADOR-01)", + ) + p.add_argument( + "--token", + default=os.getenv("BOT_API_TOKEN", "dev-token-change-me"), + help="Bearer token (BOT_API_TOKEN)", + ) + p.add_argument( + "--app-path", + default=os.getenv("DIGITADOR_APP_PATH", DEFAULT_APP_PATH), + help="Ruta al acceso directo / ejecutable de Integración Total", + ) + p.add_argument( + "--poll", + type=float, + default=8.0, + help="Segundos entre heartbeat/claim", + ) + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + agent = WorkerAgent( + api_url=args.api_url, + worker_id=args.worker_id, + token=args.token, + poll_seconds=args.poll, + app_path=args.app_path, + ) + try: + agent.run_forever() + except KeyboardInterrupt: + print("\nDetenido por usuario") + agent.stop() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bots/bot_digitador/main.py b/bots/bot_digitador/main.py new file mode 100644 index 0000000..7c93134 --- /dev/null +++ b/bots/bot_digitador/main.py @@ -0,0 +1,106 @@ +"""CLI local del Bot Digitador (sin API remota).""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +from bot_digitador.core.coords import DEFAULT_APP_PATH +from bot_digitador.core.digitador_service import DigitadorService +from bot_digitador.models.events import DigitadorEvent + + +def _print_event(event: DigitadorEvent) -> None: + print(event.display_message(), flush=True) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Digita radicados en Integración Total desde un CSV local." + ) + parser.add_argument( + "--csv", + required=True, + help="Ruta al CSV de entrada (coma o pipe)", + ) + parser.add_argument( + "--output-dir", + default=None, + help="Carpeta de reportes (default: /reportes_bot/)", + ) + parser.add_argument( + "--delimiter", + choices=[",", "|"], + default="|", + help="Delimitador CSV (se auto-detecta si no coincide con columnas)", + ) + parser.add_argument( + "--date-format", + choices=["dd/mm/yyyy", "mm/dd/yyyy"], + default="dd/mm/yyyy", + help="Formato preferido de fechas en el CSV", + ) + parser.add_argument( + "--app-path", + default=os.getenv("DIGITADOR_APP_PATH", DEFAULT_APP_PATH), + help="Ruta al acceso directo / ejecutable de Integración Total", + ) + parser.add_argument( + "--reinicio-cada", + type=int, + default=6, + help="Reinicio completo de la app cada N registros", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + + csv_path = Path(args.csv).resolve() + if not csv_path.exists(): + print(f"CSV no encontrado: {csv_path}", file=sys.stderr) + return 2 + + if args.output_dir: + output_dir = Path(args.output_dir).resolve() + else: + from datetime import datetime + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + output_dir = csv_path.parent / "reportes_bot" / stamp + + output_dir.mkdir(parents=True, exist_ok=True) + + service = DigitadorService( + csv_path=str(csv_path), + output_dir=str(output_dir), + date_format=args.date_format, + delimiter=args.delimiter, + app_path=args.app_path, + reinicio_cada=args.reinicio_cada, + event_callback=_print_event, + ) + summary = service.run() + + print( + f"Resumen: OK={summary.exitosos}/{summary.total}, " + f"fallidos={summary.fallidos}, existentes={summary.existentes}, " + f"omitidos={summary.omitidos}, timeouts={summary.timeouts}, " + f"cancelled={summary.cancelled}", + flush=True, + ) + print(f"Reportes en: {summary.output_dir}", flush=True) + + if summary.fatal_error: + print(f"Error fatal: {summary.fatal_error}", file=sys.stderr) + return 1 + if summary.cancelled: + return 130 + return 0 if summary.fallidos == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bots/bot_digitador/requirements.txt b/bots/bot_digitador/requirements.txt new file mode 100644 index 0000000..ffa2f98 --- /dev/null +++ b/bots/bot_digitador/requirements.txt @@ -0,0 +1,8 @@ +httpx>=0.27.0 +boto3>=1.34.0 +pywinauto>=0.6.8 +opencv-python-headless>=4.8.0 +numpy>=1.24.0 +customtkinter>=5.2.0 +python-dotenv>=1.0.0 +Pillow>=10.0.0 diff --git a/bots/bot_digitador/run_gui.py b/bots/bot_digitador/run_gui.py new file mode 100644 index 0000000..147178b --- /dev/null +++ b/bots/bot_digitador/run_gui.py @@ -0,0 +1,12 @@ +"""Punto de entrada de la interfaz gráfica del bot digitador.""" + +from bot_digitador.ui.dashboard import BotDashboard + + +def main(): + app = BotDashboard() + app.mainloop() + + +if __name__ == "__main__": + main() diff --git a/bots/bot_digitador/template/README.md b/bots/bot_digitador/template/README.md new file mode 100644 index 0000000..63139f6 --- /dev/null +++ b/bots/bot_digitador/template/README.md @@ -0,0 +1,16 @@ +# Plantillas OpenCV + +Copia aquí los PNG necesarios desde la carpeta `template/` del script original +(`bot_saludtotal.py` / proyecto de digitación): + +- `error.png` +- `success.png` +- `bien.png` +- `flecha_diferencias_abierto.png` +- `flecha_diferencias_cerrado.png` +- `fecha_vacia_slash.png` +- `fecha_vacia_blanco.png` +- `popup_nombres.png` +- `radicado_existente.png` + +Sin estos archivos el bot no puede validar pantallas de Integración Total. diff --git a/descarga.py b/descarga.py new file mode 100644 index 0000000..9a499f6 --- /dev/null +++ b/descarga.py @@ -0,0 +1,1847 @@ +# -*- coding: utf-8 -*- +from email.mime import base +import shutil +import os +import re +import csv +import threading +import traceback +from pathlib import Path +from datetime import datetime +import tkinter as tk +from tkinter import filedialog, messagebox, ttk + +import pandas as pd + + +# ============================================================ +# CONFIGURACIÓN GENERAL +# ============================================================ + +COLUMNAS_FINAL_IT = [ + "radicado", + "total_cargos", + "autorizacion", + "ingreso", + "tipo_documento", + "documento", + "tipo_servicio", + "copago", + "cuota_moderadora", +] + +CSV_FINAL_IT_HEADERS = [ + "RADICADO", + "BRUTO", + "AUTORIZACION", + "FECHA", + "TIPO_ID", + "NUMERO", + "SERVICIO", + "COPAGO", + "MODERADORA", +] + +CSV_FINAL_IT_RENAME = { + "radicado": "RADICADO", + "total_cargos": "BRUTO", + "autorizacion": "AUTORIZACION", + "ingreso": "FECHA", + "tipo_documento": "TIPO_ID", + "documento": "NUMERO", + "tipo_servicio": "SERVICIO", + "copago": "COPAGO", + "cuota_moderadora": "MODERADORA", +} + +# Tabla de cuotas moderadoras por año. +TABLA_MODERADORA = { + 2023: {4100, 16400, 43000}, + 2024: {4500, 18200, 47700}, + 2025: {4700, 19200, 50300}, + 2026: {5000, 20100, 52800}, +} + +# Longitudes según tabla entregada por el usuario. +# Los tipos marcados como alfanuméricos permiten letras y números. +REGLAS_DOCUMENTO = { + "CC": {"min": 3, "max": 10, "tipo": "num"}, + "CE": {"min": 3, "max": 7, "tipo": "num"}, + "TI": {"min": 10, "max": 10, "tipo": "num"}, + "RC": {"min": 10, "max": 10, "tipo": "num"}, + "CN": {"min": 9, "max": 14, "tipo": "num"}, + "PT": {"min": 7, "max": 8, "tipo": "num"}, + "PE": {"min": 15, "max": 15, "tipo": "num"}, + "PA": {"min": 3, "max": 16, "tipo": "alnum"}, + "PS": {"min": 3, "max": 16, "tipo": "alnum"}, + "AS": {"min": 3, "max": 16, "tipo": "alnum"}, + "MS": {"min": 3, "max": 16, "tipo": "alnum"}, +} + +HOMOLOGACION_TIPO_DOC = { + "CC": "CC", "C.C": "CC", "C.C.": "CC", "CEDULA": "CC", "CEDULA CIUDADANIA": "CC", + "CÉDULA": "CC", "CÉDULA CIUDADANÍA": "CC", "CEDULA DE CIUDADANIA": "CC", "Cédula Ciudadanía": "CC", "Cédula_Ciudadanía": "CC", + "TI": "TI", "T.I": "TI", "T.I.": "TI", "TARJETA IDENTIDAD": "TI", "TARJETA DE IDENTIDAD": "TI", "tarjeta identidad": "TI", "tarjeta_identidad": "TI", + "RC": "RC", "R.C": "RC", "R.C.": "RC", "REGISTRO CIVIL": "RC", "REGISTRO_CIVIL": "RC", "registro civil": "RC", "registro_civil": "RC", + "CE": "CE", "C.E": "CE", "C.E.": "CE", "CEDULA EXTRANJERIA": "CE", "CÉDULA EXTRANJERÍA": "CE", "CEDULA DE EXTRANJERIA": "CE", "Cédula Extranjería": "CE", "Cédula_Extranjería": "CE", + "PA": "PA", "PASAPORTE": "PA", + "PS": "PS", "PERMISO ESPECIAL": "PS", + "PT": "PT", "PERMISO PROTECCION TEMPORAL": "PT", "PERMISO POR PROTECCION TEMPORAL": "PT", "PERMISO PROTECCION TEMPORAL": "PT", "Permiso Protección Temporal": "PT", "Permiso_Protección_Temporal": "PT", + "PE": "PE", "PERMISO ESPECIAL": "PE", "PERMISO ESPECIAL DE PERMANENCIA": "PE", "Permiso Especial de Permanencia": "PE", "Permiso_Especial_de_Permanencia": "PE", + "CN": "CN", "CERTIFICADO NACIDO VIVO": "CN", "NACIDO VIVO": "CN", "CERTIFICADO NACIDO": "CN", "Certificado Nacido Vivo": "CN", "Certificado_Nacido_Vivo": "CN", + "AS": "AS", "ADULTO SIN IDENTIFICACION": "AS", "ADULTO_SIN_IDENTIFICACION": "AS", + "MS": "MS", "MENOR SIN IDENTIFICACION": "MS", "MENOR_SIN_IDENTIFICACION": "MS", "MENOR SIN IDENTIFICACIÓN": "MS", "Menor Sin Identificación": "MS", "Menor_Sin_Identificación": "MS", +} + +# Diccionario inicial basado en los ejemplos que compartiste. +# Se puede ampliar fácilmente. +DICCIONARIO_COLUMNAS = { + "radicado": [ + "radicado", "radicacion", "radicación", "numero radicado", "numero_radicado", + "numero radicacion", "numero_radicacion", "numeroradicacion", "numero de radicacion", + "no radicado", "nro radicado", "num radicado", "id radicado" + ], + "autorizacion": [ + "autorizacion", "autorización", "autorrizacion", "autoriz", "autoriza", + "numero autorizacion", "num autorizacion", "no autorizacion", "nro autorizacion", + "aut", "numaut", "no_aut", "autoritiponumero" + ], + "ingreso": [ + "ingreso", "fecha ingreso", "fec ingreso", "fec.ingreso", "fecha_ingreso", + "fecha de ingreso", "fecha servicio", "fec servicio", "fecha_servicio", + "fecha", "ingeso" + ], + "tipo_documento": [ + "tipo_documento", "tipo documento", "tipo de documento", "tipo doc", "tipodoc", + "tipo", "td", "doc tipo", "documenoto tipo" + ], + "documento": [ + "documento", "docuemnto", "numero documento", "número documento", "no documento", + "no. documento", "num documento", "identificacion", "identificación", + "cedula", "cédula", "id paciente", "paciente id", "id" + ], + "tipo_documento_documento": [ + "tipo numero", "tiponumero", "tiponum", "tipnumero", "tipnum", "tipnumebr", "tipnumingraut", + "tipo / documento", "tipo/documento", "tipo muner", "documento tipo", "autoritiponumero", + "tipo y documento", "tipo-documento", "tipo doc documento", + "tipo identificacion", "tipo identificación", "tipo id", + "identificacion paciente", "identificación paciente", "identipo", + "tipo num doc", "tipo nro doc", "tipo no doc" + ], + "total_cargos": [ + "total_cargos", "total cargos", "bruto", "brutos", "brruto", "valor bruto", + "valorbruto", "valor factura", "valor_factura", "total factura", + "totalfactura", "valor facturado", "valor_facturado" + ], + "neto": [ + "neto", "valor neto", "valorneto", "netos", "valor pagado", "valor_pagado" + ], + "copago": [ + "copago", "copagos", "cop", "valor copago", "valor_copago" + ], + "cuota_moderadora": [ + "moderadora", "cuota moderadora", "cuota_moderadora", "cm", + "valor moderadora", "valor_moderadora", "moderradora" + ], + "cop_mod": [ + "cop/mod", "copago/moderadora", "copago cm", "copago/cm", "copago cuota moderadora", + "copago/CM", "copago cm", "valor usuario", "valor_usuario", "abono paciente", + "cuota usuario", "pago usuario", "pagousuario", "copago/CM" + ], + "tipo_servicio": [ + "tipo_servicio", "tipo servicio", "servicio", "tipo de servicio", + "origen_servicio", "origen servicio", "ambito", "ámbito" + ], +} + + +# ============================================================ +# UTILIDADES DE LECTURA +# ============================================================ + +def normalizar_nombre_columna(nombre): + if nombre is None: + return "" + s = str(nombre).strip().lower() + s = s.replace("\ufeff", "") + s = s.replace("_", " ") + s = re.sub(r"\s+", " ", s) + # quitar tildes básicas sin depender de librerías externas + trans = str.maketrans("áéíóúüñ", "aeiouun") + s = s.translate(trans) + return s + + +def detectar_delimitador(path): + try: + with open(path, "r", encoding="utf-8-sig", errors="ignore") as f: + muestra = f.read(4096) + delimitadores = ["|", ";", ",", "\t"] + conteos = {d: muestra.count(d) for d in delimitadores} + mejor = max(conteos, key=conteos.get) + return mejor if conteos[mejor] > 0 else "," + except Exception: + return "," + + +def leer_csv_flexible(path): + sep = detectar_delimitador(path) + intentos = [ + {"sep": sep, "encoding": "utf-8-sig"}, + {"sep": sep, "encoding": "latin1"}, + {"sep": ";", "encoding": "utf-8-sig"}, + {"sep": ",", "encoding": "utf-8-sig"}, + {"sep": "|", "encoding": "utf-8-sig"}, + ] + + ultimo_error = None + for opt in intentos: + try: + df = pd.read_csv( + path, + sep=opt["sep"], + encoding=opt["encoding"], + dtype=str, + engine="python", + keep_default_na=False, + on_bad_lines="skip", + ) + df.columns = [str(c).strip() for c in df.columns] + return df + except Exception as e: + ultimo_error = e + + raise RuntimeError(f"No se pudo leer CSV {path.name}: {ultimo_error}") + + +def encontrar_columna(df, campo): + cols_norm = {normalizar_nombre_columna(c): c for c in df.columns} + equivalencias = DICCIONARIO_COLUMNAS.get(campo, []) + + for eq in equivalencias: + eq_norm = normalizar_nombre_columna(eq) + if eq_norm in cols_norm: + return cols_norm[eq_norm] + + # búsqueda por contiene, útil para encabezados largos + for eq in equivalencias: + eq_norm = normalizar_nombre_columna(eq) + for c_norm, c_original in cols_norm.items(): + if eq_norm and (eq_norm in c_norm or c_norm in eq_norm): + return c_original + + return None + + +# ============================================================ +# LIMPIEZA Y NORMALIZACIÓN +# ============================================================ + +def limpiar_texto(valor): + if valor is None: + return "" + s = str(valor).strip() + if s.lower() in {"nan", "none", "null"}: + return "" + return s + + +def limpiar_numero_entero(valor, default=""): + """ + Limpia valores monetarios sin inflar decimales. + + Ejemplos: + - $ 150.000,00 -> 150000 + - 150000,00 -> 150000 + - 150000.00 -> 150000 + - 1.250.000,00 -> 1250000 + - 1,250,000.00 -> 1250000 + - 150.000 -> 150000 + - 150,000 -> 150000 + """ + s = limpiar_texto(valor) + if not s: + return default + + s = s.replace("$", "").replace("COP", "").replace("cop", "").strip() + s = re.sub(r"[^\d,.\-]", "", s) + + if not s: + return default + + # Para este proceso esperamos valores positivos. + s = s.replace("-", "") + + # Caso con coma y punto: el último separador define el decimal. + if "," in s and "." in s: + last_comma = s.rfind(",") + last_dot = s.rfind(".") + + if last_comma > last_dot: + # Formato colombiano: 1.250.000,00 + entero = s[:last_comma] + decimal = s[last_comma + 1:] + if len(decimal) <= 2: + s = entero.replace(".", "").replace(",", "") + else: + s = s.replace(".", "").replace(",", "") + else: + # Formato inglés: 1,250,000.00 + entero = s[:last_dot] + decimal = s[last_dot + 1:] + if len(decimal) <= 2: + s = entero.replace(",", "").replace(".", "") + else: + s = s.replace(",", "").replace(".", "") + + elif "," in s: + partes = s.split(",") + ultima = partes[-1] + + # 150000,00 -> decimal. Se elimina la parte decimal. + if len(ultima) in (1, 2): + s = "".join(partes[:-1]) + else: + # 150,000 -> miles + s = "".join(partes) + + elif "." in s: + partes = s.split(".") + ultima = partes[-1] + + # 150000.00 -> decimal. + # 150.000 -> miles porque la última parte tiene 3 dígitos. + if len(ultima) in (1, 2): + s = "".join(partes[:-1]) + else: + s = "".join(partes) + + s = re.sub(r"\D", "", s) + return s if s else default + + +def to_int(valor, default=0): + s = limpiar_numero_entero(valor, default="") + if not s: + return default + try: + return int(s) + except Exception: + return default + + +def homologar_tipo_documento(valor): + """ + Homologa tipo de documento aunque venga con texto adicional. + + Ejemplos: + - C.C. -> CC + - CÉDULA CIUDADANIA -> CC + - CEDULA CIUDAD 1021676637 -> CC + - CC 1012333072 -> CC + - RC 1188975098 -> RC + """ + s = limpiar_texto(valor).upper() + if not s: + return "" + + trans = str.maketrans("ÁÉÍÓÚÜÑ", "AEIOUUN") + s = s.translate(trans) + + s = s.replace(".", "").replace("-", " ").replace("_", " ") + s = re.sub(r"\s+", " ", s).strip() + + # Casos textuales de cédula + if "CEDULA" in s and ("CIUDAD" in s or "CIUDADANIA" in s): + return "CC" + + # Si empieza por tipo válido y luego trae documento/nombre + m = re.match(r"^(CC|TI|RC|CE|PA|PS|PT|PE|CN|AS|MS)\b", s) + if m: + return m.group(1) + + return HOMOLOGACION_TIPO_DOC.get(s, s) + + +def limpiar_documento(valor, tipo_doc): + s = limpiar_texto(valor).upper() + if tipo_doc in {"PA", "PS", "AS", "MS"}: + s = re.sub(r"[^A-Z0-9]", "", s) + else: + s = re.sub(r"\D", "", s) + return s + + +def validar_documento(tipo_doc, documento): + tipo_doc = homologar_tipo_documento(tipo_doc) + documento = limpiar_documento(documento, tipo_doc) + + # Si el tipo está vacío, pero el documento numérico es usable, pasa porque el bot procesa el número. + if not tipo_doc: + if documento and documento.isdigit() and 3 <= len(documento) <= 16: + return True, "" + return False, "Tipo de documento vacío y documento inválido" + + if tipo_doc not in REGLAS_DOCUMENTO: + return False, f"Tipo de documento no parametrizado: {tipo_doc}" + + regla = REGLAS_DOCUMENTO[tipo_doc] + largo = len(documento) + + if largo < regla["min"] or largo > regla["max"]: + if regla["min"] == regla["max"]: + return False, f"Longitud inválida para {tipo_doc}. Debe tener {regla['min']} dígitos." + return False, f"Longitud inválida para {tipo_doc}. Debe tener entre {regla['min']} y {regla['max']} dígitos." + + if regla["tipo"] == "num" and not documento.isdigit(): + return False, f"Documento inválido para {tipo_doc}. Debe ser numérico." + + if regla["tipo"] == "alnum" and not re.match(r"^[A-Z0-9]+$", documento): + return False, f"Documento inválido para {tipo_doc}. Debe ser alfanumérico." + + return True, "" + + +def normalizar_fecha(valor): + """ + Normaliza ingreso a DD/MM/YYYY. + + Soporta: + - 2026-05-13 00:00 + - 13/05/2026 + - 13-05-2026 + - 29 abr. 2026 06:52 p. m. + - ATENCION 07/05/2026. 8/05/2026 -> toma la primera fecha válida + """ + s = limpiar_texto(valor) + if not s: + return "", "fecha ingreso vacia" + + s_original = s + s = s.strip() + s_upper = s.upper() + if any(marker in s_upper for marker in ["SIN_FECHA", "SIN FECHA", "NO FECHA", "SINFECHA"]): + return "", "sin fecha" + + trans = str.maketrans("ÁÉÍÓÚÜÑáéíóúüñ", "AEIOUUNaeiouun") + s_norm = s.translate(trans) + s_norm = re.sub(r"\s+", " ", s_norm).strip() + s_norm = s_norm.upper() + s_norm = re.sub(r"[^A-Z0-9/\-.:\s]", " ", s_norm) + + meses = { + "ENE": "01", "ENERO": "01", + "FEB": "02", "FEBRERO": "02", + "MAR": "03", "MARZO": "03", + "ABR": "04", "ABRIL": "04", + "MAY": "05", "MAYO": "05", + "JUN": "06", "JUNIO": "06", + "JUL": "07", "JULIO": "07", + "AGO": "08", "AGOSTO": "08", + "SEP": "09", "SEPT": "09", "SEPTIEMBRE": "09", + "OCT": "10", "OCTUBRE": "10", + "NOV": "11", "NOVIEMBRE": "11", + "DIC": "12", "DICIEMBRE": "12", + } + + def reemplazar_mes(match): + token = re.sub(r"[^A-Z]", "", match.group(0).upper()) + if token in meses: + return meses[token] + if token.endswith("BR"): + return meses["ABR"] + return match.group(0) + + s_mes = re.sub(r"\b[\w\.]{3,12}\b", reemplazar_mes, s_norm) + s_mes = re.sub(r"\bP\s*\.\s*M\b", "PM", s_mes) + s_mes = re.sub(r"\bA\s*\.\s*M\b", "AM", s_mes) + s_mes = re.sub(r"\bP\s*M\b", "PM", s_mes) + s_mes = re.sub(r"\bA\s*M\b", "AM", s_mes) + s_mes = re.sub(r"\s+", " ", s_mes).strip() + + # Buscar primero fechas numéricas dentro del texto y tomar la primera válida. + fechas_en_texto = re.findall(r"\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b", s_mes) + for f in fechas_en_texto: + try: + fecha = pd.to_datetime(f, dayfirst=True, errors="raise") + if 1900 <= fecha.year <= 2100: + return fecha.strftime("%d/%m/%Y"), "" + except Exception: + pass + + # Intentar parsear después de reemplazar meses en español. + for dayfirst in [True, False]: + try: + fecha = pd.to_datetime(s_mes, dayfirst=dayfirst, errors="raise") + if pd.isna(fecha): + continue + if 1900 <= fecha.year <= 2100: + return fecha.strftime("%d/%m/%Y"), "" + except Exception: + pass + + # Serial de Excel aproximado + if re.fullmatch(r"\d{5}", s_mes): + try: + fecha = pd.to_datetime(float(s_mes), unit="D", origin="1899-12-30") + return fecha.strftime("%d/%m/%Y"), "" + except Exception: + pass + + # Formato compacto YYYYMMDD o DDMMYYYY + solo_digitos = re.sub(r"\D", "", s_mes) + if len(solo_digitos) == 8: + candidatos = [ + solo_digitos[:4] + "-" + solo_digitos[4:6] + "-" + solo_digitos[6:], + solo_digitos[:2] + "/" + solo_digitos[2:4] + "/" + solo_digitos[4:], + ] + for c in candidatos: + try: + fecha = pd.to_datetime(c, dayfirst=("/" in c), errors="raise") + if 1900 <= fecha.year <= 2100: + return fecha.strftime("%d/%m/%Y"), "" + except Exception: + pass + + # ISO claro + if re.match(r"^\d{4}-\d{1,2}-\d{1,2}(?:[ T]\d{1,2}:\d{2}(?::\d{2})?(?:[-+]\d{2}:?\d{2})?)?$", s_mes): + try: + fecha = pd.to_datetime(s_mes, dayfirst=False, errors="raise") + if 1900 <= fecha.year <= 2100: + return fecha.strftime("%d/%m/%Y"), "" + except Exception: + pass + + return "", f"Fecha ingreso inválida: {s_original}" + + +def obtener_anio_fecha(fecha_ddmmyyyy): + try: + return int(fecha_ddmmyyyy[-4:]) + except Exception: + return None + + +def normalizar_autorizacion(valor): + """ + Normaliza autorización. + + Reglas: + - Si viene vacía o con marcador de ausencia -> observación. + - Si viene con guion -> se conserva. + - Si viene pegada sin guion -> se agrega guion después de los primeros 5 caracteres. + Ejemplo: + 128242624857057 -> 12824-2624857057 + - Después de normalizar debe quedar con largo 16 incluyendo el guion. + """ + s_raw = limpiar_texto(valor) + s = s_raw.upper() + if not s: + return "", "autorizacion vacia" + + if any(marker in s for marker in ["SIN_AUTORIZACION", "SIN AUTORIZACION", "NO AUTORIZACION", "NO AUTORIZACIÓN", "NO AUTORIZACION"]): + return "", "sin autorizacion" + + s = s.replace("AUTORIZACION", "").replace("AUTORIZACIÓN", "") + s = s.replace(":", " ").strip() + s = re.sub(r"[^A-Z0-9\-]", "", s) + + if not s: + return "", "autorizacion vacia" + + if s.isdigit() and len(s) == 10: + return s, "autorizacion incompleta" + + # Si viene sin guion y con caracteres suficientes, agregar guion después del 5.º carácter. + if "-" not in s and len(s) > 5: + s = s[:5] + "-" + s[5:] + + if len(s) == 16: + return s, "" + + return s, "autorizacion invalida" + + +def separar_copago_moderadora(valor, anio): + """ + Si el valor coincide con tabla moderadora del año => cuota_moderadora. + Si no coincide => copago. + """ + monto = to_int(valor, default=0) + if monto == 0: + return 0, 0 + + if anio in TABLA_MODERADORA and monto in TABLA_MODERADORA[anio]: + return 0, monto + + return monto, 0 + + +def normalizar_tipo_servicio(valor): + s = limpiar_texto(valor).upper() + s = re.sub(r"\s+", " ", s).strip() + # Se respeta porque el usuario indicó que ya lo manejan bien. + return s + + +# ============================================================ +# DETECCIÓN Y EXTRACCIÓN ESPECIAL +# ============================================================ + +def es_archivo_especial(df): + """ + Heurística: + - Alguna columna tiene textos largos. + - Y contiene palabras clave típicas del archivo especial: + AUTORIZACION, PACIENTE, INGRESO, ABONO. + """ + if df.empty: + return False + + palabras = ["AUTORIZACION", "AUTORIZACIÓN", "PACIENTE", "INGRESO", "ABONO"] + for col in df.columns: + serie = df[col].astype(str).head(50) + textos = " ".join(serie.tolist()).upper() + largo_promedio = serie.map(lambda x: len(str(x))).mean() + hits = sum(1 for p in palabras if p in textos) + + if largo_promedio >= 45 and hits >= 2: + return True + + return False + + +def extraer_tipo_doc_y_documento_desde_texto(texto): + """ + Extrae tipo y número de documento cuando vienen mezclados. + + Soporta: + - CC 1065138068 + - CC-1065138068 + - C.C. 1065138068 + - CÉDULA CIUDAD 1021676637 NOMBRE PACIENTE + - CEDULA CIUDADANIA 1021676637 NOMBRE PACIENTE + - PACIENTE: RC-1253158282-NOMBRE + - RC 1.067.645.729 + - TI 1.066.894.756 + - solo número: 1023876746 => tipo vacío, documento 1023876746 + """ + t = limpiar_texto(texto).upper() + if not t: + return "", "" + + trans = str.maketrans("ÁÉÍÓÚÜÑ", "AEIOUUN") + t_norm = t.translate(trans) + t_norm = t_norm.replace("_", " ") + + t_norm = re.sub(r"\bC\s*\.?\s*C\.?\b", "CC", t_norm) + t_norm = re.sub(r"\bT\s*\.?\s*I\.?\b", "TI", t_norm) + t_norm = re.sub(r"\bR\s*\.?\s*C\.?\b", "RC", t_norm) + t_norm = re.sub(r"\bC\s*\.?\s*E\.?\b", "CE", t_norm) + + def limpiar_doc_texto(valor): + return re.sub(r"[^A-Z0-9]", "", valor.upper()) + + m = re.search(r"\bCEDULA(?:\s+DE)?\s+CIUDAD(?:ANIA)?\s+([0-9]{3,16})\b", t_norm) + if m: + return "CC", m.group(1) + + m = re.search(r"\bCEDULA\s+([0-9]{3,16})\b", t_norm) + if m: + return "CC", m.group(1) + + m = re.match(r"^([A-Z ]+?)\s+([0-9\.\s]+)$", t_norm) + if m: + tipo_candidato = homologar_tipo_documento(m.group(1)) + if tipo_candidato in REGLAS_DOCUMENTO: + return tipo_candidato, limpiar_doc_texto(m.group(2)) + + m = re.search(r"PACIENTE\s*:\s*(CC|TI|RC|CE|PA|PS|PT|PE|CN|AS|MS)\s*[-:/ ]+\s*([0-9\.\s]+)", t_norm) + if m: + return homologar_tipo_documento(m.group(1)), limpiar_doc_texto(m.group(2)) + + m = re.search( + r"(?:TIPO\s*/?\s*DOCUMENTO|TIPO\s*/?\s*NUMERO|TIPO\s+NUMERO|DOCUMENTO)\s*[:\- ]+\s*" + r"(CC|TI|RC|CE|PA|PS|PT|PE|CN|AS|MS)\s*[-:/ ]+\s*([0-9\.\s]+)", + t_norm + ) + if m: + return homologar_tipo_documento(m.group(1)), limpiar_doc_texto(m.group(2)) + + m = re.search(r"\b(CC|TI|RC|CE|PA|PS|PT|PE|CN|AS|MS)\s*[-:/ ]+\s*([0-9\.\s]+)\b", t_norm) + if m: + return homologar_tipo_documento(m.group(1)), limpiar_doc_texto(m.group(2)) + + m = re.search(r"\b([0-9\.\s]+)\s*[-:/ ]+\s*(CC|TI|RC|CE|PA|PS|PT|PE|CN|AS|MS)\b", t_norm) + if m: + return homologar_tipo_documento(m.group(2)), limpiar_doc_texto(m.group(1)) + + # Solo número: se permite tipo vacío si el documento es usable para el bot. + # Evita extraer números desde textos largos con nombres/observaciones. + if re.fullmatch(r"[0-9\s.,;:/\-]+", t_norm): + cleaned = limpiar_doc_texto(t_norm) + if 6 <= len(cleaned) <= 16: + return "", cleaned + + numeros = re.findall(r"\d{5,20}", cleaned) + if numeros: + candidatos = [n for n in numeros if 6 <= len(n) <= 10] + if candidatos: + candidatos = sorted(candidatos, key=len) + return "", candidatos[0] + return "", numeros[-1] + + # Si el texto contiene solo tipo y número con puntos, limpiar y reintentar. + m = re.search(r"\b(CC|TI|RC|CE|PA|PS|PT|PE|CN|AS|MS)\s+([0-9\.\s]+)\b", t_norm) + if m: + return homologar_tipo_documento(m.group(1)), limpiar_doc_texto(m.group(2)) + + return "", "" + + +def extraer_autorizacion_desde_texto(texto): + t = limpiar_texto(texto).upper() + m = re.search(r"AUTORIZACI[ÓO]N\s*:\s*([A-Z0-9\-]+)", t) + if m: + return m.group(1) + return "" + + +def extraer_ingreso_desde_texto(texto): + t = limpiar_texto(texto).upper() + # Caso especial mencionado: INGRESO: 9 + m = re.search(r"INGRESO\s*:\s*([0-9/\-.]+)", t) + if m: + return m.group(1) + return "" + + +def extraer_abono_desde_texto(texto): + t = limpiar_texto(texto).upper() + m = re.search(r"ABONO\s+PACIENTE\s*:\s*([0-9.,]+)", t) + if m: + return m.group(1) + return "" + + +def obtener_serie_columna_especial(df): + """ + Devuelve la columna que parece contener el texto mezclado. + """ + mejor_col = None + mejor_score = -1 + palabras = ["AUTORIZACION", "AUTORIZACIÓN", "PACIENTE", "INGRESO", "ABONO"] + + for col in df.columns: + serie = df[col].astype(str) + textos = " ".join(serie.head(50).tolist()).upper() + score = sum(textos.count(p) for p in palabras) + int(serie.map(lambda x: len(str(x))).mean()) + if score > mejor_score: + mejor_score = score + mejor_col = col + + return mejor_col + + +def construir_base_normal(df, archivo_nombre): + """ + Construye dataframe base con columnas estándar a partir de archivo normal. + """ + base = pd.DataFrame() + for campo in COLUMNAS_FINAL_IT: + base[campo] = "" + + # columnas origen + col_radicado = encontrar_columna(df, "radicado") + col_tipo_doc = encontrar_columna(df, "tipo_documento") + col_doc = encontrar_columna(df, "documento") + col_tipo_doc_doc = encontrar_columna(df, "tipo_documento_documento") + col_ingreso = encontrar_columna(df, "ingreso") + col_total = encontrar_columna(df, "total_cargos") + col_aut = encontrar_columna(df, "autorizacion") + col_serv = encontrar_columna(df, "tipo_servicio") + col_copago = encontrar_columna(df, "copago") + col_mod = encontrar_columna(df, "cuota_moderadora") + col_cop_mod = encontrar_columna(df, "cop_mod") + col_neto = encontrar_columna(df, "neto") + + n = len(df) + base = pd.DataFrame(index=df.index) + base["archivo_origen"] = archivo_nombre + base["tipo_archivo"] = "NORMAL" + + base["radicado"] = df[col_radicado] if col_radicado else "" + base["tipo_documento"] = df[col_tipo_doc] if col_tipo_doc else "" + base["documento"] = df[col_doc] if col_doc else "" + + # Si viene tipo y documento mezclado en una misma columna, intentar partir. + if col_tipo_doc_doc: + columnas_para_extraer = [col_tipo_doc_doc] + else: + # Fallback: revisar columnas probables sin incluir radicado ni montos. + columnas_para_extraer = [ + c for c in df.columns + if c not in { + col_radicado, col_ingreso, col_total, col_aut, + col_serv, col_copago, col_mod, col_cop_mod, col_neto + } + ] + + for col_mix in columnas_para_extraer: + for i, val in df[col_mix].items(): + tipo, doc = extraer_tipo_doc_y_documento_desde_texto(val) + current_tipo = limpiar_texto(base.at[i, "tipo_documento"]) + current_doc = limpiar_texto(base.at[i, "documento"]) + raw_val = limpiar_texto(val) + + if col_mix == col_tipo_doc_doc and (current_tipo == raw_val or current_doc == raw_val): + if tipo: + base.at[i, "tipo_documento"] = tipo + if doc: + base.at[i, "documento"] = doc + continue + + if col_mix != col_tipo_doc_doc and current_tipo and current_doc: + continue + + if tipo and not current_tipo: + base.at[i, "tipo_documento"] = tipo + if doc and not current_doc: + base.at[i, "documento"] = doc + + # Si no se detectó columna radicado explícita, intentar inferir radicado desde cualquier columna + # buscando secuencias largas de dígitos (heurística). No sobrescribe documento ya detectado. + for col in df.columns: + for i, val in df[col].items(): + if limpiar_texto(base.at[i, "radicado"]): + continue + s = limpiar_texto(val) + if not s: + continue + # Buscar secuencia de al menos 7 dígitos hasta 13 + m = re.search(r"(\d{7,13})", str(s)) + if not m: + continue + candidate = m.group(1) + # Evitar tomar si coincide con documento ya detectado + if limpiar_texto(base.at[i, "documento"]) == candidate: + continue + # Evitar sobrescribir si ya hay tipo_documento (probablemente documento legítimo) + if limpiar_texto(base.at[i, "tipo_documento"]): + continue + base.at[i, "radicado"] = candidate + + base["ingreso"] = df[col_ingreso] if col_ingreso else "" + base["total_cargos"] = df[col_total] if col_total else "" + base["autorizacion"] = df[col_aut] if col_aut else "" + base["tipo_servicio"] = df[col_serv] if col_serv else "" + base["copago_origen"] = df[col_copago] if col_copago else "" + base["moderadora_origen"] = df[col_mod] if col_mod else "" + base["cop_mod_origen"] = df[col_cop_mod] if col_cop_mod else "" + base["neto_origen"] = df[col_neto] if col_neto else "" + + return base + + +def construir_base_especial(df, archivo_nombre): + """ + Reconstruye columnas estándar desde texto mezclado. + """ + col_texto = obtener_serie_columna_especial(df) + col_radicado = encontrar_columna(df, "radicado") + col_total = encontrar_columna(df, "total_cargos") + col_serv = encontrar_columna(df, "tipo_servicio") + + base = pd.DataFrame(index=df.index) + base["archivo_origen"] = archivo_nombre + base["tipo_archivo"] = "ESPECIAL" + + base["radicado"] = df[col_radicado] if col_radicado else "" + base["tipo_documento"] = "" + base["documento"] = "" + base["ingreso"] = "" + base["total_cargos"] = df[col_total] if col_total else "" + base["autorizacion"] = "" + base["tipo_servicio"] = df[col_serv] if col_serv else "" + base["copago_origen"] = "" + base["moderadora_origen"] = "" + base["cop_mod_origen"] = "" + base["neto_origen"] = "" + + if col_texto: + for i, texto in df[col_texto].items(): + tipo, doc = extraer_tipo_doc_y_documento_desde_texto(texto) + aut = extraer_autorizacion_desde_texto(texto) + ingreso = extraer_ingreso_desde_texto(texto) + abono = extraer_abono_desde_texto(texto) + + base.at[i, "tipo_documento"] = tipo + base.at[i, "documento"] = doc + base.at[i, "autorizacion"] = aut + base.at[i, "ingreso"] = ingreso + base.at[i, "cop_mod_origen"] = abono + + return base + + +# ============================================================ +# NORMALIZACIÓN FINAL Y VALIDACIONES +# ============================================================ + + +def resolver_tipo_y_documento(tipo_doc_raw, documento_raw): + """ + Resuelve casos donde el tipo y documento vienen mezclados en cualquiera de las dos columnas. + + Regla: + - Si tipo_documento trae "CC 1012333072", separa. + - Si tipo_documento trae "CÉDULA CIUDADANIA" y documento viene aparte, homologa a CC. + - Si documento trae "RC 1188975098", separa. + - Si solo viene número, permite tipo vacío y documento numérico. + """ + tipo_raw = limpiar_texto(tipo_doc_raw) + doc_raw = limpiar_texto(documento_raw) + + # 1. Intentar extraer desde la columna tipo_documento si viene mezclada. + tipo_mix, doc_mix = extraer_tipo_doc_y_documento_desde_texto(tipo_raw) + if tipo_mix or doc_mix: + tipo_final = homologar_tipo_documento(tipo_mix) + doc_final = limpiar_documento(doc_mix, tipo_final) + + # Si solo encontró tipo, usar documento de la otra columna. + if tipo_final and not doc_final and doc_raw: + doc_final = limpiar_documento(doc_raw, tipo_final) + + if tipo_final or doc_final: + return tipo_final, doc_final + + # 2. Intentar extraer desde la columna documento si viene mezclada. + tipo_mix, doc_mix = extraer_tipo_doc_y_documento_desde_texto(doc_raw) + if tipo_mix or doc_mix: + tipo_final = homologar_tipo_documento(tipo_mix) + doc_final = limpiar_documento(doc_mix, tipo_final) + + # Si el tipo no salió en documento, usar tipo_raw homologado. + if not tipo_final: + tipo_final = homologar_tipo_documento(tipo_raw) + + if tipo_final or doc_final: + return tipo_final, doc_final + + # 3. Caso normal. + tipo_final = homologar_tipo_documento(tipo_raw) + doc_final = limpiar_documento(doc_raw, tipo_final) + return tipo_final, doc_final + + +def normalizar_registros(base): + registros = [] + errores = [] + + for idx, row in base.iterrows(): + obs = [] + + archivo = limpiar_texto(row.get("archivo_origen", "")) + tipo_archivo = limpiar_texto(row.get("tipo_archivo", "")) + + radicado = limpiar_texto(row.get("radicado", "")) + + # IMPORTANTE: + # Primero separar tipo/documento si vienen mezclados, luego homologar y validar. + tipo_doc, documento = resolver_tipo_y_documento( + row.get("tipo_documento", ""), + row.get("documento", "") + ) + + ingreso, err_fecha = normalizar_fecha(row.get("ingreso", "")) + # No detener el proceso si la fecha no se reconoce. + if err_fecha: + ingreso = "" + obs.append(err_fecha) + + anio = obtener_anio_fecha(ingreso) + + total_cargos = limpiar_numero_entero(row.get("total_cargos", ""), default="") + neto_origen = limpiar_numero_entero(row.get("neto_origen", ""), default="") + if not total_cargos and neto_origen: + total_cargos = neto_origen + + if not total_cargos: + obs.append("Total cargos / bruto vacío o inválido") + + autorizacion, err_aut = normalizar_autorizacion(row.get("autorizacion", "")) + # No detener el proceso si la autorización no se reconoce. + if err_aut: + autorizacion = autorizacion or "" + obs.append(err_aut) + + tipo_servicio = normalizar_tipo_servicio(row.get("tipo_servicio", "")) + + ok_doc, err_doc = validar_documento(tipo_doc, documento) + if not ok_doc: + obs.append(err_doc) + + # Copago / moderadora: + copago = to_int(row.get("copago_origen", ""), default=0) + moderadora = to_int(row.get("moderadora_origen", ""), default=0) + + cop_mod = to_int(row.get("cop_mod_origen", ""), default=0) + + # Si hay cop_mod mezclado, clasificar por tabla. + if cop_mod > 0: + c, m = separar_copago_moderadora(cop_mod, anio) + copago += c + moderadora += m + + # Si hay neto y bruto, usar diferencia para inferir cop/mod. + neto = to_int(row.get("neto_origen", ""), default=0) + bruto = to_int(total_cargos, default=0) + if bruto > 0 and neto > 0 and bruto >= neto: + diferencia = bruto - neto + if diferencia > 0 and copago == 0 and moderadora == 0: + c, m = separar_copago_moderadora(diferencia, anio) + copago = c + moderadora = m + + final = { + "archivo_origen": archivo, + "tipo_archivo": tipo_archivo, + "radicado": radicado, + "tipo_documento": tipo_doc, + "documento": documento, + "ingreso": ingreso, + "total_cargos": total_cargos, + "autorizacion": autorizacion, + "tipo_servicio": tipo_servicio, + "copago": str(copago), + "cuota_moderadora": str(moderadora), + "observacion": " | ".join(obs), + } + + if obs: + errores.append(final) + else: + registros.append(final) + + return pd.DataFrame(registros), pd.DataFrame(errores) + + +def separar_duplicados(df_ok): + """ + Lógica tipo CONTAR.SI del usuario: + + Si un radicado aparece 2 o más veces: + → TODOS esos registros van a DUPLICADOS.xlsx + → NINGUNO pasa al CSV_FINAL_IT + """ + if df_ok.empty: + return df_ok, pd.DataFrame() + + # Limpiar radicado temporalmente + temp = df_ok.copy() + temp["_radicado_clean"] = temp["radicado"].astype(str).str.strip() + + # Conteo tipo CONTAR.SI + conteo = temp.groupby("_radicado_clean")["_radicado_clean"].transform("count") + + # Duplicado = aparece 2 o más veces + dup_mask = (conteo >= 2) & temp["_radicado_clean"].ne("") + + duplicados = temp[dup_mask].copy() + unicos = temp[~dup_mask].copy() + + if not duplicados.empty: + duplicados["cantidad_duplicados"] = conteo[dup_mask] + duplicados["observacion"] = ( + "Radicado duplicado detectado por CONTAR.SI. " + "Cantidad: " + duplicados["cantidad_duplicados"].astype(str) + ) + + # Quitar columnas temporales + for df_ in [duplicados, unicos]: + if "_radicado_clean" in df_.columns: + df_.drop(columns=["_radicado_clean"], inplace=True, errors="ignore") + + return unicos, duplicados + + + +def clasificar_errores_por_hoja(df_errores): + """ + Separa los errores en hojas por categoría. + Un mismo registro puede aparecer en varias hojas si tiene varios errores. + """ + if df_errores.empty or "observacion" not in df_errores.columns: + return {} + + categorias = { + "Autorizaciones": [ + "autorización", "autorizacion" + ], + "Fechas": [ + "fecha ingreso", "fecha" + ], + "Documentos": [ + "documento inválido", "documento invalido", "longitud inválida", + "longitud invalida", "tipo de documento" + ], + "Total_Bruto": [ + "total cargos", "bruto" + ], + "Tipo_Documento": [ + "tipo de documento vacío", "tipo de documento vacio", + "tipo de documento no parametrizado" + ], + "Otros": [] + } + + hojas = {} + obs_norm = df_errores["observacion"].astype(str).str.lower() + + usados = pd.Series(False, index=df_errores.index) + + for hoja, palabras in categorias.items(): + if hoja == "Otros": + continue + + mask = pd.Series(False, index=df_errores.index) + for palabra in palabras: + mask = mask | obs_norm.str.contains(palabra, na=False, regex=False) + + if mask.any(): + hojas[hoja] = df_errores[mask].copy() + usados = usados | mask + + otros = df_errores[~usados].copy() + if not otros.empty: + hojas["Otros"] = otros + + return hojas + + +def escribir_errores_por_hojas(writer, df_err_total): + """ + Escribe ERRORES_NORMALIZACION.xlsx con: + - Hoja Todos + - Hojas por categoría: Autorizaciones, Fechas, Documentos, Total_Bruto, Tipo_Documento, Otros + """ + columnas_base = [ + "archivo_origen", "tipo_archivo", + *COLUMNAS_FINAL_IT, + "observacion" + ] + + if df_err_total.empty: + pd.DataFrame(columns=columnas_base).to_excel(writer, index=False, sheet_name="Todos") + return + + df_err_total.to_excel(writer, index=False, sheet_name="Todos") + + hojas = clasificar_errores_por_hoja(df_err_total) + for nombre_hoja, df_hoja in hojas.items(): + # Excel permite máximo 31 caracteres en nombre de hoja + nombre_seguro = str(nombre_hoja)[:31] + df_hoja.to_excel(writer, index=False, sheet_name=nombre_seguro) + + + + + +def leer_archivo_flexible(path): + """Lee CSV o Excel manteniendo todo como texto.""" + path = Path(path) + ext = path.suffix.lower() + if ext == ".csv": + return leer_csv_flexible(path) + if ext in {".xlsx", ".xlsm", ".xls"}: + return pd.read_excel(path, dtype=str, keep_default_na=False) + raise RuntimeError(f"Formato no soportado: {path.name}. Usa CSV o Excel.") + + +def listar_archivos_entrada(carpeta): + carpeta = Path(carpeta) + archivos = [] + for patron in ("*.csv", "*.xlsx", "*.xlsm", "*.xls"): + archivos.extend(carpeta.glob(patron)) + + # Evitar leer salidas anteriores o temporales. + archivos_limpios = [] + for a in sorted(archivos): + nombre = a.name.upper() + if a.parent.name.upper() == "SALIDA_NORMALIZADOR_IT": + continue + if nombre.startswith("CONSOLIDADO_BOT_IT"): + continue + if "ORIGEN_SERVICIO" in nombre: + continue + if nombre.startswith("~$"): + continue + archivos_limpios.append(a) + return archivos_limpios + + +def normalizar_clave_radicado(valor): + """ + Normaliza el radicado para poder cruzarlo contra Origen Servicio. + Evita fallos por espacios, guiones, .0 de Excel o separadores. + """ + s = limpiar_texto(valor).upper() + if not s: + return "" + + # Cuando Excel deja 100000069634188.0 + s = re.sub(r"\.0$", "", s) + + # Quitar espacios y separadores comunes + s = re.sub(r"[\s\-_.:,;/\\]+", "", s) + + # Si quedó algo tipo RAD100000..., conservar letras y números + s = re.sub(r"[^A-Z0-9]", "", s) + return s + +def cargar_origen_servicio(ruta_archivo): + """ + Carga el Excel/CSV seleccionado por el usuario y arma un diccionario: + radicado -> tipo_servicio. + + Soporta encabezados como: + - radicado / NumeroRadicacion / numero radicacion + - tipo_servicio / servicio / Origen_Servicio + """ + if not ruta_archivo: + return {} + + archivo_servicio = Path(ruta_archivo) + + if not archivo_servicio.exists(): + return {} + + df_serv = leer_archivo_flexible(archivo_servicio) + df_serv.columns = [str(c).strip() for c in df_serv.columns] + + col_rad = encontrar_columna(df_serv, "radicado") + col_serv = encontrar_columna(df_serv, "tipo_servicio") + + # Refuerzo por si el encabezado viene pegado tipo NumeroRadicacion + if not col_rad: + for c in df_serv.columns: + c_norm = normalizar_nombre_columna(c).replace(" ", "") + if "radic" in c_norm: + col_rad = c + break + + # Refuerzo por si el encabezado viene pegado tipo OrigenServicio + if not col_serv: + for c in df_serv.columns: + c_norm = normalizar_nombre_columna(c).replace(" ", "") + if "servicio" in c_norm or "origenservicio" in c_norm: + col_serv = c + break + + if not col_rad or not col_serv: + return {} + + mapa = {} + for _, row in df_serv.iterrows(): + rad_original = limpiar_texto(row.get(col_rad, "")) + serv = limpiar_texto(row.get(col_serv, "")) + + if rad_original and serv: + # Guarda varias llaves para que cruce aunque Excel venga con espacios, guiones o .0 + mapa[rad_original] = serv + mapa[normalizar_clave_radicado(rad_original)] = serv + + solo_digitos = re.sub(r"\D", "", rad_original) + if solo_digitos: + mapa[solo_digitos] = serv + + return mapa + +def simplificar_observacion(obs): + """Convierte observaciones largas a códigos rápidos para filtrar.""" + obs = limpiar_texto(obs) + if not obs: + return "" + + partes_salida = [] + obs_norm = normalizar_nombre_columna(obs) + + reglas = [ + ("SIN_AUTORIZACION", ["autorizacion vacia", "sin autorizacion"]), + ("AUTORIZACION_INCOMPLETA", ["autorizacion incompleta"]), + ("AUTORIZACION_INVALIDA", ["autorizacion invalida"]), + ("SIN_FECHA", ["sin fecha"]), + ("FECHA_INVALIDA", ["fecha ingreso vacia", "fecha ingreso invalida", "fecha"]), + ("DOCUMENTO_INVALIDO", ["documento invalido", "longitud invalida", "tipo de documento vacio", "tipo de documento no parametrizado"]), + ("VALOR_INVALIDO", ["total cargos", "bruto"]), + ] + + for codigo, palabras in reglas: + if any(p in obs_norm for p in palabras): + partes_salida.append(codigo) + + if not partes_salida: + partes_salida.append(obs.upper()) + + # Quitar repetidos conservando orden. + vistos = set() + final = [] + for p in partes_salida: + if p not in vistos: + vistos.add(p) + final.append(p) + return " | ".join(final) + + +def preparar_consolidado(df_ok_total, df_err_total): + columnas = [*COLUMNAS_FINAL_IT, "OBSERVACION", "archivo_origen", "tipo_archivo"] + + partes = [] + + if df_ok_total is not None and not df_ok_total.empty: + ok = df_ok_total.copy() + ok["OBSERVACION"] = "" + partes.append(ok) + + if df_err_total is not None and not df_err_total.empty: + err = df_err_total.copy() + err["OBSERVACION"] = err.get("observacion", "").apply(simplificar_observacion) + partes.append(err) + + if partes: + cons = pd.concat(partes, ignore_index=True) + else: + cons = pd.DataFrame(columns=columnas) + + for c in COLUMNAS_FINAL_IT: + if c not in cons.columns: + cons[c] = "" + + # Detectar duplicado sobre TODO el consolidado, así esté bueno o malo. + if not cons.empty and "radicado" in cons.columns: + rad = cons["radicado"].astype(str).str.strip() + conteo = rad.map(rad.value_counts()) + mask_dup = rad.ne("") & conteo.ge(2) + cons.loc[mask_dup, "OBSERVACION"] = cons.loc[mask_dup, "OBSERVACION"].apply( + lambda x: "RADICADO_DUPLICADO" if not limpiar_texto(x) else f"{x} | RADICADO_DUPLICADO" + ) + + cons["OBSERVACION"] = cons["OBSERVACION"].apply(lambda x: "PASO_BOT" if not limpiar_texto(x) else x) + + # Limpieza de columnas auxiliares. + for c in ["observacion", "cantidad_duplicados"]: + if c in cons.columns: + cons.drop(columns=[c], inplace=True, errors="ignore") + + for c in columnas: + if c not in cons.columns: + cons[c] = "" + + return cons[columnas] + + +def preparar_resumen(df_consolidado, resumen_archivos): + resumen_obs = pd.DataFrame(columns=["OBSERVACION", "CANTIDAD"]) + if df_consolidado is not None and not df_consolidado.empty: + resumen_obs = ( + df_consolidado["OBSERVACION"] + .fillna("") + .astype(str) + .str.split(" | ") + .explode() + .str.strip() + .replace("", pd.NA) + .dropna() + .value_counts() + .rename_axis("OBSERVACION") + .reset_index(name="CANTIDAD") + ) + total = pd.DataFrame([{"OBSERVACION": "TOTAL", "CANTIDAD": len(df_consolidado)}]) + resumen_obs = pd.concat([resumen_obs, total], ignore_index=True) + + resumen_arch = pd.DataFrame(resumen_archivos) + return resumen_obs, resumen_arch + + +def autoajustar_excel(writer, hojas): + try: + from openpyxl.styles import Font, PatternFill, Alignment + for nombre_hoja in hojas: + ws = writer.book[nombre_hoja] + ws.freeze_panes = "A2" + ws.auto_filter.ref = ws.dimensions + for cell in ws[1]: + cell.font = Font(bold=True) + cell.fill = PatternFill("solid", fgColor="D9EAF7") + cell.alignment = Alignment(horizontal="center") + for col in ws.columns: + max_len = 0 + letter = col[0].column_letter + for cell in col[:200]: + max_len = max(max_len, len(str(cell.value or ""))) + ws.column_dimensions[letter].width = min(max(max_len + 2, 12), 45) + except Exception: + pass + + +def escribir_excel_consolidado(path_salida, df_original, df_consolidado, resumen_obs, resumen_archivos): + with pd.ExcelWriter(path_salida, engine="openpyxl") as writer: + df_original.to_excel(writer, index=False, sheet_name="Original") + df_consolidado.to_excel(writer, index=False, sheet_name="Consolidado") + resumen_obs.to_excel(writer, index=False, sheet_name="Resumen", startrow=0) + + # Debajo del resumen por observación se deja el detalle por archivo. + start = len(resumen_obs) + 3 + pd.DataFrame([{"": "DETALLE POR ARCHIVO"}]).to_excel( + writer, index=False, header=False, sheet_name="Resumen", startrow=start + ) + resumen_archivos.to_excel(writer, index=False, sheet_name="Resumen", startrow=start + 2) + + autoajustar_excel(writer, ["Original", "Consolidado", "Resumen"]) + + +def procesar_archivo_unico(archivo_entrada, log_callback=None, ruta_origen_servicio=None): + archivo_entrada = Path(archivo_entrada) + carpeta_temp = archivo_entrada.parent / "__TEMP_ARCHIVO_UNICO__" + carpeta_temp.mkdir(exist_ok=True) + + copia = carpeta_temp / archivo_entrada.name + shutil.copy2(archivo_entrada, copia) + + return procesar_carpeta_incremental(carpeta_temp, log_callback=log_callback, ruta_origen_servicio=ruta_origen_servicio) + + +def procesar_carpeta_incremental(carpeta, log_callback=None, ruta_origen_servicio=None): + """ + Genera un Excel único con 3 hojas: + 1. Original + 2. Consolidado + 3. Resumen + + También conserva CSV_FINAL_IT.csv solo con registros PASO_BOT para el bot. + """ + carpeta = Path(carpeta) + archivos = listar_archivos_entrada(carpeta) + + if not archivos: + raise RuntimeError("No se encontraron archivos CSV o Excel en la carpeta seleccionada.") + + salida = carpeta / "SALIDA_NORMALIZADOR_IT" + salida.mkdir(exist_ok=True) + + excel_consolidado = salida / "CONSOLIDADO_BOT_IT.xlsx" + csv_final = salida / "CSV_FINAL_IT.csv" + + todos_original = [] + todos_ok = [] + todos_errores = [] + resumen = [] + + def log(msg): + if log_callback: + log_callback(msg) + + log("Procesando archivos y generando Excel único: Original | Consolidado | Resumen...") + mapa_servicios = cargar_origen_servicio(ruta_origen_servicio) + + if mapa_servicios: + log(f"Origen_Servicio cargado: {len(mapa_servicios)} radicados con servicio.") + else: + log("No se seleccionó Origen Servicio o no tenía datos válidos.") + + for idx_archivo, path in enumerate(archivos, start=1): + try: + log(f"[{idx_archivo}/{len(archivos)}] Leyendo: {path.name}") + df = leer_archivo_flexible(path) + df.columns = [str(c).strip() for c in df.columns] + filas_leidas = len(df) + + original = df.copy() + original.insert(0, "archivo_origen", path.name) + todos_original.append(original) + + if df.empty: + resumen.append({ + "archivo": path.name, + "estado": "SIN_DATOS", + "tipo": "", + "filas_leidas": 0, + "paso_bot": 0, + "con_error": 0, + "duplicados": 0, + "detalle": "Archivo vacío", + }) + continue + + especial = es_archivo_especial(df) + if especial: + log(" → Detectado como ESPECIAL") + base = construir_base_especial(df, path.name) + else: + log(" → Detectado como NORMAL") + base = construir_base_normal(df, path.name) + # Llenar tipo_servicio desde el Excel/CSV Origen Servicio usando el radicado + if mapa_servicios and "radicado" in base.columns: + encontrados_servicio = 0 + sin_cruce_servicio = 0 + + for i, row in base.iterrows(): + rad_original = limpiar_texto(row.get("radicado", "")) + rad_norm = normalizar_clave_radicado(rad_original) + rad_digitos = re.sub(r"\D", "", rad_original) + + servicio_excel = ( + mapa_servicios.get(rad_original, "") + or mapa_servicios.get(rad_norm, "") + or mapa_servicios.get(rad_digitos, "") + ) + + if servicio_excel: + base.at[i, "tipo_servicio"] = servicio_excel + encontrados_servicio += 1 + else: + sin_cruce_servicio += 1 + + log(f" → Origen Servicio aplicado: {encontrados_servicio} filas con servicio / {sin_cruce_servicio} sin cruce.") + + ok, err = normalizar_registros(base) + todos_ok.append(ok) + todos_errores.append(err) + + resumen.append({ + "archivo": path.name, + "estado": "PROCESADO", + "tipo": "ESPECIAL" if especial else "NORMAL", + "filas_leidas": filas_leidas, + "paso_bot": 0, # se actualiza después + "con_error": len(err), + "duplicados": 0, # se actualiza después + "detalle": "", + }) + + log(f" → Leídas: {filas_leidas} | OK preliminar: {len(ok)} | Con error: {len(err)}") + + except Exception as e: + detalle = f"{type(e).__name__}: {e}" + log(f" [ERROR] {path.name}: {detalle}") + resumen.append({ + "archivo": path.name, + "estado": "ERROR_LECTURA", + "tipo": "", + "filas_leidas": 0, + "paso_bot": 0, + "con_error": 0, + "duplicados": 0, + "detalle": detalle, + }) + + df_original = pd.concat(todos_original, ignore_index=True, sort=False) if todos_original else pd.DataFrame() + df_ok_total = pd.concat(todos_ok, ignore_index=True) if todos_ok else pd.DataFrame() + df_err_total = pd.concat(todos_errores, ignore_index=True) if todos_errores else pd.DataFrame() + + df_consolidado = preparar_consolidado(df_ok_total, df_err_total) + + # CSV para el bot: solo PASO_BOT. + df_final_bot = df_consolidado[df_consolidado["OBSERVACION"].eq("PASO_BOT")].copy() + if df_final_bot.empty: + pd.DataFrame(columns=CSV_FINAL_IT_HEADERS).to_csv(csv_final, sep="|", index=False, encoding="utf-8-sig") + else: + df_out = df_final_bot.reindex(columns=COLUMNAS_FINAL_IT).fillna("") + df_out = df_out.rename(columns=CSV_FINAL_IT_RENAME) + df_out = df_out.reindex(columns=CSV_FINAL_IT_HEADERS) + df_out.to_csv(csv_final, sep="|", index=False, encoding="utf-8-sig") + + # Actualizar resumen por archivo desde el consolidado real. + if not df_consolidado.empty and "archivo_origen" in df_consolidado.columns: + paso_por_archivo = df_consolidado[df_consolidado["OBSERVACION"].eq("PASO_BOT")].groupby("archivo_origen").size().to_dict() + dup_por_archivo = df_consolidado[df_consolidado["OBSERVACION"].str.contains("RADICADO_DUPLICADO", na=False)].groupby("archivo_origen").size().to_dict() + error_por_archivo = df_consolidado[~df_consolidado["OBSERVACION"].eq("PASO_BOT")].groupby("archivo_origen").size().to_dict() + for r in resumen: + a = r.get("archivo", "") + r["paso_bot"] = int(paso_por_archivo.get(a, 0)) + r["duplicados"] = int(dup_por_archivo.get(a, 0)) + r["con_error"] = int(error_por_archivo.get(a, 0)) + + resumen_obs, resumen_arch = preparar_resumen(df_consolidado, resumen) + escribir_excel_consolidado(excel_consolidado, df_original, df_consolidado, resumen_obs, resumen_arch) + + total_duplicados = int(df_consolidado["OBSERVACION"].str.contains("RADICADO_DUPLICADO", na=False).sum()) if not df_consolidado.empty else 0 + total_errores = int((~df_consolidado["OBSERVACION"].eq("PASO_BOT")).sum()) if not df_consolidado.empty else 0 + + return { + "carpeta_salida": str(salida), + "excel_consolidado": str(excel_consolidado), + "csv_final": str(csv_final), + "total_final": len(df_final_bot), + "total_errores": total_errores, + "total_duplicados": total_duplicados, + "archivos": len(archivos), + "modo": "excel_consolidado", + } + + +def procesar_carpeta(carpeta, log_callback=None, ruta_origen_servicio=None): + # Se deja como alias para que el modo masivo también genere el nuevo formato. + return procesar_carpeta_incremental(carpeta, log_callback=log_callback, ruta_origen_servicio=ruta_origen_servicio) + + +# ============================================================ +# INTERFAZ GRÁFICA +# ============================================================ + +class NormalizadorITApp: + def __init__(self, root): + self.root = root + self.root.title("Normalizador CSV IT - Salud Total") + self.root.geometry("900x600") + self.root.minsize(850, 550) + + self.carpeta_var = tk.StringVar() + self.archivo_var = tk.StringVar() + self.ruta_origen_servicio = tk.StringVar() + + self.modo_entrada = tk.StringVar(value="carpeta") + self.modo_proceso = tk.StringVar(value="incremental") + + self._build_ui() + + def _build_ui(self): + frame = ttk.Frame(self.root, padding=15) + frame.pack(fill="both", expand=True) + + title = ttk.Label( + frame, + text="Normalizador CSV para Bot IT", + font=("Segoe UI", 16, "bold") + ) + title.pack(anchor="w") + + subtitle = ttk.Label( + frame, + text="Genera Excel único: Original, Consolidado y Resumen; además crea CSV_FINAL_IT.csv", + font=("Segoe UI", 10) + ) + subtitle.pack(anchor="w", pady=(0, 15)) + + entrada_frame = ttk.LabelFrame(frame, text="Tipo de entrada") + entrada_frame.pack(fill="x", pady=5) + + ttk.Radiobutton( + entrada_frame, + text="Procesar carpeta completa", + variable=self.modo_entrada, + value="carpeta" + ).pack(anchor="w", padx=8, pady=2) + + ttk.Radiobutton( + entrada_frame, + text="Procesar un solo archivo CSV", + variable=self.modo_entrada, + value="archivo" + ).pack(anchor="w", padx=8, pady=2) + + row = ttk.Frame(frame) + row.pack(fill="x", pady=5) + + ttk.Label(row, text="Ruta:").pack(side="left") + + self.entry_ruta = ttk.Entry(row, textvariable=self.carpeta_var) + self.entry_ruta.pack(side="left", fill="x", expand=True, padx=8) + + ttk.Button(row, text="Seleccionar", command=self.seleccionar_entrada).pack(side="left") + + row_servicio = ttk.Frame(frame) + row_servicio.pack(fill="x", pady=5) + + ttk.Label(row_servicio, text="Origen Servicio:").pack(side="left") + + self.entry_origen_servicio = ttk.Entry(row_servicio, textvariable=self.ruta_origen_servicio) + self.entry_origen_servicio.pack(side="left", fill="x", expand=True, padx=8) + + ttk.Button( + row_servicio, + text="Seleccionar Excel", + command=self.seleccionar_origen_servicio + ).pack(side="left") + + ttk.Button( + row_servicio, + text="Limpiar", + command=lambda: self.ruta_origen_servicio.set("") + ).pack(side="left", padx=(8, 0)) + + modo_frame = ttk.LabelFrame(frame, text="Modo de procesamiento") + modo_frame.pack(fill="x", pady=8) + + ttk.Radiobutton( + modo_frame, + text="Procesar uno por uno y consolidar (recomendado)", + variable=self.modo_proceso, + value="incremental" + ).pack(anchor="w", padx=8, pady=2) + + ttk.Radiobutton( + modo_frame, + text="Procesar todo junto (modo anterior)", + variable=self.modo_proceso, + value="masivo" + ).pack(anchor="w", padx=8, pady=2) + + buttons = ttk.Frame(frame) + buttons.pack(fill="x", pady=10) + + self.btn_procesar = ttk.Button(buttons, text="Procesar carpeta", command=self.iniciar_proceso) + self.btn_procesar.pack(side="left") + + ttk.Button(buttons, text="Abrir salida", command=self.abrir_salida).pack(side="left", padx=8) + + self.progress = ttk.Progressbar(frame, mode="indeterminate") + self.progress.pack(fill="x", pady=8) + + ttk.Label(frame, text="Log del proceso:").pack(anchor="w") + + self.txt_log = tk.Text(frame, height=22, wrap="word") + self.txt_log.pack(fill="both", expand=True) + + scrollbar = ttk.Scrollbar(self.txt_log, command=self.txt_log.yview) + self.txt_log.configure(yscrollcommand=scrollbar.set) + + footer = ttk.Label( + frame, + text="Salidas: CONSOLIDADO_BOT_IT.xlsx (Original, Consolidado, Resumen) | CSV_FINAL_IT.csv", + font=("Segoe UI", 9) + ) + footer.pack(anchor="w", pady=(8, 0)) + + def log(self, msg): + self.txt_log.insert("end", f"{datetime.now().strftime('%H:%M:%S')} {msg}\n") + self.txt_log.see("end") + self.root.update_idletasks() + + def seleccionar_entrada(self): + modo = self.modo_entrada.get() + + if modo == "archivo": + archivo = filedialog.askopenfilename( + title="Selecciona un archivo CSV o Excel", + filetypes=[("CSV o Excel", "*.csv *.xlsx *.xlsm *.xls"), ("CSV", "*.csv"), ("Excel", "*.xlsx *.xlsm *.xls")] + ) + + if archivo: + self.archivo_var.set(archivo) + self.carpeta_var.set(archivo) + + else: + carpeta = filedialog.askdirectory( + title="Selecciona carpeta con CSV o Excel" + ) + + if carpeta: + self.carpeta_var.set(carpeta) + + def seleccionar_origen_servicio(self): + archivo = filedialog.askopenfilename( + title="Seleccionar Excel/CSV de Origen Servicio", + filetypes=[ + ("Excel o CSV", "*.xlsx *.xlsm *.xls *.csv"), + ("Excel", "*.xlsx *.xlsm *.xls"), + ("CSV", "*.csv"), + ("Todos", "*.*"), + ] + ) + + if archivo: + self.ruta_origen_servicio.set(archivo) + self.log(f"Origen Servicio seleccionado: {archivo}") + + def abrir_salida(self): + carpeta = self.carpeta_var.get().strip() + if not carpeta: + messagebox.showwarning("Sin carpeta", "Selecciona primero una carpeta.") + return + + salida = Path(carpeta) / "SALIDA_NORMALIZADOR_IT" + if not salida.exists(): + messagebox.showinfo("Sin salida", "Todavía no existe la carpeta de salida.") + return + + try: + os.startfile(str(salida)) + except Exception as e: + messagebox.showerror("Error", f"No se pudo abrir la salida:\n{e}") + + def iniciar_proceso(self): + ruta = self.carpeta_var.get().strip() + + if not ruta: + messagebox.showwarning("Sin ruta", "Selecciona carpeta o archivo.") + return + + if not Path(ruta).exists(): + messagebox.showerror("Error", "La ruta seleccionada no existe.") + return + + self.btn_procesar.config(state="disabled") + self.progress.start(10) + self.txt_log.delete("1.0", "end") + self.log("Iniciando normalización...") + if self.ruta_origen_servicio.get().strip(): + self.log(f"Usando Origen Servicio: {self.ruta_origen_servicio.get().strip()}") + else: + self.log("Origen Servicio: no seleccionado (se usará el servicio del archivo original si existe).") + + hilo = threading.Thread(target=self._procesar_thread, args=(ruta,), daemon=True) + hilo.start() + + def _procesar_thread(self, ruta): + try: + modo_entrada = self.modo_entrada.get() + modo_proceso = self.modo_proceso.get() + + if modo_entrada == "archivo": + resultado = procesar_archivo_unico(ruta, log_callback=self.log, ruta_origen_servicio=self.ruta_origen_servicio.get().strip()) + + else: + if modo_proceso == "incremental": + resultado = procesar_carpeta_incremental(ruta, log_callback=self.log, ruta_origen_servicio=self.ruta_origen_servicio.get().strip()) + else: + resultado = procesar_carpeta(ruta, log_callback=self.log, ruta_origen_servicio=self.ruta_origen_servicio.get().strip()) + + self.log("") + self.log("Proceso terminado correctamente.") + self.log(f"Archivos leídos: {resultado['archivos']}") + self.log(f"Registros finales IT: {resultado['total_final']}") + self.log(f"Errores normalización: {resultado['total_errores']}") + self.log(f"Duplicados: {resultado['total_duplicados']}") + self.log(f"Salida: {resultado['carpeta_salida']}") + if resultado.get("excel_consolidado"): + self.log(f"Excel consolidado: {resultado['excel_consolidado']}") + + messagebox.showinfo( + "Proceso terminado", + "Normalización finalizada.\n\n" + f"Registros finales IT: {resultado['total_final']}\n" + f"Errores: {resultado['total_errores']}\n" + f"Duplicados: {resultado['total_duplicados']}\n\n" + f"Salida:\n{resultado['carpeta_salida']}" + ) + + except Exception as e: + self.log("[ERROR GENERAL]") + self.log(str(e)) + self.log(traceback.format_exc()) + messagebox.showerror("Error", f"Ocurrió un error:\n{e}") + + finally: + self.progress.stop() + self.btn_procesar.config(state="normal") + + +def main(): + root = tk.Tk() + app = NormalizadorITApp(root) + root.mainloop() + + +if __name__ == "__main__": + main() diff --git a/docker-compose.yml b/docker-compose.yml index 0c3b2ca..ffc12e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -199,6 +199,36 @@ services: retries: 5 start_period: 20s + bot-digitador: + build: + context: ./services/bot_digitador + dockerfile: Dockerfile + container_name: ipsextractor-bot-digitador + restart: unless-stopped + depends_on: + database: + condition: service_healthy + poblacion_apta: + condition: service_started + environment: + BOT_API_TOKEN: ${BOT_API_TOKEN:-dev-token-change-me} + BOT_API_HOST: 0.0.0.0 + BOT_API_PORT: "8210" + DATABASE_URL: postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor + POBLACION_APTA_URL: http://poblacion_apta:8003 + MINIO_ENDPOINT: ${MINIO_ENDPOINT:-https://s3.builker.xyz} + MINIO_BUCKET: ${MINIO_BUCKET:-pys-cuentas-medicas} + MINIO_ANONYMOUS: ${MINIO_ANONYMOUS:-1} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-} + MINIO_REGION: ${MINIO_REGION:-us-east-1} + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8210/health"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s + api_gateway: build: context: ./services/api_gateway @@ -215,12 +245,15 @@ services: condition: service_healthy bot-descarga: condition: service_healthy + bot-digitador: + condition: service_healthy environment: EXTRACTION_URL: http://extraction:8001 CLEANING_URL: http://cleaning:8002 POBLACION_APTA_URL: http://poblacion_apta:8003 COUNT_URL: http://count-api:8080 BOT_DESCARGA_URL: http://bot-descarga:8200 + BOT_DIGITADOR_URL: http://bot-digitador:8210 ports: # Único puerto expuesto al host. Los demás servicios solo son accesibles en la red Docker. - "0.0.0.0:8189:8000" diff --git a/services/api_gateway/api/main.py b/services/api_gateway/api/main.py index 4e5cef5..645e34c 100644 --- a/services/api_gateway/api/main.py +++ b/services/api_gateway/api/main.py @@ -20,6 +20,7 @@ CLEANING_URL = os.getenv("CLEANING_URL", "http://cleaning:8002").rstrip("/") POBLACION_APTA_URL = os.getenv("POBLACION_APTA_URL", "http://poblacion_apta:8003").rstrip("/") COUNT_URL = os.getenv("COUNT_URL", "http://count-api:8080").rstrip("/") BOT_DESCARGA_URL = os.getenv("BOT_DESCARGA_URL", "http://bot-descarga:8200").rstrip("/") +BOT_DIGITADOR_URL = os.getenv("BOT_DIGITADOR_URL", "http://bot-digitador:8210").rstrip("/") # Timeouts: JSON ligero vs uploads OCR/conteo _TIMEOUT_JSON = httpx.Timeout(connect=5.0, read=30.0, write=30.0, pool=5.0) @@ -36,12 +37,13 @@ async def lifespan(app: FastAPI): limits=httpx.Limits(max_connections=100, max_keepalive_connections=20), ) _log.info( - "Gateway iniciado — extraction=%s cleaning=%s poblacion_apta=%s count=%s bot_descarga=%s", + "Gateway iniciado — extraction=%s cleaning=%s poblacion_apta=%s count=%s bot_descarga=%s bot_digitador=%s", EXTRACTION_URL, CLEANING_URL, POBLACION_APTA_URL, COUNT_URL, BOT_DESCARGA_URL, + BOT_DIGITADOR_URL, ) yield await _http_client.aclose() @@ -84,6 +86,7 @@ def _is_upload_path(path: str) -> bool: or path.startswith("/poblacion-apta/jobs/from-csv") or path.startswith("/poblacion-apta/jobs/from-cleaning") or path.startswith("/bot-descarga/jobs") + or path.startswith("/bot-digitador/jobs") ) @@ -105,6 +108,7 @@ async def health() -> dict[str, Any]: poblacion_apta = await _safe_health(client, POBLACION_APTA_URL) counting = await _safe_health(client, COUNT_URL) bot_descarga = await _safe_health(client, BOT_DESCARGA_URL) + bot_digitador = await _safe_health(client, BOT_DIGITADOR_URL) return { "status": "ok" if extraction["ok"] and cleaning["ok"] and poblacion_apta["ok"] else "degraded", "services": { @@ -113,6 +117,7 @@ async def health() -> dict[str, Any]: "poblacion_apta": poblacion_apta, "counting": counting, "bot_descarga": bot_descarga, + "bot_digitador": bot_digitador, }, } @@ -183,6 +188,11 @@ async def bot_descarga_proxy(path: str, request: Request) -> Response: return await _proxy_request(request, f"{BOT_DESCARGA_URL}/{path}") +@app.api_route("/bot-digitador/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"]) +async def bot_digitador_proxy(path: str, request: Request) -> Response: + return await _proxy_request(request, f"{BOT_DIGITADOR_URL}/{path}") + + @app.api_route("/jobs", methods=["GET", "POST"]) async def jobs_root_proxy(request: Request) -> Response: return await _proxy_request(request, f"{EXTRACTION_URL}/jobs") diff --git a/services/bot_descarga/api/main.py b/services/bot_descarga/api/main.py index 7800d14..5454da6 100644 --- a/services/bot_descarga/api/main.py +++ b/services/bot_descarga/api/main.py @@ -87,26 +87,30 @@ def claim_pending( @app.post("/jobs") async def create_job( - worker_id: str = Form(...), + worker_id: str = Form(""), + worker_ids: str = Form(""), job_id: str = Form(...), title: str = Form(""), file: UploadFile = File(...), _: str = Depends(require_bearer), ) -> dict: - if not worker_id.strip(): - raise HTTPException(status_code=400, detail="worker_id obligatorio") + """Crea una corrida de descarga. + + - 1 worker → flujo clásico (un solo job claimable). + - N workers (worker_ids JSON o todos online) → padre + N shards, + mismo downloads_prefix del padre. + """ + import json as _json + + from domain.excel_split import parse_radicado_rows, rows_to_xlsx_bytes, split_rows + from infrastructure.storage_minio import downloads_prefix as mk_prefix + parent_job_id = job_id.strip() if not parent_job_id: raise HTTPException(status_code=400, detail="job_id (Job Desktop) obligatorio") if not store.parent_job_exists(parent_job_id): raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado") - if not store.is_worker_online(worker_id): - raise HTTPException( - status_code=409, - detail=f"Worker '{worker_id}' no está online. Enciende el agent primero.", - ) - data = await file.read() if not data: raise HTTPException(status_code=400, detail="Archivo vacío") @@ -115,45 +119,224 @@ async def create_job( content_type = file.content_type or "application/octet-stream" title_clean = title.strip() - job = store.create_job( - worker_id=worker_id.strip(), + # Resolver workers + selected: List[str] = [] + raw_ids = (worker_ids or "").strip() + if raw_ids: + try: + parsed = _json.loads(raw_ids) + if isinstance(parsed, list): + selected = [str(x).strip() for x in parsed if str(x).strip()] + elif isinstance(parsed, str) and parsed.strip(): + selected = [parsed.strip()] + except _json.JSONDecodeError: + selected = [x.strip() for x in raw_ids.split(",") if x.strip()] + if not selected and worker_id.strip(): + selected = [worker_id.strip()] + + online = store.list_online_worker_ids() + online_set = set(online) + if not selected: + selected = list(online) + else: + missing = [w for w in selected if w not in online_set] + if missing: + raise HTTPException( + status_code=409, + detail=f"Workers no online: {', '.join(missing)}. Enciende el agent primero.", + ) + + if not selected: + raise HTTPException( + status_code=409, + detail="No hay bots online. Enciende al menos un agent.", + ) + + # Dedup preservando orden + seen: set[str] = set() + workers_unique: List[str] = [] + for w in selected: + if w not in seen: + seen.add(w) + workers_unique.append(w) + + # Parse filas + try: + rows = parse_radicado_rows(data, original) + except Exception as e: + raise HTTPException(status_code=400, detail=f"No se pudo leer el Excel/CSV: {e}") from e + if not rows: + raise HTTPException(status_code=400, detail="El archivo no tiene filas de radicado/NIT") + + # Un solo bot (o una sola fila) → flujo actual + n_workers = min(len(workers_unique), len(rows)) + if n_workers <= 1: + only = workers_unique[0] + job = store.create_job( + worker_id=only, + excel_key="", + original_filename=original, + title=title_clean, + job_id=parent_job_id, + ) + task_id = store.create_downloads_task( + parent_job_id=parent_job_id, + download_job_id=job.id, + worker_id=only, + title=title_clean or f"Bot descarga ({original})", + original_filename=original, + worker_ids=[only], + mode="single", + ) + try: + key = minio.upload_excel(job.id, data, original, content_type=content_type) + except Exception as e: + store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}") + raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e + + job = store.update_excel_key(job.id, key) + assert job is not None + store.add_event( + job.id, + "run_started", + f"Job creado y asignado a {only}", + metadata={ + "excel_key": key, + "worker_id": only, + "job_id": parent_job_id, + "task_id": task_id, + "mode": "single", + "total_rows": len(rows), + }, + ) + return job.to_dict() + + # Distribuido: padre + shards + workers_unique = workers_unique[:n_workers] + chunks = split_rows(rows, n_workers) + n = len(chunks) + workers_unique = workers_unique[:n] + + parent = store.create_job( + worker_id=None, excel_key="", original_filename=original, - title=title_clean, + title=title_clean or f"Descarga distribuida ({original})", job_id=parent_job_id, - ) - task_id = store.create_downloads_task( - parent_job_id=parent_job_id, - download_job_id=job.id, - worker_id=worker_id.strip(), - title=title_clean or f"Bot descarga ({original})", - original_filename=original, - ) - job = store.get_job(job.id) - assert job is not None - - try: - key = minio.upload_excel(job.id, data, original, content_type=content_type) - except Exception as e: - store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}") - raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e - - job = store.update_excel_key(job.id, key) - assert job is not None - store.add_event( - job.id, - "run_started", - f"Job creado y asignado a {worker_id}", - metadata={ - "excel_key": key, - "worker_id": worker_id, - "job_id": parent_job_id, - "task_id": task_id, + shard_count=n, + status=JobStatus.PENDING.value, + result={ + "mode": "distributed", + "shard_ids": [], + "total_rows": len(rows), + "total": len(rows), + "processed": 0, + "exitosos": 0, + "fallidos": 0, }, ) + prefix = parent.downloads_prefix or mk_prefix(parent.id) + + task_id = store.create_downloads_task( + parent_job_id=parent_job_id, + download_job_id=parent.id, + worker_id=None, + title=title_clean or f"Bot descarga ×{n} ({original})", + original_filename=original, + worker_ids=workers_unique, + mode="distributed", + ) + + try: + full_key = minio.upload_excel(parent.id, data, original, content_type=content_type) + except Exception as e: + store.patch_job(parent.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}") + raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e + + store.update_excel_key(parent.id, full_key) + + shard_ids: List[str] = [] + for i, (chunk, wid) in enumerate(zip(chunks, workers_unique)): + shard_bytes = rows_to_xlsx_bytes(chunk) + shard = store.create_job( + worker_id=wid, + excel_key="", + original_filename=f"shard_{i + 1}_of_{n}.xlsx", + title=f"{title_clean or 'Descarga'} · shard {i + 1}/{n}", + job_id=parent_job_id, + task_id=task_id, + downloads_prefix_override=prefix, + parent_download_job_id=parent.id, + shard_index=i, + shard_count=n, + result={"total": len(chunk), "processed": 0}, + ) + try: + shard_key = minio.upload_excel( + shard.id, + shard_bytes, + f"shard_{i + 1}_of_{n}.xlsx", + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + except Exception as e: + store.patch_job( + parent.id, + status=JobStatus.FAILED.value, + error=f"MinIO shard upload failed: {e}", + ) + raise HTTPException( + status_code=502, detail=f"Error subiendo shard a MinIO: {e}" + ) from e + store.update_excel_key(shard.id, shard_key) + shard_ids.append(shard.id) + store.add_event( + parent.id, + "info", + f"Shard {i + 1}/{n} → {wid} ({len(chunk)} filas)", + metadata={"shard_id": shard.id, "worker_id": wid, "rows": len(chunk)}, + ) + + store.patch_job( + parent.id, + result={ + "mode": "distributed", + "shard_ids": shard_ids, + "total_rows": len(rows), + "total": len(rows), + "processed": 0, + "exitosos": 0, + "fallidos": 0, + "downloads_prefix": prefix, + "worker_ids": workers_unique, + }, + ) + store.add_event( + parent.id, + "run_started", + f"Corrida distribuida: {n} bots, {len(rows)} filas", + metadata={ + "excel_key": full_key, + "worker_ids": workers_unique, + "job_id": parent_job_id, + "task_id": task_id, + "mode": "distributed", + "shard_ids": shard_ids, + "total_rows": len(rows), + }, + ) + job = store.get_job(parent.id) + assert job is not None return job.to_dict() +@app.post("/internal/reclaim") +def reclaim_jobs(_: str = Depends(require_bearer)) -> dict: + n = store.reclaim_stale_running_jobs() + return {"reclaimed": n} + + @app.get("/jobs") def list_jobs( limit: int = Query(100, ge=1, le=500), @@ -168,7 +351,15 @@ def get_job(download_job_id: str, _: str = Depends(require_bearer)) -> dict: job = store.get_job(download_job_id) if not job: raise HTTPException(status_code=404, detail="Job no encontrado") - return job.to_dict() + # Refrescar consolidación si es padre distribuido + if job.is_distributed_parent: + refreshed = store.refresh_parent_from_shards(download_job_id) + if refreshed: + job = refreshed + payload = job.to_dict() + if job.is_distributed_parent: + payload["shards"] = [s.to_dict() for s in store.list_shards(job.id)] + return payload @app.get("/jobs/{download_job_id}/files") @@ -279,7 +470,8 @@ def post_event( body: EventBody, _: str = Depends(require_bearer), ) -> dict: - if not store.get_job(download_job_id): + job = store.get_job(download_job_id) + if not job: raise HTTPException(status_code=404, detail="Job no encontrado") event = store.add_event( download_job_id, @@ -292,8 +484,7 @@ def post_event( metadata=body.metadata, timestamp=body.timestamp, ) - job = store.get_job(download_job_id) - if job and body.index is not None and body.total is not None: + if body.index is not None and body.total is not None: result = dict(job.result or {}) result["processed"] = body.index result["total"] = body.total @@ -327,4 +518,7 @@ def cancel_flag(download_job_id: str, _: str = Depends(require_bearer)) -> dict: job = store.get_job(download_job_id) if not job: raise HTTPException(status_code=404, detail="Job no encontrado") - return {"cancel_requested": job.cancel_requested, "status": job.status} + return { + "cancel_requested": store.is_cancel_requested(download_job_id), + "status": job.status, + } diff --git a/services/bot_descarga/domain/excel_split.py b/services/bot_descarga/domain/excel_split.py new file mode 100644 index 0000000..02f4464 --- /dev/null +++ b/services/bot_descarga/domain/excel_split.py @@ -0,0 +1,99 @@ +"""Parseo y partición de Excel/CSV de radicados para descarga distribuida.""" +from __future__ import annotations + +import csv +import io +import re +from typing import Any + +from openpyxl import Workbook, load_workbook + +_HEADER_NAMES = ("radicado", "proceso", "numero", "número") + + +def _clean_nit(valor: Any) -> str: + valor = str(valor).strip() + valor = re.sub(r'[\\/:*?"<>|]', "_", valor) + return valor if valor else "SIN_NIT" + + +def parse_radicado_rows(data: bytes, filename: str = "input.xlsx") -> list[tuple[str, str]]: + """Devuelve lista de (radicado, nit).""" + name = (filename or "input.xlsx").lower() + if name.endswith(".csv"): + return _parse_csv(data) + return _parse_xlsx(data) + + +def _parse_csv(data: bytes) -> list[tuple[str, str]]: + text = None + for enc in ("utf-8-sig", "utf-8", "latin-1", "cp1252"): + try: + text = data.decode(enc) + break + except UnicodeDecodeError: + continue + if text is None: + text = data.decode("utf-8", errors="replace") + sample = text[:4096] + try: + dialect = csv.Sniffer().sniff(sample, delimiters="|,;\t") + delim = dialect.delimiter + except csv.Error: + delim = "|" if sample.count("|") >= sample.count(",") else "," + reader = csv.reader(io.StringIO(text), delimiter=delim) + out: list[tuple[str, str]] = [] + for fila in reader: + if not fila: + continue + radicado = fila[0] if len(fila) > 0 else None + nit = fila[1] if len(fila) > 1 else None + if not radicado or str(radicado).strip() in ("", "None"): + continue + radicado_s = str(radicado).strip() + if radicado_s.lower() in _HEADER_NAMES: + continue + out.append((radicado_s, _clean_nit(nit))) + return out + + +def _parse_xlsx(data: bytes) -> list[tuple[str, str]]: + wb = load_workbook(io.BytesIO(data), data_only=True) + ws = wb.active + out: list[tuple[str, str]] = [] + for fila in ws.iter_rows(values_only=True): + radicado = fila[0] if len(fila) > 0 else None + nit = fila[1] if len(fila) > 1 else None + if not radicado or str(radicado).strip() in ("", "None"): + continue + radicado_s = str(radicado).strip() + if radicado_s.lower() in _HEADER_NAMES: + continue + out.append((radicado_s, _clean_nit(nit))) + return out + + +def split_rows(rows: list[tuple[str, str]], n: int) -> list[list[tuple[str, str]]]: + """Parte filas en n trozos lo más equilibrados posible (chunks contiguos).""" + if n <= 1 or not rows: + return [rows] + n = min(n, len(rows)) + size, rem = divmod(len(rows), n) + chunks: list[list[tuple[str, str]]] = [] + idx = 0 + for i in range(n): + take = size + (1 if i < rem else 0) + chunks.append(rows[idx : idx + take]) + idx += take + return [c for c in chunks if c] + + +def rows_to_xlsx_bytes(rows: list[tuple[str, str]]) -> bytes: + wb = Workbook() + ws = wb.active + ws.append(["radicado", "nit"]) + for radicado, nit in rows: + ws.append([radicado, nit]) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() diff --git a/services/bot_descarga/domain/models.py b/services/bot_descarga/domain/models.py index 5953ef6..b65a94a 100644 --- a/services/bot_descarga/domain/models.py +++ b/services/bot_descarga/domain/models.py @@ -40,7 +40,7 @@ class Worker: @dataclass class Job: id: str - worker_id: str + worker_id: Optional[str] status: str = JobStatus.PENDING.value title: str = "" excel_key: str = "" @@ -54,10 +54,25 @@ class Job: finished_at: Optional[str] = None job_id: Optional[str] = None task_id: Optional[str] = None + parent_download_job_id: Optional[str] = None + shard_index: Optional[int] = None + shard_count: Optional[int] = None def to_dict(self) -> dict: return asdict(self) + @property + def is_distributed_parent(self) -> bool: + return ( + self.parent_download_job_id is None + and self.shard_count is not None + and int(self.shard_count) > 1 + ) + + @property + def is_shard(self) -> bool: + return bool(self.parent_download_job_id) + @dataclass class JobEventRecord: diff --git a/services/bot_descarga/infrastructure/database.py b/services/bot_descarga/infrastructure/database.py index 281c986..df957c4 100644 --- a/services/bot_descarga/infrastructure/database.py +++ b/services/bot_descarga/infrastructure/database.py @@ -103,7 +103,12 @@ class DownloadJobRow(Base): task_id: Mapped[str | None] = mapped_column( String(36), ForeignKey("tasks.id"), nullable=True, index=True ) - worker_id: Mapped[str] = mapped_column(String(36), index=True) + worker_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) + parent_download_job_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("download_jobs.id"), nullable=True, index=True + ) + shard_index: Mapped[int | None] = mapped_column(Integer, nullable=True) + shard_count: Mapped[int | None] = mapped_column(Integer, nullable=True) status: Mapped[str] = mapped_column(String(32), default="pending", index=True) title: Mapped[str | None] = mapped_column(Text, nullable=True) excel_key: Mapped[str] = mapped_column(Text, default="") diff --git a/services/bot_descarga/infrastructure/storage_minio.py b/services/bot_descarga/infrastructure/storage_minio.py index 7471c21..0d58712 100644 --- a/services/bot_descarga/infrastructure/storage_minio.py +++ b/services/bot_descarga/infrastructure/storage_minio.py @@ -194,21 +194,25 @@ class MinioStorage: Si el manifest falta, falla o queda vacío, hace fallback a list_keys bajo el prefijo (cubre varias carpetas NIT planas). + En corridas distribuidas varios shards sobrescriben el manifest parcial: + siempre se fusiona con list_keys del prefijo. """ prefix = prefix or downloads_prefix(job_id) - files: List[dict] = [] + by_key: dict[str, dict] = {} try: raw = self.get_bytes(manifest_key(job_id)) payload = json.loads(raw.decode("utf-8")) for key in payload.get("files") or []: entry = self._file_entry(str(key)) if entry is not None: - files.append(entry) + by_key[entry["key"]] = entry except Exception: - files = [] + pass - if not files: - files = self._list_pdf_entries(prefix) + for entry in self._list_pdf_entries(prefix): + by_key[entry["key"]] = entry + + files = list(by_key.values()) return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)} def _list_pdf_entries(self, prefix: str) -> List[dict]: diff --git a/services/bot_descarga/infrastructure/store.py b/services/bot_descarga/infrastructure/store.py index c0a0899..a18ff3a 100644 --- a/services/bot_descarga/infrastructure/store.py +++ b/services/bot_descarga/infrastructure/store.py @@ -73,6 +73,9 @@ class JobStore: finished_at=_iso(row.finished_at), job_id=row.job_id, task_id=row.task_id, + parent_download_job_id=row.parent_download_job_id, + shard_index=row.shard_index, + shard_count=row.shard_count, ) def _refresh_worker_status(self, worker: Worker) -> Worker: @@ -129,7 +132,13 @@ class JobStore: WorkerStatus.BUSY.value if row.current_job_id else WorkerStatus.ONLINE.value ) session.flush() - return self._row_worker(row) + worker = self._row_worker(row) + # Reclaim fuera de la sesión del heartbeat para no bloquear + try: + self.reclaim_stale_running_jobs() + except Exception: + pass + return worker def get_worker(self, worker_id: str) -> Optional[Worker]: with session_scope() as session: @@ -139,12 +148,23 @@ class JobStore: return self._row_worker(row) def list_workers(self) -> List[Worker]: + try: + self.reclaim_stale_running_jobs() + except Exception: + pass with session_scope() as session: rows = session.scalars( select(DownloadWorkerRow).order_by(DownloadWorkerRow.id) ).all() return [self._row_worker(r) for r in rows] + def list_online_worker_ids(self) -> List[str]: + return [ + w.id + for w in self.list_workers() + if w.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value) + ] + def is_worker_online(self, worker_id: str) -> bool: worker = self.get_worker(worker_id) if not worker: @@ -162,31 +182,41 @@ class JobStore: def create_job( self, - worker_id: str, + worker_id: Optional[str], excel_key: str, original_filename: str = "", title: str = "", *, job_id: Optional[str] = None, task_id: Optional[str] = None, + downloads_prefix_override: Optional[str] = None, + parent_download_job_id: Optional[str] = None, + shard_index: Optional[int] = None, + shard_count: Optional[int] = None, + result: Optional[Dict[str, Any]] = None, + status: Optional[str] = None, + download_id: Optional[str] = None, ) -> Job: import uuid as uuid_mod - download_id = str(uuid_mod.uuid4()) - prefix = downloads_prefix(download_id) + download_id = download_id or str(uuid_mod.uuid4()) + prefix = downloads_prefix_override or downloads_prefix(download_id) with session_scope() as session: row = DownloadJobRow( id=download_id, job_id=job_id, task_id=task_id, worker_id=worker_id, - status=JobStatus.PENDING.value, + parent_download_job_id=parent_download_job_id, + shard_index=shard_index, + shard_count=shard_count, + status=status or JobStatus.PENDING.value, title=title or f"Descarga {original_filename}", excel_key=excel_key, downloads_prefix=prefix, original_filename=original_filename, cancel_requested=False, - result={}, + result=result or {}, created_at=utcnow(), ) session.add(row) @@ -205,34 +235,57 @@ class JobStore: limit: int = 100, *, job_id: Optional[str] = None, + parents_only: bool = True, ) -> List[Job]: with session_scope() as session: - stmt = select(DownloadJobRow).order_by(DownloadJobRow.created_at.desc()).limit(limit) + stmt = select(DownloadJobRow).order_by(DownloadJobRow.created_at.desc()) if job_id: - stmt = ( - select(DownloadJobRow) - .where(DownloadJobRow.job_id == job_id) - .order_by(DownloadJobRow.created_at.desc()) - .limit(limit) - ) + stmt = stmt.where(DownloadJobRow.job_id == job_id) + if parents_only: + stmt = stmt.where(DownloadJobRow.parent_download_job_id.is_(None)) + stmt = stmt.limit(limit) rows = session.scalars(stmt).all() return [self._row_job(r) for r in rows] + def list_shards(self, parent_id: str) -> List[Job]: + with session_scope() as session: + rows = session.scalars( + select(DownloadJobRow) + .where(DownloadJobRow.parent_download_job_id == parent_id) + .order_by(DownloadJobRow.shard_index.asc()) + ).all() + return [self._row_job(r) for r in rows] + def claim_pending(self, worker_id: str) -> Optional[Job]: + try: + self.reclaim_stale_running_jobs() + except Exception: + pass with session_scope() as session: worker = session.get(DownloadWorkerRow, worker_id) if not worker or worker.current_job_id: return None - row = session.scalars( + # No claimar padres orquestadores (shard_count > 1 sin parent) + rows = session.scalars( select(DownloadJobRow) .where( DownloadJobRow.worker_id == worker_id, DownloadJobRow.status == JobStatus.PENDING.value, ) .order_by(DownloadJobRow.created_at.asc()) - .limit(1) - ).first() + ).all() + row = None + for candidate in rows: + is_parent = ( + candidate.parent_download_job_id is None + and candidate.shard_count is not None + and int(candidate.shard_count) > 1 + ) + if is_parent: + continue + row = candidate + break if not row: return None @@ -243,9 +296,19 @@ class JobStore: worker.status = WorkerStatus.BUSY.value worker.last_heartbeat = now - if row.task_id: - task = session.get(Task, row.task_id) - if task: + # Task del padre (shards heredan task_id del padre) + task_id = row.task_id + if row.parent_download_job_id: + parent = session.get(DownloadJobRow, row.parent_download_job_id) + if parent: + task_id = parent.task_id + if parent.status == JobStatus.PENDING.value: + parent.status = JobStatus.RUNNING.value + parent.started_at = parent.started_at or now + + if task_id: + task = session.get(Task, task_id) + if task and task.status == JobStatus.PENDING.value: task.status = JobStatus.RUNNING.value task.started_at = now task.updated_at = now @@ -271,12 +334,44 @@ class JobStore: row.status = JobStatus.CANCELLED.value row.finished_at = now self._sync_task(session, row, status=JobStatus.CANCELLED.value) + + # Cancelar shards hijos si es padre distribuido + if row.parent_download_job_id is None and row.shard_count and int(row.shard_count) > 1: + children = session.scalars( + select(DownloadJobRow).where( + DownloadJobRow.parent_download_job_id == job_id + ) + ).all() + for child in children: + if child.status in ( + JobStatus.COMPLETED.value, + JobStatus.FAILED.value, + JobStatus.CANCELLED.value, + ): + continue + child.cancel_requested = True + if child.status == JobStatus.PENDING.value: + child.status = JobStatus.CANCELLED.value + child.finished_at = now + worker = session.get(DownloadWorkerRow, child.worker_id) if child.worker_id else None + if worker and worker.current_job_id == child.id: + worker.current_job_id = None + worker.status = WorkerStatus.ONLINE.value + session.flush() return self._row_job(row) def is_cancel_requested(self, job_id: str) -> bool: job = self.get_job(job_id) - return bool(job and job.cancel_requested) + if not job: + return False + if job.cancel_requested: + return True + if job.parent_download_job_id: + parent = self.get_job(job.parent_download_job_id) + if parent and parent.cancel_requested: + return True + return False def _sync_task( self, @@ -287,9 +382,15 @@ class JobStore: result: Optional[Dict[str, Any]] = None, error: Optional[str] = None, ) -> None: - if not row.task_id: + # Sync siempre contra el padre si es shard + target = row + if row.parent_download_job_id: + parent = session.get(DownloadJobRow, row.parent_download_job_id) + if parent: + target = parent + if not target.task_id: return - task = session.get(Task, row.task_id) + task = session.get(Task, target.task_id) if not task: return now = utcnow() @@ -298,7 +399,7 @@ class JobStore: if result is not None: task.result = { **dict(task.result or {}), - "download_job_id": row.id, + "download_job_id": target.id, **result, } if error is not None: @@ -319,6 +420,7 @@ class JobStore: result: Optional[Dict[str, Any]] = None, error: Optional[str] = None, ) -> Optional[Job]: + parent_id: Optional[str] = None with session_scope() as session: row = session.get(DownloadJobRow, job_id) if not row: @@ -340,22 +442,263 @@ class JobStore: row.finished_at = row.finished_at or utcnow() finished = True - self._sync_task( - session, - row, - status=new_status if status else None, - result=result, - error=error, - ) + # No sync task directo en shards; se consolida en el padre + if not row.parent_download_job_id: + self._sync_task( + session, + row, + status=new_status if status else None, + result=result, + error=error, + ) if finished: - worker = session.get(DownloadWorkerRow, row.worker_id) + worker = session.get(DownloadWorkerRow, row.worker_id) if row.worker_id else None if worker and worker.current_job_id == job_id: worker.current_job_id = None worker.status = WorkerStatus.ONLINE.value + parent_id = row.parent_download_job_id session.flush() - return self._row_job(row) + job = self._row_job(row) + + if parent_id: + self.refresh_parent_from_shards(parent_id) + refreshed = self.get_job(job_id) + return refreshed or job + return job + + def refresh_parent_from_shards(self, parent_id: str) -> Optional[Job]: + with session_scope() as session: + parent = session.get(DownloadJobRow, parent_id) + if not parent: + return None + children = list( + session.scalars( + select(DownloadJobRow).where( + DownloadJobRow.parent_download_job_id == parent_id + ) + ).all() + ) + if not children: + return self._row_job(parent) + + statuses = [c.status for c in children] + terminal = { + JobStatus.COMPLETED.value, + JobStatus.FAILED.value, + JobStatus.CANCELLED.value, + } + any_active = any( + s in (JobStatus.PENDING.value, JobStatus.RUNNING.value) for s in statuses + ) + all_terminal = all(s in terminal for s in statuses) + + total = 0 + exitosos = 0 + fallidos = 0 + uploaded = 0 + for c in children: + r = dict(c.result or {}) + total += int(r.get("total") or 0) + exitosos += int(r.get("exitosos") or 0) + fallidos += int(r.get("fallidos") or 0) + uploaded += int(r.get("uploaded_count") or 0) + # Progreso parcial mientras corre + if c.status == JobStatus.RUNNING.value: + total = max(total, int(r.get("total") or 0)) + exitosos = max(exitosos, int(r.get("processed") or r.get("exitosos") or 0)) + + # Agregar progreso en vuelo (processed) de shards running + processed = 0 + for c in children: + r = dict(c.result or {}) + if c.status in terminal: + processed += int(r.get("exitosos") or 0) + int(r.get("fallidos") or 0) + else: + processed += int(r.get("processed") or 0) + + shard_ids = [c.id for c in children] + parent_result = dict(parent.result or {}) + parent_result.update( + { + "mode": "distributed", + "shard_ids": shard_ids, + "total": total or int(parent_result.get("total_rows") or 0), + "total_rows": int(parent_result.get("total_rows") or total or 0), + "exitosos": exitosos, + "fallidos": fallidos, + "processed": processed, + "uploaded_count": uploaded, + "downloads_prefix": parent.downloads_prefix, + "shards": [ + { + "id": c.id, + "worker_id": c.worker_id, + "status": c.status, + "shard_index": c.shard_index, + "result": dict(c.result or {}), + "error": c.error, + } + for c in children + ], + } + ) + parent.result = parent_result + + now = utcnow() + if any_active: + parent.status = JobStatus.RUNNING.value + parent.started_at = parent.started_at or now + parent.finished_at = None + parent.error = None + self._sync_task( + session, + parent, + status=JobStatus.RUNNING.value, + result=parent_result, + ) + elif all_terminal: + any_failed = any(s == JobStatus.FAILED.value for s in statuses) + any_cancelled = any(s == JobStatus.CANCELLED.value for s in statuses) + all_cancelled = all(s == JobStatus.CANCELLED.value for s in statuses) + if all_cancelled: + parent.status = JobStatus.CANCELLED.value + elif any_failed: + parent.status = JobStatus.FAILED.value + parent.error = next( + (c.error for c in children if c.error), + "Uno o más shards fallaron", + ) + elif any_cancelled and not any_failed: + # mezclado completed+cancelled → completed parcial + parent.status = JobStatus.COMPLETED.value + else: + parent.status = JobStatus.COMPLETED.value + parent.finished_at = parent.finished_at or now + self._sync_task( + session, + parent, + status=parent.status, + result=parent_result, + error=parent.error, + ) + + session.flush() + return self._row_job(parent) + + def reclaim_stale_running_jobs(self) -> int: + """Jobs running cuyo worker está offline → pending + reasignar a otro online.""" + reclaimed = 0 + with session_scope() as session: + now = utcnow() + cutoff = now - timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS) + workers = list(session.scalars(select(DownloadWorkerRow)).all()) + offline_ids: set[str] = set() + online_free: List[DownloadWorkerRow] = [] + for w in workers: + hb = w.last_heartbeat + if hb is None: + offline_ids.add(w.id) + continue + if hb.tzinfo is None: + hb = hb.replace(tzinfo=timezone.utc) + if hb < cutoff: + offline_ids.add(w.id) + if w.current_job_id: + w.current_job_id = None + w.status = WorkerStatus.OFFLINE.value + elif not w.current_job_id: + online_free.append(w) + + if not offline_ids: + session.flush() + return 0 + + running = session.scalars( + select(DownloadJobRow).where( + DownloadJobRow.status == JobStatus.RUNNING.value, + DownloadJobRow.worker_id.in_(list(offline_ids)), + ) + ).all() + + rr = 0 + for row in running: + # Solo shards o jobs simples (no padres orquestadores) + is_parent = ( + row.parent_download_job_id is None + and row.shard_count is not None + and int(row.shard_count) > 1 + ) + if is_parent: + continue + + row.status = JobStatus.PENDING.value + row.started_at = None + row.cancel_requested = False + + if online_free: + target = online_free[rr % len(online_free)] + rr += 1 + row.worker_id = target.id + # Si no hay online free, queda pending con el mismo worker_id + # (se reasignará cuando haya free en el próximo reclaim) + + reclaimed += 1 + session.add( + DownloadJobEventRow( + download_job_id=row.id, + event_type="warning", + message=( + f"Worker offline; job reclaimed" + + (f" → {row.worker_id}" if row.worker_id else "") + ), + timestamp=now, + metadata_json={"reclaimed": True, "worker_id": row.worker_id}, + ) + ) + if row.parent_download_job_id: + session.add( + DownloadJobEventRow( + download_job_id=row.parent_download_job_id, + event_type="warning", + message=f"Shard {row.shard_index} reclamado (worker offline)", + timestamp=now, + metadata_json={ + "shard_id": row.id, + "worker_id": row.worker_id, + }, + ) + ) + + # Reasignar pending de workers offline a online free + if online_free: + pending_orphan = session.scalars( + select(DownloadJobRow).where( + DownloadJobRow.status == JobStatus.PENDING.value, + DownloadJobRow.worker_id.in_(list(offline_ids)), + ) + ).all() + for row in pending_orphan: + is_parent = ( + row.parent_download_job_id is None + and row.shard_count is not None + and int(row.shard_count) > 1 + ) + if is_parent: + continue + target = online_free[rr % len(online_free)] + rr += 1 + if row.worker_id != target.id: + row.worker_id = target.id + reclaimed += 1 + + session.flush() + + # Refrescar padres afectados + # (se hace en siguiente patch; aquí no hace falta) + + return reclaimed def add_event( self, @@ -385,6 +728,28 @@ class JobStore: ) session.add(ev) session.flush() + # Mirror a padre si es shard + row = session.get(DownloadJobRow, job_id) + if row and row.parent_download_job_id: + session.add( + DownloadJobEventRow( + download_job_id=row.parent_download_job_id, + event_type=event_type, + message=message, + timestamp=ts, + radicado=radicado, + nit=nit, + idx=index, + total=total, + metadata_json={ + **(metadata or {}), + "shard_id": job_id, + "shard_index": row.shard_index, + "worker_id": row.worker_id, + }, + ) + ) + session.flush() return JobEventRecord( id=int(ev.id), job_id=job_id, @@ -429,9 +794,11 @@ class JobStore: *, parent_job_id: str, download_job_id: str, - worker_id: str, + worker_id: Optional[str], title: str, original_filename: str, + worker_ids: Optional[List[str]] = None, + mode: str = "single", ) -> str: """Crea Task(type=downloads) y la asocia al download_job. Devuelve task_id.""" with session_scope() as session: @@ -443,10 +810,12 @@ class JobStore: title=title or f"Bot descarga ({original_filename})", request={ "worker_id": worker_id, + "worker_ids": worker_ids or ([worker_id] if worker_id else []), "original_filename": original_filename, "download_job_id": download_job_id, + "mode": mode, }, - result={"download_job_id": download_job_id}, + result={"download_job_id": download_job_id, "mode": mode}, created_at=now, updated_at=now, ) diff --git a/services/bot_descarga/requirements.txt b/services/bot_descarga/requirements.txt index 4a41418..7b13fff 100644 --- a/services/bot_descarga/requirements.txt +++ b/services/bot_descarga/requirements.txt @@ -3,6 +3,7 @@ uvicorn[standard]>=0.27.0 python-multipart>=0.0.9 boto3>=1.34.0 httpx>=0.27.0 +openpyxl>=3.1.0 pydantic>=2.0.0 sqlalchemy>=2.0.0 psycopg[binary]>=3.1.0 diff --git a/services/bot_descarga/tests/test_excel_split.py b/services/bot_descarga/tests/test_excel_split.py new file mode 100644 index 0000000..96875ea --- /dev/null +++ b/services/bot_descarga/tests/test_excel_split.py @@ -0,0 +1,33 @@ +"""Tests de partición Excel para descarga distribuida.""" +from __future__ import annotations + +from domain.excel_split import parse_radicado_rows, rows_to_xlsx_bytes, split_rows + + +def test_split_rows_balanced(): + rows = [(str(i), "NIT") for i in range(100)] + chunks = split_rows(rows, 2) + assert len(chunks) == 2 + assert len(chunks[0]) == 50 + assert len(chunks[1]) == 50 + + +def test_split_rows_uneven(): + rows = [(str(i), "N") for i in range(10)] + chunks = split_rows(rows, 3) + assert len(chunks) == 3 + assert sum(len(c) for c in chunks) == 10 + assert [len(c) for c in chunks] == [4, 3, 3] + + +def test_roundtrip_xlsx(): + rows = [("RAD-1", "900"), ("RAD-2", "901")] + data = rows_to_xlsx_bytes(rows) + parsed = parse_radicado_rows(data, "shard.xlsx") + assert parsed == rows + + +def test_split_more_workers_than_rows(): + rows = [("A", "1"), ("B", "2")] + chunks = split_rows(rows, 5) + assert len(chunks) == 2 diff --git a/services/bot_digitador/Dockerfile b/services/bot_digitador/Dockerfile new file mode 100644 index 0000000..2e70b55 --- /dev/null +++ b/services/bot_digitador/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY domain /app/domain +COPY infrastructure /app/infrastructure +COPY api /app/api + +ENV BOT_API_HOST=0.0.0.0 +ENV BOT_API_PORT=8210 + +EXPOSE 8210 + +HEALTHCHECK --interval=10s --timeout=5s --retries=5 --start-period=15s \ + CMD curl -f http://127.0.0.1:8210/health || exit 1 + +CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8210"] diff --git a/services/bot_digitador/api/__init__.py b/services/bot_digitador/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/bot_digitador/api/main.py b/services/bot_digitador/api/main.py new file mode 100644 index 0000000..94256b9 --- /dev/null +++ b/services/bot_digitador/api/main.py @@ -0,0 +1,600 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any, Dict, List, Optional + +from fastapi import Depends, FastAPI, File, Form, HTTPException, Query, UploadFile +from fastapi.responses import Response +from pydantic import BaseModel, Field + +from domain.models import JobStatus +from infrastructure.auth import require_bearer +from infrastructure.storage_minio import MinioSettings, MinioStorage +from infrastructure.store import JobStore + +store = JobStore() +minio = MinioStorage(MinioSettings.from_env()) + +POBLACION_APTA_URL = os.getenv("POBLACION_APTA_URL", "http://poblacion_apta:8003").rstrip("/") + +app = FastAPI(title="BotDigitador Remote API", version="1.0.0") + + +class RegisterWorkerBody(BaseModel): + worker_id: str + hostname: str = "" + max_concurrent: int = 1 + + +class HeartbeatBody(BaseModel): + worker_id: str + + +class EventBody(BaseModel): + event_type: str + message: str + radicado: Optional[str] = None + nit: Optional[str] = None + index: Optional[int] = None + total: Optional[int] = None + metadata: Dict[str, Any] = Field(default_factory=dict) + timestamp: Optional[str] = None + + +class PatchJobBody(BaseModel): + status: Optional[str] = None + result: Optional[Dict[str, Any]] = None + error: Optional[str] = None + + +def _media_type_for_key(key: str) -> str: + lower = key.lower() + if lower.endswith(".csv"): + return "text/csv; charset=utf-8" + if lower.endswith(".log") or lower.endswith(".txt"): + return "text/plain; charset=utf-8" + if lower.endswith(".json"): + return "application/json" + return "application/octet-stream" + + +@app.get("/health") +def health() -> dict: + return {"status": "ok", "service": "bot-digitador"} + + +@app.post("/workers/register") +def register_worker(body: RegisterWorkerBody, _: str = Depends(require_bearer)) -> dict: + worker = store.register_worker( + body.worker_id, + hostname=body.hostname, + max_concurrent=body.max_concurrent, + ) + return worker.to_dict() + + +@app.post("/workers/heartbeat") +def heartbeat(body: HeartbeatBody, _: str = Depends(require_bearer)) -> dict: + worker = store.heartbeat(body.worker_id) + if not worker: + raise HTTPException(status_code=404, detail="Worker no registrado") + return worker.to_dict() + + +@app.get("/workers") +def list_workers(_: str = Depends(require_bearer)) -> List[dict]: + return [w.to_dict() for w in store.list_workers()] + + +@app.get("/workers/me/pending") +def claim_pending( + worker_id: str = Query(...), + _: str = Depends(require_bearer), +) -> dict: + job = store.claim_pending(worker_id) + if not job: + return {"job": None} + return {"job": job.to_dict()} + + +@app.post("/jobs") +async def create_job( + job_id: str = Form(...), + worker_id: str = Form(""), + worker_ids: str = Form(""), + title: str = Form(""), + file: Optional[UploadFile] = File(None), + poblacion_apta_job_id: str = Form(""), + subset: str = Form("aptos"), + _: str = Depends(require_bearer), +) -> dict: + """Crea una corrida de digitación. + + Exactamente uno de: file (CSV) o poblacion_apta_job_id. + - 1 worker → flujo clásico. + - N workers → padre + N shards CSV, mismo reports_prefix del padre. + """ + import json as _json + + import httpx + + from domain.csv_split import parse_digitador_rows, rows_to_csv_bytes, split_rows + from infrastructure.storage_minio import reports_prefix as mk_prefix + + parent_job_id = job_id.strip() + if not parent_job_id: + raise HTTPException(status_code=400, detail="job_id (Job Desktop) obligatorio") + if not store.parent_job_exists(parent_job_id): + raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado") + + pa_id = (poblacion_apta_job_id or "").strip() + data: bytes = b"" + original = "input.csv" + content_type = "text/csv" + source_type = "upload" + + file_bytes: bytes = b"" + if file is not None: + file_bytes = await file.read() + + if pa_id and file_bytes: + raise HTTPException( + status_code=400, + detail="Proporcione exactamente uno de: file o poblacion_apta_job_id", + ) + + if pa_id: + source_type = "poblacion_apta" + subset_norm = (subset or "aptos").strip().lower() + if subset_norm != "aptos": + raise HTTPException( + status_code=400, + detail="Solo se digitan aptos (PASO_BOT). Use subset=aptos.", + ) + url = f"{POBLACION_APTA_URL}/jobs/{pa_id}/export.csv" + try: + async with httpx.AsyncClient(timeout=120.0) as client: + resp = await client.get(url, params={"subset": subset_norm}) + except Exception as e: + raise HTTPException( + status_code=502, + detail=f"No se pudo contactar poblacion_apta: {e}", + ) from e + if resp.status_code == 404: + raise HTTPException(status_code=404, detail="Job de población apta no encontrado") + if not resp.is_success: + raise HTTPException( + status_code=502, + detail=f"Error exportando CSV de población apta: HTTP {resp.status_code}", + ) + data = resp.content + original = f"poblacion_apta_{subset_norm}_{pa_id[:8]}.csv" + content_type = "text/csv; charset=utf-8" + elif file_bytes: + data = file_bytes + original = (file.filename if file else None) or "input.csv" + content_type = (file.content_type if file else None) or "text/csv" + source_type = "upload" + else: + raise HTTPException( + status_code=400, + detail="Proporcione exactamente uno de: file o poblacion_apta_job_id", + ) + + if not data: + raise HTTPException(status_code=400, detail="CSV vacío") + + title_clean = title.strip() + + # Resolver workers + selected: List[str] = [] + raw_ids = (worker_ids or "").strip() + if raw_ids: + try: + parsed = _json.loads(raw_ids) + if isinstance(parsed, list): + selected = [str(x).strip() for x in parsed if str(x).strip()] + elif isinstance(parsed, str) and parsed.strip(): + selected = [parsed.strip()] + except _json.JSONDecodeError: + selected = [x.strip() for x in raw_ids.split(",") if x.strip()] + if not selected and worker_id.strip(): + selected = [worker_id.strip()] + + online = store.list_online_worker_ids() + online_set = set(online) + if not selected: + selected = list(online) + else: + missing = [w for w in selected if w not in online_set] + if missing: + raise HTTPException( + status_code=409, + detail=f"Workers no online: {', '.join(missing)}. Enciende el agent primero.", + ) + + if not selected: + raise HTTPException( + status_code=409, + detail="No hay bots online. Enciende al menos un agent.", + ) + + seen: set[str] = set() + workers_unique: List[str] = [] + for w in selected: + if w not in seen: + seen.add(w) + workers_unique.append(w) + + try: + rows = parse_digitador_rows(data, original) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + except Exception as e: + raise HTTPException(status_code=400, detail=f"No se pudo leer el CSV: {e}") from e + if not rows: + raise HTTPException(status_code=400, detail="El CSV no tiene filas para digitar") + + # Normalizar a CSV canónico del bot (siempre pipe + headers snake_case) + canonical_bytes = rows_to_csv_bytes(rows) + + n_workers = min(len(workers_unique), len(rows)) + if n_workers <= 1: + only = workers_unique[0] + job = store.create_job( + worker_id=only, + csv_key="", + original_filename=original, + title=title_clean, + job_id=parent_job_id, + result={"source_type": source_type, "total_rows": len(rows), "total": len(rows)}, + ) + task_id = store.create_digitador_task( + parent_job_id=parent_job_id, + digitador_job_id=job.id, + worker_id=only, + title=title_clean or f"Bot digitador ({original})", + original_filename=original, + worker_ids=[only], + mode="single", + source_type=source_type, + ) + try: + key = minio.upload_csv(job.id, canonical_bytes, original, content_type=content_type) + except Exception as e: + store.patch_job(job.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}") + raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e + + job = store.update_csv_key(job.id, key) + assert job is not None + store.add_event( + job.id, + "run_started", + f"Job creado y asignado a {only}", + metadata={ + "csv_key": key, + "worker_id": only, + "job_id": parent_job_id, + "task_id": task_id, + "mode": "single", + "total_rows": len(rows), + "source_type": source_type, + }, + ) + return job.to_dict() + + workers_unique = workers_unique[:n_workers] + chunks = split_rows(rows, n_workers) + n = len(chunks) + workers_unique = workers_unique[:n] + + parent = store.create_job( + worker_id=None, + csv_key="", + original_filename=original, + title=title_clean or f"Digitación distribuida ({original})", + job_id=parent_job_id, + shard_count=n, + status=JobStatus.PENDING.value, + result={ + "mode": "distributed", + "shard_ids": [], + "total_rows": len(rows), + "total": len(rows), + "processed": 0, + "ok": 0, + "error": 0, + "timeout": 0, + "existentes": 0, + "source_type": source_type, + }, + ) + prefix = parent.reports_prefix or mk_prefix(parent.id) + + task_id = store.create_digitador_task( + parent_job_id=parent_job_id, + digitador_job_id=parent.id, + worker_id=None, + title=title_clean or f"Bot digitador ×{n} ({original})", + original_filename=original, + worker_ids=workers_unique, + mode="distributed", + source_type=source_type, + ) + + try: + full_key = minio.upload_csv(parent.id, canonical_bytes, original, content_type=content_type) + except Exception as e: + store.patch_job(parent.id, status=JobStatus.FAILED.value, error=f"MinIO upload failed: {e}") + raise HTTPException(status_code=502, detail=f"Error subiendo a MinIO: {e}") from e + + store.update_csv_key(parent.id, full_key) + + shard_ids: List[str] = [] + for i, (chunk, wid) in enumerate(zip(chunks, workers_unique)): + shard_bytes = rows_to_csv_bytes(chunk) + shard = store.create_job( + worker_id=wid, + csv_key="", + original_filename=f"shard_{i + 1}_of_{n}.csv", + title=f"{title_clean or 'Digitador'} · shard {i + 1}/{n}", + job_id=parent_job_id, + task_id=task_id, + reports_prefix_override=prefix, + parent_digitador_job_id=parent.id, + shard_index=i, + shard_count=n, + result={"total": len(chunk), "processed": 0, "source_type": source_type}, + ) + try: + shard_key = minio.upload_csv( + shard.id, + shard_bytes, + f"shard_{i + 1}_of_{n}.csv", + content_type="text/csv; charset=utf-8", + ) + except Exception as e: + store.patch_job( + parent.id, + status=JobStatus.FAILED.value, + error=f"MinIO shard upload failed: {e}", + ) + raise HTTPException( + status_code=502, detail=f"Error subiendo shard a MinIO: {e}" + ) from e + store.update_csv_key(shard.id, shard_key) + shard_ids.append(shard.id) + store.add_event( + parent.id, + "info", + f"Shard {i + 1}/{n} → {wid} ({len(chunk)} filas)", + metadata={"shard_id": shard.id, "worker_id": wid, "rows": len(chunk)}, + ) + + store.patch_job( + parent.id, + result={ + "mode": "distributed", + "shard_ids": shard_ids, + "total_rows": len(rows), + "total": len(rows), + "processed": 0, + "ok": 0, + "error": 0, + "timeout": 0, + "existentes": 0, + "reports_prefix": prefix, + "worker_ids": workers_unique, + "source_type": source_type, + }, + ) + store.add_event( + parent.id, + "run_started", + f"Corrida distribuida: {n} bots, {len(rows)} filas", + metadata={ + "csv_key": full_key, + "worker_ids": workers_unique, + "job_id": parent_job_id, + "task_id": task_id, + "mode": "distributed", + "shard_ids": shard_ids, + "total_rows": len(rows), + "source_type": source_type, + }, + ) + job = store.get_job(parent.id) + assert job is not None + return job.to_dict() + + +@app.post("/internal/reclaim") +def reclaim_jobs(_: str = Depends(require_bearer)) -> dict: + n = store.reclaim_stale_running_jobs() + return {"reclaimed": n} + + +@app.get("/jobs") +def list_jobs( + limit: int = Query(100, ge=1, le=500), + job_id: Optional[str] = Query(None, description="Filtrar por Job Desktop"), + _: str = Depends(require_bearer), +) -> List[dict]: + return [j.to_dict() for j in store.list_jobs(limit=limit, job_id=job_id)] + + +@app.get("/jobs/{digitador_job_id}") +def get_job(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict: + job = store.get_job(digitador_job_id) + if not job: + raise HTTPException(status_code=404, detail="Job no encontrado") + if job.is_distributed_parent: + refreshed = store.refresh_parent_from_shards(digitador_job_id) + if refreshed: + job = refreshed + payload = job.to_dict() + if job.is_distributed_parent: + payload["shards"] = [s.to_dict() for s in store.list_shards(job.id)] + return payload + + +@app.get("/jobs/{digitador_job_id}/files") +def list_job_files(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict: + job = store.get_job(digitador_job_id) + if not job: + raise HTTPException(status_code=404, detail="Job no encontrado") + try: + return minio.list_report_files(digitador_job_id, prefix=job.reports_prefix) + except Exception as e: + raise HTTPException(status_code=502, detail=f"Error listando MinIO: {e}") from e + + +@app.get("/jobs/{digitador_job_id}/object") +def get_job_object( + digitador_job_id: str, + key: str = Query(..., min_length=1, description="Object key bajo reports_prefix"), + _: str = Depends(require_bearer), +): + """Stream de un reporte MinIO (csv/log/json bajo el prefijo del job).""" + job = store.get_job(digitador_job_id) + if not job: + raise HTTPException(status_code=404, detail="Job no encontrado") + + prefix = (job.reports_prefix or f"digitados/{digitador_job_id}/").lstrip("/") + if not prefix.endswith("/"): + prefix += "/" + normalized = key.lstrip("/") + if not normalized.startswith(prefix): + raise HTTPException(status_code=403, detail="Key fuera del prefijo del job") + lower = normalized.lower() + if not (lower.endswith(".csv") or lower.endswith(".log") or lower.endswith(".json")): + raise HTTPException(status_code=400, detail="Solo se permiten objetos csv/log/json") + if ".." in normalized.split("/"): + raise HTTPException(status_code=400, detail="Key no permitida") + + try: + data = minio.get_bytes(normalized) + except Exception as e: + raise HTTPException(status_code=502, detail=f"Error leyendo MinIO: {e}") from e + + filename = Path(normalized).name + return Response( + content=data, + media_type=_media_type_for_key(normalized), + headers={"Content-Disposition": f'inline; filename="{filename}"'}, + ) + + +@app.get("/jobs/{digitador_job_id}/events") +def get_events( + digitador_job_id: str, + since: int = Query(0, ge=0), + _: str = Depends(require_bearer), +) -> dict: + if not store.get_job(digitador_job_id): + raise HTTPException(status_code=404, detail="Job no encontrado") + events = store.list_events(digitador_job_id, since_id=since) + return {"events": [e.to_dict() for e in events]} + + +@app.post("/jobs/{digitador_job_id}/cancel") +def cancel_job(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict: + job = store.request_cancel(digitador_job_id) + if not job: + raise HTTPException(status_code=404, detail="Job no encontrado") + store.add_event(digitador_job_id, "warning", "Cancelación solicitada por el usuario") + return job.to_dict() + + +@app.get("/jobs/{digitador_job_id}/file") +def get_job_file(digitador_job_id: str, _: str = Depends(require_bearer)): + job = store.get_job(digitador_job_id) + if not job: + raise HTTPException(status_code=404, detail="Job no encontrado") + if not job.csv_key: + raise HTTPException(status_code=404, detail="CSV no disponible") + + try: + data = minio.get_bytes(job.csv_key) + except Exception as e: + raise HTTPException(status_code=502, detail=f"Error leyendo MinIO: {e}") from e + + filename = Path(job.csv_key).name + return Response( + content=data, + media_type="text/csv; charset=utf-8", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +@app.get("/jobs/{digitador_job_id}/file-url") +def get_job_file_url(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict: + job = store.get_job(digitador_job_id) + if not job or not job.csv_key: + raise HTTPException(status_code=404, detail="Job/archivo no encontrado") + try: + url = minio.presigned_get(job.csv_key) + except Exception as e: + raise HTTPException(status_code=502, detail=str(e)) from e + return {"url": url, "csv_key": job.csv_key} + + +@app.post("/jobs/{digitador_job_id}/events") +def post_event( + digitador_job_id: str, + body: EventBody, + _: str = Depends(require_bearer), +) -> dict: + job = store.get_job(digitador_job_id) + if not job: + raise HTTPException(status_code=404, detail="Job no encontrado") + event = store.add_event( + digitador_job_id, + body.event_type, + body.message, + radicado=body.radicado, + nit=body.nit, + index=body.index, + total=body.total, + metadata=body.metadata, + timestamp=body.timestamp, + ) + if body.index is not None and body.total is not None: + result = dict(job.result or {}) + result["processed"] = body.index + result["total"] = body.total + if body.nit: + result["nit_actual"] = body.nit + store.patch_job(digitador_job_id, result=result) + return event.to_dict() + + +@app.patch("/jobs/{digitador_job_id}") +def patch_job( + digitador_job_id: str, + body: PatchJobBody, + _: str = Depends(require_bearer), +) -> dict: + if body.status and body.status not in {s.value for s in JobStatus}: + raise HTTPException(status_code=400, detail="status inválido") + job = store.patch_job( + digitador_job_id, + status=body.status, + result=body.result, + error=body.error, + ) + if not job: + raise HTTPException(status_code=404, detail="Job no encontrado") + return job.to_dict() + + +@app.get("/jobs/{digitador_job_id}/cancel-flag") +def cancel_flag(digitador_job_id: str, _: str = Depends(require_bearer)) -> dict: + job = store.get_job(digitador_job_id) + if not job: + raise HTTPException(status_code=404, detail="Job no encontrado") + return { + "cancel_requested": store.is_cancel_requested(digitador_job_id), + "status": job.status, + } diff --git a/services/bot_digitador/domain/__init__.py b/services/bot_digitador/domain/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/bot_digitador/domain/csv_split.py b/services/bot_digitador/domain/csv_split.py new file mode 100644 index 0000000..8a33098 --- /dev/null +++ b/services/bot_digitador/domain/csv_split.py @@ -0,0 +1,157 @@ +"""Parseo y partición de CSV de filas a digitar (bot digitador).""" +from __future__ import annotations + +import csv +import io +from typing import Any + +REQUIRED_BOT_COLUMNS = ( + "radicado", + "documento", + "ingreso", + "total_cargos", + "autorizacion", + "tipo_documento", + "tipo_servicio", + "copago", + "cuota_moderadora", +) + +IT_TO_BOT = { + "RADICADO": "radicado", + "NUMERO": "documento", + "FECHA": "ingreso", + "BRUTO": "total_cargos", + "AUTORIZACION": "autorizacion", + "TIPO_ID": "tipo_documento", + "SERVICIO": "tipo_servicio", + "COPAGO": "copago", + "MODERADORA": "cuota_moderadora", +} + + +def _normalize_header(name: str) -> str: + raw = str(name or "").strip() + if not raw: + return "" + upper = raw.upper() + if upper in IT_TO_BOT: + return IT_TO_BOT[upper] + lower = raw.lower().strip() + # Aceptar snake_case ya normalizado + if lower in REQUIRED_BOT_COLUMNS: + return lower + # Alias comunes + aliases = { + "número": "documento", + "numero": "documento", + "fecha": "ingreso", + "bruto": "total_cargos", + "tipo_id": "tipo_documento", + "servicio": "tipo_servicio", + "moderadora": "cuota_moderadora", + } + return aliases.get(lower, lower) + + +def _decode_text(data: bytes) -> str: + for enc in ("utf-8-sig", "utf-8", "latin-1", "cp1252"): + try: + return data.decode(enc) + except UnicodeDecodeError: + continue + return data.decode("utf-8", errors="replace") + + +def _detect_delimiter(sample: str) -> str: + try: + dialect = csv.Sniffer().sniff(sample, delimiters="|,;\t") + return dialect.delimiter + except csv.Error: + return "|" if sample.count("|") >= sample.count(",") else "," + + +def _validate_columns(headers: list[str]) -> None: + present = set(headers) + missing = [c for c in REQUIRED_BOT_COLUMNS if c not in present] + if missing: + raise ValueError(f"Columnas faltantes: {', '.join(missing)}") + + +def parse_digitador_rows(data: bytes, filename: str = "input.csv") -> list[dict[str, str]]: + """Devuelve filas normalizadas con headers del bot (snake_case).""" + _ = filename # reservado por firma / diagnóstico futuro + text = _decode_text(data) + if not text.strip(): + return [] + + delim = _detect_delimiter(text[:4096]) + reader = csv.DictReader(io.StringIO(text), delimiter=delim) + if not reader.fieldnames: + raise ValueError("CSV sin encabezados") + + rename_map: dict[str, str] = {} + normalized_headers: list[str] = [] + for raw in reader.fieldnames: + norm = _normalize_header(raw) + rename_map[raw] = norm + if norm and norm not in normalized_headers: + normalized_headers.append(norm) + + _validate_columns(normalized_headers) + + out: list[dict[str, str]] = [] + for row in reader: + if not row: + continue + mapped: dict[str, str] = {c: "" for c in REQUIRED_BOT_COLUMNS} + empty = True + for raw_key, value in row.items(): + if raw_key is None: + continue + col = rename_map.get(raw_key) or _normalize_header(raw_key) + if col not in REQUIRED_BOT_COLUMNS: + continue + cell = "" if value is None else str(value).strip() + mapped[col] = cell + if cell: + empty = False + if empty: + continue + # Filtrar filas que parecen ser un segundo header + if mapped.get("radicado", "").lower() in ("radicado", "radicado"): + if mapped.get("documento", "").lower() in ("documento", "numero", "número", ""): + continue + out.append(mapped) + return out + + +def split_rows(rows: list[dict[str, Any]], n: int) -> list[list[dict[str, Any]]]: + """Parte filas en n trozos lo más equilibrados posible (chunks contiguos).""" + if n <= 1 or not rows: + return [rows] + n = min(n, len(rows)) + size, rem = divmod(len(rows), n) + chunks: list[list[dict[str, Any]]] = [] + idx = 0 + for i in range(n): + take = size + (1 if i < rem else 0) + chunks.append(rows[idx : idx + take]) + idx += take + return [c for c in chunks if c] + + +def rows_to_csv_bytes(rows: list[dict[str, Any]]) -> bytes: + """Serializa filas a CSV pipe-delimited con headers del bot (utf-8-sig).""" + buf = io.StringIO() + writer = csv.DictWriter( + buf, + fieldnames=list(REQUIRED_BOT_COLUMNS), + delimiter="|", + lineterminator="\n", + extrasaction="ignore", + ) + writer.writeheader() + for row in rows: + writer.writerow({c: str(row.get(c, "") or "") for c in REQUIRED_BOT_COLUMNS}) + return buf.getvalue().encode("utf-8-sig") diff --git a/services/bot_digitador/domain/models.py b/services/bot_digitador/domain/models.py new file mode 100644 index 0000000..33875fa --- /dev/null +++ b/services/bot_digitador/domain/models.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from enum import Enum +from typing import Any, Dict, Optional + + +def utc_now() -> str: + return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + + +class WorkerStatus(str, Enum): + ONLINE = "online" + OFFLINE = "offline" + BUSY = "busy" + + +class JobStatus(str, Enum): + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +@dataclass +class Worker: + id: str + hostname: str = "" + max_concurrent: int = 1 + status: str = WorkerStatus.ONLINE.value + last_heartbeat: str = field(default_factory=utc_now) + current_job_id: Optional[str] = None + + def to_dict(self) -> dict: + return asdict(self) + + +@dataclass +class Job: + id: str + worker_id: Optional[str] + status: str = JobStatus.PENDING.value + title: str = "" + csv_key: str = "" + reports_prefix: str = "" + original_filename: str = "" + cancel_requested: bool = False + result: Dict[str, Any] = field(default_factory=dict) + error: Optional[str] = None + created_at: str = field(default_factory=utc_now) + started_at: Optional[str] = None + finished_at: Optional[str] = None + job_id: Optional[str] = None + task_id: Optional[str] = None + parent_digitador_job_id: Optional[str] = None + shard_index: Optional[int] = None + shard_count: Optional[int] = None + + def to_dict(self) -> dict: + return asdict(self) + + @property + def is_distributed_parent(self) -> bool: + return ( + self.parent_digitador_job_id is None + and self.shard_count is not None + and int(self.shard_count) > 1 + ) + + @property + def is_shard(self) -> bool: + return bool(self.parent_digitador_job_id) + + +@dataclass +class JobEventRecord: + id: int + job_id: str + event_type: str + message: str + timestamp: str + radicado: Optional[str] = None + nit: Optional[str] = None + index: Optional[int] = None + total: Optional[int] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict: + return asdict(self) diff --git a/services/bot_digitador/infrastructure/__init__.py b/services/bot_digitador/infrastructure/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/bot_digitador/infrastructure/auth.py b/services/bot_digitador/infrastructure/auth.py new file mode 100644 index 0000000..968cbc6 --- /dev/null +++ b/services/bot_digitador/infrastructure/auth.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import os +from typing import Optional + +from fastapi import Header, HTTPException + + +def get_api_token() -> str: + return os.getenv("BOT_API_TOKEN", "dev-token-change-me") + + +def require_bearer(authorization: Optional[str] = Header(default=None)) -> str: + expected = get_api_token() + if not authorization or not authorization.lower().startswith("bearer "): + raise HTTPException(status_code=401, detail="Authorization Bearer requerido") + token = authorization.split(" ", 1)[1].strip() + if token != expected: + raise HTTPException(status_code=401, detail="Token inválido") + return token diff --git a/services/bot_digitador/infrastructure/database.py b/services/bot_digitador/infrastructure/database.py new file mode 100644 index 0000000..2289c8a --- /dev/null +++ b/services/bot_digitador/infrastructure/database.py @@ -0,0 +1,164 @@ +"""Persistencia Postgres del servicio bot-digitador (DB compartida IPSExtractor).""" +from __future__ import annotations + +import os +import uuid +from contextlib import contextmanager +from datetime import datetime, timezone +from typing import Any, Iterator + +from sqlalchemy import ( + JSON, + Boolean, + DateTime, + ForeignKey, + Integer, + String, + Text, + create_engine, + text, +) +from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, sessionmaker + +DATABASE_URL = os.getenv( + "DATABASE_URL", + "postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor", +) + +engine = create_engine(DATABASE_URL, pool_pre_ping=True) +SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False) + + +class Base(DeclarativeBase): + pass + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def new_id() -> str: + return str(uuid.uuid4()) + + +def _iso(dt: datetime | None) -> str | None: + if dt is None: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone().isoformat(timespec="seconds") + + +class Job(Base): + """Contenedor Desktop (lectura / FK).""" + + __tablename__ = "jobs" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + status: Mapped[str] = mapped_column(String(32), default="pending", index=True) + title: Mapped[str | None] = mapped_column(Text, nullable=True) + job_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) + + +class Task(Base): + __tablename__ = "tasks" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + job_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("jobs.id"), nullable=True, index=True + ) + type: Mapped[str] = mapped_column(String(64), index=True) + status: Mapped[str] = mapped_column(String(32), default="pending", index=True) + title: Mapped[str | None] = mapped_column(Text, nullable=True) + request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + owner_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + +class DigitadorWorkerRow(Base): + __tablename__ = "digitador_workers" + + id: Mapped[str] = mapped_column(String(36), primary_key=True) + hostname: Mapped[str] = mapped_column(Text, default="") + max_concurrent: Mapped[int] = mapped_column(Integer, default=1) + status: Mapped[str] = mapped_column(String(32), index=True) + last_heartbeat: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) + current_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + + +class DigitadorJobRow(Base): + __tablename__ = "digitador_jobs" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + job_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("jobs.id"), nullable=True, index=True + ) + task_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("tasks.id"), nullable=True, index=True + ) + worker_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) + parent_digitador_job_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("digitador_jobs.id"), nullable=True, index=True + ) + shard_index: Mapped[int | None] = mapped_column(Integer, nullable=True) + shard_count: Mapped[int | None] = mapped_column(Integer, nullable=True) + status: Mapped[str] = mapped_column(String(32), default="pending", index=True) + title: Mapped[str | None] = mapped_column(Text, nullable=True) + csv_key: Mapped[str] = mapped_column(Text, default="") + reports_prefix: Mapped[str] = mapped_column(Text, default="") + original_filename: Mapped[str] = mapped_column(Text, default="") + cancel_requested: Mapped[bool] = mapped_column(Boolean, default=False) + result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + +class DigitadorJobEventRow(Base): + __tablename__ = "digitador_job_events" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + digitador_job_id: Mapped[str] = mapped_column( + String(36), ForeignKey("digitador_jobs.id"), index=True + ) + event_type: Mapped[str] = mapped_column(String(64)) + message: Mapped[str] = mapped_column(Text) + timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) + radicado: Mapped[str | None] = mapped_column(String(255), nullable=True) + nit: Mapped[str | None] = mapped_column(String(64), nullable=True) + idx: Mapped[int | None] = mapped_column(Integer, nullable=True) + total: Mapped[int | None] = mapped_column(Integer, nullable=True) + metadata_json: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict) + + +def init_db() -> None: + if DATABASE_URL.startswith("postgresql"): + with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection: + connection.execute(text("SELECT pg_advisory_lock(74512005)")) + try: + Base.metadata.create_all(bind=connection) + finally: + connection.execute(text("SELECT pg_advisory_unlock(74512005)")) + return + Base.metadata.create_all(bind=engine) + + +@contextmanager +def session_scope() -> Iterator[Session]: + session = SessionLocal() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() diff --git a/services/bot_digitador/infrastructure/storage_minio.py b/services/bot_digitador/infrastructure/storage_minio.py new file mode 100644 index 0000000..4149ba8 --- /dev/null +++ b/services/bot_digitador/infrastructure/storage_minio.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO, List, Optional + + +@dataclass(frozen=True) +class MinioSettings: + endpoint: str = "https://s3.builker.xyz" + bucket: str = "pys-cuentas-medicas" + access_key: str = "" + secret_key: str = "" + region: str = "us-east-1" + anonymous: bool = True + + @classmethod + def from_env(cls) -> "MinioSettings": + access_key = (os.getenv("MINIO_ACCESS_KEY") or "").strip() + secret_key = (os.getenv("MINIO_SECRET_KEY") or "").strip() + anon_flag = (os.getenv("MINIO_ANONYMOUS") or "").strip().lower() + if anon_flag in ("0", "false", "no"): + anonymous = False + elif anon_flag in ("1", "true", "yes"): + anonymous = True + else: + anonymous = not (access_key and secret_key) + return cls( + endpoint=os.getenv("MINIO_ENDPOINT", "https://s3.builker.xyz").rstrip("/"), + bucket=os.getenv("MINIO_BUCKET", "pys-cuentas-medicas"), + access_key=access_key, + secret_key=secret_key, + region=os.getenv("MINIO_REGION", "us-east-1"), + anonymous=anonymous, + ) + + +def csv_key(job_id: str, filename: str = "input.csv") -> str: + return f"uploads/{job_id}/{filename}" + + +def reports_prefix(job_id: str) -> str: + return f"digitados/{job_id}/" + + +def meta_key(job_id: str) -> str: + return f"uploads/{job_id}/meta.json" + + +def manifest_key(job_id: str) -> str: + return f"digitados/{job_id}/_manifest.json" + + +_REPORT_NAMES = { + "resultados.csv", + "omitidos.csv", + "ejecucion_bot.log", +} + + +def _s3_client(settings: MinioSettings): + import boto3 + from botocore import UNSIGNED + from botocore.client import Config + + s3_cfg = { + "addressing_style": "path", + "request_checksum_calculation": "when_required", + "response_checksum_validation": "when_required", + } + if settings.anonymous: + return boto3.client( + "s3", + endpoint_url=settings.endpoint, + region_name=settings.region, + config=Config(signature_version=UNSIGNED, s3=s3_cfg), + ) + return boto3.client( + "s3", + endpoint_url=settings.endpoint, + aws_access_key_id=settings.access_key, + aws_secret_access_key=settings.secret_key, + region_name=settings.region, + config=Config(signature_version="s3v4", s3=s3_cfg), + ) + + +class MinioStorage: + """Cliente S3-compatible para MinIO (uploads/digitados por job_id).""" + + def __init__(self, settings: Optional[MinioSettings] = None): + self.settings = settings or MinioSettings.from_env() + self._client = None + + @property + def client(self): + if self._client is None: + self._client = _s3_client(self.settings) + return self._client + + @property + def bucket(self) -> str: + return self.settings.bucket + + def put_bytes(self, key: str, data: bytes, content_type: str = "application/octet-stream") -> str: + self.client.put_object( + Bucket=self.bucket, + Key=key, + Body=data, + ContentType=content_type, + ) + return key + + def put_file(self, key: str, path: str | Path, content_type: str = "application/octet-stream") -> str: + path = Path(path) + return self.put_bytes(key, path.read_bytes(), content_type=content_type) + + def put_stream(self, key: str, stream: BinaryIO, content_type: str = "application/octet-stream") -> str: + data = stream.read() + return self.put_bytes(key, data, content_type=content_type) + + def get_bytes(self, key: str) -> bytes: + obj = self.client.get_object(Bucket=self.bucket, Key=key) + return obj["Body"].read() + + def download_file(self, key: str, dest: str | Path) -> Path: + dest = Path(dest) + dest.parent.mkdir(parents=True, exist_ok=True) + self.client.download_file(self.bucket, key, str(dest)) + return dest + + def exists(self, key: str) -> bool: + try: + self.client.head_object(Bucket=self.bucket, Key=key) + return True + except Exception: + return False + + def list_keys(self, prefix: str) -> List[str]: + keys: List[str] = [] + token = None + while True: + kwargs = {"Bucket": self.bucket, "Prefix": prefix} + if token: + kwargs["ContinuationToken"] = token + resp = self.client.list_objects_v2(**kwargs) + for item in resp.get("Contents") or []: + keys.append(item["Key"]) + if not resp.get("IsTruncated"): + break + token = resp.get("NextContinuationToken") + return keys + + def presigned_get(self, key: str, expires_in: int = 3600) -> str: + if self.settings.anonymous: + return f"{self.settings.endpoint}/{self.bucket}/{key}" + return self.client.generate_presigned_url( + "get_object", + Params={"Bucket": self.bucket, "Key": key}, + ExpiresIn=expires_in, + ) + + def upload_csv( + self, + job_id: str, + data: bytes, + original_filename: str, + content_type: str = "text/csv", + ) -> str: + ext = Path(original_filename).suffix.lower() or ".csv" + canonical = "input.csv" if ext in (".csv", ".txt") else "input.csv" + key = csv_key(job_id, canonical) + self.put_bytes(key, data, content_type=content_type) + meta = { + "original_filename": original_filename, + "size": len(data), + "canonical": canonical, + } + self.put_bytes(meta_key(job_id), json.dumps(meta).encode("utf-8"), "application/json") + return key + + def write_manifest(self, job_id: str, keys: List[str]) -> str: + key = manifest_key(job_id) + payload = {"job_id": job_id, "files": keys, "count": len(keys)} + self.put_bytes(key, json.dumps(payload, ensure_ascii=False).encode("utf-8"), "application/json") + return key + + def list_report_files(self, job_id: str, prefix: str | None = None) -> dict: + """Lista reportes de digitación (RESULTADOS/OMITIDOS/log + list_keys). + + Fusiona manifest con listado del prefijo (cubriendo shards distribuidos). + """ + prefix = prefix or reports_prefix(job_id) + by_key: dict[str, dict] = {} + try: + raw = self.get_bytes(manifest_key(job_id)) + payload = json.loads(raw.decode("utf-8")) + for key in payload.get("files") or []: + entry = self._file_entry(str(key)) + if entry is not None: + by_key[entry["key"]] = entry + except Exception: + pass + + for entry in self._list_report_entries(prefix): + by_key[entry["key"]] = entry + + files = list(by_key.values()) + return {"job_id": job_id, "prefix": prefix, "files": files, "count": len(files)} + + def _list_report_entries(self, prefix: str) -> List[dict]: + files: List[dict] = [] + for key in self.list_keys(prefix): + entry = self._file_entry(key) + if entry is not None: + files.append(entry) + return files + + @staticmethod + def _file_entry(key: str) -> dict | None: + name = Path(key).name + lower = name.lower() + if lower.startswith("_manifest") and lower.endswith(".json"): + return None + # RESULTADOS.csv, OMITIDOS.csv, ejecucion_bot.log (+ variantes _shard_N) + if lower in _REPORT_NAMES or lower.endswith(".csv") or lower.endswith(".log"): + return {"key": key, "name": name} + return None diff --git a/services/bot_digitador/infrastructure/store.py b/services/bot_digitador/infrastructure/store.py new file mode 100644 index 0000000..3d7dd74 --- /dev/null +++ b/services/bot_digitador/infrastructure/store.py @@ -0,0 +1,833 @@ +"""Repositorio sobre PostgreSQL (tablas digitador_* + sync de tasks).""" +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional + +from sqlalchemy import select + +from domain.models import ( + Job, + JobEventRecord, + JobStatus, + Worker, + WorkerStatus, + utc_now, +) +from infrastructure.database import ( + DigitadorJobEventRow, + DigitadorJobRow, + DigitadorWorkerRow, + Task, + _iso, + init_db, + session_scope, + utcnow, +) +from infrastructure.storage_minio import reports_prefix + +HEARTBEAT_OFFLINE_SECONDS = 45 + + +def _parse_iso(value: str | None) -> datetime | None: + if not value: + return None + dt = datetime.fromisoformat(value) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt + + +class JobStore: + """API de persistencia compatible con el patrón bot_descarga.""" + + def __init__(self, db_path: str | None = None): + # db_path ignorado (compat); usa DATABASE_URL. + init_db() + + def _row_worker(self, row: DigitadorWorkerRow) -> Worker: + worker = Worker( + id=row.id, + hostname=row.hostname or "", + max_concurrent=int(row.max_concurrent or 1), + status=row.status, + last_heartbeat=_iso(row.last_heartbeat) or utc_now(), + current_job_id=row.current_job_id, + ) + return self._refresh_worker_status(worker) + + def _row_job(self, row: DigitadorJobRow) -> Job: + return Job( + id=row.id, + worker_id=row.worker_id, + status=row.status, + title=row.title or "", + csv_key=row.csv_key or "", + reports_prefix=row.reports_prefix or "", + original_filename=row.original_filename or "", + cancel_requested=bool(row.cancel_requested), + result=dict(row.result or {}), + error=row.error, + created_at=_iso(row.created_at) or utc_now(), + started_at=_iso(row.started_at), + finished_at=_iso(row.finished_at), + job_id=row.job_id, + task_id=row.task_id, + parent_digitador_job_id=row.parent_digitador_job_id, + shard_index=row.shard_index, + shard_count=row.shard_count, + ) + + def _refresh_worker_status(self, worker: Worker) -> Worker: + try: + hb = datetime.fromisoformat(worker.last_heartbeat) + if hb.tzinfo is None: + hb = hb.replace(tzinfo=timezone.utc) + age = datetime.now(timezone.utc) - hb.astimezone(timezone.utc) + if age > timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS): + worker.status = WorkerStatus.OFFLINE.value + elif worker.current_job_id: + worker.status = WorkerStatus.BUSY.value + else: + worker.status = WorkerStatus.ONLINE.value + except Exception: + worker.status = WorkerStatus.OFFLINE.value + return worker + + def register_worker( + self, + worker_id: str, + hostname: str = "", + max_concurrent: int = 1, + ) -> Worker: + with session_scope() as session: + row = session.get(DigitadorWorkerRow, worker_id) + now = utcnow() + if row: + row.hostname = hostname + row.max_concurrent = max_concurrent + row.status = WorkerStatus.ONLINE.value + row.last_heartbeat = now + row.current_job_id = None + else: + row = DigitadorWorkerRow( + id=worker_id, + hostname=hostname, + max_concurrent=max_concurrent, + status=WorkerStatus.ONLINE.value, + last_heartbeat=now, + current_job_id=None, + ) + session.add(row) + session.flush() + return self._row_worker(row) + + def heartbeat(self, worker_id: str) -> Optional[Worker]: + with session_scope() as session: + row = session.get(DigitadorWorkerRow, worker_id) + if not row: + return None + row.last_heartbeat = utcnow() + row.status = ( + WorkerStatus.BUSY.value if row.current_job_id else WorkerStatus.ONLINE.value + ) + session.flush() + worker = self._row_worker(row) + try: + self.reclaim_stale_running_jobs() + except Exception: + pass + return worker + + def get_worker(self, worker_id: str) -> Optional[Worker]: + with session_scope() as session: + row = session.get(DigitadorWorkerRow, worker_id) + if not row: + return None + return self._row_worker(row) + + def list_workers(self) -> List[Worker]: + try: + self.reclaim_stale_running_jobs() + except Exception: + pass + with session_scope() as session: + rows = session.scalars( + select(DigitadorWorkerRow).order_by(DigitadorWorkerRow.id) + ).all() + return [self._row_worker(r) for r in rows] + + def list_online_worker_ids(self) -> List[str]: + return [ + w.id + for w in self.list_workers() + if w.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value) + ] + + def is_worker_online(self, worker_id: str) -> bool: + worker = self.get_worker(worker_id) + if not worker: + return False + return worker.status in (WorkerStatus.ONLINE.value, WorkerStatus.BUSY.value) + + def update_csv_key(self, job_id: str, csv_key: str) -> Optional[Job]: + with session_scope() as session: + row = session.get(DigitadorJobRow, job_id) + if not row: + return None + row.csv_key = csv_key + session.flush() + return self._row_job(row) + + def create_job( + self, + worker_id: Optional[str], + csv_key: str, + original_filename: str = "", + title: str = "", + *, + job_id: Optional[str] = None, + task_id: Optional[str] = None, + reports_prefix_override: Optional[str] = None, + parent_digitador_job_id: Optional[str] = None, + shard_index: Optional[int] = None, + shard_count: Optional[int] = None, + result: Optional[Dict[str, Any]] = None, + status: Optional[str] = None, + digitador_id: Optional[str] = None, + ) -> Job: + import uuid as uuid_mod + + digitador_id = digitador_id or str(uuid_mod.uuid4()) + prefix = reports_prefix_override or reports_prefix(digitador_id) + with session_scope() as session: + row = DigitadorJobRow( + id=digitador_id, + job_id=job_id, + task_id=task_id, + worker_id=worker_id, + parent_digitador_job_id=parent_digitador_job_id, + shard_index=shard_index, + shard_count=shard_count, + status=status or JobStatus.PENDING.value, + title=title or f"Digitador {original_filename}", + csv_key=csv_key, + reports_prefix=prefix, + original_filename=original_filename, + cancel_requested=False, + result=result or {}, + created_at=utcnow(), + ) + session.add(row) + session.flush() + return self._row_job(row) + + def get_job(self, job_id: str) -> Optional[Job]: + with session_scope() as session: + row = session.get(DigitadorJobRow, job_id) + if not row: + return None + return self._row_job(row) + + def list_jobs( + self, + limit: int = 100, + *, + job_id: Optional[str] = None, + parents_only: bool = True, + ) -> List[Job]: + with session_scope() as session: + stmt = select(DigitadorJobRow).order_by(DigitadorJobRow.created_at.desc()) + if job_id: + stmt = stmt.where(DigitadorJobRow.job_id == job_id) + if parents_only: + stmt = stmt.where(DigitadorJobRow.parent_digitador_job_id.is_(None)) + stmt = stmt.limit(limit) + rows = session.scalars(stmt).all() + return [self._row_job(r) for r in rows] + + def list_shards(self, parent_id: str) -> List[Job]: + with session_scope() as session: + rows = session.scalars( + select(DigitadorJobRow) + .where(DigitadorJobRow.parent_digitador_job_id == parent_id) + .order_by(DigitadorJobRow.shard_index.asc()) + ).all() + return [self._row_job(r) for r in rows] + + def claim_pending(self, worker_id: str) -> Optional[Job]: + try: + self.reclaim_stale_running_jobs() + except Exception: + pass + with session_scope() as session: + worker = session.get(DigitadorWorkerRow, worker_id) + if not worker or worker.current_job_id: + return None + + rows = session.scalars( + select(DigitadorJobRow) + .where( + DigitadorJobRow.worker_id == worker_id, + DigitadorJobRow.status == JobStatus.PENDING.value, + ) + .order_by(DigitadorJobRow.created_at.asc()) + ).all() + row = None + for candidate in rows: + is_parent = ( + candidate.parent_digitador_job_id is None + and candidate.shard_count is not None + and int(candidate.shard_count) > 1 + ) + if is_parent: + continue + row = candidate + break + if not row: + return None + + now = utcnow() + row.status = JobStatus.RUNNING.value + row.started_at = now + worker.current_job_id = row.id + worker.status = WorkerStatus.BUSY.value + worker.last_heartbeat = now + + task_id = row.task_id + if row.parent_digitador_job_id: + parent = session.get(DigitadorJobRow, row.parent_digitador_job_id) + if parent: + task_id = parent.task_id + if parent.status == JobStatus.PENDING.value: + parent.status = JobStatus.RUNNING.value + parent.started_at = parent.started_at or now + + if task_id: + task = session.get(Task, task_id) + if task and task.status == JobStatus.PENDING.value: + task.status = JobStatus.RUNNING.value + task.started_at = now + task.updated_at = now + + session.flush() + return self._row_job(row) + + def request_cancel(self, job_id: str) -> Optional[Job]: + with session_scope() as session: + row = session.get(DigitadorJobRow, job_id) + if not row: + return None + if row.status in ( + JobStatus.COMPLETED.value, + JobStatus.FAILED.value, + JobStatus.CANCELLED.value, + ): + return self._row_job(row) + + now = utcnow() + row.cancel_requested = True + if row.status == JobStatus.PENDING.value: + row.status = JobStatus.CANCELLED.value + row.finished_at = now + self._sync_task(session, row, status=JobStatus.CANCELLED.value) + + if row.parent_digitador_job_id is None and row.shard_count and int(row.shard_count) > 1: + children = session.scalars( + select(DigitadorJobRow).where( + DigitadorJobRow.parent_digitador_job_id == job_id + ) + ).all() + for child in children: + if child.status in ( + JobStatus.COMPLETED.value, + JobStatus.FAILED.value, + JobStatus.CANCELLED.value, + ): + continue + child.cancel_requested = True + if child.status == JobStatus.PENDING.value: + child.status = JobStatus.CANCELLED.value + child.finished_at = now + worker = ( + session.get(DigitadorWorkerRow, child.worker_id) + if child.worker_id + else None + ) + if worker and worker.current_job_id == child.id: + worker.current_job_id = None + worker.status = WorkerStatus.ONLINE.value + + session.flush() + return self._row_job(row) + + def is_cancel_requested(self, job_id: str) -> bool: + job = self.get_job(job_id) + if not job: + return False + if job.cancel_requested: + return True + if job.parent_digitador_job_id: + parent = self.get_job(job.parent_digitador_job_id) + if parent and parent.cancel_requested: + return True + return False + + def _sync_task( + self, + session, + row: DigitadorJobRow, + *, + status: Optional[str] = None, + result: Optional[Dict[str, Any]] = None, + error: Optional[str] = None, + ) -> None: + target = row + if row.parent_digitador_job_id: + parent = session.get(DigitadorJobRow, row.parent_digitador_job_id) + if parent: + target = parent + if not target.task_id: + return + task = session.get(Task, target.task_id) + if not task: + return + now = utcnow() + if status: + task.status = status + if result is not None: + task.result = { + **dict(task.result or {}), + "digitador_job_id": target.id, + **result, + } + if error is not None: + task.error = error + task.updated_at = now + if status in ( + JobStatus.COMPLETED.value, + JobStatus.FAILED.value, + JobStatus.CANCELLED.value, + ): + task.finished_at = now + + def patch_job( + self, + job_id: str, + *, + status: Optional[str] = None, + result: Optional[Dict[str, Any]] = None, + error: Optional[str] = None, + ) -> Optional[Job]: + parent_id: Optional[str] = None + with session_scope() as session: + row = session.get(DigitadorJobRow, job_id) + if not row: + return None + + new_status = status or row.status + if result is not None: + row.result = result + if error is not None: + row.error = error + row.status = new_status + + finished = False + if new_status in ( + JobStatus.COMPLETED.value, + JobStatus.FAILED.value, + JobStatus.CANCELLED.value, + ): + row.finished_at = row.finished_at or utcnow() + finished = True + + if not row.parent_digitador_job_id: + self._sync_task( + session, + row, + status=new_status if status else None, + result=result, + error=error, + ) + + if finished: + worker = session.get(DigitadorWorkerRow, row.worker_id) if row.worker_id else None + if worker and worker.current_job_id == job_id: + worker.current_job_id = None + worker.status = WorkerStatus.ONLINE.value + + parent_id = row.parent_digitador_job_id + session.flush() + job = self._row_job(row) + + if parent_id: + self.refresh_parent_from_shards(parent_id) + refreshed = self.get_job(job_id) + return refreshed or job + return job + + def refresh_parent_from_shards(self, parent_id: str) -> Optional[Job]: + with session_scope() as session: + parent = session.get(DigitadorJobRow, parent_id) + if not parent: + return None + children = list( + session.scalars( + select(DigitadorJobRow).where( + DigitadorJobRow.parent_digitador_job_id == parent_id + ) + ).all() + ) + if not children: + return self._row_job(parent) + + statuses = [c.status for c in children] + terminal = { + JobStatus.COMPLETED.value, + JobStatus.FAILED.value, + JobStatus.CANCELLED.value, + } + any_active = any( + s in (JobStatus.PENDING.value, JobStatus.RUNNING.value) for s in statuses + ) + all_terminal = all(s in terminal for s in statuses) + + total = 0 + ok = 0 + error_count = 0 + timeout = 0 + existentes = 0 + for c in children: + r = dict(c.result or {}) + total += int(r.get("total") or 0) + ok += int(r.get("ok") or 0) + error_count += int(r.get("error") or 0) + timeout += int(r.get("timeout") or 0) + existentes += int(r.get("existentes") or 0) + + processed = 0 + for c in children: + r = dict(c.result or {}) + if c.status in terminal: + processed += ( + int(r.get("ok") or 0) + + int(r.get("error") or 0) + + int(r.get("timeout") or 0) + + int(r.get("existentes") or 0) + ) + else: + processed += int(r.get("processed") or 0) + + shard_ids = [c.id for c in children] + parent_result = dict(parent.result or {}) + parent_result.update( + { + "mode": "distributed", + "shard_ids": shard_ids, + "total": total or int(parent_result.get("total_rows") or 0), + "total_rows": int(parent_result.get("total_rows") or total or 0), + "ok": ok, + "error": error_count, + "timeout": timeout, + "existentes": existentes, + "processed": processed, + "reports_prefix": parent.reports_prefix, + "shards": [ + { + "id": c.id, + "worker_id": c.worker_id, + "status": c.status, + "shard_index": c.shard_index, + "result": dict(c.result or {}), + "error": c.error, + } + for c in children + ], + } + ) + parent.result = parent_result + + now = utcnow() + if any_active: + parent.status = JobStatus.RUNNING.value + parent.started_at = parent.started_at or now + parent.finished_at = None + parent.error = None + self._sync_task( + session, + parent, + status=JobStatus.RUNNING.value, + result=parent_result, + ) + elif all_terminal: + any_failed = any(s == JobStatus.FAILED.value for s in statuses) + any_cancelled = any(s == JobStatus.CANCELLED.value for s in statuses) + all_cancelled = all(s == JobStatus.CANCELLED.value for s in statuses) + if all_cancelled: + parent.status = JobStatus.CANCELLED.value + elif any_failed: + parent.status = JobStatus.FAILED.value + parent.error = next( + (c.error for c in children if c.error), + "Uno o más shards fallaron", + ) + elif any_cancelled and not any_failed: + parent.status = JobStatus.COMPLETED.value + else: + parent.status = JobStatus.COMPLETED.value + parent.finished_at = parent.finished_at or now + self._sync_task( + session, + parent, + status=parent.status, + result=parent_result, + error=parent.error, + ) + + session.flush() + return self._row_job(parent) + + def reclaim_stale_running_jobs(self) -> int: + """Jobs running cuyo worker está offline → pending + reasignar a otro online.""" + reclaimed = 0 + with session_scope() as session: + now = utcnow() + cutoff = now - timedelta(seconds=HEARTBEAT_OFFLINE_SECONDS) + workers = list(session.scalars(select(DigitadorWorkerRow)).all()) + offline_ids: set[str] = set() + online_free: List[DigitadorWorkerRow] = [] + for w in workers: + hb = w.last_heartbeat + if hb is None: + offline_ids.add(w.id) + continue + if hb.tzinfo is None: + hb = hb.replace(tzinfo=timezone.utc) + if hb < cutoff: + offline_ids.add(w.id) + if w.current_job_id: + w.current_job_id = None + w.status = WorkerStatus.OFFLINE.value + elif not w.current_job_id: + online_free.append(w) + + if not offline_ids: + session.flush() + return 0 + + running = session.scalars( + select(DigitadorJobRow).where( + DigitadorJobRow.status == JobStatus.RUNNING.value, + DigitadorJobRow.worker_id.in_(list(offline_ids)), + ) + ).all() + + rr = 0 + for row in running: + is_parent = ( + row.parent_digitador_job_id is None + and row.shard_count is not None + and int(row.shard_count) > 1 + ) + if is_parent: + continue + + row.status = JobStatus.PENDING.value + row.started_at = None + row.cancel_requested = False + + if online_free: + target = online_free[rr % len(online_free)] + rr += 1 + row.worker_id = target.id + + reclaimed += 1 + session.add( + DigitadorJobEventRow( + digitador_job_id=row.id, + event_type="warning", + message=( + "Worker offline; job reclaimed" + + (f" → {row.worker_id}" if row.worker_id else "") + ), + timestamp=now, + metadata_json={"reclaimed": True, "worker_id": row.worker_id}, + ) + ) + if row.parent_digitador_job_id: + session.add( + DigitadorJobEventRow( + digitador_job_id=row.parent_digitador_job_id, + event_type="warning", + message=f"Shard {row.shard_index} reclamado (worker offline)", + timestamp=now, + metadata_json={ + "shard_id": row.id, + "worker_id": row.worker_id, + }, + ) + ) + + if online_free: + pending_orphan = session.scalars( + select(DigitadorJobRow).where( + DigitadorJobRow.status == JobStatus.PENDING.value, + DigitadorJobRow.worker_id.in_(list(offline_ids)), + ) + ).all() + for row in pending_orphan: + is_parent = ( + row.parent_digitador_job_id is None + and row.shard_count is not None + and int(row.shard_count) > 1 + ) + if is_parent: + continue + target = online_free[rr % len(online_free)] + rr += 1 + if row.worker_id != target.id: + row.worker_id = target.id + reclaimed += 1 + + session.flush() + + return reclaimed + + def add_event( + self, + job_id: str, + event_type: str, + message: str, + *, + radicado: Optional[str] = None, + nit: Optional[str] = None, + index: Optional[int] = None, + total: Optional[int] = None, + metadata: Optional[Dict[str, Any]] = None, + timestamp: Optional[str] = None, + ) -> JobEventRecord: + with session_scope() as session: + ts = _parse_iso(timestamp) or utcnow() + ev = DigitadorJobEventRow( + digitador_job_id=job_id, + event_type=event_type, + message=message, + timestamp=ts, + radicado=radicado, + nit=nit, + idx=index, + total=total, + metadata_json=metadata or {}, + ) + session.add(ev) + session.flush() + row = session.get(DigitadorJobRow, job_id) + if row and row.parent_digitador_job_id: + session.add( + DigitadorJobEventRow( + digitador_job_id=row.parent_digitador_job_id, + event_type=event_type, + message=message, + timestamp=ts, + radicado=radicado, + nit=nit, + idx=index, + total=total, + metadata_json={ + **(metadata or {}), + "shard_id": job_id, + "shard_index": row.shard_index, + "worker_id": row.worker_id, + }, + ) + ) + session.flush() + return JobEventRecord( + id=int(ev.id), + job_id=job_id, + event_type=event_type, + message=message, + timestamp=_iso(ts) or utc_now(), + radicado=radicado, + nit=nit, + index=index, + total=total, + metadata=metadata or {}, + ) + + def list_events(self, job_id: str, since_id: int = 0) -> List[JobEventRecord]: + with session_scope() as session: + rows = session.scalars( + select(DigitadorJobEventRow) + .where( + DigitadorJobEventRow.digitador_job_id == job_id, + DigitadorJobEventRow.id > since_id, + ) + .order_by(DigitadorJobEventRow.id.asc()) + ).all() + return [ + JobEventRecord( + id=int(r.id), + job_id=r.digitador_job_id, + event_type=r.event_type, + message=r.message, + timestamp=_iso(r.timestamp) or utc_now(), + radicado=r.radicado, + nit=r.nit, + index=r.idx, + total=r.total, + metadata=dict(r.metadata_json or {}), + ) + for r in rows + ] + + def create_digitador_task( + self, + *, + parent_job_id: str, + digitador_job_id: str, + worker_id: Optional[str], + title: str, + original_filename: str, + worker_ids: Optional[List[str]] = None, + mode: str = "single", + source_type: Optional[str] = None, + ) -> str: + """Crea Task(type=digitador) y la asocia al digitador_job. Devuelve task_id.""" + with session_scope() as session: + now = utcnow() + request: Dict[str, Any] = { + "worker_id": worker_id, + "worker_ids": worker_ids or ([worker_id] if worker_id else []), + "original_filename": original_filename, + "digitador_job_id": digitador_job_id, + "mode": mode, + } + result: Dict[str, Any] = {"digitador_job_id": digitador_job_id, "mode": mode} + if source_type: + request["source_type"] = source_type + result["source_type"] = source_type + task = Task( + job_id=parent_job_id, + type="digitador", + status=JobStatus.PENDING.value, + title=title or f"Bot digitador ({original_filename})", + request=request, + result=result, + created_at=now, + updated_at=now, + ) + session.add(task) + session.flush() + row = session.get(DigitadorJobRow, digitador_job_id) + if row: + row.task_id = task.id + row.job_id = parent_job_id + session.flush() + return task.id + + def parent_job_exists(self, parent_job_id: str) -> bool: + from infrastructure.database import Job as IpsJob + + with session_scope() as session: + return session.get(IpsJob, parent_job_id) is not None diff --git a/services/bot_digitador/requirements.txt b/services/bot_digitador/requirements.txt new file mode 100644 index 0000000..7b13fff --- /dev/null +++ b/services/bot_digitador/requirements.txt @@ -0,0 +1,10 @@ +fastapi>=0.110.0 +uvicorn[standard]>=0.27.0 +python-multipart>=0.0.9 +boto3>=1.34.0 +httpx>=0.27.0 +openpyxl>=3.1.0 +pydantic>=2.0.0 +sqlalchemy>=2.0.0 +psycopg[binary]>=3.1.0 +pytest>=7.0.0 diff --git a/services/bot_digitador/tests/test_csv_split.py b/services/bot_digitador/tests/test_csv_split.py new file mode 100644 index 0000000..4a5b989 --- /dev/null +++ b/services/bot_digitador/tests/test_csv_split.py @@ -0,0 +1,96 @@ +"""Tests de parseo/partición CSV para digitador.""" +from __future__ import annotations + +import pytest + +from domain.csv_split import ( + REQUIRED_BOT_COLUMNS, + parse_digitador_rows, + rows_to_csv_bytes, + split_rows, +) + + +def _it_csv_bytes() -> bytes: + header = "|".join( + [ + "RADICADO", + "NUMERO", + "FECHA", + "BRUTO", + "AUTORIZACION", + "TIPO_ID", + "SERVICIO", + "COPAGO", + "MODERADORA", + ] + ) + row = "|".join( + ["R-1", "123", "2024-01-01", "1000", "A-9", "CC", "AMB", "0", "4500"] + ) + return f"{header}\n{row}\n".encode("utf-8-sig") + + +def _snake_csv_bytes() -> bytes: + header = "|".join(REQUIRED_BOT_COLUMNS) + row = "|".join( + ["R-2", "999", "2024-02-02", "2000", "A-1", "TI", "HOS", "100", "0"] + ) + return f"{header}\n{row}\n".encode("utf-8") + + +def test_it_to_bot_mapping(): + rows = parse_digitador_rows(_it_csv_bytes(), "aptos.csv") + assert len(rows) == 1 + assert rows[0]["radicado"] == "R-1" + assert rows[0]["documento"] == "123" + assert rows[0]["ingreso"] == "2024-01-01" + assert rows[0]["total_cargos"] == "1000" + assert rows[0]["autorizacion"] == "A-9" + assert rows[0]["tipo_documento"] == "CC" + assert rows[0]["tipo_servicio"] == "AMB" + assert rows[0]["copago"] == "0" + assert rows[0]["cuota_moderadora"] == "4500" + + +def test_snake_case_headers(): + rows = parse_digitador_rows(_snake_csv_bytes(), "input.csv") + assert len(rows) == 1 + assert rows[0]["radicado"] == "R-2" + assert rows[0]["documento"] == "999" + + +def test_missing_columns_raises(): + bad = b"RADICADO|NUMERO\nR1|1\n" + with pytest.raises(ValueError) as exc: + parse_digitador_rows(bad, "bad.csv") + assert "Columnas faltantes" in str(exc.value) + + +def test_split_rows_balanced(): + rows = [{"radicado": str(i)} for i in range(100)] + chunks = split_rows(rows, 2) + assert len(chunks) == 2 + assert len(chunks[0]) == 50 + assert len(chunks[1]) == 50 + + +def test_split_rows_uneven(): + rows = [{"radicado": str(i)} for i in range(10)] + chunks = split_rows(rows, 3) + assert len(chunks) == 3 + assert sum(len(c) for c in chunks) == 10 + assert [len(c) for c in chunks] == [4, 3, 3] + + +def test_roundtrip_csv(): + rows = parse_digitador_rows(_it_csv_bytes(), "aptos.csv") + data = rows_to_csv_bytes(rows) + parsed = parse_digitador_rows(data, "shard.csv") + assert parsed == rows + + +def test_split_more_workers_than_rows(): + rows = [{"radicado": "A"}, {"radicado": "B"}] + chunks = split_rows(rows, 5) + assert len(chunks) == 2 diff --git a/services/cleaning/domain/csv_cleaning/apply.py b/services/cleaning/domain/csv_cleaning/apply.py index 2c1f861..de697e0 100644 --- a/services/cleaning/domain/csv_cleaning/apply.py +++ b/services/cleaning/domain/csv_cleaning/apply.py @@ -49,6 +49,11 @@ def apply_rules( if col not in result.columns: log.append(f"[WARN] Columna no encontrada: {col}") return + # Eliminar columna: no tiene sentido transformar y luego borrar. + if "drop_column" in rule_ids: + result.drop(columns=[col], inplace=True) + log.append(f"[OK] '{col}' <- {get_rule_name('drop_column')}") + return split_has_homologation = ( "doc_id_split_tipo_numero" in rule_ids and "doc_type_homologate" in rule_ids ) diff --git a/services/cleaning/domain/csv_cleaning/rules/builtins.py b/services/cleaning/domain/csv_cleaning/rules/builtins.py index 8e0bbd9..cfd2ef7 100644 --- a/services/cleaning/domain/csv_cleaning/rules/builtins.py +++ b/services/cleaning/domain/csv_cleaning/rules/builtins.py @@ -140,5 +140,14 @@ register_rule( ), fn=special._rule_passthrough_series, ) +register_rule( + "drop_column", + name="Eliminar columna", + description=( + "Quita por completo esta columna del resultado limpio. " + "Útil para campos que no se necesitan en el export o en pasos posteriores." + ), + fn=special._rule_passthrough_series, +) rebuild_cleaning_rules() diff --git a/services/cleaning/domain/csv_cleaning/type_inference.py b/services/cleaning/domain/csv_cleaning/type_inference.py index 4cfdaa6..d61867e 100644 --- a/services/cleaning/domain/csv_cleaning/type_inference.py +++ b/services/cleaning/domain/csv_cleaning/type_inference.py @@ -99,7 +99,12 @@ def _infer_column_type( def _applicable_rules_for_type(inferred_type: str) -> list[str]: """Identificadores de reglas a mostrar según el tipo inferido (sin redundancias).""" - base = ["strip_special_spaces", "strip_special_keep_date_seps", "strip_spaces"] + base = [ + "strip_special_spaces", + "strip_special_keep_date_seps", + "strip_spaces", + "drop_column", + ] if inferred_type == TYPE_INTEGER: return base + ["cop_mod_split", "cop_mod_bruto_neto"] if inferred_type == TYPE_DECIMAL: diff --git a/services/db/alembic/versions/0007_download_job_shards.py b/services/db/alembic/versions/0007_download_job_shards.py new file mode 100644 index 0000000..94059e2 --- /dev/null +++ b/services/db/alembic/versions/0007_download_job_shards.py @@ -0,0 +1,83 @@ +"""Download jobs: parent/shard fields for distributed bots. + +Revision ID: 0007 +Revises: 0006 +Create Date: 2026-08-09 + +""" +from __future__ import annotations + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0007" +down_revision: Union[str, None] = "0006" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + conn = op.get_bind() + inspector = sa.inspect(conn) + if not inspector.has_table("download_jobs"): + return + cols = {c["name"] for c in inspector.get_columns("download_jobs")} + + with op.batch_alter_table("download_jobs") as batch: + if "parent_download_job_id" not in cols: + batch.add_column( + sa.Column( + "parent_download_job_id", + sa.String(36), + sa.ForeignKey("download_jobs.id"), + nullable=True, + ) + ) + if "shard_index" not in cols: + batch.add_column(sa.Column("shard_index", sa.Integer(), nullable=True)) + if "shard_count" not in cols: + batch.add_column(sa.Column("shard_count", sa.Integer(), nullable=True)) + + # Padre orquestador no tiene worker asignado + op.alter_column( + "download_jobs", + "worker_id", + existing_type=sa.String(36), + nullable=True, + ) + + existing_indexes = {ix["name"] for ix in inspector.get_indexes("download_jobs")} + if "ix_download_jobs_parent" not in existing_indexes: + op.create_index( + "ix_download_jobs_parent", + "download_jobs", + ["parent_download_job_id"], + ) + + +def downgrade() -> None: + conn = op.get_bind() + inspector = sa.inspect(conn) + if not inspector.has_table("download_jobs"): + return + existing_indexes = {ix["name"] for ix in inspector.get_indexes("download_jobs")} + if "ix_download_jobs_parent" in existing_indexes: + op.drop_index("ix_download_jobs_parent", table_name="download_jobs") + + cols = {c["name"] for c in inspector.get_columns("download_jobs")} + with op.batch_alter_table("download_jobs") as batch: + if "shard_count" in cols: + batch.drop_column("shard_count") + if "shard_index" in cols: + batch.drop_column("shard_index") + if "parent_download_job_id" in cols: + batch.drop_column("parent_download_job_id") + + op.alter_column( + "download_jobs", + "worker_id", + existing_type=sa.String(36), + nullable=False, + ) diff --git a/services/db/alembic/versions/0008_bot_digitador_tables.py b/services/db/alembic/versions/0008_bot_digitador_tables.py new file mode 100644 index 0000000..d20629e --- /dev/null +++ b/services/db/alembic/versions/0008_bot_digitador_tables.py @@ -0,0 +1,152 @@ +"""Bot digitador: digitador_workers, digitador_jobs, digitador_job_events. + +Revision ID: 0008 +Revises: 0007 +Create Date: 2026-08-09 + +""" +from __future__ import annotations + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0008" +down_revision: Union[str, None] = "0007" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _table_exists(conn: sa.engine.Connection, table: str) -> bool: + return sa.inspect(conn).has_table(table) + + +def upgrade() -> None: + conn = op.get_bind() + json_type = ( + postgresql.JSONB() + if conn.dialect.name == "postgresql" + else sa.JSON() + ) + + if not _table_exists(conn, "digitador_workers"): + op.create_table( + "digitador_workers", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("hostname", sa.Text(), nullable=False, server_default=""), + sa.Column("max_concurrent", sa.Integer(), nullable=False, server_default="1"), + sa.Column("status", sa.String(32), nullable=False), + sa.Column("last_heartbeat", sa.DateTime(timezone=True), nullable=False), + sa.Column("current_job_id", sa.String(36), nullable=True), + ) + op.create_index("ix_digitador_workers_status", "digitador_workers", ["status"]) + + if not _table_exists(conn, "digitador_jobs"): + op.create_table( + "digitador_jobs", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True), + sa.Column("task_id", sa.String(36), sa.ForeignKey("tasks.id"), nullable=True), + sa.Column("worker_id", sa.String(36), nullable=True), + sa.Column( + "parent_digitador_job_id", + sa.String(36), + sa.ForeignKey("digitador_jobs.id"), + nullable=True, + ), + sa.Column("shard_index", sa.Integer(), nullable=True), + sa.Column("shard_count", sa.Integer(), nullable=True), + sa.Column("status", sa.String(32), nullable=False, server_default="pending"), + sa.Column("title", sa.Text(), nullable=True), + sa.Column("csv_key", sa.Text(), nullable=False, server_default=""), + sa.Column("reports_prefix", sa.Text(), nullable=False, server_default=""), + sa.Column("original_filename", sa.Text(), nullable=False, server_default=""), + sa.Column( + "cancel_requested", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + sa.Column( + "result", + json_type, + nullable=False, + server_default=( + sa.text("'{}'::jsonb") + if conn.dialect.name == "postgresql" + else sa.text("'{}'") + ), + ), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index("ix_digitador_jobs_job_id", "digitador_jobs", ["job_id"]) + op.create_index("ix_digitador_jobs_task_id", "digitador_jobs", ["task_id"]) + op.create_index("ix_digitador_jobs_worker_id", "digitador_jobs", ["worker_id"]) + op.create_index("ix_digitador_jobs_status", "digitador_jobs", ["status"]) + op.create_index( + "ix_digitador_jobs_parent", + "digitador_jobs", + ["parent_digitador_job_id"], + ) + op.create_index( + "ix_digitador_jobs_worker_status", + "digitador_jobs", + ["worker_id", "status"], + ) + + if not _table_exists(conn, "digitador_job_events"): + op.create_table( + "digitador_job_events", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column( + "digitador_job_id", + sa.String(36), + sa.ForeignKey("digitador_jobs.id"), + nullable=False, + ), + sa.Column("event_type", sa.String(64), nullable=False), + sa.Column("message", sa.Text(), nullable=False), + sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False), + sa.Column("radicado", sa.String(255), nullable=True), + sa.Column("nit", sa.String(64), nullable=True), + sa.Column("idx", sa.Integer(), nullable=True), + sa.Column("total", sa.Integer(), nullable=True), + sa.Column( + "metadata", + json_type, + nullable=False, + server_default=( + sa.text("'{}'::jsonb") + if conn.dialect.name == "postgresql" + else sa.text("'{}'") + ), + ), + ) + op.create_index( + "ix_digitador_job_events_job_id", + "digitador_job_events", + ["digitador_job_id", "id"], + ) + + +def downgrade() -> None: + conn = op.get_bind() + if _table_exists(conn, "digitador_job_events"): + op.drop_index("ix_digitador_job_events_job_id", table_name="digitador_job_events") + op.drop_table("digitador_job_events") + if _table_exists(conn, "digitador_jobs"): + op.drop_index("ix_digitador_jobs_worker_status", table_name="digitador_jobs") + op.drop_index("ix_digitador_jobs_parent", table_name="digitador_jobs") + op.drop_index("ix_digitador_jobs_status", table_name="digitador_jobs") + op.drop_index("ix_digitador_jobs_worker_id", table_name="digitador_jobs") + op.drop_index("ix_digitador_jobs_task_id", table_name="digitador_jobs") + op.drop_index("ix_digitador_jobs_job_id", table_name="digitador_jobs") + op.drop_table("digitador_jobs") + if _table_exists(conn, "digitador_workers"): + op.drop_index("ix_digitador_workers_status", table_name="digitador_workers") + op.drop_table("digitador_workers") diff --git a/services/desktop_app/src/renderer/App.tsx b/services/desktop_app/src/renderer/App.tsx index 01d9cef..4f09ca2 100644 --- a/services/desktop_app/src/renderer/App.tsx +++ b/services/desktop_app/src/renderer/App.tsx @@ -5,6 +5,7 @@ import { Cpu, Download, Hash, + Keyboard, Link2, Package, Settings, @@ -35,6 +36,7 @@ import "./modules/extraction/ExtractionModule"; import "./modules/cleaning/CleaningModule"; import "./modules/poblacion_apta/PoblacionAptaModule"; import "./modules/downloads/DownloadsModule"; +import "./modules/digitador/DigitadorModule"; const BOT_PACKAGE_URL = "http://s3.builker.xyz/apps-versions/BotDescargaPYSCuentasMedicas/latest/bot_descarga.zip"; @@ -403,6 +405,14 @@ function App() { Población apta + + + ); + } + + return ( +
+
+
+ + + {selected && ( + + {selected.status} + + )} +
+
+ + +
+
+ + {error && ( +
+ + {error} +
+ )} + + {selected?.error && ( +
+ + {selected.error} +
+ )} + + {dlStatus && ( +
+ + {dlStatus} +
+ )} + + {selected && ( + <> +
+
+ {exitosos} + OK +
+
+ {fallidos} + Fallidos +
+
+ {existentes} + Existentes +
+
+ {timeouts} + Timeouts +
+
+ {total || "—"} + Total +
+
+ +

+ {selected.shard_count && Number(selected.shard_count) > 1 + ? `${selected.shard_count} bots (distribuido)` + : ( + <> + Worker {selected.worker_id ?? "—"} + + )} + {selected.original_filename ? ` · ${selected.original_filename}` : ""} + {selected.reports_prefix ? ( + <> + {" "} + · Prefijo {selected.reports_prefix} + + ) : null} +

+ + {(() => { + const shardList = Array.isArray(selected.shards) + ? selected.shards + : Array.isArray(result.shards) + ? (result.shards as Array<{ + id?: string; + worker_id?: string | null; + status?: string; + shard_index?: number | null; + }>) + : []; + if (shardList.length === 0) return null; + return ( +
+

Estado por bot

+
+ {shardList.map((s, i) => ( +
+
+ + Shard {(s.shard_index ?? i) + 1} + + {s.worker_id || "—"} +
+
+ Estado + {s.status || "—"} +
+
+ ))} +
+
+ ); + })()} + +
+
+

+ Reportes en MinIO + {files.length > 0 ? ` · ${files.length} archivo${files.length === 1 ? "" : "s"}` : ""} +

+
+ + {filesError && ( +
+ + No se pudo listar MinIO: {filesError} +
+ )} + + {!filesError && files.length === 0 ? ( +

Sin reportes listados aún (RESULTADOS / OMITIDOS).

+ ) : ( + + + + + + + + + {files.map((f) => { + const fileBusy = dlBusyKey === `file:${f.key}`; + const name = reportBasename(f); + return ( + + + + + ); + })} + +
Archivo
+ {name} + + +
+ )} +
+ +
+

Log

+
+ {events.length === 0 ? ( + Sin eventos. + ) : ( + events.map((ev) => ( +
+ [{ev.timestamp}] {ev.message} +
+ )) + )} +
+
+ + )} +
+ ); +} diff --git a/services/desktop_app/src/renderer/modules/digitador/DigitadorWizardModule.tsx b/services/desktop_app/src/renderer/modules/digitador/DigitadorWizardModule.tsx new file mode 100644 index 0000000..6364be8 --- /dev/null +++ b/services/desktop_app/src/renderer/modules/digitador/DigitadorWizardModule.tsx @@ -0,0 +1,760 @@ +import { + AlertCircle, + Bot, + CheckCircle2, + FileSpreadsheet, + Keyboard, + Loader2, + Play, + RefreshCw, + Square, + Upload, + Users, +} from "lucide-react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import type { ApiClient } from "../../lib/apiClient"; +import type { + DigitadorJob, + DigitadorJobEvent, + DigitadorWorker, + Job, + PoblacionAptaJob, +} from "../../../shared/types"; + +export type DigitadorSourceMode = "aptos" | "archivo"; + +export type DigitadorInitialSource = { + poblacionAptaJobId: string; +}; + +interface Props { + client: ApiClient; + job: Job; + initialSource?: DigitadorInitialSource | null; +} + +type DigitadorStep = 1 | 2; + +type ShardInfo = { + id?: string; + worker_id?: string | null; + status?: string; + shard_index?: number | null; + result?: Record; + error?: string | null; +}; + +export function DigitadorWizardModule({ client, job, initialSource = null }: Props) { + const [step, setStep] = useState(1); + const [workers, setWorkers] = useState([]); + const [selectedIds, setSelectedIds] = useState([]); + const [useAllOnline, setUseAllOnline] = useState(true); + const [sourceMode, setSourceMode] = useState( + initialSource?.poblacionAptaJobId ? "aptos" : "aptos", + ); + const [paJobs, setPaJobs] = useState([]); + const [selectedPaId, setSelectedPaId] = useState(initialSource?.poblacionAptaJobId ?? ""); + const [loadingPa, setLoadingPa] = useState(false); + const [file, setFile] = useState(null); + const [title, setTitle] = useState(""); + const [busy, setBusy] = useState(false); + const [refreshing, setRefreshing] = useState(false); + const [error, setError] = useState(null); + const [activeJob, setActiveJob] = useState(null); + const [events, setEvents] = useState([]); + const sinceRef = useRef(0); + const pollRef = useRef | null>(null); + const fileInputRef = useRef(null); + const initializedSelectRef = useRef(false); + const appliedInitialRef = useRef(null); + + const onlineWorkers = useMemo( + () => workers.filter((w) => w.status === "online" || w.status === "busy"), + [workers], + ); + + useEffect(() => { + if (!initialSource?.poblacionAptaJobId) return; + const key = initialSource.poblacionAptaJobId; + if (appliedInitialRef.current === key) return; + appliedInitialRef.current = key; + setSourceMode("aptos"); + setSelectedPaId(initialSource.poblacionAptaJobId); + setStep(1); + }, [initialSource?.poblacionAptaJobId]); + + const refreshWorkers = useCallback(async () => { + setRefreshing(true); + try { + const list = await client.listDigitadorWorkers(); + setWorkers(list); + setError(null); + const online = list.filter((w) => w.status === "online" || w.status === "busy"); + if (!initializedSelectRef.current && online.length > 0) { + initializedSelectRef.current = true; + setUseAllOnline(online.length >= 2); + setSelectedIds(online.map((w) => w.id)); + } else if (useAllOnline) { + setSelectedIds(online.map((w) => w.id)); + } else { + setSelectedIds((prev) => prev.filter((id) => online.some((w) => w.id === id))); + } + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } finally { + setRefreshing(false); + } + }, [client, useAllOnline]); + + useEffect(() => { + void refreshWorkers(); + const t = setInterval(() => void refreshWorkers(), 5000); + return () => clearInterval(t); + }, [refreshWorkers]); + + const loadPaJobs = useCallback(async () => { + setLoadingPa(true); + try { + const list = await client.poblacionAptaJobs(job.id); + const completed = list + .filter((j) => j.status === "completed") + .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); + setPaJobs(completed); + setSelectedPaId((prev) => { + if (prev && completed.some((j) => j.id === prev)) return prev; + return completed[0]?.id ?? ""; + }); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } finally { + setLoadingPa(false); + } + }, [client, job.id]); + + useEffect(() => { + if (sourceMode !== "aptos") return; + void loadPaJobs(); + }, [sourceMode, loadPaJobs]); + + useEffect(() => { + if (!activeJob || ["completed", "failed", "cancelled"].includes(activeJob.status)) { + if (pollRef.current) { + clearInterval(pollRef.current); + pollRef.current = null; + } + return; + } + + async function poll() { + if (!activeJob) return; + try { + const nextJob = await client.getDigitadorJob(activeJob.id); + setActiveJob(nextJob); + const { events: next } = await client.getDigitadorEvents(activeJob.id, sinceRef.current); + if (next.length) { + setEvents((prev) => [...prev, ...next]); + sinceRef.current = next[next.length - 1].id; + } + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } + } + + void poll(); + pollRef.current = setInterval(() => void poll(), 3000); + return () => { + if (pollRef.current) clearInterval(pollRef.current); + }; + }, [activeJob?.id, activeJob?.status, client]); + + function toggleWorker(id: string) { + if (useAllOnline) return; + setSelectedIds((prev) => + prev.includes(id) ? prev.filter((x) => x !== id) : [...prev, id], + ); + } + + const selectedPa = paJobs.find((j) => j.id === selectedPaId) ?? null; + const aptosCount = Number(selectedPa?.summary?.paso_bot ?? 0); + + async function startJob() { + const targets = useAllOnline ? onlineWorkers.map((w) => w.id) : selectedIds; + if (targets.length === 0) { + setError("Necesitas al menos un bot online."); + return; + } + if (sourceMode === "archivo") { + if (!file) { + setError("Selecciona un CSV."); + return; + } + } else { + if (!selectedPaId) { + setError("Selecciona una corrida de población apta."); + return; + } + if (aptosCount <= 0) { + setError("Esa corrida no tiene aptos (PASO_BOT) para digitar."); + return; + } + } + + setBusy(true); + setError(null); + try { + const form = new FormData(); + form.append("job_id", job.id); + if (sourceMode === "archivo" && file) { + form.append("file", file); + } else { + form.append("poblacion_apta_job_id", selectedPaId); + form.append("subset", "aptos"); + } + if (title.trim()) form.append("title", title.trim()); + if (targets.length === 1) { + form.append("worker_id", targets[0]); + } else { + form.append("worker_ids", JSON.stringify(targets)); + } + const created = await client.createDigitadorJob(form); + sinceRef.current = 0; + setEvents([]); + setActiveJob(created); + setStep(2); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } finally { + setBusy(false); + } + } + + async function cancelJob() { + if (!activeJob) return; + setBusy(true); + try { + const updated = await client.cancelDigitadorJob(activeJob.id); + setActiveJob(updated); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } finally { + setBusy(false); + } + } + + function resetWizard() { + setStep(1); + setActiveJob(null); + setEvents([]); + setFile(null); + setTitle(""); + setError(null); + sinceRef.current = 0; + if (fileInputRef.current) fileInputRef.current.value = ""; + } + + const result = activeJob?.result ?? {}; + const processed = Number(result.processed ?? result.exitosos ?? 0); + const total = Number(result.total ?? result.total_rows ?? 0); + const pct = total > 0 ? Math.min(100, Math.round((processed / total) * 100)) : 0; + const running = Boolean( + activeJob && !["completed", "failed", "cancelled"].includes(activeJob.status), + ); + const isDistributed = + Boolean(activeJob?.shard_count && Number(activeJob.shard_count) > 1) || + result.mode === "distributed"; + const shards: ShardInfo[] = Array.isArray(activeJob?.shards) + ? (activeJob!.shards as ShardInfo[]) + : Array.isArray(result.shards) + ? (result.shards as ShardInfo[]) + : []; + const effectiveTargets = useAllOnline ? onlineWorkers.map((w) => w.id) : selectedIds; + const sourceReady = + sourceMode === "archivo" ? Boolean(file) : Boolean(selectedPaId) && aptosCount > 0; + const canStart = effectiveTargets.length > 0 && sourceReady && !busy; + + return ( +
+ + +
+ {step === 1 && ( +
+
+

+ + Bot digitador +

+

+ Digita en Integración Total solo los aptos (PASO_BOT), o un CSV + propio con el mismo formato. +

+
+ + {/* 1. Origen */} +
+

1. Origen

+
+ + +
+ + + setTitle(e.target.value)} + placeholder="Ej. Digitación lote mayo 2026" + style={{ width: "100%", marginBottom: 14 }} + /> + + {sourceMode === "aptos" ? ( + <> +
+ + +
+ + {loadingPa && paJobs.length === 0 ? ( +

+ Cargando corridas… +

+ ) : paJobs.length === 0 ? ( +
+

+ No hay corridas de población apta completadas en este trabajo. +

+

+ Ejecuta primero Población apta, o cambia a «Subir CSV». +

+
+ ) : ( + <> + + + {selectedPa && ( +
+ {aptosCount > 0 ? ( + + ) : ( + + )} +
+
+ {aptosCount > 0 + ? `${aptosCount} registros aptos listos para digitar` + : "Sin aptos en esta corrida"} +
+
+ Solo se envían filas PASO_BOT. Los no aptos no se digitan. +
+
+
+ )} + + )} + + ) : ( + <> + +
+
+ {file ? ( + <> + + {file.name} + + ) : ( + Ningún archivo seleccionado + )} +
+ + setFile(e.target.files?.[0] ?? null)} + /> +
+

+ Columnas: radicado, documento, ingreso, total_cargos, autorizacion, + tipo_documento, tipo_servicio, copago, cuota_moderadora (también acepta + headers IT). +

+ + )} +
+ + {/* 2. Bots */} +
+
+

+ 2. Bots + {onlineWorkers.length > 0 && ( + {onlineWorkers.length} online + )} +

+ +
+

+ El agent debe estar encendido en la PC con Integración Total. Con varios bots, las + filas se reparte automáticamente. +

+ + {onlineWorkers.length >= 2 && ( + + )} + + {workers.length === 0 ? ( +
+
+ +
+
Ningún bot conectado
+

+ En la PC del digitador ejecuta run_gui.py del agent y conéctalo + al gateway. +

+
+
+
+ ) : ( +
+ {workers.map((w) => { + const online = w.status === "online" || w.status === "busy"; + const selected = effectiveTargets.includes(w.id); + return ( + + ); + })} +
+ )} + {onlineWorkers.length === 0 && workers.length > 0 && ( +

+ Hay workers registrados pero ninguno online. Revisa el agent. +

+ )} + {effectiveTargets.length > 1 && ( +

+ Se repartirá entre {effectiveTargets.length} bots. +

+ )} +
+
+ )} + + {step === 2 && activeJob && ( +
+
+
+ {running ? ( + + ) : activeJob.status === "completed" ? ( + + ) : activeJob.status === "failed" ? ( + + ) : ( + + )} +
+

+ {running + ? isDistributed + ? "Digitación distribuida en curso" + : "Digitación en curso" + : activeJob.status === "completed" + ? "Digitación completada" + : activeJob.status === "cancelled" + ? "Digitación cancelada" + : activeJob.status === "failed" + ? "Digitación fallida" + : "Digitación finalizada"} +

+

+ {isDistributed + ? `${Number(activeJob.shard_count) || shards.length || "?"} bots` + : `Bot ${activeJob.worker_id ?? "—"}`} + {activeJob.original_filename ? ` · ${activeJob.original_filename}` : ""} +

+
+ + {activeJob.error && ( +
+ + {activeJob.error} +
+ )} + +
+
+ + {Number(result.exitosos ?? processed)}/{total || "?"} filas + + {pct}% +
+
+
+
+
+ +
+
+ {String(result.exitosos ?? processed)} + OK +
+
+ {String(result.fallidos ?? 0)} + Fallidos +
+
+ {String(result.existentes ?? 0)} + Existentes +
+
+ {String(result.timeouts ?? 0)} + Timeouts +
+
+ + {shards.length > 0 && ( +
+

Shards por bot

+
+ {shards.map((s, i) => ( +
+
+ + Shard {(s.shard_index ?? i) + 1} + + {s.worker_id || "—"} +
+
+ Estado + {s.status || "—"} +
+
+ ))} +
+
+ )} + +
+

Log del agent

+
+ {events.length === 0 ? ( + Esperando eventos… + ) : ( + events.map((ev) => ( +
+ [{ev.timestamp}] {ev.message} +
+ )) + )} +
+
+
+ )} +
+ + {error && ( +
+ + {error} +
+ )} + +
+ {step === 2 ? ( + + ) : ( + + )} + +
+
+ + {step === 1 && ( + + )} + {step === 2 && running && ( + + )} + {step === 2 && !running && ( + + )} +
+
+ ); +} diff --git a/services/desktop_app/src/renderer/modules/poblacion_apta/PoblacionAptaResults.tsx b/services/desktop_app/src/renderer/modules/poblacion_apta/PoblacionAptaResults.tsx index 380c24c..9a14cc9 100644 --- a/services/desktop_app/src/renderer/modules/poblacion_apta/PoblacionAptaResults.tsx +++ b/services/desktop_app/src/renderer/modules/poblacion_apta/PoblacionAptaResults.tsx @@ -1,4 +1,4 @@ -import { Download, Loader2, RefreshCw, Users } from "lucide-react"; +import { Download, Keyboard, Loader2, RefreshCw, Users } from "lucide-react"; import { useEffect, useState } from "react"; import type { PoblacionAptaJob } from "../../../shared/types"; import { PoblacionAptaResultsTable } from "../../components/PoblacionAptaResultsTable"; @@ -7,7 +7,7 @@ import type { ModuleResultsProps } from "../registry"; type Row = Record; -export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps) { +export function PoblacionAptaResults({ client, task, job, onActivateModule }: ModuleResultsProps) { const [paJob, setPaJob] = useState(null); const [rows, setRows] = useState([]); const [loading, setLoading] = useState(true); @@ -60,6 +60,16 @@ export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps) downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text); } + async function exportTodos() { + const paJobId = String( + (task.result as Record)?.poblacion_apta_job_id ?? "", + ); + if (!paJobId) return; + const blob = await client.poblacionAptaExportCsv(paJobId, "todos"); + const text = await blob.text(); + downloadCsv(buildExportCsvFilename("poblacion_apta_todos", job.title), text); + } + if (loading) { return (
@@ -88,6 +98,10 @@ export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps) } const summary = paJob?.summary ?? {}; + const paJobId = String( + (task.result as Record)?.poblacion_apta_job_id ?? paJob?.id ?? "", + ); + const canDigitador = paJob?.status === "completed" && Boolean(paJobId) && Boolean(onActivateModule); return (
@@ -108,6 +122,23 @@ export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps) + + {canDigitador && ( + + )}
diff --git a/services/desktop_app/src/renderer/modules/poblacion_apta/PoblacionAptaWizard.tsx b/services/desktop_app/src/renderer/modules/poblacion_apta/PoblacionAptaWizard.tsx index 3063eb8..a918170 100644 --- a/services/desktop_app/src/renderer/modules/poblacion_apta/PoblacionAptaWizard.tsx +++ b/services/desktop_app/src/renderer/modules/poblacion_apta/PoblacionAptaWizard.tsx @@ -3,6 +3,7 @@ import { CheckSquare, Download, FileSpreadsheet, + Keyboard, Loader2, Square, Upload, @@ -235,6 +236,13 @@ export function PoblacionAptaWizard({ downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text); } + async function exportTodos() { + if (!resultJob) return; + const blob = await client.poblacionAptaExportCsv(resultJob.id, "todos"); + const text = await blob.text(); + downloadCsv(buildExportCsvFilename("poblacion_apta_todos", job.title), text); + } + function reset() { setDone(false); setResultJob(null); @@ -271,6 +279,23 @@ export function PoblacionAptaWizard({ + + {onActivateModule && ( + + )} diff --git a/services/desktop_app/src/renderer/types/jobSection.ts b/services/desktop_app/src/renderer/types/jobSection.ts index 2116738..2aed4da 100644 --- a/services/desktop_app/src/renderer/types/jobSection.ts +++ b/services/desktop_app/src/renderer/types/jobSection.ts @@ -4,4 +4,5 @@ export type JobSection = | "cleaning" | "poblacion_apta" | "downloads" + | "digitador" | "results"; diff --git a/services/desktop_app/src/renderer/views/JobResultsView.tsx b/services/desktop_app/src/renderer/views/JobResultsView.tsx index d93516c..60b5aaf 100644 --- a/services/desktop_app/src/renderer/views/JobResultsView.tsx +++ b/services/desktop_app/src/renderer/views/JobResultsView.tsx @@ -5,6 +5,7 @@ import { Cpu, Download, Hash, + Keyboard, Loader2, Plus, RefreshCw, @@ -29,8 +30,9 @@ import type { ApiClient } from "../lib/apiClient"; import { downloadCsv, toCsv, buildExportCsvFilename } from "../lib/csv"; import { taskTimingLabel, timingDuration } from "../lib/time"; import { DownloadsResultsPanel } from "../modules/downloads/DownloadsResultsPanel"; +import { DigitadorResultsPanel } from "../modules/digitador/DigitadorResultsPanel"; -type ResultTab = "counting" | "extraction" | "cleaning" | "poblacion_apta" | "downloads"; +type ResultTab = "counting" | "extraction" | "cleaning" | "poblacion_apta" | "downloads" | "digitador"; interface Props { client: ApiClient; @@ -46,11 +48,15 @@ interface Props { onStartCleaning: (ctx?: Record) => void; onStartPoblacionApta: () => void; onStartDownload: () => void; + onStartDigitador: () => void; onContinueToCounting?: (ctx: { downloadJobId: string; downloadsPrefix?: string; files?: DownloadJobFile[]; }) => void; + onContinueToDigitador?: (ctx: { + poblacionAptaJobId: string; + }) => void; } export function JobResultsView({ @@ -67,7 +73,9 @@ export function JobResultsView({ onStartCleaning, onStartPoblacionApta, onStartDownload, + onStartDigitador, onContinueToCounting, + onContinueToDigitador, }: Props) { const [activeTab, setActiveTab] = useState("counting"); const [refreshing, setRefreshing] = useState(false); @@ -82,6 +90,8 @@ export function JobResultsView({ .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); const downloadTasks = tasks.filter((t) => t.type === "downloads") .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); + const digitadorTasks = tasks.filter((t) => t.type === "digitador") + .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); async function refresh() { setRefreshing(true); @@ -164,6 +174,17 @@ export function JobResultsView({ {downloadTasks.length} )} +
); @@ -780,12 +810,16 @@ function PoblacionAptaResultsSection({ tasks, liveProgress, onStartPoblacionApta, + onContinueToDigitador, }: { client: ApiClient; job: Job; tasks: Task[]; liveProgress?: BackgroundProgress | null; onStartPoblacionApta: () => void; + onContinueToDigitador?: (ctx: { + poblacionAptaJobId: string; + }) => void; }) { const [selectedTask, setSelectedTask] = useState(tasks[0] ?? null); const [paJob, setPaJob] = useState(null); @@ -853,6 +887,16 @@ function PoblacionAptaResultsSection({ downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text); } + async function exportTodos() { + const paJobId = String( + (selectedTask?.result as Record)?.poblacion_apta_job_id ?? "", + ); + if (!paJobId) return; + const blob = await client.poblacionAptaExportCsv(paJobId, "todos"); + const text = await blob.text(); + downloadCsv(buildExportCsvFilename("poblacion_apta_todos", job.title), text); + } + if (tasks.length === 0 && !liveProgress?.running) { return (
@@ -874,6 +918,13 @@ function PoblacionAptaResultsSection({ } const summary = paJob?.summary ?? {}; + const paJobId = String( + (selectedTask?.result as Record)?.poblacion_apta_job_id ?? paJob?.id ?? "", + ); + const canDigitador = + Boolean(onContinueToDigitador) && + paJob?.status === "completed" && + Boolean(paJobId); return (
@@ -917,6 +968,21 @@ function PoblacionAptaResultsSection({ + + {canDigitador && ( + + )} diff --git a/services/desktop_app/src/renderer/views/JobView.tsx b/services/desktop_app/src/renderer/views/JobView.tsx index 44a9fb6..09c7663 100644 --- a/services/desktop_app/src/renderer/views/JobView.tsx +++ b/services/desktop_app/src/renderer/views/JobView.tsx @@ -9,6 +9,7 @@ import { CleaningWizardModule } from "../modules/cleaning/CleaningWizardModule"; import { PoblacionAptaWizardModule } from "../modules/poblacion_apta/PoblacionAptaWizardModule"; import { JobExtractionWizard } from "../modules/extraction/JobExtractionWizard"; import { DownloadsWizardModule } from "../modules/downloads/DownloadsWizardModule"; +import { DigitadorWizardModule } from "../modules/digitador/DigitadorWizardModule"; import { JobResultsView } from "./JobResultsView"; interface Props { @@ -127,6 +128,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio {section === "cleaning" && "Limpieza de datos"} {section === "poblacion_apta" && "Población apta"} {section === "downloads" && "Bot de descarga"} + {section === "digitador" && "Bot digitador"} {section === "results" && "Resultados del trabajo"}
@@ -201,6 +203,19 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
+
+ +
+
) => onActivateSection("cleaning", ctx)} onStartPoblacionApta={() => onActivateSection("poblacion_apta")} onStartDownload={() => onActivateSection("downloads")} + onStartDigitador={() => onActivateSection("digitador")} onContinueToCounting={(ctx) => onActivateSection("counting", ctx)} + onContinueToDigitador={(ctx) => onActivateSection("digitador", ctx)} />
diff --git a/services/desktop_app/src/shared/types.ts b/services/desktop_app/src/shared/types.ts index d41f1f0..ddf5516 100644 --- a/services/desktop_app/src/shared/types.ts +++ b/services/desktop_app/src/shared/types.ts @@ -373,7 +373,7 @@ export interface DownloadWorker { export interface DownloadJob { id: string; - worker_id: string; + worker_id: string | null; status: "pending" | "running" | "completed" | "failed" | "cancelled" | string; title: string; excel_key: string; @@ -389,6 +389,11 @@ export interface DownloadJob { job_id?: string | null; /** Task type=downloads */ task_id?: string | null; + parent_download_job_id?: string | null; + shard_index?: number | null; + shard_count?: number | null; + /** Presente en GET de padre distribuido */ + shards?: DownloadJob[]; } export interface DownloadJobFile { @@ -417,6 +422,63 @@ export interface DownloadJobEvent { metadata?: Record; } +/** Worker del bot digitador (control remoto). */ +export interface DigitadorWorker { + id: string; + hostname: string; + max_concurrent: number; + status: "online" | "offline" | "busy" | string; + last_heartbeat: string; + current_job_id: string | null; +} + +export interface DigitadorJob { + id: string; + worker_id: string | null; + status: "pending" | "running" | "completed" | "failed" | "cancelled" | string; + title: string; + csv_key: string; + reports_prefix: string; + original_filename: string; + cancel_requested: boolean; + result: Record; + error: string | null; + created_at: string; + started_at: string | null; + finished_at: string | null; + job_id?: string | null; + task_id?: string | null; + parent_digitador_job_id?: string | null; + shard_index?: number | null; + shard_count?: number | null; + shards?: DigitadorJob[]; +} + +export interface DigitadorJobFile { + key: string; + name?: string | null; +} + +export interface DigitadorJobFilesResponse { + job_id: string; + prefix: string; + files: DigitadorJobFile[]; + count: number; +} + +export interface DigitadorJobEvent { + id: number; + job_id: string; + event_type: string; + message: string; + timestamp: string; + radicado?: string | null; + nit?: string | null; + index?: number | null; + total?: number | null; + metadata?: Record; +} + export interface DesktopBridge { getRuntimeConfig(): Promise; selectDirectory(): Promise; diff --git a/services/poblacion_apta/api/main.py b/services/poblacion_apta/api/main.py index 8b00363..4bcce38 100644 --- a/services/poblacion_apta/api/main.py +++ b/services/poblacion_apta/api/main.py @@ -15,6 +15,7 @@ from domain.verificador_st import ( consolidado_to_records, export_no_aptos_csv, export_paso_bot_csv, + export_todos_csv, process_cleaning_sources, ) from domain.verificador_st.core import cargar_origen_servicio_desde_bytes @@ -483,11 +484,13 @@ def get_rows( @app.get("/jobs/{job_id}/export.csv") def export_csv( job_id: str, - subset: str = Query("aptos", description="aptos | no_aptos"), + subset: str = Query("aptos", description="aptos | no_aptos | todos"), ) -> Response: subset_norm = (subset or "aptos").strip().lower() - if subset_norm not in {"aptos", "no_aptos", "errores"}: - raise HTTPException(status_code=400, detail="subset debe ser aptos o no_aptos") + if subset_norm in {"all", "completo", "consolidado"}: + subset_norm = "todos" + if subset_norm not in {"aptos", "no_aptos", "errores", "todos"}: + raise HTTPException(status_code=400, detail="subset debe ser aptos, no_aptos o todos") if subset_norm == "errores": subset_norm = "no_aptos" @@ -504,6 +507,9 @@ def export_csv( if subset_norm == "aptos": csv_text = export_paso_bot_csv(payload) filename = f"poblacion_apta_aptos_{job_id[:8]}.csv" + elif subset_norm == "todos": + csv_text = export_todos_csv(payload) + filename = f"poblacion_apta_todos_{job_id[:8]}.csv" else: csv_text = export_no_aptos_csv(payload) filename = f"poblacion_apta_no_aptos_{job_id[:8]}.csv" diff --git a/services/poblacion_apta/domain/verificador_st/__init__.py b/services/poblacion_apta/domain/verificador_st/__init__.py index e9e7da9..578e288 100644 --- a/services/poblacion_apta/domain/verificador_st/__init__.py +++ b/services/poblacion_apta/domain/verificador_st/__init__.py @@ -3,6 +3,7 @@ from domain.verificador_st.pipeline import ( consolidado_to_records, export_no_aptos_csv, export_paso_bot_csv, + export_todos_csv, process_cleaning_sources, ) @@ -10,5 +11,6 @@ __all__ = [ "consolidado_to_records", "export_no_aptos_csv", "export_paso_bot_csv", + "export_todos_csv", "process_cleaning_sources", ] diff --git a/services/poblacion_apta/domain/verificador_st/core.py b/services/poblacion_apta/domain/verificador_st/core.py index bc76df4..e68b0e2 100644 --- a/services/poblacion_apta/domain/verificador_st/core.py +++ b/services/poblacion_apta/domain/verificador_st/core.py @@ -51,6 +51,8 @@ TABLA_MODERADORA = { 2025: {4700, 19200, 50300}, 2026: {5000, 20100, 52800}, } +# Todos los montos de la tabla; la clasificación no filtra por año. +VALORES_MODERADORA = {v for vals in TABLA_MODERADORA.values() for v in vals} # Longitudes según tabla entregada por el usuario. # Los tipos marcados como alfanuméricos permiten letras y números. @@ -554,14 +556,15 @@ def normalizar_autorizacion(valor): def separar_copago_moderadora(valor, anio): """ - Si el valor coincide con tabla moderadora del año => cuota_moderadora. - Si no coincide => copago. + Si el valor coincide con cualquier cuota de la tabla (sin importar el año) + => cuota_moderadora. Si no coincide => copago. + El parámetro anio se conserva por compatibilidad de firma. """ monto = to_int(valor, default=0) if monto == 0: return 0, 0 - if anio in TABLA_MODERADORA and monto in TABLA_MODERADORA[anio]: + if monto in VALORES_MODERADORA: return 0, monto return monto, 0 diff --git a/services/poblacion_apta/domain/verificador_st/pipeline.py b/services/poblacion_apta/domain/verificador_st/pipeline.py index 5129c99..60bed73 100644 --- a/services/poblacion_apta/domain/verificador_st/pipeline.py +++ b/services/poblacion_apta/domain/verificador_st/pipeline.py @@ -223,3 +223,21 @@ def export_no_aptos_csv(rows: list[dict[str, Any]]) -> str: df = df.rename(columns=CSV_FINAL_IT_RENAME) df = df.reindex(columns=headers) return df.to_csv(sep="|", index=False, encoding="utf-8-sig") + + +def export_todos_csv(rows: list[dict[str, Any]]) -> str: + """CSV pipe-separated con todas las filas (aptos y no aptos) e OBSERVACION.""" + headers = [*CSV_FINAL_IT_HEADERS, "OBSERVACION"] + if not rows: + df = pd.DataFrame(columns=headers) + else: + df = pd.DataFrame(rows) + for c in COLUMNAS_FINAL_IT: + if c not in df.columns: + df[c] = "" + if "OBSERVACION" not in df.columns: + df["OBSERVACION"] = "" + df = df.reindex(columns=[*COLUMNAS_FINAL_IT, "OBSERVACION"]).fillna("") + df = df.rename(columns=CSV_FINAL_IT_RENAME) + df = df.reindex(columns=headers) + return df.to_csv(sep="|", index=False, encoding="utf-8-sig")