add logger for more strucutred logging in python

crawler/logger.py

@@ -1,10 +1,49 @@
+from functools import lru_cache
 import logging
+import sys
+from pathlib import Path
 
 
-def createLogger(name):
-    logging.basicConfig(
-        level=logging.INFO,
-        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-        handlers=[logging.FileHandler("app.log"), logging.StreamHandler()],
+@lru_cache(maxsize=None)
+def get_logger(
+    name: str,
+    log_file: str = "app.log",
+    console_level: str = "INFO",
+    file_level: str = "DEBUG",
+):
+    """
+    Configure a logger with console and file handlers.
+
+    Args:
+        name: Logger name (usually __name__).
+        log_file: Path to the log file.
+        console_level: Console logging level (INFO, DEBUG, etc.).
+        file_level: File logging level.
+    """
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.DEBUG)  # Set to lowest level (handlers filter further)
+
+    # Clear existing handlers (avoid duplicates in Jupyter/reloads)
+    logger.handlers.clear()
+
+    # ---- Console Handler ----
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(console_level.upper())
+    console_format = logging.Formatter(
+        "[%(asctime)s] %(levelname)s in %(module)s:%(lineno)d - %(message)s"
+        # "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
     )
-    return logging.getLogger(name)
+    console_handler.setFormatter(console_format)
+    logger.addHandler(console_handler)
+
+    # ---- File Handler ----
+    # Path(log_file).parent.mkdir(parents=True, exist_ok=True)  # Ensure dir exists
+    # file_handler = logging.FileHandler(log_file)
+    # file_handler.setLevel(file_level.upper())
+    # file_format = logging.Formatter(
+    #     "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"
+    # )
+    # file_handler.setFormatter(file_format)
+    # logger.addHandler(file_handler) # skip files for now
+
+    return logger