add logger for more strucutred logging in python

crawler/api/app.py

@@ -15,6 +15,7 @@ from api.worker import (
 )
 from fastapi import Depends, FastAPI, HTTPException, Query
 from api.auth import User
+from logger import get_logger
 from models.listing import QueryParameters
 from repositories.listing_repository import ListingRepository
 from repositories.listing_repository import ListingRepository
@@ -25,15 +26,17 @@ from alembic import command
 from alembic.config import Config
 from contextlib import asynccontextmanager
 
+logger = get_logger(__file__)
+
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     alembic_cfg = Config("./alembic.ini")
-    print("Running alembic migrations")
+    logger.info("Running alembic migrations")
     command.upgrade(alembic_cfg, "head")
-    print("Finished running alembic migrations")
+    logger.info("Finished running alembic migrations")
     yield
-    print("Shutting down")
+    logger.warning("Shutting down")
 
 
 app = FastAPI(lifespan=lifespan)
@@ -55,6 +58,7 @@ app.add_middleware(
 async def get_listing(user: Annotated[User, Depends(get_current_user)]):
     repository = ListingRepository(engine)
     listings = await repository.get_listings(limit=5)
+    logger.info(f"Fetched {len(listings)} listings")
     return {"listings": listings}
 
 

crawler/logger.py

@@ -1,10 +1,49 @@
+from functools import lru_cache
 import logging
+import sys
+from pathlib import Path
 
 
-def createLogger(name):
+@lru_cache(maxsize=None)
-    logging.basicConfig(
+def get_logger(
-        level=logging.INFO,
+    name: str,
-        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    log_file: str = "app.log",
-        handlers=[logging.FileHandler("app.log"), logging.StreamHandler()],
+    console_level: str = "INFO",
+    file_level: str = "DEBUG",
+):
+    """
+    Configure a logger with console and file handlers.
+
+    Args:
+        name: Logger name (usually __name__).
+        log_file: Path to the log file.
+        console_level: Console logging level (INFO, DEBUG, etc.).
+        file_level: File logging level.
+    """
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.DEBUG)  # Set to lowest level (handlers filter further)
+
+    # Clear existing handlers (avoid duplicates in Jupyter/reloads)
+    logger.handlers.clear()
+
+    # ---- Console Handler ----
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(console_level.upper())
+    console_format = logging.Formatter(
+        "[%(asctime)s] %(levelname)s in %(module)s:%(lineno)d - %(message)s"
+        # "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
     )
-    return logging.getLogger(name)
+    console_handler.setFormatter(console_format)
+    logger.addHandler(console_handler)
+
+    # ---- File Handler ----
+    # Path(log_file).parent.mkdir(parents=True, exist_ok=True)  # Ensure dir exists
+    # file_handler = logging.FileHandler(log_file)
+    # file_handler.setLevel(file_level.upper())
+    # file_format = logging.Formatter(
+    #     "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"
+    # )
+    # file_handler.setFormatter(file_format)
+    # logger.addHandler(file_handler) # skip files for now
+
+    return logger