replace pymysql with mysqlclient as it is "better"; also fix an issue in the ui exporter that had wrong imports

crawler/api/app.py

@@ -18,6 +18,15 @@ from fastapi.middleware.cors import CORSMiddleware
 
 from tasks import listing_tasks
 from ui_exporter import export_immoweb
+from alembic import command
+from alembic.config import Config
+from contextlib import asynccontextmanager
+from celery.exceptions import TaskRevokedError
+from celery_app import app as celery_app
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from api.metrics import metrics_app  # Import the Prometheus ASGI app
+from opentelemetry.metrics import get_meter
+
 
 load_dotenv()
 logger = logging.getLogger("uvicorn")
@@ -35,6 +44,16 @@ logger = logging.getLogger("uvicorn")
 
 # app = FastAPI(lifespan=lifespan)
 app = FastAPI()
+app.mount("/metrics", metrics_app)
+meter = get_meter(__name__)
+request_counter = meter.create_counter(
+    name="custom_request_count",
+    description="Number of times /hello was called",
+)
+hist = meter.create_histogram(
+    name="custom_request_duration",
+    description="Duration of /hello requests in seconds",
+)
 
 
 # Allow CORS (for React frontend)
@@ -48,6 +67,8 @@ app.add_middleware(
 
 @app.get("/api/status")
 async def get_status():
+    request_counter.add(1, {"method": "GET", "path": "/status"})
+    hist.record(1.5, {"method": "GET", "path": "/status"})
     return {"status": "OK"}
 
 
@@ -125,3 +146,6 @@ async def get_districts(
     user: Annotated[User, Depends(get_current_user)],
 ) -> dict[str, str]:
     return districts.get_districts()
+
+
+FastAPIInstrumentor.instrument_app(app)

crawler/poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand.
+# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand.
 
 [[package]]
 name = "aiohappyeyeballs"
@@ -2709,6 +2709,24 @@ files = [
     {file = "multidict-6.5.0.tar.gz", hash = "sha256:942bd8002492ba819426a8d7aefde3189c1b87099cdf18aaaefefcf7f3f7b6d2"},
 ]
 
+[[package]]
+name = "mysqlclient"
+version = "2.2.7"
+description = "Python interface to MySQL"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "mysqlclient-2.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:2e3c11f7625029d7276ca506f8960a7fd3c5a0a0122c9e7404e6a8fe961b3d22"},
+    {file = "mysqlclient-2.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:a22d99d26baf4af68ebef430e3131bb5a9b722b79a9fcfac6d9bbf8a88800687"},
+    {file = "mysqlclient-2.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:4b4c0200890837fc64014cc938ef2273252ab544c1b12a6c1d674c23943f3f2e"},
+    {file = "mysqlclient-2.2.7-cp313-cp313-win_amd64.whl", hash = "sha256:201a6faa301011dd07bca6b651fe5aaa546d7c9a5426835a06c3172e1056a3c5"},
+    {file = "mysqlclient-2.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:199dab53a224357dd0cb4d78ca0e54018f9cee9bf9ec68d72db50e0a23569076"},
+    {file = "mysqlclient-2.2.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:92af368ed9c9144737af569c86d3b6c74a012a6f6b792eb868384787b52bb585"},
+    {file = "mysqlclient-2.2.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:977e35244fe6ef44124e9a1c2d1554728a7b76695598e4b92b37dc2130503069"},
+    {file = "mysqlclient-2.2.7.tar.gz", hash = "sha256:24ae22b59416d5fcce7e99c9d37548350b4565baac82f95e149cac6ce4163845"},
+]
+
 [[package]]
 name = "nbclient"
 version = "0.10.2"
@@ -3728,22 +3746,6 @@ dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pyte
 docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"]
 tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"]
 
-[[package]]
-name = "pymysql"
-version = "1.1.2"
-description = "Pure Python MySQL Driver"
-optional = false
-python-versions = ">=3.8"
-groups = ["main"]
-files = [
-    {file = "pymysql-1.1.2-py3-none-any.whl", hash = "sha256:e6b1d89711dd51f8f74b1631fe08f039e7d76cf67a42a323d3178f0f25762ed9"},
-    {file = "pymysql-1.1.2.tar.gz", hash = "sha256:4961d3e165614ae65014e361811a724e2044ad3ea3739de9903ae7c21f539f03"},
-]
-
-[package.extras]
-ed25519 = ["PyNaCl (>=1.4.0)"]
-rsa = ["cryptography"]
-
 [[package]]
 name = "pyparsing"
 version = "3.2.3"
@@ -5703,4 +5705,4 @@ type = ["pytest-mypy"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">3.11"
-content-hash = "55c0df08e3ac4416b1c435d2a107d9289cb2fe20a628598a2dec3f838f1a3320"
+content-hash = "5f53cec7fc3cc93d494341e9fd6562076c1a8952f83075f671a3507c50fcb334"

crawler/pyproject.toml

@@ -36,7 +36,7 @@ opentelemetry-sdk = "^1.36.0"
 opentelemetry-exporter-prometheus = "^0.57b0"
 opentelemetry-instrumentation-fastapi = "^0.57b0"
 opentelemetry-instrumentation-sqlalchemy = "^0.57b0"
-pymysql = "^1.1.2"
+mysqlclient = "^2.2.7"
 
 [tool.poetry.group.dev.dependencies]
 ipdb = "^0.13.13"

crawler/ui_exporter.py

@@ -2,7 +2,7 @@ import json
 import logging
 import pathlib
 
-from rec.query import QueryParameters
+from models.listing import QueryParameters
 from repositories.listing_repository import ListingRepository
 
 logger = logging.getLogger("uvicorn.error")