Fix deps and move to a better local environment

- Cleaned up some deps and moved them to the dev section
- Moved from mysqlclient to pymysql which is a python native one which does not require the OS to have the correct mysql lib
- Added a podman compose file so we can have all dependencies in one place easily without the need to install redis or a database locally

For podman install
- podman
- podman-compose (do a poetry sync I think?)
- podman-compose up to start the containers

crawler/podman-compose.yml

@@ -0,0 +1,24 @@
+version: "3.8"
+
+services:
+  redis:
+    image: redis:8
+    container_name: redis-container
+    ports:
+      - "6379:6379"
+    volumes:
+      - ./data/redis:/data
+    command: ["redis-server", "--appendonly", "yes"]
+
+  mysql:
+    image: mysql:9
+    container_name: mysql-container
+    ports:
+      - "3306:3306"
+    environment:
+      MYSQL_ROOT_PASSWORD: wtfviktordidyoubuildsomuch
+      MYSQL_DATABASE: wrongmove
+      MYSQL_USER: wrongmoveuser
+      MYSQL_PASSWORD: wrongmovepass
+    volumes:
+      - ./data/mysql:/var/lib/mysql

crawler/pyproject.toml

@@ -14,7 +14,6 @@ pillow = "^10.2.0"
 numpy = "^1.26.4"
 transformers = "^4.38.2"
 pytesseract = "^0.3.10"
-jupyterlab = "^4.1.4"
 pandas = "^2.2.1"
 geopy = "^2.4.1"
 matplotlib = "^3.10.0"
@@ -28,7 +27,6 @@ tenacity = "^9.1.2"
 fastapi = {extras = ["standard"], version = "^0.115.12"}
 pyjwt = "^2.10.1"
 cryptography = "^45.0.4"
-mysqlclient = "^2.2.7"
 celery = "^5.5.3"
 redis = "^6.2.0"
 watchdog = "^6.0.0"
@@ -38,9 +36,12 @@ opentelemetry-sdk = "^1.36.0"
 opentelemetry-exporter-prometheus = "^0.57b0"
 opentelemetry-instrumentation-fastapi = "^0.57b0"
 opentelemetry-instrumentation-sqlalchemy = "^0.57b0"
+pymysql = "^1.1.2"
 
 [tool.poetry.group.dev.dependencies]
 ipdb = "^0.13.13"
+jupyterlab = "^4.4.7"
+podman-compose = "^1.5.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]