Add intelligent query splitting to maximize Rightmove data extraction

crawler/pyproject.toml

@@ -20,6 +20,7 @@ matplotlib = "^3.10.0"
 opencv-python = "^4.11.0.86"
 click = "^8.2.0"
 aiohttp = "^3.11.18"
+aiohttp-socks = "^0.8.4"
 sqlmodel = "^0.0.24"
 alembic = "^1.16.1"
 sqlalchemy = {extras = ["asyncio"], version = "^2.0.41"}
@@ -42,6 +43,15 @@ mysqlclient = "^2.2.7"
 ipdb = "^0.13.13"
 jupyterlab = "^4.4.7"
 podman-compose = "^1.5.0"
+pytest = "^8.0.0"
+pytest-asyncio = "^0.23.0"
+pytest-cov = "^4.1.0"
+httpx = "^0.27.0"
+aioresponses = "^0.7.6"
+fakeredis = "^2.21.0"
+mypy = "^1.8.0"
+types-requests = "^2.31.0"
+types-redis = "^4.6.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
@@ -52,4 +62,23 @@ build-backend = "poetry.core.masonry.api"
 lint.ignore = [
     "E741", # Ambigious name
 ]
-exclude = ["*.ipynb"]
+exclude = ["*.ipynb"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+asyncio_default_fixture_loop_scope = "function"
+
+[tool.mypy]
+python_version = "3.11"
+warn_return_any = true
+warn_unused_ignores = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+strict_optional = true
+plugins = ["pydantic.mypy"]
+
+[[tool.mypy.overrides]]
+module = ["transformers.*", "pytesseract.*", "cv2.*", "celery.*", "tqdm.*", "aiohttp.*", "aiohttp_socks.*", "tenacity.*", "geopy.*", "pandas.*", "numpy.*", "PIL.*", "sqlmodel.*", "sqlalchemy.*", "alembic.*", "apprise.*", "opentelemetry.*"]
+ignore_missing_imports = true