wrongmove/crawler/pyproject.toml

[tool.poetry]
name = "rec"
version = "0.1.0"
description = ""
authors = ["Kadir Tugan <git@k8n.dev>"]

[tool.poetry.dependencies]
python = ">3.11"
requests = "^2.31.0"
cachetools = "^5.3.2"
diskcache = "^5.6.3"
tqdm = "^4.66.2"
pillow = "^10.2.0"
numpy = "^1.26.4"
transformers = "^4.38.2"
pytesseract = "^0.3.10"
jupyterlab = "^4.1.4"
pandas = "^2.2.1"
geopy = "^2.4.1"
matplotlib = "^3.10.0"
opencv-python = "^4.11.0.86"
click = "^8.2.0"
aiohttp = "^3.11.18"
sqlmodel = "^0.0.24"
alembic = "^1.16.1"
sqlalchemy = {extras = ["asyncio"], version = "^2.0.41"}
tenacity = "^9.1.2"

[tool.poetry.group.dev.dependencies]
ipdb = "^0.13.13"

[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
Initial commit: Rightmove fetch, database, caching, poetry etc. - Fetching rightmove listing api - Memoizing query - Writing to sqlite database with sqlalchemy - Poetry dependencies 2023-11-06 00:31:58 +00:00			`[tool.poetry]`
updating poetry 2023-11-18 12:15:34 +02:00			`name = "rec"`
Initial commit: Rightmove fetch, database, caching, poetry etc. - Fetching rightmove listing api - Memoizing query - Writing to sqlite database with sqlalchemy - Poetry dependencies 2023-11-06 00:31:58 +00:00			`version = "0.1.0"`
			`description = ""`
			`authors = ["Kadir Tugan <git@k8n.dev>"]`

			`[tool.poetry.dependencies]`
Real crawling scripts and floorplan detection 1. get all listings 2. get all detail jsons 3. get all images 4. get all floorplans 5. detecting floorplans Also updating dependencies for huggingface etc. 2024-03-10 18:49:39 +00:00			`python = ">3.11"`
Initial commit: Rightmove fetch, database, caching, poetry etc. - Fetching rightmove listing api - Memoizing query - Writing to sqlite database with sqlalchemy - Poetry dependencies 2023-11-06 00:31:58 +00:00			`requests = "^2.31.0"`
			`cachetools = "^5.3.2"`
			`diskcache = "^5.6.3"`
Real crawling scripts and floorplan detection 1. get all listings 2. get all detail jsons 3. get all images 4. get all floorplans 5. detecting floorplans Also updating dependencies for huggingface etc. 2024-03-10 18:49:39 +00:00			`tqdm = "^4.66.2"`
			`pillow = "^10.2.0"`
			`numpy = "^1.26.4"`
			`transformers = "^4.38.2"`
adding tesseract OCR for floorplan detection 2024-03-10 22:32:34 +00:00			`pytesseract = "^0.3.10"`
fixing bugs, adding properties for querying and analysis§ 2024-03-11 09:44:37 +00:00			`jupyterlab = "^4.1.4"`
			`pandas = "^2.2.1"`
Reduced need for routing by limiting to a radius of 7 miles 2024-04-01 15:21:31 +02:00			`geopy = "^2.4.1"`
adding new dependencies 2025-03-02 17:11:48 +00:00			`matplotlib = "^3.10.0"`
			`opencv-python = "^4.11.0.86"`
[1/n] click-ify - add entrypoint for click script and add 1_dump_listings command run via: poetry run python main.py --step dump_listings 2025-05-11 18:59:41 +00:00			`click = "^8.2.0"`
use aiohttp to fetch details concurrently 2025-05-17 22:34:27 +00:00			`aiohttp = "^3.11.18"`
add sqlmodel + alembic + setup models skeleton to slowly enable transition towards a db 2025-06-03 20:00:30 +00:00			`sqlmodel = "^0.0.24"`
			`alembic = "^1.16.1"`
			`sqlalchemy = {extras = ["asyncio"], version = "^2.0.41"}`
add tenacity to retry transient blockouts by rightmove 2025-06-08 20:59:04 +00:00			`tenacity = "^9.1.2"`
Initial commit: Rightmove fetch, database, caching, poetry etc. - Fetching rightmove listing api - Memoizing query - Writing to sqlite database with sqlalchemy - Poetry dependencies 2023-11-06 00:31:58 +00:00
add ipdb to the dev deps 2025-05-17 22:11:51 +00:00			`[tool.poetry.group.dev.dependencies]`
			`ipdb = "^0.13.13"`

Initial commit: Rightmove fetch, database, caching, poetry etc. - Fetching rightmove listing api - Memoizing query - Writing to sqlite database with sqlalchemy - Poetry dependencies 2023-11-06 00:31:58 +00:00			`[build-system]`
			`requires = ["poetry-core>=1.0.0"]`
			`build-backend = "poetry.core.masonry.api"`