viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

migrate processing to a pipeline approach where each listing is processed in a pipeline in parallel and status reported back to track progress

Browse source
This commit is contained in:
Viktor Barzin 2025-07-27 18:33:39 +00:00
parent 4fa09e31c8
commit 91a0436f7f
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
6 changed files with 347 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crawler/3_dump_images.py
View file

@ -27,7 +27,7 @@ async def dump_images(
@retry(wait=wait_random(min=1, max=2), stop=stop_after_attempt(3))
async def dump_images_for_listing(listing: Listing, base_path: Path) -> Listing | None:
all_floorplans = listing.additional_info["property"]["floorplans"]
all_floorplans = listing.additional_info.get("property", {}).get("floorplans", [])
for floorplan in all_floorplans:
url = floorplan["url"]
picname = url.split("/")[-1]