viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

migrate dump images command to use model listings

Browse source
This commit is contained in:
Viktor Barzin 2025-06-07 13:56:00 +00:00
parent 4f5a934fa9
commit ba87d07cd2
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
6 changed files with 99 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

41
crawler/3_dump_images.py
View file

@ -1,36 +1,45 @@
import asyncio
import json
from pathlib import Path
import aiohttp
from repositories import ListingRepository
from tqdm.asyncio import tqdm
from data_access import Listing
# from data_access import Listing
from models import Listing
# Setting this too high either crashes rightmove or gets us blocked
semaphore = asyncio.Semaphore(10)
async def dump_images(listing_paths: list[str]):
listings = Listing.get_all_listings(listing_paths)
await tqdm.gather(*[dump_images_for_listing(listing) for listing in listings])
async def dump_images(repository: ListingRepository, image_base_path: Path):
listings = await repository.get_listings()
updated_listings = await tqdm.gather(
*[dump_images_for_listing(listing, image_base_path) for listing in listings]
)
await repository.upsert_listings(
[listing for listing in updated_listings if listing is not None]
)
async def dump_images_for_listing(listing: Listing):
with open(listing.path_detail_json()) as f:
detail = json.load(f)
for photo in detail["property"]["floorplans"]:
url = photo["url"]
async def dump_images_for_listing(listing: Listing, base_path: Path) -> Listing | None:
all_floorplans = listing.additional_info["property"]["floorplans"]
for floorplan in all_floorplans:
url = floorplan["url"]
picname = url.split("/")[-1]
order = photo["order"]
p = listing.path_floorplan_file(order, picname)
if p.exists():
floorplan_path = Path(base_path, str(listing.id), "floorplans", picname)
if floorplan_path.exists():
continue
try:
async with aiohttp.ClientSession() as session:
async with semaphore:
async with semaphore:
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
if response.status != 200:
raise Exception(f"Error for {url}: {response.status}")
with open(p, "wb") as f:
floorplan_path.parent.mkdir(parents=True, exist_ok=True)
with open(floorplan_path, "wb") as f:
f.write(await response.read())
listing.floorplan_image_paths.append(str(floorplan_path))
return listing
except Exception as e:
tqdm.write(f"Error for {url}: {e}")