wrongmove/crawler/2_dump_detail.py

import asyncio
import json
from rec.query import detail_query
from tqdm.asyncio import tqdm

from data_access import Listing

# Setting this too high either crashes rightmove or gets us blocked
semaphore = asyncio.Semaphore(10)


async def dump_detail(listing_paths: list[str]):
    listings = Listing.get_all_listings(listing_paths)
    filtered_listings = await tqdm.gather(
        *[_dump_detail_for_listing(listing) for listing in listings]
    )
    return filtered_listings


async def _dump_detail_for_listing(listing: Listing):
    if listing.path_detail_json().exists():
        return

    # for listing in tqdm(filtered_listings):
    async with semaphore:
        d = await detail_query(listing.identifier)
    with open(listing.path_detail_json(), "w") as f:
        json.dump(d, f)