diff --git a/crawler/1_dump_listings.py b/crawler/1_dump_listings.py index f3751ab..8f44e22 100644 --- a/crawler/1_dump_listings.py +++ b/crawler/1_dump_listings.py @@ -47,7 +47,8 @@ async def dump_listings( # if listing is already in db, do not fetch details again all_listings = await repository.get_listings( - only_ids=[listing.identifier for listing in listings] + only_ids=[listing.identifier for listing in listings], + query_parameters=parameters, ) all_listing_ids = {listing.id for listing in all_listings} @@ -58,7 +59,7 @@ async def dump_listings( *[ _fetch_detail_with_semaphore(semaphore, listing.identifier) for listing in listings_without_details - # if listing.identifier not in all_listing_ids # One day we will rely solely on the model data + if listing.identifier not in all_listing_ids ], desc="Fetching details", )