viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use cached data when fetching details

Browse source
This commit is contained in:
Viktor Barzin 2025-06-08 18:23:25 +00:00
parent 3785d01009
commit 831c45e1f5
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
1 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
crawler/1_dump_listings.py
View file

@ -47,7 +47,8 @@ async def dump_listings(
# if listing is already in db, do not fetch details again # if listing is already in db, do not fetch details again
all_listings = await repository.get_listings( all_listings = await repository.get_listings(
only_ids=[listing.identifier for listing in listings] only_ids=[listing.identifier for listing in listings],
query_parameters=parameters,
) )
all_listing_ids = {listing.id for listing in all_listings} all_listing_ids = {listing.id for listing in all_listings}
@ -58,7 +59,7 @@ async def dump_listings(
*[ *[
_fetch_detail_with_semaphore(semaphore, listing.identifier) _fetch_detail_with_semaphore(semaphore, listing.identifier)
for listing in listings_without_details for listing in listings_without_details
# if listing.identifier not in all_listing_ids # One day we will rely solely on the model data if listing.identifier not in all_listing_ids
], ],
desc="Fetching details", desc="Fetching details",
) )