viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use cached data when fetching details

Browse source
This commit is contained in:
Viktor Barzin 2025-06-08 18:23:25 +00:00
parent 3785d01009
commit 831c45e1f5
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
1 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
crawler/1_dump_listings.py
View file

@ -47,7 +47,8 @@ async def dump_listings(
# if listing is already in db, do not fetch details again
all_listings = await repository.get_listings(
only_ids=[listing.identifier for listing in listings]
only_ids=[listing.identifier for listing in listings],
query_parameters=parameters,
)
all_listing_ids = {listing.id for listing in all_listings}
@ -58,7 +59,7 @@ async def dump_listings(
*[
_fetch_detail_with_semaphore(semaphore, listing.identifier)
for listing in listings_without_details
# if listing.identifier not in all_listing_ids # One day we will rely solely on the model data
if listing.identifier not in all_listing_ids
],
desc="Fetching details",
)