viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

bugfix fetching transactions to make use of db cache

Browse source
This commit is contained in:
Viktor Barzin 2025-06-09 21:23:54 +00:00
parent 9b2653ce91
commit 296a4e7603
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
5 changed files with 71 additions and 586656 deletions
Download patch file Download diff file Expand all files Collapse all files

33
crawler/1_dump_listings.py
View file

@ -45,28 +45,24 @@ async def dump_listings(
listings.append(listing)
# if listing is already in db, do not fetch details again
all_listings = await repository.get_listings(
only_ids=[listing.identifier for listing in listings],
query_parameters=parameters,
)
all_listing_ids = {listing.id for listing in all_listings}
listings_without_details = [
listing for listing in listings if not listing.path_detail_json().exists()
all_listing_ids = [l.id for l in await repository.get_listings()]
missing_listing = [
listing for listing in listings if listing.identifier not in all_listing_ids
]
listing_details = await tqdm.gather(
*[
_fetch_detail_with_semaphore(semaphore, listing.identifier)
for listing in listings_without_details
if listing.identifier not in all_listing_ids
for listing in missing_listing
],
desc="Fetching details (only missing)",
)
for listing, detail in zip(listings_without_details, listing_details):
for listing, detail in zip(missing_listing, listing_details):
listing._details_object = detail
await dump_listings_to_fs(listings)
model_listings = await repository.upsert_listings_legacy(listings) # upsert in db
await dump_listings_to_fs(missing_listing)
model_listings = await repository.upsert_listings_legacy(
missing_listing
) # upsert in db
return model_listings
@ -104,9 +100,8 @@ async def _fetch_detail_with_semaphore(
async def dump_listings_to_fs(listings: list[Listing]) -> None:
for listing in listings:
if not listing.path_listing_json().exists():
listing.dump_listing()
if not listing.path_detail_json().exists():
with open(listing.path_detail_json(), "w") as f:
json.dump(listing._details_object, f, indent=4)
for listing in tqdm(listings, desc="Dumping listings to FS"):
listing.dump_listing()
# if not listing.path_detail_json().exists():
with open(listing.path_detail_json(), "w") as f:
json.dump(listing._details_object, f, indent=4)