viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

changing detail downloads to prefiltering first. Making the progress bar more accurate and frontloading

Browse source
This commit is contained in:
Kadir 2024-04-01 20:28:37 +02:00
parent b16fee0648
commit 5305451fe8
1 changed files with 7 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

14
crawler/2_dump_detail.py
View file

@ -4,11 +4,13 @@ from tqdm import tqdm
from data_access import Listing from data_access import Listing
counter = 0 listings = Listing.get_all_listings()
for listing in tqdm(Listing.get_all_listings()): filtered_listings = []
if listing.path_detail_json().exists(): for listing in listings:
continue if not listing.path_detail_json().exists():
counter+=1 filtered_listings.append(listing)
for listing in tqdm(filtered_listings):
try: try:
d = detail_query(listing.identifier) d = detail_query(listing.identifier)
with open(listing.path_detail_json(), "w") as f: with open(listing.path_detail_json(), "w") as f:
@ -16,5 +18,3 @@ for listing in tqdm(Listing.get_all_listings()):
except: except:
print("Failed at: ", listing.identifier) print("Failed at: ", listing.identifier)
raise raise
print('crawled new: ', counter)