viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add counter to 2_dump_details to understand how many new are crawled

Browse source
This commit is contained in:
Kadir 2024-03-30 19:24:03 +01:00
parent 5720e68547
commit acc67192c9
1 changed files with 4 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

5
crawler/2_dump_detail.py
View file

@ -4,10 +4,11 @@ from tqdm import tqdm
from data_access import Listing from data_access import Listing
counter = 0
for listing in tqdm(Listing.get_all_listings()): for listing in tqdm(Listing.get_all_listings()):
if listing.path_detail_json().exists(): if listing.path_detail_json().exists():
continue continue
counter+=1
try: try:
d = detail_query(listing.identifier) d = detail_query(listing.identifier)
with open(listing.path_detail_json(), "w") as f: with open(listing.path_detail_json(), "w") as f:
@ -15,3 +16,5 @@ for listing in tqdm(Listing.get_all_listings()):
except: except:
print("Failed at: ", listing.identifier) print("Failed at: ", listing.identifier)
raise raise
print('crawled new: ', counter)