diff --git a/crawler/2_dump_detail.py b/crawler/2_dump_detail.py index 13644bf..5b8bbd9 100644 --- a/crawler/2_dump_detail.py +++ b/crawler/2_dump_detail.py @@ -4,10 +4,11 @@ from tqdm import tqdm from data_access import Listing +counter = 0 for listing in tqdm(Listing.get_all_listings()): if listing.path_detail_json().exists(): continue - + counter+=1 try: d = detail_query(listing.identifier) with open(listing.path_detail_json(), "w") as f: @@ -15,3 +16,5 @@ for listing in tqdm(Listing.get_all_listings()): except: print("Failed at: ", listing.identifier) raise + +print('crawled new: ', counter) \ No newline at end of file