20 lines
No EOL
486 B
Python
20 lines
No EOL
486 B
Python
import json
|
|
from rec.query import detail_query
|
|
from tqdm import tqdm
|
|
|
|
from data_access import Listing
|
|
|
|
counter = 0
|
|
for listing in tqdm(Listing.get_all_listings()):
|
|
if listing.path_detail_json().exists():
|
|
continue
|
|
counter+=1
|
|
try:
|
|
d = detail_query(listing.identifier)
|
|
with open(listing.path_detail_json(), "w") as f:
|
|
json.dump(d, f)
|
|
except:
|
|
print("Failed at: ", listing.identifier)
|
|
raise
|
|
|
|
print('crawled new: ', counter) |