wrongmove/crawler/2_dump_detail.py

20 lines
538 B
Python

import json
from rec.query import detail_query
from tqdm import tqdm
from data_access import Listing
listings = Listing.get_all_listings()
filtered_listings = []
for listing in listings:
if not listing.path_detail_json().exists():
filtered_listings.append(listing)
for listing in tqdm(filtered_listings):
try:
d = detail_query(listing.identifier)
with open(listing.path_detail_json(), "w") as f:
json.dump(d, f)
except:
print("Failed at: ", listing.identifier)
raise