wrongmove/crawler/1_dump_listings.py

23 lines
608 B
Python
Raw Normal View History

from rec.query import listing_query
import pathlib
import json
d = listing_query(1, 1, 2, 15, 0, 800000)
folder = pathlib.Path("data/rs/")
for i in range(1, 10000):
try:
print(f"page {i}")
d = listing_query(i, 1, 2, 15, 0, 800000)
except:
break
for property in d['properties']:
identifier = property['identifier']
listing_folder = folder / str(identifier)
listing_folder.mkdir(exist_ok=True, parents=True)
listing_path = listing_folder / f"listing.json"
with open(listing_path, 'w') as f:
json.dump(property, f)