wrongmove/crawler/1_dump_listings.py

21 lines
561 B
Python

from rec.query import listing_query
import pathlib
import json
from data_access import Listing
d = listing_query(1, 3, 3, 15, 0, 800000)
folder = pathlib.Path("data/rs/")
for i in range(1, 10000):
try:
print(f"page {i}")
d = listing_query(i, 3, 3, 15, 0, 800000, max_days_since_added=3)
except:
break
for property in d['properties']:
identifier = property['identifier']
listing = Listing(identifier)
with open(listing.path_listing_json(), 'w') as f:
json.dump(property, f)