wrongmove/crawler/1_dump_listings.py

from rec.query import listing_query
import pathlib
import json
from data_access import Listing

d = listing_query(1, 3, 3, 15, 0, 800000, max_days_since_added=7)
folder = pathlib.Path("data/rs/")

for i in range(1, 10000):
    try:
        print(f"page {i}")
        d = listing_query(i, 3, 3, 15, 0, 800000, max_days_since_added=1)
    except:
        break

    for property in d["properties"]:
        identifier = property["identifier"]

        listing = Listing(identifier)
        with open(listing.path_listing_json(), "w") as f:
            json.dump(property, f)