wrongmove/crawler/1_dump_listings.py

from rec.query import listing_query
from rec.districts import get_districts
import pathlib
from data_access import Listing

folder = pathlib.Path("data/rs/")
districts = get_districts()

for district, locid in districts.items():
    print("#### District:", district)
    for i in [1, 2]:
        try:
            d = listing_query(
                page=i,
                min_bedrooms=1,
                max_bedrooms=4,
                radius=0,
                min_price=0,
                max_price=1000000,
                location_id=locid,
                page_size=500,
                max_days_since_added=None,
            )
        except Exception as e:
            print(e)
            break
        if i == 1:
            print("totalAvailableResults: ", d["totalAvailableResults"])
        if len(d["properties"]) == 0:
            break
        print(f"page {i}", end=", ", flush=True)

        for property in d["properties"]:
            identifier = property["identifier"]

            listing = Listing(identifier)
            listing.dump_listing(property)
    print()  # break line as we used end=, above.