wrongmove/crawler/1_dump_listings.py

from rec.query import listing_query
from rec.districts import get_districts
from data_access import Listing


def dump_listings(district_names: set[str] | None = None):
    districts = get_districts() if district_names is None else {
        district: locid
        for district, locid in get_districts().items()
        if district in district_names
    }
    print("Valid districts to scrape:", districts.keys())
    for district, locid in districts.items():
        print("#### District:", district)
        for i in [1, 2]:
            try:
                d = listing_query(
                    page=i,
                    min_bedrooms=1,
                    max_bedrooms=4,
                    radius=0,
                    min_price=0,
                    max_price=1000000,
                    location_id=locid,
                    page_size=500,
                    max_days_since_added=None,
                )
            except Exception as e:
                print(e)
                break
            if i == 1:
                print("totalAvailableResults: ", d["totalAvailableResults"])
            if len(d["properties"]) == 0:
                break
            print(f"page {i}", end=", ", flush=True)

            for property in d["properties"]:
                identifier = property["identifier"]

                listing = Listing(identifier)
                listing.dump_listing(property)
        print()  # break line as we used end=, above.


def main():
    dump_listings()


if __name__ == "__main__":
    main()