wrongmove/crawler/main_tmp.py

def record():
    from rec.query import listing_query, detail_query
    import json

    page = 1
    listing = listing_query(page, 2, 2, 5, 200000, 500000)
    with open(
        f"/Users/kadir/code/realestate/crawler/code/json/queries/listing{page}.json",
        "w",
    ) as f:
        json.dump(listing, f)

    for prop in listing["properties"]:
        identifier = prop["identifier"]
        resp = detail_query(identifier)
        # print(identifier, resp.status_code)
        with open(
            f"/Users/kadir/code/realestate/crawler/code/json/queries/detail_{identifier}.json",
            "w",
        ) as f:
            json.dump(resp, f)


def process():
    import json
    import pathlib

    path = pathlib.Path("/Users/kadir/code/realestate/crawler/code/json/queries/")

    detailjsons = list(path.glob("detail_*json"))
    for file in detailjsons:
        with open(file) as f:
            js = json.load(f)

        for floorplan in js["property"]["floorplans"]:
            print(floorplan["url"])


# record()
process()