def record(): from rec.query import listing_query, detail_query import json page = 1 listing = listing_query(page, 2, 2, 5, 200000, 500000) with open(f'/Users/kadir/code/realestate/crawler/code/json/queries/listing{page}.json', 'w') as f: json.dump(listing, f) for prop in listing['properties']: identifier = prop['identifier'] resp = detail_query(identifier) # print(identifier, resp.status_code) with open(f'/Users/kadir/code/realestate/crawler/code/json/queries/detail_{identifier}.json', 'w') as f: json.dump(resp, f) def process(): import json import pathlib path = pathlib.Path('/Users/kadir/code/realestate/crawler/code/json/queries/') detailjsons = list(path.glob('detail_*json')) for file in detailjsons: with open(file) as f: js = json.load(f) for floorplan in js['property']['floorplans']: print(floorplan['url']) # record() process()