32 lines
1,016 B
Python
32 lines
1,016 B
Python
|
def record():
|
||
|
|
from rec.query import listing_query, detail_query
|
||
|
|
import json
|
||
|
|
|
||
|
|
page = 1
|
||
|
|
listing = listing_query(page, 2, 2, 5, 200000, 500000)
|
||
|
|
with open(f'/Users/kadir/code/realestate/crawler/code/json/queries/listing{page}.json', 'w') as f:
|
||
|
|
json.dump(listing, f)
|
||
|
|
|
||
|
|
for prop in listing['properties']:
|
||
|
|
identifier = prop['identifier']
|
||
|
|
resp = detail_query(identifier)
|
||
|
|
# print(identifier, resp.status_code)
|
||
|
|
with open(f'/Users/kadir/code/realestate/crawler/code/json/queries/detail_{identifier}.json', 'w') as f:
|
||
|
|
json.dump(resp, f)
|
||
|
|
|
||
|
|
def process():
|
||
|
|
import json
|
||
|
|
import pathlib
|
||
|
|
path = pathlib.Path('/Users/kadir/code/realestate/crawler/code/json/queries/')
|
||
|
|
|
||
|
|
detailjsons = list(path.glob('detail_*json'))
|
||
|
|
for file in detailjsons:
|
||
|
|
|
||
|
|
with open(file) as f:
|
||
|
|
js = json.load(f)
|
||
|
|
|
||
|
|
for floorplan in js['property']['floorplans']:
|
||
|
|
print(floorplan['url'])
|
||
|
|
|
||
|
|
# record()
|
||
|
|
process()
|