adding floorplans, detail json, refactored the folders
This commit is contained in:
parent
e2f7998ee9
commit
46bb641026
215 changed files with 481 additions and 23 deletions
32
crawler/main_tmp.py
Normal file
32
crawler/main_tmp.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
def record():
|
||||
from rec.query import listing_query, detail_query
|
||||
import json
|
||||
|
||||
page = 1
|
||||
listing = listing_query(page, 2, 2, 5, 200000, 500000)
|
||||
with open(f'/Users/kadir/code/realestate/crawler/code/json/queries/listing{page}.json', 'w') as f:
|
||||
json.dump(listing, f)
|
||||
|
||||
for prop in listing['properties']:
|
||||
identifier = prop['identifier']
|
||||
resp = detail_query(identifier)
|
||||
# print(identifier, resp.status_code)
|
||||
with open(f'/Users/kadir/code/realestate/crawler/code/json/queries/detail_{identifier}.json', 'w') as f:
|
||||
json.dump(resp, f)
|
||||
|
||||
def process():
|
||||
import json
|
||||
import pathlib
|
||||
path = pathlib.Path('/Users/kadir/code/realestate/crawler/code/json/queries/')
|
||||
|
||||
detailjsons = list(path.glob('detail_*json'))
|
||||
for file in detailjsons:
|
||||
|
||||
with open(file) as f:
|
||||
js = json.load(f)
|
||||
|
||||
for floorplan in js['property']['floorplans']:
|
||||
print(floorplan['url'])
|
||||
|
||||
# record()
|
||||
process()
|
||||
Loading…
Add table
Add a link
Reference in a new issue