wrongmove/crawler/3_dump_images.py

import json
from urllib.request import urlretrieve
from tqdm import tqdm
from data_access import Listing

for listing in tqdm(Listing.get_all_listings()):
    with open(listing.path_detail_json()) as f:
        detail = json.load(f)
    
    
    for photo in detail['property']['photos']:
        url = photo['maxSizeUrl']
        picname = url.split('/')[-1]
        order = photo['order']
        p = listing.path_pic_file(order, picname)
        if p.exists():
            continue
        tqdm.write(str(p))
        urlretrieve(url, p)
        
    for photo in detail['property']['floorplans']:
        url = photo['url']
        picname = url.split('/')[-1]
        order = photo['order']
        p = listing.path_floorplan_file(order, picname)
        if p.exists():
            continue
        tqdm.write(str(p))
        urlretrieve(url, p)