wrongmove/crawler/3_dump_images.py

import json
from urllib.request import urlretrieve
from tqdm import tqdm
from data_access import Listing

for listing in tqdm(Listing.get_all_listings()):
    with open(listing.path_detail_json()) as f:
        detail = json.load(f)

    # for photo in detail["property"]["photos"]:
    #     url = photo["maxSizeUrl"]
    #     picname = url.split("/")[-1]
    #     order = photo["order"]
    #     p = listing.path_pic_file(order, picname)
    #     if p.exists():
    #         continue
    #     tqdm.write(str(p))
    #     urlretrieve(url, p)

    for photo in detail["property"]["floorplans"]:
        url = photo["url"]
        picname = url.split("/")[-1]
        order = photo["order"]
        p = listing.path_floorplan_file(order, picname)
        if p.exists():
            continue
        tqdm.write(str(p))
        urlretrieve(url, p)