viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[3/n] click-ify add dump images command

Browse source 
run with
poetry run python main.py --step dump_images
This commit is contained in:
Viktor Barzin 2025-05-11 19:04:19 +00:00
parent c2196c15c1
commit 70e8ef9f95
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
2 changed files with 36 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

58
crawler/3_dump_images.py
View file

@ -3,29 +3,39 @@ from urllib.request import urlretrieve
from tqdm import tqdm
from data_access import Listing
for listing in tqdm(Listing.get_all_listings()):
with open(listing.path_detail_json()) as f:
detail = json.load(f)
# for photo in detail["property"]["photos"]:
# url = photo["maxSizeUrl"]
# picname = url.split("/")[-1]
# order = photo["order"]
# p = listing.path_pic_file(order, picname)
# if p.exists():
# continue
# tqdm.write(str(p))
# urlretrieve(url, p)
def dump_images():
for listing in tqdm(Listing.get_all_listings()):
with open(listing.path_detail_json()) as f:
detail = json.load(f)
for photo in detail["property"]["floorplans"]:
url = photo["url"]
picname = url.split("/")[-1]
order = photo["order"]
p = listing.path_floorplan_file(order, picname)
if p.exists():
continue
tqdm.write(str(p))
try:
urlretrieve(url, p)
except:
tqdm.write(f"404 for {url}")
# for photo in detail["property"]["photos"]:
# url = photo["maxSizeUrl"]
# picname = url.split("/")[-1]
# order = photo["order"]
# p = listing.path_pic_file(order, picname)
# if p.exists():
# continue
# tqdm.write(str(p))
# urlretrieve(url, p)
for photo in detail["property"]["floorplans"]:
url = photo["url"]
picname = url.split("/")[-1]
order = photo["order"]
p = listing.path_floorplan_file(order, picname)
if p.exists():
continue
tqdm.write(str(p))
try:
urlretrieve(url, p)
except:
tqdm.write(f"404 for {url}")
def main():
dump_images()
if __name__ == "__main__":
main()