viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[3/n] click-ify add dump images command

Browse source 
run with
poetry run python main.py --step dump_images
This commit is contained in:
Viktor Barzin 2025-05-11 19:04:19 +00:00
parent c2196c15c1
commit 70e8ef9f95
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
2 changed files with 36 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

10
crawler/3_dump_images.py
View file

@ -3,6 +3,8 @@ from urllib.request import urlretrieve
from tqdm import tqdm from tqdm import tqdm
from data_access import Listing from data_access import Listing
def dump_images():
for listing in tqdm(Listing.get_all_listings()): for listing in tqdm(Listing.get_all_listings()):
with open(listing.path_detail_json()) as f: with open(listing.path_detail_json()) as f:
detail = json.load(f) detail = json.load(f)
@ -29,3 +31,11 @@ for listing in tqdm(Listing.get_all_listings()):
urlretrieve(url, p) urlretrieve(url, p)
except: except:
tqdm.write(f"404 for {url}") tqdm.write(f"404 for {url}")
def main():
dump_images()
if __name__ == "__main__":
main()

2
crawler/main.py
View file

@ -3,10 +3,12 @@ import importlib
dump_listings_module = importlib.import_module('1_dump_listings') dump_listings_module = importlib.import_module('1_dump_listings')
dump_detail_module = importlib.import_module('2_dump_detail') dump_detail_module = importlib.import_module('2_dump_detail')
dump_images_module = importlib.import_module('3_dump_images')
steps_to_handlers = { steps_to_handlers = {
'dump_listings': dump_listings_module.dump_listings, 'dump_listings': dump_listings_module.dump_listings,
'dump_detail': dump_detail_module.dump_detail, 'dump_detail': dump_detail_module.dump_detail,
'dump_images': dump_images_module.dump_images,
} }