viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix types and format

Browse source
This commit is contained in:
Viktor Barzin 2025-05-18 12:27:26 +00:00
parent 91d3237516
commit b873eaf203
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
8 changed files with 117 additions and 172 deletions
Download patch file Download diff file Expand all files Collapse all files

16
crawler/3_dump_images.py
View file

@ -1,6 +1,5 @@
import asyncio
import json
import pathlib
import aiohttp
from tqdm.asyncio import tqdm
from data_access import Listing
@ -11,7 +10,8 @@ semaphore = asyncio.Semaphore(10)
async def dump_images(listing_paths: list[str]):
listings = Listing.get_all_listings(listing_paths)
await tqdm.gather(*[dump_images_for_listing(listing) for listing in listings])
await tqdm.gather(
*[dump_images_for_listing(listing) for listing in listings])
async def dump_images_for_listing(listing: Listing):
@ -30,17 +30,9 @@ async def dump_images_for_listing(listing: Listing):
async with semaphore:
async with session.get(url) as response:
if response.status != 200:
raise Exception(f"Error for {url}: {response.status}")
raise Exception(
f"Error for {url}: {response.status}")
with open(p, "wb") as f:
f.write(await response.read())
except Exception as e:
tqdm.write(f"Error for {url}: {e}")
def main():
listing_paths = sorted(list(pathlib.Path("data/rs").glob("*/listing.json")))
dump_images(listing_paths)
if __name__ == "__main__":
main()