viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

convert listings dump to asyncio

Browse source
This commit is contained in:
Viktor Barzin 2025-05-17 21:55:42 +00:00
parent df24c2c1b7
commit ad879f2d4f
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
3 changed files with 35 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

5
crawler/main.py
View file

@ -1,3 +1,4 @@
import asyncio
import pathlib
import click
import importlib
@ -107,12 +108,12 @@ def dump_listings(
f'{query_parameters}'
)
data_dir_path = pathlib.Path(data_dir)
dump_listings_module.dump_listings(query_parameters, data_dir_path)
asyncio.run(dump_listings_module.dump_listings(query_parameters, data_dir_path))
@cli.command()
@click.pass_context
def dump_detail(ctx: click.core.Context):
def dump_details(ctx: click.core.Context):
data_dir = ctx.obj['data_dir']
click.echo(f'Running dump_detail for listings stored in {data_dir}')
listing_paths = sorted(list(pathlib.Path(data_dir).glob("*/listing.json")))