[1/n] click-ify - add entrypoint for click script and add

1_dump_listings command

run via:
poetry run python main.py --step dump_listings
This commit is contained in:
Viktor Barzin 2025-05-11 18:59:41 +00:00
parent 0a66efa48a
commit 90b531f5d9
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
4 changed files with 272 additions and 64 deletions

26
crawler/main.py Normal file
View file

@ -0,0 +1,26 @@
import click
import importlib
dump_listings_module = importlib.import_module('1_dump_listings')
steps_to_handlers = {
'dump_listings': dump_listings_module.dump_listings,
}
@click.command()
@click.option(
'--step',
default=[],
help='Scraping step to run',
multiple=True,
type=click.Choice(steps_to_handlers.keys())
)
def main(step: list[str]):
for s in step:
click.echo(f'Calling handler for step: {s}')
steps_to_handlers[s]()
if __name__ == '__main__':
main()