[1/n] click-ify - add entrypoint for click script and add
1_dump_listings command run via: poetry run python main.py --step dump_listings
This commit is contained in:
parent
0a66efa48a
commit
90b531f5d9
4 changed files with 272 additions and 64 deletions
26
crawler/main.py
Normal file
26
crawler/main.py
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import click
|
||||
import importlib
|
||||
|
||||
dump_listings_module = importlib.import_module('1_dump_listings')
|
||||
|
||||
steps_to_handlers = {
|
||||
'dump_listings': dump_listings_module.dump_listings,
|
||||
}
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
'--step',
|
||||
default=[],
|
||||
help='Scraping step to run',
|
||||
multiple=True,
|
||||
type=click.Choice(steps_to_handlers.keys())
|
||||
)
|
||||
def main(step: list[str]):
|
||||
for s in step:
|
||||
click.echo(f'Calling handler for step: {s}')
|
||||
steps_to_handlers[s]()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue