wrongmove/crawler/main.py

import click
import importlib

dump_listings_module = importlib.import_module('1_dump_listings')
dump_detail_module = importlib.import_module('2_dump_detail')

steps_to_handlers = {
    'dump_listings': dump_listings_module.dump_listings,
    'dump_detail': dump_detail_module.dump_detail,
}


@click.command()
@click.option(
    '--step',
    default=[],
    help='Scraping step to run',
    multiple=True,
    type=click.Choice(steps_to_handlers.keys())
)
def main(step: list[str]):
    for s in step:
        click.echo(f'Calling handler for step: {s}')
        steps_to_handlers[s]()


if __name__ == '__main__':
    main()