wrongmove/crawler/main.py

import click
import importlib

from rec.districts import get_districts

dump_listings_module = importlib.import_module('1_dump_listings')
dump_detail_module = importlib.import_module('2_dump_detail')
dump_images_module = importlib.import_module('3_dump_images')
detect_floorplan_module = importlib.import_module('4_detect_floorplan')
routing_module = importlib.import_module('5_routing')

steps_to_handlers = {
    'dump_listings': dump_listings_module.dump_listings,
    'dump_detail': dump_detail_module.dump_detail,
    'dump_images': dump_images_module.dump_images,
    'detect_floorplan': detect_floorplan_module.detect_floorplan,
    'routing': routing_module.calculate_route,
}


@click.group()
def cli():
    pass


@cli.command()
@click.option(
    '--district',
    default=None,
    help='Districts to scrape',
    type=click.Choice(get_districts().keys(), case_sensitive=False),
    multiple=True,
)
def dump_listings(district: list[str]):
    click.echo(f'Running dump_listings for districts {district}')
    dump_listings_module.dump_listings(set(district))


@cli.command()
def dump_detail():
    click.echo('Running dump_detail')
    dump_detail_module.dump_detail()


@cli.command()
def dump_images():
    click.echo('Running dump_images')
    dump_images_module.dump_images()


@cli.command()
def detect_floorplan():
    click.echo('Running detect_floorplan')
    detect_floorplan_module.detect_floorplan()


@cli.command()
def routing():
    click.echo('Running routing')
    routing_module.calculate_route()


if __name__ == '__main__':
    cli()