[1/n] click-ify - add entrypoint for click script and add

1_dump_listings command

run via:
poetry run python main.py --step dump_listings

crawler/1_dump_listings.py

@@ -1,12 +1,11 @@
 from rec.query import listing_query
 from rec.districts import get_districts
-import pathlib
 from data_access import Listing
 
-folder = pathlib.Path("data/rs/")
-districts = get_districts()
 
-for district, locid in districts.items():
+def dump_listings():
+    districts = get_districts()
+    for district, locid in districts.items():
         print("#### District:", district)
         for i in [1, 2]:
             try:
@@ -36,3 +35,11 @@ for district, locid in districts.items():
                 listing = Listing(identifier)
                 listing.dump_listing(property)
         print()  # break line as we used end=, above.
+
+
+def main():
+    dump_listings()
+
+
+if __name__ == "__main__":
+    main()

crawler/main.py

@@ -0,0 +1,26 @@
+import click
+import importlib
+
+dump_listings_module = importlib.import_module('1_dump_listings')
+
+steps_to_handlers = {
+    'dump_listings': dump_listings_module.dump_listings,
+}
+
+
+@click.command()
+@click.option(
+    '--step',
+    default=[],
+    help='Scraping step to run',
+    multiple=True,
+    type=click.Choice(steps_to_handlers.keys())
+)
+def main(step: list[str]):
+    for s in step:
+        click.echo(f'Calling handler for step: {s}')
+        steps_to_handlers[s]()
+
+
+if __name__ == '__main__':
+    main()

crawler/pyproject.toml

@@ -21,8 +21,7 @@ pandas = "^2.2.1"
 geopy = "^2.4.1"
 matplotlib = "^3.10.0"
 opencv-python = "^4.11.0.86"
-
+click = "^8.2.0"
-[tool.poetry.group.dev.dependencies]
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]