[2/n] click-ify - add 2_dump_detail command

run with
poetry run python main.py --step dump_detail

crawler/2_dump_detail.py

@@ -4,12 +4,12 @@ from tqdm import tqdm
 
 from data_access import Listing
 
-incremental = True
 
-
+def dump_detail():
-listings = Listing.get_all_listings()
+    incremental = True
-filtered_listings = []
+    listings = Listing.get_all_listings()
-for listing in listings:
+    filtered_listings = []
+    for listing in listings:
         # We introduced last_seen later, so not all entries have it.
         # If it doesnt exist then its on the platform anymore. So skip
         last_seen = listing.last_seen
@@ -22,10 +22,18 @@ for listing in listings:
         if incremental and not listing.path_detail_json().exists():
             filtered_listings.append(listing)
 
-for listing in tqdm(filtered_listings):
+    for listing in tqdm(filtered_listings):
         try:
             d = detail_query(listing.identifier)
             with open(listing.path_detail_json(), "w") as f:
                 json.dump(d, f)
         except Exception as e:
             print(e)
+
+
+def main():
+    dump_detail()
+
+
+if __name__ == "__main__":
+    main()

crawler/main.py

@@ -2,9 +2,11 @@ import click
 import importlib
 
 dump_listings_module = importlib.import_module('1_dump_listings')
+dump_detail_module = importlib.import_module('2_dump_detail')
 
 steps_to_handlers = {
     'dump_listings': dump_listings_module.dump_listings,
+    'dump_detail': dump_detail_module.dump_detail,
 }