migrate routing command to use the models and store data there

crawler/5_routing.py

@@ -1,45 +1,67 @@
-from data_access import Listing
-from tqdm import tqdm
+from models.listing import DestinationMode, Route, RouteLegStep
+from repositories.listing_repository import ListingRepository
+from tqdm.asyncio import tqdm
 from rec import routing
+from models import Listing
 
 
 async def calculate_route(
-    listing_paths: list[str],
+    repository: ListingRepository,
     destination_address: str,
     travel_mode: routing.TravelMode,
+    limit: int | None = None,
 ) -> None:
+    listings = await repository.get_listings()
 
-    listings = Listing.get_all_listings(listing_paths)
+    if limit is not None:
+        listings = listings[:limit]
 
-    # reduce listings to everything within 7 miles
-    filtered_listings = []
-    for listing in listings:
-        print(f"Processing {listing.identifier}")
-        if listing.isRemoved:
-            print(f"Removed-Skip: Skipping {listing.identifier} " "is already removed.")
-            continue
-        sqm_ocr = await listing.sqm_ocr()
-        if sqm_ocr is None or sqm_ocr < 30 or sqm_ocr > 200:
-            print(
-                (
-                    f"Floorplan-Skip: Skipping {listing.identifier} as "
-                    f"sqm_ocr is {sqm_ocr}"
+    destimation_mode = DestinationMode(destination_address, travel_mode)
+    updated_listings = await tqdm.gather(
+        *[update_routing_info(listing, destimation_mode) for listing in listings],
+        total=len(listings),
+        desc="Updating routing info",
+    )
+    await repository.upsert_listings(
+        [listing for listing in updated_listings if listing is not None]
+    )
+
+
+async def update_routing_info(
+    listing: Listing, destination_mode: DestinationMode
+) -> Listing | None:
+    if listing.routing_info.get(destination_mode) is not None:
+        # already calculated, do not recompute to save API calls
+        return None
+
+    routes_data = routing.transit_route(
+        listing.latitude,
+        listing.longtitude,
+        destination_mode.destination_address,
+        destination_mode.travel_mode,
+    )
+
+    route_data = routes_data["routes"][0]
+    routes = []
+    for route_data in routes_data["routes"]:
+        duration_s = int(route_data["duration"].split("s")[0])
+        route = Route(
+            legs=[
+                RouteLegStep(
+                    distance_meters=step_data["distanceMeters"],
+                    duration_s=int(step_data["staticDuration"].split("s")[0]),
+                    travel_mode=routing.TravelMode(step_data["travelMode"]),
                 )
-            )
-            continue
-        filtered_listings.append(listing)
-
-    print(f"Filtered listings from {len(listings)} to {len(filtered_listings)}")
-
-    for listing in tqdm(filtered_listings):
-        listing.calculate_route(
-            destination_address,
-            travel_mode,
-            recalculate=False,
+                for step_data in route_data["legs"][0]["steps"]
+            ],
+            distance_meters=route_data["distanceMeters"],
+            duration_s=duration_s,
         )
-        traveltime = listing.travel_time(destination_address, travel_mode)[0]
-        duration_minutes = traveltime["duration"] / 60.0
-        tqdm.write(f"{listing.identifier} {duration_minutes}")
+        routes.append(route)
+    listing.routing_info_json = listing.serialize_routing_info(
+        {**listing.routing_info, **{destination_mode: routes}}
+    )
+    return listing
 
 
 # async def geocode_address(