Add services layer, tests, streaming UI, and cleanup legacy code

crawler/data_access.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 import json
 import pathlib
 from typing import Any, List
+import warnings
 from models.listing import ListingSite, PriceHistoryItem
 from rec import floorplan, routing
 import re
@@ -12,6 +13,12 @@ import datetime
 
 @dataclass()
 class Listing:
+    """Legacy Listing class for filesystem-based data access.
+
+    .. deprecated::
+        Use models.listing.RentListing or models.listing.BuyListing instead.
+        This class is kept for backwards compatibility with the populate_db command.
+    """
     identifier: int
     _details_object: dict[str, Any] | None = None
     _listing_object: dict[str, Any] | None = None
@@ -36,6 +43,14 @@ class Listing:
         "council_tax_band",
     ]
 
+    def __post_init__(self) -> None:
+        warnings.warn(
+            "data_access.Listing is deprecated. Use models.listing.RentListing "
+            "or models.listing.BuyListing instead.",
+            DeprecationWarning,
+            stacklevel=3,
+        )
+
     @staticmethod
     def get_all_listings(
         listing_paths: list[pathlib.Path],
@@ -144,39 +159,6 @@ class Listing:
         # todo add check if return is image
         return images
 
-    def calculate_sqm_model(self):
-        objs = []
-        for floorplan_path in self.list_floorplans():
-            estimated_sqm, model_output, predictions = floorplan.calculate_model(
-                floorplan_path
-            )
-            objs.append(
-                {
-                    "floorplan_path": str(floorplan_path),
-                    "estimated_sqm": estimated_sqm,
-                    "model_output": model_output,
-                    "no_predictions": len(
-                        predictions
-                    ),  # cant serialize the predictions itself since its a tensor
-                }
-            )
-
-        with open(self.path_floorplan_model_json(), "w") as f:
-            json.dump(objs, f)
-
-    @property
-    def sqm_model(self, recalculate=True) -> float:
-        if not self.path_floorplan_model_json().exists() or recalculate:
-            self.calculate_sqm_model()
-
-        with open(self.path_floorplan_json()) as f:
-            objs = json.load(f)
-
-        max_sqm = max(
-            [o["estimated_sqm"] for o in objs if o is None]
-        )  # filter out Nones
-        return max_sqm
-
     async def calculate_sqm_ocr(self, recalculate=True):
         objs = []
         if self.path_floorplan_ocr_json().exists():
@@ -405,63 +387,6 @@ class Listing:
     def listing_site(self) -> ListingSite:
         return ListingSite.RIGHTMOVE  # this class supports only right move
 
-    async def dict_nicely(self):
-        travel_time_fastest = {}
-        travel_time_second = {}
-        if self.path_routing_json().exists():
-            with open(self.path_routing_json(), "r") as f:
-                travel_times = json.load(f)
-            for destination_mode in travel_times.keys():
-                destination_mode_clean = destination_mode.replace(" ", "_").replace(
-                    ",", "_"
-                )
-                destination, travel_mode = self.__from_routing_cache_key(
-                    destination_mode
-                )
-                travel_time_fastest[destination_mode_clean] = self.travel_time(
-                    destination, travel_mode
-                )[0]["duration"]
-                travel_time_second[destination_mode_clean] = self.travel_time(
-                    destination, travel_mode
-                )[1]["duration"]
-
-        return {
-            "identifier": self.identifier,
-            "sqm_ocr": await self.sqm_ocr(),
-            "price": self.price,
-            "price_per_sqm": await self.price_per_sqm(),
-            "url": self.url,
-            "bedrooms": self.bedrooms,
-            "travel_time_fastest": ":".join(
-                sorted(
-                    f"{dest} in {travel_mode//60}min"
-                    for dest, travel_mode in travel_time_fastest.items()
-                )
-            ),
-            "travel_time_second": ":".join(
-                sorted(
-                    f"{dest} in {travel_mode//60}min"
-                    for dest, travel_mode in travel_time_second.items()
-                )
-            ),
-            "lease_left": self.leaseLeft,
-            "service_charge": self.serviceCharge,
-            "development": self.development,
-            "tenure_type": self.tenure_type,
-            "updated_days": self.updateDaysAgo,
-            "status": self.status,
-            "last_seen": self.last_seen,
-            "agency": self.agency,
-            "council_tax_band": self.councilTaxBand,
-            "photo_thumbnail": self.photoThumbnail,
-            "let_date_available": (
-                self.letDateAvailable.strftime("%d/%m/%Y")
-                if self.letDateAvailable
-                else "Ask agent"
-            ),
-            "price_history": self.priceHistory,
-        }
-
     def __routing_cache_key(
         self,
         dest_address: str,