merge dump listings and dump details commands - fetch both details and listings in the same command

crawler/1_dump_listings.py

@@ -1,13 +1,18 @@
 import asyncio
+import json
 import pathlib
 from typing import Any
-from rec.query import listing_query, QueryParameters
+from rec.query import detail_query, listing_query, QueryParameters
 from rec.districts import get_districts
+import repositories
+from sqlalchemy import Engine
+from tqdm.asyncio import tqdm
 from data_access import Listing
 
 
 async def dump_listings(
     parameters: QueryParameters,
+    db_engine: Engine,
     data_dir: pathlib.Path = pathlib.Path("data/rs/"),
 ) -> list[Listing]:
     if parameters.district_names:
@@ -19,23 +24,20 @@ async def dump_listings(
     else:
         districts = get_districts()
     print("Valid districts to scrape:", districts.keys())
-    listings = []
 
     semaphore = asyncio.Semaphore(5)  # if too high, rightmove drops connections
-    json_responses = await asyncio.gather(
+    json_responses = await tqdm.gather(
         *[
             _dump_listings_with_semaphore(semaphore, i, parameters, locid)
             for locid in districts.values()
             for i in [1, 2]
-        ]
+        ],
+        desc="Fetching listings",
     )
-    listings = []
+    listings: list[Listing] = []
     for response_json in json_responses:
         if response_json["totalAvailableResults"] == 0:
-            print("No results found")
             continue
-        if response_json["totalAvailableResults"] > 0:
-            print("totalAvailableResults: ", response_json["totalAvailableResults"])
         for property in response_json["properties"]:
             identifier = property["identifier"]
 
@@ -43,6 +45,21 @@ async def dump_listings(
             listing.dump_listing(property)
             listings.append(listing)
 
+    # if listing is already in db, do not fetch details again
+    repository = repositories.ListingRepository(db_engine)
+    all_listings = await repository.get_listings(
+        only_ids=[listing.identifier for listing in listings]
+    )
+    all_listing_ids = {listing.id for listing in all_listings}
+
+    await tqdm.gather(
+        *[
+            _dump_detail_with_semaphore(semaphore, listing)
+            for listing in listings
+            # if listing.identifier not in all_listing_ids # One day we will rely solely on the model data
+        ],
+        desc="Fetching details",
+    )
     return listings
 
 
@@ -53,7 +70,7 @@ async def _dump_listings_with_semaphore(
     location_id: str,
 ) -> dict[str, Any]:
     async with semaphore:
-        listing = await listing_query(
+        listing_query_result = await listing_query(
             page=page_id,
             channel=parameters.listing_type,
             min_bedrooms=parameters.min_bedrooms,
@@ -66,4 +83,16 @@ async def _dump_listings_with_semaphore(
             max_days_since_added=parameters.max_days_since_added,
             furnish_types=parameters.furnish_types or [],
         )
-    return listing
+
+    return listing_query_result
+
+
+async def _dump_detail_with_semaphore(semaphore: asyncio.Semaphore, listing: Listing):
+    if listing.path_detail_json().exists():
+        return
+
+    # for listing in tqdm(filtered_listings):
+    async with semaphore:
+        d = await detail_query(listing.identifier)
+    with open(listing.path_detail_json(), "w") as f:
+        json.dump(d, f)