bugfix fetching transactions to make use of db cache

crawler/1_dump_listings.py

@@ -45,28 +45,24 @@ async def dump_listings(
             listings.append(listing)
 
     # if listing is already in db, do not fetch details again
-    all_listings = await repository.get_listings(
+    all_listing_ids = [l.id for l in await repository.get_listings()]
-        only_ids=[listing.identifier for listing in listings],
+    missing_listing = [
-        query_parameters=parameters,
+        listing for listing in listings if listing.identifier not in all_listing_ids
-    )
-    all_listing_ids = {listing.id for listing in all_listings}
-
-    listings_without_details = [
-        listing for listing in listings if not listing.path_detail_json().exists()
     ]
     listing_details = await tqdm.gather(
         *[
             _fetch_detail_with_semaphore(semaphore, listing.identifier)
-            for listing in listings_without_details
+            for listing in missing_listing
-            if listing.identifier not in all_listing_ids
         ],
         desc="Fetching details (only missing)",
     )
-    for listing, detail in zip(listings_without_details, listing_details):
+    for listing, detail in zip(missing_listing, listing_details):
         listing._details_object = detail
 
-    await dump_listings_to_fs(listings)
+    await dump_listings_to_fs(missing_listing)
-    model_listings = await repository.upsert_listings_legacy(listings)  # upsert in db
+    model_listings = await repository.upsert_listings_legacy(
+        missing_listing
+    )  # upsert in db
 
     return model_listings
 
@@ -104,9 +100,8 @@ async def _fetch_detail_with_semaphore(
 
 
 async def dump_listings_to_fs(listings: list[Listing]) -> None:
-    for listing in listings:
+    for listing in tqdm(listings, desc="Dumping listings to FS"):
-        if not listing.path_listing_json().exists():
+        listing.dump_listing()
-            listing.dump_listing()
+        # if not listing.path_detail_json().exists():
-        if not listing.path_detail_json().exists():
+        with open(listing.path_detail_json(), "w") as f:
-            with open(listing.path_detail_json(), "w") as f:
+            json.dump(listing._details_object, f, indent=4)
-                json.dump(listing._details_object, f, indent=4)

crawler/alembic/versions/042751f52538_add_more_fields_to_tables.py

@@ -0,0 +1,38 @@
+"""add more fields to tables
+
+Revision ID: 042751f52538
+Revises: 72b7410ff3e6
+Create Date: 2025-06-08 17:49:25.347510
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import sqlite
+
+# revision identifiers, used by Alembic.
+revision: str = '042751f52538'
+down_revision: Union[str, None] = '72b7410ff3e6'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('buylisting', sa.Column('price_history_json', sa.String(), nullable=False))
+    op.drop_column('buylisting', 'price_history')
+    op.add_column('rentlisting', sa.Column('price_history_json', sa.String(), nullable=False))
+    op.drop_column('rentlisting', 'price_history')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('rentlisting', sa.Column('price_history', sqlite.JSON(), nullable=False))
+    op.drop_column('rentlisting', 'price_history_json')
+    op.add_column('buylisting', sa.Column('price_history', sqlite.JSON(), nullable=False))
+    op.drop_column('buylisting', 'price_history_json')
+    # ### end Alembic commands ###

crawler/data_access.py

@@ -259,17 +259,16 @@ class Listing:
 
     @property
     def detailobject(self) -> dict[str, Any]:
-        if self._details_object is None:
+        if self._details_object is not None:
-            if self.path_detail_json().exists():
-                with open(self.path_detail_json()) as f:
-                    self._details_object = json.load(f)
-                    return self._details_object  # type: ignore
-            else:
-                raise ValueError(
-                    f"Detail object for listing {self.identifier} not found."
-                )
-        else:
             return self._details_object
+        if (
+            self.path_detail_json().exists()
+            and json.load(self.path_detail_json().open()).get("property") is not None
+        ):
+            with open(self.path_detail_json()) as f:
+                self._details_object = json.load(f)
+                return self._details_object  # type: ignore
+        raise ValueError(f"Detail object for listing {self.identifier} not found.")
 
     @property
     def price(self) -> float:

immoweb/script.js

@@ -149,6 +149,7 @@ var map = new mapboxgl.Map({
 });
 
 
+console.log('ekekek');
 map.on("load", function () {
     var crossData = data.features.map(function (d, i) {
         //clone properties
@@ -215,28 +216,31 @@ function getListingDialogHTML(properties) {
     return styledResult;
 }
 function getPropertyHTML(property) {
-    const priceHistoryHTMLs = property.properties.info.price_history.map((d) => {
+    const priceHistoryHTMLs = property.properties.price_history.map((d) => {
         return `<li>${d.last_seen.split('T')[0]}: £${d.price}</li>`;
     });
+
     let priceHistoryHTML = '';
     if (priceHistoryHTMLs.length > 1) {
         priceHistoryHTML = `
+        <strong>Price history:</strong>
         <ul>
             ${priceHistoryHTMLs.join('')}
         </ul>
         <br />
         `
     }
+    const lastSeenStr = property.properties.last_seen.split('T')[0];
+    const lastSeenDays = Math.round((new Date() - new Date(lastSeenStr)) / (1000 * 60 * 60 * 24));
 
     return `
         <div>
-            <img src="${property.properties.info.photo_thumbnail}" style="width:100%; height:auto;">
+            <img src="${property.properties.photo_thumbnail}" style="width:100%; height:auto;">
             <p>
-                <strong>Available from:</strong> ${property.properties.info.let_date_available}
+                <strong>Available from:</strong> ${property.properties.info.property.let_date_available}
                 <br />
                 <strong>Price:</strong> £${property.properties.total_price}
                 <br />
-                <strong>Price history:</strong>
                 ${priceHistoryHTML}
                 <strong>Rooms:</strong> ${property.properties.rooms}
                 <br />
@@ -244,9 +248,9 @@ function getPropertyHTML(property) {
                 <br />
                 <strong>Price per area:</strong> £${property.properties.qmprice}/m²
                 <br />
-                <strong>Last seen:</strong> ${property.properties.info.last_seen} days ago
+                <strong>Last seen:</strong> ${lastSeenDays} days ago
                 <br />
-                <strong>Agency:</strong> ${property.properties.info.agency}
+                <strong>Agency:</strong> ${property.properties.agency}
                 <br />
                 <a href="${property.properties.url}" target="_blank">View Listing</a>
             </p>