add apprise and send notification when refreshing listings

crawler/api/app.py

@@ -15,6 +15,7 @@ from dotenv import load_dotenv
 from fastapi import Depends, FastAPI, HTTPException, Query
 from api.auth import User
 from models.listing import QueryParameters
+from notifications import send_notification
 from rec import districts
 from redis_repository import RedisRepository
 from repositories.listing_repository import ListingRepository
@@ -82,6 +83,7 @@ async def refresh_listings(
     user: Annotated[User, Depends(get_current_user)],
     query_parameters: Annotated[QueryParameters, Query()],
 ) -> dict[str, str]:
+    await send_notification(f"{user.email} refreshing listings with query parameters {query_parameters.model_dump_json()}")
     # TODO: rate limit
     expiry_time = datetime.now() + timedelta(minutes=10)
     task = listing_tasks.dump_listings_task.apply_async(

crawler/data_access.py

@@ -110,7 +110,7 @@ class Listing:
 
         # some places list pw in price and others pcm
         price = max(
-            self._listing_object["price"],
+            self._listing_object["price"] or 0,
             self._listing_object.get("monthlyRent", 0) or 0,
         )
         self.append_price_history(price)

crawler/notifications.py

@@ -0,0 +1,27 @@
+from abc import abstractmethod
+from enum import StrEnum
+import apprise
+from functools import lru_cache
+
+
+class Surface:
+    @abstractmethod
+    def connection_string(self) -> str | None:
+        ...
+
+class Slack(Surface):
+    def connection_string(self) -> str | None:
+        return "https://hooks.slack.com/services/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"
+
+
+@lru_cache(maxsize=None)
+def get_notifier(surfaces: list[Surface] | None = None) -> apprise.Apprise:
+    surfaces = surfaces or [Slack()]
+    obj = apprise.Apprise()
+    for surface in surfaces:
+        obj.add(surface.connection_string())
+    return obj
+
+async def send_notification( body: str, title: str='') -> bool:
+    notifier = get_notifier()
+    return await notifier.async_notify(body=body, title=title)

crawler/poetry.lock

@@ -217,6 +217,26 @@ files = [
     {file = "appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee"},
 ]
 
+[[package]]
+name = "apprise"
+version = "1.9.3"
+description = "Push Notifications that work with just about every platform!"
+optional = false
+python-versions = ">=3.6"
+groups = ["main"]
+files = [
+    {file = "apprise-1.9.3-py3-none-any.whl", hash = "sha256:e9b5abb73244c21a30ee493860f8d4ae80665d225b1b436179d48db4f6fc5b9e"},
+    {file = "apprise-1.9.3.tar.gz", hash = "sha256:f583667ea35b8899cd46318c6cb26f0faf6a4605b119174c2523a012590c65a6"},
+]
+
+[package.dependencies]
+certifi = "*"
+click = ">=5.0"
+markdown = "*"
+PyYAML = "*"
+requests = "*"
+requests-oauthlib = "*"
+
 [[package]]
 name = "argon2-cffi"
 version = "25.1.0"
@@ -2263,6 +2283,22 @@ babel = ["Babel"]
 lingua = ["lingua"]
 testing = ["pytest"]
 
+[[package]]
+name = "markdown"
+version = "3.8.2"
+description = "Python implementation of John Gruber's Markdown."
+optional = false
+python-versions = ">=3.9"
+groups = ["main"]
+files = [
+    {file = "markdown-3.8.2-py3-none-any.whl", hash = "sha256:5c83764dbd4e00bdd94d85a19b8d55ccca20fe35b2e678a1422b380324dd5f24"},
+    {file = "markdown-3.8.2.tar.gz", hash = "sha256:247b9a70dd12e27f67431ce62523e675b866d254f900c4fe75ce3dda62237c45"},
+]
+
+[package.extras]
+docs = ["mdx_gh_links (>=0.2)", "mkdocs (>=1.6)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"]
+testing = ["coverage", "pyyaml"]
+
 [[package]]
 name = "markdown-it-py"
 version = "3.0.0"
@@ -2752,6 +2788,23 @@ files = [
     {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"},
 ]
 
+[[package]]
+name = "oauthlib"
+version = "3.3.1"
+description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1"},
+    {file = "oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9"},
+]
+
+[package.extras]
+rsa = ["cryptography (>=3.0.0)"]
+signals = ["blinker (>=1.4.0)"]
+signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"]
+
 [[package]]
 name = "opencv-python"
 version = "4.11.0.86"
@@ -3876,6 +3929,25 @@ urllib3 = ">=1.21.1,<3"
 socks = ["PySocks (>=1.5.6,!=1.5.7)"]
 use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 
+[[package]]
+name = "requests-oauthlib"
+version = "2.0.0"
+description = "OAuthlib authentication support for Requests."
+optional = false
+python-versions = ">=3.4"
+groups = ["main"]
+files = [
+    {file = "requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9"},
+    {file = "requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36"},
+]
+
+[package.dependencies]
+oauthlib = ">=3.0.0"
+requests = ">=2.0.0"
+
+[package.extras]
+rsa = ["oauthlib[signedtoken] (>=3.0.0)"]
+
 [[package]]
 name = "rfc3339-validator"
 version = "0.1.4"
@@ -5172,4 +5244,4 @@ propcache = ">=0.2.1"
 [metadata]
 lock-version = "2.1"
 python-versions = ">3.11"
-content-hash = "d8fefd01d3b4a213cf389c63829e901ce921eb931cb7034af6c869a47a557a59"
+content-hash = "110fae166c8a5b2f9c178f822097ee5f939d22f04445bab6ca945612e08a4ed8"

crawler/pyproject.toml

@@ -32,6 +32,7 @@ mysqlclient = "^2.2.7"
 celery = "^5.5.3"
 redis = "^6.2.0"
 watchdog = "^6.0.0"
+apprise = "^1.9.3"
 
 [tool.poetry.group.dev.dependencies]
 ipdb = "^0.13.13"

crawler/repositories/listing_repository.py

@@ -77,7 +77,7 @@ class ListingRepository:
         if query_parameters.furnish_types:
             query = query.where(model.furnish_type.in_(query_parameters.furnish_types))
         if (
-            isinstance(model, BuyListing)
+            isinstance(model, RentListing)
             and query_parameters.let_date_available_from is not None
         ):
             query = query.where(
@@ -120,9 +120,6 @@ class ListingRepository:
                 try:
                     model_listing = await self._get_concrete_listing(listing)
                 except Exception as e:  # WHY SO MANY ERORRS??
-                    import ipdb
-
-                    ipdb.set_trace()
                     # If for whatever reason we cannot add listing, ignore and retry
                     print(f"Error converting listing {listing.identifier}: {e}")
                     failed_to_upsert.append(listing)