diff --git a/crawler/api/app.py b/crawler/api/app.py index 73ba146..eec5d9a 100644 --- a/crawler/api/app.py +++ b/crawler/api/app.py @@ -15,6 +15,7 @@ from dotenv import load_dotenv from fastapi import Depends, FastAPI, HTTPException, Query from api.auth import User from models.listing import QueryParameters +from notifications import send_notification from rec import districts from redis_repository import RedisRepository from repositories.listing_repository import ListingRepository @@ -82,6 +83,7 @@ async def refresh_listings( user: Annotated[User, Depends(get_current_user)], query_parameters: Annotated[QueryParameters, Query()], ) -> dict[str, str]: + await send_notification(f"{user.email} refreshing listings with query parameters {query_parameters.model_dump_json()}") # TODO: rate limit expiry_time = datetime.now() + timedelta(minutes=10) task = listing_tasks.dump_listings_task.apply_async( diff --git a/crawler/data_access.py b/crawler/data_access.py index 2e6fa54..3b9bf68 100644 --- a/crawler/data_access.py +++ b/crawler/data_access.py @@ -110,7 +110,7 @@ class Listing: # some places list pw in price and others pcm price = max( - self._listing_object["price"], + self._listing_object["price"] or 0, self._listing_object.get("monthlyRent", 0) or 0, ) self.append_price_history(price) diff --git a/crawler/notifications.py b/crawler/notifications.py new file mode 100644 index 0000000..a5eb656 --- /dev/null +++ b/crawler/notifications.py @@ -0,0 +1,27 @@ +from abc import abstractmethod +from enum import StrEnum +import apprise +from functools import lru_cache + + +class Surface: + @abstractmethod + def connection_string(self) -> str | None: + ... + +class Slack(Surface): + def connection_string(self) -> str | None: + return "https://hooks.slack.com/services/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" + + +@lru_cache(maxsize=None) +def get_notifier(surfaces: list[Surface] | None = None) -> apprise.Apprise: + surfaces = surfaces or [Slack()] + obj = apprise.Apprise() + for surface in surfaces: + obj.add(surface.connection_string()) + return obj + +async def send_notification( body: str, title: str='') -> bool: + notifier = get_notifier() + return await notifier.async_notify(body=body, title=title) diff --git a/crawler/poetry.lock b/crawler/poetry.lock index 78834c6..ef1119a 100644 --- a/crawler/poetry.lock +++ b/crawler/poetry.lock @@ -217,6 +217,26 @@ files = [ {file = "appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee"}, ] +[[package]] +name = "apprise" +version = "1.9.3" +description = "Push Notifications that work with just about every platform!" +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "apprise-1.9.3-py3-none-any.whl", hash = "sha256:e9b5abb73244c21a30ee493860f8d4ae80665d225b1b436179d48db4f6fc5b9e"}, + {file = "apprise-1.9.3.tar.gz", hash = "sha256:f583667ea35b8899cd46318c6cb26f0faf6a4605b119174c2523a012590c65a6"}, +] + +[package.dependencies] +certifi = "*" +click = ">=5.0" +markdown = "*" +PyYAML = "*" +requests = "*" +requests-oauthlib = "*" + [[package]] name = "argon2-cffi" version = "25.1.0" @@ -2263,6 +2283,22 @@ babel = ["Babel"] lingua = ["lingua"] testing = ["pytest"] +[[package]] +name = "markdown" +version = "3.8.2" +description = "Python implementation of John Gruber's Markdown." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "markdown-3.8.2-py3-none-any.whl", hash = "sha256:5c83764dbd4e00bdd94d85a19b8d55ccca20fe35b2e678a1422b380324dd5f24"}, + {file = "markdown-3.8.2.tar.gz", hash = "sha256:247b9a70dd12e27f67431ce62523e675b866d254f900c4fe75ce3dda62237c45"}, +] + +[package.extras] +docs = ["mdx_gh_links (>=0.2)", "mkdocs (>=1.6)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"] +testing = ["coverage", "pyyaml"] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -2752,6 +2788,23 @@ files = [ {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, ] +[[package]] +name = "oauthlib" +version = "3.3.1" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1"}, + {file = "oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + [[package]] name = "opencv-python" version = "4.11.0.86" @@ -3876,6 +3929,25 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requests-oauthlib" +version = "2.0.0" +description = "OAuthlib authentication support for Requests." +optional = false +python-versions = ">=3.4" +groups = ["main"] +files = [ + {file = "requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9"}, + {file = "requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36"}, +] + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + [[package]] name = "rfc3339-validator" version = "0.1.4" @@ -5172,4 +5244,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = ">3.11" -content-hash = "d8fefd01d3b4a213cf389c63829e901ce921eb931cb7034af6c869a47a557a59" +content-hash = "110fae166c8a5b2f9c178f822097ee5f939d22f04445bab6ca945612e08a4ed8" diff --git a/crawler/pyproject.toml b/crawler/pyproject.toml index e7e7961..92e1a7f 100644 --- a/crawler/pyproject.toml +++ b/crawler/pyproject.toml @@ -32,6 +32,7 @@ mysqlclient = "^2.2.7" celery = "^5.5.3" redis = "^6.2.0" watchdog = "^6.0.0" +apprise = "^1.9.3" [tool.poetry.group.dev.dependencies] ipdb = "^0.13.13" diff --git a/crawler/repositories/listing_repository.py b/crawler/repositories/listing_repository.py index 61a5818..69c1219 100644 --- a/crawler/repositories/listing_repository.py +++ b/crawler/repositories/listing_repository.py @@ -77,7 +77,7 @@ class ListingRepository: if query_parameters.furnish_types: query = query.where(model.furnish_type.in_(query_parameters.furnish_types)) if ( - isinstance(model, BuyListing) + isinstance(model, RentListing) and query_parameters.let_date_available_from is not None ): query = query.where( @@ -120,9 +120,6 @@ class ListingRepository: try: model_listing = await self._get_concrete_listing(listing) except Exception as e: # WHY SO MANY ERORRS?? - import ipdb - - ipdb.set_trace() # If for whatever reason we cannot add listing, ignore and retry print(f"Error converting listing {listing.identifier}: {e}") failed_to_upsert.append(listing)