diff --git a/crawler/rec/query.py b/crawler/rec/query.py index 47f4fd7..600bebb 100644 --- a/crawler/rec/query.py +++ b/crawler/rec/query.py @@ -2,10 +2,6 @@ import enum from typing import Any import aiohttp -import requests -import urllib3 - -urllib3.disable_warnings() # type: ignore class ListingType(enum.StrEnum): @@ -44,12 +40,12 @@ async def detail_query(detail_id: int): } url = f"https://api.rightmove.co.uk/api/property/{detail_id}" async with aiohttp.ClientSession() as session: - async with session.get(url, params=params, - headers=headers) as response: + async with session.get(url, params=params, headers=headers) as response: if response.status != 200: raise Exception( f"""id: {detail_id}. Status Code: {response.status}.""" - f"""Failed due to: {await response.text()}""") + f"""Failed due to: {await response.text()}""" + ) return await response.json() @@ -89,7 +85,10 @@ async def listing_query( if len(property_type) > 0: params["propertyTypes"] = ",".join(property_type) if max_days_since_added is not None and max_days_since_added not in [ - 1, 3, 7, 14 + 1, + 3, + 7, + 14, ]: raise Exception("Invalid max days. Can only be", [1, 3, 7, 14]) params["maxDaysSinceAdded"] = str(max_days_since_added) @@ -104,15 +103,15 @@ async def listing_query( "Host": "api.rightmove.co.uk", "Accept-Encoding": "gzip, deflate, br", "User-Agent": "okhttp/4.12.0", - "Connection": "keep-alive" + "Connection": "keep-alive", } - response = requests.get( - "https://api.rightmove.co.uk/api/property-listing", - params=params, - headers=headers, - verify=False, - ) - if response.status_code != 200: - raise Exception("Failed due to: ", response.text) - return response.json() + async with aiohttp.ClientSession() as session: + async with session.get( + "https://api.rightmove.co.uk/api/property-listing", + params=params, + headers=headers, + ) as response: + if response.status != 200: + raise Exception(f"Failed due to: {await response.text()}") + return await response.json()