viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use aiohttp for dumping listing query

Browse source
This commit is contained in:
Viktor Barzin 2025-05-31 23:48:45 +00:00
parent de1416de7e
commit 1122f5a96f
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
1 changed files with 17 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

35
crawler/rec/query.py
View file

@ -2,10 +2,6 @@
import enum import enum
from typing import Any from typing import Any
import aiohttp import aiohttp
import requests
import urllib3
urllib3.disable_warnings() # type: ignore
class ListingType(enum.StrEnum): class ListingType(enum.StrEnum):
@ -44,12 +40,12 @@ async def detail_query(detail_id: int):
} }
url = f"https://api.rightmove.co.uk/api/property/{detail_id}" url = f"https://api.rightmove.co.uk/api/property/{detail_id}"
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.get(url, params=params, async with session.get(url, params=params, headers=headers) as response:
headers=headers) as response:
if response.status != 200: if response.status != 200:
raise Exception( raise Exception(
f"""id: {detail_id}. Status Code: {response.status}.""" f"""id: {detail_id}. Status Code: {response.status}."""
f"""Failed due to: {await response.text()}""") f"""Failed due to: {await response.text()}"""
)
return await response.json() return await response.json()
@ -89,7 +85,10 @@ async def listing_query(
if len(property_type) > 0: if len(property_type) > 0:
params["propertyTypes"] = ",".join(property_type) params["propertyTypes"] = ",".join(property_type)
if max_days_since_added is not None and max_days_since_added not in [ if max_days_since_added is not None and max_days_since_added not in [
1, 3, 7, 14 1,
3,
7,
14,
]: ]:
raise Exception("Invalid max days. Can only be", [1, 3, 7, 14]) raise Exception("Invalid max days. Can only be", [1, 3, 7, 14])
params["maxDaysSinceAdded"] = str(max_days_since_added) params["maxDaysSinceAdded"] = str(max_days_since_added)
@ -104,15 +103,15 @@ async def listing_query(
"Host": "api.rightmove.co.uk", "Host": "api.rightmove.co.uk",
"Accept-Encoding": "gzip, deflate, br", "Accept-Encoding": "gzip, deflate, br",
"User-Agent": "okhttp/4.12.0", "User-Agent": "okhttp/4.12.0",
"Connection": "keep-alive" "Connection": "keep-alive",
} }
response = requests.get(
"https://api.rightmove.co.uk/api/property-listing",
params=params,
headers=headers,
verify=False,
)
if response.status_code != 200:
raise Exception("Failed due to: ", response.text)
return response.json() async with aiohttp.ClientSession() as session:
async with session.get(
"https://api.rightmove.co.uk/api/property-listing",
params=params,
headers=headers,
) as response:
if response.status != 200:
raise Exception(f"Failed due to: {await response.text()}")
return await response.json()