From 24bf44caf95cf4abde96fa3ab892089c9c638dd1 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sun, 1 Jun 2025 00:12:31 +0000 Subject: [PATCH] trust env when fetching details and listings. this allows setting proxy env vars. I use this in conjunction with a tor proxy to avoid ip-bans from rightmove --- crawler/rec/query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crawler/rec/query.py b/crawler/rec/query.py index 600bebb..edeed2c 100644 --- a/crawler/rec/query.py +++ b/crawler/rec/query.py @@ -39,7 +39,7 @@ async def detail_query(detail_id: int): "appVersion": "3.70.0", } url = f"https://api.rightmove.co.uk/api/property/{detail_id}" - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(trust_env=True) as session: async with session.get(url, params=params, headers=headers) as response: if response.status != 200: raise Exception( @@ -106,7 +106,7 @@ async def listing_query( "Connection": "keep-alive", } - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(trust_env=True) as session: async with session.get( "https://api.rightmove.co.uk/api/property-listing", params=params,