viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

reduce concurrency when fetching images + add retries

Browse source
This commit is contained in:
Viktor Barzin 2025-07-01 16:12:06 +00:00
parent 59c33428c2
commit 20ff91d663
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
3 changed files with 8 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

5
crawler/rec/query.py
View file

@ -6,7 +6,7 @@ from typing import Any
import aiohttp
from models.listing import FurnishType, ListingType, QueryParameters
from rec import districts
from tenacity import retry, wait_random
from tenacity import retry, stop_after_attempt, wait_random
headers = {
@ -27,7 +27,7 @@ class PropertyType(enum.StrEnum):
TERRACED = "terraced"
@retry(wait=wait_random(min=1, max=2))
@retry(wait=wait_random(min=1, max=2), stop=stop_after_attempt(3))
async def detail_query(detail_id: int) -> dict[str, Any]:
params = {
"apiApplication": "ANDROID",
@ -44,6 +44,7 @@ async def detail_query(detail_id: int) -> dict[str, Any]:
return await response.json()
@retry(wait=wait_random(min=1, max=2), stop=stop_after_attempt(3))
async def listing_query(
*,
page: int,