viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

format progress to 2 digits and add status updates before starting

Browse source
This commit is contained in:
Viktor Barzin 2025-07-27 18:47:09 +00:00
parent 91a0436f7f
commit 87efe0694c
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
1 changed files with 4 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

7
crawler/tasks/listing_tasks.py
View file

@ -21,8 +21,8 @@ logger = logging.getLogger("uvicorn.error")
@app.task(bind=True, pydantic=True) @app.task(bind=True, pydantic=True)
def dump_listings_task(self: Task, parameters_json: str) -> dict[str, Any]: def dump_listings_task(self: Task, parameters_json: str) -> dict[str, Any]:
parsed_parameters = QueryParameters.model_validate_json(parameters_json) parsed_parameters = QueryParameters.model_validate_json(parameters_json)
asyncio.run(dump_listings_full(task=self, parameters=parsed_parameters))
self.update_state(state="Starting...", meta={"progress": 0}) self.update_state(state="Starting...", meta={"progress": 0})
asyncio.run(dump_listings_full(task=self, parameters=parsed_parameters))
return {"progress": 0} return {"progress": 0}
@ -38,6 +38,7 @@ async def dump_listings_full(
"""Fetches all listings, images as well as detects floorplans""" """Fetches all listings, images as well as detects floorplans"""
repository = ListingRepository(engine) repository = ListingRepository(engine)
task.update_state(state="Identifying missing listings", meta={"progress": 0})
missing_ids = await get_missing_listing_ids(parameters, repository) missing_ids = await get_missing_listing_ids(parameters, repository)
logger.info(f"Found {len(missing_ids)} missing listings") logger.info(f"Found {len(missing_ids)} missing listings")
@ -60,7 +61,7 @@ async def dump_listings_and_monitor(
async def monitor() -> None: async def monitor() -> None:
while (progress := sum(task_progress.values())) < len(missing_ids): while (progress := sum(task_progress.values())) < len(missing_ids):
progress_ratio = progress / len(missing_ids) progress_ratio = round(progress / len(missing_ids), 2)
logger.error( logger.error(
f"Task progress: {task_progress}% ({progress} out of {len(missing_ids)})" f"Task progress: {task_progress}% ({progress} out of {len(missing_ids)})"
) )
@ -151,7 +152,7 @@ async def _fetch_listings_with_semaphore(
# we do 10 queries each with an increment in price range so we send more queries but each # we do 10 queries each with an increment in price range so we send more queries but each
# has a smaller chance of returning more than 1.5k results # has a smaller chance of returning more than 1.5k results
number_of_steps = 1 number_of_steps = 10
price_step = parameters.max_price // number_of_steps price_step = parameters.max_price // number_of_steps
for step in range(number_of_steps): for step in range(number_of_steps):