Remove watchdog and tqdm dependencies, replace with logging
- Remove watchdog (unused) and tqdm from pyproject.toml dependencies - Replace tqdm.gather() with asyncio.gather() + logger.info() in image_fetcher, floorplan_detector, and route_calculator services - Replace tqdm progress bar with logger.info() in listing_repository - Remove tqdm from mypy ignore_missing_imports overrides
This commit is contained in:
parent
d488208a26
commit
cde3540a1e
5 changed files with 19 additions and 13 deletions
|
|
@ -1,10 +1,12 @@
|
|||
"""Floorplan detector service - OCR-based square meter detection."""
|
||||
import asyncio
|
||||
import logging
|
||||
from config.scraper_config import MAX_OCR_WORKERS
|
||||
from models import Listing
|
||||
from rec import floorplan
|
||||
from repositories.listing_repository import ListingRepository
|
||||
from tqdm.asyncio import tqdm
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def detect_floorplan(repository: ListingRepository) -> None:
|
||||
|
|
@ -12,13 +14,15 @@ async def detect_floorplan(repository: ListingRepository) -> None:
|
|||
listings = await repository.get_listings()
|
||||
semaphore = asyncio.Semaphore(MAX_OCR_WORKERS)
|
||||
|
||||
logger.info("Detecting floorplans for %d listings", len(listings))
|
||||
updated_listings = [
|
||||
listing
|
||||
for listing in await tqdm.gather(
|
||||
for listing in await asyncio.gather(
|
||||
*[_calculate_sqm_ocr(listing, semaphore) for listing in listings]
|
||||
)
|
||||
if listing is not None
|
||||
]
|
||||
logger.info("Finished floorplan detection, %d listings updated", len(updated_listings))
|
||||
await repository.upsert_listings(updated_listings)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import aiohttp
|
|||
from rec.exceptions import FloorplanDownloadError
|
||||
from repositories import ListingRepository
|
||||
from tenacity import retry, stop_after_attempt, wait_random
|
||||
from tqdm.asyncio import tqdm
|
||||
|
||||
from models import Listing
|
||||
|
||||
|
|
@ -26,13 +25,15 @@ async def dump_images(
|
|||
) -> None:
|
||||
"""Download floorplan images for all listings."""
|
||||
listings = await repository.get_listings()
|
||||
logger.info("Downloading images for %d listings", len(listings))
|
||||
async with aiohttp.ClientSession() as session:
|
||||
updated_listings = await tqdm.gather(
|
||||
updated_listings = await asyncio.gather(
|
||||
*[
|
||||
dump_images_for_listing(listing, image_base_path, session=session)
|
||||
for listing in listings
|
||||
]
|
||||
)
|
||||
logger.info("Finished downloading images for %d listings", len(listings))
|
||||
await repository.upsert_listings(
|
||||
[listing for listing in updated_listings if listing is not None]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
"""Route calculator service - calculates transit routes using Google Maps API."""
|
||||
import asyncio
|
||||
import logging
|
||||
from models.listing import DestinationMode, Route, RouteLegStep
|
||||
from repositories.listing_repository import ListingRepository
|
||||
from tqdm.asyncio import tqdm
|
||||
from rec import routing
|
||||
from models import Listing
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _parse_duration(duration_str: str) -> int:
|
||||
"""Parse a duration string like '123s' to integer seconds."""
|
||||
|
|
@ -24,11 +27,11 @@ async def calculate_route(
|
|||
listings = listings[:limit]
|
||||
|
||||
destination_mode = DestinationMode(destination_address, travel_mode)
|
||||
updated_listings = await tqdm.gather(
|
||||
logger.info("Calculating routes for %d listings", len(listings))
|
||||
updated_listings = await asyncio.gather(
|
||||
*[update_routing_info(listing, destination_mode) for listing in listings],
|
||||
total=len(listings),
|
||||
desc="Updating routing info",
|
||||
)
|
||||
logger.info("Finished route calculation for %d listings", len(listings))
|
||||
await repository.upsert_listings(
|
||||
[listing for listing in updated_listings if listing is not None]
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue