diff --git a/pyproject.toml b/pyproject.toml index 85c7aeb..29e3bb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,6 @@ authors = ["Kadir Tugan "] python = ">3.11" requests = "^2.31.0" cachetools = "^5.3.2" -tqdm = "^4.66.2" pillow = "^10.2.0" numpy = "^1.26.4" pytesseract = "^0.3.10" @@ -26,7 +25,6 @@ pyjwt = "^2.10.1" cryptography = "^45.0.4" celery = "^5.5.3" redis = "^6.2.0" -watchdog = "^6.0.0" apprise = "^1.9.3" opentelemetry-api = "^1.36.0" opentelemetry-sdk = "^1.36.0" @@ -83,5 +81,5 @@ strict_optional = true plugins = ["pydantic.mypy"] [[tool.mypy.overrides]] -module = ["pytesseract.*", "cv2.*", "celery.*", "tqdm.*", "aiohttp.*", "aiohttp_socks.*", "tenacity.*", "pandas.*", "numpy.*", "PIL.*", "sqlmodel.*", "sqlalchemy.*", "alembic.*", "apprise.*", "opentelemetry.*", "webauthn.*"] +module = ["pytesseract.*", "cv2.*", "celery.*", "aiohttp.*", "aiohttp_socks.*", "tenacity.*", "pandas.*", "numpy.*", "PIL.*", "sqlmodel.*", "sqlalchemy.*", "alembic.*", "apprise.*", "opentelemetry.*", "webauthn.*"] ignore_missing_imports = true \ No newline at end of file diff --git a/repositories/listing_repository.py b/repositories/listing_repository.py index f73e571..b02f7e8 100644 --- a/repositories/listing_repository.py +++ b/repositories/listing_repository.py @@ -12,7 +12,6 @@ from models.listing import ( ) from sqlalchemy import Engine, func, select as sa_select from sqlmodel import Session, select -from tqdm import tqdm logger = logging.getLogger("uvicorn.error") @@ -263,7 +262,8 @@ class ListingRepository: models = [] failed_to_upsert = [] with Session(self.engine) as session: - for listing in tqdm(listings, desc="Upserting listings"): + logger.info("Upserting %d listings", len(listings)) + for listing in listings: # Convert legacy Listing to the appropriate SQLModel entity try: model_listing = await self._get_concrete_listing(listing) diff --git a/services/floorplan_detector.py b/services/floorplan_detector.py index 490b88f..01d28ec 100644 --- a/services/floorplan_detector.py +++ b/services/floorplan_detector.py @@ -1,10 +1,12 @@ """Floorplan detector service - OCR-based square meter detection.""" import asyncio +import logging from config.scraper_config import MAX_OCR_WORKERS from models import Listing from rec import floorplan from repositories.listing_repository import ListingRepository -from tqdm.asyncio import tqdm + +logger = logging.getLogger(__name__) async def detect_floorplan(repository: ListingRepository) -> None: @@ -12,13 +14,15 @@ async def detect_floorplan(repository: ListingRepository) -> None: listings = await repository.get_listings() semaphore = asyncio.Semaphore(MAX_OCR_WORKERS) + logger.info("Detecting floorplans for %d listings", len(listings)) updated_listings = [ listing - for listing in await tqdm.gather( + for listing in await asyncio.gather( *[_calculate_sqm_ocr(listing, semaphore) for listing in listings] ) if listing is not None ] + logger.info("Finished floorplan detection, %d listings updated", len(updated_listings)) await repository.upsert_listings(updated_listings) diff --git a/services/image_fetcher.py b/services/image_fetcher.py index 085cba8..9562db9 100644 --- a/services/image_fetcher.py +++ b/services/image_fetcher.py @@ -8,7 +8,6 @@ import aiohttp from rec.exceptions import FloorplanDownloadError from repositories import ListingRepository from tenacity import retry, stop_after_attempt, wait_random -from tqdm.asyncio import tqdm from models import Listing @@ -26,13 +25,15 @@ async def dump_images( ) -> None: """Download floorplan images for all listings.""" listings = await repository.get_listings() + logger.info("Downloading images for %d listings", len(listings)) async with aiohttp.ClientSession() as session: - updated_listings = await tqdm.gather( + updated_listings = await asyncio.gather( *[ dump_images_for_listing(listing, image_base_path, session=session) for listing in listings ] ) + logger.info("Finished downloading images for %d listings", len(listings)) await repository.upsert_listings( [listing for listing in updated_listings if listing is not None] ) diff --git a/services/route_calculator.py b/services/route_calculator.py index 1e04902..27c7c25 100644 --- a/services/route_calculator.py +++ b/services/route_calculator.py @@ -1,10 +1,13 @@ """Route calculator service - calculates transit routes using Google Maps API.""" +import asyncio +import logging from models.listing import DestinationMode, Route, RouteLegStep from repositories.listing_repository import ListingRepository -from tqdm.asyncio import tqdm from rec import routing from models import Listing +logger = logging.getLogger(__name__) + def _parse_duration(duration_str: str) -> int: """Parse a duration string like '123s' to integer seconds.""" @@ -24,11 +27,11 @@ async def calculate_route( listings = listings[:limit] destination_mode = DestinationMode(destination_address, travel_mode) - updated_listings = await tqdm.gather( + logger.info("Calculating routes for %d listings", len(listings)) + updated_listings = await asyncio.gather( *[update_routing_info(listing, destination_mode) for listing in listings], - total=len(listings), - desc="Updating routing info", ) + logger.info("Finished route calculation for %d listings", len(listings)) await repository.upsert_listings( [listing for listing in updated_listings if listing is not None] )