diff --git a/crawler/1_dump_listings.py b/crawler/1_dump_listings.py index aeef2c5..38f65cc 100644 --- a/crawler/1_dump_listings.py +++ b/crawler/1_dump_listings.py @@ -6,10 +6,9 @@ import logging import pathlib from typing import Any from listing_processor import ListingProcessor -from rec.query import detail_query, listing_query, QueryParameters +from rec.query import listing_query, QueryParameters from rec.districts import get_districts from repositories import ListingRepository -import requests from tqdm.asyncio import tqdm from data_access import Listing from models import Listing as modelListing diff --git a/crawler/3_dump_images.py b/crawler/3_dump_images.py index 1c8f3f5..afc3fd5 100644 --- a/crawler/3_dump_images.py +++ b/crawler/3_dump_images.py @@ -1,5 +1,4 @@ import asyncio -import json from pathlib import Path import aiohttp from repositories import ListingRepository diff --git a/crawler/alembic/env.py b/crawler/alembic/env.py index 7d2c55c..168bdd8 100644 --- a/crawler/alembic/env.py +++ b/crawler/alembic/env.py @@ -1,12 +1,8 @@ from logging.config import fileConfig -from sqlalchemy import engine_from_config -from sqlalchemy import pool from alembic import context -from models import Listing, User # Import all models here from database import engine -import sqlmodel from sqlmodel import SQLModel # this is the Alembic Config object, which provides diff --git a/crawler/alembic/versions/8220f657bae5_add_indices_to_commonly_searched_.py b/crawler/alembic/versions/8220f657bae5_add_indices_to_commonly_searched_.py index 5230e61..1b6500a 100644 --- a/crawler/alembic/versions/8220f657bae5_add_indices_to_commonly_searched_.py +++ b/crawler/alembic/versions/8220f657bae5_add_indices_to_commonly_searched_.py @@ -8,7 +8,6 @@ Create Date: 2025-06-30 22:54:11.706618 from typing import Sequence, Union from alembic import op -import sqlalchemy as sa # revision identifiers, used by Alembic. diff --git a/crawler/api/app.py b/crawler/api/app.py index 6f6cffd..3be40c4 100644 --- a/crawler/api/app.py +++ b/crawler/api/app.py @@ -1,35 +1,23 @@ -import asyncio -import dataclasses from datetime import datetime, timedelta import json import logging import logging.config -from pathlib import Path -import queue -from threading import Thread from typing import Annotated -import uuid from api.auth import get_current_user from api.config import DEV_TIER_ORIGINS, PROD_TIER_ORIGINS from dotenv import load_dotenv -from fastapi import Depends, FastAPI, HTTPException, Query +from fastapi import Depends, FastAPI, Query from api.auth import User from models.listing import QueryParameters from notifications import send_notification from rec import districts from redis_repository import RedisRepository from repositories.listing_repository import ListingRepository -from repositories.listing_repository import ListingRepository from database import engine from fastapi.middleware.cors import CORSMiddleware from tasks import listing_tasks from ui_exporter import export_immoweb -from alembic import command -from alembic.config import Config -from contextlib import asynccontextmanager -from celery.exceptions import TaskRevokedError -from celery_app import app as celery_app load_dotenv() logger = logging.getLogger("uvicorn") diff --git a/crawler/api/auth.py b/crawler/api/auth.py index b510957..2aa7210 100644 --- a/crawler/api/auth.py +++ b/crawler/api/auth.py @@ -1,4 +1,3 @@ -from datetime import timedelta from api.config import AUTHENTIK_URL, OIDC_CACHE_TTL, OIDC_CLIENT_ID, OIDC_METADATA_URL from cachetools import TTLCache from fastapi import Depends, HTTPException diff --git a/crawler/data_access.py b/crawler/data_access.py index 3b9bf68..db30b0c 100644 --- a/crawler/data_access.py +++ b/crawler/data_access.py @@ -3,7 +3,7 @@ from collections import defaultdict from dataclasses import dataclass import json import pathlib -from typing import Any, List, Dict +from typing import Any, List from models.listing import ListingSite, PriceHistoryItem from rec import floorplan, routing import re diff --git a/crawler/main.py b/crawler/main.py index fbe10ea..6c05f4e 100644 --- a/crawler/main.py +++ b/crawler/main.py @@ -1,12 +1,10 @@ import asyncio from datetime import datetime -import json import os import pathlib import click import importlib -import listing_processor from models.listing import FurnishType, ListingType, QueryParameters from rec.districts import get_districts from data_access import Listing diff --git a/crawler/models/listing.py b/crawler/models/listing.py index 10ca7f4..024bd8b 100644 --- a/crawler/models/listing.py +++ b/crawler/models/listing.py @@ -4,11 +4,10 @@ import dataclasses from datetime import datetime, timedelta import enum import json -from pathlib import Path from typing import Any, Dict, List from pydantic import BaseModel from rec import routing -from sqlmodel import JSON, TEXT, SQLModel, Field, String +from sqlmodel import JSON, TEXT, SQLModel, Field @dataclass(frozen=True) diff --git a/crawler/notifications.py b/crawler/notifications.py index 50c7ee3..2d40276 100644 --- a/crawler/notifications.py +++ b/crawler/notifications.py @@ -1,5 +1,4 @@ from abc import abstractmethod -from enum import StrEnum import apprise from functools import lru_cache import os diff --git a/crawler/rec/query.py b/crawler/rec/query.py index aa16c33..b589876 100644 --- a/crawler/rec/query.py +++ b/crawler/rec/query.py @@ -1,10 +1,7 @@ -import asyncio -from dataclasses import dataclass -from datetime import datetime import enum from typing import Any import aiohttp -from models.listing import FurnishType, ListingType, QueryParameters +from models.listing import FurnishType, ListingType from rec import districts from tenacity import retry, stop_after_attempt, wait_random diff --git a/crawler/repositories/listing_repository.py b/crawler/repositories/listing_repository.py index b4c7e65..fa66597 100644 --- a/crawler/repositories/listing_repository.py +++ b/crawler/repositories/listing_repository.py @@ -10,7 +10,7 @@ from models.listing import ( RentListing, ) from sqlalchemy import Engine -from sqlmodel import Sequence, Session, and_, col, select +from sqlmodel import Session, select from sqlmodel.sql.expression import SelectOfScalar from tqdm import tqdm