Add services layer, tests, streaming UI, and cleanup legacy code

crawler/tests/conftest.py

@@ -0,0 +1,186 @@
+"""Shared pytest fixtures for the test suite."""
+from datetime import datetime
+from typing import AsyncGenerator, Generator
+import pytest
+from sqlalchemy import Engine
+from sqlmodel import SQLModel, Session, create_engine
+from httpx import ASGITransport, AsyncClient
+
+from models.listing import (
+    BuyListing,
+    FurnishType,
+    ListingSite,
+    RentListing,
+    Listing,
+)
+from repositories.listing_repository import ListingRepository
+from api.auth import User
+
+
+@pytest.fixture
+def in_memory_engine() -> Generator[Engine, None, None]:
+    """Create an in-memory SQLite engine for testing."""
+    engine = create_engine(
+        "sqlite:///:memory:",
+        echo=False,
+        connect_args={"check_same_thread": False},
+    )
+    SQLModel.metadata.create_all(engine)
+    yield engine
+    SQLModel.metadata.drop_all(engine)
+
+
+@pytest.fixture
+def listing_repository(in_memory_engine: Engine) -> ListingRepository:
+    """Create a ListingRepository with the in-memory engine."""
+    return ListingRepository(engine=in_memory_engine)
+
+
+@pytest.fixture
+def sample_rent_listing() -> RentListing:
+    """Create a sample RentListing for testing."""
+    return RentListing(
+        id=12345678,
+        price=2500.0,
+        number_of_bedrooms=2,
+        square_meters=65.0,
+        agency="Test Agency",
+        council_tax_band="C",
+        longitude=-0.1276,
+        latitude=51.5074,
+        price_history_json="[]",
+        listing_site=ListingSite.RIGHTMOVE,
+        last_seen=datetime.now(),
+        photo_thumbnail="https://example.com/photo.jpg",
+        floorplan_image_paths=[],
+        additional_info={"property": {"visible": True}},
+        routing_info_json=None,
+        furnish_type=FurnishType.FURNISHED,
+        available_from=datetime.now(),
+    )
+
+
+@pytest.fixture
+def sample_buy_listing() -> BuyListing:
+    """Create a sample BuyListing for testing."""
+    return BuyListing(
+        id=87654321,
+        price=450000.0,
+        number_of_bedrooms=3,
+        square_meters=95.0,
+        agency="Test Estate Agents",
+        council_tax_band="D",
+        longitude=-0.1180,
+        latitude=51.5100,
+        price_history_json="[]",
+        listing_site=ListingSite.RIGHTMOVE,
+        last_seen=datetime.now(),
+        photo_thumbnail="https://example.com/buy_photo.jpg",
+        floorplan_image_paths=[],
+        additional_info={"property": {"visible": True}},
+        routing_info_json=None,
+        service_charge=1500.0,
+        lease_left=90,
+    )
+
+
+@pytest.fixture
+def sample_rent_listings() -> list[RentListing]:
+    """Create multiple sample RentListings for testing filters."""
+    now = datetime.now()
+    return [
+        RentListing(
+            id=1,
+            price=1500.0,
+            number_of_bedrooms=1,
+            square_meters=40.0,
+            agency="Agency A",
+            council_tax_band="B",
+            longitude=-0.1,
+            latitude=51.5,
+            price_history_json="[]",
+            listing_site=ListingSite.RIGHTMOVE,
+            last_seen=now,
+            photo_thumbnail=None,
+            floorplan_image_paths=[],
+            additional_info={"property": {"visible": True}},
+            routing_info_json=None,
+            furnish_type=FurnishType.FURNISHED,
+            available_from=now,
+        ),
+        RentListing(
+            id=2,
+            price=2000.0,
+            number_of_bedrooms=2,
+            square_meters=55.0,
+            agency="Agency B",
+            council_tax_band="C",
+            longitude=-0.12,
+            latitude=51.51,
+            price_history_json="[]",
+            listing_site=ListingSite.RIGHTMOVE,
+            last_seen=now,
+            photo_thumbnail=None,
+            floorplan_image_paths=[],
+            additional_info={"property": {"visible": True}},
+            routing_info_json=None,
+            furnish_type=FurnishType.UNFURNISHED,
+            available_from=now,
+        ),
+        RentListing(
+            id=3,
+            price=3000.0,
+            number_of_bedrooms=3,
+            square_meters=80.0,
+            agency="Agency C",
+            council_tax_band="D",
+            longitude=-0.14,
+            latitude=51.52,
+            price_history_json="[]",
+            listing_site=ListingSite.RIGHTMOVE,
+            last_seen=now,
+            photo_thumbnail=None,
+            floorplan_image_paths=[],
+            additional_info={"property": {"visible": True}},
+            routing_info_json=None,
+            furnish_type=FurnishType.FURNISHED,
+            available_from=now,
+        ),
+    ]
+
+
+@pytest.fixture
+def mock_user() -> User:
+    """Create a mock user for API tests."""
+    return User(
+        sub="test-user-id",
+        email="test@example.com",
+        name="Test User",
+    )
+
+
+@pytest.fixture
+async def async_client(
+    in_memory_engine: Engine, mock_user: User
+) -> AsyncGenerator[AsyncClient, None]:
+    """Create an AsyncClient for API testing with mock auth."""
+    from api.app import app
+    from api.auth import get_current_user
+
+    # Override dependencies
+    app.dependency_overrides[get_current_user] = lambda: mock_user
+
+    # Patch the engine used by the repository
+    original_engine = None
+    try:
+        from database import engine as db_engine
+        original_engine = db_engine
+    except Exception:
+        pass
+
+    transport = ASGITransport(app=app)
+    async with AsyncClient(transport=transport, base_url="http://test") as client:
+        yield client
+
+    # Clean up dependency overrides
+    app.dependency_overrides.clear()