Add services layer, tests, streaming UI, and cleanup legacy code

crawler/tests/test_listing_geojson.py

@@ -0,0 +1,299 @@
+"""Tests for the listing_geojson API endpoint and QueryParameters parsing."""
+import json
+import pytest
+from datetime import datetime
+from unittest.mock import patch, MagicMock, AsyncMock
+
+
+class TestQueryParametersModel:
+    """Test QueryParameters model directly."""
+
+    def test_datetime_parsing_z_suffix(self):
+        """Test that datetime with Z suffix is parsed correctly."""
+        from models.listing import QueryParameters, ListingType
+
+        params = QueryParameters(
+            listing_type=ListingType.RENT,
+            let_date_available_from="2026-02-01T11:33:01.248Z",
+        )
+        assert params.let_date_available_from is not None
+        assert params.let_date_available_from.year == 2026
+
+    def test_datetime_parsing_offset(self):
+        """Test that datetime with offset is parsed correctly."""
+        from models.listing import QueryParameters, ListingType
+
+        params = QueryParameters(
+            listing_type=ListingType.RENT,
+            let_date_available_from="2026-02-01T11:33:01.248+00:00",
+        )
+        assert params.let_date_available_from is not None
+
+    def test_defaults_work(self):
+        """Test that default values are applied correctly."""
+        from models.listing import QueryParameters, ListingType
+
+        params = QueryParameters(listing_type=ListingType.RENT)
+        assert params.min_bedrooms == 1
+        assert params.max_bedrooms == 999
+        assert params.min_price == 0
+        assert params.max_price == 10_000_000
+        assert params.district_names == set()
+        assert params.let_date_available_from is None
+
+    def test_full_frontend_params(self):
+        """Test with all parameters as sent by frontend."""
+        from models.listing import QueryParameters, ListingType
+
+        params = QueryParameters(
+            listing_type=ListingType.RENT,
+            min_bedrooms=1,
+            max_bedrooms=3,
+            max_price=3000,
+            min_price=2000,
+            min_sqm=50,
+            last_seen_days=28,
+            let_date_available_from="2026-02-01T11:19:22.072Z",
+        )
+        assert params.listing_type == ListingType.RENT
+        assert params.min_bedrooms == 1
+        assert params.max_bedrooms == 3
+        assert params.min_sqm == 50
+
+
+class TestGetQueryParametersDependency:
+    """Test the get_query_parameters FastAPI dependency."""
+
+    def test_parses_datetime_correctly(self):
+        """Test that the dependency parses datetime Z suffix."""
+        from api.app import get_query_parameters
+        from models.listing import ListingType
+
+        params = get_query_parameters(
+            listing_type=ListingType.RENT,
+            let_date_available_from=datetime(2026, 2, 1, 11, 33, 1),
+        )
+        assert params.let_date_available_from is not None
+
+    def test_defaults_applied(self):
+        """Test that defaults are applied when not provided."""
+        from api.app import get_query_parameters
+        from models.listing import ListingType
+
+        params = get_query_parameters(listing_type=ListingType.RENT)
+        assert params.min_bedrooms == 1
+        assert params.max_bedrooms == 999
+
+
+class TestListingGeoJsonEndpoint:
+    """Test the /api/listing_geojson endpoint."""
+
+    @pytest.fixture
+    def client(self):
+        """Create test client with mocked auth."""
+        from fastapi.testclient import TestClient
+        from api.app import app, get_current_user
+        from api.auth import User
+
+        # Override auth dependency
+        async def mock_auth():
+            return User(email="test@example.com", name="Test User")
+
+        app.dependency_overrides[get_current_user] = mock_auth
+        yield TestClient(app)
+        app.dependency_overrides.clear()
+
+    @pytest.fixture
+    def mock_export(self):
+        """Mock the export service."""
+        with patch("api.app.export_service.export_to_geojson") as mock:
+            mock.return_value = MagicMock(
+                data={"type": "FeatureCollection", "features": [{"type": "Feature"}]}
+            )
+            yield mock
+
+    def test_minimal_params_no_422(self, client, mock_export):
+        """Test that minimal params don't cause 422."""
+        response = client.get("/api/listing_geojson?listing_type=RENT")
+        assert response.status_code != 422, f"Got 422: {response.json()}"
+
+    def test_with_datetime_z_suffix_no_422(self, client, mock_export):
+        """Test datetime parsing with Z suffix doesn't cause 422."""
+        response = client.get(
+            "/api/listing_geojson?"
+            "listing_type=RENT"
+            "&let_date_available_from=2026-02-01T11:33:01.248Z"
+        )
+        assert response.status_code != 422, f"Got 422: {response.json()}"
+
+    def test_full_frontend_params_no_422(self, client, mock_export):
+        """Test with all parameters as sent by frontend."""
+        response = client.get(
+            "/api/listing_geojson?"
+            "listing_type=RENT"
+            "&min_bedrooms=1"
+            "&max_bedrooms=3"
+            "&max_price=3000"
+            "&min_price=2000"
+            "&min_sqm=50"
+            "&last_seen_days=28"
+            "&let_date_available_from=2026-02-01T11:19:22.072Z"
+        )
+        assert response.status_code != 422, f"Got 422: {response.json()}"
+
+    def test_returns_geojson_structure(self, client, mock_export):
+        """Test that endpoint returns proper GeoJSON structure."""
+        response = client.get("/api/listing_geojson?listing_type=RENT")
+        assert response.status_code == 200
+        data = response.json()
+        assert "type" in data
+        assert data["type"] == "FeatureCollection"
+        assert "features" in data
+
+
+class TestStreamingEndpoint:
+    """Test the /api/listing_geojson/stream endpoint."""
+
+    @pytest.fixture
+    def client(self):
+        """Create test client with mocked auth."""
+        from fastapi.testclient import TestClient
+        from api.app import app
+        from api.auth import get_current_user, User
+
+        async def mock_auth():
+            return User(sub="test-id", email="test@example.com", name="Test User")
+
+        app.dependency_overrides[get_current_user] = mock_auth
+        yield TestClient(app)
+        app.dependency_overrides.clear()
+
+    @pytest.fixture
+    def mock_repository(self):
+        """Mock the repository methods."""
+        with patch("api.app.ListingRepository") as MockRepo:
+            mock_instance = MagicMock()
+            mock_instance.count_listings.return_value = 3
+            mock_instance.stream_listings_optimized.return_value = iter([
+                {
+                    'id': 1,
+                    'price': 2000.0,
+                    'number_of_bedrooms': 2,
+                    'square_meters': 50.0,
+                    'longitude': -0.1,
+                    'latitude': 51.5,
+                    'photo_thumbnail': 'https://example.com/1.jpg',
+                    'last_seen': datetime.now(),
+                    'agency': 'Test Agency',
+                    'price_history_json': '[]',
+                    'available_from': datetime.now(),
+                },
+                {
+                    'id': 2,
+                    'price': 2500.0,
+                    'number_of_bedrooms': 2,
+                    'square_meters': 60.0,
+                    'longitude': -0.12,
+                    'latitude': 51.51,
+                    'photo_thumbnail': 'https://example.com/2.jpg',
+                    'last_seen': datetime.now(),
+                    'agency': 'Test Agency 2',
+                    'price_history_json': '[]',
+                    'available_from': None,
+                },
+                {
+                    'id': 3,
+                    'price': 3000.0,
+                    'number_of_bedrooms': 3,
+                    'square_meters': None,
+                    'longitude': -0.14,
+                    'latitude': 51.52,
+                    'photo_thumbnail': None,
+                    'last_seen': datetime.now(),
+                    'agency': None,
+                    'price_history_json': '[{"first_seen": "2026-01-01", "last_seen": "2026-01-15", "price": 2800}]',
+                    'available_from': None,
+                },
+            ])
+            MockRepo.return_value = mock_instance
+            yield mock_instance
+
+    def test_streaming_returns_ndjson(self, client, mock_repository):
+        """Test that streaming endpoint returns NDJSON format."""
+        response = client.get("/api/listing_geojson/stream?listing_type=RENT&limit=10")
+        assert response.status_code == 200
+        assert response.headers["content-type"] == "application/x-ndjson"
+
+    def test_streaming_metadata_includes_total_expected(self, client, mock_repository):
+        """Test that first line includes total_expected count."""
+        response = client.get("/api/listing_geojson/stream?listing_type=RENT&limit=10")
+        lines = response.text.strip().split("\n")
+        assert len(lines) >= 1
+
+        metadata = json.loads(lines[0])
+        assert metadata["type"] == "metadata"
+        assert "total_expected" in metadata
+        assert metadata["total_expected"] == 3
+        assert "batch_size" in metadata
+
+    def test_streaming_returns_batches_and_complete(self, client, mock_repository):
+        """Test that streaming returns batch and complete messages."""
+        response = client.get("/api/listing_geojson/stream?listing_type=RENT&limit=10")
+        lines = response.text.strip().split("\n")
+
+        # Parse all lines
+        messages = [json.loads(line) for line in lines]
+
+        # First should be metadata
+        assert messages[0]["type"] == "metadata"
+
+        # Should have at least one batch
+        batch_messages = [m for m in messages if m["type"] == "batch"]
+        assert len(batch_messages) >= 1
+
+        # Last should be complete
+        assert messages[-1]["type"] == "complete"
+        assert "total" in messages[-1]
+
+    def test_streaming_features_have_correct_structure(self, client, mock_repository):
+        """Test that streamed features have correct GeoJSON structure."""
+        response = client.get("/api/listing_geojson/stream?listing_type=RENT&batch_size=10&limit=10")
+        lines = response.text.strip().split("\n")
+        messages = [json.loads(line) for line in lines]
+
+        batch_messages = [m for m in messages if m["type"] == "batch"]
+        assert len(batch_messages) >= 1
+
+        features = batch_messages[0]["features"]
+        assert len(features) > 0
+
+        feature = features[0]
+        assert feature["type"] == "Feature"
+        assert "properties" in feature
+        assert "geometry" in feature
+        assert feature["geometry"]["type"] == "Point"
+        assert "coordinates" in feature["geometry"]
+
+        # Check properties
+        props = feature["properties"]
+        assert "total_price" in props
+        assert "rooms" in props
+        assert "url" in props
+        assert "last_seen" in props
+
+    def test_streaming_handles_null_square_meters(self, client, mock_repository):
+        """Test that null square_meters doesn't cause errors."""
+        response = client.get("/api/listing_geojson/stream?listing_type=RENT&batch_size=10&limit=10")
+        assert response.status_code == 200
+
+        lines = response.text.strip().split("\n")
+        messages = [json.loads(line) for line in lines]
+
+        # Find feature with id=3 (has null square_meters)
+        for msg in messages:
+            if msg["type"] == "batch":
+                for feature in msg["features"]:
+                    if feature["properties"]["url"].endswith("/3"):
+                        assert feature["properties"]["qm"] is None
+                        assert feature["properties"]["qmprice"] is None
+