viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Flatten repo structure: move crawler/ to root, remove vqa/ and immoweb/

Browse source 
The crawler subdirectory was the only active project. Moving it to the
repo root simplifies paths and removes the unnecessary nesting. The
vqa/ and immoweb/ directories were legacy/unused and have been removed.

Updated .drone.yml, .gitignore, .claude/ docs, and skills to reflect
the new flat structure.
This commit is contained in:
Viktor Barzin 2026-02-07 23:01:20 +00:00
parent e2247be700
commit eafbc1ac52
No known key found for this signature in database
GPG key ID: 0EB088298288D958
221 changed files with 70 additions and 146140 deletions
Download patch file Download diff file Expand all files Collapse all files

1
tests/integration/__init__.py Normal file
View file

@ -0,0 +1 @@
# Integration tests package

315
tests/integration/test_api.py Normal file
View file

@ -0,0 +1,315 @@
"""Integration tests for API endpoints."""
import json
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from httpx import AsyncClient
from api.auth import User
class TestStatusEndpoint:
"""Tests for the /api/status endpoint."""
async def test_status_endpoint_returns_ok(
self, async_client: AsyncClient
) -> None:
"""Test that status endpoint returns OK status."""
response = await async_client.get("/api/status")
assert response.status_code == 200
assert response.json() == {"status": "OK"}
class TestListingEndpoint:
"""Tests for the /api/listing endpoint."""
async def test_listing_endpoint_requires_auth(self) -> None:
"""Test that listing endpoint requires authentication."""
from api.app import app
from httpx import ASGITransport, AsyncClient
# Clear any dependency overrides to test auth requirement
app.dependency_overrides.clear()
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get("/api/listing")
# Should return 401 or 403 without valid auth
assert response.status_code in (401, 403)
async def test_listing_endpoint_with_auth(
self, async_client: AsyncClient
) -> None:
"""Test that listing endpoint works with authentication."""
# Mock the repository to return empty list
with patch(
"api.app.ListingRepository.get_listings",
new_callable=AsyncMock,
return_value=[],
):
response = await async_client.get("/api/listing")
assert response.status_code == 200
data = response.json()
assert "listings" in data
class TestListingGeoJsonEndpoint:
"""Tests for the /api/listing_geojson endpoint."""
async def test_listing_geojson_requires_auth(self) -> None:
"""Test that listing_geojson endpoint requires authentication."""
from api.app import app
from httpx import ASGITransport, AsyncClient
# Clear any dependency overrides to test auth requirement
app.dependency_overrides.clear()
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get(
"/api/listing_geojson",
params={"listing_type": "RENT"},
)
# Should return 401 or 403 without valid auth
assert response.status_code in (401, 403)
async def test_listing_geojson_with_filters(
self, async_client: AsyncClient
) -> None:
"""Test that listing_geojson accepts filter parameters."""
mock_result = MagicMock()
mock_result.data = {"type": "FeatureCollection", "features": []}
with patch(
"api.app.export_service.export_to_geojson",
new_callable=AsyncMock,
return_value=mock_result,
):
response = await async_client.get(
"/api/listing_geojson",
params={
"listing_type": "RENT",
"min_bedrooms": 2,
"max_bedrooms": 3,
"min_price": 1500,
"max_price": 3000,
},
)
assert response.status_code == 200
data = response.json()
assert data["type"] == "FeatureCollection"
class TestGetDistrictsEndpoint:
"""Tests for the /api/get_districts endpoint."""
async def test_get_districts_requires_auth(self) -> None:
"""Test that get_districts endpoint requires authentication."""
from api.app import app
from httpx import ASGITransport, AsyncClient
# Clear any dependency overrides to test auth requirement
app.dependency_overrides.clear()
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get("/api/get_districts")
# Should return 401 or 403 without valid auth
assert response.status_code in (401, 403)
async def test_get_districts_returns_dict(
self, async_client: AsyncClient
) -> None:
"""Test that get_districts returns a dictionary of districts."""
response = await async_client.get("/api/get_districts")
assert response.status_code == 200
data = response.json()
assert isinstance(data, dict)
# Check some known districts exist
assert "London" in data
assert "Westminster" in data
assert "Camden" in data
async def test_get_districts_values_are_region_ids(
self, async_client: AsyncClient
) -> None:
"""Test that district values are REGION identifiers."""
response = await async_client.get("/api/get_districts")
data = response.json()
# All values should be REGION^... format
for district_name, region_id in data.items():
assert region_id.startswith("REGION^"), (
f"District {district_name} has invalid region ID: {region_id}"
)
class TestRefreshListingsEndpoint:
"""Tests for the /api/refresh_listings endpoint."""
async def test_refresh_listings_requires_auth(self) -> None:
"""Test that refresh_listings endpoint requires authentication."""
from api.app import app
from httpx import ASGITransport, AsyncClient
# Clear any dependency overrides to test auth requirement
app.dependency_overrides.clear()
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.post(
"/api/refresh_listings",
params={"listing_type": "RENT"},
)
# Should return 401 or 403 without valid auth
assert response.status_code in (401, 403)
class TestTaskStatusEndpoint:
"""Tests for the /api/task_status endpoint."""
async def test_task_status_requires_auth(self) -> None:
"""Test that task_status endpoint requires authentication."""
from api.app import app
from httpx import ASGITransport, AsyncClient
# Clear any dependency overrides to test auth requirement
app.dependency_overrides.clear()
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get(
"/api/task_status",
params={"task_id": "test-task-id"},
)
# Should return 401 or 403 without valid auth
assert response.status_code in (401, 403)
class TestStreamListingGeoJsonEndpoint:
"""Tests for the /api/listing_geojson/stream endpoint."""
async def test_stream_returns_ndjson_with_metadata(
self, async_client: AsyncClient
) -> None:
"""Test that the stream endpoint returns valid NDJSON starting with a metadata message."""
fake_features = [
{"type": "Feature", "properties": {"id": 1}, "geometry": {"type": "Point", "coordinates": [0, 0]}},
{"type": "Feature", "properties": {"id": 2}, "geometry": {"type": "Point", "coordinates": [1, 1]}},
]
with patch("api.app.get_cached_count", return_value=2), \
patch("api.app.get_cached_features", return_value=iter([fake_features])):
response = await async_client.get(
"/api/listing_geojson/stream",
params={"listing_type": "RENT", "batch_size": 50},
)
assert response.status_code == 200
assert response.headers["content-type"] == "application/x-ndjson"
lines = [line for line in response.text.strip().split("\n") if line]
assert len(lines) >= 2 # at least metadata + complete
metadata = json.loads(lines[0])
assert metadata["type"] == "metadata"
assert "batch_size" in metadata
assert "total_expected" in metadata
complete = json.loads(lines[-1])
assert complete["type"] == "complete"
assert "total" in complete
async def test_stream_cache_hit_path(
self, async_client: AsyncClient
) -> None:
"""Test that cache-hit path returns cached: True in metadata."""
fake_features = [
{"type": "Feature", "properties": {"id": 1}, "geometry": {"type": "Point", "coordinates": [0, 0]}},
]
with patch("api.app.get_cached_count", return_value=1), \
patch("api.app.get_cached_features", return_value=iter([fake_features])):
response = await async_client.get(
"/api/listing_geojson/stream",
params={"listing_type": "RENT"},
)
assert response.status_code == 200
lines = [line for line in response.text.strip().split("\n") if line]
metadata = json.loads(lines[0])
assert metadata["cached"] is True
assert metadata["total_expected"] == 1
batch_msg = json.loads(lines[1])
assert batch_msg["type"] == "batch"
assert len(batch_msg["features"]) == 1
async def test_stream_cache_miss_path(
self, async_client: AsyncClient
) -> None:
"""Test that cache-miss path queries DB and returns cached: False."""
from datetime import datetime
fake_rows = [
{
"id": 100,
"price": 2000.0,
"number_of_bedrooms": 2,
"square_meters": 50.0,
"longitude": -0.1,
"latitude": 51.5,
"photo_thumbnail": None,
"last_seen": datetime(2024, 1, 1),
"agency": "Test Agency",
"price_history_json": "[]",
"available_from": None,
},
]
mock_repo = MagicMock()
mock_repo.count_listings.return_value = 1
mock_repo.stream_listings_optimized.return_value = iter(fake_rows)
with patch("api.app.get_cached_count", return_value=None), \
patch("api.app.ListingRepository", return_value=mock_repo), \
patch("api.app.cache_features_batch"):
response = await async_client.get(
"/api/listing_geojson/stream",
params={"listing_type": "RENT"},
)
assert response.status_code == 200
lines = [line for line in response.text.strip().split("\n") if line]
metadata = json.loads(lines[0])
assert metadata["cached"] is False
assert metadata["total_expected"] == 1
batch_msg = json.loads(lines[1])
assert batch_msg["type"] == "batch"
assert len(batch_msg["features"]) == 1
assert batch_msg["features"][0]["type"] == "Feature"
assert batch_msg["features"][0]["properties"]["total_price"] == 2000.0
complete = json.loads(lines[-1])
assert complete["type"] == "complete"
assert complete["total"] == 1
async def test_stream_with_limit(
self, async_client: AsyncClient
) -> None:
"""Test that the limit parameter caps the number of streamed features."""
fake_features = [
{"type": "Feature", "properties": {"id": i}, "geometry": {"type": "Point", "coordinates": [0, 0]}}
for i in range(5)
]
with patch("api.app.get_cached_count", return_value=5), \
patch("api.app.get_cached_features", return_value=iter([fake_features])):
response = await async_client.get(
"/api/listing_geojson/stream",
params={"listing_type": "RENT", "limit": 3},
)
assert response.status_code == 200
lines = [line for line in response.text.strip().split("\n") if line]
metadata = json.loads(lines[0])
assert metadata["total_expected"] == 3
complete = json.loads(lines[-1])
assert complete["type"] == "complete"
assert complete["total"] == 3

311
tests/integration/test_throttle_integration.py Normal file
View file

@ -0,0 +1,311 @@
"""Integration tests for throttle detection and circuit breaker."""
import asyncio
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from aiohttp import ClientResponse
from config.scraper_config import ScraperConfig
from rec.exceptions import (
CircuitBreakerOpenError,
RateLimitError,
ServiceUnavailableError,
ThrottlingError,
)
from rec.query import (
detail_query,
listing_query,
probe_query,
get_circuit_breaker,
reset_circuit_breaker,
)
from rec.throttle_detector import reset_throttle_metrics, get_throttle_metrics
from rec.circuit_breaker import CircuitBreaker, CircuitState
from models.listing import ListingType
@pytest.fixture
def config() -> ScraperConfig:
"""Create a test configuration."""
return ScraperConfig(
max_concurrent_requests=5,
request_delay_ms=10,
slow_response_threshold=2.0,
enable_circuit_breaker=True,
circuit_breaker_failure_threshold=3,
circuit_breaker_recovery_timeout=0.5,
)
@pytest.fixture(autouse=True)
def reset_globals() -> None:
"""Reset global state before each test."""
reset_throttle_metrics()
reset_circuit_breaker()
class MockResponse:
"""Mock aiohttp response."""
def __init__(
self,
status: int = 200,
json_data: dict | None = None,
text: str = "",
):
self.status = status
self._json_data = json_data or {}
self._text = text
async def json(self) -> dict:
return self._json_data
async def text(self) -> str:
return self._text
async def __aenter__(self) -> "MockResponse":
return self
async def __aexit__(self, *args: object) -> None:
pass
class TestThrottlingRetryBehavior:
"""Test retry behavior for throttling errors."""
@pytest.mark.asyncio
async def test_rate_limit_triggers_retry(self, config: ScraperConfig) -> None:
"""Test that 429 responses trigger retry with backoff."""
call_count = 0
def mock_get(*args: object, **kwargs: object) -> MockResponse:
nonlocal call_count
call_count += 1
if call_count < 3:
return MockResponse(status=429)
return MockResponse(
status=200,
json_data={"totalAvailableResults": 10, "properties": []},
)
mock_session = MagicMock()
mock_session.get = mock_get
# Mock district lookup
with patch("rec.query.districts.get_districts", return_value={"Test": "LOC1"}):
# The retry decorator will catch RateLimitError and retry
# We need to patch the tenacity wait to speed up the test
with patch("tenacity.wait_exponential.__call__", return_value=0):
result = await probe_query(
session=mock_session,
channel=ListingType.RENT,
min_bedrooms=1,
max_bedrooms=2,
radius=1.0,
min_price=1000,
max_price=2000,
district="Test",
config=config,
)
assert result["totalAvailableResults"] == 10
assert call_count == 3
@pytest.mark.asyncio
async def test_service_unavailable_triggers_retry(
self, config: ScraperConfig
) -> None:
"""Test that 503 responses trigger retry."""
call_count = 0
def mock_get(*args: object, **kwargs: object) -> MockResponse:
nonlocal call_count
call_count += 1
if call_count < 2:
return MockResponse(status=503)
return MockResponse(
status=200,
json_data={"totalAvailableResults": 5, "properties": []},
)
mock_session = MagicMock()
mock_session.get = mock_get
with patch("rec.query.districts.get_districts", return_value={"Test": "LOC1"}):
with patch("tenacity.wait_exponential.__call__", return_value=0):
result = await probe_query(
session=mock_session,
channel=ListingType.RENT,
min_bedrooms=1,
max_bedrooms=2,
radius=1.0,
min_price=1000,
max_price=2000,
district="Test",
config=config,
)
assert call_count == 2
class TestCircuitBreakerIntegration:
"""Test circuit breaker integration with queries."""
@pytest.mark.asyncio
async def test_circuit_breaker_opens_after_failures(
self, config: ScraperConfig
) -> None:
"""Test that circuit breaker opens after consecutive failures."""
call_count = 0
def mock_get(*args: object, **kwargs: object) -> MockResponse:
nonlocal call_count
call_count += 1
return MockResponse(status=429)
mock_session = MagicMock()
mock_session.get = mock_get
with patch("rec.query.districts.get_districts", return_value={"Test": "LOC1"}):
# After enough failures, circuit should open
with pytest.raises((RateLimitError, CircuitBreakerOpenError)):
with patch("tenacity.wait_exponential.__call__", return_value=0):
await probe_query(
session=mock_session,
channel=ListingType.RENT,
min_bedrooms=1,
max_bedrooms=2,
radius=1.0,
min_price=1000,
max_price=2000,
district="Test",
config=config,
)
# Check circuit breaker state
cb = get_circuit_breaker(config)
assert cb is not None
# After many failures, the circuit should be open
assert cb.failure_count >= config.circuit_breaker_failure_threshold
@pytest.mark.asyncio
async def test_circuit_breaker_blocks_requests_when_open(
self, config: ScraperConfig
) -> None:
"""Test that open circuit breaker blocks requests immediately."""
# Force open the circuit breaker
cb = get_circuit_breaker(config)
assert cb is not None
for _ in range(config.circuit_breaker_failure_threshold):
cb.record_failure()
assert cb.is_open
mock_session = MagicMock()
with patch("rec.query.districts.get_districts", return_value={"Test": "LOC1"}):
with pytest.raises(CircuitBreakerOpenError):
await probe_query(
session=mock_session,
channel=ListingType.RENT,
min_bedrooms=1,
max_bedrooms=2,
radius=1.0,
min_price=1000,
max_price=2000,
district="Test",
config=config,
)
class TestMetricsTracking:
"""Test throttle metrics are properly tracked."""
@pytest.mark.asyncio
async def test_metrics_tracked_on_rate_limit(self, config: ScraperConfig) -> None:
"""Test that rate limit errors are tracked in metrics."""
def mock_get(*args: object, **kwargs: object) -> MockResponse:
return MockResponse(status=429)
mock_session = MagicMock()
mock_session.get = mock_get
with patch("rec.query.districts.get_districts", return_value={"Test": "LOC1"}):
with pytest.raises((RateLimitError, CircuitBreakerOpenError)):
with patch("tenacity.wait_exponential.__call__", return_value=0):
await probe_query(
session=mock_session,
channel=ListingType.RENT,
min_bedrooms=1,
max_bedrooms=2,
radius=1.0,
min_price=1000,
max_price=2000,
district="Test",
config=config,
)
metrics = get_throttle_metrics()
assert metrics.rate_limit_count > 0
@pytest.mark.asyncio
async def test_metrics_tracked_on_success(self, config: ScraperConfig) -> None:
"""Test that successful requests are tracked in metrics."""
def mock_get(*args: object, **kwargs: object) -> MockResponse:
return MockResponse(
status=200,
json_data={"totalAvailableResults": 10, "properties": []},
)
mock_session = MagicMock()
mock_session.get = mock_get
with patch("rec.query.districts.get_districts", return_value={"Test": "LOC1"}):
await probe_query(
session=mock_session,
channel=ListingType.RENT,
min_bedrooms=1,
max_bedrooms=2,
radius=1.0,
min_price=1000,
max_price=2000,
district="Test",
config=config,
)
metrics = get_throttle_metrics()
assert metrics.total_requests == 1
assert metrics.total_throttling_events == 0
class TestConfigIntegration:
"""Test configuration integration."""
def test_config_from_env_includes_throttle_settings(self) -> None:
"""Test that config loads throttle settings from environment."""
import os
original_env = os.environ.copy()
try:
os.environ["RIGHTMOVE_SLOW_RESPONSE_THRESHOLD"] = "5.0"
os.environ["RIGHTMOVE_ENABLE_CIRCUIT_BREAKER"] = "false"
os.environ["RIGHTMOVE_CIRCUIT_BREAKER_FAILURES"] = "10"
os.environ["RIGHTMOVE_CIRCUIT_BREAKER_TIMEOUT"] = "120.0"
config = ScraperConfig.from_env()
assert config.slow_response_threshold == 5.0
assert config.enable_circuit_breaker is False
assert config.circuit_breaker_failure_threshold == 10
assert config.circuit_breaker_recovery_timeout == 120.0
finally:
os.environ.clear()
os.environ.update(original_env)
def test_circuit_breaker_disabled_returns_none(self) -> None:
"""Test that disabled circuit breaker returns None."""
config = ScraperConfig(
enable_circuit_breaker=False,
)
reset_circuit_breaker()
cb = get_circuit_breaker(config)
assert cb is None