Flatten repo structure: move crawler/ to root, remove vqa/ and immoweb/
The crawler subdirectory was the only active project. Moving it to the repo root simplifies paths and removes the unnecessary nesting. The vqa/ and immoweb/ directories were legacy/unused and have been removed. Updated .drone.yml, .gitignore, .claude/ docs, and skills to reflect the new flat structure.
This commit is contained in:
Viktor Barzin
parent
e2247be700
commit
eafbc1ac52
221 changed files with 70 additions and 146140 deletions
388
tests/unit/test_cli.py
Normal file
388
tests/unit/test_cli.py
Normal file
|
|
@ -0,0 +1,388 @@
|
|||
"""Characterization and unit tests for the CLI (main.py)."""
|
||||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import click
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from models.listing import FurnishType, ListingType, QueryParameters
|
||||
from main import build_query_parameters, cli, listing_filter_options
|
||||
|
||||
|
||||
class TestBuildQueryParameters:
|
||||
"""Tests for build_query_parameters()."""
|
||||
|
||||
def test_typical_rent_inputs(self) -> None:
|
||||
qp = build_query_parameters(
|
||||
type="RENT",
|
||||
district=["London", "Camden"],
|
||||
min_bedrooms=2,
|
||||
max_bedrooms=4,
|
||||
min_price=1000,
|
||||
max_price=3000,
|
||||
furnish_types=["FURNISHED"],
|
||||
available_from=datetime(2025, 6, 1),
|
||||
last_seen_days=7,
|
||||
min_sqm=50,
|
||||
)
|
||||
assert qp.listing_type == ListingType.RENT
|
||||
assert qp.district_names == {"London", "Camden"}
|
||||
assert qp.min_bedrooms == 2
|
||||
assert qp.max_bedrooms == 4
|
||||
assert qp.min_price == 1000
|
||||
assert qp.max_price == 3000
|
||||
assert qp.furnish_types == [FurnishType.FURNISHED]
|
||||
assert qp.let_date_available_from == datetime(2025, 6, 1)
|
||||
assert qp.last_seen_days == 7
|
||||
assert qp.min_sqm == 50
|
||||
|
||||
def test_typical_buy_inputs(self) -> None:
|
||||
qp = build_query_parameters(
|
||||
type="BUY",
|
||||
district=["Barnet"],
|
||||
min_bedrooms=3,
|
||||
max_bedrooms=5,
|
||||
min_price=200000,
|
||||
max_price=500000,
|
||||
furnish_types=[],
|
||||
available_from=None,
|
||||
last_seen_days=14,
|
||||
)
|
||||
assert qp.listing_type == ListingType.BUY
|
||||
assert qp.district_names == {"Barnet"}
|
||||
assert qp.furnish_types is None
|
||||
assert qp.let_date_available_from is None
|
||||
assert qp.min_sqm is None
|
||||
|
||||
def test_empty_districts_yields_empty_set(self) -> None:
|
||||
qp = build_query_parameters(
|
||||
type="RENT",
|
||||
district=[],
|
||||
min_bedrooms=1,
|
||||
max_bedrooms=10,
|
||||
min_price=0,
|
||||
max_price=999999,
|
||||
furnish_types=[],
|
||||
available_from=None,
|
||||
last_seen_days=14,
|
||||
)
|
||||
assert qp.district_names == set()
|
||||
|
||||
def test_none_districts_yields_empty_set(self) -> None:
|
||||
qp = build_query_parameters(
|
||||
type="RENT",
|
||||
district=None,
|
||||
min_bedrooms=1,
|
||||
max_bedrooms=10,
|
||||
min_price=0,
|
||||
max_price=999999,
|
||||
furnish_types=[],
|
||||
available_from=None,
|
||||
last_seen_days=14,
|
||||
)
|
||||
assert qp.district_names == set()
|
||||
|
||||
def test_furnish_types_conversion(self) -> None:
|
||||
qp = build_query_parameters(
|
||||
type="RENT",
|
||||
district=["London"],
|
||||
min_bedrooms=1,
|
||||
max_bedrooms=10,
|
||||
min_price=0,
|
||||
max_price=999999,
|
||||
furnish_types=["FURNISHED", "UNFURNISHED"],
|
||||
available_from=None,
|
||||
last_seen_days=14,
|
||||
)
|
||||
assert qp.furnish_types == [FurnishType.FURNISHED, FurnishType.UNFURNISHED]
|
||||
|
||||
def test_empty_furnish_types_yields_none(self) -> None:
|
||||
qp = build_query_parameters(
|
||||
type="RENT",
|
||||
district=["London"],
|
||||
min_bedrooms=1,
|
||||
max_bedrooms=10,
|
||||
min_price=0,
|
||||
max_price=999999,
|
||||
furnish_types=[],
|
||||
available_from=None,
|
||||
last_seen_days=14,
|
||||
)
|
||||
assert qp.furnish_types is None
|
||||
|
||||
def test_default_optional_parameters(self) -> None:
|
||||
qp = build_query_parameters(
|
||||
type="RENT",
|
||||
district=["London"],
|
||||
min_bedrooms=1,
|
||||
max_bedrooms=10,
|
||||
min_price=0,
|
||||
max_price=999999,
|
||||
furnish_types=[],
|
||||
available_from=None,
|
||||
last_seen_days=14,
|
||||
)
|
||||
assert qp.radius == 0
|
||||
assert qp.page_size == 500
|
||||
assert qp.max_days_since_added == 14
|
||||
|
||||
|
||||
class TestListingFilterOptionsDecorator:
|
||||
"""Tests for the listing_filter_options decorator."""
|
||||
|
||||
def test_applies_all_expected_options(self) -> None:
|
||||
@click.command()
|
||||
@listing_filter_options
|
||||
def dummy_cmd(**kwargs: object) -> None:
|
||||
pass
|
||||
|
||||
expected_option_names = {
|
||||
"type",
|
||||
"min_bedrooms",
|
||||
"max_bedrooms",
|
||||
"min_price",
|
||||
"max_price",
|
||||
"district",
|
||||
"furnish_types",
|
||||
"available_from",
|
||||
"last_seen_days",
|
||||
"min_sqm",
|
||||
}
|
||||
param_names = {p.name for p in dummy_cmd.params}
|
||||
assert expected_option_names.issubset(param_names), (
|
||||
f"Missing options: {expected_option_names - param_names}"
|
||||
)
|
||||
|
||||
def test_type_option_is_required(self) -> None:
|
||||
@click.command()
|
||||
@listing_filter_options
|
||||
def dummy_cmd(**kwargs: object) -> None:
|
||||
pass
|
||||
|
||||
type_param = next(p for p in dummy_cmd.params if p.name == "type")
|
||||
assert type_param.required is True
|
||||
|
||||
def test_produces_query_parameters_kwarg(self) -> None:
|
||||
"""After refactoring, the decorator should produce a query_parameters kwarg."""
|
||||
captured: dict = {}
|
||||
|
||||
@click.command()
|
||||
@listing_filter_options
|
||||
def dummy_cmd(query_parameters: QueryParameters) -> None:
|
||||
captured["qp"] = query_parameters
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(dummy_cmd, ["--type", "RENT"])
|
||||
assert result.exit_code == 0, f"Command failed: {result.output}"
|
||||
assert isinstance(captured["qp"], QueryParameters)
|
||||
assert captured["qp"].listing_type == ListingType.RENT
|
||||
|
||||
|
||||
class TestDumpListingsCommand:
|
||||
"""Tests for the dump-listings CLI command."""
|
||||
|
||||
@patch("main.listing_service.refresh_listings", new_callable=AsyncMock)
|
||||
@patch("main.engine", new_callable=MagicMock)
|
||||
def test_calls_refresh_listings_with_correct_params(
|
||||
self,
|
||||
mock_engine: MagicMock,
|
||||
mock_refresh: AsyncMock,
|
||||
) -> None:
|
||||
from services.listing_service import RefreshResult
|
||||
|
||||
mock_refresh.return_value = RefreshResult(
|
||||
task_id=None,
|
||||
new_listings_count=5,
|
||||
message="Fetched 5 new listings",
|
||||
)
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"dump-listings",
|
||||
"--type", "RENT",
|
||||
"--min-bedrooms", "2",
|
||||
"--max-bedrooms", "4",
|
||||
"--min-price", "1000",
|
||||
"--max-price", "3000",
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0, f"CLI failed: {result.output}"
|
||||
mock_refresh.assert_called_once()
|
||||
call_args = mock_refresh.call_args
|
||||
qp: QueryParameters = call_args.args[1]
|
||||
assert qp.listing_type == ListingType.RENT
|
||||
assert qp.min_bedrooms == 2
|
||||
assert qp.max_bedrooms == 4
|
||||
assert qp.min_price == 1000
|
||||
assert qp.max_price == 3000
|
||||
assert call_args.kwargs.get("full") is not True
|
||||
|
||||
@patch("main.listing_service.refresh_listings", new_callable=AsyncMock)
|
||||
@patch("main.engine", new_callable=MagicMock)
|
||||
def test_include_processing_flag_passes_full_true(
|
||||
self,
|
||||
mock_engine: MagicMock,
|
||||
mock_refresh: AsyncMock,
|
||||
) -> None:
|
||||
from services.listing_service import RefreshResult
|
||||
|
||||
mock_refresh.return_value = RefreshResult(
|
||||
task_id=None,
|
||||
new_listings_count=0,
|
||||
message="Fetched 0 new listings",
|
||||
)
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"dump-listings",
|
||||
"--type", "RENT",
|
||||
"--include-processing",
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0, f"CLI failed: {result.output}"
|
||||
mock_refresh.assert_called_once()
|
||||
call_kwargs = mock_refresh.call_args.kwargs
|
||||
assert call_kwargs.get("full") is True
|
||||
|
||||
@patch("main.listing_service.refresh_listings", new_callable=AsyncMock)
|
||||
@patch("main.engine", new_callable=MagicMock)
|
||||
def test_include_processing_short_flag(
|
||||
self,
|
||||
mock_engine: MagicMock,
|
||||
mock_refresh: AsyncMock,
|
||||
) -> None:
|
||||
from services.listing_service import RefreshResult
|
||||
|
||||
mock_refresh.return_value = RefreshResult(
|
||||
task_id=None,
|
||||
new_listings_count=0,
|
||||
message="Fetched 0 new listings",
|
||||
)
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"dump-listings",
|
||||
"--type", "RENT",
|
||||
"-p",
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0, f"CLI failed: {result.output}"
|
||||
mock_refresh.assert_called_once()
|
||||
call_kwargs = mock_refresh.call_args.kwargs
|
||||
assert call_kwargs.get("full") is True
|
||||
|
||||
|
||||
class TestExportCsvCommand:
|
||||
"""Tests for the export-csv CLI command."""
|
||||
|
||||
@patch("main.export_service.export_to_csv", new_callable=AsyncMock)
|
||||
@patch("main.engine", new_callable=MagicMock)
|
||||
def test_calls_export_to_csv(
|
||||
self,
|
||||
mock_engine: MagicMock,
|
||||
mock_export: AsyncMock,
|
||||
) -> None:
|
||||
from services.export_service import ExportResult
|
||||
|
||||
mock_export.return_value = ExportResult(
|
||||
success=True,
|
||||
output_path="/tmp/test.csv",
|
||||
data=None,
|
||||
record_count=10,
|
||||
message="Exported 10 listings to /tmp/test.csv",
|
||||
)
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"export-csv",
|
||||
"--output-file", "/tmp/test.csv",
|
||||
"--type", "RENT",
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0, f"CLI failed: {result.output}"
|
||||
mock_export.assert_called_once()
|
||||
call_args = mock_export.call_args
|
||||
qp = call_args[0][2]
|
||||
assert qp.listing_type == ListingType.RENT
|
||||
|
||||
|
||||
class TestExportImmowebCommand:
|
||||
"""Tests for the export-immoweb CLI command."""
|
||||
|
||||
@patch("main.export_service.export_to_geojson", new_callable=AsyncMock)
|
||||
@patch("main.engine", new_callable=MagicMock)
|
||||
def test_calls_export_to_geojson(
|
||||
self,
|
||||
mock_engine: MagicMock,
|
||||
mock_export: AsyncMock,
|
||||
) -> None:
|
||||
from services.export_service import ExportResult
|
||||
|
||||
mock_export.return_value = ExportResult(
|
||||
success=True,
|
||||
output_path="/tmp/test.geojson",
|
||||
data=None,
|
||||
record_count=5,
|
||||
message="Exported 5 listings to /tmp/test.geojson",
|
||||
)
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"export-immoweb",
|
||||
"--output-file", "/tmp/test.geojson",
|
||||
"--type", "RENT",
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0, f"CLI failed: {result.output}"
|
||||
mock_export.assert_called_once()
|
||||
|
||||
|
||||
class TestListDistrictsCommand:
|
||||
"""Tests for the list-districts CLI command."""
|
||||
|
||||
@patch("main.engine", new_callable=MagicMock)
|
||||
def test_outputs_district_names(self, mock_engine: MagicMock) -> None:
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["list-districts"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "London" in result.output
|
||||
assert "Camden" in result.output
|
||||
assert "Available districts" in result.output
|
||||
|
||||
|
||||
class TestRoutingCommand:
|
||||
"""Tests for the routing CLI command."""
|
||||
|
||||
@patch("main.engine", new_callable=MagicMock)
|
||||
def test_requires_api_key_env_var(self, mock_engine: MagicMock) -> None:
|
||||
runner = CliRunner(env={"ROUTING_API_KEY": None})
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"routing",
|
||||
"--destination-address", "London Bridge",
|
||||
"--travel-mode", "transit",
|
||||
"--limit", "1",
|
||||
],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
|
||||
assert result.exit_code != 0
|
||||
assert "ROUTING_API_KEY" in result.output
|
||||
Loading…
Add table
Add a link
Reference in a new issue