Refactor codebase following Clean Code principles and add 229 tests

- Extract helpers to reduce function sizes (listing_tasks, app.py, query.py, listing_fetcher) - Replace nonlocal mutations with _PipelineState dataclass in listing_tasks - Fix bugs: isinstance→equality check in repository, verify_exp for OIDC tokens - Consolidate duplicate filter methods in listing_repository - Move hardcoded config to env vars with backward-compatible defaults - Simplify CLI decorator to auto-build QueryParameters - Add deprecation docstring to data_access.py - Test count: 158 → 387 (all passing)
2026-02-07 20:19:57 +00:00 · 2026-02-07 20:19:57 +00:00 · 150342bb9e
commit 150342bb9e
parent 7e05b3c971
48 changed files with 5029 additions and 990 deletions
--- a/crawler/tests/unit/test_task_service.py
+++ b/crawler/tests/unit/test_task_service.py
@ -0,0 +1,306 @@
+"""Unit tests for services/task_service.py."""
+from unittest.mock import MagicMock, patch
+import pytest
+
+from services.task_service import (
+    TaskStatus,
+    _extract_progress_info,
+    _extract_result,
+    _make_system_user,
+    get_task_status,
+)
+
+
+class TestMakeSystemUser:
+    """Tests for _make_system_user helper."""
+
+    def test_creates_user_with_email(self) -> None:
+        user = _make_system_user("test@example.com")
+        assert user.email == "test@example.com"
+        assert user.sub == ""
+        assert user.name == ""
+
+    def test_different_emails_create_different_users(self) -> None:
+        u1 = _make_system_user("a@b.com")
+        u2 = _make_system_user("c@d.com")
+        assert u1.email != u2.email
+
+
+class TestExtractResult:
+    """Tests for _extract_result helper."""
+
+    def test_failed_task_returns_error(self) -> None:
+        mock_result = MagicMock()
+        mock_result.failed.return_value = True
+        mock_result.result = Exception("something broke")
+
+        result, error = _extract_result(mock_result)
+        assert result is None
+        assert error is not None
+        assert "something broke" in error
+
+    def test_failed_task_with_no_result(self) -> None:
+        mock_result = MagicMock()
+        mock_result.failed.return_value = True
+        mock_result.result = None
+
+        result, error = _extract_result(mock_result)
+        assert result is None
+        assert error is None
+
+    def test_successful_json_serializable_result(self) -> None:
+        mock_result = MagicMock()
+        mock_result.failed.return_value = False
+        mock_result.result = {"count": 42, "status": "done"}
+
+        result, error = _extract_result(mock_result)
+        assert result == {"count": 42, "status": "done"}
+        assert error is None
+
+    def test_non_serializable_result_falls_back_to_str(self) -> None:
+        mock_result = MagicMock()
+        mock_result.failed.return_value = False
+        mock_result.result = object()  # not JSON-serializable
+
+        result, error = _extract_result(mock_result)
+        assert isinstance(result, str)
+        assert error is None
+
+    def test_none_result_stays_none(self) -> None:
+        mock_result = MagicMock()
+        mock_result.failed.return_value = False
+        mock_result.result = None
+
+        result, error = _extract_result(mock_result)
+        assert result is None
+        assert error is None
+
+
+class TestExtractProgressInfo:
+    """Tests for _extract_progress_info helper."""
+
+    def test_extracts_progress_fields(self) -> None:
+        mock_result = MagicMock()
+        mock_result.info = {"progress": 0.5, "processed": 50, "total": 100}
+        mock_result.status = "STARTED"
+
+        info = _extract_progress_info(mock_result)
+        assert info["progress"] == 0.5
+        assert info["processed"] == 50
+        assert info["total"] == 100
+        assert info["message"] is None
+
+    def test_extracts_message_from_info(self) -> None:
+        mock_result = MagicMock()
+        mock_result.info = {"message": "Processing page 3"}
+        mock_result.status = "STARTED"
+
+        info = _extract_progress_info(mock_result)
+        assert info["message"] == "Processing page 3"
+
+    def test_falls_back_to_reason_for_skipped(self) -> None:
+        mock_result = MagicMock()
+        mock_result.info = {"reason": "Already running"}
+        mock_result.status = "SKIPPED"
+
+        info = _extract_progress_info(mock_result)
+        assert info["message"] == "Already running"
+
+    def test_custom_state_used_as_message(self) -> None:
+        mock_result = MagicMock()
+        mock_result.info = {}
+        mock_result.status = "Fetching listings"
+
+        info = _extract_progress_info(mock_result)
+        assert info["message"] == "Fetching listings"
+
+    def test_standard_state_not_used_as_message(self) -> None:
+        mock_result = MagicMock()
+        mock_result.info = {}
+        mock_result.status = "PENDING"
+
+        info = _extract_progress_info(mock_result)
+        assert info["message"] is None
+
+    def test_none_info_returns_all_none(self) -> None:
+        mock_result = MagicMock()
+        mock_result.info = None
+        mock_result.status = "PENDING"
+
+        info = _extract_progress_info(mock_result)
+        assert info == {"progress": None, "processed": None, "total": None, "message": None}
+
+
+class TestGetTaskStatus:
+    """Tests for get_task_status."""
+
+    def test_pending_task(self) -> None:
+        """Test status for a pending task."""
+        mock_result = MagicMock()
+        mock_result.status = "PENDING"
+        mock_result.failed.return_value = False
+        mock_result.result = None
+        mock_result.info = None
+        mock_result.traceback = None
+
+        with patch("services.task_service.dump_listings_task", create=True) as mock_task:
+            mock_task.AsyncResult.return_value = mock_result
+            # Patch the lazy import
+            with patch.dict("sys.modules", {"tasks.listing_tasks": MagicMock(dump_listings_task=mock_task)}):
+                status = get_task_status("test-id")
+                assert status.task_id == "test-id"
+                assert status.status == "PENDING"
+                assert status.error is None
+
+    def test_failed_task(self) -> None:
+        """Test status for a failed task."""
+        mock_result = MagicMock()
+        mock_result.status = "FAILURE"
+        mock_result.failed.return_value = True
+        mock_result.result = Exception("something broke")
+        mock_result.info = None
+        mock_result.traceback = "Traceback..."
+
+        with patch("services.task_service.dump_listings_task", create=True) as mock_task:
+            mock_task.AsyncResult.return_value = mock_result
+            with patch.dict("sys.modules", {"tasks.listing_tasks": MagicMock(dump_listings_task=mock_task)}):
+                status = get_task_status("test-id")
+                assert status.status == "FAILURE"
+                assert status.error is not None
+                assert status.traceback == "Traceback..."
+
+    def test_custom_state_with_progress(self) -> None:
+        """Test that custom states with progress info are extracted correctly."""
+        mock_result = MagicMock()
+        mock_result.status = "Fetching listings"
+        mock_result.failed.return_value = False
+        mock_result.result = None
+        mock_result.info = {"progress": 0.5, "processed": 50, "total": 100}
+        mock_result.traceback = None
+
+        with patch("services.task_service.dump_listings_task", create=True) as mock_task:
+            mock_task.AsyncResult.return_value = mock_result
+            with patch.dict("sys.modules", {"tasks.listing_tasks": MagicMock(dump_listings_task=mock_task)}):
+                status = get_task_status("test-id")
+                assert status.progress == 0.5
+                assert status.processed == 50
+                assert status.total == 100
+
+    def test_successful_task(self) -> None:
+        """Test status for a successful task."""
+        mock_result = MagicMock()
+        mock_result.status = "SUCCESS"
+        mock_result.failed.return_value = False
+        mock_result.result = {"listings_count": 42}
+        mock_result.info = None
+        mock_result.traceback = None
+
+        with patch("services.task_service.dump_listings_task", create=True) as mock_task:
+            mock_task.AsyncResult.return_value = mock_result
+            with patch.dict("sys.modules", {"tasks.listing_tasks": MagicMock(dump_listings_task=mock_task)}):
+                status = get_task_status("test-id")
+                assert status.status == "SUCCESS"
+                assert status.result == {"listings_count": 42}
+                assert status.error is None
+
+
+class TestGetUserTasks:
+    """Tests for get_user_tasks."""
+
+    def test_returns_task_list(self) -> None:
+        mock_redis = MagicMock()
+        mock_redis.get_tasks_for_user.return_value = ["task-1", "task-2"]
+
+        with patch("services.task_service.RedisRepository", create=True) as MockRedisRepo:
+            MockRedisRepo.instance.return_value = mock_redis
+            with patch.dict("sys.modules", {"redis_repository": MagicMock(RedisRepository=MockRedisRepo)}):
+                from services.task_service import get_user_tasks
+                result = get_user_tasks("test@example.com")
+                assert result == ["task-1", "task-2"]
+
+    def test_returns_empty_list_for_unknown_user(self) -> None:
+        mock_redis = MagicMock()
+        mock_redis.get_tasks_for_user.return_value = []
+
+        with patch("services.task_service.RedisRepository", create=True) as MockRedisRepo:
+            MockRedisRepo.instance.return_value = mock_redis
+            with patch.dict("sys.modules", {"redis_repository": MagicMock(RedisRepository=MockRedisRepo)}):
+                from services.task_service import get_user_tasks
+                result = get_user_tasks("nobody@example.com")
+                assert result == []
+
+
+class TestCancelTask:
+    """Tests for cancel_task."""
+
+    def test_cancel_revokes_and_removes(self) -> None:
+        mock_celery = MagicMock()
+        mock_redis = MagicMock()
+        mock_redis.remove_task_for_user.return_value = True
+
+        with patch.dict("sys.modules", {
+            "celery_app": MagicMock(app=mock_celery),
+            "redis_repository": MagicMock(RedisRepository=MagicMock(instance=MagicMock(return_value=mock_redis))),
+        }):
+            from services.task_service import cancel_task
+            result = cancel_task("task-123", user_email="test@example.com")
+            assert result is True
+            mock_celery.control.revoke.assert_called_once_with("task-123", terminate=True)
+
+    def test_cancel_without_user_email(self) -> None:
+        mock_celery = MagicMock()
+
+        with patch.dict("sys.modules", {"celery_app": MagicMock(app=mock_celery)}):
+            from services.task_service import cancel_task
+            result = cancel_task("task-456")
+            assert result is True
+            mock_celery.control.revoke.assert_called_once_with("task-456", terminate=True)
+
+
+class TestClearAllTasks:
+    """Tests for clear_all_tasks."""
+
+    def test_clear_with_revoke(self) -> None:
+        mock_celery = MagicMock()
+        mock_redis = MagicMock()
+        mock_redis.get_tasks_for_user.return_value = ["t1", "t2"]
+        mock_redis.clear_tasks_for_user.return_value = 2
+
+        with patch.dict("sys.modules", {
+            "celery_app": MagicMock(app=mock_celery),
+            "redis_repository": MagicMock(RedisRepository=MagicMock(instance=MagicMock(return_value=mock_redis))),
+        }):
+            from services.task_service import clear_all_tasks
+            count = clear_all_tasks("test@example.com", revoke=True)
+            assert count == 2
+            assert mock_celery.control.revoke.call_count == 2
+
+    def test_clear_without_revoke(self) -> None:
+        mock_celery = MagicMock()
+        mock_redis = MagicMock()
+        mock_redis.clear_tasks_for_user.return_value = 3
+
+        with patch.dict("sys.modules", {
+            "celery_app": MagicMock(app=mock_celery),
+            "redis_repository": MagicMock(RedisRepository=MagicMock(instance=MagicMock(return_value=mock_redis))),
+        }):
+            from services.task_service import clear_all_tasks
+            count = clear_all_tasks("test@example.com", revoke=False)
+            assert count == 3
+            mock_celery.control.revoke.assert_not_called()
+
+    def test_revoke_failure_logs_warning_and_continues(self) -> None:
+        mock_celery = MagicMock()
+        mock_celery.control.revoke.side_effect = Exception("connection lost")
+        mock_redis = MagicMock()
+        mock_redis.get_tasks_for_user.return_value = ["t1"]
+        mock_redis.clear_tasks_for_user.return_value = 1
+
+        with patch.dict("sys.modules", {
+            "celery_app": MagicMock(app=mock_celery),
+            "redis_repository": MagicMock(RedisRepository=MagicMock(instance=MagicMock(return_value=mock_redis))),
+        }):
+            from services.task_service import clear_all_tasks
+            # Should not raise despite revoke failure
+            count = clear_all_tasks("test@example.com", revoke=True)
+            assert count == 1