Flatten repo structure: move crawler/ to root, remove vqa/ and immoweb/

The crawler subdirectory was the only active project. Moving it to the
repo root simplifies paths and removes the unnecessary nesting. The
vqa/ and immoweb/ directories were legacy/unused and have been removed.

Updated .drone.yml, .gitignore, .claude/ docs, and skills to reflect
the new flat structure.

api/app.py

@@ -0,0 +1,318 @@
+"""FastAPI application for the Real Estate Crawler API."""
+from datetime import datetime, timedelta
+import json
+import logging
+import logging.config
+from typing import Annotated, AsyncGenerator, Optional
+from api.auth import get_current_user
+from api.config import DEV_TIER_ORIGINS, PROD_TIER_ORIGINS
+from api.passkey_routes import passkey_router
+from dotenv import load_dotenv
+from fastapi import Depends, FastAPI, Query
+from fastapi.responses import StreamingResponse
+from api.auth import User
+from models.listing import QueryParameters, ListingType, FurnishType
+from notifications import send_notification
+from repositories.listing_repository import ListingRepository
+from database import engine
+from fastapi.middleware.cors import CORSMiddleware
+from ui_exporter import convert_to_geojson_feature, convert_row_to_geojson
+
+from services import listing_service, export_service, district_service, task_service
+from services.listing_cache import (
+    get_cached_count,
+    get_cached_features,
+    cache_features_batch,
+)
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from api.metrics import metrics_app
+from opentelemetry.metrics import get_meter
+
+
+load_dotenv()
+logger = logging.getLogger("uvicorn")
+
+DEFAULT_BATCH_SIZE = 50
+
+
+def get_query_parameters(
+    listing_type: ListingType,
+    min_bedrooms: int = 1,
+    max_bedrooms: int = 999,
+    min_price: int = 0,
+    max_price: int = 10_000_000,
+    min_sqm: Optional[int] = None,
+    last_seen_days: Optional[int] = None,
+    let_date_available_from: Optional[datetime] = None,
+    furnish_types: Optional[str] = None,  # comma-separated list
+) -> QueryParameters:
+    """Parse query parameters into QueryParameters model."""
+    parsed_furnish_types = None
+    if furnish_types:
+        parsed_furnish_types = [FurnishType(f.strip()) for f in furnish_types.split(",")]
+
+    return QueryParameters(
+        listing_type=listing_type,
+        min_bedrooms=min_bedrooms,
+        max_bedrooms=max_bedrooms,
+        min_price=min_price,
+        max_price=max_price,
+        min_sqm=min_sqm,
+        last_seen_days=last_seen_days,
+        let_date_available_from=let_date_available_from,
+        furnish_types=parsed_furnish_types,
+    )
+
+
+app = FastAPI()
+app.include_router(passkey_router)
+app.mount("/metrics", metrics_app)
+meter = get_meter(__name__)
+request_counter = meter.create_counter(
+    name="custom_request_count",
+    description="Number of times /hello was called",
+)
+hist = meter.create_histogram(
+    name="custom_request_duration",
+    description="Duration of /hello requests in seconds",
+)
+
+
+# Allow CORS (for React frontend)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=[*DEV_TIER_ORIGINS, *PROD_TIER_ORIGINS],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/api/status")
+async def get_status() -> dict[str, str]:
+    request_counter.add(1, {"method": "GET", "path": "/status"})
+    hist.record(1.5, {"method": "GET", "path": "/status"})
+    return {"status": "OK"}
+
+
+@app.get("/api/listing")
+async def get_listing(
+    user: Annotated[User, Depends(get_current_user)],
+    limit: int = 5,
+) -> dict[str, list]:
+    """Get listings from the database."""
+    repository = ListingRepository(engine)
+    result = await listing_service.get_listings(repository, limit=limit)
+    logger.info(f"Fetched {result.total_count} listings for {user.email}")
+    return {"listings": result.listings}
+
+
+@app.get("/api/listing_geojson")
+async def get_listing_geojson(
+    user: Annotated[User, Depends(get_current_user)],
+    query_parameters: Annotated[QueryParameters, Depends(get_query_parameters)],
+    limit: int | None = None,
+) -> dict:
+    """Get listings as GeoJSON for map display."""
+    repository = ListingRepository(engine)
+    result = await export_service.export_to_geojson(
+        repository,
+        query_parameters=query_parameters,
+        limit=limit,
+    )
+    return result.data
+
+
+
+async def _stream_from_cache(
+    query_parameters: QueryParameters,
+    batch_size: int,
+    limit: int | None,
+) -> AsyncGenerator[str, None]:
+    """Stream GeoJSON features from the Redis cache (cache-hit path)."""
+    cached_count = get_cached_count(query_parameters)
+    effective_total = min(limit, cached_count) if limit and cached_count else cached_count
+
+    yield json.dumps({
+        "type": "metadata",
+        "batch_size": batch_size,
+        "total_expected": effective_total,
+        "cached": True,
+    }) + "\n"
+
+    count = 0
+    for feature_batch in get_cached_features(query_parameters, batch_size=batch_size):
+        if limit and count + len(feature_batch) > limit:
+            feature_batch = feature_batch[:limit - count]
+        count += len(feature_batch)
+        yield json.dumps({"type": "batch", "features": feature_batch}) + "\n"
+        if limit and count >= limit:
+            break
+
+    yield json.dumps({"type": "complete", "total": count}) + "\n"
+
+
+async def _stream_from_db(
+    query_parameters: QueryParameters,
+    batch_size: int,
+    limit: int | None,
+) -> AsyncGenerator[str, None]:
+    """Stream GeoJSON features from the database, populating the cache as we go."""
+    repository = ListingRepository(engine)
+
+    total = repository.count_listings(query_parameters)
+    effective_total = min(limit, total) if limit else total
+
+    yield json.dumps({
+        "type": "metadata",
+        "batch_size": batch_size,
+        "total_expected": effective_total,
+        "cached": False,
+    }) + "\n"
+
+    count = 0
+    batch: list[dict] = []
+    for row in repository.stream_listings_optimized(
+        query_parameters, limit=limit, page_size=batch_size
+    ):
+        feature = convert_row_to_geojson(row, query_parameters.listing_type.value)
+        batch.append(feature)
+        count += 1
+
+        if len(batch) >= batch_size:
+            cache_features_batch(query_parameters, batch)
+            yield json.dumps({"type": "batch", "features": batch}) + "\n"
+            batch = []
+
+    if batch:
+        cache_features_batch(query_parameters, batch)
+        yield json.dumps({"type": "batch", "features": batch}) + "\n"
+
+    yield json.dumps({"type": "complete", "total": count}) + "\n"
+
+
+@app.get("/api/listing_geojson/stream")
+async def stream_listing_geojson(
+    user: Annotated[User, Depends(get_current_user)],
+    query_parameters: Annotated[QueryParameters, Depends(get_query_parameters)],
+    batch_size: int = DEFAULT_BATCH_SIZE,
+    limit: int | None = None,
+) -> StreamingResponse:
+    """Stream listings as NDJSON for progressive map loading.
+
+    Returns newline-delimited JSON with three message types:
+    - metadata: Initial message with batch_size and total_expected count
+    - batch: Array of GeoJSON features
+    - complete: Final message with total count
+    """
+    cached_count = get_cached_count(query_parameters)
+    if cached_count is not None and cached_count > 0:
+        generator = _stream_from_cache(query_parameters, batch_size, limit)
+    else:
+        generator = _stream_from_db(query_parameters, batch_size, limit)
+
+    return StreamingResponse(
+        generator,
+        media_type="application/x-ndjson",
+        headers={
+            "Cache-Control": "no-cache",
+            "X-Accel-Buffering": "no",  # Disable nginx buffering
+        }
+    )
+
+
+@app.post("/api/refresh_listings")
+async def refresh_listings(
+    user: Annotated[User, Depends(get_current_user)],
+    query_parameters: Annotated[QueryParameters, Depends(get_query_parameters)],
+) -> dict[str, str]:
+    """Trigger a background task to refresh listings."""
+    await send_notification(
+        f"{user.email} refreshing listings with query parameters {query_parameters.model_dump_json()}"
+    )
+
+    repository = ListingRepository(engine)
+    result = await listing_service.refresh_listings(
+        repository,
+        query_parameters,
+        async_mode=True,
+        user_email=user.email,
+    )
+
+    # Track task for user
+    if result.task_id:
+        task_service.add_task_for_user(user.email, result.task_id)
+
+    return {"task_id": result.task_id or "", "message": result.message}
+
+
+@app.get("/api/task_status")
+async def get_task_status(
+    user: Annotated[User, Depends(get_current_user)],
+    task_id: str,
+) -> dict[str, str | int | float | None]:
+    """Get the status of a background task."""
+    status = task_service.get_task_status(task_id)
+    return {
+        "task_id": status.task_id,
+        "status": status.status,
+        "result": json.dumps(status.result) if status.result else None,
+        "progress": status.progress,
+        "processed": status.processed,
+        "total": status.total,
+        "message": status.message,
+        "error": status.error,
+        "traceback": status.traceback,
+    }
+
+
+@app.get("/api/tasks_for_user")
+async def get_tasks_for_user(
+    user: Annotated[User, Depends(get_current_user)],
+) -> list[str]:
+    """Get all task IDs for the current user."""
+    return task_service.get_user_tasks(user.email)
+
+
+@app.post("/api/cancel_task")
+async def cancel_task(
+    user: Annotated[User, Depends(get_current_user)],
+    task_id: str = Query(..., description="The task ID to cancel"),
+) -> dict[str, str | bool]:
+    """Cancel a running task and remove it from the user's task list."""
+    # Verify user owns this task
+    user_tasks = task_service.get_user_tasks(user.email)
+    if task_id not in user_tasks:
+        return {"success": False, "message": "Task not found or not owned by user"}
+
+    try:
+        task_service.cancel_task(task_id, user_email=user.email)
+        logger.info(f"Task {task_id} cancelled by {user.email}")
+        return {"success": True, "message": "Task cancelled"}
+    except Exception as e:
+        logger.error(f"Failed to cancel task {task_id}: {e}")
+        return {"success": False, "message": str(e)}
+
+
+@app.post("/api/clear_all_tasks")
+async def clear_all_tasks(
+    user: Annotated[User, Depends(get_current_user)],
+) -> dict[str, str | int | bool]:
+    """Clear all tasks for the current user."""
+    try:
+        count = task_service.clear_all_tasks(user.email)
+        logger.info(f"Cleared {count} tasks for {user.email}")
+        return {"success": True, "count": count, "message": f"Cleared {count} tasks"}
+    except Exception as e:
+        logger.error(f"Failed to clear tasks for {user.email}: {e}")
+        return {"success": False, "count": 0, "message": str(e)}
+
+
+@app.get("/api/get_districts")
+async def get_districts(
+    user: Annotated[User, Depends(get_current_user)],
+) -> dict[str, str]:
+    """Get all available districts."""
+    return district_service.get_all_districts()
+
+
+FastAPIInstrumentor.instrument_app(app)

api/auth.py

@@ -0,0 +1,99 @@
+from api.config import (
+    AUTHENTIK_URL,
+    OIDC_CACHE_TTL,
+    OIDC_CLIENT_ID,
+    OIDC_METADATA_URL,
+    JWT_SECRET,
+    JWT_ALGORITHM,
+    JWT_ISSUER,
+)
+from cachetools import TTLCache
+from fastapi import Depends, HTTPException
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from httpx import AsyncClient
+import jwt
+from pydantic import BaseModel
+
+
+# HTTPBearer scheme (provider-agnostic, works for both OIDC and passkey JWTs)
+http_bearer = HTTPBearer()
+
+JWKS_CACHE = TTLCache(maxsize=1, ttl=OIDC_CACHE_TTL)
+OIDC_METADATA_CACHE = TTLCache(maxsize=1, ttl=OIDC_CACHE_TTL)
+
+
+class User(BaseModel):
+    sub: str  # User ID
+    email: str
+    name: str
+
+
+async def get_oidc_metadata() -> dict:  # type: ignore[type-arg]
+    if "oidc_metadata" not in OIDC_METADATA_CACHE:
+        async with AsyncClient() as client:
+            resp = await client.get(OIDC_METADATA_URL, follow_redirects=True)
+        OIDC_METADATA_CACHE["oidc_metadata"] = resp.json()
+    return OIDC_METADATA_CACHE["oidc_metadata"]
+
+
+async def get_cached_jwks_client() -> jwt.PyJWKClient:
+    if "jwks_client" not in JWKS_CACHE:
+        metadata = await get_oidc_metadata()
+        jwks_url = metadata["jwks_uri"]
+        JWKS_CACHE["jwks_client"] = jwt.PyJWKClient(
+            jwks_url,
+            cache_keys=True,  # PyJWT's built-in key caching
+            max_cached_keys=5,
+        )
+    return JWKS_CACHE["jwks_client"]
+
+
+async def _verify_authentik_token(token: str) -> User:
+    """Verify a token issued by Authentik (RS256 via JWKS)."""
+    metadata = await get_oidc_metadata()
+    signing_key = (await get_cached_jwks_client()).get_signing_key_from_jwt(token)
+
+    payload = jwt.decode(
+        token,
+        signing_key,
+        algorithms=["RS256"],
+        audience=OIDC_CLIENT_ID,
+        issuer=metadata["issuer"],
+    )
+    return User(**payload)
+
+
+def _verify_passkey_token(token: str) -> User:
+    """Verify a token issued by the passkey service (HS256)."""
+    payload = jwt.decode(
+        token,
+        JWT_SECRET,
+        algorithms=[JWT_ALGORITHM],
+        issuer=JWT_ISSUER,
+    )
+    return User(
+        sub=payload["sub"],
+        email=payload["email"],
+        name=payload.get("name", payload["email"]),
+    )
+
+
+async def get_current_user(
+    credentials: HTTPAuthorizationCredentials = Depends(http_bearer),
+) -> User:
+    token = credentials.credentials
+    try:
+        # Decode WITHOUT verification just to read the "iss" claim for routing.
+        # This is safe: we only use the issuer to decide which verified decode
+        # path to take next; the actual security check happens in the branch below.
+        unverified = jwt.decode(
+            token, options={"verify_signature": False, "verify_exp": False}
+        )
+        issuer = unverified.get("iss", "")
+
+        if issuer == JWT_ISSUER:
+            return _verify_passkey_token(token)
+        else:
+            return await _verify_authentik_token(token)
+    except jwt.PyJWTError as e:
+        raise HTTPException(status_code=401, detail=f"Invalid token: {e}")

api/config.py

@@ -0,0 +1,33 @@
+from datetime import timedelta
+import logging
+import os
+
+
+_logger = logging.getLogger(__name__)
+
+# Authentik OIDC Configuration
+AUTHENTIK_URL = os.getenv("AUTHENTIK_URL", "https://authentik.viktorbarzin.me")
+OIDC_CLIENT_ID = os.getenv("OIDC_CLIENT_ID", "5AJKRgcdgVm1OyApBzFkadDFfStW9a555zwv2MOe")
+OIDC_METADATA_URL = (
+    f"{AUTHENTIK_URL}/application/o/wrongmove/.well-known/openid-configuration"
+)
+
+OIDC_CACHE_TTL = timedelta(
+    hours=1
+).total_seconds()  # Cache to avoid spamming authentik with requests
+
+DEV_TIER_ORIGINS = ["https://localhost/"]
+PROD_TIER_ORIGINS = ["https://wrongmove.viktorbarzin.me/"]
+
+# WebAuthn / Passkey Configuration
+WEBAUTHN_RP_ID = os.getenv("WEBAUTHN_RP_ID", "localhost")
+WEBAUTHN_RP_NAME = os.getenv("WEBAUTHN_RP_NAME", "Wrongmove")
+WEBAUTHN_ORIGIN = os.getenv("WEBAUTHN_ORIGIN", "https://localhost")
+
+# JWT Configuration (for passkey-issued tokens)
+JWT_SECRET = os.getenv("JWT_SECRET", "change-me-in-production")
+if JWT_SECRET == "change-me-in-production":
+    _logger.warning("JWT_SECRET is using the default value. Set JWT_SECRET env var in production.")
+JWT_ALGORITHM = os.getenv("JWT_ALGORITHM", "HS256")
+JWT_EXPIRATION_HOURS = int(os.getenv("JWT_EXPIRATION_HOURS", "24"))
+JWT_ISSUER = os.getenv("JWT_ISSUER", "wrongmove")

api/metrics.py

@@ -0,0 +1,17 @@
+# metrics.py
+from opentelemetry.metrics import set_meter_provider
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry.exporter.prometheus import PrometheusMetricReader
+from prometheus_client import make_asgi_app
+
+# Set up Prometheus reader and meter provider
+reader = PrometheusMetricReader()
+provider = MeterProvider(
+    metric_readers=[reader],
+    resource=Resource.create({SERVICE_NAME: "fastapi-metrics-app"}),
+)
+set_meter_provider(provider)
+
+# Expose the Prometheus metrics endpoint
+metrics_app = make_asgi_app()  # Exposes /metrics

api/passkey_routes.py

@@ -0,0 +1,93 @@
+import logging
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel, EmailStr
+
+from database import engine
+from repositories.user_repository import UserRepository
+from services import passkey_service
+
+logger = logging.getLogger("uvicorn")
+
+passkey_router = APIRouter(prefix="/api/passkey", tags=["passkey"])
+
+
+class RegisterBeginRequest(BaseModel):
+    email: EmailStr
+
+
+class RegisterBeginResponse(BaseModel):
+    options: dict  # type: ignore[type-arg]
+    session_id: str
+
+
+class CeremonyCompleteRequest(BaseModel):
+    session_id: str
+    credential: dict  # type: ignore[type-arg]
+
+
+class AuthTokenResponse(BaseModel):
+    token: str
+
+
+class LoginBeginResponse(BaseModel):
+    options: dict  # type: ignore[type-arg]
+    session_id: str
+
+
+@passkey_router.post("/register/begin", response_model=RegisterBeginResponse)
+async def register_begin(body: RegisterBeginRequest) -> RegisterBeginResponse:
+    """Start passkey registration ceremony."""
+    try:
+        user_repo = UserRepository(engine)
+        options, session_id = passkey_service.begin_registration(
+            body.email, user_repo
+        )
+        return RegisterBeginResponse(options=options, session_id=session_id)
+    except Exception as e:
+        logger.error(f"Registration begin failed: {e}")
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@passkey_router.post("/register/complete", response_model=AuthTokenResponse)
+async def register_complete(body: CeremonyCompleteRequest) -> AuthTokenResponse:
+    """Complete passkey registration ceremony."""
+    try:
+        user_repo = UserRepository(engine)
+        token = passkey_service.complete_registration(
+            body.session_id, body.credential, user_repo
+        )
+        return AuthTokenResponse(token=token)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Registration complete failed: {e}")
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@passkey_router.post("/login/begin", response_model=LoginBeginResponse)
+async def login_begin() -> LoginBeginResponse:
+    """Start passkey authentication ceremony."""
+    try:
+        user_repo = UserRepository(engine)
+        options, session_id = passkey_service.begin_authentication(user_repo)
+        return LoginBeginResponse(options=options, session_id=session_id)
+    except Exception as e:
+        logger.error(f"Login begin failed: {e}")
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@passkey_router.post("/login/complete", response_model=AuthTokenResponse)
+async def login_complete(body: CeremonyCompleteRequest) -> AuthTokenResponse:
+    """Complete passkey authentication ceremony."""
+    try:
+        user_repo = UserRepository(engine)
+        token = passkey_service.complete_authentication(
+            body.session_id, body.credential, user_repo
+        )
+        return AuthTokenResponse(token=token)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Login complete failed: {e}")
+        raise HTTPException(status_code=400, detail=str(e))