Add structured JSON logging, OTel business metrics, and Grafana dashboard
Structured logging via JsonFormatter replaces uvicorn's default format so Loki can parse timestamps and fields. 14 business metrics (scrape stats, throttle events, circuit breaker state, cache hit rate, OCR success rate, Celery task lifecycle) are defined in a shared metrics module and instrumented across the scraper pipeline, API, and workers. Celery workers expose a Prometheus HTTP endpoint on configurable ports.
This commit is contained in:
Viktor Barzin
parent
a1829957c1
commit
d6edb747d2
12 changed files with 742 additions and 49 deletions
23
api/app.py
23
api/app.py
|
|
@ -39,12 +39,13 @@ from services.listing_cache import (
|
|||
from repositories.poi_repository import POIRepository
|
||||
from repositories.user_repository import UserRepository
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
from api.metrics import metrics_app
|
||||
from opentelemetry.metrics import get_meter
|
||||
from api.metrics import init_metrics, get_metrics_asgi_app, geojson_cache_operations
|
||||
from logging_config import configure_logging
|
||||
|
||||
|
||||
load_dotenv()
|
||||
logger = logging.getLogger("uvicorn")
|
||||
configure_logging("api")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_BATCH_SIZE = 50
|
||||
_rate_limit_config = RateLimitConfig.from_env()
|
||||
|
|
@ -99,16 +100,8 @@ app = FastAPI(
|
|||
app.include_router(passkey_router)
|
||||
app.include_router(poi_router)
|
||||
app.include_router(ws_router)
|
||||
app.mount("/metrics", metrics_app)
|
||||
meter = get_meter(__name__)
|
||||
request_counter = meter.create_counter(
|
||||
name="custom_request_count",
|
||||
description="Number of times /hello was called",
|
||||
)
|
||||
hist = meter.create_histogram(
|
||||
name="custom_request_duration",
|
||||
description="Duration of /hello requests in seconds",
|
||||
)
|
||||
init_metrics("realestate-crawler-api")
|
||||
app.mount("/metrics", get_metrics_asgi_app())
|
||||
|
||||
|
||||
# Allow CORS (for React frontend)
|
||||
|
|
@ -146,8 +139,6 @@ async def unhandled_exception_handler(request: Request, exc: Exception) -> JSONR
|
|||
|
||||
@app.get("/api/status")
|
||||
async def get_status() -> dict[str, str]:
|
||||
request_counter.add(1, {"method": "GET", "path": "/status"})
|
||||
hist.record(1.5, {"method": "GET", "path": "/status"})
|
||||
return {"status": "OK"}
|
||||
|
||||
|
||||
|
|
@ -333,8 +324,10 @@ async def stream_listing_geojson(
|
|||
|
||||
cached_count = get_cached_count(query_parameters)
|
||||
if cached_count is not None and cached_count > 0 and not include_poi_distances:
|
||||
geojson_cache_operations.add(1, {"result": "hit"})
|
||||
generator = _stream_from_cache(query_parameters, batch_size, limit)
|
||||
else:
|
||||
geojson_cache_operations.add(1, {"result": "miss"})
|
||||
generator = _stream_from_db(
|
||||
query_parameters, batch_size, limit, poi_distances_lookup,
|
||||
skip_cache=include_poi_distances,
|
||||
|
|
|
|||
|
|
@ -51,13 +51,15 @@ class AuditLogMiddleware(BaseHTTPMiddleware):
|
|||
|
||||
duration_ms = (time.monotonic() - start) * 1000
|
||||
audit_logger.info(
|
||||
"method=%s path=%s query=%s user=%s ip=%s status=%d duration_ms=%.1f",
|
||||
request.method,
|
||||
path,
|
||||
query,
|
||||
identity,
|
||||
ip,
|
||||
response.status_code,
|
||||
duration_ms,
|
||||
"API request",
|
||||
extra={
|
||||
"method": request.method,
|
||||
"path": path,
|
||||
"query": query,
|
||||
"user": identity,
|
||||
"ip": ip,
|
||||
"status": response.status_code,
|
||||
"duration_ms": round(duration_ms, 1),
|
||||
},
|
||||
)
|
||||
return response
|
||||
|
|
|
|||
157
api/metrics.py
157
api/metrics.py
|
|
@ -1,17 +1,152 @@
|
|||
# metrics.py
|
||||
from opentelemetry.metrics import set_meter_provider
|
||||
"""OpenTelemetry metrics with Prometheus export.
|
||||
|
||||
Provides ``init_metrics()`` to lazily initialise the MeterProvider and all
|
||||
business metric instruments. Safe to call from both the API and Celery
|
||||
workers — the provider is created at most once per process.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from opentelemetry.metrics import (
|
||||
Counter,
|
||||
Histogram,
|
||||
Meter,
|
||||
UpDownCounter,
|
||||
get_meter,
|
||||
set_meter_provider,
|
||||
)
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.resources import SERVICE_NAME, Resource
|
||||
from opentelemetry.exporter.prometheus import PrometheusMetricReader
|
||||
from prometheus_client import make_asgi_app
|
||||
|
||||
# Set up Prometheus reader and meter provider
|
||||
reader = PrometheusMetricReader()
|
||||
provider = MeterProvider(
|
||||
metric_readers=[reader],
|
||||
resource=Resource.create({SERVICE_NAME: "fastapi-metrics-app"}),
|
||||
)
|
||||
set_meter_provider(provider)
|
||||
_reader: PrometheusMetricReader | None = None
|
||||
_meter: Meter | None = None
|
||||
|
||||
# Expose the Prometheus metrics endpoint
|
||||
metrics_app = make_asgi_app() # Exposes /metrics
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scrape metrics
|
||||
# ---------------------------------------------------------------------------
|
||||
scrape_listings_found: Counter
|
||||
scrape_listings_processed: Counter
|
||||
scrape_listings_failed: Counter
|
||||
scrape_duration_seconds: Histogram
|
||||
scrape_pages_fetched: Counter
|
||||
scrape_subqueries_total: Counter
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Throttle / circuit-breaker metrics
|
||||
# ---------------------------------------------------------------------------
|
||||
throttle_events_total: Counter
|
||||
# circuit_breaker_state is registered as an ObservableGauge in circuit_breaker.py
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API / cache metrics
|
||||
# ---------------------------------------------------------------------------
|
||||
geojson_cache_operations: Counter
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OCR metrics
|
||||
# ---------------------------------------------------------------------------
|
||||
ocr_attempts: Counter
|
||||
ocr_successes: Counter
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Celery task metrics
|
||||
# ---------------------------------------------------------------------------
|
||||
celery_tasks_total: Counter
|
||||
celery_task_duration_seconds: Histogram
|
||||
celery_tasks_active: UpDownCounter
|
||||
|
||||
|
||||
def init_metrics(service_name: str = "realestate-crawler") -> PrometheusMetricReader:
|
||||
"""Initialise the OTel MeterProvider and define all instruments.
|
||||
|
||||
Returns the ``PrometheusMetricReader`` so the API can mount the ASGI app.
|
||||
Calling this more than once is a no-op (returns the existing reader).
|
||||
"""
|
||||
global _reader, _meter
|
||||
global scrape_listings_found, scrape_listings_processed, scrape_listings_failed
|
||||
global scrape_duration_seconds, scrape_pages_fetched, scrape_subqueries_total
|
||||
global throttle_events_total
|
||||
global geojson_cache_operations
|
||||
global ocr_attempts, ocr_successes
|
||||
global celery_tasks_total, celery_task_duration_seconds, celery_tasks_active
|
||||
|
||||
if _reader is not None:
|
||||
return _reader
|
||||
|
||||
_reader = PrometheusMetricReader()
|
||||
provider = MeterProvider(
|
||||
metric_readers=[_reader],
|
||||
resource=Resource.create({SERVICE_NAME: service_name}),
|
||||
)
|
||||
set_meter_provider(provider)
|
||||
_meter = get_meter(__name__)
|
||||
|
||||
# -- Scrape --
|
||||
scrape_listings_found = _meter.create_counter(
|
||||
"scrape_listings_found_total",
|
||||
description="Total listings discovered during scrape runs",
|
||||
)
|
||||
scrape_listings_processed = _meter.create_counter(
|
||||
"scrape_listings_processed_total",
|
||||
description="Total listings successfully processed",
|
||||
)
|
||||
scrape_listings_failed = _meter.create_counter(
|
||||
"scrape_listings_failed_total",
|
||||
description="Total listings that failed processing",
|
||||
)
|
||||
scrape_duration_seconds = _meter.create_histogram(
|
||||
"scrape_duration_seconds",
|
||||
description="Duration of a full scrape run in seconds",
|
||||
)
|
||||
scrape_pages_fetched = _meter.create_counter(
|
||||
"scrape_pages_fetched_total",
|
||||
description="Total API pages fetched during scraping",
|
||||
)
|
||||
scrape_subqueries_total = _meter.create_counter(
|
||||
"scrape_subqueries_total",
|
||||
description="Total subqueries executed after query splitting",
|
||||
)
|
||||
|
||||
# -- Throttle --
|
||||
throttle_events_total = _meter.create_counter(
|
||||
"throttle_events_total",
|
||||
description="Total throttling events by type",
|
||||
)
|
||||
|
||||
# -- Cache --
|
||||
geojson_cache_operations = _meter.create_counter(
|
||||
"geojson_cache_operations_total",
|
||||
description="GeoJSON cache operations (hit/miss)",
|
||||
)
|
||||
|
||||
# -- OCR --
|
||||
ocr_attempts = _meter.create_counter(
|
||||
"ocr_attempts_total",
|
||||
description="Total OCR detection attempts",
|
||||
)
|
||||
ocr_successes = _meter.create_counter(
|
||||
"ocr_successes_total",
|
||||
description="Total OCR detections that found square meters",
|
||||
)
|
||||
|
||||
# -- Celery --
|
||||
celery_tasks_total = _meter.create_counter(
|
||||
"celery_tasks_total",
|
||||
description="Total Celery tasks by name and status",
|
||||
)
|
||||
celery_task_duration_seconds = _meter.create_histogram(
|
||||
"celery_task_duration_seconds",
|
||||
description="Duration of Celery tasks in seconds",
|
||||
)
|
||||
celery_tasks_active = _meter.create_up_down_counter(
|
||||
"celery_tasks_active",
|
||||
description="Currently active Celery tasks",
|
||||
)
|
||||
|
||||
return _reader
|
||||
|
||||
|
||||
def get_metrics_asgi_app(): # type: ignore[no-untyped-def]
|
||||
"""Return the Prometheus ASGI app for mounting at /metrics."""
|
||||
return make_asgi_app()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue