wrongmove/frontend/src/services/__tests__/streamingService.test.ts

257 lines
8.3 KiB
TypeScript
Raw Normal View History

import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { mockUser, createMockFeature } from '@/__tests__/helpers';
wrongmove: round-3 fix sweep — scrape pipeline, BUY tab, filter URL state, render hygiene, map polish Coordinated fix across 31 bugs found in a parallel QA pass. Findings docs at /tmp/wrongmove-bugs/qa-round-3/qa{1,2,3,4}-*.md. ## Backend / scrape (Fix-1) — 8 bugs - B1 [P0] Scrape totally broken on prod: pod UID 100 vs NFS dir 1000:1000 mode 775 → PermissionError on every never-seen listing. Switched Dockerfile to explicit `useradd --uid 1000 --gid 1000`; added securityContext + chown initContainer to k8s/{api,celery-beat}-deployment.yaml. Celery worker manifest lives outside this repo — Dockerfile UID change is the load-bearing fix. - B4 [P1] Celery broker reaped every ~30s by Redis HAProxy idle timeout. Added `broker_transport_options` / `result_backend_transport_options` with `socket_keepalive=True, health_check_interval=25` in celery_app.py + same kwargs on every redis.from_url/Redis call across services/, utils/redis_lock.py, redis_repository.py. - B5 [P1] dump_listings_task never published terminal FAILURE to the task_progress pub/sub channel — UI polled forever. Wrap body in try/except that publishes FAILURE before re-raising. - B6 [P1] _process_worker had no per-listing exception handler — one bad listing killed the whole scrape via asyncio.gather. Wrap loop body in try/except Exception (re-raises CancelledError). - B20 [P2] dump_listings_task gained time_limit=3600, soft_time_limit=3500, acks_late=True. - B21 [P2] RedisRepository moved off shared db0 (was alongside paperless-ngx) to db3 via REDIS_USER_DB env var; keys prefixed `wrongmove:user:`. - B32 [P3] redis_lock now uses uuid4() owner token + Lua compare-and-delete. - B33 [P3] Slack notify in refresh_listings → asyncio.create_task (fire-and-forget). ## Frontend filter system (Fix-2) — 7 bugs - B2 [P0] BUY tab click triggered "Maximum update depth exceeded" → ErrorBoundary. Replaced the three mutually-triggering useEffects in FilterBar with a single one-way controlled-value flow (URL → parent state → form), guarded by previousListingTypeRef so price-defaults fires once per real transition. - B3 [P0] Filter values never reached the URL. Wired useFilterParams.setFilterValues into FilterBar/FilterPanel onSubmit + handleRemoveChip + new handleResetAllFilters; fed parsed filterValues into both forms' defaultValues; added URL→form sync via form.reset on browser back/forward. - B8 [P1] Chip removal now resets form state via new FilterBar onFormReady callback — More badge no longer sticks. - B12 [P2] Desktop swipe-review FAB added next to header (mobile FAB unchanged). - B17 [P2] "Reset all" affordance on chip strip. - B22 [P2] formatPrice precision: 1500 → £1.5k, 2500 → £2.5k (no longer collides with £2k/£3k defaults). - B30 [P3] last_seen_days input gained min={0}. ## Frontend render hygiene + data integrity (Fix-3) — 8 bugs - B7 [P1] streamingService bails on first non-NDJSON chunk (HTML response = backend down) and throws StreamParseError so the existing AlertError dialog surfaces a single user-visible error instead of 18× console.error spam. - B9 [P1] formatDuration widened to (null|undefined|number): returns "—" for non-finite or negative, caps implausibly large values. - B10 [P1] PropertyCard / PropertyCardCompact / SwipeCard JSX leaves render "—" for null total_price/qm/qmprice (was "£0/0 m²/£0/m²" — looked like free listings). - B13 [P2] hexgrid worker reduceAverage uses Number.isFinite filter instead of !isNaN (which incorrectly accepted null → 0, biasing per-hex averages low). - B14 [P2] ListingDetail Overview wraps agency in "Listed by" labelled block so it can't collapse to a bare agency name. - B15 [P2] Compact POIDistanceBadges iterates all three travel modes with "—" for missing, matching the detail-sheet Travel table. - B24 [P3] Drawer.Description (sr-only) added to ListingDetailSheet + MobileBottomSheet to silence Radix a11y warning. - B25 [P3] lastSeenDays clamped to ≥0 so future timestamps don't render as "-7d ago". ## Frontend map / carousel / tasks polish (Fix-4) — 8 bugs - B11 [P2] HexgridHeatmapClient destroy race: Map.tsx adds .catch() + ref guard so post-destroy promise rejections are silent no-ops. Verified by browser smoke (24 rapid Map↔List toggles → 0 pageErrors). - B16 [P2] PhotoCarousel + inner CardCarousel gained keyboard nav (Arrow keys). - B18 [P2] Default map center moved from Czech Republic to London (zoom 10). - B19+B29 [P2/P3] Mapbox token: no longer hard-coded fallback; reads env-only and shows a clear "Map unavailable — set VITE_MAPBOX_TOKEN" banner when missing. - B23 [P3] PhotoCarousel suppresses "1/1" counter for single-photo listings; added onError fallback for broken URLs. - B26 [P3] PhotoCarousel only enables loop when photos.length > 1. - B27 [P3] TaskIndicator cancel/clear-all buttons gained aria-label + data-testid. - B28 [P3] useTaskProgress strips terminal-local task IDs from the polling union — no more forever-poll on completed tasks. ## Tests 74 new vitest tests + 18 new pytest tests. Local: tsc clean, 201 vitest tests pass, 633 pytest tests pass. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-10 22:27:29 +00:00
import { streamListingGeoJSON, StreamParseError } from '@/services/streamingService';
import { ApiError } from '@/types';
import type { ParameterValues } from '@/components/FilterPanel';
const defaultParams: ParameterValues = {
metric: 'qmprice',
listing_type: 'RENT',
min_bedrooms: 1,
max_bedrooms: 3,
min_price: 1000,
max_price: 5000,
district: '',
};
function createMockResponse(lines: string[]) {
const body = lines.join('\n') + '\n';
const encoder = new TextEncoder();
const stream = new ReadableStream({
start(controller) {
controller.enqueue(encoder.encode(body));
controller.close();
},
});
return new Response(stream, {
status: 200,
headers: { 'Content-Type': 'application/x-ndjson' },
});
}
describe('streamingService', () => {
const originalFetch = globalThis.fetch;
afterEach(() => {
globalThis.fetch = originalFetch;
});
it('yields feature batches', async () => {
const features = [createMockFeature(), createMockFeature({ rooms: 3 })];
globalThis.fetch = vi.fn().mockResolvedValue(
createMockResponse([
JSON.stringify({ type: 'metadata', total_expected: 2 }),
JSON.stringify({ type: 'batch', features }),
JSON.stringify({ type: 'complete', total: 2 }),
]),
);
const batches: unknown[][] = [];
for await (const batch of streamListingGeoJSON(mockUser(), defaultParams)) {
batches.push(batch);
}
expect(batches).toHaveLength(1);
expect(batches[0]).toHaveLength(2);
});
it('calls onProgress with metadata', async () => {
const onProgress = vi.fn();
globalThis.fetch = vi.fn().mockResolvedValue(
createMockResponse([
JSON.stringify({ type: 'metadata', total_expected: 42 }),
JSON.stringify({ type: 'complete', total: 0 }),
]),
);
const gen = streamListingGeoJSON(mockUser(), defaultParams, onProgress);
// Consume the generator
for await (const _ of gen) { /* drain */ }
expect(onProgress).toHaveBeenCalledWith({ count: 0, total: 42 });
});
it('calls onProgress with incrementing count on batches', async () => {
const features1 = [createMockFeature()];
const features2 = [createMockFeature(), createMockFeature()];
const onProgress = vi.fn();
globalThis.fetch = vi.fn().mockResolvedValue(
createMockResponse([
JSON.stringify({ type: 'metadata', total_expected: 3 }),
JSON.stringify({ type: 'batch', features: features1 }),
JSON.stringify({ type: 'batch', features: features2 }),
JSON.stringify({ type: 'complete', total: 3 }),
]),
);
for await (const _ of streamListingGeoJSON(mockUser(), defaultParams, onProgress)) { /* drain */ }
// metadata: count=0, batch1: count=1, batch2: count=3, complete: count=3
expect(onProgress).toHaveBeenCalledWith({ count: 1 });
expect(onProgress).toHaveBeenCalledWith({ count: 3 });
});
it('calls onProgress on complete message', async () => {
const onProgress = vi.fn();
globalThis.fetch = vi.fn().mockResolvedValue(
createMockResponse([
JSON.stringify({ type: 'metadata', total_expected: 5 }),
JSON.stringify({ type: 'complete', total: 5 }),
]),
);
for await (const _ of streamListingGeoJSON(mockUser(), defaultParams, onProgress)) { /* drain */ }
expect(onProgress).toHaveBeenCalledWith({ count: 5, total: 5 });
});
it('handles multiple batches and yields each separately', async () => {
const batch1 = [createMockFeature({ rooms: 1 })];
const batch2 = [createMockFeature({ rooms: 2 })];
globalThis.fetch = vi.fn().mockResolvedValue(
createMockResponse([
JSON.stringify({ type: 'metadata', total_expected: 2 }),
JSON.stringify({ type: 'batch', features: batch1 }),
JSON.stringify({ type: 'batch', features: batch2 }),
JSON.stringify({ type: 'complete', total: 2 }),
]),
);
const batches: unknown[][] = [];
for await (const batch of streamListingGeoJSON(mockUser(), defaultParams)) {
batches.push(batch);
}
expect(batches).toHaveLength(2);
expect(batches[0]).toHaveLength(1);
expect(batches[1]).toHaveLength(1);
});
it('handles abort signal', async () => {
const controller = new AbortController();
controller.abort();
globalThis.fetch = vi.fn().mockRejectedValue(new DOMException('Aborted', 'AbortError'));
const batches: unknown[][] = [];
try {
for await (const batch of streamListingGeoJSON(mockUser(), defaultParams, undefined, { signal: controller.signal })) {
batches.push(batch);
}
} catch {
// Expected: fetch throws on aborted signal
}
expect(batches).toHaveLength(0);
});
it('handles malformed JSON lines without crashing', async () => {
const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
const features = [createMockFeature()];
globalThis.fetch = vi.fn().mockResolvedValue(
createMockResponse([
JSON.stringify({ type: 'metadata', total_expected: 1 }),
'this is not valid json{{{',
JSON.stringify({ type: 'batch', features }),
JSON.stringify({ type: 'complete', total: 1 }),
]),
);
const batches: unknown[][] = [];
for await (const batch of streamListingGeoJSON(mockUser(), defaultParams)) {
batches.push(batch);
}
// Should still yield the valid batch
expect(batches).toHaveLength(1);
expect(consoleSpy).toHaveBeenCalled();
consoleSpy.mockRestore();
});
it('handles empty stream with no batches', async () => {
globalThis.fetch = vi.fn().mockResolvedValue(
createMockResponse([
JSON.stringify({ type: 'metadata', total_expected: 0 }),
JSON.stringify({ type: 'complete', total: 0 }),
]),
);
const batches: unknown[][] = [];
for await (const batch of streamListingGeoJSON(mockUser(), defaultParams)) {
batches.push(batch);
}
expect(batches).toHaveLength(0);
});
it('throws ApiError on HTTP error response', async () => {
globalThis.fetch = vi.fn().mockResolvedValue(
new Response(null, { status: 401 }),
);
await expect(async () => {
for await (const _ of streamListingGeoJSON(mockUser(), defaultParams)) { /* drain */ }
}).rejects.toThrow(ApiError);
});
it('throws when response body is null', async () => {
const response = new Response(null, { status: 200 });
// Force body to null by creating a response without a body that reports ok
Object.defineProperty(response, 'body', { value: null });
globalThis.fetch = vi.fn().mockResolvedValue(response);
await expect(async () => {
for await (const _ of streamListingGeoJSON(mockUser(), defaultParams)) { /* drain */ }
}).rejects.toThrow('No response body');
});
wrongmove: round-3 fix sweep — scrape pipeline, BUY tab, filter URL state, render hygiene, map polish Coordinated fix across 31 bugs found in a parallel QA pass. Findings docs at /tmp/wrongmove-bugs/qa-round-3/qa{1,2,3,4}-*.md. ## Backend / scrape (Fix-1) — 8 bugs - B1 [P0] Scrape totally broken on prod: pod UID 100 vs NFS dir 1000:1000 mode 775 → PermissionError on every never-seen listing. Switched Dockerfile to explicit `useradd --uid 1000 --gid 1000`; added securityContext + chown initContainer to k8s/{api,celery-beat}-deployment.yaml. Celery worker manifest lives outside this repo — Dockerfile UID change is the load-bearing fix. - B4 [P1] Celery broker reaped every ~30s by Redis HAProxy idle timeout. Added `broker_transport_options` / `result_backend_transport_options` with `socket_keepalive=True, health_check_interval=25` in celery_app.py + same kwargs on every redis.from_url/Redis call across services/, utils/redis_lock.py, redis_repository.py. - B5 [P1] dump_listings_task never published terminal FAILURE to the task_progress pub/sub channel — UI polled forever. Wrap body in try/except that publishes FAILURE before re-raising. - B6 [P1] _process_worker had no per-listing exception handler — one bad listing killed the whole scrape via asyncio.gather. Wrap loop body in try/except Exception (re-raises CancelledError). - B20 [P2] dump_listings_task gained time_limit=3600, soft_time_limit=3500, acks_late=True. - B21 [P2] RedisRepository moved off shared db0 (was alongside paperless-ngx) to db3 via REDIS_USER_DB env var; keys prefixed `wrongmove:user:`. - B32 [P3] redis_lock now uses uuid4() owner token + Lua compare-and-delete. - B33 [P3] Slack notify in refresh_listings → asyncio.create_task (fire-and-forget). ## Frontend filter system (Fix-2) — 7 bugs - B2 [P0] BUY tab click triggered "Maximum update depth exceeded" → ErrorBoundary. Replaced the three mutually-triggering useEffects in FilterBar with a single one-way controlled-value flow (URL → parent state → form), guarded by previousListingTypeRef so price-defaults fires once per real transition. - B3 [P0] Filter values never reached the URL. Wired useFilterParams.setFilterValues into FilterBar/FilterPanel onSubmit + handleRemoveChip + new handleResetAllFilters; fed parsed filterValues into both forms' defaultValues; added URL→form sync via form.reset on browser back/forward. - B8 [P1] Chip removal now resets form state via new FilterBar onFormReady callback — More badge no longer sticks. - B12 [P2] Desktop swipe-review FAB added next to header (mobile FAB unchanged). - B17 [P2] "Reset all" affordance on chip strip. - B22 [P2] formatPrice precision: 1500 → £1.5k, 2500 → £2.5k (no longer collides with £2k/£3k defaults). - B30 [P3] last_seen_days input gained min={0}. ## Frontend render hygiene + data integrity (Fix-3) — 8 bugs - B7 [P1] streamingService bails on first non-NDJSON chunk (HTML response = backend down) and throws StreamParseError so the existing AlertError dialog surfaces a single user-visible error instead of 18× console.error spam. - B9 [P1] formatDuration widened to (null|undefined|number): returns "—" for non-finite or negative, caps implausibly large values. - B10 [P1] PropertyCard / PropertyCardCompact / SwipeCard JSX leaves render "—" for null total_price/qm/qmprice (was "£0/0 m²/£0/m²" — looked like free listings). - B13 [P2] hexgrid worker reduceAverage uses Number.isFinite filter instead of !isNaN (which incorrectly accepted null → 0, biasing per-hex averages low). - B14 [P2] ListingDetail Overview wraps agency in "Listed by" labelled block so it can't collapse to a bare agency name. - B15 [P2] Compact POIDistanceBadges iterates all three travel modes with "—" for missing, matching the detail-sheet Travel table. - B24 [P3] Drawer.Description (sr-only) added to ListingDetailSheet + MobileBottomSheet to silence Radix a11y warning. - B25 [P3] lastSeenDays clamped to ≥0 so future timestamps don't render as "-7d ago". ## Frontend map / carousel / tasks polish (Fix-4) — 8 bugs - B11 [P2] HexgridHeatmapClient destroy race: Map.tsx adds .catch() + ref guard so post-destroy promise rejections are silent no-ops. Verified by browser smoke (24 rapid Map↔List toggles → 0 pageErrors). - B16 [P2] PhotoCarousel + inner CardCarousel gained keyboard nav (Arrow keys). - B18 [P2] Default map center moved from Czech Republic to London (zoom 10). - B19+B29 [P2/P3] Mapbox token: no longer hard-coded fallback; reads env-only and shows a clear "Map unavailable — set VITE_MAPBOX_TOKEN" banner when missing. - B23 [P3] PhotoCarousel suppresses "1/1" counter for single-photo listings; added onError fallback for broken URLs. - B26 [P3] PhotoCarousel only enables loop when photos.length > 1. - B27 [P3] TaskIndicator cancel/clear-all buttons gained aria-label + data-testid. - B28 [P3] useTaskProgress strips terminal-local task IDs from the polling union — no more forever-poll on completed tasks. ## Tests 74 new vitest tests + 18 new pytest tests. Local: tsc clean, 201 vitest tests pass, 633 pytest tests pass. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-10 22:27:29 +00:00
// B7 regression: HTML response (e.g. proxy SPA fallback when backend is down)
// should bail on the first parse error instead of looping 18× per stream.
it('throws StreamParseError on the first unparseable line when nothing has parsed yet', async () => {
const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
const htmlLines = [
'<!doctype html>',
'<html>',
' <head><title>App</title></head>',
' <body><div id="root"></div></body>',
'</html>',
];
globalThis.fetch = vi.fn().mockResolvedValue(createMockResponse(htmlLines));
let caught: unknown = null;
try {
for await (const _ of streamListingGeoJSON(mockUser(), defaultParams)) { /* drain */ }
} catch (e) {
caught = e;
}
expect(caught).toBeInstanceOf(StreamParseError);
// Critical: no console.error spam — bail immediately on first failure.
expect(consoleSpy).not.toHaveBeenCalled();
consoleSpy.mockRestore();
});
it('StreamParseError captures a snippet of the offending input', async () => {
globalThis.fetch = vi.fn().mockResolvedValue(
createMockResponse(['<!doctype html><html><head>...</head>']),
);
let caught: StreamParseError | null = null;
try {
for await (const _ of streamListingGeoJSON(mockUser(), defaultParams)) { /* drain */ }
} catch (e) {
caught = e as StreamParseError;
}
expect(caught).toBeInstanceOf(StreamParseError);
expect(caught?.snippet).toContain('<!doctype html>');
});
});