viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add proper buy listing support with type-aware UI filters and display

Browse source
This commit is contained in:
Viktor Barzin 2026-02-01 19:13:29 +00:00
parent c7ac448f15
commit 6d8f69610f
6 changed files with 416 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crawler/api/app.py
View file

@ -146,7 +146,7 @@ async def stream_listing_geojson(
for row in repository.stream_listings_optimized(
query_parameters, limit=limit, page_size=batch_size
):
feature = convert_row_to_geojson(row)
feature = convert_row_to_geojson(row, query_parameters.listing_type.value)
batch.append(feature)
count += 1

107
crawler/frontend/src/components/FilterPanel.tsx
View file

@ -101,6 +101,24 @@ export function FilterPanel({ onSubmit, isLoading, listingCount }: FilterPanelPr
},
});
// Watch listing_type to make filters type-aware
const watchedListingType = form.watch('listing_type');
// Update price defaults when listing type changes
useEffect(() => {
if (watchedListingType === ListingType.BUY) {
form.setValue('min_price', 300000);
form.setValue('max_price', 600000);
} else {
form.setValue('min_price', 2000);
form.setValue('max_price', 3000);
}
// Clear furnish types when switching to BUY
if (watchedListingType === ListingType.BUY) {
setSelectedFurnishTypes([]);
}
}, [watchedListingType, form]);
const handleFormSubmit = (action: 'fetch-data' | 'visualize') => {
return form.handleSubmit((values) => {
const params: ParameterValues = {
@ -400,29 +418,31 @@ export function FilterPanel({ onSubmit, isLoading, listingCount }: FilterPanelPr
)}
/>
</div>
<div>
<FormLabel className="text-xs">Furnishing</FormLabel>
<div className="flex flex-wrap gap-2 mt-2">
{[
{ value: FurnishType.FURNISHED, label: 'Furnished' },
{ value: FurnishType.PART_FURNISHED, label: 'Part' },
{ value: FurnishType.UNFURNISHED, label: 'Unfurn.' },
].map((option) => (
<button
key={option.value}
type="button"
onClick={() => toggleFurnishType(option.value)}
className={`px-2 py-1 text-xs rounded-md border transition-colors ${
selectedFurnishTypes.includes(option.value)
? 'bg-primary text-primary-foreground border-primary'
: 'bg-background hover:bg-muted border-input'
}`}
>
{option.label}
</button>
))}
{watchedListingType === ListingType.RENT && (
<div>
<FormLabel className="text-xs">Furnishing</FormLabel>
<div className="flex flex-wrap gap-2 mt-2">
{[
{ value: FurnishType.FURNISHED, label: 'Furnished' },
{ value: FurnishType.PART_FURNISHED, label: 'Part' },
{ value: FurnishType.UNFURNISHED, label: 'Unfurn.' },
].map((option) => (
<button
key={option.value}
type="button"
onClick={() => toggleFurnishType(option.value)}
className={`px-2 py-1 text-xs rounded-md border transition-colors ${
selectedFurnishTypes.includes(option.value)
? 'bg-primary text-primary-foreground border-primary'
: 'bg-background hover:bg-muted border-input'
}`}
>
{option.label}
</button>
))}
</div>
</div>
</div>
)}
</div>
</AccordionContent>
</AccordionItem>
@ -456,33 +476,32 @@ export function FilterPanel({ onSubmit, isLoading, listingCount }: FilterPanelPr
</AccordionContent>
</AccordionItem>
{/* Availability */}
{/* Availability / Recency */}
<AccordionItem value="availability">
<AccordionTrigger className="py-2 text-sm font-medium">
Availability
{watchedListingType === ListingType.RENT ? 'Availability' : 'Recency'}
</AccordionTrigger>
<AccordionContent>
<div className="space-y-4">
<FormField
control={form.control}
name="available_from"
render={({ field }) => (
<FormItem>
<FormLabel className="text-xs">Available From</FormLabel>
<FormControl>
<Calendar29
onSelect={field.onChange}
selected={field.value}
rawInputValue={availableFromRawInput}
onChangeRawInputValue={setAvailableFromRawInput}
/>
</FormControl>
<FormDescription className="text-xs">
Rental listings only
</FormDescription>
</FormItem>
)}
/>
{watchedListingType === ListingType.RENT && (
<FormField
control={form.control}
name="available_from"
render={({ field }) => (
<FormItem>
<FormLabel className="text-xs">Available From</FormLabel>
<FormControl>
<Calendar29
onSelect={field.onChange}
selected={field.value}
rawInputValue={availableFromRawInput}
onChangeRawInputValue={setAvailableFromRawInput}
/>
</FormControl>
</FormItem>
)}
/>
)}
<FormField
control={form.control}
name="last_seen_days"

24
crawler/frontend/src/components/PropertyCard.tsx
View file

@ -59,7 +59,9 @@ export function PropertyCard({
<div className="flex items-start justify-between gap-2">
<div className="font-semibold text-base truncate">
£{property.total_price.toLocaleString()}
<span className="text-muted-foreground font-normal text-sm">/mo</span>
{property.listing_type !== 'BUY' && (
<span className="text-muted-foreground font-normal text-sm">/mo</span>
)}
</div>
{priceIndicator && (
<span className={`text-xs px-1.5 py-0.5 rounded ${priceIndicator.color}`}>
@ -119,7 +121,9 @@ export function PropertyCard({
<div>
<div className="font-semibold text-xl">
£{property.total_price.toLocaleString()}
<span className="text-muted-foreground font-normal text-sm">/mo</span>
{property.listing_type !== 'BUY' && (
<span className="text-muted-foreground font-normal text-sm">/mo</span>
)}
</div>
{priceIndicator && (
<span className={`inline-block mt-1 text-xs px-2 py-0.5 rounded ${priceIndicator.color}`}>
@ -145,10 +149,18 @@ export function PropertyCard({
<PoundSterling className="h-4 w-4 text-muted-foreground" />
<span><strong>£{property.qmprice}</strong>/m²</span>
</div>
<div className="flex items-center gap-2 text-sm">
<Clock className="h-4 w-4 text-muted-foreground" />
<span>Available <strong>{property.available_from}</strong></span>
</div>
{property.listing_type !== 'BUY' && property.available_from && (
<div className="flex items-center gap-2 text-sm">
<Clock className="h-4 w-4 text-muted-foreground" />
<span>Available <strong>{property.available_from}</strong></span>
</div>
)}
{property.listing_type === 'BUY' && (
<div className="flex items-center gap-2 text-sm">
<Clock className="h-4 w-4 text-muted-foreground" />
<span>Seen <strong>{lastSeenDays}d</strong> ago</span>
</div>
)}
</div>
{/* Agency and last seen */}

71
crawler/frontend/src/types/index.ts Normal file
View file

@ -0,0 +1,71 @@
// TypeScript types for the frontend application
// GeoJSON types
export interface PropertyPriceHistory {
id: number;
price: number;
last_seen: string;
}
export interface PropertyProperties {
url: string;
city: string;
country: string;
qm: number;
qmprice: number;
total_price: number;
rooms: number;
agency: string;
available_from: string;
last_seen: string;
photo_thumbnail: string;
price_history: PropertyPriceHistory[];
listing_type?: 'RENT' | 'BUY';
}
export interface PropertyFeature {
type: 'Feature';
geometry: {
type: 'Point';
coordinates: [number, number]; // [longitude, latitude]
};
properties: PropertyProperties;
}
export interface GeoJSONFeatureCollection {
type: 'FeatureCollection';
features: PropertyFeature[];
}
// Task status types
export enum TaskStatus {
PENDING = 'PENDING',
STARTED = 'STARTED',
SUCCESS = 'SUCCESS',
FAILURE = 'FAILURE',
REVOKED = 'REVOKED',
}
export interface TaskStatusResponse {
status: TaskStatus;
result: string; // JSON string containing { progress: number }
}
export interface TaskResult {
progress: number;
}
export interface RefreshListingsResponse {
task_id: string;
}
// API error type
export class ApiError extends Error {
constructor(
message: string,
public statusCode: number
) {
super(message);
this.name = 'ApiError';
}
}

153
crawler/repositories/listing_repository.py
View file

@ -1,5 +1,6 @@
from datetime import datetime, timedelta
import logging
from typing import Generator
from data_access import Listing
from models.listing import (
BuyListing,
@ -9,13 +10,20 @@ from models.listing import (
QueryParameters,
RentListing,
)
from sqlalchemy import Engine
from sqlalchemy import Engine, func, select as sa_select
from sqlmodel import Session, select
from sqlmodel.sql.expression import SelectOfScalar
from tqdm import tqdm
logger = logging.getLogger("uvicorn.error")
# Columns needed for GeoJSON streaming (excludes routing_info_json, additional_info)
STREAMING_COLUMNS = [
'id', 'price', 'number_of_bedrooms', 'square_meters',
'longitude', 'latitude', 'photo_thumbnail', 'last_seen',
'agency', 'price_history_json', 'available_from'
]
class ListingRepository:
engine: Engine
@ -58,6 +66,147 @@ class ListingRepository:
logging.debug(f"Found {len(rows)} listings")
return rows
def stream_listings(
self,
query_parameters: QueryParameters | None = None,
limit: int | None = None,
chunk_size: int = 100,
) -> Generator[modelListing, None, None]:
"""Yield listings one at a time for streaming.
Uses yield_per for memory-efficient iteration over large result sets.
Args:
query_parameters: Filtering parameters
limit: Maximum number of listings to yield
chunk_size: Number of rows to fetch at a time from the database
"""
model = RentListing # if no query params, default to renting listings
if query_parameters:
model = (
RentListing
if query_parameters.listing_type == ListingType.RENT
else BuyListing
)
query = select(model)
query = self._add_where_from_query_parameters(query, model, query_parameters)
if limit:
query = query.limit(limit)
with Session(self.engine) as session:
for listing in session.exec(query).yield_per(chunk_size):
yield listing
def _get_model_for_query(
self, query_parameters: QueryParameters | None
) -> type[RentListing] | type[BuyListing]:
"""Get the appropriate model class based on query parameters."""
if query_parameters and query_parameters.listing_type == ListingType.BUY:
return BuyListing
return RentListing
def count_listings(self, query_parameters: QueryParameters | None = None) -> int:
"""Fast count for progress estimation."""
model = self._get_model_for_query(query_parameters)
query = sa_select(func.count(model.id))
query = self._add_where_from_query_parameters_raw(query, model, query_parameters)
with Session(self.engine) as session:
return session.execute(query).scalar() or 0
def stream_listings_optimized(
self,
query_parameters: QueryParameters | None = None,
limit: int | None = None,
page_size: int = 100,
) -> Generator[dict, None, None]:
"""Stream listings with keyset pagination and column projection.
Uses keyset pagination for O(1) performance at any offset, and only
fetches columns needed for GeoJSON (excludes large JSON blobs).
Args:
query_parameters: Filtering parameters
limit: Maximum number of listings to yield
page_size: Number of rows to fetch per database round-trip
"""
model = self._get_model_for_query(query_parameters)
# Select only needed columns (excludes routing_info_json, additional_info)
columns = [
getattr(model, col) for col in STREAMING_COLUMNS if hasattr(model, col)
]
last_id: int | None = None
total_yielded = 0
while True:
if limit and total_yielded >= limit:
break
query = sa_select(*columns)
query = self._add_where_from_query_parameters_raw(
query, model, query_parameters
)
# Keyset pagination: WHERE id > last_id (O(1) performance)
if last_id is not None:
query = query.where(model.id > last_id)
batch_limit = page_size
if limit:
batch_limit = min(page_size, limit - total_yielded)
query = query.order_by(model.id).limit(batch_limit)
with Session(self.engine) as session:
results = session.execute(query).fetchall()
if not results:
break
for row in results:
yield row._asdict()
last_id = row.id
total_yielded += 1
if len(results) < page_size:
break
def _add_where_from_query_parameters_raw(
self,
query,
model: type[RentListing] | type[BuyListing],
query_parameters: QueryParameters | None = None,
):
"""Add WHERE clauses from query parameters (for raw SQLAlchemy selects)."""
if query_parameters is None:
return query
query = query.where(
model.number_of_bedrooms.between(
query_parameters.min_bedrooms, query_parameters.max_bedrooms
),
model.price.between(query_parameters.min_price, query_parameters.max_price),
)
if query_parameters.min_sqm is not None:
query = query.where(model.square_meters >= query_parameters.min_sqm)
if query_parameters.furnish_types and model == RentListing:
query = query.where(model.furnish_type.in_(query_parameters.furnish_types))
if (
model == RentListing
and query_parameters.let_date_available_from is not None
):
query = query.where(
model.available_from >= query_parameters.let_date_available_from
)
if query_parameters.last_seen_days is not None:
last_seen_threshold = datetime.now() - timedelta(
days=query_parameters.last_seen_days
)
query = query.where(model.last_seen >= last_seen_threshold)
return query
def _add_where_from_query_parameters(
self,
query: SelectOfScalar[Listing],
@ -74,7 +223,7 @@ class ListingRepository:
)
if query_parameters.min_sqm is not None:
query = query.where(model.square_meters >= query_parameters.min_sqm)
if query_parameters.furnish_types:
if query_parameters.furnish_types and model == RentListing:
query = query.where(model.furnish_type.in_(query_parameters.furnish_types))
if (
isinstance(model, RentListing)

146
crawler/ui_exporter.py
View file

@ -1,13 +1,122 @@
import json
import logging
import pathlib
from typing import Any
from models.listing import QueryParameters
from models.listing import QueryParameters, RentListing, BuyListing
from repositories.listing_repository import ListingRepository
logger = logging.getLogger("uvicorn.error")
def convert_row_to_geojson(row: dict[str, Any], listing_type: str = "RENT") -> dict[str, Any]:
"""Convert a projected row dict to GeoJSON Feature format.
This function handles dict rows from stream_listings_optimized(),
which uses column projection and returns dicts instead of model instances.
Args:
row: A dict with keys matching STREAMING_COLUMNS
Returns:
A GeoJSON Feature dict with properties and geometry
"""
# Parse price history from JSON string
price_history = []
if row.get('price_history_json'):
parsed = json.loads(row['price_history_json'])
price_history = [
{
"first_seen": p["first_seen"],
"last_seen": p["last_seen"],
"price": p["price"]
}
for p in parsed
]
sqm = row.get('square_meters')
price = row['price']
# Handle available_from which may be a datetime or None
available_from_val = row.get('available_from')
available_from_str = None
if available_from_val is not None:
if hasattr(available_from_val, 'isoformat'):
available_from_str = available_from_val.isoformat()
else:
available_from_str = str(available_from_val)
# Handle last_seen which should be a datetime
last_seen_val = row['last_seen']
if hasattr(last_seen_val, 'isoformat'):
last_seen_str = last_seen_val.isoformat()
else:
last_seen_str = str(last_seen_val)
return {
"type": "Feature",
"properties": {
"listing_type": listing_type,
"city": "London",
"country": "United Kingdom",
"qm": sqm,
"qmprice": round(price / sqm, 2) if sqm else None,
"rooms": row['number_of_bedrooms'],
"total_price": price,
"url": f"https://www.rightmove.co.uk/properties/{row['id']}",
"photo_thumbnail": row.get('photo_thumbnail'),
"last_seen": last_seen_str,
"price_history": price_history,
"agency": row.get('agency'),
"available_from": available_from_str,
},
"geometry": {
"coordinates": [row['longitude'], row['latitude']],
"type": "Point",
},
}
def convert_to_geojson_feature(listing: RentListing | BuyListing) -> dict[str, Any]:
"""Convert a single listing to GeoJSON Feature format.
Args:
listing: A RentListing or BuyListing model instance
Returns:
A GeoJSON Feature dict with properties and geometry
"""
# Safely access nested additional_info
property_info = listing.additional_info.get("property", {}) if listing.additional_info else {}
listing_type = "RENT" if isinstance(listing, RentListing) else "BUY"
return {
"type": "Feature",
"properties": {
"listing_type": listing_type,
"city": "London", # change me
"country": "United Kingdom",
"qm": listing.square_meters,
"qmprice": listing.price_per_square_meter,
"rooms": listing.number_of_bedrooms,
"total_price": listing.price,
"url": listing.url,
"photo_thumbnail": listing.photo_thumbnail,
"last_seen": listing.last_seen.isoformat(),
"price_history": [item.to_dict() for item in listing.price_history],
"agency": listing.agency,
"available_from": property_info.get("letDateAvailable", None),
},
"geometry": {
"coordinates": [
listing.longitude,
listing.latitude,
],
"type": "Point",
},
}
async def export_immoweb(
repository: ListingRepository,
output_file: str | None = None,
@ -20,39 +129,8 @@ async def export_immoweb(
)
logger.info(f"Fetched {len(listings)} listings")
# Convert listings to immoweb format
immoweb_listings = []
for listing in listings:
immoweb_listing = {
"type": "Feature",
"properties": {
"city": "London", # change me
"country": "United Kingdom",
"qm": listing.square_meters,
"qmprice": listing.price_per_square_meter,
"rooms": listing.number_of_bedrooms,
"total_price": listing.price,
"url": listing.url,
"photo_thumbnail": listing.photo_thumbnail,
"last_seen": listing.last_seen.isoformat(),
"price_history": [item.to_dict() for item in listing.price_history],
"agency": listing.agency,
"available_from": listing.additional_info["property"].get(
"letDateAvailable", None
),
# All other crap can be found in additional_info
# Prefer pulling out fields here instead of exporting the entire additional_info
# "info": listing.additional_info,
},
"geometry": {
"coordinates": [
listing.longitude,
listing.latitude,
],
"type": "Point",
},
}
immoweb_listings.append(immoweb_listing)
# Convert listings to GeoJSON features using the helper function
immoweb_listings = [convert_to_geojson_feature(listing) for listing in listings]
prefix = "var data = "
serialized_data = {"type": "FeatureCollection", "features": immoweb_listings}