From 69d15e9a16295a699e1ce32a43fa1f1ef0c00a96 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Wed, 18 Jun 2025 20:38:50 +0000 Subject: [PATCH] add support for querying buying listings as well as by max price --- crawler/api/app.py | 12 +++- crawler/frontend/src/App.tsx | 17 ++++-- .../frontend/src/components/Parameters.tsx | 55 +++++++++++++++++-- crawler/models/listing.py | 3 +- crawler/repositories/listing_repository.py | 39 +++++++------ 5 files changed, 95 insertions(+), 31 deletions(-) diff --git a/crawler/api/app.py b/crawler/api/app.py index b702ce8..8536940 100644 --- a/crawler/api/app.py +++ b/crawler/api/app.py @@ -1,8 +1,9 @@ from typing import Annotated from api.auth import get_current_user from api.config import DEV_TIER_ORIGINS, PROD_TIER_ORIGINS -from fastapi import Depends, FastAPI +from fastapi import Depends, FastAPI, Query from api.auth import User +from models.listing import QueryParameters from repositories.listing_repository import ListingRepository from repositories.listing_repository import ListingRepository from database import engine @@ -29,7 +30,12 @@ async def get_listing(user: Annotated[User, Depends(get_current_user)]): @app.get("/api/listing_geojson") -async def get_listing_geojson(user: Annotated[User, Depends(get_current_user)]): +async def get_listing_geojson( + user: Annotated[User, Depends(get_current_user)], + query_parameters: Annotated[QueryParameters, Query()], +): repository = ListingRepository(engine) - geojson_data = await export_immoweb(repository, limit=None) + geojson_data = await export_immoweb( + repository, query_parameters=query_parameters, limit=None + ) return geojson_data diff --git a/crawler/frontend/src/App.tsx b/crawler/frontend/src/App.tsx index 7348fa8..e736633 100644 --- a/crawler/frontend/src/App.tsx +++ b/crawler/frontend/src/App.tsx @@ -31,11 +31,15 @@ function App() { const [isParametersModalOpen, setIsParametersModalOpen] = useState(true) const [error, setError] = useState('') const [queryParameters, setQueryParameters] = useState(null) - const fetchData = async () => { - + const fetchData = async (parameters: ParameterValues) => { + const accessToken = user?.access_token; + const queryString = new URLSearchParams(); + queryString.append('listing_type', parameters.listing_type) + if (parameters.max_price) { + queryString.append("max_price", parameters.max_price.toString()); + } try { - const accessToken = user?.access_token; - const response = await fetch('/api/listing_geojson', + const response = await fetch("/api/listing_geojson?" + queryString, { method: 'GET', headers: { @@ -49,14 +53,15 @@ function App() { return data; } catch (err) { setError('Failed to fetch data: ' + err); - alert(error) + alert(JSON.stringify(err)) } finally { } }; const onSubmit = async (parameters: ParameterValues) => { // Fetch listing data setQueryParameters(parameters) - const data = await fetchData(); + const data = await fetchData(parameters); + console.log(data) if (data) { setListingData(data); } diff --git a/crawler/frontend/src/components/Parameters.tsx b/crawler/frontend/src/components/Parameters.tsx index fd6d9a6..e46f5dd 100644 --- a/crawler/frontend/src/components/Parameters.tsx +++ b/crawler/frontend/src/components/Parameters.tsx @@ -4,7 +4,8 @@ import { useForm } from "react-hook-form"; import { z } from "zod"; import { Button } from "./ui/button"; import { Dialog, DialogContent, DialogTrigger } from "./ui/dialog"; -import { Form, FormControl, FormDescription, FormField, FormItem, FormLabel, FormMessage } from "./ui/form"; +import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from "./ui/form"; +import { Input } from "./ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "./ui/select"; @@ -14,10 +15,16 @@ export enum Metric { qm = 'qm', price = 'total_price', } +export enum ListingType { + RENT = 'RENT', + BUY = 'BUY' +} export interface ParameterValues { metric: Metric + listing_type: ListingType + max_price?: number } export function Parameters( @@ -33,17 +40,21 @@ export function Parameters( const formSchema = z.object({ metric: z.nativeEnum(Metric, { required_error: "Metric is required" }), + listing_type: z.nativeEnum(ListingType, { required_error: "Listing Type is required" }), + max_price: z.number().optional(), }) const form = useForm>({ resolver: zodResolver(formSchema), defaultValues: { metric: Metric.qmprice, + max_price: 3000, }, }) // 2. Define a submit handler. function onSubmit(values: z.infer) { // Do something with the form values. // ✅ This will be type-safe and validated. + console.log(values) props.onSubmit(values) } @@ -79,13 +90,49 @@ export function Parameters( Price - - This is your public display name. - )} /> + ( + + Listing Type + + + + + + + )} + /> + ( + + Max price + + field.onChange(Number(e.target.value))} /> + + + + + )} + /> + diff --git a/crawler/models/listing.py b/crawler/models/listing.py index 6feb0f7..463a4e2 100644 --- a/crawler/models/listing.py +++ b/crawler/models/listing.py @@ -6,6 +6,7 @@ import enum import json from pathlib import Path from typing import Any, Dict, List +from pydantic import BaseModel from rec import routing from sqlmodel import JSON, SQLModel, Field, String @@ -230,7 +231,7 @@ class ListingType(enum.StrEnum): @dataclass(frozen=True) -class QueryParameters: +class QueryParameters(BaseModel): listing_type: ListingType min_bedrooms: int = 1 max_bedrooms: int = 999 diff --git a/crawler/repositories/listing_repository.py b/crawler/repositories/listing_repository.py index c34bc3a..d8ffc41 100644 --- a/crawler/repositories/listing_repository.py +++ b/crawler/repositories/listing_repository.py @@ -4,6 +4,7 @@ from models.listing import ( BuyListing, FurnishType, Listing as modelListing, + ListingType, QueryParameters, RentListing, ) @@ -32,12 +33,19 @@ class ListingRepository: """ only_ids = only_ids or [] - query = select( - RentListing - ) # TODO: one nice day I will think of a way to query both rent and buy + model = RentListing # if no query params, default to renting listings + if query_parameters: + model = ( + RentListing + if query_parameters.listing_type == ListingType.RENT + else BuyListing + # else RentListing + ) + + query = select(model) if only_ids: - query = query.where(RentListing.id.in_(only_ids)) # type: ignore - query = self._add_where_from_query_parameters(query, query_parameters) + query = query.where(model.id.in_(only_ids)) # type: ignore + query = self._add_where_from_query_parameters(query, model, query_parameters) if limit: query = query.limit(limit) @@ -47,34 +55,31 @@ class ListingRepository: def _add_where_from_query_parameters( self, - query: SelectOfScalar[RentListing], + query: SelectOfScalar[Listing], + model: type[Listing], query_parameters: QueryParameters | None = None, - ) -> SelectOfScalar[RentListing]: + ) -> SelectOfScalar[Listing]: if query_parameters is None: return query query = query.where( - RentListing.number_of_bedrooms.between( + model.number_of_bedrooms.between( query_parameters.min_bedrooms, query_parameters.max_bedrooms ), - RentListing.price.between( - query_parameters.min_price, query_parameters.max_price - ), + model.price.between(query_parameters.min_price, query_parameters.max_price), ) if query_parameters.min_sqm is not None: - query = query.where(RentListing.square_meters >= query_parameters.min_sqm) + query = query.where(model.square_meters >= query_parameters.min_sqm) if query_parameters.furnish_types: - query = query.where( - RentListing.furnish_type.in_(query_parameters.furnish_types) - ) + query = query.where(model.furnish_type.in_(query_parameters.furnish_types)) if query_parameters.let_date_available_from is not None: query = query.where( - RentListing.available_from >= query_parameters.let_date_available_from + model.available_from >= query_parameters.let_date_available_from ) if query_parameters.last_seen_days is not None: last_seen_threshold = datetime.now() - timedelta( days=query_parameters.last_seen_days ) - query = query.where(RentListing.last_seen >= last_seen_threshold) + query = query.where(model.last_seen >= last_seen_threshold) return query async def upsert_listings(