viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

reuse query params when exporting to immoweb and allow filtering from available date

Browse source
This commit is contained in:
Viktor Barzin 2025-06-01 15:17:14 +00:00
parent a23a5ae192
commit 11315359d2
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
6 changed files with 10207 additions and 42335 deletions
Download patch file Download diff file Expand all files Collapse all files

50
crawler/ui_exporter.py
View file

@ -1,47 +1,53 @@
import json
import pathlib
from data_access import Listing
from rec.query import QueryParameters, filter_listings
async def export_immoweb(ctx, output_file: str):
data_dir = ctx.obj['data_dir']
output_file_path = pathlib.Path(output_file)
async def export_immoweb(
ctx,
output_file: str,
query_parameters: QueryParameters | None = None,
):
data_dir = ctx.obj["data_dir"]
output_file_path = pathlib.Path(output_file)
listing_paths = sorted(list(pathlib.Path(data_dir).glob("*/listing.json")))
# listing_paths = listing_paths[:10]
listings = Listing.get_all_listings([str(path) for path in listing_paths])
if query_parameters is not None:
listings = await filter_listings(listings, query_parameters)
# Convert listings to immoweb format
immoweb_listings = []
for listing in listings:
immoweb_listing = {
'type': 'Feature',
'properties': {
'city': 'London', # change me
'country': 'United Kingdom',
'qm': await listing.sqm_ocr(),
'qmprice': round(await listing.price_per_sqm(), 2),
'rooms': listing.bedrooms,
'total_price': listing.price,
'url': listing.url,
"type": "Feature",
"properties": {
"city": "London", # change me
"country": "United Kingdom",
"qm": await listing.sqm_ocr(),
"qmprice": round(await listing.price_per_sqm(), 2),
"rooms": listing.bedrooms,
"total_price": listing.price,
"url": listing.url,
# Additional info; the above is GeoJSON format
# Below is all other crap we want in the UI
'info': await listing.dict_nicely(),
"info": await listing.dict_nicely(),
},
'geometry': {
'coordinates': [
"geometry": {
"coordinates": [
listing.longitude,
listing.latitude,
],
'type': 'Point',
}
"type": "Point",
},
}
immoweb_listings.append(immoweb_listing)
prefix = 'var data = '
prefix = "var data = "
serialized_data = {"type": "FeatureCollection", "features": immoweb_listings}
result = prefix + json.dumps(serialized_data, indent=4)
with open(output_file_path, 'w') as f:
with open(output_file_path, "w") as f:
f.write(result)
# json.dump(serialized_data, f, indent=4)