viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add services layer, tests, streaming UI, and cleanup legacy code

Browse source
This commit is contained in:
Viktor Barzin 2026-02-06 20:55:10 +00:00
parent 5514fa6381
commit d205d15c74
62 changed files with 3729 additions and 1024 deletions
Download patch file Download diff file Expand all files Collapse all files

50
crawler/csv_exporter.py
View file

@ -1,6 +1,6 @@
from pathlib import Path
import pandas as pd
from rec.query import QueryParameters
from models.listing import QueryParameters
from repositories.listing_repository import ListingRepository
@ -10,7 +10,7 @@ async def export_to_csv(
query_parameters: QueryParameters | None = None,
) -> None:
listings = await repository.get_listings(query_parameters=query_parameters)
ds = [*[listing.__dict__ for listing in listings]]
ds = [listing.__dict__ for listing in listings]
df = pd.DataFrame(ds)
# read decisions on file
@ -22,37 +22,19 @@ async def export_to_csv(
drop_columns = ["_sa_instance_state", "additional_info"]
df = df.drop(columns=drop_columns)
# remove all entries where we didnt calculate transit time (probably due to a too far distance)
# df2 = df[df.travel_time_fastest.notna()]
df2 = df
# fill in gap values for service charge and lease left for Excel filters
if "service_charge" not in df.columns:
df.loc[:, "service_charge"] = -1
df.loc[:, "service_charge"] = df.service_charge.fillna(-1)
if "lease_left" not in df.columns:
df.loc[:, "lease_left"] = -1
df.loc[:, "lease_left"] = df.lease_left.fillna(-1)
if "square_meters" not in df.columns:
df.loc[:, "square_meters"] = -1
df.loc[:, "square_meters"] = df.square_meters.fillna(-1)
# drop columns
# dropcolumns = ['distance_per_transit', 'duration_static', 'distance']
# s1 = df2['travel_time_fastest'].apply(pd.Series).drop(dropcolumns, axis=1)
# s1 = df2
# Add price per sqm column
df.loc[:, "price_per_sqm"] = df.price / df.square_meters
# fill in gap values for service charge and lease left. This is for excel so we can use filters better there
if "service_charge" not in df2.columns:
df2.loc[:, "service_charge"] = -1
df2.loc[:, "service_charge"] = df2.service_charge.fillna(-1)
if "lease_left" not in df2.columns:
df2.loc[:, "lease_left"] = -1
df2.loc[:, "lease_left"] = df2.lease_left.fillna(-1)
if "square_meters" not in df2.columns:
df2.loc[:, "square_meters"] = -1
df2.loc[:, "square_meters"] = df2.square_meters.fillna(-1)
df3 = df2
# df3 = pd.concat([df2.drop(['travel_time_fastest', 'travel_time_second'], axis=1), s1], axis=1)
# df3.loc[:, 'duration'] = (df3.loc[:, ['duration']].min(axis=1) / 60).round()
df3.shape
df4 = df3
# df5 = df4[columns]
# Add some interesting columns
df4.loc[:, "price_per_sqm"] = df4.price / df4.square_meters
df5 = df4
df6 = df5.sort_values(by=["price_per_sqm"], ascending=True)
df6.to_csv(str(output_file), index=False)
df = df.sort_values(by=["price_per_sqm"], ascending=True)
df.to_csv(str(output_file), index=False)