viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

make the csv exporter use the filtering params to allow exporting customizable reports

Browse source
This commit is contained in:
Viktor Barzin 2025-06-01 20:11:00 +00:00
parent 0acd417d34
commit 0bdfeec195
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
2 changed files with 41 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crawler/csv_exporter.py
View file

@ -2,13 +2,17 @@ import asyncio
from pathlib import Path from pathlib import Path
from data_access import Listing from data_access import Listing
import pandas as pd import pandas as pd
from rec.query import QueryParameters, filter_listings
async def export_to_csv( async def export_to_csv(
listings: list[Listing], listings: list[Listing],
output_file: Path, output_file: Path,
columns: list[str], columns: list[str],
query_parameters: QueryParameters | None = None,
) -> None: ) -> None:
if query_parameters is not None:
listings = await filter_listings(listings, query_parameters)
ds = await asyncio.gather(*[listing.dict_nicely() for listing in listings]) ds = await asyncio.gather(*[listing.dict_nicely() for listing in listings])
df = pd.DataFrame(ds) df = pd.DataFrame(ds)
# read decisions on file # read decisions on file

40
crawler/main.py
View file

@ -258,14 +258,48 @@ def routing(
), ),
) )
@click.pass_context @click.pass_context
def export_csv(ctx: click.core.Context, output_file: str, columns: tuple[str]): @listing_filter_options
def export_csv(
ctx: click.core.Context,
output_file: str,
columns: tuple[str],
district: list[str],
min_bedrooms: int,
max_bedrooms: int,
min_price: int,
max_price: int,
type: str,
furnish_types: list[str],
available_from: datetime | None,
last_seen_days: int,
min_sqm: int | None = None,
):
data_dir = ctx.obj["data_dir"] data_dir = ctx.obj["data_dir"]
click.echo(f"Exporting data to {output_file} using {data_dir=}") query_parameters = QueryParameters(
listing_type=ListingType[type],
district_names=set(district),
min_bedrooms=min_bedrooms,
max_bedrooms=max_bedrooms,
min_price=min_price,
max_price=max_price,
furnish_types=[FurnishType[furnish_type] for furnish_type in furnish_types],
let_date_available_from=available_from,
last_seen_days=last_seen_days,
min_sqm=min_sqm,
)
click.echo(
f"Exporting data to {output_file} using {data_dir=} and query parameters: {query_parameters}"
)
output_file_path = pathlib.Path(output_file) output_file_path = pathlib.Path(output_file)
listing_paths = sorted(list(pathlib.Path(data_dir).glob("*/listing.json"))) listing_paths = sorted(list(pathlib.Path(data_dir).glob("*/listing.json")))
listings = Listing.get_all_listings([str(path) for path in listing_paths]) listings = Listing.get_all_listings([str(path) for path in listing_paths])
asyncio.run( asyncio.run(
csv_exporter.export_to_csv(listings, output_file_path, list(columns)), csv_exporter.export_to_csv(
listings,
output_file_path,
list(columns),
query_parameters=query_parameters,
),
) )