viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add filter for last seen days

Browse source
This commit is contained in:
Viktor Barzin 2025-06-01 15:26:38 +00:00
parent 11315359d2
commit 8b90ecde11
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
3 changed files with 333 additions and 115480 deletions
Download patch file Download diff file Expand all files Collapse all files

10
crawler/main.py
View file

@ -82,6 +82,12 @@ def listing_filter_options(func):
default=None,
type=click.DateTime(),
)
@click.option(
"--last-seen-days",
help="Last seen (days). If set, only listings that were seen in the last N days will be included.",
default=14,
type=int,
)
@wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
@ -121,6 +127,7 @@ def dump_listings(
type: str,
furnish_types: list[str],
available_from: datetime | None,
last_seen_days: int,
):
data_dir: str = ctx.obj["data_dir"]
query_parameters = QueryParameters(
@ -132,6 +139,7 @@ def dump_listings(
max_price=max_price,
furnish_types=[FurnishType[furnish_type] for furnish_type in furnish_types],
let_date_available_from=available_from,
last_seen_days=last_seen_days,
)
click.echo(
f"Running dump_listings for districts {district}, data dir {data_dir} and parameters: "
@ -279,6 +287,7 @@ def export_immoweb(
type: str,
furnish_types: list[str],
available_from: datetime | None,
last_seen_days: int,
):
query_parameters = QueryParameters(
listing_type=ListingType[type],
@ -289,6 +298,7 @@ def export_immoweb(
max_price=max_price,
furnish_types=[FurnishType[furnish_type] for furnish_type in furnish_types],
let_date_available_from=available_from,
last_seen_days=last_seen_days,
)
click.echo(
f"Exporting data to {output_file} that matches the query parameters: {query_parameters}"