viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

extract common listing filtering options into a decorator to enable reuse between commands

Browse source
This commit is contained in:
Viktor Barzin 2025-06-01 12:11:15 +00:00
parent c543be7ff6
commit a23a5ae192
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
1 changed files with 64 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

115
crawler/main.py
View file

@ -11,6 +11,8 @@ import csv_exporter
from rec.query import ListingType, FurnishType from rec.query import ListingType, FurnishType
from rec.routing import API_KEY_ENVIRONMENT_VARIABLE, TravelMode from rec.routing import API_KEY_ENVIRONMENT_VARIABLE, TravelMode
from ui_exporter import export_immoweb as export_immoweb_ui from ui_exporter import export_immoweb as export_immoweb_ui
from functools import wraps
dump_listings_module = importlib.import_module("1_dump_listings") dump_listings_module = importlib.import_module("1_dump_listings")
dump_detail_module = importlib.import_module("2_dump_detail") dump_detail_module = importlib.import_module("2_dump_detail")
@ -19,6 +21,67 @@ detect_floorplan_module = importlib.import_module("4_detect_floorplan")
routing_module = importlib.import_module("5_routing") routing_module = importlib.import_module("5_routing")
def listing_filter_options(func):
"""Decorator to add common options for filtering listings."""
@click.option(
"--type",
"-t",
help="Type of listing to scrape",
type=click.Choice(
ListingType.__members__.keys(),
case_sensitive=False,
),
required=True,
)
@click.option(
"--min-bedrooms",
default=1,
help="Minimum number of bedrooms",
type=click.IntRange(min=1),
)
@click.option(
"--max-bedrooms",
default=5,
help="Maximum number of bedrooms",
type=click.IntRange(min=1),
)
@click.option(
"--min-price",
default=0,
help="Minimum price",
type=click.IntRange(min=0),
)
@click.option(
"--max-price",
default=1000000,
help="Maximum price",
type=click.IntRange(min=0),
)
@click.option(
"--district",
default=None,
help="Districts to scrape",
type=click.Choice(get_districts().keys(), case_sensitive=False),
multiple=True,
)
@click.option(
"--furnish-types",
"-f",
help="Furnish types for rented listings",
type=click.Choice(
[furnish_type.name for furnish_type in FurnishType.__members__.values()],
case_sensitive=False,
),
multiple=True,
)
@wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
return wrapper
@click.group() @click.group()
@click.option( @click.option(
"--data-dir", "--data-dir",
@ -39,57 +102,7 @@ def cli(ctx, data_dir: str):
@cli.command() @cli.command()
@click.option( @listing_filter_options
"--type",
"-t",
help="Type of listing to scrape",
type=click.Choice(
ListingType.__members__.keys(),
case_sensitive=False,
),
required=True,
)
@click.option(
"--min-bedrooms",
default=1,
help="Minimum number of bedrooms",
type=click.IntRange(min=1),
)
@click.option(
"--max-bedrooms",
default=5,
help="Maximum number of bedrooms",
type=click.IntRange(min=1),
)
@click.option(
"--min-price",
default=0,
help="Minimum price",
type=click.IntRange(min=0),
)
@click.option(
"--max-price",
default=1000000,
help="Maximum price",
type=click.IntRange(min=0),
)
@click.option(
"--district",
default=None,
help="Districts to scrape",
type=click.Choice(get_districts().keys(), case_sensitive=False),
multiple=True,
)
@click.option(
"--furnish-types",
"-f",
help="Furnish types for rented listings",
type=click.Choice(
[furnish_type.name for furnish_type in FurnishType.__members__.values()],
case_sensitive=False,
),
multiple=True,
)
@click.pass_context @click.pass_context
def dump_listings( def dump_listings(
ctx: click.core.Context, ctx: click.core.Context,