extract common listing filtering options into a decorator to enable reuse between commands
This commit is contained in:
parent
c543be7ff6
commit
a23a5ae192
1 changed files with 64 additions and 51 deletions
115
crawler/main.py
115
crawler/main.py
|
|
@ -11,6 +11,8 @@ import csv_exporter
|
|||
from rec.query import ListingType, FurnishType
|
||||
from rec.routing import API_KEY_ENVIRONMENT_VARIABLE, TravelMode
|
||||
from ui_exporter import export_immoweb as export_immoweb_ui
|
||||
from functools import wraps
|
||||
|
||||
|
||||
dump_listings_module = importlib.import_module("1_dump_listings")
|
||||
dump_detail_module = importlib.import_module("2_dump_detail")
|
||||
|
|
@ -19,6 +21,67 @@ detect_floorplan_module = importlib.import_module("4_detect_floorplan")
|
|||
routing_module = importlib.import_module("5_routing")
|
||||
|
||||
|
||||
def listing_filter_options(func):
|
||||
"""Decorator to add common options for filtering listings."""
|
||||
|
||||
@click.option(
|
||||
"--type",
|
||||
"-t",
|
||||
help="Type of listing to scrape",
|
||||
type=click.Choice(
|
||||
ListingType.__members__.keys(),
|
||||
case_sensitive=False,
|
||||
),
|
||||
required=True,
|
||||
)
|
||||
@click.option(
|
||||
"--min-bedrooms",
|
||||
default=1,
|
||||
help="Minimum number of bedrooms",
|
||||
type=click.IntRange(min=1),
|
||||
)
|
||||
@click.option(
|
||||
"--max-bedrooms",
|
||||
default=5,
|
||||
help="Maximum number of bedrooms",
|
||||
type=click.IntRange(min=1),
|
||||
)
|
||||
@click.option(
|
||||
"--min-price",
|
||||
default=0,
|
||||
help="Minimum price",
|
||||
type=click.IntRange(min=0),
|
||||
)
|
||||
@click.option(
|
||||
"--max-price",
|
||||
default=1000000,
|
||||
help="Maximum price",
|
||||
type=click.IntRange(min=0),
|
||||
)
|
||||
@click.option(
|
||||
"--district",
|
||||
default=None,
|
||||
help="Districts to scrape",
|
||||
type=click.Choice(get_districts().keys(), case_sensitive=False),
|
||||
multiple=True,
|
||||
)
|
||||
@click.option(
|
||||
"--furnish-types",
|
||||
"-f",
|
||||
help="Furnish types for rented listings",
|
||||
type=click.Choice(
|
||||
[furnish_type.name for furnish_type in FurnishType.__members__.values()],
|
||||
case_sensitive=False,
|
||||
),
|
||||
multiple=True,
|
||||
)
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
@click.group()
|
||||
@click.option(
|
||||
"--data-dir",
|
||||
|
|
@ -39,57 +102,7 @@ def cli(ctx, data_dir: str):
|
|||
|
||||
|
||||
@cli.command()
|
||||
@click.option(
|
||||
"--type",
|
||||
"-t",
|
||||
help="Type of listing to scrape",
|
||||
type=click.Choice(
|
||||
ListingType.__members__.keys(),
|
||||
case_sensitive=False,
|
||||
),
|
||||
required=True,
|
||||
)
|
||||
@click.option(
|
||||
"--min-bedrooms",
|
||||
default=1,
|
||||
help="Minimum number of bedrooms",
|
||||
type=click.IntRange(min=1),
|
||||
)
|
||||
@click.option(
|
||||
"--max-bedrooms",
|
||||
default=5,
|
||||
help="Maximum number of bedrooms",
|
||||
type=click.IntRange(min=1),
|
||||
)
|
||||
@click.option(
|
||||
"--min-price",
|
||||
default=0,
|
||||
help="Minimum price",
|
||||
type=click.IntRange(min=0),
|
||||
)
|
||||
@click.option(
|
||||
"--max-price",
|
||||
default=1000000,
|
||||
help="Maximum price",
|
||||
type=click.IntRange(min=0),
|
||||
)
|
||||
@click.option(
|
||||
"--district",
|
||||
default=None,
|
||||
help="Districts to scrape",
|
||||
type=click.Choice(get_districts().keys(), case_sensitive=False),
|
||||
multiple=True,
|
||||
)
|
||||
@click.option(
|
||||
"--furnish-types",
|
||||
"-f",
|
||||
help="Furnish types for rented listings",
|
||||
type=click.Choice(
|
||||
[furnish_type.name for furnish_type in FurnishType.__members__.values()],
|
||||
case_sensitive=False,
|
||||
),
|
||||
multiple=True,
|
||||
)
|
||||
@listing_filter_options
|
||||
@click.pass_context
|
||||
def dump_listings(
|
||||
ctx: click.core.Context,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue