viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor main.py click to use click commands to allow passing parameters to commands and enable fetching districts by district name

Browse source
This commit is contained in:
Viktor Barzin 2025-05-14 19:41:13 +00:00
parent bb2488a63b
commit ea56555884
3 changed files with 73 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

9
crawler/1_dump_listings.py
View file

@ -3,8 +3,13 @@ from rec.districts import get_districts
from data_access import Listing from data_access import Listing
def dump_listings(): def dump_listings(district_names: set[str] | None = None):
districts = get_districts() districts = get_districts() if district_names is None else {
district: locid
for district, locid in get_districts().items()
if district in district_names
}
print("Valid districts to scrape:", districts.keys())
for district, locid in districts.items(): for district, locid in districts.items():
print("#### District:", district) print("#### District:", district)
for i in [1, 2]: for i in [1, 2]:

51
crawler/main.py
View file

@ -1,6 +1,8 @@
import click import click
import importlib import importlib
from rec.districts import get_districts
dump_listings_module = importlib.import_module('1_dump_listings') dump_listings_module = importlib.import_module('1_dump_listings')
dump_detail_module = importlib.import_module('2_dump_detail') dump_detail_module = importlib.import_module('2_dump_detail')
dump_images_module = importlib.import_module('3_dump_images') dump_images_module = importlib.import_module('3_dump_images')
@ -16,20 +18,47 @@ steps_to_handlers = {
} }
@click.command() @click.group()
def cli():
pass
@cli.command()
@click.option( @click.option(
'--step', '--district',
default=[], default=None,
help='Scraping step to run', help='Districts to scrape',
type=click.Choice(get_districts().keys(), case_sensitive=False),
multiple=True, multiple=True,
type=click.Choice(steps_to_handlers.keys())
) )
def main(step: list[str]): def dump_listings(district: list[str]):
click.echo(f'Running steps: {step}') click.echo(f'Running dump_listings for districts {district}')
for s in step: dump_listings_module.dump_listings(set(district))
click.echo(f'Calling handler for step: {s}')
steps_to_handlers[s]()
@cli.command()
def dump_detail():
click.echo('Running dump_detail')
dump_detail_module.dump_detail()
@cli.command()
def dump_images():
click.echo('Running dump_images')
dump_images_module.dump_images()
@cli.command()
def detect_floorplan():
click.echo('Running detect_floorplan')
detect_floorplan_module.detect_floorplan()
@cli.command()
def routing():
click.echo('Running routing')
routing_module.calculate_route()
if __name__ == '__main__': if __name__ == '__main__':
main() cli()

51
crawler/rec/districts.py
View file

@ -1,36 +1,37 @@
def get_districts(): def get_districts():
return { return {
# "Barking and Dagenham": "REGION^61400", "Barking and Dagenham": "REGION^61400",
# "Barnet": "REGION^93929", "Barnet": "REGION^93929",
# "Bexley": "REGION^93932", "Bexley": "REGION^93932",
# "Brent": "REGION^93935", "Brent": "REGION^93935",
# "Bromley": "REGION^93938", "Bromley": "REGION^93938",
"Camden": "REGION^93941", "Camden": "REGION^93941",
"City of London": "REGION^61224", "City of London": "REGION^61224",
# "Croydon": "REGION^93944", "Croydon": "REGION^93944",
# "Ealing": "REGION^93947", "Ealing": "REGION^93947",
# "Enfield": "REGION^93950", "Enfield": "REGION^93950",
# "Greenwich": "REGION^61226", "Greenwich": "REGION^61226",
"Hackney": "REGION^93953", "Hackney": "REGION^93953",
"Hammersmith and Fulham": "REGION^61407", "Hammersmith and Fulham": "REGION^61407",
# "Haringey": "REGION^61227", "Haringey": "REGION^61227",
# "Harrow": "REGION^93956", "Harrow": "REGION^93956",
# "Havering": "REGION^61228", "Havering": "REGION^61228",
# "Hillingdon": "REGION^93959", "Hillingdon": "REGION^93959",
# "Hounslow": "REGION^93962", "Hounslow": "REGION^93962",
"Islington": "REGION^93965", "Islington": "REGION^93965",
"London": "REGION^87490",
"Kensington and Chelsea": "REGION^61229", "Kensington and Chelsea": "REGION^61229",
# "Kingston upon Thames": "REGION^93968", "Kingston upon Thames": "REGION^93968",
# "Lambeth": "REGION^93971", "Lambeth": "REGION^93971",
# "Lewisham": "REGION^61413", "Lewisham": "REGION^61413",
# "Merton": "REGION^61414", "Merton": "REGION^61414",
# "Newham": "REGION^61231", "Newham": "REGION^61231",
# "Redbridge": "REGION^61537", "Redbridge": "REGION^61537",
# "Richmond upon Thames": "REGION^61415", "Richmond upon Thames": "REGION^61415",
# "Southwark": "REGION^61518", "Southwark": "REGION^61518",
# "Sutton": "REGION^93974", "Sutton": "REGION^93974",
"Tower Hamlets": "REGION^61417", "Tower Hamlets": "REGION^61417",
# "Waltham Forest": "REGION^61232", "Waltham Forest": "REGION^61232",
# "Wandsworth": "REGION^93977", "Wandsworth": "REGION^93977",
"Westminster": "REGION^93980", "Westminster": "REGION^93980",
} }