viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

expose rightmove query parameters as cli options

Browse source
This commit is contained in:
Viktor Barzin 2025-05-17 20:40:44 +00:00
parent 96562c0895
commit bb9afc76fe
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
2 changed files with 70 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

40
crawler/1_dump_listings.py
View file

@ -1,33 +1,47 @@
from dataclasses import dataclass
import pathlib
from rec.query import listing_query
from rec.districts import get_districts
from data_access import Listing
@dataclass(frozen=True)
class QueryParameters:
min_bedrooms: int
max_bedrooms: int
min_price: int
max_price: int
district_names: set[str]
radius: float = 0
page_size: int = 500 # items per page
max_days_since_added: int | None = None
def dump_listings(
district_names: set[str] | None = None,
data_dir: pathlib.Path = pathlib.Path("data/rs/")
):
districts = get_districts() if district_names is None else {
parameters: QueryParameters,
data_dir: pathlib.Path = pathlib.Path("data/rs/"),
) -> list[Listing]:
districts = {
district: locid
for district, locid in get_districts().items()
if district in district_names
if district in parameters.district_names
}
print("Valid districts to scrape:", districts.keys())
listings = []
for district, locid in districts.items():
print("#### District:", district)
for i in [1, 2]:
try:
d = listing_query(
page=i,
min_bedrooms=1,
max_bedrooms=4,
radius=0,
min_price=0,
max_price=1000000,
min_bedrooms=parameters.min_bedrooms,
max_bedrooms=parameters.max_bedrooms,
radius=parameters.radius,
min_price=parameters.min_price,
max_price=parameters.max_price,
location_id=locid,
page_size=500,
max_days_since_added=None,
page_size=parameters.page_size,
max_days_since_added=parameters.max_days_since_added,
)
except Exception as e:
print(e)
@ -43,7 +57,9 @@ def dump_listings(
listing = Listing(identifier, data_dir=data_dir)
listing.dump_listing(property)
listings.append(listing)
print() # break line as we used end=, above.
return listings
def main():