viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

adding better exception messages and interrupt free crawling

Browse source
This commit is contained in:
Kadir 2024-05-06 18:54:55 +01:00
parent 874fed3e8f
commit f98cd02696
3 changed files with 8 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

5
crawler/2_dump_detail.py
View file

@ -15,6 +15,5 @@ for listing in tqdm(filtered_listings):
d = detail_query(listing.identifier) d = detail_query(listing.identifier)
with open(listing.path_detail_json(), "w") as f: with open(listing.path_detail_json(), "w") as f:
json.dump(d, f) json.dump(d, f)
except: except Exception as e:
print("Failed at: ", listing.identifier) print(e)
raise

5
crawler/data_access.py
View file

@ -161,6 +161,10 @@ class Listing:
@property @property
def price(self) -> float: def price(self) -> float:
return self.detailobject["property"]["price"] return self.detailobject["property"]["price"]
@property
def tenure_type(self) -> str:
return self.detailobject["property"]["tenureType"]
@property @property
def price_per_sqm(self) -> float: def price_per_sqm(self) -> float:
@ -228,6 +232,7 @@ class Listing:
"lease_left": self.leaseLeft, "lease_left": self.leaseLeft,
"service_charge": self.serviceCharge, "service_charge": self.serviceCharge,
"development": self.development, "development": self.development,
"tenure_type": self.tenure_type,
} }

2
crawler/rec/query.py
View file

@ -36,7 +36,7 @@ def detail_query(detail_id: int):
url = f"https://api.rightmove.co.uk/api/property/{detail_id}" url = f"https://api.rightmove.co.uk/api/property/{detail_id}"
response = requests.get(url, params=params, headers=headers, verify=False) response = requests.get(url, params=params, headers=headers, verify=False)
if response.status_code != 200: if response.status_code != 200:
raise Exception("Failed due to: ", response.text) raise Exception(f"id: {detail_id}. Status Code: {response.status_code}. Failed due to: {response.text}")
return response.json() return response.json()