viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ruff format

Browse source
This commit is contained in:
Kadir 2024-03-25 20:48:48 +00:00
parent 37e3e8ad6f
commit d777558b34
17 changed files with 411 additions and 368 deletions
Download patch file Download diff file Expand all files Collapse all files

13
crawler/rec/floorplan.py
View file

@ -3,21 +3,22 @@ from PIL import Image
from transformers import Pix2StructProcessor, Pix2StructForConditionalGeneration
import pytesseract
def inference(image_path):
image = Image.open(image_path)
question = "How many living rooms are displayed on this floor plan?" # not sure if it even has an effect
processor = Pix2StructProcessor.from_pretrained('google/deplot')
model = Pix2StructForConditionalGeneration.from_pretrained('google/deplot')
question = "How many living rooms are displayed on this floor plan?" # not sure if it even has an effect
processor = Pix2StructProcessor.from_pretrained("google/deplot")
model = Pix2StructForConditionalGeneration.from_pretrained("google/deplot")
inputs = processor(images=image, text=question, return_tensors="pt")
predictions = model.generate(**inputs, max_new_tokens=512)
output = processor.decode(predictions[0], skip_special_tokens=True)
return output, predictions
def extract_total_sqm(deplot_input_str):
sqmregex = r'(\d+\.\d*) ?(sq ?m|sq. ?m)'
sqmregex = r"(\d+\.\d*) ?(sq ?m|sq. ?m)"
matches = re.findall(sqmregex, deplot_input_str.lower())
if len(matches) == 0:
return None

36
crawler/rec/query.py
View file

@ -18,10 +18,10 @@ headers = {
def detail_query(detail_id: int):
params = {
'apiApplication': 'ANDROID',
'appVersion': '3.70.0',
"apiApplication": "ANDROID",
"appVersion": "3.70.0",
}
url = f'https://api.rightmove.co.uk/api/property/{detail_id}'
url = f"https://api.rightmove.co.uk/api/property/{detail_id}"
response = requests.get(url, params=params, headers=headers, verify=False)
if response.status_code != 200:
raise Exception("Failed due to: ", response.text)
@ -30,7 +30,16 @@ def detail_query(detail_id: int):
# @cache.memoize()
def listing_query(page: int, min_bedrooms: int, max_bedrooms: int, radius: float, min_price: int, max_price: int, mustNewHome: bool = False, max_days_since_added: int = None) -> dict:
def listing_query(
page: int,
min_bedrooms: int,
max_bedrooms: int,
radius: float,
min_price: int,
max_price: int,
mustNewHome: bool = False,
max_days_since_added: int = None,
) -> dict:
params = {
"locationIdentifier": "POSTCODE^4228216",
"channel": "BUY",
@ -49,12 +58,12 @@ def listing_query(page: int, min_bedrooms: int, max_bedrooms: int, radius: float
"appVersion": "3.70.0",
}
if max_days_since_added:
if max_days_since_added not in [1,3,7,14]:
raise Exception("Invalid max days. Can only be", [1,3,7,14])
params['maxDaysSinceAdded'] = max_days_since_added
if max_days_since_added not in [1, 3, 7, 14]:
raise Exception("Invalid max days. Can only be", [1, 3, 7, 14])
params["maxDaysSinceAdded"] = max_days_since_added
if mustNewHome:
params['mustHave'] = 'newHome'
params["mustHave"] = "newHome"
response = requests.get(
"https://api.rightmove.co.uk/api/property-listing",
@ -69,7 +78,14 @@ def listing_query(page: int, min_bedrooms: int, max_bedrooms: int, radius: float
if __name__ == "__main__":
response = listing_query(page=1, min_bedrooms=2, max_bedrooms=2, radius=5.0, min_price=150000, max_price=700000)
response = listing_query(
page=1,
min_bedrooms=2,
max_bedrooms=2,
radius=5.0,
min_price=150000,
max_price=700000,
)
resp = response
for d in resp["properties"]:
rl = RightmoveListing(

172
crawler/rec/routing.py
View file

@ -2,100 +2,101 @@ import requests
from rec.utils import nextMonday
from collections import defaultdict
API_KEY = 'AIzaSyBoBHzeQFgR7O-NlNsuHXQcC1B7ccEHpl8'
API_KEY = "AIzaSyBoBHzeQFgR7O-NlNsuHXQcC1B7ccEHpl8"
url = "https://routes.googleapis.com/directions/v2:computeRoutes"
def transit_route(origin_lat:float, origin_lon:float, dest_lat:float, dest_lon:float, compute_alternative_routes=True):
monday9am = nextMonday()
def transit_route(
origin_lat: float,
origin_lon: float,
dest_lat: float,
dest_lon: float,
compute_alternative_routes=True,
):
monday9am = nextMonday()
header = {
"X-Goog-Api-Key": API_KEY,
"Content-Type": "application/json",
"X-Goog-FieldMask": "routes.distanceMeters,routes.duration,routes.staticDuration,routes.legs.steps.distanceMeters,routes.legs.steps.staticDuration,routes.legs.steps.travelMode",
}
header = {
"X-Goog-Api-Key": API_KEY,
"Content-Type": "application/json",
"X-Goog-FieldMask": "routes.distanceMeters,routes.duration,routes.staticDuration,routes.legs.steps.distanceMeters,routes.legs.steps.staticDuration,routes.legs.steps.travelMode",
}
body = {
"origin":{
"location":{
"latLng":{
"latitude": origin_lat,
"longitude": origin_lon
}
}
},
"destination":{
"location":{
"latLng":{
"latitude": dest_lat,
"longitude": dest_lon
}
}
},
"travelMode": "TRANSIT",
# "2023-10-15T15:01:23.045123456Z"
"departureTime": monday9am.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
"computeAlternativeRoutes": compute_alternative_routes,
# "routeModifiers": {
# "avoidTolls": false,
# "avoidHighways": false,
# "avoidFerries": false
# },
"languageCode": "en-US",
"units": "METRIC"
}
body = {
"origin": {
"location": {"latLng": {"latitude": origin_lat, "longitude": origin_lon}}
},
"destination": {
"location": {"latLng": {"latitude": dest_lat, "longitude": dest_lon}}
},
"travelMode": "TRANSIT",
# "2023-10-15T15:01:23.045123456Z"
"departureTime": monday9am.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
"computeAlternativeRoutes": compute_alternative_routes,
# "routeModifiers": {
# "avoidTolls": false,
# "avoidHighways": false,
# "avoidFerries": false
# },
"languageCode": "en-US",
"units": "METRIC",
}
r = requests.post(url, json=body, headers=header)
if r.status_code == 200:
return r.json()
raise Exception(r.json())
r = requests.post(url, json=body, headers=header)
if r.status_code == 200:
return r.json()
raise Exception(r.json())
def extract_time(d, limit: int = 2):
res = []
for route in d["routes"]:
distance = route["distanceMeters"]
duration = int(route["duration"].strip("s"))
duration_static = int(route["staticDuration"].strip("s"))
steps = route["legs"][0]["steps"]
initial_walk_duration = 0
used_transit = False
duration_per_transit = defaultdict(lambda: 0)
distance_per_transit = defaultdict(lambda: 0)
number_of_transit_stops = 0
for step in steps:
if used_transit == False and step["travelMode"] == "WALK":
initial_walk_duration += int(step["staticDuration"].strip("s"))
else:
used_transit = True
duration_per_transit[step["travelMode"]] += int(
step["staticDuration"].strip("s")
)
distance_per_transit[step["travelMode"]] += step.get("distanceMeters", 0)
if step["travelMode"] == "TRANSIT":
number_of_transit_stops += 1
res.append(
{
"duration": duration,
"distance": distance,
"duration_static": duration_static,
"initial_walk_duration": initial_walk_duration,
"duration_per_transit": dict(duration_per_transit),
"distance_per_transit": dict(distance_per_transit),
"number_of_transit_stops": number_of_transit_stops,
}
)
return res[:limit]
def extract_time(d, limit:int=2):
res = []
for route in d['routes']:
distance = route['distanceMeters']
duration = int(route['duration'].strip('s'))
duration_static = int(route['staticDuration'].strip('s'))
steps = route['legs'][0]['steps']
initial_walk_duration = 0
used_transit = False
duration_per_transit = defaultdict(lambda: 0)
distance_per_transit = defaultdict(lambda: 0)
number_of_transit_stops = 0
for step in steps:
if used_transit == False and step['travelMode'] == 'WALK':
initial_walk_duration += int(step['staticDuration'].strip('s'))
else:
used_transit = True
duration_per_transit[step['travelMode']] += int(step['staticDuration'].strip('s'))
distance_per_transit[step['travelMode']] += step.get('distanceMeters', 0)
if step['travelMode'] == 'TRANSIT':
number_of_transit_stops += 1
res.append({
'duration': duration,
'distance': distance,
'duration_static': duration_static,
'initial_walk_duration': initial_walk_duration,
'duration_per_transit': dict(duration_per_transit),
'distance_per_transit': dict(distance_per_transit),
'number_of_transit_stops': number_of_transit_stops,
})
return res[:limit]
if __name__ == "__main__":
import json
with open('code/json/routing_routeapi.json', 'r') as f:
d = json.load(f)
extract_time(d)
import json
with open("code/json/routing_routeapi.json", "r") as f:
d = json.load(f)
extract_time(d)
# if __name__ == "__main__":
# origin = 51.5635664310333, -0.1107173751570373 # home
@ -104,4 +105,3 @@ if __name__ == "__main__":
# import json
# with open('code/json/routing_routeapi.json', 'w') as f:
# json.dump(d, f)

8
crawler/rec/utils.py
View file

@ -1,5 +1,6 @@
from datetime import datetime, timedelta, timezone
def nextMonday():
"""
I think this function doesnt work when the day is monday itself.
@ -10,8 +11,11 @@ def nextMonday():
now = datetime.now(timezone.utc)
days_until_monday = (0 - now.weekday() + 7) % 7
monday = now + timedelta(days=days_until_monday)
monday_9am = monday.replace(hour=9, minute=0, second=0, microsecond=0, tzinfo=timezone.utc)
monday_9am = monday.replace(
hour=9, minute=0, second=0, microsecond=0, tzinfo=timezone.utc
)
return monday_9am
if __name__ == '__main__':
if __name__ == "__main__":
print(nextMonday())