adding floorplans, detail json, refactored the folders

crawler/proof_of_concept/image.py

@@ -0,0 +1,13 @@
+import requests
+
+headers = {
+    'Host': 'media.rightmove.co.uk',
+    # 'Accept-Encoding': 'gzip, deflate, br',
+    'User-Agent': 'okhttp/4.10.0',
+}
+
+response = requests.get(
+    'https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_IMG_00_0000.jpeg',
+    headers=headers,
+    verify=False,
+)

crawler/proof_of_concept/listings.py

@@ -0,0 +1,57 @@
+import requests
+
+headers = {
+    'Host': 'api.rightmove.co.uk',
+    # 'Accept-Encoding': 'gzip, deflate, br',
+    'User-Agent': 'okhttp/4.10.0',
+    'Connection': 'close',
+}
+
+params = {
+    'locationIdentifier': 'POSTCODE^4228216',
+    'channel': 'BUY',
+    'page': '1',
+    'numberOfPropertiesPerPage': '25',
+    'radius': '3.0',
+    'sortBy': 'distance',
+    'includeUnavailableProperties': 'false',
+    'propertyTypes': 'flat',
+    'dontShow': 'sharedOwnership,retirement',
+    'minPrice': '150000',
+    'maxPrice': '500000',
+    'minBedrooms': '2',
+    'maxBedrooms': '2',
+    'apiApplication': 'ANDROID',
+    'appVersion': '3.70.0',
+}
+
+response = requests.get('https://api.rightmove.co.uk/api/property-listing', params=params, headers=headers, verify=False)
+
+import requests
+
+headers = {
+    'Host': 'api.rightmove.co.uk',
+    # 'Accept-Encoding': 'gzip, deflate, br',
+    'User-Agent': 'okhttp/4.10.0',
+    'Connection': 'close',
+}
+
+params = {
+    'locationIdentifier': 'POSTCODE^4228216',
+    'channel': 'BUY',
+    'page': '2',
+    'numberOfPropertiesPerPage': '25',
+    'radius': '3.0',
+    'sortBy': 'distance',
+    'includeUnavailableProperties': 'false',
+    'propertyTypes': 'flat',
+    'dontShow': 'sharedOwnership,retirement',
+    'minPrice': '150000',
+    'maxPrice': '600000',
+    'minBedrooms': '2',
+    'maxBedrooms': '2',
+    'apiApplication': 'ANDROID',
+    'appVersion': '3.70.0',
+}
+
+response = requests.get('https://api.rightmove.co.uk/api/property-listing', params=params, headers=headers, verify=False)

crawler/proof_of_concept/routing_distancematrix.py

@@ -0,0 +1,23 @@
+import requests
+
+API_KEY = 'AIzaSyBoBHzeQFgR7O-NlNsuHXQcC1B7ccEHpl8'
+url = "https://maps.googleapis.com/maps/api/distancematrix/json"
+origin = '51.5636306598907,-0.11061106079085892'
+dest = "51.53836609846008,-0.12743940233824352"
+
+params = {
+   "origins": origin,
+   "destinations": dest,
+   "key": API_KEY,
+   "departure_time": "", # timstamp, optional
+   "mode": "transit",  
+}
+
+r = requests.get(url, params=params)
+print(r.status_code)
+
+print(r.json())
+
+with open('code/json/routing_distancematrix.json', 'w') as f:
+   f.write(r.text)
+

crawler/proof_of_concept/routing_routing.py

@@ -0,0 +1,90 @@
+import requests
+from utils import nextMonday
+from collections import defaultdict
+
+API_KEY = 'AIzaSyBoBHzeQFgR7O-NlNsuHXQcC1B7ccEHpl8'
+url = "https://routes.googleapis.com/directions/v2:computeRoutes"
+
+
+def travel_time(origin_lat:float, origin_lon:float, dest_lat:float, dest_lon:float):   
+   monday9am = nextMonday()
+
+   header = {
+      "X-Goog-Api-Key": API_KEY,
+      "Content-Type": "application/json",
+      "X-Goog-FieldMask": "routes.distanceMeters,routes.duration,routes.staticDuration,routes.legs.steps.distanceMeters,routes.legs.steps.staticDuration,routes.legs.steps.travelMode",
+   }
+
+   body = {
+   "origin":{
+      "location":{
+         "latLng":{
+         "latitude": origin_lat,
+         "longitude": origin_lon
+         }
+      }
+   },
+   "destination":{
+      "location":{
+         "latLng":{
+         "latitude": dest_lat,
+         "longitude": dest_lon
+         }
+      }
+   },
+   "travelMode": "TRANSIT",
+   # "2023-10-15T15:01:23.045123456Z"
+   "departureTime": monday9am.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
+   "computeAlternativeRoutes": False,
+   #   "routeModifiers": {
+   #     "avoidTolls": false,
+   #     "avoidHighways": false,
+   #     "avoidFerries": false
+   #   },
+   "languageCode": "en-US",
+   "units": "METRIC"
+   }
+
+   r = requests.post(url, json=body, headers=header)
+   if r.status_code == 200:
+      return r.json()
+   
+   raise Exception(r.json())
+
+def extract_time(d):
+   r = d['routes'][0]
+   print(r.keys())
+   distance = r['distanceMeters']
+   duration = r['duration']
+   duration_static = r['staticDuration']
+   
+   steps = r['legs'][0]['steps']
+   # print(steps)
+   duration_per_transit = defaultdict(lambda: 0)
+   distance_per_transit = defaultdict(lambda: 0)
+   
+   for step in steps:
+      duration_per_transit[step['travelMode']] += int(step['staticDuration'].strip('s'))
+      distance_per_transit[step['travelMode']] += step.get('distanceMeters', 0)
+   
+   
+   print(f"dis {distance}, dur {duration}, duration per transit {dict(duration_per_transit)}, distance per transit {dict(distance_per_transit)}")
+  
+
+if __name__ == "__main__":
+   import json
+   with open('code/json/routing_routeapi.json', 'r') as f:
+      d = json.load(f)
+      
+   extract_time(d)
+   
+    
+
+# if __name__ == "__main__":
+#    origin = 51.5635664310333, -0.1107173751570373 # home
+#    dest = 51.50475678313417, 0.04915321000190009 # london city airport
+#    d = travel_time(origin[0], origin[1], dest[0], dest[1])
+#    import json
+#    with open('code/json/routing_routeapi.json', 'w') as f:
+#       json.dump(d, f)
+

crawler/proof_of_concept/single-query.py

@@ -0,0 +1,15 @@
+import requests
+
+headers = {
+    'Host': 'api.rightmove.co.uk',
+    # 'Accept-Encoding': 'gzip, deflate, br',
+    'User-Agent': 'okhttp/4.10.0',
+    'Connection': 'close',
+}
+
+params = {
+    'apiApplication': 'ANDROID',
+    'appVersion': '3.70.0',
+}
+
+response = requests.get('https://api.rightmove.co.uk/api/property/119578451', params=params, headers=headers, verify=False)

crawler/proof_of_concept/utils.py

@@ -0,0 +1,17 @@
+from datetime import datetime, timedelta, timezone
+
+def nextMonday():
+    """
+    I think this function doesnt work when the day is monday itself.
+
+    Returns:
+        _type_: _description_
+    """
+    now = datetime.now(timezone.utc)
+    days_until_monday = (0 - now.weekday() + 7) % 7
+    monday = now + timedelta(days=days_until_monday)
+    monday_9am = monday.replace(hour=9, minute=0, second=0, microsecond=0, tzinfo=timezone.utc)
+    return monday_9am
+
+if __name__ == '__main__':
+    print(nextMonday())