{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "38e8690a-f6f7-4e14-a657-f20605477afd", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/Users/kadir/code/realestate/crawler/venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "from data_access import Listing\n", "import pandas as pd" ] }, { "cell_type": "markdown", "id": "cfe2ab03-3204-4fd8-b76a-a734f6b87d75", "metadata": {}, "source": [ "### Fetch previous decisions" ] }, { "cell_type": "code", "execution_count": 2, "id": "424501ab-ecc6-42f5-b87e-b0d2871bdc74", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/var/folders/wl/kx43lvyn6yv7lq988gwrkq_m0000gn/T/ipykernel_85865/3290399543.py:3: FutureWarning: The behavior of 'to_datetime' with 'unit' when parsing strings is deprecated. In a future version, strings will be parsed as datetime strings, matching the behavior without a 'unit'. To retain the old behavior, explicitly cast ints or floats to numeric type before calling to_datetime.\n", " decisions = pd.read_json(decisions_path)\n", "/var/folders/wl/kx43lvyn6yv7lq988gwrkq_m0000gn/T/ipykernel_85865/3290399543.py:3: FutureWarning: The behavior of 'to_datetime' with 'unit' when parsing strings is deprecated. In a future version, strings will be parsed as datetime strings, matching the behavior without a 'unit'. To retain the old behavior, explicitly cast ints or floats to numeric type before calling to_datetime.\n", " decisions = pd.read_json(decisions_path)\n" ] } ], "source": [ "# read decisions on file\n", "decisions_path = 'data/decisions.json'\n", "decisions = pd.read_json(decisions_path)" ] }, { "cell_type": "code", "execution_count": null, "id": "ed170ba4-700a-4e0e-9950-a80765cd751c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 3, "id": "354237e5-a07e-4c0b-a775-223d5f701f2c", "metadata": {}, "outputs": [], "source": [ "# read new decisions\n", "try:\n", " newdecisions = pd.read_clipboard()\n", " newdecisions = newdecisions.loc[newdecisions.decision.notna(), ['identifier', 'decision']]\n", " # drop old decision rows and concat\n", " decisions = decisions[~decisions.identifier.isin(newdecisions.identifier)]\n", " decisions = pd.concat([decisions, newdecisions])\n", " # save to json\n", " decisions.to_json(decisions_path)\n", " print(decisions.shape)\n", " decisions.head()\n", "except:\n", " newdecisions = pd.DataFrame()" ] }, { "cell_type": "code", "execution_count": 4, "id": "86224a20-53e1-403c-8d9f-71b9a9df750c", "metadata": {}, "outputs": [], "source": [ "\"\"\"\n", "output:\n", "{145699277: 'n',\n", " 144642851: 'n',\n", " 145394765: 'n',\n", " 145418669: 'removed',\n", " 143205230: 'n',\n", " 140628560: 'eigentlich geil',\n", " ...\n", "}\n", "\"\"\"\n", "decisions = decisions.set_index('identifier').decision.to_dict()" ] }, { "cell_type": "code", "execution_count": null, "id": "ec257220-f170-41b8-9f9d-b8ef61512acf", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 5, "id": "6dbd25bd-802d-4953-83c3-f01640174353", "metadata": {}, "outputs": [], "source": [ "# Use if we want to skip at the bottom\n", "# decisions = {}" ] }, { "cell_type": "markdown", "id": "7c1ee5eb-1000-4ced-983c-df47fb6ceae8", "metadata": {}, "source": [ "### Get all data prepped for sheets" ] }, { "cell_type": "code", "execution_count": null, "id": "f20bddee-1e7c-4c46-a17a-c7bb6c13f30c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 6, "id": "b1101088-9613-465f-81fd-79801e0202b8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "18508\n" ] } ], "source": [ "ls = Listing.get_all_listings()\n", "filtered = []\n", "for l in ls:\n", " last_seen = l.last_seen\n", " if last_seen is None or last_seen > 30:\n", " continue\n", " filtered.append(l)\n", "print(len(filtered))" ] }, { "cell_type": "code", "execution_count": 7, "id": "63e61601-7e3f-4d58-89f6-1794e4868cc3", "metadata": {}, "outputs": [], "source": [ "ds = [l.dict_nicely() for l in filtered]" ] }, { "cell_type": "code", "execution_count": 8, "id": "1c222721-f426-42c0-9ac5-badc1f7a2034", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
identifiersqm_ocrpriceprice_per_sqmurlbedroomstravel_time_fastesttravel_time_secondlease_leftservice_chargedevelopmenttenure_typeupdated_daysstatuslast_seendecision
0101369066NaN875000.0NaNhttps://www.rightmove.co.uk/properties/1013690663{'duration': 2252, 'distance': 7140, 'duration...{'duration': 2465, 'distance': 7502, 'duration...0.0NaNFalseShare of Freehold12None0None
110548477245.7325000.07111.597374https://www.rightmove.co.uk/properties/1054847721{'duration': 1983, 'distance': 10095, 'duratio...{'duration': 2043, 'distance': 10083, 'duratio...104.0641.53FalseLeasehold36None0None
210582712658.5950000.016239.316239https://www.rightmove.co.uk/properties/1058271261{'duration': 2478, 'distance': 9584, 'duration...{'duration': 2478, 'distance': 9584, 'duration...NaNNaNTrueLeasehold2None0None
3105836849NaN400000.0NaNhttps://www.rightmove.co.uk/properties/1058368493{'duration': 2565, 'distance': 14070, 'duratio...{'duration': 2565, 'distance': 14070, 'duratio...NaNNaNFalseLeasehold393None20None
410810247653.7515000.09590.316574https://www.rightmove.co.uk/properties/1081024761{'duration': 1266, 'distance': 4042, 'duration...{'duration': 1861, 'distance': 4548, 'duration...104.0NaNFalseLeasehold16None0None
...................................................
185039420608049.6899000.018125.000000https://www.rightmove.co.uk/properties/942060801{'duration': 1125, 'distance': 4637, 'duration...{'duration': 1125, 'distance': 4641, 'duration...NaNNaNTrueLeasehold256None0None
1850494206329NaN700000.0NaNhttps://www.rightmove.co.uk/properties/942063291{'duration': 2172, 'distance': 12497, 'duratio...{'duration': 2112, 'distance': 12497, 'duratio...NaNNaNFalseLeasehold256None20None
185059450830694.01000000.010638.297872https://www.rightmove.co.uk/properties/945083062{'duration': 1046, 'distance': 2193, 'duration...{'duration': 1046, 'distance': 2193, 'duration...977.0NaNFalseLeasehold149None0None
1850695975483NaN800000.0NaNhttps://www.rightmove.co.uk/properties/959754832{'duration': 2281, 'distance': 7262, 'duration...{'duration': 2815, 'distance': 5607, 'duration...999.00.00FalseLeasehold3None0None
185079677399670.81000000.014124.293785https://www.rightmove.co.uk/properties/967739962{'duration': 1608, 'distance': 8301, 'duration...{'duration': 1608, 'distance': 8301, 'duration...992.04716.36TrueLeasehold227None20None
\n", "

18508 rows × 16 columns

\n", "
" ], "text/plain": [ " identifier sqm_ocr price price_per_sqm \\\n", "0 101369066 NaN 875000.0 NaN \n", "1 105484772 45.7 325000.0 7111.597374 \n", "2 105827126 58.5 950000.0 16239.316239 \n", "3 105836849 NaN 400000.0 NaN \n", "4 108102476 53.7 515000.0 9590.316574 \n", "... ... ... ... ... \n", "18503 94206080 49.6 899000.0 18125.000000 \n", "18504 94206329 NaN 700000.0 NaN \n", "18505 94508306 94.0 1000000.0 10638.297872 \n", "18506 95975483 NaN 800000.0 NaN \n", "18507 96773996 70.8 1000000.0 14124.293785 \n", "\n", " url bedrooms \\\n", "0 https://www.rightmove.co.uk/properties/101369066 3 \n", "1 https://www.rightmove.co.uk/properties/105484772 1 \n", "2 https://www.rightmove.co.uk/properties/105827126 1 \n", "3 https://www.rightmove.co.uk/properties/105836849 3 \n", "4 https://www.rightmove.co.uk/properties/108102476 1 \n", "... ... ... \n", "18503 https://www.rightmove.co.uk/properties/94206080 1 \n", "18504 https://www.rightmove.co.uk/properties/94206329 1 \n", "18505 https://www.rightmove.co.uk/properties/94508306 2 \n", "18506 https://www.rightmove.co.uk/properties/95975483 2 \n", "18507 https://www.rightmove.co.uk/properties/96773996 2 \n", "\n", " travel_time_fastest \\\n", "0 {'duration': 2252, 'distance': 7140, 'duration... \n", "1 {'duration': 1983, 'distance': 10095, 'duratio... \n", "2 {'duration': 2478, 'distance': 9584, 'duration... \n", "3 {'duration': 2565, 'distance': 14070, 'duratio... \n", "4 {'duration': 1266, 'distance': 4042, 'duration... \n", "... ... \n", "18503 {'duration': 1125, 'distance': 4637, 'duration... \n", "18504 {'duration': 2172, 'distance': 12497, 'duratio... \n", "18505 {'duration': 1046, 'distance': 2193, 'duration... \n", "18506 {'duration': 2281, 'distance': 7262, 'duration... \n", "18507 {'duration': 1608, 'distance': 8301, 'duration... \n", "\n", " travel_time_second lease_left \\\n", "0 {'duration': 2465, 'distance': 7502, 'duration... 0.0 \n", "1 {'duration': 2043, 'distance': 10083, 'duratio... 104.0 \n", "2 {'duration': 2478, 'distance': 9584, 'duration... NaN \n", "3 {'duration': 2565, 'distance': 14070, 'duratio... NaN \n", "4 {'duration': 1861, 'distance': 4548, 'duration... 104.0 \n", "... ... ... \n", "18503 {'duration': 1125, 'distance': 4641, 'duration... NaN \n", "18504 {'duration': 2112, 'distance': 12497, 'duratio... NaN \n", "18505 {'duration': 1046, 'distance': 2193, 'duration... 977.0 \n", "18506 {'duration': 2815, 'distance': 5607, 'duration... 999.0 \n", "18507 {'duration': 1608, 'distance': 8301, 'duration... 992.0 \n", "\n", " service_charge development tenure_type updated_days status \\\n", "0 NaN False Share of Freehold 12 None \n", "1 641.53 False Leasehold 36 None \n", "2 NaN True Leasehold 2 None \n", "3 NaN False Leasehold 393 None \n", "4 NaN False Leasehold 16 None \n", "... ... ... ... ... ... \n", "18503 NaN True Leasehold 256 None \n", "18504 NaN False Leasehold 256 None \n", "18505 NaN False Leasehold 149 None \n", "18506 0.00 False Leasehold 3 None \n", "18507 4716.36 True Leasehold 227 None \n", "\n", " last_seen decision \n", "0 0 None \n", "1 0 None \n", "2 0 None \n", "3 20 None \n", "4 0 None \n", "... ... ... \n", "18503 0 None \n", "18504 20 None \n", "18505 0 None \n", "18506 0 None \n", "18507 20 None \n", "\n", "[18508 rows x 16 columns]" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = pd.DataFrame(ds)\n", "df.loc[:, 'decision'] = df.identifier.apply(lambda x: decisions.get(x))\n", "df" ] }, { "cell_type": "code", "execution_count": 9, "id": "d80d9911-9a6d-4608-a6da-11dc864ee32b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(18508, 16)" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.shape" ] }, { "cell_type": "code", "execution_count": null, "id": "51e2770c-7633-4bd3-9a63-11ea705f0694", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "127c5377-594d-450f-81c5-235acd8ca863", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 10, "id": "7b37ad6b-9b0a-444e-b8c3-6fe4e43e42cb", "metadata": {}, "outputs": [], "source": [ "# dropcolumns = ['distance_per_transit', 'duration_static', 'distance']\n", "# s1 = df['travel_time_fastest'].apply(pd.Series).drop(dropcolumns, axis=1)\n", "# s1.columns = ['a_' + c for c in s1.columns]\n", "\n", "# s2 = df['travel_time_second'].apply(pd.Series).drop(dropcolumns, axis=1)\n", "# s2.columns = ['b_' + c for c in s2.columns]\n", "\n", "# df2 = pd.concat([df.drop(['travel_time_fastest', 'travel_time_second'], axis=1), s1, s2], axis=1)\n", "# df2.loc[:, 'min_duration'] = (df2.loc[:, ['a_duration', 'b_duration']].min(axis=1) / 60).round()\n", "# df2.head(2)" ] }, { "cell_type": "code", "execution_count": 11, "id": "8c75aaa6-6113-482f-809b-11e405510184", "metadata": {}, "outputs": [], "source": [ "# df2.to_clipboard()" ] }, { "cell_type": "code", "execution_count": 12, "id": "90500b06-9eb9-49e9-a0e0-6adef0b8effd", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(17217, 18)" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# remove all entries where we didnt calculate transit time (probably due to a too far distance)\n", "df2 = df[df.travel_time_fastest.notna()]\n", "\n", "# drop columns\n", "dropcolumns = ['distance_per_transit', 'duration_static', 'distance']\n", "s1 = df2['travel_time_fastest'].apply(pd.Series).drop(dropcolumns, axis=1)\n", "\n", "# fill in gap values for service charge and lease left. This is for excel so we can use filters better there\n", "df2.loc[:, 'service_charge'] = df2.service_charge.fillna(-1)\n", "df2.loc[:, 'lease_left'] = df2.lease_left.fillna(-1)\n", "df2.loc[:, 'sqm_ocr'] = df2.sqm_ocr.fillna(-1)\n", "\n", "\n", "df3 = pd.concat([df2.drop(['travel_time_fastest', 'travel_time_second'], axis=1), s1], axis=1)\n", "df3.loc[:, 'duration'] = (df3.loc[:, ['duration']].min(axis=1) / 60).round()\n", "df3.shape" ] }, { "cell_type": "code", "execution_count": null, "id": "2735f38e-7f05-4b5d-b835-1f3f31745ec7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 13, "id": "227f434a-7daf-4f9b-944b-b22ce0216b13", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0 None\n", "1 None\n", "2 None\n", "3 None\n", "4 None\n", " ... \n", "18503 None\n", "18504 None\n", "18505 None\n", "18506 None\n", "18507 None\n", "Name: status, Length: 17217, dtype: object" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df3.status" ] }, { "cell_type": "code", "execution_count": 14, "id": "cc96e017-1a6f-4e27-b128-e3ef9ff9cb27", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(10396, 17)" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# filter out undesirable\n", "available = df3.status.isna()\n", "near = df3.duration < 40\n", "last_seen_under_30 = df3.last_seen < 30\n", "df4 = df3[available & near & last_seen_under_30].drop(axis=1, columns=[\"status\"])\n", "df4.shape" ] }, { "cell_type": "code", "execution_count": 15, "id": "7c1e9779-b3f8-4098-b2a4-4547ac8ca8f9", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
identifiersqm_ocrpriceprice_per_sqmurlbedroomslease_leftservice_chargedevelopmenttenure_typeupdated_dayslast_seendecisiondurationinitial_walk_durationduration_per_transitnumber_of_transit_stops
0101369066-1.0875000.0NaNhttps://www.rightmove.co.uk/properties/10136906630.0-1.00FalseShare of Freehold120None38142{'WALK': 797, 'TRANSIT': 1227}2
110548477245.7325000.07111.597374https://www.rightmove.co.uk/properties/1054847721104.0641.53FalseLeasehold360None33372{'WALK': 609, 'TRANSIT': 1109}2
\n", "
" ], "text/plain": [ " identifier sqm_ocr price price_per_sqm \\\n", "0 101369066 -1.0 875000.0 NaN \n", "1 105484772 45.7 325000.0 7111.597374 \n", "\n", " url bedrooms lease_left \\\n", "0 https://www.rightmove.co.uk/properties/101369066 3 0.0 \n", "1 https://www.rightmove.co.uk/properties/105484772 1 104.0 \n", "\n", " service_charge development tenure_type updated_days last_seen \\\n", "0 -1.00 False Share of Freehold 12 0 \n", "1 641.53 False Leasehold 36 0 \n", "\n", " decision duration initial_walk_duration duration_per_transit \\\n", "0 None 38 142 {'WALK': 797, 'TRANSIT': 1227} \n", "1 None 33 372 {'WALK': 609, 'TRANSIT': 1109} \n", "\n", " number_of_transit_stops \n", "0 2 \n", "1 2 " ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df4.to_clipboard()\n", "df4.head(2)" ] }, { "cell_type": "code", "execution_count": null, "id": "7e9ab326-78d3-449a-831c-b23644d06e64", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "4f65084c-cd24-4405-988f-8443110dfaaa", "metadata": {}, "source": [ "# single query" ] }, { "cell_type": "code", "execution_count": 25, "id": "7ff34697-478e-4cd0-8e80-f7e286b04754", "metadata": {}, "outputs": [], "source": [ "import requests\n", "headers = {\n", " \"Host\": \"api.rightmove.co.uk\",\n", " # 'Accept-Encoding': 'gzip, deflate, br',\n", " \"User-Agent\": \"okhttp/4.10.0\",\n", " \"Connection\": \"close\",\n", "}\n", "params = {\n", " \"apiApplication\": \"ANDROID\",\n", " \"appVersion\": \"4.15.0\",\n", "}\n", "url = f\"https://api.rightmove.co.uk/api/property/152614085\"\n", "response = requests.get(url, params=params, headers=headers, verify=False)\n" ] }, { "cell_type": "code", "execution_count": 27, "id": "b4c179ff-2665-4db5-90ea-f5ec8fbcf51a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'property': {'identifier': 152614085,\n", " 'updateDate': 1731327705000,\n", " 'development': False,\n", " 'onlineViewing': False,\n", " 'price': 400000.0,\n", " 'priceQualifier': 'Guide Price',\n", " 'shouldShowPrice': True,\n", " 'premiumDisplay': False,\n", " 'transactionTypeId': 1,\n", " 'visible': True,\n", " 'bedrooms': 2,\n", " 'photoCount': 12,\n", " 'address': 'Mercury House, 2 Jude Street, London, E16',\n", " 'status': None,\n", " 'outcode': 'E16',\n", " 'summary': '2 bedroom 2 bathroom apartment on the 4th floor, secure allocated parking space included, 24hour concierge and offered to the market chain free.',\n", " 'fullDescription': 'Guide Price £400,000-£425,000

Offered with secure underground parking is this two bedroom, two bathroom apartment positioned on the 4th floor of a popular development moments from Canning Town. Station.

Offered with vacant possession and chain free and fully furnished to a high interior specification the property comprises approximately 695 sq ft to include a bright reception room with private balcony, open-plan kitchen fully fitted with quality appliances, granite work surfaces and ample storage, two well proportioned double bedrooms - one with en-suite shower room, and luxury bathroom suite.

The development is set around a landscaped courtyard and further benefits from a 24 hour concierge service. Situated 0.2 mile from Canning Town Jubilee and DLR stations the location offers impressive transport links to Canary Wharf, City & West End.

Service £3500 approx. per annum
Ground Rent £300 per annum
Council Tax Band C Newham £1,532.74
EPC B





',\n", " 'branch': {'identifier': 200531,\n", " 'updateDate': 1731327705000,\n", " 'name': 'Royal Wharf - Sales',\n", " 'brandName': 'Life Residential',\n", " 'branchLogo': 'https://media.rightmove.co.uk/201k/200531/branch_logo_200531_0000.jpeg',\n", " 'largeBranchLogo': 'https://media.rightmove.co.uk/201k/200531/branch_logo_200531_0000.jpeg',\n", " 'brandPlusResale': True,\n", " 'brandPlusLettings': True,\n", " 'address': '16 Royal Crest Avenue,\\r\\nLondon\\r\\nE16 2TQ',\n", " 'development': False},\n", " 'telephoneNumber': '020 3668 1030',\n", " 'listingUpdateReason': 'Reduced today',\n", " 'letFurnishType': None,\n", " 'letType': None,\n", " 'letDateAvailable': None,\n", " 'letBond': None,\n", " 'showLettingFeesMessage': False,\n", " 'lettingFeesMessage': None,\n", " 'floorplanCount': 1,\n", " 'showMap': True,\n", " 'latitude': 51.512047,\n", " 'longitude': 0.012448,\n", " 'exactLocationAvailable': True,\n", " 'showStreetView': True,\n", " 'streetViewLatitude': 51.512047,\n", " 'streetViewLongitude': 0.012448,\n", " 'streetViewHeading': None,\n", " 'streetViewPitch': None,\n", " 'streetViewZoom': None,\n", " 'saved': False,\n", " 'publicsiteUrl': 'https://www.rightmove.co.uk/properties/152614085',\n", " 'mobileStreetViewUrl': 'https://www.rightmove.co.uk/apps/streetview.html?propertyId=152614085',\n", " 'mobilePropertyMapViewUrl': 'https://www.rightmove.co.uk/apps/property-mapview.html?propertyId=152614085',\n", " 'schoolCheckerUrl': 'https://www.rightmove.co.uk/property-for-sale/nearby-schools/property-152614085.html',\n", " 'soldPricesUrl': 'https://www.rightmove.co.uk/house-prices/e16-1ff.html',\n", " 'marketInfoUrl': 'https://www.rightmove.co.uk/house-prices/e16-1ff.html',\n", " 'propertyDisclaimer': 'Disclaimer - Property reference EAS180199. The information displayed about this property comprises a property advertisement. Rightmove.co.uk makes no warranty as to the accuracy or completeness of the advertisement or any linked or associated information, and Rightmove has no control over the content. This property advertisement does not constitute property particulars. The information is provided and maintained by Life Residential, Royal Wharf - Sales. Please contact the selling agent or developer directly to obtain any information which may be available under the terms of The Energy Performance of Buildings (Certificates and Inspections) (England and Wales) Regulations 2007 or the Home Report if in relation to a residential property in Scotland.',\n", " 'propertyPhrase': '2 bedroom apartment',\n", " 'tenureType': 'Leasehold',\n", " 'buildToRent': False,\n", " 'propertySubtype': 'Apartment',\n", " 'contactMethod': 'EMAIL',\n", " 'misInfo': {'propertyId': 152614085,\n", " 'branchId': 200531,\n", " 'offerAdvertStampTypeId': None,\n", " 'brandPlus': True,\n", " 'featuredProperty': True,\n", " 'channel': 'BUY',\n", " 'premiumDisplay': False,\n", " 'premiumDisplayStampId': None,\n", " 'countryCode': 'GB'},\n", " 'mortgageCalculator': {'price': 400000,\n", " 'propertyTypeAlias': 'flats_apartments'},\n", " 'brochure': {'showBrochureLead': False,\n", " 'brochures': [{'url': 'https://assets.reapit.net/lir/live/pdf.php?p=EAS180199&t=S',\n", " 'caption': 'Particulars'}]},\n", " 'analyticsInfo': {'branchId': '200531',\n", " 'propertyId': '152614085',\n", " 'onlineViewing': 'F',\n", " 'imageCount': '12',\n", " 'floorplanCount': '1',\n", " 'beds': '2',\n", " 'postcode': 'E16 1FF',\n", " 'propertyType': 'Flats / Apartments',\n", " 'propertySubType': 'Apartment',\n", " 'added': '20240918',\n", " 'price': '400000',\n", " 'tenure': 'Leasehold',\n", " 'bathrooms': '2',\n", " 'sharedOwnership': 'F'},\n", " 'displayPrices': [{'displayPrice': '£400,000',\n", " 'displayPriceQualifier': 'Guide Price'}],\n", " 'stations': [{'station': 'Canning Town Station',\n", " 'distance': 0.2,\n", " 'type': '2,4'},\n", " {'station': 'Royal Victoria Station', 'distance': 0.3, 'type': '4'},\n", " {'station': 'Emirates Royal Docks Station', 'distance': 0.4, 'type': '7'}],\n", " 'features': [{'featureDescription': 'Guide Price £400,000-£425,000'},\n", " {'featureDescription': '2 Bedroom /2 /Bathroom'},\n", " {'featureDescription': 'Allocated Parking'},\n", " {'featureDescription': 'Private Balcony'},\n", " {'featureDescription': 'Concierge'},\n", " {'featureDescription': 'Courtyard Gardens'},\n", " {'featureDescription': '0.2mi to Canning Town Station'}],\n", " 'photos': [{'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_00_0000_max_656x437.jpeg',\n", " 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_00_0000_max_135x100.jpeg',\n", " 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_00_0000.jpeg',\n", " 'caption': 'Picture No. 33',\n", " 'order': 0},\n", " {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_01_0000_max_656x437.jpeg',\n", " 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_01_0000_max_135x100.jpeg',\n", " 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_01_0000.jpeg',\n", " 'caption': 'Picture No. 35',\n", " 'order': 1},\n", " {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_03_0000_max_656x437.jpeg',\n", " 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_03_0000_max_135x100.jpeg',\n", " 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_03_0000.jpeg',\n", " 'caption': 'Picture No. 34',\n", " 'order': 2},\n", " {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_02_0000_max_656x437.jpeg',\n", " 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_02_0000_max_135x100.jpeg',\n", " 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_02_0000.jpeg',\n", " 'caption': 'Picture No. 36',\n", " 'order': 3},\n", " {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_04_0000_max_656x437.jpeg',\n", " 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_04_0000_max_135x100.jpeg',\n", " 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_04_0000.jpeg',\n", " 'caption': 'Picture No. 39',\n", " 'order': 4},\n", " {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_05_0000_max_656x437.jpeg',\n", " 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_05_0000_max_135x100.jpeg',\n", " 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_05_0000.jpeg',\n", " 'caption': 'Picture No. 29',\n", " 'order': 5},\n", " {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_06_0000_max_656x437.jpeg',\n", " 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_06_0000_max_135x100.jpeg',\n", " 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_06_0000.jpeg',\n", " 'caption': 'Picture No. 30',\n", " 'order': 6},\n", " {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_07_0000_max_656x437.jpeg',\n", " 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_07_0000_max_135x100.jpeg',\n", " 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_07_0000.jpeg',\n", " 'caption': 'Picture No. 40',\n", " 'order': 7},\n", " {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_08_0000_max_656x437.jpeg',\n", " 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_08_0000_max_135x100.jpeg',\n", " 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_08_0000.jpeg',\n", " 'caption': 'Picture No. 31',\n", " 'order': 8},\n", " {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_09_0000_max_656x437.jpeg',\n", " 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_09_0000_max_135x100.jpeg',\n", " 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_09_0000.jpeg',\n", " 'caption': 'Picture No. 41',\n", " 'order': 9},\n", " {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_10_0000_max_656x437.jpeg',\n", " 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_10_0000_max_135x100.jpeg',\n", " 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_10_0000.jpeg',\n", " 'caption': 'Picture No. 32',\n", " 'order': 10},\n", " {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_11_0000_max_656x437.jpeg',\n", " 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_11_0000_max_135x100.jpeg',\n", " 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_11_0000.jpeg',\n", " 'caption': 'Picture No. 44',\n", " 'order': 11}],\n", " 'virtualTours': [],\n", " 'epcs': [{'url': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_EPCGRAPH_00_0000.png',\n", " 'caption': 'EPC Rating Graph',\n", " 'order': 0}],\n", " 'floorplans': [{'order': 0,\n", " 'url': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_FLP_00_0000.jpeg',\n", " 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_FLP_00_0000_max_296x197.jpeg',\n", " 'caption': 'Floorplan'}],\n", " 'note': None,\n", " 'affordableBuyingScheme': False,\n", " 'lettingsInfo': {'title': '', 'content': []},\n", " 'propertyDetailsInfo': {'title': '',\n", " 'content': [{'type': 'propertyType',\n", " 'title': 'Property type',\n", " 'value': 'Apartment'},\n", " {'type': 'bedrooms', 'title': 'Bedrooms', 'value': '2'},\n", " {'type': 'bathrooms', 'title': 'Bathrooms', 'value': '2'},\n", " {'type': 'tenure', 'title': 'Tenure', 'value': 'Leasehold'}]},\n", " 'tenureInfo': {'title': 'Leasehold',\n", " 'content': [{'type': 'groundRent',\n", " 'title': 'Ground rent',\n", " 'value': '£300 per year\\n(Ask agent about the review period)'},\n", " {'type': 'annualServiceCharge',\n", " 'title': 'Service charge',\n", " 'value': '£3,500 per year'},\n", " {'type': 'lengthOfLease',\n", " 'title': 'Length of lease',\n", " 'value': '133 years left'}]},\n", " 'sharedOwnershipInfo': {'title': '', 'content': []},\n", " 'councilTaxInfo': {'title': 'Council tax',\n", " 'content': [{'type': 'councilTaxBand',\n", " 'title': 'Council tax band',\n", " 'value': 'C'}]},\n", " 'domesticRatesInfo': {'title': '', 'content': []},\n", " 'linkToGlossary': 'https://www.rightmove.co.uk/guides/property-details-glossary/',\n", " 'enquiredTimestamp': None,\n", " 'stampDutyCalculator': {'country': 'ENGLAND',\n", " 'price': 400000,\n", " 'buyerType': None,\n", " 'result': None}}}" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "response.json()" ] }, { "cell_type": "markdown", "id": "98f8e950-2a3b-4856-aa62-3bc758e2fd42", "metadata": {}, "source": [ "# Find out the proper radius we want to use" ] }, { "cell_type": "code", "execution_count": 18, "id": "04bb61d5-cba7-4739-9568-b00342c1b636", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Filtered listings from 86437 to 46209\n" ] } ], "source": [ "from data_access import Listing\n", "from geopy.distance import geodesic\n", "\n", "listings = Listing.get_all_listings()\n", "BROCK_STREET_LAT_LONG = 51.52570434674584, -0.13956495005056113\n", "\n", "# reduce listings to everything within 7 miles\n", "filtered_listings = []\n", "for listing in listings:\n", " miles = geodesic(BROCK_STREET_LAT_LONG, (listing.latitude, listing.longitude)).miles\n", " if miles <= 7:\n", " filtered_listings.append(listing)\n", "\n", "print(f\"Filtered listings from {len(listings)} to {len(filtered_listings)}\")" ] }, { "cell_type": "markdown", "id": "a73fba2d-afeb-4194-8421-eff8e84a14e9", "metadata": {}, "source": [ "# Typeahead / fetch all boroughs" ] }, { "cell_type": "code", "execution_count": 20, "id": "61844fe2-408d-4b89-995f-c31110a850f6", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'key': 'CAMD',\n", " 'term': 'CAMD',\n", " 'typeAheadLocations': [{'displayName': 'Camden, North West London',\n", " 'locationIdentifier': 'REGION^85261',\n", " 'normalisedSearchTerm': 'CAMDEN NORTH WEST LONDON'},\n", " {'displayName': 'Camden (London Borough)',\n", " 'locationIdentifier': 'REGION^93941',\n", " 'normalisedSearchTerm': 'CAMDEN LONDON BOROUGH'},\n", " {'displayName': 'Camden Town, North West London',\n", " 'locationIdentifier': 'REGION^85262',\n", " 'normalisedSearchTerm': 'CAMDEN TOWN NORTH WEST LONDON'},\n", " {'displayName': 'Camden Town Station',\n", " 'locationIdentifier': 'STATION^1712',\n", " 'normalisedSearchTerm': 'CAMDEN TOWN STATION'},\n", " {'displayName': 'Camden Road Station',\n", " 'locationIdentifier': 'STATION^1709',\n", " 'normalisedSearchTerm': 'CAMDEN ROAD STATION'},\n", " {'displayName': 'Camden Town, Gosport, Hampshire',\n", " 'locationIdentifier': 'REGION^76577',\n", " 'normalisedSearchTerm': 'CAMDEN TOWN GOSPORT HAMPSHIRE'},\n", " {'displayName': 'Camderry, Omagh, County Tyrone, Northern Ireland',\n", " 'locationIdentifier': 'REGION^73327',\n", " 'normalisedSearchTerm': 'CAMDERRY OMAGH COUNTY TYRONE NORTHERN IRELAND'}],\n", " 'isComplete': True}" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import requests\n", "\n", "cookies = {\n", " 'permuserid': '240330LSEXEOANG04Q2VA3OZCIQ8TTSQ',\n", " 'TS019c0ed0': '012f990cd3494097746bc0b10b8d61bc6237319024e80701e0e8d735bd7d62a792529823c3164c771f223b0cade5ec9ae4b7fc001c',\n", " 'beta_optin': 'N:36:-1',\n", " 'RM_Register': 'C',\n", " 'JSESSIONID': '0BE8E261D81387C9BC530DB1A5F28955',\n", " 'svr': '3111',\n", " 'permuserid': '240330LSEXEOANG04Q2VA3OZCIQ8TTSQ',\n", " 'TS01ec61d1': '012f990cd3161dd68e4ed69b9d64f7d4de2356c651edef96201cc7facd11b28ef9338596a2022bc2ed7a56f09c5dee5aa1711de2d1',\n", " 'rmsessionid': '7bc54ce6-da97-42cf-8719-4e3e9c53e276',\n", " 'TS01821201': '012f990cd35255a563a541cfe06e4a774c129628165af71838ccdb7a17919672962514804459a0d9d9a90fe7b8feeec66145e30b98',\n", " 'TS01826437': '012f990cd3161dd68e4ed69b9d64f7d4de2356c651edef96201cc7facd11b28ef9338596a2022bc2ed7a56f09c5dee5aa1711de2d1',\n", " 'TPCmaxPrice': '800000',\n", " 'TS01a07bd2': '012f990cd3161dd68e4ed69b9d64f7d4de2356c651edef96201cc7facd11b28ef9338596a2022bc2ed7a56f09c5dee5aa1711de2d1',\n", " 'TPCminPrice': '700000',\n", "}\n", "\n", "headers = {\n", " 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0',\n", " 'Accept': 'application/json, text/javascript',\n", " 'Accept-Language': 'en-GB,en;q=0.5',\n", " # 'Accept-Encoding': 'gzip, deflate, br',\n", " 'Referer': 'https://www.rightmove.co.uk/property-for-sale/find.html?locationIdentifier=REGION%5E87515&maxBedrooms=3&minBedrooms=1&maxPrice=800000&minPrice=750001&propertyTypes=&includeSSTC=false&mustHave=&dontShow=retirement%2CsharedOwnership&furnishTypes=&keywords=',\n", " 'X-Correlation-Text-Val': 'source=search',\n", " 'Content-Type': 'application/x-www-form-urlencoded',\n", " 'traceparent': '00-d3020142d839bf1ef2b172fa596acea8-605693e1c4c5cbb5-00',\n", " 'DNT': '1',\n", " 'Sec-GPC': '1',\n", " 'Connection': 'keep-alive',\n", " # 'Cookie': 'permuserid=240330LSEXEOANG04Q2VA3OZCIQ8TTSQ; TS019c0ed0=012f990cd3494097746bc0b10b8d61bc6237319024e80701e0e8d735bd7d62a792529823c3164c771f223b0cade5ec9ae4b7fc001c; beta_optin=N:36:-1; RM_Register=C; JSESSIONID=0BE8E261D81387C9BC530DB1A5F28955; svr=3111; permuserid=240330LSEXEOANG04Q2VA3OZCIQ8TTSQ; TS01ec61d1=012f990cd3161dd68e4ed69b9d64f7d4de2356c651edef96201cc7facd11b28ef9338596a2022bc2ed7a56f09c5dee5aa1711de2d1; rmsessionid=7bc54ce6-da97-42cf-8719-4e3e9c53e276; TS01821201=012f990cd35255a563a541cfe06e4a774c129628165af71838ccdb7a17919672962514804459a0d9d9a90fe7b8feeec66145e30b98; TS01826437=012f990cd3161dd68e4ed69b9d64f7d4de2356c651edef96201cc7facd11b28ef9338596a2022bc2ed7a56f09c5dee5aa1711de2d1; TPCmaxPrice=800000; TS01a07bd2=012f990cd3161dd68e4ed69b9d64f7d4de2356c651edef96201cc7facd11b28ef9338596a2022bc2ed7a56f09c5dee5aa1711de2d1; TPCminPrice=700000',\n", " 'Sec-Fetch-Dest': 'empty',\n", " 'Sec-Fetch-Mode': 'cors',\n", " 'Sec-Fetch-Site': 'same-origin',\n", "}\n", "\n", "response = requests.get('https://www.rightmove.co.uk/typeAhead/uknostreet/CA/MD/EN/', cookies=cookies, headers=headers)\n", "response.json()" ] }, { "cell_type": "code", "execution_count": 39, "id": "eba907d0-680d-4374-a56e-95bd4abf0053", "metadata": {}, "outputs": [], "source": [ "import requests\n", "from textwrap import wrap\n", "\n", "def query_loc_identifier(name: str)->str:\n", " \"\"\"\n", " \n", " \"\"\"\n", " name = name.upper()\n", " name = '/'.join(wrap(name,2))\n", " \n", " headers = {\n", " 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0',\n", " }\n", " \n", " response = requests.get(f'https://www.rightmove.co.uk/typeAhead/uknostreet/{name}', headers=headers)\n", " return response.json()" ] }, { "cell_type": "code", "execution_count": null, "id": "d185e013-9beb-4e57-9d8b-a830e14339c3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "cabe5d3f-ad0f-49c2-9fbf-686539a05bd1", "metadata": {}, "outputs": [], "source": [ "\n" ] }, { "cell_type": "code", "execution_count": 30, "id": "ba500fe6-fb18-466e-a697-403d28181674", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 46, "id": "56c49b50-ff31-4785-9088-45ff5a39545e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'Greenwich': 'REGION^61226', 'Hillingdon': 'REGION^93959', 'Ealing': 'REGION^93947', 'Richmond upon Thames': 'REGION^61415', 'Sutton': 'REGION^93974', 'Wandsworth': 'REGION^93977', 'Camden': 'REGION^93941', 'Enfield': 'REGION^93950', 'Croydon': 'REGION^93944', 'Hackney': 'REGION^93953', 'Kingston upon Thames': 'REGION^93968', 'Kensington and Chelsea': 'REGION^61229', 'Bromley': 'REGION^93938', 'Brent': 'REGION^93935', 'Waltham Forest': 'REGION^61232', 'Southwark': 'REGION^61518', 'Harrow': 'REGION^93956', 'Lewisham': 'REGION^61413', 'Barnet': 'REGION^93929', 'Islington': 'REGION^93965', 'Haringey': 'REGION^61227', 'Lambeth': 'REGION^93971', 'Westminster': '', 'Tower Hamlets': 'REGION^61417', 'Havering': 'REGION^61228', 'Barking and Dagenham': 'REGION^61400', 'Hammersmith and Fulham': 'REGION^61407', 'Bexley': 'REGION^93932', 'Redbridge': 'REGION^61537', 'Newham': 'REGION^61231', 'Merton': 'REGION^61414', 'Hounslow': 'REGION^93962'}\n" ] } ], "source": [ "{\n", " \"City of London\": \"REGION^61224\",\n", " \"Greenwich\": \"REGION^61226\",\n", " \"Hillingdon\": \"REGION^93959\",\n", " \"Ealing\": \"REGION^93947\",\n", " \"Richmond upon Thames\": \"REGION^61415\",\n", " \"Sutton\": \"REGION^93974\",\n", " \"Wandsworth\": \"REGION^93977\",\n", " \"Camden\": \"REGION^93941\",\n", " \"Enfield\": \"REGION^93950\",\n", " \"Croydon\": \"REGION^93944\",\n", " \"Hackney\": \"REGION^93953\",\n", " \"Kingston upon Thames\": \"REGION^93968\",\n", " \"Kensington and Chelsea\": \"REGION^61229\",\n", " \"Bromley\": \"REGION^93938\",\n", " \"Brent\": \"REGION^93935\",\n", " \"Waltham Forest\": \"REGION^61232\",\n", " \"Southwark\": \"REGION^61518\",\n", " \"Harrow\": \"REGION^93956\",\n", " \"Lewisham\": \"REGION^61413\",\n", " \"Barnet\": \"REGION^93929\",\n", " \"Islington\": \"REGION^93965\",\n", " \"Haringey\": \"REGION^61227\",\n", " \"Lambeth\": \"REGION^93971\",\n", " \"Westminster\": \"REGION^93980\",\n", " \"Tower Hamlets\": \"REGION^61417\",\n", " \"Havering\": \"REGION^61228\",\n", " \"Barking and Dagenham\": \"REGION^61400\",\n", " \"Hammersmith and Fulham\": \"REGION^61407\",\n", " \"Bexley\": \"REGION^93932\",\n", " \"Redbridge\": \"REGION^61537\",\n", " \"Newham\": \"REGION^61231\",\n", " \"Merton\": \"REGION^61414\",\n", " \"Hounslow\": \"REGION^93962\",\n", "}\n", "\n", "bor_to_locid = {}\n", "\n", "for borough in boroughs:\n", " bor_to_locid[borough] = ''\n", " d = query_loc_identifier(borough)\n", " locs = d['typeAheadLocations']\n", " filtered = [l for l in locs if 'Borough' in l['displayName']]\n", " if len(filtered)>1:\n", " print(f\"{borough} has more entries: {len(filtered)}!\")\n", " if filtered:\n", " bor_to_locid[borough] = filtered[0]['locationIdentifier']\n", "\n", "Westminster (City of)\n", "print(bor_to_locid)" ] }, { "cell_type": "code", "execution_count": null, "id": "157e20a2-d137-45b8-802b-948fa8e04ba3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "386578dc-1ad5-4b8a-8905-29b0c47a6174", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "10c17fdf-f424-40cb-9d8c-9218f8d4ab53", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "88d99eb7-8c92-4817-86ce-ba0738331dba", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6c8b4488-ae2b-41ab-9c95-e3c85f9fb77e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c3072907-7ad8-4618-92ab-818e392218d9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b20f6f16-3236-4772-b1a3-2d4a3b1925a6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1a7230dc-1a0f-43e2-bd15-0c85ea445733", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "aeaf84bf-8514-48c6-88ce-2c6828bdcdf2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c888d4e6-d192-45df-b9b6-5e2d39bca344", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 1, "id": "cd71db7f-ba11-4d5d-a183-768ed4db23ba", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/Users/kadir/code/realestate/crawler/venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "from data_access import Listing" ] }, { "cell_type": "code", "execution_count": 19, "id": "d0ced84b-ee91-4642-b2ff-dd32d9f1e437", "metadata": {}, "outputs": [], "source": [ "l = Listing(141932357)" ] }, { "cell_type": "code", "execution_count": 30, "id": "7157f5f7-65b3-4232-bcae-26b93e5d93e6", "metadata": {}, "outputs": [], "source": [ "import json\n", "with open(l.path_listing_json()) as f:\n", " js = json.load(f)" ] }, { "cell_type": "code", "execution_count": 31, "id": "3f453f9c-bdaa-4713-8220-c504f1a436ae", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'identifier': 141932357,\n", " 'bedrooms': 1,\n", " 'address': 'Moreton Street, Pimlico',\n", " 'propertyType': 'Flat',\n", " 'status': None,\n", " 'transactionTypeId': 1,\n", " 'photoCount': 8,\n", " 'floorplanCount': 1,\n", " 'price': 595000.0,\n", " 'monthlyRent': None,\n", " 'priceQualifier': '',\n", " 'photoThumbnailUrl': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_00_0000_max_200x138.jpeg',\n", " 'photoThumbnail2Url': None,\n", " 'photoThumbnail3Url': None,\n", " 'photoThumbnail4Url': None,\n", " 'photoLargeThumbnailUrl': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_00_0000_max_656x437.jpeg',\n", " 'displayPrices': [{'displayPrice': '£595,000', 'displayPriceQualifier': ''}],\n", " 'thumbnailPhotos': [{'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_00_0000_max_656x437.jpeg'},\n", " {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_01_0000_max_656x437.jpeg'},\n", " {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_02_0000_max_656x437.jpeg'},\n", " {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_03_0000_max_656x437.jpeg'},\n", " {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_04_0000_max_656x437.jpeg'},\n", " {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_05_0000_max_656x437.jpeg'},\n", " {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_06_0000_max_656x437.jpeg'},\n", " {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_07_0000_max_656x437.jpeg'}],\n", " 'keywords': [],\n", " 'development': False,\n", " 'autoEmailReasonType': 'price_reduced',\n", " 'sortDate': 1718034274000,\n", " 'onlineViewing': False,\n", " 'buildToRent': False,\n", " 'summary': 'Newly refurbished! Beautifully presented with courtyard garden and air conditioning. This one/two bedroom spacious apartment has a large open plan kitchen/reception room with two good size vaults, reception room and a bathroom.',\n", " 'premiumDisplay': False,\n", " 'latitude': 51.488754,\n", " 'longitude': -0.137302,\n", " 'showMap': True,\n", " 'distance': None,\n", " 'featuredProperty': False,\n", " 'branch': {'identifier': 151016,\n", " 'branchLogo': 'https://media.rightmove.co.uk:443/brand/brand_rmchoice_logo_39479_0013.png',\n", " 'brandPlusResale': True,\n", " 'brandPlusLettings': True,\n", " 'brandName': 'Dexters',\n", " 'development': False,\n", " 'name': 'Westminster',\n", " 'updateDate': 1718803632000,\n", " 'hideReducedPropsFlag': False,\n", " 'contactTelephoneNumber': '020 7590 9570'},\n", " 'updateDate': 1718677927000,\n", " 'listingUpdateReason': 'Reduced on 10/06/2024',\n", " 'saved': False,\n", " 'hidden': False,\n", " 'premiumDisplayStickerName': '',\n", " 'enquiredTimestamp': None,\n", " 'visible': True,\n", " 'hasVideoContent': False}" ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "js" ] }, { "cell_type": "code", "execution_count": null, "id": "99d1a790-19d4-4686-b916-ee74fb3b2411", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a4a2937c-1509-4c52-8aec-7bc86372092f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 33, "id": "7cd815ac-a4e6-4540-a286-829b0364a860", "metadata": {}, "outputs": [], "source": [ "l = Listing(101369066)" ] }, { "cell_type": "code", "execution_count": 34, "id": "eb8e2bd4-3e8d-4e81-b569-9b17b84ca45c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "l.last_seen" ] }, { "cell_type": "code", "execution_count": null, "id": "a437a351-e3aa-458a-b692-3b630e598fa0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 24, "id": "90e94a48-f456-440b-b122-e58135b43614", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'2025-01-26T21:37:07.580704'" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import datetime\n", "import json\n", "datetime.datetime.now().isoformat()" ] }, { "cell_type": "code", "execution_count": 25, "id": "a44b384c-4803-44cc-a6e4-528b169795e0", "metadata": {}, "outputs": [], "source": [ "with open('/tmp/la.json', 'w') as f:\n", " json.dump(datetime.datetime.now().isoformat(), f)" ] }, { "cell_type": "code", "execution_count": 26, "id": "8b8bc05a-46a8-4f39-8b68-22ad62d13fe1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\"2025-01-26T21:37:07.744971\"" ] } ], "source": [ "!cat /tmp/la.json" ] }, { "cell_type": "code", "execution_count": null, "id": "341fe82c-fda8-4fc6-8bc0-081b285a5330", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.0" } }, "nbformat": 4, "nbformat_minor": 5 }