1661 lines
63 KiB
Text
1661 lines
63 KiB
Text
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "38e8690a-f6f7-4e14-a657-f20605477afd",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"/Users/kadir/code/realestate/crawler/venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
||
" from .autonotebook import tqdm as notebook_tqdm\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"from data_access import Listing\n",
|
||
"import pandas as pd"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "cfe2ab03-3204-4fd8-b76a-a734f6b87d75",
|
||
"metadata": {},
|
||
"source": [
|
||
"### Fetch previous decisions"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"id": "424501ab-ecc6-42f5-b87e-b0d2871bdc74",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# read decisions on file\n",
|
||
"decisions_path = 'data/decisions.json'\n",
|
||
"decisions = pd.read_json(decisions_path)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "ed170ba4-700a-4e0e-9950-a80765cd751c",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"id": "354237e5-a07e-4c0b-a775-223d5f701f2c",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# read new decisions\n",
|
||
"try:\n",
|
||
" newdecisions = pd.read_clipboard()\n",
|
||
" newdecisions = newdecisions.loc[newdecisions.decision.notna(), ['identifier', 'decision']]\n",
|
||
" # drop old decision rows and concat\n",
|
||
" decisions = decisions[~decisions.identifier.isin(newdecisions.identifier)]\n",
|
||
" decisions = pd.concat([decisions, newdecisions])\n",
|
||
" # save to json\n",
|
||
" decisions.to_json(decisions_path)\n",
|
||
" print(decisions.shape)\n",
|
||
" decisions.head()\n",
|
||
"except:\n",
|
||
" newdecisions = pd.DataFrame()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"id": "86224a20-53e1-403c-8d9f-71b9a9df750c",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"\"\"\"\n",
|
||
"output:\n",
|
||
"{145699277: 'n',\n",
|
||
" 144642851: 'n',\n",
|
||
" 145394765: 'n',\n",
|
||
" 145418669: 'removed',\n",
|
||
" 143205230: 'n',\n",
|
||
" 140628560: 'eigentlich geil',\n",
|
||
" ...\n",
|
||
"}\n",
|
||
"\"\"\"\n",
|
||
"decisions = decisions.set_index('identifier').decision.to_dict()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "ec257220-f170-41b8-9f9d-b8ef61512acf",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"id": "6dbd25bd-802d-4953-83c3-f01640174353",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Use if we want to skip at the bottom\n",
|
||
"# decisions = {}"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "7c1ee5eb-1000-4ced-983c-df47fb6ceae8",
|
||
"metadata": {},
|
||
"source": [
|
||
"### Get all data prepped for sheets"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "f20bddee-1e7c-4c46-a17a-c7bb6c13f30c",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 6,
|
||
"id": "b1101088-9613-465f-81fd-79801e0202b8",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"10574\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"ls = Listing.get_all_listings()\n",
|
||
"filtered = []\n",
|
||
"for l in ls:\n",
|
||
" last_seen = l.last_seen\n",
|
||
" if last_seen is None or last_seen > 30:\n",
|
||
" continue\n",
|
||
" filtered.append(l)\n",
|
||
"print(len(filtered))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 7,
|
||
"id": "63e61601-7e3f-4d58-89f6-1794e4868cc3",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"ds = [l.dict_nicely() for l in filtered]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 8,
|
||
"id": "1c222721-f426-42c0-9ac5-badc1f7a2034",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>identifier</th>\n",
|
||
" <th>sqm_ocr</th>\n",
|
||
" <th>price</th>\n",
|
||
" <th>price_per_sqm</th>\n",
|
||
" <th>url</th>\n",
|
||
" <th>bedrooms</th>\n",
|
||
" <th>travel_time_fastest</th>\n",
|
||
" <th>travel_time_second</th>\n",
|
||
" <th>lease_left</th>\n",
|
||
" <th>service_charge</th>\n",
|
||
" <th>development</th>\n",
|
||
" <th>tenure_type</th>\n",
|
||
" <th>updated_days</th>\n",
|
||
" <th>status</th>\n",
|
||
" <th>last_seen</th>\n",
|
||
" <th>decision</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>105484772</td>\n",
|
||
" <td>45.7</td>\n",
|
||
" <td>325000.0</td>\n",
|
||
" <td>7111.597374</td>\n",
|
||
" <td>https://www.rightmove.co.uk/properties/105484772</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>{'duration': 1983, 'distance': 10095, 'duratio...</td>\n",
|
||
" <td>{'duration': 2043, 'distance': 10083, 'duratio...</td>\n",
|
||
" <td>104.0</td>\n",
|
||
" <td>641.53</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>Leasehold</td>\n",
|
||
" <td>116</td>\n",
|
||
" <td>None</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1</th>\n",
|
||
" <td>105827126</td>\n",
|
||
" <td>58.5</td>\n",
|
||
" <td>950000.0</td>\n",
|
||
" <td>16239.316239</td>\n",
|
||
" <td>https://www.rightmove.co.uk/properties/105827126</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>{'duration': 2478, 'distance': 9584, 'duration...</td>\n",
|
||
" <td>{'duration': 2478, 'distance': 9584, 'duration...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>Leasehold</td>\n",
|
||
" <td>83</td>\n",
|
||
" <td>None</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td>108102476</td>\n",
|
||
" <td>53.7</td>\n",
|
||
" <td>515000.0</td>\n",
|
||
" <td>9590.316574</td>\n",
|
||
" <td>https://www.rightmove.co.uk/properties/108102476</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>{'duration': 1266, 'distance': 4042, 'duration...</td>\n",
|
||
" <td>{'duration': 1861, 'distance': 4548, 'duration...</td>\n",
|
||
" <td>104.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>Leasehold</td>\n",
|
||
" <td>97</td>\n",
|
||
" <td>None</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3</th>\n",
|
||
" <td>108171770</td>\n",
|
||
" <td>45.0</td>\n",
|
||
" <td>650000.0</td>\n",
|
||
" <td>14444.444444</td>\n",
|
||
" <td>https://www.rightmove.co.uk/properties/108171770</td>\n",
|
||
" <td>2</td>\n",
|
||
" <td>{'duration': 1591, 'distance': 7827, 'duration...</td>\n",
|
||
" <td>{'duration': 1591, 'distance': 7827, 'duration...</td>\n",
|
||
" <td>962.0</td>\n",
|
||
" <td>2000.00</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>Leasehold</td>\n",
|
||
" <td>261</td>\n",
|
||
" <td>None</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4</th>\n",
|
||
" <td>109595123</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1000000.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>https://www.rightmove.co.uk/properties/109595123</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>{'duration': 2463, 'distance': 9565, 'duration...</td>\n",
|
||
" <td>{'duration': 2463, 'distance': 9565, 'duration...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>Please confirm if this is a freehold or leaseh...</td>\n",
|
||
" <td>96</td>\n",
|
||
" <td>None</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>...</th>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>10569</th>\n",
|
||
" <td>88731877</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>570000.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>https://www.rightmove.co.uk/properties/88731877</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>{'duration': 912, 'distance': 6329, 'duration_...</td>\n",
|
||
" <td>{'duration': 852, 'distance': 6329, 'duration_...</td>\n",
|
||
" <td>998.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>Leasehold</td>\n",
|
||
" <td>407</td>\n",
|
||
" <td>None</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>10570</th>\n",
|
||
" <td>89825950</td>\n",
|
||
" <td>48.9</td>\n",
|
||
" <td>680000.0</td>\n",
|
||
" <td>13905.930470</td>\n",
|
||
" <td>https://www.rightmove.co.uk/properties/89825950</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>{'duration': 273, 'distance': 762, 'duration_s...</td>\n",
|
||
" <td>{'duration': 273, 'distance': 762, 'duration_s...</td>\n",
|
||
" <td>112.0</td>\n",
|
||
" <td>1700.00</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>Leasehold</td>\n",
|
||
" <td>113</td>\n",
|
||
" <td>None</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>10571</th>\n",
|
||
" <td>94206080</td>\n",
|
||
" <td>49.6</td>\n",
|
||
" <td>899000.0</td>\n",
|
||
" <td>18125.000000</td>\n",
|
||
" <td>https://www.rightmove.co.uk/properties/94206080</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>{'duration': 1125, 'distance': 4637, 'duration...</td>\n",
|
||
" <td>{'duration': 1125, 'distance': 4641, 'duration...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>Leasehold</td>\n",
|
||
" <td>337</td>\n",
|
||
" <td>None</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>10572</th>\n",
|
||
" <td>94508306</td>\n",
|
||
" <td>94.0</td>\n",
|
||
" <td>1000000.0</td>\n",
|
||
" <td>10638.297872</td>\n",
|
||
" <td>https://www.rightmove.co.uk/properties/94508306</td>\n",
|
||
" <td>2</td>\n",
|
||
" <td>{'duration': 1046, 'distance': 2193, 'duration...</td>\n",
|
||
" <td>{'duration': 1046, 'distance': 2193, 'duration...</td>\n",
|
||
" <td>977.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>Leasehold</td>\n",
|
||
" <td>230</td>\n",
|
||
" <td>None</td>\n",
|
||
" <td>9</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>10573</th>\n",
|
||
" <td>95975483</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>800000.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>https://www.rightmove.co.uk/properties/95975483</td>\n",
|
||
" <td>2</td>\n",
|
||
" <td>{'duration': 2281, 'distance': 7262, 'duration...</td>\n",
|
||
" <td>{'duration': 2815, 'distance': 5607, 'duration...</td>\n",
|
||
" <td>999.0</td>\n",
|
||
" <td>0.00</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>Leasehold</td>\n",
|
||
" <td>84</td>\n",
|
||
" <td>None</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"<p>10574 rows × 16 columns</p>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" identifier sqm_ocr price price_per_sqm \\\n",
|
||
"0 105484772 45.7 325000.0 7111.597374 \n",
|
||
"1 105827126 58.5 950000.0 16239.316239 \n",
|
||
"2 108102476 53.7 515000.0 9590.316574 \n",
|
||
"3 108171770 45.0 650000.0 14444.444444 \n",
|
||
"4 109595123 NaN 1000000.0 NaN \n",
|
||
"... ... ... ... ... \n",
|
||
"10569 88731877 NaN 570000.0 NaN \n",
|
||
"10570 89825950 48.9 680000.0 13905.930470 \n",
|
||
"10571 94206080 49.6 899000.0 18125.000000 \n",
|
||
"10572 94508306 94.0 1000000.0 10638.297872 \n",
|
||
"10573 95975483 NaN 800000.0 NaN \n",
|
||
"\n",
|
||
" url bedrooms \\\n",
|
||
"0 https://www.rightmove.co.uk/properties/105484772 1 \n",
|
||
"1 https://www.rightmove.co.uk/properties/105827126 1 \n",
|
||
"2 https://www.rightmove.co.uk/properties/108102476 1 \n",
|
||
"3 https://www.rightmove.co.uk/properties/108171770 2 \n",
|
||
"4 https://www.rightmove.co.uk/properties/109595123 1 \n",
|
||
"... ... ... \n",
|
||
"10569 https://www.rightmove.co.uk/properties/88731877 1 \n",
|
||
"10570 https://www.rightmove.co.uk/properties/89825950 1 \n",
|
||
"10571 https://www.rightmove.co.uk/properties/94206080 1 \n",
|
||
"10572 https://www.rightmove.co.uk/properties/94508306 2 \n",
|
||
"10573 https://www.rightmove.co.uk/properties/95975483 2 \n",
|
||
"\n",
|
||
" travel_time_fastest \\\n",
|
||
"0 {'duration': 1983, 'distance': 10095, 'duratio... \n",
|
||
"1 {'duration': 2478, 'distance': 9584, 'duration... \n",
|
||
"2 {'duration': 1266, 'distance': 4042, 'duration... \n",
|
||
"3 {'duration': 1591, 'distance': 7827, 'duration... \n",
|
||
"4 {'duration': 2463, 'distance': 9565, 'duration... \n",
|
||
"... ... \n",
|
||
"10569 {'duration': 912, 'distance': 6329, 'duration_... \n",
|
||
"10570 {'duration': 273, 'distance': 762, 'duration_s... \n",
|
||
"10571 {'duration': 1125, 'distance': 4637, 'duration... \n",
|
||
"10572 {'duration': 1046, 'distance': 2193, 'duration... \n",
|
||
"10573 {'duration': 2281, 'distance': 7262, 'duration... \n",
|
||
"\n",
|
||
" travel_time_second lease_left \\\n",
|
||
"0 {'duration': 2043, 'distance': 10083, 'duratio... 104.0 \n",
|
||
"1 {'duration': 2478, 'distance': 9584, 'duration... NaN \n",
|
||
"2 {'duration': 1861, 'distance': 4548, 'duration... 104.0 \n",
|
||
"3 {'duration': 1591, 'distance': 7827, 'duration... 962.0 \n",
|
||
"4 {'duration': 2463, 'distance': 9565, 'duration... NaN \n",
|
||
"... ... ... \n",
|
||
"10569 {'duration': 852, 'distance': 6329, 'duration_... 998.0 \n",
|
||
"10570 {'duration': 273, 'distance': 762, 'duration_s... 112.0 \n",
|
||
"10571 {'duration': 1125, 'distance': 4641, 'duration... NaN \n",
|
||
"10572 {'duration': 1046, 'distance': 2193, 'duration... 977.0 \n",
|
||
"10573 {'duration': 2815, 'distance': 5607, 'duration... 999.0 \n",
|
||
"\n",
|
||
" service_charge development \\\n",
|
||
"0 641.53 False \n",
|
||
"1 NaN True \n",
|
||
"2 NaN False \n",
|
||
"3 2000.00 False \n",
|
||
"4 NaN True \n",
|
||
"... ... ... \n",
|
||
"10569 NaN False \n",
|
||
"10570 1700.00 False \n",
|
||
"10571 NaN True \n",
|
||
"10572 NaN False \n",
|
||
"10573 0.00 False \n",
|
||
"\n",
|
||
" tenure_type updated_days status \\\n",
|
||
"0 Leasehold 116 None \n",
|
||
"1 Leasehold 83 None \n",
|
||
"2 Leasehold 97 None \n",
|
||
"3 Leasehold 261 None \n",
|
||
"4 Please confirm if this is a freehold or leaseh... 96 None \n",
|
||
"... ... ... ... \n",
|
||
"10569 Leasehold 407 None \n",
|
||
"10570 Leasehold 113 None \n",
|
||
"10571 Leasehold 337 None \n",
|
||
"10572 Leasehold 230 None \n",
|
||
"10573 Leasehold 84 None \n",
|
||
"\n",
|
||
" last_seen decision \n",
|
||
"0 0 None \n",
|
||
"1 0 None \n",
|
||
"2 0 None \n",
|
||
"3 0 None \n",
|
||
"4 0 None \n",
|
||
"... ... ... \n",
|
||
"10569 0 None \n",
|
||
"10570 0 None \n",
|
||
"10571 0 None \n",
|
||
"10572 9 None \n",
|
||
"10573 0 None \n",
|
||
"\n",
|
||
"[10574 rows x 16 columns]"
|
||
]
|
||
},
|
||
"execution_count": 8,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"df = pd.DataFrame(ds)\n",
|
||
"df.loc[:, 'decision'] = df.identifier.apply(lambda x: decisions.get(x))\n",
|
||
"df"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 9,
|
||
"id": "d80d9911-9a6d-4608-a6da-11dc864ee32b",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"(10574, 16)"
|
||
]
|
||
},
|
||
"execution_count": 9,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"df.shape"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "51e2770c-7633-4bd3-9a63-11ea705f0694",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "127c5377-594d-450f-81c5-235acd8ca863",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 10,
|
||
"id": "7b37ad6b-9b0a-444e-b8c3-6fe4e43e42cb",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# dropcolumns = ['distance_per_transit', 'duration_static', 'distance']\n",
|
||
"# s1 = df['travel_time_fastest'].apply(pd.Series).drop(dropcolumns, axis=1)\n",
|
||
"# s1.columns = ['a_' + c for c in s1.columns]\n",
|
||
"\n",
|
||
"# s2 = df['travel_time_second'].apply(pd.Series).drop(dropcolumns, axis=1)\n",
|
||
"# s2.columns = ['b_' + c for c in s2.columns]\n",
|
||
"\n",
|
||
"# df2 = pd.concat([df.drop(['travel_time_fastest', 'travel_time_second'], axis=1), s1, s2], axis=1)\n",
|
||
"# df2.loc[:, 'min_duration'] = (df2.loc[:, ['a_duration', 'b_duration']].min(axis=1) / 60).round()\n",
|
||
"# df2.head(2)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 11,
|
||
"id": "8c75aaa6-6113-482f-809b-11e405510184",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# df2.to_clipboard()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 12,
|
||
"id": "90500b06-9eb9-49e9-a0e0-6adef0b8effd",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"(9494, 18)"
|
||
]
|
||
},
|
||
"execution_count": 12,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"# remove all entries where we didnt calculate transit time (probably due to a too far distance)\n",
|
||
"df2 = df[df.travel_time_fastest.notna()]\n",
|
||
"\n",
|
||
"# drop columns\n",
|
||
"dropcolumns = ['distance_per_transit', 'duration_static', 'distance']\n",
|
||
"s1 = df2['travel_time_fastest'].apply(pd.Series).drop(dropcolumns, axis=1)\n",
|
||
"\n",
|
||
"# fill in gap values for service charge and lease left. This is for excel so we can use filters better there\n",
|
||
"df2.loc[:, 'service_charge'] = df2.service_charge.fillna(-1)\n",
|
||
"df2.loc[:, 'lease_left'] = df2.lease_left.fillna(-1)\n",
|
||
"df2.loc[:, 'sqm_ocr'] = df2.sqm_ocr.fillna(-1)\n",
|
||
"\n",
|
||
"\n",
|
||
"df3 = pd.concat([df2.drop(['travel_time_fastest', 'travel_time_second'], axis=1), s1], axis=1)\n",
|
||
"df3.loc[:, 'duration'] = (df3.loc[:, ['duration']].min(axis=1) / 60).round()\n",
|
||
"df3.shape"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "2735f38e-7f05-4b5d-b835-1f3f31745ec7",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 13,
|
||
"id": "227f434a-7daf-4f9b-944b-b22ce0216b13",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"0 None\n",
|
||
"1 None\n",
|
||
"2 None\n",
|
||
"3 None\n",
|
||
"4 None\n",
|
||
" ... \n",
|
||
"10569 None\n",
|
||
"10570 None\n",
|
||
"10571 None\n",
|
||
"10572 None\n",
|
||
"10573 None\n",
|
||
"Name: status, Length: 9494, dtype: object"
|
||
]
|
||
},
|
||
"execution_count": 13,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"df3.status"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 14,
|
||
"id": "cc96e017-1a6f-4e27-b128-e3ef9ff9cb27",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"(6578, 17)"
|
||
]
|
||
},
|
||
"execution_count": 14,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"# filter out undesirable\n",
|
||
"available = df3.status.isna()\n",
|
||
"near = df3.duration < 40\n",
|
||
"last_seen_under_30 = df3.last_seen < 30\n",
|
||
"df4 = df3[available & near & last_seen_under_30].drop(axis=1, columns=[\"status\"])\n",
|
||
"df4.shape"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 15,
|
||
"id": "7c1e9779-b3f8-4098-b2a4-4547ac8ca8f9",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>identifier</th>\n",
|
||
" <th>sqm_ocr</th>\n",
|
||
" <th>price</th>\n",
|
||
" <th>price_per_sqm</th>\n",
|
||
" <th>url</th>\n",
|
||
" <th>bedrooms</th>\n",
|
||
" <th>lease_left</th>\n",
|
||
" <th>service_charge</th>\n",
|
||
" <th>development</th>\n",
|
||
" <th>tenure_type</th>\n",
|
||
" <th>updated_days</th>\n",
|
||
" <th>last_seen</th>\n",
|
||
" <th>decision</th>\n",
|
||
" <th>duration</th>\n",
|
||
" <th>initial_walk_duration</th>\n",
|
||
" <th>duration_per_transit</th>\n",
|
||
" <th>number_of_transit_stops</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>105484772</td>\n",
|
||
" <td>45.7</td>\n",
|
||
" <td>325000.0</td>\n",
|
||
" <td>7111.597374</td>\n",
|
||
" <td>https://www.rightmove.co.uk/properties/105484772</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>104.0</td>\n",
|
||
" <td>641.53</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>Leasehold</td>\n",
|
||
" <td>116</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>None</td>\n",
|
||
" <td>33</td>\n",
|
||
" <td>372</td>\n",
|
||
" <td>{'WALK': 609, 'TRANSIT': 1109}</td>\n",
|
||
" <td>2</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td>108102476</td>\n",
|
||
" <td>53.7</td>\n",
|
||
" <td>515000.0</td>\n",
|
||
" <td>9590.316574</td>\n",
|
||
" <td>https://www.rightmove.co.uk/properties/108102476</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>104.0</td>\n",
|
||
" <td>-1.00</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>Leasehold</td>\n",
|
||
" <td>97</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>None</td>\n",
|
||
" <td>21</td>\n",
|
||
" <td>593</td>\n",
|
||
" <td>{'WALK': 819, 'TRANSIT': 445}</td>\n",
|
||
" <td>1</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" identifier sqm_ocr price price_per_sqm \\\n",
|
||
"0 105484772 45.7 325000.0 7111.597374 \n",
|
||
"2 108102476 53.7 515000.0 9590.316574 \n",
|
||
"\n",
|
||
" url bedrooms lease_left \\\n",
|
||
"0 https://www.rightmove.co.uk/properties/105484772 1 104.0 \n",
|
||
"2 https://www.rightmove.co.uk/properties/108102476 1 104.0 \n",
|
||
"\n",
|
||
" service_charge development tenure_type updated_days last_seen decision \\\n",
|
||
"0 641.53 False Leasehold 116 0 None \n",
|
||
"2 -1.00 False Leasehold 97 0 None \n",
|
||
"\n",
|
||
" duration initial_walk_duration duration_per_transit \\\n",
|
||
"0 33 372 {'WALK': 609, 'TRANSIT': 1109} \n",
|
||
"2 21 593 {'WALK': 819, 'TRANSIT': 445} \n",
|
||
"\n",
|
||
" number_of_transit_stops \n",
|
||
"0 2 \n",
|
||
"2 1 "
|
||
]
|
||
},
|
||
"execution_count": 15,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"df4.to_clipboard()\n",
|
||
"df4.head(2)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "7e9ab326-78d3-449a-831c-b23644d06e64",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "4f65084c-cd24-4405-988f-8443110dfaaa",
|
||
"metadata": {},
|
||
"source": [
|
||
"# single query"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 25,
|
||
"id": "7ff34697-478e-4cd0-8e80-f7e286b04754",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import requests\n",
|
||
"headers = {\n",
|
||
" \"Host\": \"api.rightmove.co.uk\",\n",
|
||
" # 'Accept-Encoding': 'gzip, deflate, br',\n",
|
||
" \"User-Agent\": \"okhttp/4.10.0\",\n",
|
||
" \"Connection\": \"close\",\n",
|
||
"}\n",
|
||
"params = {\n",
|
||
" \"apiApplication\": \"ANDROID\",\n",
|
||
" \"appVersion\": \"4.15.0\",\n",
|
||
"}\n",
|
||
"url = f\"https://api.rightmove.co.uk/api/property/152614085\"\n",
|
||
"response = requests.get(url, params=params, headers=headers, verify=False)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 27,
|
||
"id": "b4c179ff-2665-4db5-90ea-f5ec8fbcf51a",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"{'property': {'identifier': 152614085,\n",
|
||
" 'updateDate': 1731327705000,\n",
|
||
" 'development': False,\n",
|
||
" 'onlineViewing': False,\n",
|
||
" 'price': 400000.0,\n",
|
||
" 'priceQualifier': 'Guide Price',\n",
|
||
" 'shouldShowPrice': True,\n",
|
||
" 'premiumDisplay': False,\n",
|
||
" 'transactionTypeId': 1,\n",
|
||
" 'visible': True,\n",
|
||
" 'bedrooms': 2,\n",
|
||
" 'photoCount': 12,\n",
|
||
" 'address': 'Mercury House, 2 Jude Street, London, E16',\n",
|
||
" 'status': None,\n",
|
||
" 'outcode': 'E16',\n",
|
||
" 'summary': '2 bedroom 2 bathroom apartment on the 4th floor, secure allocated parking space included, 24hour concierge and offered to the market chain free.',\n",
|
||
" 'fullDescription': 'Guide Price £400,000-£425,000 <br /><br />Offered with secure underground parking is this two bedroom, two bathroom apartment positioned on the 4th floor of a popular development moments from Canning Town. Station.<br /><br />Offered with vacant possession and chain free and fully furnished to a high interior specification the property comprises approximately 695 sq ft to include a bright reception room with private balcony, open-plan kitchen fully fitted with quality appliances, granite work surfaces and ample storage, two well proportioned double bedrooms - one with en-suite shower room, and luxury bathroom suite. <br /><br />The development is set around a landscaped courtyard and further benefits from a 24 hour concierge service. Situated 0.2 mile from Canning Town Jubilee and DLR stations the location offers impressive transport links to Canary Wharf, City & West End.<br /><br />Service £3500 approx. per annum <br />Ground Rent £300 per annum<br />Council Tax Band C Newham £1,532.74<br />EPC B<br /><br /><br /><br /><br /><br />',\n",
|
||
" 'branch': {'identifier': 200531,\n",
|
||
" 'updateDate': 1731327705000,\n",
|
||
" 'name': 'Royal Wharf - Sales',\n",
|
||
" 'brandName': 'Life Residential',\n",
|
||
" 'branchLogo': 'https://media.rightmove.co.uk/201k/200531/branch_logo_200531_0000.jpeg',\n",
|
||
" 'largeBranchLogo': 'https://media.rightmove.co.uk/201k/200531/branch_logo_200531_0000.jpeg',\n",
|
||
" 'brandPlusResale': True,\n",
|
||
" 'brandPlusLettings': True,\n",
|
||
" 'address': '16 Royal Crest Avenue,\\r\\nLondon\\r\\nE16 2TQ',\n",
|
||
" 'development': False},\n",
|
||
" 'telephoneNumber': '020 3668 1030',\n",
|
||
" 'listingUpdateReason': 'Reduced today',\n",
|
||
" 'letFurnishType': None,\n",
|
||
" 'letType': None,\n",
|
||
" 'letDateAvailable': None,\n",
|
||
" 'letBond': None,\n",
|
||
" 'showLettingFeesMessage': False,\n",
|
||
" 'lettingFeesMessage': None,\n",
|
||
" 'floorplanCount': 1,\n",
|
||
" 'showMap': True,\n",
|
||
" 'latitude': 51.512047,\n",
|
||
" 'longitude': 0.012448,\n",
|
||
" 'exactLocationAvailable': True,\n",
|
||
" 'showStreetView': True,\n",
|
||
" 'streetViewLatitude': 51.512047,\n",
|
||
" 'streetViewLongitude': 0.012448,\n",
|
||
" 'streetViewHeading': None,\n",
|
||
" 'streetViewPitch': None,\n",
|
||
" 'streetViewZoom': None,\n",
|
||
" 'saved': False,\n",
|
||
" 'publicsiteUrl': 'https://www.rightmove.co.uk/properties/152614085',\n",
|
||
" 'mobileStreetViewUrl': 'https://www.rightmove.co.uk/apps/streetview.html?propertyId=152614085',\n",
|
||
" 'mobilePropertyMapViewUrl': 'https://www.rightmove.co.uk/apps/property-mapview.html?propertyId=152614085',\n",
|
||
" 'schoolCheckerUrl': 'https://www.rightmove.co.uk/property-for-sale/nearby-schools/property-152614085.html',\n",
|
||
" 'soldPricesUrl': 'https://www.rightmove.co.uk/house-prices/e16-1ff.html',\n",
|
||
" 'marketInfoUrl': 'https://www.rightmove.co.uk/house-prices/e16-1ff.html',\n",
|
||
" 'propertyDisclaimer': '<b>Disclaimer</b> - Property reference EAS180199. The information displayed about this property comprises a property advertisement. Rightmove.co.uk makes no warranty as to the accuracy or completeness of the advertisement or any linked or associated information, and Rightmove has no control over the content. This property advertisement does not constitute property particulars. The information is provided and maintained by <b>Life Residential, Royal Wharf - Sales</b>. Please contact the selling agent or developer directly to obtain any information which may be available under the terms of The Energy Performance of Buildings (Certificates and Inspections) (England and Wales) Regulations 2007 or the Home Report if in relation to a residential property in Scotland.',\n",
|
||
" 'propertyPhrase': '2 bedroom apartment',\n",
|
||
" 'tenureType': 'Leasehold',\n",
|
||
" 'buildToRent': False,\n",
|
||
" 'propertySubtype': 'Apartment',\n",
|
||
" 'contactMethod': 'EMAIL',\n",
|
||
" 'misInfo': {'propertyId': 152614085,\n",
|
||
" 'branchId': 200531,\n",
|
||
" 'offerAdvertStampTypeId': None,\n",
|
||
" 'brandPlus': True,\n",
|
||
" 'featuredProperty': True,\n",
|
||
" 'channel': 'BUY',\n",
|
||
" 'premiumDisplay': False,\n",
|
||
" 'premiumDisplayStampId': None,\n",
|
||
" 'countryCode': 'GB'},\n",
|
||
" 'mortgageCalculator': {'price': 400000,\n",
|
||
" 'propertyTypeAlias': 'flats_apartments'},\n",
|
||
" 'brochure': {'showBrochureLead': False,\n",
|
||
" 'brochures': [{'url': 'https://assets.reapit.net/lir/live/pdf.php?p=EAS180199&t=S',\n",
|
||
" 'caption': 'Particulars'}]},\n",
|
||
" 'analyticsInfo': {'branchId': '200531',\n",
|
||
" 'propertyId': '152614085',\n",
|
||
" 'onlineViewing': 'F',\n",
|
||
" 'imageCount': '12',\n",
|
||
" 'floorplanCount': '1',\n",
|
||
" 'beds': '2',\n",
|
||
" 'postcode': 'E16 1FF',\n",
|
||
" 'propertyType': 'Flats / Apartments',\n",
|
||
" 'propertySubType': 'Apartment',\n",
|
||
" 'added': '20240918',\n",
|
||
" 'price': '400000',\n",
|
||
" 'tenure': 'Leasehold',\n",
|
||
" 'bathrooms': '2',\n",
|
||
" 'sharedOwnership': 'F'},\n",
|
||
" 'displayPrices': [{'displayPrice': '£400,000',\n",
|
||
" 'displayPriceQualifier': 'Guide Price'}],\n",
|
||
" 'stations': [{'station': 'Canning Town Station',\n",
|
||
" 'distance': 0.2,\n",
|
||
" 'type': '2,4'},\n",
|
||
" {'station': 'Royal Victoria Station', 'distance': 0.3, 'type': '4'},\n",
|
||
" {'station': 'Emirates Royal Docks Station', 'distance': 0.4, 'type': '7'}],\n",
|
||
" 'features': [{'featureDescription': 'Guide Price £400,000-£425,000'},\n",
|
||
" {'featureDescription': '2 Bedroom /2 /Bathroom'},\n",
|
||
" {'featureDescription': 'Allocated Parking'},\n",
|
||
" {'featureDescription': 'Private Balcony'},\n",
|
||
" {'featureDescription': 'Concierge'},\n",
|
||
" {'featureDescription': 'Courtyard Gardens'},\n",
|
||
" {'featureDescription': '0.2mi to Canning Town Station'}],\n",
|
||
" 'photos': [{'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_00_0000_max_656x437.jpeg',\n",
|
||
" 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_00_0000_max_135x100.jpeg',\n",
|
||
" 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_00_0000.jpeg',\n",
|
||
" 'caption': 'Picture No. 33',\n",
|
||
" 'order': 0},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_01_0000_max_656x437.jpeg',\n",
|
||
" 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_01_0000_max_135x100.jpeg',\n",
|
||
" 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_01_0000.jpeg',\n",
|
||
" 'caption': 'Picture No. 35',\n",
|
||
" 'order': 1},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_03_0000_max_656x437.jpeg',\n",
|
||
" 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_03_0000_max_135x100.jpeg',\n",
|
||
" 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_03_0000.jpeg',\n",
|
||
" 'caption': 'Picture No. 34',\n",
|
||
" 'order': 2},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_02_0000_max_656x437.jpeg',\n",
|
||
" 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_02_0000_max_135x100.jpeg',\n",
|
||
" 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_02_0000.jpeg',\n",
|
||
" 'caption': 'Picture No. 36',\n",
|
||
" 'order': 3},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_04_0000_max_656x437.jpeg',\n",
|
||
" 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_04_0000_max_135x100.jpeg',\n",
|
||
" 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_04_0000.jpeg',\n",
|
||
" 'caption': 'Picture No. 39',\n",
|
||
" 'order': 4},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_05_0000_max_656x437.jpeg',\n",
|
||
" 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_05_0000_max_135x100.jpeg',\n",
|
||
" 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_05_0000.jpeg',\n",
|
||
" 'caption': 'Picture No. 29',\n",
|
||
" 'order': 5},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_06_0000_max_656x437.jpeg',\n",
|
||
" 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_06_0000_max_135x100.jpeg',\n",
|
||
" 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_06_0000.jpeg',\n",
|
||
" 'caption': 'Picture No. 30',\n",
|
||
" 'order': 6},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_07_0000_max_656x437.jpeg',\n",
|
||
" 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_07_0000_max_135x100.jpeg',\n",
|
||
" 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_07_0000.jpeg',\n",
|
||
" 'caption': 'Picture No. 40',\n",
|
||
" 'order': 7},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_08_0000_max_656x437.jpeg',\n",
|
||
" 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_08_0000_max_135x100.jpeg',\n",
|
||
" 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_08_0000.jpeg',\n",
|
||
" 'caption': 'Picture No. 31',\n",
|
||
" 'order': 8},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_09_0000_max_656x437.jpeg',\n",
|
||
" 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_09_0000_max_135x100.jpeg',\n",
|
||
" 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_09_0000.jpeg',\n",
|
||
" 'caption': 'Picture No. 41',\n",
|
||
" 'order': 9},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_10_0000_max_656x437.jpeg',\n",
|
||
" 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_10_0000_max_135x100.jpeg',\n",
|
||
" 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_10_0000.jpeg',\n",
|
||
" 'caption': 'Picture No. 32',\n",
|
||
" 'order': 10},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_11_0000_max_656x437.jpeg',\n",
|
||
" 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_IMG_11_0000_max_135x100.jpeg',\n",
|
||
" 'maxSizeUrl': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_IMG_11_0000.jpeg',\n",
|
||
" 'caption': 'Picture No. 44',\n",
|
||
" 'order': 11}],\n",
|
||
" 'virtualTours': [],\n",
|
||
" 'epcs': [{'url': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_EPCGRAPH_00_0000.png',\n",
|
||
" 'caption': 'EPC Rating Graph',\n",
|
||
" 'order': 0}],\n",
|
||
" 'floorplans': [{'order': 0,\n",
|
||
" 'url': 'https://media.rightmove.co.uk/201k/200531/152614085/200531_EAS180199_FLP_00_0000.jpeg',\n",
|
||
" 'thumbnailUrl': 'https://media.rightmove.co.uk/dir/201k/200531/152614085/200531_EAS180199_FLP_00_0000_max_296x197.jpeg',\n",
|
||
" 'caption': 'Floorplan'}],\n",
|
||
" 'note': None,\n",
|
||
" 'affordableBuyingScheme': False,\n",
|
||
" 'lettingsInfo': {'title': '', 'content': []},\n",
|
||
" 'propertyDetailsInfo': {'title': '',\n",
|
||
" 'content': [{'type': 'propertyType',\n",
|
||
" 'title': 'Property type',\n",
|
||
" 'value': 'Apartment'},\n",
|
||
" {'type': 'bedrooms', 'title': 'Bedrooms', 'value': '2'},\n",
|
||
" {'type': 'bathrooms', 'title': 'Bathrooms', 'value': '2'},\n",
|
||
" {'type': 'tenure', 'title': 'Tenure', 'value': 'Leasehold'}]},\n",
|
||
" 'tenureInfo': {'title': 'Leasehold',\n",
|
||
" 'content': [{'type': 'groundRent',\n",
|
||
" 'title': 'Ground rent',\n",
|
||
" 'value': '£300 per year\\n(Ask agent about the review period)'},\n",
|
||
" {'type': 'annualServiceCharge',\n",
|
||
" 'title': 'Service charge',\n",
|
||
" 'value': '£3,500 per year'},\n",
|
||
" {'type': 'lengthOfLease',\n",
|
||
" 'title': 'Length of lease',\n",
|
||
" 'value': '133 years left'}]},\n",
|
||
" 'sharedOwnershipInfo': {'title': '', 'content': []},\n",
|
||
" 'councilTaxInfo': {'title': 'Council tax',\n",
|
||
" 'content': [{'type': 'councilTaxBand',\n",
|
||
" 'title': 'Council tax band',\n",
|
||
" 'value': 'C'}]},\n",
|
||
" 'domesticRatesInfo': {'title': '', 'content': []},\n",
|
||
" 'linkToGlossary': 'https://www.rightmove.co.uk/guides/property-details-glossary/',\n",
|
||
" 'enquiredTimestamp': None,\n",
|
||
" 'stampDutyCalculator': {'country': 'ENGLAND',\n",
|
||
" 'price': 400000,\n",
|
||
" 'buyerType': None,\n",
|
||
" 'result': None}}}"
|
||
]
|
||
},
|
||
"execution_count": 27,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"response.json()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "98f8e950-2a3b-4856-aa62-3bc758e2fd42",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Find out the proper radius we want to use"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 18,
|
||
"id": "04bb61d5-cba7-4739-9568-b00342c1b636",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Filtered listings from 86437 to 46209\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"from data_access import Listing\n",
|
||
"from geopy.distance import geodesic\n",
|
||
"\n",
|
||
"listings = Listing.get_all_listings()\n",
|
||
"BROCK_STREET_LAT_LONG = 51.52570434674584, -0.13956495005056113\n",
|
||
"\n",
|
||
"# reduce listings to everything within 7 miles\n",
|
||
"filtered_listings = []\n",
|
||
"for listing in listings:\n",
|
||
" miles = geodesic(BROCK_STREET_LAT_LONG, (listing.latitude, listing.longitude)).miles\n",
|
||
" if miles <= 7:\n",
|
||
" filtered_listings.append(listing)\n",
|
||
"\n",
|
||
"print(f\"Filtered listings from {len(listings)} to {len(filtered_listings)}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "a73fba2d-afeb-4194-8421-eff8e84a14e9",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Typeahead / fetch all boroughs"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 20,
|
||
"id": "61844fe2-408d-4b89-995f-c31110a850f6",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"{'key': 'CAMD',\n",
|
||
" 'term': 'CAMD',\n",
|
||
" 'typeAheadLocations': [{'displayName': 'Camden, North West London',\n",
|
||
" 'locationIdentifier': 'REGION^85261',\n",
|
||
" 'normalisedSearchTerm': 'CAMDEN NORTH WEST LONDON'},\n",
|
||
" {'displayName': 'Camden (London Borough)',\n",
|
||
" 'locationIdentifier': 'REGION^93941',\n",
|
||
" 'normalisedSearchTerm': 'CAMDEN LONDON BOROUGH'},\n",
|
||
" {'displayName': 'Camden Town, North West London',\n",
|
||
" 'locationIdentifier': 'REGION^85262',\n",
|
||
" 'normalisedSearchTerm': 'CAMDEN TOWN NORTH WEST LONDON'},\n",
|
||
" {'displayName': 'Camden Town Station',\n",
|
||
" 'locationIdentifier': 'STATION^1712',\n",
|
||
" 'normalisedSearchTerm': 'CAMDEN TOWN STATION'},\n",
|
||
" {'displayName': 'Camden Road Station',\n",
|
||
" 'locationIdentifier': 'STATION^1709',\n",
|
||
" 'normalisedSearchTerm': 'CAMDEN ROAD STATION'},\n",
|
||
" {'displayName': 'Camden Town, Gosport, Hampshire',\n",
|
||
" 'locationIdentifier': 'REGION^76577',\n",
|
||
" 'normalisedSearchTerm': 'CAMDEN TOWN GOSPORT HAMPSHIRE'},\n",
|
||
" {'displayName': 'Camderry, Omagh, County Tyrone, Northern Ireland',\n",
|
||
" 'locationIdentifier': 'REGION^73327',\n",
|
||
" 'normalisedSearchTerm': 'CAMDERRY OMAGH COUNTY TYRONE NORTHERN IRELAND'}],\n",
|
||
" 'isComplete': True}"
|
||
]
|
||
},
|
||
"execution_count": 20,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"import requests\n",
|
||
"\n",
|
||
"cookies = {\n",
|
||
" 'permuserid': '240330LSEXEOANG04Q2VA3OZCIQ8TTSQ',\n",
|
||
" 'TS019c0ed0': '012f990cd3494097746bc0b10b8d61bc6237319024e80701e0e8d735bd7d62a792529823c3164c771f223b0cade5ec9ae4b7fc001c',\n",
|
||
" 'beta_optin': 'N:36:-1',\n",
|
||
" 'RM_Register': 'C',\n",
|
||
" 'JSESSIONID': '0BE8E261D81387C9BC530DB1A5F28955',\n",
|
||
" 'svr': '3111',\n",
|
||
" 'permuserid': '240330LSEXEOANG04Q2VA3OZCIQ8TTSQ',\n",
|
||
" 'TS01ec61d1': '012f990cd3161dd68e4ed69b9d64f7d4de2356c651edef96201cc7facd11b28ef9338596a2022bc2ed7a56f09c5dee5aa1711de2d1',\n",
|
||
" 'rmsessionid': '7bc54ce6-da97-42cf-8719-4e3e9c53e276',\n",
|
||
" 'TS01821201': '012f990cd35255a563a541cfe06e4a774c129628165af71838ccdb7a17919672962514804459a0d9d9a90fe7b8feeec66145e30b98',\n",
|
||
" 'TS01826437': '012f990cd3161dd68e4ed69b9d64f7d4de2356c651edef96201cc7facd11b28ef9338596a2022bc2ed7a56f09c5dee5aa1711de2d1',\n",
|
||
" 'TPCmaxPrice': '800000',\n",
|
||
" 'TS01a07bd2': '012f990cd3161dd68e4ed69b9d64f7d4de2356c651edef96201cc7facd11b28ef9338596a2022bc2ed7a56f09c5dee5aa1711de2d1',\n",
|
||
" 'TPCminPrice': '700000',\n",
|
||
"}\n",
|
||
"\n",
|
||
"headers = {\n",
|
||
" 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0',\n",
|
||
" 'Accept': 'application/json, text/javascript',\n",
|
||
" 'Accept-Language': 'en-GB,en;q=0.5',\n",
|
||
" # 'Accept-Encoding': 'gzip, deflate, br',\n",
|
||
" 'Referer': 'https://www.rightmove.co.uk/property-for-sale/find.html?locationIdentifier=REGION%5E87515&maxBedrooms=3&minBedrooms=1&maxPrice=800000&minPrice=750001&propertyTypes=&includeSSTC=false&mustHave=&dontShow=retirement%2CsharedOwnership&furnishTypes=&keywords=',\n",
|
||
" 'X-Correlation-Text-Val': 'source=search',\n",
|
||
" 'Content-Type': 'application/x-www-form-urlencoded',\n",
|
||
" 'traceparent': '00-d3020142d839bf1ef2b172fa596acea8-605693e1c4c5cbb5-00',\n",
|
||
" 'DNT': '1',\n",
|
||
" 'Sec-GPC': '1',\n",
|
||
" 'Connection': 'keep-alive',\n",
|
||
" # 'Cookie': 'permuserid=240330LSEXEOANG04Q2VA3OZCIQ8TTSQ; TS019c0ed0=012f990cd3494097746bc0b10b8d61bc6237319024e80701e0e8d735bd7d62a792529823c3164c771f223b0cade5ec9ae4b7fc001c; beta_optin=N:36:-1; RM_Register=C; JSESSIONID=0BE8E261D81387C9BC530DB1A5F28955; svr=3111; permuserid=240330LSEXEOANG04Q2VA3OZCIQ8TTSQ; TS01ec61d1=012f990cd3161dd68e4ed69b9d64f7d4de2356c651edef96201cc7facd11b28ef9338596a2022bc2ed7a56f09c5dee5aa1711de2d1; rmsessionid=7bc54ce6-da97-42cf-8719-4e3e9c53e276; TS01821201=012f990cd35255a563a541cfe06e4a774c129628165af71838ccdb7a17919672962514804459a0d9d9a90fe7b8feeec66145e30b98; TS01826437=012f990cd3161dd68e4ed69b9d64f7d4de2356c651edef96201cc7facd11b28ef9338596a2022bc2ed7a56f09c5dee5aa1711de2d1; TPCmaxPrice=800000; TS01a07bd2=012f990cd3161dd68e4ed69b9d64f7d4de2356c651edef96201cc7facd11b28ef9338596a2022bc2ed7a56f09c5dee5aa1711de2d1; TPCminPrice=700000',\n",
|
||
" 'Sec-Fetch-Dest': 'empty',\n",
|
||
" 'Sec-Fetch-Mode': 'cors',\n",
|
||
" 'Sec-Fetch-Site': 'same-origin',\n",
|
||
"}\n",
|
||
"\n",
|
||
"response = requests.get('https://www.rightmove.co.uk/typeAhead/uknostreet/CA/MD/EN/', cookies=cookies, headers=headers)\n",
|
||
"response.json()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 39,
|
||
"id": "eba907d0-680d-4374-a56e-95bd4abf0053",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import requests\n",
|
||
"from textwrap import wrap\n",
|
||
"\n",
|
||
"def query_loc_identifier(name: str)->str:\n",
|
||
" \"\"\"\n",
|
||
" \n",
|
||
" \"\"\"\n",
|
||
" name = name.upper()\n",
|
||
" name = '/'.join(wrap(name,2))\n",
|
||
" \n",
|
||
" headers = {\n",
|
||
" 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0',\n",
|
||
" }\n",
|
||
" \n",
|
||
" response = requests.get(f'https://www.rightmove.co.uk/typeAhead/uknostreet/{name}', headers=headers)\n",
|
||
" return response.json()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "d185e013-9beb-4e57-9d8b-a830e14339c3",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "cabe5d3f-ad0f-49c2-9fbf-686539a05bd1",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 30,
|
||
"id": "ba500fe6-fb18-466e-a697-403d28181674",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 46,
|
||
"id": "56c49b50-ff31-4785-9088-45ff5a39545e",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"{'Greenwich': 'REGION^61226', 'Hillingdon': 'REGION^93959', 'Ealing': 'REGION^93947', 'Richmond upon Thames': 'REGION^61415', 'Sutton': 'REGION^93974', 'Wandsworth': 'REGION^93977', 'Camden': 'REGION^93941', 'Enfield': 'REGION^93950', 'Croydon': 'REGION^93944', 'Hackney': 'REGION^93953', 'Kingston upon Thames': 'REGION^93968', 'Kensington and Chelsea': 'REGION^61229', 'Bromley': 'REGION^93938', 'Brent': 'REGION^93935', 'Waltham Forest': 'REGION^61232', 'Southwark': 'REGION^61518', 'Harrow': 'REGION^93956', 'Lewisham': 'REGION^61413', 'Barnet': 'REGION^93929', 'Islington': 'REGION^93965', 'Haringey': 'REGION^61227', 'Lambeth': 'REGION^93971', 'Westminster': '', 'Tower Hamlets': 'REGION^61417', 'Havering': 'REGION^61228', 'Barking and Dagenham': 'REGION^61400', 'Hammersmith and Fulham': 'REGION^61407', 'Bexley': 'REGION^93932', 'Redbridge': 'REGION^61537', 'Newham': 'REGION^61231', 'Merton': 'REGION^61414', 'Hounslow': 'REGION^93962'}\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"{\n",
|
||
" \"City of London\": \"REGION^61224\",\n",
|
||
" \"Greenwich\": \"REGION^61226\",\n",
|
||
" \"Hillingdon\": \"REGION^93959\",\n",
|
||
" \"Ealing\": \"REGION^93947\",\n",
|
||
" \"Richmond upon Thames\": \"REGION^61415\",\n",
|
||
" \"Sutton\": \"REGION^93974\",\n",
|
||
" \"Wandsworth\": \"REGION^93977\",\n",
|
||
" \"Camden\": \"REGION^93941\",\n",
|
||
" \"Enfield\": \"REGION^93950\",\n",
|
||
" \"Croydon\": \"REGION^93944\",\n",
|
||
" \"Hackney\": \"REGION^93953\",\n",
|
||
" \"Kingston upon Thames\": \"REGION^93968\",\n",
|
||
" \"Kensington and Chelsea\": \"REGION^61229\",\n",
|
||
" \"Bromley\": \"REGION^93938\",\n",
|
||
" \"Brent\": \"REGION^93935\",\n",
|
||
" \"Waltham Forest\": \"REGION^61232\",\n",
|
||
" \"Southwark\": \"REGION^61518\",\n",
|
||
" \"Harrow\": \"REGION^93956\",\n",
|
||
" \"Lewisham\": \"REGION^61413\",\n",
|
||
" \"Barnet\": \"REGION^93929\",\n",
|
||
" \"Islington\": \"REGION^93965\",\n",
|
||
" \"Haringey\": \"REGION^61227\",\n",
|
||
" \"Lambeth\": \"REGION^93971\",\n",
|
||
" \"Westminster\": \"REGION^93980\",\n",
|
||
" \"Tower Hamlets\": \"REGION^61417\",\n",
|
||
" \"Havering\": \"REGION^61228\",\n",
|
||
" \"Barking and Dagenham\": \"REGION^61400\",\n",
|
||
" \"Hammersmith and Fulham\": \"REGION^61407\",\n",
|
||
" \"Bexley\": \"REGION^93932\",\n",
|
||
" \"Redbridge\": \"REGION^61537\",\n",
|
||
" \"Newham\": \"REGION^61231\",\n",
|
||
" \"Merton\": \"REGION^61414\",\n",
|
||
" \"Hounslow\": \"REGION^93962\",\n",
|
||
"}\n",
|
||
"\n",
|
||
"bor_to_locid = {}\n",
|
||
"\n",
|
||
"for borough in boroughs:\n",
|
||
" bor_to_locid[borough] = ''\n",
|
||
" d = query_loc_identifier(borough)\n",
|
||
" locs = d['typeAheadLocations']\n",
|
||
" filtered = [l for l in locs if 'Borough' in l['displayName']]\n",
|
||
" if len(filtered)>1:\n",
|
||
" print(f\"{borough} has more entries: {len(filtered)}!\")\n",
|
||
" if filtered:\n",
|
||
" bor_to_locid[borough] = filtered[0]['locationIdentifier']\n",
|
||
"\n",
|
||
"Westminster (City of)\n",
|
||
"print(bor_to_locid)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "157e20a2-d137-45b8-802b-948fa8e04ba3",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "386578dc-1ad5-4b8a-8905-29b0c47a6174",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "10c17fdf-f424-40cb-9d8c-9218f8d4ab53",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "88d99eb7-8c92-4817-86ce-ba0738331dba",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "6c8b4488-ae2b-41ab-9c95-e3c85f9fb77e",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c3072907-7ad8-4618-92ab-818e392218d9",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "b20f6f16-3236-4772-b1a3-2d4a3b1925a6",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "1a7230dc-1a0f-43e2-bd15-0c85ea445733",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "aeaf84bf-8514-48c6-88ce-2c6828bdcdf2",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c888d4e6-d192-45df-b9b6-5e2d39bca344",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "cd71db7f-ba11-4d5d-a183-768ed4db23ba",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"/Users/kadir/code/realestate/crawler/venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
||
" from .autonotebook import tqdm as notebook_tqdm\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"from data_access import Listing"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 19,
|
||
"id": "d0ced84b-ee91-4642-b2ff-dd32d9f1e437",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"l = Listing(141932357)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 30,
|
||
"id": "7157f5f7-65b3-4232-bcae-26b93e5d93e6",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import json\n",
|
||
"with open(l.path_listing_json()) as f:\n",
|
||
" js = json.load(f)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 31,
|
||
"id": "3f453f9c-bdaa-4713-8220-c504f1a436ae",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"{'identifier': 141932357,\n",
|
||
" 'bedrooms': 1,\n",
|
||
" 'address': 'Moreton Street, Pimlico',\n",
|
||
" 'propertyType': 'Flat',\n",
|
||
" 'status': None,\n",
|
||
" 'transactionTypeId': 1,\n",
|
||
" 'photoCount': 8,\n",
|
||
" 'floorplanCount': 1,\n",
|
||
" 'price': 595000.0,\n",
|
||
" 'monthlyRent': None,\n",
|
||
" 'priceQualifier': '',\n",
|
||
" 'photoThumbnailUrl': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_00_0000_max_200x138.jpeg',\n",
|
||
" 'photoThumbnail2Url': None,\n",
|
||
" 'photoThumbnail3Url': None,\n",
|
||
" 'photoThumbnail4Url': None,\n",
|
||
" 'photoLargeThumbnailUrl': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_00_0000_max_656x437.jpeg',\n",
|
||
" 'displayPrices': [{'displayPrice': '£595,000', 'displayPriceQualifier': ''}],\n",
|
||
" 'thumbnailPhotos': [{'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_00_0000_max_656x437.jpeg'},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_01_0000_max_656x437.jpeg'},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_02_0000_max_656x437.jpeg'},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_03_0000_max_656x437.jpeg'},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_04_0000_max_656x437.jpeg'},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_05_0000_max_656x437.jpeg'},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_06_0000_max_656x437.jpeg'},\n",
|
||
" {'url': 'https://media.rightmove.co.uk/dir/152k/151016/141932357/151016_32721026_IMG_07_0000_max_656x437.jpeg'}],\n",
|
||
" 'keywords': [],\n",
|
||
" 'development': False,\n",
|
||
" 'autoEmailReasonType': 'price_reduced',\n",
|
||
" 'sortDate': 1718034274000,\n",
|
||
" 'onlineViewing': False,\n",
|
||
" 'buildToRent': False,\n",
|
||
" 'summary': 'Newly refurbished! Beautifully presented with courtyard garden and air conditioning. This one/two bedroom spacious apartment has a large open plan kitchen/reception room with two good size vaults, reception room and a bathroom.',\n",
|
||
" 'premiumDisplay': False,\n",
|
||
" 'latitude': 51.488754,\n",
|
||
" 'longitude': -0.137302,\n",
|
||
" 'showMap': True,\n",
|
||
" 'distance': None,\n",
|
||
" 'featuredProperty': False,\n",
|
||
" 'branch': {'identifier': 151016,\n",
|
||
" 'branchLogo': 'https://media.rightmove.co.uk:443/brand/brand_rmchoice_logo_39479_0013.png',\n",
|
||
" 'brandPlusResale': True,\n",
|
||
" 'brandPlusLettings': True,\n",
|
||
" 'brandName': 'Dexters',\n",
|
||
" 'development': False,\n",
|
||
" 'name': 'Westminster',\n",
|
||
" 'updateDate': 1718803632000,\n",
|
||
" 'hideReducedPropsFlag': False,\n",
|
||
" 'contactTelephoneNumber': '020 7590 9570'},\n",
|
||
" 'updateDate': 1718677927000,\n",
|
||
" 'listingUpdateReason': 'Reduced on 10/06/2024',\n",
|
||
" 'saved': False,\n",
|
||
" 'hidden': False,\n",
|
||
" 'premiumDisplayStickerName': '',\n",
|
||
" 'enquiredTimestamp': None,\n",
|
||
" 'visible': True,\n",
|
||
" 'hasVideoContent': False}"
|
||
]
|
||
},
|
||
"execution_count": 31,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"js"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "99d1a790-19d4-4686-b916-ee74fb3b2411",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "a4a2937c-1509-4c52-8aec-7bc86372092f",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 33,
|
||
"id": "7cd815ac-a4e6-4540-a286-829b0364a860",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"l = Listing(101369066)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 34,
|
||
"id": "eb8e2bd4-3e8d-4e81-b569-9b17b84ca45c",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"0"
|
||
]
|
||
},
|
||
"execution_count": 34,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"l.last_seen"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "a437a351-e3aa-458a-b692-3b630e598fa0",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 24,
|
||
"id": "90e94a48-f456-440b-b122-e58135b43614",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"'2025-01-26T21:37:07.580704'"
|
||
]
|
||
},
|
||
"execution_count": 24,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"import datetime\n",
|
||
"import json\n",
|
||
"datetime.datetime.now().isoformat()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 25,
|
||
"id": "a44b384c-4803-44cc-a6e4-528b169795e0",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"with open('/tmp/la.json', 'w') as f:\n",
|
||
" json.dump(datetime.datetime.now().isoformat(), f)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 26,
|
||
"id": "8b8bc05a-46a8-4f39-8b68-22ad62d13fe1",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\"2025-01-26T21:37:07.744971\""
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"!cat /tmp/la.json"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "341fe82c-fda8-4fc6-8bc0-081b285a5330",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3 (ipykernel)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.12.0"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|