viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

crawling for 3 and refactoring to allow incremental crawls

Browse source
This commit is contained in:
Kadir 2024-03-11 14:43:53 +00:00
parent de2639f9c3
commit 36258d877f
5 changed files with 310 additions and 167 deletions
Download patch file Download diff file Expand all files Collapse all files

13
crawler/1_dump_listings.py
View file

@ -1,22 +1,21 @@
from rec.query import listing_query
import pathlib
import json
from data_access import Listing
d = listing_query(1, 1, 2, 15, 0, 800000)
d = listing_query(1, 3, 3, 15, 0, 800000)
folder = pathlib.Path("data/rs/")
for i in range(1, 10000):
try:
print(f"page {i}")
d = listing_query(i, 1, 2, 15, 0, 800000)
d = listing_query(i, 3, 3, 15, 0, 800000)
except:
break
for property in d['properties']:
identifier = property['identifier']
listing_folder = folder / str(identifier)
listing_folder.mkdir(exist_ok=True, parents=True)
listing_path = listing_folder / f"listing.json"
with open(listing_path, 'w') as f:
listing = Listing(identifier)
with open(listing.path_listing_json(), 'w') as f:
json.dump(property, f)

28
crawler/2_dump_detail.py
View file

@ -1,25 +1,17 @@
import pathlib
import json
from rec.query import detail_query
from tqdm import tqdm
folder = pathlib.Path('data/rs/')
listings = folder.glob('*/listing.json')
from data_access import Listing
for listing_path in listings:
with open(listing_path) as f:
listing = json.load(f)
identifier = listing['identifier']
for listing in tqdm(Listing.get_all_listings()):
if listing.path_detail_json().exists():
continue
try:
d = detail_query(identifier)
d = detail_query(listing.identifier)
with open(listing.path_detail_json(), 'w') as f:
json.dump(d, f)
except:
print('Failed at: ', identifier)
print('Failed at: ', listing.identifier)
raise
print(identifier)
detail_path = pathlib.Path(f'data/rs/{identifier}/detail.json')
with open(detail_path, 'w') as f:
json.dump(d, f)

32
crawler/3_dump_images.py
View file

@ -2,41 +2,31 @@ import pathlib
import json
from urllib.request import urlretrieve
from tqdm import tqdm
from data_access import Listing
folder = pathlib.Path('data/rs/')
details = folder.glob('*/detail.json')
for detail_path in tqdm(list(details)):
with open(detail_path) as f:
for listing in tqdm(Listing.get_all_listings()):
with open(listing.path_detail_json()) as f:
detail = json.load(f)
identifier = detail['property']['identifier']
rsfolder = folder / str(identifier)
for photo in detail['property']['photos']:
url = photo['maxSizeUrl']
picname = url.split('/')[-1]
order = photo['order']
filename = f'{order}_{picname}'
fullpicpath = rsfolder / 'pics' / filename
if fullpicpath.exists():
p = listing.path_pic_file(order, picname)
if p.exists():
continue
fullpicpath.parent.mkdir(parents=True, exist_ok=True) # create the 'pics' folder
tqdm.write(str(fullpicpath))
urlretrieve(url, fullpicpath)
tqdm.write(str(p))
urlretrieve(url, p)
for photo in detail['property']['floorplans']:
url = photo['url']
picname = url.split('/')[-1]
order = photo['order']
filename = f'{order}_{picname}'
fullpicpath = rsfolder / 'floorplans' / filename
if fullpicpath.exists():
p = listing.path_floorplan_file(order, picname)
if p.exists():
continue
fullpicpath.parent.mkdir(parents=True, exist_ok=True) # create the 'floorplans' folder
tqdm.write(str(fullpicpath))
urlretrieve(url, fullpicpath)
tqdm.write(str(p))
urlretrieve(url, p)

22
crawler/data_access.py
View file

@ -10,6 +10,7 @@ _DATA_DIR = pathlib.Path('data/rs/')
class Listing():
identifier: int
_cached: Dict = None
@staticmethod
def get_all_listings() -> List['Listing']:
@ -23,7 +24,9 @@ class Listing():
return identifiers
def path_listing(self) -> pathlib.Path:
return _DATA_DIR / str(self.identifier)
p = _DATA_DIR / str(self.identifier)
p.mkdir(parents=True, exist_ok=True)
return p
def path_listing_json(self) -> pathlib.Path:
return self.path_listing() / 'listing.json'
@ -40,15 +43,16 @@ class Listing():
def path_pic_folder(self) -> pathlib.Path:
return self.path_listing() / 'pics'
def path_pic_folder(self, order, name) -> pathlib.Path:
def path_pic_file(self, order, name) -> pathlib.Path:
self.path_pic_folder().mkdir(parents=True, exist_ok=True)
return self.path_pic_folder() / f'{order}_{name}'
def path_floorplan_folder(self) -> pathlib.Path:
return self.path_listing() / 'floorplans'
def path_floorplan_file(self, order, name) -> pathlib.Path:
return self.path_pic_folder() / f'{order}_{name}'
self.path_floorplan_folder().mkdir(parents=True, exist_ok=True)
return self.path_floorplan_folder() / f'{order}_{name}'
def list_floorplans(self):
images = list(self.path_floorplan_folder().glob('*'))
@ -80,7 +84,10 @@ class Listing():
max_sqm = max([o['estimated_sqm'] for o in objs if o is None]) # filter out Nones
return max_sqm
def calculate_sqm_ocr(self):
def calculate_sqm_ocr(self, recalculate=True):
if not recalculate and self.path_floorplan_ocr_json().exists():
return
objs = []
for floorplan_path in self.list_floorplans():
estimated_sqm, model_output = floorplan.calculate_ocr(floorplan_path)
@ -129,12 +136,17 @@ class Listing():
return None
return self.price / self.sqm_ocr
@property
def bedrooms(self) -> int:
return self.detailobject['property']['bedrooms']
def dict_nicely(self):
return {
'sqm_ocr': self.sqm_ocr,
'price': self.price,
'price_per_sqm': self.price_per_sqm,
'url': self.url,
'bedrooms': self.bedrooms,
}

382
crawler/exploration.ipynb
View file

@ -71,6 +71,7 @@
" <th>price</th>\n",
" <th>price_per_sqm</th>\n",
" <th>url</th>\n",
" <th>bedrooms</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
@ -80,6 +81,7 @@
" <td>350000.0</td>\n",
" <td>44814.340589</td>\n",
" <td>https://www.rightmove.co.uk/properties/102360773</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
@ -87,6 +89,7 @@
" <td>695000.0</td>\n",
" <td>118197.278912</td>\n",
" <td>https://www.rightmove.co.uk/properties/115530848</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
@ -94,6 +97,7 @@
" <td>575000.0</td>\n",
" <td>8881.680568</td>\n",
" <td>https://www.rightmove.co.uk/properties/117095606</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
@ -101,6 +105,7 @@
" <td>790000.0</td>\n",
" <td>14962.121212</td>\n",
" <td>https://www.rightmove.co.uk/properties/118752221</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
@ -108,6 +113,7 @@
" <td>425000.0</td>\n",
" <td>93201.754386</td>\n",
" <td>https://www.rightmove.co.uk/properties/119578451</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
@ -115,45 +121,51 @@
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1020</th>\n",
" <th>1021</th>\n",
" <td>NaN</td>\n",
" <td>220000.0</td>\n",
" <td>NaN</td>\n",
" <td>https://www.rightmove.co.uk/properties/86809926</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1021</th>\n",
" <th>1022</th>\n",
" <td>49.00</td>\n",
" <td>450000.0</td>\n",
" <td>9183.673469</td>\n",
" <td>https://www.rightmove.co.uk/properties/86811141</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1022</th>\n",
" <th>1023</th>\n",
" <td>58.20</td>\n",
" <td>550000.0</td>\n",
" <td>9450.171821</td>\n",
" <td>https://www.rightmove.co.uk/properties/86811177</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1023</th>\n",
" <th>1024</th>\n",
" <td>3.00</td>\n",
" <td>475000.0</td>\n",
" <td>158333.333333</td>\n",
" <td>https://www.rightmove.co.uk/properties/86812494</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1024</th>\n",
" <th>1025</th>\n",
" <td>81.60</td>\n",
" <td>790000.0</td>\n",
" <td>9681.372549</td>\n",
" <td>https://www.rightmove.co.uk/properties/86972726</td>\n",
" <td>2</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>1025 rows × 4 columns</p>\n",
"<p>1026 rows × 5 columns</p>\n",
"</div>"
],
"text/plain": [
@ -164,26 +176,26 @@
"3 52.80 790000.0 14962.121212 \n",
"4 4.56 425000.0 93201.754386 \n",
"... ... ... ... \n",
"1020 NaN 220000.0 NaN \n",
"1021 49.00 450000.0 9183.673469 \n",
"1022 58.20 550000.0 9450.171821 \n",
"1023 3.00 475000.0 158333.333333 \n",
"1024 81.60 790000.0 9681.372549 \n",
"1021 NaN 220000.0 NaN \n",
"1022 49.00 450000.0 9183.673469 \n",
"1023 58.20 550000.0 9450.171821 \n",
"1024 3.00 475000.0 158333.333333 \n",
"1025 81.60 790000.0 9681.372549 \n",
"\n",
" url \n",
"0 https://www.rightmove.co.uk/properties/102360773 \n",
"1 https://www.rightmove.co.uk/properties/115530848 \n",
"2 https://www.rightmove.co.uk/properties/117095606 \n",
"3 https://www.rightmove.co.uk/properties/118752221 \n",
"4 https://www.rightmove.co.uk/properties/119578451 \n",
"... ... \n",
"1020 https://www.rightmove.co.uk/properties/86809926 \n",
"1021 https://www.rightmove.co.uk/properties/86811141 \n",
"1022 https://www.rightmove.co.uk/properties/86811177 \n",
"1023 https://www.rightmove.co.uk/properties/86812494 \n",
"1024 https://www.rightmove.co.uk/properties/86972726 \n",
" url bedrooms \n",
"0 https://www.rightmove.co.uk/properties/102360773 1 \n",
"1 https://www.rightmove.co.uk/properties/115530848 1 \n",
"2 https://www.rightmove.co.uk/properties/117095606 2 \n",
"3 https://www.rightmove.co.uk/properties/118752221 1 \n",
"4 https://www.rightmove.co.uk/properties/119578451 2 \n",
"... ... ... \n",
"1021 https://www.rightmove.co.uk/properties/86809926 1 \n",
"1022 https://www.rightmove.co.uk/properties/86811141 1 \n",
"1023 https://www.rightmove.co.uk/properties/86811177 2 \n",
"1024 https://www.rightmove.co.uk/properties/86812494 1 \n",
"1025 https://www.rightmove.co.uk/properties/86972726 2 \n",
"\n",
"[1025 rows x 4 columns]"
"[1026 rows x 5 columns]"
]
},
"execution_count": 4,
@ -198,7 +210,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 5,
"id": "99c5b304-3d13-466b-a9f5-83a5db6311b5",
"metadata": {},
"outputs": [
@ -227,6 +239,7 @@
" <th>price</th>\n",
" <th>price_per_sqm</th>\n",
" <th>url</th>\n",
" <th>bedrooms</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
@ -236,6 +249,7 @@
" <td>550000.0</td>\n",
" <td>7.418157</td>\n",
" <td>https://www.rightmove.co.uk/properties/145546538</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>249</th>\n",
@ -243,6 +257,7 @@
" <td>725000.0</td>\n",
" <td>33.180626</td>\n",
" <td>https://www.rightmove.co.uk/properties/140821736</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>177</th>\n",
@ -250,6 +265,7 @@
" <td>695000.0</td>\n",
" <td>5054.545455</td>\n",
" <td>https://www.rightmove.co.uk/properties/139354259</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>352</th>\n",
@ -257,6 +273,7 @@
" <td>475000.0</td>\n",
" <td>5555.555556</td>\n",
" <td>https://www.rightmove.co.uk/properties/142142348</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>36</th>\n",
@ -264,6 +281,7 @@
" <td>475000.0</td>\n",
" <td>5729.794934</td>\n",
" <td>https://www.rightmove.co.uk/properties/128925950</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>700</th>\n",
@ -271,6 +289,7 @@
" <td>650000.0</td>\n",
" <td>6435.643564</td>\n",
" <td>https://www.rightmove.co.uk/properties/144591572</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>823</th>\n",
@ -278,6 +297,7 @@
" <td>525000.0</td>\n",
" <td>6529.850746</td>\n",
" <td>https://www.rightmove.co.uk/properties/145051769</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>214</th>\n",
@ -285,6 +305,7 @@
" <td>650000.0</td>\n",
" <td>7103.048847</td>\n",
" <td>https://www.rightmove.co.uk/properties/140326307</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>171</th>\n",
@ -292,6 +313,7 @@
" <td>650000.0</td>\n",
" <td>7135.016465</td>\n",
" <td>https://www.rightmove.co.uk/properties/139245428</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>598</th>\n",
@ -299,6 +321,7 @@
" <td>795000.0</td>\n",
" <td>7266.910420</td>\n",
" <td>https://www.rightmove.co.uk/properties/144034655</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>146</th>\n",
@ -306,6 +329,7 @@
" <td>700000.0</td>\n",
" <td>7464.278098</td>\n",
" <td>https://www.rightmove.co.uk/properties/138510395</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>711</th>\n",
@ -313,6 +337,7 @@
" <td>750000.0</td>\n",
" <td>7476.821852</td>\n",
" <td>https://www.rightmove.co.uk/properties/144620303</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>592</th>\n",
@ -320,6 +345,7 @@
" <td>650000.0</td>\n",
" <td>7515.319690</td>\n",
" <td>https://www.rightmove.co.uk/properties/143987669</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>624</th>\n",
@ -327,6 +353,7 @@
" <td>775000.0</td>\n",
" <td>7646.768624</td>\n",
" <td>https://www.rightmove.co.uk/properties/144217922</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>55</th>\n",
@ -334,6 +361,7 @@
" <td>750000.0</td>\n",
" <td>7684.426230</td>\n",
" <td>https://www.rightmove.co.uk/properties/132564737</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>851</th>\n",
@ -341,6 +369,7 @@
" <td>625000.0</td>\n",
" <td>7715.096902</td>\n",
" <td>https://www.rightmove.co.uk/properties/145172504</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>356</th>\n",
@ -348,6 +377,7 @@
" <td>695000.0</td>\n",
" <td>7808.988764</td>\n",
" <td>https://www.rightmove.co.uk/properties/142185623</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>748</th>\n",
@ -355,6 +385,7 @@
" <td>675000.0</td>\n",
" <td>7857.974389</td>\n",
" <td>https://www.rightmove.co.uk/properties/144771281</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>204</th>\n",
@ -362,6 +393,7 @@
" <td>650000.0</td>\n",
" <td>7888.349515</td>\n",
" <td>https://www.rightmove.co.uk/properties/140173319</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>654</th>\n",
@ -369,6 +401,7 @@
" <td>665000.0</td>\n",
" <td>7980.319213</td>\n",
" <td>https://www.rightmove.co.uk/properties/144361100</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>514</th>\n",
@ -376,6 +409,7 @@
" <td>700000.0</td>\n",
" <td>8000.000000</td>\n",
" <td>https://www.rightmove.co.uk/properties/143460365</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>762</th>\n",
@ -383,6 +417,7 @@
" <td>750000.0</td>\n",
" <td>8073.196986</td>\n",
" <td>https://www.rightmove.co.uk/properties/144846725</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>963</th>\n",
@ -390,6 +425,7 @@
" <td>750000.0</td>\n",
" <td>8073.196986</td>\n",
" <td>https://www.rightmove.co.uk/properties/145565252</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>938</th>\n",
@ -397,6 +433,7 @@
" <td>699000.0</td>\n",
" <td>8118.466899</td>\n",
" <td>https://www.rightmove.co.uk/properties/145491137</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>403</th>\n",
@ -404,6 +441,7 @@
" <td>750000.0</td>\n",
" <td>8278.145695</td>\n",
" <td>https://www.rightmove.co.uk/properties/142704416</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>302</th>\n",
@ -411,6 +449,7 @@
" <td>795000.0</td>\n",
" <td>8520.900322</td>\n",
" <td>https://www.rightmove.co.uk/properties/141595433</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>671</th>\n",
@ -418,6 +457,7 @@
" <td>800000.0</td>\n",
" <td>8577.248847</td>\n",
" <td>https://www.rightmove.co.uk/properties/144429140</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>349</th>\n",
@ -425,6 +465,7 @@
" <td>695000.0</td>\n",
" <td>8626.039469</td>\n",
" <td>https://www.rightmove.co.uk/properties/142115918</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>224</th>\n",
@ -432,6 +473,7 @@
" <td>800000.0</td>\n",
" <td>8879.023307</td>\n",
" <td>https://www.rightmove.co.uk/properties/140464481</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>740</th>\n",
@ -439,6 +481,7 @@
" <td>735000.0</td>\n",
" <td>8996.328029</td>\n",
" <td>https://www.rightmove.co.uk/properties/144722414</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>326</th>\n",
@ -446,6 +489,7 @@
" <td>800000.0</td>\n",
" <td>9163.802978</td>\n",
" <td>https://www.rightmove.co.uk/properties/141846023</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>319</th>\n",
@ -453,6 +497,7 @@
" <td>750000.0</td>\n",
" <td>9194.556822</td>\n",
" <td>https://www.rightmove.co.uk/properties/141797357</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>558</th>\n",
@ -460,6 +505,7 @@
" <td>800000.0</td>\n",
" <td>9219.776420</td>\n",
" <td>https://www.rightmove.co.uk/properties/143758763</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>712</th>\n",
@ -467,6 +513,7 @@
" <td>800000.0</td>\n",
" <td>9269.988413</td>\n",
" <td>https://www.rightmove.co.uk/properties/144622157</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>473</th>\n",
@ -474,6 +521,7 @@
" <td>800000.0</td>\n",
" <td>9302.325581</td>\n",
" <td>https://www.rightmove.co.uk/properties/143210102</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>105</th>\n",
@ -481,13 +529,15 @@
" <td>750000.0</td>\n",
" <td>9328.358209</td>\n",
" <td>https://www.rightmove.co.uk/properties/136988726</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1002</th>\n",
" <th>1003</th>\n",
" <td>80.27</td>\n",
" <td>750000.0</td>\n",
" <td>9343.465803</td>\n",
" <td>https://www.rightmove.co.uk/properties/86775291</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>235</th>\n",
@ -495,6 +545,7 @@
" <td>775000.0</td>\n",
" <td>9359.903382</td>\n",
" <td>https://www.rightmove.co.uk/properties/140611055</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>65</th>\n",
@ -502,6 +553,7 @@
" <td>800000.0</td>\n",
" <td>9400.705053</td>\n",
" <td>https://www.rightmove.co.uk/properties/134116232</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>30</th>\n",
@ -509,13 +561,15 @@
" <td>795000.0</td>\n",
" <td>9498.207885</td>\n",
" <td>https://www.rightmove.co.uk/properties/127787960</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1024</th>\n",
" <th>1025</th>\n",
" <td>81.60</td>\n",
" <td>790000.0</td>\n",
" <td>9681.372549</td>\n",
" <td>https://www.rightmove.co.uk/properties/86972726</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>88</th>\n",
@ -523,6 +577,7 @@
" <td>800000.0</td>\n",
" <td>9785.932722</td>\n",
" <td>https://www.rightmove.co.uk/properties/136012193</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>454</th>\n",
@ -530,6 +585,7 @@
" <td>800000.0</td>\n",
" <td>9900.990099</td>\n",
" <td>https://www.rightmove.co.uk/properties/143138867</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>343</th>\n",
@ -537,6 +593,7 @@
" <td>800000.0</td>\n",
" <td>9920.634921</td>\n",
" <td>https://www.rightmove.co.uk/properties/142032935</td>\n",
" <td>2</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
@ -580,63 +637,63 @@
"712 86.30 800000.0 9269.988413 \n",
"473 86.00 800000.0 9302.325581 \n",
"105 80.40 750000.0 9328.358209 \n",
"1002 80.27 750000.0 9343.465803 \n",
"1003 80.27 750000.0 9343.465803 \n",
"235 82.80 775000.0 9359.903382 \n",
"65 85.10 800000.0 9400.705053 \n",
"30 83.70 795000.0 9498.207885 \n",
"1024 81.60 790000.0 9681.372549 \n",
"1025 81.60 790000.0 9681.372549 \n",
"88 81.75 800000.0 9785.932722 \n",
"454 80.80 800000.0 9900.990099 \n",
"343 80.64 800000.0 9920.634921 \n",
"\n",
" url \n",
"953 https://www.rightmove.co.uk/properties/145546538 \n",
"249 https://www.rightmove.co.uk/properties/140821736 \n",
"177 https://www.rightmove.co.uk/properties/139354259 \n",
"352 https://www.rightmove.co.uk/properties/142142348 \n",
"36 https://www.rightmove.co.uk/properties/128925950 \n",
"700 https://www.rightmove.co.uk/properties/144591572 \n",
"823 https://www.rightmove.co.uk/properties/145051769 \n",
"214 https://www.rightmove.co.uk/properties/140326307 \n",
"171 https://www.rightmove.co.uk/properties/139245428 \n",
"598 https://www.rightmove.co.uk/properties/144034655 \n",
"146 https://www.rightmove.co.uk/properties/138510395 \n",
"711 https://www.rightmove.co.uk/properties/144620303 \n",
"592 https://www.rightmove.co.uk/properties/143987669 \n",
"624 https://www.rightmove.co.uk/properties/144217922 \n",
"55 https://www.rightmove.co.uk/properties/132564737 \n",
"851 https://www.rightmove.co.uk/properties/145172504 \n",
"356 https://www.rightmove.co.uk/properties/142185623 \n",
"748 https://www.rightmove.co.uk/properties/144771281 \n",
"204 https://www.rightmove.co.uk/properties/140173319 \n",
"654 https://www.rightmove.co.uk/properties/144361100 \n",
"514 https://www.rightmove.co.uk/properties/143460365 \n",
"762 https://www.rightmove.co.uk/properties/144846725 \n",
"963 https://www.rightmove.co.uk/properties/145565252 \n",
"938 https://www.rightmove.co.uk/properties/145491137 \n",
"403 https://www.rightmove.co.uk/properties/142704416 \n",
"302 https://www.rightmove.co.uk/properties/141595433 \n",
"671 https://www.rightmove.co.uk/properties/144429140 \n",
"349 https://www.rightmove.co.uk/properties/142115918 \n",
"224 https://www.rightmove.co.uk/properties/140464481 \n",
"740 https://www.rightmove.co.uk/properties/144722414 \n",
"326 https://www.rightmove.co.uk/properties/141846023 \n",
"319 https://www.rightmove.co.uk/properties/141797357 \n",
"558 https://www.rightmove.co.uk/properties/143758763 \n",
"712 https://www.rightmove.co.uk/properties/144622157 \n",
"473 https://www.rightmove.co.uk/properties/143210102 \n",
"105 https://www.rightmove.co.uk/properties/136988726 \n",
"1002 https://www.rightmove.co.uk/properties/86775291 \n",
"235 https://www.rightmove.co.uk/properties/140611055 \n",
"65 https://www.rightmove.co.uk/properties/134116232 \n",
"30 https://www.rightmove.co.uk/properties/127787960 \n",
"1024 https://www.rightmove.co.uk/properties/86972726 \n",
"88 https://www.rightmove.co.uk/properties/136012193 \n",
"454 https://www.rightmove.co.uk/properties/143138867 \n",
"343 https://www.rightmove.co.uk/properties/142032935 "
" url bedrooms \n",
"953 https://www.rightmove.co.uk/properties/145546538 2 \n",
"249 https://www.rightmove.co.uk/properties/140821736 2 \n",
"177 https://www.rightmove.co.uk/properties/139354259 2 \n",
"352 https://www.rightmove.co.uk/properties/142142348 2 \n",
"36 https://www.rightmove.co.uk/properties/128925950 2 \n",
"700 https://www.rightmove.co.uk/properties/144591572 1 \n",
"823 https://www.rightmove.co.uk/properties/145051769 2 \n",
"214 https://www.rightmove.co.uk/properties/140326307 2 \n",
"171 https://www.rightmove.co.uk/properties/139245428 2 \n",
"598 https://www.rightmove.co.uk/properties/144034655 2 \n",
"146 https://www.rightmove.co.uk/properties/138510395 2 \n",
"711 https://www.rightmove.co.uk/properties/144620303 2 \n",
"592 https://www.rightmove.co.uk/properties/143987669 2 \n",
"624 https://www.rightmove.co.uk/properties/144217922 1 \n",
"55 https://www.rightmove.co.uk/properties/132564737 2 \n",
"851 https://www.rightmove.co.uk/properties/145172504 2 \n",
"356 https://www.rightmove.co.uk/properties/142185623 2 \n",
"748 https://www.rightmove.co.uk/properties/144771281 2 \n",
"204 https://www.rightmove.co.uk/properties/140173319 2 \n",
"654 https://www.rightmove.co.uk/properties/144361100 2 \n",
"514 https://www.rightmove.co.uk/properties/143460365 1 \n",
"762 https://www.rightmove.co.uk/properties/144846725 1 \n",
"963 https://www.rightmove.co.uk/properties/145565252 2 \n",
"938 https://www.rightmove.co.uk/properties/145491137 2 \n",
"403 https://www.rightmove.co.uk/properties/142704416 2 \n",
"302 https://www.rightmove.co.uk/properties/141595433 2 \n",
"671 https://www.rightmove.co.uk/properties/144429140 2 \n",
"349 https://www.rightmove.co.uk/properties/142115918 2 \n",
"224 https://www.rightmove.co.uk/properties/140464481 2 \n",
"740 https://www.rightmove.co.uk/properties/144722414 2 \n",
"326 https://www.rightmove.co.uk/properties/141846023 2 \n",
"319 https://www.rightmove.co.uk/properties/141797357 2 \n",
"558 https://www.rightmove.co.uk/properties/143758763 2 \n",
"712 https://www.rightmove.co.uk/properties/144622157 2 \n",
"473 https://www.rightmove.co.uk/properties/143210102 2 \n",
"105 https://www.rightmove.co.uk/properties/136988726 2 \n",
"1003 https://www.rightmove.co.uk/properties/86775291 2 \n",
"235 https://www.rightmove.co.uk/properties/140611055 2 \n",
"65 https://www.rightmove.co.uk/properties/134116232 2 \n",
"30 https://www.rightmove.co.uk/properties/127787960 1 \n",
"1025 https://www.rightmove.co.uk/properties/86972726 2 \n",
"88 https://www.rightmove.co.uk/properties/136012193 1 \n",
"454 https://www.rightmove.co.uk/properties/143138867 2 \n",
"343 https://www.rightmove.co.uk/properties/142032935 2 "
]
},
"execution_count": 7,
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
@ -648,7 +705,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 6,
"id": "52545cfa-0932-46fe-ba7e-961fd43f2786",
"metadata": {},
"outputs": [
@ -677,6 +734,7 @@
" <th>price</th>\n",
" <th>price_per_sqm</th>\n",
" <th>url</th>\n",
" <th>bedrooms</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
@ -686,6 +744,7 @@
" <td>400000.0</td>\n",
" <td>5449.591281</td>\n",
" <td>https://www.rightmove.co.uk/properties/142186991</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>293</th>\n",
@ -693,6 +752,7 @@
" <td>425000.0</td>\n",
" <td>5519.480519</td>\n",
" <td>https://www.rightmove.co.uk/properties/141437783</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>352</th>\n",
@ -700,6 +760,7 @@
" <td>475000.0</td>\n",
" <td>5555.555556</td>\n",
" <td>https://www.rightmove.co.uk/properties/142142348</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>685</th>\n",
@ -707,6 +768,7 @@
" <td>425000.0</td>\n",
" <td>5589.898724</td>\n",
" <td>https://www.rightmove.co.uk/properties/144494012</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>36</th>\n",
@ -714,6 +776,7 @@
" <td>475000.0</td>\n",
" <td>5729.794934</td>\n",
" <td>https://www.rightmove.co.uk/properties/128925950</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>491</th>\n",
@ -721,13 +784,15 @@
" <td>450000.0</td>\n",
" <td>6277.901786</td>\n",
" <td>https://www.rightmove.co.uk/properties/143315840</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1019</th>\n",
" <th>1020</th>\n",
" <td>73.67</td>\n",
" <td>495000.0</td>\n",
" <td>6719.152980</td>\n",
" <td>https://www.rightmove.co.uk/properties/86807916</td>\n",
" <td>2</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
@ -741,19 +806,19 @@
"685 76.03 425000.0 5589.898724 \n",
"36 82.90 475000.0 5729.794934 \n",
"491 71.68 450000.0 6277.901786 \n",
"1019 73.67 495000.0 6719.152980 \n",
"1020 73.67 495000.0 6719.152980 \n",
"\n",
" url \n",
"359 https://www.rightmove.co.uk/properties/142186991 \n",
"293 https://www.rightmove.co.uk/properties/141437783 \n",
"352 https://www.rightmove.co.uk/properties/142142348 \n",
"685 https://www.rightmove.co.uk/properties/144494012 \n",
"36 https://www.rightmove.co.uk/properties/128925950 \n",
"491 https://www.rightmove.co.uk/properties/143315840 \n",
"1019 https://www.rightmove.co.uk/properties/86807916 "
" url bedrooms \n",
"359 https://www.rightmove.co.uk/properties/142186991 2 \n",
"293 https://www.rightmove.co.uk/properties/141437783 2 \n",
"352 https://www.rightmove.co.uk/properties/142142348 2 \n",
"685 https://www.rightmove.co.uk/properties/144494012 2 \n",
"36 https://www.rightmove.co.uk/properties/128925950 2 \n",
"491 https://www.rightmove.co.uk/properties/143315840 2 \n",
"1020 https://www.rightmove.co.uk/properties/86807916 2 "
]
},
"execution_count": 8,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
@ -765,7 +830,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 7,
"id": "d0246926-13ef-4110-8e3a-fb676a55c2a6",
"metadata": {},
"outputs": [
@ -794,6 +859,7 @@
" <th>price</th>\n",
" <th>price_per_sqm</th>\n",
" <th>url</th>\n",
" <th>bedrooms</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
@ -803,6 +869,7 @@
" <td>550000.0</td>\n",
" <td>7.418157</td>\n",
" <td>https://www.rightmove.co.uk/properties/145546538</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>823</th>\n",
@ -810,6 +877,7 @@
" <td>525000.0</td>\n",
" <td>6529.850746</td>\n",
" <td>https://www.rightmove.co.uk/properties/145051769</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>492</th>\n",
@ -817,6 +885,7 @@
" <td>525000.0</td>\n",
" <td>7104.194858</td>\n",
" <td>https://www.rightmove.co.uk/properties/143317361</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>561</th>\n",
@ -824,6 +893,7 @@
" <td>550000.0</td>\n",
" <td>7170.795306</td>\n",
" <td>https://www.rightmove.co.uk/properties/143780789</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>69</th>\n",
@ -831,13 +901,15 @@
" <td>525000.0</td>\n",
" <td>7181.942544</td>\n",
" <td>https://www.rightmove.co.uk/properties/134574563</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>987</th>\n",
" <th>988</th>\n",
" <td>72.40</td>\n",
" <td>525000.0</td>\n",
" <td>7251.381215</td>\n",
" <td>https://www.rightmove.co.uk/properties/86648925</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>272</th>\n",
@ -845,6 +917,7 @@
" <td>575000.0</td>\n",
" <td>7382.205675</td>\n",
" <td>https://www.rightmove.co.uk/properties/141131297</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>521</th>\n",
@ -852,6 +925,7 @@
" <td>525000.0</td>\n",
" <td>7406.884876</td>\n",
" <td>https://www.rightmove.co.uk/properties/143514149</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>324</th>\n",
@ -859,13 +933,15 @@
" <td>575000.0</td>\n",
" <td>7555.847569</td>\n",
" <td>https://www.rightmove.co.uk/properties/141831353</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1017</th>\n",
" <th>1018</th>\n",
" <td>71.80</td>\n",
" <td>550000.0</td>\n",
" <td>7660.167131</td>\n",
" <td>https://www.rightmove.co.uk/properties/86804832</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>526</th>\n",
@ -873,6 +949,7 @@
" <td>600000.0</td>\n",
" <td>7692.307692</td>\n",
" <td>https://www.rightmove.co.uk/properties/143552156</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>817</th>\n",
@ -880,6 +957,7 @@
" <td>550000.0</td>\n",
" <td>7727.975270</td>\n",
" <td>https://www.rightmove.co.uk/properties/145035929</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>843</th>\n",
@ -887,6 +965,7 @@
" <td>600000.0</td>\n",
" <td>7741.935484</td>\n",
" <td>https://www.rightmove.co.uk/properties/145144988</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>337</th>\n",
@ -894,6 +973,7 @@
" <td>550000.0</td>\n",
" <td>7790.368272</td>\n",
" <td>https://www.rightmove.co.uk/properties/141904286</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>233</th>\n",
@ -901,6 +981,7 @@
" <td>600000.0</td>\n",
" <td>7926.023778</td>\n",
" <td>https://www.rightmove.co.uk/properties/140582213</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>763</th>\n",
@ -908,6 +989,7 @@
" <td>600000.0</td>\n",
" <td>8000.000000</td>\n",
" <td>https://www.rightmove.co.uk/properties/144862070</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>315</th>\n",
@ -915,6 +997,7 @@
" <td>590000.0</td>\n",
" <td>8274.894811</td>\n",
" <td>https://www.rightmove.co.uk/properties/141669686</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>899</th>\n",
@ -922,6 +1005,7 @@
" <td>595000.0</td>\n",
" <td>8325.171401</td>\n",
" <td>https://www.rightmove.co.uk/properties/145362911</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>147</th>\n",
@ -929,13 +1013,15 @@
" <td>600000.0</td>\n",
" <td>8391.608392</td>\n",
" <td>https://www.rightmove.co.uk/properties/138537527</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>972</th>\n",
" <th>973</th>\n",
" <td>70.89</td>\n",
" <td>595000.0</td>\n",
" <td>8393.285372</td>\n",
" <td>https://www.rightmove.co.uk/properties/86296491</td>\n",
" <td>2</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
@ -948,11 +1034,11 @@
"492 73.90 525000.0 7104.194858 \n",
"561 76.70 550000.0 7170.795306 \n",
"69 73.10 525000.0 7181.942544 \n",
"987 72.40 525000.0 7251.381215 \n",
"988 72.40 525000.0 7251.381215 \n",
"272 77.89 575000.0 7382.205675 \n",
"521 70.88 525000.0 7406.884876 \n",
"324 76.10 575000.0 7555.847569 \n",
"1017 71.80 550000.0 7660.167131 \n",
"1018 71.80 550000.0 7660.167131 \n",
"526 78.00 600000.0 7692.307692 \n",
"817 71.17 550000.0 7727.975270 \n",
"843 77.50 600000.0 7741.935484 \n",
@ -962,32 +1048,32 @@
"315 71.30 590000.0 8274.894811 \n",
"899 71.47 595000.0 8325.171401 \n",
"147 71.50 600000.0 8391.608392 \n",
"972 70.89 595000.0 8393.285372 \n",
"973 70.89 595000.0 8393.285372 \n",
"\n",
" url \n",
"953 https://www.rightmove.co.uk/properties/145546538 \n",
"823 https://www.rightmove.co.uk/properties/145051769 \n",
"492 https://www.rightmove.co.uk/properties/143317361 \n",
"561 https://www.rightmove.co.uk/properties/143780789 \n",
"69 https://www.rightmove.co.uk/properties/134574563 \n",
"987 https://www.rightmove.co.uk/properties/86648925 \n",
"272 https://www.rightmove.co.uk/properties/141131297 \n",
"521 https://www.rightmove.co.uk/properties/143514149 \n",
"324 https://www.rightmove.co.uk/properties/141831353 \n",
"1017 https://www.rightmove.co.uk/properties/86804832 \n",
"526 https://www.rightmove.co.uk/properties/143552156 \n",
"817 https://www.rightmove.co.uk/properties/145035929 \n",
"843 https://www.rightmove.co.uk/properties/145144988 \n",
"337 https://www.rightmove.co.uk/properties/141904286 \n",
"233 https://www.rightmove.co.uk/properties/140582213 \n",
"763 https://www.rightmove.co.uk/properties/144862070 \n",
"315 https://www.rightmove.co.uk/properties/141669686 \n",
"899 https://www.rightmove.co.uk/properties/145362911 \n",
"147 https://www.rightmove.co.uk/properties/138537527 \n",
"972 https://www.rightmove.co.uk/properties/86296491 "
" url bedrooms \n",
"953 https://www.rightmove.co.uk/properties/145546538 2 \n",
"823 https://www.rightmove.co.uk/properties/145051769 2 \n",
"492 https://www.rightmove.co.uk/properties/143317361 2 \n",
"561 https://www.rightmove.co.uk/properties/143780789 2 \n",
"69 https://www.rightmove.co.uk/properties/134574563 2 \n",
"988 https://www.rightmove.co.uk/properties/86648925 2 \n",
"272 https://www.rightmove.co.uk/properties/141131297 2 \n",
"521 https://www.rightmove.co.uk/properties/143514149 2 \n",
"324 https://www.rightmove.co.uk/properties/141831353 2 \n",
"1018 https://www.rightmove.co.uk/properties/86804832 2 \n",
"526 https://www.rightmove.co.uk/properties/143552156 1 \n",
"817 https://www.rightmove.co.uk/properties/145035929 2 \n",
"843 https://www.rightmove.co.uk/properties/145144988 2 \n",
"337 https://www.rightmove.co.uk/properties/141904286 2 \n",
"233 https://www.rightmove.co.uk/properties/140582213 2 \n",
"763 https://www.rightmove.co.uk/properties/144862070 2 \n",
"315 https://www.rightmove.co.uk/properties/141669686 2 \n",
"899 https://www.rightmove.co.uk/properties/145362911 2 \n",
"147 https://www.rightmove.co.uk/properties/138537527 1 \n",
"973 https://www.rightmove.co.uk/properties/86296491 2 "
]
},
"execution_count": 9,
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
@ -1004,6 +1090,70 @@
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 10,
"id": "edd9fa24-cad2-4448-9b17-c6d514564f41",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>sqm_ocr</th>\n",
" <th>price</th>\n",
" <th>price_per_sqm</th>\n",
" <th>url</th>\n",
" <th>bedrooms</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"Empty DataFrame\n",
"Columns: [sqm_ocr, price, price_per_sqm, url, bedrooms]\n",
"Index: []"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df[df.bedrooms > 2]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "002b2a3a-3ecc-45c1-8c2f-c143380ee0d5",
"metadata": {},
"outputs": [],
"source": [
"\n"
]
}
],
"metadata": {