Initial commit: Rightmove fetch, database, caching, poetry etc.
- Fetching rightmove listing api - Memoizing query - Writing to sqlite database with sqlalchemy - Poetry dependencies
This commit is contained in:
Kadir Tugan
commit
4ee7ae16c4
12 changed files with 3236 additions and 0 deletions
5
.gitignore
vendored
Normal file
5
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
_cache/
|
||||
cert/
|
||||
venv/
|
||||
__pycache__/
|
||||
sqlite.db
|
||||
47
GUIDE
Normal file
47
GUIDE
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
package name: com.rightmove.android
|
||||
|
||||
frida --codeshare pcipolloni/universal-android-ssl-pinning-bypass-with-frida -f com.rightmove.android
|
||||
|
||||
|
||||
|
||||
1. install burp
|
||||
2. Add listener 8282
|
||||
3. Export certificate in the DER format
|
||||
4. convert certificate with command
|
||||
```
|
||||
# converts from DER to PEM
|
||||
openssl x509 -inform DER -in burp.der -out burp.pem
|
||||
```
|
||||
5. Copy cert to android with the proper name
|
||||
```
|
||||
# According to https://codeshare.frida.re/@pcipolloni/universal-android-ssl-pinning-bypass-with-frida/ the cert path is hardcoded
|
||||
adb push burp.pem /data/local/tmp/cert-der.crt
|
||||
```
|
||||
6. Add the proxy in the android wifi settings
|
||||
```
|
||||
# find your own local network ip
|
||||
ip addr
|
||||
# Open the wifi you are connected to, edit and add port 8282 (from above) and the ip
|
||||
|
||||
```
|
||||
192.168.0.211/24
|
||||
|
||||
|
||||
1. Install frida server on android
|
||||
|
||||
|
||||
4. run Frida
|
||||
```
|
||||
adb shell "/data/local/tmp/frida-server &"
|
||||
```
|
||||
5. Check if it runs with
|
||||
```
|
||||
frida-ps -U
|
||||
```
|
||||
|
||||
6. pin rightmove
|
||||
frida -U --codeshare pcipolloni/universal-android-ssl-pinning-bypass-with-frida -f com.rightmove.android
|
||||
|
||||
|
||||
|
||||
|
||||
13
code/image.py
Normal file
13
code/image.py
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
import requests
|
||||
|
||||
headers = {
|
||||
'Host': 'media.rightmove.co.uk',
|
||||
# 'Accept-Encoding': 'gzip, deflate, br',
|
||||
'User-Agent': 'okhttp/4.10.0',
|
||||
}
|
||||
|
||||
response = requests.get(
|
||||
'https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_IMG_00_0000.jpeg',
|
||||
headers=headers,
|
||||
verify=False,
|
||||
)
|
||||
305
code/json/detail.json
Normal file
305
code/json/detail.json
Normal file
|
|
@ -0,0 +1,305 @@
|
|||
{
|
||||
"property": {
|
||||
"identifier": 138680705,
|
||||
"updateDate": 1697679094000,
|
||||
"development": false,
|
||||
"onlineViewing": false,
|
||||
"price": 375000.0,
|
||||
"priceQualifier": "Offers in Excess of",
|
||||
"shouldShowPrice": true,
|
||||
"premiumDisplay": false,
|
||||
"transactionTypeId": 1,
|
||||
"visible": true,
|
||||
"bedrooms": 2,
|
||||
"photoCount": 11,
|
||||
"address": "Clinger Court, Hobbs Place Estate, N1",
|
||||
"status": null,
|
||||
"outcode": "N1",
|
||||
"summary": "An well presented, bright two bedroom apartment situated on the Fifth floor of this purpose built development. The property measures approximately 642 Sq. ft and has a bright and contemporary interior which comprises of a Large living room with balcony, a modern fully-fitted separate kitchen and...",
|
||||
"fullDescription": "An well presented, bright two bedroom apartment situated on the Fifth floor of this purpose built development. The property measures approximately 642 Sq. ft and has a bright and contemporary interior which comprises of a Large living room with balcony, a modern fully-fitted separate kitchen and two generously sized double bedrooms and a contemporary finished bathroom.<br /><br />Located in trendy Hoxton and moments from the restaurants, bars and boutique shops of Hoxton Square and the Shoreditch Triangle. Approx. 0.8 miles to Old Street Station (Northern Line & National Rail), 0.6 miles to Hoxton Station (Overground) and 1.3 miles to Liverpool Street Station (Central, Circle, Metropolitan, Hammersmith & City Lines & National Rail).<br /><br />",
|
||||
"branch": {
|
||||
"identifier": 46001,
|
||||
"updateDate": 1697679094000,
|
||||
"name": "Stoke Newington",
|
||||
"brandName": "Oakwood",
|
||||
"branchLogo": "https://media.rightmove.co.uk/47k/46001/branch_logo_46001_0005.png",
|
||||
"largeBranchLogo": "https://media.rightmove.co.uk/47k/46001/branch_logo_46001_0005.png",
|
||||
"brandPlusResale": true,
|
||||
"brandPlusLettings": true,
|
||||
"address": "48 Stoke Newington Church Street,\r\nLondon,\r\nN16 0NB",
|
||||
"development": false
|
||||
},
|
||||
"telephoneNumber": "020 3835 3657",
|
||||
"listingUpdateReason": "Reduced on 11/10/2023",
|
||||
"letFurnishType": null,
|
||||
"letType": null,
|
||||
"letDateAvailable": null,
|
||||
"letBond": null,
|
||||
"showLettingFeesMessage": false,
|
||||
"lettingFeesMessage": null,
|
||||
"floorplanCount": 1,
|
||||
"showMap": true,
|
||||
"latitude": 51.535007,
|
||||
"longitude": -0.082629,
|
||||
"exactLocationAvailable": true,
|
||||
"showStreetView": true,
|
||||
"streetViewLatitude": 51.535015,
|
||||
"streetViewLongitude": -0.082695,
|
||||
"streetViewHeading": 149.77,
|
||||
"streetViewPitch": 17.08,
|
||||
"streetViewZoom": 1,
|
||||
"saved": false,
|
||||
"publicsiteUrl": "https://www.rightmove.co.uk/properties/138680705",
|
||||
"mobileStreetViewUrl": "https://www.rightmove.co.uk/apps/streetview.html?propertyId=138680705",
|
||||
"mobilePropertyMapViewUrl": "https://www.rightmove.co.uk/apps/property-mapview.html?propertyId=138680705",
|
||||
"schoolCheckerUrl": "https://www.rightmove.co.uk/property-for-sale/nearby-schools/property-138680705.html",
|
||||
"soldPricesUrl": "https://www.rightmove.co.uk/house-prices/n1-5ja.html",
|
||||
"marketInfoUrl": "https://www.rightmove.co.uk/property-for-sale/market-information/property-138680705.html",
|
||||
"propertyDisclaimer": "<b>Disclaimer</b> - Property reference 32532509. The information displayed about this property comprises a property advertisement. Rightmove.co.uk makes no warranty as to the accuracy or completeness of the advertisement or any linked or associated information, and Rightmove has no control over the content. This property advertisement does not constitute property particulars. The information is provided and maintained by <b>Oakwood, Stoke Newington</b>. Please contact the selling agent or developer directly to obtain any information which may be available under the terms of The Energy Performance of Buildings (Certificates and Inspections) (England and Wales) Regulations 2007 or the Home Report if in relation to a residential property in Scotland.",
|
||||
"propertyPhrase": "2 bedroom flat",
|
||||
"tenureType": "Leasehold",
|
||||
"buildToRent": false,
|
||||
"propertySubtype": "Flat",
|
||||
"contactMethod": "EMAIL",
|
||||
"misInfo": {
|
||||
"propertyId": 138680705,
|
||||
"branchId": 46001,
|
||||
"offerAdvertStampTypeId": null,
|
||||
"brandPlus": true,
|
||||
"featuredProperty": false,
|
||||
"channel": "BUY",
|
||||
"premiumDisplay": false,
|
||||
"premiumDisplayStampId": null,
|
||||
"countryCode": "GB"
|
||||
},
|
||||
"mortgageCalculator": {
|
||||
"price": 375000,
|
||||
"propertyTypeAlias": "flats_apartments"
|
||||
},
|
||||
"brochure": {
|
||||
"showBrochureLead": false,
|
||||
"brochures": [
|
||||
{
|
||||
"url": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_DOC_02_0001.pdf",
|
||||
"caption": "Clinger Court, Hobbs Place Estate, N1"
|
||||
},
|
||||
{
|
||||
"url": "https://www.vebra.com/details/property/32532509",
|
||||
"caption": "Brochure"
|
||||
}
|
||||
]
|
||||
},
|
||||
"analyticsInfo": {
|
||||
"branchId": "46001",
|
||||
"propertyId": "138680705",
|
||||
"onlineViewing": "F",
|
||||
"imageCount": "11",
|
||||
"floorplanCount": "1",
|
||||
"beds": "2",
|
||||
"postcode": "N1 5JA",
|
||||
"propertyType": "Flats / Apartments",
|
||||
"propertySubType": "Flat",
|
||||
"added": "20230815",
|
||||
"price": "375000",
|
||||
"tenure": "Leasehold",
|
||||
"bathrooms": "1",
|
||||
"sharedOwnership": "F"
|
||||
},
|
||||
"displayPrices": [
|
||||
{
|
||||
"displayPrice": "£375,000",
|
||||
"displayPriceQualifier": "Offers in Excess of"
|
||||
}
|
||||
],
|
||||
"stations": [
|
||||
{
|
||||
"station": "Hoxton Station",
|
||||
"distance": 0.3,
|
||||
"type": "6"
|
||||
},
|
||||
{
|
||||
"station": "Haggerston Station",
|
||||
"distance": 0.4,
|
||||
"type": "6"
|
||||
},
|
||||
{
|
||||
"station": "Old Street Station",
|
||||
"distance": 0.7,
|
||||
"type": "1,2"
|
||||
}
|
||||
],
|
||||
"features": [],
|
||||
"photos": [
|
||||
{
|
||||
"url": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_00_0000_max_656x437.jpeg",
|
||||
"thumbnailUrl": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_00_0000_max_135x100.jpeg",
|
||||
"maxSizeUrl": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_IMG_00_0000.jpeg",
|
||||
"caption": "Clinger Court (1).jpg",
|
||||
"order": 0
|
||||
},
|
||||
{
|
||||
"url": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_01_0000_max_656x437.jpeg",
|
||||
"thumbnailUrl": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_01_0000_max_135x100.jpeg",
|
||||
"maxSizeUrl": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_IMG_01_0000.jpeg",
|
||||
"caption": "Clinger Court (7).jpg",
|
||||
"order": 1
|
||||
},
|
||||
{
|
||||
"url": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_02_0000_max_656x437.jpeg",
|
||||
"thumbnailUrl": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_02_0000_max_135x100.jpeg",
|
||||
"maxSizeUrl": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_IMG_02_0000.jpeg",
|
||||
"caption": "Clinger Court (3).jpg",
|
||||
"order": 2
|
||||
},
|
||||
{
|
||||
"url": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_03_0000_max_656x437.jpeg",
|
||||
"thumbnailUrl": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_03_0000_max_135x100.jpeg",
|
||||
"maxSizeUrl": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_IMG_03_0000.jpeg",
|
||||
"caption": "Clinger Court (5).jpg",
|
||||
"order": 3
|
||||
},
|
||||
{
|
||||
"url": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_04_0000_max_656x437.jpeg",
|
||||
"thumbnailUrl": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_04_0000_max_135x100.jpeg",
|
||||
"maxSizeUrl": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_IMG_04_0000.jpeg",
|
||||
"caption": "Clinger Court (4).jpg",
|
||||
"order": 4
|
||||
},
|
||||
{
|
||||
"url": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_05_0000_max_656x437.jpeg",
|
||||
"thumbnailUrl": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_05_0000_max_135x100.jpeg",
|
||||
"maxSizeUrl": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_IMG_05_0000.jpeg",
|
||||
"caption": "Clinger Court (8).jpg",
|
||||
"order": 5
|
||||
},
|
||||
{
|
||||
"url": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_06_0000_max_656x437.jpeg",
|
||||
"thumbnailUrl": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_06_0000_max_135x100.jpeg",
|
||||
"maxSizeUrl": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_IMG_06_0000.jpeg",
|
||||
"caption": "Clinger Court (9).jpg",
|
||||
"order": 6
|
||||
},
|
||||
{
|
||||
"url": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_07_0000_max_656x437.jpeg",
|
||||
"thumbnailUrl": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_07_0000_max_135x100.jpeg",
|
||||
"maxSizeUrl": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_IMG_07_0000.jpeg",
|
||||
"caption": "Clinger Court (10).jpg",
|
||||
"order": 7
|
||||
},
|
||||
{
|
||||
"url": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_08_0000_max_656x437.jpeg",
|
||||
"thumbnailUrl": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_08_0000_max_135x100.jpeg",
|
||||
"maxSizeUrl": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_IMG_08_0000.jpeg",
|
||||
"caption": "Clinger Court (2).jpg",
|
||||
"order": 8
|
||||
},
|
||||
{
|
||||
"url": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_09_0000_max_656x437.jpeg",
|
||||
"thumbnailUrl": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_09_0000_max_135x100.jpeg",
|
||||
"maxSizeUrl": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_IMG_09_0000.jpeg",
|
||||
"caption": "Clinger Court (6).jpg",
|
||||
"order": 9
|
||||
},
|
||||
{
|
||||
"url": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_10_0000_max_656x437.jpeg",
|
||||
"thumbnailUrl": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_IMG_10_0000_max_135x100.jpeg",
|
||||
"maxSizeUrl": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_IMG_10_0000.jpeg",
|
||||
"caption": "Clinger Court (11).jpg",
|
||||
"order": 10
|
||||
}
|
||||
],
|
||||
"virtualTours": [],
|
||||
"epcs": [
|
||||
{
|
||||
"url": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_EPCGRAPH_00_0000.png",
|
||||
"caption": "EE Rating",
|
||||
"order": 0
|
||||
}
|
||||
],
|
||||
"floorplans": [
|
||||
{
|
||||
"order": 0,
|
||||
"url": "https://media.rightmove.co.uk/47k/46001/138680705/46001_32532509_FLP_00_0000.jpeg",
|
||||
"thumbnailUrl": "https://media.rightmove.co.uk/dir/47k/46001/138680705/46001_32532509_FLP_00_0000_max_296x197.jpeg",
|
||||
"caption": "floorplan.jpg"
|
||||
}
|
||||
],
|
||||
"note": null,
|
||||
"affordableBuyingScheme": false,
|
||||
"lettingsInfo": {
|
||||
"title": "",
|
||||
"content": []
|
||||
},
|
||||
"propertyDetailsInfo": {
|
||||
"title": "",
|
||||
"content": [
|
||||
{
|
||||
"type": "propertyType",
|
||||
"title": "Property type",
|
||||
"value": "Flat"
|
||||
},
|
||||
{
|
||||
"type": "bedrooms",
|
||||
"title": "Bedrooms",
|
||||
"value": "x2"
|
||||
},
|
||||
{
|
||||
"type": "bathrooms",
|
||||
"title": "Bathrooms",
|
||||
"value": "x1"
|
||||
},
|
||||
{
|
||||
"type": "tenure",
|
||||
"title": "Tenure",
|
||||
"value": "Leasehold"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tenureInfo": {
|
||||
"title": "Leasehold",
|
||||
"content": [
|
||||
{
|
||||
"type": "groundRent",
|
||||
"title": "Ground rent",
|
||||
"value": "Ask agent"
|
||||
},
|
||||
{
|
||||
"type": "annualServiceCharge",
|
||||
"title": "Service charge",
|
||||
"value": "£2,195.76 per year"
|
||||
},
|
||||
{
|
||||
"type": "lengthOfLease",
|
||||
"title": "Length of lease",
|
||||
"value": "93 years left"
|
||||
}
|
||||
]
|
||||
},
|
||||
"sharedOwnershipInfo": {
|
||||
"title": "",
|
||||
"content": []
|
||||
},
|
||||
"councilTaxInfo": {
|
||||
"title": "Council tax",
|
||||
"content": [
|
||||
{
|
||||
"type": "councilTaxBand",
|
||||
"title": "Council tax band",
|
||||
"value": "B"
|
||||
}
|
||||
]
|
||||
},
|
||||
"domesticRatesInfo": {
|
||||
"title": "",
|
||||
"content": []
|
||||
},
|
||||
"linkToGlossary": "https://www.rightmove.co.uk/guides/property-details-glossary/",
|
||||
"enquiredTimestamp": null,
|
||||
"stampDutyCalculator": {
|
||||
"country": "ENGLAND",
|
||||
"price": 375000,
|
||||
"buyerType": null,
|
||||
"result": null
|
||||
}
|
||||
}
|
||||
}
|
||||
2526
code/json/listings.json
Normal file
2526
code/json/listings.json
Normal file
File diff suppressed because it is too large
Load diff
57
code/listings.py
Normal file
57
code/listings.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import requests
|
||||
|
||||
headers = {
|
||||
'Host': 'api.rightmove.co.uk',
|
||||
# 'Accept-Encoding': 'gzip, deflate, br',
|
||||
'User-Agent': 'okhttp/4.10.0',
|
||||
'Connection': 'close',
|
||||
}
|
||||
|
||||
params = {
|
||||
'locationIdentifier': 'POSTCODE^4228216',
|
||||
'channel': 'BUY',
|
||||
'page': '1',
|
||||
'numberOfPropertiesPerPage': '25',
|
||||
'radius': '3.0',
|
||||
'sortBy': 'distance',
|
||||
'includeUnavailableProperties': 'false',
|
||||
'propertyTypes': 'flat',
|
||||
'dontShow': 'sharedOwnership,retirement',
|
||||
'minPrice': '150000',
|
||||
'maxPrice': '500000',
|
||||
'minBedrooms': '2',
|
||||
'maxBedrooms': '2',
|
||||
'apiApplication': 'ANDROID',
|
||||
'appVersion': '3.70.0',
|
||||
}
|
||||
|
||||
response = requests.get('https://api.rightmove.co.uk/api/property-listing', params=params, headers=headers, verify=False)
|
||||
|
||||
import requests
|
||||
|
||||
headers = {
|
||||
'Host': 'api.rightmove.co.uk',
|
||||
# 'Accept-Encoding': 'gzip, deflate, br',
|
||||
'User-Agent': 'okhttp/4.10.0',
|
||||
'Connection': 'close',
|
||||
}
|
||||
|
||||
params = {
|
||||
'locationIdentifier': 'POSTCODE^4228216',
|
||||
'channel': 'BUY',
|
||||
'page': '2',
|
||||
'numberOfPropertiesPerPage': '25',
|
||||
'radius': '3.0',
|
||||
'sortBy': 'distance',
|
||||
'includeUnavailableProperties': 'false',
|
||||
'propertyTypes': 'flat',
|
||||
'dontShow': 'sharedOwnership,retirement',
|
||||
'minPrice': '150000',
|
||||
'maxPrice': '500000',
|
||||
'minBedrooms': '2',
|
||||
'maxBedrooms': '2',
|
||||
'apiApplication': 'ANDROID',
|
||||
'appVersion': '3.70.0',
|
||||
}
|
||||
|
||||
response = requests.get('https://api.rightmove.co.uk/api/property-listing', params=params, headers=headers, verify=False)
|
||||
15
code/single-query.py
Normal file
15
code/single-query.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
import requests
|
||||
|
||||
headers = {
|
||||
'Host': 'api.rightmove.co.uk',
|
||||
# 'Accept-Encoding': 'gzip, deflate, br',
|
||||
'User-Agent': 'okhttp/4.10.0',
|
||||
'Connection': 'close',
|
||||
}
|
||||
|
||||
params = {
|
||||
'apiApplication': 'ANDROID',
|
||||
'appVersion': '3.70.0',
|
||||
}
|
||||
|
||||
response = requests.get('https://api.rightmove.co.uk/api/property/119578451', params=params, headers=headers, verify=False)
|
||||
34
db.py
Normal file
34
db.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import Column, Integer, String, JSON, FLOAT
|
||||
|
||||
engine = create_engine("sqlite:///sqlite.db", echo=True)
|
||||
session = Session(engine)
|
||||
|
||||
from sqlalchemy.orm import declarative_base
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
class RightmoveListing(Base):
|
||||
__tablename__ = "rightmove"
|
||||
id = Column(Integer, primary_key=True)
|
||||
price = Column(FLOAT)
|
||||
listing_json = Column(JSON, nullable=True)
|
||||
detail_json = Column(JSON, nullable=True)
|
||||
updated_timestamp = Column(Integer, nullable=True)
|
||||
distance_to_office_minutes = Column(FLOAT, nullable=True)
|
||||
|
||||
def save(self):
|
||||
session.add(self)
|
||||
session.commit()
|
||||
|
||||
def __repr__(self):
|
||||
return "<User(name='%s', fullname='%s', nickname='%s')>" % (
|
||||
self.name,
|
||||
self.fullname,
|
||||
self.nickname,
|
||||
)
|
||||
|
||||
if __name__ == '__main__':
|
||||
Base.metadata.create_all(engine)
|
||||
145
poetry.lock
generated
Normal file
145
poetry.lock
generated
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
[[package]]
|
||||
name = "cachetools"
|
||||
version = "5.3.2"
|
||||
description = "Extensible memoizing collections and decorators"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2023.7.22"
|
||||
description = "Python package for providing Mozilla's CA Bundle."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
|
||||
[[package]]
|
||||
name = "charset-normalizer"
|
||||
version = "3.3.2"
|
||||
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7.0"
|
||||
|
||||
[[package]]
|
||||
name = "diskcache"
|
||||
version = "5.6.3"
|
||||
description = "Disk Cache -- Disk and file backed persistent cache."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3"
|
||||
|
||||
[[package]]
|
||||
name = "greenlet"
|
||||
version = "3.0.1"
|
||||
description = "Lightweight in-process concurrent programming"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.extras]
|
||||
docs = ["sphinx"]
|
||||
test = ["objgraph", "psutil"]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "3.4"
|
||||
description = "Internationalized Domain Names in Applications (IDNA)"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.5"
|
||||
|
||||
[[package]]
|
||||
name = "requests"
|
||||
version = "2.31.0"
|
||||
description = "Python HTTP for Humans."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
certifi = ">=2017.4.17"
|
||||
charset-normalizer = ">=2,<4"
|
||||
idna = ">=2.5,<4"
|
||||
urllib3 = ">=1.21.1,<3"
|
||||
|
||||
[package.extras]
|
||||
socks = ["PySocks (>=1.5.6,!=1.5.7)"]
|
||||
use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"]
|
||||
|
||||
[[package]]
|
||||
name = "sqlalchemy"
|
||||
version = "2.0.23"
|
||||
description = "Database Abstraction Library"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""}
|
||||
typing-extensions = ">=4.2.0"
|
||||
|
||||
[package.extras]
|
||||
aiomysql = ["greenlet (!=0.4.17)", "aiomysql (>=0.2.0)"]
|
||||
aioodbc = ["greenlet (!=0.4.17)", "aioodbc"]
|
||||
aiosqlite = ["greenlet (!=0.4.17)", "aiosqlite", "typing-extensions (!=3.10.0.1)"]
|
||||
asyncio = ["greenlet (!=0.4.17)"]
|
||||
asyncmy = ["greenlet (!=0.4.17)", "asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)"]
|
||||
mariadb_connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"]
|
||||
mssql = ["pyodbc"]
|
||||
mssql_pymssql = ["pymssql"]
|
||||
mssql_pyodbc = ["pyodbc"]
|
||||
mypy = ["mypy (>=0.910)"]
|
||||
mysql = ["mysqlclient (>=1.4.0)"]
|
||||
mysql_connector = ["mysql-connector-python"]
|
||||
oracle = ["cx-oracle (>=8)"]
|
||||
oracle_oracledb = ["oracledb (>=1.0.1)"]
|
||||
postgresql = ["psycopg2 (>=2.7)"]
|
||||
postgresql_asyncpg = ["greenlet (!=0.4.17)", "asyncpg"]
|
||||
postgresql_pg8000 = ["pg8000 (>=1.29.1)"]
|
||||
postgresql_psycopg = ["psycopg (>=3.0.7)"]
|
||||
postgresql_psycopg2binary = ["psycopg2-binary"]
|
||||
postgresql_psycopg2cffi = ["psycopg2cffi"]
|
||||
postgresql_psycopgbinary = ["psycopg[binary] (>=3.0.7)"]
|
||||
pymysql = ["pymysql"]
|
||||
sqlcipher = ["sqlcipher3-binary"]
|
||||
|
||||
[[package]]
|
||||
name = "typing-extensions"
|
||||
version = "4.8.0"
|
||||
description = "Backported and Experimental Type Hints for Python 3.8+"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
|
||||
[[package]]
|
||||
name = "urllib3"
|
||||
version = "2.0.7"
|
||||
description = "HTTP library with thread-safe connection pooling, file post, and more."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.extras]
|
||||
brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"]
|
||||
secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"]
|
||||
socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]
|
||||
zstd = ["zstandard (>=0.18.0)"]
|
||||
|
||||
[metadata]
|
||||
lock-version = "1.1"
|
||||
python-versions = "^3.9"
|
||||
content-hash = "03e44da9afee6f14612fc4705f1c2b55b19e63af013140eb9580c02792218b84"
|
||||
|
||||
[metadata.files]
|
||||
cachetools = []
|
||||
certifi = []
|
||||
charset-normalizer = []
|
||||
diskcache = []
|
||||
greenlet = []
|
||||
idna = []
|
||||
requests = []
|
||||
sqlalchemy = []
|
||||
typing-extensions = []
|
||||
urllib3 = []
|
||||
18
pyproject.toml
Normal file
18
pyproject.toml
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
[tool.poetry]
|
||||
name = "rightmove-crawler"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = ["Kadir Tugan <git@k8n.dev>"]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.9"
|
||||
SQLAlchemy = "^2.0.23"
|
||||
requests = "^2.31.0"
|
||||
cachetools = "^5.3.2"
|
||||
diskcache = "^5.6.3"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core>=1.0.0"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
60
query.py
Normal file
60
query.py
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
import pickle
|
||||
|
||||
import cachetools
|
||||
from diskcache import Cache
|
||||
import requests
|
||||
from db import RightmoveListing
|
||||
|
||||
import urllib3
|
||||
urllib3.disable_warnings()
|
||||
|
||||
cache = Cache(r'_cache')
|
||||
|
||||
|
||||
headers = {
|
||||
'Host': 'api.rightmove.co.uk',
|
||||
# 'Accept-Encoding': 'gzip, deflate, br',
|
||||
'User-Agent': 'okhttp/4.10.0',
|
||||
'Connection': 'close',
|
||||
}
|
||||
|
||||
@cache.memoize()
|
||||
def listing_query(page: int, min_bedrooms: int, max_bedrooms: int):
|
||||
print("Executing")
|
||||
params = {
|
||||
'locationIdentifier': 'POSTCODE^4228216',
|
||||
'channel': 'BUY',
|
||||
'page': str(page),
|
||||
'numberOfPropertiesPerPage': '25',
|
||||
'radius': '5.0',
|
||||
'sortBy': 'distance',
|
||||
'includeUnavailableProperties': 'false',
|
||||
'propertyTypes': 'flat',
|
||||
'dontShow': 'sharedOwnership,retirement',
|
||||
'minPrice': '150000',
|
||||
'maxPrice': '500000',
|
||||
'minBedrooms': str(min_bedrooms),
|
||||
'maxBedrooms': str(max_bedrooms),
|
||||
'apiApplication': 'ANDROID',
|
||||
'appVersion': '3.70.0',
|
||||
}
|
||||
|
||||
response = requests.get('https://api.rightmove.co.uk/api/property-listing', params=params, headers=headers,
|
||||
verify=False)
|
||||
if response.status_code != 200:
|
||||
raise Exception("Failed due to: ", response.text)
|
||||
|
||||
return response.json()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
response = listing_query(1, 2, 2)
|
||||
resp = response
|
||||
for d in resp['properties']:
|
||||
rl = RightmoveListing(
|
||||
id=d['identifier'],
|
||||
listing_json=d,
|
||||
price=d['price'],
|
||||
updated_timestamp = d['updateDate'],
|
||||
)
|
||||
rl.save()
|
||||
11
rightmove_parser.py
Normal file
11
rightmove_parser.py
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
|
||||
def parse_listing_json_entry(d):
|
||||
id = d['identifier']
|
||||
# address = d['address']
|
||||
propertyType = d['propertyType']
|
||||
price = d['price']
|
||||
latitude = d['latitude']
|
||||
longitude = d['longitude']
|
||||
updated_date = d['updateDate']
|
||||
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue