add minimal working version where we fetch data and visualize it corectly

This commit is contained in:
Viktor Barzin 2025-06-15 12:48:52 +00:00
parent b995bc2286
commit 4c7fe8927b
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
8 changed files with 1110 additions and 293 deletions

View file

@ -1,13 +1,18 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Vite + React + TS</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Vite + React + TS</title>
<!-- <link href='https://api.mapbox.com/mapbox-gl-js/v3.12.0/mapbox-gl.css' rel='stylesheet' /> -->
<script src="/HexgridHeatmap.js"> </script>
</head>
<body>
<div id="root" style="width: 100%; height: 100%"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>

File diff suppressed because it is too large Load diff

View file

@ -15,10 +15,13 @@
"@types/d3": "^7.4.3",
"@types/mapbox-gl": "^3.4.1",
"@types/turf": "^3.5.32",
"d3": "^7.9.0",
"mapbox-gl": "^3.12.0",
"oidc-client-ts": "^3.2.1",
"react": "^19.1.0",
"react-dom": "^19.1.0",
"react-oidc-context": "^3.3.0",
"rivets": "^0.9.6",
"tailwindcss": "^4.1.10"
},
"devDependencies": {
@ -26,6 +29,7 @@
"@types/node": "^24.0.1",
"@types/react": "^19.1.2",
"@types/react-dom": "^19.1.2",
"@types/rivets": "^0.9.5",
"@vitejs/plugin-react-swc": "^3.9.0",
"eslint": "^9.25.0",
"eslint-plugin-react-hooks": "^5.2.0",

View file

@ -1,261 +0,0 @@
// rivet
var filter = { city: 'London', country: null, mode: 'qmprice' };
filter['countries'] = Array.from(new Set(data.features.map(function (d) { return d['properties']['country'] })));
rivets.bind(document.getElementById('overlay'), { filter: filter });
console.log('kekeke')
function clone(d) {
return JSON.parse(JSON.stringify(d));
}
function percentile(arr, p) {
if (arr.length === 0) return 0;
if (typeof p !== 'number') throw new TypeError('p must be a number');
if (p <= 0) return arr[0];
if (p >= 1) return arr[arr.length - 1];
var index = arr.length * p,
lower = Math.floor(index),
upper = lower + 1,
weight = index % 1;
if (upper >= arr.length) return arr[lower];
return arr[lower] * (1 - weight) + arr[upper] * weight;
}
function update() {
// close overlay
var modal = document.getElementById('modal_overlay');
modal.classList.toggle('modal-open');
// init heatmap
heatmap = new HexgridHeatmap(map, "hexgrid-heatmap", "waterway-label");
heatmap.setIntensity(9); // dunno yet
heatmap.setSpread(0.05); // dunno yet
heatmap.setCellDensity(0.5); // small value == bigger hexagons
heatmap.setPropertyName(filter.mode);
if (filter.mode === 'qmprice') {
// if we visualize sqm based data, remove properties where we have no data
qmDim.filter(function (d) { return d > 0; });
}
// set filter
if (filter.city) {
cityDim.filterExact(filter.city);
} else if (filter.country) {
countryDim.filterExact(filter.country);
} else {
alert('nothing loadable');
}
filter.count = cityDim.top(Infinity).length;
var subset = { "type": "FeatureCollection", "features": [] };
indexDim.top(Infinity).forEach(function (i) {
subset.features.push(data.features[i.index]);
});
loadData(subset);
}
function loadData(subset) {
heatmap.setData(subset);
var values = subset.features.map(function (d) { return d['properties'][filter.mode] });
values = values.sort(function (a, b) { return a - b; });
// setting the color stops, min is at 5th percentile, max at 95percentile
var min = values[Math.round(values.length * 0.05)];
var max = values[Math.round(values.length * 0.95)];
var colorStopsPerc = [
[0, "rgba(0,185,243,0)"],
[25, "rgba(0,185,243,0.24)"],
[60, "rgba(255,223,0,0.3)"],
[100, "rgba(255,105,0,0.3)"],
];
makeLegend(colorStopsPerc, min, max);
var colorStopsValue = colorStopsPerc.map(function (d) {
return [min + d[0] * (max - min) / 100, d[1]];
});
heatmap.setColorStops(colorStopsValue);
heatmap.update();
//get bounding box and zoom to that area
// we use a 1% percentile since some data can be corrupt
var longitudes = subset.features.map(function (d) { return d.geometry.coordinates[0]; }).sort(function (a, b) { return a - b; });
var latitudes = subset.features.map(function (d) { return d.geometry.coordinates[1]; }).sort(function (a, b) { return a - b; });
var minlng = percentile(longitudes, 0.01);
var maxlng = percentile(longitudes, 0.99);
var minlat = percentile(latitudes, 0.01);
var maxlat = percentile(latitudes, 0.99);
map.fitBounds([
[minlng, minlat],
[maxlng, maxlat]
]);
}
function makeLegend(colorstops, minValue, maxValue) {
/**
* colorstops: [[0, 'green'], [100, 'red']]
* @type {number}
*/
var svg_height = 300, svg_width = 70;
var svg = d3.select('#svg');
var defs = svg
.attr('height', svg_height)
.attr('width', svg_width);
var linearGradient = svg.append("defs")
.append("linearGradient")
.attr("id", "linear-gradient");
linearGradient
.attr("x1", "0%")
.attr("y1", "100%")
.attr("x2", "0%")
.attr("y2", "0%");
svg.append("rect")
.attr("width", svg_width * 0.4)
.attr("height", svg_height)
.attr('rx', 4)
.style("fill", "url(#linear-gradient)");
colorstops.forEach(function (d) {
linearGradient.append("stop")
.attr("offset", d[0] + "%")
.attr("stop-color", d[1]);
});
var xScale = d3.scaleLinear().range([svg_height - 20, 0]).domain([minValue, maxValue]);
var xAxis = d3.axisRight(xScale).ticks(5);
svg.append("g")
.attr("class", "axis")
.attr("transform", "translate(" + svg_width / 2 + "," + (10) + ")")
.call(xAxis);
}
// ORIGINAL BLOCK
mapboxgl.accessToken = 'pk.eyJ1IjoiZGktdG8iLCJhIjoiY2o0bnBoYXcxMW1mNzJ3bDhmc2xiNWttaiJ9.ZccatVk_4shzoAsEUXXecA';
var map = new mapboxgl.Map({
container: 'map',
style: 'mapbox://styles/mapbox/light-v9',
center: [13.38032, 49.994210],
zoom: 5
});
console.log('ekekek');
map.on("load", function () {
var crossData = data.features.map(function (d, i) {
//clone properties
var props = clone(d['properties']);
props['index'] = i;
return props;
});
cf = crossfilter(crossData);
qmDim = cf.dimension(function (d) { return d.qm; });
cityDim = cf.dimension(function (d) { return d.city; });
countryDim = cf.dimension(function (d) { return d.country; });
rentDim = cf.dimension(function (d) { return d.total_price; });
roomsDim = cf.dimension(function (d) { return d.rooms; });
urlDim = cf.dimension(function (d) { return d.url; });
indexDim = cf.dimension(function (d) { return d.index; });
});
map.on('click', (e) => {
// {
// lngLat: {
// lng: 40.203,
// lat: -74.451
// },
// originalEvent: {...},
// point: {
// x: 266,
// y: 464
// },
// target: {...},
// type: "click"
// }
openListingsDialog(e.lngLat.lng, e.lngLat.lat);
});
function openListingsDialog(longtitude, latitude) {
const searchBuffer = 0.001 // ~100m
const properties = heatmap._tree.search({
minX: longtitude - searchBuffer,
maxX: longtitude + searchBuffer,
minY: latitude - searchBuffer,
maxY: latitude + searchBuffer
})
const html = getListingDialogHTML(properties);
if (properties.length > 0) {
new mapboxgl.Popup()
.setLngLat([longtitude, latitude])
.setHTML(html)
.setMaxWidth("500px")
.addTo(map);
}
}
function getListingDialogHTML(properties) {
let listinHTMLs = [];
for (let property of properties) {
listinHTMLs.push(getPropertyHTML(property));
}
// separate them with a line
const result = listinHTMLs.join('<hr>');
const styledResult = `
<div class="scrollable-panel">
${result}
</div>
`
return styledResult;
}
function getPropertyHTML(property) {
const priceHistoryHTMLs = property.properties.price_history.map((d) => {
return `<li>${d.last_seen.split('T')[0]}: £${d.price}</li>`;
});
let priceHistoryHTML = '';
if (priceHistoryHTMLs.length > 1) {
priceHistoryHTML = `
<strong>Price history:</strong>
<ul>
${priceHistoryHTMLs.join('')}
</ul>
<br />
`
}
const lastSeenStr = property.properties.last_seen.split('T')[0];
const lastSeenDays = Math.round((new Date() - new Date(lastSeenStr)) / (1000 * 60 * 60 * 24));
return `
<div>
<img src="${property.properties.photo_thumbnail}" style="width:100%; height:auto;">
<p>
<strong>Available from:</strong> ${property.properties.available_from}
<br />
<strong>Price:</strong> £${property.properties.total_price}
<br />
${priceHistoryHTML}
<strong>Rooms:</strong> ${property.properties.rooms}
<br />
<strong>Area:</strong> ${property.properties.qm} m²
<br />
<strong>Price per area:</strong> £${property.properties.qmprice}/m²
<br />
<strong>Last seen:</strong> ${lastSeenDays} days ago
<br />
<strong>Agency:</strong> ${property.properties.agency}
<br />
<a href="${property.properties.url}" target="_blank">View Listing</a>
</p>
</div>
`;
}

View file

@ -6,16 +6,7 @@ import { Map } from './components/Map';
import { Parameters } from './components/Parameters';
function App() {
useEffect(() => {
const script = document.createElement('script');
script.src = './script.js';
script.async = true;
document.body.appendChild(script);
return () => {
document.body.removeChild(script); // Cleanup
};
}, []);
const [listingData, setListingData] = useState({});
const [user, setUser] = useState<User | null>(null);
useEffect(() => {
@ -65,8 +56,15 @@ function App() {
<p className="read-the-docs">
Click on the Vite and React logos to learn more
</p> */}
<Parameters />
<Map />
{/* <link href='https://api.mapbox.com/mapbox-gl-js/v3.12.0/mapbox-gl.css' rel='stylesheet' /> */}
<Parameters setListingData={setListingData} />
{Object.keys(listingData).length > 0 &&
<div style={{ width: '100%' }}>
<Map listingData={listingData} />
</div>
}
</>
)
}

View file

@ -1,6 +1,269 @@
export function Map() {
import crossfilter from "crossfilter2";
import * as d3 from "d3";
import mapboxgl from "mapbox-gl";
// import 'mapbox-gl/dist/mapbox-gl.css';
import { useEffect, useRef } from "react";
export function Map(
props: {
listingData: any;
}
) {
const data = props.listingData;
var crossData = data.features.map(function (d, i) {
//clone properties
var props = clone(d['properties']);
props['index'] = i;
return props;
});
const cf = crossfilter(crossData);
const qmDim = cf.dimension(function (d) { return d.qm; });
const cityDim = cf.dimension(function (d) { return d.city; });
const countryDim = cf.dimension(function (d) { return d.country; });
const rentDim = cf.dimension(function (d) { return d.total_price; });
const roomsDim = cf.dimension(function (d) { return d.rooms; });
const urlDim = cf.dimension(function (d) { return d.url; });
const indexDim = cf.dimension(function (d) { return d.index; });
let heatmap = null;
// rivet
var filter = { city: 'London', country: null, mode: 'qmprice' };
// filter['countries'] = Array.from(new Set(data.features.map(function (d) { return d['properties']['country'] })));
// rivets.bind(document.getElementById('overlay'), { filter: filter });
const mapRef = useRef(mapboxgl.Map)
const mapContainerRef = useRef('map')
useEffect(() => {
mapboxgl.accessToken = 'pk.eyJ1IjoiZGktdG8iLCJhIjoiY2o0bnBoYXcxMW1mNzJ3bDhmc2xiNWttaiJ9.ZccatVk_4shzoAsEUXXecA';
mapRef.current = new mapboxgl.Map({
container: mapContainerRef.current,
style: 'mapbox://styles/mapbox/light-v9',
center: [13.38032, 49.994210],
zoom: 5
});
mapRef.current.on('load', function () {
update()
})
mapRef.current.on('click', function (e) {
openListingsDialog(e.lngLat.lng, e.lngLat.lat);
})
return () => {
mapRef.current.remove()
}
}, [])
function clone(d) {
return JSON.parse(JSON.stringify(d));
}
function percentile(arr, p) {
if (arr.length === 0) return 0;
if (typeof p !== 'number') throw new TypeError('p must be a number');
if (p <= 0) return arr[0];
if (p >= 1) return arr[arr.length - 1];
var index = arr.length * p,
lower = Math.floor(index),
upper = lower + 1,
weight = index % 1;
if (upper >= arr.length) return arr[lower];
return arr[lower] * (1 - weight) + arr[upper] * weight;
}
function update() {
// close overlay
var modal = document.getElementById('modal_overlay');
modal.classList.toggle('modal-open');
// init heatmap
heatmap = new HexgridHeatmap(mapRef.current, "hexgrid-heatmap", "waterway-label");
heatmap.setIntensity(9); // dunno yet
heatmap.setSpread(0.05); // dunno yet
heatmap.setCellDensity(0.5); // small value == bigger hexagons
heatmap.setPropertyName(filter.mode);
if (filter.mode === 'qmprice') {
// if we visualize sqm based data, remove properties where we have no data
qmDim.filter(function (d) { return d > 0; });
}
// set filter
if (filter.city) {
cityDim.filterExact(filter.city);
} else if (filter.country) {
countryDim.filterExact(filter.country);
} else {
alert('nothing loadable');
}
filter.count = cityDim.top(Infinity).length;
var subset = { "type": "FeatureCollection", "features": [] };
indexDim.top(Infinity).forEach(function (i) {
subset.features.push(data.features[i.index]);
});
loadData(heatmap, subset);
}
function loadData(heatmap, subset) {
heatmap.setData(subset);
var values = subset.features.map(function (d) { return d['properties'][filter.mode] });
values = values.sort(function (a, b) { return a - b; });
// setting the color stops, min is at 5th percentile, max at 95percentile
var min = values[Math.round(values.length * 0.05)];
var max = values[Math.round(values.length * 0.95)];
var colorStopsPerc = [
[0, "rgba(0,185,243,0)"],
[25, "rgba(0,185,243,0.24)"],
[60, "rgba(255,223,0,0.3)"],
[100, "rgba(255,105,0,0.3)"],
];
makeLegend(colorStopsPerc, min, max);
var colorStopsValue = colorStopsPerc.map(function (d) {
return [min + d[0] * (max - min) / 100, d[1]];
});
heatmap.setColorStops(colorStopsValue);
heatmap.update();
//get bounding box and zoom to that area
// we use a 1% percentile since some data can be corrupt
var longitudes = subset.features.map(function (d) { return d.geometry.coordinates[0]; }).sort(function (a, b) { return a - b; });
var latitudes = subset.features.map(function (d) { return d.geometry.coordinates[1]; }).sort(function (a, b) { return a - b; });
var minlng = percentile(longitudes, 0.01);
var maxlng = percentile(longitudes, 0.99);
var minlat = percentile(latitudes, 0.01);
var maxlat = percentile(latitudes, 0.99);
mapRef.current.fitBounds([
[minlng, minlat],
[maxlng, maxlat]
]);
}
function makeLegend(colorstops, minValue, maxValue) {
/**
* colorstops: [[0, 'green'], [100, 'red']]
* @type {number}
*/
var svg_height = 300, svg_width = 70;
var svg = d3.select('#svg');
var defs = svg
.attr('height', svg_height)
.attr('width', svg_width);
var linearGradient = svg.append("defs")
.append("linearGradient")
.attr("id", "linear-gradient");
linearGradient
.attr("x1", "0%")
.attr("y1", "100%")
.attr("x2", "0%")
.attr("y2", "0%");
svg.append("rect")
.attr("width", svg_width * 0.4)
.attr("height", svg_height)
.attr('rx', 4)
.style("fill", "url(#linear-gradient)");
colorstops.forEach(function (d) {
linearGradient.append("stop")
.attr("offset", d[0] + "%")
.attr("stop-color", d[1]);
});
var xScale = d3.scaleLinear().range([svg_height - 20, 0]).domain([minValue, maxValue]);
var xAxis = d3.axisRight(xScale).ticks(5);
svg.append("g")
.attr("class", "axis")
.attr("transform", "translate(" + svg_width / 2 + "," + (10) + ")")
.call(xAxis);
}
function openListingsDialog(longtitude, latitude) {
const searchBuffer = 0.001 // ~100m
const properties = heatmap._tree.search({
minX: longtitude - searchBuffer,
maxX: longtitude + searchBuffer,
minY: latitude - searchBuffer,
maxY: latitude + searchBuffer
})
const html = getListingDialogHTML(properties);
if (properties.length > 0) {
new mapboxgl.Popup()
.setLngLat([longtitude, latitude])
.setHTML(html)
.setMaxWidth("500px")
.addTo(mapRef.current);
}
}
function getListingDialogHTML(properties) {
let listinHTMLs = [];
for (let property of properties) {
listinHTMLs.push(getPropertyHTML(property));
}
// separate them with a line
const result = listinHTMLs.join('<hr>');
const styledResult = `
<div class="scrollable-panel">
${result}
</div>
`
return styledResult;
}
function getPropertyHTML(property) {
const priceHistoryHTMLs = property.properties.price_history.map((d) => {
return `<li>${d.last_seen.split('T')[0]}: £${d.price}</li>`;
});
let priceHistoryHTML = '';
if (priceHistoryHTMLs.length > 1) {
priceHistoryHTML = `
<strong>Price history:</strong>
<ul>
${priceHistoryHTMLs.join('')}
</ul>
<br />
`
}
const lastSeenStr = property.properties.last_seen.split('T')[0];
const lastSeenDays = Math.round((new Date() - new Date(lastSeenStr)) / (1000 * 60 * 60 * 24));
return `
<div>
<img src="${property.properties.photo_thumbnail}" style="width:100%; height:auto;">
<p>
<strong>Available from:</strong> ${property.properties.available_from}
<br />
<strong>Price:</strong> £${property.properties.total_price}
<br />
${priceHistoryHTML}
<strong>Rooms:</strong> ${property.properties.rooms}
<br />
<strong>Area:</strong> ${property.properties.qm} m²
<br />
<strong>Price per area:</strong> £${property.properties.qmprice}/m²
<br />
<strong>Last seen:</strong> ${lastSeenDays} days ago
<br />
<strong>Agency:</strong> ${property.properties.agency}
<br />
<a href="${property.properties.url}" target="_blank">View Listing</a>
</p>
</div>
`;
}
return <>
<div id='map'></div>
<div id='map' ref={mapContainerRef}></div>
<div id="legend">
<svg id="svg">

View file

@ -1,8 +1,11 @@
import { useState } from "react";
import { useAuth } from "react-oidc-context";
export function Parameters() {
const [data, setData] = useState({});
export function Parameters(
props: {
setListingData: (data: any) => void,
}
) {
const [error, setError] = useState('')
const [isLoading, setIsLoading] = useState(false)
const { user } = useAuth(); // Get user data (includes token)
@ -11,7 +14,7 @@ export function Parameters() {
try {
const accessToken = user?.access_token;
const response = await fetch('/api/listing',
const response = await fetch('/api/listing_geojson',
{
method: 'GET',
headers: {
@ -22,14 +25,14 @@ export function Parameters() {
);
if (!response.ok) throw new Error('Error: ' + response.json());
const data: Response = await response.json();
setData(data);
props.setListingData(data);
} catch (err) {
setError('Failed to fetch data: ' + err);
alert(error)
} finally {
setIsLoading(false);
}
};
console.log(data)
return <>
<div className="modal modal-open" id="modal_overlay">
@ -65,8 +68,8 @@ export function Parameters() {
<hr className="modal-buttons-seperator" />
<label >What to visualize?</label>
<select className="u-full-width" id="heatmap_type" rv-value="filter.mode">
<option selected={true} value="qmprice">Price per squaremeter</option>
<select defaultValue={'qmprice'} className="u-full-width" id="heatmap_type" rv-value="filter.mode">
<option value="qmprice">Price per squaremeter</option>
<option value="rooms">Number of rooms</option>
<option value="qm">Squaremeter</option>
<option value="total_price">Price</option>

View file

@ -34,6 +34,5 @@
"include": [
"src",
"public/HexgridHeatmap.js",
"public/script.js"
]
}