rename frontend directory to frontend

crawler/frontend/public/robots.txt

@@ -0,0 +1,4 @@
+# Block ALL bots from ALL pages
+User-agent: *
+Disallow: /
+

crawler/frontend/public/script.js

@@ -0,0 +1,261 @@
+
+// rivet
+var filter = { city: 'London', country: null, mode: 'qmprice' };
+filter['countries'] = Array.from(new Set(data.features.map(function (d) { return d['properties']['country'] })));
+rivets.bind(document.getElementById('overlay'), { filter: filter });
+console.log('kekeke')
+
+
+function clone(d) {
+    return JSON.parse(JSON.stringify(d));
+}
+
+function percentile(arr, p) {
+    if (arr.length === 0) return 0;
+    if (typeof p !== 'number') throw new TypeError('p must be a number');
+    if (p <= 0) return arr[0];
+    if (p >= 1) return arr[arr.length - 1];
+
+    var index = arr.length * p,
+        lower = Math.floor(index),
+        upper = lower + 1,
+        weight = index % 1;
+
+    if (upper >= arr.length) return arr[lower];
+    return arr[lower] * (1 - weight) + arr[upper] * weight;
+}
+
+function update() {
+    // close overlay
+    var modal = document.getElementById('modal_overlay');
+    modal.classList.toggle('modal-open');
+
+    // init heatmap
+    heatmap = new HexgridHeatmap(map, "hexgrid-heatmap", "waterway-label");
+    heatmap.setIntensity(9); // dunno yet
+    heatmap.setSpread(0.05); // dunno yet
+    heatmap.setCellDensity(0.5); // small value == bigger hexagons
+    heatmap.setPropertyName(filter.mode);
+
+    if (filter.mode === 'qmprice') {
+        // if we visualize sqm based data, remove properties where we have no data
+        qmDim.filter(function (d) { return d > 0; });
+    }
+
+
+    // set filter
+    if (filter.city) {
+        cityDim.filterExact(filter.city);
+    } else if (filter.country) {
+        countryDim.filterExact(filter.country);
+    } else {
+        alert('nothing loadable');
+    }
+    filter.count = cityDim.top(Infinity).length;
+
+    var subset = { "type": "FeatureCollection", "features": [] };
+    indexDim.top(Infinity).forEach(function (i) {
+        subset.features.push(data.features[i.index]);
+    });
+
+    loadData(subset);
+}
+
+function loadData(subset) {
+    heatmap.setData(subset);
+    var values = subset.features.map(function (d) { return d['properties'][filter.mode] });
+    values = values.sort(function (a, b) { return a - b; });
+
+    // setting the color stops, min is at 5th percentile, max at 95percentile
+    var min = values[Math.round(values.length * 0.05)];
+    var max = values[Math.round(values.length * 0.95)];
+    var colorStopsPerc = [
+        [0, "rgba(0,185,243,0)"],
+        [25, "rgba(0,185,243,0.24)"],
+        [60, "rgba(255,223,0,0.3)"],
+        [100, "rgba(255,105,0,0.3)"],
+    ];
+    makeLegend(colorStopsPerc, min, max);
+    var colorStopsValue = colorStopsPerc.map(function (d) {
+        return [min + d[0] * (max - min) / 100, d[1]];
+    });
+    heatmap.setColorStops(colorStopsValue);
+    heatmap.update();
+
+    //get bounding box and zoom to that area
+    // we use a 1% percentile since some data can be corrupt
+    var longitudes = subset.features.map(function (d) { return d.geometry.coordinates[0]; }).sort(function (a, b) { return a - b; });
+    var latitudes = subset.features.map(function (d) { return d.geometry.coordinates[1]; }).sort(function (a, b) { return a - b; });
+    var minlng = percentile(longitudes, 0.01);
+    var maxlng = percentile(longitudes, 0.99);
+    var minlat = percentile(latitudes, 0.01);
+    var maxlat = percentile(latitudes, 0.99);
+    map.fitBounds([
+        [minlng, minlat],
+        [maxlng, maxlat]
+    ]);
+}
+
+function makeLegend(colorstops, minValue, maxValue) {
+    /**
+     * colorstops: [[0, 'green'], [100, 'red']]
+     * @type {number}
+     */
+    var svg_height = 300, svg_width = 70;
+    var svg = d3.select('#svg');
+    var defs = svg
+        .attr('height', svg_height)
+        .attr('width', svg_width);
+
+    var linearGradient = svg.append("defs")
+        .append("linearGradient")
+        .attr("id", "linear-gradient");
+
+    linearGradient
+        .attr("x1", "0%")
+        .attr("y1", "100%")
+        .attr("x2", "0%")
+        .attr("y2", "0%");
+
+    svg.append("rect")
+        .attr("width", svg_width * 0.4)
+        .attr("height", svg_height)
+        .attr('rx', 4)
+        .style("fill", "url(#linear-gradient)");
+
+    colorstops.forEach(function (d) {
+        linearGradient.append("stop")
+            .attr("offset", d[0] + "%")
+            .attr("stop-color", d[1]);
+    });
+
+
+    var xScale = d3.scaleLinear().range([svg_height - 20, 0]).domain([minValue, maxValue]);
+    var xAxis = d3.axisRight(xScale).ticks(5);
+
+    svg.append("g")
+        .attr("class", "axis")
+        .attr("transform", "translate(" + svg_width / 2 + "," + (10) + ")")
+        .call(xAxis);
+}
+
+// ORIGINAL BLOCK
+
+mapboxgl.accessToken = 'pk.eyJ1IjoiZGktdG8iLCJhIjoiY2o0bnBoYXcxMW1mNzJ3bDhmc2xiNWttaiJ9.ZccatVk_4shzoAsEUXXecA';
+var map = new mapboxgl.Map({
+    container: 'map',
+    style: 'mapbox://styles/mapbox/light-v9',
+    center: [13.38032, 49.994210],
+    zoom: 5
+});
+
+
+console.log('ekekek');
+map.on("load", function () {
+    var crossData = data.features.map(function (d, i) {
+        //clone properties
+        var props = clone(d['properties']);
+        props['index'] = i;
+        return props;
+    });
+    cf = crossfilter(crossData);
+    qmDim = cf.dimension(function (d) { return d.qm; });
+    cityDim = cf.dimension(function (d) { return d.city; });
+    countryDim = cf.dimension(function (d) { return d.country; });
+    rentDim = cf.dimension(function (d) { return d.total_price; });
+    roomsDim = cf.dimension(function (d) { return d.rooms; });
+    urlDim = cf.dimension(function (d) { return d.url; });
+    indexDim = cf.dimension(function (d) { return d.index; });
+});
+map.on('click', (e) => {
+    // {
+    //     lngLat: {
+    //         lng: 40.203,
+    //         lat: -74.451
+    //     },
+    //     originalEvent: {...},
+    //     point: {
+    //         x: 266,
+    //         y: 464
+    //     },
+    //      target: {...},
+    //      type: "click"
+    // }
+    openListingsDialog(e.lngLat.lng, e.lngLat.lat);
+});
+function openListingsDialog(longtitude, latitude) {
+    const searchBuffer = 0.001 // ~100m
+    const properties = heatmap._tree.search({
+        minX: longtitude - searchBuffer,
+        maxX: longtitude + searchBuffer,
+        minY: latitude - searchBuffer,
+        maxY: latitude + searchBuffer
+    })
+    const html = getListingDialogHTML(properties);
+    if (properties.length > 0) {
+        new mapboxgl.Popup()
+            .setLngLat([longtitude, latitude])
+            .setHTML(html)
+            .setMaxWidth("500px")
+            .addTo(map);
+    }
+}
+
+function getListingDialogHTML(properties) {
+    let listinHTMLs = [];
+    for (let property of properties) {
+        listinHTMLs.push(getPropertyHTML(property));
+    }
+    // separate them with a line
+    const result = listinHTMLs.join('<hr>');
+    const styledResult = `
+    <div class="scrollable-panel">
+        ${result}
+    </div>
+    `
+
+    return styledResult;
+}
+function getPropertyHTML(property) {
+    const priceHistoryHTMLs = property.properties.price_history.map((d) => {
+        return `<li>${d.last_seen.split('T')[0]}: £${d.price}</li>`;
+    });
+
+    let priceHistoryHTML = '';
+    if (priceHistoryHTMLs.length > 1) {
+        priceHistoryHTML = `
+        <strong>Price history:</strong>
+        <ul>
+            ${priceHistoryHTMLs.join('')}
+        </ul>
+        <br />
+        `
+    }
+    const lastSeenStr = property.properties.last_seen.split('T')[0];
+    const lastSeenDays = Math.round((new Date() - new Date(lastSeenStr)) / (1000 * 60 * 60 * 24));
+
+    return `
+        <div>
+            <img src="${property.properties.photo_thumbnail}" style="width:100%; height:auto;">
+            <p>
+                <strong>Available from:</strong> ${property.properties.available_from}
+                <br />
+                <strong>Price:</strong> £${property.properties.total_price}
+                <br />
+                ${priceHistoryHTML}
+                <strong>Rooms:</strong> ${property.properties.rooms}
+                <br />
+                <strong>Area:</strong> ${property.properties.qm} m²
+                <br />
+                <strong>Price per area:</strong> £${property.properties.qmprice}/m²
+                <br />
+                <strong>Last seen:</strong> ${lastSeenDays} days ago
+                <br />
+                <strong>Agency:</strong> ${property.properties.agency}
+                <br />
+                <a href="${property.properties.url}" target="_blank">View Listing</a>
+            </p>
+        </div>
+    `;
+
+}

crawler/frontend/public/vite.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>