wrongmove/crawler/runall.sh
2025-06-08 18:18:38 +00:00

18 lines
1.1 KiB
Bash
Executable file

#!/usr/bin/env bash
set -euxo pipefail
DATA_DIR="data/rs/test"
LISTING_FILTER_OPTIONS="--min-price 2000 --max-price 4000 --min-bedrooms 2 --max-bedrooms 4 -t rent --available-from $(date +%Y-%m-%d) --last-seen-days 7 --furnish-types furnished"
#LISTING_FILTER_OPTIONS="--min-price 2000 --max-price 2500 --min-bedrooms 2 --max-bedrooms 4 -t rent --available-from $(date +%Y-%m-%d) --last-seen-days 7 --furnish-types furnished --district Islington" # DEBUG: UNCOMMENT ME WHEN TESTING
poetry install
alembic upgrade head # init db
python main.py --data-dir $DATA_DIR dump-listings $LISTING_FILTER_OPTIONS
python main.py --data-dir $DATA_DIR dump-images
python main.py --data-dir $DATA_DIR detect-floorplan
#python main.py --data-dir $DATA_DIR routing --destination-address 'Meta Brock Street' -m transit # NOTE: THIS CONSUMES API CALLS; USE CAREFULLY; add -l to limit number of entries
# python main.py --data-dir $DATA_DIR export-csv -O data/listings.csv # not used anymore - immoweb exporter is used to visualize data instead of csv
python main.py --data-dir $DATA_DIR export-immoweb -O ../immoweb/data/london_geojs.js $LISTING_FILTER_OPTIONS