wrongmove/crawler/runall.sh

#!/usr/bin/env bash

set -euxo pipefail

DATA_DIR="data/rs"

LISTING_FILTER_OPTIONS="--min-price 2000 --max-price 4000 --min-bedrooms 2 -t rent --available-from $(date +%Y-%m-%d) --last-seen-days 7 --furnish-types furnished"
#LISTING_FILTER_OPTIONS="--min-price 2000 --max-price 2500 --min-bedrooms 2 --max-bedrooms 4 -t rent --available-from $(date +%Y-%m-%d) --last-seen-days 7 --furnish-types furnished --district Islington" # DEBUG: UNCOMMENT ME WHEN TESTING


poetry install
alembic upgrade head # init db 
python main.py --data-dir $DATA_DIR dump-listings $LISTING_FILTER_OPTIONS
python main.py --data-dir $DATA_DIR dump-images 
python main.py --data-dir $DATA_DIR detect-floorplan
#python main.py --data-dir $DATA_DIR routing --destination-address 'Meta Brock Street' -m transit # NOTE: THIS CONSUMES API CALLS; USE CAREFULLY; add -l to limit number of entries
# python main.py --data-dir $DATA_DIR export-csv -O data/listings.csv # not used anymore - immoweb exporter is used to visualize data instead of csv
python main.py --data-dir $DATA_DIR export-immoweb -O ../immoweb/data/london_geojs.js $LISTING_FILTER_OPTIONS
add runall script, update parameters to 4 bed etc and allow incremental updating 2024-11-23 22:57:22 +00:00			`#!/usr/bin/env bash`

			`set -euxo pipefail`

some cleanups 2025-06-08 20:58:28 +00:00			`DATA_DIR="data/rs"`
migrate immoweb exporter to use models 2025-06-08 18:18:38 +00:00
some cleanups 2025-06-08 20:58:28 +00:00			`LISTING_FILTER_OPTIONS="--min-price 2000 --max-price 4000 --min-bedrooms 2 -t rent --available-from $(date +%Y-%m-%d) --last-seen-days 7 --furnish-types furnished"`
migrate immoweb exporter to use models 2025-06-08 18:18:38 +00:00			`#LISTING_FILTER_OPTIONS="--min-price 2000 --max-price 2500 --min-bedrooms 2 --max-bedrooms 4 -t rent --available-from $(date +%Y-%m-%d) --last-seen-days 7 --furnish-types furnished --district Islington" # DEBUG: UNCOMMENT ME WHEN TESTING`
merge dump listings and dump details commands - fetch both details and listings in the same command 2025-06-07 12:00:23 +00:00
update runall script to use the click entrypoint 2025-05-17 23:14:00 +00:00
migrate immoweb exporter to use models 2025-06-08 18:18:38 +00:00			`poetry install`
			`alembic upgrade head # init db`
refactor runall script to share filter options when dumping listings and when exporting to immoweb 2025-06-01 18:10:09 +00:00			`python main.py --data-dir $DATA_DIR dump-listings $LISTING_FILTER_OPTIONS`
update runall script to use the click entrypoint 2025-05-17 23:14:00 +00:00			`python main.py --data-dir $DATA_DIR dump-images`
			`python main.py --data-dir $DATA_DIR detect-floorplan`
update runall script to use new command for sourcing routes 2025-05-18 21:32:16 +00:00			`#python main.py --data-dir $DATA_DIR routing --destination-address 'Meta Brock Street' -m transit # NOTE: THIS CONSUMES API CALLS; USE CAREFULLY; add -l to limit number of entries`
refactor runall script to share filter options when dumping listings and when exporting to immoweb 2025-06-01 18:10:09 +00:00			`# python main.py --data-dir $DATA_DIR export-csv -O data/listings.csv # not used anymore - immoweb exporter is used to visualize data instead of csv`
			`python main.py --data-dir $DATA_DIR export-immoweb -O ../immoweb/data/london_geojs.js $LISTING_FILTER_OPTIONS`