viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

update runall script to use the click entrypoint

Browse source
This commit is contained in:
Viktor Barzin 2025-05-17 23:14:00 +00:00
parent b1e0ed170b
commit 47347543d2
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
2 changed files with 10 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crawler/3_dump_images.py
View file

@ -2,7 +2,7 @@ import asyncio
import json
import pathlib
import aiohttp
from tqdm import tqdm
from tqdm.asyncio import tqdm
from data_access import Listing
# Setting this too high either crashes rightmove or gets us blocked

15
crawler/runall.sh
View file

@ -2,9 +2,12 @@
set -euxo pipefail
mkdir -p /tmp/re/
python 1_dump_listings.py | tee -a /tmp/re/1.log | tee -a /tmp/re/log.log
python 2_dump_detail.py | tee -a /tmp/re/2.log | tee -a /tmp/re/log.log
python 3_dump_images.py | tee -a /tmp/re/3.log | tee -a /tmp/re/log.log
python 4_detect_floorplan.py | tee -a /tmp/re/4.log | tee -a /tmp/re/log.log
python 5_routing.py | tee -a /tmp/re/5.log | tee -a /tmp/re/log.log
DATA_DIR="data/rs"
python main.py --data-dir $DATA_DIR dump-listings --max-price 3500 --min-bedrooms 2 --max-bedrooms 4 --district islington -t rent
python main.py --data-dir $DATA_DIR dump-details
python main.py --data-dir $DATA_DIR dump-images
python main.py --data-dir $DATA_DIR detect-floorplan
#python 5_routing.py | tee -a /tmp/re/5.log | tee -a /tmp/re/log.log
python main.py --data-dir $DATA_DIR export-csv -O data/listings.csv