viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

parameterize dump images step to work with custom data paths

Browse source
This commit is contained in:
Viktor Barzin 2025-05-14 21:01:58 +00:00
parent e424361ed9
commit 07fef7fbab
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
3 changed files with 19 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

7
crawler/data_access.py
View file

@ -19,7 +19,12 @@ class Listing:
for listing_path in listing_paths:
with open(listing_path) as f:
d = json.load(f)
identifiers.append(Listing(d["identifier"]))
# data_dir is the first directory before the listing_path
data_dir = pathlib.Path(listing_path)
while str(d['identifier']) in str(data_dir.resolve().absolute()):
data_dir = data_dir.parent
identifiers.append(Listing(d["identifier"], data_dir=data_dir))
return identifiers