Flatten repo structure: move crawler/ to root, remove vqa/ and immoweb/
The crawler subdirectory was the only active project. Moving it to the repo root simplifies paths and removes the unnecessary nesting. The vqa/ and immoweb/ directories were legacy/unused and have been removed. Updated .drone.yml, .gitignore, .claude/ docs, and skills to reflect the new flat structure.
This commit is contained in:
parent
e2247be700
commit
eafbc1ac52
221 changed files with 70 additions and 146140 deletions
|
|
@ -1,50 +0,0 @@
|
|||
"""Redis-based distributed locking for task coordination."""
|
||||
import logging
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from typing import Generator
|
||||
|
||||
import redis
|
||||
|
||||
logger = logging.getLogger("uvicorn.error")
|
||||
|
||||
|
||||
def get_redis_client() -> redis.Redis:
|
||||
"""Get Redis client from Celery broker URL."""
|
||||
broker_url = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0")
|
||||
return redis.from_url(broker_url, decode_responses=True)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def redis_lock(
|
||||
lock_name: str, timeout: int = 3600 * 4
|
||||
) -> Generator[bool, None, None]:
|
||||
"""Distributed lock using Redis.
|
||||
|
||||
Args:
|
||||
lock_name: Unique name for the lock
|
||||
timeout: Lock expiration time in seconds (default: 4 hours)
|
||||
|
||||
Yields:
|
||||
bool: True if lock was acquired, False otherwise
|
||||
|
||||
Example:
|
||||
with redis_lock("scrape_listings") as acquired:
|
||||
if not acquired:
|
||||
logger.warning("Another scrape is already running")
|
||||
return
|
||||
# ... do work ...
|
||||
"""
|
||||
client = get_redis_client()
|
||||
lock_key = f"lock:{lock_name}"
|
||||
|
||||
# Try to acquire the lock
|
||||
acquired = client.set(lock_key, "1", nx=True, ex=timeout)
|
||||
|
||||
try:
|
||||
yield bool(acquired)
|
||||
finally:
|
||||
# Release the lock only if we acquired it
|
||||
if acquired:
|
||||
client.delete(lock_key)
|
||||
logger.info(f"Released lock: {lock_name}")
|
||||
Loading…
Add table
Add a link
Reference in a new issue