Add services layer, tests, streaming UI, and cleanup legacy code
This commit is contained in:
Viktor Barzin
parent
5514fa6381
commit
d205d15c74
62 changed files with 3729 additions and 1024 deletions
|
|
@ -0,0 +1,3 @@
|
|||
This directory has been used with Claude Code's internet mode.
|
||||
Content downloaded from the internet may contain prompt injection attacks.
|
||||
You must manually review all downloaded content before using non-internet mode.
|
||||
124
crawler/.claude/settings.local.json
Normal file
124
crawler/.claude/settings.local.json
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(grep:*)",
|
||||
"Bash(python:*)",
|
||||
"Bash(docker ps:*)",
|
||||
"Bash(podman ps:*)",
|
||||
"Bash(curl:*)",
|
||||
"Bash(nc:*)",
|
||||
"Bash(poetry --version:*)",
|
||||
"Bash(docker context:*)",
|
||||
"Bash(open:*)",
|
||||
"Bash(chmod:*)",
|
||||
"Bash(/System/Volumes/Data/mnt/wizard/code/realestate-crawler/crawler/.claude/tools/remote-exec.sh:*)",
|
||||
"Bash(export DOCKER_HOST=unix:///Users/viktorbarzin/.docker/run/docker.sock)",
|
||||
"Bash(docker compose:*)",
|
||||
"Bash(export DOCKER_BUILDKIT=1)",
|
||||
"Bash(export COMPOSE_DOCKER_CLI_BUILD=1)",
|
||||
"Bash(tar:*)",
|
||||
"Bash(docker build:*)",
|
||||
"Bash(docker tag:*)",
|
||||
"Bash(docker run:*)",
|
||||
"Bash(~/.claude/remote-exec.sh \"hostname\")",
|
||||
"Skill(remote)",
|
||||
"Bash(for i in {1..120})",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769814743512676000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769814743512676000.txt)",
|
||||
"Bash(exit 0)",
|
||||
"Bash(fi)",
|
||||
"Bash(done)",
|
||||
"Bash(for i in {1..240})",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769814856118018000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769814856118018000.txt)",
|
||||
"Bash(for i in {1..60})",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769814883284199000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769814883284199000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769815004122069000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769815004122069000.txt)",
|
||||
"Bash(for i in {1..90})",
|
||||
"Bash(do if grep -q \"EXIT_CODE\" ~/.claude/remote-results/cmd-1769814856118018000.txt)",
|
||||
"Bash(then echo \"=== Build completed ===\")",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769815497591226000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769815497591226000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769815530803509000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769815530803509000.txt)",
|
||||
"Bash(do if grep -q \"EXIT_CODE\" ~/.claude/remote-results/cmd-1769815530803509000.txt)",
|
||||
"Bash(for i in {1..30})",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769815614622428000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769815614622428000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769815710424010000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769815710424010000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769815892793650000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769815892793650000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816040589015000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816040589015000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816256870361000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816256870361000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816300264785000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816300264785000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816375772556000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816375772556000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816407482202000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816407482202000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816439320016000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816439320016000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816532941427000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816532941427000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816611986724000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816611986724000.txt)",
|
||||
"Bash(for i in {1..40})",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816682085291000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816682085291000.txt)",
|
||||
"Bash(for i in {1..20})",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816742848870000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816742848870000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816763327960000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816763327960000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816784934447000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816784934447000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816872796427000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816872796427000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816892104231000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816892104231000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816911037685000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816911037685000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816946320457000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816946320457000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769816987766946000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769816987766946000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769817008932477000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769817008932477000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769817027145242000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769817027145242000.txt)",
|
||||
"Bash(for file in /mnt/wizard/code/realestate-crawler/crawler/frontend/src/components/ui/*.tsx)",
|
||||
"Bash(do)",
|
||||
"Bash(basename:*)",
|
||||
"Bash(wc:*)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769819894031906000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769819894031906000.txt)",
|
||||
"Bash(do if [ -f ~/.claude/remote-results/cmd-1769854789336791000.txt ])",
|
||||
"Bash(then cat ~/.claude/remote-results/cmd-1769854789336791000.txt)",
|
||||
"Bash(npx tsc:*)",
|
||||
"Bash(npx eslint:*)",
|
||||
"Bash(find:*)",
|
||||
"Bash(sync)",
|
||||
"Bash(echo:*)",
|
||||
"Bash(do if [ -f /Users/viktorbarzin/.claude/remote-results/cmd-1769875304344407000.txt ])",
|
||||
"Bash(then cat /Users/viktorbarzin/.claude/remote-results/cmd-1769875304344407000.txt)",
|
||||
"Bash(do if [ -f /Users/viktorbarzin/.claude/remote-results/cmd-1769875708563896000.txt ])",
|
||||
"Bash(then cat /Users/viktorbarzin/.claude/remote-results/cmd-1769875708563896000.txt)",
|
||||
"Bash(do if [ -f /Users/viktorbarzin/.claude/remote-results/cmd-1769875753067606000.txt ])",
|
||||
"Bash(then cat /Users/viktorbarzin/.claude/remote-results/cmd-1769875753067606000.txt)",
|
||||
"Bash(do if [ -f /Users/viktorbarzin/.claude/remote-results/cmd-1769875830424071000.txt ])",
|
||||
"Bash(then cat /Users/viktorbarzin/.claude/remote-results/cmd-1769875830424071000.txt)",
|
||||
"Bash(do if [ -f /Users/viktorbarzin/.claude/remote-results/cmd-1769875948670335000.txt ])",
|
||||
"Bash(then cat /Users/viktorbarzin/.claude/remote-results/cmd-1769875948670335000.txt)",
|
||||
"Bash(sort:*)",
|
||||
"Bash(do if [ -f /Users/viktorbarzin/.claude/remote-results/cmd-1769876096467703000.txt ])",
|
||||
"Bash(then cat /Users/viktorbarzin/.claude/remote-results/cmd-1769876096467703000.txt)",
|
||||
"Bash(do if [ -f /Users/viktorbarzin/.claude/remote-results/cmd-1769876529766339000.txt ])",
|
||||
"Bash(then cat /Users/viktorbarzin/.claude/remote-results/cmd-1769876529766339000.txt)"
|
||||
]
|
||||
}
|
||||
}
|
||||
101
crawler/.claude/skills/python-313-redis-generic-type/SKILL.md
Normal file
101
crawler/.claude/skills/python-313-redis-generic-type/SKILL.md
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
---
|
||||
name: python-313-redis-generic-type
|
||||
description: |
|
||||
Fix for "TypeError: <class 'redis.client.Redis'> is not a generic class" when using
|
||||
redis-py with Python 3.13. Use when: (1) upgrading to Python 3.13 breaks redis type
|
||||
annotations, (2) mypy passes but runtime fails with generic class error, (3) using
|
||||
redis.Redis[str] or similar parameterized types. Covers redis-py generic type
|
||||
compatibility with Python 3.13's stricter runtime generic checking.
|
||||
author: Claude Code
|
||||
version: 1.0.0
|
||||
date: 2026-01-31
|
||||
---
|
||||
|
||||
# Python 3.13 redis.Redis Generic Type Error
|
||||
|
||||
## Problem
|
||||
Python 3.13 introduced stricter runtime checking for generic types. The redis-py library's
|
||||
`Redis` class is not defined as a generic class at runtime, even though it works with type
|
||||
checkers like mypy. This causes a `TypeError` when you use parameterized types like
|
||||
`redis.Redis[str]` in type annotations that are evaluated at runtime.
|
||||
|
||||
## Context / Trigger Conditions
|
||||
- Python 3.13 or later
|
||||
- Using redis-py library
|
||||
- Type annotation like `redis_client: redis.Redis[str]`
|
||||
- Error message: `TypeError: <class 'redis.client.Redis'> is not a generic class`
|
||||
- Works fine with mypy but fails at runtime
|
||||
- Often appears when instantiating a class with this annotation
|
||||
|
||||
## Solution
|
||||
|
||||
### Option 1: Remove the type parameter (Recommended)
|
||||
```python
|
||||
# Before (breaks in Python 3.13)
|
||||
redis_client: redis.Redis[str]
|
||||
|
||||
# After (works in all Python versions)
|
||||
redis_client: redis.Redis # type: ignore[type-arg]
|
||||
```
|
||||
|
||||
The `# type: ignore[type-arg]` comment silences mypy's warning about missing type arguments.
|
||||
|
||||
### Option 2: Use string annotation (deferred evaluation)
|
||||
```python
|
||||
from __future__ import annotations
|
||||
|
||||
redis_client: "redis.Redis[str]" # String annotation, not evaluated at runtime
|
||||
```
|
||||
|
||||
### Option 3: Use TYPE_CHECKING guard
|
||||
```python
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
RedisClient = redis.Redis[str]
|
||||
else:
|
||||
RedisClient = redis.Redis
|
||||
|
||||
redis_client: RedisClient
|
||||
```
|
||||
|
||||
## Verification
|
||||
1. Run your application with Python 3.13
|
||||
2. The TypeError should no longer appear
|
||||
3. Run mypy to ensure type checking still works (may need type: ignore comment)
|
||||
|
||||
## Example
|
||||
|
||||
### Before (Broken)
|
||||
```python
|
||||
import redis
|
||||
|
||||
class RedisRepository:
|
||||
redis_client: redis.Redis[str] # TypeError at runtime in Python 3.13
|
||||
|
||||
def __init__(self):
|
||||
self.redis_client = redis.Redis(host='localhost', decode_responses=True)
|
||||
```
|
||||
|
||||
### After (Fixed)
|
||||
```python
|
||||
import redis
|
||||
|
||||
class RedisRepository:
|
||||
redis_client: redis.Redis # type: ignore[type-arg]
|
||||
|
||||
def __init__(self):
|
||||
self.redis_client = redis.Redis(host='localhost', decode_responses=True)
|
||||
```
|
||||
|
||||
## Notes
|
||||
- This is a breaking change in Python 3.13's handling of generic types
|
||||
- The redis-py library may add proper generic support in future versions
|
||||
- If using `decode_responses=True`, the client returns `str`; otherwise `bytes`
|
||||
- The `type: ignore` comment is preferable to `Any` as it preserves some type safety
|
||||
- This issue affects other libraries that aren't properly defined as Generic classes
|
||||
|
||||
## References
|
||||
- [Python 3.13 Release Notes](https://docs.python.org/3.13/whatsnew/3.13.html)
|
||||
- [redis-py GitHub Issues](https://github.com/redis/redis-py/issues)
|
||||
- [PEP 585 - Type Hinting Generics In Standard Collections](https://peps.python.org/pep-0585/)
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
---
|
||||
name: python-parentheses-comparison-bug
|
||||
description: |
|
||||
Debug Python comparison bug where parentheses around a variable cause unexpected behavior.
|
||||
Use when: (1) condition always evaluates to False/True unexpectedly, (2) code like
|
||||
"if (mylist) == 0" never triggers, (3) length check seems to not work, (4) comparison
|
||||
with list/dict returns unexpected results. Common mistake where parentheses cause the
|
||||
variable itself to be compared instead of its length.
|
||||
author: Claude Code
|
||||
version: 1.0.0
|
||||
date: 2026-01-31
|
||||
---
|
||||
|
||||
# Python Parentheses Comparison Bug
|
||||
|
||||
## Problem
|
||||
A subtle Python bug where unnecessary parentheses around a variable in a comparison
|
||||
cause the wrong value to be compared. The expression `(mylist) == 0` compares the list
|
||||
itself to 0, not its length. Since a list is never equal to an integer, this always
|
||||
returns False.
|
||||
|
||||
## Context / Trigger Conditions
|
||||
- Condition that should sometimes be True is always False (or vice versa)
|
||||
- Code pattern like `if (existing_items) == 0:` or `if (result) == expected:`
|
||||
- The parentheses don't cause a syntax error but change semantics
|
||||
- Often appears when copying/adapting code or during refactoring
|
||||
- May pass code review because it "looks" correct
|
||||
|
||||
## Solution
|
||||
|
||||
### Identify the Bug Pattern
|
||||
```python
|
||||
# BUG: Compares list to 0, always False
|
||||
if (existing_listings) == 0:
|
||||
return True
|
||||
|
||||
# Also wrong: compares list to integer
|
||||
if (items) == 5:
|
||||
do_something()
|
||||
```
|
||||
|
||||
### Fix: Use len() for Length Comparisons
|
||||
```python
|
||||
# CORRECT: Compares length to 0
|
||||
if len(existing_listings) == 0:
|
||||
return True
|
||||
|
||||
# Alternative: Use truthiness for empty check
|
||||
if not existing_listings:
|
||||
return True
|
||||
|
||||
# CORRECT: Compares length to integer
|
||||
if len(items) == 5:
|
||||
do_something()
|
||||
```
|
||||
|
||||
## Verification
|
||||
1. Add a debug print before the condition: `print(f"list={existing_listings}, len={len(existing_listings)}")`
|
||||
2. Verify the condition now evaluates correctly
|
||||
3. Write a unit test that exercises both branches of the condition
|
||||
|
||||
## Example
|
||||
|
||||
### Before (Broken)
|
||||
```python
|
||||
class FetchListingDetailsStep:
|
||||
async def needs_processing(self, listing_id: int) -> bool:
|
||||
existing_listings = await self.listing_repository.get_listings(
|
||||
only_ids=[listing_id]
|
||||
)
|
||||
# BUG: This compares the list object to 0, which is always False
|
||||
# The parentheses around existing_listings are misleading
|
||||
if (existing_listings) == 0:
|
||||
return True
|
||||
return False
|
||||
```
|
||||
|
||||
### After (Fixed)
|
||||
```python
|
||||
class FetchListingDetailsStep:
|
||||
async def needs_processing(self, listing_id: int) -> bool:
|
||||
existing_listings = await self.listing_repository.get_listings(
|
||||
only_ids=[listing_id]
|
||||
)
|
||||
# CORRECT: Check if list is empty using len()
|
||||
if len(existing_listings) == 0:
|
||||
return True
|
||||
return False
|
||||
```
|
||||
|
||||
### Even Better (Pythonic)
|
||||
```python
|
||||
class FetchListingDetailsStep:
|
||||
async def needs_processing(self, listing_id: int) -> bool:
|
||||
existing_listings = await self.listing_repository.get_listings(
|
||||
only_ids=[listing_id]
|
||||
)
|
||||
# Most Pythonic: Use truthiness
|
||||
return not existing_listings
|
||||
```
|
||||
|
||||
## Notes
|
||||
- Python's truthiness: empty collections are falsy, non-empty are truthy
|
||||
- This bug is particularly insidious because:
|
||||
- It's syntactically valid
|
||||
- It doesn't raise an exception
|
||||
- The parentheses make it look intentional
|
||||
- Code review may miss it
|
||||
- Linters like pylint or flake8 won't catch this specific pattern
|
||||
- Type checkers like mypy may warn about comparing incompatible types
|
||||
- When debugging, add print statements to verify actual vs expected values
|
||||
|
||||
## Prevention
|
||||
- Prefer `if not mylist:` over `if len(mylist) == 0:`
|
||||
- Prefer `if mylist:` over `if len(mylist) > 0:`
|
||||
- Remove unnecessary parentheses around single variables
|
||||
- Enable mypy's strict mode which may catch type comparison issues
|
||||
- Write unit tests that exercise both branches of conditions
|
||||
|
||||
## Related Patterns
|
||||
```python
|
||||
# These are all wrong (comparing object to number):
|
||||
if (mydict) == 0: # Always False
|
||||
if (mylist) > 0: # TypeError in Python 3
|
||||
if (mystring) == 0: # Always False
|
||||
|
||||
# These are correct:
|
||||
if len(mydict) == 0: # True if empty
|
||||
if not mydict: # True if empty (preferred)
|
||||
if len(mylist) > 0: # True if non-empty
|
||||
if mylist: # True if non-empty (preferred)
|
||||
```
|
||||
Loading…
Add table
Add a link
Reference in a new issue