Flatten repo structure: move crawler/ to root, remove vqa/ and immoweb/
The crawler subdirectory was the only active project. Moving it to the repo root simplifies paths and removes the unnecessary nesting. The vqa/ and immoweb/ directories were legacy/unused and have been removed. Updated .drone.yml, .gitignore, .claude/ docs, and skills to reflect the new flat structure.
This commit is contained in:
Viktor Barzin
parent
e2247be700
commit
eafbc1ac52
221 changed files with 70 additions and 146140 deletions
82
alembic/versions/6363b18a22ca_initial.py
Normal file
82
alembic/versions/6363b18a22ca_initial.py
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
"""initial
|
||||
|
||||
Revision ID: 6363b18a22ca
|
||||
Revises:
|
||||
Create Date: 2025-06-22 15:27:23.187594
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
import sqlmodel
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '6363b18a22ca'
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('buylisting',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('price', sa.Float(), nullable=False),
|
||||
sa.Column('number_of_bedrooms', sa.Integer(), nullable=False),
|
||||
sa.Column('square_meters', sa.Float(), nullable=True),
|
||||
sa.Column('agency', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
||||
sa.Column('council_tax_band', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
||||
sa.Column('longitude', sa.Float(), nullable=False),
|
||||
sa.Column('latitude', sa.Float(), nullable=False),
|
||||
sa.Column('price_history_json', sa.TEXT(), nullable=False),
|
||||
sa.Column('listing_site', sa.Enum('RIGHTMOVE', name='listingsite'), nullable=False),
|
||||
sa.Column('last_seen', sa.DateTime(), nullable=False),
|
||||
sa.Column('photo_thumbnail', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
||||
sa.Column('floorplan_image_paths', sa.JSON(), nullable=False),
|
||||
sa.Column('additional_info', sa.JSON(), nullable=False),
|
||||
sa.Column('routing_info_json', sa.TEXT(), nullable=True),
|
||||
sa.Column('service_charge', sa.Float(), nullable=True),
|
||||
sa.Column('lease_left', sa.Integer(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('rentlisting',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('price', sa.Float(), nullable=False),
|
||||
sa.Column('number_of_bedrooms', sa.Integer(), nullable=False),
|
||||
sa.Column('square_meters', sa.Float(), nullable=True),
|
||||
sa.Column('agency', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
||||
sa.Column('council_tax_band', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
||||
sa.Column('longitude', sa.Float(), nullable=False),
|
||||
sa.Column('latitude', sa.Float(), nullable=False),
|
||||
sa.Column('price_history_json', sa.TEXT(), nullable=False),
|
||||
sa.Column('listing_site', sa.Enum('RIGHTMOVE', name='listingsite'), nullable=False),
|
||||
sa.Column('last_seen', sa.DateTime(), nullable=False),
|
||||
sa.Column('photo_thumbnail', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
||||
sa.Column('floorplan_image_paths', sa.JSON(), nullable=False),
|
||||
sa.Column('additional_info', sa.JSON(), nullable=False),
|
||||
sa.Column('routing_info_json', sa.TEXT(), nullable=True),
|
||||
sa.Column('available_from', sa.DateTime(), nullable=True),
|
||||
sa.Column('furnish_type', sa.Enum('FURNISHED', 'UNFURNISHED', 'PART_FURNISHED', 'ASK_LANDLORD', 'UNKNOWN', name='furnishtype'), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('user',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('email', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
||||
sa.Column('password', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_user_email'), 'user', ['email'], unique=True)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(op.f('ix_user_email'), table_name='user')
|
||||
op.drop_table('user')
|
||||
op.drop_table('rentlisting')
|
||||
op.drop_table('buylisting')
|
||||
# ### end Alembic commands ###
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
"""add indices to commonly searched numeric columns
|
||||
|
||||
Revision ID: 8220f657bae5
|
||||
Revises: 6363b18a22ca
|
||||
Create Date: 2025-06-30 22:54:11.706618
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '8220f657bae5'
|
||||
down_revision: Union[str, None] = '6363b18a22ca'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_index(op.f('ix_buylisting_last_seen'), 'buylisting', ['last_seen'], unique=False)
|
||||
op.create_index(op.f('ix_buylisting_number_of_bedrooms'), 'buylisting', ['number_of_bedrooms'], unique=False)
|
||||
op.create_index(op.f('ix_buylisting_price'), 'buylisting', ['price'], unique=False)
|
||||
op.create_index(op.f('ix_buylisting_square_meters'), 'buylisting', ['square_meters'], unique=False)
|
||||
op.create_index(op.f('ix_rentlisting_last_seen'), 'rentlisting', ['last_seen'], unique=False)
|
||||
op.create_index(op.f('ix_rentlisting_number_of_bedrooms'), 'rentlisting', ['number_of_bedrooms'], unique=False)
|
||||
op.create_index(op.f('ix_rentlisting_price'), 'rentlisting', ['price'], unique=False)
|
||||
op.create_index(op.f('ix_rentlisting_square_meters'), 'rentlisting', ['square_meters'], unique=False)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(op.f('ix_rentlisting_square_meters'), table_name='rentlisting')
|
||||
op.drop_index(op.f('ix_rentlisting_price'), table_name='rentlisting')
|
||||
op.drop_index(op.f('ix_rentlisting_number_of_bedrooms'), table_name='rentlisting')
|
||||
op.drop_index(op.f('ix_rentlisting_last_seen'), table_name='rentlisting')
|
||||
op.drop_index(op.f('ix_buylisting_square_meters'), table_name='buylisting')
|
||||
op.drop_index(op.f('ix_buylisting_price'), table_name='buylisting')
|
||||
op.drop_index(op.f('ix_buylisting_number_of_bedrooms'), table_name='buylisting')
|
||||
op.drop_index(op.f('ix_buylisting_last_seen'), table_name='buylisting')
|
||||
# ### end Alembic commands ###
|
||||
56
alembic/versions/a1b2c3d4e5f6_add_streaming_indexes.py
Normal file
56
alembic/versions/a1b2c3d4e5f6_add_streaming_indexes.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
"""add streaming indexes for query optimization
|
||||
|
||||
Revision ID: a1b2c3d4e5f6
|
||||
Revises: e5f1bc4e3323
|
||||
Create Date: 2026-02-01 12:00:00.000000
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'a1b2c3d4e5f6'
|
||||
down_revision: Union[str, None] = 'e5f1bc4e3323'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add composite and single-column indexes for streaming query optimization."""
|
||||
# Composite index for main query pattern (bedrooms, price, last_seen filtering)
|
||||
op.create_index(
|
||||
'ix_rentlisting_query_composite',
|
||||
'rentlisting',
|
||||
['number_of_bedrooms', 'price', 'last_seen'],
|
||||
unique=False
|
||||
)
|
||||
op.create_index(
|
||||
'ix_buylisting_query_composite',
|
||||
'buylisting',
|
||||
['number_of_bedrooms', 'price', 'last_seen'],
|
||||
unique=False
|
||||
)
|
||||
|
||||
# Missing single-column indexes for frequently filtered columns
|
||||
op.create_index(
|
||||
'ix_rentlisting_furnish_type',
|
||||
'rentlisting',
|
||||
['furnish_type'],
|
||||
unique=False
|
||||
)
|
||||
op.create_index(
|
||||
'ix_rentlisting_available_from',
|
||||
'rentlisting',
|
||||
['available_from'],
|
||||
unique=False
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove streaming indexes."""
|
||||
op.drop_index('ix_rentlisting_available_from', table_name='rentlisting')
|
||||
op.drop_index('ix_rentlisting_furnish_type', table_name='rentlisting')
|
||||
op.drop_index('ix_buylisting_query_composite', table_name='buylisting')
|
||||
op.drop_index('ix_rentlisting_query_composite', table_name='rentlisting')
|
||||
66
alembic/versions/b4c7d8e9f0a1_add_passkey_auth.py
Normal file
66
alembic/versions/b4c7d8e9f0a1_add_passkey_auth.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
"""add passkey auth
|
||||
|
||||
Revision ID: b4c7d8e9f0a1
|
||||
Revises: a1b2c3d4e5f6
|
||||
Create Date: 2025-07-15 10:00:00.000000
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
import sqlmodel
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'b4c7d8e9f0a1'
|
||||
down_revision: Union[str, None] = 'a1b2c3d4e5f6'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# Make user.password nullable
|
||||
op.alter_column('user', 'password',
|
||||
existing_type=sqlmodel.sql.sqltypes.AutoString(),
|
||||
nullable=True)
|
||||
|
||||
# Add created_at to user table
|
||||
op.add_column('user',
|
||||
sa.Column('created_at', sa.DateTime(),
|
||||
nullable=True,
|
||||
server_default=sa.func.now()))
|
||||
|
||||
# Create passkeycredential table
|
||||
op.create_table('passkeycredential',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('credential_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
||||
sa.Column('public_key', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
||||
sa.Column('sign_count', sa.Integer(), nullable=False),
|
||||
sa.Column('transports', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
||||
sa.Column('user_id', sa.Integer(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True,
|
||||
server_default=sa.func.now()),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['user.id']),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
)
|
||||
op.create_index(op.f('ix_passkeycredential_credential_id'),
|
||||
'passkeycredential', ['credential_id'], unique=True)
|
||||
op.create_index(op.f('ix_passkeycredential_user_id'),
|
||||
'passkeycredential', ['user_id'], unique=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
op.drop_index(op.f('ix_passkeycredential_user_id'),
|
||||
table_name='passkeycredential')
|
||||
op.drop_index(op.f('ix_passkeycredential_credential_id'),
|
||||
table_name='passkeycredential')
|
||||
op.drop_table('passkeycredential')
|
||||
|
||||
op.drop_column('user', 'created_at')
|
||||
|
||||
op.alter_column('user', 'password',
|
||||
existing_type=sqlmodel.sql.sqltypes.AutoString(),
|
||||
nullable=False)
|
||||
30
alembic/versions/e5f1bc4e3323_fix_typo_in_logitude_column.py
Normal file
30
alembic/versions/e5f1bc4e3323_fix_typo_in_logitude_column.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
"""fix typo in logitude column
|
||||
|
||||
Revision ID: e5f1bc4e3323
|
||||
Revises: 8220f657bae5
|
||||
Create Date: 2025-10-18 20:31:29.558034
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import mysql
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'e5f1bc4e3323'
|
||||
down_revision: Union[str, None] = '8220f657bae5'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema - this migration is now a no-op since tables already have correct column name."""
|
||||
# The tables were created with 'longitude' (correct spelling) in the initial migration.
|
||||
# This migration was incorrectly auto-generated and has been fixed to be a no-op.
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema - no-op since upgrade is no-op."""
|
||||
pass
|
||||
Loading…
Add table
Add a link
Reference in a new issue