viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

make a few columns in the listing model indices to help with search

Browse source
This commit is contained in:
Viktor Barzin 2025-06-30 22:57:41 +00:00
parent 92c8403157
commit 5adffc8dcf
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
3 changed files with 58 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

46
crawler/alembic/versions/8220f657bae5_add_indices_to_commonly_searched_.py Normal file
View file

@ -0,0 +1,46 @@
"""add indices to commonly searched numeric columns
Revision ID: 8220f657bae5
Revises: 6363b18a22ca
Create Date: 2025-06-30 22:54:11.706618
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '8220f657bae5'
down_revision: Union[str, None] = '6363b18a22ca'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_index(op.f('ix_buylisting_last_seen'), 'buylisting', ['last_seen'], unique=False)
op.create_index(op.f('ix_buylisting_number_of_bedrooms'), 'buylisting', ['number_of_bedrooms'], unique=False)
op.create_index(op.f('ix_buylisting_price'), 'buylisting', ['price'], unique=False)
op.create_index(op.f('ix_buylisting_square_meters'), 'buylisting', ['square_meters'], unique=False)
op.create_index(op.f('ix_rentlisting_last_seen'), 'rentlisting', ['last_seen'], unique=False)
op.create_index(op.f('ix_rentlisting_number_of_bedrooms'), 'rentlisting', ['number_of_bedrooms'], unique=False)
op.create_index(op.f('ix_rentlisting_price'), 'rentlisting', ['price'], unique=False)
op.create_index(op.f('ix_rentlisting_square_meters'), 'rentlisting', ['square_meters'], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_rentlisting_square_meters'), table_name='rentlisting')
op.drop_index(op.f('ix_rentlisting_price'), table_name='rentlisting')
op.drop_index(op.f('ix_rentlisting_number_of_bedrooms'), table_name='rentlisting')
op.drop_index(op.f('ix_rentlisting_last_seen'), table_name='rentlisting')
op.drop_index(op.f('ix_buylisting_square_meters'), table_name='buylisting')
op.drop_index(op.f('ix_buylisting_price'), table_name='buylisting')
op.drop_index(op.f('ix_buylisting_number_of_bedrooms'), table_name='buylisting')
op.drop_index(op.f('ix_buylisting_last_seen'), table_name='buylisting')
# ### end Alembic commands ###

7
crawler/database.py
View file

@ -1,6 +1,10 @@
import os
from sqlmodel import create_engine, SQLModel
from sqlalchemy.orm import sessionmaker
from dotenv import load_dotenv
load_dotenv()
# PostgreSQL example (or use "sqlite:///database.db" for SQLite)
# DATABASE_URL = "postgresql://user:password@localhost/db_name"
@ -9,7 +13,8 @@ from sqlalchemy.orm import sessionmaker
DATABASE_URL = os.environ["DB_CONNECTION_STRING"]
engine = create_engine(DATABASE_URL, echo=False) # `echo=True` for debug logs
debug = os.getenv("ENV", "dev") == "prod"
engine = create_engine(DATABASE_URL, echo=debug) # `echo=True` for debug logs
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)

10
crawler/models/listing.py
View file

@ -55,9 +55,9 @@ class ListingSite(enum.StrEnum):
class Listing(SQLModel, table=False):
id: int = Field(primary_key=True)
price: float = Field(nullable=False)
number_of_bedrooms: int = Field(nullable=False)
square_meters: float | None = Field(default=None, nullable=True)
price: float = Field(nullable=False, index=True)
number_of_bedrooms: int = Field(nullable=False, index=True)
square_meters: float | None = Field(default=None, nullable=True, index=True)
agency: str | None = Field(default=None, nullable=True)
council_tax_band: str | None = Field(default=None, nullable=True)
longtitude: float = Field(nullable=False)
@ -65,7 +65,9 @@ class Listing(SQLModel, table=False):
# price_history: List[Dict[str, Any]] = Field(default_factory=list, sa_type=JSON)
price_history_json: str = Field(sa_type=TEXT)
listing_site: ListingSite = Field(nullable=False)
last_seen: datetime = Field(default_factory=datetime.now, nullable=False)
last_seen: datetime = Field(
default_factory=datetime.now, nullable=False, index=True
)
photo_thumbnail: str | None = Field(default=None, nullable=True)
floorplan_image_paths: List[str] = Field(
default_factory=list, sa_type=JSON, nullable=False