From 5adffc8dcf9919a146397e68802a3fbabd9e7b74 Mon Sep 17 00:00:00 2001
From: Viktor Barzin <vbarzin@gmail.com>
Date: Mon, 30 Jun 2025 22:57:41 +0000
Subject: [PATCH] make a few columns in the listing model indices to help with
 search

---
 ...7bae5_add_indices_to_commonly_searched_.py | 46 +++++++++++++++++++
 crawler/database.py                           |  7 ++-
 crawler/models/listing.py                     | 10 ++--
 3 files changed, 58 insertions(+), 5 deletions(-)
 create mode 100644 crawler/alembic/versions/8220f657bae5_add_indices_to_commonly_searched_.py

diff --git a/crawler/alembic/versions/8220f657bae5_add_indices_to_commonly_searched_.py b/crawler/alembic/versions/8220f657bae5_add_indices_to_commonly_searched_.py
new file mode 100644
index 0000000..5230e61
--- /dev/null
+++ b/crawler/alembic/versions/8220f657bae5_add_indices_to_commonly_searched_.py
@@ -0,0 +1,46 @@
+"""add indices to commonly searched numeric columns
+
+Revision ID: 8220f657bae5
+Revises: 6363b18a22ca
+Create Date: 2025-06-30 22:54:11.706618
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '8220f657bae5'
+down_revision: Union[str, None] = '6363b18a22ca'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_index(op.f('ix_buylisting_last_seen'), 'buylisting', ['last_seen'], unique=False)
+    op.create_index(op.f('ix_buylisting_number_of_bedrooms'), 'buylisting', ['number_of_bedrooms'], unique=False)
+    op.create_index(op.f('ix_buylisting_price'), 'buylisting', ['price'], unique=False)
+    op.create_index(op.f('ix_buylisting_square_meters'), 'buylisting', ['square_meters'], unique=False)
+    op.create_index(op.f('ix_rentlisting_last_seen'), 'rentlisting', ['last_seen'], unique=False)
+    op.create_index(op.f('ix_rentlisting_number_of_bedrooms'), 'rentlisting', ['number_of_bedrooms'], unique=False)
+    op.create_index(op.f('ix_rentlisting_price'), 'rentlisting', ['price'], unique=False)
+    op.create_index(op.f('ix_rentlisting_square_meters'), 'rentlisting', ['square_meters'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_rentlisting_square_meters'), table_name='rentlisting')
+    op.drop_index(op.f('ix_rentlisting_price'), table_name='rentlisting')
+    op.drop_index(op.f('ix_rentlisting_number_of_bedrooms'), table_name='rentlisting')
+    op.drop_index(op.f('ix_rentlisting_last_seen'), table_name='rentlisting')
+    op.drop_index(op.f('ix_buylisting_square_meters'), table_name='buylisting')
+    op.drop_index(op.f('ix_buylisting_price'), table_name='buylisting')
+    op.drop_index(op.f('ix_buylisting_number_of_bedrooms'), table_name='buylisting')
+    op.drop_index(op.f('ix_buylisting_last_seen'), table_name='buylisting')
+    # ### end Alembic commands ###
diff --git a/crawler/database.py b/crawler/database.py
index e6b9d46..8d977de 100644
--- a/crawler/database.py
+++ b/crawler/database.py
@@ -1,6 +1,10 @@
 import os
 from sqlmodel import create_engine, SQLModel
 from sqlalchemy.orm import sessionmaker
+from dotenv import load_dotenv
+
+load_dotenv()
+
 
 # PostgreSQL example (or use "sqlite:///database.db" for SQLite)
 # DATABASE_URL = "postgresql://user:password@localhost/db_name"
@@ -9,7 +13,8 @@ from sqlalchemy.orm import sessionmaker
 DATABASE_URL = os.environ["DB_CONNECTION_STRING"]
 
 
-engine = create_engine(DATABASE_URL, echo=False)  # `echo=True` for debug logs
+debug = os.getenv("ENV", "dev") == "prod"
+engine = create_engine(DATABASE_URL, echo=debug)  # `echo=True` for debug logs
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 
diff --git a/crawler/models/listing.py b/crawler/models/listing.py
index 0a4b144..10ca7f4 100644
--- a/crawler/models/listing.py
+++ b/crawler/models/listing.py
@@ -55,9 +55,9 @@ class ListingSite(enum.StrEnum):
 
 class Listing(SQLModel, table=False):
     id: int = Field(primary_key=True)
-    price: float = Field(nullable=False)
-    number_of_bedrooms: int = Field(nullable=False)
-    square_meters: float | None = Field(default=None, nullable=True)
+    price: float = Field(nullable=False, index=True)
+    number_of_bedrooms: int = Field(nullable=False, index=True)
+    square_meters: float | None = Field(default=None, nullable=True, index=True)
     agency: str | None = Field(default=None, nullable=True)
     council_tax_band: str | None = Field(default=None, nullable=True)
     longtitude: float = Field(nullable=False)
@@ -65,7 +65,9 @@ class Listing(SQLModel, table=False):
     # price_history: List[Dict[str, Any]] = Field(default_factory=list, sa_type=JSON)
     price_history_json: str = Field(sa_type=TEXT)
     listing_site: ListingSite = Field(nullable=False)
-    last_seen: datetime = Field(default_factory=datetime.now, nullable=False)
+    last_seen: datetime = Field(
+        default_factory=datetime.now, nullable=False, index=True
+    )
     photo_thumbnail: str | None = Field(default=None, nullable=True)
     floorplan_image_paths: List[str] = Field(
         default_factory=list, sa_type=JSON, nullable=False