feat(refactor): Document refactoring progress and phases in markdown
feat(scripts): Add backfill script for content_hash in cache tables feat(scripts): Create recompute script for analysis_cache population test(tests): Implement comprehensive tests for analysis module functions fix(tests): Update CLI tests to assert errors on stderr instead of stdout fix(tests): Adjust MCP integration tests to pass context parameter correctly fix(tests): Modify service tests to return hash on save functions for consistency
This commit is contained in:
+205
-5
@@ -80,12 +80,14 @@ def init_db(path: str | None = None) -> sqlite3.Connection:
|
||||
url TEXT,
|
||||
payload TEXT NOT NULL,
|
||||
content_hash TEXT,
|
||||
fetched_at TEXT NOT NULL
|
||||
fetched_at TEXT NOT NULL,
|
||||
last_verified_at TEXT
|
||||
)
|
||||
"""
|
||||
)
|
||||
# Migration: add content_hash column if the table already existed without it.
|
||||
# Migrations: add columns if the table already existed without them.
|
||||
_add_column_if_missing(cursor, "finn_ads", "content_hash", "TEXT")
|
||||
_add_column_if_missing(cursor, "finn_ads", "last_verified_at", "TEXT")
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
@@ -136,6 +138,50 @@ def init_db(path: str | None = None) -> sqlite3.Connection:
|
||||
"""
|
||||
)
|
||||
|
||||
# New tables for Phase 2 enhancements
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS user_feedback (
|
||||
finnkode TEXT PRIMARY KEY,
|
||||
verdict TEXT NOT NULL,
|
||||
notes TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS price_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
finnkode TEXT NOT NULL,
|
||||
total_price INTEGER,
|
||||
asking_price INTEGER,
|
||||
sale_status TEXT,
|
||||
recorded_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_price_history_finnkode_recorded ON price_history(finnkode, recorded_at)")
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS search_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
search_url TEXT NOT NULL,
|
||||
finnkodes TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_search_runs_url_created ON search_runs(search_url, created_at)")
|
||||
|
||||
# Create indexes for efficient staleness queries
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_finn_ads_verified ON finn_ads(last_verified_at)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_eiendom_units_fetched ON eiendom_units(fetched_at)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_units_fetched ON similar_units(fetched_at)")
|
||||
|
||||
conn.commit()
|
||||
return conn
|
||||
|
||||
@@ -258,6 +304,8 @@ def save_finn_ad(conn: sqlite3.Connection, ad: FinnAd) -> tuple[str, bool]:
|
||||
if ad.detail_fetched_at
|
||||
else datetime.now(UTC).isoformat()
|
||||
)
|
||||
# Update last_verified_at to now when saving (indicates we just checked the data)
|
||||
last_verified_at = datetime.now(UTC).isoformat()
|
||||
|
||||
# Check existing hash before writing.
|
||||
cursor.execute(
|
||||
@@ -270,9 +318,9 @@ def save_finn_ad(conn: sqlite3.Connection, ad: FinnAd) -> tuple[str, bool]:
|
||||
|
||||
cursor.execute(
|
||||
"INSERT OR REPLACE INTO finn_ads"
|
||||
" (finnkode, url, payload, content_hash, fetched_at)"
|
||||
" VALUES (?, ?, ?, ?, ?)",
|
||||
(ad.finnkode, ad.url, json.dumps(payload, default=_json_default), new_hash, fetched_at),
|
||||
" (finnkode, url, payload, content_hash, fetched_at, last_verified_at)"
|
||||
" VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(ad.finnkode, ad.url, json.dumps(payload, default=_json_default), new_hash, fetched_at, last_verified_at),
|
||||
)
|
||||
conn.commit()
|
||||
logger.debug("finn_ad %s saved (hash=%s)", ad.finnkode, new_hash[:8])
|
||||
@@ -522,6 +570,158 @@ def invalidate_analysis(conn: sqlite3.Connection, finnkode: str) -> None:
|
||||
conn.commit()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# User feedback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def save_feedback(
|
||||
conn: sqlite3.Connection, finnkode: str, verdict: str, notes: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Store user feedback/verdict for a FINN listing."""
|
||||
cursor = conn.cursor()
|
||||
now = datetime.now(UTC).isoformat()
|
||||
cursor.execute(
|
||||
"INSERT OR REPLACE INTO user_feedback"
|
||||
" (finnkode, verdict, notes, created_at, updated_at)"
|
||||
" VALUES (?, ?, ?, ?, ?)",
|
||||
(finnkode, verdict, notes, now, now),
|
||||
)
|
||||
conn.commit()
|
||||
logger.debug("feedback saved for %s (verdict=%s)", finnkode, verdict)
|
||||
return {"finnkode": finnkode, "verdict": verdict, "notes": notes}
|
||||
|
||||
|
||||
def get_feedback(conn: sqlite3.Connection, finnkode: str) -> dict[str, Any] | None:
|
||||
"""Retrieve stored feedback for a FINN listing."""
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT finnkode, verdict, notes, created_at, updated_at FROM user_feedback WHERE finnkode = ?",
|
||||
(finnkode,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return {
|
||||
"finnkode": row["finnkode"],
|
||||
"verdict": row["verdict"],
|
||||
"notes": row["notes"],
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
|
||||
|
||||
def get_feedback_by_verdict(
|
||||
conn: sqlite3.Connection, verdict: str, limit: int = 100
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Retrieve all stored feedback with a given verdict."""
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT finnkode, verdict, notes, created_at, updated_at FROM user_feedback"
|
||||
" WHERE verdict = ? ORDER BY updated_at DESC LIMIT ?",
|
||||
(verdict, limit),
|
||||
)
|
||||
return [
|
||||
{
|
||||
"finnkode": row["finnkode"],
|
||||
"verdict": row["verdict"],
|
||||
"notes": row["notes"],
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
for row in cursor.fetchall()
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Price history
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def save_price_history(
|
||||
conn: sqlite3.Connection,
|
||||
finnkode: str,
|
||||
total_price: int | None = None,
|
||||
asking_price: int | None = None,
|
||||
sale_status: str | None = None,
|
||||
) -> None:
|
||||
"""Record a price/status snapshot for a listing."""
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"INSERT INTO price_history (finnkode, total_price, asking_price, sale_status, recorded_at)"
|
||||
" VALUES (?, ?, ?, ?, ?)",
|
||||
(finnkode, total_price, asking_price, sale_status, datetime.now(UTC).isoformat()),
|
||||
)
|
||||
conn.commit()
|
||||
logger.debug("price_history recorded for %s (total=%s, asking=%s)", finnkode, total_price, asking_price)
|
||||
|
||||
|
||||
def get_price_history(conn: sqlite3.Connection, finnkode: str, limit: int = 100) -> list[dict[str, Any]]:
|
||||
"""Retrieve price history for a listing."""
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT total_price, asking_price, sale_status, recorded_at FROM price_history"
|
||||
" WHERE finnkode = ? ORDER BY recorded_at DESC LIMIT ?",
|
||||
(finnkode, limit),
|
||||
)
|
||||
return [
|
||||
{
|
||||
"total_price": row["total_price"],
|
||||
"asking_price": row["asking_price"],
|
||||
"sale_status": row["sale_status"],
|
||||
"recorded_at": row["recorded_at"],
|
||||
}
|
||||
for row in cursor.fetchall()
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Search runs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def save_search_run(
|
||||
conn: sqlite3.Connection, search_url: str, finnkodes: list[str]
|
||||
) -> None:
|
||||
"""Record a search run with the finnkodes found."""
|
||||
cursor = conn.cursor()
|
||||
finnkodes_json = json.dumps(finnkodes)
|
||||
cursor.execute(
|
||||
"INSERT INTO search_runs (search_url, finnkodes, created_at)"
|
||||
" VALUES (?, ?, ?)",
|
||||
(search_url, finnkodes_json, datetime.now(UTC).isoformat()),
|
||||
)
|
||||
conn.commit()
|
||||
logger.debug("search_run recorded for %s (%d finnkodes)", search_url, len(finnkodes))
|
||||
|
||||
|
||||
def get_latest_search_run(conn: sqlite3.Connection, search_url: str) -> dict[str, Any] | None:
|
||||
"""Retrieve the most recent search run for a URL."""
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT search_url, finnkodes, created_at FROM search_runs"
|
||||
" WHERE search_url = ? ORDER BY created_at DESC LIMIT 1",
|
||||
(search_url,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return {
|
||||
"search_url": row["search_url"],
|
||||
"finnkodes": json.loads(row["finnkodes"]),
|
||||
"created_at": row["created_at"],
|
||||
}
|
||||
|
||||
|
||||
def delete_feedback(conn: sqlite3.Connection, finnkode: str) -> dict[str, Any]:
|
||||
"""Delete stored feedback for a FINN listing."""
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM user_feedback WHERE finnkode = ?", (finnkode,))
|
||||
conn.commit()
|
||||
logger.debug("feedback deleted for %s", finnkode)
|
||||
return {"finnkode": finnkode, "deleted": True}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user