initial
This commit is contained in:
@@ -0,0 +1,243 @@
|
||||
"""SQLite cache and persistence for FINN and Eiendom.no data."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from .config import FINN_CACHE_PATH
|
||||
from .models import EiendomUnit, FinnAd, FinnSearchCard, SimilarUnit
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_connection(path: str | None = None) -> sqlite3.Connection:
|
||||
db_path = path or FINN_CACHE_PATH
|
||||
conn = sqlite3.connect(str(db_path), detect_types=sqlite3.PARSE_DECLTYPES)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
|
||||
def init_db(path: str | None = None) -> sqlite3.Connection:
|
||||
conn = get_connection(path)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS finn_ads (
|
||||
finnkode TEXT PRIMARY KEY,
|
||||
url TEXT,
|
||||
payload TEXT NOT NULL,
|
||||
fetched_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS eiendom_units (
|
||||
unit_code TEXT PRIMARY KEY,
|
||||
payload TEXT NOT NULL,
|
||||
fetched_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS similar_units (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
unit_code TEXT NOT NULL,
|
||||
listing_status TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
fetched_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS cache_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
expires_at TEXT
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.commit()
|
||||
return conn
|
||||
|
||||
|
||||
def cache_get(conn: sqlite3.Connection, key: str) -> dict[str, Any] | None:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT value, expires_at FROM cache_meta WHERE key = ?", (key,))
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
|
||||
expires_at = row["expires_at"]
|
||||
if expires_at and datetime.fromisoformat(expires_at) < datetime.now(UTC):
|
||||
cursor.execute("DELETE FROM cache_meta WHERE key = ?", (key,))
|
||||
conn.commit()
|
||||
return None
|
||||
|
||||
return json.loads(row["value"])
|
||||
|
||||
|
||||
def cache_set(
|
||||
conn: sqlite3.Connection,
|
||||
key: str,
|
||||
payload: dict[str, Any],
|
||||
ttl_hours: int | None = None,
|
||||
ttl_minutes: int | None = None,
|
||||
) -> None:
|
||||
expires_at = None
|
||||
if ttl_minutes is not None:
|
||||
expires_at = (datetime.now(UTC) + timedelta(minutes=ttl_minutes)).isoformat()
|
||||
elif ttl_hours is not None:
|
||||
expires_at = (datetime.now(UTC) + timedelta(hours=ttl_hours)).isoformat()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"INSERT OR REPLACE INTO cache_meta (key, value, expires_at) VALUES (?, ?, ?)",
|
||||
(key, json.dumps(payload), expires_at),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _is_fresh(fetched_at: str, ttl_hours: int | None) -> bool:
|
||||
if ttl_hours is None:
|
||||
return True
|
||||
return datetime.fromisoformat(fetched_at) >= datetime.now(UTC) - timedelta(hours=ttl_hours)
|
||||
|
||||
|
||||
def save_search_page(
|
||||
conn: sqlite3.Connection,
|
||||
url: str,
|
||||
html: str,
|
||||
ttl_minutes: int = 60,
|
||||
) -> None:
|
||||
cache_set(conn, f"search_page:{url}", {"html": html}, ttl_minutes=ttl_minutes)
|
||||
|
||||
|
||||
def get_search_page(conn: sqlite3.Connection, url: str) -> str | None:
|
||||
payload = cache_get(conn, f"search_page:{url}")
|
||||
if not payload:
|
||||
return None
|
||||
return payload.get("html")
|
||||
|
||||
|
||||
def save_search_cards(
|
||||
conn: sqlite3.Connection,
|
||||
url: str,
|
||||
cards: list[FinnSearchCard],
|
||||
ttl_minutes: int = 60,
|
||||
) -> None:
|
||||
cache_set(
|
||||
conn,
|
||||
f"search_cards:{url}",
|
||||
[card.model_dump(mode="json") for card in cards],
|
||||
ttl_minutes=ttl_minutes,
|
||||
)
|
||||
|
||||
|
||||
def get_search_cards(conn: sqlite3.Connection, url: str) -> list[FinnSearchCard]:
|
||||
payload = cache_get(conn, f"search_cards:{url}")
|
||||
if not payload:
|
||||
return []
|
||||
return [FinnSearchCard.model_validate(item) for item in payload]
|
||||
|
||||
|
||||
def save_finn_ad(conn: sqlite3.Connection, ad: FinnAd) -> None:
|
||||
cursor = conn.cursor()
|
||||
payload = ad.model_dump(mode="json")
|
||||
cursor.execute(
|
||||
"INSERT OR REPLACE INTO finn_ads (finnkode, url, payload, fetched_at) VALUES (?, ?, ?, ?)",
|
||||
(
|
||||
ad.finnkode,
|
||||
ad.url,
|
||||
json.dumps(payload),
|
||||
ad.detail_fetched_at.isoformat()
|
||||
if ad.detail_fetched_at
|
||||
else datetime.now(UTC).isoformat(),
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_finn_ad(
|
||||
conn: sqlite3.Connection, finnkode: str, ttl_hours: int | None = None
|
||||
) -> FinnAd | None:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT payload, fetched_at FROM finn_ads WHERE finnkode = ?", (finnkode,))
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
if ttl_hours is not None and not _is_fresh(row["fetched_at"], ttl_hours):
|
||||
return None
|
||||
return FinnAd.model_validate(json.loads(row["payload"]))
|
||||
|
||||
|
||||
def save_eiendom_unit(conn: sqlite3.Connection, unit: EiendomUnit) -> None:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"INSERT OR REPLACE INTO eiendom_units (unit_code, payload, fetched_at) VALUES (?, ?, ?)",
|
||||
(unit.unit_code, json.dumps(unit.model_dump(mode="json")), unit.fetched_at.isoformat()),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_eiendom_unit(
|
||||
conn: sqlite3.Connection,
|
||||
unit_code: str,
|
||||
ttl_hours: int | None = None,
|
||||
) -> EiendomUnit | None:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT payload, fetched_at FROM eiendom_units WHERE unit_code = ?",
|
||||
(unit_code,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
if ttl_hours is not None and not _is_fresh(row["fetched_at"], ttl_hours):
|
||||
return None
|
||||
return EiendomUnit.model_validate(json.loads(row["payload"]))
|
||||
|
||||
|
||||
def save_similar_units(
|
||||
conn: sqlite3.Connection,
|
||||
unit_code: str,
|
||||
listing_status: str,
|
||||
similar_units: list[SimilarUnit],
|
||||
) -> None:
|
||||
cursor = conn.cursor()
|
||||
payload = json.dumps([item.model_dump(mode="json") for item in similar_units])
|
||||
cursor.execute(
|
||||
(
|
||||
"INSERT INTO similar_units"
|
||||
" (unit_code, listing_status, payload, fetched_at)"
|
||||
" VALUES (?, ?, ?, ?)"
|
||||
),
|
||||
(unit_code, listing_status, payload, datetime.now(UTC).isoformat()),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_similar_units(
|
||||
conn: sqlite3.Connection,
|
||||
unit_code: str,
|
||||
listing_status: str,
|
||||
ttl_hours: int | None = None,
|
||||
) -> list[SimilarUnit]:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
(
|
||||
"SELECT payload, fetched_at FROM similar_units"
|
||||
" WHERE unit_code = ? AND listing_status = ?"
|
||||
" ORDER BY id DESC LIMIT 1"
|
||||
),
|
||||
(unit_code, listing_status),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
return []
|
||||
if ttl_hours is not None and not _is_fresh(row["fetched_at"], ttl_hours):
|
||||
return []
|
||||
return [SimilarUnit.model_validate(item) for item in json.loads(row["payload"])]
|
||||
Reference in New Issue
Block a user