Refactor and enhance various components of the FINN real estate analysis tool

- Updated docker-compose files to use local data volumes for development.
- Refactored analysis.py to improve code readability and performance, including changes to cache age calculations and hash computations.
- Enhanced cache.py to ensure the database directory is created if it doesn't exist and improved SQL query formatting.
- Modified cli.py to improve logging and statistics reporting for finn_ads.
- Updated config.py to streamline environment variable handling.
- Initialized the database eagerly in http_server.py to prevent runtime errors.
- Refactored mcp_server.py to slim down data structures and improve response formatting for API calls.
- Enhanced service.py to improve feedback handling and shortlist retrieval, ensuring enriched data is returned.
- Updated recompute_analysis_cache.py for better SQL query formatting.
This commit is contained in:
Ole
2026-05-29 15:17:11 +00:00
parent 55d93894ac
commit eb95b98111
10 changed files with 295 additions and 343 deletions
+13 -15
View File
@@ -132,9 +132,7 @@ def _compute_deps_hash(
"""
ad_hash = get_finn_ad_hash(conn, finnkode)
unit_hash = get_eiendom_unit_hash(conn, unit_code) if unit_code else None
comps_hash = (
get_similar_units_hash(conn, unit_code, listing_status) if unit_code else None
)
comps_hash = get_similar_units_hash(conn, unit_code, listing_status) if unit_code else None
return combine_hashes(ad_hash, unit_hash, comps_hash)
@@ -225,7 +223,7 @@ async def analyze_ad(
from datetime import datetime, UTC, timedelta
price_history = get_price_history(conn, finn_ad.finnkode, limit=20)
# Compute cache age: how long since we last fetched this ad
cursor = conn.cursor()
cursor.execute(
@@ -239,11 +237,11 @@ async def analyze_ad(
last_verified = db_row["last_verified_at"]
if last_verified:
last_verified_at = datetime.fromisoformat(last_verified)
structural_age_days = (datetime.now(UTC) - fetched_at).days
price_age_hours = (datetime.now(UTC) - last_verified_at).total_seconds() / 3600
structural_age_mins = (datetime.now(UTC) - fetched_at).total_seconds() / 60
price_age_mins = (datetime.now(UTC) - last_verified_at).total_seconds() / 60
cache_age = {
"structural_days": structural_age_days,
"price_hours": round(price_age_hours, 1),
"structural_minutes": round(structural_age_mins, 1),
"price_minutes": round(price_age_mins, 1),
}
result = {
@@ -282,6 +280,7 @@ async def analyze_ad(
# Round-trip through JSON to guarantee all values are serialisable
# (catches any datetime that survives model_dump, e.g. from scoring).
import json as _json
result = _json.loads(_json.dumps(result, default=str))
save_analysis(conn, finn_ad.finnkode, deps_hash, result)
@@ -301,7 +300,9 @@ async def _fetch_card_to_db(
treats None as a skip without aborting the whole batch.
"""
try:
finn_ad = cache.get_finn_ad(conn, card.finnkode, ttl_hours=FINN_CACHE_TTL_AD_STRUCTURAL_DAYS * 24)
finn_ad = cache.get_finn_ad(
conn, card.finnkode, ttl_hours=FINN_CACHE_TTL_AD_STRUCTURAL_DAYS * 24
)
if finn_ad is None:
finn_ad = await ad_module.fetch_ad_details(card.finnkode, client=client)
save_finn_ad(conn, finn_ad)
@@ -364,9 +365,7 @@ async def analyze_search(
skipped_count = len(cards[:detail_limit]) - len(resale_cards)
if ctx is not None:
await ctx.info(
f"Found {len(cards)} listings, {len(resale_cards)} resale ads to fetch."
)
await ctx.info(f"Found {len(cards)} listings, {len(resale_cards)} resale ads to fetch.")
# ------------------------------------------------------------------
# Phase 1: parallel fetch to DB
@@ -424,8 +423,7 @@ async def analyze_search(
if ctx is not None:
await ctx.info(
f"Done. {len(results)} analyzed, {enriched_count} enriched, "
f"{skipped_count} skipped."
f"Done. {len(results)} analyzed, {enriched_count} enriched, {skipped_count} skipped."
)
# Record this search run in the database
@@ -442,4 +440,4 @@ async def analyze_search(
"skipped_listings": skipped_count,
"eiendom_enriched": enriched_count,
},
}
}