"""Orchestration for FINN search + Eiendom.no enrichment + scoring. Analysis caching ---------------- ``analyze_ad`` caches its result under a ``deps_hash`` that is the SHA-256 of the combined raw payloads of the ad, the eiendom unit, and the comparable sales used to produce it. On a subsequent call the function: 1. Reads the three raw content hashes from the DB (no deserialisation). 2. Derives the same deps_hash from those hashes. 3. Checks analysis_cache for a matching (finnkode, deps_hash) row. 4. Returns the cached result immediately if found. 5. Otherwise runs the full scoring pipeline and writes to analysis_cache. The cached result is invalidated automatically the moment any piece of underlying data changes, because the deps_hash will differ. """ import logging from . import ad as ad_module from . import cache, eiendom_no, scoring, search from .cache import ( combine_hashes, get_analysis, get_eiendom_unit_hash, get_finn_ad_hash, get_similar_units_hash, invalidate_analysis, save_analysis, save_eiendom_unit, save_finn_ad, save_similar_units, ) from .config import ( EIENDOM_NO_CACHE_TTL_HOURS, FINN_CACHE_PATH, FINN_CACHE_TTL_AD_HOURS, FINN_DETAIL_LIMIT, FINN_MAX_SEARCH_PAGES, ) from .models import EiendomUnit, FinnAd, SimilarUnit logger = logging.getLogger(__name__) def _normalize_description(text: str | None) -> str: return text.lower() if text else "" def _is_resale_listing(url: str) -> bool: """True for ordinary resale ads. Project / new-build ads use different URL paths that fetch_ad_details cannot resolve (it builds a /homes/ URL).""" return "/realestate/homes/" in url def _build_ad_summary( ad: FinnAd, enriched: EiendomUnit | None, similar_units: list[SimilarUnit], scores: dict, categories: list[str], ) -> dict: description = _normalize_description(ad.listing_description) reasons = [] risks = [] next_steps = [ "Open the FINN listing and condition report.", "Review the Eiendom.no estimate and comparable sales.", "Ask the broker about renovation status and approvals.", ] if enriched and enriched.estimated_selling_price and ad.total_price: if ad.total_price < enriched.estimated_selling_price: reasons.append("Listing price is below Eiendom.no estimate.") elif ad.total_price <= enriched.estimated_selling_price_upper: reasons.append("Price sits within the local estimate range.") else: reasons.append("Listing price is above the estimate range.") else: reasons.append("Eiendom.no enrichment is unavailable or incomplete.") if "utsikt" in description or ad.has_balcony or ad.has_terrace: reasons.append("Outdoor space or view potential is positive.") if "hybel" in description or "leie" in description: reasons.append("Potential hybel/rental opportunity is mentioned.") if "potensial" in description or "renover" in description: reasons.append("Renovation or improvement potential is highlighted.") if scores.get("risk", 0.0) < 0: risks.append("Risk flags are detected in description or metadata.") if ad.common_costs and ad.common_costs > 5000: risks.append("Common costs are relatively high and should be reviewed.") if enriched and enriched.sale_status and enriched.sale_status.upper() != "FOR_SALE": risks.append("Eiendom.no sale status does not indicate an active sale.") if not enriched: risks.append("Missing Eiendom.no data increases uncertainty.") if not any("Eiendom.no" in step for step in next_steps): next_steps.append("Verify the property on Eiendom.no and reconcile any mismatches.") if similar_units: next_steps.append("Review the comparable units and average sqm prices.") else: next_steps.append("Comparable sales are unavailable; treat valuation with caution.") return { "why_interesting": reasons, "risks": risks, "next_steps": next_steps, "shortlist_reason": ", ".join(reasons[:3]) if reasons else "Review details and seller disclosures.", } def _compute_deps_hash( conn, finnkode: str, unit_code: str | None, listing_status: str = "RECENTLY_SOLD", ) -> str: """Derive a deps_hash from the three stored raw content hashes. Reads only the hash column -- no payload deserialisation. """ ad_hash = get_finn_ad_hash(conn, finnkode) unit_hash = get_eiendom_unit_hash(conn, unit_code) if unit_code else None comps_hash = ( get_similar_units_hash(conn, unit_code, listing_status) if unit_code else None ) return combine_hashes(ad_hash, unit_hash, comps_hash) async def analyze_ad( finn_ad: FinnAd, unit_code: str | None = None, ) -> dict: """Enrich a FinnAd and compute score summary. Result is cached in analysis_cache keyed by deps_hash. Recomputation happens only when the underlying raw data has actually changed. """ conn = cache.init_db(FINN_CACHE_PATH) # ------------------------------------------------------------------ # 1. Ensure the ad is in the DB so we have a stable hash to key on. # ------------------------------------------------------------------ ad_hash, ad_changed = save_finn_ad(conn, finn_ad) # ------------------------------------------------------------------ # 2. Fetch / refresh Eiendom.no data (cache-aware). # ------------------------------------------------------------------ enriched: EiendomUnit | None = None unit_hash_changed = False if unit_code: enriched = cache.get_eiendom_unit(conn, unit_code) if enriched is None: enriched = await eiendom_no.enrich_ad_with_eiendom_no(finn_ad, unit_code) if enriched is not None: _, unit_hash_changed = save_eiendom_unit(conn, enriched) # If already cached, unit_hash_changed stays False -- no new write. # ------------------------------------------------------------------ # 3. Fetch / refresh similar units (cache-aware). # ------------------------------------------------------------------ similar_units: list[SimilarUnit] = [] comps_hash_changed = False if enriched: similar_units = cache.get_similar_units( conn, enriched.unit_code, "RECENTLY_SOLD", ttl_hours=EIENDOM_NO_CACHE_TTL_HOURS ) if not similar_units: vector = enriched.unit_vector or eiendom_no.build_unit_vector(enriched) if vector: similar_units = await eiendom_no.get_similar_units(vector) if similar_units: _, comps_hash_changed = save_similar_units( conn, enriched.unit_code, "RECENTLY_SOLD", similar_units ) # ------------------------------------------------------------------ # 4. Derive deps_hash and check analysis_cache. # ------------------------------------------------------------------ deps_hash = _compute_deps_hash(conn, finn_ad.finnkode, unit_code) cached_analysis = get_analysis(conn, finn_ad.finnkode, deps_hash) if cached_analysis is not None: logger.debug("analysis_cache hit for %s -- skipping recompute", finn_ad.finnkode) return cached_analysis # ------------------------------------------------------------------ # 5. Cache miss: compute, store, return. # ------------------------------------------------------------------ logger.debug( "analysis_cache miss for %s (ad_changed=%s, unit_changed=%s, comps_changed=%s)", finn_ad.finnkode, ad_changed, unit_hash_changed, comps_hash_changed, ) scores = scoring.score_ad(finn_ad, enriched, similar_units) categories = scoring.classify_ad(scores) summary = _build_ad_summary(finn_ad, enriched, similar_units, scores, categories) result = { "finnkode": finn_ad.finnkode, "title": finn_ad.title, "address": finn_ad.address, "score": scores, "categories": categories, "summary": summary, "eiendom_unit": enriched.model_dump(mode="json") if enriched else None, "similar_units": [unit.model_dump(mode="json") for unit in similar_units], } # Round-trip through JSON to guarantee all values are serialisable # (catches any datetime that survives model_dump, e.g. from scoring). import json as _json result = _json.loads(_json.dumps(result, default=str)) save_analysis(conn, finn_ad.finnkode, deps_hash, result) return result async def _analyze_card(card, conn, *, include_eiendom_no: bool, client) -> dict: """Fetch details + enrich a single search card. Raises on unrecoverable errors; the caller is responsible for catching and skipping.""" finn_ad = cache.get_finn_ad(conn, card.finnkode, ttl_hours=FINN_CACHE_TTL_AD_HOURS) if finn_ad is None: finn_ad = await ad_module.fetch_ad_details(card.finnkode, client=client) unit_code = None if include_eiendom_no: try: matched_unit = await eiendom_no.search_unit_from_finn_url(card.url) unit_code = matched_unit.unit_code if matched_unit else None except Exception as exc: # A failed unit resolution is non-fatal -- proceed without enrichment. logger.warning("Eiendom.no unit search failed for %s: %s", card.finnkode, exc) unit_code = None return await analyze_ad(finn_ad, unit_code=unit_code) async def analyze_search( search_url: str, max_pages: int = FINN_MAX_SEARCH_PAGES, fetch_details: bool = True, detail_limit: int = FINN_DETAIL_LIMIT, include_eiendom_no: bool = True, client=None, use_cache: bool = True, ) -> dict: """Analyze a FINN search URL and enrich matching listings. Search-level results are NOT cached as a whole (the search page itself is cached at the HTML level). Individual ad analyses ARE cached via ``analyze_ad``, so re-running a search only re-scores ads whose underlying data has changed. """ conn = cache.init_db(FINN_CACHE_PATH) cards = await search.fetch_search_pages( search_url, max_pages=max_pages, client=client, use_cache=use_cache, ) results = [] enriched_count = 0 skipped_count = 0 cache_hits = 0 if fetch_details: for card in cards[:detail_limit]: # Project / new-build ads are not resale listings and fetch_ad_details # cannot resolve them -- skip up front rather than 404 mid-run. if not _is_resale_listing(card.url): logger.info("Skipping non-resale card %s (%s)", card.finnkode, card.url) skipped_count += 1 continue # One bad card (stale finnkode, removed ad, transient network error) # must not abort the whole search -- isolate each card. try: result = await _analyze_card( card, conn, include_eiendom_no=include_eiendom_no, client=client ) except Exception as exc: logger.warning("Skipping card %s: %s", card.finnkode, exc) skipped_count += 1 continue if result.get("eiendom_unit"): enriched_count += 1 # Track analysis cache hits via the absence of recompute logging # (the flag is not propagated up here; rely on debug logs). results.append(result) results.sort(key=lambda item: item["score"].get("total", 0.0), reverse=True) return { "search_url": search_url, "search_cards": [card.model_dump(mode="json") for card in cards], "analysis": results, "summary": { "total_listings": len(cards), "analyzed_listings": len(results), "skipped_listings": skipped_count, "eiendom_enriched": enriched_count, }, }