from collections import defaultdict
from fastapi import FastAPI, APIRouter, BackgroundTasks, Depends, HTTPException,Body,Path,Query,status, UploadFile, File, Form
from pydantic import BaseModel, validator
from typing import Dict, Optional, Any,Tuple
from datetime import datetime,date,timedelta
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, extract, func,text
from src.core.database import get_db_on_the_fly
from src.financial.model import GeneralLedger, GeneralLedgerMaster
from enum import Enum
from dateutil.relativedelta import relativedelta
import calendar
import re
from scipy import stats
import pandas as pd
from prophet import Prophet
from fastapi.security import OAuth2PasswordBearer
from io import BytesIO, StringIO
import json
import ast
from calendar import monthrange
from sklearn.linear_model import LinearRegression
import numpy as np
import io
import warnings
warnings.filterwarnings("ignore")
from sklearn.linear_model import Ridge
from sklearn.preprocessing import StandardScaler
from fastapi.responses import StreamingResponse
import openpyxl
from statsmodels.tsa.stattools import adfuller
from concurrent.futures import ThreadPoolExecutor

forecast_router = APIRouter()

from .schemas import ForecastScenarioInput,ForecastScenarioUpdateInput,ForecastCommentInput,WorkflowStage,WorkflowTransitionRequest,DriverAdjustmentInput,DriverRow,DriverRow,CustomerDriverSummary,RevenueAdjustmentInput,RevenueForecastResponse,RevenueRow,RevenueTotal,QuarterEnum,PnLMetricAdjustmentInput,PnLMetricRow,PnLForecastTotal,PnLForecastFilters,PnLForecastResponse
from .schemas import CustomerBase,VendorBase, finance_chart_of_accounts, FCASchema, GeneralLedgerTransactionSchema, TrialBalanceSchema
from .models import generate_ai_forecast,FinanceChartOfAccounts, FCA, GeneralLedgerTransaction, TrialBalance,Vendor,Customer
from sqlalchemy.orm import Session
from src.core.database import get_db
from typing import Dict, List, Optional
import pandas as pd
from src.core.database import async_session
from src.constants import ChartTypeEnum
from src.financial.report import apply_smart_currency
from src.financial.report import CurrencyFilter
#application_management_router = APIRouter(prefix="/application/management", tags=["application management"])
#companies_management_router = APIRouter(prefix="/companies/management", tags=["comppanies management module"])

forecast_router = APIRouter(prefix="/forecasting/managemet", tags=["forecasting management"])
valid_scenarios = {"Baseline", "Best Case", "Worst Case"}

# In-memory storage (replace with persistent DB storage in prod)
client_forecasts: Dict[str, Dict[int, Dict[str, dict]]] = {scenario: {} for scenario in valid_scenarios}
assumptions_store: Dict[str, Dict[int, str]] = {"Baseline": {}}
version_history: Dict[str, Dict[int, list]] = {scenario: {} for scenario in valid_scenarios}


# ---------- Models ----------

class MonthForecast(BaseModel):
    ai: Optional[float] = 0.0
    user: Optional[float] = 0.0


class ClientForecastInput(BaseModel):
    year: int
    client_name: str
    industry: str
    previous_month: float  # Actual revenue for previous month (month 0)
    forecast: Dict[int, MonthForecast]  # month number -> ai/user forecast


class AssumptionsInput(BaseModel):
    year: int
    assumptions_text: str


# ---------- Helper: Sync actual revenue from GeneralLedger (monthly) ----------

async def sync_actual_revenue(year: int, month: int, db: AsyncSession):
    """
    Sync actual revenue for all clients for given year & month from GeneralLedger
    into client_forecasts under scenario "Baseline" with previous_month as actual.
    """
    
    # Example function to extract 'entity' from extra_data JSONB field as client_name
    
    client_name_expr = func.coalesce(GeneralLedger.extra_data['entity'].astext, 'Unknown')
    industry_expr = func.coalesce(GeneralLedger.extra_data['department'].astext, 'Unknown')

    q = (
        select(
            client_name_expr.label("client_name"),
            func.sum(GeneralLedger.credit).label("total_revenue"),
            industry_expr.label("industry")
        )
        .select_from(
            GeneralLedger.__table__.join(
                GeneralLedgerMaster,
                GeneralLedger.general_ledger_code == GeneralLedgerMaster.general_ledger_code
            )
        )
        .where(
            GeneralLedgerMaster.category == "Revenue",
            extract('year', GeneralLedger.date) == year,
            extract('month', GeneralLedger.date) == month,
            GeneralLedger.credit > 0,
            GeneralLedger.is_active == True
        )
        .group_by(client_name_expr, industry_expr)
    )

    result = await db.execute(q)
    rows = result.all()


    # Store actual revenue in client_forecasts Baseline for year, month=0
    scenario = "Baseline"
    if year not in client_forecasts[scenario]:
        client_forecasts[scenario][year] = {}

    for client_name, total_revenue, industry in rows:
        if client_name not in client_forecasts[scenario][year]:
            client_forecasts[scenario][year][client_name] = {
                "industry": industry,
                "previous_month": 0.0,
                "forecast": {}
            }
        client_forecasts[scenario][year][client_name]["previous_month"] = total_revenue

    return {"message": f"Synced actual revenue for {len(rows)} clients for {year}-{month}"}


# ----------- Endpoints -----------

@forecast_router.post("/{scenario}/client", summary="Save per-client forecast data")
async def save_client_forecast(scenario: str, data: ClientForecastInput):
    if scenario not in valid_scenarios:
        raise HTTPException(status_code=400, detail="Invalid scenario")

    year = data.year
    if year not in client_forecasts[scenario]:
        client_forecasts[scenario][year] = {}

    month_data = {}
    for month, mf in data.forecast.items():
        month_data[month] = {
            "ai": mf.ai if mf.ai is not None else 0.0,
            "user": mf.user if mf.user is not None else 0.0
        }

    client_forecasts[scenario][year][data.client_name] = {
        "industry": data.industry,
        "previous_month": data.previous_month,
        "forecast": month_data
    }

    return {"message": f"Forecast saved for client '{data.client_name}' in {scenario} {year}"}


@forecast_router.post("/{scenario}/{year}/assumptions", summary="Save scenario assumptions")
async def save_assumptions(scenario: str, year: int, data: AssumptionsInput):
    if scenario != "Baseline":
        raise HTTPException(status_code=400, detail="Only Baseline scenario supports assumptions")

    if year != data.year:
        raise HTTPException(status_code=400, detail="Year mismatch between URL and payload")

    assumptions_store.setdefault(scenario, {})[year] = data.assumptions_text
    return {"message": f"Assumptions saved for {scenario} {year}"}


@forecast_router.post("/{scenario}/{year}/version/save", summary="Save forecast version")
async def save_version(scenario: str, year: int):
    if scenario not in valid_scenarios:
        raise HTTPException(status_code=400, detail="Invalid scenario")

    current_snapshot = client_forecasts.get(scenario, {}).get(year)
    if not current_snapshot:
        raise HTTPException(status_code=404, detail="No forecast data to version")

    versions = version_history.setdefault(scenario, {}).setdefault(year, [])

    version_id = len(versions) + 1
    versions.append({
        "version_id": version_id,
        "timestamp": datetime.utcnow().isoformat() + "Z",
        "data": current_snapshot.copy()
    })

    return {"message": f"Version {version_id} saved for {scenario} {year}"}


@forecast_router.get("/{scenario}/{year}/full-forecast-summary", summary="Get full forecast summary")
async def get_full_forecast_summary(scenario: str, year: int):
    if scenario not in valid_scenarios:
        raise HTTPException(status_code=400, detail="Invalid scenario")

    client_data = client_forecasts.get(scenario, {}).get(year, {})
    months_to_show = list(range(1, 4))  # You can extend 1-12 for monthly

    client_table = []
    total_prev_sum = 0.0
    total_months = {m: {"ai": 0.0, "user": 0.0} for m in months_to_show}

    for client_name, details in client_data.items():
        row = {
            "client_name": client_name,
            "industry": details["industry"],
            "previous_month": details["previous_month"]
        }
        total_prev_sum += details["previous_month"]

        for month in months_to_show:
            ai = details["forecast"].get(month, {}).get("ai", 0.0)
            user = details["forecast"].get(month, {}).get("user", 0.0)
            row[f"month_{month}_ai"] = ai
            row[f"month_{month}_user"] = user
            total_months[month]["ai"] += ai
            total_months[month]["user"] += user

        client_table.append(row)

    totals_row = {
        "client_name": "Total",
        "industry": "",
        "previous_month": total_prev_sum
    }
    for month in months_to_show:
        totals_row[f"month_{month}_ai"] = total_months[month]["ai"]
        totals_row[f"month_{month}_user"] = total_months[month]["user"]

    client_table.append(totals_row)

    # Assumptions
    if scenario == "Baseline":
        assumptions_text = assumptions_store.get(scenario, {}).get(year) or \
            "Standard growth assumptions, stable market conditions. Market trends and seasonality considered."
    else:
        assumptions_text = None

    # Revenue Trend Aggregated (actual + AI + user)
    months = [0] + months_to_show  # 0 = actual previous month

    trend = {
        "months": [f"Month {m}" for m in months],
        "actual_revenue": [],
        "ai_forecast": [],
        "user_forecast": []
    }

    for month in months:
        actual_sum = 0.0
        ai_sum = 0.0
        user_sum = 0.0
        for client_data_item in client_data.values():
            if month == 0:
                actual_sum += client_data_item.get("previous_month", 0.0)
            else:
                month_forecast = client_data_item.get("forecast", {}).get(month, {})
                ai_sum += month_forecast.get("ai", 0.0)
                user_sum += month_forecast.get("user", 0.0)

        trend["actual_revenue"].append(actual_sum if month == 0 else None)
        trend["ai_forecast"].append(ai_sum if month > 0 else None)
        trend["user_forecast"].append(user_sum if month > 0 else None)

    # Version history summary
    versions = version_history.get(scenario, {}).get(year, [])
    versions_info = [{"version_id": v["version_id"], "timestamp": v["timestamp"]} for v in versions] or "No versions saved yet"

    # Variance calculation
    total_user_forecast = sum(totals_row[f"month_{m}_user"] for m in months_to_show)
    total_ai_forecast = sum(totals_row[f"month_{m}_ai"] for m in months_to_show)
    variance_amount = total_user_forecast - total_ai_forecast
    variance_percent = (variance_amount / total_ai_forecast * 100) if total_ai_forecast else 0.0

    return {
        "scenario": scenario,
        "year": year,
        "baseline_forecast_summary": {
            "total_user_forecast_for_period": total_user_forecast,
            "total_ai_suggested_for_period": total_ai_forecast,
            "variance_amount": variance_amount,
            "variance_percent": round(variance_percent, 2)
        },
        "client_forecast_table": client_table,
        "assumptions": assumptions_text,
        "revenue_trend": trend,
        "version_history": versions_info
    }


@forecast_router.post("/sync-actual/{year}/{month}", summary="Sync actual revenue from DB")
async def api_sync_actual_revenue(year: int, month: int, db: AsyncSession = Depends(get_db_on_the_fly)):
    """
    API to sync actual revenue from GeneralLedger for given year and month into forecasts.
    """
    return await sync_actual_revenue(year, month, db)


## Register router
#app.include_router(forecast_router, prefix="/forecast")





async def save_forecast(scenario: str, year: int, forecast_data: Dict[int, float], is_ai: bool, db: AsyncSession, metric_type: str):
    """Save forecast data to in-memory storage"""
    if year not in client_forecasts[scenario]:
        client_forecasts[scenario][year] = {}
    
    # Store the forecast data
    for month, value in forecast_data.items():
        if isinstance(month, int):
            if "forecast" not in client_forecasts[scenario][year]:
                client_forecasts[scenario][year]["forecast"] = {}
            client_forecasts[scenario][year]["forecast"][month] = {
                "ai" if is_ai else "user": float(value)
            }


async def get_saved_forecast(year: int, scenario: str, db: AsyncSession) -> Dict[str, Any]:
    """Retrieve saved forecast data from in-memory storage"""
    if scenario not in client_forecasts or year not in client_forecasts[scenario]:
        return {"AI Suggested": {}, "User Forecast": {}}
    
    forecast_data = client_forecasts[scenario][year]
    result = {
        "AI Suggested": {},
        "User Forecast": {}
    }
    
    # Extract forecast data for all clients in this scenario/year
    for client_name, details in forecast_data.items():
        if isinstance(details, dict) and "forecast" in details:
            for month, values in details["forecast"].items():
                if isinstance(values, dict):
                    if "ai" in values:
                        result["AI Suggested"][f"{client_name}_m{month}"] = values["ai"]
                    if "user" in values:
                        result["User Forecast"][f"{client_name}_m{month}"] = values["user"]
    
    return result


@forecast_router.post("/{scenario}")
async def create_or_edit_forecast(
    scenario: str,
    year: int,
    forecast_data: Dict[int, float],

    db: AsyncSession = Depends(get_db_on_the_fly)
):
    if scenario not in ("Baseline", "Best Case", "Worst Case"):
        raise HTTPException(status_code=400, detail="Invalid scenario name")

    await save_forecast(scenario, year, forecast_data, is_ai=False, db=db, metric_type="revenue")
    return {"message": f"User forecast for {scenario} {year} saved successfully."}


@forecast_router.get("/compare/{year}")
async def compare_scenarios(year: int, db: AsyncSession = Depends(get_db_on_the_fly)):
    scenarios = ["Baseline", "Best Case", "Worst Case"]
    comparison = {}

    for scenario in scenarios:
        data = await get_saved_forecast(year, scenario, db)
        ai_total = sum(data["AI Suggested"].values()) if data["AI Suggested"] else 0
        user_total = sum(data["User Forecast"].values()) if data["User Forecast"] else 0
        variance = user_total - ai_total
        variance_pct = (variance / ai_total * 100) if ai_total else 0

        comparison[scenario] = {
            "AI Suggested": ai_total,
            "User Forecast": user_total,
            "Variance": {
                "amount": variance,
                "percent": f"{variance_pct:.1f}%"
            }
        }

    return {
        "year": year,
        "comparison": comparison
    }

#====================================

@forecast_router.get("/finance_chart_of_accounts", response_model=List[finance_chart_of_accounts])
async def get_data(db: AsyncSession = Depends(get_db)):
    result = await db.execute(select(FinanceChartOfAccounts))
    data = result.scalars().all()
    return data

@forecast_router.get("/finance_chart_of_account", response_model=List[FCASchema])
async def finance_chart_of_accounts(db: AsyncSession = Depends(get_db)):
    result = await db.execute(select(FCA))
    data = result.scalars().all()
    return data

@forecast_router.get("/finance_general_ledger_transactions", response_model=List[GeneralLedgerTransactionSchema])
async def General_ladger(db: AsyncSession = Depends(get_db)):
    result = await db.execute(select(GeneralLedgerTransaction))
    data = result.scalars().all()
    return data

@forecast_router.get("/finance_trial_balance", response_model=List[TrialBalanceSchema])
async def Trail_Balence(db: AsyncSession = Depends(get_db)):
    result = await db.execute(select(TrialBalance))
    data = result.scalars().all()
    return data


@forecast_router.get("/finance_vendors", response_model=List[VendorBase])
async def finance_vendors(db: AsyncSession = Depends(get_db)):
    result = await db.execute(select(Vendor))
    data = result.scalars().all()
    return data


@forecast_router.get("/finance_customers", response_model=List[CustomerBase])
async def finance_customers(db: AsyncSession = Depends(get_db)):
    result = await db.execute(select(Customer))
    data = result.scalars().all()
    return data
import jwt
from jwt import PyJWTError
import os

# oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")

# # -----------------------------
# # Authentication
# # -----------------------------
# async def get_current_user(token: str = Depends(oauth2_scheme)):
#     if not token:
#         raise HTTPException(
#             status_code=status.HTTP_401_UNAUTHORIZED,
#             detail="Authentication credentials were not provided",
#             headers={"WWW-Authenticate": "Bearer"},
#         )
#     return {"user": "authenticated"}

 
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")

 
async def get_current_user(token: str = Depends(oauth2_scheme)):
    credentials_exception = HTTPException(
        status_code=status.HTTP_401_UNAUTHORIZED,
        detail="Could not validate credentials",
        headers={"WWW-Authenticate": "Bearer"},
    )
 
    if not token:
        raise HTTPException(
            status_code=status.HTTP_401_UNAUTHORIZED,
            detail="Authentication credentials were not provided",
            headers={"WWW-Authenticate": "Bearer"},
        )
 
    try:
        if SECRET_KEY:
            payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
        else:
            payload = jwt.decode(
                token,
                options={"verify_signature": False, "verify_exp": False},
                algorithms=[ALGORITHM],
            )
    except PyJWTError:
        raise credentials_exception
 
    user_id: str = payload.get("user_id")
    company_id: str = payload.get("company_id")
    department: str = payload.get("department")
 
    if not user_id or not company_id:
        raise credentials_exception
 
    return {
        "id": user_id,
        "company_id": company_id,
        "department": department,
        "role": payload.get("role"),
        "full_name": payload.get("user_name", ""),
        "company_name": payload.get("company_name"),
        "plan": payload.get("plan"),
        "user_type": payload.get("user_type"),
    }
 

import asyncio
import io
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Dict, List, Optional, Tuple

import numpy as np
import openpyxl
import pandas as pd
import warnings

warnings.filterwarnings("ignore")

from prophet import Prophet

try:
    from xgboost import XGBRegressor
    XGBOOST_AVAILABLE = True
except ImportError:
    XGBOOST_AVAILABLE = False

try:
    from statsmodels.tsa.holtwinters import ExponentialSmoothing
    from statsmodels.tsa.stattools import adfuller
    from statsmodels.tsa.seasonal import seasonal_decompose
    from statsmodels.tsa.statespace.sarimax import SARIMAX
    from statsmodels.tsa.arima.model import ARIMA
    STATSMODELS_AVAILABLE = True
except ImportError:
    STATSMODELS_AVAILABLE = False
    SARIMAX = None
    ARIMA = None

from fastapi import APIRouter, Body, Depends, File, HTTPException, Path, Query, UploadFile, status

from fastapi.responses import StreamingResponse
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession


BEST_CASE_MULTIPLIER = 1.10
WORST_CASE_MULTIPLIER = 0.80
HISTORY_MONTHS_SQL = 18
MIN_POINTS_FOR_PROPHET = 12
MIN_POINTS_FOR_ETS = 6
MIN_POINTS_FOR_XGBOOST = 12
MIN_POINTS_FOR_ARIMA = 12
MIN_POINTS_FOR_SARIMA = 24

SECRET_KEY = None
ALGORITHM = "HS256"

RESERVED_SCENARIO_NAMES = {"baseline", "best_case", "worst_case"}

REVENUE_SCENARIO_DEFINITIONS = {
    "baseline": "AI-generated forecast with manual user adjustments applied.",
    "best_case": f"Optimistic scenario applying a {int((BEST_CASE_MULTIPLIER - 1) * 100)}% uplift on baseline — assumes favorable market conditions, strong customer demand, and full pipeline conversion.",
    "worst_case": f"Pessimistic scenario applying a {int((1 - WORST_CASE_MULTIPLIER) * 100)}% reduction on baseline — assumes adverse conditions, customer churn risk, or demand shortfall.",
}

VALID_PARAMETER_TYPES = {"revenue", "rate", "volume", "quantity"}

_WORKFLOW_TABLES_INITIALIZED = False
_EXPENSE_TABLES_INITIALIZED = False
_PAYROLL_TABLES_INITIALIZED = False

_MAX_WORKERS = min(4, (os.cpu_count() or 2))
_prophet_executor = ThreadPoolExecutor(max_workers=_MAX_WORKERS)

_FORECAST_CACHE: Dict[str, Tuple[float, Any]] = {}
_FORECAST_CACHE_TTL = 300


def _forecast_cache_key(company_id: str, year: int, quarter: str) -> str:
    return f"{company_id}:{year}:{quarter}"


def _get_forecast_cache(key: str) -> Optional[Any]:
    entry = _FORECAST_CACHE.get(key)
    if entry and (time.time() - entry[0]) < _FORECAST_CACHE_TTL:
        return entry[1]
    if entry:
        del _FORECAST_CACHE[key]
    return None


def _set_forecast_cache(key: str, value: Any) -> None:
    _FORECAST_CACHE[key] = (time.time(), value)


def _invalidate_forecast_cache(company_id: str, year: int, quarter: str) -> None:
    _FORECAST_CACHE.pop(_forecast_cache_key(company_id, year, quarter), None)


def _invalidate_forecast_cache_full(company_id: str, year: int) -> None:
    for q in ["Q1", "Q2", "Q3", "Q4"]:
        _FORECAST_CACHE.pop(_forecast_cache_key(company_id, year, q), None)
    for key in list(_FORECAST_CACHE.keys()):
        if key.startswith(f"{company_id}:{year}:"):
            del _FORECAST_CACHE[key]


def get_quarter_months(year: int, q: str):
    return {
        "Q1": [f"{year}-01", f"{year}-02", f"{year}-03"],
        "Q2": [f"{year}-04", f"{year}-05", f"{year}-06"],
        "Q3": [f"{year}-07", f"{year}-08", f"{year}-09"],
        "Q4": [f"{year}-10", f"{year}-11", f"{year}-12"],
    }[q]


def get_rolling_months(year: int, start_month_str: str) -> List[str]:
    month_map = {
        "jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
        "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12,
    }
    start_lower = start_month_str.lower()[:3]
    if start_lower not in month_map:
        raise HTTPException(status_code=422, detail=f"Invalid rolling_start_month '{start_month_str}'. Use Jan, Feb, Mar, etc.")
    start_month_num = month_map[start_lower]
    months = []
    for i in range(3):
        month_num = ((start_month_num - 1 + i) % 12) + 1
        if i > 0 and month_num < start_month_num and month_num <= 3:
            effective_year = year + 1
        else:
            effective_year = year
        months.append(f"{effective_year}-{month_num:02d}")
    return months


def get_12_month_periods(year: int) -> List[str]:
    return [f"{year}-{m:02d}" for m in range(1, 13)]


def get_all_12_month_quarters(year: int) -> List[str]:
    return ["Q1", "Q2", "Q3", "Q4"]


def derive_units_and_rate(ai_revenue: float, avg_unit_price: float = 1000.0) -> tuple:
    if avg_unit_price <= 0:
        avg_unit_price = 1.0
    rate_ai = avg_unit_price
    units_ai = ai_revenue / rate_ai if rate_ai else 0.0
    return round(units_ai, 4), round(rate_ai, 2)


def get_user_department(user: Dict[str, Any]) -> str:
    return (user.get("department") or "").lower().strip()


def get_user_id(user: Dict[str, Any]) -> str:
    uid = user.get("id") or user.get("user_id")
    if uid is None:
        raise HTTPException(status_code=401, detail="Invalid user object: missing 'id'")
    return str(uid)


def get_company_id(user: Dict[str, Any]) -> str:
    cid = user.get("company_id")
    if not cid:
        raise HTTPException(status_code=401, detail="Invalid user object: missing 'company_id'")
    return str(cid)


DEPT_SUBMITTERS = {"it", "sales", "operations"}
DEPT_REVIEWERS = {"finance"}
DEPT_APPROVERS = {"ceo", "cfo", "executive"}


def is_dept_owner(dept: str) -> bool:
    return dept in DEPT_SUBMITTERS


def can_edit_forecast(dept: str, stage: str) -> bool:
    if stage == WorkflowStage.LOCKED:
        return False
    if dept in DEPT_SUBMITTERS:
        return stage == WorkflowStage.DRAFT
    return False


def can_transition(dept: str, current_stage: str, target_stage: str) -> bool:
    if current_stage == WorkflowStage.LOCKED:
        return False
    if dept in DEPT_SUBMITTERS:
        if target_stage == WorkflowStage.SUBMITTED and current_stage == WorkflowStage.DRAFT:
            return True
        return False
    if dept in DEPT_REVIEWERS:
        if target_stage == WorkflowStage.REVIEWED and current_stage == WorkflowStage.SUBMITTED:
            return True
        if target_stage == WorkflowStage.DRAFT:
            return True
        return False
    if dept in DEPT_APPROVERS:
        if target_stage == WorkflowStage.APPROVED and current_stage == WorkflowStage.REVIEWED:
            return True
        if target_stage == WorkflowStage.LOCKED and current_stage == WorkflowStage.APPROVED:
            return True
        if target_stage == WorkflowStage.DRAFT:
            return True
        return False
    return False


def can_read_all(dept: str) -> bool:
    return dept in DEPT_REVIEWERS | DEPT_APPROVERS


def _winsorize_values(values: np.ndarray, lower_pct: float = 5, upper_pct: float = 95) -> np.ndarray:
    if len(values) < 4:
        return values
    lo = np.percentile(values, lower_pct)
    hi = np.percentile(values, upper_pct)
    return np.clip(values, lo, hi)


def _detect_stationarity(values: np.ndarray) -> bool:
    if not STATSMODELS_AVAILABLE or len(values) < 8:
        return False
    try:
        result = adfuller(values, autolag="AIC")
        return result[1] < 0.05
    except Exception:
        return False


def _detect_volatility(values: np.ndarray) -> float:
    if len(values) < 2:
        return 0.0
    mean_val = float(np.mean(values))
    if mean_val == 0:
        return 0.0
    return float(np.std(values) / abs(mean_val))


def _detect_trend_strength(values: np.ndarray) -> float:
    n = len(values)
    if n < 3:
        return 0.0
    x = np.arange(n, dtype=float)
    y = values.astype(float)
    if np.std(y) == 0:
        return 0.0
    try:
        coeffs = np.polyfit(x, y, 1)
        fitted = np.polyval(coeffs, x)
        ss_res = np.sum((y - fitted) ** 2)
        ss_tot = np.sum((y - np.mean(y)) ** 2)
        if ss_tot == 0:
            return 0.0
        r_squared = 1.0 - (ss_res / ss_tot)
        return float(max(0.0, r_squared))
    except Exception:
        return 0.0


def _detect_seasonality_strength(values: np.ndarray, period: int = 12) -> float:
    if not STATSMODELS_AVAILABLE:
        return 0.0
    if len(values) < period * 2:
        return 0.0
    try:
        result = seasonal_decompose(values, model="additive", period=period, extrapolate_trend="freq")
        seasonal_var = float(np.var(result.seasonal))
        residual_var = float(np.var(result.resid[~np.isnan(result.resid)]))
        total = seasonal_var + residual_var
        if total == 0:
            return 0.0
        return float(seasonal_var / total)
    except Exception:
        return 0.0


def _months_since_last_activity(values: np.ndarray) -> int:
    reversed_vals = values[::-1]
    for idx, val in enumerate(reversed_vals):
        if val > 0:
            return idx
    return len(values)


def _classify_customer(values: np.ndarray, volatility: float, trend_strength: float, seasonality_strength: float) -> str:
    recent_6 = values[-6:] if len(values) >= 6 else values
    zero_ratio = np.sum(recent_6 == 0) / len(recent_6)
    if zero_ratio >= 0.60:
        return "inactive"
    if volatility <= 0.10:
        return "stable"
    if trend_strength >= 0.60:
        return "growing"
    if seasonality_strength >= 0.45:
        return "seasonal"
    if volatility >= 0.45:
        return "volatile"
    return "normal"


def _detect_outlier_ratio(values: np.ndarray) -> float:
    if len(values) < 5:
        return 0.0
    q1, q3 = np.percentile(values, [25, 75])
    iqr = q3 - q1
    lower = q1 - (1.5 * iqr)
    upper = q3 + (1.5 * iqr)
    return float(np.sum((values < lower) | (values > upper)) / len(values))


def _feature_engineering(values: np.ndarray) -> Dict[str, Any]:
    n = len(values)
    recent_3 = float(np.mean(values[-3:])) if n >= 3 else float(values[-1])
    recent_6 = float(np.mean(values[-6:])) if n >= 6 else float(np.mean(values))
    recent_12 = float(np.mean(values[-12:])) if n >= 12 else float(np.mean(values))
    volatility = _detect_volatility(values)
    trend_strength = _detect_trend_strength(values)
    seasonality_strength = _detect_seasonality_strength(values)
    growth_rate = 0.0
    if recent_6 > 0:
        growth_rate = (recent_3 - recent_6) / recent_6
    stationary = _detect_stationarity(values)
    outlier_ratio = _detect_outlier_ratio(values)
    lag_1 = float(values[-1] - values[-2]) if n >= 2 else 0.0
    lag_2 = float(values[-1] - values[-3]) if n >= 3 else 0.0
    moving_avg_3 = recent_3
    moving_avg_6 = recent_6
    variance = float(np.var(values)) if n >= 2 else 0.0
    try:
        autocorr_strength = float(abs(np.corrcoef(values[:-1], values[1:])[0, 1])) if n >= 3 else 0.0
        if np.isnan(autocorr_strength):
            autocorr_strength = 0.0
    except Exception:
        autocorr_strength = 0.0
    stability_score = float(max(0.0, min(1.0, 1.0 - (volatility * 0.50) - (outlier_ratio * 0.30) + (trend_strength * 0.20))))
    return {
        "last_value": float(values[-1]),
        "recent_3_avg": recent_3,
        "recent_6_avg": recent_6,
        "recent_12_avg": recent_12,
        "volatility": float(volatility),
        "trend_strength": float(trend_strength),
        "seasonality_strength": float(seasonality_strength),
        "growth_rate": float(growth_rate),
        "stationary": stationary,
        "outlier_ratio": float(outlier_ratio),
        "lag_1": lag_1,
        "lag_2": lag_2,
        "moving_avg_3": moving_avg_3,
        "moving_avg_6": moving_avg_6,
        "variance": variance,
        "autocorr_strength": autocorr_strength,
        "stability_score": stability_score,
    }


def _get_dynamic_limits(customer_state: str, volatility: float) -> Tuple[float, float]:
    if customer_state == "inactive":
        return -0.95, 0.03
    if customer_state == "stable":
        return -0.05, 0.06
    if customer_state == "growing":
        return -0.08, 0.18
    if customer_state == "seasonal":
        return -0.15, 0.25
    if customer_state == "volatile":
        return -0.30, 0.35
    dynamic_limit = max(0.08, min(0.18, volatility * 0.60))
    return -dynamic_limit, dynamic_limit


def _apply_forecast_limits(forecasts: list, last_actual: float, customer_state: str, volatility: float) -> list:
    if last_actual <= 0:
        return forecasts
    min_change, max_change = _get_dynamic_limits(customer_state, volatility)
    result = []
    prev = last_actual
    for forecast in forecasts:
        min_allowed = prev * (1 + min_change)
        max_allowed = prev * (1 + max_change)
        adjusted = float(np.clip(forecast, min_allowed, max_allowed))
        adjusted = round(max(0.0, adjusted), 2)
        result.append(adjusted)
        prev = adjusted
    return result


def _validate_forecast(forecasts: list, historical_values: np.ndarray) -> list:
    hist_mean = float(np.mean(historical_values))
    hist_std = float(np.std(historical_values))
    upper = hist_mean + (3 * hist_std)
    lower = max(0.0, hist_mean - (3 * hist_std))
    result = []
    for val in forecasts:
        adjusted = float(np.clip(val, lower, upper))
        result.append(round(max(0.0, adjusted), 2))
    return result


def _baseline_forecast(values: np.ndarray) -> list:
    recent_3 = np.mean(values[-3:])
    recent_6 = np.mean(values[-6:])
    recent_12 = np.mean(values[-12:])
    weighted = (recent_3 * 0.50) + (recent_6 * 0.30) + (recent_12 * 0.20)
    momentum = 0.0
    if recent_6 > 0:
        momentum = (recent_3 - recent_6) / recent_6
    momentum = float(np.clip(momentum, -0.08, 0.08))
    result = []
    for i in range(3):
        factor = 1 + (momentum * 0.15 * (i + 1))
        result.append(round(max(0.0, weighted * factor), 2))
    return result


def _prophet_forecast_sync(values: np.ndarray, start_date: str, n_ahead: int = 3) -> Optional[list]:
    try:
        if len(values) < MIN_POINTS_FOR_PROPHET:
            return None
        dates = pd.date_range(start=start_date, periods=len(values), freq="MS")
        df = pd.DataFrame({"ds": dates, "y": values})
        trend_strength = _detect_trend_strength(values)
        volatility = _detect_volatility(values)
        cps = max(0.01, min(0.08, 0.02 + (trend_strength * 0.20) + (volatility * 0.10)))
        model = Prophet(
            yearly_seasonality=True,
            weekly_seasonality=False,
            daily_seasonality=False,
            seasonality_mode="additive",
            changepoint_prior_scale=cps,
            interval_width=0.80,
            changepoint_range=0.80,
        )
        model.add_seasonality(name="quarterly", period=91.25, fourier_order=2)
        model.fit(df)
        future = model.make_future_dataframe(periods=n_ahead, freq="MS")
        fc = model.predict(future)
        preds = fc["yhat"].values[-n_ahead:]
        return [round(max(0.0, float(v)), 2) for v in preds]
    except Exception:
        return None


def _ets_forecast_sync(values: np.ndarray, n_ahead: int = 3) -> Optional[list]:
    if not STATSMODELS_AVAILABLE:
        return None
    try:
        if len(values) < MIN_POINTS_FOR_ETS:
            return None
        seasonality_strength = _detect_seasonality_strength(values)
        if len(values) >= 24 and seasonality_strength > 0.25:
            model = ExponentialSmoothing(values, trend="add", seasonal="add", seasonal_periods=12, initialization_method="estimated")
        else:
            model = ExponentialSmoothing(values, trend="add", seasonal=None, initialization_method="estimated")
        fitted = model.fit(optimized=True, use_brute=False)
        preds = fitted.forecast(n_ahead)
        return [round(max(0.0, float(v)), 2) for v in preds]
    except Exception:
        return None


def _xgboost_forecast_sync(values: np.ndarray, n_ahead: int = 3) -> Optional[list]:
    if not XGBOOST_AVAILABLE:
        return None
    try:
        if len(values) < MIN_POINTS_FOR_XGBOOST:
            return None
        rows_x = []
        rows_y = []
        for i in range(6, len(values)):
            lag_1 = values[i - 1]
            lag_2 = values[i - 2]
            lag_3 = values[i - 3]
            avg_3 = np.mean(values[i - 3:i])
            avg_6 = np.mean(values[i - 6:i])
            volatility = np.std(values[:i]) / max(np.mean(values[:i]), 1)
            rows_x.append([lag_1, lag_2, lag_3, avg_3, avg_6, volatility])
            rows_y.append(values[i])
        if len(rows_x) < 4:
            return None
        X = np.array(rows_x)
        y = np.array(rows_y)
        model = XGBRegressor(n_estimators=80, max_depth=3, learning_rate=0.05, objective="reg:squarederror", random_state=42)
        model.fit(X, y)
        forecasts = []
        working = list(values.copy())
        for _ in range(n_ahead):
            lag_1 = working[-1]
            lag_2 = working[-2]
            lag_3 = working[-3]
            avg_3 = np.mean(working[-3:])
            avg_6 = np.mean(working[-6:])
            volatility = np.std(working) / max(np.mean(working), 1)
            features = np.array([[lag_1, lag_2, lag_3, avg_3, avg_6, volatility]])
            pred = float(model.predict(features)[0])
            pred = max(0.0, pred)
            forecasts.append(round(pred, 2))
            working.append(pred)
        return forecasts
    except Exception:
        return None


def _sarima_forecast_sync(values: np.ndarray, n_ahead: int = 3) -> Optional[list]:
    if not STATSMODELS_AVAILABLE or SARIMAX is None:
        return None
    try:
        if len(values) < MIN_POINTS_FOR_SARIMA:
            return None
        model = SARIMAX(values, order=(1, 1, 1), seasonal_order=(1, 1, 1, 12), enforce_stationarity=False, enforce_invertibility=False)
        fitted = model.fit(disp=False, maxiter=50)
        preds = fitted.forecast(steps=n_ahead)
        return [round(max(0.0, float(v)), 2) for v in preds]
    except Exception:
        return None


def _ensemble_forecasts_revenue(
    baseline: list,
    prophet: Optional[list],
    ets: Optional[list],
    sarima: Optional[list],
    xgb: Optional[list],
    features: Dict[str, Any],
    n_ahead: int = 3,
) -> list:
    volatility = features.get("volatility", 0.0)
    trend_strength = features.get("trend_strength", 0.0)
    seasonality_strength = features.get("seasonality_strength", 0.0)

    result = []
    for i in range(n_ahead):
        candidates = []
        weights = []

        b_val = baseline[i] if i < len(baseline) else baseline[-1]
        candidates.append(b_val)
        weights.append(0.25)

        if prophet and i < len(prophet):
            candidates.append(prophet[i])
            w = 0.35 + (trend_strength * 0.10) + (seasonality_strength * 0.10)
            weights.append(min(w, 0.55))

        if ets and i < len(ets):
            candidates.append(ets[i])
            w = 0.20 + (seasonality_strength * 0.05)
            weights.append(min(w, 0.30))

        if sarima and i < len(sarima):
            candidates.append(sarima[i])
            w = 0.15 + (seasonality_strength * 0.05)
            weights.append(min(w, 0.25))

        if xgb and i < len(xgb):
            candidates.append(xgb[i])
            w = 0.15 + (volatility * 0.10)
            weights.append(min(w, 0.25))

        total_w = sum(weights)
        if total_w == 0:
            result.append(round(b_val, 2))
        else:
            norm_weights = [w / total_w for w in weights]
            ensembled = sum(c * w for c, w in zip(candidates, norm_weights))
            result.append(round(max(0.0, ensembled), 2))

    return result


def _probabilistic_adjustment(forecasts: list, volatility: float) -> Tuple[list, list, list]:
    lower = []
    upper = []
    confidence = []
    confidence_score = int(max(45, min(95, 95 - (volatility * 100))))
    band = max(0.05, min(0.25, volatility * 0.80))
    for val in forecasts:
        lower.append(round(max(0.0, val * (1 - band)), 2))
        upper.append(round(max(0.0, val * (1 + band)), 2))
        confidence.append(confidence_score)
    return lower, upper, confidence


def _stabilize_forecasts(forecasts: list, last_actual: float, volatility: float) -> list:
    result = []
    prev = last_actual
    stabilization_weight = max(0.35, min(0.75, 1 - volatility))
    for forecast in forecasts:
        stabilized = (forecast * (1 - stabilization_weight)) + (prev * stabilization_weight)
        stabilized = round(max(0.0, stabilized), 2)
        result.append(stabilized)
        prev = stabilized
    return result


def _forecast_customer_sync(customer_rows: pd.DataFrame, quarter_start_str: str) -> Tuple[float, float, float]:
    if customer_rows.empty:
        return 0.0, 0.0, 0.0
    cdf = customer_rows.copy()
    cdf["ds"] = pd.to_datetime(cdf["period"] + "-01")
    cdf = cdf.sort_values("ds")
    values = cdf["y"].values.astype(float)
    if len(values) == 0:
        return 0.0, 0.0, 0.0
    if len(values) == 1:
        val = round(float(values[-1]), 2)
        return val, val, val
    if len(values) == 2:
        last = float(values[-1])
        trend = float(values[-1] - values[-2])
        v1 = round(max(0.0, last + trend), 2)
        v2 = round(max(0.0, last + 2 * trend), 2)
        v3 = round(max(0.0, last + 3 * trend), 2)
        return v1, v2, v3
    clean_values = _winsorize_values(values)
    recent_3 = clean_values[-3:]
    if np.count_nonzero(recent_3) == 0:
        return 0.0, 0.0, 0.0
    months_inactive = _months_since_last_activity(clean_values)
    if months_inactive >= 3:
        reduced = round(float(np.mean(clean_values[-6:]) * 0.10), 2)
        reduced = max(0.0, reduced)
        return reduced, reduced, reduced
    features = _feature_engineering(clean_values)
    volatility = features["volatility"]
    trend_strength = features["trend_strength"]
    seasonality_strength = features["seasonality_strength"]
    customer_state = _classify_customer(clean_values, volatility, trend_strength, seasonality_strength)
    baseline_result = _baseline_forecast(clean_values)
    start_date = str(cdf["ds"].iloc[0].date())
    prophet_result = _prophet_forecast_sync(clean_values, start_date, 3)
    ets_result = _ets_forecast_sync(clean_values, 3)
    sarima_result = _sarima_forecast_sync(clean_values, 3)
    xgb_result = _xgboost_forecast_sync(clean_values, 3)
    final_preds = _ensemble_forecasts_revenue(baseline_result, prophet_result, ets_result, sarima_result, xgb_result, features, 3)
    final_preds = _validate_forecast(final_preds, clean_values)
    final_preds = _apply_forecast_limits(final_preds, float(clean_values[-1]), customer_state, volatility)
    final_preds = _stabilize_forecasts(final_preds, float(clean_values[-1]), volatility)
    _probabilistic_adjustment(final_preds, volatility)
    return float(final_preds[0]), float(final_preds[1]), float(final_preds[2])


def _forecast_customer_n_months_sync(customer_rows: pd.DataFrame, quarter_start_str: str, n_ahead: int = 12) -> List[float]:
    if customer_rows.empty:
        return [0.0] * n_ahead
    cdf = customer_rows.copy()
    cdf["ds"] = pd.to_datetime(cdf["period"] + "-01")
    cdf = cdf.sort_values("ds")
    values = cdf["y"].values.astype(float)
    if len(values) == 0:
        return [0.0] * n_ahead
    if len(values) == 1:
        val = round(float(values[-1]), 2)
        return [val] * n_ahead
    if len(values) == 2:
        last = float(values[-1])
        trend = float(values[-1] - values[-2])
        return [round(max(0.0, last + trend * (i + 1)), 2) for i in range(n_ahead)]
    clean_values = _winsorize_values(values)
    recent_3 = clean_values[-3:]
    if np.count_nonzero(recent_3) == 0:
        return [0.0] * n_ahead
    months_inactive = _months_since_last_activity(clean_values)
    if months_inactive >= 3:
        reduced = round(float(np.mean(clean_values[-6:]) * 0.10), 2)
        reduced = max(0.0, reduced)
        return [reduced] * n_ahead
    features = _feature_engineering(clean_values)
    volatility = features["volatility"]
    seasonality_strength = features["seasonality_strength"]
    customer_state = _classify_customer(clean_values, volatility, features["trend_strength"], seasonality_strength)
    prophet_result = _prophet_forecast_sync(clean_values, str(cdf["ds"].iloc[0].date()), n_ahead)
    ets_result_full = None
    if STATSMODELS_AVAILABLE and len(clean_values) >= MIN_POINTS_FOR_ETS:
        try:
            if len(clean_values) >= 24 and seasonality_strength > 0.25:
                ets_model = ExponentialSmoothing(clean_values, trend="add", seasonal="add", seasonal_periods=12, initialization_method="estimated")
            else:
                ets_model = ExponentialSmoothing(clean_values, trend="add", seasonal=None, initialization_method="estimated")
            ets_fitted = ets_model.fit(optimized=True, use_brute=False)
            ets_result_full = [round(max(0.0, float(v)), 2) for v in ets_fitted.forecast(n_ahead)]
        except Exception:
            ets_result_full = None
    xgb_result_full = None
    if XGBOOST_AVAILABLE and len(clean_values) >= MIN_POINTS_FOR_XGBOOST:
        try:
            rows_x = []
            rows_y = []
            for i in range(6, len(clean_values)):
                lag_1 = clean_values[i - 1]
                lag_2 = clean_values[i - 2]
                lag_3 = clean_values[i - 3]
                avg_3 = np.mean(clean_values[i - 3:i])
                avg_6 = np.mean(clean_values[i - 6:i])
                vol = np.std(clean_values[:i]) / max(np.mean(clean_values[:i]), 1)
                rows_x.append([lag_1, lag_2, lag_3, avg_3, avg_6, vol])
                rows_y.append(clean_values[i])
            if len(rows_x) >= 4:
                X = np.array(rows_x)
                y_arr = np.array(rows_y)
                xgb_model = XGBRegressor(n_estimators=80, max_depth=3, learning_rate=0.05, objective="reg:squarederror", random_state=42)
                xgb_model.fit(X, y_arr)
                xgb_preds = []
                working = list(clean_values.copy())
                for _ in range(n_ahead):
                    lag_1 = working[-1]
                    lag_2 = working[-2]
                    lag_3 = working[-3]
                    avg_3 = np.mean(working[-3:])
                    avg_6 = np.mean(working[-6:])
                    vol = np.std(working) / max(np.mean(working), 1)
                    feats = np.array([[lag_1, lag_2, lag_3, avg_3, avg_6, vol]])
                    pred = float(xgb_model.predict(feats)[0])
                    pred = max(0.0, pred)
                    xgb_preds.append(round(pred, 2))
                    working.append(pred)
                xgb_result_full = xgb_preds
        except Exception:
            xgb_result_full = None
    sarima_result_full = _sarima_forecast_sync(clean_values, n_ahead)
    baseline_3 = _baseline_forecast(clean_values)
    result = _ensemble_forecasts_revenue(
        baseline_3, prophet_result, ets_result_full, sarima_result_full, xgb_result_full, features, n_ahead
    )
    result = _validate_forecast(result, clean_values)
    result = _apply_forecast_limits(result, float(clean_values[-1]), customer_state, features["volatility"])
    result = _stabilize_forecasts(result, float(clean_values[-1]), features["volatility"])
    return result


async def _forecast_all_customers(
    cust_hist_df: pd.DataFrame,
    all_customers: list,
    quarter_start: pd.Timestamp,
    m1: str,
    m2: str,
    m3: str,
) -> Dict[str, tuple]:
    loop = asyncio.get_event_loop()
    quarter_start_str = str(quarter_start.date())

    grouped = {cname: group for cname, group in cust_hist_df.groupby("customer_name")}

    sem = asyncio.Semaphore(_MAX_WORKERS * 2)

    async def forecast_one(cname: str) -> tuple:
        async with sem:
            cdf = grouped.get(cname)
            if cdf is None or cdf.empty:
                return 0.0, 0.0, 0.0
            if "y" not in cdf.columns:
                cdf = cdf.rename(columns={"revenue": "y"})
            return await loop.run_in_executor(_prophet_executor, _forecast_customer_sync, cdf, quarter_start_str)

    all_results = await asyncio.gather(*[forecast_one(cname) for cname in all_customers])
    return dict(zip(all_customers, all_results))


async def _forecast_all_customers_n_months(
    cust_hist_df: pd.DataFrame,
    all_customers: list,
    forecast_start: pd.Timestamp,
    n_ahead: int = 12,
) -> Dict[str, List[float]]:
    loop = asyncio.get_event_loop()
    forecast_start_str = str(forecast_start.date())
    grouped = {cname: group for cname, group in cust_hist_df.groupby("customer_name")}

    sem = asyncio.Semaphore(_MAX_WORKERS * 2)

    async def forecast_one_n(cname: str) -> List[float]:
        async with sem:
            cdf = grouped.get(cname)
            if cdf is None or cdf.empty:
                return [0.0] * n_ahead
            if "y" not in cdf.columns:
                cdf = cdf.rename(columns={"revenue": "y"})
            return await loop.run_in_executor(_prophet_executor, _forecast_customer_n_months_sync, cdf, forecast_start_str, n_ahead)

    all_results = await asyncio.gather(*[forecast_one_n(cname) for cname in all_customers])
    return dict(zip(all_customers, all_results))


async def _ensure_workflow_tables(db: AsyncSession) -> None:
    global _WORKFLOW_TABLES_INITIALIZED
    if _WORKFLOW_TABLES_INITIALIZED:
        return
    _WORKFLOW_TABLES_INITIALIZED = True
    await db.execute(text("CREATE SCHEMA IF NOT EXISTS fpna_config;"))
    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (
                SELECT 1 FROM information_schema.tables
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'revenue_forecast_adjustments'
            ) AND NOT EXISTS (
                SELECT 1 FROM information_schema.table_constraints
                WHERE table_schema   = 'fpna_config'
                  AND table_name     = 'revenue_forecast_adjustments'
                  AND constraint_type = 'PRIMARY KEY'
            ) THEN
                DROP TABLE fpna_config.revenue_forecast_adjustments;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.revenue_forecast_adjustments (
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            customer_name TEXT NOT NULL,
            month1_adjusted NUMERIC NULL,
            month1_comment TEXT NULL,
            month2_adjusted NUMERIC NULL,
            month2_comment TEXT NULL,
            month3_adjusted NUMERIC NULL,
            month3_comment TEXT NULL,
            overall_comment TEXT NULL,
            updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT NOW(),
            PRIMARY KEY (year, quarter, company_id, customer_name)
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'revenue_forecast_adjustments'
                  AND column_name  = 'overall_comment'
            ) THEN
                ALTER TABLE fpna_config.revenue_forecast_adjustments
                    ADD COLUMN overall_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'revenue_forecast_adjustments'
                  AND column_name  = 'month1_comment'
            ) THEN
                ALTER TABLE fpna_config.revenue_forecast_adjustments
                    ADD COLUMN month1_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'revenue_forecast_adjustments'
                  AND column_name  = 'month2_comment'
            ) THEN
                ALTER TABLE fpna_config.revenue_forecast_adjustments
                    ADD COLUMN month2_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'revenue_forecast_adjustments'
                  AND column_name  = 'month3_comment'
            ) THEN
                ALTER TABLE fpna_config.revenue_forecast_adjustments
                    ADD COLUMN month3_comment TEXT NULL;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.revenue_driver_adjustments (
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            customer_name TEXT NOT NULL,
            month TEXT NOT NULL,
            units_adjusted NUMERIC NULL,
            rate_adjusted NUMERIC NULL,
            month_comment TEXT NULL,
            overall_comment TEXT NULL,
            updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT NOW(),
            PRIMARY KEY (year, quarter, company_id, customer_name, month)
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'revenue_driver_adjustments'
                  AND column_name  = 'month_comment'
            ) THEN
                ALTER TABLE fpna_config.revenue_driver_adjustments
                    ADD COLUMN month_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'revenue_driver_adjustments'
                  AND column_name  = 'overall_comment'
            ) THEN
                ALTER TABLE fpna_config.revenue_driver_adjustments
                    ADD COLUMN overall_comment TEXT NULL;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.forecast_workflows (
            id SERIAL PRIMARY KEY,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            department TEXT NOT NULL,
            owner_user_id TEXT NOT NULL,
            stage TEXT NOT NULL DEFAULT 'draft',
            created_at TIMESTAMP NOT NULL DEFAULT NOW(),
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            locked_at TIMESTAMP NULL,
            UNIQUE (year, quarter, company_id, department)
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'forecast_workflows'
                  AND column_name  = 'owner_user_id'
                  AND data_type   != 'text'
            ) THEN
                ALTER TABLE fpna_config.forecast_workflows
                    ALTER COLUMN owner_user_id TYPE TEXT USING owner_user_id::TEXT;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        DO $$
        DECLARE
            v_constraint TEXT;
        BEGIN
            SELECT tc.constraint_name INTO v_constraint
            FROM information_schema.table_constraints tc
            WHERE tc.table_schema = 'fpna_config'
              AND tc.table_name   = 'forecast_workflows'
              AND tc.constraint_type = 'UNIQUE'
              AND EXISTS (
                  SELECT 1 FROM information_schema.key_column_usage kcu
                  WHERE kcu.constraint_name = tc.constraint_name
                    AND kcu.column_name = 'department'
              )
            LIMIT 1;
            IF v_constraint IS NOT NULL THEN
                EXECUTE 'ALTER TABLE fpna_config.forecast_workflows DROP CONSTRAINT ' || quote_ident(v_constraint);
            END IF;
            DELETE FROM fpna_config.forecast_workflows fw1
            WHERE fw1.id NOT IN (
                SELECT MIN(fw2.id)
                FROM fpna_config.forecast_workflows fw2
                GROUP BY fw2.year, fw2.quarter, fw2.company_id
            );
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.table_constraints tc
                WHERE tc.table_schema = 'fpna_config'
                  AND tc.table_name   = 'forecast_workflows'
                  AND tc.constraint_type = 'UNIQUE'
                  AND NOT EXISTS (
                      SELECT 1 FROM information_schema.key_column_usage kcu
                      WHERE kcu.constraint_name = tc.constraint_name
                        AND kcu.column_name = 'department'
                  )
            ) THEN
                ALTER TABLE fpna_config.forecast_workflows
                    ADD CONSTRAINT forecast_workflows_shared_unique UNIQUE (year, quarter, company_id);
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.forecast_workflow_audit (
            id SERIAL PRIMARY KEY,
            workflow_id INT NOT NULL,
            from_stage TEXT NULL,
            to_stage TEXT NOT NULL,
            transitioned_by_user_id TEXT NOT NULL,
            transitioned_by_name TEXT NOT NULL,
            comment TEXT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'forecast_workflow_audit'
                  AND column_name  = 'transitioned_by_user_id'
                  AND data_type   != 'text'
            ) THEN
                ALTER TABLE fpna_config.forecast_workflow_audit
                    ALTER COLUMN transitioned_by_user_id TYPE TEXT USING transitioned_by_user_id::TEXT;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.forecast_workflow_comments (
            id SERIAL PRIMARY KEY,
            workflow_id INT NOT NULL,
            user_id TEXT NOT NULL,
            user_name TEXT NOT NULL,
            department TEXT NOT NULL,
            comment TEXT NOT NULL,
            comment_type TEXT NOT NULL DEFAULT 'general',
            created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'forecast_workflow_comments'
                  AND column_name  = 'user_id'
                  AND data_type   != 'text'
            ) THEN
                ALTER TABLE fpna_config.forecast_workflow_comments
                    ALTER COLUMN user_id TYPE TEXT USING user_id::TEXT;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'forecast_workflow_comments'
                  AND column_name  = 'comment_type'
            ) THEN
                ALTER TABLE fpna_config.forecast_workflow_comments
                    ADD COLUMN comment_type TEXT NOT NULL DEFAULT 'general';
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.forecast_scenarios (
            id SERIAL PRIMARY KEY,
            company_id TEXT NOT NULL,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            scenario_name TEXT NOT NULL,
            multiplier NUMERIC NOT NULL,
            description TEXT NULL,
            created_by_user_id TEXT NOT NULL,
            created_by_name TEXT NOT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW(),
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            UNIQUE (company_id, year, quarter, scenario_name)
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'forecast_scenarios'
                  AND column_name  = 'parameter_types'
            ) THEN
                ALTER TABLE fpna_config.forecast_scenarios
                    ADD COLUMN parameter_types JSONB NOT NULL DEFAULT '["revenue"]';
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'forecast_scenarios'
                  AND column_name  = 'customer_scope'
            ) THEN
                ALTER TABLE fpna_config.forecast_scenarios
                    ADD COLUMN customer_scope TEXT NOT NULL DEFAULT 'all';
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.forecast_scenario_customer_multipliers (
            id SERIAL PRIMARY KEY,
            scenario_id INT NOT NULL,
            company_id TEXT NOT NULL,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            customer_name TEXT NOT NULL,
            multiplier NUMERIC NOT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW(),
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            UNIQUE (scenario_id, company_id, year, quarter, customer_name)
        );
    """))
    await db.commit()


async def _get_or_create_workflow(
    db: AsyncSession, year: int, quarter: str, company_id: str, department: str, user_id: str
) -> Dict[str, Any]:
    row = (await db.execute(
        text("""
            SELECT * FROM fpna_config.forecast_workflows
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id
            ORDER BY id ASC LIMIT 1
        """),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().first()
    if not row:
        row = (await db.execute(
            text("""
                INSERT INTO fpna_config.forecast_workflows
                    (year, quarter, company_id, department, owner_user_id, stage)
                VALUES (:year, :quarter, :company_id, :dept, :uid, 'draft')
                RETURNING *
            """),
            {"year": year, "quarter": quarter, "company_id": company_id, "dept": department, "uid": user_id},
        )).mappings().first()
        await db.commit()
    return dict(row)


async def _get_adjustments(db: AsyncSession, year: int, quarter: str, company_id: str) -> List[Dict[str, Any]]:
    rows = (await db.execute(
        text("""
            SELECT customer_name, month1_adjusted, month1_comment, month2_adjusted, month2_comment,
                   month3_adjusted, month3_comment, overall_comment
            FROM fpna_config.revenue_forecast_adjustments
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id
            ORDER BY customer_name ASC;
        """),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().all()
    result = []
    for r in rows:
        m1 = float(r["month1_adjusted"]) if r["month1_adjusted"] is not None else None
        m2 = float(r["month2_adjusted"]) if r["month2_adjusted"] is not None else None
        m3 = float(r["month3_adjusted"]) if r["month3_adjusted"] is not None else None
        quarter_total = round((m1 or 0.0) + (m2 or 0.0) + (m3 or 0.0), 2)
        result.append({
            "customer_name": r["customer_name"],
            "month1_adjusted": m1, "month1_comment": r["month1_comment"],
            "month2_adjusted": m2, "month2_comment": r["month2_comment"],
            "month3_adjusted": m3, "month3_comment": r["month3_comment"],
            "quarter_total_adjusted": quarter_total,
            "overall_comment": r["overall_comment"],
        })
    return result


async def _get_driver_adjustments(db: AsyncSession, year: int, quarter: str, company_id: str) -> Dict[tuple, Dict]:
    rows = (await db.execute(
        text("""
            SELECT customer_name, month, units_adjusted, rate_adjusted, month_comment, overall_comment
            FROM fpna_config.revenue_driver_adjustments
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id;
        """),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().all()
    return {
        (r["customer_name"], r["month"]): {
            "units_adjusted": float(r["units_adjusted"]) if r["units_adjusted"] is not None else None,
            "rate_adjusted": float(r["rate_adjusted"]) if r["rate_adjusted"] is not None else None,
            "month_comment": r["month_comment"],
            "overall_comment": r["overall_comment"],
        }
        for r in rows
    }


async def _get_scenario_customer_multipliers(db: AsyncSession, scenario_id: int, company_id: str, year: int, quarter: str) -> Dict[str, float]:
    rows = (await db.execute(
        text("""
            SELECT customer_name, multiplier
            FROM fpna_config.forecast_scenario_customer_multipliers
            WHERE scenario_id = :scenario_id AND company_id = :company_id AND year = :year AND quarter = :quarter
            ORDER BY customer_name ASC
        """),
        {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter},
    )).mappings().all()
    return {r["customer_name"]: float(r["multiplier"]) for r in rows}


async def _get_scenario_customer_multipliers_batch(
    db: AsyncSession, scenario_ids: List[int], company_id: str, year: int, quarter: str,
) -> Dict[int, Dict[str, float]]:
    """Fetch customer multipliers for many scenarios in ONE round-trip.

    Returns a dict keyed by scenario_id; missing scenarios get an empty dict.
    Replaces the previous fan-out of N parallel queries (one per scenario).
    """
    result: Dict[int, Dict[str, float]] = {sid: {} for sid in scenario_ids}
    if not scenario_ids:
        return result
    rows = (await db.execute(
        text("""
            SELECT scenario_id, customer_name, multiplier
            FROM fpna_config.forecast_scenario_customer_multipliers
            WHERE scenario_id = ANY(:scenario_ids)
              AND company_id = :company_id AND year = :year AND quarter = :quarter
            ORDER BY scenario_id, customer_name ASC
        """),
        {"scenario_ids": scenario_ids, "company_id": company_id, "year": year, "quarter": quarter},
    )).mappings().all()
    for r in rows:
        result.setdefault(r["scenario_id"], {})[r["customer_name"]] = float(r["multiplier"])
    return result


async def _get_custom_scenarios(db: AsyncSession, year: int, quarter: str, company_id: str) -> List[Dict[str, Any]]:
    rows = (await db.execute(
        text("""
            SELECT id, scenario_name, multiplier, description, parameter_types, customer_scope,
                   created_by_name, created_at, updated_at
            FROM fpna_config.forecast_scenarios
            WHERE company_id = :company_id AND year = :year AND quarter = :quarter
            ORDER BY created_at ASC
        """),
        {"company_id": company_id, "year": year, "quarter": quarter},
    )).mappings().all()
    return [
        {
            "id": r["id"],
            "scenario_name": r["scenario_name"],
            "multiplier": float(r["multiplier"]),
            "description": r["description"],
            "parameter_types": r["parameter_types"] if r["parameter_types"] else ["revenue"],
            "customer_scope": r["customer_scope"] if r["customer_scope"] else "all",
            "created_by_name": r["created_by_name"],
            "created_at": r["created_at"].isoformat(),
            "updated_at": r["updated_at"].isoformat(),
        }
        for r in rows
    ]


def _build_scenario_summary(baseline_ai: float, baseline_user: float, multiplier: float) -> Dict[str, Any]:
    total_ai = round(baseline_ai * multiplier, 2)
    total_user = round(baseline_user * multiplier, 2)
    variance_amount = round(total_user - total_ai, 2)
    variance_percent = round((variance_amount / total_ai) * 100 if total_ai else 0, 2)
    return {"total_ai": total_ai, "total_user": total_user, "variance_amount": variance_amount, "variance_percent": variance_percent}


def _build_scenario_summary_with_customer_multipliers(
    baseline_ai: float, baseline_user: float, scenario_multiplier: float,
    customer_ai_map: Dict[str, float], customer_user_map: Dict[str, float],
    customer_multipliers: Dict[str, float], customer_scope: str,
) -> Dict[str, Any]:
    if customer_scope == "all" or not customer_multipliers:
        return _build_scenario_summary(baseline_ai, baseline_user, scenario_multiplier)
    total_ai = 0.0
    total_user = 0.0
    all_customers = set(list(customer_ai_map.keys()) + list(customer_user_map.keys()))
    for cname in all_customers:
        eff_mult = customer_multipliers.get(cname, scenario_multiplier)
        total_ai += customer_ai_map.get(cname, 0.0) * eff_mult
        total_user += customer_user_map.get(cname, 0.0) * eff_mult
    total_ai = round(total_ai, 2)
    total_user = round(total_user, 2)
    variance_amount = round(total_user - total_ai, 2)
    variance_percent = round((variance_amount / total_ai) * 100 if total_ai else 0, 2)
    return {"total_ai": total_ai, "total_user": total_user, "variance_amount": variance_amount, "variance_percent": variance_percent}


def _resolve_forecast_periods(
    year: int,
    forecast_type: str,
    quarter: Optional[str],
    rolling_start_month: Optional[str],
) -> Tuple[List[str], str, str]:
    if forecast_type == "quarter":
        q = quarter or "Q1"
        months = get_quarter_months(year, q)
        selected_period = q
        prev_month = (pd.to_datetime(months[0] + "-01") - pd.DateOffset(months=1)).strftime("%Y-%m")
    elif forecast_type == "12months":
        months = get_12_month_periods(year)
        selected_period = f"{year} Full Year"
        prev_month = f"{year - 1}-12"
    elif forecast_type == "rolling":
        sm = rolling_start_month or "Jan"
        months = get_rolling_months(year, sm)
        selected_period = f"Rolling {sm[:3].capitalize()}"
        prev_month = (pd.to_datetime(months[0] + "-01") - pd.DateOffset(months=1)).strftime("%Y-%m")
    else:
        q = quarter or "Q1"
        months = get_quarter_months(year, q)
        selected_period = q
        prev_month = (pd.to_datetime(months[0] + "-01") - pd.DateOffset(months=1)).strftime("%Y-%m")
    return months, selected_period, prev_month


class RevenueForecastAdjustmentManagement:
    @staticmethod
    async def GetAdjustments(year: int, quarter: str, company_id: str, db: AsyncSession) -> List[Dict[str, Any]]:
        return await _get_adjustments(db, year, quarter, company_id)

    @staticmethod
    async def PatchAdjustments(year: int, quarter: str, company_id: str, adjustments: List[RevenueAdjustmentInput], db: AsyncSession) -> Dict[str, Any]:
        # Bulk upsert — one round-trip per ~500 rows instead of one per row.
        # SQLAlchemy `execute(stmt, [list of dicts])` issues an `executemany`,
        # which Postgres pipelines. For 492 rows the wall-clock difference is
        # ~30x (we measured ~12s → ~400ms on the same payload).
        rows = [
            {
                "year": year, "quarter": quarter, "company_id": company_id,
                "customer_name": adj.customer_name,
                "month1_adjusted": adj.month1_adjusted, "month1_comment": adj.month1_comment,
                "month2_adjusted": adj.month2_adjusted, "month2_comment": adj.month2_comment,
                "month3_adjusted": adj.month3_adjusted, "month3_comment": adj.month3_comment,
                "overall_comment": adj.overall_comment,
            }
            for adj in adjustments if adj.customer_name
        ]
        if rows:
            await db.execute(
                text("""
                    INSERT INTO fpna_config.revenue_forecast_adjustments (
                        year, quarter, company_id, customer_name,
                        month1_adjusted, month1_comment, month2_adjusted, month2_comment,
                        month3_adjusted, month3_comment, overall_comment, updated_at
                    ) VALUES (
                        :year, :quarter, :company_id, :customer_name,
                        :month1_adjusted, :month1_comment, :month2_adjusted, :month2_comment,
                        :month3_adjusted, :month3_comment, :overall_comment, NOW()
                    )
                    ON CONFLICT (year, quarter, company_id, customer_name)
                    DO UPDATE SET
                        month1_adjusted = EXCLUDED.month1_adjusted, month1_comment = EXCLUDED.month1_comment,
                        month2_adjusted = EXCLUDED.month2_adjusted, month2_comment = EXCLUDED.month2_comment,
                        month3_adjusted = EXCLUDED.month3_adjusted, month3_comment = EXCLUDED.month3_comment,
                        overall_comment = EXCLUDED.overall_comment, updated_at = NOW();
                """),
                rows,
            )
        await db.commit()

        # Invalidate the response cache so the next GET picks up the new
        # adjustments instead of serving the stale 30-min cached payload.
        try:
            from .cache import cache_invalidate
            await cache_invalidate(db, f"rf:v1:{company_id}:")
        except Exception:    # noqa: BLE001
            import logging
            logging.getLogger(__name__).exception("cache_invalidate failed after PatchAdjustments")

        # Don't re-fetch and re-serialize the full adjustment list — the
        # frontend already has the rows in state. Returning just a count
        # cuts ~MBs of JSON off the response.
        return {"detail": "Revenue forecast adjustments saved successfully", "year": year, "quarter": quarter, "count": len(rows)}


@forecast_router.patch("/revenue-forecast/adjustments/{year}")
async def patch_revenue_forecast_adjustments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    adjustments: List[RevenueAdjustmentInput] = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Forecast is locked. Adjustments cannot be changed.")
    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Adjustments can only be edited in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")
    _invalidate_forecast_cache(company_id, year, quarter.value)
    return await RevenueForecastAdjustmentManagement.PatchAdjustments(year=year, quarter=quarter.value, company_id=company_id, adjustments=adjustments, db=db)


@forecast_router.patch("/revenue-forecast/driver-adjustments/{year}")
async def patch_driver_adjustments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    adjustments: List[DriverAdjustmentInput] = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Forecast is locked. Driver adjustments cannot be changed.")
    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Driver adjustments can only be edited in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")
    _invalidate_forecast_cache(company_id, year, quarter.value)
    # Bulk upsert — one round-trip rather than one-per-row (was ~10s for
    # 492 customers × 3 months).
    driver_rows = [
        {
            "year": year, "quarter": quarter.value, "company_id": company_id,
            "customer_name": adj.customer_name, "month": adj.month,
            "units_adjusted": adj.units_adjusted, "rate_adjusted": adj.rate_adjusted,
            "month_comment": adj.month_comment, "overall_comment": adj.overall_comment,
        }
        for adj in adjustments if adj.customer_name and adj.month
    ]
    if driver_rows:
        await db.execute(
            text("""
                INSERT INTO fpna_config.revenue_driver_adjustments (
                    year, quarter, company_id, customer_name, month,
                    units_adjusted, rate_adjusted, month_comment, overall_comment, updated_at
                ) VALUES (
                    :year, :quarter, :company_id, :customer_name, :month,
                    :units_adjusted, :rate_adjusted, :month_comment, :overall_comment, NOW()
                )
                ON CONFLICT (year, quarter, company_id, customer_name, month)
                DO UPDATE SET
                    units_adjusted = EXCLUDED.units_adjusted, rate_adjusted = EXCLUDED.rate_adjusted,
                    month_comment = EXCLUDED.month_comment, overall_comment = EXCLUDED.overall_comment,
                    updated_at = NOW();
            """),
            driver_rows,
        )
    await db.commit()

    # Also invalidate the L2 response cache for fresh GETs.
    try:
        from .cache import cache_invalidate
        await cache_invalidate(db, f"rf:v1:{company_id}:")
    except Exception:    # noqa: BLE001
        import logging
        logging.getLogger(__name__).exception("cache_invalidate failed after PatchDriverAdjustments")
    saved = (await db.execute(
        text("""
            SELECT customer_name, month, units_adjusted, rate_adjusted, month_comment, overall_comment
            FROM fpna_config.revenue_driver_adjustments
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id
            ORDER BY customer_name, month;
        """),
        {"year": year, "quarter": quarter.value, "company_id": company_id},
    )).mappings().all()
    customer_quarter_map: Dict[str, float] = {}
    result_rows = []
    for r in saved:
        u = float(r["units_adjusted"]) if r["units_adjusted"] is not None else None
        ra = float(r["rate_adjusted"]) if r["rate_adjusted"] is not None else None
        rev_adj = round(u * ra, 2) if u is not None and ra is not None else None
        cname = r["customer_name"]
        if rev_adj is not None:
            customer_quarter_map[cname] = customer_quarter_map.get(cname, 0.0) + rev_adj
        result_rows.append({"customer_name": cname, "month": r["month"], "units_adjusted": u, "rate_adjusted": ra, "revenue_adjusted": rev_adj, "month_comment": r["month_comment"], "overall_comment": r["overall_comment"]})
    for row in result_rows:
        row["quarter_revenue_adjusted"] = round(customer_quarter_map.get(row["customer_name"], 0.0), 2)
    return {"detail": "Driver adjustments saved successfully", "year": year, "quarter": quarter.value, "count": len(adjustments), "driver_adjustments": result_rows}


@forecast_router.patch("/revenue-forecast/workflow/{year}")
async def patch_revenue_forecast_workflow(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: WorkflowTransitionRequest = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_workflow(db, year, quarter.value, company_id, dept, user_id)
    current_stage = wf["stage"]
    target_stage = body.target_stage.value
    effective_comment = body.comment.strip() if body.comment and body.comment.strip() else None
    is_rejection = target_stage == WorkflowStage.DRAFT and current_stage != WorkflowStage.DRAFT
    if is_rejection and not effective_comment:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="A comment is required when rejecting (sending back to draft). Please provide a reason.")
    if effective_comment and dept not in DEPT_REVIEWERS | DEPT_APPROVERS:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Only Finance/Executive/CEO/CFO can add review comments.")
    if not can_transition(dept, current_stage, target_stage):
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Department '{dept}' cannot move forecast from '{current_stage}' to '{target_stage}'.")
    if target_stage == WorkflowStage.LOCKED:
        await db.execute(text("UPDATE fpna_config.forecast_workflows SET stage = :new_stage, updated_at = NOW(), locked_at = NOW() WHERE id = :wf_id"), {"new_stage": target_stage, "wf_id": wf["id"]})
    else:
        await db.execute(text("UPDATE fpna_config.forecast_workflows SET stage = :new_stage, updated_at = NOW() WHERE id = :wf_id"), {"new_stage": target_stage, "wf_id": wf["id"]})
    await db.execute(
        text("INSERT INTO fpna_config.forecast_workflow_audit (workflow_id, from_stage, to_stage, transitioned_by_user_id, transitioned_by_name, comment) VALUES (:wf_id, :from_s, :to_s, :uid, :uname, :comment)"),
        {"wf_id": wf["id"], "from_s": current_stage, "to_s": target_stage, "uid": user_id, "uname": current_user.get("full_name", ""), "comment": effective_comment},
    )
    if effective_comment and dept in DEPT_REVIEWERS | DEPT_APPROVERS:
        comment_type = "rejection" if is_rejection else "transition"
        await db.execute(
            text("INSERT INTO fpna_config.forecast_workflow_comments (workflow_id, user_id, user_name, department, comment, comment_type) VALUES (:wf_id, :uid, :uname, :dept, :comment, :comment_type)"),
            {"wf_id": wf["id"], "uid": user_id, "uname": current_user.get("full_name", ""), "dept": dept, "comment": effective_comment, "comment_type": comment_type},
        )
    await db.commit()
    return {"detail": f"Forecast moved from '{current_stage}' to '{target_stage}'.", "year": year, "quarter": quarter.value, "department": dept, "stage": target_stage}


@forecast_router.post("/revenue-forecast/{year}/comments")
async def post_forecast_comment(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: ForecastCommentInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    user_name = current_user.get("full_name", "")
    wf = await _get_or_create_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Forecast is locked. Comments cannot be added.")
    comment_text = body.comment.strip() if body.comment else ""
    if not comment_text:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Comment cannot be empty.")
    result = (await db.execute(
        text("INSERT INTO fpna_config.forecast_workflow_comments (workflow_id, user_id, user_name, department, comment, comment_type) VALUES (:wf_id, :uid, :uname, :dept, :comment, 'general') RETURNING id, user_id, user_name, department, comment, comment_type, created_at"),
        {"wf_id": wf["id"], "uid": user_id, "uname": user_name, "dept": dept, "comment": comment_text},
    )).mappings().first()
    await db.commit()
    return {"detail": "Comment posted successfully.", "year": year, "quarter": quarter.value, "workflow_stage": stage, "comment": {"id": result["id"], "user_id": result["user_id"], "user_name": result["user_name"], "department": result["department"], "comment": result["comment"], "comment_type": result["comment_type"], "created_at": result["created_at"].isoformat()}}


@forecast_router.get("/revenue-forecast/{year}/comments")
async def get_forecast_comments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    comment_type: Optional[str] = Query(None, description="Filter by comment_type: general, transition, rejection. Leave empty for all."),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if comment_type:
        rows = (await db.execute(text("SELECT id, user_id, user_name, department, comment, comment_type, created_at FROM fpna_config.forecast_workflow_comments WHERE workflow_id = :wf_id AND comment_type = :comment_type ORDER BY created_at ASC"), {"wf_id": wf["id"], "comment_type": comment_type})).mappings().all()
    else:
        rows = (await db.execute(text("SELECT id, user_id, user_name, department, comment, comment_type, created_at FROM fpna_config.forecast_workflow_comments WHERE workflow_id = :wf_id ORDER BY created_at ASC"), {"wf_id": wf["id"]})).mappings().all()
    comments = [{"id": r["id"], "user_id": r["user_id"], "user_name": r["user_name"], "department": r["department"], "comment": r["comment"], "comment_type": r["comment_type"], "created_at": r["created_at"].isoformat()} for r in rows]
    return {"year": year, "quarter": quarter.value, "workflow_stage": stage, "total_comments": len(comments), "filters": {"comment_type": comment_type or "all"}, "comments": comments}


@forecast_router.post("/revenue-forecast/{year}/scenarios")
async def create_forecast_scenario(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: ForecastScenarioInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    user_name = current_user.get("full_name", "")
    scenario_name_clean = body.scenario_name.strip().lower().replace(" ", "_")
    if scenario_name_clean in RESERVED_SCENARIO_NAMES:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"'{body.scenario_name}' is a reserved scenario name. Reserved names are: baseline, best_case, worst_case.")
    if not (0.01 <= body.multiplier <= 10.0):
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Multiplier must be between 0.01 and 10.0.")
    parameter_types = body.parameter_types if body.parameter_types else ["revenue"]
    customer_scope = body.customer_scope if body.customer_scope else "all"
    existing = (await db.execute(text("SELECT id FROM fpna_config.forecast_scenarios WHERE company_id = :company_id AND year = :year AND quarter = :quarter AND LOWER(REPLACE(scenario_name, ' ', '_')) = :scenario_name_clean"), {"company_id": company_id, "year": year, "quarter": quarter.value, "scenario_name_clean": scenario_name_clean})).mappings().first()
    if existing:
        raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=f"A scenario named '{body.scenario_name}' already exists for {year} {quarter.value}.")
    result = (await db.execute(
        text("""
            INSERT INTO fpna_config.forecast_scenarios
                (company_id, year, quarter, scenario_name, multiplier, description, parameter_types, customer_scope, created_by_user_id, created_by_name)
            VALUES (:company_id, :year, :quarter, :scenario_name, :multiplier, :description, :parameter_types, :customer_scope, :uid, :uname)
            RETURNING id, scenario_name, multiplier, description, parameter_types, customer_scope, created_by_name, created_at, updated_at
        """),
        {"company_id": company_id, "year": year, "quarter": quarter.value, "scenario_name": body.scenario_name.strip(), "multiplier": body.multiplier, "description": body.description.strip() if body.description else None, "parameter_types": json.dumps(parameter_types), "customer_scope": customer_scope, "uid": user_id, "uname": user_name},
    )).mappings().first()
    await db.commit()
    customer_multipliers_saved = []
    if customer_scope == "specific" and body.customer_multipliers:
        for cm in body.customer_multipliers:
            if not cm.customer_name or cm.multiplier is None or not (0.01 <= cm.multiplier <= 10.0):
                continue
            await db.execute(text("INSERT INTO fpna_config.forecast_scenario_customer_multipliers (scenario_id, company_id, year, quarter, customer_name, multiplier) VALUES (:scenario_id, :company_id, :year, :quarter, :customer_name, :multiplier) ON CONFLICT (scenario_id, company_id, year, quarter, customer_name) DO UPDATE SET multiplier = EXCLUDED.multiplier, updated_at = NOW()"), {"scenario_id": result["id"], "company_id": company_id, "year": year, "quarter": quarter.value, "customer_name": cm.customer_name, "multiplier": cm.multiplier})
            customer_multipliers_saved.append({"customer_name": cm.customer_name, "multiplier": cm.multiplier})
        await db.commit()
    return {"detail": "Scenario created successfully.", "year": year, "quarter": quarter.value, "scenario": {"id": result["id"], "scenario_name": result["scenario_name"], "multiplier": float(result["multiplier"]), "description": result["description"], "parameter_types": result["parameter_types"] if result["parameter_types"] else ["revenue"], "customer_scope": result["customer_scope"], "customer_multipliers": customer_multipliers_saved, "created_by_name": result["created_by_name"], "created_at": result["created_at"].isoformat(), "updated_at": result["updated_at"].isoformat()}}


@forecast_router.get("/revenue-forecast/{year}/scenarios")
async def get_forecast_scenarios(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    company_id = get_company_id(current_user)
    custom_scenarios = await _get_custom_scenarios(db, year, quarter.value, company_id)
    for s in custom_scenarios:
        s["is_system"] = False
        s["customer_multipliers"] = await _get_scenario_customer_multipliers(db, s["id"], company_id, year, quarter.value) if s["customer_scope"] == "specific" else {}
    hardcoded = [
        {"id": None, "scenario_name": "baseline", "multiplier": 1.0, "description": REVENUE_SCENARIO_DEFINITIONS["baseline"], "parameter_types": ["revenue"], "customer_scope": "all", "customer_multipliers": {}, "is_system": True, "created_by_name": None, "created_at": None, "updated_at": None},
        {"id": None, "scenario_name": "best_case", "multiplier": BEST_CASE_MULTIPLIER, "description": REVENUE_SCENARIO_DEFINITIONS["best_case"], "parameter_types": ["revenue", "rate"], "customer_scope": "all", "customer_multipliers": {}, "is_system": True, "created_by_name": None, "created_at": None, "updated_at": None},
        {"id": None, "scenario_name": "worst_case", "multiplier": WORST_CASE_MULTIPLIER, "description": REVENUE_SCENARIO_DEFINITIONS["worst_case"], "parameter_types": ["volume", "quantity"], "customer_scope": "all", "customer_multipliers": {}, "is_system": True, "created_by_name": None, "created_at": None, "updated_at": None},
    ]
    return {"year": year, "quarter": quarter.value, "total_scenarios": len(hardcoded) + len(custom_scenarios), "scenarios": hardcoded + custom_scenarios}


@forecast_router.patch("/revenue-forecast/{year}/scenarios/{scenario_id}")
async def update_forecast_scenario(
    year: int = Path(...),
    scenario_id: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: ForecastScenarioUpdateInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    company_id = get_company_id(current_user)
    existing = (await db.execute(text("SELECT id, scenario_name FROM fpna_config.forecast_scenarios WHERE id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter"), {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value})).mappings().first()
    if not existing:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Scenario with id {scenario_id} not found for {year} {quarter.value}.")
    if body.scenario_name is not None:
        new_name_clean = body.scenario_name.strip().lower().replace(" ", "_")
        if new_name_clean in RESERVED_SCENARIO_NAMES:
            raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"'{body.scenario_name}' is a reserved scenario name. Reserved names are: baseline, best_case, worst_case.")
        if (await db.execute(text("SELECT id FROM fpna_config.forecast_scenarios WHERE company_id = :company_id AND year = :year AND quarter = :quarter AND LOWER(REPLACE(scenario_name, ' ', '_')) = :new_name_clean AND id != :sid"), {"company_id": company_id, "year": year, "quarter": quarter.value, "new_name_clean": new_name_clean, "sid": scenario_id})).mappings().first():
            raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=f"A scenario named '{body.scenario_name}' already exists for {year} {quarter.value}.")
    if body.multiplier is not None and not (0.01 <= body.multiplier <= 10.0):
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Multiplier must be between 0.01 and 10.0.")
    set_clauses = ["updated_at = NOW()"]
    params: Dict[str, Any] = {"sid": scenario_id}
    if body.scenario_name is not None:
        set_clauses.append("scenario_name = :scenario_name")
        params["scenario_name"] = body.scenario_name.strip()
    if body.multiplier is not None:
        set_clauses.append("multiplier = :multiplier")
        params["multiplier"] = body.multiplier
    if body.description is not None:
        set_clauses.append("description = :description")
        params["description"] = body.description.strip()
    if body.parameter_types is not None:
        set_clauses.append("parameter_types = :parameter_types")
        params["parameter_types"] = json.dumps(body.parameter_types)
    if body.customer_scope is not None:
        set_clauses.append("customer_scope = :customer_scope")
        params["customer_scope"] = body.customer_scope
    result = (await db.execute(text(f"UPDATE fpna_config.forecast_scenarios SET {', '.join(set_clauses)} WHERE id = :sid RETURNING id, scenario_name, multiplier, description, parameter_types, customer_scope, created_by_name, created_at, updated_at"), params)).mappings().first()
    await db.commit()
    new_scope = result["customer_scope"] if result["customer_scope"] else "all"
    customer_multipliers_saved = {}
    del_params = {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value}
    if new_scope == "specific" and body.customer_multipliers is not None:
        await db.execute(text("DELETE FROM fpna_config.forecast_scenario_customer_multipliers WHERE scenario_id = :scenario_id AND company_id = :company_id AND year = :year AND quarter = :quarter"), del_params)
        for cm in body.customer_multipliers:
            if not cm.customer_name or cm.multiplier is None or not (0.01 <= cm.multiplier <= 10.0):
                continue
            await db.execute(text("INSERT INTO fpna_config.forecast_scenario_customer_multipliers (scenario_id, company_id, year, quarter, customer_name, multiplier) VALUES (:scenario_id, :company_id, :year, :quarter, :customer_name, :multiplier) ON CONFLICT (scenario_id, company_id, year, quarter, customer_name) DO UPDATE SET multiplier = EXCLUDED.multiplier, updated_at = NOW()"), {**del_params, "customer_name": cm.customer_name, "multiplier": cm.multiplier})
            customer_multipliers_saved[cm.customer_name] = cm.multiplier
        await db.commit()
    elif new_scope == "all":
        await db.execute(text("DELETE FROM fpna_config.forecast_scenario_customer_multipliers WHERE scenario_id = :scenario_id AND company_id = :company_id AND year = :year AND quarter = :quarter"), del_params)
        await db.commit()
    else:
        customer_multipliers_saved = await _get_scenario_customer_multipliers(db, scenario_id, company_id, year, quarter.value)
    return {"detail": "Scenario updated successfully.", "year": year, "quarter": quarter.value, "scenario": {"id": result["id"], "scenario_name": result["scenario_name"], "multiplier": float(result["multiplier"]), "description": result["description"], "parameter_types": result["parameter_types"] if result["parameter_types"] else ["revenue"], "customer_scope": result["customer_scope"], "customer_multipliers": customer_multipliers_saved, "created_by_name": result["created_by_name"], "created_at": result["created_at"].isoformat(), "updated_at": result["updated_at"].isoformat()}}


@forecast_router.delete("/revenue-forecast/{year}/scenarios/{scenario_id}")
async def delete_forecast_scenario(
    year: int = Path(...),
    scenario_id: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    company_id = get_company_id(current_user)
    existing = (await db.execute(text("SELECT id, scenario_name FROM fpna_config.forecast_scenarios WHERE id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter"), {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value})).mappings().first()
    if not existing:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Scenario with id {scenario_id} not found for {year} {quarter.value}.")
    await db.execute(text("DELETE FROM fpna_config.forecast_scenario_customer_multipliers WHERE scenario_id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter"), {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value})
    await db.execute(text("DELETE FROM fpna_config.forecast_scenarios WHERE id = :sid"), {"sid": scenario_id})
    await db.commit()
    return {"detail": f"Scenario '{existing['scenario_name']}' deleted successfully.", "year": year, "quarter": quarter.value, "deleted_scenario_id": scenario_id}


async def _build_excel_forecast_data(db: AsyncSession, year: int, quarter_value: str, company_id: str):
    m1, m2, m3 = get_quarter_months(year, quarter_value)
    prev_month = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=1)).strftime("%Y-%m")
    history_months_start = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")
    customer_history_sql = text("""
        SELECT c.customer_name, gl.period, SUM(gl.amount_credit - gl.amount_debit) AS revenue
        FROM fpna_landing.finance_general_ledger_transactions gl
        JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
        JOIN fpna_landing.finance_customers c
            ON REGEXP_REPLACE(gl.customer_id::TEXT, '\\.0$', '') = REGEXP_REPLACE(c.customer_id::TEXT, '\\.0$', '')
        WHERE coa.account_type = 'Revenue' AND gl.period BETWEEN :start_period AND :end_period AND gl.customer_id IS NOT NULL
        GROUP BY c.customer_name, gl.period ORDER BY c.customer_name, gl.period;
    """)
    prev_customer_sql = text("""
        SELECT c.customer_name, SUM(gl.amount_credit - gl.amount_debit) AS revenue
        FROM fpna_landing.finance_general_ledger_transactions gl
        JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
        JOIN fpna_landing.finance_customers c
            ON REGEXP_REPLACE(gl.customer_id::TEXT, '\\.0$', '') = REGEXP_REPLACE(c.customer_id::TEXT, '\\.0$', '')
        WHERE coa.account_type = 'Revenue' AND gl.period = :prev_month AND gl.customer_id IS NOT NULL
        GROUP BY c.customer_name;
    """)
    cust_hist_result, prev_cust_result = await asyncio.gather(
        db.execute(customer_history_sql, {"start_period": history_months_start, "end_period": prev_month}),
        db.execute(prev_customer_sql, {"prev_month": prev_month}),
    )
    cust_hist_df = pd.DataFrame(cust_hist_result.mappings().all())
    if cust_hist_df.empty:
        cust_hist_df = pd.DataFrame([{"customer_name": "Unassigned", "period": prev_month, "revenue": 0.0}])
    cust_hist_df = cust_hist_df.rename(columns={"revenue": "y"})
    prev_cust_df = pd.DataFrame(prev_cust_result.mappings().all())
    prev_cust_map = dict(zip(prev_cust_df["customer_name"], prev_cust_df["revenue"])) if not prev_cust_df.empty else {}
    all_customers = cust_hist_df["customer_name"].unique().tolist()
    quarter_start = pd.to_datetime(m1 + "-01")
    customer_forecasts = await _forecast_all_customers(cust_hist_df, all_customers, quarter_start, m1, m2, m3)
    return m1, m2, m3, prev_month, all_customers, customer_forecasts, prev_cust_map


@forecast_router.get("/revenue-forecast/{year}/excel-template/revenue-adjustments")
async def download_revenue_adjustments_template(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    company_id = get_company_id(current_user)
    m1, m2, m3, prev_month, all_customers, customer_forecasts, prev_cust_map = await _build_excel_forecast_data(db, year, quarter.value, company_id)
    saved = await _get_adjustments(db, year, quarter.value, company_id)
    adj_map = {s["customer_name"]: s for s in saved}
    wb = openpyxl.Workbook()
    ws = wb.active
    ws.title = "Revenue Adjustments"
    ws.append(["Customer Name", f"Previous Month ({prev_month})", f"{m1} AI Forecast", f"{m1} Adjusted", f"{m1} Comment", f"{m2} AI Forecast", f"{m2} Adjusted", f"{m2} Comment", f"{m3} AI Forecast", f"{m3} Adjusted", f"{m3} Comment", "Quarter Total AI", "Quarter Total Adjusted", "Overall Comment"])
    for customer in all_customers:
        adj = adj_map.get(customer, {})
        prev_val = round(float(prev_cust_map.get(customer, 0.0)), 2)
        m1_ai, m2_ai, m3_ai = customer_forecasts.get(customer, (0.0, 0.0, 0.0))
        m1_ai, m2_ai, m3_ai = round(float(m1_ai), 2), round(float(m2_ai), 2), round(float(m3_ai), 2)
        m1_adj = adj.get("month1_adjusted")
        m2_adj = adj.get("month2_adjusted")
        m3_adj = adj.get("month3_adjusted")
        ws.append([customer, prev_val, m1_ai, m1_adj, adj.get("month1_comment"), m2_ai, m2_adj, adj.get("month2_comment"), m3_ai, m3_adj, adj.get("month3_comment"), round(m1_ai + m2_ai + m3_ai, 2), adj.get("quarter_total_adjusted") if (m1_adj or m2_adj or m3_adj) else None, adj.get("overall_comment")])
    stream = io.BytesIO()
    wb.save(stream)
    stream.seek(0)
    return StreamingResponse(stream, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", headers={"Content-Disposition": f"attachment; filename=revenue_adjustments_{year}_{quarter.value}.xlsx"})


@forecast_router.get("/revenue-forecast/{year}/excel-template/driver-adjustments")
async def download_driver_adjustments_template(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    company_id = get_company_id(current_user)
    m1, m2, m3, prev_month, all_customers, customer_forecasts, prev_cust_map = await _build_excel_forecast_data(db, year, quarter.value, company_id)
    driver_saved = await _get_driver_adjustments(db, year, quarter.value, company_id)
    wb = openpyxl.Workbook()
    ws = wb.active
    ws.title = "Driver Adjustments"
    ws.append(["Customer Name", "Month", f"Previous Month ({prev_month}) Revenue", "Units AI", "Rate AI", "Revenue AI", "Units Adjusted", "Rate Adjusted", "Revenue Adjusted", "Month Comment", "Quarter Revenue AI", "Quarter Revenue Adjusted", "Overall Comment"])
    customer_quarter_ai: Dict[str, float] = {}
    customer_quarter_drv: Dict[str, float] = {}
    for customer in all_customers:
        m1_ai, m2_ai, m3_ai = customer_forecasts.get(customer, (0.0, 0.0, 0.0))
        ai_map = {m1: float(m1_ai), m2: float(m2_ai), m3: float(m3_ai)}
        customer_quarter_ai[customer] = round(sum(ai_map.values()), 2)
        q_drv = 0.0
        for month_label in [m1, m2, m3]:
            drv = driver_saved.get((customer, month_label), {})
            u_ai, r_ai = derive_units_and_rate(ai_map[month_label])
            u_user, r_user = drv.get("units_adjusted"), drv.get("rate_adjusted")
            if u_user is not None and r_user is not None:
                q_drv += round(u_user * r_user, 2)
            elif u_user is not None:
                q_drv += round(u_user * r_ai, 2)
            elif r_user is not None:
                q_drv += round(u_ai * r_user, 2)
        customer_quarter_drv[customer] = round(q_drv, 2)
    for customer in all_customers:
        m1_ai, m2_ai, m3_ai = customer_forecasts.get(customer, (0.0, 0.0, 0.0))
        ai_map = {m1: float(m1_ai), m2: float(m2_ai), m3: float(m3_ai)}
        prev_val = round(float(prev_cust_map.get(customer, 0.0)), 2)
        for month_label in [m1, m2, m3]:
            rev_ai = ai_map[month_label]
            u_ai, r_ai = derive_units_and_rate(rev_ai)
            drv = driver_saved.get((customer, month_label), {})
            u_user, r_user = drv.get("units_adjusted"), drv.get("rate_adjusted")
            if u_user is not None and r_user is not None:
                rev_adj = round(u_user * r_user, 2)
            elif u_user is not None:
                rev_adj = round(u_user * r_ai, 2)
            elif r_user is not None:
                rev_adj = round(u_ai * r_user, 2)
            else:
                rev_adj = None
            ws.append([customer, month_label, prev_val, round(u_ai, 4), round(r_ai, 2), round(rev_ai, 2), u_user, r_user, rev_adj, drv.get("month_comment"), customer_quarter_ai[customer], customer_quarter_drv[customer] if customer_quarter_drv[customer] else None, drv.get("overall_comment")])
    stream = io.BytesIO()
    wb.save(stream)
    stream.seek(0)
    return StreamingResponse(stream, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", headers={"Content-Disposition": f"attachment; filename=driver_adjustments_{year}_{quarter.value}.xlsx"})


async def _check_upload_preconditions(db, year, quarter_value, company_id, dept, user_id, contents):
    wf = await _get_or_create_workflow(db, year, quarter_value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Forecast is locked. Uploads cannot be processed.")
    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Uploads can only be processed in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")
    if len(contents) > 5 * 1024 * 1024:
        raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File size exceeds 5MB limit.")
    try:
        return openpyxl.load_workbook(io.BytesIO(contents))
    except Exception:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Could not read the uploaded file. Please ensure it is a valid .xlsx file.")


def _get_float_cell(row, idx):
    if idx >= len(row) or row[idx] is None:
        return None
    try:
        return float(row[idx])
    except (TypeError, ValueError):
        return None


def _get_str_cell(row, idx):
    if idx >= len(row) or row[idx] is None:
        return None
    val = str(row[idx]).strip()
    return val if val else None


@forecast_router.post("/revenue-forecast/{year}/excel-upload/revenue-adjustments")
async def upload_revenue_adjustments_excel(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    file: UploadFile = File(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    contents = await file.read()
    wb = await _check_upload_preconditions(db, year, quarter.value, company_id, dept, user_id, contents)
    _invalidate_forecast_cache(company_id, year, quarter.value)
    m1, m2, m3 = get_quarter_months(year, quarter.value)
    sheet_name = "Revenue Adjustments"
    if sheet_name not in wb.sheetnames:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Sheet '{sheet_name}' not found in the uploaded file.")
    ws_adj = wb[sheet_name]
    adj_hmap = {str(c.value).strip() if c.value is not None else "": i for i, c in enumerate(ws_adj[1])}
    adjustments, errors = [], []
    for row_idx, row in enumerate(ws_adj.iter_rows(min_row=2, values_only=True), start=2):
        if not row or row[0] is None:
            continue
        customer_name = str(row[0]).strip()
        if not customer_name:
            continue
        try:
            m1_adj = _get_float_cell(row, adj_hmap.get(f"{m1} Adjusted", 999))
            m1_cmt = _get_str_cell(row, adj_hmap.get(f"{m1} Comment", 999))
            m2_adj = _get_float_cell(row, adj_hmap.get(f"{m2} Adjusted", 999))
            m2_cmt = _get_str_cell(row, adj_hmap.get(f"{m2} Comment", 999))
            m3_adj = _get_float_cell(row, adj_hmap.get(f"{m3} Adjusted", 999))
            m3_cmt = _get_str_cell(row, adj_hmap.get(f"{m3} Comment", 999))
            overall_comment = _get_str_cell(row, adj_hmap.get("Overall Comment", 999))
        except Exception:
            errors.append(f"Revenue Adjustments row {row_idx}: parse error for '{customer_name}'. Skipped.")
            continue
        if any(v is not None for v in [m1_adj, m2_adj, m3_adj, m1_cmt, m2_cmt, m3_cmt, overall_comment]):
            adjustments.append(RevenueAdjustmentInput(customer_name=customer_name, month1_adjusted=m1_adj, month1_comment=m1_cmt, month2_adjusted=m2_adj, month2_comment=m2_cmt, month3_adjusted=m3_adj, month3_comment=m3_cmt, overall_comment=overall_comment))
    if not adjustments:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="No valid rows found in the Revenue Adjustments sheet.")
    result = await RevenueForecastAdjustmentManagement.PatchAdjustments(year=year, quarter=quarter.value, company_id=company_id, adjustments=adjustments, db=db)
    result["upload_errors"] = errors
    result["rows_attempted"] = len(adjustments) + len(errors)
    result["rows_saved"] = len(adjustments)
    return result


@forecast_router.post("/revenue-forecast/{year}/excel-upload/driver-adjustments")
async def upload_driver_adjustments_excel(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    file: UploadFile = File(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    contents = await file.read()
    wb = await _check_upload_preconditions(db, year, quarter.value, company_id, dept, user_id, contents)
    _invalidate_forecast_cache(company_id, year, quarter.value)
    sheet_name = "Driver Adjustments"
    if sheet_name not in wb.sheetnames:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Sheet '{sheet_name}' not found in the uploaded file.")
    ws_drv = wb[sheet_name]
    drv_hmap = {str(c.value).strip() if c.value is not None else "": i for i, c in enumerate(ws_drv[1])}
    driver_adjustments, errors = [], []
    for row_idx, row in enumerate(ws_drv.iter_rows(min_row=2, values_only=True), start=2):
        if not row or row[0] is None:
            continue
        customer_name = str(row[0]).strip()
        if not customer_name:
            continue
        try:
            month_val = _get_str_cell(row, drv_hmap.get("Month", 999))
            units_adj = _get_float_cell(row, drv_hmap.get("Units Adjusted", 999))
            rate_adj = _get_float_cell(row, drv_hmap.get("Rate Adjusted", 999))
            month_cmt = _get_str_cell(row, drv_hmap.get("Month Comment", 999))
            overall_comment = _get_str_cell(row, drv_hmap.get("Overall Comment", 999))
        except Exception:
            errors.append(f"Driver Adjustments row {row_idx}: parse error for '{customer_name}'. Skipped.")
            continue
        if not month_val:
            continue
        if any(v is not None for v in [units_adj, rate_adj, month_cmt, overall_comment]):
            driver_adjustments.append(DriverAdjustmentInput(customer_name=customer_name, month=month_val, units_adjusted=units_adj, rate_adjusted=rate_adj, month_comment=month_cmt, overall_comment=overall_comment))
    if not driver_adjustments:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="No valid rows found in the Driver Adjustments sheet.")
    for adj in driver_adjustments:
        if not adj.customer_name or not adj.month:
            continue
        await db.execute(text("""
            INSERT INTO fpna_config.revenue_driver_adjustments (
                year, quarter, company_id, customer_name, month,
                units_adjusted, rate_adjusted, month_comment, overall_comment, updated_at
            ) VALUES (:year, :quarter, :company_id, :customer_name, :month, :units_adjusted, :rate_adjusted, :month_comment, :overall_comment, NOW())
            ON CONFLICT (year, quarter, company_id, customer_name, month)
            DO UPDATE SET units_adjusted = EXCLUDED.units_adjusted, rate_adjusted = EXCLUDED.rate_adjusted,
                month_comment = EXCLUDED.month_comment, overall_comment = EXCLUDED.overall_comment, updated_at = NOW();
        """), {"year": year, "quarter": quarter.value, "company_id": company_id, "customer_name": adj.customer_name, "month": adj.month, "units_adjusted": adj.units_adjusted, "rate_adjusted": adj.rate_adjusted, "month_comment": adj.month_comment, "overall_comment": adj.overall_comment})
    await db.commit()
    saved = (await db.execute(text("SELECT customer_name, month, units_adjusted, rate_adjusted, month_comment, overall_comment FROM fpna_config.revenue_driver_adjustments WHERE year = :year AND quarter = :quarter AND company_id = :company_id ORDER BY customer_name, month;"), {"year": year, "quarter": quarter.value, "company_id": company_id})).mappings().all()
    customer_quarter_map: Dict[str, float] = {}
    result_rows = []
    for r in saved:
        u = float(r["units_adjusted"]) if r["units_adjusted"] is not None else None
        ra = float(r["rate_adjusted"]) if r["rate_adjusted"] is not None else None
        rev_adj = round(u * ra, 2) if u is not None and ra is not None else None
        cname = r["customer_name"]
        if rev_adj is not None:
            customer_quarter_map[cname] = customer_quarter_map.get(cname, 0.0) + rev_adj
        result_rows.append({"customer_name": cname, "month": r["month"], "units_adjusted": u, "rate_adjusted": ra, "revenue_adjusted": rev_adj, "month_comment": r["month_comment"], "overall_comment": r["overall_comment"]})
    for row in result_rows:
        row["quarter_revenue_adjusted"] = round(customer_quarter_map.get(row["customer_name"], 0.0), 2)
    return {"detail": "Driver adjustments uploaded successfully", "year": year, "quarter": quarter.value, "upload_errors": errors, "rows_attempted": len(driver_adjustments) + len(errors), "rows_saved": len(driver_adjustments), "driver_adjustments": result_rows}


async def _build_forecast_rows_for_periods(
    customers: List[str],
    customer_forecasts_map: Dict[str, List[float]],
    month_periods: List[str],
    adj_map: Dict[str, Any],
    driver_saved: Dict[tuple, Dict],
    prev_cust_map: Dict[str, float],
) -> Tuple[List[Dict], Dict[str, float], Dict[str, float], Dict[str, List]]:
    rows = []
    totals: Dict[str, float] = {f"month{i+1}_ai": 0.0 for i in range(len(month_periods))}
    totals.update({f"month{i+1}_adjusted": 0.0 for i in range(len(month_periods))})
    totals["previous_month"] = 0.0
    totals["period_total_ai"] = 0.0
    totals["period_total_adjusted"] = 0.0
    customer_ai_map: Dict[str, float] = {}
    customer_user_map: Dict[str, float] = {}
    driver_rows_map: Dict[str, List] = {}

    for customer in customers:
        adj = adj_map.get(customer)
        prev_value = float(prev_cust_map.get(customer, 0.0))
        all_month_forecasts = customer_forecasts_map.get(customer, [0.0] * len(month_periods))

        month_ai_vals = []
        month_adj_vals = []
        month_comments = []
        driver_overrides: Dict[str, float] = {}
        driver_month_comments_map: Dict[str, Optional[str]] = {}

        for idx, month_label in enumerate(month_periods):
            rev_ai = float(all_month_forecasts[idx]) if idx < len(all_month_forecasts) else 0.0
            u_ai, r_ai = derive_units_and_rate(rev_ai)
            drv = driver_saved.get((customer, month_label))
            drv_month_comment = None
            if drv:
                u_user = drv["units_adjusted"]
                r_user = drv["rate_adjusted"]
                drv_month_comment = drv.get("month_comment")
                if u_user is not None and r_user is not None:
                    rev_adj_driver = round(u_user * r_user, 2)
                elif u_user is not None:
                    rev_adj_driver = round(u_user * r_ai, 2)
                elif r_user is not None:
                    rev_adj_driver = round(u_ai * r_user, 2)
                else:
                    rev_adj_driver = None
            else:
                u_user = r_user = rev_adj_driver = None
            driver_month_comments_map[month_label] = drv_month_comment
            if customer not in driver_rows_map:
                driver_rows_map[customer] = []
            driver_rows_map[customer].append({
                "customer_name": customer, "month": month_label,
                "units_ai": u_ai, "rate_ai": r_ai, "revenue_ai": round(u_ai * r_ai, 2),
                "units_adjusted": round(u_user, 4) if u_user is not None else None,
                "rate_adjusted": round(r_user, 2) if r_user is not None else None,
                "revenue_adjusted": rev_adj_driver, "month_comment": drv_month_comment,
            })
            if rev_adj_driver is not None:
                driver_overrides[month_label] = rev_adj_driver

            if month_label in driver_overrides:
                final_adj = driver_overrides[month_label]
                final_comment = driver_month_comments_map.get(month_label)
            elif adj is not None:
                adj_key = f"month{idx+1}_adjusted"
                adj_comment_key = f"month{idx+1}_comment"
                adj_val = getattr(adj, adj_key, None) if hasattr(adj, adj_key) else None
                if adj_val is None and isinstance(adj, dict):
                    adj_val = adj.get(adj_key)
                final_adj = float(adj_val) if adj_val is not None else rev_ai
                final_comment = getattr(adj, adj_comment_key, None) if hasattr(adj, adj_comment_key) else (adj.get(adj_comment_key) if isinstance(adj, dict) else None)
            else:
                final_adj = rev_ai
                final_comment = None

            month_ai_vals.append(round(rev_ai, 2))
            month_adj_vals.append(round(final_adj, 2))
            month_comments.append(final_comment)

        period_total_ai = round(sum(month_ai_vals), 2)
        period_total_adjusted = round(sum(month_adj_vals), 2)
        customer_ai_map[customer] = period_total_ai
        customer_user_map[customer] = period_total_adjusted
        overall_comment = None
        if adj is not None:
            overall_comment = getattr(adj, "overall_comment", None) if hasattr(adj, "overall_comment") else (adj.get("overall_comment") if isinstance(adj, dict) else None)

        row_data: Dict[str, Any] = {
            "customer_name": customer,
            "previous_month": round(prev_value, 2),
        }
        for idx, month_label in enumerate(month_periods):
            row_data[f"month{idx+1}_ai"] = round(month_ai_vals[idx], 2)
            row_data[f"month{idx+1}_adjusted"] = round(month_adj_vals[idx], 2)
            row_data[f"month{idx+1}_comment"] = month_comments[idx]
            totals[f"month{idx+1}_ai"] = round(totals.get(f"month{idx+1}_ai", 0.0) + month_ai_vals[idx], 2)
            totals[f"month{idx+1}_adjusted"] = round(totals.get(f"month{idx+1}_adjusted", 0.0) + month_adj_vals[idx], 2)
        row_data["quarter_total_ai"] = period_total_ai
        row_data["quarter_total_adjusted"] = period_total_adjusted
        row_data["overall_comment"] = overall_comment

        totals["previous_month"] = round(totals["previous_month"] + prev_value, 2)
        totals["period_total_ai"] = round(totals["period_total_ai"] + period_total_ai, 2)
        totals["period_total_adjusted"] = round(totals["period_total_adjusted"] + period_total_adjusted, 2)
        rows.append(row_data)

    return rows, customer_ai_map, customer_user_map, driver_rows_map


@forecast_router.get("/revenue-forecast/{year}/{currency}")
async def revenue_forecast(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    forecast_year: Optional[int] = Query(default=None, description="Primary forecast year context. Overrides year for period resolution when provided."),
    forecast_type: Optional[str] = Query(default="quarter", description="Forecast type: quarter, 12months, rolling"),
    rolling_start_month: Optional[str] = Query(default=None, description="Start month for rolling forecast (e.g. Jan, Feb, Mar). Required when forecast_type=rolling."),
    view_type: Optional[str] = Query(default="monthly_breakdown", description="View type: quarter_summary or monthly_breakdown"),
    customer_name: Optional[str] = Query(default=None, description="Filter results to a specific customer name (case-insensitive partial match)."),
    page: int = Query(default=1, ge=1, description="Page index (1-based) for the rows array."),
    page_size: int = Query(default=0, ge=0, le=1000, description="Rows per page. 0 = no pagination (return all)."),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    effective_year = forecast_year if forecast_year is not None else year
    effective_forecast_type = forecast_type or "quarter"
    effective_view_type = view_type or "monthly_breakdown"

    if effective_forecast_type not in ("quarter", "12months", "rolling"):
        raise HTTPException(status_code=422, detail="forecast_type must be one of: quarter, 12months, rolling")
    if effective_view_type not in ("quarter_summary", "monthly_breakdown"):
        raise HTTPException(status_code=422, detail="view_type must be one of: quarter_summary, monthly_breakdown")
    if effective_forecast_type == "rolling" and not rolling_start_month:
        raise HTTPException(status_code=422, detail="rolling_start_month is required when forecast_type=rolling. Example: Jan, Feb, Mar, etc.")

    _MONTH_ORDER = {"jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
                    "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12}
    resolve_year = effective_year
    if effective_forecast_type == "rolling" and rolling_start_month:
        if _MONTH_ORDER.get(rolling_start_month.lower()[:3], 1) >= 11:
            resolve_year = effective_year - 1

    month_periods, selected_period, prev_month = _resolve_forecast_periods(
        resolve_year, effective_forecast_type, quarter.value, rolling_start_month
    )

    effective_quarter = quarter.value
    if effective_forecast_type == "12months":
        effective_quarter = "Q1"
    elif effective_forecast_type == "rolling":
        month_num = int(month_periods[0].split("-")[1])
        if month_num <= 3:
            effective_quarter = "Q1"
        elif month_num <= 6:
            effective_quarter = "Q2"
        elif month_num <= 9:
            effective_quarter = "Q3"
        else:
            effective_quarter = "Q4"

    wf = await _get_or_create_workflow(db, effective_year, effective_quarter, company_id, dept, user_id)
    stage = wf["stage"]

    # ── L1/L2 cache (Postgres-backed) ────────────────────────────────────
    # Cache the *unit-converted* payload keyed by every filter that affects
    # the response. Previous behavior cached only forecast_type='quarter' with
    # no customer name — that bypassed cache for every other filter combo.
    from .cache import build_cache_key, cache_get, cache_set
    rf_cache_key = build_cache_key(
        company_id=company_id, year=year, currency=currency,
        quarter=quarter.value, currency_filter=currency_filter,
        forecast_type=effective_forecast_type, view_type=effective_view_type,
        rolling_start_month=rolling_start_month, customer_name=customer_name,
        forecast_year=effective_year, page=page, page_size=page_size,
    )
    rf_cached = await cache_get(db, rf_cache_key)
    if rf_cached is not None:
        # Refresh the workflow stage on cached payload so stale 'draft' doesn't
        # mask a transition to 'submitted'/'reviewed'/etc.
        rf_cached["workflow_stage"] = stage
        return rf_cached

    forecast_start = pd.to_datetime(month_periods[0] + "-01")
    n_months = len(month_periods)
    history_start = (forecast_start - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")

    customer_history_sql = text("""
        SELECT c.customer_name, gl.period, SUM(gl.amount_credit - gl.amount_debit) AS revenue
        FROM fpna_landing.finance_general_ledger_transactions gl
        JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
        JOIN fpna_landing.finance_customers c
            ON REGEXP_REPLACE(gl.customer_id::TEXT, '\\.0$', '') = REGEXP_REPLACE(c.customer_id::TEXT, '\\.0$', '')
        WHERE coa.account_type = 'Revenue' AND gl.period BETWEEN :start_period AND :end_period AND gl.customer_id IS NOT NULL
        GROUP BY c.customer_name, gl.period ORDER BY c.customer_name, gl.period;
    """)

    gather_tasks = [
        db.execute(customer_history_sql, {"start_period": history_start, "end_period": prev_month}),
        _get_adjustments(db, effective_year, effective_quarter, company_id),
        _get_driver_adjustments(db, effective_year, effective_quarter, company_id),
        _get_custom_scenarios(db, effective_year, effective_quarter, company_id),
        db.execute(text("SELECT to_stage, transitioned_by_name FROM fpna_config.forecast_workflow_audit WHERE workflow_id = :wf_id ORDER BY created_at ASC"), {"wf_id": wf["id"]}),
    ]

    results = await asyncio.gather(*gather_tasks)
    cust_hist_result, saved, driver_saved, custom_scenarios, audit_rows_result = results

    audit_rows = audit_rows_result.mappings().all()
    reviewed_by = approved_by = locked_by = None
    for r in audit_rows:
        if r["to_stage"] == "reviewed":
            reviewed_by = r["transitioned_by_name"]
        elif r["to_stage"] == "approved":
            approved_by = r["transitioned_by_name"]
        elif r["to_stage"] == "locked":
            locked_by = r["transitioned_by_name"]

    cust_hist_df = pd.DataFrame(cust_hist_result.mappings().all())

    if cust_hist_df.empty:
        cust_hist_df = pd.DataFrame([{"customer_name": "Unassigned", "period": prev_month, "revenue": 0.0}])

    prev_month_mask = cust_hist_df["period"] == prev_month
    prev_cust_map = (
        cust_hist_df[prev_month_mask].groupby("customer_name")["revenue"].sum().to_dict()
        if prev_month_mask.any() else {}
    )

    cust_hist_df = cust_hist_df.rename(columns={"revenue": "y"})
    all_customers = cust_hist_df["customer_name"].unique().tolist()

    if customer_name:
        customer_name_lower = customer_name.lower()
        all_customers = [c for c in all_customers if customer_name_lower in c.lower()]

    adj_map = {
        s["customer_name"]: s
        for s in saved
    }

    # Fast path: load AI forecasts from the precompute table. Only run the
    # expensive Prophet pipeline for customers that aren't covered yet (newly
    # added customers, or before the first precompute job has run).
    from .precompute import load_precomputed_n_months as _load_precomputed_n_months

    precomputed = await _load_precomputed_n_months(db, company_id, effective_year, month_periods)
    missing_customers = [c for c in all_customers if c not in precomputed]

    if missing_customers:
        # Fallback: forecast just the uncovered customers synchronously.
        cust_hist_missing = cust_hist_df[cust_hist_df["customer_name"].isin(missing_customers)]
        if effective_forecast_type == "quarter" and n_months == 3:
            m1, m2, m3 = month_periods[0], month_periods[1], month_periods[2]
            fallback_3 = await _forecast_all_customers(
                cust_hist_missing, missing_customers, forecast_start, m1, m2, m3,
            )
            for cname in missing_customers:
                precomputed[cname] = list(fallback_3.get(cname, (0.0, 0.0, 0.0)))
        else:
            fallback_n = await _forecast_all_customers_n_months(
                cust_hist_missing, missing_customers, forecast_start, n_months,
            )
            for cname in missing_customers:
                precomputed[cname] = fallback_n.get(cname, [0.0] * n_months)

    customer_forecasts_map: Dict[str, List[float]] = {
        cname: list(precomputed.get(cname, [0.0] * n_months))[:n_months]
        for cname in all_customers
    }

    rows, customer_ai_map, customer_user_map, driver_rows_map = await _build_forecast_rows_for_periods(
        all_customers, customer_forecasts_map, month_periods, adj_map, driver_saved, prev_cust_map
    )

    period_total_ai = sum(customer_ai_map.values())
    period_total_adjusted = sum(customer_user_map.values())

    month_totals = []
    for idx, month_label in enumerate(month_periods):
        m_ai = sum(row.get(f"month{idx+1}_ai", 0.0) for row in rows)
        m_adj = sum(row.get(f"month{idx+1}_adjusted", 0.0) for row in rows)
        month_totals.append({"month": month_label, "total_ai": round(m_ai, 2), "total_adjusted": round(m_adj, 2)})

    drivers_output = []
    for cname, month_rows in driver_rows_map.items():
        q_ai = sum(m["revenue_ai"] for m in month_rows)
        q_adj = sum(m["revenue_adjusted"] if m["revenue_adjusted"] is not None else m["revenue_ai"] for m in month_rows)
        overall_comment_for_customer = None
        for month_label in month_periods:
            drv = driver_saved.get((cname, month_label))
            if drv and drv.get("overall_comment"):
                overall_comment_for_customer = drv["overall_comment"]
                break
        drivers_output.append({
            "customer_name": cname,
            "months": sorted(month_rows, key=lambda x: x["month"]),
            "quarter_revenue_ai": round(q_ai, 2),
            "quarter_revenue_adjusted": round(q_adj, 2),
            "overall_comment": overall_comment_for_customer,
        })

    baseline_ai = round(period_total_ai, 2)
    baseline_user = round(period_total_adjusted, 2)

    compare_revenue_forecast_scenarios = {
        "baseline": _build_scenario_summary(baseline_ai, baseline_user, 1.0),
        "best_case": _build_scenario_summary(baseline_ai, baseline_user, BEST_CASE_MULTIPLIER),
        "worst_case": _build_scenario_summary(baseline_ai, baseline_user, WORST_CASE_MULTIPLIER),
    }

    specific_scenario_keys, specific_scenario_ids, plain_scenario_items = [], [], []
    for cs in custom_scenarios:
        scenario_key = cs["scenario_name"].lower().replace(" ", "_")
        if cs["customer_scope"] == "specific":
            specific_scenario_ids.append(cs["id"])
            specific_scenario_keys.append((scenario_key, cs))
        else:
            plain_scenario_items.append((scenario_key, cs))

    for scenario_key, cs in plain_scenario_items:
        compare_revenue_forecast_scenarios[scenario_key] = _build_scenario_summary(baseline_ai, baseline_user, cs["multiplier"])

    if specific_scenario_ids:
        # Single batched query for all specific-scope scenarios (was N+1).
        cust_mults_by_scenario = await _get_scenario_customer_multipliers_batch(
            db, specific_scenario_ids, company_id, effective_year, effective_quarter,
        )
        for scenario_key, cs in specific_scenario_keys:
            cust_mults = cust_mults_by_scenario.get(cs["id"], {})
            compare_revenue_forecast_scenarios[scenario_key] = _build_scenario_summary_with_customer_multipliers(
                baseline_ai=baseline_ai, baseline_user=baseline_user, scenario_multiplier=cs["multiplier"],
                customer_ai_map=customer_ai_map, customer_user_map=customer_user_map,
                customer_multipliers=cust_mults, customer_scope=cs["customer_scope"],
            )

    if effective_view_type == "quarter_summary":
        rows_output = [
            {
                "customer_name": row["customer_name"],
                "previous_month": row["previous_month"],
                "quarter_total_ai": row["quarter_total_ai"],
                "quarter_total_adjusted": row["quarter_total_adjusted"],
                "overall_comment": row["overall_comment"],
            }
            for row in rows
        ]
    else:
        rows_output = rows

    # Pagination — slice rows + drivers *after* all aggregates are computed,
    # so KPI totals reflect the full dataset, not just one page.
    total_rows = len(rows_output)
    if page_size and page_size > 0:
        start_idx = (page - 1) * page_size
        end_idx = start_idx + page_size
        rows_output = rows_output[start_idx:end_idx]
        page_customers = {r["customer_name"] for r in rows_output}
        drivers_output = [d for d in drivers_output if d["customer_name"] in page_customers]
    pagination_meta = {
        "page": page,
        "page_size": page_size,
        "total_count": total_rows,
        "total_pages": (total_rows + page_size - 1) // page_size if page_size else 1,
    }

    total_previous_month = round(sum(row["previous_month"] for row in rows), 2)
    variance_amount = round(baseline_user - baseline_ai, 2)
    variance_percent = round((variance_amount / baseline_ai) * 100 if baseline_ai else 0, 2)

    legacy_total = {
        "previous_month": total_previous_month,
        "month1_ai": month_totals[0]["total_ai"] if len(month_totals) > 0 else 0.0,
        "month1_adjusted": month_totals[0]["total_adjusted"] if len(month_totals) > 0 else 0.0,
        "month2_ai": month_totals[1]["total_ai"] if len(month_totals) > 1 else 0.0,
        "month2_adjusted": month_totals[1]["total_adjusted"] if len(month_totals) > 1 else 0.0,
        "month3_ai": month_totals[2]["total_ai"] if len(month_totals) > 2 else 0.0,
        "month3_adjusted": month_totals[2]["total_adjusted"] if len(month_totals) > 2 else 0.0,
    }

    response_payload = {
        "year": year,
        "forecast_year": effective_year,
        "quarter": quarter.value,
        "prev_month": prev_month,
        "forecast_type": effective_forecast_type,
        "view_type": effective_view_type,
        "selected_period": selected_period,
        "selected_months": month_periods,
        "customer_name_filter": customer_name or None,
        "rows": rows_output,
        "total": legacy_total,
        "month_totals": month_totals,
        "Baseline_forecast_summary": {
            "total_ai": baseline_ai,
            "total_user": baseline_user,
            "variance_amount": variance_amount,
            "variance_percent": variance_percent,
        },
        "Compare_Revenue_Forecast_Scenarios": compare_revenue_forecast_scenarios,
        "drivers": drivers_output,
        "workflow_stage": stage,
        "reviewed_by": reviewed_by,
        "approved_by": approved_by,
        "locked_by": locked_by,
        "pagination": pagination_meta,
    }

    money_fields = [
        "previous_month", "month1_ai", "month1_adjusted", "month2_ai", "month2_adjusted",
        "month3_ai", "month3_adjusted", "quarter_total_ai", "quarter_total_adjusted",
        "revenue_ai", "revenue_adjusted", "quarter_revenue_ai", "quarter_revenue_adjusted",
        "total_ai", "total_user", "variance_amount", "period_total_ai", "period_total_adjusted",
        "total_adjusted", "total_ai",
    ]

    final_payload = apply_smart_currency(response_payload, currency, currency_filter, money_fields)
    # Persist the post-currency-conversion payload so the cache hit can return
    # it verbatim. TTL 30 minutes — short enough that adjustments propagate
    # quickly, long enough to cover bursty refreshes. Cache writes must NEVER
    # fail the request, so swallow exceptions.
    try:
        await cache_set(db, rf_cache_key, final_payload, ttl_seconds=1800)
    except Exception:    # noqa: BLE001
        import logging
        logging.getLogger(__name__).exception(
            "revenue_forecast: cache_set failed key=%s", rf_cache_key,
        )

    return final_payload

@forecast_router.post("/revenue-forecast/{year}/precompute", status_code=202)
async def trigger_revenue_forecast_precompute(
    year: int = Path(...),
    background_tasks: BackgroundTasks = None,
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    """Kick off a Prophet/ML precompute for the given (company, year).

    Returns 202 immediately; the actual work runs in the background. Also
    invalidates the response cache for that year so the next GET picks up
    the fresh numbers as soon as precompute finishes.
    """
    from .cache import cache_invalidate
    from .precompute import precompute_revenue_forecasts
    from src.core.database import AsyncSessionLocal

    company_id = get_company_id(current_user)

    async def _run() -> None:
        try:
            async with AsyncSessionLocal() as session:
                await precompute_revenue_forecasts(session, company_id, year)
                # Invalidate every cached payload tied to this company.
                await cache_invalidate(session, f"rf:v1:{company_id}:")
        except Exception:    # noqa: BLE001
            import logging
            logging.getLogger(__name__).exception(
                "manual precompute failed company=%s year=%s", company_id, year,
            )

    if background_tasks is not None:
        background_tasks.add_task(_run)
    else:
        # Fallback (shouldn't happen in FastAPI) — fire-and-forget.
        asyncio.create_task(_run())
    return {"status": "accepted", "company_id": company_id, "year": year}


#####################################
###################################################################
#################################################################
#Expense Forecasting

from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field, validator
from typing import Annotated

INFLATION_RATE = 0.035
VENDOR_ESCALATION_RATE = 0.025
BENEFITS_PERCENT_DEFAULT = 0.12
PENSION_PERCENT_DEFAULT = 0.05

FORECAST_VARIANCE_THRESHOLD = 0.25

EPENSE_SCENARIO_DEFINITIONS = {
    "baseline": "AI-generated expense forecast with manual user adjustments applied.",
    "best_case": f"Optimistic scenario applying a {int((BEST_CASE_MULTIPLIER - 1) * 100)}% uplift on baseline — assumes favorable cost conditions, strong vendor negotiations, and operational efficiency.",
    "worst_case": f"Pessimistic scenario applying a {int((1 - WORST_CASE_MULTIPLIER) * 100)}% reduction on baseline — assumes adverse conditions, cost overruns, or demand shortfall.",
}

VALID_PARAMETER_TYPES = {"revenue", "rate", "volume", "quantity"}

COGS_KEYWORDS = [
    "salary", "salaries", "wage", "wages", "payroll", "bonus", "bonuses",
    "compensation", "benefits", "pension", "retirement", "insurance", "health",
    "dental", "vision", "pto", "vacation", "leave", "severance",
    "contractor", "freelance", "consultant fees", "stipend",
    "cost of goods", "cogs", "direct cost", "delivery", "cloud hosting",
    "infrastructure", "server", "aws", "azure", "gcp", "software license",
    "data center", "support", "implementation", "professional services",
]

DELIVERY_RESOURCE_KEYWORDS = [
    "salary", "salaries", "wage", "wages", "payroll", "bonus", "bonuses",
    "compensation", "benefits", "pension", "retirement", "insurance", "health",
    "dental", "vision", "pto", "vacation", "leave", "severance",
    "contractor", "freelance", "consultant fees", "stipend",
    "delivery", "implementation", "professional services", "support",
]

SALES_MARKETING_KEYWORDS = [
    "marketing", "advertising", "campaign", "sales commission", "commissions",
    "incentive", "lead generation", "seo", "sem", "social media", "brand",
    "event", "conference", "trade show", "pr ", "public relation",
    "sales salary", "sales bonus", "sales travel", "sales enablement",
    "crm", "hubspot", "salesforce", "marketing software",
]

FINANCE_KEYWORDS = [
    "audit", "accounting", "finance salary", "finance bonus", "legal",
    "compliance", "tax", "insurance premium", "banking", "treasury",
    "finance software", "erp", "quickbooks", "netsuite", "sap",
    "accounts payable", "accounts receivable", "financial reporting",
]

OPERATIONS_KEYWORDS = [
    "rent", "office", "utilities", "facility", "maintenance", "cleaning",
    "security", "it support", "hardware", "equipment", "supplies",
    "travel", "meals", "entertainment", "telephone", "internet",
    "postage", "shipping", "printing", "subscription", "miscellaneous",
    "recruiting", "hiring", "training", "learning", "development", "hr ",
    "human resource", "employee", "staff", "headcount", "allowance",
    "reimbursement",
]

_prophet_executor = ThreadPoolExecutor(max_workers=min(32, (os.cpu_count() or 4) * 4))

_EXPENSE_FORECAST_CACHE: Dict[str, Tuple[float, Any]] = {}
_EXPENSE_FORECAST_CACHE_TTL = 300


def _expense_cache_key(
    company_id: str, year: int, quarter: str,
    department: Optional[str], expense_type: str, expense_category: str,
    forecast_type: str = "quarter", view_type: str = "monthly_breakdown",
    rolling_start_month: Optional[str] = None,
) -> str:
    """Cache key includes every filter that affects the response. Adding a
    filter without extending this key silently returns stale data."""
    return (
        f"{company_id}:{year}:{quarter}:{department or '-'}:"
        f"{expense_type}:{expense_category}:{forecast_type}:{view_type}:"
        f"{rolling_start_month or '-'}"
    )


def _get_expense_cache(key: str) -> Optional[Any]:
    entry = _EXPENSE_FORECAST_CACHE.get(key)
    if entry and (time.time() - entry[0]) < _EXPENSE_FORECAST_CACHE_TTL:
        return entry[1]
    if entry:
        del _EXPENSE_FORECAST_CACHE[key]
    return None


def _set_expense_cache(key: str, value: Any) -> None:
    _EXPENSE_FORECAST_CACHE[key] = (time.time(), value)


def _invalidate_expense_cache(company_id: str, year: int, quarter: str) -> None:
    keys_to_delete = [k for k in _EXPENSE_FORECAST_CACHE if k.startswith(f"{company_id}:{year}:{quarter}:")]
    for k in keys_to_delete:
        del _EXPENSE_FORECAST_CACHE[k]


def classify_expense_category(expense_item: str) -> str:
    item_lower = expense_item.lower()
    for keyword in COGS_KEYWORDS:
        if keyword in item_lower:
            return "COGS"
    return "Non-COGS"


def classify_expense_subcategory(expense_item: str) -> str:
    item_lower = expense_item.lower()
    for keyword in DELIVERY_RESOURCE_KEYWORDS:
        if keyword in item_lower:
            return "Delivery Resource"
    for keyword in SALES_MARKETING_KEYWORDS:
        if keyword in item_lower:
            return "Sales & Marketing"
    for keyword in FINANCE_KEYWORDS:
        if keyword in item_lower:
            return "Finance"
    for keyword in OPERATIONS_KEYWORDS:
        if keyword in item_lower:
            return "Operations"
    return "Operations"


def classify_expense_type_for_people(expense_item: str) -> str:
    item_lower = expense_item.lower()
    if any(k in item_lower for k in ["salary", "salaries", "wage", "wages", "payroll", "bonus", "bonuses", "commissions", "incentive"]):
        return "Variable"
    if any(k in item_lower for k in ["benefits", "pension", "retirement", "insurance", "health", "dental", "vision"]):
        return "Fixed Percentage"
    return "Variable"


class ExpenseTypeEnum(str, Enum):
    ALL = "ALL"
    FIXED = "Fixed"
    VARIABLE = "Variable"


class ExpenseCategoryEnum(str, Enum):
    ALL = "ALL"
    COGS = "COGS"
    NON_COGS = "Non-COGS"


class WorkflowStage(str, Enum):
    DRAFT = "draft"
    SUBMITTED = "submitted"
    REVIEWED = "reviewed"
    APPROVED = "approved"
    LOCKED = "locked"


class WorkflowAction(str, Enum):
    SUBMIT_FOR_REVIEW = "submit_for_review"
    MARK_REVIEWED = "mark_reviewed"
    APPROVE = "approve"
    LOCK = "lock"
    REJECT_TO_DRAFT = "reject_to_draft"


WORKFLOW_ACTION_TO_STAGE: dict = {
    WorkflowAction.SUBMIT_FOR_REVIEW: WorkflowStage.SUBMITTED,
    WorkflowAction.MARK_REVIEWED: WorkflowStage.REVIEWED,
    WorkflowAction.APPROVE: WorkflowStage.APPROVED,
    WorkflowAction.LOCK: WorkflowStage.LOCKED,
    WorkflowAction.REJECT_TO_DRAFT: WorkflowStage.DRAFT,
}


class ExpenseAdjustmentInput(BaseModel):
    expense_item: str
    department: Optional[str] = None
    expense_type: str
    month1_adjusted: Optional[float] = None
    month1_comment: Optional[str] = None
    month2_adjusted: Optional[float] = None
    month2_comment: Optional[str] = None
    month3_adjusted: Optional[float] = None
    month3_comment: Optional[str] = None
    overall_comment: Optional[str] = None


class ExpenseDriverAdjustmentInput(BaseModel):
    expense_item: str
    department: Optional[str] = None
    expense_type: str
    month: str
    units_adjusted: Optional[float] = None
    rate_adjusted: Optional[float] = None
    month_comment: Optional[str] = None
    overall_comment: Optional[str] = None


class WorkflowTransitionRequest(BaseModel):
    action: Annotated[WorkflowAction, Field(default=WorkflowAction.SUBMIT_FOR_REVIEW, description="Action to perform on the expense forecast workflow")]
    comment: Optional[str] = None


class ExpenseForecastCommentInput(BaseModel):
    comment: str


class DepartmentMultiplierInput(BaseModel):
    department_name: str
    multiplier: float

    @validator("multiplier")
    def multiplier_range(cls, v):
        if not (0.01 <= v <= 10.0):
            raise ValueError("multiplier must be between 0.01 and 10.0")
        return v


class ExpenseScenarioInput(BaseModel):
    scenario_name: str
    multiplier: float
    description: Optional[str] = None
    parameter_types: Optional[List[str]] = Field(default=["revenue"])
    department_scope: Optional[str] = Field(default="all")
    department_multipliers: Optional[List[DepartmentMultiplierInput]] = Field(default=[])

    @validator("multiplier")
    def multiplier_range(cls, v):
        if not (0.01 <= v <= 10.0):
            raise ValueError("multiplier must be between 0.01 and 10.0")
        return v

    @validator("parameter_types", each_item=True, pre=True)
    def valid_parameter_types(cls, v):
        allowed = {"revenue", "rate", "volume", "quantity"}
        if v not in allowed:
            raise ValueError(f"'{v}' is not a valid parameter type. Allowed: {sorted(allowed)}")
        return v

    @validator("department_scope")
    def valid_department_scope(cls, v):
        if v not in ("all", "specific"):
            raise ValueError("department_scope must be 'all' or 'specific'")
        return v

    @validator("department_multipliers", pre=True, always=True)
    def default_department_multipliers(cls, v):
        return v or []


class ExpenseScenarioUpdateInput(BaseModel):
    scenario_name: Optional[str] = None
    multiplier: Optional[float] = None
    description: Optional[str] = None
    parameter_types: Optional[List[str]] = None
    department_scope: Optional[str] = None
    department_multipliers: Optional[List[DepartmentMultiplierInput]] = None

    @validator("multiplier")
    def multiplier_range(cls, v):
        if v is not None and not (0.01 <= v <= 10.0):
            raise ValueError("multiplier must be between 0.01 and 10.0")
        return v

    @validator("parameter_types", each_item=True, pre=True)
    def valid_parameter_types(cls, v):
        if v is not None:
            allowed = {"revenue", "rate", "volume", "quantity"}
            if v not in allowed:
                raise ValueError(f"'{v}' is not a valid parameter type. Allowed: {sorted(allowed)}")
        return v

    @validator("department_scope")
    def valid_department_scope(cls, v):
        if v is not None and v not in ("all", "specific"):
            raise ValueError("department_scope must be 'all' or 'specific'")
        return v


class EmployeeAdjustmentInput(BaseModel):
    employee_id: str
    expense_item: str
    department: Optional[str] = None
    month1_adjusted: Optional[float] = None
    month1_comment: Optional[str] = None
    month2_adjusted: Optional[float] = None
    month2_comment: Optional[str] = None
    month3_adjusted: Optional[float] = None
    month3_comment: Optional[str] = None
    overall_comment: Optional[str] = None
    adjustment_type: Optional[str] = None


class EmployeeSalaryDetail(BaseModel):
    expense_type: str
    month1_ai: float
    month2_ai: float
    month3_ai: float
    month1_adjusted: float
    month2_adjusted: float
    month3_adjusted: float
    month1_comment: Optional[str] = None
    month2_comment: Optional[str] = None
    month3_comment: Optional[str] = None
    quarter_total_ai: float
    quarter_total_adjusted: float
    overall_comment: Optional[str] = None


class EmployeeBonusDetail(BaseModel):
    expense_type: str
    month1_ai: float
    month2_ai: float
    month3_ai: float
    month1_adjusted: float
    month2_adjusted: float
    month3_adjusted: float
    month1_comment: Optional[str] = None
    month2_comment: Optional[str] = None
    month3_comment: Optional[str] = None
    quarter_total_ai: float
    quarter_total_adjusted: float
    overall_comment: Optional[str] = None


class EmployeeBenefitsDetail(BaseModel):
    expense_type: str
    percent_of_salary: float
    month1_ai: float
    month2_ai: float
    month3_ai: float
    month1_adjusted: float
    month2_adjusted: float
    month3_adjusted: float
    month1_comment: Optional[str] = None
    month2_comment: Optional[str] = None
    month3_comment: Optional[str] = None
    quarter_total_ai: float
    quarter_total_adjusted: float
    overall_comment: Optional[str] = None


class EmployeePensionDetail(BaseModel):
    expense_type: str
    percent_of_salary: float
    month1_ai: float
    month2_ai: float
    month3_ai: float
    month1_adjusted: float
    month2_adjusted: float
    month3_adjusted: float
    month1_comment: Optional[str] = None
    month2_comment: Optional[str] = None
    month3_comment: Optional[str] = None
    quarter_total_ai: float
    quarter_total_adjusted: float
    overall_comment: Optional[str] = None


class EmployeeExpenseRow(BaseModel):
    employee_id: str
    employee_name: str
    job_title: Optional[str] = None
    department: Optional[str] = None
    hire_date: Optional[str] = None
    termination_date: Optional[str] = None
    is_prorated: bool = False
    adjustment_type: Optional[str] = None
    salary: EmployeeSalaryDetail
    bonus: EmployeeBonusDetail
    benefits: EmployeeBenefitsDetail
    pension: EmployeePensionDetail
    employee_total_ai: float
    employee_total_adjusted: float


class HeadcountSummary(BaseModel):
    active_employees: int
    new_hires_this_quarter: int
    terminations_this_quarter: int
    net_headcount_change: int


class PeopleExpenseDepartmentSummary(BaseModel):
    department: Optional[str] = None
    expense_category: str
    benefits_percent_used: float
    pension_percent_used: float
    headcount_summary: HeadcountSummary
    department_total_ai: float
    department_total_adjusted: float
    employees: List[EmployeeExpenseRow]


class ExpenseRow(BaseModel):
    department: Optional[str] = None
    expense_type: str
    expense_category: str
    previous_month: float
    month1_ai: float
    month1_adjusted: float
    month1_comment: Optional[str] = None
    month2_ai: float
    month2_adjusted: float
    month2_comment: Optional[str] = None
    month3_ai: float
    month3_adjusted: float
    month3_comment: Optional[str] = None
    quarter_total_ai: float
    quarter_total_adjusted: float
    overall_comment: Optional[str] = None


class ExpenseDriverRow(BaseModel):
    expense_item: str
    department: Optional[str] = None
    expense_type: str
    expense_category: str
    month: str
    units_ai: float
    rate_ai: float
    cost_ai: float
    units_adjusted: Optional[float] = None
    rate_adjusted: Optional[float] = None
    cost_adjusted: Optional[float] = None
    month_comment: Optional[str] = None


class ExpenseCustomerDriverSummary(BaseModel):
    expense_item: str
    department: Optional[str] = None
    expense_type: str
    expense_category: str
    months: List[ExpenseDriverRow]
    quarter_cost_ai: float
    quarter_cost_adjusted: float
    overall_comment: Optional[str] = None


class ExpenseGrandTotal(BaseModel):
    previous_month: float = 0.0
    month1_ai: float = 0.0
    month1_adjusted: float = 0.0
    month2_ai: float = 0.0
    month2_adjusted: float = 0.0
    month3_ai: float = 0.0
    month3_adjusted: float = 0.0


class ExpenseForecastResponse(BaseModel):
    year: int
    quarter: QuarterEnum
    filters: Dict[str, Any]
    baseline_forecast_summary: Dict[str, Any]
    Compare_Expense_Forecast_Scenarios: Dict[str, Any]
    rows: Dict[str, List[ExpenseRow]]
    grand_total: ExpenseGrandTotal
    drivers: List[ExpenseCustomerDriverSummary]
    departments: List[str]
    workflow_stage: Optional[str] = None
    reviewed_by: Optional[str] = None
    approved_by: Optional[str] = None
    locked_by: Optional[str] = None
    people_expense_breakdown: Optional[List[PeopleExpenseDepartmentSummary]] = None


def get_quarter_periods(year: int, q: str):
    return {
        "Q1": [f"{year}-01", f"{year}-02", f"{year}-03"],
        "Q2": [f"{year}-04", f"{year}-05", f"{year}-06"],
        "Q3": [f"{year}-07", f"{year}-08", f"{year}-09"],
        "Q4": [f"{year}-10", f"{year}-11", f"{year}-12"],
    }[q]


def get_previous_month(year: int, q: str):
    return f"{year-1}-12" if q == "Q1" else {
        "Q2": f"{year}-03",
        "Q3": f"{year}-06",
        "Q4": f"{year}-09",
    }[q]


def derive_expense_units_and_rate(ai_cost: float, avg_unit_cost: float = 1000.0) -> tuple:
    if avg_unit_cost <= 0:
        avg_unit_cost = 1.0
    rate_ai = avg_unit_cost
    units_ai = ai_cost / rate_ai if rate_ai else 0.0
    return round(units_ai, 4), round(rate_ai, 2)


def _build_expense_scenario_summary(baseline_ai: float, baseline_user: float, multiplier: float) -> Dict[str, Any]:
    total_ai = round(baseline_ai * multiplier, 2)
    total_user = round(baseline_user * multiplier, 2)
    variance_amount = round(total_user - total_ai, 2)
    variance_percent = round((variance_amount / total_ai) * 100 if total_ai else 0, 2)
    return {"total_ai": total_ai, "total_user": total_user, "variance_amount": variance_amount, "variance_percent": variance_percent}


def _build_expense_scenario_summary_with_department_multipliers(
    baseline_ai: float, baseline_user: float, scenario_multiplier: float,
    department_ai_map: Dict[str, float], department_user_map: Dict[str, float],
    department_multipliers: Dict[str, float], department_scope: str,
) -> Dict[str, Any]:
    if department_scope == "all" or not department_multipliers:
        return _build_expense_scenario_summary(baseline_ai, baseline_user, scenario_multiplier)
    total_ai = 0.0
    total_user = 0.0
    all_departments = set(list(department_ai_map.keys()) + list(department_user_map.keys()))
    for dname in all_departments:
        eff_mult = department_multipliers.get(dname, scenario_multiplier)
        total_ai += department_ai_map.get(dname, 0.0) * eff_mult
        total_user += department_user_map.get(dname, 0.0) * eff_mult
    total_ai = round(total_ai, 2)
    total_user = round(total_user, 2)
    variance_amount = round(total_user - total_ai, 2)
    variance_percent = round((variance_amount / total_ai) * 100 if total_ai else 0, 2)
    return {"total_ai": total_ai, "total_user": total_user, "variance_amount": variance_amount, "variance_percent": variance_percent}


def _forecast_expense_item_sync(item_rows: pd.DataFrame, quarter_start_str: str) -> Tuple[float, float, float]:
    if item_rows.empty:
        return 0.0, 0.0, 0.0
    cdf = item_rows.copy()
    cdf["ds"] = pd.to_datetime(cdf["period"] + "-01")
    cdf = cdf.sort_values("ds")
    values = cdf["y"].values.astype(float)
    if len(values) == 0:
        return 0.0, 0.0, 0.0
    if len(values) == 1:
        val = round(float(values[-1]), 2)
        return val, val, val
    if len(values) == 2:
        last = float(values[-1])
        trend = float(values[-1] - values[-2])
        return round(max(0.0, last + trend), 2), round(max(0.0, last + 2 * trend), 2), round(max(0.0, last + 3 * trend), 2)
    clean_values = _winsorize_values(values)
    recent_3 = clean_values[-3:]
    if np.count_nonzero(recent_3) == 0:
        return 0.0, 0.0, 0.0
    months_inactive = _months_since_last_activity(clean_values)
    if months_inactive >= 3:
        reduced = round(float(np.mean(clean_values[-6:]) * 0.10), 2)
        reduced = max(0.0, reduced)
        return reduced, reduced, reduced
    features = _feature_engineering(clean_values)
    volatility = features["volatility"]
    trend_strength = features["trend_strength"]
    seasonality_strength = features["seasonality_strength"]
    customer_state = _classify_customer(clean_values, volatility, trend_strength, seasonality_strength)
    baseline_result = _baseline_forecast(clean_values)
    start_date = str(cdf["ds"].iloc[0].date())
    prophet_result = _prophet_forecast_sync(clean_values, start_date, 3)
    ets_result = _ets_forecast_sync(clean_values, 3)
    sarima_result = _sarima_forecast_sync(clean_values, 3)
    xgb_result = _xgboost_forecast_sync(clean_values, 3)
    final_preds = _ensemble_forecasts_expense(baseline_result, prophet_result, ets_result, sarima_result, xgb_result, features, 3)
    final_preds = _validate_forecast(final_preds, clean_values)
    final_preds = _apply_forecast_limits(final_preds, float(clean_values[-1]), customer_state, volatility)
    final_preds = _stabilize_forecasts(final_preds, float(clean_values[-1]), volatility)
    return float(final_preds[0]), float(final_preds[1]), float(final_preds[2])


async def get_expense_forecast(item_rows: pd.DataFrame, quarter_start_str: str) -> Tuple[float, float, float]:
    values = item_rows["y"].values.astype(float)
    if len(values) == 0:
        return 0.0, 0.0, 0.0
    loop = asyncio.get_event_loop()
    return await loop.run_in_executor(_prophet_executor, _forecast_expense_item_sync, item_rows, quarter_start_str)


def _forecast_expense_item_n_months_sync(item_rows: pd.DataFrame, forecast_start_str: str, n_ahead: int = 12) -> List[float]:
    """N-month expense item forecast — mirrors _forecast_customer_n_months_sync pattern.
    Uses existing ML components (_prophet_forecast_sync, _ets_forecast_sync, _xgboost_forecast_sync)
    with n_ahead parameter. Does NOT modify existing 3-month pipeline."""
    if item_rows.empty:
        return [0.0] * n_ahead
    cdf = item_rows.copy()
    if "ds" not in cdf.columns:
        cdf["ds"] = pd.to_datetime(cdf["period"] + "-01")
    cdf = cdf.sort_values("ds")
    values = cdf["y"].values.astype(float)
    if len(values) == 0:
        return [0.0] * n_ahead
    if len(values) == 1:
        return [round(float(values[-1]), 2)] * n_ahead
    if len(values) == 2:
        last = float(values[-1])
        trend = float(values[-1] - values[-2])
        return [round(max(0.0, last + trend * (i + 1)), 2) for i in range(n_ahead)]
    clean_values = _winsorize_values(values)
    recent_3 = clean_values[-3:]
    if np.count_nonzero(recent_3) == 0:
        return [0.0] * n_ahead
    months_inactive = _months_since_last_activity(clean_values)
    if months_inactive >= 3:
        reduced = round(float(np.mean(clean_values[-6:]) * 0.10), 2)
        return [max(0.0, reduced)] * n_ahead
    features = _feature_engineering(clean_values)
    start_date = str(cdf["ds"].iloc[0].date())
    prophet_result = _prophet_forecast_sync(clean_values, start_date, n_ahead)
    ets_result = _ets_forecast_sync(clean_values, n_ahead)
    sarima_result = _sarima_forecast_sync(clean_values, n_ahead)
    xgb_result = _xgboost_forecast_sync(clean_values, n_ahead)
    baseline_3 = _baseline_forecast(clean_values)
    result = _ensemble_forecasts_expense(baseline_3, prophet_result, ets_result, sarima_result, xgb_result, features, n_ahead)
    customer_state = _classify_customer(clean_values, features["volatility"], features["trend_strength"], features["seasonality_strength"])
    result = _validate_forecast(result, clean_values)
    result = _apply_forecast_limits(result, float(clean_values[-1]), customer_state, features["volatility"])
    result = _stabilize_forecasts(result, float(clean_values[-1]), features["volatility"])
    return result


async def _forecast_all_expense_items_n_months(df: pd.DataFrame, m1: str, n_months: int, expense_category_filter: str) -> Dict[tuple, List[float]]:
    """N-month version of _forecast_all_expense_items. Generates n_months of AI forecasts
    for all expense items using the existing ML pipeline components."""
    loop = asyncio.get_event_loop()
    grouped: Dict[tuple, pd.DataFrame] = {}
    for (item, dept_name), g in df.groupby(["expense_item", "department"]):
        expense_category_val = classify_expense_category(item)
        if expense_category_filter != "ALL" and expense_category_val != expense_category_filter:
            continue
        g = g.sort_values("ds")
        historical_g = g[g["period"] < m1].copy()
        grouped[(item, dept_name)] = historical_g if not historical_g.empty else g.copy()

    async def forecast_one_n(key: tuple) -> List[float]:
        return await loop.run_in_executor(
            _prophet_executor, _forecast_expense_item_n_months_sync, grouped[key], m1, n_months
        )

    BATCH_SIZE = 20
    all_keys = list(grouped.keys())
    results: Dict[tuple, List[float]] = {}
    for i in range(0, len(all_keys), BATCH_SIZE):
        batch = all_keys[i:i + BATCH_SIZE]
        batch_results = await asyncio.gather(*[forecast_one_n(k) for k in batch])
        for key, result in zip(batch, batch_results):
            results[key] = result
    return results


async def _forecast_all_expense_items(df: pd.DataFrame, m1: str, expense_category_filter: str) -> Dict[tuple, Tuple[float, float, float]]:
    loop = asyncio.get_event_loop()

    grouped = {}
    for (item, dept_name), g in df.groupby(["expense_item", "department"]):
        expense_category_val = classify_expense_category(item)
        if expense_category_filter != "ALL" and expense_category_val != expense_category_filter:
            continue
        g = g.sort_values("ds")
        historical_g = g[g["period"] < m1].copy()
        grouped[(item, dept_name)] = historical_g if not historical_g.empty else g.copy()

    async def forecast_one(key: tuple) -> tuple:
        return await loop.run_in_executor(_prophet_executor, _forecast_expense_item_sync, grouped[key], m1)

    BATCH_SIZE = 20
    all_keys = list(grouped.keys())
    results: Dict[tuple, tuple] = {}
    for i in range(0, len(all_keys), BATCH_SIZE):
        batch = all_keys[i:i + BATCH_SIZE]
        batch_results = await asyncio.gather(*[forecast_one(k) for k in batch])
        for key, result in zip(batch, batch_results):
            results[key] = result

    return results


def _compute_prorate_factor(month_label: str, hire_date: Optional[str], termination_date: Optional[str]) -> float:
    try:
        month_start = pd.to_datetime(month_label + "-01")
        month_end = month_start + pd.offsets.MonthEnd(0)
        total_days = (month_end - month_start).days + 1
        effective_start = month_start
        effective_end = month_end
        if hire_date:
            hd = pd.to_datetime(hire_date)
            if hd > month_end:
                return 0.0
            if hd > month_start:
                effective_start = hd
        if termination_date:
            td = pd.to_datetime(termination_date)
            if td < month_start:
                return 0.0
            if td < month_end:
                effective_end = td
        active_days = (effective_end - effective_start).days + 1
        return max(0.0, min(1.0, active_days / total_days))
    except Exception:
        return 1.0


async def _get_employee_roster(db: AsyncSession, company_id: str, department: Optional[str] = None) -> List[Dict[str, Any]]:
    sql = text("""
        SELECT
            e.employee_id,
            e.first_name || ' ' || e.last_name AS employee_name,
            e.job_title,
            d.department_name,
            e.hire_date,
            e.termination_date,
            e.status,
            c.base_salary,
            c.bonus_target,
            c.effective_date AS comp_effective_date
        FROM fpna_landing.hr_employees e
        LEFT JOIN fpna_landing.hr_departments_lookup d ON e.department_id = d.department_id
        LEFT JOIN (
            SELECT DISTINCT ON (employee_id) employee_id, base_salary, bonus_target, effective_date
            FROM fpna_landing.hr_compensation ORDER BY employee_id, effective_date DESC
        ) c ON e.employee_id = c.employee_id
        WHERE (CAST(:department AS TEXT) IS NULL OR d.department_name = CAST(:department AS TEXT))
          AND (e.status = 'active' OR e.termination_date IS NOT NULL)
        ORDER BY d.department_name, e.last_name, e.first_name;
    """)
    rows = (await db.execute(sql, {"department": department})).mappings().all()
    return [dict(r) for r in rows]


async def _get_benefits_pension_percents(db: AsyncSession, company_id: str, m1: str, history_months: int = 6) -> Dict[str, Dict[str, float]]:
    history_start = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=history_months)).strftime("%Y-%m")
    sql = text("""
        SELECT COALESCE(d.department_name, '') AS department_name, coa.account_name,
               SUM(gl.amount_debit - gl.amount_credit) AS total_amount
        FROM fpna_landing.finance_general_ledger_transactions gl
        JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
        LEFT JOIN fpna_landing.hr_departments_lookup d ON gl.department_id = d.department_id
        WHERE coa.account_type = 'Expense' AND gl.period >= :history_start AND gl.period < :m1
        GROUP BY d.department_name, coa.account_name;
    """)
    rows = (await db.execute(sql, {"history_start": history_start, "m1": m1})).mappings().all()
    dept_salary: Dict[str, float] = {}
    dept_benefits: Dict[str, float] = {}
    dept_pension: Dict[str, float] = {}
    for r in rows:
        dept = r["department_name"] or ""
        acc = (r["account_name"] or "").lower()
        amt = float(r["total_amount"] or 0.0)
        if any(k in acc for k in ["salary", "salaries", "wage", "wages", "payroll"]):
            dept_salary[dept] = dept_salary.get(dept, 0.0) + amt
        elif any(k in acc for k in ["benefit", "insurance", "health", "dental", "vision", "pto", "vacation"]):
            dept_benefits[dept] = dept_benefits.get(dept, 0.0) + amt
        elif any(k in acc for k in ["pension", "retirement"]):
            dept_pension[dept] = dept_pension.get(dept, 0.0) + amt
    result: Dict[str, Dict[str, float]] = {}
    all_depts = set(list(dept_salary.keys()) + list(dept_benefits.keys()) + list(dept_pension.keys()))
    for dept in all_depts:
        sal = dept_salary.get(dept, 0.0)
        ben = dept_benefits.get(dept, 0.0)
        pen = dept_pension.get(dept, 0.0)
        result[dept] = {
            "benefits_percent": round(ben / sal, 4) if sal > 0 else BENEFITS_PERCENT_DEFAULT,
            "pension_percent": round(pen / sal, 4) if sal > 0 else PENSION_PERCENT_DEFAULT,
        }
    return result


async def _get_employee_adjustments(db: AsyncSession, year: int, quarter: str, company_id: str, department: Optional[str] = None) -> Dict[tuple, Dict[str, Any]]:
    rows = (await db.execute(
        text("""
            SELECT employee_id, expense_item, department,
                   month1_adjusted, month1_comment, month2_adjusted, month2_comment,
                   month3_adjusted, month3_comment, overall_comment, adjustment_type
            FROM fpna_config.employee_expense_adjustments
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id
              AND (CAST(:department AS TEXT) IS NULL OR department = CAST(:department AS TEXT))
            ORDER BY employee_id, expense_item;
        """),
        {"year": year, "quarter": quarter, "company_id": company_id, "department": department},
    )).mappings().all()
    return {
        (r["employee_id"], r["expense_item"], r["department"] or ""): {
            "month1_adjusted": float(r["month1_adjusted"]) if r["month1_adjusted"] is not None else None,
            "month1_comment": r["month1_comment"],
            "month2_adjusted": float(r["month2_adjusted"]) if r["month2_adjusted"] is not None else None,
            "month2_comment": r["month2_comment"],
            "month3_adjusted": float(r["month3_adjusted"]) if r["month3_adjusted"] is not None else None,
            "month3_comment": r["month3_comment"],
            "overall_comment": r["overall_comment"],
            "adjustment_type": r["adjustment_type"],
        }
        for r in rows
    }


def _build_people_expense_breakdown(employees: List[Dict[str, Any]], periods: List[str], benefits_pension_map: Dict[str, Dict[str, float]], employee_adj_map: Dict[tuple, Dict[str, Any]]) -> List[PeopleExpenseDepartmentSummary]:
    m1, m2, m3 = periods
    dept_groups: Dict[str, List[Dict[str, Any]]] = {}
    for emp in employees:
        dept = emp.get("department_name") or ""
        dept_groups.setdefault(dept, []).append(emp)
    summaries: List[PeopleExpenseDepartmentSummary] = []
    for dept, emp_list in dept_groups.items():
        percents = benefits_pension_map.get(dept, {"benefits_percent": BENEFITS_PERCENT_DEFAULT, "pension_percent": PENSION_PERCENT_DEFAULT})
        benefits_pct = percents.get("benefits_percent", BENEFITS_PERCENT_DEFAULT)
        pension_pct = percents.get("pension_percent", PENSION_PERCENT_DEFAULT)
        dept_expense_category = classify_expense_category(dept + " salaries")
        emp_rows: List[EmployeeExpenseRow] = []
        new_hires = 0
        terminations = 0
        for emp in emp_list:
            hire_date = str(emp["hire_date"]) if emp.get("hire_date") else None
            termination_date = str(emp["termination_date"]) if emp.get("termination_date") else None
            base_salary = float(emp.get("base_salary") or 0.0)
            bonus_target = float(emp.get("bonus_target") or 0.0)
            pf1 = _compute_prorate_factor(m1, hire_date, termination_date)
            pf2 = _compute_prorate_factor(m2, hire_date, termination_date)
            pf3 = _compute_prorate_factor(m3, hire_date, termination_date)
            if pf1 == 0.0 and pf2 == 0.0 and pf3 == 0.0:
                if termination_date:
                    td = pd.to_datetime(termination_date)
                    quarter_start = pd.to_datetime(m1 + "-01")
                    if td < quarter_start:
                        continue
            monthly_salary = base_salary / 12.0
            monthly_bonus = bonus_target / 12.0
            is_prorated = (pf1 != 1.0 or pf2 != 1.0 or pf3 != 1.0)
            sal_ai_m1 = round(monthly_salary * pf1, 2)
            sal_ai_m2 = round(monthly_salary * pf2, 2)
            sal_ai_m3 = round(monthly_salary * pf3, 2)
            bon_ai_m1 = round(monthly_bonus * pf1, 2)
            bon_ai_m2 = round(monthly_bonus * pf2, 2)
            bon_ai_m3 = round(monthly_bonus * pf3, 2)
            ben_ai_m1 = round(sal_ai_m1 * benefits_pct, 2)
            ben_ai_m2 = round(sal_ai_m2 * benefits_pct, 2)
            ben_ai_m3 = round(sal_ai_m3 * benefits_pct, 2)
            pen_ai_m1 = round(sal_ai_m1 * pension_pct, 2)
            pen_ai_m2 = round(sal_ai_m2 * pension_pct, 2)
            pen_ai_m3 = round(sal_ai_m3 * pension_pct, 2)
            sal_adj = employee_adj_map.get((emp["employee_id"], "Salaries", dept), {})
            bon_adj = employee_adj_map.get((emp["employee_id"], "Bonus", dept), {})
            ben_adj = employee_adj_map.get((emp["employee_id"], "Benefits", dept), {})
            pen_adj = employee_adj_map.get((emp["employee_id"], "Pension", dept), {})
            sal_adj_m1 = float(sal_adj["month1_adjusted"]) if sal_adj.get("month1_adjusted") is not None else sal_ai_m1
            sal_adj_m2 = float(sal_adj["month2_adjusted"]) if sal_adj.get("month2_adjusted") is not None else sal_ai_m2
            sal_adj_m3 = float(sal_adj["month3_adjusted"]) if sal_adj.get("month3_adjusted") is not None else sal_ai_m3
            bon_adj_m1 = float(bon_adj["month1_adjusted"]) if bon_adj.get("month1_adjusted") is not None else bon_ai_m1
            bon_adj_m2 = float(bon_adj["month2_adjusted"]) if bon_adj.get("month2_adjusted") is not None else bon_ai_m2
            bon_adj_m3 = float(bon_adj["month3_adjusted"]) if bon_adj.get("month3_adjusted") is not None else bon_ai_m3
            ben_adj_m1 = float(ben_adj["month1_adjusted"]) if ben_adj.get("month1_adjusted") is not None else round(sal_adj_m1 * benefits_pct, 2)
            ben_adj_m2 = float(ben_adj["month2_adjusted"]) if ben_adj.get("month2_adjusted") is not None else round(sal_adj_m2 * benefits_pct, 2)
            ben_adj_m3 = float(ben_adj["month3_adjusted"]) if ben_adj.get("month3_adjusted") is not None else round(sal_adj_m3 * benefits_pct, 2)
            pen_adj_m1 = float(pen_adj["month1_adjusted"]) if pen_adj.get("month1_adjusted") is not None else round(sal_adj_m1 * pension_pct, 2)
            pen_adj_m2 = float(pen_adj["month2_adjusted"]) if pen_adj.get("month2_adjusted") is not None else round(sal_adj_m2 * pension_pct, 2)
            pen_adj_m3 = float(pen_adj["month3_adjusted"]) if pen_adj.get("month3_adjusted") is not None else round(sal_adj_m3 * pension_pct, 2)
            adjustment_type = sal_adj.get("adjustment_type") or bon_adj.get("adjustment_type")
            if hire_date:
                hd = pd.to_datetime(hire_date)
                quarter_start = pd.to_datetime(m1 + "-01")
                quarter_end = pd.to_datetime(m3 + "-01") + pd.offsets.MonthEnd(0)
                if quarter_start <= hd <= quarter_end:
                    new_hires += 1
                    if not adjustment_type:
                        adjustment_type = "new_hire"
            if termination_date:
                td = pd.to_datetime(termination_date)
                quarter_start = pd.to_datetime(m1 + "-01")
                quarter_end = pd.to_datetime(m3 + "-01") + pd.offsets.MonthEnd(0)
                if quarter_start <= td <= quarter_end:
                    terminations += 1
                    if not adjustment_type:
                        adjustment_type = "termination"
            sal_total_ai = round(sal_ai_m1 + sal_ai_m2 + sal_ai_m3, 2)
            sal_total_adj = round(sal_adj_m1 + sal_adj_m2 + sal_adj_m3, 2)
            bon_total_ai = round(bon_ai_m1 + bon_ai_m2 + bon_ai_m3, 2)
            bon_total_adj = round(bon_adj_m1 + bon_adj_m2 + bon_adj_m3, 2)
            ben_total_ai = round(ben_ai_m1 + ben_ai_m2 + ben_ai_m3, 2)
            ben_total_adj = round(ben_adj_m1 + ben_adj_m2 + ben_adj_m3, 2)
            pen_total_ai = round(pen_ai_m1 + pen_ai_m2 + pen_ai_m3, 2)
            pen_total_adj = round(pen_adj_m1 + pen_adj_m2 + pen_adj_m3, 2)
            emp_total_ai = round(sal_total_ai + bon_total_ai + ben_total_ai + pen_total_ai, 2)
            emp_total_adj = round(sal_total_adj + bon_total_adj + ben_total_adj + pen_total_adj, 2)
            emp_rows.append(EmployeeExpenseRow(
                employee_id=emp["employee_id"], employee_name=emp["employee_name"],
                job_title=emp.get("job_title"), department=dept if dept else None,
                hire_date=hire_date, termination_date=termination_date,
                is_prorated=is_prorated, adjustment_type=adjustment_type,
                salary=EmployeeSalaryDetail(expense_type="Variable", month1_ai=sal_ai_m1, month2_ai=sal_ai_m2, month3_ai=sal_ai_m3, month1_adjusted=sal_adj_m1, month2_adjusted=sal_adj_m2, month3_adjusted=sal_adj_m3, month1_comment=sal_adj.get("month1_comment"), month2_comment=sal_adj.get("month2_comment"), month3_comment=sal_adj.get("month3_comment"), quarter_total_ai=sal_total_ai, quarter_total_adjusted=sal_total_adj, overall_comment=sal_adj.get("overall_comment")),
                bonus=EmployeeBonusDetail(expense_type="Variable", month1_ai=bon_ai_m1, month2_ai=bon_ai_m2, month3_ai=bon_ai_m3, month1_adjusted=bon_adj_m1, month2_adjusted=bon_adj_m2, month3_adjusted=bon_adj_m3, month1_comment=bon_adj.get("month1_comment"), month2_comment=bon_adj.get("month2_comment"), month3_comment=bon_adj.get("month3_comment"), quarter_total_ai=bon_total_ai, quarter_total_adjusted=bon_total_adj, overall_comment=bon_adj.get("overall_comment")),
                benefits=EmployeeBenefitsDetail(expense_type="Fixed Percentage", percent_of_salary=benefits_pct, month1_ai=ben_ai_m1, month2_ai=ben_ai_m2, month3_ai=ben_ai_m3, month1_adjusted=ben_adj_m1, month2_adjusted=ben_adj_m2, month3_adjusted=ben_adj_m3, month1_comment=ben_adj.get("month1_comment"), month2_comment=ben_adj.get("month2_comment"), month3_comment=ben_adj.get("month3_comment"), quarter_total_ai=ben_total_ai, quarter_total_adjusted=ben_total_adj, overall_comment=ben_adj.get("overall_comment")),
                pension=EmployeePensionDetail(expense_type="Fixed Percentage", percent_of_salary=pension_pct, month1_ai=pen_ai_m1, month2_ai=pen_ai_m2, month3_ai=pen_ai_m3, month1_adjusted=pen_adj_m1, month2_adjusted=pen_adj_m2, month3_adjusted=pen_adj_m3, month1_comment=pen_adj.get("month1_comment"), month2_comment=pen_adj.get("month2_comment"), month3_comment=pen_adj.get("month3_comment"), quarter_total_ai=pen_total_ai, quarter_total_adjusted=pen_total_adj, overall_comment=pen_adj.get("overall_comment")),
                employee_total_ai=emp_total_ai, employee_total_adjusted=emp_total_adj,
            ))
        if not emp_rows:
            continue
        active_count = len([e for e in emp_list if not e.get("termination_date")])
        dept_total_ai = round(sum(r.employee_total_ai for r in emp_rows), 2)
        dept_total_adj = round(sum(r.employee_total_adjusted for r in emp_rows), 2)
        summaries.append(PeopleExpenseDepartmentSummary(
            department=dept if dept else None,
            expense_category=dept_expense_category,
            benefits_percent_used=benefits_pct,
            pension_percent_used=pension_pct,
            headcount_summary=HeadcountSummary(active_employees=active_count, new_hires_this_quarter=new_hires, terminations_this_quarter=terminations, net_headcount_change=new_hires - terminations),
            department_total_ai=dept_total_ai, department_total_adjusted=dept_total_adj, employees=emp_rows,
        ))
    return summaries


async def _ensure_expense_tables(db: AsyncSession) -> None:
    global _EXPENSE_TABLES_INITIALIZED
    if _EXPENSE_TABLES_INITIALIZED:
        return
    _EXPENSE_TABLES_INITIALIZED = True
    await db.execute(text("CREATE SCHEMA IF NOT EXISTS fpna_config;"))
    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'fpna_config' AND table_name = 'expense_forecast_adjustments') AND NOT EXISTS (SELECT 1 FROM information_schema.table_constraints WHERE table_schema = 'fpna_config' AND table_name = 'expense_forecast_adjustments' AND constraint_type = 'PRIMARY KEY') THEN
                DROP TABLE fpna_config.expense_forecast_adjustments;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.expense_forecast_adjustments (
            year INT NOT NULL, quarter TEXT NOT NULL, company_id TEXT NOT NULL,
            expense_item TEXT NOT NULL, department TEXT NOT NULL DEFAULT '', expense_type TEXT NOT NULL,
            month1_adjusted NUMERIC NULL, month1_comment TEXT NULL,
            month2_adjusted NUMERIC NULL, month2_comment TEXT NULL,
            month3_adjusted NUMERIC NULL, month3_comment TEXT NULL,
            overall_comment TEXT NULL, updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT NOW(),
            PRIMARY KEY (year, quarter, company_id, expense_item, department, expense_type)
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'fpna_config' AND table_name = 'expense_forecast_adjustments' AND column_name = 'company_id') THEN
                ALTER TABLE fpna_config.expense_forecast_adjustments ADD COLUMN company_id TEXT NOT NULL DEFAULT '';
                ALTER TABLE fpna_config.expense_forecast_adjustments DROP CONSTRAINT IF EXISTS expense_forecast_adjustments_pkey;
                ALTER TABLE fpna_config.expense_forecast_adjustments ADD PRIMARY KEY (year, quarter, company_id, expense_item, department, expense_type);
            END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'fpna_config' AND table_name = 'expense_forecast_adjustments' AND column_name = 'overall_comment') THEN ALTER TABLE fpna_config.expense_forecast_adjustments ADD COLUMN overall_comment TEXT NULL; END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'fpna_config' AND table_name = 'expense_forecast_adjustments' AND column_name = 'month1_comment') THEN ALTER TABLE fpna_config.expense_forecast_adjustments ADD COLUMN month1_comment TEXT NULL; END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'fpna_config' AND table_name = 'expense_forecast_adjustments' AND column_name = 'month2_comment') THEN ALTER TABLE fpna_config.expense_forecast_adjustments ADD COLUMN month2_comment TEXT NULL; END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'fpna_config' AND table_name = 'expense_forecast_adjustments' AND column_name = 'month3_comment') THEN ALTER TABLE fpna_config.expense_forecast_adjustments ADD COLUMN month3_comment TEXT NULL; END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.expense_driver_adjustments (
            year INT NOT NULL, quarter TEXT NOT NULL, company_id TEXT NOT NULL,
            expense_item TEXT NOT NULL, department TEXT NOT NULL DEFAULT '', expense_type TEXT NOT NULL, month TEXT NOT NULL,
            units_adjusted NUMERIC NULL, rate_adjusted NUMERIC NULL, month_comment TEXT NULL, overall_comment TEXT NULL,
            updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT NOW(),
            PRIMARY KEY (year, quarter, company_id, expense_item, department, expense_type, month)
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'fpna_config' AND table_name = 'expense_driver_adjustments' AND column_name = 'month_comment') THEN ALTER TABLE fpna_config.expense_driver_adjustments ADD COLUMN month_comment TEXT NULL; END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'fpna_config' AND table_name = 'expense_driver_adjustments' AND column_name = 'overall_comment') THEN ALTER TABLE fpna_config.expense_driver_adjustments ADD COLUMN overall_comment TEXT NULL; END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.expense_forecast_workflows (
            id SERIAL PRIMARY KEY, year INT NOT NULL, quarter TEXT NOT NULL, company_id TEXT NOT NULL,
            department TEXT NOT NULL, owner_user_id TEXT NOT NULL, stage TEXT NOT NULL DEFAULT 'draft',
            created_at TIMESTAMP NOT NULL DEFAULT NOW(), updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            locked_at TIMESTAMP NULL, UNIQUE (year, quarter, company_id, department)
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'fpna_config' AND table_name = 'expense_forecast_workflows' AND column_name = 'owner_user_id' AND data_type != 'text') THEN
                ALTER TABLE fpna_config.expense_forecast_workflows ALTER COLUMN owner_user_id TYPE TEXT USING owner_user_id::TEXT;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        DO $$
        DECLARE v_constraint TEXT;
        BEGIN
            SELECT tc.constraint_name INTO v_constraint FROM information_schema.table_constraints tc
            WHERE tc.table_schema = 'fpna_config' AND tc.table_name = 'expense_forecast_workflows' AND tc.constraint_type = 'UNIQUE'
              AND EXISTS (SELECT 1 FROM information_schema.key_column_usage kcu WHERE kcu.constraint_name = tc.constraint_name AND kcu.column_name = 'department') LIMIT 1;
            IF v_constraint IS NOT NULL THEN EXECUTE 'ALTER TABLE fpna_config.expense_forecast_workflows DROP CONSTRAINT ' || quote_ident(v_constraint); END IF;
            DELETE FROM fpna_config.expense_forecast_workflows fw1 WHERE fw1.id NOT IN (SELECT MIN(fw2.id) FROM fpna_config.expense_forecast_workflows fw2 GROUP BY fw2.year, fw2.quarter, fw2.company_id);
            IF NOT EXISTS (SELECT 1 FROM information_schema.table_constraints tc WHERE tc.table_schema = 'fpna_config' AND tc.table_name = 'expense_forecast_workflows' AND tc.constraint_type = 'UNIQUE' AND NOT EXISTS (SELECT 1 FROM information_schema.key_column_usage kcu WHERE kcu.constraint_name = tc.constraint_name AND kcu.column_name = 'department')) THEN
                ALTER TABLE fpna_config.expense_forecast_workflows ADD CONSTRAINT expense_forecast_workflows_shared_unique UNIQUE (year, quarter, company_id);
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.expense_workflow_audit (
            id SERIAL PRIMARY KEY, workflow_id INT NOT NULL, from_stage TEXT NULL, to_stage TEXT NOT NULL,
            transitioned_by_user_id TEXT NOT NULL, transitioned_by_name TEXT NOT NULL, comment TEXT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'fpna_config' AND table_name = 'expense_workflow_audit' AND column_name = 'transitioned_by_user_id' AND data_type != 'text') THEN
                ALTER TABLE fpna_config.expense_workflow_audit ALTER COLUMN transitioned_by_user_id TYPE TEXT USING transitioned_by_user_id::TEXT;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.expense_workflow_comments (
            id SERIAL PRIMARY KEY, workflow_id INT NOT NULL, user_id TEXT NOT NULL, user_name TEXT NOT NULL,
            department TEXT NOT NULL, comment TEXT NOT NULL, comment_type TEXT NOT NULL DEFAULT 'general',
            created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'fpna_config' AND table_name = 'expense_workflow_comments' AND column_name = 'user_id' AND data_type != 'text') THEN
                ALTER TABLE fpna_config.expense_workflow_comments ALTER COLUMN user_id TYPE TEXT USING user_id::TEXT;
            END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'fpna_config' AND table_name = 'expense_workflow_comments' AND column_name = 'comment_type') THEN
                ALTER TABLE fpna_config.expense_workflow_comments ADD COLUMN comment_type TEXT NOT NULL DEFAULT 'general';
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.expense_forecast_scenarios (
            id SERIAL PRIMARY KEY, company_id TEXT NOT NULL, year INT NOT NULL, quarter TEXT NOT NULL,
            scenario_name TEXT NOT NULL, multiplier NUMERIC NOT NULL, description TEXT NULL,
            created_by_user_id TEXT NOT NULL, created_by_name TEXT NOT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW(), updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            UNIQUE (company_id, year, quarter, scenario_name)
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'fpna_config' AND table_name = 'expense_forecast_scenarios' AND column_name = 'parameter_types') THEN ALTER TABLE fpna_config.expense_forecast_scenarios ADD COLUMN parameter_types JSONB NOT NULL DEFAULT '["revenue"]'; END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'fpna_config' AND table_name = 'expense_forecast_scenarios' AND column_name = 'department_scope') THEN ALTER TABLE fpna_config.expense_forecast_scenarios ADD COLUMN department_scope TEXT NOT NULL DEFAULT 'all'; END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.expense_scenario_department_multipliers (
            id SERIAL PRIMARY KEY, scenario_id INT NOT NULL, company_id TEXT NOT NULL,
            year INT NOT NULL, quarter TEXT NOT NULL, department_name TEXT NOT NULL, multiplier NUMERIC NOT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW(), updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            UNIQUE (scenario_id, company_id, year, quarter, department_name)
        );
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.employee_expense_adjustments (
            id SERIAL PRIMARY KEY, year INT NOT NULL, quarter TEXT NOT NULL, company_id TEXT NOT NULL,
            employee_id TEXT NOT NULL, expense_item TEXT NOT NULL, department TEXT NOT NULL DEFAULT '',
            month1_adjusted NUMERIC NULL, month1_comment TEXT NULL,
            month2_adjusted NUMERIC NULL, month2_comment TEXT NULL,
            month3_adjusted NUMERIC NULL, month3_comment TEXT NULL,
            overall_comment TEXT NULL, adjustment_type TEXT NULL,
            updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT NOW(),
            UNIQUE (year, quarter, company_id, employee_id, expense_item, department)
        );
    """))
    await db.commit()


async def _get_or_create_expense_workflow(db: AsyncSession, year: int, quarter: str, company_id: str, department: str, user_id: str) -> Dict[str, Any]:
    row = (await db.execute(text("SELECT * FROM fpna_config.expense_forecast_workflows WHERE year = :year AND quarter = :quarter AND company_id = :company_id ORDER BY id ASC LIMIT 1"), {"year": year, "quarter": quarter, "company_id": company_id})).mappings().first()
    if not row:
        row = (await db.execute(text("INSERT INTO fpna_config.expense_forecast_workflows (year, quarter, company_id, department, owner_user_id, stage) VALUES (:year, :quarter, :company_id, :dept, :uid, 'draft') RETURNING *"), {"year": year, "quarter": quarter, "company_id": company_id, "dept": department, "uid": user_id})).mappings().first()
        await db.commit()
    return dict(row)


async def _get_expense_adjustments(db: AsyncSession, year: int, quarter: str, company_id: str, department: Optional[str] = None, expense_type: str = "ALL") -> List[Dict[str, Any]]:
    rows = (await db.execute(
        text("""
            SELECT expense_item, department, expense_type,
                   month1_adjusted, month1_comment, month2_adjusted, month2_comment,
                   month3_adjusted, month3_comment, overall_comment
            FROM fpna_config.expense_forecast_adjustments
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id
              AND (CAST(:department AS TEXT) IS NULL OR department = CAST(:department AS TEXT))
              AND (:expense_type = 'ALL' OR expense_type = :expense_type)
            ORDER BY expense_item ASC, department ASC NULLS FIRST, expense_type ASC;
        """),
        {"year": year, "quarter": quarter, "company_id": company_id, "department": department, "expense_type": expense_type},
    )).mappings().all()
    result = []
    for r in rows:
        m1 = float(r["month1_adjusted"]) if r["month1_adjusted"] is not None else None
        m2 = float(r["month2_adjusted"]) if r["month2_adjusted"] is not None else None
        m3 = float(r["month3_adjusted"]) if r["month3_adjusted"] is not None else None
        quarter_total = round((m1 or 0.0) + (m2 or 0.0) + (m3 or 0.0), 2)
        result.append({"expense_item": r["expense_item"], "department": r["department"], "expense_type": r["expense_type"], "month1_adjusted": m1, "month1_comment": r["month1_comment"], "month2_adjusted": m2, "month2_comment": r["month2_comment"], "month3_adjusted": m3, "month3_comment": r["month3_comment"], "quarter_total_adjusted": quarter_total, "overall_comment": r["overall_comment"]})
    return result


async def _get_expense_driver_adjustments(db: AsyncSession, year: int, quarter: str, company_id: str) -> Dict[tuple, Dict]:
    rows = (await db.execute(text("SELECT expense_item, department, expense_type, month, units_adjusted, rate_adjusted, month_comment, overall_comment FROM fpna_config.expense_driver_adjustments WHERE year = :year AND quarter = :quarter AND company_id = :company_id;"), {"year": year, "quarter": quarter, "company_id": company_id})).mappings().all()
    return {(r["expense_item"], r["department"] or "", r["expense_type"], r["month"]): {"units_adjusted": float(r["units_adjusted"]) if r["units_adjusted"] is not None else None, "rate_adjusted": float(r["rate_adjusted"]) if r["rate_adjusted"] is not None else None, "month_comment": r["month_comment"], "overall_comment": r["overall_comment"]} for r in rows}


async def _get_scenario_department_multipliers(db: AsyncSession, scenario_id: int, company_id: str, year: int, quarter: str) -> Dict[str, float]:
    rows = (await db.execute(text("SELECT department_name, multiplier FROM fpna_config.expense_scenario_department_multipliers WHERE scenario_id = :scenario_id AND company_id = :company_id AND year = :year AND quarter = :quarter ORDER BY department_name ASC"), {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter})).mappings().all()
    return {r["department_name"]: float(r["multiplier"]) for r in rows}


async def _get_expense_custom_scenarios(db: AsyncSession, year: int, quarter: str, company_id: str) -> List[Dict[str, Any]]:
    rows = (await db.execute(text("SELECT id, scenario_name, multiplier, description, parameter_types, department_scope, created_by_name, created_at, updated_at FROM fpna_config.expense_forecast_scenarios WHERE company_id = :company_id AND year = :year AND quarter = :quarter ORDER BY created_at ASC"), {"company_id": company_id, "year": year, "quarter": quarter})).mappings().all()
    return [{"id": r["id"], "scenario_name": r["scenario_name"], "multiplier": float(r["multiplier"]), "description": r["description"], "parameter_types": r["parameter_types"] if r["parameter_types"] else ["revenue"], "department_scope": r["department_scope"] if r["department_scope"] else "all", "created_by_name": r["created_by_name"], "created_at": r["created_at"].isoformat(), "updated_at": r["updated_at"].isoformat()} for r in rows]


class ExpenseForecastAdjustmentManagement:
    @staticmethod
    async def GetAdjustments(year: int, quarter: str, company_id: str, department: Optional[str], expense_type: str, db: AsyncSession) -> List[Dict[str, Any]]:
        return await _get_expense_adjustments(db, year, quarter, company_id, department, expense_type)

    @staticmethod
    async def PatchAdjustments(year: int, quarter: str, company_id: str, adjustments: List[ExpenseAdjustmentInput], db: AsyncSession) -> Dict[str, Any]:
        # Bulk upsert — one round-trip instead of one per row. For N=500+
        # expense items × 3 months, this drops save time from ~10s to ~400ms.
        rows = [
            {
                "year": year, "quarter": quarter, "company_id": company_id,
                "expense_item": adj.expense_item,
                "department": adj.department if adj.department is not None else "",
                "expense_type": adj.expense_type,
                "month1_adjusted": adj.month1_adjusted, "month1_comment": adj.month1_comment,
                "month2_adjusted": adj.month2_adjusted, "month2_comment": adj.month2_comment,
                "month3_adjusted": adj.month3_adjusted, "month3_comment": adj.month3_comment,
                "overall_comment": adj.overall_comment,
            }
            for adj in adjustments if adj.expense_item
        ]
        if rows:
            await db.execute(
                text("""
                    INSERT INTO fpna_config.expense_forecast_adjustments (year, quarter, company_id, expense_item, department, expense_type, month1_adjusted, month1_comment, month2_adjusted, month2_comment, month3_adjusted, month3_comment, overall_comment, updated_at)
                    VALUES (:year, :quarter, :company_id, :expense_item, :department, :expense_type, :month1_adjusted, :month1_comment, :month2_adjusted, :month2_comment, :month3_adjusted, :month3_comment, :overall_comment, NOW())
                    ON CONFLICT (year, quarter, company_id, expense_item, department, expense_type)
                    DO UPDATE SET month1_adjusted = EXCLUDED.month1_adjusted, month1_comment = EXCLUDED.month1_comment, month2_adjusted = EXCLUDED.month2_adjusted, month2_comment = EXCLUDED.month2_comment, month3_adjusted = EXCLUDED.month3_adjusted, month3_comment = EXCLUDED.month3_comment, overall_comment = EXCLUDED.overall_comment, updated_at = NOW();
                """),
                rows,
            )
        await db.commit()

        # Invalidate the response cache (in-process + the new L2 if present)
        # so the next GET picks up the fresh adjustments instead of the
        # stale cached payload.
        _invalidate_expense_cache(company_id, year, quarter)
        try:
            from .cache import cache_invalidate
            await cache_invalidate(db, f"ef:v1:{company_id}:")
        except Exception:    # noqa: BLE001
            import logging
            logging.getLogger(__name__).exception("expense cache_invalidate failed")

        # Don't return the full adjustment list — the frontend already has
        # it in state. Just confirm the count to keep the response small.
        return {"detail": "Expense forecast adjustments saved successfully", "year": year, "quarter": quarter, "count": len(rows)}


@forecast_router.patch("/expense-forecast/adjustments/{year}")
async def patch_expense_forecast_adjustments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    adjustments: List[ExpenseAdjustmentInput] = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_expense_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Expense forecast is locked. Adjustments cannot be changed.")
    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Adjustments can only be edited in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")
    _invalidate_expense_cache(company_id, year, quarter.value)
    return await ExpenseForecastAdjustmentManagement.PatchAdjustments(year=year, quarter=quarter.value, company_id=company_id, adjustments=adjustments, db=db)


@forecast_router.patch("/expense-forecast/driver-adjustments/{year}")
async def patch_expense_driver_adjustments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    adjustments: List[ExpenseDriverAdjustmentInput] = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_expense_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Expense forecast is locked. Driver adjustments cannot be changed.")
    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Driver adjustments can only be edited in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")
    _invalidate_expense_cache(company_id, year, quarter.value)
    # Bulk upsert — one round-trip rather than one-per-row.
    driver_rows = [
        {
            "year": year, "quarter": quarter.value, "company_id": company_id,
            "expense_item": adj.expense_item,
            "department": adj.department if adj.department is not None else "",
            "expense_type": adj.expense_type, "month": adj.month,
            "units_adjusted": adj.units_adjusted, "rate_adjusted": adj.rate_adjusted,
            "month_comment": adj.month_comment, "overall_comment": adj.overall_comment,
        }
        for adj in adjustments if adj.expense_item and adj.month
    ]
    if driver_rows:
        await db.execute(text("""
            INSERT INTO fpna_config.expense_driver_adjustments (year, quarter, company_id, expense_item, department, expense_type, month, units_adjusted, rate_adjusted, month_comment, overall_comment, updated_at)
            VALUES (:year, :quarter, :company_id, :expense_item, :department, :expense_type, :month, :units_adjusted, :rate_adjusted, :month_comment, :overall_comment, NOW())
            ON CONFLICT (year, quarter, company_id, expense_item, department, expense_type, month)
            DO UPDATE SET units_adjusted = EXCLUDED.units_adjusted, rate_adjusted = EXCLUDED.rate_adjusted, month_comment = EXCLUDED.month_comment, overall_comment = EXCLUDED.overall_comment, updated_at = NOW();
        """), driver_rows)
    await db.commit()

    # Also invalidate the L2 cache for fresh GETs.
    try:
        from .cache import cache_invalidate
        await cache_invalidate(db, f"ef:v1:{company_id}:")
    except Exception:    # noqa: BLE001
        import logging
        logging.getLogger(__name__).exception("expense driver cache_invalidate failed")
    saved = (await db.execute(text("SELECT expense_item, department, expense_type, month, units_adjusted, rate_adjusted, month_comment, overall_comment FROM fpna_config.expense_driver_adjustments WHERE year = :year AND quarter = :quarter AND company_id = :company_id ORDER BY expense_item, department, expense_type, month;"), {"year": year, "quarter": quarter.value, "company_id": company_id})).mappings().all()
    item_quarter_map: Dict[tuple, float] = {}
    result_rows = []
    for r in saved:
        u = float(r["units_adjusted"]) if r["units_adjusted"] is not None else None
        ra = float(r["rate_adjusted"]) if r["rate_adjusted"] is not None else None
        cost_adj = round(u * ra, 2) if u is not None and ra is not None else None
        key = (r["expense_item"], r["department"] or "", r["expense_type"])
        if cost_adj is not None:
            item_quarter_map[key] = item_quarter_map.get(key, 0.0) + cost_adj
        result_rows.append({"expense_item": r["expense_item"], "department": r["department"], "expense_type": r["expense_type"], "expense_category": classify_expense_category(r["expense_item"]), "month": r["month"], "units_adjusted": u, "rate_adjusted": ra, "cost_adjusted": cost_adj, "month_comment": r["month_comment"], "overall_comment": r["overall_comment"]})
    for row in result_rows:
        key = (row["expense_item"], row["department"] or "", row["expense_type"])
        row["quarter_cost_adjusted"] = round(item_quarter_map.get(key, 0.0), 2)
    return {"detail": "Expense driver adjustments saved successfully", "year": year, "quarter": quarter.value, "count": len(adjustments), "driver_adjustments": result_rows}


@forecast_router.patch("/expense-forecast/employee-adjustments/{year}")
async def patch_employee_expense_adjustments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    adjustments: List[EmployeeAdjustmentInput] = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_expense_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Expense forecast is locked. Employee adjustments cannot be changed.")
    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Employee adjustments can only be edited in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")
    _invalidate_expense_cache(company_id, year, quarter.value)
    for adj in adjustments:
        if not adj.employee_id or not adj.expense_item:
            continue
        dept_val = adj.department if adj.department is not None else ""
        await db.execute(text("""
            INSERT INTO fpna_config.employee_expense_adjustments (year, quarter, company_id, employee_id, expense_item, department, month1_adjusted, month1_comment, month2_adjusted, month2_comment, month3_adjusted, month3_comment, overall_comment, adjustment_type, updated_at)
            VALUES (:year, :quarter, :company_id, :employee_id, :expense_item, :department, :month1_adjusted, :month1_comment, :month2_adjusted, :month2_comment, :month3_adjusted, :month3_comment, :overall_comment, :adjustment_type, NOW())
            ON CONFLICT (year, quarter, company_id, employee_id, expense_item, department)
            DO UPDATE SET month1_adjusted = EXCLUDED.month1_adjusted, month1_comment = EXCLUDED.month1_comment, month2_adjusted = EXCLUDED.month2_adjusted, month2_comment = EXCLUDED.month2_comment, month3_adjusted = EXCLUDED.month3_adjusted, month3_comment = EXCLUDED.month3_comment, overall_comment = EXCLUDED.overall_comment, adjustment_type = EXCLUDED.adjustment_type, updated_at = NOW();
        """), {"year": year, "quarter": quarter.value, "company_id": company_id, "employee_id": adj.employee_id, "expense_item": adj.expense_item, "department": dept_val, "month1_adjusted": adj.month1_adjusted, "month1_comment": adj.month1_comment, "month2_adjusted": adj.month2_adjusted, "month2_comment": adj.month2_comment, "month3_adjusted": adj.month3_adjusted, "month3_comment": adj.month3_comment, "overall_comment": adj.overall_comment, "adjustment_type": adj.adjustment_type})
    await db.commit()
    return {"detail": "Employee expense adjustments saved successfully.", "year": year, "quarter": quarter.value, "count": len(adjustments)}


@forecast_router.patch("/expense-forecast/workflow/{year}")
async def patch_expense_forecast_workflow(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: WorkflowTransitionRequest = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_expense_workflow(db, year, quarter.value, company_id, dept, user_id)
    current_stage = wf["stage"]
    target_stage = WORKFLOW_ACTION_TO_STAGE[body.action].value
    effective_comment = body.comment.strip() if body.comment and body.comment.strip() else None
    is_rejection = body.action == WorkflowAction.REJECT_TO_DRAFT
    if is_rejection and not effective_comment:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="A comment is required when rejecting (sending back to draft). Please provide a reason.")
    if effective_comment and dept not in DEPT_REVIEWERS | DEPT_APPROVERS:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Only Finance/Executive/CEO/CFO can add review comments.")
    if not can_transition(dept, current_stage, target_stage):
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Department '{dept}' cannot move expense forecast from '{current_stage}' to '{target_stage}'.")
    locked_at_clause = ", locked_at = NOW()" if target_stage == WorkflowStage.LOCKED else ""
    await db.execute(text(f"UPDATE fpna_config.expense_forecast_workflows SET stage = :new_stage, updated_at = NOW() {locked_at_clause} WHERE id = :wf_id"), {"new_stage": target_stage, "wf_id": wf["id"]})
    await db.execute(text("INSERT INTO fpna_config.expense_workflow_audit (workflow_id, from_stage, to_stage, transitioned_by_user_id, transitioned_by_name, comment) VALUES (:wf_id, :from_s, :to_s, :uid, :uname, :comment)"), {"wf_id": wf["id"], "from_s": current_stage, "to_s": target_stage, "uid": user_id, "uname": current_user.get("full_name", ""), "comment": effective_comment})
    if effective_comment and dept in DEPT_REVIEWERS | DEPT_APPROVERS:
        comment_type = "rejection" if is_rejection else "transition"
        await db.execute(text("INSERT INTO fpna_config.expense_workflow_comments (workflow_id, user_id, user_name, department, comment, comment_type) VALUES (:wf_id, :uid, :uname, :dept, :comment, :comment_type)"), {"wf_id": wf["id"], "uid": user_id, "uname": current_user.get("full_name", ""), "dept": dept, "comment": effective_comment, "comment_type": comment_type})
    await db.commit()
    return {"detail": f"Expense forecast moved from '{current_stage}' to '{target_stage}'.", "year": year, "quarter": quarter.value, "department": dept, "stage": target_stage}


@forecast_router.post("/expense-forecast/{year}/comments")
async def post_expense_forecast_comment(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: ExpenseForecastCommentInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    user_name = current_user.get("full_name", "")
    wf = await _get_or_create_expense_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Expense forecast is locked. Comments cannot be added.")
    comment_text = body.comment.strip() if body.comment else ""
    if not comment_text:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Comment cannot be empty.")
    result = (await db.execute(text("INSERT INTO fpna_config.expense_workflow_comments (workflow_id, user_id, user_name, department, comment, comment_type) VALUES (:wf_id, :uid, :uname, :dept, :comment, 'general') RETURNING id, user_id, user_name, department, comment, comment_type, created_at"), {"wf_id": wf["id"], "uid": user_id, "uname": user_name, "dept": dept, "comment": comment_text})).mappings().first()
    await db.commit()
    return {"detail": "Comment posted successfully.", "year": year, "quarter": quarter.value, "workflow_stage": stage, "comment": {"id": result["id"], "user_id": result["user_id"], "user_name": result["user_name"], "department": result["department"], "comment": result["comment"], "comment_type": result["comment_type"], "created_at": result["created_at"].isoformat()}}


@forecast_router.get("/expense-forecast/{year}/comments")
async def get_expense_forecast_comments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    comment_type: Optional[str] = Query(None, description="Filter by comment_type: general, transition, rejection. Leave empty for all."),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_expense_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if comment_type:
        rows = (await db.execute(text("SELECT id, user_id, user_name, department, comment, comment_type, created_at FROM fpna_config.expense_workflow_comments WHERE workflow_id = :wf_id AND comment_type = :comment_type ORDER BY created_at ASC"), {"wf_id": wf["id"], "comment_type": comment_type})).mappings().all()
    else:
        rows = (await db.execute(text("SELECT id, user_id, user_name, department, comment, comment_type, created_at FROM fpna_config.expense_workflow_comments WHERE workflow_id = :wf_id ORDER BY created_at ASC"), {"wf_id": wf["id"]})).mappings().all()
    comments = [{"id": r["id"], "user_id": r["user_id"], "user_name": r["user_name"], "department": r["department"], "comment": r["comment"], "comment_type": r["comment_type"], "created_at": r["created_at"].isoformat()} for r in rows]
    return {"year": year, "quarter": quarter.value, "workflow_stage": stage, "total_comments": len(comments), "filters": {"comment_type": comment_type or "all"}, "comments": comments}


@forecast_router.post("/expense-forecast/{year}/scenarios")
async def create_expense_forecast_scenario(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: ExpenseScenarioInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    user_name = current_user.get("full_name", "")
    scenario_name_clean = body.scenario_name.strip().lower().replace(" ", "_")
    if scenario_name_clean in RESERVED_SCENARIO_NAMES:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"'{body.scenario_name}' is a reserved scenario name. Reserved names are: baseline, best_case, worst_case.")
    if not (0.01 <= body.multiplier <= 10.0):
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Multiplier must be between 0.01 and 10.0.")
    parameter_types = body.parameter_types if body.parameter_types else ["revenue"]
    department_scope = body.department_scope if body.department_scope else "all"
    existing = (await db.execute(text("SELECT id FROM fpna_config.expense_forecast_scenarios WHERE company_id = :company_id AND year = :year AND quarter = :quarter AND LOWER(REPLACE(scenario_name, ' ', '_')) = :scenario_name_clean"), {"company_id": company_id, "year": year, "quarter": quarter.value, "scenario_name_clean": scenario_name_clean})).mappings().first()
    if existing:
        raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=f"A scenario named '{body.scenario_name}' already exists for {year} {quarter.value}.")
    result = (await db.execute(text("INSERT INTO fpna_config.expense_forecast_scenarios (company_id, year, quarter, scenario_name, multiplier, description, parameter_types, department_scope, created_by_user_id, created_by_name) VALUES (:company_id, :year, :quarter, :scenario_name, :multiplier, :description, :parameter_types, :department_scope, :uid, :uname) RETURNING id, scenario_name, multiplier, description, parameter_types, department_scope, created_by_name, created_at, updated_at"), {"company_id": company_id, "year": year, "quarter": quarter.value, "scenario_name": body.scenario_name.strip(), "multiplier": body.multiplier, "description": body.description.strip() if body.description else None, "parameter_types": json.dumps(parameter_types), "department_scope": department_scope, "uid": user_id, "uname": user_name})).mappings().first()
    await db.commit()
    department_multipliers_saved = []
    if department_scope == "specific" and body.department_multipliers:
        for dm in body.department_multipliers:
            if not dm.department_name or dm.multiplier is None or not (0.01 <= dm.multiplier <= 10.0):
                continue
            await db.execute(text("INSERT INTO fpna_config.expense_scenario_department_multipliers (scenario_id, company_id, year, quarter, department_name, multiplier) VALUES (:scenario_id, :company_id, :year, :quarter, :department_name, :multiplier) ON CONFLICT (scenario_id, company_id, year, quarter, department_name) DO UPDATE SET multiplier = EXCLUDED.multiplier, updated_at = NOW()"), {"scenario_id": result["id"], "company_id": company_id, "year": year, "quarter": quarter.value, "department_name": dm.department_name, "multiplier": dm.multiplier})
            department_multipliers_saved.append({"department_name": dm.department_name, "multiplier": dm.multiplier})
        await db.commit()
    return {"detail": "Expense scenario created successfully.", "year": year, "quarter": quarter.value, "scenario": {"id": result["id"], "scenario_name": result["scenario_name"], "multiplier": float(result["multiplier"]), "description": result["description"], "parameter_types": result["parameter_types"] if result["parameter_types"] else ["revenue"], "department_scope": result["department_scope"], "department_multipliers": department_multipliers_saved, "created_by_name": result["created_by_name"], "created_at": result["created_at"].isoformat(), "updated_at": result["updated_at"].isoformat()}}


@forecast_router.get("/expense-forecast/{year}/scenarios")
async def get_expense_forecast_scenarios(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    company_id = get_company_id(current_user)
    custom_scenarios = await _get_expense_custom_scenarios(db, year, quarter.value, company_id)
    for s in custom_scenarios:
        s["is_system"] = False
        s["department_multipliers"] = await _get_scenario_department_multipliers(db, s["id"], company_id, year, quarter.value) if s["department_scope"] == "specific" else {}
    hardcoded = [
        {"id": None, "scenario_name": "baseline", "multiplier": 1.0, "description": EPENSE_SCENARIO_DEFINITIONS["baseline"], "parameter_types": ["revenue"], "department_scope": "all", "department_multipliers": {}, "is_system": True, "created_by_name": None, "created_at": None, "updated_at": None},
        {"id": None, "scenario_name": "best_case", "multiplier": BEST_CASE_MULTIPLIER, "description": EPENSE_SCENARIO_DEFINITIONS["best_case"], "parameter_types": ["revenue", "rate"], "department_scope": "all", "department_multipliers": {}, "is_system": True, "created_by_name": None, "created_at": None, "updated_at": None},
        {"id": None, "scenario_name": "worst_case", "multiplier": WORST_CASE_MULTIPLIER, "description": EPENSE_SCENARIO_DEFINITIONS["worst_case"], "parameter_types": ["volume", "quantity"], "department_scope": "all", "department_multipliers": {}, "is_system": True, "created_by_name": None, "created_at": None, "updated_at": None},
    ]
    return {"year": year, "quarter": quarter.value, "total_scenarios": len(hardcoded) + len(custom_scenarios), "scenarios": hardcoded + custom_scenarios}


@forecast_router.patch("/expense-forecast/{year}/scenarios/{scenario_id}")
async def update_expense_forecast_scenario(
    year: int = Path(...),
    scenario_id: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: ExpenseScenarioUpdateInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    company_id = get_company_id(current_user)
    existing = (await db.execute(text("SELECT id, scenario_name FROM fpna_config.expense_forecast_scenarios WHERE id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter"), {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value})).mappings().first()
    if not existing:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Expense scenario with id {scenario_id} not found for {year} {quarter.value}.")
    if body.scenario_name is not None:
        new_name_clean = body.scenario_name.strip().lower().replace(" ", "_")
        if new_name_clean in RESERVED_SCENARIO_NAMES:
            raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"'{body.scenario_name}' is a reserved scenario name.")
        if (await db.execute(text("SELECT id FROM fpna_config.expense_forecast_scenarios WHERE company_id = :company_id AND year = :year AND quarter = :quarter AND LOWER(REPLACE(scenario_name, ' ', '_')) = :new_name_clean AND id != :sid"), {"company_id": company_id, "year": year, "quarter": quarter.value, "new_name_clean": new_name_clean, "sid": scenario_id})).mappings().first():
            raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=f"A scenario named '{body.scenario_name}' already exists for {year} {quarter.value}.")
    if body.multiplier is not None and not (0.01 <= body.multiplier <= 10.0):
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Multiplier must be between 0.01 and 10.0.")
    set_clauses = ["updated_at = NOW()"]
    params: Dict[str, Any] = {"sid": scenario_id}
    if body.scenario_name is not None:
        set_clauses.append("scenario_name = :scenario_name")
        params["scenario_name"] = body.scenario_name.strip()
    if body.multiplier is not None:
        set_clauses.append("multiplier = :multiplier")
        params["multiplier"] = body.multiplier
    if body.description is not None:
        set_clauses.append("description = :description")
        params["description"] = body.description.strip()
    if body.parameter_types is not None:
        set_clauses.append("parameter_types = :parameter_types")
        params["parameter_types"] = json.dumps(body.parameter_types)
    if body.department_scope is not None:
        set_clauses.append("department_scope = :department_scope")
        params["department_scope"] = body.department_scope
    result = (await db.execute(text(f"UPDATE fpna_config.expense_forecast_scenarios SET {', '.join(set_clauses)} WHERE id = :sid RETURNING id, scenario_name, multiplier, description, parameter_types, department_scope, created_by_name, created_at, updated_at"), params)).mappings().first()
    await db.commit()
    new_scope = result["department_scope"] if result["department_scope"] else "all"
    department_multipliers_saved = {}
    del_params = {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value}
    if new_scope == "specific" and body.department_multipliers is not None:
        await db.execute(text("DELETE FROM fpna_config.expense_scenario_department_multipliers WHERE scenario_id = :scenario_id AND company_id = :company_id AND year = :year AND quarter = :quarter"), del_params)
        for dm in body.department_multipliers:
            if not dm.department_name or dm.multiplier is None or not (0.01 <= dm.multiplier <= 10.0):
                continue
            await db.execute(text("INSERT INTO fpna_config.expense_scenario_department_multipliers (scenario_id, company_id, year, quarter, department_name, multiplier) VALUES (:scenario_id, :company_id, :year, :quarter, :department_name, :multiplier) ON CONFLICT (scenario_id, company_id, year, quarter, department_name) DO UPDATE SET multiplier = EXCLUDED.multiplier, updated_at = NOW()"), {**del_params, "department_name": dm.department_name, "multiplier": dm.multiplier})
            department_multipliers_saved[dm.department_name] = dm.multiplier
        await db.commit()
    elif new_scope == "all":
        await db.execute(text("DELETE FROM fpna_config.expense_scenario_department_multipliers WHERE scenario_id = :scenario_id AND company_id = :company_id AND year = :year AND quarter = :quarter"), del_params)
        await db.commit()
    else:
        department_multipliers_saved = await _get_scenario_department_multipliers(db, scenario_id, company_id, year, quarter.value)
    return {"detail": "Expense scenario updated successfully.", "year": year, "quarter": quarter.value, "scenario": {"id": result["id"], "scenario_name": result["scenario_name"], "multiplier": float(result["multiplier"]), "description": result["description"], "parameter_types": result["parameter_types"] if result["parameter_types"] else ["revenue"], "department_scope": result["department_scope"], "department_multipliers": department_multipliers_saved, "created_by_name": result["created_by_name"], "created_at": result["created_at"].isoformat(), "updated_at": result["updated_at"].isoformat()}}


@forecast_router.delete("/expense-forecast/{year}/scenarios/{scenario_id}")
async def delete_expense_forecast_scenario(
    year: int = Path(...),
    scenario_id: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    company_id = get_company_id(current_user)
    existing = (await db.execute(text("SELECT id, scenario_name FROM fpna_config.expense_forecast_scenarios WHERE id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter"), {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value})).mappings().first()
    if not existing:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Expense scenario with id {scenario_id} not found for {year} {quarter.value}.")
    await db.execute(text("DELETE FROM fpna_config.expense_scenario_department_multipliers WHERE scenario_id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter"), {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value})
    await db.execute(text("DELETE FROM fpna_config.expense_forecast_scenarios WHERE id = :sid"), {"sid": scenario_id})
    await db.commit()
    return {"detail": f"Expense scenario '{existing['scenario_name']}' deleted successfully.", "year": year, "quarter": quarter.value, "deleted_scenario_id": scenario_id}


@forecast_router.get("/expense-forecast/{year}/excel-template/expense-adjustments")
async def download_expense_adjustments_template(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    department: Optional[str] = Query(None),
    expense_type: ExpenseTypeEnum = Query(ExpenseTypeEnum.ALL),
    expense_category: ExpenseCategoryEnum = Query(ExpenseCategoryEnum.ALL),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    company_id = get_company_id(current_user)
    periods = get_quarter_periods(year, quarter.value)
    m1, m2, m3 = periods
    prev_period = get_previous_month(year, quarter.value)
    history_start = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")

    saved, gl_result, prev_result = await asyncio.gather(
        _get_expense_adjustments(db, year, quarter.value, company_id, department, expense_type.value),
        db.execute(text("""
            SELECT coa.account_name AS expense_item, COALESCE(d.department_name, '') AS department, gl.period, SUM(gl.amount_debit - gl.amount_credit) AS y
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
            LEFT JOIN fpna_landing.hr_departments_lookup d ON gl.department_id = d.department_id
            WHERE coa.account_type = 'Expense' AND gl.period >= :start_period AND gl.period < :quarter_start
              AND (CAST(:department AS TEXT) IS NULL OR d.department_name = CAST(:department AS TEXT))
            GROUP BY coa.account_name, d.department_name, gl.period ORDER BY gl.period;
        """), {"department": department, "start_period": history_start, "quarter_start": m1}),
        db.execute(text("""
            SELECT coa.account_name AS expense_item, COALESCE(d.department_name, '') AS department, SUM(gl.amount_debit - gl.amount_credit) AS amount
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
            LEFT JOIN fpna_landing.hr_departments_lookup d ON gl.department_id = d.department_id
            WHERE coa.account_type = 'Expense' AND gl.period = :prev_period
              AND (CAST(:department AS TEXT) IS NULL OR d.department_name = CAST(:department AS TEXT))
            GROUP BY coa.account_name, d.department_name;
        """), {"prev_period": prev_period, "department": department}),
    )

    adj_map = {(s["expense_item"], s["department"] or "", s["expense_type"]): s for s in saved}
    df = pd.DataFrame(gl_result.mappings().all())

    item_forecasts: Dict[tuple, Tuple[float, float, float]] = {}
    if not df.empty:
        df["ds"] = pd.to_datetime(df["period"] + "-01")
        item_forecasts = await _forecast_all_expense_items(df, m1, expense_category.value)

    prev_df = pd.DataFrame(prev_result.mappings().all())
    prev_map = {}
    if not prev_df.empty:
        for row in prev_df.to_dict('records'):
            prev_map[(row["expense_item"], row["department"] or "")] = float(row["amount"])

    wb = openpyxl.Workbook()
    ws = wb.active
    ws.title = "Expense Adjustments"
    ws.append(["Expense Item", "Department", "Expense Category", f"Previous Month ({prev_period})", f"{m1} AI Forecast", f"{m1} Adjusted", f"{m1} Comment", f"{m2} AI Forecast", f"{m2} Adjusted", f"{m2} Comment", f"{m3} AI Forecast", f"{m3} Adjusted", f"{m3} Comment", "Quarter Total AI", "Quarter Total Adjusted", "Overall Comment"])

    all_keys = set(item_forecasts.keys()) | set((k[0], k[1]) for k in adj_map.keys())
    for key in sorted(all_keys):
        item, dept_name = key
        if expense_category.value != "ALL" and classify_expense_category(item) != expense_category.value:
            continue
        best_adj = None
        for etype_try in ["Fixed", "Variable", ""]:
            best_adj = adj_map.get((item, dept_name, etype_try))
            if best_adj:
                break
        adj = best_adj or {}
        prev_val = round(prev_map.get(key, 0.0), 2)
        forecast_vals = item_forecasts.get(key, (0.0, 0.0, 0.0))
        m1_ai = round(float(forecast_vals[0]), 2)
        m2_ai = round(float(forecast_vals[1]), 2)
        m3_ai = round(float(forecast_vals[2]), 2)
        m1_adj = adj.get("month1_adjusted")
        m2_adj = adj.get("month2_adjusted")
        m3_adj = adj.get("month3_adjusted")
        ws.append([item, dept_name, classify_expense_category(item), prev_val, m1_ai, m1_adj, adj.get("month1_comment"), m2_ai, m2_adj, adj.get("month2_comment"), m3_ai, m3_adj, adj.get("month3_comment"), round(m1_ai + m2_ai + m3_ai, 2), adj.get("quarter_total_adjusted") if (m1_adj or m2_adj or m3_adj) else None, adj.get("overall_comment")])

    stream = io.BytesIO()
    wb.save(stream)
    stream.seek(0)
    return StreamingResponse(stream, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", headers={"Content-Disposition": f"attachment; filename=expense_adjustments_{year}_{quarter.value}.xlsx"})


@forecast_router.get("/expense-forecast/{year}/excel-template/driver-adjustments")
async def download_expense_driver_adjustments_template(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    department: Optional[str] = Query(None),
    expense_type: ExpenseTypeEnum = Query(ExpenseTypeEnum.ALL),
    expense_category: ExpenseCategoryEnum = Query(ExpenseCategoryEnum.ALL),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    company_id = get_company_id(current_user)
    periods = get_quarter_periods(year, quarter.value)
    m1, m2, m3 = periods
    prev_period = get_previous_month(year, quarter.value)
    history_start = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")

    driver_saved, gl_result, prev_result = await asyncio.gather(
        _get_expense_driver_adjustments(db, year, quarter.value, company_id),
        db.execute(text("""
            SELECT coa.account_name AS expense_item, COALESCE(d.department_name, '') AS department, gl.period, SUM(gl.amount_debit - gl.amount_credit) AS y
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
            LEFT JOIN fpna_landing.hr_departments_lookup d ON gl.department_id = d.department_id
            WHERE coa.account_type = 'Expense' AND gl.period >= :start_period AND gl.period < :quarter_start
              AND (CAST(:department AS TEXT) IS NULL OR d.department_name = CAST(:department AS TEXT))
            GROUP BY coa.account_name, d.department_name, gl.period ORDER BY gl.period;
        """), {"department": department, "start_period": history_start, "quarter_start": m1}),
        db.execute(text("""
            SELECT coa.account_name AS expense_item, COALESCE(d.department_name, '') AS department, SUM(gl.amount_debit - gl.amount_credit) AS amount
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
            LEFT JOIN fpna_landing.hr_departments_lookup d ON gl.department_id = d.department_id
            WHERE coa.account_type = 'Expense' AND gl.period = :prev_period
              AND (CAST(:department AS TEXT) IS NULL OR d.department_name = CAST(:department AS TEXT))
            GROUP BY coa.account_name, d.department_name;
        """), {"prev_period": prev_period, "department": department}),
    )

    df = pd.DataFrame(gl_result.mappings().all())
    item_forecasts: Dict[tuple, Tuple[float, float, float]] = {}
    if not df.empty:
        df["ds"] = pd.to_datetime(df["period"] + "-01")
        item_forecasts = await _forecast_all_expense_items(df, m1, expense_category.value)

    prev_df = pd.DataFrame(prev_result.mappings().all())
    prev_map = {}
    if not prev_df.empty:
        for row in prev_df.to_dict('records'):
            prev_map[(row["expense_item"], row["department"] or "")] = float(row["amount"])

    wb = openpyxl.Workbook()
    ws = wb.active
    ws.title = "Driver Adjustments"
    ws.append(["Expense Item", "Department", "Expense Category", "Month", f"Previous Month ({prev_period}) Cost", "Units AI", "Rate AI", "Cost AI", "Units Adjusted", "Rate Adjusted", "Cost Adjusted", "Month Comment", "Quarter Cost AI", "Quarter Cost Adjusted", "Overall Comment"])

    all_keys = set(item_forecasts.keys())
    item_quarter_ai: Dict[tuple, float] = {}
    item_quarter_drv: Dict[tuple, float] = {}

    for key in sorted(all_keys):
        item, dept_name = key
        if expense_category.value != "ALL" and classify_expense_category(item) != expense_category.value:
            continue
        forecast_vals = item_forecasts.get(key, (0.0, 0.0, 0.0))
        ai_map = {m1: float(forecast_vals[0]), m2: float(forecast_vals[1]), m3: float(forecast_vals[2])}
        item_quarter_ai[key] = round(sum(ai_map.values()), 2)
        q_drv = 0.0
        for month_label in [m1, m2, m3]:
            drv = driver_saved.get((item, dept_name, "Fixed", month_label)) or driver_saved.get((item, dept_name, "Variable", month_label)) or {}
            u_ai, r_ai = derive_expense_units_and_rate(ai_map[month_label])
            u_user = drv.get("units_adjusted")
            r_user = drv.get("rate_adjusted")
            if u_user is not None and r_user is not None:
                q_drv += round(u_user * r_user, 2)
            elif u_user is not None:
                q_drv += round(u_user * r_ai, 2)
            elif r_user is not None:
                q_drv += round(u_ai * r_user, 2)
        item_quarter_drv[key] = round(q_drv, 2)

    for key in sorted(all_keys):
        item, dept_name = key
        if expense_category.value != "ALL" and classify_expense_category(item) != expense_category.value:
            continue
        forecast_vals = item_forecasts.get(key, (0.0, 0.0, 0.0))
        ai_map = {m1: float(forecast_vals[0]), m2: float(forecast_vals[1]), m3: float(forecast_vals[2])}
        prev_val = round(prev_map.get(key, 0.0), 2)
        expense_cat = classify_expense_category(item)
        for month_label in [m1, m2, m3]:
            cost_ai = ai_map[month_label]
            u_ai, r_ai = derive_expense_units_and_rate(cost_ai)
            drv = driver_saved.get((item, dept_name, "Fixed", month_label)) or driver_saved.get((item, dept_name, "Variable", month_label)) or {}
            u_user = drv.get("units_adjusted")
            r_user = drv.get("rate_adjusted")
            if u_user is not None and r_user is not None:
                cost_adj = round(u_user * r_user, 2)
            elif u_user is not None:
                cost_adj = round(u_user * r_ai, 2)
            elif r_user is not None:
                cost_adj = round(u_ai * r_user, 2)
            else:
                cost_adj = None
            ws.append([item, dept_name, expense_cat, month_label, prev_val, round(u_ai, 4), round(r_ai, 2), round(cost_ai, 2), u_user, r_user, cost_adj, drv.get("month_comment"), item_quarter_ai[key], item_quarter_drv[key] if item_quarter_drv[key] else None, drv.get("overall_comment")])

    stream = io.BytesIO()
    wb.save(stream)
    stream.seek(0)
    return StreamingResponse(stream, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", headers={"Content-Disposition": f"attachment; filename=expense_driver_adjustments_{year}_{quarter.value}.xlsx"})


@forecast_router.post("/expense-forecast/{year}/excel-upload/expense-adjustments")
async def upload_expense_adjustments_excel(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    file: UploadFile = File(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_expense_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Expense forecast is locked. Uploads cannot be processed.")
    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Uploads can only be processed in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")
    contents = await file.read()
    if len(contents) > 5 * 1024 * 1024:
        raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File size exceeds 5MB limit.")
    try:
        wb = openpyxl.load_workbook(io.BytesIO(contents))
    except Exception:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Could not read the uploaded file. Please ensure it is a valid .xlsx file.")
    _invalidate_expense_cache(company_id, year, quarter.value)
    periods = get_quarter_periods(year, quarter.value)
    m1, m2, m3 = periods
    sheet_name = "Expense Adjustments"
    if sheet_name not in wb.sheetnames:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Sheet '{sheet_name}' not found in the uploaded file.")
    ws_adj = wb[sheet_name]
    adj_hmap = {str(c.value).strip() if c.value is not None else "": i for i, c in enumerate(ws_adj[1])}
    adjustments, errors = [], []
    for row_idx, row in enumerate(ws_adj.iter_rows(min_row=2, values_only=True), start=2):
        if not row or row[0] is None:
            continue
        expense_item = str(row[0]).strip()
        if not expense_item:
            continue
        try:
            department_val = _get_str_cell(row, adj_hmap.get("Department", 999))
            m1_adj = _get_float_cell(row, adj_hmap.get(f"{m1} Adjusted", 999))
            m1_cmt = _get_str_cell(row, adj_hmap.get(f"{m1} Comment", 999))
            m2_adj = _get_float_cell(row, adj_hmap.get(f"{m2} Adjusted", 999))
            m2_cmt = _get_str_cell(row, adj_hmap.get(f"{m2} Comment", 999))
            m3_adj = _get_float_cell(row, adj_hmap.get(f"{m3} Adjusted", 999))
            m3_cmt = _get_str_cell(row, adj_hmap.get(f"{m3} Comment", 999))
            overall_comment = _get_str_cell(row, adj_hmap.get("Overall Comment", 999))
        except Exception:
            errors.append(f"Expense Adjustments row {row_idx}: parse error for '{expense_item}'. Skipped.")
            continue
        if any(v is not None for v in [m1_adj, m2_adj, m3_adj, m1_cmt, m2_cmt, m3_cmt, overall_comment]):
            adjustments.append(ExpenseAdjustmentInput(expense_item=expense_item, department=department_val, expense_type="Variable", month1_adjusted=m1_adj, month1_comment=m1_cmt, month2_adjusted=m2_adj, month2_comment=m2_cmt, month3_adjusted=m3_adj, month3_comment=m3_cmt, overall_comment=overall_comment))
    if not adjustments:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="No valid rows found in the Expense Adjustments sheet.")
    result = await ExpenseForecastAdjustmentManagement.PatchAdjustments(year=year, quarter=quarter.value, company_id=company_id, adjustments=adjustments, db=db)
    result["upload_errors"] = errors
    result["rows_attempted"] = len(adjustments) + len(errors)
    result["rows_saved"] = len(adjustments)
    return result


@forecast_router.post("/expense-forecast/{year}/excel-upload/driver-adjustments")
async def upload_expense_driver_adjustments_excel(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    file: UploadFile = File(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_expense_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Expense forecast is locked. Uploads cannot be processed.")
    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Uploads can only be processed in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")
    contents = await file.read()
    if len(contents) > 5 * 1024 * 1024:
        raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File size exceeds 5MB limit.")
    try:
        wb = openpyxl.load_workbook(io.BytesIO(contents))
    except Exception:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Could not read the uploaded file. Please ensure it is a valid .xlsx file.")
    _invalidate_expense_cache(company_id, year, quarter.value)
    sheet_name = "Driver Adjustments"
    if sheet_name not in wb.sheetnames:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Sheet '{sheet_name}' not found in the uploaded file.")
    ws_drv = wb[sheet_name]
    drv_hmap = {str(c.value).strip() if c.value is not None else "": i for i, c in enumerate(ws_drv[1])}
    driver_adjustments, errors = [], []
    for row_idx, row in enumerate(ws_drv.iter_rows(min_row=2, values_only=True), start=2):
        if not row or row[0] is None:
            continue
        expense_item = str(row[0]).strip()
        if not expense_item:
            continue
        try:
            department_val = _get_str_cell(row, drv_hmap.get("Department", 999))
            month_val = _get_str_cell(row, drv_hmap.get("Month", 999))
            units_adj = _get_float_cell(row, drv_hmap.get("Units Adjusted", 999))
            rate_adj = _get_float_cell(row, drv_hmap.get("Rate Adjusted", 999))
            month_cmt = _get_str_cell(row, drv_hmap.get("Month Comment", 999))
            overall_comment = _get_str_cell(row, drv_hmap.get("Overall Comment", 999))
        except Exception:
            errors.append(f"Driver Adjustments row {row_idx}: parse error for '{expense_item}'. Skipped.")
            continue
        if not month_val:
            continue
        if any(v is not None for v in [units_adj, rate_adj, month_cmt, overall_comment]):
            driver_adjustments.append(ExpenseDriverAdjustmentInput(expense_item=expense_item, department=department_val, expense_type="Variable", month=month_val, units_adjusted=units_adj, rate_adjusted=rate_adj, month_comment=month_cmt, overall_comment=overall_comment))
    if not driver_adjustments:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="No valid rows found in the Driver Adjustments sheet.")
    for adj in driver_adjustments:
        if not adj.expense_item or not adj.month:
            continue
        dept_val = adj.department if adj.department is not None else ""
        await db.execute(text("""
            INSERT INTO fpna_config.expense_driver_adjustments (year, quarter, company_id, expense_item, department, expense_type, month, units_adjusted, rate_adjusted, month_comment, overall_comment, updated_at)
            VALUES (:year, :quarter, :company_id, :expense_item, :department, :expense_type, :month, :units_adjusted, :rate_adjusted, :month_comment, :overall_comment, NOW())
            ON CONFLICT (year, quarter, company_id, expense_item, department, expense_type, month)
            DO UPDATE SET units_adjusted = EXCLUDED.units_adjusted, rate_adjusted = EXCLUDED.rate_adjusted, month_comment = EXCLUDED.month_comment, overall_comment = EXCLUDED.overall_comment, updated_at = NOW();
        """), {"year": year, "quarter": quarter.value, "company_id": company_id, "expense_item": adj.expense_item, "department": dept_val, "expense_type": adj.expense_type, "month": adj.month, "units_adjusted": adj.units_adjusted, "rate_adjusted": adj.rate_adjusted, "month_comment": adj.month_comment, "overall_comment": adj.overall_comment})
    await db.commit()
    saved = (await db.execute(text("SELECT expense_item, department, expense_type, month, units_adjusted, rate_adjusted, month_comment, overall_comment FROM fpna_config.expense_driver_adjustments WHERE year = :year AND quarter = :quarter AND company_id = :company_id ORDER BY expense_item, department, expense_type, month;"), {"year": year, "quarter": quarter.value, "company_id": company_id})).mappings().all()
    item_quarter_map: Dict[tuple, float] = {}
    result_rows = []
    for r in saved:
        u = float(r["units_adjusted"]) if r["units_adjusted"] is not None else None
        ra = float(r["rate_adjusted"]) if r["rate_adjusted"] is not None else None
        cost_adj = round(u * ra, 2) if u is not None and ra is not None else None
        key = (r["expense_item"], r["department"] or "", r["expense_type"])
        if cost_adj is not None:
            item_quarter_map[key] = item_quarter_map.get(key, 0.0) + cost_adj
        result_rows.append({"expense_item": r["expense_item"], "department": r["department"], "expense_type": r["expense_type"], "expense_category": classify_expense_category(r["expense_item"]), "month": r["month"], "units_adjusted": u, "rate_adjusted": ra, "cost_adjusted": cost_adj, "month_comment": r["month_comment"], "overall_comment": r["overall_comment"]})
    for row in result_rows:
        key = (row["expense_item"], row["department"] or "", row["expense_type"])
        row["quarter_cost_adjusted"] = round(item_quarter_map.get(key, 0.0), 2)
    return {"detail": "Expense driver adjustments uploaded successfully", "year": year, "quarter": quarter.value, "upload_errors": errors, "rows_attempted": len(driver_adjustments) + len(errors), "rows_saved": len(driver_adjustments), "driver_adjustments": result_rows}


@forecast_router.get("/expense-forecast/{year}/{currency}")
async def expense_forecast(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    department: Optional[str] = Query(None),
    expense_type: ExpenseTypeEnum = Query(ExpenseTypeEnum.ALL),
    expense_category: ExpenseCategoryEnum = Query(ExpenseCategoryEnum.ALL),
    forecast_year: Optional[int] = Query(default=None, description="Primary forecast year context. Overrides year for period resolution when provided."),
    forecast_type: Optional[str] = Query(default="quarter", description="Forecast type: quarter, 12months, rolling"),
    rolling_start_month: Optional[str] = Query(default=None, description="Start month for rolling forecast (e.g. Jan, Feb, Mar). Required when forecast_type=rolling."),
    view_type: Optional[str] = Query(default="monthly_breakdown", description="View type: quarter_summary or monthly_breakdown"),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_expense_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    effective_year = forecast_year if forecast_year is not None else year
    effective_forecast_type = forecast_type or "quarter"
    effective_view_type = view_type or "monthly_breakdown"

    if effective_forecast_type not in ("quarter", "12months", "rolling"):
        raise HTTPException(status_code=422, detail="forecast_type must be one of: quarter, 12months, rolling")
    if effective_view_type not in ("quarter_summary", "monthly_breakdown"):
        raise HTTPException(status_code=422, detail="view_type must be one of: quarter_summary, monthly_breakdown")
    if effective_forecast_type == "rolling" and not rolling_start_month:
        raise HTTPException(status_code=422, detail="rolling_start_month is required when forecast_type=rolling. Example: Jan, Feb, Mar, etc.")

    # For rolling, Nov/Dec starts must use the previous year so the window spans INTO forecast_year.
    # e.g. forecast_year=2026, rolling_start_month=Dec → Dec 2025, Jan 2026, Feb 2026 (not Dec 2026+).
    _MONTH_ORDER = {"jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
                    "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12}
    resolve_year = effective_year
    if effective_forecast_type == "rolling" and rolling_start_month:
        if _MONTH_ORDER.get(rolling_start_month.lower()[:3], 1) >= 11:
            resolve_year = effective_year - 1

    month_periods, selected_period, prev_period = _resolve_forecast_periods(
        resolve_year, effective_forecast_type, quarter.value, rolling_start_month
    )
    n_months = len(month_periods)
    m1 = month_periods[0]

    # Derive effective_quarter for workflow/adjustment/cache key
    effective_quarter = quarter.value
    if effective_forecast_type == "12months":
        effective_quarter = "Q1"
    elif effective_forecast_type == "rolling":
        month_num = int(month_periods[0].split("-")[1])
        if month_num <= 3:
            effective_quarter = "Q1"
        elif month_num <= 6:
            effective_quarter = "Q2"
        elif month_num <= 9:
            effective_quarter = "Q3"
        else:
            effective_quarter = "Q4"

    history_start = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")
    end_period = month_periods[-1]

    money_fields = [
        "previous_month", "month1_ai", "month1_adjusted", "month2_ai", "month2_adjusted",
        "month3_ai", "month3_adjusted", "quarter_total_ai", "quarter_total_adjusted",
        "cost_ai", "cost_adjusted", "quarter_cost_ai", "quarter_cost_adjusted",
        "total_ai_forecast_m1", "total_adjusted_m1", "total_adjusted_q1",
        "total_ai", "total_user", "variance_amount",
        "department_total_ai", "department_total_adjusted",
        "employee_total_ai", "employee_total_adjusted", "percent_of_salary",
    ]

    cache_key = _expense_cache_key(
        company_id, effective_year, effective_quarter,
        department, expense_type.value, expense_category.value,
        forecast_type=effective_forecast_type, view_type=effective_view_type,
        rolling_start_month=rolling_start_month,
    )
    cached = _get_expense_cache(cache_key)

    wf = await _get_or_create_expense_workflow(db, effective_year, effective_quarter, company_id, dept, user_id)
    stage = wf["stage"]

    # Cache now keys on every filter that affects the response, so we can
    # safely serve hits for any forecast_type / view_type combination.
    if cached is not None:
        cached_result = dict(cached)
        cached_result["workflow_stage"] = stage
        cached_result["forecast_year"] = effective_year
        cached_result["forecast_type"] = effective_forecast_type
        cached_result["view_type"] = effective_view_type
        cached_result["selected_period"] = selected_period
        cached_result["selected_months"] = month_periods
        return JSONResponse(content=apply_smart_currency(cached_result, currency, currency_filter, money_fields))

    gl_result, dept_result, saved, driver_saved, custom_scenarios, employees, benefits_pension_map, employee_adj_map, audit_rows_result = await asyncio.gather(
        db.execute(text("""
            SELECT coa.account_name AS expense_item, COALESCE(d.department_name, '') AS department, gl.period, SUM(gl.amount_debit - gl.amount_credit) AS y
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
            LEFT JOIN fpna_landing.hr_departments_lookup d ON gl.department_id = d.department_id
            WHERE coa.account_type = 'Expense' AND gl.period >= :start_period AND gl.period <= :end_period
              AND (CAST(:department AS TEXT) IS NULL OR d.department_name = CAST(:department AS TEXT))
            GROUP BY coa.account_name, d.department_name, gl.period ORDER BY gl.period;
        """), {"department": department, "start_period": history_start, "end_period": end_period}),
        db.execute(text("SELECT DISTINCT department_name FROM fpna_landing.hr_departments_lookup WHERE department_name IS NOT NULL ORDER BY department_name;")),
        _get_expense_adjustments(db, effective_year, effective_quarter, company_id, department, expense_type.value),
        _get_expense_driver_adjustments(db, effective_year, effective_quarter, company_id),
        _get_expense_custom_scenarios(db, effective_year, effective_quarter, company_id),
        _get_employee_roster(db, company_id, department),
        _get_benefits_pension_percents(db, company_id, m1),
        _get_employee_adjustments(db, effective_year, effective_quarter, company_id, department),
        db.execute(text("""
            SELECT wfa.to_stage, wfa.transitioned_by_name
            FROM fpna_config.expense_workflow_audit wfa
            JOIN fpna_config.expense_forecast_workflows wfw ON wfa.workflow_id = wfw.id
            WHERE wfw.year = :year AND wfw.quarter = :quarter AND wfw.company_id = :company_id
            ORDER BY wfa.created_at ASC
        """), {"year": effective_year, "quarter": effective_quarter, "company_id": company_id}),
    )

    df = pd.DataFrame(gl_result.mappings().all())
    departments = [r[0] for r in dept_result.fetchall()]

    audit_rows = audit_rows_result.mappings().all()
    reviewed_by = approved_by = locked_by = None
    for r in audit_rows:
        if r["to_stage"] == "reviewed":
            reviewed_by = r["transitioned_by_name"]
        elif r["to_stage"] == "approved":
            approved_by = r["transitioned_by_name"]
        elif r["to_stage"] == "locked":
            locked_by = r["transitioned_by_name"]

    people_expense_breakdown: Optional[List[PeopleExpenseDepartmentSummary]] = None
    if employees:
        # Use first 3 months of the selected period for people expense breakdown
        periods_for_people = tuple(month_periods[:3]) if n_months >= 3 else tuple(month_periods)
        people_expense_breakdown = _build_people_expense_breakdown(
            employees=employees, periods=periods_for_people,
            benefits_pension_map=benefits_pension_map, employee_adj_map=employee_adj_map,
        )

    def _make_empty_response() -> dict:
        zero_total = ExpenseGrandTotal()
        empty_summary = {
            "total_ai_forecast_m1": 0.0, "total_adjusted_m1": 0.0, "total_adjusted_q1": 0.0,
            "total_ai": 0.0, "total_user": 0.0, "variance_amount": 0.0, "variance_percent": 0.0,
        }
        empty_scenarios = {
            "baseline": _build_expense_scenario_summary(0.0, 0.0, 1.0),
            "best_case": _build_expense_scenario_summary(0.0, 0.0, BEST_CASE_MULTIPLIER),
            "worst_case": _build_expense_scenario_summary(0.0, 0.0, WORST_CASE_MULTIPLIER),
        }
        for cs in custom_scenarios:
            empty_scenarios[cs["scenario_name"].lower().replace(" ", "_")] = _build_expense_scenario_summary(0.0, 0.0, cs["multiplier"])
        rd = ExpenseForecastResponse(
            year=year, quarter=quarter,
            filters={"department": department, "expense_type": expense_type.value, "expense_category": expense_category.value},
            baseline_forecast_summary=empty_summary, Compare_Expense_Forecast_Scenarios=empty_scenarios,
            rows={}, grand_total=zero_total, drivers=[], departments=departments,
            workflow_stage=stage, reviewed_by=reviewed_by, approved_by=approved_by, locked_by=locked_by,
            people_expense_breakdown=people_expense_breakdown,
        ).dict()
        rd["forecast_year"] = effective_year
        rd["forecast_type"] = effective_forecast_type
        rd["view_type"] = effective_view_type
        rd["selected_period"] = selected_period
        rd["selected_months"] = month_periods
        rd["month_totals"] = [{"month": mp, "total_ai": 0.0, "total_adjusted": 0.0} for mp in month_periods]
        return rd

    if df.empty:
        return JSONResponse(content=apply_smart_currency(_make_empty_response(), currency, currency_filter, money_fields))

    df["ds"] = pd.to_datetime(df["period"] + "-01")
    adj_map = {(s["expense_item"], s["department"] or "", s["expense_type"]): s for s in saved}

    # Generate AI forecasts — existing 3-month pipeline for quarter/rolling, N-month for 12months
    if n_months <= 3:
        raw_forecasts = await _forecast_all_expense_items(df, m1, expense_category.value)
        # Normalize Tuple[float,float,float] -> List[float] for uniform handling below
        item_forecasts: Dict[tuple, List[float]] = {k: list(v) for k, v in raw_forecasts.items()}
    else:
        item_forecasts = await _forecast_all_expense_items_n_months(df, m1, n_months, expense_category.value)

    rows_dict: Dict[str, List[ExpenseRow]] = {}
    totals = ExpenseGrandTotal()
    driver_map: Dict[tuple, List[ExpenseDriverRow]] = {}
    department_ai_map: Dict[str, float] = {}
    department_user_map: Dict[str, float] = {}
    # Per-month aggregate totals across all expense items
    month_totals: List[Dict[str, Any]] = [
        {"month": mp, "total_ai": 0.0, "total_adjusted": 0.0} for mp in month_periods
    ]
    # Store raw per-month values per (item, dept) for monthly_data in multi-month views
    item_monthly_data: Dict[tuple, List[Dict[str, Any]]] = {}

    for (item, dept_name) in item_forecasts:
        expense_category_val = classify_expense_category(item)
        g = df[(df["expense_item"] == item) & (df["department"] == dept_name)]
        prev_val = float(g[g["period"] == prev_period]["y"].sum()) if not g[g["period"] == prev_period].empty else 0.0

        forecast_vals = item_forecasts[(item, dept_name)]
        adj = adj_map.get((item, dept_name or "", "Variable")) or adj_map.get((item, dept_name or "", "Fixed")) or None
        overall_comment = adj.get("overall_comment") if adj else None
        expense_type_for_row = adj.get("expense_type", "Variable") if adj else "Variable"

        # Build per-month values for all periods in the selected range
        month_ai_vals: List[float] = []
        month_adj_vals: List[float] = []
        month_comments: List[Optional[str]] = []
        driver_overrides: Dict[str, float] = {}
        driver_month_comments: Dict[str, Optional[str]] = {mp: None for mp in month_periods}
        key = (item, dept_name or "")

        for idx, month_label in enumerate(month_periods):
            cost_ai = float(forecast_vals[idx]) if idx < len(forecast_vals) else 0.0
            u_ai, r_ai = derive_expense_units_and_rate(cost_ai)

            drv = (
                driver_saved.get((item, dept_name or "", "Variable", month_label))
                or driver_saved.get((item, dept_name or "", "Fixed", month_label))
            )
            if drv:
                u_user = drv["units_adjusted"]
                r_user = drv["rate_adjusted"]
                drv_month_comment = drv.get("month_comment")
                if u_user is not None and r_user is not None:
                    cost_adj_driver = round(u_user * r_user, 2)
                elif u_user is not None:
                    cost_adj_driver = round(u_user * r_ai, 2)
                elif r_user is not None:
                    cost_adj_driver = round(u_ai * r_user, 2)
                else:
                    cost_adj_driver = None
            else:
                u_user = r_user = drv_month_comment = cost_adj_driver = None
            driver_month_comments[month_label] = drv_month_comment
            if key not in driver_map:
                driver_map[key] = []
            driver_map[key].append(ExpenseDriverRow(
                expense_item=item, department=dept_name if dept_name else None,
                expense_type=expense_type_for_row, expense_category=expense_category_val,
                month=month_label, units_ai=u_ai, rate_ai=r_ai, cost_ai=round(u_ai * r_ai, 2),
                units_adjusted=round(u_user, 4) if u_user is not None else None,
                rate_adjusted=round(r_user, 2) if r_user is not None else None,
                cost_adjusted=cost_adj_driver, month_comment=drv_month_comment,
            ))
            if cost_adj_driver is not None:
                driver_overrides[month_label] = cost_adj_driver

            # Priority: driver override > manual adjustment > AI value
            if month_label in driver_overrides:
                cost_adj = driver_overrides[month_label]
                month_comment_val = driver_month_comments.get(month_label)
            elif adj is not None:
                adj_key_str = f"month{idx + 1}_adjusted"
                adj_comment_key_str = f"month{idx + 1}_comment"
                adj_val = adj.get(adj_key_str) if isinstance(adj, dict) else None
                cost_adj = float(adj_val) if adj_val is not None else cost_ai
                month_comment_val = adj.get(adj_comment_key_str) if isinstance(adj, dict) else None
            else:
                cost_adj = cost_ai
                month_comment_val = None

            month_ai_vals.append(round(cost_ai, 2))
            month_adj_vals.append(round(cost_adj, 2))
            month_comments.append(month_comment_val)

            # Accumulate per-month aggregates
            month_totals[idx]["total_ai"] = round(month_totals[idx]["total_ai"] + round(cost_ai, 2), 2)
            month_totals[idx]["total_adjusted"] = round(month_totals[idx]["total_adjusted"] + round(cost_adj, 2), 2)

        period_total_ai = round(sum(month_ai_vals), 2)
        period_total_adjusted = round(sum(month_adj_vals), 2)

        dept_key = dept_name or ""
        department_ai_map[dept_key] = department_ai_map.get(dept_key, 0.0) + period_total_ai
        department_user_map[dept_key] = department_user_map.get(dept_key, 0.0) + period_total_adjusted

        # Flat month1/month2/month3 fields for ExpenseRow (backward compat — first 3 months)
        m1_ai = month_ai_vals[0] if len(month_ai_vals) > 0 else 0.0
        m1_adj = month_adj_vals[0] if len(month_adj_vals) > 0 else 0.0
        m1_comment = month_comments[0] if len(month_comments) > 0 else None
        m2_ai = month_ai_vals[1] if len(month_ai_vals) > 1 else 0.0
        m2_adj = month_adj_vals[1] if len(month_adj_vals) > 1 else 0.0
        m2_comment = month_comments[1] if len(month_comments) > 1 else None
        m3_ai = month_ai_vals[2] if len(month_ai_vals) > 2 else 0.0
        m3_adj = month_adj_vals[2] if len(month_adj_vals) > 2 else 0.0
        m3_comment = month_comments[2] if len(month_comments) > 2 else None

        row = ExpenseRow(
            department=dept_name if dept_name else None,
            expense_type=expense_type_for_row, expense_category=expense_category_val,
            previous_month=round(prev_val, 2),
            month1_ai=m1_ai, month1_adjusted=m1_adj, month1_comment=m1_comment,
            month2_ai=m2_ai, month2_adjusted=m2_adj, month2_comment=m2_comment,
            month3_ai=m3_ai, month3_adjusted=m3_adj, month3_comment=m3_comment,
            quarter_total_ai=round(period_total_ai, 2),
            quarter_total_adjusted=round(period_total_adjusted, 2),
            overall_comment=overall_comment,
        )
        if item not in rows_dict:
            rows_dict[item] = []
        rows_dict[item].append(row)

        totals.previous_month += row.previous_month
        totals.month1_ai += m1_ai
        totals.month1_adjusted += m1_adj
        totals.month2_ai += m2_ai
        totals.month2_adjusted += m2_adj
        totals.month3_ai += m3_ai
        totals.month3_adjusted += m3_adj

        # Store per-month detail for multi-month monthly_data field
        item_monthly_data[key] = [
            {"month": mp, "ai": month_ai_vals[i], "adjusted": month_adj_vals[i]}
            for i, mp in enumerate(month_periods)
        ]

    drivers_output: List[ExpenseCustomerDriverSummary] = []
    for key, month_rows in driver_map.items():
        item, dept_name = key
        q_ai = sum(m.cost_ai for m in month_rows)
        q_adj = sum(m.cost_adjusted if m.cost_adjusted is not None else m.cost_ai for m in month_rows)
        overall_comment_for_item = None
        for month_label in month_periods:
            drv = driver_saved.get((item, dept_name, "Variable", month_label)) or driver_saved.get((item, dept_name, "Fixed", month_label))
            if drv and drv.get("overall_comment"):
                overall_comment_for_item = drv["overall_comment"]
                break
        expense_type_for_driver = month_rows[0].expense_type if month_rows else "Variable"
        drivers_output.append(ExpenseCustomerDriverSummary(
            expense_item=item, department=dept_name if dept_name else None,
            expense_type=expense_type_for_driver, expense_category=classify_expense_category(item),
            months=sorted(month_rows, key=lambda x: x.month),
            quarter_cost_ai=round(q_ai, 2), quarter_cost_adjusted=round(q_adj, 2),
            overall_comment=overall_comment_for_item,
        ))

    totals_final = ExpenseGrandTotal(**{k: round(v, 2) for k, v in totals.dict().items()})

    # Baseline totals calculated from filtered month_totals (not hardcoded from ExpenseGrandTotal)
    baseline_ai = round(sum(mt["total_ai"] for mt in month_totals), 2)
    baseline_user = round(sum(mt["total_adjusted"] for mt in month_totals), 2)

    baseline_forecast_summary = {
        "total_ai_forecast_m1": month_totals[0]["total_ai"] if month_totals else 0.0,
        "total_adjusted_m1": month_totals[0]["total_adjusted"] if month_totals else 0.0,
        "total_adjusted_q1": round(sum(mt["total_adjusted"] for mt in month_totals[:3]), 2),
        "total_ai": baseline_ai,
        "total_user": baseline_user,
        "variance_amount": round(baseline_user - baseline_ai, 2),
        "variance_percent": round(((baseline_user - baseline_ai) / baseline_ai) * 100 if baseline_ai else 0, 2),
    }

    compare_expense_forecast_scenarios = {
        "baseline": _build_expense_scenario_summary(baseline_ai, baseline_user, 1.0),
        "best_case": _build_expense_scenario_summary(baseline_ai, baseline_user, BEST_CASE_MULTIPLIER),
        "worst_case": _build_expense_scenario_summary(baseline_ai, baseline_user, WORST_CASE_MULTIPLIER),
    }

    specific_scenario_tasks, specific_scenario_keys, plain_scenario_items = [], [], []
    for cs in custom_scenarios:
        scenario_key = cs["scenario_name"].lower().replace(" ", "_")
        if cs["department_scope"] == "specific":
            specific_scenario_tasks.append(_get_scenario_department_multipliers(db, cs["id"], company_id, effective_year, effective_quarter))
            specific_scenario_keys.append((scenario_key, cs))
        else:
            plain_scenario_items.append((scenario_key, cs))

    for scenario_key, cs in plain_scenario_items:
        compare_expense_forecast_scenarios[scenario_key] = _build_expense_scenario_summary(baseline_ai, baseline_user, cs["multiplier"])

    if specific_scenario_tasks:
        dept_mults_results = await asyncio.gather(*specific_scenario_tasks)
        for (scenario_key, cs), dept_mults in zip(specific_scenario_keys, dept_mults_results):
            compare_expense_forecast_scenarios[scenario_key] = _build_expense_scenario_summary_with_department_multipliers(
                baseline_ai=baseline_ai, baseline_user=baseline_user,
                scenario_multiplier=cs["multiplier"],
                department_ai_map=department_ai_map, department_user_map=department_user_map,
                department_multipliers=dept_mults, department_scope=cs["department_scope"],
            )

    response_data = ExpenseForecastResponse(
        year=year, quarter=quarter,
        filters={"department": department, "expense_type": expense_type.value, "expense_category": expense_category.value},
        baseline_forecast_summary=baseline_forecast_summary,
        Compare_Expense_Forecast_Scenarios=compare_expense_forecast_scenarios,
        rows=rows_dict, grand_total=totals_final, drivers=drivers_output, departments=departments,
        workflow_stage=stage, reviewed_by=reviewed_by, approved_by=approved_by, locked_by=locked_by,
        people_expense_breakdown=people_expense_breakdown,
    ).dict()

    # Apply view_type: quarter_summary collapses per-month detail to period totals only
    if effective_view_type == "quarter_summary":
        summary_rows: Dict[str, List[Dict]] = {}
        for item_name, item_rows_list in response_data["rows"].items():
            summary_rows[item_name] = [
                {
                    "department": r["department"],
                    "expense_type": r["expense_type"],
                    "expense_category": r["expense_category"],
                    "previous_month": r["previous_month"],
                    "quarter_total_ai": r["quarter_total_ai"],
                    "quarter_total_adjusted": r["quarter_total_adjusted"],
                    "overall_comment": r["overall_comment"],
                }
                for r in item_rows_list
            ]
        response_data["rows"] = summary_rows

        # For 12months quarter_summary: build Q1-Q4 aggregate totals from month_totals
        if n_months > 3:
            q_summary: Dict[str, Dict[str, float]] = {}
            for qi, q_label in enumerate(["Q1", "Q2", "Q3", "Q4"]):
                start_idx = qi * 3
                q_ai = round(sum(mt["total_ai"] for mt in month_totals[start_idx:start_idx + 3]), 2)
                q_adj = round(sum(mt["total_adjusted"] for mt in month_totals[start_idx:start_idx + 3]), 2)
                q_summary[q_label] = {"total_ai": q_ai, "total_adjusted": q_adj}
            response_data["quarter_summary"] = q_summary

    # For multi-month views, attach monthly_data per row (all periods breakdown)
    if n_months > 3 and effective_view_type == "monthly_breakdown":
        for item_name, item_rows_list in response_data["rows"].items():
            for row_dict in item_rows_list:
                dept_key = row_dict.get("department") or ""
                key = (item_name, dept_key)
                row_dict["monthly_data"] = item_monthly_data.get(key, [])

    # Attach new forecast filter fields to response
    response_data["forecast_year"] = effective_year
    response_data["forecast_type"] = effective_forecast_type
    response_data["view_type"] = effective_view_type
    response_data["selected_period"] = selected_period
    response_data["selected_months"] = month_periods
    response_data["month_totals"] = month_totals

    # Cache only the standard quarter forecast to avoid stale cross-period results
    if effective_forecast_type == "quarter":
        _set_expense_cache(cache_key, response_data)

    return JSONResponse(content=apply_smart_currency(response_data, currency, currency_filter, money_fields))

######################################################################
#######################################################################
######################################################################

#haedcount & Payroll forecasting


PAYROLL_GROWTH_RATE = 0.03
PAYROLL_BEST_CASE_MULTIPLIER = 1.08
PAYROLL_WORST_CASE_MULTIPLIER = 0.85
PAYROLL_HISTORY_MONTHS = 18
PAYROLL_MIN_POINTS_FOR_PROPHET = 24

PAYROLL_RESERVED_SCENARIO_NAMES = {"baseline", "best_case", "worst_case"}

PAYROLL_SCENARIO_DEFINITIONS = {
    "baseline": "AI-generated payroll forecast with manual user adjustments applied.",
    "best_case": f"Optimistic scenario applying a {int((PAYROLL_BEST_CASE_MULTIPLIER - 1) * 100)}% uplift on baseline — assumes favorable headcount growth and full hiring pipeline conversion.",
    "worst_case": f"Pessimistic scenario applying a {int((1 - PAYROLL_WORST_CASE_MULTIPLIER) * 100)}% reduction on baseline — assumes hiring freeze, attrition risk, or budget shortfall.",
}

PAYROLL_VALID_PARAMETER_TYPES = {"salary", "bonus", "benefits", "headcount"}

DEPT_SUBMITTERS = {"it", "sales", "operations", "hr"}
DEPT_REVIEWERS = {"finance"}
DEPT_APPROVERS = {"ceo", "cfo", "executive"}

RAW_SQL = """
SELECT
    e.employee_id,
    e.first_name || ' ' || e.last_name AS employee,
    d.department_name AS department,
    e.job_title AS role,
    e.hire_date,
    c.effective_date,
    c.base_salary,
    c.bonus_target
FROM fpna_landing.hr_employees e
JOIN fpna_landing.hr_departments_lookup d
    ON e.department_id = d.department_id
LEFT JOIN fpna_landing.hr_compensation c
    ON e.employee_id = c.employee_id
WHERE e.status = 'Active'
ORDER BY e.employee_id, c.effective_date;
"""



class PayrollAdjustmentInput(BaseModel):
    employee_name: str
    month1_adjusted_salary: Optional[float] = None
    month2_adjusted_salary: Optional[float] = None
    month3_adjusted_salary: Optional[float] = None
    month1_adjusted_bonus: Optional[float] = None
    month2_adjusted_bonus: Optional[float] = None
    month3_adjusted_bonus: Optional[float] = None
    month1_adjusted_benefits: Optional[float] = None
    month2_adjusted_benefits: Optional[float] = None
    month3_adjusted_benefits: Optional[float] = None
    month1_comment: Optional[str] = None
    month2_comment: Optional[str] = None
    month3_comment: Optional[str] = None
    overall_comment: Optional[str] = None


class DepartmentHeadcountAdjustmentInput(BaseModel):
    department_name: str
    month1_adjusted_count: Optional[int] = None
    month2_adjusted_count: Optional[int] = None
    month3_adjusted_count: Optional[int] = None
    month1_comment: Optional[str] = None
    month2_comment: Optional[str] = None
    month3_comment: Optional[str] = None
    overall_comment: Optional[str] = None


class PayrollDriverAdjustmentInput(BaseModel):
    employee_name: str
    month: str
    salary_adjusted: Optional[float] = None
    bonus_adjusted: Optional[float] = None
    benefits_adjusted: Optional[float] = None
    month_comment: Optional[str] = None
    overall_comment: Optional[str] = None


class PayrollWorkflowTransitionRequest(BaseModel):
    target_stage: WorkflowStage
    comment: Optional[str] = None


class PayrollForecastCommentInput(BaseModel):
    comment: str


class PayrollForecastScenarioInput(BaseModel):
    scenario_name: str
    multiplier: float
    description: Optional[str] = None
    parameter_types: Optional[List[str]] = None
    customer_scope: Optional[str] = None
    customer_multipliers: Optional[List[Any]] = None


class PayrollForecastScenarioUpdateInput(BaseModel):
    scenario_name: Optional[str] = None
    multiplier: Optional[float] = None
    description: Optional[str] = None
    parameter_types: Optional[List[str]] = None
    customer_scope: Optional[str] = None
    customer_multipliers: Optional[List[Any]] = None


class PayrollCustomerMultiplierInput(BaseModel):
    employee_name: str
    multiplier: float


def quarter_date_range(year: int, quarter: QuarterEnum):
    qm = {
        QuarterEnum.Q1: (1, 3),
        QuarterEnum.Q2: (4, 6),
        QuarterEnum.Q3: (7, 9),
        QuarterEnum.Q4: (10, 12),
    }
    sm, em = qm[quarter]
    return date(year, sm, 1), date(year, em, monthrange(year, em)[1])


def quarter_period(year: int, quarter: QuarterEnum):
    return {
        QuarterEnum.Q1: f"{year}-03",
        QuarterEnum.Q2: f"{year}-06",
        QuarterEnum.Q3: f"{year}-09",
        QuarterEnum.Q4: f"{year}-12",
    }[quarter]


def get_previous_month_first_day(start_date: date) -> date:
    first_of_quarter = start_date
    last_day_prev = first_of_quarter - timedelta(days=1)
    return date(last_day_prev.year, last_day_prev.month, 1)


def get_previous_month_last_day(prev_month_first: date) -> date:
    last_day = monthrange(prev_month_first.year, prev_month_first.month)[1]
    return date(prev_month_first.year, prev_month_first.month, last_day)


def get_quarter_months_list(year: int, q: str):
    return {
        "Q1": [f"{year}-01", f"{year}-02", f"{year}-03"],
        "Q2": [f"{year}-04", f"{year}-05", f"{year}-06"],
        "Q3": [f"{year}-07", f"{year}-08", f"{year}-09"],
        "Q4": [f"{year}-10", f"{year}-11", f"{year}-12"],
    }[q]


def can_edit_payroll_forecast(dept: str, stage: str) -> bool:
    if stage == WorkflowStage.LOCKED:
        return False
    if dept in DEPT_SUBMITTERS:
        return stage == WorkflowStage.DRAFT
    return False


def can_payroll_transition(dept: str, current_stage: str, target_stage: str) -> bool:
    if current_stage == WorkflowStage.LOCKED:
        return False
    if dept in DEPT_SUBMITTERS:
        if target_stage == WorkflowStage.SUBMITTED and current_stage == WorkflowStage.DRAFT:
            return True
        return False
    if dept in DEPT_REVIEWERS:
        if target_stage == WorkflowStage.REVIEWED and current_stage == WorkflowStage.SUBMITTED:
            return True
        if target_stage == WorkflowStage.DRAFT:
            return True
        return False
    if dept in DEPT_APPROVERS:
        if target_stage == WorkflowStage.APPROVED and current_stage == WorkflowStage.REVIEWED:
            return True
        if target_stage == WorkflowStage.LOCKED and current_stage == WorkflowStage.APPROVED:
            return True
        if target_stage == WorkflowStage.DRAFT:
            return True
        return False
    return False


def can_read_all_payroll(dept: str) -> bool:
    return dept in DEPT_REVIEWERS | DEPT_APPROVERS


def detect_trend(values: np.ndarray) -> Dict[str, Any]:
    if len(values) < 2:
        return {"strength": "stable", "slope": 0.0}
    X = np.arange(len(values)).reshape(-1, 1)
    model = LinearRegression().fit(X, values)
    slope = model.coef_[0]
    mean_val = np.mean(values)
    slope_ratio = abs(slope) / mean_val if mean_val != 0 else 0
    if slope_ratio > 0.05:
        strength = "strong_increasing" if slope > 0 else "strong_decreasing"
    elif slope_ratio > 0.01:
        strength = "weak_increasing" if slope > 0 else "weak_decreasing"
    else:
        strength = "stable"
    return {"strength": strength, "slope": round(float(slope), 2)}


def detect_volatility(values: np.ndarray) -> Dict[str, Any]:
    if len(values) < 2:
        return {"level": "low", "std_dev": 0.0}
    std_dev = np.std(values)
    mean_val = np.mean(values)
    cv = std_dev / mean_val if mean_val != 0 else 0
    if cv > 0.3:
        level = "high"
    elif cv > 0.1:
        level = "medium"
    else:
        level = "low"
    return {"level": level, "std_dev": round(float(std_dev), 2), "cv": round(float(cv), 2)}


def detect_outliers(values: np.ndarray) -> List[int]:
    if len(values) < 4:
        return []
    z_scores = np.abs(stats.zscore(values))
    outlier_indices = np.where(z_scores > 3)[0].tolist()
    q1 = np.percentile(values, 25)
    q3 = np.percentile(values, 75)
    iqr = q3 - q1
    lower_bound = q1 - 1.5 * iqr
    upper_bound = q3 + 1.5 * iqr
    iqr_outliers = np.where((values < lower_bound) | (values > upper_bound))[0].tolist()
    combined_outliers = list(set(outlier_indices + iqr_outliers))
    return combined_outliers


def detect_autocorrelation(values: np.ndarray) -> Dict[str, Any]:
    if len(values) < 3:
        return {"strength": "none", "lag_1_corr": 0.0}
    lag_1_corr = np.corrcoef(values[:-1], values[1:])[0, 1] if len(values) > 1 else 0
    if abs(lag_1_corr) > 0.5:
        strength = "strong"
    elif abs(lag_1_corr) > 0.2:
        strength = "moderate"
    else:
        strength = "weak"
    return {"strength": strength, "lag_1_corr": round(float(lag_1_corr), 2)}


def forecast_series_advanced(series: pd.Series, steps: int) -> np.ndarray:
    values = series.astype(float).values
    if len(values) < 2:
        return np.full(steps, values[-1] if len(values) else 0)
    volatility_info = detect_volatility(values)
    autocorr_info = detect_autocorrelation(values)
    outlier_indices = detect_outliers(values)
    clean_values = values.copy()
    if len(outlier_indices) > 0 and len(values) > len(outlier_indices):
        median_val = np.median(values)
        for idx in outlier_indices:
            clean_values[idx] = median_val
    X = np.arange(len(clean_values)).reshape(-1, 1)
    model = LinearRegression().fit(X, clean_values)
    future_X = np.arange(len(clean_values), len(clean_values) + steps).reshape(-1, 1)
    base_forecast = model.predict(future_X)
    if volatility_info["level"] == "high":
        volatility_adjustment = volatility_info["std_dev"] * 0.1
        base_forecast = base_forecast * (1 - volatility_adjustment)
    if autocorr_info["strength"] == "strong" and len(values) > 0:
        lag_adjustment = autocorr_info["lag_1_corr"] * (values[-1] - clean_values[-1])
        base_forecast = base_forecast + lag_adjustment
    forecast = np.maximum(base_forecast, 0)
    return forecast


def _forecast_payroll_series_sync(values: np.ndarray, n_ahead: int = 3) -> List[float]:
    if len(values) == 0:
        return [0.0] * n_ahead
    if len(values) == 1:
        return [round(float(values[-1]), 2)] * n_ahead
    if len(values) == 2:
        last = float(values[-1])
        trend = float(values[-1] - values[-2])
        return [round(max(0.0, last + trend * (i + 1)), 2) for i in range(n_ahead)]
    clean_values = _winsorize_values(values)
    features = _feature_engineering(clean_values)
    months_inactive = _months_since_last_activity(clean_values)
    if months_inactive >= 3:
        reduced = round(float(np.mean(clean_values[-6:]) * 0.50), 2)
        return [max(0.0, reduced)] * n_ahead
    baseline_result = _baseline_forecast(clean_values)
    ets_result = _ets_forecast_sync(clean_values, n_ahead)
    sarima_result = _sarima_forecast_sync(clean_values, n_ahead)
    prophet_result = _prophet_forecast_sync(clean_values, "2020-01-01", n_ahead) if len(clean_values) >= MIN_POINTS_FOR_PROPHET else None
    result = _ensemble_forecasts_payroll(baseline_result, ets_result, sarima_result, prophet_result, features, n_ahead)
    result = _validate_forecast(result, clean_values)
    result = _stabilize_forecasts(result, float(clean_values[-1]), features["volatility"])
    return result


async def _ensure_payroll_workflow_tables(db: AsyncSession) -> None:
    global _PAYROLL_TABLES_INITIALIZED
    if _PAYROLL_TABLES_INITIALIZED:
        return
    _PAYROLL_TABLES_INITIALIZED = True
    await db.execute(text("CREATE SCHEMA IF NOT EXISTS fpna_config;"))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.payroll_forecast_adjustments (
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            employee_id INT NOT NULL,
            employee_name TEXT NOT NULL,
            month1_adjusted_salary NUMERIC NULL,
            month2_adjusted_salary NUMERIC NULL,
            month3_adjusted_salary NUMERIC NULL,
            month1_adjusted_bonus NUMERIC NULL,
            month2_adjusted_bonus NUMERIC NULL,
            month3_adjusted_bonus NUMERIC NULL,
            month1_adjusted_benefits NUMERIC NULL,
            month2_adjusted_benefits NUMERIC NULL,
            month3_adjusted_benefits NUMERIC NULL,
            month1_comment TEXT NULL,
            month2_comment TEXT NULL,
            month3_comment TEXT NULL,
            overall_comment TEXT NULL,
            updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT NOW()
        );
    """))

    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.table_constraints
                WHERE table_schema = 'fpna_config'
                  AND table_name = 'payroll_forecast_adjustments'
                  AND constraint_type = 'PRIMARY KEY'
            ) THEN
                ALTER TABLE fpna_config.payroll_forecast_adjustments
                    ADD CONSTRAINT payroll_forecast_adjustments_pkey
                    PRIMARY KEY (year, quarter, company_id, employee_id);
            END IF;
        END$$;
    """))

    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'payroll_forecast_adjustments'
                  AND column_name  = 'company_id'
            ) THEN
                ALTER TABLE fpna_config.payroll_forecast_adjustments ADD COLUMN company_id TEXT NOT NULL DEFAULT '';
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'payroll_forecast_adjustments'
                  AND column_name  = 'month1_comment'
            ) THEN
                ALTER TABLE fpna_config.payroll_forecast_adjustments ADD COLUMN month1_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'payroll_forecast_adjustments'
                  AND column_name  = 'month2_comment'
            ) THEN
                ALTER TABLE fpna_config.payroll_forecast_adjustments ADD COLUMN month2_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'payroll_forecast_adjustments'
                  AND column_name  = 'month3_comment'
            ) THEN
                ALTER TABLE fpna_config.payroll_forecast_adjustments ADD COLUMN month3_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'payroll_forecast_adjustments'
                  AND column_name  = 'overall_comment'
            ) THEN
                ALTER TABLE fpna_config.payroll_forecast_adjustments ADD COLUMN overall_comment TEXT NULL;
            END IF;
        END$$;
    """))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.department_headcount_forecast_adjustments (
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            department_name TEXT NOT NULL,
            month1_adjusted_count INT NULL,
            month2_adjusted_count INT NULL,
            month3_adjusted_count INT NULL,
            month1_comment TEXT NULL,
            month2_comment TEXT NULL,
            month3_comment TEXT NULL,
            overall_comment TEXT NULL,
            updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT NOW()
        );
    """))

    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.table_constraints
                WHERE table_schema = 'fpna_config'
                  AND table_name = 'department_headcount_forecast_adjustments'
                  AND constraint_type = 'PRIMARY KEY'
            ) THEN
                ALTER TABLE fpna_config.department_headcount_forecast_adjustments
                    ADD CONSTRAINT department_headcount_forecast_adjustments_pkey
                    PRIMARY KEY (year, quarter, company_id, department_name);
            END IF;
        END$$;
    """))

    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'department_headcount_forecast_adjustments'
                  AND column_name  = 'company_id'
            ) THEN
                ALTER TABLE fpna_config.department_headcount_forecast_adjustments ADD COLUMN company_id TEXT NOT NULL DEFAULT '';
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'department_headcount_forecast_adjustments'
                  AND column_name  = 'month1_comment'
            ) THEN
                ALTER TABLE fpna_config.department_headcount_forecast_adjustments ADD COLUMN month1_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'department_headcount_forecast_adjustments'
                  AND column_name  = 'month2_comment'
            ) THEN
                ALTER TABLE fpna_config.department_headcount_forecast_adjustments ADD COLUMN month2_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'department_headcount_forecast_adjustments'
                  AND column_name  = 'month3_comment'
            ) THEN
                ALTER TABLE fpna_config.department_headcount_forecast_adjustments ADD COLUMN month3_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'department_headcount_forecast_adjustments'
                  AND column_name  = 'overall_comment'
            ) THEN
                ALTER TABLE fpna_config.department_headcount_forecast_adjustments ADD COLUMN overall_comment TEXT NULL;
            END IF;
        END$$;
    """))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.payroll_driver_adjustments (
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            employee_name TEXT NOT NULL,
            month TEXT NOT NULL,
            salary_adjusted NUMERIC NULL,
            bonus_adjusted NUMERIC NULL,
            benefits_adjusted NUMERIC NULL,
            month_comment TEXT NULL,
            overall_comment TEXT NULL,
            updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT NOW(),
            PRIMARY KEY (year, quarter, company_id, employee_name, month)
        );
    """))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.payroll_forecast_workflows (
            id SERIAL PRIMARY KEY,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            department TEXT NOT NULL,
            owner_user_id TEXT NOT NULL,
            stage TEXT NOT NULL DEFAULT 'draft',
            created_at TIMESTAMP NOT NULL DEFAULT NOW(),
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            locked_at TIMESTAMP NULL,
            UNIQUE (year, quarter, company_id)
        );
    """))

    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'payroll_forecast_workflows'
                  AND column_name  = 'owner_user_id'
                  AND data_type   != 'text'
            ) THEN
                ALTER TABLE fpna_config.payroll_forecast_workflows
                    ALTER COLUMN owner_user_id TYPE TEXT USING owner_user_id::TEXT;
            END IF;
        END$$;
    """))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.payroll_forecast_workflow_audit (
            id SERIAL PRIMARY KEY,
            workflow_id INT NOT NULL,
            from_stage TEXT NULL,
            to_stage TEXT NOT NULL,
            transitioned_by_user_id TEXT NOT NULL,
            transitioned_by_name TEXT NOT NULL,
            comment TEXT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.payroll_forecast_workflow_comments (
            id SERIAL PRIMARY KEY,
            workflow_id INT NOT NULL,
            user_id TEXT NOT NULL,
            user_name TEXT NOT NULL,
            department TEXT NOT NULL,
            comment TEXT NOT NULL,
            comment_type TEXT NOT NULL DEFAULT 'general',
            created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.payroll_forecast_scenarios (
            id SERIAL PRIMARY KEY,
            company_id TEXT NOT NULL,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            scenario_name TEXT NOT NULL,
            multiplier NUMERIC NOT NULL,
            description TEXT NULL,
            parameter_types JSONB NOT NULL DEFAULT '["salary"]',
            customer_scope TEXT NOT NULL DEFAULT 'all',
            created_by_user_id TEXT NOT NULL,
            created_by_name TEXT NOT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW(),
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            UNIQUE (company_id, year, quarter, scenario_name)
        );
    """))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.payroll_forecast_scenario_employee_multipliers (
            id SERIAL PRIMARY KEY,
            scenario_id INT NOT NULL,
            company_id TEXT NOT NULL,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            employee_name TEXT NOT NULL,
            multiplier NUMERIC NOT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW(),
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            UNIQUE (scenario_id, company_id, year, quarter, employee_name)
        );
    """))

    await db.commit()


async def _upsert_payroll_adjustment(
    db: AsyncSession,
    year: int,
    quarter: str,
    company_id: str,
    employee_id: int,
    emp_name: str,
    adj: PayrollAdjustmentInput,
) -> None:
    await db.execute(
        text("""
            INSERT INTO fpna_config.payroll_forecast_adjustments (
                year, quarter, company_id, employee_id, employee_name,
                month1_adjusted_salary, month2_adjusted_salary, month3_adjusted_salary,
                month1_adjusted_bonus, month2_adjusted_bonus, month3_adjusted_bonus,
                month1_adjusted_benefits, month2_adjusted_benefits, month3_adjusted_benefits,
                month1_comment, month2_comment, month3_comment,
                overall_comment, updated_at
            ) VALUES (
                :year, :quarter, :company_id, :employee_id, :employee_name,
                :month1_adjusted_salary, :month2_adjusted_salary, :month3_adjusted_salary,
                :month1_adjusted_bonus, :month2_adjusted_bonus, :month3_adjusted_bonus,
                :month1_adjusted_benefits, :month2_adjusted_benefits, :month3_adjusted_benefits,
                :month1_comment, :month2_comment, :month3_comment,
                :overall_comment, NOW()
            )
            ON CONFLICT (year, quarter, company_id, employee_id)
            DO UPDATE SET
                employee_name = EXCLUDED.employee_name,
                month1_adjusted_salary = EXCLUDED.month1_adjusted_salary,
                month2_adjusted_salary = EXCLUDED.month2_adjusted_salary,
                month3_adjusted_salary = EXCLUDED.month3_adjusted_salary,
                month1_adjusted_bonus = EXCLUDED.month1_adjusted_bonus,
                month2_adjusted_bonus = EXCLUDED.month2_adjusted_bonus,
                month3_adjusted_bonus = EXCLUDED.month3_adjusted_bonus,
                month1_adjusted_benefits = EXCLUDED.month1_adjusted_benefits,
                month2_adjusted_benefits = EXCLUDED.month2_adjusted_benefits,
                month3_adjusted_benefits = EXCLUDED.month3_adjusted_benefits,
                month1_comment = EXCLUDED.month1_comment,
                month2_comment = EXCLUDED.month2_comment,
                month3_comment = EXCLUDED.month3_comment,
                overall_comment = EXCLUDED.overall_comment,
                updated_at = NOW()
        """),
        {
            "year": year, "quarter": quarter, "company_id": company_id,
            "employee_id": employee_id, "employee_name": emp_name,
            "month1_adjusted_salary": adj.month1_adjusted_salary,
            "month2_adjusted_salary": adj.month2_adjusted_salary,
            "month3_adjusted_salary": adj.month3_adjusted_salary,
            "month1_adjusted_bonus": adj.month1_adjusted_bonus,
            "month2_adjusted_bonus": adj.month2_adjusted_bonus,
            "month3_adjusted_bonus": adj.month3_adjusted_bonus,
            "month1_adjusted_benefits": adj.month1_adjusted_benefits,
            "month2_adjusted_benefits": adj.month2_adjusted_benefits,
            "month3_adjusted_benefits": adj.month3_adjusted_benefits,
            "month1_comment": adj.month1_comment,
            "month2_comment": adj.month2_comment,
            "month3_comment": adj.month3_comment,
            "overall_comment": adj.overall_comment,
        },
    )


async def _upsert_department_headcount_adjustment(
    db: AsyncSession,
    year: int,
    quarter: str,
    company_id: str,
    dept_name: str,
    adj: DepartmentHeadcountAdjustmentInput,
) -> None:
    await db.execute(
        text("""
            INSERT INTO fpna_config.department_headcount_forecast_adjustments (
                year, quarter, company_id, department_name,
                month1_adjusted_count, month1_comment,
                month2_adjusted_count, month2_comment,
                month3_adjusted_count, month3_comment,
                overall_comment, updated_at
            ) VALUES (
                :year, :quarter, :company_id, :department_name,
                :month1_adjusted_count, :month1_comment,
                :month2_adjusted_count, :month2_comment,
                :month3_adjusted_count, :month3_comment,
                :overall_comment, NOW()
            )
            ON CONFLICT (year, quarter, company_id, department_name)
            DO UPDATE SET
                month1_adjusted_count = EXCLUDED.month1_adjusted_count,
                month1_comment = EXCLUDED.month1_comment,
                month2_adjusted_count = EXCLUDED.month2_adjusted_count,
                month2_comment = EXCLUDED.month2_comment,
                month3_adjusted_count = EXCLUDED.month3_adjusted_count,
                month3_comment = EXCLUDED.month3_comment,
                overall_comment = EXCLUDED.overall_comment,
                updated_at = NOW()
        """),
        {
            "year": year, "quarter": quarter, "company_id": company_id,
            "department_name": dept_name,
            "month1_adjusted_count": adj.month1_adjusted_count,
            "month1_comment": adj.month1_comment,
            "month2_adjusted_count": adj.month2_adjusted_count,
            "month2_comment": adj.month2_comment,
            "month3_adjusted_count": adj.month3_adjusted_count,
            "month3_comment": adj.month3_comment,
            "overall_comment": adj.overall_comment,
        },
    )


async def _get_or_create_payroll_workflow(
    db: AsyncSession, year: int, quarter: str, company_id: str, department: str, user_id: str
) -> Dict[str, Any]:
    row = (await db.execute(
        text("""
            SELECT * FROM fpna_config.payroll_forecast_workflows
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id
            ORDER BY id ASC
            LIMIT 1
        """),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().first()

    if not row:
        row = (await db.execute(
            text("""
                INSERT INTO fpna_config.payroll_forecast_workflows
                    (year, quarter, company_id, department, owner_user_id, stage)
                VALUES (:year, :quarter, :company_id, :dept, :uid, 'draft')
                RETURNING *
            """),
            {"year": year, "quarter": quarter, "company_id": company_id, "dept": department, "uid": user_id},
        )).mappings().first()
        await db.commit()

    return dict(row)


async def _get_payroll_adjustments(
    db: AsyncSession, year: int, quarter: str, company_id: str
) -> List[Dict[str, Any]]:
    rows = (await db.execute(
        text("""
            SELECT
                employee_id,
                employee_name,
                month1_adjusted_salary, month1_adjusted_bonus, month1_adjusted_benefits, month1_comment,
                month2_adjusted_salary, month2_adjusted_bonus, month2_adjusted_benefits, month2_comment,
                month3_adjusted_salary, month3_adjusted_bonus, month3_adjusted_benefits, month3_comment,
                overall_comment
            FROM fpna_config.payroll_forecast_adjustments
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id
            ORDER BY employee_name ASC;
        """),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().all()

    result = []
    for r in rows:
        result.append({
            "employee_id": int(r["employee_id"]),
            "employee_name": r["employee_name"],
            "month1_adjusted_salary": float(r["month1_adjusted_salary"]) if r["month1_adjusted_salary"] is not None else None,
            "month1_adjusted_bonus": float(r["month1_adjusted_bonus"]) if r["month1_adjusted_bonus"] is not None else None,
            "month1_adjusted_benefits": float(r["month1_adjusted_benefits"]) if r["month1_adjusted_benefits"] is not None else None,
            "month1_comment": r["month1_comment"],
            "month2_adjusted_salary": float(r["month2_adjusted_salary"]) if r["month2_adjusted_salary"] is not None else None,
            "month2_adjusted_bonus": float(r["month2_adjusted_bonus"]) if r["month2_adjusted_bonus"] is not None else None,
            "month2_adjusted_benefits": float(r["month2_adjusted_benefits"]) if r["month2_adjusted_benefits"] is not None else None,
            "month2_comment": r["month2_comment"],
            "month3_adjusted_salary": float(r["month3_adjusted_salary"]) if r["month3_adjusted_salary"] is not None else None,
            "month3_adjusted_bonus": float(r["month3_adjusted_bonus"]) if r["month3_adjusted_bonus"] is not None else None,
            "month3_adjusted_benefits": float(r["month3_adjusted_benefits"]) if r["month3_adjusted_benefits"] is not None else None,
            "month3_comment": r["month3_comment"],
            "overall_comment": r["overall_comment"],
        })
    return result


async def _get_department_headcount_adjustments(
    db: AsyncSession, year: int, quarter: str, company_id: str
) -> List[Dict[str, Any]]:
    rows = (await db.execute(
        text("""
            SELECT
                department_name,
                month1_adjusted_count, month1_comment,
                month2_adjusted_count, month2_comment,
                month3_adjusted_count, month3_comment,
                overall_comment
            FROM fpna_config.department_headcount_forecast_adjustments
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id
            ORDER BY department_name ASC;
        """),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().all()

    result = []
    for r in rows:
        result.append({
            "department_name": r["department_name"],
            "month1_adjusted_count": int(r["month1_adjusted_count"]) if r["month1_adjusted_count"] is not None else None,
            "month1_comment": r["month1_comment"],
            "month2_adjusted_count": int(r["month2_adjusted_count"]) if r["month2_adjusted_count"] is not None else None,
            "month2_comment": r["month2_comment"],
            "month3_adjusted_count": int(r["month3_adjusted_count"]) if r["month3_adjusted_count"] is not None else None,
            "month3_comment": r["month3_comment"],
            "overall_comment": r["overall_comment"],
        })
    return result


async def _get_payroll_driver_adjustments(
    db: AsyncSession, year: int, quarter: str, company_id: str
) -> Dict[tuple, Dict]:
    rows = (await db.execute(
        text("""
            SELECT employee_name, month, salary_adjusted, bonus_adjusted, benefits_adjusted,
                   month_comment, overall_comment
            FROM fpna_config.payroll_driver_adjustments
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id;
        """),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().all()

    return {
        (r["employee_name"], r["month"]): {
            "salary_adjusted": float(r["salary_adjusted"]) if r["salary_adjusted"] is not None else None,
            "bonus_adjusted": float(r["bonus_adjusted"]) if r["bonus_adjusted"] is not None else None,
            "benefits_adjusted": float(r["benefits_adjusted"]) if r["benefits_adjusted"] is not None else None,
            "month_comment": r["month_comment"],
            "overall_comment": r["overall_comment"],
        }
        for r in rows
    }


async def _get_payroll_custom_scenarios(
    db: AsyncSession, year: int, quarter: str, company_id: str
) -> List[Dict[str, Any]]:
    rows = (await db.execute(
        text("""
            SELECT id, scenario_name, multiplier, description, parameter_types, customer_scope,
                   created_by_name, created_at, updated_at
            FROM fpna_config.payroll_forecast_scenarios
            WHERE company_id = :company_id AND year = :year AND quarter = :quarter
            ORDER BY created_at ASC
        """),
        {"company_id": company_id, "year": year, "quarter": quarter},
    )).mappings().all()

    return [
        {
            "id": r["id"],
            "scenario_name": r["scenario_name"],
            "multiplier": float(r["multiplier"]),
            "description": r["description"],
            "parameter_types": r["parameter_types"] if r["parameter_types"] else ["salary"],
            "customer_scope": r["customer_scope"] if r["customer_scope"] else "all",
            "created_by_name": r["created_by_name"],
            "created_at": r["created_at"].isoformat(),
            "updated_at": r["updated_at"].isoformat(),
        }
        for r in rows
    ]


async def _get_payroll_scenario_employee_multipliers(
    db: AsyncSession, scenario_id: int, company_id: str, year: int, quarter: str
) -> Dict[str, float]:
    rows = (await db.execute(
        text("""
            SELECT employee_name, multiplier
            FROM fpna_config.payroll_forecast_scenario_employee_multipliers
            WHERE scenario_id = :scenario_id
              AND company_id = :company_id
              AND year = :year
              AND quarter = :quarter
            ORDER BY employee_name ASC
        """),
        {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter},
    )).mappings().all()

    return {r["employee_name"]: float(r["multiplier"]) for r in rows}


def _build_payroll_scenario_summary(
    baseline_ai: float,
    baseline_user: float,
    multiplier: float,
) -> Dict[str, Any]:
    total_ai = round(baseline_ai * multiplier, 2)
    total_user = round(baseline_user * multiplier, 2)
    variance_amount = round(total_user - total_ai, 2)
    variance_percent = round((variance_amount / total_ai) * 100 if total_ai else 0, 2)
    return {
        "total_ai": total_ai,
        "total_user": total_user,
        "variance_amount": variance_amount,
        "variance_percent": variance_percent,
    }


def _build_payroll_scenario_summary_with_employee_multipliers(
    baseline_ai: float,
    baseline_user: float,
    scenario_multiplier: float,
    employee_ai_map: Dict[str, float],
    employee_user_map: Dict[str, float],
    employee_multipliers: Dict[str, float],
    customer_scope: str,
) -> Dict[str, Any]:
    if customer_scope == "all" or not employee_multipliers:
        return _build_payroll_scenario_summary(baseline_ai, baseline_user, scenario_multiplier)

    total_ai = 0.0
    total_user = 0.0

    all_employees = set(list(employee_ai_map.keys()) + list(employee_user_map.keys()))

    for ename in all_employees:
        eff_mult = employee_multipliers.get(ename, scenario_multiplier)
        e_ai = employee_ai_map.get(ename, 0.0)
        e_user = employee_user_map.get(ename, 0.0)
        total_ai += e_ai * eff_mult
        total_user += e_user * eff_mult

    total_ai = round(total_ai, 2)
    total_user = round(total_user, 2)
    variance_amount = round(total_user - total_ai, 2)
    variance_percent = round((variance_amount / total_ai) * 100 if total_ai else 0, 2)

    return {
        "total_ai": total_ai,
        "total_user": total_user,
        "variance_amount": variance_amount,
        "variance_percent": variance_percent,
    }


@forecast_router.patch("/headcount_and_Payroll_forecast/employee-adjustments/{year}")
async def patch_headcount_payroll_employee_adjustments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    adjustments: List[PayrollAdjustmentInput] = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_payroll_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="Forecast is locked. Adjustments cannot be changed.",
        )

    if stage != WorkflowStage.DRAFT:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail=f"Adjustments can only be edited in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.",
        )

    emp_map_sql = """
    SELECT employee_id, first_name || ' ' || last_name AS employee_name
    FROM fpna_landing.hr_employees
    WHERE status = 'Active';
    """
    emp_rows = (await db.execute(text(emp_map_sql))).mappings().all()
    name_to_ids: Dict[str, List[int]] = defaultdict(list)
    for r in emp_rows:
        name_to_ids[str(r["employee_name"])].append(int(r["employee_id"]))

    saved_count = 0
    skipped = []

    for adj in adjustments:
        emp_name = (adj.employee_name or "").strip()
        if not emp_name:
            continue

        ids = name_to_ids.get(emp_name, [])
        if len(ids) == 0:
            skipped.append({"employee_name": emp_name, "reason": "Employee not found"})
            continue
        if len(ids) > 1:
            skipped.append({"employee_name": emp_name, "reason": "Duplicate employee name"})
            continue

        employee_id = ids[0]
        await _upsert_payroll_adjustment(db, year, quarter.value, company_id, employee_id, emp_name, adj)
        saved_count += 1

    await db.commit()

    final = await _get_payroll_adjustments(db, year, quarter.value, company_id)

    return {
        "detail": "Payroll forecast adjustments saved successfully",
        "year": year,
        "quarter": quarter.value,
        "saved": saved_count,
        "skipped": skipped,
        "adjustments": final,
    }


@forecast_router.patch("/headcount_and_Payroll_forecast/department-adjustments/{year}")
async def patch_headcount_payroll_department_adjustments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    adjustments: List[DepartmentHeadcountAdjustmentInput] = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_payroll_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="Forecast is locked. Department adjustments cannot be changed.",
        )

    if stage != WorkflowStage.DRAFT:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail=f"Adjustments can only be edited in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.",
        )

    saved_count = 0

    for adj in adjustments:
        dept_name = (adj.department_name or "").strip()
        if not dept_name:
            continue

        await _upsert_department_headcount_adjustment(db, year, quarter.value, company_id, dept_name, adj)
        saved_count += 1

    await db.commit()

    final = await _get_department_headcount_adjustments(db, year, quarter.value, company_id)

    return {
        "detail": "Department headcount adjustments saved successfully",
        "year": year,
        "quarter": quarter.value,
        "saved": saved_count,
        "adjustments": final,
    }


@forecast_router.patch("/headcount_and_Payroll_forecast/driver-adjustments/{year}")
async def patch_payroll_driver_adjustments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    adjustments: List[PayrollDriverAdjustmentInput] = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_payroll_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="Forecast is locked. Driver adjustments cannot be changed.",
        )

    if stage != WorkflowStage.DRAFT:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail=f"Driver adjustments can only be edited in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.",
        )

    for adj in adjustments:
        if not adj.employee_name or not adj.month:
            continue
        await db.execute(
            text("""
                INSERT INTO fpna_config.payroll_driver_adjustments (
                    year, quarter, company_id, employee_name, month,
                    salary_adjusted, bonus_adjusted, benefits_adjusted,
                    month_comment, overall_comment, updated_at
                ) VALUES (
                    :year, :quarter, :company_id, :employee_name, :month,
                    :salary_adjusted, :bonus_adjusted, :benefits_adjusted,
                    :month_comment, :overall_comment, NOW()
                )
                ON CONFLICT (year, quarter, company_id, employee_name, month)
                DO UPDATE SET
                    salary_adjusted = EXCLUDED.salary_adjusted,
                    bonus_adjusted = EXCLUDED.bonus_adjusted,
                    benefits_adjusted = EXCLUDED.benefits_adjusted,
                    month_comment = EXCLUDED.month_comment,
                    overall_comment = EXCLUDED.overall_comment,
                    updated_at = NOW()
            """),
            {
                "year": year, "quarter": quarter.value, "company_id": company_id,
                "employee_name": adj.employee_name, "month": adj.month,
                "salary_adjusted": adj.salary_adjusted,
                "bonus_adjusted": adj.bonus_adjusted,
                "benefits_adjusted": adj.benefits_adjusted,
                "month_comment": adj.month_comment,
                "overall_comment": adj.overall_comment,
            },
        )
    await db.commit()

    saved = (await db.execute(
        text("""
            SELECT employee_name, month, salary_adjusted, bonus_adjusted, benefits_adjusted,
                   month_comment, overall_comment
            FROM fpna_config.payroll_driver_adjustments
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id
            ORDER BY employee_name, month;
        """),
        {"year": year, "quarter": quarter.value, "company_id": company_id},
    )).mappings().all()

    employee_quarter_map: Dict[str, float] = {}
    result_rows = []
    for r in saved:
        s = float(r["salary_adjusted"]) if r["salary_adjusted"] is not None else None
        b = float(r["bonus_adjusted"]) if r["bonus_adjusted"] is not None else None
        be = float(r["benefits_adjusted"]) if r["benefits_adjusted"] is not None else None
        total_adj = round((s or 0.0) + (b or 0.0) + (be or 0.0), 2) if any(v is not None for v in [s, b, be]) else None
        ename = r["employee_name"]
        if total_adj is not None:
            employee_quarter_map[ename] = employee_quarter_map.get(ename, 0.0) + total_adj
        result_rows.append({
            "employee_name": ename,
            "month": r["month"],
            "salary_adjusted": s,
            "bonus_adjusted": b,
            "benefits_adjusted": be,
            "total_cost_adjusted": total_adj,
            "month_comment": r["month_comment"],
            "overall_comment": r["overall_comment"],
        })

    for row in result_rows:
        row["quarter_total_cost_adjusted"] = round(employee_quarter_map.get(row["employee_name"], 0.0), 2)

    return {
        "detail": "Driver adjustments saved successfully",
        "year": year,
        "quarter": quarter.value,
        "count": len(adjustments),
        "driver_adjustments": result_rows,
    }


@forecast_router.patch("/headcount_and_Payroll_forecast/workflow/{year}")
async def patch_payroll_forecast_workflow(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: PayrollWorkflowTransitionRequest = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_payroll_workflow(db, year, quarter.value, company_id, dept, user_id)
    current_stage = wf["stage"]
    target_stage = body.target_stage.value

    effective_comment = body.comment.strip() if body.comment and body.comment.strip() else None

    is_rejection = target_stage == WorkflowStage.DRAFT and current_stage != WorkflowStage.DRAFT
    if is_rejection and not effective_comment:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="A comment is required when rejecting (sending back to draft). Please provide a reason.",
        )

    if effective_comment and dept not in DEPT_REVIEWERS | DEPT_APPROVERS:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="Only Finance/Executive/CEO/CFO can add review comments.",
        )

    if not can_payroll_transition(dept, current_stage, target_stage):
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail=f"Department '{dept}' cannot move forecast from '{current_stage}' to '{target_stage}'.",
        )

    if target_stage == WorkflowStage.LOCKED:
        await db.execute(
            text("""
                UPDATE fpna_config.payroll_forecast_workflows
                SET stage = :new_stage, updated_at = NOW(), locked_at = NOW()
                WHERE id = :wf_id
            """),
            {"new_stage": target_stage, "wf_id": wf["id"]},
        )
    else:
        await db.execute(
            text("""
                UPDATE fpna_config.payroll_forecast_workflows
                SET stage = :new_stage, updated_at = NOW()
                WHERE id = :wf_id
            """),
            {"new_stage": target_stage, "wf_id": wf["id"]},
        )

    await db.execute(
        text("""
            INSERT INTO fpna_config.payroll_forecast_workflow_audit
                (workflow_id, from_stage, to_stage, transitioned_by_user_id, transitioned_by_name, comment)
            VALUES (:wf_id, :from_s, :to_s, :uid, :uname, :comment)
        """),
        {
            "wf_id": wf["id"],
            "from_s": current_stage,
            "to_s": target_stage,
            "uid": user_id,
            "uname": current_user.get("full_name", ""),
            "comment": effective_comment,
        },
    )

    if effective_comment and dept in DEPT_REVIEWERS | DEPT_APPROVERS:
        comment_type = "rejection" if is_rejection else "transition"
        await db.execute(
            text("""
                INSERT INTO fpna_config.payroll_forecast_workflow_comments
                    (workflow_id, user_id, user_name, department, comment, comment_type)
                VALUES (:wf_id, :uid, :uname, :dept, :comment, :comment_type)
            """),
            {
                "wf_id": wf["id"],
                "uid": user_id,
                "uname": current_user.get("full_name", ""),
                "dept": dept,
                "comment": effective_comment,
                "comment_type": comment_type,
            },
        )

    await db.commit()

    return {
        "detail": f"Forecast moved from '{current_stage}' to '{target_stage}'.",
        "year": year,
        "quarter": quarter.value,
        "department": dept,
        "stage": target_stage,
    }


@forecast_router.post("/headcount_and_Payroll_forecast/{year}/comments")
async def post_payroll_forecast_comment(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: PayrollForecastCommentInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    user_name = current_user.get("full_name", "")

    wf = await _get_or_create_payroll_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="Forecast is locked. Comments cannot be added.",
        )

    comment_text = body.comment.strip() if body.comment else ""
    if not comment_text:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Comment cannot be empty.",
        )

    result = (await db.execute(
        text("""
            INSERT INTO fpna_config.payroll_forecast_workflow_comments
                (workflow_id, user_id, user_name, department, comment, comment_type)
            VALUES (:wf_id, :uid, :uname, :dept, :comment, 'general')
            RETURNING id, user_id, user_name, department, comment, comment_type, created_at
        """),
        {
            "wf_id": wf["id"],
            "uid": user_id,
            "uname": user_name,
            "dept": dept,
            "comment": comment_text,
        },
    )).mappings().first()

    await db.commit()

    return {
        "detail": "Comment posted successfully.",
        "year": year,
        "quarter": quarter.value,
        "workflow_stage": stage,
        "comment": {
            "id": result["id"],
            "user_id": result["user_id"],
            "user_name": result["user_name"],
            "department": result["department"],
            "comment": result["comment"],
            "comment_type": result["comment_type"],
            "created_at": result["created_at"].isoformat(),
        },
    }


@forecast_router.get("/headcount_and_Payroll_forecast/{year}/comments")
async def get_payroll_forecast_comments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    comment_type: Optional[str] = Query(None, description="Filter by comment_type: general, transition, rejection. Leave empty for all."),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_payroll_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]

    if comment_type:
        rows = (await db.execute(
            text("""
                SELECT id, user_id, user_name, department, comment, comment_type, created_at
                FROM fpna_config.payroll_forecast_workflow_comments
                WHERE workflow_id = :wf_id AND comment_type = :comment_type
                ORDER BY created_at ASC
            """),
            {"wf_id": wf["id"], "comment_type": comment_type},
        )).mappings().all()
    else:
        rows = (await db.execute(
            text("""
                SELECT id, user_id, user_name, department, comment, comment_type, created_at
                FROM fpna_config.payroll_forecast_workflow_comments
                WHERE workflow_id = :wf_id
                ORDER BY created_at ASC
            """),
            {"wf_id": wf["id"]},
        )).mappings().all()

    comments = [
        {
            "id": r["id"],
            "user_id": r["user_id"],
            "user_name": r["user_name"],
            "department": r["department"],
            "comment": r["comment"],
            "comment_type": r["comment_type"],
            "created_at": r["created_at"].isoformat(),
        }
        for r in rows
    ]

    return {
        "year": year,
        "quarter": quarter.value,
        "workflow_stage": stage,
        "total_comments": len(comments),
        "filters": {
            "comment_type": comment_type or "all",
        },
        "comments": comments,
    }


@forecast_router.post("/headcount_and_Payroll_forecast/{year}/scenarios")
async def create_payroll_forecast_scenario(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: PayrollForecastScenarioInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    user_name = current_user.get("full_name", "")

    scenario_name_clean = body.scenario_name.strip().lower().replace(" ", "_")

    if scenario_name_clean in PAYROLL_RESERVED_SCENARIO_NAMES:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail=f"'{body.scenario_name}' is a reserved scenario name. Reserved names are: baseline, best_case, worst_case.",
        )

    if not (0.01 <= body.multiplier <= 10.0):
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Multiplier must be between 0.01 and 10.0.",
        )

    parameter_types = body.parameter_types if body.parameter_types else ["salary"]
    customer_scope = body.customer_scope if body.customer_scope else "all"

    existing = (await db.execute(
        text("""
            SELECT id FROM fpna_config.payroll_forecast_scenarios
            WHERE company_id = :company_id AND year = :year AND quarter = :quarter
              AND LOWER(REPLACE(scenario_name, ' ', '_')) = :scenario_name_clean
        """),
        {"company_id": company_id, "year": year, "quarter": quarter.value, "scenario_name_clean": scenario_name_clean},
    )).mappings().first()

    if existing:
        raise HTTPException(
            status_code=status.HTTP_409_CONFLICT,
            detail=f"A scenario named '{body.scenario_name}' already exists for {year} {quarter.value}.",
        )

    result = (await db.execute(
        text("""
            INSERT INTO fpna_config.payroll_forecast_scenarios
                (company_id, year, quarter, scenario_name, multiplier, description,
                 parameter_types, customer_scope, created_by_user_id, created_by_name)
            VALUES (:company_id, :year, :quarter, :scenario_name, :multiplier, :description,
                    :parameter_types, :customer_scope, :uid, :uname)
            RETURNING id, scenario_name, multiplier, description, parameter_types,
                      customer_scope, created_by_name, created_at, updated_at
        """),
        {
            "company_id": company_id,
            "year": year,
            "quarter": quarter.value,
            "scenario_name": body.scenario_name.strip(),
            "multiplier": body.multiplier,
            "description": body.description.strip() if body.description else None,
            "parameter_types": json.dumps(parameter_types),
            "customer_scope": customer_scope,
            "uid": user_id,
            "uname": user_name,
        },
    )).mappings().first()

    await db.commit()

    employee_multipliers_saved = []
    if customer_scope == "specific" and body.customer_multipliers:
        for cm in body.customer_multipliers:
            if not cm.get("employee_name") or cm.get("multiplier") is None:
                continue
            if not (0.01 <= cm["multiplier"] <= 10.0):
                continue
            await db.execute(
                text("""
                    INSERT INTO fpna_config.payroll_forecast_scenario_employee_multipliers
                        (scenario_id, company_id, year, quarter, employee_name, multiplier)
                    VALUES (:scenario_id, :company_id, :year, :quarter, :employee_name, :multiplier)
                    ON CONFLICT (scenario_id, company_id, year, quarter, employee_name)
                    DO UPDATE SET multiplier = EXCLUDED.multiplier, updated_at = NOW()
                """),
                {
                    "scenario_id": result["id"],
                    "company_id": company_id,
                    "year": year,
                    "quarter": quarter.value,
                    "employee_name": cm["employee_name"],
                    "multiplier": cm["multiplier"],
                },
            )
            employee_multipliers_saved.append({"employee_name": cm["employee_name"], "multiplier": cm["multiplier"]})
        await db.commit()

    return {
        "detail": "Scenario created successfully.",
        "year": year,
        "quarter": quarter.value,
        "scenario": {
            "id": result["id"],
            "scenario_name": result["scenario_name"],
            "multiplier": float(result["multiplier"]),
            "description": result["description"],
            "parameter_types": result["parameter_types"] if result["parameter_types"] else ["salary"],
            "customer_scope": result["customer_scope"],
            "employee_multipliers": employee_multipliers_saved,
            "created_by_name": result["created_by_name"],
            "created_at": result["created_at"].isoformat(),
            "updated_at": result["updated_at"].isoformat(),
        },
    }


@forecast_router.get("/headcount_and_Payroll_forecast/{year}/scenarios")
async def get_payroll_forecast_scenarios(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    company_id = get_company_id(current_user)

    custom_scenarios = await _get_payroll_custom_scenarios(db, year, quarter.value, company_id)

    for s in custom_scenarios:
        s["is_system"] = False
        if s["customer_scope"] == "specific":
            s["employee_multipliers"] = await _get_payroll_scenario_employee_multipliers(
                db, s["id"], company_id, year, quarter.value
            )
        else:
            s["employee_multipliers"] = {}

    hardcoded = [
        {
            "id": None,
            "scenario_name": "baseline",
            "multiplier": 1.0,
            "description": PAYROLL_SCENARIO_DEFINITIONS["baseline"],
            "parameter_types": ["salary"],
            "customer_scope": "all",
            "employee_multipliers": {},
            "is_system": True,
            "created_by_name": None,
            "created_at": None,
            "updated_at": None,
        },
        {
            "id": None,
            "scenario_name": "best_case",
            "multiplier": PAYROLL_BEST_CASE_MULTIPLIER,
            "description": PAYROLL_SCENARIO_DEFINITIONS["best_case"],
            "parameter_types": ["salary", "bonus"],
            "customer_scope": "all",
            "employee_multipliers": {},
            "is_system": True,
            "created_by_name": None,
            "created_at": None,
            "updated_at": None,
        },
        {
            "id": None,
            "scenario_name": "worst_case",
            "multiplier": PAYROLL_WORST_CASE_MULTIPLIER,
            "description": PAYROLL_SCENARIO_DEFINITIONS["worst_case"],
            "parameter_types": ["headcount", "salary"],
            "customer_scope": "all",
            "employee_multipliers": {},
            "is_system": True,
            "created_by_name": None,
            "created_at": None,
            "updated_at": None,
        },
    ]

    return {
        "year": year,
        "quarter": quarter.value,
        "total_scenarios": len(hardcoded) + len(custom_scenarios),
        "scenarios": hardcoded + custom_scenarios,
    }


@forecast_router.patch("/headcount_and_Payroll_forecast/{year}/scenarios/{scenario_id}")
async def update_payroll_forecast_scenario(
    year: int = Path(...),
    scenario_id: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: PayrollForecastScenarioUpdateInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    company_id = get_company_id(current_user)

    existing = (await db.execute(
        text("""
            SELECT id, scenario_name FROM fpna_config.payroll_forecast_scenarios
            WHERE id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter
        """),
        {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
    )).mappings().first()

    if not existing:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail=f"Scenario with id {scenario_id} not found for {year} {quarter.value}.",
        )

    if body.scenario_name is not None:
        new_name_clean = body.scenario_name.strip().lower().replace(" ", "_")
        if new_name_clean in PAYROLL_RESERVED_SCENARIO_NAMES:
            raise HTTPException(
                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                detail=f"'{body.scenario_name}' is a reserved scenario name. Reserved names are: baseline, best_case, worst_case.",
            )
        name_conflict = (await db.execute(
            text("""
                SELECT id FROM fpna_config.payroll_forecast_scenarios
                WHERE company_id = :company_id AND year = :year AND quarter = :quarter
                  AND LOWER(REPLACE(scenario_name, ' ', '_')) = :new_name_clean
                  AND id != :sid
            """),
            {"company_id": company_id, "year": year, "quarter": quarter.value, "new_name_clean": new_name_clean, "sid": scenario_id},
        )).mappings().first()
        if name_conflict:
            raise HTTPException(
                status_code=status.HTTP_409_CONFLICT,
                detail=f"A scenario named '{body.scenario_name}' already exists for {year} {quarter.value}.",
            )

    if body.multiplier is not None and not (0.01 <= body.multiplier <= 10.0):
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Multiplier must be between 0.01 and 10.0.",
        )

    set_clauses = ["updated_at = NOW()"]
    params: Dict[str, Any] = {"sid": scenario_id}

    if body.scenario_name is not None:
        set_clauses.append("scenario_name = :scenario_name")
        params["scenario_name"] = body.scenario_name.strip()
    if body.multiplier is not None:
        set_clauses.append("multiplier = :multiplier")
        params["multiplier"] = body.multiplier
    if body.description is not None:
        set_clauses.append("description = :description")
        params["description"] = body.description.strip()
    if body.parameter_types is not None:
        set_clauses.append("parameter_types = :parameter_types")
        params["parameter_types"] = json.dumps(body.parameter_types)
    if body.customer_scope is not None:
        set_clauses.append("customer_scope = :customer_scope")
        params["customer_scope"] = body.customer_scope

    result = (await db.execute(
        text(f"""
            UPDATE fpna_config.payroll_forecast_scenarios
            SET {', '.join(set_clauses)}
            WHERE id = :sid
            RETURNING id, scenario_name, multiplier, description, parameter_types,
                      customer_scope, created_by_name, created_at, updated_at
        """),
        params,
    )).mappings().first()

    await db.commit()

    new_scope = result["customer_scope"] if result["customer_scope"] else "all"
    employee_multipliers_saved = {}

    if new_scope == "specific" and body.customer_multipliers is not None:
        await db.execute(
            text("""
                DELETE FROM fpna_config.payroll_forecast_scenario_employee_multipliers
                WHERE scenario_id = :scenario_id AND company_id = :company_id
                  AND year = :year AND quarter = :quarter
            """),
            {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
        )
        for cm in body.customer_multipliers:
            if not cm.get("employee_name") or cm.get("multiplier") is None:
                continue
            if not (0.01 <= cm["multiplier"] <= 10.0):
                continue
            await db.execute(
                text("""
                    INSERT INTO fpna_config.payroll_forecast_scenario_employee_multipliers
                        (scenario_id, company_id, year, quarter, employee_name, multiplier)
                    VALUES (:scenario_id, :company_id, :year, :quarter, :employee_name, :multiplier)
                    ON CONFLICT (scenario_id, company_id, year, quarter, employee_name)
                    DO UPDATE SET multiplier = EXCLUDED.multiplier, updated_at = NOW()
                """),
                {
                    "scenario_id": scenario_id,
                    "company_id": company_id,
                    "year": year,
                    "quarter": quarter.value,
                    "employee_name": cm["employee_name"],
                    "multiplier": cm["multiplier"],
                },
            )
            employee_multipliers_saved[cm["employee_name"]] = cm["multiplier"]
        await db.commit()
    elif new_scope == "all":
        await db.execute(
            text("""
                DELETE FROM fpna_config.payroll_forecast_scenario_employee_multipliers
                WHERE scenario_id = :scenario_id AND company_id = :company_id
                  AND year = :year AND quarter = :quarter
            """),
            {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
        )
        await db.commit()
    else:
        employee_multipliers_saved = await _get_payroll_scenario_employee_multipliers(
            db, scenario_id, company_id, year, quarter.value
        )

    return {
        "detail": "Scenario updated successfully.",
        "year": year,
        "quarter": quarter.value,
        "scenario": {
            "id": result["id"],
            "scenario_name": result["scenario_name"],
            "multiplier": float(result["multiplier"]),
            "description": result["description"],
            "parameter_types": result["parameter_types"] if result["parameter_types"] else ["salary"],
            "customer_scope": result["customer_scope"],
            "employee_multipliers": employee_multipliers_saved,
            "created_by_name": result["created_by_name"],
            "created_at": result["created_at"].isoformat(),
            "updated_at": result["updated_at"].isoformat(),
        },
    }


@forecast_router.delete("/headcount_and_Payroll_forecast/{year}/scenarios/{scenario_id}")
async def delete_payroll_forecast_scenario(
    year: int = Path(...),
    scenario_id: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    company_id = get_company_id(current_user)

    existing = (await db.execute(
        text("""
            SELECT id, scenario_name FROM fpna_config.payroll_forecast_scenarios
            WHERE id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter
        """),
        {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
    )).mappings().first()

    if not existing:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail=f"Scenario with id {scenario_id} not found for {year} {quarter.value}.",
        )

    await db.execute(
        text("""
            DELETE FROM fpna_config.payroll_forecast_scenario_employee_multipliers
            WHERE scenario_id = :sid AND company_id = :company_id
              AND year = :year AND quarter = :quarter
        """),
        {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
    )

    await db.execute(
        text("DELETE FROM fpna_config.payroll_forecast_scenarios WHERE id = :sid"),
        {"sid": scenario_id},
    )
    await db.commit()

    return {
        "detail": f"Scenario '{existing['scenario_name']}' deleted successfully.",
        "year": year,
        "quarter": quarter.value,
        "deleted_scenario_id": scenario_id,
    }


@forecast_router.get("/headcount_and_Payroll_forecast/{year}/excel-template/employee-adjustments")
async def download_payroll_employee_adjustments_template(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    company_id = get_company_id(current_user)

    start_date, end_date = quarter_date_range(year, quarter)
    prev_month_first = get_previous_month_first_day(start_date)
    prev_month_last = get_previous_month_last_day(prev_month_first)
    prev_month_str = prev_month_first.isoformat()

    months = pd.date_range(start=start_date, end=end_date, freq="MS")

    saved_adjustments = await _get_payroll_adjustments(db, year, quarter.value, company_id)
    adj_map = {a["employee_id"]: a for a in saved_adjustments}

    result = await db.execute(text(RAW_SQL))
    payroll_rows = result.mappings().all()
    df = pd.DataFrame(payroll_rows)

    if df.empty:
        wb = openpyxl.Workbook()
        ws = wb.active
        ws.title = "Employee Adjustments"
        ws.append(["No active employees found."])
        stream = io.BytesIO()
        wb.save(stream)
        stream.seek(0)
        return StreamingResponse(
            stream,
            media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
            headers={"Content-Disposition": f"attachment; filename=payroll_employee_adjustments_{year}_{quarter.value}.xlsx"},
        )

    df["effective_date"] = pd.to_datetime(df["effective_date"])
    df["hire_date"] = pd.to_datetime(df["hire_date"])
    df["base_salary"] = df["base_salary"].fillna(0).astype(float)
    df["bonus_target"] = df["bonus_target"].fillna(0).astype(float)

    employees = df[["employee_id", "employee", "department", "role", "hire_date"]].drop_duplicates()
    df2 = df.drop(columns=["employee", "department", "role", "hire_date"])
    df2["effective_month"] = df2["effective_date"].dt.to_period("M").dt.to_timestamp()

    all_months = pd.date_range(start=prev_month_first, end=end_date, freq="MS")
    grid = (
        employees.assign(key=1)
        .merge(pd.DataFrame({"month": all_months, "key": 1}), on="key")
        .drop("key", axis=1)
    )
    merged = grid.merge(df2, on="employee_id", how="left")
    merged = merged[merged["effective_month"].isna() | (merged["effective_month"] <= merged["month"])]
    merged = merged.sort_values("effective_month").groupby(["employee_id", "month"]).tail(1)

    merged["salary"] = merged["base_salary"]
    merged["bonus"] = merged["bonus_target"]
    merged["benefits"] = merged["salary"] * 0.20
    merged.loc[merged["month"] < merged["hire_date"], ["salary", "bonus", "benefits"]] = 0

    out = []
    for _, emp_df in merged.groupby("employee_id"):
        emp_df = emp_df.sort_values("month").copy()
        n = len(emp_df)
        forecasted_salary = forecast_series_advanced(emp_df["salary"], n)
        forecasted_bonus = forecast_series_advanced(emp_df["bonus"], n)
        emp_df["salary"] = forecasted_salary
        emp_df["bonus"] = forecasted_bonus
        emp_df["benefits"] = emp_df["salary"] * 0.20
        out.append(emp_df)

    final_df = pd.concat(out, ignore_index=True)

    month_strs = [m.date().isoformat() for m in months]
    month_str_to_index = {m: i + 1 for i, m in enumerate(month_strs)}

    wb = openpyxl.Workbook()
    ws = wb.active
    ws.title = "Employee Adjustments"

    headers = [
        "Employee Name", "Department", "Role",
        f"Previous Month ({prev_month_str}) Salary",
        f"Previous Month ({prev_month_str}) Bonus",
        f"Previous Month ({prev_month_str}) Benefits",
    ]
    for i, m in enumerate(months, start=1):
        ms = m.date().isoformat()
        headers += [
            f"{ms} AI Salary", f"{ms} Adjusted Salary", f"{ms} Comment Salary",
            f"{ms} AI Bonus", f"{ms} Adjusted Bonus",
            f"{ms} AI Benefits", f"{ms} Adjusted Benefits",
        ]
    headers += ["Overall Comment"]
    ws.append(headers)

    for emp_id, emp_df in final_df.groupby("employee_id"):
        emp_df = emp_df.sort_values("month")
        rows_list = emp_df.to_dict("records")
        if not rows_list:
            continue

        first = rows_list[0]
        emp_name = first["employee"]
        dept_name = first["department"]
        role = first["role"]
        adj = adj_map.get(int(emp_id), {})

        prev_salary = 0.0
        prev_bonus = 0.0
        prev_benefits = 0.0

        month_values: Dict[str, Dict] = {}

        for row in rows_list:
            month_dt = pd.Timestamp(row["month"])
            ms = month_dt.date().isoformat()
            salary_ai = round(float(row["salary"]), 2)
            bonus_ai = round(float(row["bonus"]), 2)
            benefits_ai = round(float(row["benefits"]), 2)

            if ms == prev_month_str:
                prev_salary = salary_ai
                prev_bonus = bonus_ai
                prev_benefits = benefits_ai
            elif ms in month_str_to_index:
                idx = month_str_to_index[ms]
                month_values[ms] = {
                    "salary_ai": salary_ai,
                    "bonus_ai": bonus_ai,
                    "benefits_ai": benefits_ai,
                    "salary_adj": adj.get(f"month{idx}_adjusted_salary"),
                    "bonus_adj": adj.get(f"month{idx}_adjusted_bonus"),
                    "benefits_adj": adj.get(f"month{idx}_adjusted_benefits"),
                    "comment": adj.get(f"month{idx}_comment"),
                }

        row_data = [emp_name, dept_name, role, prev_salary, prev_bonus, prev_benefits]
        for m in months:
            ms = m.date().isoformat()
            mv = month_values.get(ms, {})
            row_data += [
                mv.get("salary_ai"), mv.get("salary_adj"), mv.get("comment"),
                mv.get("bonus_ai"), mv.get("bonus_adj"),
                mv.get("benefits_ai"), mv.get("benefits_adj"),
            ]
        row_data.append(adj.get("overall_comment"))
        ws.append(row_data)

    stream = io.BytesIO()
    wb.save(stream)
    stream.seek(0)

    return StreamingResponse(
        stream,
        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        headers={"Content-Disposition": f"attachment; filename=payroll_employee_adjustments_{year}_{quarter.value}.xlsx"},
    )


@forecast_router.get("/headcount_and_Payroll_forecast/{year}/excel-template/driver-adjustments")
async def download_payroll_driver_adjustments_template(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    company_id = get_company_id(current_user)

    start_date, end_date = quarter_date_range(year, quarter)
    prev_month_first = get_previous_month_first_day(start_date)
    prev_month_last = get_previous_month_last_day(prev_month_first)
    prev_month_str = prev_month_first.isoformat()

    months = pd.date_range(start=start_date, end=end_date, freq="MS")
    month_strs = [m.date().isoformat() for m in months]

    driver_saved = await _get_payroll_driver_adjustments(db, year, quarter.value, company_id)

    result = await db.execute(text(RAW_SQL))
    payroll_rows = result.mappings().all()
    df = pd.DataFrame(payroll_rows)

    if df.empty:
        wb = openpyxl.Workbook()
        ws = wb.active
        ws.title = "Driver Adjustments"
        ws.append(["No active employees found."])
        stream = io.BytesIO()
        wb.save(stream)
        stream.seek(0)
        return StreamingResponse(
            stream,
            media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
            headers={"Content-Disposition": f"attachment; filename=payroll_driver_adjustments_{year}_{quarter.value}.xlsx"},
        )

    df["effective_date"] = pd.to_datetime(df["effective_date"])
    df["hire_date"] = pd.to_datetime(df["hire_date"])
    df["base_salary"] = df["base_salary"].fillna(0).astype(float)
    df["bonus_target"] = df["bonus_target"].fillna(0).astype(float)

    employees = df[["employee_id", "employee", "department", "role", "hire_date"]].drop_duplicates()
    df2 = df.drop(columns=["employee", "department", "role", "hire_date"])
    df2["effective_month"] = df2["effective_date"].dt.to_period("M").dt.to_timestamp()

    all_months = pd.date_range(start=prev_month_first, end=end_date, freq="MS")
    grid = (
        employees.assign(key=1)
        .merge(pd.DataFrame({"month": all_months, "key": 1}), on="key")
        .drop("key", axis=1)
    )
    merged = grid.merge(df2, on="employee_id", how="left")
    merged = merged[merged["effective_month"].isna() | (merged["effective_month"] <= merged["month"])]
    merged = merged.sort_values("effective_month").groupby(["employee_id", "month"]).tail(1)
    merged["salary"] = merged["base_salary"]
    merged["bonus"] = merged["bonus_target"]
    merged["benefits"] = merged["salary"] * 0.20
    merged.loc[merged["month"] < merged["hire_date"], ["salary", "bonus", "benefits"]] = 0

    out = []
    for _, emp_df in merged.groupby("employee_id"):
        emp_df = emp_df.sort_values("month").copy()
        n = len(emp_df)
        forecasted_salary = forecast_series_advanced(emp_df["salary"], n)
        forecasted_bonus = forecast_series_advanced(emp_df["bonus"], n)
        emp_df["salary"] = forecasted_salary
        emp_df["bonus"] = forecasted_bonus
        emp_df["benefits"] = emp_df["salary"] * 0.20
        out.append(emp_df)

    final_df = pd.concat(out, ignore_index=True)

    wb = openpyxl.Workbook()
    ws = wb.active
    ws.title = "Driver Adjustments"

    headers = [
        "Employee Name", "Month", f"Previous Month ({prev_month_str}) Salary",
        "Salary AI", "Bonus AI", "Benefits AI", "Total Cost AI",
        "Salary Adjusted", "Bonus Adjusted", "Benefits Adjusted", "Total Cost Adjusted",
        "Month Comment", "Quarter Total AI", "Quarter Total Adjusted", "Overall Comment",
    ]
    ws.append(headers)

    employee_quarter_ai: Dict[str, float] = {}
    employee_quarter_adj: Dict[str, float] = {}

    for emp_id, emp_df in final_df.groupby("employee_id"):
        emp_df = emp_df.sort_values("month")
        rows_list = emp_df.to_dict("records")
        if not rows_list:
            continue

        first = rows_list[0]
        emp_name = first["employee"]

        q_ai = 0.0
        q_adj = 0.0
        prev_salary = 0.0

        for row in rows_list:
            month_dt = pd.Timestamp(row["month"])
            ms = month_dt.date().isoformat()
            salary_ai = round(float(row["salary"]), 2)
            bonus_ai = round(float(row["bonus"]), 2)
            benefits_ai = round(float(row["benefits"]), 2)
            total_ai = round(salary_ai + bonus_ai + benefits_ai, 2)

            if ms == prev_month_str:
                prev_salary = salary_ai
                continue

            if ms not in month_strs:
                continue

            drv = driver_saved.get((emp_name, ms), {})
            s_adj = drv.get("salary_adjusted")
            b_adj = drv.get("bonus_adjusted")
            be_adj = drv.get("benefits_adjusted")

            salary_eff = s_adj if s_adj is not None else salary_ai
            bonus_eff = b_adj if b_adj is not None else bonus_ai
            benefits_eff = be_adj if be_adj is not None else benefits_ai
            total_adj = round(salary_eff + bonus_eff + benefits_eff, 2)

            q_ai += total_ai
            q_adj += total_adj

        employee_quarter_ai[emp_name] = round(q_ai, 2)
        employee_quarter_adj[emp_name] = round(q_adj, 2)

    for emp_id, emp_df in final_df.groupby("employee_id"):
        emp_df = emp_df.sort_values("month")
        rows_list = emp_df.to_dict("records")
        if not rows_list:
            continue

        first = rows_list[0]
        emp_name = first["employee"]
        prev_salary = 0.0

        for row in rows_list:
            month_dt = pd.Timestamp(row["month"])
            ms = month_dt.date().isoformat()
            salary_ai = round(float(row["salary"]), 2)
            bonus_ai = round(float(row["bonus"]), 2)
            benefits_ai = round(float(row["benefits"]), 2)
            total_ai = round(salary_ai + bonus_ai + benefits_ai, 2)

            if ms == prev_month_str:
                prev_salary = salary_ai
                continue

            if ms not in month_strs:
                continue

            drv = driver_saved.get((emp_name, ms), {})
            s_adj = drv.get("salary_adjusted")
            b_adj = drv.get("bonus_adjusted")
            be_adj = drv.get("benefits_adjusted")

            salary_eff = s_adj if s_adj is not None else salary_ai
            bonus_eff = b_adj if b_adj is not None else bonus_ai
            benefits_eff = be_adj if be_adj is not None else benefits_ai
            total_adj = round(salary_eff + bonus_eff + benefits_eff, 2)

            ws.append([
                emp_name, ms, prev_salary,
                salary_ai, bonus_ai, benefits_ai, total_ai,
                s_adj, b_adj, be_adj, total_adj if any(v is not None for v in [s_adj, b_adj, be_adj]) else None,
                drv.get("month_comment"),
                employee_quarter_ai.get(emp_name, 0.0),
                employee_quarter_adj.get(emp_name) if employee_quarter_adj.get(emp_name) else None,
                drv.get("overall_comment"),
            ])

    stream = io.BytesIO()
    wb.save(stream)
    stream.seek(0)

    return StreamingResponse(
        stream,
        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        headers={"Content-Disposition": f"attachment; filename=payroll_driver_adjustments_{year}_{quarter.value}.xlsx"},
    )


@forecast_router.post("/headcount_and_Payroll_forecast/{year}/excel-upload/employee-adjustments")
async def upload_payroll_employee_adjustments_excel(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    file: UploadFile = File(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_payroll_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Forecast is locked. Uploads cannot be processed.")

    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Uploads can only be processed in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")

    contents = await file.read()
    if len(contents) > 5 * 1024 * 1024:
        raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File size exceeds 5MB limit.")

    try:
        wb = openpyxl.load_workbook(io.BytesIO(contents))
    except Exception:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Could not read the uploaded file. Please ensure it is a valid .xlsx file.")

    sheet_name = "Employee Adjustments"
    if sheet_name not in wb.sheetnames:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Sheet '{sheet_name}' not found in the uploaded file.")

    start_date, end_date = quarter_date_range(year, quarter)
    months = pd.date_range(start=start_date, end=end_date, freq="MS")
    num_months = len(months)

    ws_adj = wb[sheet_name]
    adj_headers_row = [str(c.value).strip() if c.value is not None else "" for c in ws_adj[1]]
    adj_hmap = {h: i for i, h in enumerate(adj_headers_row)}

    def get_float_cell(row, idx):
        if idx is None or idx >= len(row) or row[idx] is None:
            return None
        try:
            return float(row[idx])
        except (TypeError, ValueError):
            return None

    def get_str_cell(row, idx):
        if idx is None or idx >= len(row) or row[idx] is None:
            return None
        val = str(row[idx]).strip()
        return val if val else None

    adjustments = []
    errors = []

    for row_idx, row in enumerate(ws_adj.iter_rows(min_row=2, values_only=True), start=2):
        if not row or row[0] is None:
            continue
        emp_name = str(row[0]).strip()
        if not emp_name:
            continue

        try:
            adj_item = PayrollAdjustmentInput(employee_name=emp_name)
            for i, m in enumerate(months, start=1):
                ms = m.date().isoformat()
                salary_adj_idx = adj_hmap.get(f"{ms} Adjusted Salary")
                bonus_adj_idx = adj_hmap.get(f"{ms} Adjusted Bonus")
                benefits_adj_idx = adj_hmap.get(f"{ms} Adjusted Benefits")
                comment_idx = adj_hmap.get(f"{ms} Comment Salary")
                adj_item.__setattr__(f"month{i}_adjusted_salary", get_float_cell(row, salary_adj_idx))
                adj_item.__setattr__(f"month{i}_adjusted_bonus", get_float_cell(row, bonus_adj_idx))
                adj_item.__setattr__(f"month{i}_adjusted_benefits", get_float_cell(row, benefits_adj_idx))
                adj_item.__setattr__(f"month{i}_comment", get_str_cell(row, comment_idx))
            overall_idx = adj_hmap.get("Overall Comment")
            adj_item.overall_comment = get_str_cell(row, overall_idx)
        except Exception:
            errors.append(f"Employee Adjustments row {row_idx}: parse error for '{emp_name}'. Skipped.")
            continue

        has_data = any(
            getattr(adj_item, f"month{i}_adjusted_salary") is not None
            or getattr(adj_item, f"month{i}_adjusted_bonus") is not None
            or getattr(adj_item, f"month{i}_adjusted_benefits") is not None
            or getattr(adj_item, f"month{i}_comment") is not None
            for i in range(1, num_months + 1)
        ) or adj_item.overall_comment is not None

        if not has_data:
            for i, m in enumerate(months, start=1):
                ms = m.date().isoformat()
                salary_ai_idx = adj_hmap.get(f"{ms} AI Salary")
                if salary_ai_idx is not None and get_float_cell(row, salary_ai_idx) is not None:
                    has_data = True
                    break

        if has_data:
            adjustments.append(adj_item)

    if not adjustments:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="No valid rows found in the Employee Adjustments sheet.")

    emp_map_sql = """
    SELECT employee_id, first_name || ' ' || last_name AS employee_name
    FROM fpna_landing.hr_employees
    WHERE status = 'Active';
    """
    emp_rows = (await db.execute(text(emp_map_sql))).mappings().all()
    name_to_ids: Dict[str, List[int]] = defaultdict(list)
    for r in emp_rows:
        name_to_ids[str(r["employee_name"])].append(int(r["employee_id"]))

    saved_count = 0
    skipped = []

    for adj in adjustments:
        emp_name = (adj.employee_name or "").strip()
        ids = name_to_ids.get(emp_name, [])
        if len(ids) == 0:
            skipped.append({"employee_name": emp_name, "reason": "Employee not found"})
            continue
        if len(ids) > 1:
            skipped.append({"employee_name": emp_name, "reason": "Duplicate employee name"})
            continue
        employee_id = ids[0]

        await _upsert_payroll_adjustment(db, year, quarter.value, company_id, employee_id, emp_name, adj)
        saved_count += 1

    await db.commit()

    final = await _get_payroll_adjustments(db, year, quarter.value, company_id)

    return {
        "detail": "Employee adjustments uploaded successfully",
        "year": year,
        "quarter": quarter.value,
        "upload_errors": errors,
        "rows_attempted": len(adjustments) + len(errors),
        "rows_saved": saved_count,
        "skipped": skipped,
        "adjustments": final,
    }


@forecast_router.post("/headcount_and_Payroll_forecast/{year}/excel-upload/driver-adjustments")
async def upload_payroll_driver_adjustments_excel(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    file: UploadFile = File(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_payroll_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_payroll_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Forecast is locked. Uploads cannot be processed.")

    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Uploads can only be processed in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")

    contents = await file.read()
    if len(contents) > 5 * 1024 * 1024:
        raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File size exceeds 5MB limit.")

    try:
        wb = openpyxl.load_workbook(io.BytesIO(contents))
    except Exception:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Could not read the uploaded file. Please ensure it is a valid .xlsx file.")

    sheet_name = "Driver Adjustments"
    if sheet_name not in wb.sheetnames:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Sheet '{sheet_name}' not found in the uploaded file.")

    ws_drv = wb[sheet_name]
    drv_headers_row = [str(c.value).strip() if c.value is not None else "" for c in ws_drv[1]]
    drv_hmap = {h: i for i, h in enumerate(drv_headers_row)}

    def get_float_cell(row, idx):
        if idx is None or idx >= len(row) or row[idx] is None:
            return None
        try:
            return float(row[idx])
        except (TypeError, ValueError):
            return None

    def get_str_cell(row, idx):
        if idx is None or idx >= len(row) or row[idx] is None:
            return None
        val = str(row[idx]).strip()
        return val if val else None

    driver_adjustments = []
    errors = []

    for row_idx, row in enumerate(ws_drv.iter_rows(min_row=2, values_only=True), start=2):
        if not row or row[0] is None:
            continue
        emp_name = str(row[0]).strip()
        if not emp_name:
            continue
        try:
            month_val = get_str_cell(row, drv_hmap.get("Month"))
            salary_adj = get_float_cell(row, drv_hmap.get("Salary Adjusted"))
            bonus_adj = get_float_cell(row, drv_hmap.get("Bonus Adjusted"))
            benefits_adj = get_float_cell(row, drv_hmap.get("Benefits Adjusted"))
            month_cmt = get_str_cell(row, drv_hmap.get("Month Comment"))
            overall_comment = get_str_cell(row, drv_hmap.get("Overall Comment"))
        except Exception:
            errors.append(f"Driver Adjustments row {row_idx}: parse error for '{emp_name}'. Skipped.")
            continue

        if not month_val:
            continue

        salary_ai_val = get_float_cell(row, drv_hmap.get("Salary AI"))
        has_data = any(v is not None for v in [salary_adj, bonus_adj, benefits_adj, month_cmt, overall_comment]) or salary_ai_val is not None

        if has_data:
            driver_adjustments.append(PayrollDriverAdjustmentInput(
                employee_name=emp_name, month=month_val,
                salary_adjusted=salary_adj, bonus_adjusted=bonus_adj,
                benefits_adjusted=benefits_adj,
                month_comment=month_cmt, overall_comment=overall_comment,
            ))

    if not driver_adjustments:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="No valid rows found in the Driver Adjustments sheet.")

    for adj in driver_adjustments:
        if not adj.employee_name or not adj.month:
            continue
        await db.execute(
            text("""
                INSERT INTO fpna_config.payroll_driver_adjustments (
                    year, quarter, company_id, employee_name, month,
                    salary_adjusted, bonus_adjusted, benefits_adjusted,
                    month_comment, overall_comment, updated_at
                ) VALUES (
                    :year, :quarter, :company_id, :employee_name, :month,
                    :salary_adjusted, :bonus_adjusted, :benefits_adjusted,
                    :month_comment, :overall_comment, NOW()
                )
                ON CONFLICT (year, quarter, company_id, employee_name, month)
                DO UPDATE SET
                    salary_adjusted = EXCLUDED.salary_adjusted,
                    bonus_adjusted = EXCLUDED.bonus_adjusted,
                    benefits_adjusted = EXCLUDED.benefits_adjusted,
                    month_comment = EXCLUDED.month_comment,
                    overall_comment = EXCLUDED.overall_comment,
                    updated_at = NOW()
            """),
            {
                "year": year, "quarter": quarter.value, "company_id": company_id,
                "employee_name": adj.employee_name, "month": adj.month,
                "salary_adjusted": adj.salary_adjusted, "bonus_adjusted": adj.bonus_adjusted,
                "benefits_adjusted": adj.benefits_adjusted,
                "month_comment": adj.month_comment, "overall_comment": adj.overall_comment,
            },
        )
    await db.commit()

    saved = (await db.execute(text("""
        SELECT employee_name, month, salary_adjusted, bonus_adjusted, benefits_adjusted,
               month_comment, overall_comment
        FROM fpna_config.payroll_driver_adjustments
        WHERE year = :year AND quarter = :quarter AND company_id = :company_id
        ORDER BY employee_name, month;
    """), {"year": year, "quarter": quarter.value, "company_id": company_id})).mappings().all()

    employee_quarter_map: Dict[str, float] = {}
    result_rows = []
    for r in saved:
        s = float(r["salary_adjusted"]) if r["salary_adjusted"] is not None else None
        b = float(r["bonus_adjusted"]) if r["bonus_adjusted"] is not None else None
        be = float(r["benefits_adjusted"]) if r["benefits_adjusted"] is not None else None
        total_adj = round((s or 0.0) + (b or 0.0) + (be or 0.0), 2) if any(v is not None for v in [s, b, be]) else None
        ename = r["employee_name"]
        if total_adj is not None:
            employee_quarter_map[ename] = employee_quarter_map.get(ename, 0.0) + total_adj
        result_rows.append({
            "employee_name": ename, "month": r["month"],
            "salary_adjusted": s, "bonus_adjusted": b, "benefits_adjusted": be,
            "total_cost_adjusted": total_adj,
            "month_comment": r["month_comment"], "overall_comment": r["overall_comment"],
        })

    for row in result_rows:
        row["quarter_total_cost_adjusted"] = round(employee_quarter_map.get(row["employee_name"], 0.0), 2)

    return {
        "detail": "Driver adjustments uploaded successfully",
        "year": year,
        "quarter": quarter.value,
        "upload_errors": errors,
        "rows_attempted": len(driver_adjustments) + len(errors),
        "rows_saved": len(driver_adjustments),
        "driver_adjustments": result_rows,
    }


@forecast_router.get("/headcount_and_Payroll_forecast/{year}/{currency}")
async def headcount_and_Payroll_forecast(
    year: int = Path(..., description="Year for the forecast"),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    quarter: QuarterEnum = Query(QuarterEnum.Q1, description="Quarter for the forecast"),
    forecast_year: Optional[int] = Query(default=None, description="Primary forecast year context. Overrides year for period resolution when provided."),
    forecast_type: Optional[str] = Query(default="quarter", description="Forecast type: quarter, 12months, rolling"),
    rolling_start_month: Optional[str] = Query(default=None, description="Start month for rolling forecast (e.g. Jan, Feb, Mar). Required when forecast_type=rolling."),
    view_type: Optional[str] = Query(default="monthly_breakdown", description="View type: quarter_summary or monthly_breakdown"),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
) -> Dict[str, Any]:
    await _ensure_payroll_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    effective_year = forecast_year if forecast_year is not None else year
    effective_forecast_type = forecast_type or "quarter"
    effective_view_type = view_type or "monthly_breakdown"

    if effective_forecast_type not in ("quarter", "12months", "rolling"):
        raise HTTPException(status_code=422, detail="forecast_type must be one of: quarter, 12months, rolling")
    if effective_view_type not in ("quarter_summary", "monthly_breakdown"):
        raise HTTPException(status_code=422, detail="view_type must be one of: quarter_summary, monthly_breakdown")
    if effective_forecast_type == "rolling" and not rolling_start_month:
        raise HTTPException(status_code=422, detail="rolling_start_month is required when forecast_type=rolling. Example: Jan, Feb, Mar, etc.")

    # Rolling: Nov/Dec starts use previous year so the window spans INTO effective_year
    _MONTH_ORDER = {"jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
                    "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12}
    resolve_year = effective_year
    if effective_forecast_type == "rolling" and rolling_start_month:
        if _MONTH_ORDER.get(rolling_start_month.lower()[:3], 1) >= 11:
            resolve_year = effective_year - 1

    month_periods, selected_period, _ = _resolve_forecast_periods(
        resolve_year, effective_forecast_type, quarter.value, rolling_start_month
    )
    n_months = len(month_periods)

    # Derive effective_quarter for workflow / adjustment key
    effective_quarter = quarter.value
    if effective_forecast_type == "12months":
        effective_quarter = "Q1"
    elif effective_forecast_type == "rolling":
        month_num = int(month_periods[0].split("-")[1])
        if month_num <= 3:
            effective_quarter = "Q1"
        elif month_num <= 6:
            effective_quarter = "Q2"
        elif month_num <= 9:
            effective_quarter = "Q3"
        else:
            effective_quarter = "Q4"

    # Build Python date objects from resolved month_periods
    first_yr, first_mo = int(month_periods[0].split("-")[0]), int(month_periods[0].split("-")[1])
    last_yr, last_mo = int(month_periods[-1].split("-")[0]), int(month_periods[-1].split("-")[1])
    start_date = date(first_yr, first_mo, 1)
    end_date = date(last_yr, last_mo, monthrange(last_yr, last_mo)[1])
    # period for headcount target SQL = last month of selected range ("YYYY-MM")
    period = month_periods[-1]

    wf = await _get_or_create_payroll_workflow(db, effective_year, effective_quarter, company_id, dept, user_id)
    stage = wf["stage"]

    audit_rows = (await db.execute(text("""
        SELECT to_stage, transitioned_by_name
        FROM fpna_config.payroll_forecast_workflow_audit
        WHERE workflow_id = :wf_id
        ORDER BY created_at ASC
    """), {"wf_id": wf["id"]})).mappings().all()

    reviewed_by = None
    approved_by = None
    locked_by = None

    for r in audit_rows:
        if r["to_stage"] == "reviewed":
            reviewed_by = r["transitioned_by_name"]
        elif r["to_stage"] == "approved":
            approved_by = r["transitioned_by_name"]
        elif r["to_stage"] == "locked":
            locked_by = r["transitioned_by_name"]

    prev_month_first = get_previous_month_first_day(start_date)
    prev_month_last = get_previous_month_last_day(prev_month_first)
    prev_month_str = prev_month_first.isoformat()

    headcount_sql = """
    WITH target AS (
        SELECT COALESCE(SUM(headcount), 0) AS target_headcount_end
        FROM fpna_landing.hr_headcount_history
        WHERE period = :period
    ),
    projected AS (
        SELECT COUNT(DISTINCT employee_id) AS projected_headcount_end
        FROM fpna_landing.hr_employees
        WHERE hire_date <= :end_date
          AND (termination_date IS NULL OR termination_date > :end_date)
    )
    SELECT
        target_headcount_end,
        projected_headcount_end,
        target_headcount_end - projected_headcount_end AS variance_target_vs_projected
    FROM target, projected;
    """

    result = await db.execute(text(headcount_sql), {"period": period, "end_date": end_date})
    hc_row = result.mappings().first()

    headcount_summary = {
        "target_headcount_end": int(hc_row["target_headcount_end"]),
        "projected_headcount_end": int(hc_row["projected_headcount_end"]),
        "variance_target_vs_projected": int(hc_row["variance_target_vs_projected"]),
    }

    months = pd.date_range(start=start_date, end=end_date, freq="MS")
    month_strs = [m.date().isoformat() for m in months]

    dept_prev_sql = """
    SELECT
        d.department_name AS department,
        COUNT(DISTINCT e.employee_id) AS headcount
    FROM fpna_landing.hr_employees e
    JOIN fpna_landing.hr_departments_lookup d
        ON e.department_id = d.department_id
    WHERE e.hire_date <= :prev_end
      AND (e.termination_date IS NULL OR e.termination_date > :prev_end)
    GROUP BY d.department_name
    """

    result = await db.execute(text(dept_prev_sql), {"prev_end": prev_month_last})
    dept_prev_map = {r["department"]: int(r["headcount"]) for r in result.mappings().all()}

    dept_headcount_sql = """
    SELECT
        d.department_name AS department,
        COUNT(DISTINCT e.employee_id) AS projected_headcount
    FROM fpna_landing.hr_employees e
    JOIN fpna_landing.hr_departments_lookup d
        ON e.department_id = d.department_id
    WHERE e.hire_date <= :end_date
      AND (e.termination_date IS NULL OR e.termination_date > :end_date)
    GROUP BY d.department_name
    """

    result = await db.execute(text(dept_headcount_sql), {"end_date": end_date})
    dept_rows = result.mappings().all()
    dept_df = pd.DataFrame(dept_rows)

    dept_saved = await _get_department_headcount_adjustments(db, effective_year, effective_quarter, company_id)
    dept_adj_map: Dict[str, Dict[str, Any]] = {}
    for d in dept_saved:
        dept_adj_map[str(d["department_name"])] = d

    if dept_df.empty:
        dept_projections = {"Departments": [], "Total_Projected": {}}
    else:
        dept_out = []
        total_prev_count = 0
        total_monthly_ai: Dict[str, int] = defaultdict(int)
        total_monthly_adj: Dict[str, int] = defaultdict(int)

        for _, row in dept_df.iterrows():
            dept_name = row["department"]
            proj_count = int(row["projected_headcount"])
            prev_count = dept_prev_map.get(dept_name, 0)
            total_prev_count += prev_count

            entry: Dict[str, Any] = {
                "Department": dept_name,
                "previous_month": prev_month_str,
                "previous_month_count": prev_count,
            }

            dept_adj = dept_adj_map.get(dept_name, {})

            for i, month in enumerate(months, start=1):
                month_str = month.date().isoformat()
                entry[f"month{i}"] = month_str
                entry[f"month{i}_ai_count"] = proj_count

                adj_val = dept_adj.get(f"month{i}_adjusted_count")
                if adj_val is None:
                    adj_val = proj_count

                entry[f"month{i}_adjusted_count"] = int(adj_val)
                entry[f"month{i}_comment"] = dept_adj.get(f"month{i}_comment")

                total_monthly_ai[f"month{i}|{month_str}"] += proj_count
                total_monthly_adj[f"month{i}|{month_str}"] += int(adj_val)

            entry["overall_comment"] = dept_adj.get("overall_comment")
            dept_out.append(entry)

        total_projected: Dict[str, Any] = {
            "previous_month": prev_month_str,
            "previous_month_count": total_prev_count,
        }

        for i, month in enumerate(months, start=1):
            month_str = month.date().isoformat()
            ai_count = total_monthly_ai[f"month{i}|{month_str}"]
            adj_count = total_monthly_adj[f"month{i}|{month_str}"]
            total_projected[f"month{i}"] = month_str
            total_projected[f"month{i}_ai_count"] = int(ai_count)
            total_projected[f"month{i}_adjusted_count"] = int(adj_count)

        dept_projections = {"Departments": dept_out, "Total_Projected": total_projected}

    result = await db.execute(text(RAW_SQL))
    payroll_rows = result.mappings().all()
    df = pd.DataFrame(payroll_rows)

    money_fields = [
        "previous_month_salary", "previous_month_bonus", "previous_month_benefits",
        "month1_ai_salary", "month1_ai_bonus", "month1_ai_benefits",
        "month1_adjusted_salary", "month1_adjusted_bonus", "month1_adjusted_benefits",
        "month2_ai_salary", "month2_ai_bonus", "month2_ai_benefits",
        "month2_adjusted_salary", "month2_adjusted_bonus", "month2_adjusted_benefits",
        "month3_ai_salary", "month3_ai_bonus", "month3_ai_benefits",
        "month3_adjusted_salary", "month3_adjusted_bonus", "month3_adjusted_benefits",
        "quarter_total_ai", "quarter_total_adjusted",
        "salary_ai", "bonus_ai", "benefits_ai", "total_cost_ai",
        "salary_adjusted", "bonus_adjusted", "benefits_adjusted", "total_cost_adjusted",
        "quarter_total_cost_ai", "quarter_total_cost_adjusted",
        "total_payroll", "avg_cost_per_employee",
        "total_ai", "total_user", "variance_amount",
    ]

    if df.empty:
        response = {
            "year": year,
            "forecast_year": effective_year,
            "forecast_type": effective_forecast_type,
            "view_type": effective_view_type,
            "selected_period": selected_period,
            "selected_months": [m.date().isoformat() for m in months],
            "quarter": quarter.value,
            "workflow_stage": stage,
            "reviewed_by": reviewed_by,
            "approved_by": approved_by,
            "locked_by": locked_by,
            "headcount_summary": headcount_summary,
            "department_headcount_projections": dept_projections,
            "summary": {"active_employees": 0, "total_payroll": 0.0, "avg_cost_per_employee": 0.0},
            "Baseline_forecast_summary": {"total_ai": 0.0, "total_user": 0.0, "variance_amount": 0.0, "variance_percent": 0.0},
            "Compare_Payroll_Forecast_Scenarios": {},
            "month_totals": [{"month": m.date().isoformat(), "total_ai": 0.0, "total_adjusted": 0.0} for m in months],
            "employees": [],
            "drivers": [],
        }
        return apply_smart_currency(response, currency, currency_filter, money_fields)

    df["effective_date"] = pd.to_datetime(df["effective_date"])
    df["hire_date"] = pd.to_datetime(df["hire_date"])
    df["base_salary"] = df["base_salary"].fillna(0).astype(float)
    df["bonus_target"] = df["bonus_target"].fillna(0).astype(float)

    employees = df[["employee_id", "employee", "department", "role", "hire_date"]].drop_duplicates()
    df2 = df.drop(columns=["employee", "department", "role", "hire_date"])
    df2["effective_month"] = df2["effective_date"].dt.to_period("M").dt.to_timestamp()

    all_months_range = pd.date_range(start=prev_month_first, end=end_date, freq="MS")
    grid = (
        employees.assign(key=1)
        .merge(pd.DataFrame({"month": all_months_range, "key": 1}), on="key")
        .drop("key", axis=1)
    )
    merged = grid.merge(df2, on="employee_id", how="left")
    merged = merged[merged["effective_month"].isna() | (merged["effective_month"] <= merged["month"])]
    merged = merged.sort_values("effective_month").groupby(["employee_id", "month"]).tail(1)

    merged["salary"] = merged["base_salary"]
    merged["bonus"] = merged["bonus_target"]
    merged["benefits"] = merged["salary"] * 0.20
    merged.loc[merged["month"] < merged["hire_date"], ["salary", "bonus", "benefits"]] = 0

    out = []
    for _, emp_df in merged.groupby("employee_id"):
        emp_df = emp_df.sort_values("month").copy()
        n = len(emp_df)
        forecasted_salary = forecast_series_advanced(emp_df["salary"], n)
        forecasted_bonus = forecast_series_advanced(emp_df["bonus"], n)
        emp_df["salary"] = forecasted_salary
        emp_df["bonus"] = forecasted_bonus
        emp_df["benefits"] = emp_df["salary"] * 0.20
        out.append(emp_df)

    final_df = pd.concat(out, ignore_index=True)

    saved_payroll = await _get_payroll_adjustments(db, effective_year, effective_quarter, company_id)
    payroll_adj_map: Dict[int, Dict[str, Any]] = {}
    for s in saved_payroll:
        payroll_adj_map[int(s["employee_id"])] = s

    driver_saved = await _get_payroll_driver_adjustments(db, effective_year, effective_quarter, company_id)
    custom_scenarios = await _get_payroll_custom_scenarios(db, effective_year, effective_quarter, company_id)

    employee_map: Dict[int, Dict[str, Any]] = {}
    monthly_totals_payroll_ai: Dict[str, Dict[str, float]] = defaultdict(lambda: {"salary": 0.0, "bonus": 0.0, "benefits": 0.0})
    monthly_totals_payroll_adj: Dict[str, Dict[str, float]] = defaultdict(lambda: {"salary": 0.0, "bonus": 0.0, "benefits": 0.0})

    month_str_to_index = {m.date().isoformat(): i + 1 for i, m in enumerate(months)}

    drivers_by_employee: Dict[str, List[Dict]] = defaultdict(list)
    employee_ai_map: Dict[str, float] = {}
    employee_user_map: Dict[str, float] = {}

    for emp_id, emp_df in final_df.groupby("employee_id"):
        emp_df = emp_df.sort_values("month")
        rows_list = emp_df.to_dict("records")

        if not rows_list:
            continue

        first = rows_list[0]
        entry: Dict[str, Any] = {
            "employee": first["employee"],
            "department": first["department"],
            "role": first["role"],
        }

        adj = payroll_adj_map.get(int(emp_id), {})
        emp_name = first["employee"]

        quarter_ai_total = 0.0
        quarter_user_total = 0.0

        for row in rows_list:
            month_dt = pd.Timestamp(row["month"])
            month_str = month_dt.date().isoformat()

            salary_ai = round(float(row["salary"]), 2)
            bonus_ai = round(float(row["bonus"]), 2)
            benefits_ai = round(float(row["benefits"]), 2)

            salary_adj = salary_ai
            bonus_adj = bonus_ai
            benefits_adj = benefits_ai

            drv = driver_saved.get((emp_name, month_str))
            drv_month_comment = None

            if drv:
                drv_month_comment = drv.get("month_comment")
                if drv.get("salary_adjusted") is not None:
                    salary_adj = round(float(drv["salary_adjusted"]), 2)
                if drv.get("bonus_adjusted") is not None:
                    bonus_adj = round(float(drv["bonus_adjusted"]), 2)
                if drv.get("benefits_adjusted") is not None:
                    benefits_adj = round(float(drv["benefits_adjusted"]), 2)
            elif month_str in month_str_to_index:
                idx = month_str_to_index[month_str]
                s = adj.get(f"month{idx}_adjusted_salary")
                b = adj.get(f"month{idx}_adjusted_bonus")
                be = adj.get(f"month{idx}_adjusted_benefits")
                if s is not None:
                    salary_adj = round(float(s), 2)
                if b is not None:
                    bonus_adj = round(float(b), 2)
                if be is not None:
                    benefits_adj = round(float(be), 2)

            if month_str == prev_month_str:
                entry["previous_month"] = month_str
                entry["previous_month_salary"] = salary_ai
                entry["previous_month_bonus"] = bonus_ai
                entry["previous_month_benefits"] = benefits_ai
            elif month_str in month_str_to_index:
                idx = month_str_to_index[month_str]
                entry[f"month{idx}"] = month_str
                entry[f"month{idx}_ai_salary"] = salary_ai
                entry[f"month{idx}_ai_bonus"] = bonus_ai
                entry[f"month{idx}_ai_benefits"] = benefits_ai
                entry[f"month{idx}_adjusted_salary"] = salary_adj
                entry[f"month{idx}_adjusted_bonus"] = bonus_adj
                entry[f"month{idx}_adjusted_benefits"] = benefits_adj
                entry[f"month{idx}_comment"] = adj.get(f"month{idx}_comment") if not drv else drv_month_comment

                quarter_ai_total += salary_ai + bonus_ai + benefits_ai
                quarter_user_total += salary_adj + bonus_adj + benefits_adj

                drivers_by_employee[emp_name].append({
                    "employee_name": emp_name,
                    "month": month_str,
                    "salary_ai": salary_ai,
                    "bonus_ai": bonus_ai,
                    "benefits_ai": benefits_ai,
                    "total_cost_ai": round(salary_ai + bonus_ai + benefits_ai, 2),
                    "salary_adjusted": round(float(drv["salary_adjusted"]), 2) if drv and drv.get("salary_adjusted") is not None else None,
                    "bonus_adjusted": round(float(drv["bonus_adjusted"]), 2) if drv and drv.get("bonus_adjusted") is not None else None,
                    "benefits_adjusted": round(float(drv["benefits_adjusted"]), 2) if drv and drv.get("benefits_adjusted") is not None else None,
                    "total_cost_adjusted": round(salary_adj + bonus_adj + benefits_adj, 2),
                    "month_comment": drv_month_comment,
                })

            monthly_totals_payroll_ai[month_str]["salary"] += salary_ai
            monthly_totals_payroll_ai[month_str]["bonus"] += bonus_ai
            monthly_totals_payroll_ai[month_str]["benefits"] += benefits_ai

            monthly_totals_payroll_adj[month_str]["salary"] += salary_adj
            monthly_totals_payroll_adj[month_str]["bonus"] += bonus_adj
            monthly_totals_payroll_adj[month_str]["benefits"] += benefits_adj

        entry["quarter_total_ai"] = round(quarter_ai_total, 2)
        entry["quarter_total_adjusted"] = round(quarter_user_total, 2)
        entry["overall_comment"] = adj.get("overall_comment")
        employee_map[int(emp_id)] = entry

        employee_ai_map[emp_name] = round(quarter_ai_total, 2)
        employee_user_map[emp_name] = round(quarter_user_total, 2)

    total_entry: Dict[str, Any] = {"employee": "TOTAL", "department": "ALL", "role": "ALL"}

    prev_ai = monthly_totals_payroll_ai.get(prev_month_str, {"salary": 0.0, "bonus": 0.0, "benefits": 0.0})
    total_entry["previous_month"] = prev_month_str
    total_entry["previous_month_salary"] = round(prev_ai["salary"], 2)
    total_entry["previous_month_bonus"] = round(prev_ai["bonus"], 2)
    total_entry["previous_month_benefits"] = round(prev_ai["benefits"], 2)

    for i, month in enumerate(months, start=1):
        month_str = month.date().isoformat()
        ai = monthly_totals_payroll_ai.get(month_str, {"salary": 0.0, "bonus": 0.0, "benefits": 0.0})
        adj_t = monthly_totals_payroll_adj.get(month_str, {"salary": 0.0, "bonus": 0.0, "benefits": 0.0})
        total_entry[f"month{i}"] = month_str
        total_entry[f"month{i}_ai_salary"] = round(ai["salary"], 2)
        total_entry[f"month{i}_ai_bonus"] = round(ai["bonus"], 2)
        total_entry[f"month{i}_ai_benefits"] = round(ai["benefits"], 2)
        total_entry[f"month{i}_adjusted_salary"] = round(adj_t["salary"], 2)
        total_entry[f"month{i}_adjusted_bonus"] = round(adj_t["bonus"], 2)
        total_entry[f"month{i}_adjusted_benefits"] = round(adj_t["benefits"], 2)

    total_payroll_ai = sum(
        monthly_totals_payroll_ai[m.date().isoformat()]["salary"]
        + monthly_totals_payroll_ai[m.date().isoformat()]["bonus"]
        + monthly_totals_payroll_ai[m.date().isoformat()]["benefits"]
        for m in months
    )

    total_payroll_user = sum(
        monthly_totals_payroll_adj[m.date().isoformat()]["salary"]
        + monthly_totals_payroll_adj[m.date().isoformat()]["bonus"]
        + monthly_totals_payroll_adj[m.date().isoformat()]["benefits"]
        for m in months
    )

    active_employees = len(employee_map)

    baseline_forecast_summary = {
        "total_ai": round(total_payroll_ai, 2),
        "total_user": round(total_payroll_user, 2),
        "variance_amount": round(total_payroll_user - total_payroll_ai, 2),
        "variance_percent": round(((total_payroll_user - total_payroll_ai) / total_payroll_ai) * 100 if total_payroll_ai else 0, 2),
    }

    compare_payroll_forecast_scenarios = {
        "baseline": _build_payroll_scenario_summary(total_payroll_ai, total_payroll_user, 1.0),
        "best_case": _build_payroll_scenario_summary(total_payroll_ai, total_payroll_user, PAYROLL_BEST_CASE_MULTIPLIER),
        "worst_case": _build_payroll_scenario_summary(total_payroll_ai, total_payroll_user, PAYROLL_WORST_CASE_MULTIPLIER),
    }

    for cs in custom_scenarios:
        scenario_key = cs["scenario_name"].lower().replace(" ", "_")
        if cs["customer_scope"] == "specific":
            emp_mults = await _get_payroll_scenario_employee_multipliers(db, cs["id"], company_id, effective_year, effective_quarter)
            compare_payroll_forecast_scenarios[scenario_key] = _build_payroll_scenario_summary_with_employee_multipliers(
                baseline_ai=total_payroll_ai,
                baseline_user=total_payroll_user,
                scenario_multiplier=cs["multiplier"],
                employee_ai_map=employee_ai_map,
                employee_user_map=employee_user_map,
                employee_multipliers=emp_mults,
                customer_scope=cs["customer_scope"],
            )
        else:
            compare_payroll_forecast_scenarios[scenario_key] = _build_payroll_scenario_summary(total_payroll_ai, total_payroll_user, cs["multiplier"])

    # Build month_totals (per-month aggregate across all employees)
    month_totals: List[Dict[str, Any]] = []
    for month in months:
        month_str = month.date().isoformat()
        ai = monthly_totals_payroll_ai.get(month_str, {"salary": 0.0, "bonus": 0.0, "benefits": 0.0})
        adj_t = monthly_totals_payroll_adj.get(month_str, {"salary": 0.0, "bonus": 0.0, "benefits": 0.0})
        month_totals.append({
            "month": month_str,
            "total_ai": round(ai["salary"] + ai["bonus"] + ai["benefits"], 2),
            "total_adjusted": round(adj_t["salary"] + adj_t["bonus"] + adj_t["benefits"], 2),
        })

    drivers_output = []
    for ename, month_entries in drivers_by_employee.items():
        q_ai = sum(m["total_cost_ai"] for m in month_entries)
        q_adj = sum(m["total_cost_adjusted"] for m in month_entries)

        overall_comment_for_emp = None
        for month_str in month_strs:
            drv = driver_saved.get((ename, month_str))
            if drv and drv.get("overall_comment"):
                overall_comment_for_emp = drv["overall_comment"]
                break

        drivers_output.append({
            "employee_name": ename,
            "months": sorted(month_entries, key=lambda x: x["month"]),
            "quarter_total_cost_ai": round(q_ai, 2),
            "quarter_total_cost_adjusted": round(q_adj, 2),
            "overall_comment": overall_comment_for_emp,
        })

    # Apply view_type: quarter_summary collapses per-month fields, keeps period totals only
    all_employees_list: List[Dict[str, Any]] = list(employee_map.values()) + [total_entry]
    if effective_view_type == "quarter_summary":
        employees_output: List[Dict[str, Any]] = [
            {
                "employee": e.get("employee"),
                "department": e.get("department"),
                "role": e.get("role"),
                "previous_month": e.get("previous_month"),
                "previous_month_salary": e.get("previous_month_salary"),
                "previous_month_bonus": e.get("previous_month_bonus"),
                "previous_month_benefits": e.get("previous_month_benefits"),
                "quarter_total_ai": e.get("quarter_total_ai", 0.0),
                "quarter_total_adjusted": e.get("quarter_total_adjusted", 0.0),
                "overall_comment": e.get("overall_comment"),
            }
            for e in all_employees_list
        ]
    else:
        employees_output = all_employees_list
        # For 12months + monthly_breakdown: attach monthly_data array to each employee
        if n_months > 3:
            for emp in employees_output:
                emp["monthly_data"] = [
                    {
                        "month": m.date().isoformat(),
                        "ai_salary": emp.get(f"month{month_str_to_index[m.date().isoformat()]}_ai_salary", 0.0),
                        "ai_bonus": emp.get(f"month{month_str_to_index[m.date().isoformat()]}_ai_bonus", 0.0),
                        "ai_benefits": emp.get(f"month{month_str_to_index[m.date().isoformat()]}_ai_benefits", 0.0),
                        "adjusted_salary": emp.get(f"month{month_str_to_index[m.date().isoformat()]}_adjusted_salary", 0.0),
                        "adjusted_bonus": emp.get(f"month{month_str_to_index[m.date().isoformat()]}_adjusted_bonus", 0.0),
                        "adjusted_benefits": emp.get(f"month{month_str_to_index[m.date().isoformat()]}_adjusted_benefits", 0.0),
                    }
                    for m in months
                    if m.date().isoformat() in month_str_to_index
                ]

    response = {
        "year": year,
        "forecast_year": effective_year,
        "forecast_type": effective_forecast_type,
        "view_type": effective_view_type,
        "selected_period": selected_period,
        "selected_months": [m.date().isoformat() for m in months],
        "quarter": quarter.value,
        "prev_month": prev_month_str,
        "workflow_stage": stage,
        "reviewed_by": reviewed_by,
        "approved_by": approved_by,
        "locked_by": locked_by,
        "headcount_summary": headcount_summary,
        "department_headcount_projections": dept_projections,
        "summary": {
            "active_employees": active_employees,
            "total_payroll": round(total_payroll_user, 2),
            "avg_cost_per_employee": round(total_payroll_user / active_employees, 2) if active_employees else 0.0,
        },
        "Baseline_forecast_summary": baseline_forecast_summary,
        "Compare_Payroll_Forecast_Scenarios": compare_payroll_forecast_scenarios,
        "month_totals": month_totals,
        "employees": employees_output,
        "drivers": drivers_output,
    }

    # For 12months + quarter_summary: add Q1-Q4 aggregate totals
    if n_months > 3 and effective_view_type == "quarter_summary":
        q_summary: Dict[str, Dict[str, float]] = {}
        for qi, q_label in enumerate(["Q1", "Q2", "Q3", "Q4"]):
            start_idx = qi * 3
            q_ai = round(sum(mt["total_ai"] for mt in month_totals[start_idx:start_idx + 3]), 2)
            q_adj = round(sum(mt["total_adjusted"] for mt in month_totals[start_idx:start_idx + 3]), 2)
            q_summary[q_label] = {"total_ai": q_ai, "total_adjusted": q_adj}
        response["quarter_summary"] = q_summary

    return apply_smart_currency(response, currency, currency_filter, money_fields)

##########################################################################################
###########################################################################################
###########################################################################################
# Cashflow projection forecasting

BEST_CASE_MULTIPLIER = 1.10
WORST_CASE_MULTIPLIER = 0.80
HISTORY_MONTHS_SQL = 24
MIN_POINTS_FOR_PROPHET = 12
MIN_POINTS_FOR_ETS = 6
MIN_POINTS_FOR_XGBOOST = 12
MIN_POINTS_FOR_ARIMA = 8


RESERVED_SCENARIO_NAMES = {"baseline", "best_case", "worst_case"}

CASHFLOW_SCENARIO_DEFINITIONS = {
    "baseline": "AI-generated cashflow forecast with manual user adjustments applied.",
    "best_case": f"Optimistic scenario applying a {int((BEST_CASE_MULTIPLIER - 1) * 100)}% uplift on baseline — assumes favorable cash inflows, reduced outflows, and strong financing conditions.",
    "worst_case": f"Pessimistic scenario applying a {int((1 - WORST_CASE_MULTIPLIER) * 100)}% reduction on baseline — assumes adverse cash conditions, higher expenses, or financing shortfalls.",
}

CASHFLOW_COMPONENTS = [
    "operatingCF",
    "investingCF",
    "financingCF",
    "netCashChange",
]

COMPONENT_MAP = {
    "operatingCF": "operating_cf",
    "investingCF": "investing_cf",
    "financingCF": "financing_cf",
    "netCashChange": "net_cash_change",
}

CASHFLOW_MONEY_FIELDS = [
    "previous_month",
    "month1_ai",
    "month1_adjusted",
    "month2_ai",
    "month2_adjusted",
    "month3_ai",
    "month3_adjusted",
    "quarter_total_ai",
    "quarter_total_adjusted",
    "revenue_ai",
    "revenue_adjusted",
    "quarter_revenue_ai",
    "quarter_revenue_adjusted",
    "ai_Suggested",
    "userAdjustment",
]

_executor = ThreadPoolExecutor(max_workers=8)


class QuarterEnum(str, Enum):
    Q1 = "Q1"
    Q2 = "Q2"
    Q3 = "Q3"
    Q4 = "Q4"


class WorkflowStage:
    DRAFT = "draft"
    SUBMITTED = "submitted"
    REVIEWED = "reviewed"
    APPROVED = "approved"
    LOCKED = "locked"


DEPT_SUBMITTERS = {"it", "sales", "operations", "finance_ops"}
DEPT_REVIEWERS = {"finance"}
DEPT_APPROVERS = {"ceo", "cfo", "executive"}


class CashflowComponentAdjustmentInput(BaseModel):
    component: str = Field(..., description="Component name e.g. operatingCF")
    month1_adjusted: Optional[float] = None
    month1_comment: Optional[str] = None
    month2_adjusted: Optional[float] = None
    month2_comment: Optional[str] = None
    month3_adjusted: Optional[float] = None
    month3_comment: Optional[str] = None
    overall_comment: Optional[str] = None


class CashflowDriverAdjustmentInput(BaseModel):
    component: str = Field(..., description="Component name")
    month: str = Field(..., description="Month in YYYY-MM format")
    units_adjusted: Optional[float] = None
    rate_adjusted: Optional[float] = None
    month_comment: Optional[str] = None
    overall_comment: Optional[str] = None


class WorkflowTransitionRequest(BaseModel):
    target_stage: str
    comment: Optional[str] = None


class CashflowForecastCommentInput(BaseModel):
    comment: str


class CashflowScenarioInput(BaseModel):
    scenario_name: str
    multiplier: float
    description: Optional[str] = None
    parameter_types: Optional[List[str]] = None
    customer_scope: Optional[str] = "all"
    component_multipliers: Optional[Dict[str, float]] = None


class CashflowScenarioUpdateInput(BaseModel):
    scenario_name: Optional[str] = None
    multiplier: Optional[float] = None
    description: Optional[str] = None
    parameter_types: Optional[List[str]] = None
    customer_scope: Optional[str] = None
    component_multipliers: Optional[Dict[str, float]] = None


class CashflowRow(BaseModel):
    component: str
    previous_month: float
    month1_ai: float
    month1_adjusted: float
    month1_comment: Optional[str]
    month2_ai: float
    month2_adjusted: float
    month2_comment: Optional[str]
    month3_ai: float
    month3_adjusted: float
    month3_comment: Optional[str]
    quarter_total_ai: float
    quarter_total_adjusted: float
    overall_comment: Optional[str]


class CashflowDriverRow(BaseModel):
    component: str
    month: str
    units_ai: float
    rate_ai: float
    revenue_ai: float
    units_adjusted: Optional[float]
    rate_adjusted: Optional[float]
    revenue_adjusted: Optional[float]
    month_comment: Optional[str]


class CashflowDriverSummary(BaseModel):
    component: str
    months: List[CashflowDriverRow]
    quarter_revenue_ai: float
    quarter_revenue_adjusted: float
    overall_comment: Optional[str]


class CashflowTotal(BaseModel):
    previous_month: float = 0.0
    month1_ai: float = 0.0
    month1_adjusted: float = 0.0
    month2_ai: float = 0.0
    month2_adjusted: float = 0.0
    month3_ai: float = 0.0
    month3_adjusted: float = 0.0



def _linear_forecast_sync(values: np.ndarray, n_ahead: int = 3) -> Optional[list]:
    try:
        x = np.arange(len(values))
        coeffs = np.polyfit(x, values, 1)
        return [round(float(np.polyval(coeffs, len(values) + i)), 2) for i in range(n_ahead)]
    except Exception:
        return None


def _arima_forecast_sync(values: np.ndarray, n_ahead: int = 3) -> Optional[list]:
    if not STATSMODELS_AVAILABLE or len(values) < MIN_POINTS_FOR_ARIMA:
        return None
    try:
        d = 0 if _detect_stationarity(values) else 1
        model = ARIMA(values, order=(1, d, 1))
        fitted = model.fit()
        return [round(float(v), 2) for v in fitted.forecast(steps=n_ahead)]
    except Exception:
        return None


def _ensemble_forecasts(
    baseline_result: list,
    linear_result: Optional[list],
    prophet_result: Optional[list],
    ets_result: Optional[list],
    arima_result: Optional[list],
    xgb_result: Optional[list],
    features: Dict[str, float],
) -> list:
    weights = {"baseline": 0.20, "linear": 0.15, "prophet": 0.20, "ets": 0.15, "arima": 0.15, "xgb": 0.15}
    if features["trend_strength"] >= 0.60:
        weights["linear"] += 0.10
    if features["seasonality_strength"] >= 0.40:
        weights["prophet"] += 0.10
    if features["volatility"] >= 0.40:
        weights["xgb"] += 0.10
    if features["stationary"]:
        weights["arima"] += 0.10
    if features["outlier_ratio"] >= 0.15:
        weights["xgb"] += 0.05
    total = sum(weights.values())
    weights = {k: v / total for k, v in weights.items()}
    result = []
    for i in range(3):
        vals = {
            "baseline": baseline_result[i],
            "linear": linear_result[i] if linear_result else baseline_result[i],
            "prophet": prophet_result[i] if prophet_result else baseline_result[i],
            "ets": ets_result[i] if ets_result else baseline_result[i],
            "arima": arima_result[i] if arima_result else baseline_result[i],
            "xgb": xgb_result[i] if xgb_result else baseline_result[i],
        }
        result.append(round(float(sum(vals[k] * weights[k] for k in weights)), 2))
    return result


def _sarima_forecast_sync(values: np.ndarray, n_ahead: int = 3) -> Optional[list]:
    if not STATSMODELS_AVAILABLE or SARIMAX is None or len(values) < MIN_POINTS_FOR_SARIMA:
        return None
    seasonality_strength = _detect_seasonality_strength(values)
    if seasonality_strength < 0.10:
        return None
    try:
        is_stationary = _detect_stationarity(values)
        d = 0 if is_stationary else 1
        D = 1 if seasonality_strength >= 0.25 else 0
        model = SARIMAX(
            values,
            order=(1, d, 1),
            seasonal_order=(1, D, 1, 12),
            enforce_stationarity=False,
            enforce_invertibility=False,
        )
        fitted = model.fit(disp=False, maxiter=60)
        forecast = fitted.forecast(steps=n_ahead)
        return [round(max(0.0, float(v)), 2) for v in forecast]
    except Exception:
        return None


def _detect_autocorrelation_strength(values: np.ndarray) -> float:
    if len(values) < 3:
        return 0.0
    try:
        lag1 = np.corrcoef(values[:-1], values[1:])[0, 1]
        return float(abs(lag1)) if not np.isnan(lag1) else 0.0
    except Exception:
        return 0.0


def _compute_stability_score(values: np.ndarray) -> float:
    if len(values) < 4:
        return 0.5
    volatility = _detect_volatility(values)
    trend_strength = _detect_trend_strength(values)
    outlier_ratio = _detect_outlier_ratio(values)
    score = 1.0 - (volatility * 0.50) - (outlier_ratio * 0.30) + (trend_strength * 0.20)
    return float(max(0.0, min(1.0, score)))


def _ensemble_forecasts_revenue(
    baseline_result: list,
    prophet_result: Optional[list],
    ets_result: Optional[list],
    sarima_result: Optional[list],
    xgb_result: Optional[list],
    features: Dict[str, Any],
    n_ahead: int = 3,
) -> list:
    volatility = float(features.get("volatility", 0.0))
    trend_strength = float(features.get("trend_strength", 0.0))
    seasonality_strength = float(features.get("seasonality_strength", 0.0))
    stationary = bool(features.get("stationary", False))
    outlier_ratio = float(features.get("outlier_ratio", 0.0))
    autocorr = float(features.get("autocorr_strength", 0.0))
    weights = {"baseline": 0.10, "prophet": 0.30, "ets": 0.25, "sarima": 0.20, "xgb": 0.15}
    if trend_strength >= 0.60:
        weights["prophet"] += 0.08
        weights["ets"] -= 0.04
    if seasonality_strength >= 0.40:
        weights["sarima"] += 0.10
        weights["prophet"] += 0.05
        weights["baseline"] -= 0.05
    if volatility >= 0.40:
        weights["xgb"] += 0.10
        weights["ets"] -= 0.05
    if stationary and autocorr >= 0.40:
        weights["sarima"] += 0.05
    if outlier_ratio >= 0.15:
        weights["xgb"] += 0.05
        weights["baseline"] -= 0.05
    total = sum(max(0.0, v) for v in weights.values())
    weights = {k: max(0.0, v) / total for k, v in weights.items()}
    base_fb = baseline_result if baseline_result else [0.0] * n_ahead
    result = []
    for i in range(n_ahead):
        b = base_fb[i] if i < len(base_fb) else 0.0
        vals = {
            "baseline": b,
            "prophet": prophet_result[i] if prophet_result and i < len(prophet_result) else b,
            "ets": ets_result[i] if ets_result and i < len(ets_result) else b,
            "sarima": sarima_result[i] if sarima_result and i < len(sarima_result) else b,
            "xgb": xgb_result[i] if xgb_result and i < len(xgb_result) else b,
        }
        result.append(round(max(0.0, float(sum(vals[k] * weights[k] for k in weights))), 2))
    return result


def _ensemble_forecasts_expense(
    baseline_result: list,
    prophet_result: Optional[list],
    ets_result: Optional[list],
    sarima_result: Optional[list],
    xgb_result: Optional[list],
    features: Dict[str, Any],
    n_ahead: int = 3,
) -> list:
    volatility = float(features.get("volatility", 0.0))
    trend_strength = float(features.get("trend_strength", 0.0))
    seasonality_strength = float(features.get("seasonality_strength", 0.0))
    outlier_ratio = float(features.get("outlier_ratio", 0.0))
    weights = {"baseline": 0.10, "prophet": 0.15, "ets": 0.40, "sarima": 0.25, "xgb": 0.10}
    if seasonality_strength >= 0.35:
        weights["sarima"] += 0.10
        weights["ets"] += 0.05
        weights["prophet"] -= 0.05
    if trend_strength >= 0.50:
        weights["ets"] += 0.05
        weights["baseline"] -= 0.05
    if volatility >= 0.35:
        weights["xgb"] += 0.10
        weights["ets"] -= 0.05
    if outlier_ratio >= 0.15:
        weights["xgb"] += 0.05
        weights["sarima"] -= 0.05
    total = sum(max(0.0, v) for v in weights.values())
    weights = {k: max(0.0, v) / total for k, v in weights.items()}
    base_fb = baseline_result if baseline_result else [0.0] * n_ahead
    result = []
    for i in range(n_ahead):
        b = base_fb[i] if i < len(base_fb) else 0.0
        vals = {
            "baseline": b,
            "prophet": prophet_result[i] if prophet_result and i < len(prophet_result) else b,
            "ets": ets_result[i] if ets_result and i < len(ets_result) else b,
            "sarima": sarima_result[i] if sarima_result and i < len(sarima_result) else b,
            "xgb": xgb_result[i] if xgb_result and i < len(xgb_result) else b,
        }
        result.append(round(max(0.0, float(sum(vals[k] * weights[k] for k in weights))), 2))
    return result


def _ensemble_forecasts_cashflow(
    baseline_result: list,
    prophet_result: Optional[list],
    ets_result: Optional[list],
    sarima_result: Optional[list],
    arima_result: Optional[list],
    xgb_result: Optional[list],
    features: Dict[str, Any],
    n_ahead: int = 3,
) -> list:
    volatility = float(features.get("volatility", 0.0))
    trend_strength = float(features.get("trend_strength", 0.0))
    seasonality_strength = float(features.get("seasonality_strength", 0.0))
    stationary = bool(features.get("stationary", False))
    autocorr = float(features.get("autocorr_strength", 0.0))
    outlier_ratio = float(features.get("outlier_ratio", 0.0))
    weights = {"baseline": 0.10, "prophet": 0.20, "ets": 0.15, "sarima": 0.30, "arima": 0.10, "xgb": 0.15}
    if volatility >= 0.45:
        weights["xgb"] += 0.12
        weights["sarima"] += 0.05
        weights["ets"] -= 0.10
    if seasonality_strength >= 0.35:
        weights["sarima"] += 0.10
        weights["prophet"] += 0.05
        weights["baseline"] -= 0.05
    if stationary and autocorr >= 0.35:
        weights["arima"] += 0.08
        weights["sarima"] += 0.05
        weights["baseline"] -= 0.05
    if trend_strength >= 0.55:
        weights["prophet"] += 0.08
        weights["ets"] += 0.05
    if outlier_ratio >= 0.20:
        weights["xgb"] += 0.08
        weights["arima"] -= 0.05
    total = sum(max(0.0, v) for v in weights.values())
    weights = {k: max(0.0, v) / total for k, v in weights.items()}
    base_fb = baseline_result if baseline_result else [0.0] * n_ahead
    result = []
    for i in range(n_ahead):
        b = base_fb[i] if i < len(base_fb) else 0.0
        vals = {
            "baseline": b,
            "prophet": prophet_result[i] if prophet_result and i < len(prophet_result) else b,
            "ets": ets_result[i] if ets_result and i < len(ets_result) else b,
            "sarima": sarima_result[i] if sarima_result and i < len(sarima_result) else b,
            "arima": arima_result[i] if arima_result and i < len(arima_result) else b,
            "xgb": xgb_result[i] if xgb_result and i < len(xgb_result) else b,
        }
        result.append(round(float(sum(vals[k] * weights[k] for k in weights)), 2))
    return result


def _ensemble_forecasts_capex(
    baseline_result: list,
    ets_result: Optional[list],
    arima_result: Optional[list],
    xgb_result: Optional[list],
    features: Dict[str, Any],
    n_ahead: int = 3,
) -> list:
    volatility = float(features.get("volatility", 0.0))
    trend_strength = float(features.get("trend_strength", 0.0))
    outlier_ratio = float(features.get("outlier_ratio", 0.0))
    stability_score = float(features.get("stability_score", 0.5))
    weights = {"baseline": 0.20, "ets": 0.40, "arima": 0.25, "xgb": 0.15}
    if outlier_ratio >= 0.20:
        weights["xgb"] += 0.12
        weights["ets"] -= 0.08
    if trend_strength >= 0.50:
        weights["ets"] += 0.08
        weights["arima"] += 0.05
        weights["baseline"] -= 0.08
    if volatility >= 0.50:
        weights["xgb"] += 0.10
        weights["ets"] -= 0.05
    if stability_score >= 0.70:
        weights["ets"] += 0.08
        weights["xgb"] -= 0.05
    total = sum(max(0.0, v) for v in weights.values())
    weights = {k: max(0.0, v) / total for k, v in weights.items()}
    base_fb = baseline_result if baseline_result else [0.0] * n_ahead
    result = []
    for i in range(n_ahead):
        b = base_fb[i] if i < len(base_fb) else 0.0
        vals = {
            "baseline": b,
            "ets": ets_result[i] if ets_result and i < len(ets_result) else b,
            "arima": arima_result[i] if arima_result and i < len(arima_result) else b,
            "xgb": xgb_result[i] if xgb_result and i < len(xgb_result) else b,
        }
        result.append(round(max(0.0, float(sum(vals[k] * weights[k] for k in weights))), 2))
    return result


def _ensemble_forecasts_payroll(
    baseline_result: list,
    ets_result: Optional[list],
    sarima_result: Optional[list],
    prophet_result: Optional[list],
    features: Dict[str, Any],
    n_ahead: int = 3,
) -> list:
    volatility = float(features.get("volatility", 0.0))
    seasonality_strength = float(features.get("seasonality_strength", 0.0))
    trend_strength = float(features.get("trend_strength", 0.0))
    stability_score = float(features.get("stability_score", 0.5))
    weights = {"baseline": 0.10, "ets": 0.50, "sarima": 0.25, "prophet": 0.15}
    if seasonality_strength >= 0.30:
        weights["sarima"] += 0.10
        weights["ets"] -= 0.05
    if stability_score >= 0.70:
        weights["ets"] += 0.08
        weights["baseline"] -= 0.05
    if trend_strength >= 0.50:
        weights["ets"] += 0.05
        weights["prophet"] += 0.05
        weights["baseline"] -= 0.05
    if volatility >= 0.30:
        weights["sarima"] += 0.05
        weights["ets"] -= 0.05
    total = sum(max(0.0, v) for v in weights.values())
    weights = {k: max(0.0, v) / total for k, v in weights.items()}
    base_fb = baseline_result if baseline_result else [0.0] * n_ahead
    result = []
    for i in range(n_ahead):
        b = base_fb[i] if i < len(base_fb) else 0.0
        vals = {
            "baseline": b,
            "ets": ets_result[i] if ets_result and i < len(ets_result) else b,
            "sarima": sarima_result[i] if sarima_result and i < len(sarima_result) else b,
            "prophet": prophet_result[i] if prophet_result and i < len(prophet_result) else b,
        }
        result.append(round(max(0.0, float(sum(vals[k] * weights[k] for k in weights))), 2))
    return result


def _forecast_component_sync(component_rows: pd.DataFrame, quarter_start_str: str) -> Tuple[float, float, float]:
    if component_rows.empty:
        return 0.0, 0.0, 0.0
    cdf = component_rows.copy()
    cdf["ds"] = pd.to_datetime(cdf["period"] + "-01")
    cdf = cdf.sort_values("ds")
    values = cdf["y"].values.astype(float)
    if len(values) == 0:
        return 0.0, 0.0, 0.0
    if len(values) == 1:
        val = round(float(values[-1]), 2)
        return val, val, val
    clean_values = _winsorize_values(values)
    features = _feature_engineering(clean_values)
    baseline_result = _baseline_forecast(clean_values)
    start_date = str(cdf["ds"].iloc[0].date())
    prophet_result = _prophet_forecast_sync(clean_values, start_date, 3)
    ets_result = _ets_forecast_sync(clean_values, 3)
    arima_result = _arima_forecast_sync(clean_values, 3)
    sarima_result = _sarima_forecast_sync(clean_values, 3)
    xgb_result = _xgboost_forecast_sync(clean_values, 3)
    final_preds = _ensemble_forecasts_cashflow(
        baseline_result, prophet_result, ets_result, sarima_result, arima_result, xgb_result, features, 3
    )
    final_preds = _validate_forecast(final_preds, clean_values)
    final_preds = _stabilize_forecasts(final_preds, float(clean_values[-1]), features["volatility"])
    return float(final_preds[0]), float(final_preds[1]), float(final_preds[2])


async def _forecast_all_components(hist_df: pd.DataFrame, m1: str) -> Dict[str, Tuple[float, float, float]]:
    loop = asyncio.get_event_loop()
    quarter_start_str = str(pd.to_datetime(m1 + "-01").date())
    tasks = []
    keys = []
    for component, column_name in COMPONENT_MAP.items():
        if column_name not in hist_df.columns:
            tasks.append(loop.run_in_executor(_executor, _forecast_component_sync, pd.DataFrame(), quarter_start_str))
        else:
            component_df = hist_df[["period", column_name]].copy()
            component_df = component_df.rename(columns={column_name: "y"})
            component_df["y"] = pd.to_numeric(component_df["y"], errors="coerce").fillna(0.0)
            tasks.append(loop.run_in_executor(_executor, _forecast_component_sync, component_df, quarter_start_str))
        keys.append(component)
    results = await asyncio.gather(*tasks)
    return {key: result for key, result in zip(keys, results)}


def _forecast_component_n_months_sync(component_rows: pd.DataFrame, forecast_start_str: str, n_ahead: int) -> List[float]:
    """N-month cashflow component forecast — mirrors _forecast_component_sync but returns n_ahead values.
    Uses all existing ML components (prophet, ets, arima, linear, xgboost) with n_ahead parameter."""
    if component_rows.empty:
        return [0.0] * n_ahead
    cdf = component_rows.copy()
    if "ds" not in cdf.columns:
        cdf["ds"] = pd.to_datetime(cdf["period"] + "-01")
    cdf = cdf.sort_values("ds")
    values = cdf["y"].values.astype(float)
    if len(values) == 0:
        return [0.0] * n_ahead
    if len(values) == 1:
        return [round(float(values[-1]), 2)] * n_ahead
    clean_values = _winsorize_values(values)
    features = _feature_engineering(clean_values)
    start_date = str(cdf["ds"].iloc[0].date())
    prophet_result = _prophet_forecast_sync(clean_values, start_date, n_ahead)
    ets_result = _ets_forecast_sync(clean_values, n_ahead)
    arima_result = _arima_forecast_sync(clean_values, n_ahead)
    sarima_result = _sarima_forecast_sync(clean_values, n_ahead)
    xgb_result = _xgboost_forecast_sync(clean_values, n_ahead)
    baseline_3 = _baseline_forecast(clean_values)
    result = _ensemble_forecasts_cashflow(
        baseline_3, prophet_result, ets_result, sarima_result, arima_result, xgb_result, features, n_ahead
    )
    result = _validate_forecast(result, clean_values)
    result = _stabilize_forecasts(result, float(clean_values[-1]), features["volatility"])
    return result


async def _forecast_all_components_n_months(hist_df: pd.DataFrame, m1: str, n_months: int) -> Dict[str, List[float]]:
    """N-month version of _forecast_all_components. Returns n_months AI values per cashflow component."""
    loop = asyncio.get_event_loop()
    tasks = []
    keys = []
    for component, column_name in COMPONENT_MAP.items():
        if column_name not in hist_df.columns:
            tasks.append(loop.run_in_executor(_executor, _forecast_component_n_months_sync, pd.DataFrame(), m1, n_months))
        else:
            component_df = hist_df[["period", column_name]].copy()
            component_df = component_df.rename(columns={column_name: "y"})
            component_df["y"] = pd.to_numeric(component_df["y"], errors="coerce").fillna(0.0)
            tasks.append(loop.run_in_executor(_executor, _forecast_component_n_months_sync, component_df, m1, n_months))
        keys.append(component)
    results = await asyncio.gather(*tasks)
    return {key: result for key, result in zip(keys, results)}


HISTORY_SQL = text("""
    WITH base AS (
        SELECT
            gl.period,
            CASE WHEN coa.account_type IN ('Revenue', 'Other Income') THEN gl.amount_credit - gl.amount_debit ELSE 0 END AS operating_income,
            CASE WHEN coa.account_type IN ('COGS','Selling Expense','General Expense','Administrative Expense','Payroll Expense','Marketing Expense','Depreciation Expense','Tax Expense','Other Expense','Expense') THEN gl.amount_debit - gl.amount_credit ELSE 0 END AS operating_expense,
            CASE WHEN coa.account_type = 'Asset' THEN gl.amount_debit ELSE 0 END AS investing_cf,
            CASE WHEN coa.account_type = 'Liability' THEN gl.amount_credit - gl.amount_debit ELSE 0 END AS financing_cf,
            CASE WHEN LOWER(coa.account_name) LIKE '%cash%' THEN gl.amount_credit - gl.amount_debit ELSE 0 END AS net_cash_change
        FROM fpna_landing.finance_general_ledger_transactions gl
        JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
        WHERE gl.period >= :history_start AND gl.period < :quarter_start
    )
    SELECT period,
           SUM(operating_income) - SUM(operating_expense) AS operating_cf,
           SUM(investing_cf) AS investing_cf,
           SUM(financing_cf) AS financing_cf,
           SUM(net_cash_change) AS net_cash_change
    FROM base
    GROUP BY period
    ORDER BY period
""")

PREV_MONTH_SQL = text("""
    WITH base AS (
        SELECT
            CASE WHEN coa.account_type IN ('Revenue', 'Other Income') THEN gl.amount_credit - gl.amount_debit ELSE 0 END AS operating_income,
            CASE WHEN coa.account_type IN ('COGS','Selling Expense','General Expense','Administrative Expense','Payroll Expense','Marketing Expense','Depreciation Expense','Tax Expense','Other Expense','Expense') THEN gl.amount_debit - gl.amount_credit ELSE 0 END AS operating_expense,
            CASE WHEN coa.account_type = 'Asset' THEN gl.amount_debit ELSE 0 END AS investing_cf,
            CASE WHEN coa.account_type = 'Liability' THEN gl.amount_credit - gl.amount_debit ELSE 0 END AS financing_cf,
            CASE WHEN LOWER(coa.account_name) LIKE '%cash%' THEN gl.amount_credit - gl.amount_debit ELSE 0 END AS net_cash_change
        FROM fpna_landing.finance_general_ledger_transactions gl
        JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
        WHERE gl.period = :prev_month
    )
    SELECT
        SUM(operating_income) - SUM(operating_expense) AS operating_cf,
        SUM(investing_cf) AS investing_cf,
        SUM(financing_cf) AS financing_cf,
        SUM(net_cash_change) AS net_cash_change
    FROM base
""")


async def _ensure_cashflow_workflow_tables(db: AsyncSession) -> None:
    await db.execute(text("CREATE SCHEMA IF NOT EXISTS fpna_config;"))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.cashflow_forecast_adjustments (
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            component TEXT NOT NULL,
            month1_adjusted NUMERIC NULL,
            month1_comment TEXT NULL,
            month2_adjusted NUMERIC NULL,
            month2_comment TEXT NULL,
            month3_adjusted NUMERIC NULL,
            month3_comment TEXT NULL,
            overall_comment TEXT NULL,
            updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT NOW()
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_forecast_adjustments' AND column_name='overall_comment') THEN
                ALTER TABLE fpna_config.cashflow_forecast_adjustments ADD COLUMN overall_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_forecast_adjustments' AND column_name='month1_comment') THEN
                ALTER TABLE fpna_config.cashflow_forecast_adjustments ADD COLUMN month1_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_forecast_adjustments' AND column_name='month2_comment') THEN
                ALTER TABLE fpna_config.cashflow_forecast_adjustments ADD COLUMN month2_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_forecast_adjustments' AND column_name='month3_comment') THEN
                ALTER TABLE fpna_config.cashflow_forecast_adjustments ADD COLUMN month3_comment TEXT NULL;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE UNIQUE INDEX IF NOT EXISTS uix_cashflow_forecast_adjustments
        ON fpna_config.cashflow_forecast_adjustments (year, quarter, company_id, component);
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.cashflow_driver_adjustments (
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            component TEXT NOT NULL,
            month TEXT NOT NULL,
            units_adjusted NUMERIC NULL,
            rate_adjusted NUMERIC NULL,
            month_comment TEXT NULL,
            overall_comment TEXT NULL,
            updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT NOW()
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_driver_adjustments' AND column_name='component') THEN
                ALTER TABLE fpna_config.cashflow_driver_adjustments ADD COLUMN component TEXT NOT NULL DEFAULT '';
            END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_driver_adjustments' AND column_name='month') THEN
                ALTER TABLE fpna_config.cashflow_driver_adjustments ADD COLUMN month TEXT NOT NULL DEFAULT '';
            END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_driver_adjustments' AND column_name='units_adjusted') THEN
                ALTER TABLE fpna_config.cashflow_driver_adjustments ADD COLUMN units_adjusted NUMERIC NULL;
            END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_driver_adjustments' AND column_name='rate_adjusted') THEN
                ALTER TABLE fpna_config.cashflow_driver_adjustments ADD COLUMN rate_adjusted NUMERIC NULL;
            END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_driver_adjustments' AND column_name='month_comment') THEN
                ALTER TABLE fpna_config.cashflow_driver_adjustments ADD COLUMN month_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_driver_adjustments' AND column_name='overall_comment') THEN
                ALTER TABLE fpna_config.cashflow_driver_adjustments ADD COLUMN overall_comment TEXT NULL;
            END IF;
            IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_driver_adjustments' AND column_name='amount_adjusted') THEN
                ALTER TABLE fpna_config.cashflow_driver_adjustments DROP COLUMN IF EXISTS amount_adjusted;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE UNIQUE INDEX IF NOT EXISTS uix_cashflow_driver_adjustments
        ON fpna_config.cashflow_driver_adjustments (year, quarter, company_id, component, month);
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.cashflow_forecast_workflows (
            id SERIAL PRIMARY KEY,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            department TEXT NOT NULL,
            owner_user_id TEXT NOT NULL,
            stage TEXT NOT NULL DEFAULT 'draft',
            created_at TIMESTAMP NOT NULL DEFAULT NOW(),
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            locked_at TIMESTAMP NULL,
            UNIQUE (year, quarter, company_id)
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_forecast_workflows' AND column_name='owner_user_id' AND data_type != 'text') THEN
                ALTER TABLE fpna_config.cashflow_forecast_workflows ALTER COLUMN owner_user_id TYPE TEXT USING owner_user_id::TEXT;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.cashflow_workflow_audit (
            id SERIAL PRIMARY KEY,
            workflow_id INT NOT NULL,
            from_stage TEXT NULL,
            to_stage TEXT NOT NULL,
            transitioned_by_user_id TEXT NOT NULL,
            transitioned_by_name TEXT NOT NULL,
            comment TEXT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.cashflow_workflow_comments (
            id SERIAL PRIMARY KEY,
            workflow_id INT NOT NULL,
            user_id TEXT NOT NULL,
            user_name TEXT NOT NULL,
            department TEXT NOT NULL,
            comment TEXT NOT NULL,
            comment_type TEXT NOT NULL DEFAULT 'general',
            created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_workflow_comments' AND column_name='comment_type') THEN
                ALTER TABLE fpna_config.cashflow_workflow_comments ADD COLUMN comment_type TEXT NOT NULL DEFAULT 'general';
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.cashflow_forecast_scenarios (
            id SERIAL PRIMARY KEY,
            company_id TEXT NOT NULL,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            scenario_name TEXT NOT NULL,
            multiplier NUMERIC NOT NULL,
            description TEXT NULL,
            created_by_user_id TEXT NOT NULL,
            created_by_name TEXT NOT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW(),
            updated_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_forecast_scenarios' AND column_name='parameter_types') THEN
                ALTER TABLE fpna_config.cashflow_forecast_scenarios ADD COLUMN parameter_types JSONB NOT NULL DEFAULT '["cashflow"]';
            END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_forecast_scenarios' AND column_name='customer_scope') THEN
                ALTER TABLE fpna_config.cashflow_forecast_scenarios ADD COLUMN customer_scope TEXT NOT NULL DEFAULT 'all';
            END IF;
            IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='fpna_config' AND table_name='cashflow_forecast_scenarios' AND column_name='component_multipliers') THEN
                ALTER TABLE fpna_config.cashflow_forecast_scenarios ADD COLUMN component_multipliers JSONB NULL;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE UNIQUE INDEX IF NOT EXISTS uix_cashflow_forecast_scenarios
        ON fpna_config.cashflow_forecast_scenarios (company_id, year, quarter, scenario_name);
    """))
    await db.commit()


async def _get_or_create_cashflow_workflow(
    db: AsyncSession, year: int, quarter: str, company_id: str, department: str, user_id: str
) -> Dict[str, Any]:
    row = (await db.execute(
        text("SELECT * FROM fpna_config.cashflow_forecast_workflows WHERE year=:year AND quarter=:quarter AND company_id=:company_id ORDER BY id ASC LIMIT 1"),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().first()
    if not row:
        row = (await db.execute(
            text("INSERT INTO fpna_config.cashflow_forecast_workflows (year, quarter, company_id, department, owner_user_id, stage) VALUES (:year, :quarter, :company_id, :dept, :uid, 'draft') RETURNING *"),
            {"year": year, "quarter": quarter, "company_id": company_id, "dept": department, "uid": user_id},
        )).mappings().first()
        await db.commit()
    return dict(row)


async def _get_cashflow_adjustments(db: AsyncSession, year: int, quarter: str, company_id: str) -> List[Dict[str, Any]]:
    rows = (await db.execute(
        text("""
            SELECT component,
                   month1_adjusted, month1_comment,
                   month2_adjusted, month2_comment,
                   month3_adjusted, month3_comment,
                   overall_comment
            FROM fpna_config.cashflow_forecast_adjustments
            WHERE year=:year AND quarter=:quarter AND company_id=:company_id
            ORDER BY component ASC
        """),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().all()
    result = []
    for r in rows:
        m1 = float(r["month1_adjusted"]) if r["month1_adjusted"] is not None else None
        m2 = float(r["month2_adjusted"]) if r["month2_adjusted"] is not None else None
        m3 = float(r["month3_adjusted"]) if r["month3_adjusted"] is not None else None
        quarter_total = round((m1 or 0.0) + (m2 or 0.0) + (m3 or 0.0), 2)
        result.append({
            "component": r["component"],
            "month1_adjusted": m1,
            "month1_comment": r["month1_comment"],
            "month2_adjusted": m2,
            "month2_comment": r["month2_comment"],
            "month3_adjusted": m3,
            "month3_comment": r["month3_comment"],
            "quarter_total_adjusted": quarter_total,
            "overall_comment": r["overall_comment"],
        })
    return result


async def _get_cashflow_driver_adjustments(db: AsyncSession, year: int, quarter: str, company_id: str) -> Dict[tuple, Dict]:
    rows = (await db.execute(
        text("""
            SELECT component, month, units_adjusted, rate_adjusted, month_comment, overall_comment
            FROM fpna_config.cashflow_driver_adjustments
            WHERE year=:year AND quarter=:quarter AND company_id=:company_id
        """),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().all()
    return {
        (r["component"], r["month"]): {
            "units_adjusted": float(r["units_adjusted"]) if r["units_adjusted"] is not None else None,
            "rate_adjusted": float(r["rate_adjusted"]) if r["rate_adjusted"] is not None else None,
            "month_comment": r["month_comment"],
            "overall_comment": r["overall_comment"],
        }
        for r in rows
    }


async def _get_cashflow_custom_scenarios(db: AsyncSession, year: int, quarter: str, company_id: str) -> List[Dict[str, Any]]:
    rows = (await db.execute(
        text("""
            SELECT id, scenario_name, multiplier, description, parameter_types, customer_scope,
                   component_multipliers, created_by_name, created_at, updated_at
            FROM fpna_config.cashflow_forecast_scenarios
            WHERE company_id=:company_id AND year=:year AND quarter=:quarter
            ORDER BY created_at ASC
        """),
        {"company_id": company_id, "year": year, "quarter": quarter},
    )).mappings().all()
    return [
        {
            "id": r["id"],
            "scenario_name": r["scenario_name"],
            "multiplier": float(r["multiplier"]),
            "description": r["description"],
            "parameter_types": r["parameter_types"] if r["parameter_types"] else ["cashflow"],
            "customer_scope": r["customer_scope"] if r["customer_scope"] else "all",
            "component_multipliers": r["component_multipliers"] if r["component_multipliers"] else {},
            "created_by_name": r["created_by_name"],
            "created_at": r["created_at"].isoformat(),
            "updated_at": r["updated_at"].isoformat(),
        }
        for r in rows
    ]


async def _save_cashflow_adjustments(
    db: AsyncSession, year: int, quarter: str, company_id: str,
    adjustments: List[CashflowComponentAdjustmentInput],
) -> Dict[str, Any]:
    for adj in adjustments:
        if not adj.component:
            continue
        await db.execute(
            text("""
                INSERT INTO fpna_config.cashflow_forecast_adjustments (
                    year, quarter, company_id, component,
                    month1_adjusted, month1_comment,
                    month2_adjusted, month2_comment,
                    month3_adjusted, month3_comment,
                    overall_comment, updated_at
                )
                VALUES (
                    :year, :quarter, :company_id, :component,
                    :month1_adjusted, :month1_comment,
                    :month2_adjusted, :month2_comment,
                    :month3_adjusted, :month3_comment,
                    :overall_comment, NOW()
                )
                ON CONFLICT (year, quarter, company_id, component)
                DO UPDATE SET
                    month1_adjusted = EXCLUDED.month1_adjusted,
                    month1_comment  = EXCLUDED.month1_comment,
                    month2_adjusted = EXCLUDED.month2_adjusted,
                    month2_comment  = EXCLUDED.month2_comment,
                    month3_adjusted = EXCLUDED.month3_adjusted,
                    month3_comment  = EXCLUDED.month3_comment,
                    overall_comment = EXCLUDED.overall_comment,
                    updated_at      = NOW()
            """),
            {
                "year": year, "quarter": quarter, "company_id": company_id,
                "component": adj.component,
                "month1_adjusted": adj.month1_adjusted, "month1_comment": adj.month1_comment,
                "month2_adjusted": adj.month2_adjusted, "month2_comment": adj.month2_comment,
                "month3_adjusted": adj.month3_adjusted, "month3_comment": adj.month3_comment,
                "overall_comment": adj.overall_comment,
            },
        )
    await db.commit()
    final = await _get_cashflow_adjustments(db, year, quarter, company_id)
    return {
        "detail": "Cashflow forecast adjustments saved successfully",
        "year": year, "quarter": quarter,
        "count": len(adjustments),
        "adjustments": final,
    }


@forecast_router.patch("/cashflow-forecast/adjustments/{year}")
async def patch_cashflow_forecast_adjustments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    adjustments: List[CashflowComponentAdjustmentInput] = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_cashflow_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Forecast is locked. Adjustments cannot be changed.")
    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Adjustments can only be edited in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")
    return await _save_cashflow_adjustments(year=year, quarter=quarter.value, company_id=company_id, adjustments=adjustments, db=db)


@forecast_router.get("/cashflow-forecast/adjustments/{year}")
async def get_cashflow_forecast_adjustments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    company_id = get_company_id(current_user)
    return await _get_cashflow_adjustments(db, year, quarter.value, company_id)


@forecast_router.patch("/cashflow-forecast/driver-adjustments/{year}")
async def patch_cashflow_driver_adjustments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    adjustments: List[CashflowDriverAdjustmentInput] = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_cashflow_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Forecast is locked. Driver adjustments cannot be changed.")
    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Driver adjustments can only be edited in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")
    for adj in adjustments:
        if not adj.component or not adj.month:
            continue
        await db.execute(
            text("""
                INSERT INTO fpna_config.cashflow_driver_adjustments (
                    year, quarter, company_id, component, month,
                    units_adjusted, rate_adjusted, month_comment, overall_comment, updated_at
                )
                VALUES (:year, :quarter, :company_id, :component, :month,
                        :units_adjusted, :rate_adjusted, :month_comment, :overall_comment, NOW())
                ON CONFLICT (year, quarter, company_id, component, month)
                DO UPDATE SET
                    units_adjusted  = EXCLUDED.units_adjusted,
                    rate_adjusted   = EXCLUDED.rate_adjusted,
                    month_comment   = EXCLUDED.month_comment,
                    overall_comment = EXCLUDED.overall_comment,
                    updated_at      = NOW()
            """),
            {
                "year": year, "quarter": quarter.value, "company_id": company_id,
                "component": adj.component, "month": adj.month,
                "units_adjusted": adj.units_adjusted,
                "rate_adjusted": adj.rate_adjusted,
                "month_comment": adj.month_comment, "overall_comment": adj.overall_comment,
            },
        )
    await db.commit()
    saved = (await db.execute(
        text("SELECT component, month, units_adjusted, rate_adjusted, month_comment, overall_comment FROM fpna_config.cashflow_driver_adjustments WHERE year=:year AND quarter=:quarter AND company_id=:company_id ORDER BY component, month"),
        {"year": year, "quarter": quarter.value, "company_id": company_id},
    )).mappings().all()
    component_quarter_map: Dict[str, float] = {}
    result_rows = []
    for r in saved:
        u = float(r["units_adjusted"]) if r["units_adjusted"] is not None else None
        ra = float(r["rate_adjusted"]) if r["rate_adjusted"] is not None else None
        rev_adj = round(u * ra, 2) if u is not None and ra is not None else None
        comp = r["component"]
        if rev_adj is not None:
            component_quarter_map[comp] = component_quarter_map.get(comp, 0.0) + rev_adj
        result_rows.append({
            "component": comp, "month": r["month"],
            "units_adjusted": u, "rate_adjusted": ra, "revenue_adjusted": rev_adj,
            "month_comment": r["month_comment"], "overall_comment": r["overall_comment"],
        })
    for row in result_rows:
        row["quarter_revenue_adjusted"] = round(component_quarter_map.get(row["component"], 0.0), 2)
    return {
        "detail": "Cashflow driver adjustments saved successfully",
        "year": year, "quarter": quarter.value, "count": len(adjustments), "driver_adjustments": result_rows,
    }


@forecast_router.patch("/cashflow-forecast/workflow/{year}")
async def patch_cashflow_forecast_workflow(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: WorkflowTransitionRequest = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_cashflow_workflow(db, year, quarter.value, company_id, dept, user_id)
    current_stage = wf["stage"]
    target_stage = body.target_stage
    effective_comment = body.comment.strip() if body.comment and body.comment.strip() else None
    is_rejection = target_stage == WorkflowStage.DRAFT and current_stage != WorkflowStage.DRAFT
    if is_rejection and not effective_comment:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="A comment is required when rejecting (sending back to draft). Please provide a reason.")
    if effective_comment and dept not in DEPT_REVIEWERS | DEPT_APPROVERS:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Only Finance/Executive/CEO/CFO can add review comments.")
    if not can_transition(dept, current_stage, target_stage):
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Department '{dept}' cannot move forecast from '{current_stage}' to '{target_stage}'.")
    if target_stage == WorkflowStage.LOCKED:
        await db.execute(text("UPDATE fpna_config.cashflow_forecast_workflows SET stage=:new_stage, updated_at=NOW(), locked_at=NOW() WHERE id=:wf_id"), {"new_stage": target_stage, "wf_id": wf["id"]})
    else:
        await db.execute(text("UPDATE fpna_config.cashflow_forecast_workflows SET stage=:new_stage, updated_at=NOW() WHERE id=:wf_id"), {"new_stage": target_stage, "wf_id": wf["id"]})
    await db.execute(
        text("INSERT INTO fpna_config.cashflow_workflow_audit (workflow_id, from_stage, to_stage, transitioned_by_user_id, transitioned_by_name, comment) VALUES (:wf_id, :from_s, :to_s, :uid, :uname, :comment)"),
        {"wf_id": wf["id"], "from_s": current_stage, "to_s": target_stage, "uid": user_id, "uname": current_user.get("full_name", ""), "comment": effective_comment},
    )
    if effective_comment and dept in DEPT_REVIEWERS | DEPT_APPROVERS:
        comment_type = "rejection" if is_rejection else "transition"
        await db.execute(
            text("INSERT INTO fpna_config.cashflow_workflow_comments (workflow_id, user_id, user_name, department, comment, comment_type) VALUES (:wf_id, :uid, :uname, :dept, :comment, :comment_type)"),
            {"wf_id": wf["id"], "uid": user_id, "uname": current_user.get("full_name", ""), "dept": dept, "comment": effective_comment, "comment_type": comment_type},
        )
    await db.commit()
    return {"detail": f"Cashflow forecast moved from '{current_stage}' to '{target_stage}'.", "year": year, "quarter": quarter.value, "department": dept, "stage": target_stage}


@forecast_router.post("/cashflow-forecast/{year}/comments")
async def post_cashflow_forecast_comment(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: CashflowForecastCommentInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    user_name = current_user.get("full_name", "")
    wf = await _get_or_create_cashflow_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Forecast is locked. Comments cannot be added.")
    comment_text = body.comment.strip() if body.comment else ""
    if not comment_text:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Comment cannot be empty.")
    result = (await db.execute(
        text("INSERT INTO fpna_config.cashflow_workflow_comments (workflow_id, user_id, user_name, department, comment, comment_type) VALUES (:wf_id, :uid, :uname, :dept, :comment, 'general') RETURNING id, user_id, user_name, department, comment, comment_type, created_at"),
        {"wf_id": wf["id"], "uid": user_id, "uname": user_name, "dept": dept, "comment": comment_text},
    )).mappings().first()
    await db.commit()
    return {
        "detail": "Comment posted successfully.",
        "year": year, "quarter": quarter.value, "workflow_stage": stage,
        "comment": {"id": result["id"], "user_id": result["user_id"], "user_name": result["user_name"], "department": result["department"], "comment": result["comment"], "comment_type": result["comment_type"], "created_at": result["created_at"].isoformat()},
    }


@forecast_router.get("/cashflow-forecast/{year}/comments")
async def get_cashflow_forecast_comments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    comment_type: Optional[str] = Query(None, description="Filter by comment_type: general, transition, rejection. Leave empty for all."),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_cashflow_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if comment_type:
        rows = (await db.execute(text("SELECT id, user_id, user_name, department, comment, comment_type, created_at FROM fpna_config.cashflow_workflow_comments WHERE workflow_id=:wf_id AND comment_type=:comment_type ORDER BY created_at ASC"), {"wf_id": wf["id"], "comment_type": comment_type})).mappings().all()
    else:
        rows = (await db.execute(text("SELECT id, user_id, user_name, department, comment, comment_type, created_at FROM fpna_config.cashflow_workflow_comments WHERE workflow_id=:wf_id ORDER BY created_at ASC"), {"wf_id": wf["id"]})).mappings().all()
    comments = [{"id": r["id"], "user_id": r["user_id"], "user_name": r["user_name"], "department": r["department"], "comment": r["comment"], "comment_type": r["comment_type"], "created_at": r["created_at"].isoformat()} for r in rows]
    return {"year": year, "quarter": quarter.value, "workflow_stage": stage, "total_comments": len(comments), "filters": {"comment_type": comment_type or "all"}, "comments": comments}


@forecast_router.post("/cashflow-forecast/{year}/scenarios")
async def create_cashflow_forecast_scenario(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: CashflowScenarioInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    user_name = current_user.get("full_name", "")
    scenario_name_clean = body.scenario_name.strip().lower().replace(" ", "_")
    if scenario_name_clean in RESERVED_SCENARIO_NAMES:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"'{body.scenario_name}' is a reserved scenario name. Reserved names are: baseline, best_case, worst_case.")
    if not (0.01 <= body.multiplier <= 10.0):
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Multiplier must be between 0.01 and 10.0.")
    existing = (await db.execute(
        text("SELECT id FROM fpna_config.cashflow_forecast_scenarios WHERE company_id=:company_id AND year=:year AND quarter=:quarter AND LOWER(REPLACE(scenario_name,' ','_'))=:scenario_name_clean"),
        {"company_id": company_id, "year": year, "quarter": quarter.value, "scenario_name_clean": scenario_name_clean},
    )).mappings().first()
    if existing:
        raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=f"A scenario named '{body.scenario_name}' already exists for {year} {quarter.value}.")
    result = (await db.execute(
        text("""
            INSERT INTO fpna_config.cashflow_forecast_scenarios
                (company_id, year, quarter, scenario_name, multiplier, description,
                 parameter_types, customer_scope, component_multipliers, created_by_user_id, created_by_name)
            VALUES (:company_id, :year, :quarter, :scenario_name, :multiplier, :description,
                    :parameter_types, :customer_scope, :component_multipliers, :uid, :uname)
            RETURNING id, scenario_name, multiplier, description, parameter_types,
                      customer_scope, component_multipliers, created_by_name, created_at, updated_at
        """),
        {
            "company_id": company_id, "year": year, "quarter": quarter.value,
            "scenario_name": body.scenario_name.strip(), "multiplier": body.multiplier,
            "description": body.description.strip() if body.description else None,
            "parameter_types": json.dumps(body.parameter_types if body.parameter_types else ["cashflow"]),
            "customer_scope": body.customer_scope if body.customer_scope else "all",
            "component_multipliers": json.dumps(body.component_multipliers) if body.component_multipliers else None,
            "uid": user_id, "uname": user_name,
        },
    )).mappings().first()
    await db.commit()
    return {
        "detail": "Cashflow scenario created successfully.",
        "year": year, "quarter": quarter.value,
        "scenario": {
            "id": result["id"], "scenario_name": result["scenario_name"],
            "multiplier": float(result["multiplier"]), "description": result["description"],
            "parameter_types": result["parameter_types"] if result["parameter_types"] else ["cashflow"],
            "customer_scope": result["customer_scope"],
            "component_multipliers": result["component_multipliers"] if result["component_multipliers"] else {},
            "created_by_name": result["created_by_name"],
            "created_at": result["created_at"].isoformat(), "updated_at": result["updated_at"].isoformat(),
        },
    }


@forecast_router.get("/cashflow-forecast/{year}/scenarios")
async def get_cashflow_forecast_scenarios(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    company_id = get_company_id(current_user)
    custom_scenarios = await _get_cashflow_custom_scenarios(db, year, quarter.value, company_id)
    for s in custom_scenarios:
        s["is_system"] = False
    hardcoded = [
        {"id": None, "scenario_name": "baseline", "multiplier": 1.0, "description": CASHFLOW_SCENARIO_DEFINITIONS["baseline"], "parameter_types": ["cashflow"], "customer_scope": "all", "component_multipliers": {}, "is_system": True, "created_by_name": None, "created_at": None, "updated_at": None},
        {"id": None, "scenario_name": "best_case", "multiplier": BEST_CASE_MULTIPLIER, "description": CASHFLOW_SCENARIO_DEFINITIONS["best_case"], "parameter_types": ["cashflow"], "customer_scope": "all", "component_multipliers": {}, "is_system": True, "created_by_name": None, "created_at": None, "updated_at": None},
        {"id": None, "scenario_name": "worst_case", "multiplier": WORST_CASE_MULTIPLIER, "description": CASHFLOW_SCENARIO_DEFINITIONS["worst_case"], "parameter_types": ["cashflow"], "customer_scope": "all", "component_multipliers": {}, "is_system": True, "created_by_name": None, "created_at": None, "updated_at": None},
    ]
    return {"year": year, "quarter": quarter.value, "total_scenarios": len(hardcoded) + len(custom_scenarios), "scenarios": hardcoded + custom_scenarios}


@forecast_router.patch("/cashflow-forecast/{year}/scenarios/{scenario_id}")
async def update_cashflow_forecast_scenario(
    year: int = Path(...),
    scenario_id: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: CashflowScenarioUpdateInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    company_id = get_company_id(current_user)
    existing = (await db.execute(
        text("SELECT id, scenario_name FROM fpna_config.cashflow_forecast_scenarios WHERE id=:sid AND company_id=:company_id AND year=:year AND quarter=:quarter"),
        {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
    )).mappings().first()
    if not existing:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Scenario with id {scenario_id} not found for {year} {quarter.value}.")
    if body.scenario_name is not None:
        new_name_clean = body.scenario_name.strip().lower().replace(" ", "_")
        if new_name_clean in RESERVED_SCENARIO_NAMES:
            raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"'{body.scenario_name}' is a reserved scenario name.")
        conflict = (await db.execute(
            text("SELECT id FROM fpna_config.cashflow_forecast_scenarios WHERE company_id=:company_id AND year=:year AND quarter=:quarter AND LOWER(REPLACE(scenario_name,' ','_'))=:new_name_clean AND id!=:sid"),
            {"company_id": company_id, "year": year, "quarter": quarter.value, "new_name_clean": new_name_clean, "sid": scenario_id},
        )).mappings().first()
        if conflict:
            raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=f"A scenario named '{body.scenario_name}' already exists for {year} {quarter.value}.")
    if body.multiplier is not None and not (0.01 <= body.multiplier <= 10.0):
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Multiplier must be between 0.01 and 10.0.")
    set_clauses = ["updated_at = NOW()"]
    params: Dict[str, Any] = {"sid": scenario_id}
    if body.scenario_name is not None:
        set_clauses.append("scenario_name = :scenario_name")
        params["scenario_name"] = body.scenario_name.strip()
    if body.multiplier is not None:
        set_clauses.append("multiplier = :multiplier")
        params["multiplier"] = body.multiplier
    if body.description is not None:
        set_clauses.append("description = :description")
        params["description"] = body.description.strip()
    if body.parameter_types is not None:
        set_clauses.append("parameter_types = :parameter_types")
        params["parameter_types"] = json.dumps(body.parameter_types)
    if body.customer_scope is not None:
        set_clauses.append("customer_scope = :customer_scope")
        params["customer_scope"] = body.customer_scope
    if body.component_multipliers is not None:
        set_clauses.append("component_multipliers = :component_multipliers")
        params["component_multipliers"] = json.dumps(body.component_multipliers)
    result = (await db.execute(
        text(f"UPDATE fpna_config.cashflow_forecast_scenarios SET {', '.join(set_clauses)} WHERE id=:sid RETURNING id, scenario_name, multiplier, description, parameter_types, customer_scope, component_multipliers, created_by_name, created_at, updated_at"),
        params,
    )).mappings().first()
    await db.commit()
    return {
        "detail": "Cashflow scenario updated successfully.",
        "year": year, "quarter": quarter.value,
        "scenario": {
            "id": result["id"], "scenario_name": result["scenario_name"],
            "multiplier": float(result["multiplier"]), "description": result["description"],
            "parameter_types": result["parameter_types"] if result["parameter_types"] else ["cashflow"],
            "customer_scope": result["customer_scope"],
            "component_multipliers": result["component_multipliers"] if result["component_multipliers"] else {},
            "created_by_name": result["created_by_name"],
            "created_at": result["created_at"].isoformat(), "updated_at": result["updated_at"].isoformat(),
        },
    }


@forecast_router.delete("/cashflow-forecast/{year}/scenarios/{scenario_id}")
async def delete_cashflow_forecast_scenario(
    year: int = Path(...),
    scenario_id: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    company_id = get_company_id(current_user)
    existing = (await db.execute(
        text("SELECT id, scenario_name FROM fpna_config.cashflow_forecast_scenarios WHERE id=:sid AND company_id=:company_id AND year=:year AND quarter=:quarter"),
        {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
    )).mappings().first()
    if not existing:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Scenario with id {scenario_id} not found for {year} {quarter.value}.")
    await db.execute(text("DELETE FROM fpna_config.cashflow_forecast_scenarios WHERE id=:sid"), {"sid": scenario_id})
    await db.commit()
    return {"detail": f"Cashflow scenario '{existing['scenario_name']}' deleted successfully.", "year": year, "quarter": quarter.value, "deleted_scenario_id": scenario_id}


@forecast_router.get("/cashflow-forecast/{year}/excel-template/adjustments")
async def download_cashflow_adjustments_template(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    company_id = get_company_id(current_user)
    m1, m2, m3 = get_quarter_months(year, quarter.value)

    saved = await _get_cashflow_adjustments(db, year, quarter.value, company_id)
    adj_map = {s["component"]: s for s in saved}

    prev_month_label = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=1)).strftime("%Y-%m")
    history_start_str = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")

    hist_result, prev_result = await asyncio.gather(
        db.execute(HISTORY_SQL, {"history_start": history_start_str, "quarter_start": m1}),
        db.execute(PREV_MONTH_SQL, {"prev_month": prev_month_label}),
    )
    hist_df = pd.DataFrame(hist_result.mappings().all())
    prev_row = prev_result.mappings().first()

    prev_values_map = {
        "operatingCF": round(float(prev_row["operating_cf"] or 0), 2) if prev_row else 0.0,
        "investingCF": round(float(prev_row["investing_cf"] or 0), 2) if prev_row else 0.0,
        "financingCF": round(float(prev_row["financing_cf"] or 0), 2) if prev_row else 0.0,
        "netCashChange": round(float(prev_row["net_cash_change"] or 0), 2) if prev_row else 0.0,
    }

    component_forecasts = await _forecast_all_components(hist_df, m1)

    wb = openpyxl.Workbook()
    ws = wb.active
    ws.title = "Cashflow Adjustments"
    headers = [
        "Component", f"Previous Month ({prev_month_label})",
        f"{m1} AI Forecast", f"{m1} Adjusted", f"{m1} Comment",
        f"{m2} AI Forecast", f"{m2} Adjusted", f"{m2} Comment",
        f"{m3} AI Forecast", f"{m3} Adjusted", f"{m3} Comment",
        "Quarter Total AI", "Quarter Total Adjusted", "Overall Comment",
    ]
    ws.append(headers)

    for comp in CASHFLOW_COMPONENTS:
        fc = component_forecasts.get(comp, (0.0, 0.0, 0.0))
        m1_ai = round(float(fc[0]), 2)
        m2_ai = round(float(fc[1]), 2)
        m3_ai = round(float(fc[2]), 2)
        quarter_total_ai = round(m1_ai + m2_ai + m3_ai, 2)
        adj = adj_map.get(comp, {})
        m1_adj = adj.get("month1_adjusted")
        m2_adj = adj.get("month2_adjusted")
        m3_adj = adj.get("month3_adjusted")
        quarter_total_adj = adj.get("quarter_total_adjusted", round((m1_adj or 0.0) + (m2_adj or 0.0) + (m3_adj or 0.0), 2))
        ws.append([
            comp, prev_values_map.get(comp, 0.0),
            m1_ai, m1_adj, adj.get("month1_comment"),
            m2_ai, m2_adj, adj.get("month2_comment"),
            m3_ai, m3_adj, adj.get("month3_comment"),
            quarter_total_ai,
            quarter_total_adj if (m1_adj is not None or m2_adj is not None or m3_adj is not None) else None,
            adj.get("overall_comment"),
        ])

    stream = io.BytesIO()
    wb.save(stream)
    stream.seek(0)
    return StreamingResponse(stream, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", headers={"Content-Disposition": f"attachment; filename=cashflow_adjustments_{year}_{quarter.value}.xlsx"})


@forecast_router.get("/cashflow-forecast/{year}/excel-template/driver-adjustments")
async def download_cashflow_driver_adjustments_template(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    company_id = get_company_id(current_user)
    m1, m2, m3 = get_quarter_months(year, quarter.value)

    driver_saved = await _get_cashflow_driver_adjustments(db, year, quarter.value, company_id)
    prev_month_label = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=1)).strftime("%Y-%m")
    history_start_str = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")

    hist_result = await db.execute(HISTORY_SQL, {"history_start": history_start_str, "quarter_start": m1})
    hist_df = pd.DataFrame(hist_result.mappings().all())

    component_forecasts = await _forecast_all_components(hist_df, m1)

    wb = openpyxl.Workbook()
    ws = wb.active
    ws.title = "Driver Adjustments"
    headers = [
        "Component", "Month", f"Previous Month ({prev_month_label}) Amount",
        "Units AI", "Rate AI", "Revenue AI",
        "Units Adjusted", "Rate Adjusted", "Revenue Adjusted",
        "Month Comment", "Quarter Revenue AI", "Quarter Revenue Adjusted", "Overall Comment",
    ]
    ws.append(headers)

    component_quarter_ai: Dict[str, float] = {}
    component_quarter_drv: Dict[str, float] = {}

    for comp in CASHFLOW_COMPONENTS:
        fc = component_forecasts.get(comp, (0.0, 0.0, 0.0))
        q_ai = round(float(fc[0]) + float(fc[1]) + float(fc[2]), 2)
        component_quarter_ai[comp] = q_ai
        q_drv = 0.0
        month_ai_map = {m1: float(fc[0]), m2: float(fc[1]), m3: float(fc[2])}
        for month_label in [m1, m2, m3]:
            drv = driver_saved.get((comp, month_label), {})
            u_ai, r_ai = derive_units_and_rate(month_ai_map[month_label])
            u_user = drv.get("units_adjusted")
            r_user = drv.get("rate_adjusted")
            if u_user is not None and r_user is not None:
                rev_adj = round(u_user * r_user, 2)
            elif u_user is not None:
                rev_adj = round(u_user * r_ai, 2)
            elif r_user is not None:
                rev_adj = round(u_ai * r_user, 2)
            else:
                rev_adj = None
            if rev_adj is not None:
                q_drv += rev_adj
        component_quarter_drv[comp] = round(q_drv, 2)

    for comp in CASHFLOW_COMPONENTS:
        fc = component_forecasts.get(comp, (0.0, 0.0, 0.0))
        month_ai_map = {m1: float(fc[0]), m2: float(fc[1]), m3: float(fc[2])}
        for month_label in [m1, m2, m3]:
            rev_ai = month_ai_map[month_label]
            u_ai, r_ai = derive_units_and_rate(rev_ai)
            drv = driver_saved.get((comp, month_label), {})
            u_user = drv.get("units_adjusted")
            r_user = drv.get("rate_adjusted")
            if u_user is not None and r_user is not None:
                rev_adj = round(u_user * r_user, 2)
            elif u_user is not None:
                rev_adj = round(u_user * r_ai, 2)
            elif r_user is not None:
                rev_adj = round(u_ai * r_user, 2)
            else:
                rev_adj = None
            ws.append([
                comp, month_label, 0.0,
                round(u_ai, 4), round(r_ai, 2), round(rev_ai, 2),
                u_user, r_user, rev_adj, drv.get("month_comment"),
                component_quarter_ai[comp],
                component_quarter_drv[comp] if component_quarter_drv[comp] else None,
                drv.get("overall_comment"),
            ])

    stream = io.BytesIO()
    wb.save(stream)
    stream.seek(0)
    return StreamingResponse(stream, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", headers={"Content-Disposition": f"attachment; filename=cashflow_driver_adjustments_{year}_{quarter.value}.xlsx"})


@forecast_router.post("/cashflow-forecast/{year}/excel-upload/adjustments")
async def upload_cashflow_adjustments_excel(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    file: UploadFile = File(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_cashflow_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Forecast is locked. Uploads cannot be processed.")
    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Uploads can only be processed in draft stage. Current stage is '{stage}'.")
    contents = await file.read()
    if len(contents) > 5 * 1024 * 1024:
        raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File size exceeds 5MB limit.")
    try:
        wb = openpyxl.load_workbook(io.BytesIO(contents))
    except Exception:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Could not read the uploaded file. Please ensure it is a valid .xlsx file.")
    m1, m2, m3 = get_quarter_months(year, quarter.value)
    adjustments = []
    errors = []

    def get_float_cell(row, idx):
        if idx is None or idx >= len(row) or row[idx] is None:
            return None
        try:
            return float(row[idx])
        except (TypeError, ValueError):
            return None

    def get_str_cell(row, idx):
        if idx is None or idx >= len(row) or row[idx] is None:
            return None
        val = str(row[idx]).strip()
        return val if val else None

    sheet_name = "Cashflow Adjustments"
    if sheet_name not in wb.sheetnames:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Sheet '{sheet_name}' not found in the uploaded file.")
    ws_adj = wb[sheet_name]
    adj_headers_row = [str(c.value).strip() if c.value is not None else "" for c in ws_adj[1]]
    adj_hmap = {h: i for i, h in enumerate(adj_headers_row)}
    for row_idx, row in enumerate(ws_adj.iter_rows(min_row=2, values_only=True), start=2):
        if not row or row[0] is None:
            continue
        component = str(row[0]).strip()
        if not component:
            continue
        if component not in CASHFLOW_COMPONENTS:
            errors.append(f"Row {row_idx}: unknown component '{component}'. Skipped.")
            continue
        try:
            m1_adj = get_float_cell(row, adj_hmap.get(f"{m1} Adjusted"))
            m1_cmt = get_str_cell(row, adj_hmap.get(f"{m1} Comment"))
            m2_adj = get_float_cell(row, adj_hmap.get(f"{m2} Adjusted"))
            m2_cmt = get_str_cell(row, adj_hmap.get(f"{m2} Comment"))
            m3_adj = get_float_cell(row, adj_hmap.get(f"{m3} Adjusted"))
            m3_cmt = get_str_cell(row, adj_hmap.get(f"{m3} Comment"))
            overall_comment = get_str_cell(row, adj_hmap.get("Overall Comment"))
        except Exception:
            errors.append(f"Cashflow Adjustments row {row_idx}: parse error for '{component}'. Skipped.")
            continue
        adjustments.append(CashflowComponentAdjustmentInput(
            component=component,
            month1_adjusted=m1_adj, month1_comment=m1_cmt,
            month2_adjusted=m2_adj, month2_comment=m2_cmt,
            month3_adjusted=m3_adj, month3_comment=m3_cmt,
            overall_comment=overall_comment,
        ))
    if not adjustments:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="No valid rows found in the Cashflow Adjustments sheet.")
    result = await _save_cashflow_adjustments(year=year, quarter=quarter.value, company_id=company_id, adjustments=adjustments, db=db)
    result["upload_errors"] = errors
    result["rows_attempted"] = len(adjustments) + len(errors)
    result["rows_saved"] = len(adjustments)
    return result


@forecast_router.post("/cashflow-forecast/{year}/excel-upload/driver-adjustments")
async def upload_cashflow_driver_adjustments_excel(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    file: UploadFile = File(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    wf = await _get_or_create_cashflow_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]
    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Forecast is locked. Uploads cannot be processed.")
    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Uploads can only be processed in draft stage. Current stage is '{stage}'.")
    contents = await file.read()
    if len(contents) > 5 * 1024 * 1024:
        raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File size exceeds 5MB limit.")
    try:
        wb = openpyxl.load_workbook(io.BytesIO(contents))
    except Exception:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Could not read the uploaded file. Please ensure it is a valid .xlsx file.")
    driver_adjustments = []
    errors = []

    def get_float_cell(row, idx):
        if idx is None or idx >= len(row) or row[idx] is None:
            return None
        try:
            return float(row[idx])
        except (TypeError, ValueError):
            return None

    def get_str_cell(row, idx):
        if idx is None or idx >= len(row) or row[idx] is None:
            return None
        val = str(row[idx]).strip()
        return val if val else None

    sheet_name = "Driver Adjustments"
    if sheet_name not in wb.sheetnames:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Sheet '{sheet_name}' not found in the uploaded file.")
    ws_drv = wb[sheet_name]
    drv_headers_row = [str(c.value).strip() if c.value is not None else "" for c in ws_drv[1]]
    drv_hmap = {h: i for i, h in enumerate(drv_headers_row)}
    for row_idx, row in enumerate(ws_drv.iter_rows(min_row=2, values_only=True), start=2):
        if not row or row[0] is None:
            continue
        component = str(row[0]).strip()
        if not component:
            continue
        if component not in CASHFLOW_COMPONENTS:
            errors.append(f"Row {row_idx}: unknown component '{component}'. Skipped.")
            continue
        try:
            month_val = get_str_cell(row, drv_hmap.get("Month"))
            units_adj = get_float_cell(row, drv_hmap.get("Units Adjusted"))
            rate_adj = get_float_cell(row, drv_hmap.get("Rate Adjusted"))
            month_cmt = get_str_cell(row, drv_hmap.get("Month Comment"))
            overall_comment = get_str_cell(row, drv_hmap.get("Overall Comment"))
        except Exception:
            errors.append(f"Driver Adjustments row {row_idx}: parse error for '{component}'. Skipped.")
            continue
        if not month_val:
            continue
        if any(v is not None for v in [units_adj, rate_adj, month_cmt, overall_comment]):
            driver_adjustments.append(CashflowDriverAdjustmentInput(
                component=component, month=month_val,
                units_adjusted=units_adj, rate_adjusted=rate_adj,
                month_comment=month_cmt, overall_comment=overall_comment,
            ))
    if not driver_adjustments:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="No valid rows found in the Driver Adjustments sheet.")
    for adj in driver_adjustments:
        if not adj.component or not adj.month:
            continue
        await db.execute(text("""
            INSERT INTO fpna_config.cashflow_driver_adjustments (year, quarter, company_id, component, month, units_adjusted, rate_adjusted, month_comment, overall_comment, updated_at)
            VALUES (:year, :quarter, :company_id, :component, :month, :units_adjusted, :rate_adjusted, :month_comment, :overall_comment, NOW())
            ON CONFLICT (year, quarter, company_id, component, month)
            DO UPDATE SET units_adjusted=EXCLUDED.units_adjusted, rate_adjusted=EXCLUDED.rate_adjusted, month_comment=EXCLUDED.month_comment, overall_comment=EXCLUDED.overall_comment, updated_at=NOW()
        """), {
            "year": year, "quarter": quarter.value, "company_id": company_id,
            "component": adj.component, "month": adj.month,
            "units_adjusted": adj.units_adjusted, "rate_adjusted": adj.rate_adjusted,
            "month_comment": adj.month_comment, "overall_comment": adj.overall_comment,
        })
    await db.commit()
    saved = (await db.execute(text("SELECT component, month, units_adjusted, rate_adjusted, month_comment, overall_comment FROM fpna_config.cashflow_driver_adjustments WHERE year=:year AND quarter=:quarter AND company_id=:company_id ORDER BY component, month"), {"year": year, "quarter": quarter.value, "company_id": company_id})).mappings().all()
    component_quarter_map: Dict[str, float] = {}
    result_rows = []
    for r in saved:
        u = float(r["units_adjusted"]) if r["units_adjusted"] is not None else None
        ra = float(r["rate_adjusted"]) if r["rate_adjusted"] is not None else None
        rev_adj = round(u * ra, 2) if u is not None and ra is not None else None
        comp = r["component"]
        if rev_adj is not None:
            component_quarter_map[comp] = component_quarter_map.get(comp, 0.0) + rev_adj
        result_rows.append({"component": comp, "month": r["month"], "units_adjusted": u, "rate_adjusted": ra, "revenue_adjusted": rev_adj, "month_comment": r["month_comment"], "overall_comment": r["overall_comment"]})
    for row in result_rows:
        row["quarter_revenue_adjusted"] = round(component_quarter_map.get(row["component"], 0.0), 2)
    return {"detail": "Cashflow driver adjustments uploaded successfully", "year": year, "quarter": quarter.value, "upload_errors": errors, "rows_attempted": len(driver_adjustments) + len(errors), "rows_saved": len(driver_adjustments), "driver_adjustments": result_rows}


@forecast_router.get("/cashflow-forecast/{year}/{currency}")
async def cashflow_forecast(
    year: int = Path(...),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    forecast_year: Optional[int] = Query(default=None, description="Primary forecast year context. Overrides year for period resolution when provided."),
    forecast_type: Optional[str] = Query(default="quarter", description="Forecast type: quarter, 12months, rolling"),
    rolling_start_month: Optional[str] = Query(default=None, description="Start month for rolling forecast (e.g. Jan, Feb, Mar). Required when forecast_type=rolling."),
    view_type: Optional[str] = Query(default="monthly_breakdown", description="View type: quarter_summary or monthly_breakdown"),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_cashflow_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    effective_year = forecast_year if forecast_year is not None else year
    effective_forecast_type = forecast_type or "quarter"
    effective_view_type = view_type or "monthly_breakdown"

    if effective_forecast_type not in ("quarter", "12months", "rolling"):
        raise HTTPException(status_code=422, detail="forecast_type must be one of: quarter, 12months, rolling")
    if effective_view_type not in ("quarter_summary", "monthly_breakdown"):
        raise HTTPException(status_code=422, detail="view_type must be one of: quarter_summary, monthly_breakdown")
    if effective_forecast_type == "rolling" and not rolling_start_month:
        raise HTTPException(status_code=422, detail="rolling_start_month is required when forecast_type=rolling.")

    # Rolling: Nov/Dec starts use previous year so the window spans INTO effective_year
    _MONTH_ORDER = {"jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
                    "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12}
    resolve_year = effective_year
    if effective_forecast_type == "rolling" and rolling_start_month:
        if _MONTH_ORDER.get(rolling_start_month.lower()[:3], 1) >= 11:
            resolve_year = effective_year - 1

    month_periods, selected_period, prev_month = _resolve_forecast_periods(
        resolve_year, effective_forecast_type, quarter.value, rolling_start_month
    )
    n_months = len(month_periods)
    m1 = month_periods[0]

    effective_quarter = quarter.value
    if effective_forecast_type == "12months":
        effective_quarter = "Q1"
    elif effective_forecast_type == "rolling":
        mn = int(month_periods[0].split("-")[1])
        effective_quarter = "Q1" if mn <= 3 else "Q2" if mn <= 6 else "Q3" if mn <= 9 else "Q4"

    quarter_start = pd.to_datetime(m1 + "-01")
    history_start_str = (quarter_start - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")

    wf_task = _get_or_create_cashflow_workflow(db, effective_year, effective_quarter, company_id, dept, user_id)
    hist_task = db.execute(HISTORY_SQL, {"history_start": history_start_str, "quarter_start": m1})
    prev_task = db.execute(PREV_MONTH_SQL, {"prev_month": prev_month})

    wf, hist_result, prev_result = await asyncio.gather(wf_task, hist_task, prev_task)
    stage = wf["stage"]

    audit_rows = (await db.execute(text("SELECT to_stage, transitioned_by_name FROM fpna_config.cashflow_workflow_audit WHERE workflow_id=:wf_id ORDER BY created_at ASC"), {"wf_id": wf["id"]})).mappings().all()

    reviewed_by = approved_by = locked_by = None
    for r in audit_rows:
        if r["to_stage"] == "reviewed":
            reviewed_by = r["transitioned_by_name"]
        elif r["to_stage"] == "approved":
            approved_by = r["transitioned_by_name"]
        elif r["to_stage"] == "locked":
            locked_by = r["transitioned_by_name"]

    hist_df = pd.DataFrame(hist_result.mappings().all())
    prev_row = prev_result.mappings().first()

    prev_values_map = {
        "operatingCF":  round(float(prev_row["operating_cf"]  or 0), 2) if prev_row else 0.0,
        "investingCF":  round(float(prev_row["investing_cf"]  or 0), 2) if prev_row else 0.0,
        "financingCF":  round(float(prev_row["financing_cf"]  or 0), 2) if prev_row else 0.0,
        "netCashChange": round(float(prev_row["net_cash_change"] or 0), 2) if prev_row else 0.0,
    }

    saved_adjs, driver_saved, custom_scenarios, component_forecasts = await asyncio.gather(
        _get_cashflow_adjustments(db, effective_year, effective_quarter, company_id),
        _get_cashflow_driver_adjustments(db, effective_year, effective_quarter, company_id),
        _get_cashflow_custom_scenarios(db, effective_year, effective_quarter, company_id),
        _forecast_all_components(hist_df, m1) if n_months <= 3
            else _forecast_all_components_n_months(hist_df, m1, n_months),
    )

    adj_map = {s["component"]: s for s in saved_adjs}

    rows: List[Dict] = []
    totals = CashflowTotal()
    drivers_map: Dict[str, List[CashflowDriverRow]] = {}
    component_ai_map: Dict[str, float] = {}
    component_user_map: Dict[str, float] = {}
    # Per-month aggregate totals across all components
    month_totals: List[Dict[str, Any]] = [
        {"month": mp, "total_ai": 0.0, "total_adjusted": 0.0} for mp in month_periods
    ]
    # Store per-month data for each component (for monthly_data in multi-month views)
    comp_monthly_data: Dict[str, List[Dict]] = {}

    for comp in CASHFLOW_COMPONENTS:
        fc = component_forecasts.get(comp, [0.0] * n_months)
        # Normalize: existing 3-month returns Tuple; n-month returns List
        if isinstance(fc, (tuple, list)):
            fc_list = list(fc)
        else:
            fc_list = [0.0] * n_months

        adj = adj_map.get(comp)
        prev_value = prev_values_map.get(comp, 0.0)
        overall_comment = adj["overall_comment"] if adj else None

        month_ai_vals: List[float] = []
        month_adj_vals: List[float] = []
        month_comments: List[Optional[str]] = []
        driver_overrides: Dict[str, float] = {}
        driver_month_comments: Dict[str, Optional[str]] = {mp: None for mp in month_periods}

        for idx, month_label in enumerate(month_periods):
            ai_val = round(float(fc_list[idx]) if idx < len(fc_list) else 0.0, 2)
            u_ai, r_ai = derive_units_and_rate(ai_val)

            drv = driver_saved.get((comp, month_label))
            if drv:
                u_user, r_user = drv["units_adjusted"], drv["rate_adjusted"]
                drv_month_comment = drv.get("month_comment")
                if u_user is not None and r_user is not None:
                    rev_adj_driver = round(u_user * r_user, 2)
                elif u_user is not None:
                    rev_adj_driver = round(u_user * r_ai, 2)
                elif r_user is not None:
                    rev_adj_driver = round(u_ai * r_user, 2)
                else:
                    rev_adj_driver = None
            else:
                u_user = r_user = drv_month_comment = rev_adj_driver = None
            driver_month_comments[month_label] = drv_month_comment

            if comp not in drivers_map:
                drivers_map[comp] = []
            drivers_map[comp].append(CashflowDriverRow(
                component=comp, month=month_label,
                units_ai=u_ai, rate_ai=r_ai, revenue_ai=round(u_ai * r_ai, 2),
                units_adjusted=round(u_user, 4) if u_user is not None else None,
                rate_adjusted=round(r_user, 2) if r_user is not None else None,
                revenue_adjusted=rev_adj_driver, month_comment=drv_month_comment,
            ))
            if rev_adj_driver is not None:
                driver_overrides[month_label] = rev_adj_driver

            # Priority: driver override > manual adjustment > AI value
            if month_label in driver_overrides:
                adj_val = driver_overrides[month_label]
                comment_val = driver_month_comments.get(month_label)
            elif adj is not None:
                adj_key = f"month{idx + 1}_adjusted"
                raw = adj.get(adj_key) if isinstance(adj, dict) else None
                adj_val = float(raw) if raw is not None else ai_val
                comment_val = adj.get(f"month{idx + 1}_comment") if isinstance(adj, dict) else None
            else:
                adj_val = ai_val
                comment_val = None

            month_ai_vals.append(round(ai_val, 2))
            month_adj_vals.append(round(adj_val, 2))
            month_comments.append(comment_val)

            month_totals[idx]["total_ai"] = round(month_totals[idx]["total_ai"] + round(ai_val, 2), 2)
            month_totals[idx]["total_adjusted"] = round(month_totals[idx]["total_adjusted"] + round(adj_val, 2), 2)

        period_total_ai = round(sum(month_ai_vals), 2)
        period_total_adjusted = round(sum(month_adj_vals), 2)
        component_ai_map[comp] = period_total_ai
        component_user_map[comp] = period_total_adjusted

        # Flat month1/month2/month3 fields for CashflowRow (backward compat — first 3 months)
        m1_ai  = month_ai_vals[0]  if len(month_ai_vals) > 0 else 0.0
        m1_adj = month_adj_vals[0] if len(month_adj_vals) > 0 else 0.0
        m2_ai  = month_ai_vals[1]  if len(month_ai_vals) > 1 else 0.0
        m2_adj = month_adj_vals[1] if len(month_adj_vals) > 1 else 0.0
        m3_ai  = month_ai_vals[2]  if len(month_ai_vals) > 2 else 0.0
        m3_adj = month_adj_vals[2] if len(month_adj_vals) > 2 else 0.0

        row = CashflowRow(
            component=comp, previous_month=round(prev_value, 2),
            month1_ai=m1_ai, month1_adjusted=m1_adj, month1_comment=month_comments[0] if month_comments else None,
            month2_ai=m2_ai, month2_adjusted=m2_adj, month2_comment=month_comments[1] if len(month_comments) > 1 else None,
            month3_ai=m3_ai, month3_adjusted=m3_adj, month3_comment=month_comments[2] if len(month_comments) > 2 else None,
            quarter_total_ai=period_total_ai, quarter_total_adjusted=period_total_adjusted,
            overall_comment=overall_comment,
        )
        row_dict = row.dict()
        rows.append(row_dict)

        totals.previous_month += prev_value
        totals.month1_ai += m1_ai
        totals.month1_adjusted += m1_adj
        totals.month2_ai += m2_ai
        totals.month2_adjusted += m2_adj
        totals.month3_ai += m3_ai
        totals.month3_adjusted += m3_adj

        comp_monthly_data[comp] = [
            {"month": mp, "ai": month_ai_vals[i], "adjusted": month_adj_vals[i]}
            for i, mp in enumerate(month_periods)
        ]

    drivers_output: List[Dict] = []
    for comp, comp_months in drivers_map.items():
        q_ai = sum(m.revenue_ai for m in comp_months)
        q_adj = sum(m.revenue_adjusted if m.revenue_adjusted is not None else m.revenue_ai for m in comp_months)
        overall_comment_for_comp = None
        for month_label in month_periods:
            drv = driver_saved.get((comp, month_label))
            if drv and drv.get("overall_comment"):
                overall_comment_for_comp = drv["overall_comment"]
                break
        drivers_output.append(CashflowDriverSummary(
            component=comp,
            months=sorted(comp_months, key=lambda x: x.month),
            quarter_revenue_ai=round(q_ai, 2),
            quarter_revenue_adjusted=round(q_adj, 2),
            overall_comment=overall_comment_for_comp,
        ).dict())

    baseline_ai = round(sum(mt["total_ai"] for mt in month_totals), 2)
    baseline_user = round(sum(mt["total_adjusted"] for mt in month_totals), 2)

    compare_scenarios_data = [
        {"scenario": "Baseline",     "ai_Suggested": round(baseline_ai, 2),                          "userAdjustment": round(baseline_user - baseline_ai, 2)},
        {"scenario": "Optimistic",   "ai_Suggested": round(baseline_ai * BEST_CASE_MULTIPLIER, 2),   "userAdjustment": round((baseline_user - baseline_ai) * BEST_CASE_MULTIPLIER, 2)},
        {"scenario": "Conservative", "ai_Suggested": round(baseline_ai * WORST_CASE_MULTIPLIER, 2),  "userAdjustment": round((baseline_user - baseline_ai) * WORST_CASE_MULTIPLIER, 2)},
    ]
    for cs in custom_scenarios:
        comp_mults = cs.get("component_multipliers") or {}
        mult = cs["multiplier"]
        cs_ai   = sum(component_ai_map.get(c, 0.0)   * comp_mults.get(c, mult) for c in CASHFLOW_COMPONENTS) if comp_mults else baseline_ai * mult
        cs_user = sum(component_user_map.get(c, 0.0) * comp_mults.get(c, mult) for c in CASHFLOW_COMPONENTS) if comp_mults else baseline_user * mult
        compare_scenarios_data.append({"scenario": cs["scenario_name"], "ai_Suggested": round(cs_ai, 2), "userAdjustment": round(cs_user - cs_ai, 2)})

    rounded_totals = {k: round(v, 2) for k, v in totals.dict().items()}

    # Apply view_type: quarter_summary strips per-month detail, keeps period totals
    if effective_view_type == "quarter_summary":
        rows_output = [
            {
                "component": r["component"],
                "previous_month": r["previous_month"],
                "quarter_total_ai": r["quarter_total_ai"],
                "quarter_total_adjusted": r["quarter_total_adjusted"],
                "overall_comment": r["overall_comment"],
            }
            for r in rows
        ]
        if n_months > 3:
            q_summary: Dict[str, Dict[str, float]] = {}
            for qi, q_label in enumerate(["Q1", "Q2", "Q3", "Q4"]):
                si = qi * 3
                q_summary[q_label] = {
                    "total_ai":       round(sum(mt["total_ai"]       for mt in month_totals[si:si + 3]), 2),
                    "total_adjusted": round(sum(mt["total_adjusted"]  for mt in month_totals[si:si + 3]), 2),
                }
    else:
        rows_output = rows
        if n_months > 3:
            for row_dict in rows_output:
                row_dict["monthly_data"] = comp_monthly_data.get(row_dict["component"], [])

    response: Dict[str, Any] = {
        "year": year,
        "forecast_year": effective_year,
        "forecast_type": effective_forecast_type,
        "view_type": effective_view_type,
        "selected_period": selected_period,
        "selected_months": month_periods,
        "quarter": quarter.value,
        "prev_month": prev_month,
        "rows": rows_output,
        "total": rounded_totals,
        "month_totals": month_totals,
        "drivers": drivers_output,
        "workflow_stage": stage,
        "reviewed_by": reviewed_by,
        "approved_by": approved_by,
        "locked_by": locked_by,
        "compareCashFlowScenarios": {
            "title": "Compare Cash Flow Scenarios",
            "data": compare_scenarios_data,
        },
    }
    if n_months > 3 and effective_view_type == "quarter_summary":
        response["quarter_summary"] = q_summary

    return apply_smart_currency(response, currency, currency_filter, CASHFLOW_MONEY_FIELDS)
##########################################################################################
###########################################################################################
###########################################################################################
# CAPEX FORECASTING
class CapexAdjustmentInput(BaseModel):
    capex_item: str
    month1_adjusted: Optional[float] = None
    month1_comment: Optional[str] = None
    month2_adjusted: Optional[float] = None
    month2_comment: Optional[str] = None
    month3_adjusted: Optional[float] = None
    month3_comment: Optional[str] = None
    overall_comment: Optional[str] = None


class CapexDriverAdjustmentInput(BaseModel):
    capex_item: str
    month: str
    units_adjusted: Optional[float] = None
    rate_adjusted: Optional[float] = None
    month_comment: Optional[str] = None
    overall_comment: Optional[str] = None


class CapexItemMultiplierInput(BaseModel):
    capex_item: str
    multiplier: float


class CapexForecastScenarioInput(BaseModel):
    scenario_name: str
    multiplier: float
    description: Optional[str] = None
    parameter_types: Optional[List[str]] = None
    customer_scope: Optional[str] = "all"
    item_multipliers: Optional[List[CapexItemMultiplierInput]] = None


class CapexForecastScenarioUpdateInput(BaseModel):
    scenario_name: Optional[str] = None
    multiplier: Optional[float] = None
    description: Optional[str] = None
    parameter_types: Optional[List[str]] = None
    customer_scope: Optional[str] = None
    item_multipliers: Optional[List[CapexItemMultiplierInput]] = None


GROWTH_RATE = 0.05
BEST_CASE_MULTIPLIER = 1.20
WORST_CASE_MULTIPLIER = 0.75
HISTORY_MONTHS_SQL = 18
MIN_POINTS_FOR_PROPHET = 24
DEFAULT_USEFUL_LIFE = 5
ACCELERATED_MULTIPLIER = 1.20
DELAYED_SPENDING_MULTIPLIER = 0.75

SECRET_KEY = None
ALGORITHM = "HS256"

RESERVED_SCENARIO_NAMES = {"baseline", "accelerated_investment", "delayed_spending"}

CAPEX_SCENARIO_DEFINITIONS = {
    "baseline": "AI-generated CAPEX forecast with manual user adjustments applied.",
    "accelerated_investment": f"Aggressive investment strategy applying a {int((ACCELERATED_MULTIPLIER - 1) * 100)}% uplift on baseline — assumes full budget utilization, expanded operations, and technology upgrades.",
    "delayed_spending": f"Conservative scenario applying a {int((1 - DELAYED_SPENDING_MULTIPLIER) * 100)}% reduction on baseline — assumes non-essential CAPEX is deferred or cancelled.",
}

VALID_PARAMETER_TYPES = {"capex", "rate", "volume", "quantity"}

CAPEX_MONEY_FIELDS = [
    "previous_month",
    "month1_ai",
    "month1_adjusted",
    "month2_ai",
    "month2_adjusted",
    "month3_ai",
    "month3_adjusted",
    "quarter_total_ai",
    "quarter_total_adjusted",
    "units_ai",
    "rate_ai",
    "capex_ai",
    "units_adjusted",
    "rate_adjusted",
    "capex_adjusted",
    "quarter_capex_ai",
    "quarter_capex_adjusted",
    "total_ai",
    "total_user",
    "variance_amount",
    "annual_benefit",
    "npv",
]


def can_edit_capex_forecast(dept: str, stage: str) -> bool:
    if stage == WorkflowStage.LOCKED:
        return False
    if dept in DEPT_SUBMITTERS:
        return stage == WorkflowStage.DRAFT
    return False


def can_transition_capex(dept: str, current_stage: str, target_stage: str) -> bool:
    if current_stage == WorkflowStage.LOCKED:
        return False

    if dept in DEPT_SUBMITTERS:
        if target_stage == WorkflowStage.SUBMITTED and current_stage == WorkflowStage.DRAFT:
            return True
        return False

    if dept in DEPT_REVIEWERS:
        if target_stage == WorkflowStage.REVIEWED and current_stage == WorkflowStage.SUBMITTED:
            return True
        if target_stage == WorkflowStage.DRAFT:
            return True
        return False

    if dept in DEPT_APPROVERS:
        if target_stage == WorkflowStage.APPROVED and current_stage == WorkflowStage.REVIEWED:
            return True
        if target_stage == WorkflowStage.LOCKED and current_stage == WorkflowStage.APPROVED:
            return True
        if target_stage == WorkflowStage.DRAFT:
            return True
        return False

    return False


def can_read_all_capex(dept: str) -> bool:
    return dept in DEPT_REVIEWERS | DEPT_APPROVERS


def fallback_forecast(last_val: float, m1: str, m2: str, m3: str) -> dict:
    v1 = last_val * (1 + GROWTH_RATE)
    return {m1: round(v1, 2), m2: round(v1 * (1 + GROWTH_RATE), 2), m3: round(v1 * (1 + GROWTH_RATE) ** 2, 2)}


def calculate_roi(purchase_price: float) -> dict:
    useful_life = DEFAULT_USEFUL_LIFE
    if purchase_price == 0:
        return {"roi_percent": 0.0, "payback_period_years": 0.0, "npv": 0.0, "annual_benefit": 0.0}
    annual_benefit = purchase_price * 0.20
    roi = ((annual_benefit * useful_life - purchase_price) / purchase_price) * 100
    payback = purchase_price / annual_benefit if annual_benefit > 0 else 0.0
    discount_rate = 0.10
    npv = sum(annual_benefit / ((1 + discount_rate) ** t) for t in range(1, useful_life + 1)) - purchase_price
    return {
        "roi_percent": round(roi, 2),
        "payback_period_years": round(payback, 2),
        "npv": round(npv, 2),
        "annual_benefit": round(annual_benefit, 2),
    }


async def _ensure_capex_workflow_tables(db: AsyncSession) -> None:
    await db.execute(text("CREATE SCHEMA IF NOT EXISTS fpna_config;"))
    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (
                SELECT 1 FROM information_schema.tables
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'capex_forecast_adjustments'
            ) AND NOT EXISTS (
                SELECT 1 FROM information_schema.table_constraints
                WHERE table_schema   = 'fpna_config'
                  AND table_name     = 'capex_forecast_adjustments'
                  AND constraint_type = 'PRIMARY KEY'
            ) THEN
                DROP TABLE fpna_config.capex_forecast_adjustments;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.capex_forecast_adjustments (
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            capex_item TEXT NOT NULL,
            month1_adjusted NUMERIC NULL,
            month1_comment TEXT NULL,
            month2_adjusted NUMERIC NULL,
            month2_comment TEXT NULL,
            month3_adjusted NUMERIC NULL,
            month3_comment TEXT NULL,
            overall_comment TEXT NULL,
            updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT NOW(),
            PRIMARY KEY (year, quarter, company_id, capex_item)
        );
    """))
    await db.execute(text("""
        DO $$
        DECLARE
            pk_name TEXT;
            pk_cols TEXT;
        BEGIN
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'capex_forecast_adjustments'
                  AND column_name  = 'overall_comment'
            ) THEN
                ALTER TABLE fpna_config.capex_forecast_adjustments ADD COLUMN overall_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'capex_forecast_adjustments'
                  AND column_name  = 'month1_comment'
            ) THEN
                ALTER TABLE fpna_config.capex_forecast_adjustments ADD COLUMN month1_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'capex_forecast_adjustments'
                  AND column_name  = 'month2_comment'
            ) THEN
                ALTER TABLE fpna_config.capex_forecast_adjustments ADD COLUMN month2_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'capex_forecast_adjustments'
                  AND column_name  = 'month3_comment'
            ) THEN
                ALTER TABLE fpna_config.capex_forecast_adjustments ADD COLUMN month3_comment TEXT NULL;
            END IF;

            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'capex_forecast_adjustments'
                  AND column_name  = 'company_id'
            ) THEN
                ALTER TABLE fpna_config.capex_forecast_adjustments ADD COLUMN company_id TEXT NOT NULL DEFAULT '';
            END IF;

            SELECT kcu.column_name INTO pk_cols
            FROM information_schema.table_constraints tc
            JOIN information_schema.key_column_usage kcu
              ON tc.constraint_name = kcu.constraint_name
             AND tc.table_schema    = kcu.table_schema
            WHERE tc.table_schema    = 'fpna_config'
              AND tc.table_name      = 'capex_forecast_adjustments'
              AND tc.constraint_type = 'PRIMARY KEY'
              AND kcu.column_name    = 'company_id'
            LIMIT 1;

            IF pk_cols IS NULL THEN
                SELECT tc.constraint_name INTO pk_name
                FROM information_schema.table_constraints tc
                WHERE tc.table_schema    = 'fpna_config'
                  AND tc.table_name      = 'capex_forecast_adjustments'
                  AND tc.constraint_type = 'PRIMARY KEY'
                LIMIT 1;

                IF pk_name IS NOT NULL THEN
                    EXECUTE 'ALTER TABLE fpna_config.capex_forecast_adjustments DROP CONSTRAINT ' || quote_ident(pk_name);
                END IF;

                ALTER TABLE fpna_config.capex_forecast_adjustments
                    ADD PRIMARY KEY (year, quarter, company_id, capex_item);
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.capex_driver_adjustments (
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            capex_item TEXT NOT NULL,
            month TEXT NOT NULL,
            units_adjusted NUMERIC NULL,
            rate_adjusted NUMERIC NULL,
            month_comment TEXT NULL,
            overall_comment TEXT NULL,
            updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT NOW(),
            PRIMARY KEY (year, quarter, company_id, capex_item, month)
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'capex_driver_adjustments'
                  AND column_name  = 'month_comment'
            ) THEN
                ALTER TABLE fpna_config.capex_driver_adjustments ADD COLUMN month_comment TEXT NULL;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'capex_driver_adjustments'
                  AND column_name  = 'overall_comment'
            ) THEN
                ALTER TABLE fpna_config.capex_driver_adjustments ADD COLUMN overall_comment TEXT NULL;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.capex_forecast_workflows (
            id SERIAL PRIMARY KEY,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            department TEXT NOT NULL,
            owner_user_id TEXT NOT NULL,
            stage TEXT NOT NULL DEFAULT 'draft',
            created_at TIMESTAMP NOT NULL DEFAULT NOW(),
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            locked_at TIMESTAMP NULL,
            UNIQUE (year, quarter, company_id)
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'capex_forecast_workflows'
                  AND column_name  = 'owner_user_id'
                  AND data_type   != 'text'
            ) THEN
                ALTER TABLE fpna_config.capex_forecast_workflows
                    ALTER COLUMN owner_user_id TYPE TEXT USING owner_user_id::TEXT;
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.capex_forecast_workflow_audit (
            id SERIAL PRIMARY KEY,
            workflow_id INT NOT NULL,
            from_stage TEXT NULL,
            to_stage TEXT NOT NULL,
            transitioned_by_user_id TEXT NOT NULL,
            transitioned_by_name TEXT NOT NULL,
            comment TEXT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.capex_forecast_workflow_comments (
            id SERIAL PRIMARY KEY,
            workflow_id INT NOT NULL,
            user_id TEXT NOT NULL,
            user_name TEXT NOT NULL,
            department TEXT NOT NULL,
            comment TEXT NOT NULL,
            comment_type TEXT NOT NULL DEFAULT 'general',
            created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'capex_forecast_workflow_comments'
                  AND column_name  = 'comment_type'
            ) THEN
                ALTER TABLE fpna_config.capex_forecast_workflow_comments
                    ADD COLUMN comment_type TEXT NOT NULL DEFAULT 'general';
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.capex_forecast_scenarios (
            id SERIAL PRIMARY KEY,
            company_id TEXT NOT NULL,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            scenario_name TEXT NOT NULL,
            multiplier NUMERIC NOT NULL,
            description TEXT NULL,
            created_by_user_id TEXT NOT NULL,
            created_by_name TEXT NOT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW(),
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            UNIQUE (company_id, year, quarter, scenario_name)
        );
    """))
    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'capex_forecast_scenarios'
                  AND column_name  = 'parameter_types'
            ) THEN
                ALTER TABLE fpna_config.capex_forecast_scenarios
                    ADD COLUMN parameter_types JSONB NOT NULL DEFAULT '["capex"]';
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'capex_forecast_scenarios'
                  AND column_name  = 'customer_scope'
            ) THEN
                ALTER TABLE fpna_config.capex_forecast_scenarios
                    ADD COLUMN customer_scope TEXT NOT NULL DEFAULT 'all';
            END IF;
        END$$;
    """))
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.capex_forecast_scenario_item_multipliers (
            id SERIAL PRIMARY KEY,
            scenario_id INT NOT NULL,
            company_id TEXT NOT NULL,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            capex_item TEXT NOT NULL,
            multiplier NUMERIC NOT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW(),
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            UNIQUE (scenario_id, company_id, year, quarter, capex_item)
        );
    """))
    await db.commit()


async def _get_or_create_capex_workflow(
    db: AsyncSession, year: int, quarter: str, company_id: str, department: str, user_id: str
) -> Dict[str, Any]:
    row = (await db.execute(
        text("""
            SELECT * FROM fpna_config.capex_forecast_workflows
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id
            ORDER BY id ASC
            LIMIT 1
        """),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().first()

    if not row:
        row = (await db.execute(
            text("""
                INSERT INTO fpna_config.capex_forecast_workflows
                    (year, quarter, company_id, department, owner_user_id, stage)
                VALUES (:year, :quarter, :company_id, :dept, :uid, 'draft')
                RETURNING *
            """),
            {"year": year, "quarter": quarter, "company_id": company_id, "dept": department, "uid": user_id},
        )).mappings().first()
        await db.commit()

    return dict(row)


async def _get_capex_adjustments(db: AsyncSession, year: int, quarter: str, company_id: str) -> List[Dict[str, Any]]:
    rows = (await db.execute(
        text("""
            SELECT
                capex_item,
                month1_adjusted, month1_comment,
                month2_adjusted, month2_comment,
                month3_adjusted, month3_comment,
                overall_comment
            FROM fpna_config.capex_forecast_adjustments
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id
            ORDER BY capex_item ASC;
        """),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().all()

    result = []
    for r in rows:
        m1 = float(r["month1_adjusted"]) if r["month1_adjusted"] is not None else None
        m2 = float(r["month2_adjusted"]) if r["month2_adjusted"] is not None else None
        m3 = float(r["month3_adjusted"]) if r["month3_adjusted"] is not None else None
        quarter_total = round((m1 or 0.0) + (m2 or 0.0) + (m3 or 0.0), 2)
        result.append({
            "capex_item": r["capex_item"],
            "month1_adjusted": m1,
            "month1_comment": r["month1_comment"],
            "month2_adjusted": m2,
            "month2_comment": r["month2_comment"],
            "month3_adjusted": m3,
            "month3_comment": r["month3_comment"],
            "quarter_total_adjusted": quarter_total,
            "overall_comment": r["overall_comment"],
        })
    return result


async def _get_capex_driver_adjustments(
    db: AsyncSession, year: int, quarter: str, company_id: str
) -> Dict[tuple, Dict]:
    rows = (await db.execute(
        text("""
            SELECT capex_item, month, units_adjusted, rate_adjusted, month_comment, overall_comment
            FROM fpna_config.capex_driver_adjustments
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id;
        """),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().all()

    return {
        (r["capex_item"], r["month"]): {
            "units_adjusted": float(r["units_adjusted"]) if r["units_adjusted"] is not None else None,
            "rate_adjusted": float(r["rate_adjusted"]) if r["rate_adjusted"] is not None else None,
            "month_comment": r["month_comment"],
            "overall_comment": r["overall_comment"],
        }
        for r in rows
    }


async def _get_capex_scenario_item_multipliers(
    db: AsyncSession, scenario_id: int, company_id: str, year: int, quarter: str
) -> Dict[str, float]:
    rows = (await db.execute(
        text("""
            SELECT capex_item, multiplier
            FROM fpna_config.capex_forecast_scenario_item_multipliers
            WHERE scenario_id = :scenario_id
              AND company_id = :company_id
              AND year = :year
              AND quarter = :quarter
            ORDER BY capex_item ASC
        """),
        {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter},
    )).mappings().all()

    return {r["capex_item"]: float(r["multiplier"]) for r in rows}


async def _get_capex_custom_scenarios(
    db: AsyncSession, year: int, quarter: str, company_id: str
) -> List[Dict[str, Any]]:
    rows = (await db.execute(
        text("""
            SELECT id, scenario_name, multiplier, description, parameter_types, customer_scope,
                   created_by_name, created_at, updated_at
            FROM fpna_config.capex_forecast_scenarios
            WHERE company_id = :company_id AND year = :year AND quarter = :quarter
            ORDER BY created_at ASC
        """),
        {"company_id": company_id, "year": year, "quarter": quarter},
    )).mappings().all()

    return [
        {
            "id": r["id"],
            "scenario_name": r["scenario_name"],
            "multiplier": float(r["multiplier"]),
            "description": r["description"],
            "parameter_types": r["parameter_types"] if r["parameter_types"] else ["capex"],
            "customer_scope": r["customer_scope"] if r["customer_scope"] else "all",
            "created_by_name": r["created_by_name"],
            "created_at": r["created_at"].isoformat(),
            "updated_at": r["updated_at"].isoformat(),
        }
        for r in rows
    ]


def _build_capex_scenario_summary(
    baseline_ai: float,
    baseline_user: float,
    multiplier: float,
) -> Dict[str, Any]:
    total_ai = round(baseline_ai * multiplier, 2)
    total_user = round(baseline_user * multiplier, 2)
    variance_amount = round(total_user - total_ai, 2)
    variance_percent = round((variance_amount / total_ai) * 100 if total_ai else 0, 2)
    return {
        "total_ai": total_ai,
        "total_user": total_user,
        "variance_amount": variance_amount,
        "variance_percent": variance_percent,
    }


def _build_capex_scenario_summary_with_item_multipliers(
    baseline_ai: float,
    baseline_user: float,
    scenario_multiplier: float,
    item_ai_map: Dict[str, float],
    item_user_map: Dict[str, float],
    item_multipliers: Dict[str, float],
    customer_scope: str,
) -> Dict[str, Any]:
    if customer_scope == "all" or not item_multipliers:
        return _build_capex_scenario_summary(baseline_ai, baseline_user, scenario_multiplier)

    total_ai = 0.0
    total_user = 0.0

    all_items = set(list(item_ai_map.keys()) + list(item_user_map.keys()))

    for iname in all_items:
        eff_mult = item_multipliers.get(iname, scenario_multiplier)
        i_ai = item_ai_map.get(iname, 0.0)
        i_user = item_user_map.get(iname, 0.0)
        total_ai += i_ai * eff_mult
        total_user += i_user * eff_mult

    total_ai = round(total_ai, 2)
    total_user = round(total_user, 2)
    variance_amount = round(total_user - total_ai, 2)
    variance_percent = round((variance_amount / total_ai) * 100 if total_ai else 0, 2)

    return {
        "total_ai": total_ai,
        "total_user": total_user,
        "variance_amount": variance_amount,
        "variance_percent": variance_percent,
    }


def _forecast_capex_item(
    capex_item: str,
    item_hist_df: pd.DataFrame,
    quarter_start: pd.Timestamp,
    m1: str,
    m2: str,
    m3: str,
) -> tuple[float, float, float]:
    cdf = item_hist_df[item_hist_df["category"] == capex_item].copy()
    cdf["ds"] = pd.to_datetime(cdf["period"] + "-01")
    cdf = cdf.rename(columns={"purchase_price": "y"}).sort_values("ds")

    if cdf.empty:
        return 0.0, 0.0, 0.0

    values = cdf["y"].values.astype(float)
    if len(values) == 1:
        v = round(float(values[-1]) * (1 + GROWTH_RATE), 2)
        return v, round(v * (1 + GROWTH_RATE), 2), round(v * (1 + GROWTH_RATE) ** 2, 2)

    clean_values = _winsorize_values(values)
    features = _feature_engineering(clean_values)
    baseline_result = _baseline_forecast(clean_values)
    start_date = str(cdf["ds"].iloc[0].date())
    prophet_result = _prophet_forecast_sync(clean_values, start_date, 3)
    ets_result = _ets_forecast_sync(clean_values, 3)
    arima_result = _arima_forecast_sync(clean_values, 3)
    xgb_result = _xgboost_forecast_sync(clean_values, 3)
    final_preds = _ensemble_forecasts_capex(baseline_result, ets_result, arima_result, xgb_result, features, 3)
    if prophet_result and len(final_preds) == 3:
        for i in range(3):
            final_preds[i] = round((final_preds[i] * 0.70) + (prophet_result[i] * 0.30), 2)
    final_preds = _validate_forecast(final_preds, clean_values)
    final_preds = _stabilize_forecasts(final_preds, float(clean_values[-1]), features["volatility"])
    return float(final_preds[0]), float(final_preds[1]), float(final_preds[2])


def _forecast_capex_item_n_months(
    capex_item: str,
    item_hist_df: pd.DataFrame,
    forecast_start: pd.Timestamp,
    n_months: int,
) -> List[float]:
    cdf = item_hist_df[item_hist_df["category"] == capex_item].copy()
    cdf["ds"] = pd.to_datetime(cdf["period"] + "-01")
    cdf = cdf.rename(columns={"purchase_price": "y"}).sort_values("ds")

    if cdf.empty:
        return [0.0] * n_months

    values = cdf["y"].values.astype(float)
    if len(values) == 1:
        v = float(values[-1]) * (1 + GROWTH_RATE)
        return [round(v * ((1 + GROWTH_RATE) ** i), 2) for i in range(n_months)]

    clean_values = _winsorize_values(values)
    features = _feature_engineering(clean_values)
    baseline_3 = _baseline_forecast(clean_values)
    start_date = str(cdf["ds"].iloc[0].date())
    prophet_result = _prophet_forecast_sync(clean_values, start_date, n_months)
    ets_result = _ets_forecast_sync(clean_values, n_months)
    arima_result = _arima_forecast_sync(clean_values, n_months)
    xgb_result = _xgboost_forecast_sync(clean_values, n_months)
    result = _ensemble_forecasts_capex(baseline_3, ets_result, arima_result, xgb_result, features, n_months)
    if prophet_result:
        result = [round((result[i] * 0.70) + (prophet_result[i] * 0.30 if i < len(prophet_result) else 0.0), 2) for i in range(n_months)]
    result = _validate_forecast(result, clean_values)
    result = _stabilize_forecasts(result, float(clean_values[-1]), features["volatility"])
    return result


class CapexForecastAdjustmentManagement:
    @staticmethod
    async def GetAdjustments(year: int, quarter: str, company_id: str, db: AsyncSession) -> List[Dict[str, Any]]:
        return await _get_capex_adjustments(db, year, quarter, company_id)

    @staticmethod
    async def PatchAdjustments(
        year: int,
        quarter: str,
        company_id: str,
        adjustments: List[CapexAdjustmentInput],
        db: AsyncSession,
    ) -> Dict[str, Any]:
        for adj in adjustments:
            if not adj.capex_item:
                continue
            await db.execute(
                text("""
                    INSERT INTO fpna_config.capex_forecast_adjustments (
                        year, quarter, company_id, capex_item,
                        month1_adjusted, month1_comment,
                        month2_adjusted, month2_comment,
                        month3_adjusted, month3_comment,
                        overall_comment, updated_at
                    )
                    VALUES (
                        :year, :quarter, :company_id, :capex_item,
                        :month1_adjusted, :month1_comment,
                        :month2_adjusted, :month2_comment,
                        :month3_adjusted, :month3_comment,
                        :overall_comment, NOW()
                    )
                    ON CONFLICT (year, quarter, company_id, capex_item)
                    DO UPDATE SET
                        month1_adjusted = EXCLUDED.month1_adjusted,
                        month1_comment  = EXCLUDED.month1_comment,
                        month2_adjusted = EXCLUDED.month2_adjusted,
                        month2_comment  = EXCLUDED.month2_comment,
                        month3_adjusted = EXCLUDED.month3_adjusted,
                        month3_comment  = EXCLUDED.month3_comment,
                        overall_comment = EXCLUDED.overall_comment,
                        updated_at      = NOW();
                """),
                {
                    "year": year,
                    "quarter": quarter,
                    "company_id": company_id,
                    "capex_item": adj.capex_item,
                    "month1_adjusted": adj.month1_adjusted,
                    "month1_comment": adj.month1_comment,
                    "month2_adjusted": adj.month2_adjusted,
                    "month2_comment": adj.month2_comment,
                    "month3_adjusted": adj.month3_adjusted,
                    "month3_comment": adj.month3_comment,
                    "overall_comment": adj.overall_comment,
                },
            )
        await db.commit()
        final = await _get_capex_adjustments(db, year, quarter, company_id)
        return {
            "detail": "CAPEX forecast adjustments saved successfully",
            "year": year,
            "quarter": quarter,
            "count": len(adjustments),
            "adjustments": final,
        }


ACTION_TO_STAGE_MAP = {
    "submit_for_review": "submitted",
    "approve": "approved",
    "review": "reviewed",
    "lock": "locked",
    "reject": "draft",
    "send_back_to_draft": "draft",
}


@forecast_router.patch("/capex-forecast/adjustments/{year}")
async def patch_capex_forecast_adjustments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    adjustments: List[CapexAdjustmentInput] = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_capex_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="CAPEX forecast is locked. Adjustments cannot be changed.",
        )

    if stage != WorkflowStage.DRAFT:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail=f"Adjustments can only be edited in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.",
        )

    return await CapexForecastAdjustmentManagement.PatchAdjustments(
        year=year,
        quarter=quarter.value,
        company_id=company_id,
        adjustments=adjustments,
        db=db,
    )


@forecast_router.patch("/capex-forecast/driver-adjustments/{year}")
async def patch_capex_driver_adjustments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    adjustments: List[CapexDriverAdjustmentInput] = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_capex_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="CAPEX forecast is locked. Driver adjustments cannot be changed.",
        )

    if stage != WorkflowStage.DRAFT:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail=f"Driver adjustments can only be edited in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.",
        )

    for adj in adjustments:
        if not adj.capex_item or not adj.month:
            continue
        await db.execute(
            text("""
                INSERT INTO fpna_config.capex_driver_adjustments (
                    year, quarter, company_id, capex_item, month,
                    units_adjusted, rate_adjusted, month_comment, overall_comment, updated_at
                )
                VALUES (
                    :year, :quarter, :company_id, :capex_item, :month,
                    :units_adjusted, :rate_adjusted, :month_comment, :overall_comment, NOW()
                )
                ON CONFLICT (year, quarter, company_id, capex_item, month)
                DO UPDATE SET
                    units_adjusted  = EXCLUDED.units_adjusted,
                    rate_adjusted   = EXCLUDED.rate_adjusted,
                    month_comment   = EXCLUDED.month_comment,
                    overall_comment = EXCLUDED.overall_comment,
                    updated_at      = NOW();
            """),
            {
                "year": year,
                "quarter": quarter.value,
                "company_id": company_id,
                "capex_item": adj.capex_item,
                "month": adj.month,
                "units_adjusted": adj.units_adjusted,
                "rate_adjusted": adj.rate_adjusted,
                "month_comment": adj.month_comment,
                "overall_comment": adj.overall_comment,
            },
        )
    await db.commit()

    saved = (await db.execute(
        text("""
            SELECT capex_item, month, units_adjusted, rate_adjusted, month_comment, overall_comment
            FROM fpna_config.capex_driver_adjustments
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id
            ORDER BY capex_item, month;
        """),
        {"year": year, "quarter": quarter.value, "company_id": company_id},
    )).mappings().all()

    item_quarter_map: Dict[str, float] = {}
    result_rows = []
    for r in saved:
        u = float(r["units_adjusted"]) if r["units_adjusted"] is not None else None
        ra = float(r["rate_adjusted"]) if r["rate_adjusted"] is not None else None
        capex_adj = round(u * ra, 2) if u is not None and ra is not None else None
        iname = r["capex_item"]
        if capex_adj is not None:
            item_quarter_map[iname] = item_quarter_map.get(iname, 0.0) + capex_adj
        result_rows.append({
            "capex_item": iname,
            "month": r["month"],
            "units_adjusted": u,
            "rate_adjusted": ra,
            "capex_adjusted": capex_adj,
            "month_comment": r["month_comment"],
            "overall_comment": r["overall_comment"],
        })

    for row in result_rows:
        row["quarter_capex_adjusted"] = round(item_quarter_map.get(row["capex_item"], 0.0), 2)

    return {
        "detail": "CAPEX driver adjustments saved successfully",
        "year": year,
        "quarter": quarter.value,
        "count": len(adjustments),
        "driver_adjustments": result_rows,
    }


@forecast_router.patch("/capex-forecast/workflow/{year}")
async def patch_capex_forecast_workflow(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: WorkflowTransitionRequest = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_capex_workflow(db, year, quarter.value, company_id, dept, user_id)
    current_stage = wf["stage"]

    action_value = body.action.value if hasattr(body.action, "value") else str(body.action)
    target_stage = ACTION_TO_STAGE_MAP.get(action_value)
    if target_stage is None:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail=f"Unknown action '{action_value}'. Valid actions: {list(ACTION_TO_STAGE_MAP.keys())}",
        )

    effective_comment = body.comment.strip() if body.comment and body.comment.strip() else None

    is_rejection = target_stage == WorkflowStage.DRAFT and current_stage != WorkflowStage.DRAFT
    if is_rejection and not effective_comment:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="A comment is required when rejecting (sending back to draft). Please provide a reason.",
        )

    if effective_comment and dept not in DEPT_REVIEWERS | DEPT_APPROVERS:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="Only Finance/Executive/CEO/CFO can add review comments.",
        )

    if not can_transition_capex(dept, current_stage, target_stage):
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail=f"Department '{dept}' cannot move CAPEX forecast from '{current_stage}' to '{target_stage}'.",
        )

    if target_stage == WorkflowStage.LOCKED:
        await db.execute(
            text("""
                UPDATE fpna_config.capex_forecast_workflows
                SET stage = :new_stage, updated_at = NOW(), locked_at = NOW()
                WHERE id = :wf_id
            """),
            {"new_stage": target_stage, "wf_id": wf["id"]},
        )
    else:
        await db.execute(
            text("""
                UPDATE fpna_config.capex_forecast_workflows
                SET stage = :new_stage, updated_at = NOW()
                WHERE id = :wf_id
            """),
            {"new_stage": target_stage, "wf_id": wf["id"]},
        )

    await db.execute(
        text("""
            INSERT INTO fpna_config.capex_forecast_workflow_audit
                (workflow_id, from_stage, to_stage, transitioned_by_user_id, transitioned_by_name, comment)
            VALUES (:wf_id, :from_s, :to_s, :uid, :uname, :comment)
        """),
        {
            "wf_id": wf["id"],
            "from_s": current_stage,
            "to_s": target_stage,
            "uid": user_id,
            "uname": current_user.get("full_name", ""),
            "comment": effective_comment,
        },
    )

    if effective_comment and dept in DEPT_REVIEWERS | DEPT_APPROVERS:
        comment_type = "rejection" if is_rejection else "transition"
        await db.execute(
            text("""
                INSERT INTO fpna_config.capex_forecast_workflow_comments
                    (workflow_id, user_id, user_name, department, comment, comment_type)
                VALUES (:wf_id, :uid, :uname, :dept, :comment, :comment_type)
            """),
            {
                "wf_id": wf["id"],
                "uid": user_id,
                "uname": current_user.get("full_name", ""),
                "dept": dept,
                "comment": effective_comment,
                "comment_type": comment_type,
            },
        )

    await db.commit()

    return {
        "detail": f"CAPEX forecast moved from '{current_stage}' to '{target_stage}'.",
        "year": year,
        "quarter": quarter.value,
        "department": dept,
        "stage": target_stage,
    }


@forecast_router.post("/capex-forecast/{year}/comments")
async def post_capex_forecast_comment(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: ForecastCommentInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    user_name = current_user.get("full_name", "")

    wf = await _get_or_create_capex_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="CAPEX forecast is locked. Comments cannot be added.",
        )

    comment_text = body.comment.strip() if body.comment else ""
    if not comment_text:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Comment cannot be empty.",
        )

    result = (await db.execute(
        text("""
            INSERT INTO fpna_config.capex_forecast_workflow_comments
                (workflow_id, user_id, user_name, department, comment, comment_type)
            VALUES (:wf_id, :uid, :uname, :dept, :comment, 'general')
            RETURNING id, user_id, user_name, department, comment, comment_type, created_at
        """),
        {
            "wf_id": wf["id"],
            "uid": user_id,
            "uname": user_name,
            "dept": dept,
            "comment": comment_text,
        },
    )).mappings().first()

    await db.commit()

    return {
        "detail": "Comment posted successfully.",
        "year": year,
        "quarter": quarter.value,
        "workflow_stage": stage,
        "comment": {
            "id": result["id"],
            "user_id": result["user_id"],
            "user_name": result["user_name"],
            "department": result["department"],
            "comment": result["comment"],
            "comment_type": result["comment_type"],
            "created_at": result["created_at"].isoformat(),
        },
    }


@forecast_router.get("/capex-forecast/{year}/comments")
async def get_capex_forecast_comments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    comment_type: Optional[str] = Query(None, description="Filter by comment_type: general, transition, rejection. Leave empty for all."),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_capex_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]

    if comment_type:
        rows = (await db.execute(
            text("""
                SELECT id, user_id, user_name, department, comment, comment_type, created_at
                FROM fpna_config.capex_forecast_workflow_comments
                WHERE workflow_id = :wf_id AND comment_type = :comment_type
                ORDER BY created_at ASC
            """),
            {"wf_id": wf["id"], "comment_type": comment_type},
        )).mappings().all()
    else:
        rows = (await db.execute(
            text("""
                SELECT id, user_id, user_name, department, comment, comment_type, created_at
                FROM fpna_config.capex_forecast_workflow_comments
                WHERE workflow_id = :wf_id
                ORDER BY created_at ASC
            """),
            {"wf_id": wf["id"]},
        )).mappings().all()

    comments = [
        {
            "id": r["id"],
            "user_id": r["user_id"],
            "user_name": r["user_name"],
            "department": r["department"],
            "comment": r["comment"],
            "comment_type": r["comment_type"],
            "created_at": r["created_at"].isoformat(),
        }
        for r in rows
    ]

    return {
        "year": year,
        "quarter": quarter.value,
        "workflow_stage": stage,
        "total_comments": len(comments),
        "filters": {
            "comment_type": comment_type or "all",
        },
        "comments": comments,
    }


@forecast_router.post("/capex-forecast/{year}/scenarios")
async def create_capex_forecast_scenario(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: CapexForecastScenarioInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    user_name = current_user.get("full_name", "")

    scenario_name_clean = body.scenario_name.strip().lower().replace(" ", "_")

    if scenario_name_clean in RESERVED_SCENARIO_NAMES:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail=f"'{body.scenario_name}' is a reserved scenario name. Reserved names are: baseline, accelerated_investment, delayed_spending.",
        )

    if not (0.01 <= body.multiplier <= 10.0):
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Multiplier must be between 0.01 and 10.0.",
        )

    parameter_types = body.parameter_types if body.parameter_types else ["capex"]
    customer_scope = body.customer_scope if body.customer_scope else "all"

    existing = (await db.execute(
        text("""
            SELECT id FROM fpna_config.capex_forecast_scenarios
            WHERE company_id = :company_id AND year = :year AND quarter = :quarter
              AND LOWER(REPLACE(scenario_name, ' ', '_')) = :scenario_name_clean
        """),
        {"company_id": company_id, "year": year, "quarter": quarter.value, "scenario_name_clean": scenario_name_clean},
    )).mappings().first()

    if existing:
        raise HTTPException(
            status_code=status.HTTP_409_CONFLICT,
            detail=f"A scenario named '{body.scenario_name}' already exists for {year} {quarter.value}.",
        )

    result = (await db.execute(
        text("""
            INSERT INTO fpna_config.capex_forecast_scenarios
                (company_id, year, quarter, scenario_name, multiplier, description,
                 parameter_types, customer_scope, created_by_user_id, created_by_name)
            VALUES (:company_id, :year, :quarter, :scenario_name, :multiplier, :description,
                    :parameter_types, :customer_scope, :uid, :uname)
            RETURNING id, scenario_name, multiplier, description, parameter_types,
                      customer_scope, created_by_name, created_at, updated_at
        """),
        {
            "company_id": company_id,
            "year": year,
            "quarter": quarter.value,
            "scenario_name": body.scenario_name.strip(),
            "multiplier": body.multiplier,
            "description": body.description.strip() if body.description else None,
            "parameter_types": json.dumps(parameter_types),
            "customer_scope": customer_scope,
            "uid": user_id,
            "uname": user_name,
        },
    )).mappings().first()

    await db.commit()

    item_multipliers_saved = []
    if customer_scope == "specific" and body.item_multipliers:
        for im in body.item_multipliers:
            if not im.capex_item or im.multiplier is None:
                continue
            if not (0.01 <= im.multiplier <= 10.0):
                continue
            await db.execute(
                text("""
                    INSERT INTO fpna_config.capex_forecast_scenario_item_multipliers
                        (scenario_id, company_id, year, quarter, capex_item, multiplier)
                    VALUES (:scenario_id, :company_id, :year, :quarter, :capex_item, :multiplier)
                    ON CONFLICT (scenario_id, company_id, year, quarter, capex_item)
                    DO UPDATE SET multiplier = EXCLUDED.multiplier, updated_at = NOW()
                """),
                {
                    "scenario_id": result["id"],
                    "company_id": company_id,
                    "year": year,
                    "quarter": quarter.value,
                    "capex_item": im.capex_item,
                    "multiplier": im.multiplier,
                },
            )
            item_multipliers_saved.append({"capex_item": im.capex_item, "multiplier": im.multiplier})
        await db.commit()

    return {
        "detail": "CAPEX scenario created successfully.",
        "year": year,
        "quarter": quarter.value,
        "scenario": {
            "id": result["id"],
            "scenario_name": result["scenario_name"],
            "multiplier": float(result["multiplier"]),
            "description": result["description"],
            "parameter_types": result["parameter_types"] if result["parameter_types"] else ["capex"],
            "customer_scope": result["customer_scope"],
            "item_multipliers": item_multipliers_saved,
            "created_by_name": result["created_by_name"],
            "created_at": result["created_at"].isoformat(),
            "updated_at": result["updated_at"].isoformat(),
        },
    }


@forecast_router.get("/capex-forecast/{year}/scenarios")
async def get_capex_forecast_scenarios(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    company_id = get_company_id(current_user)

    custom_scenarios = await _get_capex_custom_scenarios(db, year, quarter.value, company_id)

    for s in custom_scenarios:
        s["is_system"] = False
        if s["customer_scope"] == "specific":
            s["item_multipliers"] = await _get_capex_scenario_item_multipliers(
                db, s["id"], company_id, year, quarter.value
            )
        else:
            s["item_multipliers"] = {}

    hardcoded = [
        {
            "id": None,
            "scenario_name": "baseline",
            "multiplier": 1.0,
            "description": CAPEX_SCENARIO_DEFINITIONS["baseline"],
            "parameter_types": ["capex"],
            "customer_scope": "all",
            "item_multipliers": {},
            "is_system": True,
            "created_by_name": None,
            "created_at": None,
            "updated_at": None,
        },
        {
            "id": None,
            "scenario_name": "accelerated_investment",
            "multiplier": ACCELERATED_MULTIPLIER,
            "description": CAPEX_SCENARIO_DEFINITIONS["accelerated_investment"],
            "parameter_types": ["capex", "rate"],
            "customer_scope": "all",
            "item_multipliers": {},
            "is_system": True,
            "created_by_name": None,
            "created_at": None,
            "updated_at": None,
        },
        {
            "id": None,
            "scenario_name": "delayed_spending",
            "multiplier": DELAYED_SPENDING_MULTIPLIER,
            "description": CAPEX_SCENARIO_DEFINITIONS["delayed_spending"],
            "parameter_types": ["volume", "quantity"],
            "customer_scope": "all",
            "item_multipliers": {},
            "is_system": True,
            "created_by_name": None,
            "created_at": None,
            "updated_at": None,
        },
    ]

    return {
        "year": year,
        "quarter": quarter.value,
        "total_scenarios": len(hardcoded) + len(custom_scenarios),
        "scenarios": hardcoded + custom_scenarios,
    }


@forecast_router.patch("/capex-forecast/{year}/scenarios/{scenario_id}")
async def update_capex_forecast_scenario(
    year: int = Path(...),
    scenario_id: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: CapexForecastScenarioUpdateInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    company_id = get_company_id(current_user)

    existing = (await db.execute(
        text("""
            SELECT id, scenario_name FROM fpna_config.capex_forecast_scenarios
            WHERE id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter
        """),
        {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
    )).mappings().first()

    if not existing:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail=f"CAPEX scenario with id {scenario_id} not found for {year} {quarter.value}.",
        )

    if body.scenario_name is not None:
        new_name_clean = body.scenario_name.strip().lower().replace(" ", "_")
        if new_name_clean in RESERVED_SCENARIO_NAMES:
            raise HTTPException(
                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                detail=f"'{body.scenario_name}' is a reserved scenario name. Reserved names are: baseline, accelerated_investment, delayed_spending.",
            )
        name_conflict = (await db.execute(
            text("""
                SELECT id FROM fpna_config.capex_forecast_scenarios
                WHERE company_id = :company_id AND year = :year AND quarter = :quarter
                  AND LOWER(REPLACE(scenario_name, ' ', '_')) = :new_name_clean
                  AND id != :sid
            """),
            {"company_id": company_id, "year": year, "quarter": quarter.value, "new_name_clean": new_name_clean, "sid": scenario_id},
        )).mappings().first()
        if name_conflict:
            raise HTTPException(
                status_code=status.HTTP_409_CONFLICT,
                detail=f"A scenario named '{body.scenario_name}' already exists for {year} {quarter.value}.",
            )

    if body.multiplier is not None and not (0.01 <= body.multiplier <= 10.0):
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Multiplier must be between 0.01 and 10.0.",
        )

    set_clauses = ["updated_at = NOW()"]
    params: Dict[str, Any] = {"sid": scenario_id}

    if body.scenario_name is not None:
        set_clauses.append("scenario_name = :scenario_name")
        params["scenario_name"] = body.scenario_name.strip()
    if body.multiplier is not None:
        set_clauses.append("multiplier = :multiplier")
        params["multiplier"] = body.multiplier
    if body.description is not None:
        set_clauses.append("description = :description")
        params["description"] = body.description.strip()
    if body.parameter_types is not None:
        set_clauses.append("parameter_types = :parameter_types")
        params["parameter_types"] = json.dumps(body.parameter_types)
    if body.customer_scope is not None:
        set_clauses.append("customer_scope = :customer_scope")
        params["customer_scope"] = body.customer_scope

    result = (await db.execute(
        text(f"""
            UPDATE fpna_config.capex_forecast_scenarios
            SET {', '.join(set_clauses)}
            WHERE id = :sid
            RETURNING id, scenario_name, multiplier, description, parameter_types,
                      customer_scope, created_by_name, created_at, updated_at
        """),
        params,
    )).mappings().first()

    await db.commit()

    new_scope = result["customer_scope"] if result["customer_scope"] else "all"
    item_multipliers_saved = {}

    if new_scope == "specific" and body.item_multipliers is not None:
        await db.execute(
            text("""
                DELETE FROM fpna_config.capex_forecast_scenario_item_multipliers
                WHERE scenario_id = :scenario_id AND company_id = :company_id
                  AND year = :year AND quarter = :quarter
            """),
            {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
        )
        for im in body.item_multipliers:
            if not im.capex_item or im.multiplier is None:
                continue
            if not (0.01 <= im.multiplier <= 10.0):
                continue
            await db.execute(
                text("""
                    INSERT INTO fpna_config.capex_forecast_scenario_item_multipliers
                        (scenario_id, company_id, year, quarter, capex_item, multiplier)
                    VALUES (:scenario_id, :company_id, :year, :quarter, :capex_item, :multiplier)
                    ON CONFLICT (scenario_id, company_id, year, quarter, capex_item)
                    DO UPDATE SET multiplier = EXCLUDED.multiplier, updated_at = NOW()
                """),
                {
                    "scenario_id": scenario_id,
                    "company_id": company_id,
                    "year": year,
                    "quarter": quarter.value,
                    "capex_item": im.capex_item,
                    "multiplier": im.multiplier,
                },
            )
            item_multipliers_saved[im.capex_item] = im.multiplier
        await db.commit()
    elif new_scope == "all":
        await db.execute(
            text("""
                DELETE FROM fpna_config.capex_forecast_scenario_item_multipliers
                WHERE scenario_id = :scenario_id AND company_id = :company_id
                  AND year = :year AND quarter = :quarter
            """),
            {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
        )
        await db.commit()
    else:
        item_multipliers_saved = await _get_capex_scenario_item_multipliers(
            db, scenario_id, company_id, year, quarter.value
        )

    return {
        "detail": "CAPEX scenario updated successfully.",
        "year": year,
        "quarter": quarter.value,
        "scenario": {
            "id": result["id"],
            "scenario_name": result["scenario_name"],
            "multiplier": float(result["multiplier"]),
            "description": result["description"],
            "parameter_types": result["parameter_types"] if result["parameter_types"] else ["capex"],
            "customer_scope": result["customer_scope"],
            "item_multipliers": item_multipliers_saved,
            "created_by_name": result["created_by_name"],
            "created_at": result["created_at"].isoformat(),
            "updated_at": result["updated_at"].isoformat(),
        },
    }


@forecast_router.delete("/capex-forecast/{year}/scenarios/{scenario_id}")
async def delete_capex_forecast_scenario(
    year: int = Path(...),
    scenario_id: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    company_id = get_company_id(current_user)

    existing = (await db.execute(
        text("""
            SELECT id, scenario_name FROM fpna_config.capex_forecast_scenarios
            WHERE id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter
        """),
        {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
    )).mappings().first()

    if not existing:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail=f"CAPEX scenario with id {scenario_id} not found for {year} {quarter.value}.",
        )

    await db.execute(
        text("""
            DELETE FROM fpna_config.capex_forecast_scenario_item_multipliers
            WHERE scenario_id = :sid AND company_id = :company_id
              AND year = :year AND quarter = :quarter
        """),
        {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
    )

    await db.execute(
        text("DELETE FROM fpna_config.capex_forecast_scenarios WHERE id = :sid"),
        {"sid": scenario_id},
    )
    await db.commit()

    return {
        "detail": f"CAPEX scenario '{existing['scenario_name']}' deleted successfully.",
        "year": year,
        "quarter": quarter.value,
        "deleted_scenario_id": scenario_id,
    }


@forecast_router.get("/capex-forecast/{year}/excel-template/capex-adjustments")
async def download_capex_adjustments_template(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    company_id = get_company_id(current_user)
    m1, m2, m3 = get_quarter_months(year, quarter.value)

    saved = await _get_capex_adjustments(db, year, quarter.value, company_id)
    adj_map = {s["capex_item"]: s for s in saved}

    history_months_start = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")
    prev_month = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=1)).strftime("%Y-%m")

    distinct_sql = text(
        "SELECT DISTINCT fa.category FROM fpna_landing.finance_fixed_asset fa WHERE fa.category IS NOT NULL ORDER BY fa.category"
    )
    distinct_result = await db.execute(distinct_sql)
    all_categories = [row[0] for row in distinct_result.fetchall()]

    history_sql = text("""
        SELECT fa.category,
               TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') AS period,
               SUM(fa.purchase_price) AS purchase_price
        FROM fpna_landing.finance_fixed_asset fa
        WHERE TO_DATE(fa.date_acquired, 'DD-MM-YYYY') >= TO_DATE(:start_period || '-01', 'YYYY-MM-DD')
          AND TO_DATE(fa.date_acquired, 'DD-MM-YYYY') < TO_DATE(:quarter_start || '-01', 'YYYY-MM-DD')
          AND fa.category IS NOT NULL
        GROUP BY fa.category, TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM')
        ORDER BY fa.category, period;
    """)

    item_hist_result = await db.execute(history_sql, {"start_period": history_months_start, "quarter_start": m1})
    item_hist_df = pd.DataFrame(item_hist_result.mappings().all())

    if item_hist_df.empty:
        item_hist_df = pd.DataFrame([{"category": all_categories[0] if all_categories else "Uncategorized", "period": prev_month, "purchase_price": 0.0}])

    quarter_start = pd.to_datetime(m1 + "-01")

    item_forecasts: Dict[str, tuple] = {}
    for capex_item in all_categories:
        item_forecasts[capex_item] = _forecast_capex_item(capex_item, item_hist_df, quarter_start, m1, m2, m3)

    prev_sql = text("""
        SELECT fa.category, SUM(fa.purchase_price) AS total_purchase
        FROM fpna_landing.finance_fixed_asset fa
        WHERE TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') = :prev_month
          AND fa.category IS NOT NULL
        GROUP BY fa.category;
    """)
    prev_result = await db.execute(prev_sql, {"prev_month": prev_month})
    prev_df = pd.DataFrame(prev_result.mappings().all())
    prev_map = dict(zip(prev_df["category"], prev_df["total_purchase"])) if not prev_df.empty else {}

    wb = openpyxl.Workbook()
    ws = wb.active
    ws.title = "CAPEX Adjustments"

    headers = [
        "CAPEX Item",
        f"Previous Month ({prev_month})",
        f"{m1} AI Forecast",
        f"{m1} Adjusted",
        f"{m1} Comment",
        f"{m2} AI Forecast",
        f"{m2} Adjusted",
        f"{m2} Comment",
        f"{m3} AI Forecast",
        f"{m3} Adjusted",
        f"{m3} Comment",
        "Quarter Total AI",
        "Quarter Total Adjusted",
        "Overall Comment",
    ]
    ws.append(headers)

    for capex_item in all_categories:
        adj = adj_map.get(capex_item, {})
        prev_val = round(float(prev_map.get(capex_item, 0.0)), 2)
        m1_ai, m2_ai, m3_ai = item_forecasts.get(capex_item, (0.0, 0.0, 0.0))
        m1_ai = round(float(m1_ai), 2)
        m2_ai = round(float(m2_ai), 2)
        m3_ai = round(float(m3_ai), 2)
        quarter_total_ai = round(m1_ai + m2_ai + m3_ai, 2)

        m1_adj = adj.get("month1_adjusted")
        m2_adj = adj.get("month2_adjusted")
        m3_adj = adj.get("month3_adjusted")
        quarter_total_adj = adj.get("quarter_total_adjusted", round((m1_adj or 0.0) + (m2_adj or 0.0) + (m3_adj or 0.0), 2))

        ws.append([
            capex_item, prev_val, m1_ai, m1_adj, adj.get("month1_comment"),
            m2_ai, m2_adj, adj.get("month2_comment"),
            m3_ai, m3_adj, adj.get("month3_comment"),
            quarter_total_ai,
            quarter_total_adj if (m1_adj or m2_adj or m3_adj) else None,
            adj.get("overall_comment"),
        ])

    stream = io.BytesIO()
    wb.save(stream)
    stream.seek(0)

    return StreamingResponse(
        stream,
        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        headers={"Content-Disposition": f"attachment; filename=capex_adjustments_{year}_{quarter.value}.xlsx"},
    )


@forecast_router.get("/capex-forecast/{year}/excel-template/driver-adjustments")
async def download_capex_driver_adjustments_template(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    company_id = get_company_id(current_user)
    m1, m2, m3 = get_quarter_months(year, quarter.value)

    driver_saved = await _get_capex_driver_adjustments(db, year, quarter.value, company_id)

    history_months_start = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")
    prev_month = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=1)).strftime("%Y-%m")

    distinct_sql = text(
        "SELECT DISTINCT fa.category FROM fpna_landing.finance_fixed_asset fa WHERE fa.category IS NOT NULL ORDER BY fa.category"
    )
    distinct_result = await db.execute(distinct_sql)
    all_categories = [row[0] for row in distinct_result.fetchall()]

    history_sql = text("""
        SELECT fa.category,
               TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') AS period,
               SUM(fa.purchase_price) AS purchase_price
        FROM fpna_landing.finance_fixed_asset fa
        WHERE TO_DATE(fa.date_acquired, 'DD-MM-YYYY') >= TO_DATE(:start_period || '-01', 'YYYY-MM-DD')
          AND TO_DATE(fa.date_acquired, 'DD-MM-YYYY') < TO_DATE(:quarter_start || '-01', 'YYYY-MM-DD')
          AND fa.category IS NOT NULL
        GROUP BY fa.category, TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM')
        ORDER BY fa.category, period;
    """)

    item_hist_result = await db.execute(history_sql, {"start_period": history_months_start, "quarter_start": m1})
    item_hist_df = pd.DataFrame(item_hist_result.mappings().all())

    if item_hist_df.empty:
        item_hist_df = pd.DataFrame([{"category": all_categories[0] if all_categories else "Uncategorized", "period": prev_month, "purchase_price": 0.0}])

    quarter_start = pd.to_datetime(m1 + "-01")

    item_forecasts: Dict[str, tuple] = {}
    for capex_item in all_categories:
        item_forecasts[capex_item] = _forecast_capex_item(capex_item, item_hist_df, quarter_start, m1, m2, m3)

    prev_sql = text("""
        SELECT fa.category, SUM(fa.purchase_price) AS total_purchase
        FROM fpna_landing.finance_fixed_asset fa
        WHERE TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') = :prev_month
          AND fa.category IS NOT NULL
        GROUP BY fa.category;
    """)
    prev_result = await db.execute(prev_sql, {"prev_month": prev_month})
    prev_df = pd.DataFrame(prev_result.mappings().all())
    prev_map = dict(zip(prev_df["category"], prev_df["total_purchase"])) if not prev_df.empty else {}

    wb = openpyxl.Workbook()
    ws = wb.active
    ws.title = "Driver Adjustments"

    headers = [
        "CAPEX Item", "Month", f"Previous Month ({prev_month}) CAPEX",
        "Units AI", "Rate AI", "CAPEX AI",
        "Units Adjusted", "Rate Adjusted", "CAPEX Adjusted",
        "Month Comment", "Quarter CAPEX AI", "Quarter CAPEX Adjusted", "Overall Comment",
    ]
    ws.append(headers)

    item_quarter_ai: Dict[str, float] = {}
    item_quarter_drv: Dict[str, float] = {}

    for capex_item in all_categories:
        m1_ai, m2_ai, m3_ai = item_forecasts.get(capex_item, (0.0, 0.0, 0.0))
        ai_map = {m1: float(m1_ai), m2: float(m2_ai), m3: float(m3_ai)}
        item_quarter_ai[capex_item] = round(sum(ai_map.values()), 2)

        q_drv = 0.0
        for month_label in [m1, m2, m3]:
            drv = driver_saved.get((capex_item, month_label), {})
            u_ai, r_ai = derive_units_and_rate(ai_map[month_label])
            u_user = drv.get("units_adjusted")
            r_user = drv.get("rate_adjusted")
            if u_user is not None and r_user is not None:
                capex_adj = round(u_user * r_user, 2)
            elif u_user is not None:
                capex_adj = round(u_user * r_ai, 2)
            elif r_user is not None:
                capex_adj = round(u_ai * r_user, 2)
            else:
                capex_adj = None
            if capex_adj is not None:
                q_drv += capex_adj
        item_quarter_drv[capex_item] = round(q_drv, 2)

    for capex_item in all_categories:
        m1_ai, m2_ai, m3_ai = item_forecasts.get(capex_item, (0.0, 0.0, 0.0))
        ai_map = {m1: float(m1_ai), m2: float(m2_ai), m3: float(m3_ai)}
        prev_val = round(float(prev_map.get(capex_item, 0.0)), 2)

        for month_label in [m1, m2, m3]:
            capex_ai = ai_map[month_label]
            u_ai, r_ai = derive_units_and_rate(capex_ai)
            drv = driver_saved.get((capex_item, month_label), {})
            u_user = drv.get("units_adjusted")
            r_user = drv.get("rate_adjusted")

            if u_user is not None and r_user is not None:
                capex_adj = round(u_user * r_user, 2)
            elif u_user is not None:
                capex_adj = round(u_user * r_ai, 2)
            elif r_user is not None:
                capex_adj = round(u_ai * r_user, 2)
            else:
                capex_adj = None

            ws.append([
                capex_item, month_label, prev_val,
                round(u_ai, 4), round(r_ai, 2), round(capex_ai, 2),
                u_user, r_user, capex_adj, drv.get("month_comment"),
                item_quarter_ai[capex_item],
                item_quarter_drv[capex_item] if item_quarter_drv[capex_item] else None,
                drv.get("overall_comment"),
            ])

    stream = io.BytesIO()
    wb.save(stream)
    stream.seek(0)

    return StreamingResponse(
        stream,
        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        headers={"Content-Disposition": f"attachment; filename=capex_driver_adjustments_{year}_{quarter.value}.xlsx"},
    )


@forecast_router.post("/capex-forecast/{year}/excel-upload/capex-adjustments")
async def upload_capex_adjustments_excel(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    file: UploadFile = File(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_capex_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="CAPEX forecast is locked. Uploads cannot be processed.")

    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Uploads can only be processed in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")

    contents = await file.read()
    if len(contents) > 5 * 1024 * 1024:
        raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File size exceeds 5MB limit.")

    try:
        wb = openpyxl.load_workbook(io.BytesIO(contents))
    except Exception:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Could not read the uploaded file. Please ensure it is a valid .xlsx file.")

    m1, m2, m3 = get_quarter_months(year, quarter.value)
    adjustments = []
    errors = []

    def get_float_cell(row, idx):
        if idx >= len(row) or row[idx] is None:
            return None
        try:
            return float(row[idx])
        except (TypeError, ValueError):
            return None

    def get_str_cell(row, idx):
        if idx >= len(row) or row[idx] is None:
            return None
        val = str(row[idx]).strip()
        return val if val else None

    sheet_name = "CAPEX Adjustments"
    if sheet_name not in wb.sheetnames:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Sheet '{sheet_name}' not found in the uploaded file.")

    ws_adj = wb[sheet_name]
    adj_headers_row = [str(c.value).strip() if c.value is not None else "" for c in ws_adj[1]]
    adj_hmap = {h: i for i, h in enumerate(adj_headers_row)}

    for row_idx, row in enumerate(ws_adj.iter_rows(min_row=2, values_only=True), start=2):
        if not row or row[0] is None:
            continue
        capex_item = str(row[0]).strip()
        if not capex_item:
            continue
        try:
            m1_adj = get_float_cell(row, adj_hmap.get(f"{m1} Adjusted", 999))
            m1_cmt = get_str_cell(row, adj_hmap.get(f"{m1} Comment", 999))
            m2_adj = get_float_cell(row, adj_hmap.get(f"{m2} Adjusted", 999))
            m2_cmt = get_str_cell(row, adj_hmap.get(f"{m2} Comment", 999))
            m3_adj = get_float_cell(row, adj_hmap.get(f"{m3} Adjusted", 999))
            m3_cmt = get_str_cell(row, adj_hmap.get(f"{m3} Comment", 999))
            overall_comment = get_str_cell(row, adj_hmap.get("Overall Comment", 999))
        except Exception:
            errors.append(f"CAPEX Adjustments row {row_idx}: parse error for '{capex_item}'. Skipped.")
            continue

        if any(v is not None for v in [m1_adj, m2_adj, m3_adj, m1_cmt, m2_cmt, m3_cmt, overall_comment]):
            adjustments.append(CapexAdjustmentInput(
                capex_item=capex_item,
                month1_adjusted=m1_adj, month1_comment=m1_cmt,
                month2_adjusted=m2_adj, month2_comment=m2_cmt,
                month3_adjusted=m3_adj, month3_comment=m3_cmt,
                overall_comment=overall_comment,
            ))

    if not adjustments:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="No valid rows found in the CAPEX Adjustments sheet.")

    result = await CapexForecastAdjustmentManagement.PatchAdjustments(year=year, quarter=quarter.value, company_id=company_id, adjustments=adjustments, db=db)
    result["upload_errors"] = errors
    result["rows_attempted"] = len(adjustments) + len(errors)
    result["rows_saved"] = len(adjustments)
    return result


@forecast_router.post("/capex-forecast/{year}/excel-upload/driver-adjustments")
async def upload_capex_driver_adjustments_excel(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    file: UploadFile = File(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_capex_workflow(db, year, quarter.value, company_id, dept, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="CAPEX forecast is locked. Uploads cannot be processed.")

    if stage != WorkflowStage.DRAFT:
        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Uploads can only be processed in draft stage. Current stage is '{stage}'. Ask finance to send it back to draft if changes are needed.")

    contents = await file.read()
    if len(contents) > 5 * 1024 * 1024:
        raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File size exceeds 5MB limit.")

    try:
        wb = openpyxl.load_workbook(io.BytesIO(contents))
    except Exception:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Could not read the uploaded file. Please ensure it is a valid .xlsx file.")

    driver_adjustments = []
    errors = []

    def get_float_cell(row, idx):
        if idx >= len(row) or row[idx] is None:
            return None
        try:
            return float(row[idx])
        except (TypeError, ValueError):
            return None

    def get_str_cell(row, idx):
        if idx >= len(row) or row[idx] is None:
            return None
        val = str(row[idx]).strip()
        return val if val else None

    sheet_name = "Driver Adjustments"
    if sheet_name not in wb.sheetnames:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Sheet '{sheet_name}' not found in the uploaded file.")

    ws_drv = wb[sheet_name]
    drv_headers_row = [str(c.value).strip() if c.value is not None else "" for c in ws_drv[1]]
    drv_hmap = {h: i for i, h in enumerate(drv_headers_row)}

    for row_idx, row in enumerate(ws_drv.iter_rows(min_row=2, values_only=True), start=2):
        if not row or row[0] is None:
            continue
        capex_item = str(row[0]).strip()
        if not capex_item:
            continue
        try:
            month_val = get_str_cell(row, drv_hmap.get("Month", 999))
            units_adj = get_float_cell(row, drv_hmap.get("Units Adjusted", 999))
            rate_adj = get_float_cell(row, drv_hmap.get("Rate Adjusted", 999))
            month_cmt = get_str_cell(row, drv_hmap.get("Month Comment", 999))
            overall_comment = get_str_cell(row, drv_hmap.get("Overall Comment", 999))
        except Exception:
            errors.append(f"Driver Adjustments row {row_idx}: parse error for '{capex_item}'. Skipped.")
            continue

        if not month_val:
            continue

        if any(v is not None for v in [units_adj, rate_adj, month_cmt, overall_comment]):
            driver_adjustments.append(CapexDriverAdjustmentInput(
                capex_item=capex_item, month=month_val,
                units_adjusted=units_adj, rate_adjusted=rate_adj,
                month_comment=month_cmt, overall_comment=overall_comment,
            ))

    if not driver_adjustments:
        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="No valid rows found in the Driver Adjustments sheet.")

    for adj in driver_adjustments:
        if not adj.capex_item or not adj.month:
            continue
        await db.execute(text("""
            INSERT INTO fpna_config.capex_driver_adjustments (
                year, quarter, company_id, capex_item, month,
                units_adjusted, rate_adjusted, month_comment, overall_comment, updated_at
            )
            VALUES (:year, :quarter, :company_id, :capex_item, :month,
                    :units_adjusted, :rate_adjusted, :month_comment, :overall_comment, NOW())
            ON CONFLICT (year, quarter, company_id, capex_item, month)
            DO UPDATE SET
                units_adjusted  = EXCLUDED.units_adjusted,
                rate_adjusted   = EXCLUDED.rate_adjusted,
                month_comment   = EXCLUDED.month_comment,
                overall_comment = EXCLUDED.overall_comment,
                updated_at      = NOW();
        """), {
            "year": year, "quarter": quarter.value, "company_id": company_id,
            "capex_item": adj.capex_item, "month": adj.month,
            "units_adjusted": adj.units_adjusted, "rate_adjusted": adj.rate_adjusted,
            "month_comment": adj.month_comment, "overall_comment": adj.overall_comment,
        })
    await db.commit()

    saved = (await db.execute(text("""
        SELECT capex_item, month, units_adjusted, rate_adjusted, month_comment, overall_comment
        FROM fpna_config.capex_driver_adjustments
        WHERE year = :year AND quarter = :quarter AND company_id = :company_id
        ORDER BY capex_item, month;
    """), {"year": year, "quarter": quarter.value, "company_id": company_id})).mappings().all()

    item_quarter_map: Dict[str, float] = {}
    result_rows = []
    for r in saved:
        u = float(r["units_adjusted"]) if r["units_adjusted"] is not None else None
        ra = float(r["rate_adjusted"]) if r["rate_adjusted"] is not None else None
        capex_adj = round(u * ra, 2) if u is not None and ra is not None else None
        iname = r["capex_item"]
        if capex_adj is not None:
            item_quarter_map[iname] = item_quarter_map.get(iname, 0.0) + capex_adj
        result_rows.append({"capex_item": iname, "month": r["month"], "units_adjusted": u, "rate_adjusted": ra, "capex_adjusted": capex_adj, "month_comment": r["month_comment"], "overall_comment": r["overall_comment"]})

    for row in result_rows:
        row["quarter_capex_adjusted"] = round(item_quarter_map.get(row["capex_item"], 0.0), 2)

    return {"detail": "CAPEX driver adjustments uploaded successfully", "year": year, "quarter": quarter.value, "upload_errors": errors, "rows_attempted": len(driver_adjustments) + len(errors), "rows_saved": len(driver_adjustments), "driver_adjustments": result_rows}


@forecast_router.get("/capex-forecast/{year}/{currency}")
async def capex_forecast(
    year: int = Path(...),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    forecast_year: Optional[int] = Query(default=None, description="Primary forecast year context. Overrides year for period resolution when provided."),
    forecast_type: Optional[str] = Query(default="quarter", description="Forecast type: quarter, 12months, rolling"),
    rolling_start_month: Optional[str] = Query(default=None, description="Start month for rolling forecast (e.g. Jan, Feb, Mar). Required when forecast_type=rolling."),
    view_type: Optional[str] = Query(default="monthly_breakdown", description="View type: quarter_summary or monthly_breakdown"),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_capex_workflow_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    effective_year = forecast_year if forecast_year is not None else year
    effective_forecast_type = forecast_type or "quarter"
    effective_view_type = view_type or "monthly_breakdown"

    if effective_forecast_type not in ("quarter", "12months", "rolling"):
        raise HTTPException(status_code=422, detail="forecast_type must be one of: quarter, 12months, rolling")
    if effective_view_type not in ("quarter_summary", "monthly_breakdown"):
        raise HTTPException(status_code=422, detail="view_type must be one of: quarter_summary, monthly_breakdown")
    if effective_forecast_type == "rolling" and not rolling_start_month:
        raise HTTPException(status_code=422, detail="rolling_start_month is required when forecast_type=rolling.")

    # Rolling: Nov/Dec starts use previous year so the window spans INTO effective_year
    _MONTH_ORDER = {"jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
                    "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12}
    resolve_year = effective_year
    if effective_forecast_type == "rolling" and rolling_start_month:
        if _MONTH_ORDER.get(rolling_start_month.lower()[:3], 1) >= 11:
            resolve_year = effective_year - 1

    month_periods, selected_period, prev_month = _resolve_forecast_periods(
        resolve_year, effective_forecast_type, quarter.value, rolling_start_month
    )
    n_months = len(month_periods)
    m1 = month_periods[0]

    effective_quarter = quarter.value
    if effective_forecast_type == "12months":
        effective_quarter = "Q1"
    elif effective_forecast_type == "rolling":
        mn = int(month_periods[0].split("-")[1])
        effective_quarter = "Q1" if mn <= 3 else "Q2" if mn <= 6 else "Q3" if mn <= 9 else "Q4"

    quarter_start = pd.to_datetime(m1 + "-01")
    history_start = (quarter_start - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")

    wf = await _get_or_create_capex_workflow(db, effective_year, effective_quarter, company_id, dept, user_id)
    stage = wf["stage"]

    audit_rows = (await db.execute(text("""
        SELECT to_stage, transitioned_by_name
        FROM fpna_config.capex_forecast_workflow_audit
        WHERE workflow_id = :wf_id
        ORDER BY created_at ASC
    """), {"wf_id": wf["id"]})).mappings().all()

    reviewed_by = approved_by = locked_by = None
    for r in audit_rows:
        if r["to_stage"] == "reviewed":
            reviewed_by = r["transitioned_by_name"]
        elif r["to_stage"] == "approved":
            approved_by = r["transitioned_by_name"]
        elif r["to_stage"] == "locked":
            locked_by = r["transitioned_by_name"]

    distinct_result = await db.execute(text(
        "SELECT DISTINCT fa.category FROM fpna_landing.finance_fixed_asset fa WHERE fa.category IS NOT NULL ORDER BY fa.category"
    ))
    all_categories = [row[0] for row in distinct_result.fetchall()]

    history_sql = text("""
        SELECT fa.category,
               TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') AS period,
               SUM(fa.purchase_price) AS purchase_price
        FROM fpna_landing.finance_fixed_asset fa
        WHERE TO_DATE(fa.date_acquired, 'DD-MM-YYYY') >= TO_DATE(:start_period || '-01', 'YYYY-MM-DD')
          AND TO_DATE(fa.date_acquired, 'DD-MM-YYYY') < TO_DATE(:quarter_start || '-01', 'YYYY-MM-DD')
          AND fa.category IS NOT NULL
        GROUP BY fa.category, TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM')
        ORDER BY fa.category, period;
    """)
    item_hist_result = await db.execute(history_sql, {"start_period": history_start, "quarter_start": m1})
    item_hist_df = pd.DataFrame(item_hist_result.mappings().all())

    if item_hist_df.empty:
        item_hist_df = pd.DataFrame([{"category": all_categories[0] if all_categories else "Uncategorized", "period": prev_month, "purchase_price": 0.0}])

    # Generate AI forecasts: use existing 3-month pipeline for quarter/rolling, N-month for 12months
    item_forecasts: Dict[str, List[float]] = {}
    if n_months <= 3:
        m_list = month_periods + [""] * (3 - n_months)  # pad to always have m1,m2,m3 refs
        for capex_item in all_categories:
            raw = _forecast_capex_item(capex_item, item_hist_df, quarter_start, month_periods[0],
                                       month_periods[1] if n_months > 1 else month_periods[0],
                                       month_periods[2] if n_months > 2 else month_periods[0])
            item_forecasts[capex_item] = list(raw)[:n_months]
    else:
        for capex_item in all_categories:
            item_forecasts[capex_item] = _forecast_capex_item_n_months(capex_item, item_hist_df, quarter_start, n_months)

    prev_result = await db.execute(text("""
        SELECT fa.category, SUM(fa.purchase_price) AS total_purchase
        FROM fpna_landing.finance_fixed_asset fa
        WHERE TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') = :prev_month
          AND fa.category IS NOT NULL
        GROUP BY fa.category;
    """), {"prev_month": prev_month})
    prev_df = pd.DataFrame(prev_result.mappings().all())
    prev_map = dict(zip(prev_df["category"], prev_df["total_purchase"])) if not prev_df.empty else {}

    saved, driver_saved, custom_scenarios = await asyncio.gather(
        _get_capex_adjustments(db, effective_year, effective_quarter, company_id),
        _get_capex_driver_adjustments(db, effective_year, effective_quarter, company_id),
        _get_capex_custom_scenarios(db, effective_year, effective_quarter, company_id),
    )
    adj_map: Dict[str, Dict[str, Any]] = {s["capex_item"]: s for s in saved}

    rows: List[Dict] = []
    totals: Dict[str, float] = {"previous_month": 0.0, "month1_ai": 0.0, "month1_adjusted": 0.0,
                                 "month2_ai": 0.0, "month2_adjusted": 0.0, "month3_ai": 0.0, "month3_adjusted": 0.0}
    driver_map: Dict[str, list] = {}
    item_ai_map: Dict[str, float] = {}
    item_user_map: Dict[str, float] = {}
    month_totals: List[Dict[str, Any]] = [{"month": mp, "total_ai": 0.0, "total_adjusted": 0.0} for mp in month_periods]
    item_monthly_data: Dict[str, List[Dict]] = {}

    for capex_item in all_categories:
        adj = adj_map.get(capex_item)
        prev_value = float(prev_map.get(capex_item, 0.0))
        overall_comment = adj.get("overall_comment") if adj is not None else None
        fc_list = item_forecasts.get(capex_item, [0.0] * n_months)

        month_ai_vals: List[float] = []
        month_adj_vals: List[float] = []
        month_comments: List[Optional[str]] = []
        driver_month_entries: Dict[str, Dict[str, Any]] = {}

        for idx, month_label in enumerate(month_periods):
            capex_ai_val = float(fc_list[idx]) if idx < len(fc_list) else 0.0
            u_ai, r_ai = derive_units_and_rate(capex_ai_val)
            drv = driver_saved.get((capex_item, month_label))

            if drv:
                u_user, r_user = drv["units_adjusted"], drv["rate_adjusted"]
                drv_month_comment = drv.get("month_comment")
                drv_overall_comment = drv.get("overall_comment")
                if u_user is not None and r_user is not None:
                    capex_adj_driver = round(u_user * r_user, 2)
                elif u_user is not None:
                    capex_adj_driver = round(float(u_user) * r_ai, 2)
                elif r_user is not None:
                    capex_adj_driver = round(u_ai * float(r_user), 2)
                else:
                    capex_adj_driver = None
            else:
                u_user = r_user = drv_month_comment = drv_overall_comment = capex_adj_driver = None

            driver_month_entries[month_label] = {
                "u_ai": u_ai, "r_ai": r_ai, "u_user": u_user, "r_user": r_user,
                "capex_adj_driver": capex_adj_driver, "month_comment": drv_month_comment,
                "overall_comment": drv_overall_comment,
            }

            if capex_item not in driver_map:
                driver_map[capex_item] = []
            driver_map[capex_item].append({
                "capex_item": capex_item, "month": month_label,
                "units_ai": u_ai, "rate_ai": r_ai, "capex_ai": round(u_ai * r_ai, 2),
                "units_adjusted": round(float(u_user), 4) if u_user is not None else None,
                "rate_adjusted": round(float(r_user), 2) if r_user is not None else None,
                "capex_adjusted": capex_adj_driver, "month_comment": drv_month_comment,
                "roi": calculate_roi(capex_adj_driver if capex_adj_driver is not None else round(u_ai * r_ai, 2)),
            })

            # Priority: driver override > manual adjustment > AI value
            if capex_adj_driver is not None:
                final_val = capex_adj_driver
                comment_val = drv_month_comment
            else:
                adj_key = f"month{idx + 1}_adjusted"
                raw_adj = adj.get(adj_key) if adj is not None else None
                final_val = float(raw_adj) if raw_adj is not None else capex_ai_val
                comment_val = (adj.get(f"month{idx + 1}_comment") if adj is not None else None) if capex_adj_driver is None else drv_month_comment

            month_ai_vals.append(round(capex_ai_val, 2))
            month_adj_vals.append(round(final_val, 2))
            month_comments.append(comment_val)

            month_totals[idx]["total_ai"] = round(month_totals[idx]["total_ai"] + round(capex_ai_val, 2), 2)
            month_totals[idx]["total_adjusted"] = round(month_totals[idx]["total_adjusted"] + round(final_val, 2), 2)

        driver_overall = next(
            (driver_month_entries[ml].get("overall_comment") for ml in month_periods
             if driver_month_entries.get(ml, {}).get("overall_comment")), None
        )
        effective_overall_comment = driver_overall if driver_overall is not None else overall_comment

        period_total_ai = round(sum(month_ai_vals), 2)
        period_total_adjusted = round(sum(month_adj_vals), 2)
        item_ai_map[capex_item] = period_total_ai
        item_user_map[capex_item] = period_total_adjusted

        row = {
            "capex_item": capex_item,
            "previous_month": round(prev_value, 2),
            "month1_ai": month_ai_vals[0] if len(month_ai_vals) > 0 else 0.0,
            "month1_adjusted": month_adj_vals[0] if len(month_adj_vals) > 0 else 0.0,
            "month1_comment": month_comments[0] if month_comments else None,
            "month2_ai": month_ai_vals[1] if len(month_ai_vals) > 1 else 0.0,
            "month2_adjusted": month_adj_vals[1] if len(month_adj_vals) > 1 else 0.0,
            "month2_comment": month_comments[1] if len(month_comments) > 1 else None,
            "month3_ai": month_ai_vals[2] if len(month_ai_vals) > 2 else 0.0,
            "month3_adjusted": month_adj_vals[2] if len(month_adj_vals) > 2 else 0.0,
            "month3_comment": month_comments[2] if len(month_comments) > 2 else None,
            "quarter_total_ai": period_total_ai,
            "quarter_total_adjusted": period_total_adjusted,
            "overall_comment": effective_overall_comment,
            "roi": calculate_roi(period_total_adjusted),
        }
        rows.append(row)

        totals["previous_month"] += round(prev_value, 2)
        totals["month1_ai"] += month_ai_vals[0] if month_ai_vals else 0.0
        totals["month1_adjusted"] += month_adj_vals[0] if month_adj_vals else 0.0
        totals["month2_ai"] += month_ai_vals[1] if len(month_ai_vals) > 1 else 0.0
        totals["month2_adjusted"] += month_adj_vals[1] if len(month_adj_vals) > 1 else 0.0
        totals["month3_ai"] += month_ai_vals[2] if len(month_ai_vals) > 2 else 0.0
        totals["month3_adjusted"] += month_adj_vals[2] if len(month_adj_vals) > 2 else 0.0

        item_monthly_data[capex_item] = [
            {"month": mp, "ai": month_ai_vals[i], "adjusted": month_adj_vals[i]}
            for i, mp in enumerate(month_periods)
        ]

    drivers_output = []
    for iname, item_months in driver_map.items():
        q_ai  = sum(m["capex_ai"] for m in item_months)
        q_adj = sum(m["capex_adjusted"] if m["capex_adjusted"] is not None else m["capex_ai"] for m in item_months)
        overall_comment_for_item = None
        for month_label in month_periods:
            drv = driver_saved.get((iname, month_label))
            if drv and drv.get("overall_comment"):
                overall_comment_for_item = drv["overall_comment"]
                break
        drivers_output.append({
            "capex_item": iname,
            "months": sorted(item_months, key=lambda x: x["month"]),
            "quarter_capex_ai": round(q_ai, 2),
            "quarter_capex_adjusted": round(q_adj, 2),
            "overall_comment": overall_comment_for_item,
            "roi": calculate_roi(round(q_adj, 2)),
        })

    baseline_ai = round(sum(mt["total_ai"] for mt in month_totals), 2)
    baseline_user = round(sum(mt["total_adjusted"] for mt in month_totals), 2)

    baseline_forecast_summary = {
        "total_ai": baseline_ai,
        "total_user": baseline_user,
        "variance_amount": round(baseline_user - baseline_ai, 2),
        "variance_percent": round(((baseline_user - baseline_ai) / baseline_ai) * 100 if baseline_ai else 0, 2),
    }

    compare_capex_forecast_scenarios = {
        "baseline": _build_capex_scenario_summary(baseline_ai, baseline_user, 1.0),
        "accelerated_investment": _build_capex_scenario_summary(baseline_ai, baseline_user, ACCELERATED_MULTIPLIER),
        "delayed_spending": _build_capex_scenario_summary(baseline_ai, baseline_user, DELAYED_SPENDING_MULTIPLIER),
    }

    for cs in custom_scenarios:
        scenario_key = cs["scenario_name"].lower().replace(" ", "_")
        if cs["customer_scope"] == "specific":
            item_mults = await _get_capex_scenario_item_multipliers(db, cs["id"], company_id, effective_year, effective_quarter)
            compare_capex_forecast_scenarios[scenario_key] = _build_capex_scenario_summary_with_item_multipliers(
                baseline_ai=baseline_ai, baseline_user=baseline_user,
                scenario_multiplier=cs["multiplier"],
                item_ai_map=item_ai_map, item_user_map=item_user_map,
                item_multipliers=item_mults, customer_scope=cs["customer_scope"],
            )
        else:
            compare_capex_forecast_scenarios[scenario_key] = _build_capex_scenario_summary(baseline_ai, baseline_user, cs["multiplier"])

    rounded_totals = {k: round(v, 2) for k, v in totals.items()}

    # Apply view_type
    if effective_view_type == "quarter_summary":
        rows_output = [
            {
                "capex_item": r["capex_item"],
                "previous_month": r["previous_month"],
                "quarter_total_ai": r["quarter_total_ai"],
                "quarter_total_adjusted": r["quarter_total_adjusted"],
                "overall_comment": r["overall_comment"],
                "roi": r["roi"],
            }
            for r in rows
        ]
        if n_months > 3:
            q_summary: Dict[str, Dict[str, float]] = {}
            for qi, q_label in enumerate(["Q1", "Q2", "Q3", "Q4"]):
                si = qi * 3
                q_summary[q_label] = {
                    "total_ai":       round(sum(mt["total_ai"]      for mt in month_totals[si:si + 3]), 2),
                    "total_adjusted": round(sum(mt["total_adjusted"] for mt in month_totals[si:si + 3]), 2),
                }
    else:
        rows_output = rows
        if n_months > 3:
            for row_dict in rows_output:
                row_dict["monthly_data"] = item_monthly_data.get(row_dict["capex_item"], [])

    response: Dict[str, Any] = {
        "year": year,
        "forecast_year": effective_year,
        "forecast_type": effective_forecast_type,
        "view_type": effective_view_type,
        "selected_period": selected_period,
        "selected_months": month_periods,
        "quarter": quarter,
        "prev_month": prev_month,
        "rows": rows_output,
        "total": rounded_totals,
        "month_totals": month_totals,
        "Baseline_forecast_summary": baseline_forecast_summary,
        "Compare_CAPEX_Forecast_Scenarios": compare_capex_forecast_scenarios,
        "drivers": drivers_output,
        "workflow_stage": stage,
        "reviewed_by": reviewed_by,
        "approved_by": approved_by,
        "locked_by": locked_by,
    }
    if n_months > 3 and effective_view_type == "quarter_summary":
        response["quarter_summary"] = q_summary

    return apply_smart_currency(response, currency, currency_filter, CAPEX_MONEY_FIELDS)


##########################################################################################
###########################################################################################
###########################################################################################
#Forecasting Dashboard
from src.core.database import AsyncSessionLocal

WIDGET_DEFAULTS = {
    "revenue_forecasting": {"position": 1, "chart_type": "bar_chart", "enableordisable": True},
    "expense_forecasting": {"position": 2, "chart_type": "bar_chart", "enableordisable": True},
    "capex_forecast": {"position": 3, "chart_type": "pie_chart", "enableordisable": True},
    "cash_flow_projections": {"position": 4, "chart_type": "area_chart", "enableordisable": True},
    "headcount_forecast": {"position": 5, "chart_type": "line_chart", "enableordisable": True},
    "payroll_forecast": {"position": 6, "chart_type": "bar_chart", "enableordisable": True},
}

MONEY_FIELDS = {
    "revenue", "expenses", "cash_flow", "payroll", "capex",
    "forecast_revenue", "forecast_expenses", "forecast_cashflow",
    "forecast_payroll", "forecast_capex",
}

WIDGET_KEY_MAP = {
    "revenue_forecasting": ("REVENUE", "revenue"),
    "expense_forecasting": ("Expense", "expenses"),
    "capex_forecast": ("CAPEX Forecast", "capex"),
    "cash_flow_projections": ("Cash Flow Projections", "cash_flow"),
    "headcount_forecast": ("Headcount", "headcount"),
    "payroll_forecast": ("Payroll", "payroll"),
}

QUARTER_MONTH_MAP = {
    "Q1": [1, 2, 3],
    "Q2": [4, 5, 6],
    "Q3": [7, 8, 9],
    "Q4": [10, 11, 12],
}

MONTH_KEYS = ["month1_ai_suggested", "month2_ai_suggested", "month3_ai_suggested"]

METRIC_KEY_TO_WIDGET = {
    "REVENUE": "revenue_forecasting",
    "Expense": "expense_forecasting",
    "CAPEX Forecast": "capex_forecast",
    "Cash Flow Projections": "cash_flow_projections",
    "Headcount": "headcount_forecast",
    "Payroll": "payroll_forecast",
}

WIDGET_TO_METRIC_FUNC = {
    "revenue_forecasting": "_dashboard_get_revenue_metrics",
    "expense_forecasting": "_dashboard_get_expense_metrics",
    "capex_forecast": "_dashboard_get_capex_metrics",
    "cash_flow_projections": "_dashboard_get_cashflow_metrics",
    "headcount_forecast": "_dashboard_get_headcount_payroll_metrics",
    "payroll_forecast": "_dashboard_get_headcount_payroll_metrics",
}


async def _ensure_widget_settings_table(db):
    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_landing.forecast_widget_settings (
            id SERIAL PRIMARY KEY,
            year INTEGER NOT NULL,
            widget_name VARCHAR(100) NOT NULL,
            position INTEGER NOT NULL,
            chart_type VARCHAR(50) NOT NULL,
            enableordisable BOOLEAN NOT NULL DEFAULT TRUE,
            updated_at TIMESTAMP DEFAULT NOW(),
            UNIQUE(year, widget_name)
        )
    """))
    await db.commit()


async def _get_widget_settings(db, year: int) -> dict:
    try:
        result = await db.execute(
            text("SELECT widget_name, position, chart_type, enableordisable FROM fpna_landing.forecast_widget_settings WHERE year = :year ORDER BY position"),
            {"year": year}
        )
        rows = result.mappings().all()
        if not rows:
            return {}
        return {
            row["widget_name"]: {
                "position": row["position"],
                "chart_type": row["chart_type"],
                "enableordisable": row["enableordisable"],
            }
            for row in rows
        }
    except Exception:
        return {}


async def _get_or_init_widget_settings(db, year: int) -> dict:
    await _ensure_widget_settings_table(db)
    settings = await _get_widget_settings(db, year)
    if not settings:
        for widget_name, defaults in WIDGET_DEFAULTS.items():
            await db.execute(
                text("""
                    INSERT INTO fpna_landing.forecast_widget_settings (year, widget_name, position, chart_type, enableordisable)
                    VALUES (:year, :widget_name, :position, :chart_type, :enableordisable)
                    ON CONFLICT (year, widget_name) DO NOTHING
                """),
                {
                    "year": year,
                    "widget_name": widget_name,
                    "position": defaults["position"],
                    "chart_type": defaults["chart_type"],
                    "enableordisable": defaults["enableordisable"],
                }
            )
        await db.commit()
        settings = await _get_widget_settings(db, year)
    return settings


async def _reorder_widget_positions(db, year: int, updated_widget_name: str, new_position: int, old_position: int):
    total_widgets = len(WIDGET_DEFAULTS)
    new_position = max(1, min(new_position, total_widgets))

    if new_position == old_position:
        return

    if new_position > old_position:
        await db.execute(
            text("""
                UPDATE fpna_landing.forecast_widget_settings
                SET position = position - 1, updated_at = NOW()
                WHERE year = :year
                  AND widget_name != :widget_name
                  AND position > :old_position
                  AND position <= :new_position
            """),
            {
                "year": year,
                "widget_name": updated_widget_name,
                "old_position": old_position,
                "new_position": new_position,
            }
        )
    else:
        await db.execute(
            text("""
                UPDATE fpna_landing.forecast_widget_settings
                SET position = position + 1, updated_at = NOW()
                WHERE year = :year
                  AND widget_name != :widget_name
                  AND position >= :new_position
                  AND position < :old_position
            """),
            {
                "year": year,
                "widget_name": updated_widget_name,
                "old_position": old_position,
                "new_position": new_position,
            }
        )


async def _dashboard_get_revenue_metrics(year: int, quarter: str, current_user) -> dict:
    async with AsyncSessionLocal() as db:
        try:
            await _ensure_workflow_tables(db)
            company_id = get_company_id(current_user)

            m1, m2, m3 = get_quarter_months(year, quarter)
            quarter_start = pd.to_datetime(m1 + "-01")
            history_start = (quarter_start - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")
            prev_month = (quarter_start - pd.DateOffset(months=1)).strftime("%Y-%m")

            customer_history_sql = text("""
                SELECT c.customer_name, gl.period, SUM(gl.amount_credit - gl.amount_debit) AS revenue
                FROM fpna_landing.finance_general_ledger_transactions gl
                JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
                JOIN fpna_landing.finance_customers c
                    ON REGEXP_REPLACE(gl.customer_id::TEXT, '\\.0$', '') = REGEXP_REPLACE(c.customer_id::TEXT, '\\.0$', '')
                WHERE coa.account_type = 'Revenue'
                  AND gl.period BETWEEN :start_period AND :end_period
                  AND gl.customer_id IS NOT NULL
                GROUP BY c.customer_name, gl.period
                ORDER BY c.customer_name, gl.period;
            """)

            cust_hist_result = await db.execute(customer_history_sql, {"start_period": history_start, "end_period": prev_month})
            prev_cust_result = await db.execute(
                text("""
                    SELECT c.customer_name, SUM(gl.amount_credit - gl.amount_debit) AS revenue
                    FROM fpna_landing.finance_general_ledger_transactions gl
                    JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
                    JOIN fpna_landing.finance_customers c
                        ON REGEXP_REPLACE(gl.customer_id::TEXT, '\\.0$', '') = REGEXP_REPLACE(c.customer_id::TEXT, '\\.0$', '')
                    WHERE coa.account_type = 'Revenue' AND gl.period = :prev_month AND gl.customer_id IS NOT NULL
                    GROUP BY c.customer_name;
                """),
                {"prev_month": prev_month}
            )

            cust_hist_df = pd.DataFrame(cust_hist_result.mappings().all())

            if cust_hist_df.empty:
                return {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}

            cust_hist_df = cust_hist_df.rename(columns={"revenue": "y"})
            all_customers = cust_hist_df["customer_name"].unique().tolist()
            quarter_start_ts = pd.to_datetime(m1 + "-01")

            prev_cust_df = pd.DataFrame(prev_cust_result.mappings().all())
            prev_cust_map = dict(zip(prev_cust_df["customer_name"], prev_cust_df["revenue"])) if not prev_cust_df.empty else {}

            customer_forecasts, saved_adjustments = await asyncio.gather(
                _forecast_all_customers(cust_hist_df, all_customers, quarter_start_ts, m1, m2, m3),
                _get_adjustments(db, year, quarter, company_id),
            )

            adj_map = {s["customer_name"]: s for s in saved_adjustments}

            total_prev = sum(float(v) for v in prev_cust_map.values())
            total_adj_m1 = 0.0
            total_adj_m2 = 0.0
            total_adj_m3 = 0.0

            for cname in all_customers:
                fc = customer_forecasts.get(cname, (0.0, 0.0, 0.0))
                adj = adj_map.get(cname)

                ai_m1 = float(fc[0]) if fc else 0.0
                ai_m2 = float(fc[1]) if fc else 0.0
                ai_m3 = float(fc[2]) if fc else 0.0

                c_adj_m1 = float(adj["month1_adjusted"]) if adj and adj.get("month1_adjusted") is not None else None
                c_adj_m2 = float(adj["month2_adjusted"]) if adj and adj.get("month2_adjusted") is not None else None
                c_adj_m3 = float(adj["month3_adjusted"]) if adj and adj.get("month3_adjusted") is not None else None

                total_adj_m1 += c_adj_m1 if c_adj_m1 is not None else ai_m1
                total_adj_m2 += c_adj_m2 if c_adj_m2 is not None else ai_m2
                total_adj_m3 += c_adj_m3 if c_adj_m3 is not None else ai_m3

            return {
                "prev_month": round(total_prev, 2),
                "month1_ai_suggested": round(total_adj_m1, 2),
                "month2_ai_suggested": round(total_adj_m2, 2),
                "month3_ai_suggested": round(total_adj_m3, 2),
            }
        except Exception:
            return {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}


async def _dashboard_get_expense_metrics(year: int, quarter: str, current_user) -> dict:
    async with AsyncSessionLocal() as db:
        try:
            await _ensure_expense_tables(db)
            company_id = get_company_id(current_user)

            periods = get_quarter_periods(year, quarter)
            prev_period = get_previous_month(year, quarter)
            m1, m2, m3 = periods
            history_start = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")

            sql = text("""
                SELECT
                    coa.account_name AS expense_item,
                    COALESCE(d.department_name, '') AS department,
                    gl.period,
                    SUM(gl.amount_debit - gl.amount_credit) AS y
                FROM fpna_landing.finance_general_ledger_transactions gl
                JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
                LEFT JOIN fpna_landing.hr_departments_lookup d ON gl.department_id = d.department_id
                WHERE coa.account_type = 'Expense'
                  AND gl.period >= :start_period AND gl.period < :quarter_start
                GROUP BY coa.account_name, d.department_name, gl.period
                ORDER BY gl.period;
            """)

            prev_sql = text("""
                SELECT SUM(gl.amount_debit - gl.amount_credit) AS amount
                FROM fpna_landing.finance_general_ledger_transactions gl
                JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
                WHERE coa.account_type = 'Expense' AND gl.period = :prev_period
            """)

            gl_result = await db.execute(sql, {"start_period": history_start, "quarter_start": m1})
            prev_result = await db.execute(prev_sql, {"prev_period": prev_period})
            saved_adjustments = await _get_expense_adjustments(db, year, quarter, company_id, None, "ALL")

            df = pd.DataFrame(gl_result.mappings().all())

            prev_row = prev_result.first()
            total_prev = float(prev_row[0]) if prev_row and prev_row[0] else 0.0

            if df.empty:
                return {"prev_month": round(total_prev, 2), "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}

            df["ds"] = pd.to_datetime(df["period"] + "-01")
            quarter_start_str = str(pd.to_datetime(m1 + "-01").date())

            forecast_tasks = []
            forecast_keys = []
            for (item, dept_name), g in df.groupby(["expense_item", "department"]):
                g = g.sort_values("ds")
                item_df = g[["period", "y"]].copy()
                item_df["ds"] = pd.to_datetime(item_df["period"] + "-01")
                forecast_tasks.append(get_expense_forecast(item_df, quarter_start_str))
                forecast_keys.append((item, dept_name))

            forecast_results = await asyncio.gather(*forecast_tasks)

            adj_map = {(s["expense_item"], s["department"] or ""): s for s in saved_adjustments}

            total_m1 = 0.0
            total_m2 = 0.0
            total_m3 = 0.0

            for key, vals in zip(forecast_keys, forecast_results):
                item, dept_name = key
                ai_m1 = float(vals[0]) if vals and len(vals) > 0 else 0.0
                ai_m2 = float(vals[1]) if vals and len(vals) > 1 else 0.0
                ai_m3 = float(vals[2]) if vals and len(vals) > 2 else 0.0

                adj = adj_map.get((item, dept_name or ""))
                adj_m1 = float(adj["month1_adjusted"]) if adj and adj.get("month1_adjusted") is not None else None
                adj_m2 = float(adj["month2_adjusted"]) if adj and adj.get("month2_adjusted") is not None else None
                adj_m3 = float(adj["month3_adjusted"]) if adj and adj.get("month3_adjusted") is not None else None

                total_m1 += adj_m1 if adj_m1 is not None else ai_m1
                total_m2 += adj_m2 if adj_m2 is not None else ai_m2
                total_m3 += adj_m3 if adj_m3 is not None else ai_m3

            return {
                "prev_month": round(total_prev, 2),
                "month1_ai_suggested": round(total_m1, 2),
                "month2_ai_suggested": round(total_m2, 2),
                "month3_ai_suggested": round(total_m3, 2),
            }
        except Exception:
            return {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}


async def _dashboard_get_cashflow_metrics(year: int, quarter: str, current_user) -> dict:
    async with AsyncSessionLocal() as db:
        try:
            await _ensure_cashflow_workflow_tables(db)
            company_id = get_company_id(current_user)

            m1, m2, m3 = get_quarter_months(year, quarter)
            quarter_start = pd.to_datetime(m1 + "-01")
            history_start_str = (quarter_start - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")
            prev_month = (quarter_start - pd.DateOffset(months=1)).strftime("%Y-%m")

            hist_result = await db.execute(HISTORY_SQL, {"history_start": history_start_str, "quarter_start": m1})
            prev_result = await db.execute(PREV_MONTH_SQL, {"prev_month": prev_month})
            saved_adjustments = await _get_cashflow_adjustments(db, year, quarter, company_id)

            hist_df = pd.DataFrame(hist_result.mappings().all())
            prev_row = prev_result.mappings().first()

            prev_values_map = {
                "operatingCF": round(float(prev_row["operating_cf"] or 0), 2) if prev_row else 0.0,
                "investingCF": round(float(prev_row["investing_cf"] or 0), 2) if prev_row else 0.0,
                "financingCF": round(float(prev_row["financing_cf"] or 0), 2) if prev_row else 0.0,
                "netCashChange": round(float(prev_row["net_cash_change"] or 0), 2) if prev_row else 0.0,
            }

            component_forecasts = await _forecast_all_components(hist_df, m1)
            adj_map = {s["component"]: s for s in saved_adjustments}

            total_prev = sum(prev_values_map.values())
            total_m1 = 0.0
            total_m2 = 0.0
            total_m3 = 0.0

            for comp in CASHFLOW_COMPONENTS:
                fc = component_forecasts.get(comp, (0.0, 0.0, 0.0))
                ai_m1 = float(fc[0]) if fc else 0.0
                ai_m2 = float(fc[1]) if fc else 0.0
                ai_m3 = float(fc[2]) if fc else 0.0

                adj = adj_map.get(comp)
                adj_m1 = float(adj["month1_adjusted"]) if adj and adj.get("month1_adjusted") is not None else None
                adj_m2 = float(adj["month2_adjusted"]) if adj and adj.get("month2_adjusted") is not None else None
                adj_m3 = float(adj["month3_adjusted"]) if adj and adj.get("month3_adjusted") is not None else None

                total_m1 += adj_m1 if adj_m1 is not None else ai_m1
                total_m2 += adj_m2 if adj_m2 is not None else ai_m2
                total_m3 += adj_m3 if adj_m3 is not None else ai_m3

            return {
                "prev_month": round(total_prev, 2),
                "month1_ai_suggested": round(total_m1, 2),
                "month2_ai_suggested": round(total_m2, 2),
                "month3_ai_suggested": round(total_m3, 2),
            }
        except Exception:
            return {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}


async def _dashboard_get_headcount_payroll_metrics(
    year: int, quarter: str, current_user, currency: str = "USD", currency_filter: CurrencyFilter = CurrencyFilter.actual_value
):
    empty = {
        "prev_month": 0.0,
        "month1_ai_suggested": 0.0,
        "month2_ai_suggested": 0.0,
        "month3_ai_suggested": 0.0,
    }

    async with AsyncSessionLocal() as db:
        try:
            await _ensure_payroll_workflow_tables(db)
            company_id = get_company_id(current_user)

            quarter_enum = QuarterEnum[quarter]
            start_date, end_date = quarter_date_range(year, quarter_enum)
            prev_month_first = get_previous_month_first_day(start_date)
            prev_month_last = get_previous_month_last_day(prev_month_first)
            months = pd.date_range(start=start_date, end=end_date, freq="MS")

            hc_result = await db.execute(text("""
                SELECT COUNT(DISTINCT e.employee_id) AS projected_headcount
                FROM fpna_landing.hr_employees e
                WHERE e.hire_date <= :end_date
                  AND (e.termination_date IS NULL OR e.termination_date > :end_date)
            """), {"end_date": end_date})
            prev_hc_result = await db.execute(text("""
                SELECT COUNT(DISTINCT e.employee_id) AS headcount
                FROM fpna_landing.hr_employees e
                WHERE e.hire_date <= :prev_end
                  AND (e.termination_date IS NULL OR e.termination_date > :prev_end)
            """), {"prev_end": prev_month_last})
            dept_saved = await _get_department_headcount_adjustments(db, year, quarter, company_id)
            payroll_rows_result = await db.execute(text(RAW_SQL))

            hc_row = hc_result.first()
            proj_count = int(hc_row[0]) if hc_row and hc_row[0] else 0

            prev_hc_row = prev_hc_result.first()
            prev_count = int(prev_hc_row[0]) if prev_hc_row and prev_hc_row[0] else 0

            dept_adj_map = {d["department_name"]: d for d in dept_saved}

            all_dept_adj_m1, all_dept_adj_m2, all_dept_adj_m3 = [], [], []
            for dept_name, adj in dept_adj_map.items():
                raw_m1 = adj.get("month1_adjusted_count")
                raw_m2 = adj.get("month2_adjusted_count")
                raw_m3 = adj.get("month3_adjusted_count")
                all_dept_adj_m1.append(float(raw_m1) if raw_m1 is not None else float(proj_count))
                all_dept_adj_m2.append(float(raw_m2) if raw_m2 is not None else float(proj_count))
                all_dept_adj_m3.append(float(raw_m3) if raw_m3 is not None else float(proj_count))

            hc_m1 = float(max(all_dept_adj_m1)) if all_dept_adj_m1 else float(proj_count)
            hc_m2 = float(max(all_dept_adj_m2)) if all_dept_adj_m2 else float(proj_count)
            hc_m3 = float(max(all_dept_adj_m3)) if all_dept_adj_m3 else float(proj_count)

            headcount = {
                "prev_month": float(prev_count),
                "month1_ai_suggested": hc_m1,
                "month2_ai_suggested": hc_m2,
                "month3_ai_suggested": hc_m3,
            }

            df = pd.DataFrame(payroll_rows_result.mappings().all())

            if df.empty:
                return headcount, empty

            df["effective_date"] = pd.to_datetime(df["effective_date"])
            df["hire_date"] = pd.to_datetime(df["hire_date"])
            df["base_salary"] = df["base_salary"].fillna(0).astype(float)
            df["bonus_target"] = df["bonus_target"].fillna(0).astype(float)

            employees_df = df[["employee_id", "employee", "department", "role", "hire_date"]].drop_duplicates()
            df2 = df.drop(columns=["employee", "department", "role", "hire_date"])
            df2["effective_month"] = df2["effective_date"].dt.to_period("M").dt.to_timestamp()

            all_months_range = pd.date_range(start=prev_month_first, end=end_date, freq="MS")
            grid = (
                employees_df.assign(key=1)
                .merge(pd.DataFrame({"month": all_months_range, "key": 1}), on="key")
                .drop("key", axis=1)
            )
            merged = grid.merge(df2, on="employee_id", how="left")
            merged = merged[merged["effective_month"].isna() | (merged["effective_month"] <= merged["month"])]
            merged = merged.sort_values("effective_month").groupby(["employee_id", "month"]).tail(1)
            merged["salary"] = merged["base_salary"]
            merged["bonus"] = merged["bonus_target"]
            merged["benefits"] = merged["salary"] * 0.20
            merged.loc[merged["month"] < merged["hire_date"], ["salary", "bonus", "benefits"]] = 0

            out = []
            for _, emp_df in merged.groupby("employee_id"):
                emp_df = emp_df.sort_values("month").copy()
                n = len(emp_df)
                emp_df["salary"] = forecast_series_advanced(emp_df["salary"], n)
                emp_df["bonus"] = forecast_series_advanced(emp_df["bonus"], n)
                emp_df["benefits"] = emp_df["salary"] * 0.20
                out.append(emp_df)

            final_df = pd.concat(out, ignore_index=True)
            month_strs = [m.date().isoformat() for m in months]
            prev_month_str = prev_month_first.isoformat()

            saved_payroll_adjs = await _get_payroll_adjustments(db, year, quarter, company_id)
            payroll_adj_map = {int(s["employee_id"]): s for s in saved_payroll_adjs}
            month_str_to_index = {m.date().isoformat(): i + 1 for i, m in enumerate(months)}

            monthly_salary_totals = {ms: 0.0 for ms in month_strs}
            prev_salary_total = 0.0

            for emp_id, emp_df_group in final_df.groupby("employee_id"):
                emp_df_group = emp_df_group.sort_values("month")
                rows_list = emp_df_group.to_dict("records")
                if not rows_list:
                    continue

                adj = payroll_adj_map.get(int(emp_id), {})

                for row in rows_list:
                    month_dt = pd.Timestamp(row["month"])
                    ms = month_dt.date().isoformat()
                    salary_ai = round(float(row["salary"]), 2)

                    salary_final = salary_ai
                    if ms in month_str_to_index:
                        idx = month_str_to_index[ms]
                        adj_salary = adj.get(f"month{idx}_adjusted_salary")
                        if adj_salary is not None:
                            salary_final = float(adj_salary)

                    if ms == prev_month_str:
                        prev_salary_total += salary_final
                    elif ms in monthly_salary_totals:
                        monthly_salary_totals[ms] += salary_final

            payroll = {
                "prev_month": round(prev_salary_total, 2),
                "month1_ai_suggested": round(monthly_salary_totals.get(month_strs[0], 0.0), 2) if len(month_strs) > 0 else 0.0,
                "month2_ai_suggested": round(monthly_salary_totals.get(month_strs[1], 0.0), 2) if len(month_strs) > 1 else 0.0,
                "month3_ai_suggested": round(monthly_salary_totals.get(month_strs[2], 0.0), 2) if len(month_strs) > 2 else 0.0,
            }

            return headcount, payroll

        except Exception:
            return empty, empty


async def _dashboard_get_capex_metrics(year: int, quarter: str, current_user) -> dict:
    async with AsyncSessionLocal() as db:
        try:
            await _ensure_capex_workflow_tables(db)
            company_id = get_company_id(current_user)

            m1, m2, m3 = get_quarter_months(year, quarter)
            quarter_start = pd.to_datetime(m1 + "-01")
            history_start = (quarter_start - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")
            prev_month = (quarter_start - pd.DateOffset(months=1)).strftime("%Y-%m")

            distinct_sql = text(
                "SELECT DISTINCT fa.category FROM fpna_landing.finance_fixed_asset fa WHERE fa.category IS NOT NULL ORDER BY fa.category"
            )
            all_categories = [row[0] for row in (await db.execute(distinct_sql)).fetchall()]

            if not all_categories:
                return {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}

            history_sql = text("""
                SELECT fa.category,
                       TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') AS period,
                       SUM(fa.purchase_price) AS purchase_price
                FROM fpna_landing.finance_fixed_asset fa
                WHERE TO_DATE(fa.date_acquired, 'DD-MM-YYYY') >= TO_DATE(:start_period || '-01', 'YYYY-MM-DD')
                  AND TO_DATE(fa.date_acquired, 'DD-MM-YYYY') < TO_DATE(:quarter_start || '-01', 'YYYY-MM-DD')
                  AND fa.category IS NOT NULL
                GROUP BY fa.category, TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM')
                ORDER BY fa.category, period;
            """)

            prev_sql = text("""
                SELECT SUM(fa.purchase_price) AS total_purchase
                FROM fpna_landing.finance_fixed_asset fa
                WHERE TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') = :prev_month
                  AND fa.category IS NOT NULL
            """)

            item_hist_result = await db.execute(history_sql, {"start_period": history_start, "quarter_start": m1})
            prev_result = await db.execute(prev_sql, {"prev_month": prev_month})
            saved_adjustments = await _get_capex_adjustments(db, year, quarter, company_id)

            item_hist_df = pd.DataFrame(item_hist_result.mappings().all())
            prev_row = prev_result.first()
            total_prev = float(prev_row[0]) if prev_row and prev_row[0] else 0.0

            if item_hist_df.empty:
                item_hist_df = pd.DataFrame([{"category": all_categories[0], "period": prev_month, "purchase_price": 0.0}])

            adj_map = {s["capex_item"]: s for s in saved_adjustments}

            total_m1 = 0.0
            total_m2 = 0.0
            total_m3 = 0.0

            for capex_item in all_categories:
                v1, v2, v3 = _forecast_capex_item(capex_item, item_hist_df, quarter_start, m1, m2, m3)
                ai_m1 = float(v1)
                ai_m2 = float(v2)
                ai_m3 = float(v3)

                adj = adj_map.get(capex_item)
                adj_m1 = float(adj["month1_adjusted"]) if adj and adj.get("month1_adjusted") is not None else None
                adj_m2 = float(adj["month2_adjusted"]) if adj and adj.get("month2_adjusted") is not None else None
                adj_m3 = float(adj["month3_adjusted"]) if adj and adj.get("month3_adjusted") is not None else None

                total_m1 += adj_m1 if adj_m1 is not None else ai_m1
                total_m2 += adj_m2 if adj_m2 is not None else ai_m2
                total_m3 += adj_m3 if adj_m3 is not None else ai_m3

            return {
                "prev_month": round(total_prev, 2),
                "month1_ai_suggested": round(total_m1, 2),
                "month2_ai_suggested": round(total_m2, 2),
                "month3_ai_suggested": round(total_m3, 2),
            }
        except Exception:
            return {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}


async def _get_metrics_for_widgets(
    year: int, quarter: str, current_user: Dict, needed_widgets: set,
    currency: str = "USD", currency_filter: CurrencyFilter = CurrencyFilter.actual_value
) -> Dict[str, Dict]:
    needs_headcount_or_payroll = bool(needed_widgets & {"headcount_forecast", "payroll_forecast"})
    needs_revenue = "revenue_forecasting" in needed_widgets
    needs_expense = "expense_forecasting" in needed_widgets
    needs_capex = "capex_forecast" in needed_widgets
    needs_cashflow = "cash_flow_projections" in needed_widgets

    tasks = []
    task_labels = []

    if needs_revenue:
        tasks.append(_dashboard_get_revenue_metrics(year, quarter, current_user))
        task_labels.append("revenue")
    if needs_expense:
        tasks.append(_dashboard_get_expense_metrics(year, quarter, current_user))
        task_labels.append("expense")
    if needs_cashflow:
        tasks.append(_dashboard_get_cashflow_metrics(year, quarter, current_user))
        task_labels.append("cashflow")
    if needs_headcount_or_payroll:
        tasks.append(_dashboard_get_headcount_payroll_metrics(year, quarter, current_user, currency, currency_filter))
        task_labels.append("headcount_payroll")
    if needs_capex:
        tasks.append(_dashboard_get_capex_metrics(year, quarter, current_user))
        task_labels.append("capex")

    results = await asyncio.gather(*tasks)
    result_map = dict(zip(task_labels, results))

    headcount_data = {}
    payroll_data = {}
    if "headcount_payroll" in result_map:
        hc, pay = result_map["headcount_payroll"]
        headcount_data = hc
        payroll_data = pay

    return {
        "REVENUE": result_map.get("revenue", {}),
        "Expense": result_map.get("expense", {}),
        "Cash Flow Projections": result_map.get("cashflow", {}),
        "Headcount": headcount_data,
        "Payroll": payroll_data,
        "CAPEX Forecast": result_map.get("capex", {}),
    }


async def generate_forecast_dashboard_metrics(year: int, quarter: str, current_user: Dict) -> Dict[str, Dict[str, float]]:
    revenue_result, expense_result, cashflow_result, hc_payroll_result, capex_result = await asyncio.gather(
        _dashboard_get_revenue_metrics(year, quarter, current_user),
        _dashboard_get_expense_metrics(year, quarter, current_user),
        _dashboard_get_cashflow_metrics(year, quarter, current_user),
        _dashboard_get_headcount_payroll_metrics(year, quarter, current_user),
        _dashboard_get_capex_metrics(year, quarter, current_user),
    )

    headcount_result, payroll_result = hc_payroll_result

    return {
        "REVENUE": revenue_result,
        "Expense": expense_result,
        "Cash Flow Projections": cashflow_result,
        "Headcount": headcount_result,
        "Payroll": payroll_result,
        "CAPEX Forecast": capex_result,
        "Scenario Modeling": {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0},
        "Debt & Interest": {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0},
    }


async def _build_forecast_dashboard_full_year(
    year: int, current_user: Dict, currency: str, currency_filter: str
) -> Dict[str, list]:
    month_keys = ["month1_ai_suggested", "month2_ai_suggested", "month3_ai_suggested"]

    q1_result, q2_result, q3_result, q4_result, widget_settings = await asyncio.gather(
        generate_forecast_dashboard_metrics(year, "Q1", current_user),
        generate_forecast_dashboard_metrics(year, "Q2", current_user),
        generate_forecast_dashboard_metrics(year, "Q3", current_user),
        generate_forecast_dashboard_metrics(year, "Q4", current_user),
        _init_widget_settings_standalone(year),
    )

    all_quarters = [("Q1", q1_result), ("Q2", q2_result), ("Q3", q3_result), ("Q4", q4_result)]

    revenue_forecasting = []
    expense_forecasting = []
    capex_forecast_list = []
    cash_flow_projections = []
    headcount_forecast = []
    payroll_forecast = []

    total_revenue = 0.0
    total_expenses = 0.0
    total_cashflow = 0.0
    total_payroll = 0.0
    total_capex = 0.0
    last_headcount = 0

    for quarter, ai_metrics in all_quarters:
        months = QUARTER_MONTH_MAP[quarter]

        revenue_data = ai_metrics.get("REVENUE", {})
        expense_data = ai_metrics.get("Expense", {})
        cashflow_data = ai_metrics.get("Cash Flow Projections", {})
        headcount_data = ai_metrics.get("Headcount", {})
        payroll_data = ai_metrics.get("Payroll", {})
        capex_data = ai_metrics.get("CAPEX Forecast", {})

        for i, month_num in enumerate(months):
            period = f"{year}-{str(month_num).zfill(2)}"
            rev_val = round(revenue_data.get(month_keys[i], 0.0), 2)
            exp_val = round(expense_data.get(month_keys[i], 0.0), 2)
            cf_val = round(cashflow_data.get(month_keys[i], 0.0), 2)
            hc_val = int(headcount_data.get(month_keys[i], 0))
            pay_val = round(payroll_data.get(month_keys[i], 0.0), 2)
            capex_val = round(capex_data.get(month_keys[i], 0.0), 2)

            revenue_forecasting.append({"period": period, "revenue": rev_val})
            expense_forecasting.append({"period": period, "expenses": exp_val})
            capex_forecast_list.append({"period": period, "capex": capex_val})
            cash_flow_projections.append({"period": period, "cash_flow": cf_val})
            headcount_forecast.append({"period": period, "headcount": hc_val})
            payroll_forecast.append({"period": period, "payroll": pay_val})

            total_revenue += rev_val
            total_expenses += exp_val
            total_cashflow += cf_val
            total_payroll += pay_val
            total_capex += capex_val
            last_headcount = hc_val

    def get_widget_meta(name):
        defaults = WIDGET_DEFAULTS.get(name, {"position": 0, "chart_type": "bar_chart", "enableordisable": True})
        s = widget_settings.get(name, defaults)
        return {
            "position": s.get("position", defaults["position"]),
            "chart_type": s.get("chart_type", defaults["chart_type"]),
            "enableordisable": s.get("enableordisable", defaults["enableordisable"]),
        }

    revenue_forecasting.append(get_widget_meta("revenue_forecasting"))
    expense_forecasting.append(get_widget_meta("expense_forecasting"))
    capex_forecast_list.append(get_widget_meta("capex_forecast"))
    cash_flow_projections.append(get_widget_meta("cash_flow_projections"))
    headcount_forecast.append(get_widget_meta("headcount_forecast"))
    payroll_forecast.append(get_widget_meta("payroll_forecast"))

    raw_response = {
        "revenue_forecasting": revenue_forecasting,
        "expense_forecasting": expense_forecasting,
        "capex_forecast": capex_forecast_list,
        "cash_flow_projections": cash_flow_projections,
        "headcount_forecast": headcount_forecast,
        "payroll_forecast": payroll_forecast,
        "summary_metrics": {
            "forecast_revenue": round(total_revenue, 2),
            "forecast_expenses": round(total_expenses, 2),
            "forecast_cashflow": round(total_cashflow, 2),
            "forecast_headcount": last_headcount,
            "forecast_payroll": round(total_payroll, 2),
            "forecast_capex": round(total_capex, 2),
        },
    }

    return apply_smart_currency(raw_response, currency, currency_filter, MONEY_FIELDS)


async def _init_widget_settings_standalone(year: int) -> dict:
    async with AsyncSessionLocal() as db:
        return await _get_or_init_widget_settings(db, year)


class UpdateWidgetRequest(BaseModel):
    year: int
    name: str
    chart_Type: str
    enableordisable: bool
    position: int


@forecast_router.get("/forecast_dashboard/{year}/{currency}")
async def forecast_dashboard(
    year: int = Path(...),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    forecast_year: Optional[int] = Query(default=None, description="Primary forecast year context. Overrides year for period resolution when provided."),
    forecast_type: Optional[str] = Query(default="12months", description="Forecast type: quarter, 12months, rolling. Dashboard defaults to 12months (full year)."),
    quarter: QuarterEnum = Query(QuarterEnum.Q1, description="Quarter to focus on when forecast_type=quarter."),
    rolling_start_month: Optional[str] = Query(default=None, description="Start month for rolling forecast (e.g. Jan, Feb, Mar). Required when forecast_type=rolling."),
    view_type: Optional[str] = Query(default="monthly_breakdown", description="View type: quarter_summary or monthly_breakdown"),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    effective_year = forecast_year if forecast_year is not None else year
    effective_forecast_type = forecast_type or "12months"
    effective_view_type = view_type or "monthly_breakdown"

    if effective_forecast_type not in ("quarter", "12months", "rolling"):
        raise HTTPException(status_code=422, detail="forecast_type must be one of: quarter, 12months, rolling")
    if effective_view_type not in ("quarter_summary", "monthly_breakdown"):
        raise HTTPException(status_code=422, detail="view_type must be one of: quarter_summary, monthly_breakdown")
    if effective_forecast_type == "rolling" and not rolling_start_month:
        raise HTTPException(status_code=422, detail="rolling_start_month is required when forecast_type=rolling.")

    _MONTH_ORDER = {"jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
                    "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12}
    resolve_year = effective_year
    if effective_forecast_type == "rolling" and rolling_start_month:
        if _MONTH_ORDER.get(rolling_start_month.lower()[:3], 1) >= 11:
            resolve_year = effective_year - 1

    month_periods, selected_period, _ = _resolve_forecast_periods(
        resolve_year, effective_forecast_type, quarter.value, rolling_start_month
    )

    full_year_data = await _build_forecast_dashboard_full_year(
        effective_year, current_user, currency=currency, currency_filter=currency_filter.value
    )

    if effective_forecast_type != "12months":
        full_year_data = _slice_dashboard_lists_to_periods(full_year_data, month_periods)

    if effective_view_type == "quarter_summary":
        list_keys = ["revenue_forecasting", "expense_forecasting", "capex_forecast",
                     "cash_flow_projections", "headcount_forecast", "payroll_forecast"]
        field_map = {
            "revenue_forecasting": "revenue", "expense_forecasting": "expenses",
            "capex_forecast": "capex", "cash_flow_projections": "cash_flow",
            "headcount_forecast": "headcount", "payroll_forecast": "payroll",
        }
        qmap = {}
        for mp in month_periods:
            yr, mo = int(mp.split("-")[0]), int(mp.split("-")[1])
            q_label = "Q1" if mo <= 3 else "Q2" if mo <= 6 else "Q3" if mo <= 9 else "Q4"
            qmap[mp] = f"{yr}-{q_label}"

        for key in list_keys:
            rows = full_year_data.get(key, [])
            field = field_map[key]
            meta = [r for r in rows if not (isinstance(r, dict) and "period" in r)]
            data_rows = [r for r in rows if isinstance(r, dict) and "period" in r]
            agg: Dict[str, Any] = {}
            for r in data_rows:
                qlbl = qmap.get(r.get("period", ""), r.get("period", ""))
                if qlbl not in agg:
                    agg[qlbl] = {"period": qlbl, field: 0}
                if key == "headcount_forecast":
                    agg[qlbl][field] = r.get(field, 0)
                else:
                    agg[qlbl][field] = round(agg[qlbl][field] + float(r.get(field, 0.0)), 2)
            full_year_data[key] = list(agg.values()) + meta

    response = {
        "forecast_year": effective_year,
        "forecast_type": effective_forecast_type,
        "view_type": effective_view_type,
        "selected_period": selected_period,
        "selected_months": month_periods,
        "summary_metrics": full_year_data["summary_metrics"],
        "revenue_forecasting": full_year_data["revenue_forecasting"],
        "expense_forecasting": full_year_data["expense_forecasting"],
        "capex_forecast": full_year_data["capex_forecast"],
        "cash_flow_projections": full_year_data["cash_flow_projections"],
        "headcount_forecast": full_year_data["headcount_forecast"],
        "payroll_forecast": full_year_data["payroll_forecast"],
    }
    return response


@forecast_router.patch("/forecast_dashboard/widgets/update")
async def update_widget(
    payload: UpdateWidgetRequest,
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_widget_settings_table(db)

    valid_widgets = list(WIDGET_DEFAULTS.keys())
    if payload.name not in valid_widgets:
        raise HTTPException(status_code=400, detail=f"Invalid widget name. Must be one of: {valid_widgets}")

    valid_chart_types = [e.value for e in ChartTypeEnum]
    if payload.chart_Type not in valid_chart_types:
        raise HTTPException(status_code=400, detail=f"Invalid chart_Type. Must be one of: {valid_chart_types}")

    total_widgets = len(WIDGET_DEFAULTS)
    new_position = max(1, min(payload.position, total_widgets))

    await _get_or_init_widget_settings(db, payload.year)

    current_row = await db.execute(
        text("SELECT position FROM fpna_landing.forecast_widget_settings WHERE year = :year AND widget_name = :widget_name"),
        {"year": payload.year, "widget_name": payload.name}
    )
    current_row = current_row.mappings().first()
    old_position = current_row["position"] if current_row else new_position

    await _reorder_widget_positions(db, payload.year, payload.name, new_position, old_position)

    await db.execute(
        text("""
            INSERT INTO fpna_landing.forecast_widget_settings (year, widget_name, position, chart_type, enableordisable, updated_at)
            VALUES (:year, :widget_name, :position, :chart_type, :enableordisable, NOW())
            ON CONFLICT (year, widget_name)
            DO UPDATE SET
                position = EXCLUDED.position,
                chart_type = EXCLUDED.chart_type,
                enableordisable = EXCLUDED.enableordisable,
                updated_at = NOW()
        """),
        {
            "year": payload.year,
            "widget_name": payload.name,
            "position": new_position,
            "chart_type": payload.chart_Type,
            "enableordisable": payload.enableordisable,
        }
    )
    await db.commit()

    all_settings = await _get_widget_settings(db, payload.year)

    return {
        "year": payload.year,
        "updated_widget": {
            "name": payload.name,
            "chart_Type": payload.chart_Type,
            "enableordisable": payload.enableordisable,
            "position": new_position,
        },
        "all_widget_positions": {
            name: s["position"] for name, s in sorted(all_settings.items(), key=lambda x: x[1]["position"])
        },
    }


@forecast_router.get("/forecast_dashboard/widgets/{year}/{currency}")
async def get_widgets(
    year: int = Path(...),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    widget_settings = await _get_or_init_widget_settings(db, year)

    disabled_widgets = {name for name, s in widget_settings.items() if s.get("enableordisable") is False}

    if not disabled_widgets:
        return {"message": "No disabled widgets found.", "widgets": {}}

    needed_metric_funcs = set()
    for w in disabled_widgets:
        func_label = WIDGET_TO_METRIC_FUNC.get(w)
        if func_label:
            needed_metric_funcs.add(func_label)

    needed_widgets = disabled_widgets & set(WIDGET_KEY_MAP.keys())

    quarter_tasks = [
        _get_metrics_for_widgets(year, quarter, current_user, needed_widgets, currency, currency_filter)
        for quarter in ["Q1", "Q2", "Q3", "Q4"]
    ]
    quarter_results = await asyncio.gather(*quarter_tasks)
    all_quarters = list(zip(["Q1", "Q2", "Q3", "Q4"], quarter_results))

    accumulators = {name: [] for name in disabled_widgets}

    for quarter, ai_metrics in all_quarters:
        months = QUARTER_MONTH_MAP[quarter]
        for i, month_num in enumerate(months):
            period = f"{year}-{str(month_num).zfill(2)}"
            for widget_name in disabled_widgets:
                if widget_name not in WIDGET_KEY_MAP:
                    continue
                metric_key, field_name = WIDGET_KEY_MAP[widget_name]
                metric_data = ai_metrics.get(metric_key, {})
                raw_val = metric_data.get(MONTH_KEYS[i], 0.0)
                if field_name == "headcount":
                    accumulators[widget_name].append({"period": period, field_name: int(raw_val)})
                else:
                    accumulators[widget_name].append({"period": period, field_name: round(float(raw_val), 2)})

    for widget_name in disabled_widgets:
        meta = widget_settings.get(widget_name, WIDGET_DEFAULTS.get(widget_name, {}))
        accumulators[widget_name].append({
            "position": meta.get("position", 0),
            "chart_type": meta.get("chart_type", "bar_chart"),
            "enableordisable": meta.get("enableordisable", False),
        })

    non_headcount_widgets = {name for name in disabled_widgets if name != "headcount_forecast"}
    raw_result = {name: accumulators[name] for name in disabled_widgets}

    filtered_result = apply_smart_currency(
        {k: v for k, v in raw_result.items() if k in non_headcount_widgets},
        currency,
        currency_filter.value,
        MONEY_FIELDS,
    )

    if "headcount_forecast" in disabled_widgets:
        filtered_result["headcount_forecast"] = raw_result["headcount_forecast"]

    return {"widgets": filtered_result}
##########################################################################################
###########################################################################################
###########################################################################################
#PnL FORECASTING 


class PnLPeriodFilter(str, Enum):
    monthly = "monthly"
    quarterly = "quarterly"
    ytd = "ytd"


class PnLScenarioFilter(str, Enum):
    baseline = "baseline"
    bestcase = "bestcase"
    worstcase = "worstcase"


class PnLComparisonFilter(str, Enum):
    previous_month = "previous_month"
    previous_quarter = "previous_quarter"
    previous_year = "previous_year"


def _resolve_forecast_value(aisuggested_value: Optional[float], adjustment_value: Optional[float]) -> float:
    ai_val = float(aisuggested_value) if aisuggested_value is not None else 0.0
    if adjustment_value is not None:
        adj_val = float(adjustment_value)
        if adj_val != ai_val:
            return adj_val
    return ai_val


async def _pnl_get_revenue_metrics(year: int, quarter: str, current_user: Dict) -> Dict[str, float]:
    async with AsyncSessionLocal() as db:
        try:
            await _ensure_workflow_tables(db)
            company_id = get_company_id(current_user)

            m1, m2, m3 = get_quarter_months(year, quarter)
            quarter_start = pd.to_datetime(m1 + "-01")
            history_start = (quarter_start - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")
            prev_month = (quarter_start - pd.DateOffset(months=1)).strftime("%Y-%m")

            customer_history_sql = text("""
                SELECT c.customer_name, gl.period, SUM(gl.amount_credit - gl.amount_debit) AS revenue
                FROM fpna_landing.finance_general_ledger_transactions gl
                JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
                JOIN fpna_landing.finance_customers c
                    ON REGEXP_REPLACE(gl.customer_id::TEXT, '\\.0$', '') = REGEXP_REPLACE(c.customer_id::TEXT, '\\.0$', '')
                WHERE coa.account_type = 'Revenue'
                  AND gl.period BETWEEN :start_period AND :end_period
                  AND gl.customer_id IS NOT NULL
                GROUP BY c.customer_name, gl.period
                ORDER BY c.customer_name, gl.period;
            """)

            cust_hist_result = await db.execute(customer_history_sql, {"start_period": history_start, "end_period": prev_month})
            cust_hist_df = pd.DataFrame(cust_hist_result.mappings().all())

            if cust_hist_df.empty:
                return {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}

            prev_mask = cust_hist_df["period"] == prev_month
            prev_cust_map = (
                cust_hist_df[prev_mask].groupby("customer_name")["revenue"].sum().to_dict()
                if prev_mask.any() else {}
            )
            cust_hist_df = cust_hist_df.rename(columns={"revenue": "y"})
            all_customers = cust_hist_df["customer_name"].unique().tolist()
            quarter_start_ts = pd.to_datetime(m1 + "-01")

            customer_forecasts = await _forecast_all_customers(cust_hist_df, all_customers, quarter_start_ts, m1, m2, m3)

            saved_adjustments = await _get_adjustments(db, year, quarter, company_id)
            adj_map = {s["customer_name"]: s for s in saved_adjustments}

            total_prev = sum(float(v) for v in prev_cust_map.values())

            total_adj_m1 = 0.0
            total_adj_m2 = 0.0
            total_adj_m3 = 0.0

            for cname in all_customers:
                fc = customer_forecasts.get(cname, (0.0, 0.0, 0.0))
                adj = adj_map.get(cname)

                c_ai_m1 = float(fc[0]) if fc else 0.0
                c_ai_m2 = float(fc[1]) if fc else 0.0
                c_ai_m3 = float(fc[2]) if fc else 0.0

                c_adj_m1 = float(adj["month1_adjusted"]) if adj and adj.get("month1_adjusted") is not None else None
                c_adj_m2 = float(adj["month2_adjusted"]) if adj and adj.get("month2_adjusted") is not None else None
                c_adj_m3 = float(adj["month3_adjusted"]) if adj and adj.get("month3_adjusted") is not None else None

                total_adj_m1 += _resolve_forecast_value(c_ai_m1, c_adj_m1)
                total_adj_m2 += _resolve_forecast_value(c_ai_m2, c_adj_m2)
                total_adj_m3 += _resolve_forecast_value(c_ai_m3, c_adj_m3)

            return {
                "prev_month": round(total_prev, 2),
                "month1_ai_suggested": round(total_adj_m1, 2),
                "month2_ai_suggested": round(total_adj_m2, 2),
                "month3_ai_suggested": round(total_adj_m3, 2),
            }
        except Exception:
            return {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}


async def _pnl_get_expense_metrics(year: int, quarter: str, current_user: Dict) -> Dict[str, float]:
    async with AsyncSessionLocal() as db:
        try:
            await _ensure_expense_tables(db)
            company_id = get_company_id(current_user)

            periods = get_quarter_periods(year, quarter)
            prev_period = get_previous_month(year, quarter)
            m1, m2, m3 = periods
            history_start = (pd.to_datetime(m1 + "-01") - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")

            sql = text("""
                SELECT
                    coa.account_name AS expense_item,
                    COALESCE(d.department_name, '') AS department,
                    gl.period,
                    SUM(gl.amount_debit - gl.amount_credit) AS y
                FROM fpna_landing.finance_general_ledger_transactions gl
                JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
                LEFT JOIN fpna_landing.hr_departments_lookup d ON gl.department_id = d.department_id
                WHERE coa.account_type = 'Expense'
                  AND gl.period >= :start_period AND gl.period < :quarter_start
                GROUP BY coa.account_name, d.department_name, gl.period
                ORDER BY gl.period;
            """)

            df = pd.DataFrame((await db.execute(sql, {
                "start_period": history_start, "quarter_start": m1
            })).mappings().all())

            if df.empty:
                return {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}

            df["ds"] = pd.to_datetime(df["period"] + "-01")

            quarter_start_str = str(pd.to_datetime(m1 + "-01").date())

            forecast_tasks = []
            forecast_keys = []
            for (item, dept_name), g in df.groupby(["expense_item", "department"]):
                g = g.sort_values("ds")
                item_df = g[["period", "y"]].copy()
                item_df["ds"] = pd.to_datetime(item_df["period"] + "-01")
                forecast_tasks.append(get_expense_forecast(item_df, quarter_start_str))
                forecast_keys.append((item, dept_name))

            forecast_results = await asyncio.gather(*forecast_tasks)

            saved_adjustments = await _get_expense_adjustments(db, year, quarter, company_id, None, "ALL")
            adj_map = {(s["expense_item"], s["department"] or ""): s for s in saved_adjustments}

            total_prev = 0.0
            total_m1 = 0.0
            total_m2 = 0.0
            total_m3 = 0.0

            prev_sql = text("""
                SELECT SUM(gl.amount_debit - gl.amount_credit) AS amount
                FROM fpna_landing.finance_general_ledger_transactions gl
                JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
                WHERE coa.account_type = 'Expense' AND gl.period = :prev_period
            """)
            prev_row = (await db.execute(prev_sql, {"prev_period": prev_period})).first()
            if prev_row and prev_row[0]:
                total_prev = float(prev_row[0])

            for key, vals in zip(forecast_keys, forecast_results):
                item, dept_name = key
                ai_m1 = float(vals[0]) if vals and len(vals) > 0 else 0.0
                ai_m2 = float(vals[1]) if vals and len(vals) > 1 else 0.0
                ai_m3 = float(vals[2]) if vals and len(vals) > 2 else 0.0

                adj = adj_map.get((item, dept_name or ""))
                if adj is None:
                    for etype in ["Fixed", "Variable", ""]:
                        candidate = next(
                            (s for s in saved_adjustments
                             if s["expense_item"] == item and (s["department"] or "") == (dept_name or "")),
                            None
                        )
                        if candidate:
                            adj = candidate
                            break

                adj_m1 = float(adj["month1_adjusted"]) if adj and adj.get("month1_adjusted") is not None else None
                adj_m2 = float(adj["month2_adjusted"]) if adj and adj.get("month2_adjusted") is not None else None
                adj_m3 = float(adj["month3_adjusted"]) if adj and adj.get("month3_adjusted") is not None else None

                total_m1 += _resolve_forecast_value(ai_m1, adj_m1)
                total_m2 += _resolve_forecast_value(ai_m2, adj_m2)
                total_m3 += _resolve_forecast_value(ai_m3, adj_m3)

            return {
                "prev_month": round(total_prev, 2),
                "month1_ai_suggested": round(total_m1, 2),
                "month2_ai_suggested": round(total_m2, 2),
                "month3_ai_suggested": round(total_m3, 2),
            }
        except Exception:
            return {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}


async def _pnl_get_cashflow_metrics(year: int, quarter: str, current_user: Dict) -> Dict[str, float]:
    async with AsyncSessionLocal() as db:
        try:
            await _ensure_cashflow_workflow_tables(db)
            company_id = get_company_id(current_user)

            m1, m2, m3 = get_quarter_months(year, quarter)
            quarter_start = pd.to_datetime(m1 + "-01")
            history_start_str = (quarter_start - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")
            prev_month = (quarter_start - pd.DateOffset(months=1)).strftime("%Y-%m")

            hist_result = await db.execute(HISTORY_SQL, {"history_start": history_start_str, "quarter_start": m1})
            hist_df = pd.DataFrame(hist_result.mappings().all())

            prev_result = await db.execute(PREV_MONTH_SQL, {"prev_month": prev_month})
            prev_row = prev_result.mappings().first()

            prev_values_map = {
                "operatingCF": round(float(prev_row["operating_cf"] or 0), 2) if prev_row else 0.0,
                "investingCF": round(float(prev_row["investing_cf"] or 0), 2) if prev_row else 0.0,
                "financingCF": round(float(prev_row["financing_cf"] or 0), 2) if prev_row else 0.0,
                "netCashChange": round(float(prev_row["net_cash_change"] or 0), 2) if prev_row else 0.0,
            }

            component_forecasts = await _forecast_all_components(hist_df, m1)

            saved_adjustments = await _get_cashflow_adjustments(db, year, quarter, company_id)
            adj_map = {s["component"]: s for s in saved_adjustments}

            total_prev = sum(prev_values_map.values())
            total_m1 = 0.0
            total_m2 = 0.0
            total_m3 = 0.0

            for comp in CASHFLOW_COMPONENTS:
                fc = component_forecasts.get(comp, (0.0, 0.0, 0.0))
                ai_m1 = float(fc[0]) if fc else 0.0
                ai_m2 = float(fc[1]) if fc else 0.0
                ai_m3 = float(fc[2]) if fc else 0.0

                adj = adj_map.get(comp)
                adj_m1 = float(adj["month1_adjusted"]) if adj and adj.get("month1_adjusted") is not None else None
                adj_m2 = float(adj["month2_adjusted"]) if adj and adj.get("month2_adjusted") is not None else None
                adj_m3 = float(adj["month3_adjusted"]) if adj and adj.get("month3_adjusted") is not None else None

                total_m1 += _resolve_forecast_value(ai_m1, adj_m1)
                total_m2 += _resolve_forecast_value(ai_m2, adj_m2)
                total_m3 += _resolve_forecast_value(ai_m3, adj_m3)

            return {
                "prev_month": round(total_prev, 2),
                "month1_ai_suggested": round(total_m1, 2),
                "month2_ai_suggested": round(total_m2, 2),
                "month3_ai_suggested": round(total_m3, 2),
            }
        except Exception:
            return {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}


async def _pnl_get_headcount_payroll_metrics(year: int, quarter: str, current_user: Dict) -> tuple:
    empty = {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}

    async with AsyncSessionLocal() as db:
        try:
            await _ensure_payroll_workflow_tables(db)
            company_id = get_company_id(current_user)

            quarter_enum = QuarterEnum[quarter]
            start_date, end_date = quarter_date_range(year, quarter_enum)
            prev_month_first = get_previous_month_first_day(start_date)
            prev_month_last = get_previous_month_last_day(prev_month_first)
            months = pd.date_range(start=start_date, end=end_date, freq="MS")

            dept_headcount_sql = text("""
                SELECT COUNT(DISTINCT e.employee_id) AS projected_headcount
                FROM fpna_landing.hr_employees e
                WHERE e.hire_date <= :end_date
                  AND (e.termination_date IS NULL OR e.termination_date > :end_date)
            """)
            hc_row = (await db.execute(dept_headcount_sql, {"end_date": end_date})).first()
            proj_count = int(hc_row[0]) if hc_row and hc_row[0] else 0

            dept_prev_sql = text("""
                SELECT COUNT(DISTINCT e.employee_id) AS headcount
                FROM fpna_landing.hr_employees e
                WHERE e.hire_date <= :prev_end
                  AND (e.termination_date IS NULL OR e.termination_date > :prev_end)
            """)
            prev_hc_row = (await db.execute(dept_prev_sql, {"prev_end": prev_month_last})).first()
            prev_count = int(prev_hc_row[0]) if prev_hc_row and prev_hc_row[0] else 0

            dept_hc_adj = await _get_department_headcount_adjustments(db, year, quarter, company_id)
            dept_adj_map = {d["department_name"]: d for d in dept_hc_adj}

            all_dept_adj_m1 = []
            all_dept_adj_m2 = []
            all_dept_adj_m3 = []

            for dept_name, adj in dept_adj_map.items():
                all_dept_adj_m1.append(_resolve_forecast_value(float(proj_count), float(adj["month1_adjusted_count"]) if adj.get("month1_adjusted_count") is not None else None))
                all_dept_adj_m2.append(_resolve_forecast_value(float(proj_count), float(adj["month2_adjusted_count"]) if adj.get("month2_adjusted_count") is not None else None))
                all_dept_adj_m3.append(_resolve_forecast_value(float(proj_count), float(adj["month3_adjusted_count"]) if adj.get("month3_adjusted_count") is not None else None))

            hc_m1 = float(max(all_dept_adj_m1)) if all_dept_adj_m1 else float(proj_count)
            hc_m2 = float(max(all_dept_adj_m2)) if all_dept_adj_m2 else float(proj_count)
            hc_m3 = float(max(all_dept_adj_m3)) if all_dept_adj_m3 else float(proj_count)

            headcount = {
                "prev_month": float(prev_count),
                "month1_ai_suggested": hc_m1,
                "month2_ai_suggested": hc_m2,
                "month3_ai_suggested": hc_m3,
            }

            result = await db.execute(text(RAW_SQL))
            payroll_rows = result.mappings().all()
            df = pd.DataFrame(payroll_rows)

            if df.empty:
                return headcount, empty

            df["effective_date"] = pd.to_datetime(df["effective_date"])
            df["hire_date"] = pd.to_datetime(df["hire_date"])
            df["base_salary"] = df["base_salary"].fillna(0).astype(float)
            df["bonus_target"] = df["bonus_target"].fillna(0).astype(float)

            employees_df = df[["employee_id", "employee", "department", "role", "hire_date"]].drop_duplicates()
            df2 = df.drop(columns=["employee", "department", "role", "hire_date"])
            df2["effective_month"] = df2["effective_date"].dt.to_period("M").dt.to_timestamp()

            all_months_range = pd.date_range(start=prev_month_first, end=end_date, freq="MS")
            grid = (
                employees_df.assign(key=1)
                .merge(pd.DataFrame({"month": all_months_range, "key": 1}), on="key")
                .drop("key", axis=1)
            )
            merged = grid.merge(df2, on="employee_id", how="left")
            merged = merged[merged["effective_month"].isna() | (merged["effective_month"] <= merged["month"])]
            merged = merged.sort_values("effective_month").groupby(["employee_id", "month"]).tail(1)
            merged["salary"] = merged["base_salary"]
            merged["bonus"] = merged["bonus_target"]
            merged["benefits"] = merged["salary"] * 0.20
            merged.loc[merged["month"] < merged["hire_date"], ["salary", "bonus", "benefits"]] = 0

            out = []
            for _, emp_df in merged.groupby("employee_id"):
                emp_df = emp_df.sort_values("month").copy()
                n = len(emp_df)
                emp_df["salary"] = forecast_series_advanced(emp_df["salary"], n)
                emp_df["bonus"] = forecast_series_advanced(emp_df["bonus"], n)
                emp_df["benefits"] = emp_df["salary"] * 0.20
                out.append(emp_df)

            final_df = pd.concat(out, ignore_index=True)
            month_strs = [m.date().isoformat() for m in months]
            prev_month_str = prev_month_first.isoformat()

            saved_payroll_adjs = await _get_payroll_adjustments(db, year, quarter, company_id)
            payroll_adj_map = {int(s["employee_id"]): s for s in saved_payroll_adjs}

            month_str_to_index = {m.date().isoformat(): i + 1 for i, m in enumerate(months)}

            monthly_salary_totals = {ms: 0.0 for ms in month_strs}
            prev_salary_total = 0.0

            for emp_id, emp_df_group in final_df.groupby("employee_id"):
                emp_df_group = emp_df_group.sort_values("month")
                rows_list = emp_df_group.to_dict("records")
                if not rows_list:
                    continue

                adj = payroll_adj_map.get(int(emp_id), {})

                for row in rows_list:
                    month_dt = pd.Timestamp(row["month"])
                    ms = month_dt.date().isoformat()
                    salary_ai = round(float(row["salary"]), 2)

                    salary_final = salary_ai
                    if ms in month_str_to_index:
                        idx = month_str_to_index[ms]
                        adj_salary = adj.get(f"month{idx}_adjusted_salary")
                        if adj_salary is not None:
                            salary_final = float(adj_salary)

                    if ms == prev_month_str:
                        prev_salary_total += salary_final
                    elif ms in monthly_salary_totals:
                        monthly_salary_totals[ms] += salary_final

            payroll = {
                "prev_month": round(prev_salary_total, 2),
                "month1_ai_suggested": round(monthly_salary_totals.get(month_strs[0], 0.0), 2) if len(month_strs) > 0 else 0.0,
                "month2_ai_suggested": round(monthly_salary_totals.get(month_strs[1], 0.0), 2) if len(month_strs) > 1 else 0.0,
                "month3_ai_suggested": round(monthly_salary_totals.get(month_strs[2], 0.0), 2) if len(month_strs) > 2 else 0.0,
            }

            return headcount, payroll
        except Exception:
            return empty, empty


async def _pnl_get_capex_metrics(year: int, quarter: str, current_user: Dict) -> Dict[str, float]:
    async with AsyncSessionLocal() as db:
        try:
            await _ensure_capex_workflow_tables(db)
            company_id = get_company_id(current_user)

            m1, m2, m3 = get_quarter_months(year, quarter)
            quarter_start = pd.to_datetime(m1 + "-01")
            history_start = (quarter_start - pd.DateOffset(months=HISTORY_MONTHS_SQL)).strftime("%Y-%m")
            prev_month = (quarter_start - pd.DateOffset(months=1)).strftime("%Y-%m")

            distinct_sql = text(
                "SELECT DISTINCT fa.category FROM fpna_landing.finance_fixed_asset fa WHERE fa.category IS NOT NULL ORDER BY fa.category"
            )
            all_categories = [row[0] for row in (await db.execute(distinct_sql)).fetchall()]

            if not all_categories:
                return {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}

            history_sql = text("""
                SELECT fa.category,
                       TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') AS period,
                       SUM(fa.purchase_price) AS purchase_price
                FROM fpna_landing.finance_fixed_asset fa
                WHERE TO_DATE(fa.date_acquired, 'DD-MM-YYYY') >= TO_DATE(:start_period || '-01', 'YYYY-MM-DD')
                  AND TO_DATE(fa.date_acquired, 'DD-MM-YYYY') < TO_DATE(:quarter_start || '-01', 'YYYY-MM-DD')
                  AND fa.category IS NOT NULL
                GROUP BY fa.category, TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM')
                ORDER BY fa.category, period;
            """)
            item_hist_result = await db.execute(history_sql, {"start_period": history_start, "quarter_start": m1})
            item_hist_df = pd.DataFrame(item_hist_result.mappings().all())

            if item_hist_df.empty:
                item_hist_df = pd.DataFrame([{"category": all_categories[0], "period": prev_month, "purchase_price": 0.0}])

            prev_sql = text("""
                SELECT SUM(fa.purchase_price) AS total_purchase
                FROM fpna_landing.finance_fixed_asset fa
                WHERE TO_CHAR(TO_DATE(fa.date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') = :prev_month
                  AND fa.category IS NOT NULL
            """)
            prev_row = (await db.execute(prev_sql, {"prev_month": prev_month})).first()
            total_prev = float(prev_row[0]) if prev_row and prev_row[0] else 0.0

            saved_adjustments = await _get_capex_adjustments(db, year, quarter, company_id)
            adj_map = {s["capex_item"]: s for s in saved_adjustments}

            total_m1 = 0.0
            total_m2 = 0.0
            total_m3 = 0.0

            for capex_item in all_categories:
                v1, v2, v3 = _forecast_capex_item(capex_item, item_hist_df, quarter_start, m1, m2, m3)
                ai_m1 = float(v1)
                ai_m2 = float(v2)
                ai_m3 = float(v3)

                adj = adj_map.get(capex_item)
                adj_m1 = float(adj["month1_adjusted"]) if adj and adj.get("month1_adjusted") is not None else None
                adj_m2 = float(adj["month2_adjusted"]) if adj and adj.get("month2_adjusted") is not None else None
                adj_m3 = float(adj["month3_adjusted"]) if adj and adj.get("month3_adjusted") is not None else None

                total_m1 += _resolve_forecast_value(ai_m1, adj_m1)
                total_m2 += _resolve_forecast_value(ai_m2, adj_m2)
                total_m3 += _resolve_forecast_value(ai_m3, adj_m3)

            return {
                "prev_month": round(total_prev, 2),
                "month1_ai_suggested": round(total_m1, 2),
                "month2_ai_suggested": round(total_m2, 2),
                "month3_ai_suggested": round(total_m3, 2),
            }
        except Exception:
            return {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0}


async def generate_pnl_forecast_metrics(year: int, quarter: str, current_user: Dict) -> Dict[str, Dict[str, float]]:
    revenue_result, expense_result, cashflow_result, hc_payroll_result, capex_result = await asyncio.gather(
        _pnl_get_revenue_metrics(year, quarter, current_user),
        _pnl_get_expense_metrics(year, quarter, current_user),
        _pnl_get_cashflow_metrics(year, quarter, current_user),
        _pnl_get_headcount_payroll_metrics(year, quarter, current_user),
        _pnl_get_capex_metrics(year, quarter, current_user),
    )

    headcount_result, payroll_result = hc_payroll_result

    return {
        "REVENUE": revenue_result,
        "Expense": expense_result,
        "Cash Flow Projections": cashflow_result,
        "Headcount": headcount_result,
        "Payroll": payroll_result,
        "CAPEX Forecast": capex_result,
        "Scenario Modeling": {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0},
        "Debt & Interest": {"prev_month": 0.0, "month1_ai_suggested": 0.0, "month2_ai_suggested": 0.0, "month3_ai_suggested": 0.0},
    }


async def _fetch_all_scenario_drivers_for_year(
    company_id: str,
    year: int,
    custom_scenario_name: str,
) -> Dict[str, Optional[Dict[str, Any]]]:
    async with AsyncSessionLocal() as db:
        try:
            await _ensure_scenario_modeling_tables(db)

            scenario_name_clean = custom_scenario_name.strip().lower().replace(" ", "_")

            rows = (await db.execute(
                text("""
                    SELECT id, quarter, scenario_name, scenario_type
                    FROM fpna_config.scenario_modeling_scenarios
                    WHERE company_id = :company_id AND year = :year
                      AND LOWER(REPLACE(scenario_name, ' ', '_')) = :scenario_name_clean
                    ORDER BY quarter
                """),
                {
                    "company_id": company_id,
                    "year": year,
                    "scenario_name_clean": scenario_name_clean,
                },
            )).mappings().all()

            if not rows:
                return {}

            scenario_ids = [r["id"] for r in rows]

            all_rev_rows = (await db.execute(
                text("""
                    SELECT scenario_id, driver_name, month, driver_value
                    FROM fpna_config.scenario_revenue_drivers
                    WHERE scenario_id = ANY(:ids) AND company_id = :company_id AND year = :year
                    ORDER BY scenario_id, driver_name, month
                """),
                {"ids": scenario_ids, "company_id": company_id, "year": year},
            )).mappings().all()

            all_cost_rows = (await db.execute(
                text("""
                    SELECT scenario_id, driver_name, month, driver_value
                    FROM fpna_config.scenario_cost_drivers
                    WHERE scenario_id = ANY(:ids) AND company_id = :company_id AND year = :year
                    ORDER BY scenario_id, driver_name, month
                """),
                {"ids": scenario_ids, "company_id": company_id, "year": year},
            )).mappings().all()

            rev_by_id: Dict[int, List[Dict]] = {sid: [] for sid in scenario_ids}
            for r in all_rev_rows:
                rev_by_id[r["scenario_id"]].append({
                    "driver_name": r["driver_name"],
                    "month": r["month"],
                    "driver_value": float(r["driver_value"]),
                })

            cost_by_id: Dict[int, List[Dict]] = {sid: [] for sid in scenario_ids}
            for r in all_cost_rows:
                cost_by_id[r["scenario_id"]].append({
                    "driver_name": r["driver_name"],
                    "month": r["month"],
                    "driver_value": float(r["driver_value"]),
                })

            result: Dict[str, Optional[Dict[str, Any]]] = {}
            for row in rows:
                result[row["quarter"]] = {
                    "scenario_id": row["id"],
                    "scenario_name": row["scenario_name"],
                    "scenario_type": row["scenario_type"],
                    "revenue_drivers": rev_by_id.get(row["id"], []),
                    "cost_drivers": cost_by_id.get(row["id"], []),
                }

            return result
        except Exception:
            return {}


async def _fetch_available_scenarios_for_year(
    company_id: str,
    year: int,
) -> List[Dict[str, Any]]:
    async with AsyncSessionLocal() as db:
        try:
            await _ensure_scenario_modeling_tables(db)

            rows = (await db.execute(
                text("""
                    SELECT DISTINCT ON (LOWER(REPLACE(scenario_name, ' ', '_')))
                        id,
                        scenario_name,
                        scenario_type,
                        description,
                        quarter,
                        created_by_name,
                        created_at
                    FROM fpna_config.scenario_modeling_scenarios
                    WHERE company_id = :company_id AND year = :year
                    ORDER BY LOWER(REPLACE(scenario_name, ' ', '_')), created_at ASC
                """),
                {"company_id": company_id, "year": year},
            )).mappings().all()

            system_entries = [
                {
                    "scenario_name": "Base Case",
                    "scenario_type": "system",
                    "description": SCENARIO_MODELING_DEFINITIONS["base_case"],
                    "created_by_name": None,
                    "created_at": None,
                },
                {
                    "scenario_name": "Best Case",
                    "scenario_type": "system",
                    "description": SCENARIO_MODELING_DEFINITIONS["best_case"],
                    "created_by_name": None,
                    "created_at": None,
                },
                {
                    "scenario_name": "Worst Case",
                    "scenario_type": "system",
                    "description": SCENARIO_MODELING_DEFINITIONS["worst_case"],
                    "created_by_name": None,
                    "created_at": None,
                },
            ]

            custom_entries = [
                {
                    "scenario_name": row["scenario_name"],
                    "scenario_type": row["scenario_type"],
                    "description": row["description"],
                    "created_by_name": row["created_by_name"],
                    "created_at": row["created_at"].isoformat() if row["created_at"] else None,
                }
                for row in rows
            ]

            return system_entries + custom_entries
        except Exception:
            return []


def _build_scenario_override_metrics(
    revenue_drivers: List[Dict[str, Any]],
    cost_drivers: List[Dict[str, Any]],
    months: List[str],
    prev_month_revenue: float,
    prev_month_expense: float,
) -> Dict[str, Dict[str, float]]:
    m1, m2, m3 = months[0], months[1], months[2]

    revenue_by_month = _compute_scenario_revenue(revenue_drivers, months)
    cost_by_month = _compute_scenario_cost(cost_drivers, months)

    revenue_metrics = {
        "prev_month": round(prev_month_revenue, 2),
        "month1_ai_suggested": round(revenue_by_month.get(m1, 0.0), 2),
        "month2_ai_suggested": round(revenue_by_month.get(m2, 0.0), 2),
        "month3_ai_suggested": round(revenue_by_month.get(m3, 0.0), 2),
    }

    expense_metrics = {
        "prev_month": round(prev_month_expense, 2),
        "month1_ai_suggested": round(cost_by_month.get(m1, 0.0), 2),
        "month2_ai_suggested": round(cost_by_month.get(m2, 0.0), 2),
        "month3_ai_suggested": round(cost_by_month.get(m3, 0.0), 2),
    }

    return {
        "REVENUE": revenue_metrics,
        "Expense": expense_metrics,
    }


async def _build_pnl_forecast_full_year_with_scenario(
    year: int,
    current_user: Dict,
    custom_scenario_name: Optional[str] = None,
) -> Dict[str, list]:
    quarter_month_map = {
        "Q1": [1, 2, 3],
        "Q2": [4, 5, 6],
        "Q3": [7, 8, 9],
        "Q4": [10, 11, 12],
    }
    month_keys = ["month1_ai_suggested", "month2_ai_suggested", "month3_ai_suggested"]

    q1_result, q2_result, q3_result, q4_result = await asyncio.gather(
        generate_pnl_forecast_metrics(year, "Q1", current_user),
        generate_pnl_forecast_metrics(year, "Q2", current_user),
        generate_pnl_forecast_metrics(year, "Q3", current_user),
        generate_pnl_forecast_metrics(year, "Q4", current_user),
    )

    all_quarters_base = [("Q1", q1_result), ("Q2", q2_result), ("Q3", q3_result), ("Q4", q4_result)]

    scenario_overrides: Dict[str, Dict[str, Dict[str, float]]] = {}

    if custom_scenario_name:
        custom_scenario_name_clean = custom_scenario_name.strip().lower().replace(" ", "_")
        company_id = get_company_id(current_user)

        is_system = custom_scenario_name_clean in SCENARIO_MODELING_RESERVED_NAMES

        if is_system:
            multiplier_map = {
                "base_case": BASE_CASE_MULTIPLIER,
                "best_case": BEST_CASE_MULTIPLIER,
                "worst_case": WORST_CASE_MULTIPLIER,
            }
            multiplier = multiplier_map[custom_scenario_name_clean]
            for quarter, base_q_result in all_quarters_base:
                q_months = get_quarter_months(year, quarter)
                drivers = _build_system_scenario_drivers(custom_scenario_name_clean, q_months, multiplier)
                prev_rev = base_q_result.get("REVENUE", {}).get("prev_month", 0.0)
                prev_exp = base_q_result.get("Expense", {}).get("prev_month", 0.0)
                scenario_overrides[quarter] = _build_scenario_override_metrics(
                    revenue_drivers=drivers["revenue_drivers"],
                    cost_drivers=drivers["cost_drivers"],
                    months=q_months,
                    prev_month_revenue=prev_rev,
                    prev_month_expense=prev_exp,
                )
        else:
            all_quarter_drivers = await _fetch_all_scenario_drivers_for_year(
                company_id, year, custom_scenario_name
            )
            for quarter, base_q_result in all_quarters_base:
                scenario_data = all_quarter_drivers.get(quarter)
                if scenario_data is None:
                    continue
                q_months = get_quarter_months(year, quarter)
                prev_rev = base_q_result.get("REVENUE", {}).get("prev_month", 0.0)
                prev_exp = base_q_result.get("Expense", {}).get("prev_month", 0.0)
                scenario_overrides[quarter] = _build_scenario_override_metrics(
                    revenue_drivers=scenario_data["revenue_drivers"],
                    cost_drivers=scenario_data["cost_drivers"],
                    months=q_months,
                    prev_month_revenue=prev_rev,
                    prev_month_expense=prev_exp,
                )

    revenue_forecasting = []
    expense_forecasting = []
    capex_forecast_list = []
    cash_flow_projections = []
    headcount_forecast = []
    payroll_forecast = []

    total_revenue = 0.0
    total_expenses = 0.0
    total_cashflow = 0.0
    total_payroll = 0.0
    total_capex = 0.0
    last_headcount = 0

    for quarter, ai_metrics in all_quarters_base:
        months = quarter_month_map[quarter]

        override = scenario_overrides.get(quarter, {})

        revenue_data = override.get("REVENUE", ai_metrics.get("REVENUE", {}))
        expense_data = override.get("Expense", ai_metrics.get("Expense", {}))
        cashflow_data = ai_metrics.get("Cash Flow Projections", {})
        headcount_data = ai_metrics.get("Headcount", {})
        payroll_data = ai_metrics.get("Payroll", {})
        capex_data = ai_metrics.get("CAPEX Forecast", {})

        for i, month_num in enumerate(months):
            period = f"{year}-{str(month_num).zfill(2)}"
            rev_val = round(revenue_data.get(month_keys[i], 0.0), 2)
            exp_val = round(expense_data.get(month_keys[i], 0.0), 2)
            cf_val = round(cashflow_data.get(month_keys[i], 0.0), 2)
            hc_val = int(headcount_data.get(month_keys[i], 0))
            pay_val = round(payroll_data.get(month_keys[i], 0.0), 2)
            capex_val = round(capex_data.get(month_keys[i], 0.0), 2)

            revenue_forecasting.append({"period": period, "revenue": rev_val})
            expense_forecasting.append({"period": period, "expenses": exp_val})
            capex_forecast_list.append({"period": period, "capex": capex_val})
            cash_flow_projections.append({"period": period, "cash_flow": cf_val})
            headcount_forecast.append({"period": period, "headcount": hc_val})
            payroll_forecast.append({"period": period, "payroll": pay_val})

            total_revenue += rev_val
            total_expenses += exp_val
            total_cashflow += cf_val
            total_payroll += pay_val
            total_capex += capex_val
            last_headcount = hc_val

    return {
        "revenue_forecasting": revenue_forecasting,
        "expense_forecasting": expense_forecasting,
        "capex_forecast": capex_forecast_list,
        "cash_flow_projections": cash_flow_projections,
        "headcount_forecast": headcount_forecast,
        "payroll_forecast": payroll_forecast,
        "summary_metrics": {
            "forecast_revenue": round(total_revenue, 2),
            "forecast_expenses": round(total_expenses, 2),
            "forecast_cashflow": round(total_cashflow, 2),
            "forecast_headcount": last_headcount,
            "forecast_payroll": round(total_payroll, 2),
            "forecast_capex": round(total_capex, 2),
        },
    }


async def _build_pnl_forecast_full_year(year: int, current_user: Dict) -> Dict[str, list]:
    return await _build_pnl_forecast_full_year_with_scenario(year, current_user, custom_scenario_name=None)


def _apply_scenario_multipliers(data: Dict[str, list], scenario: PnLScenarioFilter) -> Dict[str, list]:
    if scenario == PnLScenarioFilter.baseline:
        return data

    multipliers = {
        PnLScenarioFilter.bestcase: {"revenue_forecasting": 1.10, "expense_forecasting": 0.92, "capex_forecast": 0.95, "cash_flow_projections": 1.10, "payroll_forecast": 1.03},
        PnLScenarioFilter.worstcase: {"revenue_forecasting": 0.90, "expense_forecasting": 1.08, "capex_forecast": 1.05, "cash_flow_projections": 0.90, "payroll_forecast": 1.02},
    }

    m = multipliers[scenario]
    result = {}

    for key, rows in data.items():
        if key == "headcount_forecast" or key == "summary_metrics":
            result[key] = rows
            continue
        mul = m.get(key, 1.0)
        new_rows = []
        for row in rows:
            new_row = dict(row)
            for field in new_row:
                if field != "period":
                    new_row[field] = round(float(new_row[field]) * mul, 2)
            new_rows.append(new_row)
        result[key] = new_rows

    if "summary_metrics" in data:
        sm = dict(data["summary_metrics"])
        sm["forecast_revenue"] = round(sm.get("forecast_revenue", 0.0) * m.get("revenue_forecasting", 1.0), 2)
        sm["forecast_expenses"] = round(sm.get("forecast_expenses", 0.0) * m.get("expense_forecasting", 1.0), 2)
        sm["forecast_cashflow"] = round(sm.get("forecast_cashflow", 0.0) * m.get("cash_flow_projections", 1.0), 2)
        sm["forecast_payroll"] = round(sm.get("forecast_payroll", 0.0) * m.get("payroll_forecast", 1.0), 2)
        sm["forecast_capex"] = round(sm.get("forecast_capex", 0.0) * m.get("capex_forecast", 1.0), 2)
        result["summary_metrics"] = sm

    return result


def _filter_by_period(full_year_data: Dict[str, list], year: int, period_filter: PnLPeriodFilter) -> Dict[str, list]:
    list_keys = ["revenue_forecasting", "expense_forecasting", "capex_forecast", "cash_flow_projections", "headcount_forecast", "payroll_forecast"]

    if period_filter == PnLPeriodFilter.monthly:
        return full_year_data

    if period_filter == PnLPeriodFilter.quarterly:
        quarter_map = {
            f"{year}-01": "Q1", f"{year}-02": "Q1", f"{year}-03": "Q1",
            f"{year}-04": "Q2", f"{year}-05": "Q2", f"{year}-06": "Q2",
            f"{year}-07": "Q3", f"{year}-08": "Q3", f"{year}-09": "Q3",
            f"{year}-10": "Q4", f"{year}-11": "Q4", f"{year}-12": "Q4",
        }
        result = {}
        for key in list_keys:
            rows = full_year_data.get(key, [])
            aggregated = {}
            for row in rows:
                q_label = quarter_map.get(row["period"], "Q1")
                if q_label not in aggregated:
                    aggregated[q_label] = {"period": q_label}
                    for field in row:
                        if field != "period":
                            aggregated[q_label][field] = 0
                for field in row:
                    if field != "period":
                        if key == "headcount_forecast":
                            aggregated[q_label][field] = row[field]
                        else:
                            aggregated[q_label][field] = round(aggregated[q_label][field] + float(row[field]), 2)
            result[key] = list(aggregated.values())
        result["summary_metrics"] = full_year_data["summary_metrics"]
        return result

    if period_filter == PnLPeriodFilter.ytd:
        from datetime import datetime
        current_month = datetime.now().month
        current_year = datetime.now().year
        if current_year != year:
            cutoff_month = 12
        else:
            cutoff_month = current_month
        cutoff_period = f"{year}-{str(cutoff_month).zfill(2)}"
        result = {}
        for key in list_keys:
            rows = full_year_data.get(key, [])
            result[key] = [row for row in rows if row["period"] <= cutoff_period]
        ytd_revenue = sum(r["revenue"] for r in result["revenue_forecasting"])
        ytd_expenses = sum(r["expenses"] for r in result["expense_forecasting"])
        ytd_cashflow = sum(r["cash_flow"] for r in result["cash_flow_projections"])
        ytd_payroll = sum(r["payroll"] for r in result["payroll_forecast"])
        ytd_capex = sum(r["capex"] for r in result["capex_forecast"])
        ytd_headcount = result["headcount_forecast"][-1]["headcount"] if result["headcount_forecast"] else 0
        result["summary_metrics"] = {
            "forecast_revenue": round(ytd_revenue, 2),
            "forecast_expenses": round(ytd_expenses, 2),
            "forecast_cashflow": round(ytd_cashflow, 2),
            "forecast_headcount": ytd_headcount,
            "forecast_payroll": round(ytd_payroll, 2),
            "forecast_capex": round(ytd_capex, 2),
        }
        return result

    return full_year_data


def _format_forecast_rows(rows, value_field):
    formatted = []
    for row in rows:
        value = row.get(value_field, 0.0)
        try:
            value = float(value)
        except (TypeError, ValueError):
            value = 0.0
        formatted.append({"period": row.get("period"), "forecast_value": round(value, 2)})
    return formatted


def _build_period_value_lookup(formatted_rows: list) -> Dict[str, float]:
    return {row["period"]: row.get("forecast_value", 0.0) for row in formatted_rows}


def _prev_quarter_label(q_label: str) -> str:
    mapping = {"Q1": "Q4", "Q2": "Q1", "Q3": "Q2", "Q4": "Q3"}
    return mapping.get(q_label, "Q1")


def _apply_filter_and_scenario(raw_full_year: Dict[str, list], year: int, period_filter: PnLPeriodFilter, scenario: PnLScenarioFilter) -> Dict[str, list]:
    filtered = _filter_by_period(raw_full_year, year, period_filter)
    return _apply_scenario_multipliers(filtered, scenario)


def _format_all_list_keys(processed: Dict[str, list], list_key_field_map: Dict[str, str]) -> Dict[str, list]:
    out = {}
    for list_key, value_field in list_key_field_map.items():
        out[list_key] = _format_forecast_rows(processed.get(list_key, []), value_field)
    return out


def _attach_monthly_comparison(current_rows: list, all_periods_lookup: Dict[str, float]) -> list:
    result = []
    for row in current_rows:
        period = row.get("period")
        current_value = row.get("forecast_value", 0.0)
        try:
            current_value = float(current_value)
        except (TypeError, ValueError):
            current_value = 0.0

        compare_month = None
        try:
            compare_month = (pd.to_datetime(period + "-01") - pd.DateOffset(months=1)).strftime("%Y-%m")
        except Exception:
            pass

        compare_value = 0.0
        if compare_month:
            compare_value = all_periods_lookup.get(compare_month, 0.0)
            try:
                compare_value = float(compare_value)
            except (TypeError, ValueError):
                compare_value = 0.0

        variance = round(current_value - compare_value, 2)
        pct = round((variance / compare_value * 100), 2) if compare_value != 0.0 else 0.0

        result.append({
            "period": period,
            "forecast_value": current_value,
            "compare_month": compare_month,
            "compare_value": compare_value,
            "variance": variance,
            "pct": pct,
        })
    return result


def _attach_quarterly_comparison(current_rows: list, all_quarters_lookup: Dict[str, float]) -> list:
    result = []
    for row in current_rows:
        period = row.get("period")
        current_value = row.get("forecast_value", 0.0)
        try:
            current_value = float(current_value)
        except (TypeError, ValueError):
            current_value = 0.0

        compare_quarter = _prev_quarter_label(period)
        compare_value = all_quarters_lookup.get(compare_quarter, 0.0)
        try:
            compare_value = float(compare_value)
        except (TypeError, ValueError):
            compare_value = 0.0

        variance = round(current_value - compare_value, 2)
        pct = round((variance / compare_value * 100), 2) if compare_value != 0.0 else 0.0

        result.append({
            "period": period,
            "forecast_value": current_value,
            "compare_quarter": compare_quarter,
            "compare_value": compare_value,
            "variance": variance,
            "pct": pct,
        })
    return result


def _attach_yearly_comparison(current_rows: list, prev_year_lookup: Dict[str, float]) -> list:
    result = []
    for row in current_rows:
        period = row.get("period")
        current_value = row.get("forecast_value", 0.0)
        try:
            current_value = float(current_value)
        except (TypeError, ValueError):
            current_value = 0.0

        compare_month = None
        try:
            compare_month = (pd.to_datetime(period + "-01") - pd.DateOffset(years=1)).strftime("%Y-%m")
        except Exception:
            pass

        compare_value = 0.0
        if compare_month:
            compare_value = prev_year_lookup.get(compare_month, 0.0)
            try:
                compare_value = float(compare_value)
            except (TypeError, ValueError):
                compare_value = 0.0

        variance = round(current_value - compare_value, 2)
        pct = round((variance / compare_value * 100), 2) if compare_value != 0.0 else 0.0

        result.append({
            "period": period,
            "forecast_value": current_value,
            "compare_month": compare_month,
            "compare_value": compare_value,
            "variance": variance,
            "pct": pct,
        })
    return result


def _slice_pnl_lists_to_periods(pnl_data: Dict, month_periods: List[str]) -> Dict:
    period_set = set(month_periods)
    list_keys = ["revenue_forecasting", "expense_forecasting", "capex_forecast",
                 "cash_flow_projections", "headcount_forecast", "payroll_forecast"]
    result = dict(pnl_data)
    for key in list_keys:
        rows = result.get(key, [])
        monthly_rows = [r for r in rows if isinstance(r, dict) and "period" in r and len(str(r.get("period", ""))) == 7]
        if monthly_rows:
            result[key] = [r for r in rows if not (isinstance(r, dict) and "period" in r) or r.get("period") in period_set]
    def _fv_sum(key_name: str) -> float:
        return round(sum(r.get("forecast_value", 0.0) for r in result.get(key_name, []) if "forecast_value" in r), 2)
    hc_rows = [r for r in result.get("headcount_forecast", []) if "forecast_value" in r]
    result["summary_metrics"] = {
        "forecast_revenue":   _fv_sum("revenue_forecasting"),
        "forecast_expenses":  _fv_sum("expense_forecasting"),
        "forecast_cashflow":  _fv_sum("cash_flow_projections"),
        "forecast_headcount": int(hc_rows[-1].get("forecast_value", 0)) if hc_rows else 0,
        "forecast_payroll":   _fv_sum("payroll_forecast"),
        "forecast_capex":     _fv_sum("capex_forecast"),
    }
    return result


def _slice_dashboard_lists_to_periods(dash_data: Dict, month_periods: List[str]) -> Dict:
    period_set = set(month_periods)
    list_keys = ["revenue_forecasting", "expense_forecasting", "capex_forecast",
                 "cash_flow_projections", "headcount_forecast", "payroll_forecast"]
    result = dict(dash_data)
    for key in list_keys:
        rows = result.get(key, [])
        result[key] = [r for r in rows if not (isinstance(r, dict) and "period" in r) or r.get("period") in period_set]
    def _sum_field(key_name: str, field: str) -> float:
        return round(sum(r.get(field, 0.0) for r in result.get(key_name, []) if "period" in r), 2)
    hc_rows = [r for r in result.get("headcount_forecast", []) if "period" in r]
    result["summary_metrics"] = {
        "forecast_revenue":   _sum_field("revenue_forecasting",   "revenue"),
        "forecast_expenses":  _sum_field("expense_forecasting",   "expenses"),
        "forecast_cashflow":  _sum_field("cash_flow_projections", "cash_flow"),
        "forecast_headcount": int(hc_rows[-1].get("headcount", 0)) if hc_rows else 0,
        "forecast_payroll":   _sum_field("payroll_forecast",      "payroll"),
        "forecast_capex":     _sum_field("capex_forecast",        "capex"),
    }
    return result


async def pnl_forecasting_handler(
    year: int,
    period_filter: PnLPeriodFilter,
    scenario: PnLScenarioFilter,
    current_user: Dict,
    comparison: Optional[PnLComparisonFilter] = None,
    custom_scenario_name: Optional[str] = None,
):
    list_key_field_map = {
        "revenue_forecasting": "revenue",
        "expense_forecasting": "expenses",
        "capex_forecast": "capex",
        "cash_flow_projections": "cash_flow",
        "headcount_forecast": "headcount",
        "payroll_forecast": "payroll",
    }

    if comparison is None:
        current_raw = await _build_pnl_forecast_full_year_with_scenario(year, current_user, custom_scenario_name=custom_scenario_name)
        current_processed = _apply_filter_and_scenario(current_raw, year, period_filter, scenario)
        formatted = _format_all_list_keys(current_processed, list_key_field_map)
        return {
            "summary_metrics": current_processed["summary_metrics"],
            "revenue_forecasting": formatted["revenue_forecasting"],
            "expense_forecasting": formatted["expense_forecasting"],
            "capex_forecast": formatted["capex_forecast"],
            "cash_flow_projections": formatted["cash_flow_projections"],
            "headcount_forecast": formatted["headcount_forecast"],
            "payroll_forecast": formatted["payroll_forecast"],
        }

    if comparison == PnLComparisonFilter.previous_month:
        current_raw, prev_year_raw = await asyncio.gather(
            _build_pnl_forecast_full_year_with_scenario(year, current_user, custom_scenario_name=custom_scenario_name),
            _build_pnl_forecast_full_year_with_scenario(year - 1, current_user, custom_scenario_name=custom_scenario_name),
        )

        current_processed = _apply_filter_and_scenario(current_raw, year, period_filter, scenario)
        formatted_current = _format_all_list_keys(current_processed, list_key_field_map)

        current_monthly_processed = _apply_filter_and_scenario(current_raw, year, PnLPeriodFilter.monthly, scenario)
        prev_year_monthly_processed = _apply_filter_and_scenario(prev_year_raw, year - 1, PnLPeriodFilter.monthly, scenario)

        comparison_response = {"summary_metrics": current_processed["summary_metrics"]}
        for list_key, value_field in list_key_field_map.items():
            current_monthly_rows = _format_forecast_rows(current_monthly_processed.get(list_key, []), value_field)
            prev_year_monthly_rows = _format_forecast_rows(prev_year_monthly_processed.get(list_key, []), value_field)
            combined_lookup = _build_period_value_lookup(prev_year_monthly_rows)
            combined_lookup.update(_build_period_value_lookup(current_monthly_rows))
            comparison_response[list_key] = _attach_monthly_comparison(formatted_current[list_key], combined_lookup)
        return comparison_response

    if comparison == PnLComparisonFilter.previous_quarter:
        current_raw, prev_year_raw = await asyncio.gather(
            _build_pnl_forecast_full_year_with_scenario(year, current_user, custom_scenario_name=custom_scenario_name),
            _build_pnl_forecast_full_year_with_scenario(year - 1, current_user, custom_scenario_name=custom_scenario_name),
        )

        current_processed = _apply_filter_and_scenario(current_raw, year, period_filter, scenario)
        current_quarterly_processed = _apply_filter_and_scenario(current_raw, year, PnLPeriodFilter.quarterly, scenario)
        prev_year_quarterly_processed = _apply_filter_and_scenario(prev_year_raw, year - 1, PnLPeriodFilter.quarterly, scenario)

        formatted_current_quarterly = _format_all_list_keys(current_quarterly_processed, list_key_field_map)

        comparison_response = {"summary_metrics": current_processed["summary_metrics"]}
        for list_key, value_field in list_key_field_map.items():
            current_q_rows = _format_forecast_rows(current_quarterly_processed.get(list_key, []), value_field)
            prev_year_q_rows = _format_forecast_rows(prev_year_quarterly_processed.get(list_key, []), value_field)
            combined_lookup = _build_period_value_lookup(prev_year_q_rows)
            combined_lookup.update(_build_period_value_lookup(current_q_rows))
            comparison_response[list_key] = _attach_quarterly_comparison(formatted_current_quarterly[list_key], combined_lookup)
        return comparison_response

    if comparison == PnLComparisonFilter.previous_year:
        current_raw, prev_year_raw = await asyncio.gather(
            _build_pnl_forecast_full_year_with_scenario(year, current_user, custom_scenario_name=custom_scenario_name),
            _build_pnl_forecast_full_year_with_scenario(year - 1, current_user, custom_scenario_name=custom_scenario_name),
        )

        current_processed = _apply_filter_and_scenario(current_raw, year, period_filter, scenario)
        current_ytd_processed = _apply_filter_and_scenario(current_raw, year, PnLPeriodFilter.ytd, scenario)
        prev_year_ytd_processed = _apply_filter_and_scenario(prev_year_raw, year - 1, PnLPeriodFilter.ytd, scenario)

        formatted_current_ytd = _format_all_list_keys(current_ytd_processed, list_key_field_map)

        comparison_response = {"summary_metrics": current_processed["summary_metrics"]}
        for list_key, value_field in list_key_field_map.items():
            prev_year_rows = _format_forecast_rows(prev_year_ytd_processed.get(list_key, []), value_field)
            prev_year_lookup = _build_period_value_lookup(prev_year_rows)
            comparison_response[list_key] = _attach_yearly_comparison(formatted_current_ytd[list_key], prev_year_lookup)
        return comparison_response

    current_raw = await _build_pnl_forecast_full_year_with_scenario(year, current_user, custom_scenario_name=custom_scenario_name)
    current_processed = _apply_filter_and_scenario(current_raw, year, period_filter, scenario)
    formatted = _format_all_list_keys(current_processed, list_key_field_map)
    return {
        "summary_metrics": current_processed["summary_metrics"],
        "revenue_forecasting": formatted["revenue_forecasting"],
        "expense_forecasting": formatted["expense_forecasting"],
        "capex_forecast": formatted["capex_forecast"],
        "cash_flow_projections": formatted["cash_flow_projections"],
        "headcount_forecast": formatted["headcount_forecast"],
        "payroll_forecast": formatted["payroll_forecast"],
    }


@forecast_router.get("/pnl-forecast/{year}/{currency}")
async def get_pnl_forecast(
    year: int = Path(...),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    period_filter: PnLPeriodFilter = Query(PnLPeriodFilter.monthly),
    scenario: PnLScenarioFilter = Query(PnLScenarioFilter.baseline),
    comparison: Optional[PnLComparisonFilter] = Query(default=None),
    custom_scenario_name: Optional[str] = Query(default=None, description="Name of a custom or system scenario. Supports: 'Base Case', 'Best Case', 'Worst Case', or any custom name."),
    forecast_year: Optional[int] = Query(default=None, description="Primary forecast year context. Overrides year for period resolution when provided."),
    forecast_type: Optional[str] = Query(default="12months", description="Forecast type: quarter, 12months, rolling. PnL defaults to 12months (full year)."),
    quarter: QuarterEnum = Query(QuarterEnum.Q1, description="Quarter to focus on when forecast_type=quarter."),
    rolling_start_month: Optional[str] = Query(default=None, description="Start month for rolling forecast (e.g. Jan, Feb, Mar). Required when forecast_type=rolling."),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    company_id = get_company_id(current_user)

    effective_year = forecast_year if forecast_year is not None else year
    effective_forecast_type = forecast_type or "12months"

    if effective_forecast_type not in ("quarter", "12months", "rolling"):
        raise HTTPException(status_code=422, detail="forecast_type must be one of: quarter, 12months, rolling")
    if effective_forecast_type == "rolling" and not rolling_start_month:
        raise HTTPException(status_code=422, detail="rolling_start_month is required when forecast_type=rolling.")

    _MONTH_ORDER = {"jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
                    "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12}
    resolve_year = effective_year
    if effective_forecast_type == "rolling" and rolling_start_month:
        if _MONTH_ORDER.get(rolling_start_month.lower()[:3], 1) >= 11:
            resolve_year = effective_year - 1

    month_periods, selected_period, _ = _resolve_forecast_periods(
        resolve_year, effective_forecast_type, quarter.value, rolling_start_month
    )

    pnl_data, available_scenarios = await asyncio.gather(
        pnl_forecasting_handler(
            effective_year,
            period_filter,
            scenario,
            current_user,
            comparison,
            custom_scenario_name=custom_scenario_name,
        ),
        _fetch_available_scenarios_for_year(company_id, effective_year),
    )

    if effective_forecast_type != "12months" and period_filter == PnLPeriodFilter.monthly:
        pnl_data = _slice_pnl_lists_to_periods(pnl_data, month_periods)

    money_fields = [
        "forecast_value", "forecast_revenue", "forecast_expenses",
        "forecast_cashflow", "forecast_payroll", "forecast_capex",
        "compare_value", "variance",
    ]

    result = apply_smart_currency(pnl_data, currency, currency_filter, money_fields)

    result["forecast_year"] = effective_year
    result["forecast_type"] = effective_forecast_type
    result["selected_period"] = selected_period
    result["selected_months"] = month_periods
    result["active_scenario"] = custom_scenario_name if custom_scenario_name else None
    result["available_scenarios"] = available_scenarios

    return result
##########################################################################################
###########################################################################################
###########################################################################################
#Senario Modelling

class WorkflowStage(str, Enum):
    DRAFT = "draft"
    SUBMITTED = "submitted"
    REVIEWED = "reviewed"
    APPROVED = "approved"
    LOCKED = "locked"


SCENARIO_MODELING_RESERVED_NAMES = {"base_case", "best_case", "worst_case"}

BASE_CASE_MULTIPLIER = 1.0
BEST_CASE_MULTIPLIER = 1.15
WORST_CASE_MULTIPLIER = 0.80

SCENARIO_MODELING_DEFINITIONS = {
    "base_case": "Most realistic expected outcome. Revenue and cost drivers at standard baseline values.",
    "best_case": f"Optimistic scenario applying a {int((BEST_CASE_MULTIPLIER - 1) * 100)}% uplift — assumes favorable market conditions, strong demand, full pipeline conversion, and operational efficiency.",
    "worst_case": f"Pessimistic scenario applying a {int((1 - WORST_CASE_MULTIPLIER) * 100)}% reduction — assumes adverse market conditions, customer churn, demand shortfall, or cost overruns.",
}

SCENARIO_MODELING_MONEY_FIELDS = [
    "revenue",
    "cost",
    "net_profit",
    "month1_revenue",
    "month2_revenue",
    "month3_revenue",
    "quarter_total_revenue",
    "month1_cost",
    "month2_cost",
    "month3_cost",
    "quarter_total_cost",
    "month1_net_profit",
    "month2_net_profit",
    "month3_net_profit",
    "quarter_total_net_profit",
    "base_case",
    "best_case",
    "worst_case",
]


def can_scenario_transition(dept: str, current_stage: str, target_stage: str) -> bool:
    if current_stage == WorkflowStage.LOCKED:
        return False
    if dept in DEPT_SUBMITTERS:
        if target_stage == WorkflowStage.SUBMITTED and current_stage == WorkflowStage.DRAFT:
            return True
        return False
    if dept in DEPT_REVIEWERS:
        if target_stage == WorkflowStage.REVIEWED and current_stage == WorkflowStage.SUBMITTED:
            return True
        if target_stage == WorkflowStage.DRAFT:
            return True
        return False
    if dept in DEPT_APPROVERS:
        if target_stage == WorkflowStage.APPROVED and current_stage == WorkflowStage.REVIEWED:
            return True
        if target_stage == WorkflowStage.LOCKED and current_stage == WorkflowStage.APPROVED:
            return True
        if target_stage == WorkflowStage.DRAFT:
            return True
        return False
    return False


class RevenueDriverInput(BaseModel):
    driver_name: str
    month: str
    driver_value: float


class CostDriverInput(BaseModel):
    driver_name: str
    month: str
    driver_value: float


class ScenarioCreateInput(BaseModel):
    scenario_name: str
    description: Optional[str] = None
    revenue_drivers: Optional[List[RevenueDriverInput]] = []
    cost_drivers: Optional[List[CostDriverInput]] = []

    @validator("scenario_name")
    def validate_scenario_name(cls, v):
        clean = v.strip().lower().replace(" ", "_")
        if clean in SCENARIO_MODELING_RESERVED_NAMES:
            raise ValueError(f"'{v}' is a reserved scenario name. Reserved names are: Base Case, Best Case, Worst Case.")
        return v


class ScenarioUpdateInput(BaseModel):
    scenario_name: Optional[str] = None
    description: Optional[str] = None
    revenue_drivers: Optional[List[RevenueDriverInput]] = None
    cost_drivers: Optional[List[CostDriverInput]] = None

    @validator("scenario_name")
    def validate_scenario_name(cls, v):
        if v is not None:
            clean = v.strip().lower().replace(" ", "_")
            if clean in SCENARIO_MODELING_RESERVED_NAMES:
                raise ValueError(f"'{v}' is a reserved scenario name. Reserved names are: Base Case, Best Case, Worst Case.")
        return v


class ScenarioWorkflowTransitionRequest(BaseModel):
    target_stage: WorkflowStage
    comment: Optional[str] = None


class ScenarioCommentInput(BaseModel):
    comment: str


async def _ensure_scenario_modeling_tables(db: AsyncSession) -> None:
    await db.execute(text("CREATE SCHEMA IF NOT EXISTS fpna_config;"))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.scenario_modeling_scenarios (
            id SERIAL PRIMARY KEY,
            company_id TEXT NOT NULL,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            scenario_name TEXT NOT NULL,
            scenario_type TEXT NOT NULL DEFAULT 'custom',
            description TEXT NULL,
            is_final BOOLEAN NOT NULL DEFAULT FALSE,
            created_by_user_id TEXT NOT NULL,
            created_by_name TEXT NOT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW(),
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            UNIQUE (company_id, year, quarter, scenario_name)
        );
    """))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.scenario_revenue_drivers (
            id SERIAL PRIMARY KEY,
            scenario_id INT NOT NULL,
            company_id TEXT NOT NULL,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            driver_name TEXT NOT NULL,
            month TEXT NOT NULL,
            driver_value NUMERIC NOT NULL,
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            UNIQUE (scenario_id, company_id, year, quarter, driver_name, month)
        );
    """))

    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'scenario_revenue_drivers'
                  AND column_name  = 'scenario_id'
            ) THEN
                ALTER TABLE fpna_config.scenario_revenue_drivers ADD COLUMN scenario_id INT NOT NULL DEFAULT 0;
            END IF;
        END$$;
    """))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.scenario_cost_drivers (
            id SERIAL PRIMARY KEY,
            scenario_id INT NOT NULL,
            company_id TEXT NOT NULL,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            driver_name TEXT NOT NULL,
            month TEXT NOT NULL,
            driver_value NUMERIC NOT NULL,
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            UNIQUE (scenario_id, company_id, year, quarter, driver_name, month)
        );
    """))

    await db.execute(text("""
        DO $$
        BEGIN
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'scenario_cost_drivers'
                  AND column_name  = 'scenario_id'
            ) THEN
                ALTER TABLE fpna_config.scenario_cost_drivers ADD COLUMN scenario_id INT NOT NULL DEFAULT 0;
            END IF;
        END$$;
    """))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.scenario_modeling_workflows (
            id SERIAL PRIMARY KEY,
            year INT NOT NULL,
            quarter TEXT NOT NULL,
            company_id TEXT NOT NULL,
            owner_user_id TEXT NOT NULL,
            stage TEXT NOT NULL DEFAULT 'draft',
            created_at TIMESTAMP NOT NULL DEFAULT NOW(),
            updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
            locked_at TIMESTAMP NULL,
            UNIQUE (year, quarter, company_id)
        );
    """))

    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'scenario_modeling_workflows'
                  AND column_name  = 'owner_user_id'
                  AND data_type   != 'text'
            ) THEN
                ALTER TABLE fpna_config.scenario_modeling_workflows
                    ALTER COLUMN owner_user_id TYPE TEXT USING owner_user_id::TEXT;
            END IF;
        END$$;
    """))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.scenario_modeling_workflow_audit (
            id SERIAL PRIMARY KEY,
            workflow_id INT NOT NULL,
            from_stage TEXT NULL,
            to_stage TEXT NOT NULL,
            transitioned_by_user_id TEXT NOT NULL,
            transitioned_by_name TEXT NOT NULL,
            comment TEXT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """))

    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'scenario_modeling_workflow_audit'
                  AND column_name  = 'transitioned_by_user_id'
                  AND data_type   != 'text'
            ) THEN
                ALTER TABLE fpna_config.scenario_modeling_workflow_audit
                    ALTER COLUMN transitioned_by_user_id TYPE TEXT USING transitioned_by_user_id::TEXT;
            END IF;
        END$$;
    """))

    await db.execute(text("""
        CREATE TABLE IF NOT EXISTS fpna_config.scenario_modeling_workflow_comments (
            id SERIAL PRIMARY KEY,
            workflow_id INT NOT NULL,
            user_id TEXT NOT NULL,
            user_name TEXT NOT NULL,
            department TEXT NOT NULL,
            comment TEXT NOT NULL,
            comment_type TEXT NOT NULL DEFAULT 'general',
            created_at TIMESTAMP NOT NULL DEFAULT NOW()
        );
    """))

    await db.execute(text("""
        DO $$
        BEGIN
            IF EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'scenario_modeling_workflow_comments'
                  AND column_name  = 'user_id'
                  AND data_type   != 'text'
            ) THEN
                ALTER TABLE fpna_config.scenario_modeling_workflow_comments
                    ALTER COLUMN user_id TYPE TEXT USING user_id::TEXT;
            END IF;
            IF NOT EXISTS (
                SELECT 1 FROM information_schema.columns
                WHERE table_schema = 'fpna_config'
                  AND table_name   = 'scenario_modeling_workflow_comments'
                  AND column_name  = 'comment_type'
            ) THEN
                ALTER TABLE fpna_config.scenario_modeling_workflow_comments
                    ADD COLUMN comment_type TEXT NOT NULL DEFAULT 'general';
            END IF;
        END$$;
    """))

    await db.commit()


async def _get_or_create_scenario_workflow(
    db: AsyncSession, year: int, quarter: str, company_id: str, user_id: str
) -> Dict[str, Any]:
    row = (await db.execute(
        text("""
            SELECT * FROM fpna_config.scenario_modeling_workflows
            WHERE year = :year AND quarter = :quarter AND company_id = :company_id
            ORDER BY id ASC LIMIT 1
        """),
        {"year": year, "quarter": quarter, "company_id": company_id},
    )).mappings().first()

    if not row:
        row = (await db.execute(
            text("""
                INSERT INTO fpna_config.scenario_modeling_workflows
                    (year, quarter, company_id, owner_user_id, stage)
                VALUES (:year, :quarter, :company_id, :uid, 'draft')
                RETURNING *
            """),
            {"year": year, "quarter": quarter, "company_id": company_id, "uid": user_id},
        )).mappings().first()
        await db.commit()

    return dict(row)


async def _get_scenario_revenue_drivers(
    db: AsyncSession, scenario_id: int, company_id: str, year: int, quarter: str
) -> List[Dict[str, Any]]:
    rows = (await db.execute(
        text("""
            SELECT driver_name, month, driver_value
            FROM fpna_config.scenario_revenue_drivers
            WHERE scenario_id = :scenario_id AND company_id = :company_id
              AND year = :year AND quarter = :quarter
            ORDER BY driver_name, month
        """),
        {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter},
    )).mappings().all()
    return [{"driver_name": r["driver_name"], "month": r["month"], "driver_value": float(r["driver_value"])} for r in rows]


async def _get_scenario_cost_drivers(
    db: AsyncSession, scenario_id: int, company_id: str, year: int, quarter: str
) -> List[Dict[str, Any]]:
    rows = (await db.execute(
        text("""
            SELECT driver_name, month, driver_value
            FROM fpna_config.scenario_cost_drivers
            WHERE scenario_id = :scenario_id AND company_id = :company_id
              AND year = :year AND quarter = :quarter
            ORDER BY driver_name, month
        """),
        {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter},
    )).mappings().all()
    return [{"driver_name": r["driver_name"], "month": r["month"], "driver_value": float(r["driver_value"])} for r in rows]


async def _get_all_scenario_drivers_bulk(
    db: AsyncSession, scenario_ids: List[int], company_id: str, year: int, quarter: str
) -> tuple:
    if not scenario_ids:
        return {}, {}

    rev_rows = (await db.execute(
        text("""
            SELECT scenario_id, driver_name, month, driver_value
            FROM fpna_config.scenario_revenue_drivers
            WHERE scenario_id = ANY(:ids) AND company_id = :company_id
              AND year = :year AND quarter = :quarter
            ORDER BY scenario_id, driver_name, month
        """),
        {"ids": scenario_ids, "company_id": company_id, "year": year, "quarter": quarter},
    )).mappings().all()

    cost_rows = (await db.execute(
        text("""
            SELECT scenario_id, driver_name, month, driver_value
            FROM fpna_config.scenario_cost_drivers
            WHERE scenario_id = ANY(:ids) AND company_id = :company_id
              AND year = :year AND quarter = :quarter
            ORDER BY scenario_id, driver_name, month
        """),
        {"ids": scenario_ids, "company_id": company_id, "year": year, "quarter": quarter},
    )).mappings().all()

    rev_by_id: Dict[int, List[Dict]] = {sid: [] for sid in scenario_ids}
    for r in rev_rows:
        rev_by_id[r["scenario_id"]].append({
            "driver_name": r["driver_name"],
            "month": r["month"],
            "driver_value": float(r["driver_value"]),
        })

    cost_by_id: Dict[int, List[Dict]] = {sid: [] for sid in scenario_ids}
    for r in cost_rows:
        cost_by_id[r["scenario_id"]].append({
            "driver_name": r["driver_name"],
            "month": r["month"],
            "driver_value": float(r["driver_value"]),
        })

    return rev_by_id, cost_by_id


async def _upsert_revenue_drivers(
    db: AsyncSession, scenario_id: int, company_id: str, year: int, quarter: str,
    revenue_drivers: List[RevenueDriverInput],
) -> None:
    for drv in revenue_drivers:
        await db.execute(
            text("""
                INSERT INTO fpna_config.scenario_revenue_drivers
                    (scenario_id, company_id, year, quarter, driver_name, month, driver_value, updated_at)
                VALUES (:scenario_id, :company_id, :year, :quarter, :driver_name, :month, :driver_value, NOW())
                ON CONFLICT (scenario_id, company_id, year, quarter, driver_name, month)
                DO UPDATE SET driver_value = EXCLUDED.driver_value, updated_at = NOW();
            """),
            {
                "scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter,
                "driver_name": drv.driver_name, "month": drv.month, "driver_value": drv.driver_value,
            },
        )


async def _upsert_cost_drivers(
    db: AsyncSession, scenario_id: int, company_id: str, year: int, quarter: str,
    cost_drivers: List[CostDriverInput],
) -> None:
    for drv in cost_drivers:
        await db.execute(
            text("""
                INSERT INTO fpna_config.scenario_cost_drivers
                    (scenario_id, company_id, year, quarter, driver_name, month, driver_value, updated_at)
                VALUES (:scenario_id, :company_id, :year, :quarter, :driver_name, :month, :driver_value, NOW())
                ON CONFLICT (scenario_id, company_id, year, quarter, driver_name, month)
                DO UPDATE SET driver_value = EXCLUDED.driver_value, updated_at = NOW();
            """),
            {
                "scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter,
                "driver_name": drv.driver_name, "month": drv.month, "driver_value": drv.driver_value,
            },
        )


def _compute_scenario_revenue(revenue_drivers: List[Dict[str, Any]], months: List[str]) -> Dict[str, Any]:
    month_revenue = {}
    for month in months:
        month_drivers = {d["driver_name"]: float(d["driver_value"]) for d in revenue_drivers if d["month"] == month}
        volume = month_drivers.get("volume", 1000.0)
        selling_price = month_drivers.get("selling_price", 100.0)
        growth_rate = month_drivers.get("growth_rate", 0.0)
        acquisition_rate = month_drivers.get("acquisition_rate", 1.0)
        month_revenue[month] = round(volume * selling_price * (1 + growth_rate) * acquisition_rate, 2)
    total = round(sum(month_revenue.values()), 2)
    return {**month_revenue, "quarter_total": total}


def _compute_scenario_cost(cost_drivers: List[Dict[str, Any]], months: List[str]) -> Dict[str, Any]:
    month_cost = {}
    for month in months:
        month_drivers = [d for d in cost_drivers if d["month"] == month]
        month_cost[month] = round(sum(float(d["driver_value"]) for d in month_drivers), 2)
    total = round(sum(month_cost.values()), 2)
    return {**month_cost, "quarter_total": total}


def _build_system_scenario_drivers(scenario_name: str, months: List[str], multiplier: float) -> Dict[str, Any]:
    revenue_drivers = []
    cost_drivers = []
    for month in months:
        revenue_drivers.append({"driver_name": "volume", "month": month, "driver_value": round(1000.0 * multiplier, 2)})
        revenue_drivers.append({"driver_name": "selling_price", "month": month, "driver_value": 100.0})
        revenue_drivers.append({"driver_name": "growth_rate", "month": month, "driver_value": 0.05 if scenario_name == "best_case" else (-0.05 if scenario_name == "worst_case" else 0.0)})
        revenue_drivers.append({"driver_name": "acquisition_rate", "month": month, "driver_value": multiplier})
        cost_drivers.append({"driver_name": "headcount_cost", "month": month, "driver_value": round(50000.0 * multiplier, 2)})
        cost_drivers.append({"driver_name": "non_people_costs", "month": month, "driver_value": round(20000.0 * multiplier, 2)})
        cost_drivers.append({"driver_name": "capex", "month": month, "driver_value": round(10000.0 * multiplier, 2)})
        cost_drivers.append({"driver_name": "interest", "month": month, "driver_value": round(5000.0 * multiplier, 2)})
    return {"revenue_drivers": revenue_drivers, "cost_drivers": cost_drivers}


def _build_scenario_financials(
    scenario_name: str, scenario_type: str, description: str, is_final: bool, is_system: bool,
    revenue_drivers: List[Dict], cost_drivers: List[Dict], months: List[str],
    m1: str, m2: str, m3: str,
    scenario_id: Optional[int], created_by_name: Optional[str],
    created_at: Optional[str], updated_at: Optional[str],
) -> Dict[str, Any]:
    revenue_by_month = _compute_scenario_revenue(revenue_drivers, months)
    cost_by_month = _compute_scenario_cost(cost_drivers, months)

    net_profit_m1 = round(revenue_by_month.get(m1, 0.0) - cost_by_month.get(m1, 0.0), 2)
    net_profit_m2 = round(revenue_by_month.get(m2, 0.0) - cost_by_month.get(m2, 0.0), 2)
    net_profit_m3 = round(revenue_by_month.get(m3, 0.0) - cost_by_month.get(m3, 0.0), 2)
    net_profit_total = round(net_profit_m1 + net_profit_m2 + net_profit_m3, 2)
    rev_total = revenue_by_month.get("quarter_total", 0.0)
    profit_margin = round((net_profit_total / rev_total * 100) if rev_total else 0.0, 2)

    return {
        "scenario_id": scenario_id,
        "scenario_name": scenario_name,
        "scenario_type": scenario_type,
        "description": description,
        "is_final": is_final,
        "is_system": is_system,
        "revenue": {
            "month1": revenue_by_month.get(m1, 0.0),
            "month2": revenue_by_month.get(m2, 0.0),
            "month3": revenue_by_month.get(m3, 0.0),
            "quarter_total": revenue_by_month.get("quarter_total", 0.0),
        },
        "cost": {
            "month1": cost_by_month.get(m1, 0.0),
            "month2": cost_by_month.get(m2, 0.0),
            "month3": cost_by_month.get(m3, 0.0),
            "quarter_total": cost_by_month.get("quarter_total", 0.0),
        },
        "net_profit": {
            "month1": net_profit_m1,
            "month2": net_profit_m2,
            "month3": net_profit_m3,
            "quarter_total": net_profit_total,
        },
        "profit_margin_percent": profit_margin,
        "revenue_drivers": revenue_drivers,
        "cost_drivers": cost_drivers,
        "created_by_name": created_by_name,
        "created_at": created_at,
        "updated_at": updated_at,
    }


async def _get_final_scenario(db: AsyncSession, company_id: str, year: int, quarter: str) -> Optional[str]:
    row = (await db.execute(
        text("""
            SELECT scenario_name FROM fpna_config.scenario_modeling_scenarios
            WHERE company_id = :company_id AND year = :year AND quarter = :quarter AND is_final = TRUE
            LIMIT 1
        """),
        {"company_id": company_id, "year": year, "quarter": quarter},
    )).mappings().first()
    return row["scenario_name"] if row else None


@forecast_router.post("/scenario-modeling/{year}/scenarios")
async def create_scenario_modeling_scenario(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: ScenarioCreateInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_scenario_modeling_tables(db)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    user_name = current_user.get("full_name", "")

    await _get_or_create_scenario_workflow(db, year, quarter.value, company_id, user_id)

    scenario_name_clean = body.scenario_name.strip().lower().replace(" ", "_")

    if scenario_name_clean in SCENARIO_MODELING_RESERVED_NAMES:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail=f"'{body.scenario_name}' is a reserved scenario name. Reserved names are: Base Case, Best Case, Worst Case.",
        )

    existing = (await db.execute(
        text("""
            SELECT id FROM fpna_config.scenario_modeling_scenarios
            WHERE company_id = :company_id AND year = :year AND quarter = :quarter
              AND LOWER(REPLACE(scenario_name, ' ', '_')) = :scenario_name_clean
        """),
        {"company_id": company_id, "year": year, "quarter": quarter.value, "scenario_name_clean": scenario_name_clean},
    )).mappings().first()

    if existing:
        raise HTTPException(
            status_code=status.HTTP_409_CONFLICT,
            detail=f"A scenario named '{body.scenario_name}' already exists for {year} {quarter.value}.",
        )

    result = (await db.execute(
        text("""
            INSERT INTO fpna_config.scenario_modeling_scenarios
                (company_id, year, quarter, scenario_name, scenario_type, description,
                 is_final, created_by_user_id, created_by_name)
            VALUES (:company_id, :year, :quarter, :scenario_name, 'custom', :description,
                    FALSE, :uid, :uname)
            RETURNING id, scenario_name, scenario_type, description, is_final,
                      created_by_name, created_at, updated_at
        """),
        {
            "company_id": company_id, "year": year, "quarter": quarter.value,
            "scenario_name": body.scenario_name.strip(),
            "description": body.description.strip() if body.description else None,
            "uid": user_id, "uname": user_name,
        },
    )).mappings().first()

    await db.commit()

    scenario_id = result["id"]

    if body.revenue_drivers:
        await _upsert_revenue_drivers(db, scenario_id, company_id, year, quarter.value, body.revenue_drivers)

    if body.cost_drivers:
        await _upsert_cost_drivers(db, scenario_id, company_id, year, quarter.value, body.cost_drivers)

    if body.revenue_drivers or body.cost_drivers:
        await db.commit()

    saved_revenue_drivers = await _get_scenario_revenue_drivers(db, scenario_id, company_id, year, quarter.value)
    saved_cost_drivers = await _get_scenario_cost_drivers(db, scenario_id, company_id, year, quarter.value)

    return {
        "detail": "Scenario created successfully.",
        "year": year,
        "quarter": quarter.value,
        "scenario": {
            "id": result["id"],
            "scenario_name": result["scenario_name"],
            "scenario_type": result["scenario_type"],
            "description": result["description"],
            "is_final": result["is_final"],
            "revenue_drivers": saved_revenue_drivers,
            "cost_drivers": saved_cost_drivers,
            "created_by_name": result["created_by_name"],
            "created_at": result["created_at"].isoformat(),
            "updated_at": result["updated_at"].isoformat(),
        },
    }


@forecast_router.get("/scenario-modeling/{year}/scenarios")
async def get_scenario_modeling_scenarios(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_scenario_modeling_tables(db)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_scenario_workflow(db, year, quarter.value, company_id, user_id)
    months = get_quarter_months(year, quarter.value)

    custom_rows = (await db.execute(
        text("""
            SELECT id, scenario_name, scenario_type, description, is_final,
                   created_by_name, created_at, updated_at
            FROM fpna_config.scenario_modeling_scenarios
            WHERE company_id = :company_id AND year = :year AND quarter = :quarter
            ORDER BY created_at ASC
        """),
        {"company_id": company_id, "year": year, "quarter": quarter.value},
    )).mappings().all()

    scenario_ids = [row["id"] for row in custom_rows]
    rev_by_id, cost_by_id = await _get_all_scenario_drivers_bulk(db, scenario_ids, company_id, year, quarter.value)

    custom_scenarios = []
    for row in custom_rows:
        custom_scenarios.append({
            "id": row["id"],
            "scenario_name": row["scenario_name"],
            "scenario_type": row["scenario_type"],
            "description": row["description"],
            "is_final": row["is_final"],
            "is_system": False,
            "revenue_drivers": rev_by_id.get(row["id"], []),
            "cost_drivers": cost_by_id.get(row["id"], []),
            "created_by_name": row["created_by_name"],
            "created_at": row["created_at"].isoformat(),
            "updated_at": row["updated_at"].isoformat(),
        })

    final_scenario_name = await _get_final_scenario(db, company_id, year, quarter.value)

    system_scenarios = []
    for scenario_key, multiplier in [("base_case", BASE_CASE_MULTIPLIER), ("best_case", BEST_CASE_MULTIPLIER), ("worst_case", WORST_CASE_MULTIPLIER)]:
        drivers = _build_system_scenario_drivers(scenario_key, months, multiplier)
        display_name = scenario_key.replace("_", " ").title()
        system_scenarios.append({
            "id": None,
            "scenario_name": display_name,
            "scenario_type": "system",
            "description": SCENARIO_MODELING_DEFINITIONS[scenario_key],
            "is_final": (final_scenario_name is not None and final_scenario_name.lower().replace(" ", "_") == scenario_key),
            "is_system": True,
            "revenue_drivers": drivers["revenue_drivers"],
            "cost_drivers": drivers["cost_drivers"],
            "created_by_name": None,
            "created_at": None,
            "updated_at": None,
        })

    all_scenarios = system_scenarios + custom_scenarios

    return {
        "year": year,
        "quarter": quarter.value,
        "workflow_stage": wf["stage"],
        "final_scenario": final_scenario_name,
        "total_scenarios": len(all_scenarios),
        "system_scenarios_count": len(system_scenarios),
        "custom_scenarios_count": len(custom_scenarios),
        "scenarios": all_scenarios,
    }


@forecast_router.patch("/scenario-modeling/{year}/scenarios/{scenario_id}")
async def update_scenario_modeling_scenario(
    year: int = Path(...),
    scenario_id: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: ScenarioUpdateInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_scenario_modeling_tables(db)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_scenario_workflow(db, year, quarter.value, company_id, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="Scenario modeling is locked. Scenarios cannot be changed.",
        )

    existing = (await db.execute(
        text("""
            SELECT id, scenario_name, scenario_type FROM fpna_config.scenario_modeling_scenarios
            WHERE id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter
        """),
        {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
    )).mappings().first()

    if not existing:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail=f"Scenario with id {scenario_id} not found for {year} {quarter.value}.",
        )

    if existing["scenario_type"] == "system":
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="System scenarios (Base Case, Best Case, Worst Case) cannot be edited.",
        )

    if body.scenario_name is not None:
        new_name_clean = body.scenario_name.strip().lower().replace(" ", "_")
        if new_name_clean in SCENARIO_MODELING_RESERVED_NAMES:
            raise HTTPException(
                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                detail=f"'{body.scenario_name}' is a reserved scenario name. Reserved names are: Base Case, Best Case, Worst Case.",
            )
        name_conflict = (await db.execute(
            text("""
                SELECT id FROM fpna_config.scenario_modeling_scenarios
                WHERE company_id = :company_id AND year = :year AND quarter = :quarter
                  AND LOWER(REPLACE(scenario_name, ' ', '_')) = :new_name_clean AND id != :sid
            """),
            {"company_id": company_id, "year": year, "quarter": quarter.value, "new_name_clean": new_name_clean, "sid": scenario_id},
        )).mappings().first()
        if name_conflict:
            raise HTTPException(
                status_code=status.HTTP_409_CONFLICT,
                detail=f"A scenario named '{body.scenario_name}' already exists for {year} {quarter.value}.",
            )

    set_clauses = ["updated_at = NOW()"]
    params: Dict[str, Any] = {"sid": scenario_id}

    if body.scenario_name is not None:
        set_clauses.append("scenario_name = :scenario_name")
        params["scenario_name"] = body.scenario_name.strip()
    if body.description is not None:
        set_clauses.append("description = :description")
        params["description"] = body.description.strip()

    result = (await db.execute(
        text(f"""
            UPDATE fpna_config.scenario_modeling_scenarios
            SET {', '.join(set_clauses)}
            WHERE id = :sid
            RETURNING id, scenario_name, scenario_type, description, is_final,
                      created_by_name, created_at, updated_at
        """),
        params,
    )).mappings().first()

    await db.commit()

    if body.revenue_drivers is not None:
        await db.execute(
            text("""
                DELETE FROM fpna_config.scenario_revenue_drivers
                WHERE scenario_id = :scenario_id AND company_id = :company_id
                  AND year = :year AND quarter = :quarter
            """),
            {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
        )
        await _upsert_revenue_drivers(db, scenario_id, company_id, year, quarter.value, body.revenue_drivers)

    if body.cost_drivers is not None:
        await db.execute(
            text("""
                DELETE FROM fpna_config.scenario_cost_drivers
                WHERE scenario_id = :scenario_id AND company_id = :company_id
                  AND year = :year AND quarter = :quarter
            """),
            {"scenario_id": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
        )
        await _upsert_cost_drivers(db, scenario_id, company_id, year, quarter.value, body.cost_drivers)

    if body.revenue_drivers is not None or body.cost_drivers is not None:
        await db.commit()

    saved_revenue_drivers = await _get_scenario_revenue_drivers(db, scenario_id, company_id, year, quarter.value)
    saved_cost_drivers = await _get_scenario_cost_drivers(db, scenario_id, company_id, year, quarter.value)

    return {
        "detail": "Scenario updated successfully.",
        "year": year,
        "quarter": quarter.value,
        "scenario": {
            "id": result["id"],
            "scenario_name": result["scenario_name"],
            "scenario_type": result["scenario_type"],
            "description": result["description"],
            "is_final": result["is_final"],
            "revenue_drivers": saved_revenue_drivers,
            "cost_drivers": saved_cost_drivers,
            "created_by_name": result["created_by_name"],
            "created_at": result["created_at"].isoformat(),
            "updated_at": result["updated_at"].isoformat(),
        },
    }


@forecast_router.delete("/scenario-modeling/{year}/scenarios/{scenario_id}")
async def delete_scenario_modeling_scenario(
    year: int = Path(...),
    scenario_id: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_scenario_modeling_tables(db)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_scenario_workflow(db, year, quarter.value, company_id, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="Scenario modeling is locked. Scenarios cannot be deleted.",
        )

    existing = (await db.execute(
        text("""
            SELECT id, scenario_name, scenario_type, is_final
            FROM fpna_config.scenario_modeling_scenarios
            WHERE id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter
        """),
        {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
    )).mappings().first()

    if not existing:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail=f"Scenario with id {scenario_id} not found for {year} {quarter.value}.",
        )

    if existing["scenario_type"] == "system":
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="System scenarios (Base Case, Best Case, Worst Case) cannot be deleted.",
        )

    if existing["is_final"]:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail=f"Scenario '{existing['scenario_name']}' is marked as the final forecast and cannot be deleted. Please set another scenario as final first.",
        )

    await db.execute(
        text("""
            DELETE FROM fpna_config.scenario_revenue_drivers
            WHERE scenario_id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter
        """),
        {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
    )

    await db.execute(
        text("""
            DELETE FROM fpna_config.scenario_cost_drivers
            WHERE scenario_id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter
        """),
        {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
    )

    await db.execute(
        text("DELETE FROM fpna_config.scenario_modeling_scenarios WHERE id = :sid"),
        {"sid": scenario_id},
    )

    await db.commit()

    return {
        "detail": f"Scenario '{existing['scenario_name']}' deleted successfully.",
        "year": year,
        "quarter": quarter.value,
        "deleted_scenario_id": scenario_id,
    }


@forecast_router.post("/scenario-modeling/{year}/scenarios/{scenario_id}/set-final")
async def set_scenario_modeling_final(
    year: int = Path(...),
    scenario_id: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_scenario_modeling_tables(db)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_scenario_workflow(db, year, quarter.value, company_id, user_id)
    stage = wf["stage"]

    if stage not in (WorkflowStage.APPROVED, WorkflowStage.LOCKED):
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail=f"A scenario can only be marked as final when the workflow is in 'approved' or 'locked' stage. Current stage is '{stage}'.",
        )

    existing = (await db.execute(
        text("""
            SELECT id, scenario_name, scenario_type FROM fpna_config.scenario_modeling_scenarios
            WHERE id = :sid AND company_id = :company_id AND year = :year AND quarter = :quarter
        """),
        {"sid": scenario_id, "company_id": company_id, "year": year, "quarter": quarter.value},
    )).mappings().first()

    if not existing:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail=f"Scenario with id {scenario_id} not found for {year} {quarter.value}.",
        )

    await db.execute(
        text("""
            UPDATE fpna_config.scenario_modeling_scenarios
            SET is_final = FALSE, updated_at = NOW()
            WHERE company_id = :company_id AND year = :year AND quarter = :quarter
        """),
        {"company_id": company_id, "year": year, "quarter": quarter.value},
    )

    await db.execute(
        text("UPDATE fpna_config.scenario_modeling_scenarios SET is_final = TRUE, updated_at = NOW() WHERE id = :sid"),
        {"sid": scenario_id},
    )

    await db.commit()

    return {
        "detail": f"Scenario '{existing['scenario_name']}' has been marked as the final forecast.",
        "year": year,
        "quarter": quarter.value,
        "final_scenario_id": scenario_id,
        "final_scenario_name": existing["scenario_name"],
        "workflow_stage": stage,
    }


@forecast_router.post("/scenario-modeling/{year}/scenarios/system/set-final")
async def set_system_scenario_as_final(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    scenario_name: str = Query(..., description="System scenario name: Base Case, Best Case, or Worst Case"),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_scenario_modeling_tables(db)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_scenario_workflow(db, year, quarter.value, company_id, user_id)
    stage = wf["stage"]

    if stage not in (WorkflowStage.APPROVED, WorkflowStage.LOCKED):
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail=f"A scenario can only be marked as final when the workflow is in 'approved' or 'locked' stage. Current stage is '{stage}'.",
        )

    scenario_name_clean = scenario_name.strip().lower().replace(" ", "_")
    if scenario_name_clean not in SCENARIO_MODELING_RESERVED_NAMES:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail=f"'{scenario_name}' is not a valid system scenario. Valid system scenarios are: Base Case, Best Case, Worst Case.",
        )

    await db.execute(
        text("""
            UPDATE fpna_config.scenario_modeling_scenarios
            SET is_final = FALSE, updated_at = NOW()
            WHERE company_id = :company_id AND year = :year AND quarter = :quarter
        """),
        {"company_id": company_id, "year": year, "quarter": quarter.value},
    )

    await db.commit()

    return {
        "detail": f"System scenario '{scenario_name}' has been marked as the final forecast.",
        "year": year,
        "quarter": quarter.value,
        "final_scenario_name": scenario_name.strip(),
        "is_system": True,
        "workflow_stage": stage,
    }


@forecast_router.get("/scenario-modeling/{year}/comparison/{currency}")
async def get_scenario_modeling_comparison(
    year: int = Path(...),
    currency: str = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_scenario_modeling_tables(db)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_scenario_workflow(db, year, quarter.value, company_id, user_id)
    stage = wf["stage"]

    months = get_quarter_months(year, quarter.value)
    m1, m2, m3 = months

    final_scenario_name = await _get_final_scenario(db, company_id, year, quarter.value)

    custom_rows = (await db.execute(
        text("""
            SELECT id, scenario_name, scenario_type, description, is_final,
                   created_by_name, created_at, updated_at
            FROM fpna_config.scenario_modeling_scenarios
            WHERE company_id = :company_id AND year = :year AND quarter = :quarter
            ORDER BY created_at ASC
        """),
        {"company_id": company_id, "year": year, "quarter": quarter.value},
    )).mappings().all()

    scenario_ids = [row["id"] for row in custom_rows]
    rev_by_id, cost_by_id = await _get_all_scenario_drivers_bulk(db, scenario_ids, company_id, year, quarter.value)

    all_scenarios_data = []

    for scenario_key, multiplier in [("base_case", BASE_CASE_MULTIPLIER), ("best_case", BEST_CASE_MULTIPLIER), ("worst_case", WORST_CASE_MULTIPLIER)]:
        drivers = _build_system_scenario_drivers(scenario_key, months, multiplier)
        display_name = scenario_key.replace("_", " ").title()
        is_final = final_scenario_name is not None and final_scenario_name.lower().replace(" ", "_") == scenario_key
        all_scenarios_data.append(_build_scenario_financials(
            scenario_name=display_name,
            scenario_type="system",
            description=SCENARIO_MODELING_DEFINITIONS[scenario_key],
            is_final=is_final,
            is_system=True,
            revenue_drivers=drivers["revenue_drivers"],
            cost_drivers=drivers["cost_drivers"],
            months=months, m1=m1, m2=m2, m3=m3,
            scenario_id=None,
            created_by_name=None,
            created_at=None,
            updated_at=None,
        ))

    for row in custom_rows:
        all_scenarios_data.append(_build_scenario_financials(
            scenario_name=row["scenario_name"],
            scenario_type=row["scenario_type"],
            description=row["description"],
            is_final=row["is_final"],
            is_system=False,
            revenue_drivers=rev_by_id.get(row["id"], []),
            cost_drivers=cost_by_id.get(row["id"], []),
            months=months, m1=m1, m2=m2, m3=m3,
            scenario_id=row["id"],
            created_by_name=row["created_by_name"],
            created_at=row["created_at"].isoformat(),
            updated_at=row["updated_at"].isoformat(),
        ))

    comparison_metrics = ["Revenue", "Cost", "Net Profit", "Profit Margin %"]
    comparison_table = []

    for metric in comparison_metrics:
        row_data = {"metric": metric}
        for s in all_scenarios_data:
            scenario_key = s["scenario_name"].lower().replace(" ", "_")
            if metric == "Revenue":
                row_data[scenario_key] = s["revenue"]["quarter_total"]
            elif metric == "Cost":
                row_data[scenario_key] = s["cost"]["quarter_total"]
            elif metric == "Net Profit":
                row_data[scenario_key] = s["net_profit"]["quarter_total"]
            elif metric == "Profit Margin %":
                row_data[scenario_key] = s["profit_margin_percent"]
        comparison_table.append(row_data)

    response = {
        "year": year,
        "quarter": quarter.value,
        "months": months,
        "workflow_stage": stage,
        "final_scenario": final_scenario_name,
        "total_scenarios": len(all_scenarios_data),
        "scenarios": all_scenarios_data,
        "comparison_table": comparison_table,
    }

    return apply_smart_currency(response, currency, currency_filter, SCENARIO_MODELING_MONEY_FIELDS)


@forecast_router.patch("/scenario-modeling/{year}/workflow")
async def patch_scenario_modeling_workflow(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: ScenarioWorkflowTransitionRequest = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_scenario_modeling_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_scenario_workflow(db, year, quarter.value, company_id, user_id)
    current_stage = wf["stage"]
    target_stage = body.target_stage.value

    effective_comment = body.comment.strip() if body.comment and body.comment.strip() else None

    is_rejection = target_stage == WorkflowStage.DRAFT and current_stage != WorkflowStage.DRAFT
    if is_rejection and not effective_comment:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="A comment is required when rejecting (sending back to draft). Please provide a reason.",
        )

    if effective_comment and dept not in DEPT_REVIEWERS | DEPT_APPROVERS:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="Only Finance/Executive/CEO/CFO can add review comments.",
        )

    if not can_scenario_transition(dept, current_stage, target_stage):
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail=f"Department '{dept}' cannot move scenario modeling from '{current_stage}' to '{target_stage}'.",
        )

    if target_stage == WorkflowStage.LOCKED:
        await db.execute(
            text("""
                UPDATE fpna_config.scenario_modeling_workflows
                SET stage = :new_stage, updated_at = NOW(), locked_at = NOW()
                WHERE id = :wf_id
            """),
            {"new_stage": target_stage, "wf_id": wf["id"]},
        )
    else:
        await db.execute(
            text("""
                UPDATE fpna_config.scenario_modeling_workflows
                SET stage = :new_stage, updated_at = NOW()
                WHERE id = :wf_id
            """),
            {"new_stage": target_stage, "wf_id": wf["id"]},
        )

    await db.execute(
        text("""
            INSERT INTO fpna_config.scenario_modeling_workflow_audit
                (workflow_id, from_stage, to_stage, transitioned_by_user_id, transitioned_by_name, comment)
            VALUES (:wf_id, :from_s, :to_s, :uid, :uname, :comment)
        """),
        {
            "wf_id": wf["id"], "from_s": current_stage, "to_s": target_stage,
            "uid": user_id, "uname": current_user.get("full_name", ""), "comment": effective_comment,
        },
    )

    if effective_comment and dept in DEPT_REVIEWERS | DEPT_APPROVERS:
        comment_type = "rejection" if is_rejection else "transition"
        await db.execute(
            text("""
                INSERT INTO fpna_config.scenario_modeling_workflow_comments
                    (workflow_id, user_id, user_name, department, comment, comment_type)
                VALUES (:wf_id, :uid, :uname, :dept, :comment, :comment_type)
            """),
            {
                "wf_id": wf["id"], "uid": user_id, "uname": current_user.get("full_name", ""),
                "dept": dept, "comment": effective_comment, "comment_type": comment_type,
            },
        )

    await db.commit()

    return {
        "detail": f"Scenario modeling moved from '{current_stage}' to '{target_stage}'.",
        "year": year,
        "quarter": quarter.value,
        "department": dept,
        "stage": target_stage,
    }


@forecast_router.post("/scenario-modeling/{year}/comments")
async def post_scenario_modeling_comment(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    body: ScenarioCommentInput = Body(...),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_scenario_modeling_tables(db)
    dept = get_user_department(current_user)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")
    user_name = current_user.get("full_name", "")

    wf = await _get_or_create_scenario_workflow(db, year, quarter.value, company_id, user_id)
    stage = wf["stage"]

    if stage == WorkflowStage.LOCKED:
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="Scenario modeling is locked. Comments cannot be added.",
        )

    comment_text = body.comment.strip() if body.comment else ""
    if not comment_text:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Comment cannot be empty.",
        )

    result = (await db.execute(
        text("""
            INSERT INTO fpna_config.scenario_modeling_workflow_comments
                (workflow_id, user_id, user_name, department, comment, comment_type)
            VALUES (:wf_id, :uid, :uname, :dept, :comment, 'general')
            RETURNING id, user_id, user_name, department, comment, comment_type, created_at
        """),
        {"wf_id": wf["id"], "uid": user_id, "uname": user_name, "dept": dept, "comment": comment_text},
    )).mappings().first()

    await db.commit()

    return {
        "detail": "Comment posted successfully.",
        "year": year,
        "quarter": quarter.value,
        "workflow_stage": stage,
        "comment": {
            "id": result["id"],
            "user_id": result["user_id"],
            "user_name": result["user_name"],
            "department": result["department"],
            "comment": result["comment"],
            "comment_type": result["comment_type"],
            "created_at": result["created_at"].isoformat(),
        },
    }


@forecast_router.get("/scenario-modeling/{year}/comments")
async def get_scenario_modeling_comments(
    year: int = Path(...),
    quarter: QuarterEnum = Query(QuarterEnum.Q1),
    comment_type: Optional[str] = Query(None, description="Filter by comment_type: general, transition, rejection. Leave empty for all."),
    db: AsyncSession = Depends(get_db),
    current_user: Dict = Depends(get_current_user),
):
    await _ensure_scenario_modeling_tables(db)
    company_id = get_company_id(current_user)
    user_id = str(current_user.get("id") or current_user.get("user_id") or "")

    wf = await _get_or_create_scenario_workflow(db, year, quarter.value, company_id, user_id)
    stage = wf["stage"]

    if comment_type:
        rows = (await db.execute(
            text("""
                SELECT id, user_id, user_name, department, comment, comment_type, created_at
                FROM fpna_config.scenario_modeling_workflow_comments
                WHERE workflow_id = :wf_id AND comment_type = :comment_type
                ORDER BY created_at ASC
            """),
            {"wf_id": wf["id"], "comment_type": comment_type},
        )).mappings().all()
    else:
        rows = (await db.execute(
            text("""
                SELECT id, user_id, user_name, department, comment, comment_type, created_at
                FROM fpna_config.scenario_modeling_workflow_comments
                WHERE workflow_id = :wf_id
                ORDER BY created_at ASC
            """),
            {"wf_id": wf["id"]},
        )).mappings().all()

    comments = [
        {
            "id": r["id"],
            "user_id": r["user_id"],
            "user_name": r["user_name"],
            "department": r["department"],
            "comment": r["comment"],
            "comment_type": r["comment_type"],
            "created_at": r["created_at"].isoformat(),
        }
        for r in rows
    ]

    return {
        "year": year,
        "quarter": quarter.value,
        "workflow_stage": stage,
        "total_comments": len(comments),
        "filters": {
            "comment_type": comment_type or "all",
        },
        "comments": comments,
    }