from fastapi import APIRouter, Request,Body,Path,BackgroundTasks, File, UploadFile, Depends, HTTPException,Query,Form
from typing import List, Optional, Any , Literal
from openai import BaseModel
from sqlalchemy.orm import Session
from decimal import Decimal, ROUND_HALF_UP
import pandas as pd
from io import BytesIO
from typing import Optional, Dict
import json
from sqlalchemy.orm.attributes import flag_modified
from calendar import monthrange 
from .report import apply_smart_currency
from src.HRAndCRM.models import Employee
from src.common.utils import add_delete_update_company_user_access_check
from src.application_management.models import FinancialTag
from src.common.dependencies import get_current_company_user
from src.schemas.compnay_user import CurrentCompanyUser 
from src.schemas.compnay_user import CurrentCompanyUser
from fastapi.security import OAuth2PasswordBearer
from fastapi import Query, Depends, HTTPException, status
from fastapi.encoders import jsonable_encoder 
#from .helper import dashboard_metrics
import io
import ast
import operator as op

#models
from .model import (GeneralLedgerMaster, GeneralLedger,
                    IntegrationHistory,FinancialWidget,
                    MetricDefinition,DerivedMetric, 
                    FPNAYearlyReport , FPNAYearlyReport1)
from .schemas import KPIWidgetUpdateSchema, PayrollReconciliationResponse,ProductOut,EntityOut,KeyMetrics,ProjectCostingResponse,ProjectActual,ProductQuarterly, KeyMetrics
from .schemas import ProjectOut1,RegionOut1,Project_Costing_KeyMetrics,ProductOut1,EntityOut1,DepartmentOut1,ProductOut,EntityOut,KeyMetrics,ProjectCostingResponse,ProjectActual,ProductQuarterly, KeyMetrics
from .schemas import CurrencyFilter,EntityOut,KeyMetrics,ProjectCostingResponse,ProjectActual,ProductQuarterly, KeyMetrics
from .schemas import VendorOut,ProjectOut1,RegionOut1,Project_Costing_KeyMetrics,ProductOut1,EntityOut1,DepartmentOut1,ProductOut,EntityOut,KeyMetrics,ProjectCostingResponse,ProjectActual,ProductQuarterly, KeyMetrics
from .schemas import EntityOut,KeyMetrics,ProjectCostingResponse,ProjectActual,ProductQuarterly, KeyMetrics , DashboardTimeFilter

from .schemas import CRMFunnelResponse,FunnelStage,Funnel_KeyMetrics,PayrollReconciliationResponse1,PayrollReconciliationResponse,EntityOut,KeyMetrics,ProjectCostingResponse,ProjectActual,ProductQuarterly, KeyMetrics
from .schemas import CustomerSummaryResponse,CustomerProfitabilityOut,CustomerKeyMetrics,RevenueMetricsResponse,RevenueKeyMetrics,FiltersResponse,SelectedFilters,ProductOut,EntityOut,KeyMetrics,ProjectCostingResponse,ProjectActual,ProductQuarterly, KeyMetrics
from collections import defaultdict
from sqlalchemy import and_, extract, inspect, or_,tuple_,func 
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.exc import IntegrityError
import uuid
import re
import calendar
import numpy as np
from datetime import date, datetime
from enum import Enum
import numpy as np
from datetime import date, datetime,timedelta
#db
from src.core.database import get_db,get_db_on_the_fly,get_async_db
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select,delete
from uuid import UUID
import math
from src.constants import DataIntegrationSourceEnum, PeriodType, KPIWidgetNameEnum, ChartTypeEnum
from src.financial.service import AddGLData, FinancialManagement, FinancialReport,FinancialWidgetManagement, FirstFinancialReport, KPIFinancialWidgetManagement
from fastapi.responses import StreamingResponse

#schema
from .schemas import (
    GeneralLedgerMasterDataRequest,
    GlMasterData,
    GeneralLedgerEntryUpdateRequest,
    GeneralLedgerEntryUpdateRequest,
    OverviewWidgetSchema, KPIWidgetSchema,MetricDefinitionCreate,DerivedMetricCreate,
    ReportParams , TimeFilterEnum)

#utils
from .utils import safe_eval_formula

financial_router = APIRouter(prefix="/company/financial", tags=["financial module"])
   


   

# @financial_router.get("/tags")
# async def get_tags(db: AsyncSession = Depends(get_db),currentCompanyUser: CurrentCompanyUser = Depends(get_current_company_user)):

#     try:
        
#         has_permission = add_delete_update_company_user_access_check(currentCompanyUser,action="add")

#         if not has_permission:

#             raise HTTPException(
#                 status_code=status.HTTP_403_FORBIDDEN,
#                 detail="You do not have permission to add a user."
#             )
#         result = await db.execute(select(FinancialTag))
#         tags = result.scalars().all()
#         return [{"id": tag.id, "tag_name": tag.tag_name} for tag in tags]
#     except Exception as e:
#         raise e 

@financial_router.post("/gl-master", status_code=201)
async def upload_gl_master_excel(
    file: UploadFile = File(...),
    column_mapping: str = File(...),
    db: AsyncSession = Depends(get_db_on_the_fly)
):
    # 1. Validate file extension
    if not file.filename or not file.filename.endswith(".xlsx"):
        raise HTTPException(status_code=400, detail="Only .xlsx files are supported")

    try:
        column_mapping = json.loads(column_mapping)
    except json.JSONDecodeError:
        raise HTTPException(status_code=400, detail="Invalid JSON for column_mapping.")

    # 2. Validate column_mapping is provided and is dict
    if not column_mapping or not isinstance(column_mapping, dict):
        print(column_mapping)
        raise HTTPException(status_code=400, detail="Missing or invalid column_mapping")

    # 3. Validate that provided Excel column names exist in the uploaded file
    expected_excel_columns = list(column_mapping.keys())
    print(expected_excel_columns)

    # 4. Read the uploaded Excel file
    content = await file.read()
    try:
        df = pd.read_excel(BytesIO(content))
    except Exception as e:
        raise HTTPException(status_code=400, detail=f"Error reading Excel file: {str(e)}")

    # 5. Check for missing Excel columnsg
    missing_cols = [col for col in expected_excel_columns if col not in df.columns]
    print(missing_cols,df.columns)
    if missing_cols:
        raise HTTPException(status_code=400, detail=f"Missing columns in Excel: {missing_cols}")

    # 6. Rename Excel columns to match DB fields
    df.rename(columns={k:v for k, v in column_mapping.items()}, inplace=True)
    print(df.columns)

    # 7. Prepare optional fields
    db_columns = set(column_mapping.keys())
    required_columns = {"general_ledger_code", "account_name", "category"}
    optional_columns = db_columns - required_columns
    
    
    for col in optional_columns:
        if col not in df.columns:
            df[col] = None

    for col in df.columns:
        if df[col].dtype == "object":
            print(df[col].dtype,df)
            df[col] = df[col].astype(str).str.strip().str.lower()
            if col in optional_columns:
                df.loc[df[col] == "", col] = None
    df["account_name"] = df["account_name"].str.lower()
    df["category"]= df["category"].str.lower()
    print(df.columns)
    # 8. Convert rows to GlMasterData entries
    entries: List[GlMasterData] = []
    for _, row in df.iterrows():
        try:
            tags_raw = row.get("tags")
            if isinstance(tags_raw, str):
                tags = [tag.strip() for tag in tags_raw.split(",") if tag.strip()]
            elif isinstance(tags_raw, list):
                tags = tags_raw
            elif pd.isna(tags_raw) or tags_raw is None:
                tags = []
            else:
                raise ValueError("Invalid tags format")

            entry = GlMasterData(
                general_ledger_code=int(row["general_ledger_code"]),
                account_name=row["account_name"],
                category=row["category"],
                tags=tags
            )
            
            entries.append(entry)

        except Exception as e:
            raise HTTPException(status_code=400, detail=f"Invalid data in row {row.to_dict()}: {str(e)}")

    # 9. Wrap and save data
    print(entries)
    gl_data = GeneralLedgerMasterDataRequest(data=entries)

    result = await FinancialManagement.add_gl_master_account(gl_data, db)
    return result


@financial_router.get("/gl-master/codes", status_code=200)
async def get_all_master_gl_codes(db: AsyncSession = Depends(get_db_on_the_fly)):
    print("--------------##")
    
    result  =  await FinancialManagement.get_all_master_gl_codes(db)

    print("--------------##")


    return result


@financial_router.patch('/gl-master/{general_ledger_code}')
async def update_gl_master_account(requestBody:GlMasterData,general_ledger_code:int,db:AsyncSession = Depends(get_db_on_the_fly)):

    result = await FinancialManagement.update_gl_master_account(general_ledger_code=general_ledger_code,update_data=requestBody,db=db)
    
    return result

@financial_router.delete("/gl-master/{gl_code}", status_code=200)
async def delete_master_gl_account(
    gl_code: int,
    db: AsyncSession = Depends(get_db_on_the_fly)):

    result =  await FinancialManagement.delete_master_gl_account(gl_code=gl_code,db=db)

    return result

@financial_router.post("/upload-gl/")
async def file_gl_upload(
    file: UploadFile = File(...),
    column_mapping: str = Form(...),
    db: AsyncSession = Depends(get_db_on_the_fly),
    user: CurrentCompanyUser = Depends(get_current_company_user),
):
    if not file.filename.endswith(".xlsx"):
        raise HTTPException(status_code=400, detail="Only .xlsx files allowed.")

    mapping = json.loads(column_mapping)
    contents = await file.read()

    try:
        df = pd.read_excel(BytesIO(contents), engine="calamine")

        # Validation + DB saving
        # df = AddGLData.validate_and_rename_columns(df, mapping)
        # AddGLData.validate_row_nulls(df)
        # await AddGLData.validate_master_gl_rows(df, db)

        result = await AddGLData.save_gl_data_to_db(df, mapping, user.user_id, db)

        return result
    except HTTPException as e:
        raise e
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Unexpected error: {str(e)}")
@financial_router.get("/financial_year", response_model=List[int])
async def get_years(db: AsyncSession = Depends(get_db_on_the_fly)):
    # stmt = select(extract('year', GeneralLedger.date).label("year")).distinct().order_by("year")
    # result = await db.execute(stmt)
    # years = [int(row.year) for row in result.fetchall() if row.year is not None]
    return  [2023,2024,2025] 
    stmt = select(extract('year', GeneralLedger.date).label("year")).distinct().order_by("year")
    result = await db.execute(stmt)
    years = [int(row.year) for row in result.fetchall() if row.year is not None]
    return years    
@financial_router.get("/currency" , response_model = List[str])
async def currency():
    return ["USD" ,"INR"]

@financial_router.get("/gl-entries")
async def get_gl_entries(
    page: int = Query(1, ge=1, description="Page number"),
    limit: int = Query(100, ge=1, le=500, description="Items per page (max 500)"),

    # Filters
    date: Optional[datetime] = Query(None, description="Exact date"),
    date_from: Optional[datetime] = Query(None, description="Start of date range"),
    date_to: Optional[datetime] = Query(None, description="End of date range"),
    general_ledger_code: Optional[float] = Query(None),
    account_name: Optional[str] = Query(None),
    debit_min: Optional[float] = Query(None),
    debit_max: Optional[float] = Query(None),
    credit_min: Optional[float] = Query(None),
    credit_max: Optional[float] = Query(None),
    entry_by: Optional[str] = Query(None),
    entry_system_source: Optional[str] = Query(None),

    db: AsyncSession = Depends(get_db_on_the_fly)
        ):

    stmt = select(GeneralLedger)

    # Apply filters dynamically
    filters = []

    if date:
        filters.append(GeneralLedger.date == date)
    if date_from:
        filters.append(GeneralLedger.date >= date_from)
    if date_to:
        filters.append(GeneralLedger.date <= date_to)
    if general_ledger_code is not None:
        filters.append(GeneralLedger.general_ledger_code == general_ledger_code)
    if account_name:
        filters.append(GeneralLedger.account_name.ilike(f"%{account_name}%"))
    if debit_min is not None:
        filters.append(GeneralLedger.debit >= debit_min)
    if debit_max is not None:
        filters.append(GeneralLedger.debit <= debit_max)
    if credit_min is not None:
        filters.append(GeneralLedger.credit >= credit_min)
    if credit_max is not None:
        filters.append(GeneralLedger.credit <= credit_max)
    if entry_by:
        filters.append(GeneralLedger.entry_by.ilike(f"%{entry_by}%"))
    if entry_system_source:
        filters.append(GeneralLedger.entry_system_source.ilike(f"%{entry_system_source}%"))

    if filters:
        stmt = stmt.where(*filters)

    result = await FinancialManagement.get_gl_entries(stmt=stmt,page=page,limit=limit,db=db)

    return  result
    

#it will delete all entries from db of the give code and date range from and To
@financial_router.delete("/gl-entries/{gl_code}")
async def delete_gl_entries_by_code_and_date(
    gl_code: int,
    date_from: datetime = Query(..., description="Start date in YYYY-MM-DD format (mandatory)"),
    date_to: datetime = Query(..., description="Start date in YYYY-MM-DD format (mandatory)"),
    db: AsyncSession = Depends(get_db_on_the_fly)
        ):  
    """
    Delete general ledger (GL) entries for a given GL code within a specific date range.
    YYYY-MM-DD format example date `2023-01-24`
    This endpoint deletes all GL entries that match the specified `gl_code` and fall within 
    the inclusive date range of `date_from` to `date_to`.

    :param gl_code: General Ledger code for which entries should be deleted.
    :param date_from: Start of the date range (inclusive).
    :param date_to: End of the date range (inclusive).
    :param db: Database session dependency.
    :return: The result of the deletion operation (e.g., number of rows deleted or status message).
    
    :raises HTTPException: If `date_from` is after `date_to`.
    """
    # Validate date range
    if date_from > date_to:
        raise HTTPException(status_code=400, detail="`date_from` must be before or equal to `date_to`")
    
    result = await FinancialManagement.delete_gl_entries_by_code_and_date(gl_code=gl_code,date_from=date_from,date_to=date_to,db=db)
    
    return result


@financial_router.delete("/gl-entries/item/{entry_id}")
async def delete_gl_entry_by_id(
    entry_id: UUID,
    db: AsyncSession = Depends(get_db_on_the_fly)
            ):
    result = await FinancialManagement.delete_gl_entry_by_id(entry_id=entry_id,db=db)

    return result

@financial_router.patch("/gl-entries/item/{entry_id}")
async def update_gl_entry(
    entry_id: UUID,
    update_data: GeneralLedgerEntryUpdateRequest,
    db: AsyncSession = Depends(get_db_on_the_fly)
):
    update_dict = update_data.model_dump(exclude_unset=True)

    if not update_dict:
        return {
            "warning": "No data provided to update. Please provide at least one field."
        }
    result = await FinancialManagement.update_gl_entry(entry_id=entry_id,update_dict=update_dict,db=db)

    return result

#integration
@financial_router.get("/integration-history")
async def get_integration_history(
    page: int = Query(1, ge=1),
    limit: int = Query(100, ge=1),
    db: AsyncSession = Depends(get_db_on_the_fly)):
    
    result = await FinancialManagement.get_integration_history(page=page,limit=limit,db=db)

    return result


@financial_router.get("/integration-history/{integration_id}/gl-entries")
async def get_gl_entries_by_integration_id(
    integration_id: str,
    page: int = Query(1, ge=1),
    limit: int = Query(100, ge=1),
    date: Optional[datetime] = Query(None, description="Exact date"),
    date_from: Optional[datetime] = Query(None, description="Start of date range"),
    date_to: Optional[datetime] = Query(None, description="End of date range"),
    general_ledger_code: Optional[float] = Query(None),
    account_name: Optional[str] = Query(None),
    debit_min: Optional[float] = Query(None),
    debit_max: Optional[float] = Query(None),
    credit_min: Optional[float] = Query(None),
    credit_max: Optional[float] = Query(None),
    db: AsyncSession = Depends(get_db_on_the_fly)):
    
    result = await FinancialManagement.get_gl_entries_by_integration_id(
        integration_id=integration_id,
        page=page,
        limit=limit,
        date=date,
        date_from=date_from,
        date_to=date_to,
        general_ledger_code=general_ledger_code,
        account_name=account_name,
        debit_min=debit_min,
        debit_max=debit_max,
        credit_min=credit_min,
        credit_max=credit_max,
        db=db
    )
    return result

@financial_router.delete("/integration-history/{integration_id}")
async def delete_integration_history(integration_id: str, db: AsyncSession = Depends(get_db_on_the_fly)):

    result = await FinancialManagement.delete_integration_history(integration_id=integration_id,db=db)

    return result 

@financial_router.get("/get_metric_names")
async def metric_names(db:AsyncSession = Depends(get_db)):
    stmt = text(f"""select account_code , account_name from fpna_landing.finance_chart_of_accounts""")
    result = await db.execute(stmt)
    rows = result.fetchall()
    data =[dict(row._mapping) for row in rows]
    df = pd.DataFrame(data)
    final_result = df.to_dict(orient = "records")
    return final_result


@financial_router.post("/metric/add/{year}/{currency}")
async def add_metric(
    year: int,
    currency: str,
    metric_name: str | None = None,
    account_name: str | None = None,
    db: AsyncSession = Depends(get_db),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
):

    if not account_name:
        query = text("""
            SELECT account_id, account_name
            FROM fpna_landing.finance_chart_of_accounts
            ORDER BY account_name
        """)
        result = await db.execute(query)
        rows = result.fetchall()

        return {
            "dropdown_list": [
                {
                    "account_name": row.account_name,
                    "account_id": row.account_id
                }
                for row in rows
            ]
        }

    stmt = select(FPNAYearlyReport1).where(FPNAYearlyReport1.year == year)
    result = await db.execute(stmt)
    report = result.scalar_one_or_none()

    if not report:
        return {"error": "Report not found"}

    coa_query = text("""
        SELECT account_id
        FROM fpna_landing.finance_chart_of_accounts
        WHERE account_name = :account_name
        LIMIT 1
    """)

    coa_result = await db.execute(coa_query, {"account_name": account_name})
    coa_row = coa_result.fetchone()

    if not coa_row:
        return {"error": "Invalid account_name selected"}

    gl_code = coa_row.account_id

    existing_metrics = report.metric_trend or []
    position = len(existing_metrics) + 1

    query = text("""
        SELECT period, gl_account_id, amount_debit, amount_credit
        FROM fpna_landing.finance_general_ledger_transactions
        WHERE EXTRACT(YEAR FROM TO_DATE(period, 'YYYY-MM')) = :year
    """)

    result = await db.execute(query, {"year": year})
    metric_rows = result.fetchall()
    metric_data = [dict(row._mapping) for row in metric_rows]

    metric_df = pd.DataFrame(metric_data)

    result_df = (
        metric_df[metric_df["gl_account_id"] == gl_code]
        .assign(Total_amount=lambda df: df["amount_credit"] - df["amount_debit"])
        .groupby("period", as_index=False)["Total_amount"]
        .sum()
    )

    data = result_df.to_dict(orient="records")
    data = apply_smart_currency(data, currency, currency_filter, ["Total_amount"])

    final_result = list(data)

    final_result.append({
        "chart_type": "line_chart",
        "position": position,
        "enableordisable": True
    })

    existing_metrics.append({metric_name: final_result})

    report.metric_trend = existing_metrics
    flag_modified(report, "metric_trend")

    await db.commit()
    await db.refresh(report)

    return {"message": "Metric added successfully"}



@financial_router.get("/metric/all/{year}")
async def get_all_metrics(
    year: int,
    db: AsyncSession = Depends(get_db)
):
    stmt = select(FPNAYearlyReport1).where(FPNAYearlyReport1.year == year)
    result = await db.execute(stmt)
    report = result.scalar_one_or_none()

    if not report:
        return {"error": "Report not found"}

    existing_metrics = report.metric_trend or []

    return {
        "year": year,
        "total_metrics": len(existing_metrics),
        "metrics": existing_metrics
    }


@financial_router.delete("/metric/delete/{year}/{metric_name}")
async def delete_metric(
    year: int,
    metric_name: str,
    db: AsyncSession = Depends(get_db)
):
    stmt = select(FPNAYearlyReport1).where(FPNAYearlyReport1.year == year)
    result = await db.execute(stmt)
    report = result.scalar_one_or_none()

    if not report:
        return {"error": "Report not found"}

    existing_metrics = report.metric_trend or []

    updated_metrics = []
    found = False

    for metric in existing_metrics:
        key = next(iter(metric.keys()))
        if key.lower() == metric_name.lower():
            found = True
            continue
        updated_metrics.append(metric)

    if not found:
        return {"error": "Metric not found"}

    report.metric_trend = updated_metrics
    flag_modified(report, "metric_trend")

    await db.commit()
    await db.refresh(report)

    return {"message": "Metric deleted successfully"}
import ast
import operator as op

ALLOWED_OPS = {
    ast.Add:  op.add,
    ast.Sub:  op.sub,
    ast.Mult: op.mul,
    ast.Div:  op.truediv,
    ast.Pow:  op.pow,
    ast.USub: op.neg,
    ast.UAdd: op.pos,
}


def safe_eval(expr: str, variables: dict):
    def _eval(node):
        if isinstance(node, ast.Num):
            return node.n
        if isinstance(node, ast.Constant):
            if isinstance(node.value, (int, float)):
                return node.value
            raise ValueError("Invalid constant in formula")
        if isinstance(node, ast.Name):
            return variables.get(node.id, 0)
        if isinstance(node, ast.BinOp):
            if type(node.op) not in ALLOWED_OPS:
                raise ValueError("Operator not allowed")
            left  = _eval(node.left)
            right = _eval(node.right)
            if isinstance(node.op, ast.Div) and right == 0:
                raise ValueError("Division by zero")
            return ALLOWED_OPS[type(node.op)](left, right)
        if isinstance(node, ast.UnaryOp):
            if type(node.op) not in ALLOWED_OPS:
                raise ValueError("Unary operator not allowed")
            return ALLOWED_OPS[type(node.op)](_eval(node.operand))
        raise ValueError("Invalid formula")

    tree = ast.parse(expr, mode="eval")
    return _eval(tree.body)


@financial_router.post("/derived-metric/add/{year}/{currency}")
async def add_derived_metric_to_report(
    year: int,
    currency: str,
    metric_name: str | None = None,
    formula: str | None = None,
    db: AsyncSession = Depends(get_db),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
):
    stmt = select(FPNAYearlyReport1).where(FPNAYearlyReport1.year == year)
    result = await db.execute(stmt)
    report = result.scalar_one_or_none()

    if not report:
        return {"error": "Report not found"}

    existing_metrics = report.metric_trend or []

    if not formula:
        metric_names = []
        for m in existing_metrics:
            for key in m.keys():
                metric_names.append(key)

        return {
            "dropdown_list": sorted(metric_names)
        }

    existing_metric_names = set()
    for m in existing_metrics:
        for key in m.keys():
            existing_metric_names.add(key)

    try:
        tree = ast.parse(formula, mode="eval")
    except SyntaxError as e:
        raise HTTPException(status_code=400, detail=f"Invalid formula syntax: {e}")

    formula_vars = {node.id for node in ast.walk(tree) if isinstance(node, ast.Name)}
    missing = formula_vars - existing_metric_names
    if missing:
        raise HTTPException(
            status_code=400,
            detail=f"Formula references metrics not in report: {sorted(missing)}"
        )

    period_values: dict[str, dict[str, float]] = {}

    for metric_entry in existing_metrics:
        for key, records in metric_entry.items():
            if not isinstance(records, list):
                continue
            for record in records:
                if not isinstance(record, dict):
                    continue
                period = record.get("period")
                amount = record.get("Total_amount")
                if period is None or amount is None:
                    continue
                period_values.setdefault(period, {})[key] = float(amount)

    if not period_values:
        raise HTTPException(
            status_code=422,
            detail="No period data found. Add at least one regular metric first."
        )

    derived_data = []
    for period, variables in sorted(period_values.items()):
        try:
            value = safe_eval(formula, variables)
        except Exception as e:
            raise HTTPException(
                status_code=400,
                detail=f"Formula evaluation failed for period '{period}': {e}"
            )
        derived_data.append({
            "period": period,
            "Total_amount": float(value) if value is not None else None,
        })

    derived_data = apply_smart_currency(derived_data, currency, currency_filter, ["Total_amount"])
    final_result = list(derived_data)

    position = len(existing_metrics) + 1
    final_result.append({
        "chart_type": "line_chart",
        "position": position,
        "enableordisable": True,
    })

    existing_metrics.append({metric_name: final_result})

    report.metric_trend = existing_metrics
    flag_modified(report, "metric_trend")

    await db.commit()
    await db.refresh(report)

    return {"message": f"Derived metric '{metric_name}' added successfully"}


@financial_router.get("/derived-metric/all/{year}")
async def get_all_derived_metrics(
    year: int,
    db: AsyncSession = Depends(get_db)
):
    stmt = select(FPNAYearlyReport1).where(FPNAYearlyReport1.year == year)
    result = await db.execute(stmt)
    report = result.scalar_one_or_none()

    if not report:
        return {"error": "Report not found"}

    existing_metrics = report.metric_trend or []

    # Filter only derived metrics by checking formula stored in metadata
    # Since we don't store formula separately, we return all metrics as-is
    # To distinguish, you can add "is_derived": True in final_result when saving

    return {
        "year": year,
        "total_metrics": len(existing_metrics),
        "metrics": existing_metrics
    }



@financial_router.get("/metrics/calculate")
async def calculate_metrics(year: int = Query(datetime.now().year), db: AsyncSession = Depends(get_db_on_the_fly)):
    metric_result = await db.execute(select(MetricDefinition))
    metric_definitions = metric_result.scalars().all()

    derived_result = await db.execute(select(DerivedMetric))
    derived_metrics = derived_result.scalars().all()

    gl_master_result = await db.execute(select(GeneralLedgerMaster))
    master_map = {
        float(m.general_ledger_code): {
            "tags": set((m.tags or [])),
            "category": (m.category or "").lower()
        }
        for m in gl_master_result.scalars().all()
    }

    gl_result = await db.execute(select(GeneralLedger))
    entries = [e for e in gl_result.scalars().all() if e.date.year == year]

    metric_data = {
        period: {metric.name: defaultdict(float) for metric in metric_definitions}
        for period in ["monthly", "quarterly"]
    }
    metric_data["yearly"] = {metric.name: 0.0 for metric in metric_definitions}

    for entry in entries:
        code = float(entry.general_ledger_code)
        meta = master_map.get(code)
        if not meta:
            continue
        tags = meta["tags"]
        category = meta["category"]

        for metric in metric_definitions:
            if code in (metric.gl_codes or []) or tags.intersection(set(metric.tags or [])):
                if hasattr(entry, metric.amount_field):
                    amount = float(getattr(entry, metric.amount_field))
                else:
                    continue

                mkey = entry.date.strftime("%Y-%m")
                qkey = f"Q{((entry.date.month - 1) // 3 + 1)}"
                metric_data["monthly"][metric.name][mkey] += amount
                metric_data["quarterly"][metric.name][qkey] += amount
                metric_data["yearly"][metric.name] += amount

    results = {}
    for period in ["monthly", "quarterly", "yearly"]:
        period_data = {}
        if period == "yearly":
            context = metric_data["yearly"].copy()
            for d in derived_metrics:
                if d.name not in context:
                    context[d.name] = round(safe_eval_formula(d.formula, context), 2)
            period_data = context
        else:
            all_keys = set()
            for m in metric_definitions:
                all_keys.update(metric_data[period][m.name].keys())
            for key in all_keys:
                context = {
                    m.name: metric_data[period][m.name].get(key, 0.0)
                    for m in metric_definitions
                }
                for d in derived_metrics:
                    if d.name not in context:
                        context[d.name] = round(safe_eval_formula(d.formula, context), 2)
                for k, v in context.items():
                    if k not in period_data:
                        period_data[k] = {}
                    period_data[k][key] = v
        results[period] = period_data

    return {
        "year": year,
        "metrics": results
    }













@financial_router.patch("/overview/widgets")
async def Overview_update_widgets(widget: OverviewWidgetSchema, db: AsyncSession = Depends(get_db)):
    """
    Updates an existing widget's chart type or other properties.

    Args:
        widget (WidgetSchema): The updated widget data.

    Returns:
        dict: A success message or error details.
    """
    try:
        print(widget, "widget")
        return await FinancialWidgetManagement.UpdateWidget(widget, db )
    except Exception as e:
        return {"detail": str(e)}
    

@financial_router.get("/widgets")
async def widget( year:int ,actual_vs_budget :Optional[Literal["actual_vs_budget"]] = None , db: AsyncSession = Depends(get_db) ):
    """
    Retrieves all widgets from the database and groups them by category.

    Returns:
        dict: Categorized widget data.
    """
    try:
        return await FinancialWidgetManagement.getwidgets(db , year , actual_vs_budget )
    except Exception as e:
        raise e  # Consider raising HTTPException with status code


import logging
# adjust import path if needed

logger = logging.getLogger(__name__)




def is_tagged(master_tags: set[str], master_category: str, tag: str) -> bool:
    tag = tag.lower()
    return tag in master_tags or master_category == tag


from collections import defaultdict
from sqlalchemy import func, select
from sqlalchemy.orm import aliased

import pandas as pd
from sqlalchemy import func, and_, or_
from collections import defaultdict
from fastapi import APIRouter, Depends
from sqlalchemy.ext.asyncio import AsyncSession



import pandas as pd
from sqlalchemy import func, and_, or_
from collections import defaultdict
from fastapi import APIRouter, Depends
from sqlalchemy.ext.asyncio import AsyncSession


def extract_trend_config_and_data(trend_list):
        """
        Splits mixed trend list into:
        - config dict (position, chart_type, enableordisable)
        - data list (period-based values)
        """
        if not trend_list:
            return {}, []

        config = {}
        data = []

        for item in trend_list:
            # config object
            if isinstance(item, dict) and "chart_type" or "Position" in item:
                config = {
                    "position": item.get("Position") or item.get("position"),
                    "chart_type": item.get("chart_type"),
                    "enableordisable": (
                        item.get("enableordisable")
                        )}       
            # data rows
            elif isinstance(item, dict) and "period" in item:
                data.append(item)

        return config, data 

def remove_disabled_trends(res: dict) -> dict:
    new_res = {}

    for key, value in res.items():
        # Only check trend lists
        if not key.endswith("_trend") or not isinstance(value, list):
            new_res[key] = value
            continue

        # Find enableordisable flag (if any)
        enable_flag = None
        for item in value:
            if isinstance(item, dict) and "enableordisable" in item:
                enable_flag = item["enableordisable"]
                break

        # Keep trend only if flag is not False
        if enable_flag is not False:
            new_res[key] = value

    return new_res


#######################################################################################
#######################################################################################
#######################################################################################
#financial overview
from .report import apply_entity_hierarchy_filter, all_filters_data_display, apply_smart_currency


def round_records(data, keys, decimals=2):
    if not isinstance(data, list):
        return data
    for row in data:
        for k in keys:
            if k in row and row[k] is not None:
                row[k] = round(float(row[k]), decimals)
    return data
 
 
def to_json(df):
    df = df.copy()
    df["period"] = df["period"].astype(str)
    return df.to_dict(orient="records")
 
 
def normalize_period(df, col):
    df[col] = pd.to_datetime(df[col], errors="coerce").dt.to_period("M")
    return df


async def dashboard_metrics(year, currency, currency_filter, entity, time_filter, db, actual_vs_budget):
    import pandas as pd
    import numpy as np
    from sqlalchemy import text
    all_filters_data = await all_filters_data_display(db=db)
    filters = ["period"]
    if entity:
        filters.append("entity_id")
 
    today = pd.Timestamp.today()
    month = today.month
    quarter_start_month = ((month - 1) // 3) * 3 + 1
 
    params = {"year_pattern": f"{year}-%"}
 
    cogs_accounts = ['inventory', 'Professional Services', 'IT & Software', 'Depreciation Expense']
 
    opex_accounts = [
        'Marketing', 'Salaries', 'Rent', 'Travel',
        'Employee Benefits', 'Utilities', 'Payroll Taxes', 'Insurance', 'Other Operating Expense']
 
    budget_query = text("""select period ,gl_account_id, "Type", gl_account_name,entity_id, department_id, amount from fpna_landing.finance_budget_data WHERE period LIKE :year_pattern""")
    budget_res = await db.execute(budget_query, params)
    budget_rows = budget_res.fetchall()
    budget_data = [dict(row._mapping) for row in budget_rows]
    budget_df = pd.DataFrame(budget_data)
    budget_df = budget_df.rename(columns={"Type": "account_type", "gl_account_name": "account_name", "amount": "budget_amount"})
 
    budget_result = (budget_df.groupby(filters, as_index=False).apply(lambda x: pd.Series({
        'budget_revenue': ((x['account_type'] == 'Revenue') * x['budget_amount']).sum(),
        'budget_cogs': (x['account_name'].isin(cogs_accounts) * x['budget_amount']).sum(),
        'budget_opex': (x['account_name'].isin(opex_accounts) * x['budget_amount']).sum()
    })).reset_index())
 
    revenue_query = text("""SELECT
    gl.gl_account_id,
    gl.period,
    gl.entity_id,
    gl.amount_credit,
    gl.amount_debit,
    gl.cost_center_id,
    gl.profit_center_id,
    coa.account_name,
    coa.account_type
FROM fpna_landing.finance_general_ledger_transactions gl
LEFT JOIN fpna_landing.finance_chart_of_accounts coa
    ON gl.gl_account_id = coa.account_id
WHERE coa.account_type IN ('Revenue', 'Expense')
  AND gl.period LIKE :year_pattern;
""")
    rows = (await db.execute(revenue_query, params)).mappings().all()
    df = pd.DataFrame(rows)
 
    result = (
        df.groupby(filters, as_index=False)
        .apply(lambda x: pd.Series({
            'revenue': ((x['account_type'] == 'Revenue') * (x['amount_credit'] - x['amount_debit'])).sum(),
            'cogs_accounts': (x['account_name'].isin(cogs_accounts) * (x['amount_debit'] - x['amount_credit'])).sum(),
            'op_expenditure': (x['account_name'].isin(opex_accounts) * (x['amount_debit'] - x['amount_credit'])).sum()
        })))
 
    result["budget_revenue"] = budget_result["budget_revenue"]
    result["budget_cogs"] = budget_result["budget_cogs"]
    result["budget_opex"] = budget_result["budget_opex"]
 
    actual_vs_budget_revenue = result[filters + ["revenue", "budget_revenue"]]
    actual_vs_budget_revenue = actual_vs_budget_revenue.rename(columns={"revenue": "actual_revenue"})
    actual_vs_budget_revenue = actual_vs_budget_revenue.to_dict(orient="records")
    actual_vs_budget_cogs = result[filters + ["cogs_accounts", "budget_cogs"]]
    actual_vs_budget_cogs = actual_vs_budget_cogs.rename(columns={"cogs_accounts": "actual_cogs"})
    actual_vs_budget_cogs_dict = actual_vs_budget_cogs.to_dict(orient="records")
 
    actual_vs_budget_opex = result[filters + ["op_expenditure", "budget_opex"]]
    actual_vs_budget_opex = actual_vs_budget_opex.rename(columns={"op_expenditure": "actual_operating_expense"})
    actual_vs_budget_opex_dict = actual_vs_budget_opex.to_dict(orient="records")
 
    actual_vs_budget_monthly = [{"revenue": actual_vs_budget_revenue, "cogs": actual_vs_budget_cogs_dict,
                                  "operating_expense": actual_vs_budget_opex_dict}]
 
    actual_vs_budget_monthly = apply_smart_currency(actual_vs_budget_monthly, currency, currency_filter,
                                                    money_fields=["actual_revenue", "budget_revenue", "actual_operating_expense", "budget_opex", "actual_cogs", "budget_cogs"])
 
 
    revenue_monthly = result[filters + ["revenue"]].to_dict(orient="records")
    revenue_monthly = apply_entity_hierarchy_filter(data=revenue_monthly, entity=entity, hierarchy=None, product=None, costcenter=None,
                                                    profitcenter=None, region=None, customer=None, version=None, business_unit=None)
    Total_Revenue = sum(item["revenue"] for item in revenue_monthly)
    revenue_monthly = apply_smart_currency(revenue_monthly, currency, currency_filter, money_fields=["revenue"])
 
    total_expense_monthly = (
        result.assign(total_expense=result["cogs_accounts"] + result["op_expenditure"])
        [filters + ["total_expense"]]
        .to_dict(orient="records"))
 
    Expense = sum(item["total_expense"] for item in total_expense_monthly)
    total_expense_monthly = apply_smart_currency(total_expense_monthly, currency, currency_filter, money_fields=["total_expense"])
 
    gp_monthly = (
        result.assign(gross_profit=result["revenue"] - result["cogs_accounts"])
        [filters + ["gross_profit"]]
        .to_dict(orient="records"))
    Gross_Profit = sum(item["gross_profit"] for item in gp_monthly)
    gp_monthly = apply_smart_currency(gp_monthly, currency, currency_filter, money_fields=["gross_profit"])
 
    np_monthly = (
        result.assign(net_profit=result["revenue"] - (result["cogs_accounts"] + result["op_expenditure"]))
        [filters + ["net_profit"]].to_dict(orient="records"))
    Net_Profit = sum(item["net_profit"] for item in np_monthly)
    np_monthly = apply_smart_currency(np_monthly, currency, currency_filter, money_fields=["net_profit"])
 
    c_month = date.today().month
    quarter_start_month = ((c_month - 1) // 3) * 3 + 1
    qtd_periods = [f"{year}-{m:02d}" for m in range(quarter_start_month, c_month + 1)]
    qtd_df = result[result["period"].isin(qtd_periods)]
 
    qtd_revenue = qtd_df[filters + ["revenue"]].to_dict(orient="records")
    qtd_revenue = apply_smart_currency(qtd_revenue, currency, currency_filter, money_fields=["revenue"])
 
    expense_qtd = (
        qtd_df.assign(total_expense=qtd_df["cogs_accounts"] + qtd_df["op_expenditure"])
        [filters + ["total_expense"]]
        .to_dict(orient="records"))
    expense_qtd = apply_smart_currency(expense_qtd, currency, currency_filter, money_fields=["total_expense"])
 
    gp_qtd = (
        qtd_df.assign(gross_profit=qtd_df["revenue"] - qtd_df["cogs_accounts"])
        [filters + ["gross_profit"]]
        .to_dict(orient="records"))
    gp_qtd = apply_smart_currency(gp_qtd, currency, currency_filter, money_fields=["gross_profit"])
 
    np_qtd = (
        qtd_df.assign(net_profit=qtd_df["revenue"] - (qtd_df["cogs_accounts"] + qtd_df["op_expenditure"]))
        [filters + ["net_profit"]].to_dict(orient="records"))
    np_qtd = apply_smart_currency(np_qtd, currency, currency_filter, money_fields=["net_profit"])
 
    qtd_actual_vs_budget_revenue = qtd_df[filters + ["revenue", "budget_revenue"]].to_dict(orient="records")
    qtd_actual_vs_budget_cogs = qtd_df[filters + ["cogs_accounts", "budget_cogs"]]
    qtd_actual_vs_budget_cogs = qtd_actual_vs_budget_cogs.rename(columns={"cogs_accounts": "actual_cogs"})
    qtd_actual_vs_budget_cods_dict = qtd_actual_vs_budget_cogs.to_dict(orient="records")
    qtd_actual_vs_budget_opex = qtd_df[filters + ["op_expenditure", "budget_opex"]]
    qtd_actual_vs_budget_opex = qtd_actual_vs_budget_opex.rename(columns={"op_expenditure": "operating_expense"})
    qtd_actual_vs_budget_opex_dict = qtd_actual_vs_budget_opex.to_dict(orient="records")
 
    qtd_actual_vs_budget = [{"revenue": qtd_actual_vs_budget_revenue, "cogs": qtd_actual_vs_budget_cods_dict,
                              "operating_expense": qtd_actual_vs_budget_opex_dict}]
    qtd_actual_vs_budget = apply_smart_currency(qtd_actual_vs_budget, currency, currency_filter,
                                                money_fields=["revenue", "budget_revenue", "operating_expense", "budget_opex", "actual_cogs", "budget_cogs"])
 
    result["period"] = pd.to_datetime(result["period"], format="%Y-%m")
 
    ytd_df = result[result["period"].dt.year == year]
 
    ytd_revenue = ytd_df[filters + ["revenue"]].to_dict(orient="records")
    ytd_revenue = apply_smart_currency(ytd_revenue, currency, currency_filter, money_fields=["revenue"])
 
    expense_ytd = (
        ytd_df.assign(total_expense=ytd_df["cogs_accounts"] + ytd_df["op_expenditure"])
        [filters + ["total_expense"]]
        .to_dict(orient="records"))
    expense_ytd = apply_smart_currency(expense_ytd, currency, currency_filter, money_fields=["total_expense"])
 
    gp_ytd = (
        ytd_df.assign(gross_profit=ytd_df["revenue"] - ytd_df["cogs_accounts"])
        [filters + ["gross_profit"]]
        .to_dict(orient="records"))
    gp_ytd = apply_smart_currency(gp_ytd, currency, currency_filter, money_fields=["gross_profit"])
 
    np_ytd = (
        ytd_df.assign(net_profit=ytd_df["revenue"] - (ytd_df["cogs_accounts"] + ytd_df["op_expenditure"]))
        [filters + ["net_profit"]].to_dict(orient="records"))
    np_ytd = apply_smart_currency(np_ytd, currency, currency_filter, money_fields=["net_profit"])
 
    ytd_actual_vs_budget_revenue = ytd_df[filters + ["revenue", "budget_revenue"]].to_dict(orient="records")
    ytd_actual_vs_budget_cogs = ytd_df[filters + ["cogs_accounts", "budget_cogs"]]
    ytd_actual_vs_budget_cogs = ytd_actual_vs_budget_cogs.rename(columns={"cogs_accounts": "actual_cogs"})
    ytd_actual_vs_budget_cogs_dict = ytd_actual_vs_budget_cogs.to_dict(orient="records")
    ytd_actual_vs_budget_opex = ytd_df[filters + ["op_expenditure", "budget_opex"]]
    ytd_actual_vs_budget_opex = ytd_actual_vs_budget_opex.rename(columns={"op_expenditure": "operating_expense"})
    ytd_actual_vs_budget_opex_dict = ytd_actual_vs_budget_opex.to_dict(orient="records")
 
    ytd_actual_vs_budget = [{"revenue": ytd_actual_vs_budget_revenue, "cogs": ytd_actual_vs_budget_cogs_dict,
                              "operating_expense": ytd_actual_vs_budget_opex_dict}]
    ytd_actual_vs_budget = apply_smart_currency(ytd_actual_vs_budget, currency, currency_filter,
                                                money_fields=["revenue", "budget_revenue", "operating_expense", "budget_opex", "actual_cogs", "budget_cogs"])
 
    cashflow_query = text("""
        SELECT gl.period, coa.account_type, gl.amount_credit, gl.amount_debit,gl.entity_id
        FROM fpna_landing.finance_general_ledger_transactions gl
        LEFT JOIN fpna_landing.finance_chart_of_accounts coa
        ON gl.gl_account_id = coa.account_id
        WHERE coa.account_type = 'Equity' AND gl.period LIKE :year_pattern
    """)
    cf = pd.DataFrame((await db.execute(cashflow_query, params)).mappings().all())
    if len(cf) == 0:
        cashflow = 0
        monthly_cf = []
        qtd_cf_res = []
        ytd_cf_res = []
    else:
        cf["period"] = pd.to_datetime(cf["period"]).dt.to_period("M").astype(str)
        cf["amount_credit"] = cf["amount_credit"].astype(float)
        cf["amount_debit"] = cf["amount_debit"].astype(float)
 
        financing_cf = (
            cf.groupby(filters, as_index=False)
            .apply(lambda x: (x["amount_credit"] - x["amount_debit"]).sum())
            .reset_index(name="financing_cash_flow")
        )
 
        cashflow = financing_cf["financing_cash_flow"].sum()
        monthly_cf = financing_cf.to_dict("records")
        qtd_cf_res = financing_cf[financing_cf["period"].isin(qtd_periods)].to_dict("records")
        ytd_cf_res = financing_cf[financing_cf["period"].astype(str).str.startswith(str(year))].to_dict("records")
 
    headcount_query = text("""
        SELECT COUNT(*) AS total_employees
        FROM fpna_landing.hr_employees
        WHERE hire_date >= make_date(:year, 1, 1)
        AND hire_date <  make_date(:year + 1, 1, 1)
        """)
    result_hc = await db.execute(headcount_query, {"year": year})
    count = result_hc.scalar()
 
 
    monthly_data = {
        "Revenue": revenue_monthly,
        "Grossprofit": gp_monthly,
        "expense": total_expense_monthly,
        "netprofit": np_monthly,
        "cashflow": round_records(monthly_cf, ["financing_cash_flow"]) if isinstance(monthly_cf, list) and len(monthly_cf) > 0 else [],
        "actual_vs_budget": actual_vs_budget_monthly
    }
    quarterly_data = {
        "Revenue": qtd_revenue,
        "Grossprofit": gp_qtd,
        "expense": expense_qtd,
        "netprofit": np_qtd,
        "cashflow": round_records(qtd_cf_res, ["financing_cash_flow"]) if isinstance(qtd_cf_res, list) and len(qtd_cf_res) > 0 else [],
        "actual_vs_budget": qtd_actual_vs_budget
    }
    yearly_data = {
        "Revenue": ytd_revenue,
        "Grossprofit": gp_ytd,
        "expense": expense_ytd,
        "netprofit": np_ytd,
        "cashflow": round_records(ytd_cf_res, ["financing_cash_flow"]) if isinstance(ytd_cf_res, list) and len(ytd_cf_res) > 0 else [],
        "actual_vs_budget": ytd_actual_vs_budget
    }
 
    dashboard_metrics_data = {
        "Revenue": round(Total_Revenue, 2),
        "Grossprofit": round(Gross_Profit, 2),
        "NetProfit": round(Net_Profit, 2),
        "Expense": round(Expense, 2),
        "cashflow": round(cashflow, 2),
        "headcount": count
    }
    dashboard_metrics_data = apply_smart_currency(dashboard_metrics_data, currency, currency_filter,
                                                   money_fields=["Revenue", "Grossprofit", "NetProfit", "Expense"])
 
    time_filter_map = {
        DashboardTimeFilter.monthly: monthly_data,
        DashboardTimeFilter.quarterly: quarterly_data,
        DashboardTimeFilter.year_to_date: yearly_data
    }
 
    selected = time_filter_map.get(time_filter, monthly_data)
 
    final_result = apply_entity_hierarchy_filter(data=selected, entity=entity, hierarchy=None, product=None, costcenter=None,
                                                 profitcenter=None, region=None, customer=None, version=None, business_unit=None)
    final_result["keymetrics"] = dashboard_metrics_data
    final_result["entities"] = all_filters_data["entities"]
    return final_result


async def build_actual_vs_forecast(year: int, currency: str, currency_filter, current_user, time_filter, entity, db):
    import asyncio
    import pandas as pd
    from sqlalchemy import text
    from src.forecasting.controller import _build_forecast_dashboard_full_year

    params = {"year_pattern": f"{year}-%"}
    cogs_accounts = ['inventory', 'Professional Services', 'IT & Software', 'Depreciation Expense']
    opex_accounts = [
        'Marketing', 'Salaries', 'Rent', 'Travel',
        'Employee Benefits', 'Utilities', 'Payroll Taxes', 'Insurance', 'Other Operating Expense'
    ]
    filters = ["period"]
    if entity:
        filters.append("entity_id")

    if isinstance(current_user, dict):
        current_user_dict = current_user
    else:
        try:
            current_user_dict = dict(current_user)
        except Exception:
            try:
                current_user_dict = vars(current_user)
            except Exception:
                current_user_dict = {
                    "user_id": getattr(current_user, "user_id", None),
                    "company_id": getattr(current_user, "company_id", None),
                    "company_name": getattr(current_user, "company_name", None),
                    "role": getattr(current_user, "role", None),
                    "department": getattr(current_user, "department", None),
                    "user_type": getattr(current_user, "user_type", None),
                    "plan": getattr(current_user, "plan", None),
                }

    currency_filter_str = currency_filter if isinstance(currency_filter, str) else currency_filter.value

    gl_query = text("""
        SELECT gl.period, gl.entity_id, gl.amount_credit, gl.amount_debit,
               coa.account_name, coa.account_type
        FROM fpna_landing.finance_general_ledger_transactions gl
        LEFT JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
        WHERE coa.account_type IN ('Revenue', 'Expense') AND gl.period LIKE :year_pattern
    """)
    cf_query = text("""
        SELECT gl.period, gl.entity_id, gl.amount_credit, gl.amount_debit
        FROM fpna_landing.finance_general_ledger_transactions gl
        LEFT JOIN fpna_landing.finance_chart_of_accounts coa ON gl.gl_account_id = coa.account_id
        WHERE coa.account_type = 'Equity' AND gl.period LIKE :year_pattern
    """)

    (gl_rows, cf_rows), forecast_full = await asyncio.gather(
        asyncio.gather(
            db.execute(gl_query, params),
            db.execute(cf_query, params),
        ),
        _build_forecast_dashboard_full_year(
            year=year,
            current_user=current_user_dict,
            currency=currency,
            currency_filter=currency_filter_str,
        ),
    )

    df = pd.DataFrame(gl_rows.mappings().all())
    cf = pd.DataFrame(cf_rows.mappings().all())

    result = (
        df.groupby(filters, as_index=False)
        .apply(lambda x: pd.Series({
            'revenue': ((x['account_type'] == 'Revenue') * (x['amount_credit'] - x['amount_debit'])).sum(),
            'cogs_accounts': (x['account_name'].isin(cogs_accounts) * (x['amount_debit'] - x['amount_credit'])).sum(),
            'op_expenditure': (x['account_name'].isin(opex_accounts) * (x['amount_debit'] - x['amount_credit'])).sum()
        }))
    )
    result["gross_profit"] = result["revenue"] - result["cogs_accounts"]
    result["net_profit"] = result["revenue"] - result["cogs_accounts"] - result["op_expenditure"]
    result["total_expense"] = result["cogs_accounts"] + result["op_expenditure"]
    result["period"] = result["period"].apply(lambda p: pd.to_datetime(str(p).strip() + "-01").strftime("%Y-%m"))

    actual_cashflow_map = {}
    if len(cf) > 0:
        cf["period"] = pd.to_datetime(cf["period"]).dt.to_period("M").astype(str)
        cf["amount_credit"] = cf["amount_credit"].astype(float)
        cf["amount_debit"] = cf["amount_debit"].astype(float)
        financing_cf = (
            cf.groupby(filters, as_index=False)
            .apply(lambda x: (x["amount_credit"] - x["amount_debit"]).sum())
            .reset_index(name="financing_cash_flow")
        )
        actual_cashflow_map = {row["period"]: row["financing_cash_flow"] for row in financing_cf.to_dict("records")}

    def nfk(p):
        try:
            return pd.to_datetime(str(p).strip() + "-01").strftime("%Y-%m")
        except Exception:
            return str(p)

    forecast_revenue_map = {nfk(i["period"]): i["revenue"] for i in forecast_full.get("revenue_forecasting", []) if "period" in i}
    forecast_expense_map = {nfk(i["period"]): i["expenses"] for i in forecast_full.get("expense_forecasting", []) if "period" in i}
    forecast_cashflow_map = {nfk(i["period"]): i["cash_flow"] for i in forecast_full.get("cash_flow_projections", []) if "period" in i}

    actual_revenue_list = result[filters + ["revenue"]].rename(columns={"revenue": "actual_revenue"}).to_dict(orient="records")
    actual_cogs_list = result[filters + ["cogs_accounts"]].rename(columns={"cogs_accounts": "actual_cogs"}).to_dict(orient="records")
    actual_opex_list = result[filters + ["op_expenditure"]].rename(columns={"op_expenditure": "actual_operating_expense"}).to_dict(orient="records")
    actual_gp_list = result[filters + ["gross_profit"]].rename(columns={"gross_profit": "actual_gross_profit"}).to_dict(orient="records")
    actual_np_list = result[filters + ["net_profit"]].rename(columns={"net_profit": "actual_net_profit"}).to_dict(orient="records")
    actual_expense_list = result[filters + ["total_expense"]].rename(columns={"total_expense": "actual_expense"}).to_dict(orient="records")

    for item in actual_revenue_list:
        item["forecast_revenue"] = round(float(forecast_revenue_map.get(item["period"], 0.0)), 2)
    for item in actual_cogs_list:
        item["forecast_cogs"] = round(float(forecast_expense_map.get(item["period"], 0.0)), 2)
    for item in actual_opex_list:
        item["forecast_operating_expense"] = round(float(forecast_expense_map.get(item["period"], 0.0)), 2)
    for item in actual_gp_list:
        p = item["period"]
        item["forecast_gross_profit"] = round(float(forecast_revenue_map.get(p, 0.0)) - float(forecast_expense_map.get(p, 0.0)), 2)
    for item in actual_np_list:
        p = item["period"]
        item["forecast_net_profit"] = round(float(forecast_revenue_map.get(p, 0.0)) - float(forecast_expense_map.get(p, 0.0)), 2)
    for item in actual_expense_list:
        item["forecast_expense"] = round(float(forecast_expense_map.get(item["period"], 0.0)), 2)

    all_cf_periods = sorted(set(list(actual_cashflow_map.keys()) + list(forecast_cashflow_map.keys())))
    cashflow_avf_list = [
        {
            "period": p,
            "actual_cashflow": round(float(actual_cashflow_map.get(p, 0.0)), 2),
            "forecast_cashflow": round(float(forecast_cashflow_map.get(p, 0.0)), 2),
        }
        for p in all_cf_periods
    ]

    c_month = date.today().month
    quarter_start_month = ((c_month - 1) // 3) * 3 + 1
    qtd_periods = set(f"{year}-{m:02d}" for m in range(quarter_start_month, c_month + 1))

    def filter_periods(lst, periods):
        return [i for i in lst if i.get("period") in periods]

    def filter_year(lst, yr):
        return [i for i in lst if str(i.get("period", "")).startswith(str(yr))]

    def build_avf_block(rev, cogs, opex, gp, np_, exp, cf):
        return [{
            "revenue": apply_smart_currency(rev, currency, currency_filter, money_fields=["actual_revenue", "forecast_revenue"]),
            "cogs": apply_smart_currency(cogs, currency, currency_filter, money_fields=["actual_cogs", "forecast_cogs"]),
            "operating_expense": apply_smart_currency(opex, currency, currency_filter, money_fields=["actual_operating_expense", "forecast_operating_expense"]),
            "gross_profit": apply_smart_currency(gp, currency, currency_filter, money_fields=["actual_gross_profit", "forecast_gross_profit"]),
            "net_profit": apply_smart_currency(np_, currency, currency_filter, money_fields=["actual_net_profit", "forecast_net_profit"]),
            "expense": apply_smart_currency(exp, currency, currency_filter, money_fields=["actual_expense", "forecast_expense"]),
            "cashflow": apply_smart_currency(cf, currency, currency_filter, money_fields=["actual_cashflow", "forecast_cashflow"]),
        }]

    actual_vs_forecast_monthly = build_avf_block(
        actual_revenue_list, actual_cogs_list, actual_opex_list,
        actual_gp_list, actual_np_list, actual_expense_list, cashflow_avf_list
    )
    actual_vs_forecast_quarterly = build_avf_block(
        filter_periods(actual_revenue_list, qtd_periods),
        filter_periods(actual_cogs_list, qtd_periods),
        filter_periods(actual_opex_list, qtd_periods),
        filter_periods(actual_gp_list, qtd_periods),
        filter_periods(actual_np_list, qtd_periods),
        filter_periods(actual_expense_list, qtd_periods),
        filter_periods(cashflow_avf_list, qtd_periods),
    )
    actual_vs_forecast_yearly = build_avf_block(
        filter_year(actual_revenue_list, year),
        filter_year(actual_cogs_list, year),
        filter_year(actual_opex_list, year),
        filter_year(actual_gp_list, year),
        filter_year(actual_np_list, year),
        filter_year(actual_expense_list, year),
        filter_year(cashflow_avf_list, year),
    )

    time_filter_map = {
        DashboardTimeFilter.monthly: actual_vs_forecast_monthly,
        DashboardTimeFilter.quarterly: actual_vs_forecast_quarterly,
        DashboardTimeFilter.year_to_date: actual_vs_forecast_yearly,
    }
    return time_filter_map.get(time_filter, actual_vs_forecast_monthly)


@financial_router.get("/financial-overview/{year}/{currency}")
async def financial_overview_year(
    year: int, currency: str, entity: Optional[str] = None,
    time_filter: Optional[DashboardTimeFilter] = None,
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    db: AsyncSession = Depends(get_db),
    actual_vs_budget: Optional[Literal["actual_vs_budget"]] = None,
    actual_vs_forecast: Optional[Literal["actual_vs_forecast"]] = None,
    current_user: Dict = Depends(get_current_company_user)):

    if actual_vs_forecast is not None:
        forecast_result = await build_actual_vs_forecast(
            year=year,
            currency=currency,
            currency_filter=currency_filter,
            current_user=current_user,
            time_filter=time_filter,
            entity=entity,
            db=db
        )
        return forecast_result

    financial_data = await dashboard_metrics(year=year, currency=currency,
                                             currency_filter=currency_filter, entity=entity,
                                             time_filter=time_filter, db=db, actual_vs_budget=actual_vs_budget)
 
    try:
        def normalize_tags(tags):
            flattened = []
            for t in tags or []:
                if isinstance(t, list):
                    flattened.extend(str(x).lower() for x in t)
                else:
                    flattened.append(str(t).lower())
            return set(flattened)
 
        try:
            master_result = await db.execute(select(GeneralLedgerMaster))
            master_map = {
                float(m.general_ledger_code): {
                    "tags": normalize_tags(m.tags),
                    "category": (m.category or "").lower(),
                    "account_name": (m.account_name or "").lower(),
                }
                for m in master_result.scalars().all()
            }
        except Exception as e:
            logger.error(f"Failed to load General Ledger Master data: {str(e)}")
            raise ValueError("Failed to load General Ledger Master data")
 
        def is_tagged(tags, category, check_list):
            check_set = set([c.lower() for c in check_list])
            if category.lower() in check_set:
                return True
            if tags.intersection(check_set):
                return True
            return False
 
        try:
            gl_result = await db.execute(
                select(GeneralLedger).where(func.extract("year", GeneralLedger.date) == year)
            )
            gl_entries = gl_result.scalars().all()
        except Exception as e:
            logger.error(f"Failed to fetch GL entries for year {year}: {str(e)}")
            raise ValueError(f"Failed to fetch GL entries for year {year}")
 
        financials = {
            "revenue": 0.0,
            "cogs": 0.0,
            "expenses": 0.0,
            "depreciation": 0.0,
            "ar_increase": 0.0,
            "ap_increase": 0.0,
            "inventory_change": 0.0,
            "prepaid_expenses": 0.0,
            "opening_cash": 0.0,
        }
 
        trends = {
            "rev_trend": defaultdict(float),
            "exp_trend": defaultdict(float),
            "exp_breakdown": defaultdict(float),
            "profit_analysis": defaultdict(lambda: {"gross": 0.0, "net": 0.0}),
            "headcount_trend": defaultdict(set),
            "headcount_quarterly": defaultdict(set),
        }
 
        try:
            prev_year_result = await db.execute(
                select(GeneralLedger)
                .where(
                    (func.extract("year", GeneralLedger.date) == year - 1)
                    & (GeneralLedger.account_name == "cash")
                )
                .order_by(GeneralLedger.date.desc())
                .limit(1)
            )
            prev_year_cash = prev_year_result.scalars().first()
            if prev_year_cash:
                financials["opening_cash"] = float(prev_year_cash.balance or 0)
        except Exception as e:
            logger.warning(f"Could not fetch previous year's cash balance: {str(e)}")
 
        total_headcount_set = set()
 
        for e in gl_entries:
            try:
                master = master_map.get(float(e.general_ledger_code), {})
                tags = master.get("tags", set())
                category = master.get("category", "")
                account_name = master.get("account_name", "")
                debit = float(e.debit or 0)
                credit = float(e.credit or 0)
                vendor = ""
                quarter = f"Q{((e.date.month - 1) // 3) + 1}"
                month = e.date.strftime("%b")
 
                extra_data = e.extra_data or "{}"
 
                try:
                    vendor = str(extra_data.get("vendor", "")).strip()
                except json.JSONDecodeError as ex:
                    vendor = ""
                except Exception as ex:
                    vendor = ""
 
                if is_tagged(tags, category, ["income", "revenue"]):
                    financials["revenue"] += credit
                    trends["rev_trend"][month] += credit
                    trends["profit_analysis"][quarter]["gross"] += credit
                    trends["profit_analysis"][quarter]["net"] += credit
 
                elif is_tagged(tags, category, ["cost_of_sales", "cogs"]):
                    financials["cogs"] += debit
                    trends["profit_analysis"][quarter]["gross"] -= debit
                    trends["profit_analysis"][quarter]["net"] -= debit
 
                elif category == "expense" or "expense" in tags:
                    if account_name == "depreciation":
                        financials["depreciation"] += debit
                    else:
                        financials["expenses"] += debit
                        trends["exp_breakdown"][account_name] += debit
                        trends["exp_trend"][quarter] += debit
 
                if "receivables" in tags or account_name == "accountsreceivable":
                    financials["ar_increase"] += debit - credit
                elif "payables" in tags or account_name == "accountspayable":
                    financials["ap_increase"] += credit - debit
                elif "inventory" in tags or account_name == "inventory":
                    financials["inventory_change"] += debit - credit
                elif "prepaid" in tags or account_name == "prepaidexpenses":
                    financials["prepaid_expenses"] += debit - credit
 
                is_employee = (
                    (account_name == "salaries and wages" and "payroll" in tags) or
                    (account_name == "professional services" and "consulting" in tags)
                )
 
                if is_employee and vendor:
                    total_headcount_set.add(vendor)
                    trends["headcount_trend"][month].add(vendor)
                    trends["headcount_quarterly"][quarter].add(vendor)
 
            except Exception as e:
                logger.error(f"Error processing GL entry : {str(e)}")
                continue
 
        gross_profit = financials["revenue"] - financials["cogs"]
        total_expenses = financials["expenses"] + financials["depreciation"]
        net_profit = gross_profit - financials["expenses"] - financials["depreciation"]
 
        margins = {
            "gross_margin_pct": (gross_profit / financials["revenue"] * 100) if financials["revenue"] else 0.0,
            "net_margin_pct": (net_profit / financials["revenue"] * 100) if financials["revenue"] else 0.0,
            "expense_ratio": (total_expenses / financials["revenue"] * 100) if financials["revenue"] else 0.0,
        }
 
        monthly_headcount = {k: len(v) for k, v in trends["headcount_trend"].items()}
        quarterly_headcount = {k: len(v) for k, v in trends["headcount_quarterly"].items()}
        total_headcount = len(total_headcount_set)
 
        res = {
            "year": year,
            "revenue_trend": financial_data["Revenue"],
            "expense_trend": financial_data["expense"],
            "net_profit_trend": financial_data["netprofit"],
            "gross_profit_trend": financial_data["Grossprofit"],
            "cashflow_trend": [],
            "actual_vs_budget_trend": financial_data["actual_vs_budget"]
        }
 
        res["keymetrics"] = financial_data["keymetrics"]
        res["entities"] = financial_data["entities"]
 
        stmt = select(FPNAYearlyReport1).where(FPNAYearlyReport1.year == res["year"])
        db_result = await db.execute(stmt)
        report = db_result.scalar_one_or_none()
 
        if report:
            db_trends = report.metric_trend or []
 
            trend_dict = {}
            for item in db_trends:
                for key, value in item.items():
                    trend_dict[key.lower()] = value
 
            trend_mapping = {
                "revenue_trend": res.get("revenue_trend"),
                "expense_trend": res.get("expense_trend"),
                "gross_profit_trend": res.get("gross_profit_trend"),
                "net_profit_trend": res.get("net_profit_trend"),
                "actual_vs_budget_trend": res.get("actual_vs_budget_trend"),
            }
 
            for key, new_data in trend_mapping.items():
                if new_data is not None and key in trend_dict:
                    db_trend_list = trend_dict.get(key, [])
                    if not db_trend_list:
                        continue
                    db_config = db_trend_list[-1]
                    trend_dict[key] = new_data + [db_config]
 
            if "actual_vs_budget_trend" not in trend_dict or not trend_dict["actual_vs_budget_trend"]:
                trend_dict["actual_vs_budget_trend"] = res.get("actual_vs_budget_trend", [])
 
            final_result = remove_disabled_trends(trend_dict)
            final_result["keymetrics"] = financial_data["keymetrics"]
            final_result["entities"] = financial_data["entities"]
 
            if actual_vs_budget is not None:
                return final_result.get("actual_vs_budget_trend", res.get("actual_vs_budget_trend", []))
            else:
                return final_result
 
        else:
            res["revenue_trend"].append({"chart_type": "bar_chart", "position": 1, "enableordisable": True})
            res["expense_trend"].append({"chart_type": "line_chart", "position": 2, "enableordisable": True})
            res["net_profit_trend"].append({"chart_type": "pie_chart", "position": 3, "enableordisable": True})
            res["gross_profit_trend"].append({"chart_type": "Doughnut_chart", "position": 4, "enableordisable": True})
            res["cashflow_trend"].append({"chart_type": "bubble_chart", "position": 5, "enableordisable": True})
            res["actual_vs_budget_trend"].append({"chart_type": "bubble_chart", "position": 6, "enableordisable": True})
 
            res_list = []
            res_list.append({"revenue_trend": res["revenue_trend"]})
            res_list.append({"expense_trend": res["expense_trend"]})
            res_list.append({"net_profit_trend": res["net_profit_trend"]})
            res_list.append({"gross_profit_trend": res["gross_profit_trend"]})
            res_list.append({"actual_vs_budget_trend": res["actual_vs_budget_trend"]})
            res_list.append({"cashflow_trend": res["cashflow_trend"]})
 
            stmt = insert(FPNAYearlyReport1).values(
                year=res["year"],
                metric_trend=res_list).on_conflict_do_update(
                index_elements=["year"],
                set_={
                    "metric_trend": res_list,
                    "updated_at": func.now()
                })
            await db.execute(stmt)
            await db.commit()
 
            if actual_vs_budget is not None:
                return res.get("actual_vs_budget_trend", [])
            else:
                return res
 
    except Exception as e:
        logger.exception(f"Financial overview generation failed: {str(e)}")
        return {"error": "Failed to generate financial overview", "details": str(e)}

#######################################################################################
#######################################################################################
#######################################################################################



@financial_router.get("/analytics/expenses/{year}", response_model=None)
async def get_expense_analytics(
    year: int ,
    db: AsyncSession = Depends(get_db_on_the_fly),
) -> Dict[str, any]:
    try:
        logger.info(f"Fetching expense analytics for year: {year}")

        # Load GeneralLedgerMaster metadata
        master_result = await db.execute(select(GeneralLedgerMaster))
        masters = master_result.scalars().all()
        logger.info(f"Loaded {len(masters)} GeneralLedgerMaster records")

        # Build master_map
        master_map = {}
        for m in masters:
            keys = [
                str(m.general_ledger_code),
                str(int(m.general_ledger_code)) if isinstance(m.general_ledger_code, float) else None,
                m.general_ledger_code
            ]
            for key in filter(None, keys):
                master_map[key] = {
                    "tags": [tag.lower() for tag in (m.tags or [])],
                    "category": (m.category or "").lower().strip(),
                    "account_name": m.account_name or "Unknown"
                }
                break

        logger.debug(f"Master map sample: {dict(list(master_map.items())[:3])}")

        # Fetch GL entries
        start_date = datetime(year, 1, 1)
        end_date = datetime(year + 1, 1, 1)
        query = select(GeneralLedger).where(
            and_(
                GeneralLedger.date >= start_date,
                GeneralLedger.date < end_date
            )
        )
        result = await db.execute(query)
        gl_entries = result.scalars().all()
        logger.info(f"Fetched {len(gl_entries)} GL entries for {year}")

        if not gl_entries:
            return {
                "year": year,
                "message": "No general ledger entries found for the specified year",
                "total_expense": 0,
                "monthly_expenses": {},
                "monthly_category_expenses": {},
                "quarterly_expenses": {},
                "quarterly_category_expenses": {},
                "yearly_category_expenses": {},
                "monthly_contribution_percentages": {},
                "quarterly_contribution_percentages": {},
                "yearly_contribution_percentages": {}
            }

        # Aggregation structures
        monthly_expenses = defaultdict(float)
        monthly_category_expenses = defaultdict(lambda: defaultdict(float))
        quarterly_expenses = defaultdict(float)
        quarterly_category_expenses = defaultdict(lambda: defaultdict(float))
        yearly_category_expenses = defaultdict(float)

        # Filter and process expense entries
        expense_entries = []
        unmatched_codes = set()

        for e in gl_entries:
            possible_keys = [
                str(e.general_ledger_code),
                str(int(e.general_ledger_code)) if isinstance(e.general_ledger_code, float) else None,
                e.general_ledger_code
            ]
            master_data = None
            for key in filter(None, possible_keys):
                if key in master_map:
                    master_data = master_map[key]
                    break
            if not master_data:
                unmatched_codes.add(str(e.general_ledger_code))
                continue
            if master_data["category"] == "expense" or "expense" in master_data["tags"]:
                expense_entries.append((e, master_data))

        if unmatched_codes:
            logger.warning(f"Unmatched GL codes: {unmatched_codes}")

        logger.info(f"Found {len(expense_entries)} expense entries")

        if not expense_entries:
            return {
                "year": year,
                "message": "No expense entries found for the specified year",
                "total_expense": 0,
                "monthly_expenses": {},
                "monthly_category_expenses": {},
                "quarterly_expenses": {},
                "quarterly_category_expenses": {},
                "yearly_category_expenses": {},
                "monthly_contribution_percentages": {},
                "quarterly_contribution_percentages": {},
                "yearly_contribution_percentages": {}
            }

        # Populate aggregation structures
        total_expense = 0.0
        for entry, master_data in expense_entries:
            debit = float(entry.debit or 0)
            total_expense += debit
            account_name = master_data["account_name"]
            month_key = entry.date.strftime("%Y-%m")
            quarter_key = f"Q{((entry.date.month - 1) // 3) + 1}"

            monthly_expenses[month_key] += debit
            monthly_category_expenses[month_key][account_name] += debit
            quarterly_expenses[quarter_key] += debit
            quarterly_category_expenses[quarter_key][account_name] += debit
            yearly_category_expenses[account_name] += debit

        def calculate_percentage_distribution(category_dict, total_dict):
            return {
                key: {
                    cat: round((amt / total_dict[key]) * 100, 2) if total_dict[key] else 0.0
                    for cat, amt in cats.items()
                }
                for key, cats in category_dict.items()
            }

        monthly_contribution_percentages = calculate_percentage_distribution(
            monthly_category_expenses, monthly_expenses
        )
        quarterly_contribution_percentages = calculate_percentage_distribution(
            quarterly_category_expenses, quarterly_expenses
        )
        yearly_contribution_percentages = {
            account: round((amount / total_expense) * 100, 2) if total_expense else 0.0
            for account, amount in yearly_category_expenses.items()
        }

        return {
            "year": year,
            "total_expense": total_expense,
            "monthly_expenses": dict(sorted(monthly_expenses.items())),
            "monthly_category_expenses": {
                month: dict(categories) 
                for month, categories in sorted(monthly_category_expenses.items())
            },
            "quarterly_expenses": dict(sorted(quarterly_expenses.items())),
            "quarterly_category_expenses": {
                quarter: dict(categories)
                for quarter, categories in sorted(quarterly_category_expenses.items())
            },
            "yearly_category_expenses": dict(sorted(yearly_category_expenses.items())),
            "monthly_contribution_percentages": monthly_contribution_percentages,
            "quarterly_contribution_percentages": quarterly_contribution_percentages,
            "yearly_contribution_percentages": yearly_contribution_percentages
        }

    except Exception as e:
        logger.exception(f"Error in expense analytics: {str(e)}")
        return {"error": str(e)}


@financial_router.get("/analytics/revenue/{year}")
async def get_revenue_analytics(
    year: int,
    db: AsyncSession = Depends(get_db_on_the_fly)
) -> dict:
    logger.info(f"Fetching revenue analytics for year: {year}")

    # Load GeneralLedgerMaster into memory
    master_result = await db.execute(select(GeneralLedgerMaster))
    master_map = {
        float(m.general_ledger_code): {
            "tags": [tag.lower() for tag in (m.tags or [])],
            "category": (m.category or "").lower(),
            "account_name": m.account_name
        }
        for m in master_result.scalars().all()
    }
    logger.info(f"Loaded {len(master_map)} master metadata records")

    # Load GeneralLedger entries
    result = await db.execute(select(GeneralLedger))
    gl_entries = result.scalars().all()
    logger.info(f"Fetched {len(gl_entries)} general ledger entries from DB")

    def get_master(entry):
        return master_map.get(float(entry.general_ledger_code))

    revenue_entries = [
        e for e in gl_entries
        if (m := get_master(e)) and m["category"] == "revenue" and e.date.year == year
    ]
    logger.info(f"Filtered {len(revenue_entries)} revenue entries for year {year}")

    total_revenue = sum(float(e.credit) for e in revenue_entries)
    logger.info(f"Calculated total revenue: {total_revenue}")

    monthly_revenue = defaultdict(float)
    monthly_category_revenue = defaultdict(lambda: defaultdict(float))
    quarterly_revenue = defaultdict(float)
    quarterly_category_revenue = defaultdict(lambda: defaultdict(float))
    yearly_category_revenue = defaultdict(float)

    for entry in revenue_entries:
        credit = float(entry.credit)
        master_data = get_master(entry)
        account_name = master_data["account_name"] or "Unknown"

        month_key = entry.date.strftime("%Y-%m")
        quarter_key = f"Q{((entry.date.month - 1) // 3) + 1}"

        monthly_revenue[month_key] += credit
        monthly_category_revenue[month_key][account_name] += credit

        quarterly_revenue[quarter_key] += credit
        quarterly_category_revenue[quarter_key][account_name] += credit

        yearly_category_revenue[account_name] += credit

    def calculate_percentage_distribution(category_dict, total_dict):
        return {
            key: {
                cat: round((amt / total_dict[key]) * 100, 2) if total_dict[key] else 0.0
                for cat, amt in cats.items()
            }
            for key, cats in category_dict.items()
        }

    monthly_contribution_percentages = calculate_percentage_distribution(
        monthly_category_revenue, monthly_revenue
    )
    quarterly_contribution_percentages = calculate_percentage_distribution(
        quarterly_category_revenue, quarterly_revenue
    )
    yearly_contribution_percentages = {
        account: round((amount / total_revenue) * 100, 2) if total_revenue else 0.0
        for account, amount in yearly_category_revenue.items()
    }

    logger.info("Finished computing all analytics and contribution percentages")

    return {
        "year": year,
        "total_revenue": total_revenue,
        "monthly_revenue": dict(sorted(monthly_revenue.items())),
        "monthly_category_revenue": {
            month: dict(categories) for month, categories in sorted(monthly_category_revenue.items())
        },
        "monthly_contribution_percentages": monthly_contribution_percentages,
        "quarterly_revenue": dict(sorted(quarterly_revenue.items())),
        "quarterly_category_revenue": {
            quarter: dict(categories) for quarter, categories in sorted(quarterly_category_revenue.items())
        },
        "quarterly_contribution_percentages": quarterly_contribution_percentages,
        "yearly_category_revenue": dict(sorted(yearly_category_revenue.items())),
        "yearly_contribution_percentages": yearly_contribution_percentages
    }


def get_quarter(month: int) -> str:
    return f"Q{((month - 1) // 3) + 1}"

@financial_router.get("/analytics/profit-analysis/{year}")
async def profit_analysis(
    year: int ,
    db: AsyncSession = Depends(get_db_on_the_fly)
) -> dict:
    # Fetch GL Master mapping
    master_result = await db.execute(select(GeneralLedgerMaster))
    master_map = {
        float(m.general_ledger_code): {
            "tags": [tag.lower() for tag in (m.tags or [])],
            "category": (m.category or "").lower()
        }
        for m in master_result.scalars().all()
    }

    # Fetch GL entries filtered by year
    result = await db.execute(select(GeneralLedger))
    gl_entries = [e for e in result.scalars().all() if e.date.year == year]

    monthly_revenue = defaultdict(float)
    monthly_cogs = defaultdict(float)
    monthly_expense = defaultdict(float)

    quarterly_revenue = defaultdict(float)
    quarterly_cogs = defaultdict(float)
    quarterly_expense = defaultdict(float)

    yearly_revenue = 0.0
    yearly_cogs = 0.0
    yearly_expense = 0.0

    for entry in gl_entries:
        code = float(entry.general_ledger_code)
        master = master_map.get(code)
        if not master:
            continue

        tags = set(master["tags"])
        category = master["category"]
        debit = float(entry.debit)
        credit = float(entry.credit)

        month_key = entry.date.strftime("%Y-%m")
        quarter_key = get_quarter(entry.date.month)

        if "revenue" in tags or category == "revenue":
            monthly_revenue[month_key] += credit
            quarterly_revenue[quarter_key] += credit
            yearly_revenue += credit
        elif "cost_of_sales" in tags or category == "cogs":
            monthly_cogs[month_key] += debit
            quarterly_cogs[quarter_key] += debit
            yearly_cogs += debit
        elif "expense" in tags or category == "expense":
            monthly_expense[month_key] += debit
            quarterly_expense[quarter_key] += debit
            yearly_expense += debit

    monthly_gross_profit = {m: round(monthly_revenue[m] - monthly_cogs[m], 2) for m in monthly_revenue}
    monthly_net_profit = {m: round(monthly_gross_profit[m] - monthly_expense.get(m, 0.0), 2) for m in monthly_gross_profit}

    quarterly_gross_profit = {q: round(quarterly_revenue[q] - quarterly_cogs[q], 2) for q in quarterly_revenue}
    quarterly_net_profit = {q: round(quarterly_gross_profit[q] - quarterly_expense.get(q, 0.0), 2) for q in quarterly_gross_profit}

    yearly_gross_profit = round(yearly_revenue - yearly_cogs, 2)
    yearly_net_profit = round(yearly_gross_profit - yearly_expense, 2)

    return {
        "year": year,
        "yearly_summary": {
            "revenue": round(yearly_revenue, 2),
            "cogs": round(yearly_cogs, 2),
            "expenses": round(yearly_expense, 2),
            "gross_profit": yearly_gross_profit,
            "net_profit": yearly_net_profit,
        },
        "monthly_gross_profit": monthly_gross_profit,
        "monthly_net_profit": monthly_net_profit,
        "quarterly_gross_profit": quarterly_gross_profit,
        "quarterly_net_profit": quarterly_net_profit,
    }




@financial_router.get("/analytics/cash-flow/{year}", response_model=None)
async def get_cash_flow_analytics(
    year: int ,
    db: AsyncSession = Depends(get_db_on_the_fly)
) -> Dict[str, any]:
    try:
        logger.info(f"Calculating cash flow for year: {year}")

        # Load master metadata
        master_result = await db.execute(select(GeneralLedgerMaster))
        masters = master_result.scalars().all()
        logger.info(f"Loaded {len(masters)} GeneralLedgerMaster records")

        # Build master map using multiple possible key formats
        master_map = {}
        for m in masters:
            keys = [
                str(m.general_ledger_code),
                str(int(m.general_ledger_code)) if isinstance(m.general_ledger_code, float) else None,
                m.general_ledger_code
            ]
            for key in filter(None, keys):
                master_map[key] = {
                    "tags": [tag.lower() for tag in (m.tags or [])],
                    "category": (m.category or "").lower(),
                    "account_name": m.account_name or "Unknown"
                }
                break

        logger.debug(f"Master map sample: {dict(list(master_map.items())[:3])}")

        # Fetch entries for the year
        start_date = datetime(year, 1, 1)
        end_date = datetime(year + 1, 1, 1)
        gl_result = await db.execute(
            select(GeneralLedger).where(
                and_(GeneralLedger.date >= start_date, GeneralLedger.date < end_date)
            )
        )
        entries = gl_result.scalars().all()
        logger.info(f"Fetched {len(entries)} GL entries for year {year}")

        if not entries:
            return {
                "year": year,
                "message": "No general ledger entries found for the specified year",
                "total_cash_flow": 0,
                "monthly_cash_flow": {},
                "monthly_inflow": {},
                "monthly_outflow": {},
                "monthly_category_cash_flow": {},
                "monthly_contribution_percentages": {},
                "quarterly_cash_flow": {},
                "quarterly_inflow": {},
                "quarterly_outflow": {},
                "quarterly_category_cash_flow": {},
                "quarterly_contribution_percentages": {},
                "yearly_category_cash_flow": {},
                "yearly_inflow": 0,
                "yearly_outflow": 0,
                "yearly_contribution_percentages": {}
            }

        # Initialize aggregators
        monthly_cash_flow = defaultdict(float)
        quarterly_cash_flow = defaultdict(float)
        total_cash_flow = 0.0

        monthly_inflow = defaultdict(float)
        monthly_outflow = defaultdict(float)
        quarterly_inflow = defaultdict(float)
        quarterly_outflow = defaultdict(float)
        yearly_inflow = 0.0
        yearly_outflow = 0.0

        monthly_category_cash_flow = defaultdict(lambda: defaultdict(float))
        quarterly_category_cash_flow = defaultdict(lambda: defaultdict(float))
        yearly_category_cash_flow = defaultdict(float)

        def get_quarter(month: int) -> str:
            return f"Q{((month - 1) // 3) + 1}"

        def safe_percent(part: float, whole: float) -> float:
            return round((part / whole) * 100, 2) if whole else 0.0

        unmatched_codes = set()
        matched_cash_entries = 0

        for entry in entries:
            code_keys = [
                str(entry.general_ledger_code),
                str(int(entry.general_ledger_code)) if isinstance(entry.general_ledger_code, float) else None,
                entry.general_ledger_code
            ]

            master = None
            for key in filter(None, code_keys):
                if key in master_map:
                    master = master_map[key]
                    break

            if not master:
                unmatched_codes.add(str(entry.general_ledger_code))
                continue

            if "cash" not in master["tags"]:
                continue  # Not a cash-related entry

            matched_cash_entries += 1

            credit = float(entry.credit or 0)
            debit = float(entry.debit or 0)
            net_flow = credit - debit

            month_key = entry.date.strftime("%Y-%m")
            quarter_key = get_quarter(entry.date.month)
            account_name = master["account_name"]

            monthly_cash_flow[month_key] += net_flow
            quarterly_cash_flow[quarter_key] += net_flow
            total_cash_flow += net_flow

            if net_flow > 0:
                monthly_inflow[month_key] += net_flow
                quarterly_inflow[quarter_key] += net_flow
                yearly_inflow += net_flow
            elif net_flow < 0:
                monthly_outflow[month_key] += abs(net_flow)
                quarterly_outflow[quarter_key] += abs(net_flow)
                yearly_outflow += abs(net_flow)

            monthly_category_cash_flow[month_key][account_name] += net_flow
            quarterly_category_cash_flow[quarter_key][account_name] += net_flow
            yearly_category_cash_flow[account_name] += net_flow

        if unmatched_codes:
            logger.warning(f"Unmatched GL codes (not in master): {unmatched_codes}")
        logger.info(f"Matched {matched_cash_entries} cash-related entries")

        monthly_contribution_percentages = {
            month: {
                category: safe_percent(amount, monthly_cash_flow[month])
                for category, amount in categories.items()
            }
            for month, categories in monthly_category_cash_flow.items()
        }

        quarterly_contribution_percentages = {
            quarter: {
                category: safe_percent(amount, quarterly_cash_flow[quarter])
                for category, amount in categories.items()
            }
            for quarter, categories in quarterly_category_cash_flow.items()
        }

        yearly_contribution_percentages = {
            category: safe_percent(amount, total_cash_flow)
            for category, amount in yearly_category_cash_flow.items()
        }

        return {
            "year": year,
            "total_cash_flow": total_cash_flow,
            "monthly_cash_flow": dict(sorted(monthly_cash_flow.items())),
            "monthly_inflow": dict(sorted(monthly_inflow.items())),
            "monthly_outflow": dict(sorted(monthly_outflow.items())),
            "monthly_category_cash_flow": {
                month: dict(categories) for month, categories in sorted(monthly_category_cash_flow.items())
            },
            "monthly_contribution_percentages": monthly_contribution_percentages,
            "quarterly_cash_flow": dict(sorted(quarterly_cash_flow.items())),
            "quarterly_inflow": dict(sorted(quarterly_inflow.items())),
            "quarterly_outflow": dict(sorted(quarterly_outflow.items())),
            "quarterly_category_cash_flow": {
                quarter: dict(categories) for quarter, categories in sorted(quarterly_category_cash_flow.items())
            },
            "quarterly_contribution_percentages": quarterly_contribution_percentages,
            "yearly_category_cash_flow": dict(sorted(yearly_category_cash_flow.items())),
            "yearly_inflow": yearly_inflow,
            "yearly_outflow": yearly_outflow,
            "yearly_contribution_percentages": yearly_contribution_percentages,
        }

    except Exception as e:
        logger.exception("Error calculating cash flow analytics")
        return {"error": str(e)}


from sqlalchemy import text

@financial_router.get("/test")
async def test_endpoint(db: AsyncSession = Depends(get_db_on_the_fly)):
    result = await db.execute(text("SELECT current_database();"))
    db_name = result.scalar_one()
    return {"connected_database": db_name}






#core reports

def calc_change(current: float, previous: float) -> tuple[float, float]:
    change = current - previous
    change_pct = ((change / abs(previous)) * 100) if previous else 0.0
    return change, change_pct


@financial_router.get("/financial-overview/balance-sheet-comparison")
async def balance_sheet_comparison(
    current_year: int = Query(..., description="Current year"),
    previous_year: int = Query(..., description="Previous year to compare against"),
    db: AsyncSession = Depends(get_db_on_the_fly)
):
    # Fetch GL Master mapping
    master_result = await db.execute(select(GeneralLedgerMaster))
    master_map = {
        float(m.general_ledger_code): {
            "tags": [tag.lower() for tag in (m.tags or [])],
            "category": (m.category or "").lower(),
            "account_name": m.account_name
        }
        for m in master_result.scalars().all()
    }

    # Fetch GL entries for both years
    gl_result = await db.execute(
        select(GeneralLedger).where(
            func.extract('year', GeneralLedger.date).in_([current_year, previous_year])
        )
    )
    gl_entries = gl_result.scalars().all()

    # Initialize data structures
    balance_by_account = defaultdict(lambda: {current_year: 0.0, previous_year: 0.0})
    assets = {current_year: 0.0, previous_year: 0.0}
    liabilities = {current_year: 0.0, previous_year: 0.0}
    equity = {current_year: 0.0, previous_year: 0.0}
    current_assets = {current_year: 0.0, previous_year: 0.0}
    current_liabilities = {current_year: 0.0, previous_year: 0.0}

    # Process GL entries
    for e in gl_entries:
        code = float(e.general_ledger_code)
        y = e.date.year
        if y not in [current_year, previous_year]:
            continue

        master = master_map.get(code)
        if not master:
            continue

        tags = set(master["tags"])
        account_name = master["account_name"]
        balance = float(e.debit) - float(e.credit)

        # Apply balances by category
        if  "asset" in tags:
            balance_by_account[account_name][y] += balance
            assets[y] += balance
            
            if "current_asset" in tags or "currentasset" in tags:
                current_assets[y] += balance

        elif "liability" in tags:
            balance_by_account[account_name][y] += balance
            liabilities[y] += balance
            if "current_liability" in tags or "currentliability" in tags:
                current_liabilities[y] += balance
            
        elif "equity" in tags:
            balance_by_account[account_name][y] += balance
            equity[y] += balance


    # Account-level comparison
    rows = []
    for account, data in balance_by_account.items():
        curr = data[current_year]
        prev = data[previous_year]
        change, change_pct = calc_change(curr, prev)
        rows.append({
            "account": account,
            "current": round(curr, 2),
            "previous": round(prev, 2),
            "change": round(change, 2),
            "change_percentage": round(change_pct, 2)
        })

    # Summary calculation
    asset_growth, asset_growth_pct = calc_change(assets[current_year], assets[previous_year])
    liability_growth, liability_growth_pct = calc_change(liabilities[current_year], liabilities[previous_year])
    net_equity_change, net_equity_pct = calc_change(equity[current_year], equity[previous_year])

    current_ratio = (current_assets[current_year] / current_liabilities[current_year]) if current_liabilities[current_year] else 0.0
    debt_to_equity = (liabilities[current_year] / equity[current_year]) if equity[current_year] else 0.0

    summary = {
        "Total Assets": {
            "current": round(assets[current_year], 2),
            "previous": round(assets[previous_year], 2),
            "change": round(asset_growth, 2),
            "change_percentage": round(asset_growth_pct, 2)
        },
        "Total Liabilities": {
            "current": round(liabilities[current_year], 2),
            "previous": round(liabilities[previous_year], 2),
            "change": round(liability_growth, 2),
            "change_percentage": round(liability_growth_pct, 2)
        },
        "Net Equity": {
            "current": round(equity[current_year], 2),
            "previous": round(equity[previous_year], 2),
            "change": round(net_equity_change, 2),
            "change_percentage": round(net_equity_pct, 2)
        },
        "Current Ratio": round(current_ratio, 2),
        "Debt to Equity": round(debt_to_equity, 2)
    }

    return {
        "current_year": current_year,
        "previous_year": previous_year,
        "accounts": rows,
        "summary": summary
    }

@financial_router.get("/financial-overview/accounts-receivable-payable")
async def receivable_payable_analysis(
    current_year: int = Query(...),
    previous_year: int = Query(...),
    db: AsyncSession = Depends(get_db_on_the_fly)
):
    master_result = await db.execute(select(GeneralLedgerMaster))
    master_map = {
        float(m.general_ledger_code): {
            "tags": [tag.lower() for tag in (m.tags or [])],
            "category": (m.category or '').lower(),
            "account_name": m.account_name
        }
        for m in master_result.scalars().all()
    }

    gl_result = await db.execute(
        select(GeneralLedger).where(
            func.extract('year', GeneralLedger.date).in_([current_year, previous_year])
        )
    )
    gl_entries = gl_result.scalars().all()
    print(gl_entries)
    # Aggregate data
    totals = {
        "accounts_receivable": {current_year: 0.0, previous_year: 0.0},
        "accounts_payable": {current_year: 0.0, previous_year: 0.0}
    }
    monthly = {
        "accounts_receivable": defaultdict(lambda: defaultdict(float)),
        "accounts_payable": defaultdict(lambda: defaultdict(float))
    }
    quarterly = {
        "accounts_receivable": defaultdict(lambda: defaultdict(float)),
        "accounts_payable": defaultdict(lambda: defaultdict(float))
    }

    for e in gl_entries:
        code = float(e.general_ledger_code)
        year = e.date.year
        if year not in [current_year, previous_year]:
            continue

        master = master_map.get(code)
        if not master:
            continue

        tags = set(master["tags"])
        account_type = None
        if "receivables" in tags:
            account_type = "accounts_receivable"
        elif "payables" in tags:
            account_type = "accounts_payable"

        if not account_type:
            continue

        month = e.date.strftime("%b")
        quarter = f"Q{((e.date.month - 1) // 3) + 1}"
        balance = float(e.debit) - float(e.credit)

        totals[account_type][year] += balance
        monthly[account_type][month][year] += balance
        quarterly[account_type][quarter][year] += balance

    def build_breakdown(data: dict):
        output = []
        for period, values in data.items():
            curr = values.get(current_year, 0.0)
            prev = values.get(previous_year, 0.0)
            change, change_pct = calc_change(curr, prev)
            output.append({
                "period": period,
                "current": round(curr, 2),
                "previous": round(prev, 2),
                "change": round(change, 2),
                "change_percentage": round(change_pct, 2)
            })
        return sorted(output, key=lambda x: x["period"])

    summary = {}
    for acc_type in ["accounts_receivable", "accounts_payable"]:
        curr = totals[acc_type][current_year]
        prev = totals[acc_type][previous_year]
        change, change_pct = calc_change(curr, prev)
        summary[acc_type] = {
            "current": round(curr, 2),
            "previous": round(prev, 2),
            "change": round(change, 2),
            "change_percentage": round(change_pct, 2)
        }

    return {
        "current_year": current_year,
        "previous_year": previous_year,
        "summary": summary,
        "monthly_breakdown": {
            "accounts_receivable": build_breakdown(monthly["accounts_receivable"]),
            "accounts_payable": build_breakdown(monthly["accounts_payable"])
        },
        "quarterly_breakdown": {
            "accounts_receivable": build_breakdown(quarterly["accounts_receivable"]),
            "accounts_payable": build_breakdown(quarterly["accounts_payable"])
        }
    }



from sqlalchemy import func, and_

@financial_router.get("/analytics/revenue-comparison")
async def get_revenue_comparison(
    current_year: int = Query(..., description="Current year for comparison"),
    previous_year: int = Query(..., description="Previous year for comparison"),
    db: AsyncSession = Depends(get_db_on_the_fly)
):
    logger.info(f"Comparing revenue between {previous_year} and {current_year}")

    # Load GL Master entries
    master_result = await db.execute(select(GeneralLedgerMaster))
    master_map = {
        float(m.general_ledger_code): {
            "tags": [tag.lower() for tag in (m.tags or [])],
            "category": (m.category or "").lower(),
            "account_name": m.account_name
        }
        for m in master_result.scalars().all()
    }

    # Load GL Entries
    result = await db.execute(select(GeneralLedger))
    gl_entries = result.scalars().all()

    valid_years = {current_year, previous_year}
    revenue_entries = [
        e for e in gl_entries
        if (e.date.year in valid_years) and
           (m := master_map.get(float(e.general_ledger_code))) and
           m["category"] == "revenue"
    ]

    logger.info(f"Found {len(revenue_entries)} revenue entries for both years")

    totals = {current_year: 0.0, previous_year: 0.0}
    monthly = {current_year: defaultdict(float), previous_year: defaultdict(float)}
    quarterly = {current_year: defaultdict(float), previous_year: defaultdict(float)}
    category = {current_year: defaultdict(float), previous_year: defaultdict(float)}

    for e in revenue_entries:
        year = e.date.year
        credit = float(e.credit)
        month = e.date.strftime("%b")
        quarter = f"Q{((e.date.month - 1) // 3) + 1}"
        account = master_map[float(e.general_ledger_code)]["account_name"] or "Unknown"

        totals[year] += credit
        monthly[year][month] += credit
        quarterly[year][quarter] += credit
        category[year][account] += credit

    # Contribution % per category
    contribution_percent = {
        year: {
            acc: round((amount / totals[year]) * 100, 2) if totals[year] else 0.0
            for acc, amount in category[year].items()
        }
        for year in [current_year, previous_year]
    }

    def format_comparison(data: dict):
        all_keys = set(data[current_year]) | set(data[previous_year])
        return [
            {
                "key": k,
                "current": round(data[current_year].get(k, 0.0), 2),
                "previous": round(data[previous_year].get(k, 0.0), 2),
                "change": round(data[current_year].get(k, 0.0) - data[previous_year].get(k, 0.0), 2),
                "change_percentage": round(
                    ((data[current_year].get(k, 0.0) - data[previous_year].get(k, 0.0)) /
                     abs(data[previous_year].get(k, 0.01))) * 100,
                    2
                ) if data[previous_year].get(k, 0.0) else 0.0
            }
            for k in sorted(all_keys)
        ]

    return {
        "current_year": current_year,
        "previous_year": previous_year,
        "summary": {
            "total_revenue": {
                "current": round(totals[current_year], 2),
                "previous": round(totals[previous_year], 2),
                "change": round(totals[current_year] - totals[previous_year], 2),
                "change_percentage": round(
                    ((totals[current_year] - totals[previous_year]) / abs(totals[previous_year] or 1)) * 100, 2
                ) if totals[previous_year] else 0.0
            }
        },
        "monthly_comparison": format_comparison(monthly),
        "quarterly_comparison": format_comparison(quarterly),
        "category_comparison": format_comparison(category),
        "category_contribution_percent": contribution_percent
    }






@financial_router.get("/profit_and_loss/{year}")
async def Profit_and_loss_statement(
    year:int,
    # period: PeriodType = PeriodType.YEARLY,
    
    custom_start: Optional[datetime] = Query(default=None),
    custom_end: Optional[datetime] = Query(default=None),
    entity: Optional[str] = Query(default=None),
    hierarchy: Optional[str] = Query(default=None),
    db: AsyncSession = Depends(get_db_on_the_fly)
):
    try:
        
        return await FinancialReport.profit_loss_statement(
                db,'yearly', year, custom_start, custom_end, entity ,hierarchy
     
            
        )
    except Exception as e:
        raise HTTPException(status_code=400, detail=str(e))

@financial_router.get("/balance_sheet/{year}")
async def balance_sheet(
    year:int,
    #period: PeriodType = PeriodType.YEARLY,
    
    custom_start: Optional[datetime] = Query(default=None),
    custom_end: Optional[datetime] = Query(default=None),
    entity: Optional[str] = Query(default=None),
    hierarchy: Optional[str] = Query(default=None),
    db: AsyncSession = Depends(get_db_on_the_fly)
):
    try:
        return await FinancialReport.balance_sheet(
            db, "yearly", year, custom_start, custom_end, entity=entity, hierarchy=hierarchy
        )
    except Exception as e:
        raise HTTPException(status_code=400, detail=str(e))

@financial_router.get("/revenue_report/{year}")
async def revenuereport(
    year:int,
#    period: PeriodType = PeriodType.YEARLY,
    
    custom_start: Optional[datetime] = Query(default=None),
    custom_end: Optional[datetime] = Query(default=None),
    entity: Optional[str] = Query(default=None),
    hierarchy: Optional[str] = Query(default=None),
    db: AsyncSession = Depends(get_db_on_the_fly)
):
    try:
        return await FinancialReport.get_product_revenue_report(
            db, "yearly", year, custom_start, custom_end, entity=entity, hierarchy=hierarchy
        )
    except Exception as e:
        raise HTTPException(status_code=400, detail=str(e))
@financial_router.get("/cashflow/{year}")
async def cashflow_statement(
    year:int,
    period: PeriodType = PeriodType.YEARLY,
    
    custom_start: Optional[datetime] = Query(default=None),
    custom_end: Optional[datetime] = Query(default=None),
    entity: Optional[str] = Query(default=None),
    hierarchy: Optional[str] = Query(default=None),
    db: AsyncSession = Depends(get_db_on_the_fly)
):
    try:
        return await FinancialReport.get_cashflow_statement(
            db,period,year, custom_start, custom_end,  entity=entity, hierarchy=hierarchy
        )
    except Exception as e:
        raise HTTPException(status_code=400, detail=str(e))


# @financial_router.get("/accounts_receivable/{year}")
# async def AR_aging_report(
#     year:int,
#     period: PeriodType = PeriodType.YEARLY,
    
#     custom_start: Optional[datetime] = Query(default=None),
#     custom_end: Optional[datetime] = Query(default=None),
#     entity: Optional[str] = Query(default=None),
#     hierarchy: Optional[str] = Query(default=None),
#     db: AsyncSession = Depends(get_db_on_the_fly)
# ):
#     try:
#         return await FinancialReport.get_Accounts_Receivable(
#             db,period,year, custom_start, custom_end, entity=entity, hierarchy=hierarchy
#         )
#     except Exception as e:
#         raise HTTPException(status_code=400, detail=str(e))


# @financial_router.get("/accounts_payable/{year}")
# async def Ap_aging_report(
#     year:int,
#     period: PeriodType = PeriodType.YEARLY,
    
#     custom_start: Optional[datetime] = Query(default=None),
#     custom_end: Optional[datetime] = Query(default=None),
#     entity: Optional[str] = Query(default=None),
#     hierarchy: Optional[str] = Query(default=None),
#     db: AsyncSession = Depends(get_db_on_the_fly)
# ):
#     try:
#         return await FinancialReport.get_Accounts_Payable(
#             db,
#         period,
#         year,
#         custom_start,
#         custom_end,
#         entity,
#         hierarchy
#         )
#     except Exception as e:
#         raise HTTPException(status_code=400, detail=str(e))




@financial_router.get("/operation_expense/{year}")
async def OperationExpense(
    year:int,
    #period: PeriodType = PeriodType.YEARLY,
    
    custom_start: Optional[datetime] = Query(default=None),
    custom_end: Optional[datetime] = Query(default=None),
    entity: Optional[str] = Query(default=None),
    hierarchy: Optional[str] = Query(default=None),
    department:Optional[str]=Query(default=None),
    db: AsyncSession = Depends(get_db_on_the_fly)
):
    try:
        return await FinancialReport.get_operation_expense_report(
            db,
        "yearly",
        year,
        custom_start,
        custom_end,
        department 
        )
    except Exception as e:
        raise HTTPException(status_code=400, detail=str(e))
    
@financial_router.get("/Customer_Profit_analysis/{year}")
async def Customer_profit_analysis(
         year:int,
    #period: PeriodType = PeriodType.YEARLY,
    
    custom_start: Optional[datetime] = Query(default=None),
    custom_end: Optional[datetime] = Query(default=None),
    entity: Optional[str] = Query(default=None),
    hierarchy: Optional[str] = Query(default=None),
   
    db: AsyncSession = Depends(get_db_on_the_fly)
    ):
    try:
        print(year)
        return await FinancialReport.get_customer_profit_analysis(
            db,
        "yearly",
        year,
        custom_start,
        custom_end,
        entity,
        hierarchy
        )
    except Exception as e:
        raise HTTPException(status_code=400, detail=str(e))
# @financial_router.get("/revenue")
# async def Revenue_report(
#     period: PeriodType = PeriodType.YEARLY,
#     year: int = Query(..., description="Report year"),
#     custom_start: Optional[datetime] = Query(default=None),
#     custom_end: Optional[datetime] = Query(default=None),
#     entity: Optional[str] = Query(default=None),
#     hierarchy: Optional[str] = Query(default=None),
#     db: AsyncSession = Depends(get_db_on_the_fly)
# ):
#     try:
#         return await FinancialReport.get_product_revenue_report(
#             db,year,period ,custom_start,custom_end, entity=entity, hierarchy=hierarchy
#         )
#     except Exception as e:
#         raise HTTPException(status_code=400, detail=str(e))

@financial_router.get("/fixed_assets/{year}")
async def AssertsFixedAccount(
    year:int,
    period: PeriodType = PeriodType.YEARLY,
    
    entity: Optional[str] = Query(default=None),
    custom_start: Optional[datetime] = Query(default=None),
    custom_end: Optional[datetime] = Query(default=None),
    hierarchy: Optional[str] = Query(default=None),
    db: AsyncSession = Depends(get_db_on_the_fly)
):
    try:
        return await FinancialReport.get_fixed_asset_register(
            db,period,year,custom_start,custom_end, entity=entity, hierarchy=hierarchy
        )
    except Exception as e:
        raise HTTPException(status_code=400, detail=str(e))
    
@financial_router.get("/Project_costing_and_capital_expenditure_report/{year}")
async def Project_costing_report(
    year:int,
    #period: PeriodType = PeriodType.YEARLY,
    entity: Optional[str] = Query(default=None),
    custom_start: Optional[datetime] = Query(default=None),
    custom_end: Optional[datetime] = Query(default=None),
    hierarchy: Optional[str] = Query(default=None),
    db: AsyncSession = Depends(get_db_on_the_fly)
):
    try:
        return await FinancialReport.get_project_costing_and_capexd_report(
            db,"yearly",year,custom_start,custom_end, entity=entity, hierarchy=hierarchy
        )
    except Exception as e:
        raise HTTPException(status_code=400, detail=str(e))



class ReportParams(BaseModel):
    time_period: str
    custom_start: Optional[datetime] = None
    custom_end: Optional[datetime] = None
    year: Optional[int] = None
    period: str
    start_date: Optional[datetime] = None
    end_date: Optional[datetime] = None
    entity: Optional[str] = None
    hierarchy: Optional[str] = None

@financial_router.get("/all_reports/{year}")
async def all_reports(
    year:int,
    period: PeriodType = PeriodType.YEARLY,
    
    custom_start: Optional[datetime] = Query(default=None),
    custom_end: Optional[datetime] = Query(default=None),
    entity: Optional[str] = Query(default=None),
    hierarchy: Optional[str] = Query(default=None),
    db: AsyncSession = Depends(get_db_on_the_fly)
):
    try:
        # today = datetime.now()
        # report_year = year or today.year

        # # ✅ FIX: Pass correct year (int) to get_date_range
        # from_date, to_date = FirstFinancialReport.get_date_range(
        #     period.value,
        #     year=report_year,
        #     custom_start=custom_start,
        #     custom_end=custom_end
        # )

        params = {
            "time_period": period.value,
            "custom_start": custom_start,
            "custom_end": custom_end,
            "year": year,
            "period": period.value,
            "start_date": custom_start,
            "end_date": custom_end,
            "entity": entity,
            "hierarchy": hierarchy
        }

        return await FirstFinancialReport.generate_report_data(db, params)

    except Exception as e:
        raise HTTPException(status_code=400, detail=str(e))

from decimal import Decimal

def serialize(value):
    if isinstance(value, Decimal):
        return float(value)
    return value 





#####################################################################################
####################################################################################

#CUSTOMER PROBABILITY REORT

oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")

# -----------------------------
# Authentication
# -----------------------------
async def get_current_user(token: str = Depends(oauth2_scheme)):
    if not token:
        raise HTTPException(
            status_code=status.HTTP_401_UNAUTHORIZED,
            detail="Authentication credentials were not provided",
            headers={"WWW-Authenticate": "Bearer"},
        )
    return {"user": "authenticated"}
# -----------------------------
# ENUMS
# -----------------------------
# CUSTOMER PROFITABILITY
import io
import math
import pandas as pd
from openpyxl.styles import Font, PatternFill, Alignment, Border, Side
from openpyxl.utils import get_column_letter
from fastapi.responses import StreamingResponse

ai_insights = [
    "AI-powered insights coming soon",
    "AI enhances efficiency through automated financial analysis"
]

class PeriodOption(str, Enum):
    all = "all"
    current_month = "current_month"
    last_month = "last_month"
    quarter_to_date = "quarter_to_date"
    year_to_date = "year_to_date"
    custom_range = "custom_range"

class ComparisonOption(str, Enum):
    mom = "month_on_month"
    yoy = "year_on_year"

class SortByOption(str, Enum):
    customer_name = "customer_name"
    industry = "industry"
    region = "region"
    period = "period"
    revenue = "revenue"
    total_cost = "total_cost"
    profit = "profit"
    margin = "margin"

class SortOrderOption(str, Enum):
    asc = "asc"
    desc = "desc"


def get_gl_period_filter(period: PeriodOption, year: int, date_from: str | None = None, date_to: str | None = None) -> str:
    system_date = pd.Timestamp.today()
    system_year = system_date.year
    system_month = system_date.month

    if period == PeriodOption.custom_range:
        if date_from and date_to:
            return f"AND gl.period BETWEEN '{date_from}' AND '{date_to}'"
        if date_from:
            return f"AND gl.period >= '{date_from}'"
        if date_to:
            return f"AND gl.period <= '{date_to}'"
        return f"AND gl.period LIKE '{year}-%'"

    if period == PeriodOption.all:
        return f"AND gl.period LIKE '{year}-%'"
    if period == PeriodOption.current_month:
        return f"AND gl.period = '{year}-{system_month:02d}'"
    if period == PeriodOption.last_month:
        last_month = (pd.Timestamp(year=year, month=system_month, day=1) - pd.DateOffset(months=1))
        return f"AND gl.period = '{last_month.year}-{last_month.month:02d}'"
    if period == PeriodOption.quarter_to_date:
        return f"AND gl.period BETWEEN '{year}-01' AND '{year}-03'"
    if period == PeriodOption.year_to_date:
        return f"AND gl.period BETWEEN '{year}-01' AND '{system_year}-{system_month:02d}'"
    return ""


def safe_float(value, default=0.0):
    try:
        result = float(value)
        if math.isnan(result) or math.isinf(result):
            return default
        return result
    except (TypeError, ValueError):
        return default


def safe_float_or_none(value):
    try:
        result = float(value)
        if math.isnan(result) or math.isinf(result):
            return None
        return result
    except (TypeError, ValueError):
        return None


def sanitize_for_json(obj):
    if isinstance(obj, float):
        if math.isnan(obj) or math.isinf(obj):
            return 0.0
        return obj
    if isinstance(obj, dict):
        return {k: sanitize_for_json(v) for k, v in obj.items()}
    if isinstance(obj, list):
        return [sanitize_for_json(i) for i in obj]
    return obj


def build_excel_response(df: pd.DataFrame, filename: str = "customer_profitability.xlsx") -> StreamingResponse:
    output = io.BytesIO()
    with pd.ExcelWriter(output, engine="openpyxl") as writer:
        export_df = df.copy()
        export_df.to_excel(writer, index=False, sheet_name="Customer Profitability")
        worksheet = writer.sheets["Customer Profitability"]

        header_font = Font(bold=True, color="FFFFFF", name="Arial", size=11)
        header_fill = PatternFill("solid", fgColor="1F4E79")
        header_alignment = Alignment(horizontal="center", vertical="center")
        thin_border = Border(
            left=Side(style="thin"),
            right=Side(style="thin"),
            top=Side(style="thin"),
            bottom=Side(style="thin")
        )
        number_format = '#,##0.00'
        pct_format = '0.00"%"'

        col_widths = {}
        for col_idx, col_name in enumerate(export_df.columns, start=1):
            col_letter = get_column_letter(col_idx)
            cell = worksheet.cell(row=1, column=col_idx)
            cell.font = header_font
            cell.fill = header_fill
            cell.alignment = header_alignment
            cell.border = thin_border
            col_widths[col_letter] = max(len(str(col_name)) + 4, 14)

        alt_fill = PatternFill("solid", fgColor="D6E4F0")
        for row_idx, row in enumerate(export_df.itertuples(index=False), start=2):
            for col_idx, (col_name, value) in enumerate(zip(export_df.columns, row), start=1):
                col_letter = get_column_letter(col_idx)
                cell = worksheet.cell(row=row_idx, column=col_idx, value=value)
                cell.border = thin_border
                cell.font = Font(name="Arial", size=10)
                cell.alignment = Alignment(horizontal="left")
                if row_idx % 2 == 0:
                    cell.fill = alt_fill
                col_lower = col_name.lower()
                if any(col_lower.endswith(s) for s in ("_revenue", "_total_cost", "_profit", "_variance", "total_revenue", "total_total_cost", "total_profit")):
                    cell.number_format = number_format
                    cell.alignment = Alignment(horizontal="right")
                elif "margin" in col_lower or "pct" in col_lower:
                    cell.number_format = pct_format
                    cell.alignment = Alignment(horizontal="right")
                cell_len = len(str(value)) + 4 if value is not None else 4
                if cell_len > col_widths.get(col_letter, 0):
                    col_widths[col_letter] = cell_len

        for col_letter, width in col_widths.items():
            worksheet.column_dimensions[col_letter].width = min(width, 40)

        worksheet.freeze_panes = "A2"

    output.seek(0)
    headers = {
        "Content-Disposition": f'attachment; filename="{filename}"',
        "Content-Type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
    }
    return StreamingResponse(
        output,
        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        headers=headers
    )


def filter_customer_record(record: dict, columns_to_include: list[str]) -> dict:
    return {k: v for k, v in record.items() if k in columns_to_include}


@financial_router.get(
    "/customer-profitability/{year}/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def customer_profitability_report(
    request: Request,
    year: int = Path(...),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    period: PeriodOption = Query(PeriodOption.all),
    date_from: str | None = Query(None, description="Custom range start in YYYY-MM format"),
    date_to: str | None = Query(None, description="Custom range end in YYYY-MM format"),
    comparison: ComparisonOption | None = Query(None),
    sort_by: SortByOption = Query(SortByOption.profit),
    sort_order: SortOrderOption = Query(SortOrderOption.desc),
    customer_name: str | None = Query(None),
    industry: str | None = Query(None),
    department: str | None = Query(None),
    region: str | None = Query(None),
    entity_name: str | None = Query(None),
    show_industry: bool = Query(False, description="Include industry column in response"),
    show_region: bool = Query(False, description="Include region column in response"),
    show_entity_name: bool = Query(False, description="Include entity_name column in response"),
    show_department_name: bool = Query(False, description="Include department_name column in response"),
    show_period: bool = Query(False, description="Include period column in response"),
    page: int = Query(1, ge=1),
    page_size: int = Query(2000, ge=1, le=2000),
    excel_download: bool = Query(False),
    db: AsyncSession = Depends(get_db),
):
    if date_from:
        try:
            pd.to_datetime(date_from, format="%Y-%m")
        except ValueError:
            raise HTTPException(status_code=400, detail="date_from must be in YYYY-MM format")

    if date_to:
        try:
            pd.to_datetime(date_to, format="%Y-%m")
        except ValueError:
            raise HTTPException(status_code=400, detail="date_to must be in YYYY-MM format")

    if date_from and date_to and date_from > date_to:
        raise HTTPException(status_code=400, detail="date_from must not be later than date_to")

    if (date_from or date_to) and period != PeriodOption.custom_range:
        period = PeriodOption.custom_range

    params = {}
    filters = []

    if customer_name:
        filters.append("c.customer_name = :customer_name")
        params["customer_name"] = customer_name
    if industry:
        filters.append("c.industry = :industry")
        params["industry"] = industry
    if region:
        filters.append("c.region = :region")
        params["region"] = region
    if entity_name:
        filters.append("e.entity_name = :entity_name")
        params["entity_name"] = entity_name
    if department:
        filters.append("d.department_name = :department")
        params["department"] = department

    filter_sql = "AND " + " AND ".join(filters) if filters else ""
    period_filter = get_gl_period_filter(period, year, date_from, date_to)

    show_columns = {
        "industry": show_industry,
        "region": show_region,
        "entity_name": show_entity_name,
        "department_name": show_department_name,
        "period": show_period,
    }

    sql = text(f"""
        WITH base_revenue AS (
            SELECT
                c.customer_name,
                c.industry,
                c.region,
                e.entity_name,
                d.department_name,
                gl.period,
                SUM(gl.amount_credit - gl.amount_debit) AS revenue
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa
                ON gl.gl_account_id = coa.account_id
            JOIN fpna_landing.finance_customers c
                ON REGEXP_REPLACE(gl.customer_id::TEXT,'\\.0$','')
                 = REGEXP_REPLACE(c.customer_id::TEXT,'\\.0$','')
            JOIN fpna_landing.finance_entities e
                ON gl.entity_id::TEXT = e.entity_id::TEXT
            JOIN fpna_landing.finance_departments d
                ON gl.department_id::TEXT = d.department_id::TEXT
            WHERE coa.account_type = 'Revenue'
              AND gl.customer_id IS NOT NULL
              {filter_sql}
              {period_filter}
            GROUP BY
                c.customer_name,
                c.industry,
                c.region,
                e.entity_name,
                d.department_name,
                gl.period
        ),
        company_revenue AS (
            SELECT COALESCE(SUM(gl.amount_credit - gl.amount_debit),0) AS total_revenue
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa
                ON gl.gl_account_id = coa.account_id
            WHERE coa.account_type = 'Revenue'
              {period_filter}
        ),
        company_cost AS (
            SELECT COALESCE(SUM(gl.amount_debit),0) AS total_cost
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa
                ON gl.gl_account_id = coa.account_id
            WHERE coa.account_type = 'Expense'
              {period_filter}
        )
        SELECT
            r.customer_name,
            r.industry,
            r.region,
            r.entity_name,
            r.department_name,
            r.period,
            r.revenue,
            (r.revenue / NULLIF(cr.total_revenue,0)) * cc.total_cost AS total_cost,
            r.revenue - ((r.revenue / NULLIF(cr.total_revenue,0)) * cc.total_cost) AS profit
        FROM base_revenue r
        CROSS JOIN company_revenue cr
        CROSS JOIN company_cost cc
    """)

    rows = (await db.execute(sql, params)).fetchall()

    if not rows:
        if excel_download:
            empty_df = pd.DataFrame(columns=["customer_name"])
            return build_excel_response(empty_df)
        return {
            "keymetrics": {},
            "customers": [],
            "pagination": {"page": page, "page_size": page_size, "total_records": 0, "total_pages": 0},
            "customers_list": [],
            "industries": [],
            "regions": [],
            "entities": [],
            "departments": [],
            "show_columns": show_columns
        }

    df_full = pd.DataFrame(rows, columns=[
        "customer_name", "industry", "region", "entity_name",
        "department_name", "period", "revenue", "total_cost", "profit"
    ])

    for col in ["revenue", "total_cost", "profit"]:
        df_full[col] = pd.to_numeric(df_full[col], errors="coerce").fillna(0.0)

    unique_customers = sorted(df_full["customer_name"].dropna().unique().tolist())
    unique_industries = sorted(df_full["industry"].dropna().unique().tolist())
    unique_regions = sorted(df_full["region"].dropna().unique().tolist())
    unique_entities = sorted(df_full["entity_name"].dropna().unique().tolist())
    unique_departments = sorted(df_full["department_name"].dropna().unique().tolist())

    visible_columns = ["customer_name"]
    visible_columns += [col for col, enabled in show_columns.items() if enabled and col != "period"]
    visible_columns += ["revenue", "total_cost", "profit", "pct_margin"]

    group_cols_for_pivot = ["customer_name"]
    group_cols_for_pivot += [col for col, enabled in show_columns.items() if enabled and col != "period"]

    periods_available = sorted(df_full["period"].dropna().unique().tolist())

    df_grouped = df_full.groupby(group_cols_for_pivot + ["period"], as_index=False).agg(
        revenue=("revenue", "sum"),
        total_cost=("total_cost", "sum"),
        profit=("profit", "sum")
    )

    for col in ["revenue", "total_cost", "profit"]:
        df_grouped[col] = df_grouped[col].replace([float("inf"), float("-inf")], 0.0).fillna(0.0)

    df_totals = df_grouped.groupby(group_cols_for_pivot, as_index=False).agg(
        total_revenue=("revenue", "sum"),
        total_cost_sum=("total_cost", "sum"),
        total_profit=("profit", "sum")
    )

    allowed_sort_columns = {
        "customer_name": "customer_name",
        "industry": "industry",
        "region": "region",
        "period": "total_revenue",
        "revenue": "total_revenue",
        "total_cost": "total_cost_sum",
        "profit": "total_profit",
        "margin": "total_profit"
    }
    sort_column = allowed_sort_columns.get(sort_by.value, "total_profit")
    if sort_column not in df_totals.columns:
        sort_column = "total_profit"
    df_totals = df_totals.sort_values(by=sort_column, ascending=(sort_order == SortOrderOption.asc)).reset_index(drop=True)

    raw_comparison_map = {}

    if comparison is not None and periods_available:
        if comparison == ComparisonOption.mom:
            df_grouped["compare_period"] = (
                pd.to_datetime(df_grouped["period"], format="%Y-%m") - pd.DateOffset(months=1)
            ).dt.strftime("%Y-%m")
        else:
            df_grouped["compare_period"] = (
                pd.to_datetime(df_grouped["period"], format="%Y-%m") - pd.DateOffset(years=1)
            ).dt.strftime("%Y-%m")

        compare_lookup = df_grouped[group_cols_for_pivot + ["period", "revenue", "total_cost", "profit"]].copy()
        compare_lookup = compare_lookup.rename(columns={
            "revenue": "compare_revenue",
            "total_cost": "compare_total_cost",
            "profit": "compare_profit",
            "period": "compare_period"
        })

        merged = df_grouped.merge(
            compare_lookup,
            on=group_cols_for_pivot + ["compare_period"],
            how="left"
        )

        for col in ["compare_revenue", "compare_total_cost", "compare_profit"]:
            merged[col] = pd.to_numeric(merged[col], errors="coerce").fillna(0.0).replace([float("inf"), float("-inf")], 0.0)

        merged["variance"] = (merged["revenue"] - merged["compare_revenue"]).fillna(0.0)
        merged["variance_total_cost"] = (merged["total_cost"] - merged["compare_total_cost"]).fillna(0.0)
        merged["variance_profit"] = (merged["profit"] - merged["compare_profit"]).fillna(0.0)

        merged["pct_change_revenue"] = merged.apply(
            lambda r: safe_float_or_none(round((r["variance"] / r["compare_revenue"]) * 100, 2)) if r["compare_revenue"] != 0 else None,
            axis=1
        )
        merged["pct_change_total_cost"] = merged.apply(
            lambda r: safe_float_or_none(round((r["variance_total_cost"] / r["compare_total_cost"]) * 100, 2)) if r["compare_total_cost"] != 0 else None,
            axis=1
        )
        merged["pct_change_profit"] = merged.apply(
            lambda r: safe_float_or_none(round((r["variance_profit"] / r["compare_profit"]) * 100, 2)) if r["compare_profit"] != 0 else None,
            axis=1
        )

        for _, row in merged.iterrows():
            key = tuple(row[c] for c in group_cols_for_pivot)
            p = row["period"]
            if key not in raw_comparison_map:
                raw_comparison_map[key] = {}
            raw_comparison_map[key][p] = {
                "comparison_type": comparison.value,
                "current_period": p,
                "compare_period": row["compare_period"],
                "current_revenue": safe_float(row["revenue"]),
                "compare_revenue": safe_float(row["compare_revenue"]),
                "variance": safe_float(row["variance"]),
                "pct_change_revenue": row["pct_change_revenue"],
                "current_total_cost": safe_float(row["total_cost"]),
                "compare_total_cost": safe_float(row["compare_total_cost"]),
                "variance_total_cost": safe_float(row["variance_total_cost"]),
                "pct_change_total_cost": row["pct_change_total_cost"],
                "current_profit": safe_float(row["profit"]),
                "compare_profit": safe_float(row["compare_profit"]),
                "variance_profit": safe_float(row["variance_profit"]),
                "pct_change_profit": row["pct_change_profit"],
            }

    grand_total_by_period = df_grouped.groupby("period", as_index=False).agg(
        revenue=("revenue", "sum"),
        total_cost=("total_cost", "sum"),
        profit=("profit", "sum")
    )
    grand_period_map = {row["period"]: row for _, row in grand_total_by_period.iterrows()}

    grand_total_revenue = safe_float(df_grouped["revenue"].sum())
    grand_total_cost = safe_float(df_grouped["total_cost"].sum())
    grand_total_profit = safe_float(df_grouped["profit"].sum())
    grand_total_margin = round((grand_total_profit / grand_total_revenue * 100), 2) if grand_total_revenue else 0.0

    highest_revenue = safe_float(df_totals["total_revenue"].max())
    lowest_revenue = safe_float(df_totals["total_revenue"].min())
    average_revenue = safe_float(df_totals["total_revenue"].mean())

    total_records = len(df_totals)
    total_pages = math.ceil(total_records / page_size)

    start = (page - 1) * page_size
    end = start + page_size
    df_page = df_totals.iloc[start:end]

    period_lookup = {}
    for _, row in df_grouped.iterrows():
        key = tuple(row[c] for c in group_cols_for_pivot)
        p = row["period"]
        if key not in period_lookup:
            period_lookup[key] = {}
        period_lookup[key][p] = {
            "revenue": safe_float(row["revenue"]),
            "total_cost": safe_float(row["total_cost"]),
            "profit": safe_float(row["profit"])
        }

    currency_fields_per_month = ["revenue", "total_cost", "profit"]
    currency_fields_comparison = [
        "current_revenue", "compare_revenue", "variance",
        "current_total_cost", "compare_total_cost", "variance_total_cost",
        "current_profit", "compare_profit", "variance_profit"
    ]

    customers = []
    for _, tot_row in df_page.iterrows():
        group_key = {c: tot_row[c] for c in group_cols_for_pivot}
        key = tuple(group_key[c] for c in group_cols_for_pivot)
        rows_by_period = period_lookup.get(key, {})

        out = dict(group_key)
        months = {}
        for p in periods_available:
            if p in rows_by_period:
                r = rows_by_period[p]
                rev = safe_float(r["revenue"])
                cost = safe_float(r["total_cost"])
                profit_val = safe_float(r["profit"])
                margin = round((profit_val / rev * 100), 2) if rev else 0.0
                month_entry = {
                    "revenue": rev,
                    "total_cost": cost,
                    "profit": profit_val,
                    "pct_margin": margin
                }
                if comparison is not None and key in raw_comparison_map and p in raw_comparison_map[key]:
                    raw_comp = raw_comparison_map[key][p]
                    comp_entry = dict(raw_comp)
                    fields_present = [k for k in currency_fields_comparison if k in comp_entry]
                    converted_comp = apply_smart_currency(
                        {k: comp_entry[k] for k in fields_present},
                        currency,
                        currency_filter,
                        fields_present
                    )
                    comp_entry.update(converted_comp)
                    month_entry["comparisons"] = comp_entry
                months[p] = month_entry

        for p, month_data in months.items():
            converted_month = apply_smart_currency(
                {k: month_data[k] for k in currency_fields_per_month},
                currency,
                currency_filter,
                currency_fields_per_month
            )
            month_data.update(converted_month)

        out["months"] = months

        t_rev = safe_float(tot_row["total_revenue"])
        t_cost = safe_float(tot_row["total_cost_sum"])
        t_profit = safe_float(tot_row["total_profit"])
        t_margin = round((t_profit / t_rev * 100), 2) if t_rev else 0.0

        total_converted = apply_smart_currency(
            {"revenue": t_rev, "total_cost": t_cost, "profit": t_profit},
            currency,
            currency_filter,
            ["revenue", "total_cost", "profit"]
        )
        out["total"] = {
            "revenue": safe_float(total_converted["revenue"]),
            "total_cost": safe_float(total_converted["total_cost"]),
            "profit": safe_float(total_converted["profit"]),
            "pct_margin": t_margin
        }

        out["ai_insights"] = ai_insights
        customers.append(out)

    grand_months = {}
    for p in periods_available:
        if p in grand_period_map:
            gr = grand_period_map[p]
            g_rev = safe_float(gr["revenue"])
            g_cost = safe_float(gr["total_cost"])
            g_profit = safe_float(gr["profit"])
            g_margin = round((g_profit / g_rev * 100), 2) if g_rev else 0.0
            grand_month_entry = {
                "revenue": g_rev,
                "total_cost": g_cost,
                "profit": g_profit,
                "pct_margin": g_margin
            }

            if comparison is not None:
                all_period_comps = [
                    raw_comparison_map[k][p]
                    for k in raw_comparison_map
                    if p in raw_comparison_map[k]
                ]
                if all_period_comps:
                    gt_cur_rev = safe_float(sum(safe_float(x["current_revenue"]) for x in all_period_comps))
                    gt_cmp_rev = safe_float(sum(safe_float(x["compare_revenue"]) for x in all_period_comps))
                    gt_cur_cost = safe_float(sum(safe_float(x["current_total_cost"]) for x in all_period_comps))
                    gt_cmp_cost = safe_float(sum(safe_float(x["compare_total_cost"]) for x in all_period_comps))
                    gt_cur_profit = safe_float(sum(safe_float(x["current_profit"]) for x in all_period_comps))
                    gt_cmp_profit = safe_float(sum(safe_float(x["compare_profit"]) for x in all_period_comps))
                    gt_var_rev = safe_float(gt_cur_rev - gt_cmp_rev)
                    gt_var_cost = safe_float(gt_cur_cost - gt_cmp_cost)
                    gt_var_profit = safe_float(gt_cur_profit - gt_cmp_profit)
                    gt_comp_entry = {
                        "comparison_type": comparison.value,
                        "current_period": p,
                        "compare_period": all_period_comps[0]["compare_period"],
                        "current_revenue": gt_cur_rev,
                        "compare_revenue": gt_cmp_rev,
                        "variance": gt_var_rev,
                        "pct_change_revenue": safe_float_or_none(round((gt_var_rev / gt_cmp_rev) * 100, 2)) if gt_cmp_rev != 0 else None,
                        "current_total_cost": gt_cur_cost,
                        "compare_total_cost": gt_cmp_cost,
                        "variance_total_cost": gt_var_cost,
                        "pct_change_total_cost": safe_float_or_none(round((gt_var_cost / gt_cmp_cost) * 100, 2)) if gt_cmp_cost != 0 else None,
                        "current_profit": gt_cur_profit,
                        "compare_profit": gt_cmp_profit,
                        "variance_profit": gt_var_profit,
                        "pct_change_profit": safe_float_or_none(round((gt_var_profit / gt_cmp_profit) * 100, 2)) if gt_cmp_profit != 0 else None,
                    }
                    fields_present = [k for k in currency_fields_comparison if k in gt_comp_entry]
                    converted_gc = apply_smart_currency(
                        {k: gt_comp_entry[k] for k in fields_present},
                        currency,
                        currency_filter,
                        fields_present
                    )
                    gt_comp_entry.update(converted_gc)
                    grand_month_entry["comparisons"] = gt_comp_entry

            converted_gm = apply_smart_currency(
                {k: grand_month_entry[k] for k in currency_fields_per_month},
                currency,
                currency_filter,
                currency_fields_per_month
            )
            grand_month_entry.update(converted_gm)
            grand_months[p] = grand_month_entry

    gt_total_converted = apply_smart_currency(
        {"revenue": grand_total_revenue, "total_cost": grand_total_cost, "profit": grand_total_profit},
        currency,
        currency_filter,
        ["revenue", "total_cost", "profit"]
    )

    grand_total_record = {
        "customer_name": "Grand Total",
        "months": grand_months,
        "total": {
            "revenue": safe_float(gt_total_converted["revenue"]),
            "total_cost": safe_float(gt_total_converted["total_cost"]),
            "profit": safe_float(gt_total_converted["profit"]),
            "pct_margin": grand_total_margin
        }
    }

    customers.append(grand_total_record)

    keymetrics = apply_smart_currency(
        {
            "total_revenue": grand_total_revenue,
            "total_cost": grand_total_cost,
            "total_profit": grand_total_profit,
            "margin": grand_total_margin,
            "highest_revenue": highest_revenue,
            "lowest_revenue": lowest_revenue,
            "average_revenue": average_revenue,
            "ai_insights": ai_insights
        },
        currency,
        currency_filter,
        ["total_revenue", "total_cost", "total_profit", "highest_revenue", "lowest_revenue", "average_revenue"]
    )

    if excel_download:
        excel_rows = []
        for c in customers:
            row = {"customer_name": c.get("customer_name", "")}
            for col in group_cols_for_pivot:
                if col != "customer_name" and col in c:
                    row[col] = c[col]
            for p in periods_available:
                if p in c.get("months", {}):
                    m = c["months"][p]
                    row[f"{p}_revenue"] = m.get("revenue", 0)
                    row[f"{p}_total_cost"] = m.get("total_cost", 0)
                    row[f"{p}_profit"] = m.get("profit", 0)
                    row[f"{p}_pct_margin"] = m.get("pct_margin", 0)
                else:
                    row[f"{p}_revenue"] = 0
                    row[f"{p}_total_cost"] = 0
                    row[f"{p}_profit"] = 0
                    row[f"{p}_pct_margin"] = 0
            t = c.get("total", {})
            row["total_revenue"] = t.get("revenue", 0)
            row["total_total_cost"] = t.get("total_cost", 0)
            row["total_profit"] = t.get("profit", 0)
            row["total_pct_margin"] = t.get("pct_margin", 0)
            excel_rows.append(row)

        export_df = pd.DataFrame(excel_rows)

        if period == PeriodOption.custom_range and (date_from or date_to):
            range_label = f"{date_from or 'start'}_to_{date_to or 'end'}"
            filename = f"customer_profitability_{year}_{currency}_{range_label}"
        else:
            filename = f"customer_profitability_{year}_{currency}_{period.value}"
        if comparison:
            filename = f"{filename}_{comparison.value}"
        filename = f"{filename}.xlsx"

        return build_excel_response(export_df, filename=filename)

    response_payload = sanitize_for_json({
        "keymetrics": keymetrics,
        "customers": customers,
        "pagination": {
            "page": page,
            "page_size": page_size,
            "total_records": total_records,
            "total_pages": total_pages
        },
        "customers_list": unique_customers,
        "industries": unique_industries,
        "regions": unique_regions,
        "entities": unique_entities,
        "departments": unique_departments,
        "show_columns": show_columns,
        "visible_columns": visible_columns,
        "periods": periods_available
    })

    return response_payload


######################################################
######################################################
#################################################

#customer-profitability_viewmore

def build_monthly_comparisons(monthly_values: Dict[str, float], comparison: ComparisonOption, base_periods: List[str]):
    out = []

    for period in sorted(base_periods):
        base_value = monthly_values.get(period, 0)
        compare_period = get_comparison_period(period, comparison)

        if not compare_period:
            continue

        if compare_period not in monthly_values:
            continue

        compare_value = monthly_values.get(compare_period, 0)

        if compare_value in (None, 0):
            pct = None
        else:
            pct = round(((base_value - compare_value) / compare_value) * 100, 2)

        out.append({
            "base_period": period,
            "compare_period": compare_period,
            "base_value": base_value,
            "compare_value": compare_value,
            "pct_change": pct
        })

    return out


def build_total_comparison(curr_total: float, prev_total: float, year: int):
    if prev_total in (None, 0):
        pct = None
    else:
        pct = round(((curr_total - prev_total) / prev_total) * 100, 2)

    return [{
        "base_period": str(year),
        "compare_period": str(year - 1),
        "base_value": curr_total,
        "compare_value": prev_total,
        "pct_change": pct
    }]


@financial_router.get(
    "/financials/customer-profitability_viewmore/{year}/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def customer_profitablity_viewmore(
    year: int = Path(...),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    period: PeriodOption = Query(PeriodOption.all),
    comparison: Optional[ComparisonOption] = Query(None),
    db: AsyncSession = Depends(get_db)
):
    system_date = pd.Timestamp.today()
    system_year = system_date.year
    system_month = system_date.month

    base_year_filter = get_gl_period_filter(period, year)

    compare_year_filter = ""
    extra_periods = []

    if comparison in (ComparisonOption.yoy, ComparisonOption.mom):
        compare_year_filter = get_gl_period_filter(period, year - 1)

    if comparison == ComparisonOption.mom:
        if period == PeriodOption.all:
            extra_periods.append(f"{year-1}-12")

        if period == PeriodOption.current_month:
            base_period = f"{year}-{system_month:02d}"
            prev_period = get_comparison_period(base_period, ComparisonOption.mom)
            if prev_period:
                extra_periods.append(prev_period)

        if period == PeriodOption.last_month:
            last_month_dt = (pd.Timestamp(year=year, month=system_month, day=1)
                             - pd.DateOffset(months=1))
            base_period = f"{last_month_dt.year}-{last_month_dt.month:02d}"
            prev_period = get_comparison_period(base_period, ComparisonOption.mom)
            if prev_period:
                extra_periods.append(prev_period)

        if period == PeriodOption.quarter_to_date:
            extra_periods.append(f"{year-1}-12")

        if period == PeriodOption.year_to_date:
            extra_periods.append(f"{year-1}-12")

    extra_period_clause = ""
    if extra_periods:
        extra_periods = sorted(set(extra_periods))
        extra_period_clause = " OR gl.period IN (" + ", ".join([f"'{p}'" for p in extra_periods]) + ")"

    SQL = f"""
        SELECT
            gl.period AS period,
            coa.account_type AS account_type,
            coa.account_name AS account_name,
            SUM(
                CASE
                    WHEN coa.account_type = 'Revenue'
                        THEN gl.amount_credit - gl.amount_debit
                    ELSE
                        gl.amount_debit - gl.amount_credit
                END
            ) AS amount
        FROM fpna_landing.finance_general_ledger_transactions gl
        JOIN fpna_landing.finance_chart_of_accounts coa
            ON gl.gl_account_id = coa.account_id
        WHERE coa.account_type IN ('Revenue','Expense')
          AND gl.currency = :currency
          AND (
                (1=1 {base_year_filter})
                {("OR (1=1 " + compare_year_filter + ")") if compare_year_filter else ""}
                {extra_period_clause}
              )
        GROUP BY gl.period, coa.account_type, coa.account_name
        ORDER BY gl.period
    """

    rows = (await db.execute(text(SQL), {"currency": currency})).fetchall()
    if not rows:
        return {
            "keymetrics": {},
            "revenue": [],
            "revenue_grand_total": {},
            "cost": [],
            "cost_grand_total": {},
            "profit": {},
            "margin": {}
        }

    df = pd.DataFrame(rows, columns=["period", "account_type", "account_name", "amount"])
    df["amount"] = df["amount"].astype(float)

    all_periods = sorted(df["period"].unique())

    visible_periods = all_periods
    if period == PeriodOption.all:
        visible_periods = [p for p in all_periods if p.startswith(str(year))]

    money_fields = visible_periods + ["Total"]

    def build_section(section_df: pd.DataFrame, report_name: str):
        pivot = pd.pivot_table(
            section_df,
            index="account_name",
            columns="period",
            values="amount",
            aggfunc="sum",
            fill_value=0.0
        )

        output_rows = []
        grand_total_series = {p: 0.0 for p in all_periods}

        for account, row_data in pivot.iterrows():
            series = {p: float(row_data.get(p, 0.0)) for p in all_periods}

            row = {
                "report": report_name,
                "account_name": account,
                "aiDriver": ai_insights
            }

            for p in visible_periods:
                row[p] = series.get(p, 0.0)

            row["Total"] = sum(row[p] for p in visible_periods)

            row["comparisons"] = {"monthly": [], "total": []}

            if comparison:
                row["comparisons"]["monthly"] = build_monthly_comparisons(series, comparison, visible_periods)

                prev_year_total = sum(
                    series[p] for p in series
                    if p.startswith(str(year - 1))
                )
                row["comparisons"]["total"] = build_total_comparison(row["Total"], prev_year_total, year)

            output_rows.append(row)

            for p in all_periods:
                grand_total_series[p] += series.get(p, 0.0)

        gt = {
            "report": report_name,
            "account_name": "Grand Total",
            "aiDriver": ai_insights
        }

        for p in visible_periods:
            gt[p] = grand_total_series.get(p, 0.0)

        gt["Total"] = sum(gt[p] for p in visible_periods)

        gt["comparisons"] = {"monthly": [], "total": []}

        if comparison:
            gt["comparisons"]["monthly"] = build_monthly_comparisons(grand_total_series, comparison, visible_periods)

            prev_year_gt = sum(
                grand_total_series[p] for p in grand_total_series
                if p.startswith(str(year - 1))
            )
            gt["comparisons"]["total"] = build_total_comparison(gt["Total"], prev_year_gt, year)

        return output_rows, gt, grand_total_series

    revenue_rows, revenue_gt, revenue_series = build_section(
        df[df["account_type"] == "Revenue"],
        "revenue"
    )

    cost_rows, cost_gt, cost_series = build_section(
        df[df["account_type"] == "Expense"],
        "cost"
    )

    profit_series = {p: revenue_series.get(p, 0.0) - cost_series.get(p, 0.0) for p in all_periods}

    profit = {
        "report": "profit",
        "account_name": "Grand Total",
        "aiDriver": ai_insights
    }

    for p in visible_periods:
        profit[p] = profit_series.get(p, 0.0)

    profit["Total"] = sum(profit[p] for p in visible_periods)
    profit["comparisons"] = {"monthly": [], "total": []}

    if comparison:
        profit["comparisons"]["monthly"] = build_monthly_comparisons(profit_series, comparison, visible_periods)

        prev_profit_total = sum(
            profit_series[p] for p in profit_series
            if p.startswith(str(year - 1))
        )
        profit["comparisons"]["total"] = build_total_comparison(profit["Total"], prev_profit_total, year)

    margin_series = {}
    for p in all_periods:
        rev = revenue_series.get(p, 0.0)
        prof = profit_series.get(p, 0.0)
        margin_series[p] = round((prof / rev) * 100, 2) if rev else None

    margin = {
        "report": "margin",
        "account_name": "Grand Total",
        "aiDriver": ai_insights
    }

    for p in visible_periods:
        margin[p] = margin_series.get(p)

    margin["Total"] = round((profit["Total"] / revenue_gt["Total"]) * 100, 2) if revenue_gt["Total"] else None
    margin["comparisons"] = {"monthly": [], "total": []}

    if comparison:
        margin["comparisons"]["monthly"] = build_monthly_comparisons(margin_series, comparison, visible_periods)

        prev_rev_total = sum(revenue_series[p] for p in revenue_series if p.startswith(str(year - 1)))
        prev_cost_total = sum(cost_series[p] for p in cost_series if p.startswith(str(year - 1)))
        prev_profit_total = prev_rev_total - prev_cost_total
        prev_margin_total = round((prev_profit_total / prev_rev_total) * 100, 2) if prev_rev_total else None

        margin["comparisons"]["total"] = [{
            "base_period": str(year),
            "compare_period": str(year - 1),
            "base_value": margin["Total"],
            "compare_value": prev_margin_total,
            "pct_change": None if prev_margin_total in (None, 0) else round(((margin["Total"] - prev_margin_total) / prev_margin_total) * 100, 2)
        }]

    keymetrics = {
        "total_revenue": revenue_gt["Total"],
        "total_cost": cost_gt["Total"],
        "total_profit": profit["Total"],
        "margin": margin["Total"],
        "ai_insights": ai_insights
    }

    response = {
        "keymetrics": keymetrics,
        "revenue": revenue_rows,
        "revenue_grand_total": revenue_gt,
        "cost": cost_rows,
        "cost_grand_total": cost_gt,
        "profit": profit,
        "margin": margin
    }

    response = apply_smart_currency(
        response,
        currency,
        currency_filter.value,
        money_fields + [
            "total_revenue",
            "total_cost",
            "total_profit",
            "base_value",
            "compare_value"
        ]
    )

    return response


## Revenue report by product============================================================
#=============================================================================


class RevenueComparisonOption(str, Enum):
    month_on_month = "month_on_month"
    year_on_year = "year_on_year"

class RevenueSortByOption(str, Enum):
    customer_name = "customer_name"
    product_name = "product_name"
    entity_name = "entity_name"
    region = "region"
    category = "category"
    period = "period"
    revenue = "revenue"

class SortOrderOption(str, Enum):
    asc = "asc"
    desc = "desc"

class RevenueViewMode(str, Enum):
    monthly = "monthly"
    quarterly = "quarterly"

class QuarterOption(str, Enum):
    none = "none"
    q1 = "q1"
    q2 = "q2"
    q3 = "q3"
    q4 = "q4"

QUARTER_MONTH_RANGES = {
    QuarterOption.q1: ("01", "03"),
    QuarterOption.q2: ("04", "06"),
    QuarterOption.q3: ("07", "09"),
    QuarterOption.q4: ("10", "12"),
}

QUARTER_DROPDOWN = [q.value for q in QuarterOption]

def get_revenue_period_filter(period, year, date_from=None, date_to=None, quarter=None):
    now = pd.Timestamp.today()
    system_month = now.month

    if quarter and quarter != QuarterOption.none and quarter in QUARTER_MONTH_RANGES:
        start_m, end_m = QUARTER_MONTH_RANGES[quarter]
        return f"AND gl.period BETWEEN '{year}-{start_m}' AND '{year}-{end_m}'"

    if period == PeriodOption.custom_range:
        if date_from and date_to:
            return f"AND gl.period BETWEEN '{date_from}' AND '{date_to}'"
        if date_from:
            return f"AND gl.period >= '{date_from}'"
        if date_to:
            return f"AND gl.period <= '{date_to}'"

    if period == PeriodOption.last_month:
        last_month = (pd.Timestamp(year=year, month=system_month, day=1) - pd.DateOffset(months=1))
        return f"AND gl.period = '{last_month.year}-{last_month.month:02d}'"

    return {
        PeriodOption.all: f"AND gl.period LIKE '{year}-%'",
        PeriodOption.current_month: f"AND gl.period = '{year}-{system_month:02d}'",
        PeriodOption.quarter_to_date: f"AND gl.period BETWEEN '{year}-01' AND '{year}-03'",
        PeriodOption.year_to_date: f"AND gl.period BETWEEN '{year}-01' AND '{now.year}-{system_month:02d}'"
    }.get(period, "")

def get_comparison_period(period_str: str, comparison: ComparisonOption):
    if not period_str:
        return None
    dt = pd.to_datetime(period_str, format="%Y-%m")
    if comparison == ComparisonOption.mom:
        return (dt - pd.DateOffset(months=1)).strftime("%Y-%m")
    if comparison == ComparisonOption.yoy:
        return (dt - pd.DateOffset(years=1)).strftime("%Y-%m")
    return None

def get_quarterly_compare_period(pc: str, comparison: RevenueComparisonOption):
    try:
        yr, q = pc.split("-")
        q_num = int(q[1])
        if comparison == RevenueComparisonOption.month_on_month:
            if q_num == 1:
                return f"{int(yr) - 1}-Q4"
            return f"{yr}-Q{q_num - 1}"
        if comparison == RevenueComparisonOption.year_on_year:
            return f"{int(yr) - 1}-{q}"
    except Exception:
        return None
    return None

def resolve_compare_period(pc: str, comparison: RevenueComparisonOption, view_mode: RevenueViewMode):
    if view_mode == RevenueViewMode.quarterly:
        return get_quarterly_compare_period(pc, comparison)
    return get_comparison_period(pc, comparison)

def safe_float(value, default=0.0):
    try:
        result = float(value)
        if math.isnan(result) or math.isinf(result):
            return default
        return result
    except (TypeError, ValueError):
        return default

def safe_float_or_none(value):
    try:
        result = float(value)
        if math.isnan(result) or math.isinf(result):
            return None
        return result
    except (TypeError, ValueError):
        return None

def format_revenue(value, currency, currency_filter):
    raw = safe_float(value)
    result = apply_smart_currency({"v": raw}, currency, currency_filter, ["v"])
    return result["v"]

def build_revenue_excel_response(df, filename="revenue_report.xlsx"):
    output = io.BytesIO()

    with pd.ExcelWriter(output, engine="openpyxl") as writer:
        df.to_excel(writer, index=False, sheet_name="Revenue Report")

        ws = writer.sheets["Revenue Report"]

        for cell in ws[1]:
            cell.font = Font(bold=True, color="FFFFFF")
            cell.fill = PatternFill("solid", fgColor="1F4E79")

        for col in ws.columns:
            width = max(len(str(c.value or "")) for c in col) + 5
            ws.column_dimensions[col[0].column_letter].width = min(width, 40)

        ws.freeze_panes = "A2"

    output.seek(0)

    return StreamingResponse(
        output,
        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        headers={"Content-Disposition": f'attachment; filename="{filename}"'}
    )

def apply_currency_to_comparison(comparisons: dict, currency: str, currency_filter) -> dict:
    if not comparisons:
        return comparisons

    fields_to_convert = ["current_revenue", "compare_revenue", "variance"]

    converted = apply_smart_currency(
        {k: comparisons[k] for k in fields_to_convert},
        currency,
        currency_filter,
        fields_to_convert
    )

    return {
        **comparisons,
        "current_revenue": converted["current_revenue"],
        "compare_revenue": converted["compare_revenue"],
        "variance": converted["variance"]
    }

def build_quarterly_aggregation(df, group_columns_for_agg, year):
    def get_quarter_label(period_str):
        try:
            month = int(period_str.split("-")[1])
            if month <= 3:
                return f"{year}-Q1"
            elif month <= 6:
                return f"{year}-Q2"
            elif month <= 9:
                return f"{year}-Q3"
            else:
                return f"{year}-Q4"
        except Exception:
            return f"{year}-Q?"

    df = df.copy()
    df["quarter_label"] = df["period"].apply(get_quarter_label)

    non_period_cols = [c for c in group_columns_for_agg if c != "period"]
    agg_cols = non_period_cols + ["quarter_label"]

    df_q = df.groupby(agg_cols, as_index=False)["revenue"].sum()
    df_q = df_q.rename(columns={"quarter_label": "period"})

    return df_q

def build_excel_export_pivoted(df_pivoted, period_columns, dim_columns):
    all_columns = dim_columns + period_columns + ["total_revenue"]
    export_rows = []

    for _, row in df_pivoted.iterrows():
        export_row = {}
        for col in all_columns:
            if col in row:
                export_row[col] = row[col]
        export_rows.append(export_row)

    return pd.DataFrame(export_rows)

def build_excel_export(products, grand_total, visible_columns, comparison):
    rows = []

    for p in products:
        row = {}

        for col in visible_columns:
            if col != "revenue":
                row[col] = p.get(col, "")

        row["revenue"] = p.get("revenue", 0)

        if comparison and p.get("comparisons"):
            comp = p["comparisons"]
            row["current_period"] = comp.get("current_period", "")
            row["compare_period"] = comp.get("compare_period", "")
            row["current_revenue"] = comp.get("current_revenue", 0)
            row["compare_revenue"] = comp.get("compare_revenue", 0)
            row["variance"] = comp.get("variance", 0)
            row["pct_change_revenue"] = comp.get("pct_change_revenue", None)

        rows.append(row)

    gt_row = {}
    for col in visible_columns:
        if col != "revenue":
            gt_row[col] = grand_total.get(col, "")
    gt_row["revenue"] = grand_total.get("revenue", 0)

    if comparison and grand_total.get("comparisons"):
        comp = grand_total["comparisons"]
        gt_row["current_period"] = comp.get("current_period", "")
        gt_row["compare_period"] = comp.get("compare_period", "")
        gt_row["current_revenue"] = comp.get("current_revenue", 0)
        gt_row["compare_revenue"] = comp.get("compare_revenue", 0)
        gt_row["variance"] = comp.get("variance", 0)
        gt_row["pct_change_revenue"] = comp.get("pct_change_revenue", None)

    rows.append(gt_row)

    return pd.DataFrame(rows)

@financial_router.get(
    "/Revenue_Report/{year}/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def Revenue_Report(
    request: Request,
    year: int = Path(...),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    view_mode: RevenueViewMode = Query(RevenueViewMode.monthly),
    period: PeriodOption = Query(PeriodOption.all),
    date_from: str | None = Query(None),
    date_to: str | None = Query(None),
    quarter: QuarterOption = Query(QuarterOption.none),
    comparison: RevenueComparisonOption | None = Query(None),
    sort_by: RevenueSortByOption = Query(RevenueSortByOption.revenue),
    sort_order: SortOrderOption = Query(SortOrderOption.desc),
    customer_name: str | None = Query(None),
    product_name: str | None = Query(None),
    region: str | None = Query(None),
    entity_name: str | None = Query(None),
    category: str | None = Query(None),
    show_period: bool = Query(False),
    show_region: bool = Query(False),
    show_entity: bool = Query(False),
    show_department: bool = Query(False),
    show_product: bool = Query(False),
    page: int = Query(1, ge=1),
    page_size: int = Query(3000, ge=1, le=3000),
    excel_download: bool = Query(False),
    db: AsyncSession = Depends(get_db),
):
    if date_from:
        pd.to_datetime(date_from, format="%Y-%m")

    if date_to:
        pd.to_datetime(date_to, format="%Y-%m")

    if date_from and date_to and date_from > date_to:
        raise HTTPException(status_code=400, detail="date_from must not be later than date_to")

    if (date_from or date_to) and period != PeriodOption.custom_range:
        period = PeriodOption.custom_range

    active_quarter = quarter if (view_mode == RevenueViewMode.quarterly and quarter != QuarterOption.none) else None

    filters = []
    params = {}

    filter_map = {
        "customer_name": ("c.customer_name", customer_name),
        "product_name": ("p.product_name", product_name),
        "region": ("c.region", region),
        "entity_name": ("e.entity_name", entity_name),
        "category": ("d.department_name", category)
    }

    for key, (column, value) in filter_map.items():
        if value:
            filters.append(f"{column} = :{key}")
            params[key] = value

    filter_sql = f"AND {' AND '.join(filters)}" if filters else ""

    sql = text(f"""
        WITH product_dim AS (
            SELECT
                profit_center_id,
                MAX(NULLIF(TRIM(product_name), '')) AS product_name
            FROM fpna_landing.crm_products
            GROUP BY profit_center_id
        )

        SELECT
            COALESCE(c.customer_name,'Unknown') AS customer_name,
            COALESCE(p.product_name,'UNMAPPED PRODUCT') AS product_name,
            COALESCE(e.entity_name,'Unknown') AS entity_name,
            COALESCE(c.region,'Unknown') AS region,
            COALESCE(d.department_name,'Unknown') AS category,
            gl.period,
            SUM(gl.amount_credit - gl.amount_debit) AS revenue

        FROM fpna_landing.finance_general_ledger_transactions gl

        JOIN fpna_landing.finance_chart_of_accounts coa
            ON gl.gl_account_id = coa.account_id

        LEFT JOIN fpna_landing.finance_customers c
            ON REGEXP_REPLACE(gl.customer_id::TEXT,'\\.0$','')
             = REGEXP_REPLACE(c.customer_id::TEXT,'\\.0$','')

        LEFT JOIN fpna_landing.finance_entities e
            ON gl.entity_id::TEXT = e.entity_id::TEXT

        LEFT JOIN fpna_landing.finance_departments d
            ON gl.department_id::TEXT = d.department_id::TEXT

        LEFT JOIN product_dim p
            ON gl.profit_center_id::TEXT = p.profit_center_id::TEXT

        WHERE coa.account_type = 'Revenue'
        {get_revenue_period_filter(period, year, date_from, date_to, active_quarter)}
        {filter_sql}

        GROUP BY
            c.customer_name,
            p.product_name,
            e.entity_name,
            c.region,
            d.department_name,
            gl.period
    """)

    rows = (await db.execute(sql, params)).mappings().all()

    show_columns = {
        "product_name": show_product,
        "entity_name": show_entity,
        "region": show_region,
        "category": show_department,
        "period": show_period
    }

    if not rows:
        return {
            "key_metrics": {},
            "products": [],
            "period_columns": [],
            "visible_columns": ["customer_name", "revenue"],
            "view_mode": view_mode.value,
            "active_quarter": active_quarter.value if active_quarter else None,
            "quarter_options": QUARTER_DROPDOWN,
            "customers": [],
            "products_list": [],
            "regions": [],
            "entities": [],
            "categories": [],
            "show_columns": show_columns
        }

    df_full = pd.DataFrame(rows, dtype=object)
    df_full["revenue"] = pd.to_numeric(df_full["revenue"], errors="coerce").fillna(0.0)

    unique_customers = sorted(df_full["customer_name"].dropna().unique().tolist())
    unique_products = sorted(df_full["product_name"].dropna().unique().tolist())
    unique_regions = sorted(df_full["region"].dropna().unique().tolist())
    unique_entities = sorted(df_full["entity_name"].dropna().unique().tolist())
    unique_categories = sorted(df_full["category"].dropna().unique().tolist())

    df = df_full.copy()

    dim_columns = ["customer_name"]
    if show_product:
        dim_columns.append("product_name")
    if show_entity:
        dim_columns.append("entity_name")
    if show_region:
        dim_columns.append("region")
    if show_department:
        dim_columns.append("category")

    visible_columns = dim_columns + ["revenue"]

    show_columns = {
        "product_name": show_product,
        "entity_name": show_entity,
        "region": show_region,
        "category": show_department,
        "period": show_period
    }

    if view_mode == RevenueViewMode.quarterly:
        def get_quarter_label(period_str):
            try:
                month = int(period_str.split("-")[1])
                if month <= 3:
                    return f"{year}-Q1"
                elif month <= 6:
                    return f"{year}-Q2"
                elif month <= 9:
                    return f"{year}-Q3"
                else:
                    return f"{year}-Q4"
            except Exception:
                return f"{year}-Q?"

        df["period"] = df["period"].apply(get_quarter_label)

    df_grouped = df.groupby(dim_columns + ["period"], as_index=False)["revenue"].sum()
    df_grouped["revenue"] = df_grouped["revenue"].replace([float("inf"), float("-inf")], 0.0).fillna(0.0)

    all_period_columns = sorted(df_grouped["period"].dropna().unique().tolist())

    df_pivoted = df_grouped.pivot_table(
        index=dim_columns,
        columns="period",
        values="revenue",
        aggfunc="sum",
        fill_value=0
    ).reset_index()

    df_pivoted.columns.name = None
    df_pivoted.columns = [str(c) for c in df_pivoted.columns]

    for pc in all_period_columns:
        if pc not in df_pivoted.columns:
            df_pivoted[pc] = 0.0

    df_pivoted[all_period_columns] = df_pivoted[all_period_columns].apply(pd.to_numeric, errors="coerce").fillna(0.0)

    df_pivoted["total_revenue"] = df_pivoted[all_period_columns].sum(axis=1)
    df_pivoted["total_revenue"] = df_pivoted["total_revenue"].replace([float("inf"), float("-inf")], 0.0).fillna(0.0)

    sort_col = sort_by.value
    if sort_col == "revenue" or sort_col == "period":
        sort_col = "total_revenue"
    if sort_col not in df_pivoted.columns:
        sort_col = "total_revenue"

    df_pivoted = df_pivoted.sort_values(
        by=sort_col,
        ascending=(sort_order == SortOrderOption.asc)
    ).reset_index(drop=True)

    total_revenue = safe_float(df_pivoted["total_revenue"].sum())
    highest_revenue = safe_float(df_pivoted["total_revenue"].max())
    lowest_revenue = safe_float(df_pivoted["total_revenue"].min())
    average_revenue = safe_float(df_pivoted["total_revenue"].mean())

    total_records = len(df_pivoted)
    total_pages = math.ceil(total_records / page_size)

    start = (page - 1) * page_size
    end = start + page_size

    df_page = df_pivoted.iloc[start:end]

    period_columns_response = all_period_columns if show_period else []

    products = []

    for _, row in df_page.iterrows():
        out = {}

        for dc in dim_columns:
            out[dc] = row.get(dc, "")

        period_revenues_raw = {pc: safe_float(row.get(pc, 0.0)) for pc in all_period_columns}

        if show_period:
            out["period_revenues"] = {
                pc: format_revenue(period_revenues_raw[pc], currency, currency_filter)
                for pc in all_period_columns
            }
        else:
            out["period_revenues"] = {}

        total_raw = safe_float(row.get("total_revenue", 0.0))
        out["revenue"] = format_revenue(total_raw, currency, currency_filter)
        out["ai_insights"] = ai_insights
        out["comparisons"] = {}

        if comparison is not None:
            comp_period_revenues = {}

            for pc in all_period_columns:
                current_raw = period_revenues_raw[pc]

                compare_pc = resolve_compare_period(pc, comparison, view_mode)

                compare_raw = 0.0
                if compare_pc:
                    compare_row_mask = df_pivoted.copy()
                    for dc in dim_columns:
                        compare_row_mask = compare_row_mask[compare_row_mask[dc] == row[dc]]
                    if compare_pc in df_pivoted.columns and not compare_row_mask.empty:
                        compare_raw = safe_float(compare_row_mask.iloc[0].get(compare_pc, 0.0))

                variance_raw = current_raw - compare_raw
                pct = round((variance_raw / compare_raw) * 100, 2) if compare_raw != 0 else None
                if pct is not None and (math.isnan(pct) or math.isinf(pct)):
                    pct = None

                comp_period_revenues[pc] = {
                    "comparison_type": comparison.value,
                    "current_period": pc,
                    "compare_period": compare_pc,
                    "current_revenue": format_revenue(current_raw, currency, currency_filter),
                    "compare_revenue": format_revenue(compare_raw, currency, currency_filter),
                    "variance": format_revenue(variance_raw, currency, currency_filter),
                    "pct_change_revenue": pct
                }

            out["comparisons"] = comp_period_revenues

        products.append(out)

    grand_period_revenues_raw = {}
    for pc in all_period_columns:
        col_sum = safe_float(df_pivoted[pc].sum()) if pc in df_pivoted.columns else 0.0
        grand_period_revenues_raw[pc] = col_sum

    grand_total_raw = safe_float(df_pivoted["total_revenue"].sum())

    grand_total = {
        "customer_name": "Grand Total",
        "period_revenues": (
            {pc: format_revenue(grand_period_revenues_raw[pc], currency, currency_filter) for pc in all_period_columns}
            if show_period else {}
        ),
        "revenue": format_revenue(grand_total_raw, currency, currency_filter),
        "ai_insights": ai_insights,
        "comparisons": {}
    }

    if comparison is not None:
        grand_comp_period_revenues = {}

        for pc in all_period_columns:
            current_raw = grand_period_revenues_raw.get(pc, 0.0)

            compare_pc = resolve_compare_period(pc, comparison, view_mode)

            compare_raw = 0.0
            if compare_pc and compare_pc in df_pivoted.columns:
                compare_raw = safe_float(df_pivoted[compare_pc].sum())

            variance_raw = current_raw - compare_raw
            pct = round((variance_raw / compare_raw) * 100, 2) if compare_raw != 0 else None
            if pct is not None and (math.isnan(pct) or math.isinf(pct)):
                pct = None

            grand_comp_period_revenues[pc] = {
                "comparison_type": comparison.value,
                "current_period": pc,
                "compare_period": compare_pc,
                "current_revenue": format_revenue(current_raw, currency, currency_filter),
                "compare_revenue": format_revenue(compare_raw, currency, currency_filter),
                "variance": format_revenue(variance_raw, currency, currency_filter),
                "pct_change_revenue": pct
            }

        grand_total["comparisons"] = grand_comp_period_revenues

    key_metrics = apply_smart_currency(
        {
            "total_revenue": total_revenue,
            "highest_revenue": highest_revenue,
            "lowest_revenue": lowest_revenue,
            "average_revenue": average_revenue,
            "ai_insights": ai_insights
        },
        currency,
        currency_filter,
        ["total_revenue", "highest_revenue", "lowest_revenue", "average_revenue"]
    )

    products.append(grand_total)

    if excel_download:
        export_df = build_excel_export_pivoted(df_pivoted, all_period_columns, dim_columns)
        suffix_parts = [view_mode.value]
        if active_quarter:
            suffix_parts.append(active_quarter.value)
        if comparison:
            suffix_parts.append(comparison.value)
        filename = f"revenue_report_{year}_{currency}_{'_'.join(suffix_parts)}.xlsx"
        return build_revenue_excel_response(export_df, filename)

    return {
        "key_metrics": key_metrics,
        "products": products,
        "period_columns": period_columns_response,
        "pagination": {
            "page": page,
            "page_size": page_size,
            "total_records": total_records,
            "total_pages": total_pages
        },
        "customers": unique_customers,
        "products_list": unique_products,
        "regions": unique_regions,
        "entities": unique_entities,
        "categories": unique_categories,
        "show_columns": show_columns,
        "visible_columns": visible_columns,
        "view_mode": view_mode.value,
        "active_quarter": active_quarter.value if active_quarter else None,
        "quarter_options": QUARTER_DROPDOWN
    }


########################################
######################################
###################################

#Revenue_report_viewmore

def build_display_periods(year: int, period: PeriodOption, system_month: int):
    if period == PeriodOption.current_month:
        return [f"{year}-{system_month:02d}"]

    if period == PeriodOption.last_month:
        if system_month == 1:
            return [f"{year-1}-12"]
        return [f"{year}-{system_month-1:02d}"]

    if period == PeriodOption.quarter_to_date:
        quarter_start = ((system_month - 1) // 3) * 3 + 1
        return [f"{year}-{m:02d}" for m in range(quarter_start, system_month + 1)]

    if period == PeriodOption.year_to_date:
        return [f"{year}-{m:02d}" for m in range(1, system_month + 1)]

    return [f"{year}-{m:02d}" for m in range(1, 13)]


@financial_router.get(
    "/Revenue_Report_ViewMore/{year}/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def Revenue_Report_ViewMore(
    year: int = Path(...),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    period: PeriodOption = Query(PeriodOption.all),
    comparison: ComparisonOption | None = Query(None),

    customer_name: str | None = Query(default=None),
    account_name: str | None = Query(default=None),

    db: AsyncSession = Depends(get_db)
):
    today = datetime.now()
    system_month = today.month

    display_periods = build_display_periods(year, period, system_month)

    sql_periods = set(display_periods)
    if comparison:
        for p in display_periods:
            sql_periods.add(get_comparison_period(p, comparison))

    sql_periods = sorted(filter(None, sql_periods))

    sql = text("""
        SELECT
            COALESCE(c.customer_name, 'Unknown') AS "Customer_name",
            coa.account_type,
            coa.account_name,
            gl.period,
            SUM(gl.amount_credit - gl.amount_debit) AS amount
        FROM fpna_landing.finance_general_ledger_transactions gl
        JOIN fpna_landing.finance_chart_of_accounts coa
            ON gl.gl_account_id = coa.account_id
        LEFT JOIN fpna_landing.finance_customers c
            ON CAST(gl.customer_id AS NUMERIC)::BIGINT
             = CAST(c.customer_id AS NUMERIC)::BIGINT
        WHERE coa.account_type = 'Revenue'
          AND gl.profit_center_id IS NOT NULL
          AND gl.customer_id IS NOT NULL
          AND gl.period = ANY(:periods)
        GROUP BY
            c.customer_name,
            coa.account_type,
            coa.account_name,
            gl.period
        ORDER BY
            c.customer_name,
            coa.account_name,
            gl.period
    """)

    df = pd.DataFrame(
        (await db.execute(sql, {"periods": sql_periods})).mappings().all()
    )

    if df.empty:
        return {
            "products": [],
            "grand_total": {},
            "key_metrics": {},
            "customer_dropdown": [],
            "account_dropdown": []
        }

    if customer_name:
        df = df[df["Customer_name"] == customer_name]

    if account_name:
        df = df[df["account_name"] == account_name]

    df_pivot = df.pivot_table(
        index=["Customer_name", "account_type", "account_name"],
        columns="period",
        values="amount",
        fill_value=0
    ).reset_index()

    # Ensure display periods exist
    for p in display_periods:
        if p not in df_pivot.columns:
            df_pivot[p] = 0

    # Ensure comparison periods exist
    if comparison:
        for p in display_periods:
            comp_p = get_comparison_period(p, comparison)
            if comp_p and comp_p not in df_pivot.columns:
                df_pivot[comp_p] = 0

    df_pivot["total"] = df_pivot[display_periods].sum(axis=1)

    # ------------------------------------------------------------
    # KEY METRICS (BASED ON FILTERED DATA)
    # ------------------------------------------------------------
    total_revenue = float(df_pivot["total"].sum())

    customer_totals = (
        df_pivot.groupby("Customer_name")["total"]
        .sum()
        .sort_values(ascending=False)
    )

    top_customer = None
    lowest_customer = None

    if not customer_totals.empty:
        top_customer = {
            "Customer_name": str(customer_totals.index[0]),
            "value": float(customer_totals.iloc[0])
        }

        lowest_customer = {
            "Customer_name": str(customer_totals.index[-1]),
            "value": float(customer_totals.iloc[-1])
        }

    growth_rate = 0

    if comparison:
        comp_total = 0.0
        for p in display_periods:
            comp_p = get_comparison_period(p, comparison)
            if comp_p:
                comp_total += float(df_pivot[comp_p].sum())

        if comp_total == 0:
            growth_rate = 0 if total_revenue == 0 else 100
        else:
            growth_rate = round(((total_revenue - comp_total) / comp_total) * 100, 2)
    else:
        growth_rate = 0

    key_metrics = {
        "total_revenue": total_revenue,
        "top_customer": top_customer,
        "lowest_customer": lowest_customer,
        "growth_rate": growth_rate
    }

    # ------------------------------------------------------------
    # BUILD PRODUCTS WITH CUSTOMER TOTAL ROW (pct_change BEFORE currency)
    # ------------------------------------------------------------
    products = []

    df_pivot = df_pivot.sort_values(["Customer_name", "account_name"]).reset_index(drop=True)

    for cust in df_pivot["Customer_name"].unique():
        cust_rows = df_pivot[df_pivot["Customer_name"] == cust]

        # normal rows
        for _, row in cust_rows.iterrows():
            rec = row.to_dict()
            rec["comparisons"] = {}

            if comparison:
                for p in display_periods:
                    comp_p = get_comparison_period(p, comparison)
                    prev_val = float(row.get(comp_p, 0))
                    curr_val = float(row.get(p, 0))

                    rec["comparisons"][p] = {
                        "comparison_period": comp_p,
                        "value": prev_val,
                        "pct_change": (
                            round(((curr_val - prev_val) / prev_val) * 100, 2)
                            if prev_val else None
                        )
                    }

            for k in list(rec.keys()):
                if (
                    k not in display_periods
                    and k not in ["Customer_name", "account_type", "account_name", "total", "comparisons"]
                ):
                    rec.pop(k)

            for p in display_periods:
                rec[p] = float(rec.get(p, 0))
            rec["total"] = float(rec.get("total", 0))

            products.append(rec)

        # customer total row
        cust_total = {
            "Customer_name": cust,
            "account_type": "Revenue",
            "account_name": "total",
            **{p: float(cust_rows[p].sum()) for p in display_periods},
        }
        cust_total["total"] = float(sum(cust_total[p] for p in display_periods))

        cust_total["comparisons"] = {}

        if comparison:
            for p in display_periods:
                comp_p = get_comparison_period(p, comparison)
                prev_val = float(cust_rows[comp_p].sum()) if comp_p in cust_rows.columns else 0.0
                curr_val = float(cust_total[p])

                cust_total["comparisons"][p] = {
                    "comparison_period": comp_p,
                    "value": prev_val,
                    "pct_change": (
                        round(((curr_val - prev_val) / prev_val) * 100, 2)
                        if prev_val else None
                    )
                }

        products.append(cust_total)

    # ------------------------------------------------------------
    # OVERALL GRAND TOTAL
    # ------------------------------------------------------------
    grand_total = {
        "Customer_name": "All Customers",
        "account_type": "Revenue",
        "account_name": "Revenue Total",
        **{p: float(df_pivot[p].sum()) for p in display_periods},
        "total": float(df_pivot["total"].sum())
    }

    numeric_fields = display_periods + ["total"]

    # ------------------------------------------------------------
    # APPLY CURRENCY CONVERSION
    # ------------------------------------------------------------
    products = apply_smart_currency(products, currency, currency_filter, numeric_fields)

    grand_total = apply_smart_currency([grand_total], currency, currency_filter, numeric_fields)[0]

    key_metrics = apply_smart_currency(key_metrics, currency, currency_filter, ["total_revenue"])

    if key_metrics.get("top_customer"):
        key_metrics["top_customer"] = apply_smart_currency(
            [key_metrics["top_customer"]],
            currency,
            currency_filter,
            ["value"]
        )[0]

    if key_metrics.get("lowest_customer"):
        key_metrics["lowest_customer"] = apply_smart_currency(
            [key_metrics["lowest_customer"]],
            currency,
            currency_filter,
            ["value"]
        )[0]

    # ------------------------------------------------------------
    # APPLY CURRENCY TO COMPARISON VALUES ONLY (DO NOT RECALC pct_change)
    # ------------------------------------------------------------
    if comparison:
        for rec in products:
            if "comparisons" not in rec or not isinstance(rec["comparisons"], dict):
                continue

            for p in display_periods:
                if p not in rec["comparisons"]:
                    continue

                comp_obj = rec["comparisons"][p]
                if not isinstance(comp_obj, dict):
                    continue

                comp_obj["value"] = apply_smart_currency(
                    [{"value": float(comp_obj.get("value", 0))}],
                    currency,
                    currency_filter,
                    ["value"]
                )[0]["value"]

    return {
        "products": products,
        "grand_total": grand_total,
        "key_metrics": key_metrics,
        "customer_dropdown": sorted(df["Customer_name"].unique()),
        "account_dropdown": sorted(df["account_name"].unique())
    }


#############################################################################
################################################################################
###############################################################################

#Project cost report

class StatusOption(str, Enum):
    all = "all"
    active = "active"
    closed = "closed"

class ProjectSortByOption(str, Enum):
    period = "period"
    entity_name = "entity_name"
    region = "region"
    department_name = "department_name"
    cost_center_id = "cost_center_id"
    project_name = "project_name"
    product_name = "product_name"
    actual = "actual"
    budget = "budget"
    variance = "variance"
    completion_percentage = "completion_percentage"


def normalize(val: Optional[str]) -> Optional[str]:
    if not val or val.strip().lower() == "all":
        return None
    return val


def build_project_excel_response(df: pd.DataFrame, filename: str = "project_costing_report.xlsx") -> StreamingResponse:
    output = io.BytesIO()
    with pd.ExcelWriter(output, engine="openpyxl") as writer:
        export_df = df.copy()
        export_df.to_excel(writer, index=False, sheet_name="Project Costing Report")
        worksheet = writer.sheets["Project Costing Report"]

        thin_border = Border(
            left=Side(style="thin"),
            right=Side(style="thin"),
            top=Side(style="thin"),
            bottom=Side(style="thin")
        )
        number_format = '#,##0.00'
        pct_format = '0.00"%"'
        currency_cols = {"actual", "budget", "variance"}
        comparison_currency_cols = {
            "comparison_current_actual", "comparison_previous_actual",
            "comparison_current_budget", "comparison_previous_budget",
            "comparison_current_variance", "comparison_previous_variance"
        }
        pct_cols = {"completion_percentage"}

        col_widths = {}
        for col_idx, col_name in enumerate(export_df.columns, start=1):
            col_letter = get_column_letter(col_idx)
            cell = worksheet.cell(row=1, column=col_idx)
            cell.border = thin_border
            col_widths[col_letter] = max(len(str(col_name)) + 4, 14)

        for row_idx, row in enumerate(export_df.itertuples(index=False), start=2):
            for col_idx, (col_name, value) in enumerate(zip(export_df.columns, row), start=1):
                col_letter = get_column_letter(col_idx)
                cell = worksheet.cell(row=row_idx, column=col_idx, value=value)
                cell.border = thin_border
                cell.alignment = Alignment(horizontal="left")
                if col_name in currency_cols or col_name in comparison_currency_cols:
                    cell.number_format = number_format
                    cell.alignment = Alignment(horizontal="right")
                elif col_name in pct_cols:
                    cell.number_format = pct_format
                    cell.alignment = Alignment(horizontal="right")
                cell_len = len(str(value)) + 4 if value is not None else 4
                if cell_len > col_widths.get(col_letter, 0):
                    col_widths[col_letter] = cell_len

        for col_letter, width in col_widths.items():
            worksheet.column_dimensions[col_letter].width = min(width, 40)

        worksheet.freeze_panes = "A2"

    output.seek(0)
    headers = {
        "Content-Disposition": f'attachment; filename="{filename}"',
        "Content-Type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
    }
    return StreamingResponse(
        output,
        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        headers=headers
    )


@financial_router.get("/Project_Costing_Report/{year}/{currency}")
async def project_costing_report(
    currency: str,
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    year: int = Path(...),
    period: PeriodOption = Query(PeriodOption.all),
    date_from: str | None = Query(None, description="Custom range start in YYYY-MM format"),
    date_to: str | None = Query(None, description="Custom range end in YYYY-MM format"),
    status: StatusOption = Query(StatusOption.all),
    region: Optional[str] = Query(None),
    entity_name: Optional[str] = Query(None),
    department_name: Optional[str] = Query(None),
    product_name: Optional[str] = Query(None),
    project_name: Optional[str] = Query(None),
    comparison: Optional[ComparisonOption] = Query(None, enum=[e.value for e in ComparisonOption]),
    sort_by: ProjectSortByOption = Query(ProjectSortByOption.period),
    sort_order: SortOrderOption = Query(SortOrderOption.asc),
    show_period: bool = Query(False, description="Include period column in response"),
    show_entity_name: bool = Query(False, description="Include entity_name column in response"),
    show_region: bool = Query(False, description="Include region column in response"),
    show_department_name: bool = Query(False, description="Include department_name column in response"),
    show_cost_center_id: bool = Query(False, description="Include cost_center_id column in response"),
    show_product_name: bool = Query(False, description="Include product_name column in response"),
    excel_download: bool = Query(False),
    db: AsyncSession = Depends(get_db)
):
    if date_from:
        try:
            pd.to_datetime(date_from, format="%Y-%m")
        except ValueError:
            raise HTTPException(status_code=400, detail="date_from must be in YYYY-MM format")

    if date_to:
        try:
            pd.to_datetime(date_to, format="%Y-%m")
        except ValueError:
            raise HTTPException(status_code=400, detail="date_to must be in YYYY-MM format")

    if date_from and date_to and date_from > date_to:
        raise HTTPException(status_code=400, detail="date_from must not be later than date_to")

    if (date_from or date_to) and period != PeriodOption.custom_range:
        period = PeriodOption.custom_range

    region          = normalize(region)
    entity_name     = normalize(entity_name)
    department_name = normalize(department_name)
    product_name    = normalize(product_name)
    project_name    = normalize(project_name)

    system_date  = pd.Timestamp.today()
    system_month = system_date.month
    system_year  = system_date.year

    sort_column_map = {
        ProjectSortByOption.period:                "period",
        ProjectSortByOption.entity_name:           "entity_name",
        ProjectSortByOption.region:                "region",
        ProjectSortByOption.department_name:       "department_name",
        ProjectSortByOption.cost_center_id:        "cost_center_id",
        ProjectSortByOption.project_name:          "project_name",
        ProjectSortByOption.product_name:          "product_name",
        ProjectSortByOption.actual:                "actual_cost",
        ProjectSortByOption.budget:                "budget",
        ProjectSortByOption.variance:              "variance",
        ProjectSortByOption.completion_percentage: "completion_percentage",
    }

    sort_col = sort_column_map.get(sort_by, "period")
    sort_dir = "ASC" if sort_order == SortOrderOption.asc else "DESC"

    def get_gl_period_filter(p: PeriodOption, y: int) -> str:
        if p == PeriodOption.custom_range:
            if date_from and date_to:
                return f"AND gl.period BETWEEN '{date_from}' AND '{date_to}'"
            if date_from:
                return f"AND gl.period >= '{date_from}'"
            if date_to:
                return f"AND gl.period <= '{date_to}'"
            return f"AND gl.period LIKE '{y}-%'"
        if p == PeriodOption.all:
            return f"AND gl.period LIKE '{y}-%'"
        if p == PeriodOption.current_month:
            return f"AND gl.period = '{y}-{system_month:02d}'"
        if p == PeriodOption.last_month:
            last_month = pd.Timestamp(year=y, month=system_month, day=1) - pd.DateOffset(months=1)
            return f"AND gl.period = '{last_month.year}-{last_month.month:02d}'"
        if p == PeriodOption.quarter_to_date:
            return f"AND gl.period BETWEEN '{y}-01' AND '{y}-03'"
        if p == PeriodOption.year_to_date:
            return f"AND gl.period BETWEEN '{y}-01' AND '{system_year}-{system_month:02d}'"
        return ""

    def get_budget_period_filter(p: PeriodOption, y: int) -> str:
        if p == PeriodOption.custom_range:
            if date_from and date_to:
                return f"AND period BETWEEN '{date_from}' AND '{date_to}'"
            if date_from:
                return f"AND period >= '{date_from}'"
            if date_to:
                return f"AND period <= '{date_to}'"
            return f"AND period LIKE '{y}-%'"
        if p == PeriodOption.all:
            return f"AND period LIKE '{y}-%'"
        if p == PeriodOption.current_month:
            return f"AND period = '{y}-{system_month:02d}'"
        if p == PeriodOption.last_month:
            last_month = pd.Timestamp(year=y, month=system_month, day=1) - pd.DateOffset(months=1)
            return f"AND period = '{last_month.year}-{last_month.month:02d}'"
        if p == PeriodOption.quarter_to_date:
            return f"AND period BETWEEN '{y}-01' AND '{y}-03'"
        if p == PeriodOption.year_to_date:
            return f"AND period BETWEEN '{y}-01' AND '{system_year}-{system_month:02d}'"
        return ""

    gl_period_sql     = get_gl_period_filter(period, year)
    budget_period_sql = get_budget_period_filter(period, year)

    comparison_period_label      = None
    comparison_gl_period_sql     = None
    comparison_budget_period_sql = None

    if comparison and period != PeriodOption.custom_range:
        if period == PeriodOption.current_month:
            base = f"{year}-{system_month:02d}"
            cp   = get_comparison_period(base, comparison)
            comparison_period_label      = cp
            comparison_gl_period_sql     = f"AND gl.period = '{cp}'"
            comparison_budget_period_sql = f"AND period = '{cp}'"

        elif period == PeriodOption.last_month:
            lm   = pd.Timestamp(year=year, month=system_month, day=1) - pd.DateOffset(months=1)
            base = f"{lm.year}-{lm.month:02d}"
            cp   = get_comparison_period(base, comparison)
            comparison_period_label      = cp
            comparison_gl_period_sql     = f"AND gl.period = '{cp}'"
            comparison_budget_period_sql = f"AND period = '{cp}'"

        elif period == PeriodOption.all:
            if comparison == ComparisonOption.yoy:
                cp_year = year - 1
                comparison_period_label      = str(cp_year)
                comparison_gl_period_sql     = f"AND gl.period LIKE '{cp_year}-%'"
                comparison_budget_period_sql = f"AND period LIKE '{cp_year}-%'"
            elif comparison == ComparisonOption.mom:
                cp_year  = year - 1
                cp_month = 12
                comparison_period_label      = f"{cp_year}-{cp_month:02d}"
                comparison_gl_period_sql     = f"AND gl.period LIKE '{cp_year}-%'"
                comparison_budget_period_sql = f"AND period LIKE '{cp_year}-%'"

        elif period == PeriodOption.year_to_date:
            if comparison == ComparisonOption.yoy:
                cp_year = year - 1
                comparison_period_label      = str(cp_year)
                comparison_gl_period_sql     = f"AND gl.period BETWEEN '{cp_year}-01' AND '{cp_year}-{system_month:02d}'"
                comparison_budget_period_sql = f"AND period BETWEEN '{cp_year}-01' AND '{cp_year}-{system_month:02d}'"
            elif comparison == ComparisonOption.mom:
                prev = pd.Timestamp(year=year, month=system_month, day=1) - pd.DateOffset(months=1)
                comparison_period_label      = f"{prev.year}-{prev.month:02d}"
                comparison_gl_period_sql     = f"AND gl.period BETWEEN '{prev.year}-01' AND '{prev.year}-{prev.month:02d}'"
                comparison_budget_period_sql = f"AND period BETWEEN '{prev.year}-01' AND '{prev.year}-{prev.month:02d}'"

        elif period == PeriodOption.quarter_to_date:
            if comparison == ComparisonOption.yoy:
                cp_year = year - 1
                comparison_period_label      = str(cp_year)
                comparison_gl_period_sql     = f"AND gl.period BETWEEN '{cp_year}-01' AND '{cp_year}-03'"
                comparison_budget_period_sql = f"AND period BETWEEN '{cp_year}-01' AND '{cp_year}-03'"
            elif comparison == ComparisonOption.mom:
                comparison_period_label      = f"{year-1}-Q4"
                comparison_gl_period_sql     = f"AND gl.period BETWEEN '{year-1}-10' AND '{year-1}-12'"
                comparison_budget_period_sql = f"AND period BETWEEN '{year-1}-10' AND '{year-1}-12'"

    params = {}
    entity_sql      = "AND e.entity_name = :entity_name"         if entity_name     else ""
    region_sql      = "AND e.region = :region"                   if region          else ""
    dept_sql        = "AND d.department_name = :department_name" if department_name else ""
    prod_sql        = "AND pr.product_name = :product_name"      if product_name    else ""
    project_sql     = "AND cc.name = :project_name"              if project_name    else ""

    if entity_name:     params["entity_name"]    = entity_name
    if region:          params["region"]          = region
    if department_name: params["department_name"] = department_name
    if product_name:    params["product_name"]    = product_name
    if project_name:    params["project_name"]    = project_name

    status_sql = ""
    if status == StatusOption.active:
        status_sql = "AND cc.is_active = TRUE"
    elif status == StatusOption.closed:
        status_sql = "AND cc.is_active = FALSE"

    agg_select_cols = []
    agg_group_cols  = []

    agg_select_cols.append("ae.cost_center_id")
    agg_group_cols.append("ae.cost_center_id")

    agg_select_cols.append("ae.period")
    agg_group_cols.append("ae.period")

    if show_entity_name:
        agg_select_cols.append("ae.entity_name")
        agg_group_cols.append("ae.entity_name")
    else:
        agg_select_cols.append("NULL AS entity_name")

    if show_region:
        agg_select_cols.append("ae.region")
        agg_group_cols.append("ae.region")
    else:
        agg_select_cols.append("NULL AS region")

    if show_department_name:
        agg_select_cols.append("ae.department_name")
        agg_group_cols.append("ae.department_name")
    else:
        agg_select_cols.append("NULL AS department_name")

    agg_select_cols.append("ae.project_name")
    agg_group_cols.append("ae.project_name")

    if show_product_name:
        agg_select_cols.append("ae.product_name")
        agg_group_cols.append("ae.product_name")
    else:
        agg_select_cols.append("NULL AS product_name")

    agg_select_sql = ",\n            ".join(agg_select_cols)
    agg_group_sql  = ",\n            ".join(agg_group_cols) if agg_group_cols else "1"

    def build_base_cte(gl_filter: str, budget_filter: str) -> str:
        return f"""
    WITH actuals_raw AS (
        SELECT
            gl.period,
            gl.cost_center_id,
            gl.entity_id::text        AS entity_id,
            gl.profit_center_id::text AS profit_center_id,
            SUM(gl.amount_debit)      AS actual_cost
        FROM fpna_landing.finance_general_ledger_transactions gl
        INNER JOIN fpna_landing.finance_chart_of_accounts coa
            ON coa.account_id = gl.gl_account_id
            AND coa.account_type = 'Expense'
        WHERE 1=1
            {gl_filter}
        GROUP BY gl.period, gl.cost_center_id, gl.entity_id, gl.profit_center_id
    ),
    budget_raw AS (
        SELECT
            cost_center_id::text AS cost_center_id,
            period,
            SUM(amount)          AS budget
        FROM fpna_landing.finance_budget_data
        WHERE "Type" = 'Expense'
            {budget_filter}
        GROUP BY cost_center_id, period
    ),
    actuals_enriched AS (
        SELECT
            a.period,
            a.cost_center_id,
            a.entity_id,
            SUM(a.actual_cost)                   AS actual_cost,
            COALESCE(MAX(e.entity_name), '')     AS entity_name,
            COALESCE(MAX(e.region), '')          AS region,
            COALESCE(MAX(d.department_name), '') AS department_name,
            COALESCE(MAX(cc.name), '')           AS project_name,
            COALESCE(MAX(pr.product_name), '')   AS product_name
        FROM actuals_raw a

        LEFT JOIN fpna_landing.finance_cost_centers cc
            ON cc.cost_center_id::text = a.cost_center_id
            {status_sql}

        LEFT JOIN fpna_landing.finance_departments d
            ON d.department_id::text = cc.department_id::text

        LEFT JOIN fpna_landing.finance_entities e
            ON e.entity_id::text = a.entity_id

        LEFT JOIN fpna_landing.finance_profit_centers pc
            ON pc.profit_center_id::text = a.profit_center_id

        LEFT JOIN (
            SELECT DISTINCT ON (profit_center_id) profit_center_id, product_name
            FROM fpna_landing.crm_products
            ORDER BY profit_center_id
        ) pr
            ON pr.profit_center_id::text = pc.profit_center_id::text

        WHERE 1=1
            {entity_sql}
            {region_sql}
            {dept_sql}
            {prod_sql}
            {project_sql}

        GROUP BY a.period, a.cost_center_id, a.entity_id
    ),
    actuals_enriched_with_budget AS (
        SELECT
            ae.period,
            ae.cost_center_id,
            ae.entity_id,
            ae.actual_cost,
            COALESCE(b.budget, 0) AS budget,
            ae.entity_name,
            ae.region,
            ae.department_name,
            ae.project_name,
            ae.product_name
        FROM actuals_enriched ae
        LEFT JOIN budget_raw b
            ON b.cost_center_id = ae.cost_center_id
            AND b.period = ae.period
    ),
    aggregated AS (
        SELECT
            {agg_select_sql},
            SUM(actual_cost) AS actual_cost,
            SUM(budget)      AS budget
        FROM actuals_enriched_with_budget ae
        GROUP BY
            {agg_group_sql}
    )
    """

    base_cte = build_base_cte(gl_period_sql, budget_period_sql)

    if sort_col == "variance":
        sort_expr = f"(budget - actual_cost) {sort_dir}"
    elif sort_col == "completion_percentage":
        sort_expr = f"CASE WHEN budget = 0 THEN 0 ELSE ROUND((actual_cost / budget * 100)::numeric, 2) END {sort_dir}"
    elif sort_col in ("entity_name", "region", "department_name", "project_name", "product_name", "period", "cost_center_id"):
        sort_expr = f"COALESCE({sort_col}::text, '') {sort_dir}"
    else:
        sort_expr = f"{sort_col} {sort_dir}"

    sql_all = base_cte + f"""
        SELECT
            period::text,
            cost_center_id::text,
            entity_name,
            region,
            department_name,
            project_name,
            product_name,
            actual_cost::float,
            budget::float,
            (budget - actual_cost)::float AS variance,
            CASE
                WHEN budget = 0 THEN 0::float
                ELSE ROUND((actual_cost / budget * 100)::numeric, 2)::float
            END AS completion_percentage
        FROM aggregated
        ORDER BY {sort_expr};
    """

    sql_metrics = base_cte + """
        , actuals_base AS (
            SELECT
                cost_center_id,
                SUM(actual_cost) AS actual_cost
            FROM actuals_enriched_with_budget
            GROUP BY cost_center_id
        ),
        budget_base AS (
            SELECT
                cost_center_id,
                SUM(budget) AS budget
            FROM budget_raw
            GROUP BY cost_center_id
        ),
        metrics_base AS (
            SELECT
                COALESCE(a.cost_center_id, b.cost_center_id) AS cost_center_id,
                COALESCE(a.actual_cost, 0)                   AS actual_cost,
                COALESCE(b.budget, 0)                        AS budget
            FROM actuals_base a
            FULL OUTER JOIN budget_base b
                ON b.cost_center_id = a.cost_center_id
        )
        SELECT
            COALESCE(SUM(budget), 0)::float                                    AS total_budget,
            COALESCE(SUM(actual_cost), 0)::float                               AS total_actual,
            COALESCE(SUM(budget - actual_cost), 0)::float                      AS total_variance,
            SUM(CASE WHEN actual_cost <= budget THEN 1 ELSE 0 END)::bigint     AS on_budget_projects,
            SUM(CASE WHEN actual_cost >  budget THEN 1 ELSE 0 END)::bigint     AS over_budget_projects
        FROM metrics_base;
    """

    sql_grand_total = base_cte + """
        , actuals_base AS (
            SELECT
                cost_center_id,
                SUM(actual_cost) AS actual_cost
            FROM actuals_enriched_with_budget
            GROUP BY cost_center_id
        ),
        budget_base AS (
            SELECT
                cost_center_id,
                SUM(budget) AS budget
            FROM budget_raw
            GROUP BY cost_center_id
        ),
        grand_base AS (
            SELECT
                COALESCE(a.cost_center_id, b.cost_center_id) AS cost_center_id,
                COALESCE(a.actual_cost, 0)                   AS actual_cost,
                COALESCE(b.budget, 0)                        AS budget
            FROM actuals_base a
            FULL OUTER JOIN budget_base b
                ON b.cost_center_id = a.cost_center_id
        )
        SELECT
            COALESCE(SUM(actual_cost), 0)::float          AS total_actual,
            COALESCE(SUM(budget), 0)::float               AS total_budget,
            COALESCE(SUM(budget - actual_cost), 0)::float AS total_variance
        FROM grand_base;
    """

    async with db.begin():
        all_rows        = (await db.execute(text(sql_all), params)).mappings().all()
        metrics_row     = (await db.execute(text(sql_metrics), params)).mappings().first()
        grand_total_row = (await db.execute(text(sql_grand_total), params)).mappings().first()

    cp_lookup     = {}
    cp_raw_lookup = {}
    if comparison and comparison_gl_period_sql:
        comparison_cte = build_base_cte(comparison_gl_period_sql, comparison_budget_period_sql)

        sql_cp_all = comparison_cte + f"""
            SELECT
                period::text,
                cost_center_id::text,
                entity_name,
                region,
                department_name,
                project_name,
                product_name,
                actual_cost::float,
                budget::float,
                (budget - actual_cost)::float AS variance,
                CASE
                    WHEN budget = 0 THEN 0::float
                    ELSE ROUND((actual_cost / budget * 100)::numeric, 2)::float
                END AS completion_percentage
            FROM aggregated
            ORDER BY {sort_expr};
        """

        async with db.begin():
            cp_rows = (await db.execute(text(sql_cp_all), params)).mappings().all()

        cp_raw_list = []
        cp_cids     = []
        cp_prs      = []
        cp_nums     = []
        for r in cp_rows:
            cp_cids.append(r.get("cost_center_id"))
            cp_prs.append(r.get("period"))
            act_n = float(r.get("actual_cost") or 0)
            bud_n = float(r.get("budget")      or 0)
            var_n = float(r.get("variance")    or 0)
            cp_nums.append({"actual": act_n, "budget": bud_n, "variance": var_n})
            cp_raw_list.append({
                "period":                r.get("period"),
                "actual":                act_n,
                "budget":                bud_n,
                "variance":              var_n,
                "completion_percentage": round(float(r.get("completion_percentage") or 0), 2),
            })

        cp_raw_list = apply_smart_currency(cp_raw_list, currency, currency_filter, ["actual", "budget", "variance"])
        for i, cid in enumerate(cp_cids):
            cp_period = cp_prs[i]
            if cid and cp_period:
                if cid not in cp_lookup:
                    cp_lookup[cid] = {}
                cp_lookup[cid][cp_period] = cp_raw_list[i]
                if cid not in cp_raw_lookup:
                    cp_raw_lookup[cid] = {}
                cp_raw_lookup[cid][cp_period] = cp_nums[i]

    def get_compare_period(p: str, comp: ComparisonOption) -> str:
        dt = pd.to_datetime(p + "-01")
        if comp == ComparisonOption.mom:
            return (dt - pd.DateOffset(months=1)).strftime("%Y-%m")
        return (dt - pd.DateOffset(years=1)).strftime("%Y-%m")

    excel_raw_lookup = {}
    for r in all_rows:
        r_cid = r.get("cost_center_id")
        r_p   = r.get("period")
        if r_cid and r_p:
            if r_cid not in excel_raw_lookup:
                excel_raw_lookup[r_cid] = {}
            if r_p not in excel_raw_lookup[r_cid]:
                excel_raw_lookup[r_cid][r_p] = {
                    "actual":   float(r.get("actual_cost") or 0),
                    "budget":   float(r.get("budget")      or 0),
                    "variance": float(r.get("variance")    or 0),
                }
            else:
                ex = excel_raw_lookup[r_cid][r_p]
                a  = ex["actual"] + float(r.get("actual_cost") or 0)
                b  = ex["budget"] + float(r.get("budget")      or 0)
                excel_raw_lookup[r_cid][r_p] = {"actual": a, "budget": b, "variance": b - a}

    if excel_download:
        gt_actual_raw     = float(grand_total_row["total_actual"]   or 0) if grand_total_row else 0.0
        gt_budget_raw     = float(grand_total_row["total_budget"]   or 0) if grand_total_row else 0.0
        gt_variance_raw   = float(grand_total_row["total_variance"] or 0) if grand_total_row else 0.0
        gt_completion_raw = round((gt_actual_raw / gt_budget_raw * 100), 2) if gt_budget_raw != 0 else 0.0

        if show_period:
            export_records = []
            for r in all_rows:
                cost_center_id = r.get("cost_center_id")
                record = {}
                record["project_name"] = r.get("project_name")
                if show_cost_center_id:  record["cost_center_id"]  = cost_center_id
                record["period"]                = r.get("period")
                if show_entity_name:     record["entity_name"]     = r.get("entity_name")
                if show_region:          record["region"]          = r.get("region")
                if show_department_name: record["department_name"] = r.get("department_name")
                if show_product_name:    record["product_name"]    = r.get("product_name")
                record["actual"]                = float(r.get("actual_cost") or 0)
                record["budget"]                = float(r.get("budget")      or 0)
                record["variance"]              = float(r.get("variance")    or 0)
                record["completion_percentage"] = round(float(r.get("completion_percentage") or 0), 2)
                if comparison and r.get("period"):
                    cur_p   = r.get("period")
                    cmp_p   = get_compare_period(cur_p, comparison)
                    cp_data = cp_lookup.get(cost_center_id, {}).get(cmp_p)
                    if cp_data is None and comparison == ComparisonOption.mom:
                        raw_cp = excel_raw_lookup.get(cost_center_id, {}).get(cmp_p)
                        if raw_cp:
                            cp_cv  = apply_smart_currency(
                                {"actual": raw_cp["actual"], "budget": raw_cp["budget"], "variance": raw_cp["variance"]},
                                currency, currency_filter, ["actual", "budget", "variance"]
                            )
                            cp_data = {"actual": cp_cv["actual"], "budget": cp_cv["budget"], "variance": cp_cv["variance"]}
                    if cp_data:
                        record["comparison_current_period"]   = cur_p
                        record["comparison_previous_period"]  = cmp_p
                        record["comparison_current_actual"]   = record["actual"]
                        record["comparison_previous_actual"]  = cp_data.get("actual")
                        record["comparison_current_budget"]   = record["budget"]
                        record["comparison_previous_budget"]  = cp_data.get("budget")
                        record["comparison_current_variance"] = record["variance"]
                        record["comparison_previous_variance"]= cp_data.get("variance")
                export_records.append(record)
        else:
            excel_agg = {}
            for r in all_rows:
                cost_center_id = r.get("cost_center_id")
                proj_nm  = r.get("project_name")
                key_parts = [proj_nm]
                if show_entity_name:     key_parts.append(r.get("entity_name"))
                if show_region:          key_parts.append(r.get("region"))
                if show_department_name: key_parts.append(r.get("department_name"))
                if show_cost_center_id:  key_parts.append(cost_center_id)
                if show_product_name:    key_parts.append(r.get("product_name"))
                xkey = tuple(key_parts)
                if xkey not in excel_agg:
                    excel_agg[xkey] = {
                        "project_name":  proj_nm,
                        "cost_center_id": cost_center_id,
                        "entity_name":   r.get("entity_name"),
                        "region":        r.get("region"),
                        "department_name": r.get("department_name"),
                        "product_name":  r.get("product_name"),
                        "actual": 0.0, "budget": 0.0,
                    }
                excel_agg[xkey]["actual"] += float(r.get("actual_cost") or 0)
                excel_agg[xkey]["budget"] += float(r.get("budget")      or 0)
            export_records = []
            for xkey, agg in excel_agg.items():
                cost_center_id = agg["cost_center_id"]
                act_v  = agg["actual"]
                bud_v  = agg["budget"]
                var_v  = bud_v - act_v
                compl_v = round((act_v / bud_v * 100), 2) if bud_v != 0 else 0.0
                record = {}
                record["project_name"] = agg["project_name"]
                if show_cost_center_id:  record["cost_center_id"]  = cost_center_id
                if show_entity_name:     record["entity_name"]     = agg["entity_name"]
                if show_region:          record["region"]          = agg["region"]
                if show_department_name: record["department_name"] = agg["department_name"]
                if show_product_name:    record["product_name"]    = agg["product_name"]
                record["actual"]                = act_v
                record["budget"]                = bud_v
                record["variance"]              = var_v
                record["completion_percentage"] = compl_v
                if comparison and cp_raw_lookup.get(cost_center_id):
                    cp_raw_entries = list(cp_raw_lookup[cost_center_id].values())
                    prev_actual    = sum(float(x.get("actual",   0)) for x in cp_raw_entries)
                    prev_budget    = sum(float(x.get("budget",   0)) for x in cp_raw_entries)
                    prev_variance  = prev_budget - prev_actual
                    record["comparison_current_period"]   = None
                    record["comparison_previous_period"]  = comparison_period_label
                    record["comparison_current_actual"]   = act_v
                    record["comparison_previous_actual"]  = prev_actual
                    record["comparison_current_budget"]   = bud_v
                    record["comparison_previous_budget"]  = prev_budget
                    record["comparison_current_variance"] = var_v
                    record["comparison_previous_variance"]= prev_variance
                export_records.append(record)

        export_records = apply_smart_currency(export_records, currency, currency_filter, ["actual", "budget", "variance"])

        grand_total_export = {}
        grand_total_export["project_name"] = "Grand Total"
        if show_cost_center_id:  grand_total_export["cost_center_id"]  = None
        if show_period:          grand_total_export["period"]          = None
        if show_entity_name:     grand_total_export["entity_name"]     = None
        if show_region:          grand_total_export["region"]          = None
        if show_department_name: grand_total_export["department_name"] = None
        if show_product_name:    grand_total_export["product_name"]    = None
        grand_total_export["actual"]                = gt_actual_raw
        grand_total_export["budget"]                = gt_budget_raw
        grand_total_export["variance"]              = gt_variance_raw
        grand_total_export["completion_percentage"] = gt_completion_raw

        if comparison:
            grand_total_export["comparison_current_period"]    = None
            grand_total_export["comparison_previous_period"]   = None
            grand_total_export["comparison_current_actual"]    = None
            grand_total_export["comparison_previous_actual"]   = None
            grand_total_export["comparison_current_budget"]    = None
            grand_total_export["comparison_previous_budget"]   = None
            grand_total_export["comparison_current_variance"]  = None
            grand_total_export["comparison_previous_variance"] = None

        export_records.append(grand_total_export)
        export_df = pd.DataFrame(export_records)

        if period == PeriodOption.custom_range and (date_from or date_to):
            range_label = f"{date_from or 'start'}_to_{date_to or 'end'}"
            filename = f"project_costing_report_{year}_{currency}_{range_label}.xlsx"
        else:
            period_label_file = period.value if period else "all"
            filename = f"project_costing_report_{year}_{currency}_{period_label_file}.xlsx"

        return build_project_excel_response(export_df, filename=filename)

    sql_all_entities    = "SELECT entity_id, entity_name FROM fpna_landing.finance_entities ORDER BY entity_name"
    sql_all_departments = "SELECT department_id, department_name FROM fpna_landing.finance_departments ORDER BY department_name"
    sql_all_regions     = "SELECT DISTINCT region FROM fpna_landing.finance_entities WHERE region IS NOT NULL ORDER BY region"
    sql_all_products    = "SELECT product_id, product_name FROM fpna_landing.crm_products ORDER BY product_name"
    sql_all_projects    = "SELECT cost_center_id, name AS project_name FROM fpna_landing.finance_cost_centers ORDER BY name"

    async with db.begin():
        entities_rows    = (await db.execute(text(sql_all_entities))).mappings().all()
        departments_rows = (await db.execute(text(sql_all_departments))).mappings().all()
        regions_rows     = (await db.execute(text(sql_all_regions))).mappings().all()
        products_rows    = (await db.execute(text(sql_all_products))).mappings().all()
        projects_rows    = (await db.execute(text(sql_all_projects))).mappings().all()

    metrics = {
        "total_budget":         float(metrics_row["total_budget"]       or 0),
        "total_actual":         float(metrics_row["total_actual"]       or 0),
        "total_variance":       float(metrics_row["total_variance"]     or 0),
        "on_budget_projects":   int(metrics_row["on_budget_projects"]   or 0),
        "over_budget_projects": int(metrics_row["over_budget_projects"] or 0),
    }
    key_metrics = apply_smart_currency(metrics, currency, currency_filter, ["total_budget", "total_actual", "total_variance"])

    show_columns = {
        "period":          show_period,
        "entity_name":     show_entity_name,
        "region":          show_region,
        "department_name": show_department_name,
        "cost_center_id":  show_cost_center_id,
        "product_name":    show_product_name,
    }

    periods_available = sorted(set(r.get("period") for r in all_rows if r.get("period")))

    project_period_data = {}
    project_dim_data    = {}
    project_cid_data    = {}

    for r in all_rows:
        proj_nm  = r.get("project_name")
        ent_nm   = r.get("entity_name")     if show_entity_name     else None
        reg_nm   = r.get("region")          if show_region          else None
        dept_nm  = r.get("department_name") if show_department_name else None
        cid      = r.get("cost_center_id")
        prod_nm  = r.get("product_name")    if show_product_name    else None

        key_parts = [proj_nm]
        if show_entity_name:     key_parts.append(ent_nm)
        if show_region:          key_parts.append(reg_nm)
        if show_department_name: key_parts.append(dept_nm)
        if show_cost_center_id:  key_parts.append(cid)
        if show_product_name:    key_parts.append(prod_nm)
        key = tuple(key_parts)

        period_val = r.get("period")
        actual_v   = float(r.get("actual_cost") or 0)
        budget_v   = float(r.get("budget")      or 0)
        variance_v = float(r.get("variance")    or 0)
        compl_v    = round(float(r.get("completion_percentage") or 0), 2)

        if key not in project_period_data:
            project_period_data[key] = {}
            dims = {"project_name": proj_nm}
            if show_entity_name:     dims["entity_name"]    = ent_nm
            if show_region:          dims["region"]          = reg_nm
            if show_department_name: dims["department_name"] = dept_nm
            if show_cost_center_id:  dims["cost_center_id"]  = cid
            if show_product_name:    dims["product_name"]    = prod_nm
            project_dim_data[key] = dims
            project_cid_data[key] = cid

        if period_val not in project_period_data[key]:
            project_period_data[key][period_val] = {
                "actual":                actual_v,
                "budget":                budget_v,
                "variance":              variance_v,
                "completion_percentage": compl_v,
            }
        else:
            ex         = project_period_data[key][period_val]
            new_actual = ex["actual"] + actual_v
            new_budget = ex["budget"] + budget_v
            new_var    = new_budget - new_actual
            new_compl  = round((new_actual / new_budget * 100), 2) if new_budget != 0 else 0.0
            project_period_data[key][period_val] = {
                "actual":                new_actual,
                "budget":                new_budget,
                "variance":              new_var,
                "completion_percentage": new_compl,
            }

    def get_sort_value(key):
        pd_map     = project_period_data.get(key, {})
        dims       = project_dim_data.get(key, {})
        total_act  = sum(v["actual"]  for v in pd_map.values())
        total_bud  = sum(v["budget"]  for v in pd_map.values())
        total_var  = total_bud - total_act
        total_comp = round((total_act / total_bud * 100), 2) if total_bud != 0 else 0.0
        if sort_by == ProjectSortByOption.actual:                return total_act
        if sort_by == ProjectSortByOption.budget:                return total_bud
        if sort_by == ProjectSortByOption.variance:              return total_var
        if sort_by == ProjectSortByOption.completion_percentage: return total_comp
        if sort_by == ProjectSortByOption.project_name:          return dims.get("project_name")    or ""
        if sort_by == ProjectSortByOption.entity_name:           return dims.get("entity_name")     or ""
        if sort_by == ProjectSortByOption.region:                return dims.get("region")           or ""
        if sort_by == ProjectSortByOption.department_name:       return dims.get("department_name") or ""
        if sort_by == ProjectSortByOption.cost_center_id:        return dims.get("cost_center_id")  or ""
        if sort_by == ProjectSortByOption.product_name:          return dims.get("product_name")    or ""
        if sort_by == ProjectSortByOption.period:                return periods_available[0] if periods_available else ""
        return dims.get("project_name") or ""

    sorted_keys = sorted(
        project_period_data.keys(),
        key=get_sort_value,
        reverse=(sort_order == SortOrderOption.desc)
    )

    grand_period_map = {}
    for r in all_rows:
        p = r.get("period")
        if p:
            if p not in grand_period_map:
                grand_period_map[p] = {"actual": 0.0, "budget": 0.0}
            grand_period_map[p]["actual"] += float(r.get("actual_cost") or 0)
            grand_period_map[p]["budget"] += float(r.get("budget")      or 0)

    grand_month_comparison = {}

    projects = []
    for key in sorted_keys:
        dims   = project_dim_data[key]
        pd_map = project_period_data[key]
        cid    = project_cid_data[key]

        months              = {}
        total_actual        = 0.0
        total_budget        = 0.0
        total_variance      = 0.0
        prev_total_actual   = 0.0
        prev_total_budget   = 0.0
        prev_total_variance = 0.0
        has_comparison      = False

        for p in periods_available:
            if p in pd_map:
                entry   = pd_map[p]
                act_v   = entry["actual"]
                bud_v   = entry["budget"]
                var_v   = entry["variance"]
                compl_v = entry["completion_percentage"]

                converted = apply_smart_currency(
                    {"actual": act_v, "budget": bud_v, "variance": var_v},
                    currency, currency_filter, ["actual", "budget", "variance"]
                )
                month_entry = {
                    "actual":                converted["actual"],
                    "budget":                converted["budget"],
                    "variance":              converted["variance"],
                    "completion_percentage": compl_v,
                }

                if comparison and cid:
                    cmp_p    = get_compare_period(p, comparison)
                    cp_month = cp_lookup.get(cid, {}).get(cmp_p)
                    cp_raw   = cp_raw_lookup.get(cid, {}).get(cmp_p)
                    if cp_month is None and comparison == ComparisonOption.mom:
                        raw_cp = pd_map.get(cmp_p)
                        if raw_cp:
                            cp_cv  = apply_smart_currency(
                                {"actual": raw_cp["actual"], "budget": raw_cp["budget"], "variance": raw_cp["variance"]},
                                currency, currency_filter, ["actual", "budget", "variance"]
                            )
                            cp_month = {
                                "actual":                cp_cv["actual"],
                                "budget":                cp_cv["budget"],
                                "variance":              cp_cv["variance"],
                                "completion_percentage": raw_cp["completion_percentage"],
                            }
                            cp_raw = raw_cp
                    if cp_month:
                        raw_prev_act = float(cp_raw.get("actual",   0)) if cp_raw else 0.0
                        raw_prev_bud = float(cp_raw.get("budget",   0)) if cp_raw else 0.0
                        raw_prev_var = float(cp_raw.get("variance", 0)) if cp_raw else 0.0
                        month_entry["comparison"] = {
                            "comparison_type":  comparison.value,
                            "current_period":   p,
                            "compare_period":   cmp_p,
                            "current_actual":   converted["actual"],
                            "previous_actual":  cp_month.get("actual"),
                            "current_budget":   converted["budget"],
                            "previous_budget":  cp_month.get("budget"),
                            "current_variance": converted["variance"],
                            "previous_variance":cp_month.get("variance"),
                        }
                        prev_total_actual   += raw_prev_act
                        prev_total_budget   += raw_prev_bud
                        prev_total_variance += raw_prev_var
                        has_comparison = True
                        if p not in grand_month_comparison:
                            grand_month_comparison[p] = {"actual": 0.0, "budget": 0.0, "variance": 0.0, "cmp_p": cmp_p}
                        grand_month_comparison[p]["actual"]   += raw_prev_act
                        grand_month_comparison[p]["budget"]   += raw_prev_bud
                        grand_month_comparison[p]["variance"] += raw_prev_var
                    else:
                        month_entry["comparison"] = {}
                else:
                    month_entry["comparison"] = {}

                if show_period:
                    months[p] = month_entry
                total_actual   += act_v
                total_budget   += bud_v
                total_variance += var_v

        total_completion = round((total_actual / total_budget * 100), 2) if total_budget != 0 else 0.0
        total_converted  = apply_smart_currency(
            {"actual": total_actual, "budget": total_budget, "variance": total_variance},
            currency, currency_filter, ["actual", "budget", "variance"]
        )

        record           = dict(dims)
        record["months"] = months
        record["total"]  = {
            "actual":                total_converted["actual"],
            "budget":                total_converted["budget"],
            "variance":              total_converted["variance"],
            "completion_percentage": total_completion,
        }
        record["comparison"] = {}
        if comparison and has_comparison:
            prev_conv = apply_smart_currency(
                {"actual": prev_total_actual, "budget": prev_total_budget, "variance": prev_total_variance},
                currency, currency_filter, ["actual", "budget", "variance"]
            )
            record["comparison"] = {
                "current_actual":    total_converted["actual"],
                "previous_actual":   prev_conv["actual"],
                "current_budget":    total_converted["budget"],
                "previous_budget":   prev_conv["budget"],
                "current_variance":  total_converted["variance"],
                "previous_variance": prev_conv["variance"],
            }

        projects.append(record)

    gt_actual_raw     = float(grand_total_row["total_actual"]   or 0) if grand_total_row else 0.0
    gt_budget_raw     = float(grand_total_row["total_budget"]   or 0) if grand_total_row else 0.0
    gt_variance_raw   = float(grand_total_row["total_variance"] or 0) if grand_total_row else 0.0
    gt_completion_raw = round((gt_actual_raw / gt_budget_raw * 100), 2) if gt_budget_raw != 0 else 0.0

    gt_raw       = {"actual": gt_actual_raw, "budget": gt_budget_raw, "variance": gt_variance_raw}
    gt_converted = apply_smart_currency(gt_raw, currency, currency_filter, ["actual", "budget", "variance"])

    grand_months = {}
    for p in periods_available:
        if p in grand_period_map:
            g_act   = grand_period_map[p]["actual"]
            g_bud   = grand_period_map[p]["budget"]
            g_var   = g_bud - g_act
            g_compl = round((g_act / g_bud * 100), 2) if g_bud != 0 else 0.0
            converted_gm = apply_smart_currency(
                {"actual": g_act, "budget": g_bud, "variance": g_var},
                currency, currency_filter, ["actual", "budget", "variance"]
            )
            gm_entry = {
                "actual":                converted_gm["actual"],
                "budget":                converted_gm["budget"],
                "variance":              converted_gm["variance"],
                "completion_percentage": g_compl,
            }
            if comparison and p in grand_month_comparison:
                gc      = grand_month_comparison[p]
                cmp_p   = gc["cmp_p"]
                gc_conv = apply_smart_currency(
                    {"actual": gc["actual"], "budget": gc["budget"], "variance": gc["variance"]},
                    currency, currency_filter, ["actual", "budget", "variance"]
                )
                gm_entry["comparison"] = {
                    "comparison_type":  comparison.value,
                    "current_period":   p,
                    "compare_period":   cmp_p,
                    "current_actual":   converted_gm["actual"],
                    "previous_actual":  gc_conv["actual"],
                    "current_budget":   converted_gm["budget"],
                    "previous_budget":  gc_conv["budget"],
                    "current_variance": converted_gm["variance"],
                    "previous_variance":gc_conv["variance"],
                }
            else:
                gm_entry["comparison"] = {}
            if show_period:
                grand_months[p] = gm_entry

    grand_total                         = {}
    grand_total["project_name"]         = "Grand Total"
    grand_total["months"]               = grand_months
    grand_total["total"]                = {
        "actual":                gt_converted["actual"],
        "budget":                gt_converted["budget"],
        "variance":              gt_converted["variance"],
        "completion_percentage": gt_completion_raw,
    }
    grand_total["comparison"]           = {}

    return {
        "projects":      projects,
        "grand_total":   grand_total,
        "keymetrics":    key_metrics,
        "periods":       periods_available,
        "show_columns":  show_columns,
        "entities":      [{"entity_id": str(e["entity_id"]), "entity_name": e["entity_name"]} for e in entities_rows],
        "departments":   [{"department_name": d["department_name"]} for d in departments_rows],
        "products":      [{"product_name": p["product_name"]} for p in products_rows],
        "projects_list": [{"cost_center_id": str(p["cost_center_id"]), "project_name": p["project_name"]} for p in projects_rows],
        "regions":       [{"region": r["region"]} for r in regions_rows],
    }

#############################################################################################################
#############################################################################################################
#############################################################################################################
#project costing view more


@financial_router.get("/Project_Costing_viewmore/{year}/{currency}")
async def project_costing_viewmore(
    currency: str,
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    year: int = Path(...),
    project_name: Optional[str] = Query(None),
    period: PeriodOption = Query(PeriodOption.all),
    comparison: Optional[ComparisonOption] = Query(None, enum=[e.value for e in ComparisonOption]),
    db: AsyncSession = Depends(get_db)
):
    project_name = normalize(project_name)

    system_date  = pd.Timestamp.today()
    system_month = system_date.month
    system_year  = system_date.year

    all_months = [f"{year}-{m:02d}" for m in range(1, 13)]

    def build_gl_filter(gl_filter_str: str) -> str:
        return gl_filter_str

    def get_gl_filter(y: int) -> str:
        return f"AND gl.period LIKE '{y}-%'"

    def get_budget_filter(y: int) -> str:
        return f"AND period LIKE '{y}-%'"

    project_sql = "AND cc.name = :project_name" if project_name else ""
    params      = {}
    if project_name:
        params["project_name"] = project_name

    comparison_year = None
    if comparison == ComparisonOption.yoy:
        comparison_year = year - 1
    elif comparison == ComparisonOption.mom:
        comparison_year = year - 1

    def build_viewmore_cte(y: int) -> str:
        gl_filter     = f"AND gl.period LIKE '{y}-%'"
        budget_filter = f"AND period LIKE '{y}-%'"
        return f"""
        WITH actuals_raw AS (
            SELECT
                gl.period,
                gl.cost_center_id,
                gl.entity_id::text        AS entity_id,
                gl.profit_center_id::text AS profit_center_id,
                SUM(gl.amount_debit)      AS actual_cost
            FROM fpna_landing.finance_general_ledger_transactions gl
            INNER JOIN fpna_landing.finance_chart_of_accounts coa
                ON coa.account_id = gl.gl_account_id
                AND coa.account_type = 'Expense'
            WHERE 1=1
                {gl_filter}
            GROUP BY gl.period, gl.cost_center_id, gl.entity_id, gl.profit_center_id
        ),
        budget_raw AS (
            SELECT
                cost_center_id::text AS cost_center_id,
                period,
                SUM(amount)          AS budget
            FROM fpna_landing.finance_budget_data
            WHERE "Type" = 'Expense'
                {budget_filter}
            GROUP BY cost_center_id, period
        ),
        aggregated AS (
            SELECT
                a.period,
                a.cost_center_id,
                SUM(a.actual_cost)                   AS actual_cost,
                COALESCE(MAX(b.budget), 0)           AS budget,
                COALESCE(MAX(cc.name), '')           AS project_name
            FROM actuals_raw a
            LEFT JOIN budget_raw b
                ON b.cost_center_id = a.cost_center_id
                AND b.period = a.period
            LEFT JOIN fpna_landing.finance_cost_centers cc
                ON cc.cost_center_id::text = a.cost_center_id
            WHERE 1=1
                {project_sql}
            GROUP BY a.period, a.cost_center_id
        )
    """

    sql_main = build_viewmore_cte(year) + """
        SELECT
            cost_center_id::text,
            project_name,
            period::text,
            actual_cost::float,
            budget::float
        FROM aggregated
        ORDER BY project_name, period;
    """

    async with db.begin():
        main_rows = (await db.execute(text(sql_main), params)).mappings().all()

    cp_rows_map = {}
    if comparison and comparison_year:
        sql_cp = build_viewmore_cte(comparison_year) + """
            SELECT
                cost_center_id::text,
                project_name,
                period::text,
                actual_cost::float,
                budget::float
            FROM aggregated
            ORDER BY project_name, period;
        """
        async with db.begin():
            cp_rows = (await db.execute(text(sql_cp), params)).mappings().all()
        for r in cp_rows:
            key = (r["cost_center_id"], r["period"])
            cp_rows_map[key] = r

    projects_budget  = {}
    projects_actual  = {}
    project_meta     = {}

    for r in main_rows:
        cc_id        = r["cost_center_id"]
        pname        = r["project_name"]
        prd          = r["period"]
        actual_val   = float(r["actual_cost"] or 0)
        budget_val   = float(r["budget"]      or 0)

        project_meta[cc_id] = pname

        if cc_id not in projects_budget:
            projects_budget[cc_id] = {}
        if cc_id not in projects_actual:
            projects_actual[cc_id] = {}

        projects_budget[cc_id][prd] = budget_val
        projects_actual[cc_id][prd] = actual_val

    def build_rows(data_map: dict, row_type: str, months: list, cp_map: dict, cp_months: list):
        rows        = []
        total_row   = {"type": "total", "project_name": "total", "cost_center_id": "total"}
        total_vals  = {m: 0.0 for m in months}
        grand_total = 0.0

        for cc_id, month_data in data_map.items():
            row = {
                "type":           row_type,
                "cost_center_id": cc_id,
                "project_name":   project_meta.get(cc_id, ""),
            }
            row_total = 0.0
            for m in months:
                val = month_data.get(m, 0.0)
                row[m] = val
                row_total       += val
                total_vals[m]   += val
                grand_total     += val
            row["total"] = row_total

            if comparison and cp_map:
                cp_row_total = 0.0
                for m in cp_months:
                    cp_key = (cc_id, m)
                    cp_r   = cp_map.get(cp_key)
                    if cp_r:
                        cp_val = float(cp_r["budget"] if row_type == "budget" else cp_r["actual_cost"] or 0)
                    else:
                        cp_val = 0.0
                    row[f"comparison_{m}"] = cp_val
                    cp_row_total += cp_val
                row["comparison_total"] = cp_row_total

            rows.append(row)

        total_row["total"] = grand_total
        for m in months:
            total_row[m] = total_vals[m]

        if comparison and cp_map:
            cp_grand = 0.0
            for m in cp_months:
                cp_m_total = 0.0
                for cc_id in data_map:
                    cp_key = (cc_id, m)
                    cp_r   = cp_map.get(cp_key)
                    if cp_r:
                        cp_m_total += float(cp_r["budget"] if row_type == "budget" else cp_r["actual_cost"] or 0)
                total_row[f"comparison_{m}"] = cp_m_total
                cp_grand += cp_m_total
            total_row["comparison_total"] = cp_grand

        rows.append(total_row)
        return rows

    cp_months = []
    if comparison and comparison_year:
        cp_months = [f"{comparison_year}-{m:02d}" for m in range(1, 13)]

    budget_rows = build_rows(projects_budget, "budget", all_months, cp_rows_map, cp_months)
    actual_rows = build_rows(projects_actual, "actual", all_months, cp_rows_map, cp_months)

    currency_fields = all_months + ["total"]
    if comparison and cp_months:
        currency_fields += [f"comparison_{m}" for m in cp_months] + ["comparison_total"]

    budget_rows = apply_smart_currency(budget_rows, currency, currency_filter, currency_fields)
    actual_rows = apply_smart_currency(actual_rows, currency, currency_filter, currency_fields)

    return {
        "year":       year,
        "months":     all_months,
        "comparison": {
            "type":   comparison.value if comparison else None,
            "year":   comparison_year,
            "months": cp_months,
        },
        "data":       budget_rows + actual_rows,
    }

########################################################
##################################################
################################################
#Fixed Asset

def get_fa_period_filter(period: PeriodOption, year: int) -> str:
    system_date = pd.Timestamp.today()
    system_year = system_date.year
    system_month = system_date.month
    if period == PeriodOption.all:
        return f"AND EXTRACT(YEAR FROM TO_DATE(date_acquired, 'DD-MM-YYYY')) = {year}"
    if period == PeriodOption.current_month:
        return f"AND TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'),'YYYY-MM') = '{year}-{system_month:02d}'"
    if period == PeriodOption.last_month:
        last_month = (pd.Timestamp(year=year, month=system_month, day=1) - pd.DateOffset(months=1))
        return f"AND TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'),'YYYY-MM') = '{last_month.year}-{last_month.month:02d}'"
    if period == PeriodOption.quarter_to_date:
        return f"AND TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'),'YYYY-MM') BETWEEN '{year}-01' AND '{year}-03'"
    if period == PeriodOption.year_to_date:
        return f"AND TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'),'YYYY-MM') BETWEEN '{year}-01' AND '{system_year}-{system_month:02d}'"
    return ""


def _get_fa_comparison_period(period_str: str, comparison: Optional[ComparisonOption]) -> Optional[str]:
    """
    Calculate comparison period based on period string (MM-YYYY format) and comparison type.
    
    Args:
        period_str: Period in MM-YYYY format
        comparison: Comparison type (mom, yoy) from ComparisonOption
        
    Returns:
        Comparison period in MM-YYYY format or None
    """
    if not period_str or not comparison:
        return None
    
    try:
        mm, yyyy = period_str.split("-")
        dt = pd.to_datetime(f"{yyyy}-{mm}", format="%Y-%m")
    except:
        return None
    
    if comparison == ComparisonOption.mom:
        compare_dt = dt - pd.DateOffset(months=1)
        return compare_dt.strftime("%m-%Y")
    elif comparison == ComparisonOption.yoy:
        compare_dt = dt - pd.DateOffset(years=1)
        return compare_dt.strftime("%m-%Y")
    
    return None


@financial_router.get("/fixed_asset/{year}/{currency}", dependencies=[Depends(get_current_company_user)])
async def fixed_asset_report(
    year: int,
    currency: str,
    period: PeriodOption = Query(default=PeriodOption.all),
    category: Optional[str] = Query(default=None),
    comparison: Optional[ComparisonOption] = Query(default=None),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    db: AsyncSession = Depends(get_db),
    currentCompanyUser: CurrentCompanyUser = Depends(get_current_company_user)
):
    """
    Fixed Asset Report with optional period comparison (same account_name & category)
    
    Args:
        year: Year for the report
        currency: Currency code
        period: Period filter (all, current_month, last_month, quarter_to_date, year_to_date)
        category: Optional category filter
        comparison: Optional comparison type (mom for month-over-month, yoy for year-over-year)
        currency_filter: Currency conversion filter
        db: Database session
        currentCompanyUser: Current authenticated company user
    """
    
    period_filter_sql = get_fa_period_filter(period, year)
    category_filter_sql = f"AND category = '{category}'" if category else ""

    # Fetch current period data
    query = text(f"""
        SELECT
            TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'MM-YYYY') AS period,
            'Fixed Asset' AS account_name,
            category,
            SUM(purchase_price) AS value
        FROM fpna_landing.finance_fixed_asset
        WHERE 1=1 {period_filter_sql} {category_filter_sql}
        GROUP BY TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'MM-YYYY'), category
        ORDER BY TO_NUMBER(LEFT(TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'MM-YYYY'),2) || RIGHT(TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'MM-YYYY'),4), '999999'), category
    """)
    
    result = await db.execute(query)
    rows = result.fetchall()
    columns = result.keys()
    df = pd.DataFrame(rows, columns=columns)

    # Build comparison lookup - comparing same account_name & category across periods
    comparison_lookup = {}
    if comparison and not df.empty:
        # Group by category to compare same category across periods
        for category_val in df["category"].unique():
            category_data = df[df["category"] == category_val]
            
            for _, row in category_data.iterrows():
                base_period = row["period"]
                compare_period = _get_fa_comparison_period(base_period, comparison)
                
                if compare_period:
                    # Fetch comparison period data for SAME category
                    compare_query = text(f"""
                        SELECT
                            TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'MM-YYYY') AS period,
                            'Fixed Asset' AS account_name,
                            category,
                            SUM(purchase_price) AS value
                        FROM fpna_landing.finance_fixed_asset
                        WHERE 1=1 
                            AND TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'MM-YYYY') = '{compare_period}'
                            AND category = '{category_val}'
                        GROUP BY TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'MM-YYYY'), category
                    """)
                    
                    compare_result = await db.execute(compare_query)
                    compare_rows = compare_result.fetchall()
                    
                    if compare_rows:
                        compare_df = pd.DataFrame(compare_rows, columns=columns)
                        for _, comp_row in compare_df.iterrows():
                            # Key: base_period + category (same account_name always)
                            key = (base_period, comp_row["category"])
                            comparison_lookup[key] = {
                                "compare_period": compare_period,
                                "compare_value": float(comp_row["value"] or 0),
                                "account_name": comp_row["account_name"]
                            }

    # Build fixed asset list with comparison data
    fixed_asset_list = []
    
    if not df.empty:
        for _, row in df.iterrows():
            record = {
                "period": row["period"],
                "account_name": row["account_name"],
                "category": row["category"],
                "value": float(row["value"] or 0),
                "comparison": {}
            }
            
            # Add comparison data if available (same account_name & category)
            if comparison:
                key = (row["period"], row["category"])
                compare_data = comparison_lookup.get(key, {})
                
                if compare_data:
                    current_value = float(row["value"] or 0)
                    compare_value = compare_data.get("compare_value", 0)
                    change_value = round(current_value - compare_value, 2)
                    change_pct = round(((change_value / compare_value) * 100), 2) if compare_value else None
                    
                    record["comparison"] = {
                        "current_period": row["period"],
                        "compare_period": compare_data.get("compare_period"),
                        "current_value": round(current_value, 2),
                        "compare_value": round(compare_value, 2),
                        "change_value": change_value,
                        "change_pct": change_pct
                    }
            
            fixed_asset_list.append(record)

    # Calculate key metrics
    total_asset_cost = df["value"].sum() if not df.empty else 0
    current_asset_value = total_asset_cost

    keymetrics = {
        "Total Asset Cost": total_asset_cost,
        "Current Asset Value": current_asset_value
    }

    # Fetch available categories
    category_query = text("SELECT DISTINCT category FROM fpna_landing.finance_fixed_asset ORDER BY category")
    category_result = await db.execute(category_query)
    category_rows = category_result.fetchall()
    all_categories = [row[0] for row in category_rows]

    response = {
        "year": year,
        "currency": currency,
        "period_filter": period.value,
        "selected_category": category,
        "comparison_type": comparison.value if comparison else None,
        "all_categories": all_categories,
        "fixed_asset": fixed_asset_list,
        "keymetrics": keymetrics
    }

    money_fields = ["value", "Total Asset Cost", "Current Asset Value", "current_value", "compare_value", "change_value"]
    response = apply_smart_currency(response, currency, currency_filter, money_fields)
    
    return response

@financial_router.get("/fixed_asset_viewmore/{year}/{currency}", dependencies=[Depends(get_current_company_user)])
async def fixed_asset_viewmore(
    year: int,
    currency: str,
    period: PeriodOption = Query(default=PeriodOption.all),
    category: Optional[str] = Query(default=None),
    comparison: Optional[ComparisonOption] = Query(default=None),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    db: AsyncSession = Depends(get_db)
):
    period_filter_sql = get_fa_period_filter(period, year)
    category_filter_sql = f"AND category = '{category}'" if category else ""

    query = text(f"""
        SELECT
            TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') AS month,
            'Fixed Asset' AS account_name,
            category,
            SUM(purchase_price) AS value
        FROM fpna_landing.finance_fixed_asset
        WHERE 1=1 {period_filter_sql} {category_filter_sql}
        GROUP BY TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'YYYY-MM'), category
        ORDER BY TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'YYYY-MM'), category
    """)

    result = await db.execute(query)
    rows = result.fetchall()
    columns = result.keys()
    df = pd.DataFrame(rows, columns=columns)

    if df.empty:
        return {
            "year": year,
            "period_filter": period.value,
            "selected_category": category,
            "comparison_type": comparison.value if comparison else None,
            "all_categories": [],
            "data": [],
            "keymetrics": {
                "total_asset_cost": 0,
                "current_asset_value": 0
            }
        }

    all_months = sorted(df["month"].unique().tolist())

    df["value"] = df["value"].astype(float)

    pivot_df = df.pivot_table(
        index="category",
        columns="month",
        values="value",
        aggfunc="sum",
        fill_value=0.0
    )

    for month in sorted(all_months):
        if month not in pivot_df.columns:
            pivot_df[month] = 0.0

    pivot_df = pivot_df[sorted(all_months)]

    pivot_df["total"] = pivot_df.sum(axis=1)

    data = []

    for category_name in pivot_df.index:
        row = {
            "account_name": "Fixed Asset",
            "category": str(category_name)
        }

        for month in sorted(all_months):
            value = float(pivot_df.loc[category_name, month])
            row[month] = round(value, 2)

        row["total"] = round(float(pivot_df.loc[category_name, "total"]), 2)

        row["comparisons"] = {}

        if comparison:
            for base_month in sorted(all_months):
                compare_month = _get_fa_comparison_period(base_month, comparison)

                if compare_month:
                    compare_query = text(f"""
                        SELECT
                            TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') AS month,
                            category,
                            SUM(purchase_price) AS value
                        FROM fpna_landing.finance_fixed_asset
                        WHERE 1=1
                            AND TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') = '{compare_month}'
                            AND category = '{category_name}'
                        GROUP BY TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'YYYY-MM'), category
                    """)

                    compare_result = await db.execute(compare_query)
                    compare_rows = compare_result.fetchall()

                    if compare_rows:
                        compare_value = float(compare_rows[0][2])
                        base_value = float(row.get(base_month, 0))
                        variance = base_value - compare_value
                        variance_pct = round(((variance / compare_value) * 100), 2) if compare_value else None

                        row["comparisons"][base_month] = {
                            "base_period": base_month,
                            "compare_period": compare_month,
                            "base_value": round(base_value, 2),
                            "compare_value": round(compare_value, 2),
                            "variance": round(variance, 2),
                            "variance_pct": variance_pct
                        }

        data.append(row)

    grand_total_row = {
        "account_name": "Fixed Asset",
        "category": "total"
    }

    for month in sorted(all_months):
        grand_total_row[month] = round(float(pivot_df[month].sum()), 2)

    grand_total_row["total"] = round(float(pivot_df["total"].sum()), 2)
    grand_total_row["comparisons"] = {}

    if comparison:
        for base_month in sorted(all_months):
            compare_month = _get_fa_comparison_period(base_month, comparison)

            if compare_month:
                compare_query = text(f"""
                    SELECT
                        TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') AS month,
                        SUM(purchase_price) AS value
                    FROM fpna_landing.finance_fixed_asset
                    WHERE 1=1
                        AND TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'YYYY-MM') = '{compare_month}'
                        {category_filter_sql}
                    GROUP BY TO_CHAR(TO_DATE(date_acquired, 'DD-MM-YYYY'), 'YYYY-MM')
                """)

                compare_result = await db.execute(compare_query)
                compare_rows = compare_result.fetchall()

                if compare_rows:
                    compare_value = float(compare_rows[0][1])
                    base_value = float(grand_total_row.get(base_month, 0))
                    variance = base_value - compare_value
                    variance_pct = round(((variance / compare_value) * 100), 2) if compare_value else None

                    grand_total_row["comparisons"][base_month] = {
                        "base_period": base_month,
                        "compare_period": compare_month,
                        "base_value": round(base_value, 2),
                        "compare_value": round(compare_value, 2),
                        "variance": round(variance, 2),
                        "variance_pct": variance_pct
                    }

    data.append(grand_total_row)

    total_asset_cost = grand_total_row["total"]
    current_asset_value = total_asset_cost

    keymetrics = {
        "total_asset_cost": total_asset_cost,
        "current_asset_value": current_asset_value
    }

    category_query = text("SELECT DISTINCT category FROM fpna_landing.finance_fixed_asset ORDER BY category")
    category_result = await db.execute(category_query)
    category_rows = category_result.fetchall()
    all_categories = [row[0] for row in category_rows if row[0]]

    response = {
        "year": year,
        "currency": currency,
        "period_filter": period.value,
        "selected_category": category,
        "comparison_type": comparison.value if comparison else None,
        "all_categories": all_categories,
        "data": data,
        "keymetrics": keymetrics
    }

    money_fields = sorted(all_months) + [
        "total",
        "total_asset_cost",
        "current_asset_value",
        "base_value",
        "compare_value",
        "variance"
    ]

    response = apply_smart_currency(response, currency, currency_filter, money_fields)

    return response
########################################################
##################################################
################################################
#Payroll_Reconciliation


class PayrollSortByOption(str, Enum):
    period = "period"
    department = "department"
    entity = "entity"
    region = "region"
    payroll = "payroll"
    benefits = "benefits"
    taxes = "taxes"
    total = "total"

@financial_router.get(
    "/financials/Payroll_Reconciliation/{year}/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def Payroll_Reconciliation(
    currency: str,
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    year: int = Path(...),
    period: PeriodOption = Query(PeriodOption.all),
    comparison: Optional[ComparisonOption] = Query(None),
    department_name: Optional[str] = Query(None),
    entity_name: Optional[str] = Query(None),
    region: Optional[str] = Query(None),
    sort_by: PayrollSortByOption = Query(PayrollSortByOption.period),
    sort_order: SortOrderOption = Query(SortOrderOption.asc),
    show_period: bool = Query(False),
    show_department: bool = Query(False),
    show_entity: bool = Query(False),
    show_region: bool = Query(False),
    db: AsyncSession = Depends(get_db)
):
    today = date.today()
    hr_start_date = date(year, 1, 1)
    hr_end_date = date(today.year + 1, 1, 1)

    payroll_query = """
        SELECT
            effective_date,
            base_salary
        FROM fpna_landing.hr_compensation
        WHERE effective_date >= :start_date
          AND effective_date <  :end_date
    """

    payroll_result = await db.execute(
        text(payroll_query),
        {"start_date": hr_start_date, "end_date": hr_end_date}
    )

    payroll_df = pd.DataFrame(
        payroll_result.fetchall(),
        columns=["effective_date", "base_salary"]
    )

    if payroll_df.empty:
        payroll_df = pd.DataFrame(columns=["effective_date", "base_salary"])

    payroll_df["base_salary"] = payroll_df["base_salary"].apply(lambda x: Decimal(str(x or 0)))
    payroll_df["period"] = pd.to_datetime(payroll_df["effective_date"]).dt.strftime("%Y-%m")
    payroll_monthly = payroll_df.groupby("period", as_index=False)["base_salary"].sum()
    payroll_monthly.rename(columns={"base_salary": "payroll"}, inplace=True)

    start_period = f"{year}-01"
    end_period = f"{today.year}-{today.month:02d}"

    gl_query = f"""
    SELECT
        gl.period,
        d.department_name AS department,
        e.entity_name AS entity,
        e.region AS region,
        SUM(CASE WHEN LOWER(TRIM(coa.account_name)) = 'salaries'
                 THEN gl.amount_debit - gl.amount_credit ELSE 0 END) AS payroll,
        SUM(CASE WHEN LOWER(TRIM(coa.account_name)) = 'employee benefits'
                 THEN gl.amount_debit - gl.amount_credit ELSE 0 END) AS benefits,
        SUM(CASE WHEN LOWER(TRIM(coa.account_name)) = 'payroll taxes'
                 THEN gl.amount_debit - gl.amount_credit ELSE 0 END) AS taxes
    FROM fpna_landing.finance_general_ledger_transactions gl
    JOIN fpna_landing.finance_chart_of_accounts coa
        ON gl.gl_account_id = coa.account_id
    LEFT JOIN fpna_landing.finance_departments d
        ON gl.department_id::bigint = d.department_id::bigint
    LEFT JOIN fpna_landing.finance_entities e
        ON gl.entity_id::bigint = e.entity_id::bigint
    WHERE coa.account_type = 'Expense'
      AND LOWER(TRIM(coa.account_name)) IN ('salaries', 'employee benefits', 'payroll taxes')
      AND gl.period >= :start_period
      AND gl.period <= :end_period
      { "AND d.department_name = :department" if department_name else "" }
      { "AND e.entity_name = :entity" if entity_name else "" }
      { "AND e.region = :region" if region else "" }
    GROUP BY gl.period, d.department_name, e.entity_name, e.region
    ORDER BY gl.period
    """

    gl_params = {
        "start_period": start_period,
        "end_period": end_period
    }

    if department_name:
        gl_params["department"] = department_name
    if entity_name:
        gl_params["entity"] = entity_name
    if region:
        gl_params["region"] = region

    gl_result = await db.execute(text(gl_query), gl_params)

    gl_df = pd.DataFrame(
        gl_result.fetchall(),
        columns=["period", "department", "entity", "region", "payroll", "benefits", "taxes"]
    )

    if gl_df.empty:
        gl_df = pd.DataFrame(columns=["period", "department", "entity", "region", "payroll", "benefits", "taxes"])

    gl_df["period"] = pd.to_datetime(gl_df["period"]).dt.strftime("%Y-%m")

    for col in ["payroll", "benefits", "taxes"]:
        gl_df[col] = gl_df[col].apply(lambda x: Decimal(str(x or 0)))

    gl_df["ai_insights"] = [ai_insights for _ in range(len(gl_df))]
    gl_df["total"] = gl_df["payroll"] + gl_df["benefits"] + gl_df["taxes"]

    monthly_df = gl_df[
        ["period", "department", "entity", "region", "payroll", "benefits", "taxes", "total", "ai_insights"]
    ].copy()

    system_date = pd.Timestamp.today()
    system_year = system_date.year
    system_month = system_date.month

    if period == PeriodOption.all:
        monthly_df = monthly_df[monthly_df["period"].str.startswith(f"{year}-")]

    elif period == PeriodOption.current_month:
        monthly_df = monthly_df[monthly_df["period"] == f"{year}-{system_month:02d}"]

    elif period == PeriodOption.last_month:
        last_month = (pd.Timestamp(year=year, month=system_month, day=1) - pd.DateOffset(months=1))
        monthly_df = monthly_df[monthly_df["period"] == f"{last_month.year}-{last_month.month:02d}"]

    elif period == PeriodOption.quarter_to_date:
        monthly_df = monthly_df[
            (monthly_df["period"] >= f"{year}-01") &
            (monthly_df["period"] <= f"{year}-03")
        ]

    elif period == PeriodOption.year_to_date:
        monthly_df = monthly_df[
            (monthly_df["period"] >= f"{year}-01") &
            (monthly_df["period"] <= f"{system_year}-{system_month:02d}")
        ]

    full_df = gl_df.copy()

    monthly_df["comparisons"] = [{} for _ in range(len(monthly_df))]

    if comparison:
        full_df["period"] = pd.to_datetime(full_df["period"]).dt.strftime("%Y-%m")
        full_df["department"] = full_df["department"].fillna("Unknown")
        full_df["entity"] = full_df["entity"].fillna("Unknown")
        full_df["region"] = full_df["region"].fillna("Unknown")

        monthly_df["department"] = monthly_df["department"].fillna("Unknown")
        monthly_df["entity"] = monthly_df["entity"].fillna("Unknown")
        monthly_df["region"] = monthly_df["region"].fillna("Unknown")

        compare_lookup = {}
        for _, r in full_df.iterrows():
            key = (r["period"], r["department"], r["entity"], r["region"])
            compare_lookup[key] = {
                "payroll": r["payroll"],
                "benefits": r["benefits"],
                "taxes": r["taxes"],
                "total": r["total"]
            }

        for idx, row in monthly_df.iterrows():
            compare_period = get_comparison_period(row["period"], comparison)

            if not compare_period:
                continue

            key = (compare_period, row["department"], row["entity"], row["region"])
            comp = compare_lookup.get(key)

            if not comp:
                comp = {
                    "payroll": Decimal("0"),
                    "benefits": Decimal("0"),
                    "taxes": Decimal("0"),
                    "total": Decimal("0")
                }

            payroll_var = row["payroll"] - comp["payroll"]
            benefits_var = row["benefits"] - comp["benefits"]
            taxes_var = row["taxes"] - comp["taxes"]
            total_var = row["total"] - comp["total"]

            monthly_df.at[idx, "comparisons"] = {
                "compare_period": compare_period,
                "compare_payroll": float(round(comp["payroll"], 2)),
                "payroll_varience": float(round(payroll_var, 2)),
                "compare_benefits": float(round(comp["benefits"], 2)),
                "benefits_varience": float(round(benefits_var, 2)),
                "compare_taxes": float(round(comp["taxes"], 2)),
                "taxes_varience": float(round(taxes_var, 2)),
                "compare_total": float(round(comp["total"], 2)),
                "total_varience": float(round(total_var, 2))
            }

    sort_col = sort_by.value
    ascending = sort_order == SortOrderOption.asc

    if sort_col in monthly_df.columns:
        monthly_df = monthly_df.sort_values(by=sort_col, ascending=ascending, na_position="last")

    total_payroll_raw   = monthly_df["payroll"].sum()
    total_benefits_raw  = monthly_df["benefits"].sum()
    total_taxes_raw     = monthly_df["taxes"].sum()
    grand_total_raw     = total_payroll_raw + total_benefits_raw + total_taxes_raw
    average_per_month   = grand_total_raw / Decimal("12") if grand_total_raw else Decimal("0")

    km_raw = {
        "total_payroll":    total_payroll_raw,
        "total_benefits":   total_benefits_raw,
        "total_taxes":      total_taxes_raw,
        "grand_total":      grand_total_raw,
        "average_per_month": average_per_month,
        "ai_insights":      ai_insights
    }
    keymetrics = apply_smart_currency(
        km_raw, currency, currency_filter,
        ["total_payroll", "total_benefits", "total_taxes", "grand_total", "average_per_month"]
    )

    monthly_records = monthly_df.to_dict(orient="records")

    money_fields     = ["payroll", "benefits", "taxes", "total"]
    comp_money_fields = [
        "compare_payroll", "payroll_varience",
        "compare_benefits", "benefits_varience",
        "compare_taxes", "taxes_varience",
        "compare_total", "total_varience"
    ]

    row_period_data = {}
    row_dim_data    = {}

    for row in monthly_records:
        p        = str(row.get("period", ""))
        dept     = row.get("department")
        ent      = row.get("entity")
        reg      = row.get("region")
        pay_v    = float(row.get("payroll",  0) or 0)
        ben_v    = float(row.get("benefits", 0) or 0)
        tax_v    = float(row.get("taxes",    0) or 0)
        tot_v    = float(row.get("total",    0) or 0)
        comp_raw = row.get("comparisons", {})

        key_parts = []
        if show_department: key_parts.append(dept)
        if show_entity:     key_parts.append(ent)
        if show_region:     key_parts.append(reg)
        key = tuple(key_parts)

        if key not in row_period_data:
            row_period_data[key] = {}
            dims = {}
            if show_department: dims["department"] = dept
            if show_entity:     dims["entity"]     = ent
            if show_region:     dims["region"]     = reg
            row_dim_data[key] = dims

        if p not in row_period_data[key]:
            row_period_data[key][p] = {
                "payroll":  pay_v,
                "benefits": ben_v,
                "taxes":    tax_v,
                "total":    tot_v,
                "comp_raw": comp_raw,
            }
        else:
            ex = row_period_data[key][p]
            ex["payroll"]  += pay_v
            ex["benefits"] += ben_v
            ex["taxes"]    += tax_v
            ex["total"]    += tot_v

    periods_available = sorted(set(
        p for pd_data in row_period_data.values() for p in pd_data if p
    ))

    grand_period_map  = {}
    grand_period_comp = {}

    records = []
    for key, p_data in row_period_data.items():
        dims = row_dim_data[key]

        months           = {}
        tot_payroll      = 0.0
        tot_benefits     = 0.0
        tot_taxes        = 0.0
        tot_total        = 0.0
        prev_payroll     = 0.0
        prev_benefits    = 0.0
        prev_taxes       = 0.0
        prev_total       = 0.0
        var_payroll_acc  = 0.0
        var_benefits_acc = 0.0
        var_taxes_acc    = 0.0
        var_total_acc    = 0.0
        has_comp         = False

        for p in periods_available:
            if p in p_data:
                entry    = p_data[p]
                pay_v    = entry["payroll"]
                ben_v    = entry["benefits"]
                tax_v    = entry["taxes"]
                tot_v    = entry["total"]
                comp_raw = entry.get("comp_raw", {})

                converted = apply_smart_currency(
                    {"payroll": pay_v, "benefits": ben_v, "taxes": tax_v, "total": tot_v},
                    currency, currency_filter, money_fields
                )
                month_entry = {
                    "payroll":     converted["payroll"],
                    "benefits":    converted["benefits"],
                    "taxes":       converted["taxes"],
                    "total":       converted["total"],
                    "ai_insights": ai_insights,
                }

                if comparison and comp_raw:
                    c_pay  = float(comp_raw.get("compare_payroll",   0) or 0)
                    c_ben  = float(comp_raw.get("compare_benefits",  0) or 0)
                    c_tax  = float(comp_raw.get("compare_taxes",     0) or 0)
                    c_tot  = float(comp_raw.get("compare_total",     0) or 0)
                    v_pay  = float(comp_raw.get("payroll_varience",  0) or 0)
                    v_ben  = float(comp_raw.get("benefits_varience", 0) or 0)
                    v_tax  = float(comp_raw.get("taxes_varience",    0) or 0)
                    v_tot  = float(comp_raw.get("total_varience",    0) or 0)

                    comp_conv = apply_smart_currency(
                        {
                            "compare_payroll":   c_pay, "payroll_varience":  v_pay,
                            "compare_benefits":  c_ben, "benefits_varience": v_ben,
                            "compare_taxes":     c_tax, "taxes_varience":    v_tax,
                            "compare_total":     c_tot, "total_varience":    v_tot,
                        },
                        currency, currency_filter, comp_money_fields
                    )
                    month_entry["comparison"] = {
                        "compare_period": comp_raw.get("compare_period"),
                        **comp_conv
                    }
                    prev_payroll     += c_pay
                    prev_benefits    += c_ben
                    prev_taxes       += c_tax
                    prev_total       += c_tot
                    var_payroll_acc  += v_pay
                    var_benefits_acc += v_ben
                    var_taxes_acc    += v_tax
                    var_total_acc    += v_tot
                    has_comp = True

                    if p not in grand_period_comp:
                        grand_period_comp[p] = {
                            "compare_payroll": 0.0,   "payroll_varience":  0.0,
                            "compare_benefits": 0.0,  "benefits_varience": 0.0,
                            "compare_taxes": 0.0,     "taxes_varience":    0.0,
                            "compare_total": 0.0,     "total_varience":    0.0,
                            "compare_period": comp_raw.get("compare_period"),
                        }
                    grand_period_comp[p]["compare_payroll"]   += c_pay
                    grand_period_comp[p]["payroll_varience"]  += v_pay
                    grand_period_comp[p]["compare_benefits"]  += c_ben
                    grand_period_comp[p]["benefits_varience"] += v_ben
                    grand_period_comp[p]["compare_taxes"]     += c_tax
                    grand_period_comp[p]["taxes_varience"]    += v_tax
                    grand_period_comp[p]["compare_total"]     += c_tot
                    grand_period_comp[p]["total_varience"]    += v_tot
                else:
                    month_entry["comparison"] = {}

                if show_period:
                    months[p] = month_entry

                tot_payroll  += pay_v
                tot_benefits += ben_v
                tot_taxes    += tax_v
                tot_total    += tot_v

                if p not in grand_period_map:
                    grand_period_map[p] = {"payroll": 0.0, "benefits": 0.0, "taxes": 0.0, "total": 0.0}
                grand_period_map[p]["payroll"]  += pay_v
                grand_period_map[p]["benefits"] += ben_v
                grand_period_map[p]["taxes"]    += tax_v
                grand_period_map[p]["total"]    += tot_v

        total_conv = apply_smart_currency(
            {"payroll": tot_payroll, "benefits": tot_benefits, "taxes": tot_taxes, "total": tot_total},
            currency, currency_filter, money_fields
        )
        record           = dict(dims)
        record["months"] = months
        record["total"]  = {
            "payroll":  total_conv["payroll"],
            "benefits": total_conv["benefits"],
            "taxes":    total_conv["taxes"],
            "total":    total_conv["total"],
        }
        record["comparison"] = {}
        if comparison and has_comp:
            prev_conv = apply_smart_currency(
                {
                    "compare_payroll":   prev_payroll,     "payroll_varience":  var_payroll_acc,
                    "compare_benefits":  prev_benefits,    "benefits_varience": var_benefits_acc,
                    "compare_taxes":     prev_taxes,       "taxes_varience":    var_taxes_acc,
                    "compare_total":     prev_total,       "total_varience":    var_total_acc,
                },
                currency, currency_filter, comp_money_fields
            )
            record["comparison"] = prev_conv

        records.append(record)

    grand_months         = {}
    gt_payroll           = 0.0
    gt_benefits          = 0.0
    gt_taxes             = 0.0
    gt_total             = 0.0
    gt_prev_payroll      = 0.0
    gt_prev_benefits     = 0.0
    gt_prev_taxes        = 0.0
    gt_prev_total        = 0.0
    gt_var_payroll       = 0.0
    gt_var_benefits      = 0.0
    gt_var_taxes         = 0.0
    gt_var_total         = 0.0

    for p in periods_available:
        if p in grand_period_map:
            gm    = grand_period_map[p]
            g_pay = gm["payroll"]
            g_ben = gm["benefits"]
            g_tax = gm["taxes"]
            g_tot = gm["total"]

            conv_gm = apply_smart_currency(
                {"payroll": g_pay, "benefits": g_ben, "taxes": g_tax, "total": g_tot},
                currency, currency_filter, money_fields
            )
            gm_entry = {
                "payroll":  conv_gm["payroll"],
                "benefits": conv_gm["benefits"],
                "taxes":    conv_gm["taxes"],
                "total":    conv_gm["total"],
            }

            if comparison and p in grand_period_comp:
                gc = grand_period_comp[p]
                gc_conv = apply_smart_currency(
                    {
                        "compare_payroll":   gc["compare_payroll"],  "payroll_varience":  gc["payroll_varience"],
                        "compare_benefits":  gc["compare_benefits"], "benefits_varience": gc["benefits_varience"],
                        "compare_taxes":     gc["compare_taxes"],    "taxes_varience":    gc["taxes_varience"],
                        "compare_total":     gc["compare_total"],    "total_varience":    gc["total_varience"],
                    },
                    currency, currency_filter, comp_money_fields
                )
                gm_entry["comparison"] = {
                    "compare_period": gc["compare_period"],
                    **gc_conv
                }
                gt_prev_payroll  += gc["compare_payroll"]
                gt_prev_benefits += gc["compare_benefits"]
                gt_prev_taxes    += gc["compare_taxes"]
                gt_prev_total    += gc["compare_total"]
                gt_var_payroll   += gc["payroll_varience"]
                gt_var_benefits  += gc["benefits_varience"]
                gt_var_taxes     += gc["taxes_varience"]
                gt_var_total     += gc["total_varience"]
            else:
                gm_entry["comparison"] = {}

            if show_period:
                grand_months[p] = gm_entry

            gt_payroll  += g_pay
            gt_benefits += g_ben
            gt_taxes    += g_tax
            gt_total    += g_tot

    gt_total_conv = apply_smart_currency(
        {"payroll": gt_payroll, "benefits": gt_benefits, "taxes": gt_taxes, "total": gt_total},
        currency, currency_filter, money_fields
    )
    grand_total_record = {
        "months": grand_months,
        "total": {
            "payroll":  gt_total_conv["payroll"],
            "benefits": gt_total_conv["benefits"],
            "taxes":    gt_total_conv["taxes"],
            "total":    gt_total_conv["total"],
        },
        "comparison": {},
    }
    if comparison and grand_period_comp:
        gt_prev_conv = apply_smart_currency(
            {
                "compare_payroll":   gt_prev_payroll,  "payroll_varience":  gt_var_payroll,
                "compare_benefits":  gt_prev_benefits, "benefits_varience": gt_var_benefits,
                "compare_taxes":     gt_prev_taxes,    "taxes_varience":    gt_var_taxes,
                "compare_total":     gt_prev_total,    "total_varience":    gt_var_total,
            },
            currency, currency_filter, comp_money_fields
        )
        grand_total_record["comparison"] = gt_prev_conv

    dep_result = await db.execute(
        text("SELECT department_name FROM fpna_landing.finance_departments ORDER BY department_name")
    )
    all_departments = [r.department_name for r in dep_result.fetchall()]

    ent_result = await db.execute(
        text("SELECT entity_name FROM fpna_landing.finance_entities ORDER BY entity_name")
    )
    all_entities = [r.entity_name for r in ent_result.fetchall()]

    reg_result = await db.execute(
        text("SELECT DISTINCT region FROM fpna_landing.finance_entities ORDER BY region")
    )
    all_regions = [r.region for r in reg_result.fetchall()]

    return {
        "year":        year,
        "period":      period.value,
        "filters": {
            "department": department_name,
            "entity":     entity_name,
            "region":     region
        },
        "keymetrics":  keymetrics,
        "records":     records,
        "grand_total": grand_total_record,
        "periods":     periods_available,
        "show_columns": {
            "period":     show_period,
            "department": show_department,
            "entity":     show_entity,
            "region":     show_region,
        },
        "departments": all_departments,
        "entities":    all_entities,
        "regions":     all_regions,
    }


###################################
#####################################################
##############################################

#Payroll_Reconciliation_Viewmore

@financial_router.get(
    "/financials/Payroll_Reconciliation_Viewmore/{year}/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def Payroll_Reconciliation_Viewmore(
    currency: str,
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    year: int = Path(...),
    period: PeriodOption = Query(PeriodOption.all),
    comparison: Optional[ComparisonOption] = Query(None),
    db: AsyncSession = Depends(get_db)
):
    today = date.today()

    start_period = f"{year}-01"
    end_period = f"{today.year}-{today.month:02d}"

    period_sql_filter = get_gl_period_filter(period, year)

    gl_query = f"""
    SELECT
        d.department_name AS department,
        gl.period,
        SUM(CASE WHEN LOWER(TRIM(coa.account_name)) = 'salaries'
                 THEN gl.amount_debit - gl.amount_credit ELSE 0 END) AS payroll,
        SUM(CASE WHEN LOWER(TRIM(coa.account_name)) = 'employee benefits'
                 THEN gl.amount_debit - gl.amount_credit ELSE 0 END) AS benefits,
        SUM(CASE WHEN LOWER(TRIM(coa.account_name)) = 'payroll taxes'
                 THEN gl.amount_debit - gl.amount_credit ELSE 0 END) AS taxes
    FROM fpna_landing.finance_general_ledger_transactions gl
    JOIN fpna_landing.finance_chart_of_accounts coa
        ON gl.gl_account_id = coa.account_id
    LEFT JOIN fpna_landing.finance_departments d
        ON gl.department_id::bigint = d.department_id::bigint
    WHERE coa.account_type = 'Expense'
      AND LOWER(TRIM(coa.account_name)) IN ('salaries', 'employee benefits', 'payroll taxes')
      AND gl.period >= :start_period
      AND gl.period <= :end_period
      {period_sql_filter}
    GROUP BY d.department_name, gl.period
    ORDER BY d.department_name, gl.period
    """

    gl_result = await db.execute(
        text(gl_query),
        {"start_period": start_period, "end_period": end_period}
    )

    gl_df = pd.DataFrame(
        gl_result.fetchall(),
        columns=["department", "period", "payroll", "benefits", "taxes"]
    )

    if gl_df.empty:
        gl_df = pd.DataFrame(columns=["department", "period", "payroll", "benefits", "taxes"])

    gl_df["period"] = pd.to_datetime(gl_df["period"]).dt.strftime("%Y-%m")

    for col in ["payroll", "benefits", "taxes"]:
        gl_df[col] = gl_df[col].apply(lambda x: Decimal(str(x or 0)))

    months = sorted(gl_df["period"].unique().tolist())

    types = ["payroll", "benefits", "taxes"]
    final_rows = []

    for t in types:
        df_type = gl_df[["department", "period", t]].copy()
        if df_type.empty:
            continue

        pivot = df_type.pivot_table(
            index="department",
            columns="period",
            values=t,
            aggfunc="sum",
            fill_value=Decimal("0")
        ).reset_index()

        for month in months:
            if month not in pivot.columns:
                pivot[month] = Decimal("0")

        pivot["total"] = pivot[months].sum(axis=1)
        pivot.insert(0, "type", t)
        pivot["ai_insights"] = [ai_insights for _ in range(len(pivot))]

        final_rows.append(pivot)

        total_row = {"type": t, "department": "total"}
        for month in months:
            total_row[month] = pivot[month].sum()
        total_row["total"] = pivot["total"].sum()
        total_row["ai_insights"] = ai_insights
        final_rows.append(pd.DataFrame([total_row]))

    result_df = pd.concat(final_rows, ignore_index=True) if final_rows else pd.DataFrame()

    if result_df.empty:
        result_df = pd.DataFrame(columns=["type", "department"] + months + ["total", "ai_insights"])

    result_df["comparisons"] = [{} for _ in range(len(result_df))]

    if comparison and months:
        compare_months = [get_comparison_period(m, comparison) for m in months]
        compare_months = [m for m in compare_months if m is not None]

        if compare_months:
            compare_start = min(compare_months)
            compare_end = max(compare_months)

            compare_query = f"""
            SELECT
                d.department_name AS department,
                gl.period,
                SUM(CASE WHEN LOWER(TRIM(coa.account_name)) = 'salaries'
                         THEN gl.amount_debit - gl.amount_credit ELSE 0 END) AS payroll,
                SUM(CASE WHEN LOWER(TRIM(coa.account_name)) = 'employee benefits'
                         THEN gl.amount_debit - gl.amount_credit ELSE 0 END) AS benefits,
                SUM(CASE WHEN LOWER(TRIM(coa.account_name)) = 'payroll taxes'
                         THEN gl.amount_debit - gl.amount_credit ELSE 0 END) AS taxes
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa
                ON gl.gl_account_id = coa.account_id
            LEFT JOIN fpna_landing.finance_departments d
                ON gl.department_id::bigint = d.department_id::bigint
            WHERE coa.account_type = 'Expense'
              AND LOWER(TRIM(coa.account_name)) IN ('salaries', 'employee benefits', 'payroll taxes')
              AND gl.period >= :compare_start
              AND gl.period <= :compare_end
            GROUP BY d.department_name, gl.period
            ORDER BY d.department_name, gl.period
            """

            compare_result = await db.execute(
                text(compare_query),
                {"compare_start": compare_start, "compare_end": compare_end}
            )

            compare_df = pd.DataFrame(
                compare_result.fetchall(),
                columns=["department", "period", "payroll", "benefits", "taxes"]
            )

            if compare_df.empty:
                compare_df = pd.DataFrame(columns=["department", "period", "payroll", "benefits", "taxes"])

            compare_df["period"] = pd.to_datetime(compare_df["period"]).dt.strftime("%Y-%m")

            for col in ["payroll", "benefits", "taxes"]:
                compare_df[col] = compare_df[col].apply(lambda x: Decimal(str(x or 0)))

            compare_pivot_map = {}

            for t in types:
                df_t = compare_df[["department", "period", t]].copy()
                if df_t.empty:
                    compare_pivot_map[t] = pd.DataFrame(columns=["department"] + compare_months)
                    continue

                pvt = df_t.pivot_table(
                    index="department",
                    columns="period",
                    values=t,
                    aggfunc="sum",
                    fill_value=Decimal("0")
                ).reset_index()

                for cm in compare_months:
                    if cm not in pvt.columns:
                        pvt[cm] = Decimal("0")

                compare_pivot_map[t] = pvt

            comparisons_list = []

            for _, row in result_df.iterrows():
                t = row.get("type")
                dept = row.get("department")

                if dept == "total":
                    comparisons_list.append({})
                    continue

                compare_pvt = compare_pivot_map.get(t)
                if compare_pvt is None or compare_pvt.empty:
                    comparisons_list.append({})
                    continue

                dept_row = compare_pvt[compare_pvt["department"] == dept]
                if dept_row.empty:
                    comparisons_list.append({})
                    continue

                dept_row = dept_row.iloc[0]

                month_comparisons = []

                for base_m in months:
                    compare_m = get_comparison_period(base_m, comparison)
                    if not compare_m:
                        continue

                    base_val = row.get(base_m, Decimal("0"))
                    compare_val = dept_row.get(compare_m, Decimal("0"))

                    try:
                        base_val = Decimal(str(base_val or 0))
                    except:
                        base_val = Decimal("0")

                    try:
                        compare_val = Decimal(str(compare_val or 0))
                    except:
                        compare_val = Decimal("0")

                    variance = base_val - compare_val

                    month_comparisons.append({
                        "base_period": base_m,
                        "compare_period": compare_m,
                        "base_value": round(float(base_val), 2),
                        "compare_value": round(float(compare_val), 2),
                        "variance": round(float(variance), 2)
                    })

                comparisons_list.append({
                    "comparison_type": comparison.value,
                    "months": month_comparisons
                })

            result_df["comparisons"] = comparisons_list

    total_payroll = gl_df["payroll"].sum()
    total_benefits = gl_df["benefits"].sum()
    total_taxes = gl_df["taxes"].sum()
    grand_total = total_payroll + total_benefits + total_taxes
    average_per_month = grand_total / Decimal("12") if grand_total else Decimal("0")

    key_metrics = {
        "total_payroll": total_payroll,
        "total_benefits": total_benefits,
        "total_taxes": total_taxes,
        "grand_total": grand_total,
        "average_per_month": average_per_month,
        "ai_insights": ai_insights
    }

    keymetrics = apply_smart_currency(
        key_metrics, currency, currency_filter, list(key_metrics.keys())
    )

    money_fields = months + ["total"]

    if comparison:
        money_fields = money_fields + ["base_value", "compare_value", "variance"]

    monthly_data = apply_smart_currency(
        result_df.to_dict(orient="records"),
        currency,
        currency_filter,
        money_fields
    )

    return {
        "year": year,
        "period": period.value,
        "keymetrics": keymetrics,
        "data": monthly_data
    }

    
    
##############################################################
###################################################################
################################################################
#Sales Funnel Metrics


class SalesFunnelSortByOption(str, Enum):
    period = "period"
    stage = "stage"
    product_name = "product_name"
    region = "region"
    count = "count"
    conversion_rate = "conversion_rate"


def salesfunnel_comparison_period(period_str: str, comparison: ComparisonOption):
    if not period_str:
        return None

    if not comparison:
        return None

    dt = pd.to_datetime(period_str, format="%Y-%m", errors="coerce")
    if pd.isna(dt):
        return None

    if comparison == ComparisonOption.mom:
        return (dt - pd.DateOffset(months=1)).strftime("%Y-%m")

    if comparison == ComparisonOption.yoy:
        return (dt - pd.DateOffset(years=1)).strftime("%Y-%m")

    return None


@financial_router.get(
    "/Sales_Funnel_Metrics/{year}/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def Sales_Funnel_Metrics(
    currency: str,
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    year: int = Path(...),
    period: PeriodOption = Query(PeriodOption.all),
    product: Optional[str] = Query(None),
    region: Optional[str] = Query(None),
    comparison: Optional[ComparisonOption] = Query(None),
    sort_by: SalesFunnelSortByOption = Query(SalesFunnelSortByOption.period),
    sort_order: SortOrderOption = Query(SortOrderOption.asc),
    show_period: bool = Query(False),
    show_product_name: bool = Query(False),
    show_region: bool = Query(False),
    db: AsyncSession = Depends(get_db)
):
    today = date.today()
    system_month = today.month

    if period == PeriodOption.all:
        start_date = date(year, 1, 1)
        end_date = date(year, 12, 31)

    elif period == PeriodOption.current_month:
        start_date = date(year, system_month, 1)
        end_date = date(year, system_month, monthrange(year, system_month)[1])

    elif period == PeriodOption.last_month:
        if system_month == 1:
            last_month = 12
            last_year = year - 1
        else:
            last_month = system_month - 1
            last_year = year

        start_date = date(last_year, last_month, 1)
        end_date = date(last_year, last_month, monthrange(last_year, last_month)[1])

    elif period == PeriodOption.quarter_to_date:
        start_date = date(year, 1, 1)
        end_date = date(year, 3, 31)

    elif period == PeriodOption.year_to_date:
        start_date = date(year, 1, 1)
        end_date = today

    params = {"start_date": start_date, "end_date": end_date}

    product_filter_sql = "AND p.product_name = :product" if product else ""
    region_filter_sql = "AND a.region = :region" if region else ""

    if product:
        params["product"] = product
    if region:
        params["region"] = region

    opp_query = text(f"""
        SELECT DISTINCT ON (o.opportunity_id)
            o.opportunity_id,
            o.stage,
            o.created_date,
            o.close_date,
            a.region,
            COALESCE(p.product_name, 'Unknown') AS product_name
        FROM fpna_landing.crm_opportunities o
        JOIN fpna_landing.crm_accounts a ON o.account_id = a.account_id
        LEFT JOIN fpna_landing.crm_products p 
            ON a.profit_center_id = p.profit_center_id
        WHERE o.created_date BETWEEN :start_date AND :end_date
        {product_filter_sql}
        {region_filter_sql}
    """)

    result = await db.execute(opp_query, params)
    rows = result.mappings().all()

    if not rows:
        return {
            "year": year,
            "period": period.value,
            "key_metrics": {
                "total_leads": 0,
                "total_won": 0,
                "overall_conversion": 0.0,
                "average_sales_cycle_days": 0.0
            },
            "funnel": [],
            "grand_total": {
                "period": None,
                "stage": "Grand Total",
                "product_name": None,
                "region": None,
                "count": 0,
                "conversion_rate": 0.0,
                "comparsions": {}
            },
            "regions": [],
            "products": []
        }

    df = pd.DataFrame(rows)
    df["created_date"] = pd.to_datetime(df["created_date"])
    df["close_date"] = pd.to_datetime(df["close_date"], errors="coerce")

    df["period_key"] = df["created_date"].dt.strftime("%m-%Y")

    stage_order = df["stage"].dropna().unique().tolist()

    stage_counts = (
        df.groupby(["period_key", "stage", "product_name", "region"])
        .size()
        .reset_index(name="count")
    )

    monthly_total_leads = (
        df.groupby("period_key")
        .size()
        .reset_index(name="month_total_leads")
    )

    stage_counts = stage_counts.merge(monthly_total_leads, on="period_key", how="left")

    stage_counts["conversion_rate"] = (
        stage_counts["count"] / stage_counts["month_total_leads"]
    )

    comparison_lookup = {}
    compare_period_lookup = {}

    if comparison:
        current_periods = sorted(df["period_key"].dropna().unique().tolist())

        current_periods_ym = []
        for p in current_periods:
            mm, yyyy = p.split("-")
            current_periods_ym.append(f"{yyyy}-{mm}")

        compare_periods_ym = set()
        mapping_current_to_compare = {}

        for current_ym in current_periods_ym:
            compare_ym = salesfunnel_comparison_period(current_ym, comparison)
            if compare_ym:
                compare_periods_ym.add(compare_ym)
                mapping_current_to_compare[current_ym] = compare_ym

        compare_rows_all = []

        for compare_ym in sorted(compare_periods_ym):
            compare_dt = pd.to_datetime(compare_ym, format="%Y-%m", errors="coerce")
            if pd.isna(compare_dt):
                continue

            compare_year = int(compare_dt.year)
            compare_month = int(compare_dt.month)

            compare_start_date = date(compare_year, compare_month, 1)
            compare_end_date = date(compare_year, compare_month, monthrange(compare_year, compare_month)[1])

            compare_params = {
                "start_date": compare_start_date,
                "end_date": compare_end_date
            }

            if product:
                compare_params["product"] = product
            if region:
                compare_params["region"] = region

            compare_query = text(f"""
                SELECT DISTINCT ON (o.opportunity_id)
                    o.opportunity_id,
                    o.stage,
                    o.created_date,
                    o.close_date,
                    a.region,
                    COALESCE(p.product_name, 'Unknown') AS product_name
                FROM fpna_landing.crm_opportunities o
                JOIN fpna_landing.crm_accounts a ON o.account_id = a.account_id
                LEFT JOIN fpna_landing.crm_products p 
                    ON a.profit_center_id = p.profit_center_id
                WHERE o.created_date BETWEEN :start_date AND :end_date
                {product_filter_sql}
                {region_filter_sql}
            """)

            compare_result = await db.execute(compare_query, compare_params)
            compare_rows = compare_result.mappings().all()
            compare_rows_all.extend(compare_rows)

        if compare_rows_all:
            compare_df = pd.DataFrame(compare_rows_all)
            compare_df["created_date"] = pd.to_datetime(compare_df["created_date"])
            compare_df["close_date"] = pd.to_datetime(compare_df["close_date"], errors="coerce")
            compare_df["period_key"] = compare_df["created_date"].dt.strftime("%m-%Y")

            compare_stage_counts = (
                compare_df.groupby(["period_key", "stage", "product_name", "region"])
                .size()
                .reset_index(name="count")
            )

            compare_monthly_total_leads = (
                compare_df.groupby("period_key")
                .size()
                .reset_index(name="month_total_leads")
            )

            compare_stage_counts = compare_stage_counts.merge(
                compare_monthly_total_leads, on="period_key", how="left"
            )

            compare_stage_counts["conversion_rate"] = (
                compare_stage_counts["count"] / compare_stage_counts["month_total_leads"]
            )

            for _, r in compare_stage_counts.iterrows():
                key = (r["period_key"], r["stage"], r["product_name"], r["region"])
                comparison_lookup[key] = {
                    "compare_count": int(r["count"]),
                    "compare_conversion_rate": round(float(r["conversion_rate"]) * 100, 2)
                }

            for current_ym, compare_ym in mapping_current_to_compare.items():
                c_dt = pd.to_datetime(current_ym, format="%Y-%m", errors="coerce")
                cp_dt = pd.to_datetime(compare_ym, format="%Y-%m", errors="coerce")
                if pd.isna(c_dt) or pd.isna(cp_dt):
                    continue

                current_mm_yyyy = c_dt.strftime("%m-%Y")
                compare_mm_yyyy = cp_dt.strftime("%m-%Y")
                compare_period_lookup[current_mm_yyyy] = compare_mm_yyyy

    total_leads = len(df)
    total_won   = df[df["stage"] == "Closed Won"].shape[0]

    avg_sales_cycle_days = round(
        (df.loc[df["stage"] == "Closed Won", "close_date"] -
         df.loc[df["stage"] == "Closed Won", "created_date"])
        .dt.days.mean() if total_won else 0.0,
        2
    )

    overall_conversion = round(
        (total_won / total_leads) * 100 if total_leads else 0.0,
        2
    )

    keymetrics_raw = {
        "total_leads":             total_leads,
        "total_won":               total_won,
        "overall_conversion":      overall_conversion,
        "average_sales_cycle_days": avg_sales_cycle_days
    }
    m_f = ["average_sales_cycle_days", "overall_conversion", "total_won", "total_leads"]
    key_metrics = apply_smart_currency(keymetrics_raw, currency, currency_filter, m_f)

    period_total_leads_map = {}
    for _, r in monthly_total_leads.iterrows():
        mm_r, yyyy_r = r["period_key"].split("-")
        period_total_leads_map[f"{yyyy_r}-{mm_r}"] = int(r["month_total_leads"])

    period_order = sorted(
        df["period_key"].dropna().unique().tolist(),
        key=lambda x: pd.to_datetime(x, format="%m-%Y")
    )

    row_period_data = {}
    row_dim_data    = {}

    for period_key in period_order:
        period_df = stage_counts[stage_counts["period_key"] == period_key]
        mm_pk, yyyy_pk = period_key.split("-")
        p = f"{yyyy_pk}-{mm_pk}"

        for stage in stage_order:
            stage_df = period_df[period_df["stage"] == stage]

            for _, row in stage_df.iterrows():
                stage_val = row["stage"]
                prod_val  = row["product_name"]
                reg_val   = row["region"]
                cnt_v     = int(row["count"])

                comp_period_val = None
                comp_count_val  = 0
                if comparison:
                    comp_period_key = compare_period_lookup.get(period_key)
                    if comp_period_key:
                        mm_c, yyyy_c    = comp_period_key.split("-")
                        comp_period_val = f"{yyyy_c}-{mm_c}"
                        comp_key        = (comp_period_key, stage_val, prod_val, reg_val)
                        comp_data       = comparison_lookup.get(comp_key)
                        if comp_data:
                            comp_count_val = int(comp_data["compare_count"])

                key_parts = [stage_val]
                if show_product_name: key_parts.append(prod_val)
                if show_region:       key_parts.append(reg_val)
                key = tuple(key_parts)

                if key not in row_period_data:
                    row_period_data[key] = {}
                    dims = {"stage": stage_val}
                    if show_product_name: dims["product_name"] = prod_val
                    if show_region:       dims["region"]       = reg_val
                    row_dim_data[key] = dims

                if p not in row_period_data[key]:
                    row_period_data[key][p] = {
                        "count":          cnt_v,
                        "compare_count":  comp_count_val,
                        "compare_period": comp_period_val,
                    }
                else:
                    ex = row_period_data[key][p]
                    ex["count"]         += cnt_v
                    ex["compare_count"] += comp_count_val

    periods_available = sorted(set(
        p for pd_data in row_period_data.values() for p in pd_data if p
    ))

    grand_period_map = {}

    records = []
    for key, p_data in row_period_data.items():
        dims = row_dim_data[key]

        months         = {}
        tot_count      = 0
        tot_comp_count = 0

        for p in periods_available:
            if p in p_data:
                entry        = p_data[p]
                cnt_v        = entry["count"]
                comp_cnt_v   = entry["compare_count"]
                comp_per_v   = entry["compare_period"]
                period_leads = period_total_leads_map.get(p, 0)
                conv_v       = round((cnt_v / period_leads) * 100, 2) if period_leads else 0.0
                varienc_v    = cnt_v - comp_cnt_v

                month_entry = {
                    "count":           cnt_v,
                    "conversion_rate": conv_v,
                }
                if comparison:
                    month_entry["comparison"] = {
                        "compare_period": comp_per_v,
                        "compare_count":  comp_cnt_v,
                        "varienc":        varienc_v,
                    }
                else:
                    month_entry["comparison"] = {}

                if show_period:
                    months[p] = month_entry

                tot_count      += cnt_v
                tot_comp_count += comp_cnt_v

                if p not in grand_period_map:
                    grand_period_map[p] = {
                        "count": 0, "compare_count": 0, "compare_period": comp_per_v
                    }
                grand_period_map[p]["count"]         += cnt_v
                grand_period_map[p]["compare_count"] += comp_cnt_v

        tot_conv = round((tot_count / total_leads) * 100, 2) if total_leads else 0.0

        record           = dict(dims)
        record["months"] = months
        record["total"]  = {
            "count":           tot_count,
            "conversion_rate": tot_conv,
        }
        record["comparison"] = {}
        if comparison:
            record["comparison"] = {
                "compare_count": tot_comp_count,
                "varienc":       tot_count - tot_comp_count,
            }

        records.append(record)

    ascending = sort_order == SortOrderOption.asc
    if sort_by.value in ("count", "conversion_rate"):
        records.sort(
            key=lambda x: x["total"].get(sort_by.value, 0),
            reverse=not ascending
        )
    elif sort_by.value != "period":
        records.sort(
            key=lambda x: str(x.get(sort_by.value) or ""),
            reverse=not ascending
        )

    grand_months  = {}
    gt_count      = 0
    gt_comp_count = 0

    for p in periods_available:
        if p in grand_period_map:
            gm      = grand_period_map[p]
            g_cnt   = gm["count"]
            g_comp  = gm["compare_count"]
            g_cper  = gm["compare_period"]
            p_leads = period_total_leads_map.get(p, 0)
            g_conv  = round((g_cnt / p_leads) * 100, 2) if p_leads else 0.0

            gm_entry = {"count": g_cnt, "conversion_rate": g_conv}
            if comparison:
                gm_entry["comparison"] = {
                    "compare_period": g_cper,
                    "compare_count":  g_comp,
                    "varienc":        g_cnt - g_comp,
                }
            else:
                gm_entry["comparison"] = {}

            if show_period:
                grand_months[p] = gm_entry

            gt_count      += g_cnt
            gt_comp_count += g_comp

    gt_total_leads_all = int(monthly_total_leads["month_total_leads"].sum()) if not monthly_total_leads.empty else 0
    gt_conversion_rate = round((gt_count / gt_total_leads_all) * 100, 2) if gt_total_leads_all != 0 else 0.0

    grand_total_record = {
        "months": grand_months,
        "total": {
            "count":           gt_count,
            "conversion_rate": gt_conversion_rate,
        },
        "comparison": {},
    }
    if comparison:
        grand_total_record["comparison"] = {
            "compare_count": gt_comp_count,
            "varienc":       gt_count - gt_comp_count,
        }

    return {
        "year":        year,
        "period":      period.value,
        "key_metrics": key_metrics,
        "records":     records,
        "grand_total": grand_total_record,
        "periods":     periods_available,
        "show_columns": {
            "period":       show_period,
            "product_name": show_product_name,
            "region":       show_region,
        },
        "regions":  sorted({r["region"]       for r in rows if r["region"]}),
        "products": sorted({r["product_name"] for r in rows if r["product_name"]}),
    }


###################################################################################
##################################################################################
######################################################################################

#Sales Funnel Metrics View more


@financial_router.get(
    "/Sales_Funnel_Metrics_Viewmore/{year}/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def Sales_Funnel_Metrics_Viewmore(
    currency: str,
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    year: int = Path(...),
    period: PeriodOption = Query(PeriodOption.all),
    comparison: Optional[ComparisonOption] = Query(None),
    db: AsyncSession = Depends(get_db)
):
    today = date.today()
    system_month = today.month

    if period == PeriodOption.all:
        start_date = date(year, 1, 1)
        end_date = date(year, 12, 31)

    elif period == PeriodOption.current_month:
        start_date = date(year, system_month, 1)
        end_date = date(year, system_month, monthrange(year, system_month)[1])

    elif period == PeriodOption.last_month:
        if system_month == 1:
            last_month = 12
            last_year = year - 1
        else:
            last_month = system_month - 1
            last_year = year

        start_date = date(last_year, last_month, 1)
        end_date = date(last_year, last_month, monthrange(last_year, last_month)[1])

    elif period == PeriodOption.quarter_to_date:
        start_date = date(year, 1, 1)
        end_date = date(year, 3, 31)

    elif period == PeriodOption.year_to_date:
        start_date = date(year, 1, 1)
        end_date = today

    else:
        start_date = date(year, 1, 1)
        end_date = date(year, 12, 31)

    params = {"start_date": start_date, "end_date": end_date}

    opp_query = text("""
        SELECT
            stage,
            DATE_TRUNC('month', created_date)::date AS month,
            COUNT(*) AS count
        FROM fpna_landing.crm_opportunities
        WHERE created_date BETWEEN :start_date AND :end_date
        GROUP BY stage, month
        ORDER BY stage, month
    """)

    result = await db.execute(opp_query, params)
    rows = result.mappings().all()

    if not rows:
        return {
            "year": year,
            "period": period.value,
            "key_metrics": {
                "total_leads": 0,
                "total_won": 0,
                "overall_conversion": 0.0,
                "average_sales_cycle_days": 0.0
            },
            "data": []
        }

    df = pd.DataFrame(rows)
    df["month"] = pd.to_datetime(df["month"])
    df["month_str"] = df["month"].dt.strftime("%Y-%m")

    total_leads_query = text("""
        SELECT COUNT(*) AS total_leads
        FROM fpna_landing.crm_opportunities
        WHERE created_date BETWEEN :start_date AND :end_date
    """)
    total_leads_result = await db.execute(total_leads_query, params)
    total_leads = int(total_leads_result.mappings().first()["total_leads"])

    total_won_query = text("""
        SELECT COUNT(*) AS total_won
        FROM fpna_landing.crm_opportunities
        WHERE stage = 'Closed Won'
          AND created_date BETWEEN :start_date AND :end_date
    """)
    total_won_result = await db.execute(total_won_query, params)
    total_won = int(total_won_result.mappings().first()["total_won"])

    overall_conversion = round((total_won / total_leads) * 100, 2) if total_leads else 0.0


    cycle_query = text("""
        SELECT created_date, close_date
        FROM fpna_landing.crm_opportunities
        WHERE stage = 'Closed Won'
          AND created_date BETWEEN :start_date AND :end_date
    """)
    cycle_result = await db.execute(cycle_query, params)
    cycle_rows = cycle_result.mappings().all()
    cycle_df = pd.DataFrame(cycle_rows)

    if not cycle_df.empty:
        cycle_df["created_date"] = pd.to_datetime(cycle_df["created_date"])
        cycle_df["close_date"] = pd.to_datetime(cycle_df["close_date"], errors="coerce")
        cycle_df = cycle_df.dropna(subset=["close_date"])
        avg_sales_cycle_days = round((cycle_df["close_date"] - cycle_df["created_date"]).dt.days.mean(), 2)
    else:
        avg_sales_cycle_days = 0.0

    keymetrics = {
        "total_leads": int(total_leads),
        "total_won": int(total_won),
        "overall_conversion": overall_conversion,
        "average_sales_cycle_days": avg_sales_cycle_days
    }

    pivot_df = df.pivot_table(
        index="stage",
        columns="month_str",
        values="count",
        aggfunc="sum",
        fill_value=0
    )

    all_months = pd.date_range(
        start=pd.Timestamp(start_date),
        end=pd.Timestamp(end_date),
        freq="MS"
    ).strftime("%Y-%m").tolist()

    for m in all_months:
        if m not in pivot_df.columns:
            pivot_df[m] = 0

    pivot_df = pivot_df[all_months]

    pivot_df["total"] = pivot_df.sum(axis=1)

    stage_order = ["Prospecting", "Qualification", "Proposal", "Negotiation", "Closed Won", "Closed Lost"]
    pivot_df = pivot_df.reindex(stage_order).fillna(0).astype(int)

    comparisons_by_stage = {}

    if comparison:
        min_month = pd.to_datetime(min(all_months), format="%Y-%m")
        max_month = pd.to_datetime(max(all_months), format="%Y-%m")

        if comparison == ComparisonOption.mom:
            min_compare = min_month - pd.DateOffset(months=1)
            max_compare = max_month - pd.DateOffset(months=1)

        elif comparison == ComparisonOption.yoy:
            min_compare = min_month - pd.DateOffset(years=1)
            max_compare = max_month - pd.DateOffset(years=1)

        else:
            min_compare = None
            max_compare = None

        compare_start_date = start_date
        compare_end_date = end_date

        if min_compare is not None and max_compare is not None:
            compare_start_date = date(int(min_compare.year), int(min_compare.month), 1)
            compare_end_date = date(
                int(max_compare.year),
                int(max_compare.month),
                monthrange(int(max_compare.year), int(max_compare.month))[1]
            )

        compare_params = {"start_date": compare_start_date, "end_date": compare_end_date}

        compare_query = text("""
            SELECT
                stage,
                DATE_TRUNC('month', created_date)::date AS month,
                COUNT(*) AS count
            FROM fpna_landing.crm_opportunities
            WHERE created_date BETWEEN :start_date AND :end_date
            GROUP BY stage, month
        """)

        compare_result = await db.execute(compare_query, compare_params)
        compare_rows = compare_result.mappings().all()

        compare_df = pd.DataFrame(compare_rows) if compare_rows else pd.DataFrame(columns=["stage", "month", "count"])

        if not compare_df.empty:
            compare_df["month"] = pd.to_datetime(compare_df["month"])
            compare_df["month_str"] = compare_df["month"].dt.strftime("%Y-%m")

        compare_lookup = {}
        if not compare_df.empty:
            for _, r in compare_df.iterrows():
                compare_lookup[(r["stage"], r["month_str"])] = int(r["count"])

        for stage in pivot_df.index.tolist():
            comparisons_by_stage[stage] = []

            for base_period in all_months:
                compare_period = salesfunnel_comparison_period(base_period, comparison)
                if not compare_period:
                    continue

                base_count = int(pivot_df.loc[stage, base_period]) if base_period in pivot_df.columns else 0
                compare_count = int(compare_lookup.get((stage, compare_period), 0))
                varienc = base_count - compare_count

                comparisons_by_stage[stage].append({
                    "base_period": base_period,
                    "compare_period": compare_period,
                    "base_count": base_count,
                    "compare_count": compare_count,
                    "varienc": varienc
                })

    data_list = pivot_df.reset_index().to_dict(orient="records")

    for row in data_list:
        stage = row.get("stage")
        if comparison:
            row["comparisons"] = comparisons_by_stage.get(stage, [])
        else:
            row["comparisons"] = []

    return {
        "year": year,
        "period": period.value,
        "key_metrics": keymetrics,
        "data": data_list
    }



    ##################################################
    ######################################################



@financial_router.get("/KPI/widgets")
async def KPI_get_widgets(db: AsyncSession = Depends(get_db)):
    try:
        return await KPIFinancialWidgetManagement.GetWidgets(db)
    except Exception as e:
        return {"detail": str(e)}


@financial_router.patch("/KPI/widgets")
async def KPI_update_widgets(widget: KPIWidgetUpdateSchema, db: AsyncSession = Depends(get_db)):
    try:
        return await KPIFinancialWidgetManagement.UpdateWidget(widget, db)
    except Exception as e:
        return {"detail": str(e)}


@financial_router.get("/Key_Financial_KPIs/{year}/{currency}")
async def Key_Financial_KPIs(
    request: Request,
    year: int = Path(...),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    filter: DashboardTimeFilter = Query(DashboardTimeFilter.monthly),
    db: AsyncSession = Depends(get_db),
):
    params = {"year": year, "filter": filter.value}

    kpi_query = text("""
    WITH revenue AS (
        SELECT
            CASE
                WHEN :filter = 'monthly' THEN gl.period
                WHEN :filter = 'quarterly' THEN
                    CONCAT(LEFT(gl.period,4), '-Q',
                        CEIL(CAST(RIGHT(gl.period,2) AS INT)/3.0)::int)
                WHEN :filter = 'year_to_date' THEN CONCAT(LEFT(gl.period,4), '-YTD')
            END AS time_key,
            SUM(COALESCE(gl.amount_credit,0) - COALESCE(gl.amount_debit,0)) AS revenue
        FROM fpna_landing.finance_general_ledger_transactions gl
        JOIN fpna_landing.finance_chart_of_accounts coa
          ON gl.gl_account_id = coa.account_id
        WHERE LEFT(gl.period,4)::int = CAST(:year AS INT)
          AND coa.account_type = 'Revenue'
        GROUP BY 1
    ),

    fte AS (
        SELECT
            CASE
                WHEN :filter = 'monthly' THEN period
                WHEN :filter = 'quarterly' THEN
                    CONCAT(LEFT(period,4), '-Q',
                        CEIL(CAST(RIGHT(period,2) AS INT)/3.0)::int)
                WHEN :filter = 'year_to_date' THEN CONCAT(LEFT(period,4), '-YTD')
            END AS time_key,
            AVG(COALESCE(fte_count,0)) AS fte
        FROM fpna_landing.hr_headcount_history
        WHERE LEFT(period,4)::int = CAST(:year AS INT)
        GROUP BY 1
    ),

    marketing AS (
        SELECT
            CASE
                WHEN :filter = 'monthly' THEN period
                WHEN :filter = 'quarterly' THEN
                    CONCAT(LEFT(period,4), '-Q',
                        CEIL(CAST(RIGHT(period,2) AS INT)/3.0)::int)
                WHEN :filter = 'year_to_date' THEN CONCAT(LEFT(period,4), '-YTD')
            END AS time_key,
            SUM(COALESCE(spend_amount,0)) AS spend,
            SUM(COALESCE(leads_generated,0)) AS leads
        FROM fpna_landing.crm_marketing_spend
        WHERE LEFT(period,4)::int = CAST(:year AS INT)
        GROUP BY 1
    ),

    all_keys AS (
        SELECT time_key FROM revenue
        UNION
        SELECT time_key FROM fte
        UNION
        SELECT time_key FROM marketing
    )

    SELECT
        k.time_key AS period,

        CASE
            WHEN COALESCE(f.fte,0) = 0 THEN 0
            ELSE COALESCE(r.revenue,0) / f.fte
        END AS revenue_per_employee,

        CASE
            WHEN COALESCE(m.spend,0) = 0 THEN 0
            ELSE ((COALESCE(r.revenue,0) - m.spend) / m.spend) * 100
        END AS marketing_roi,

        CASE
            WHEN COALESCE(m.leads,0) = 0 THEN 0
            ELSE COALESCE(m.spend,0) / COALESCE(m.leads,0)
        END AS cost_per_lead

    FROM all_keys k
    LEFT JOIN revenue r ON k.time_key = r.time_key
    LEFT JOIN fte f ON k.time_key = f.time_key
    LEFT JOIN marketing m ON k.time_key = m.time_key
    ORDER BY 1;
    """)

    profit_segment_query = text("""
    WITH segment_map AS (
        SELECT
            CAST(CAST(acm.customer_id AS NUMERIC) AS BIGINT) AS customer_id_clean,
            a.segment
        FROM fpna_landing.crm_account_customer_map acm
        JOIN fpna_landing.crm_accounts a
          ON acm.account_id = a.account_id
        WHERE acm.customer_id IS NOT NULL
    ),

    gl_base AS (
        SELECT
            CASE
                WHEN :filter = 'monthly' THEN gl.period
                WHEN :filter = 'quarterly' THEN
                    CONCAT(LEFT(gl.period,4), '-Q',
                        CEIL(CAST(RIGHT(gl.period,2) AS INT)/3.0)::int)
                WHEN :filter = 'year_to_date' THEN CONCAT(LEFT(gl.period,4), '-YTD')
            END AS time_key,
            COALESCE(sm.segment, 'Unknown') AS segment,
            coa.account_type,
            (COALESCE(gl.amount_credit,0) - COALESCE(gl.amount_debit,0)) AS net_amount
        FROM fpna_landing.finance_general_ledger_transactions gl
        JOIN fpna_landing.finance_chart_of_accounts coa
          ON gl.gl_account_id = coa.account_id
        LEFT JOIN segment_map sm
          ON CAST(CAST(gl.customer_id AS NUMERIC) AS BIGINT) = sm.customer_id_clean
        WHERE LEFT(gl.period,4)::int = CAST(:year AS INT)
          AND coa.account_type IN ('Revenue', 'Expense', 'COGS')
    ),

    segment_pnl AS (
        SELECT
            time_key AS period,
            segment,
            SUM(CASE WHEN account_type = 'Revenue' THEN net_amount ELSE 0 END) AS revenue,
            SUM(CASE WHEN account_type IN ('Expense','COGS') THEN ABS(net_amount) ELSE 0 END) AS cost
        FROM gl_base
        GROUP BY 1,2
    ),

    known_revenue_total AS (
        SELECT
            period,
            SUM(revenue) AS total_known_revenue
        FROM segment_pnl
        WHERE segment <> 'Unknown'
        GROUP BY 1
    ),

    unknown_cost_total AS (
        SELECT
            period,
            SUM(cost) AS total_unknown_cost
        FROM segment_pnl
        WHERE segment = 'Unknown'
        GROUP BY 1
    ),

    final_alloc AS (
        SELECT
            sp.period,
            sp.segment,
            sp.revenue,
            sp.cost AS direct_cost,
            CASE
                WHEN sp.segment = 'Unknown' THEN 0
                WHEN COALESCE(k.total_known_revenue,0) = 0 THEN 0
                ELSE COALESCE(u.total_unknown_cost,0) * (sp.revenue / k.total_known_revenue)
            END AS allocated_cost
        FROM segment_pnl sp
        LEFT JOIN known_revenue_total k ON sp.period = k.period
        LEFT JOIN unknown_cost_total u ON sp.period = u.period
    )

    SELECT
        period,
        segment,
        (revenue - (direct_cost + allocated_cost)) AS value
    FROM final_alloc
    WHERE segment <> 'Unknown'
    ORDER BY period, segment;
    """)

    kpi_rows = (await db.execute(kpi_query, params)).mappings().all()
    profit_rows = (await db.execute(profit_segment_query, params)).mappings().all()

    widgets = await KPIFinancialWidgetManagement.GetWidgets(db)
    widget_map = {w["name"]: w for w in widgets}

    metrics = []

    metrics.append({
        "metric": KPIWidgetNameEnum.REVENUM_PER_EMPLOYEE.value,
        "chart_Type": widget_map.get(KPIWidgetNameEnum.REVENUM_PER_EMPLOYEE.value, {}).get("chart_Type", ChartTypeEnum.LINE_CHART.value),
        "enableordisable": widget_map.get(KPIWidgetNameEnum.REVENUM_PER_EMPLOYEE.value, {}).get("enableordisable", True),
        "position": widget_map.get(KPIWidgetNameEnum.REVENUM_PER_EMPLOYEE.value, {}).get("position", 0),
        "trend": [{"period": r["period"], "value": float(f"{float(r['revenue_per_employee'] or 0):.2f}")} for r in kpi_rows]
    })

    metrics.append({
        "metric": KPIWidgetNameEnum.MARKETING_ROI.value,
        "chart_Type": widget_map.get(KPIWidgetNameEnum.MARKETING_ROI.value, {}).get("chart_Type", ChartTypeEnum.LINE_CHART.value),
        "enableordisable": widget_map.get(KPIWidgetNameEnum.MARKETING_ROI.value, {}).get("enableordisable", True),
        "position": widget_map.get(KPIWidgetNameEnum.MARKETING_ROI.value, {}).get("position", 1),
        "trend": [{"period": r["period"], "value": float(f"{float(r['marketing_roi'] or 0):.2f}")} for r in kpi_rows]
    })

    metrics.append({
        "metric": KPIWidgetNameEnum.COST_PER_LEAD.value,
        "chart_Type": widget_map.get(KPIWidgetNameEnum.COST_PER_LEAD.value, {}).get("chart_Type", ChartTypeEnum.LINE_CHART.value),
        "enableordisable": widget_map.get(KPIWidgetNameEnum.COST_PER_LEAD.value, {}).get("enableordisable", True),
        "position": widget_map.get(KPIWidgetNameEnum.COST_PER_LEAD.value, {}).get("position", 2),
        "trend": [{"period": r["period"], "value": float(f"{float(r['cost_per_lead'] or 0):.2f}")} for r in kpi_rows]
    })

    metrics.append({
        "metric": KPIWidgetNameEnum.PROFITABILITY_BY_SEGMENT.value,
        "chart_Type": widget_map.get(KPIWidgetNameEnum.PROFITABILITY_BY_SEGMENT.value, {}).get("chart_Type", ChartTypeEnum.BAR_CHART.value),
        "enableordisable": widget_map.get(KPIWidgetNameEnum.PROFITABILITY_BY_SEGMENT.value, {}).get("enableordisable", True),
        "position": widget_map.get(KPIWidgetNameEnum.PROFITABILITY_BY_SEGMENT.value, {}).get("position", 3),
        "trend": [{
            "period": r["period"],
            "segment": r["segment"],
            "value": float(f"{float(r['value'] or 0):.2f}")
        } for r in profit_rows]
    })

    metrics_enabled_only = [m for m in metrics if bool(m.get("enableordisable", True)) is True]
    metrics_sorted = sorted(metrics_enabled_only, key=lambda x: int(x.get("position", 0)))

    key_metrics = {}

    revenue_total_query = text("""
        WITH revenue_raw AS (
            SELECT
                SUM(COALESCE(gl.amount_credit,0) - COALESCE(gl.amount_debit,0)) AS revenue
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa
              ON gl.gl_account_id = coa.account_id
            WHERE LEFT(gl.period,4)::int = CAST(:year AS INT)
              AND coa.account_type = 'Revenue'
        ),
        fte_raw AS (
            SELECT
                AVG(COALESCE(fte_count,0)) AS fte
            FROM fpna_landing.hr_headcount_history
            WHERE LEFT(period,4)::int = CAST(:year AS INT)
        )
        SELECT
            CASE
                WHEN COALESCE(fte_raw.fte,0) = 0 THEN 0
                ELSE COALESCE(revenue_raw.revenue,0) / fte_raw.fte
            END AS value
        FROM revenue_raw, fte_raw;
    """)

    marketing_total_query = text("""
        WITH revenue_raw AS (
            SELECT
                SUM(COALESCE(gl.amount_credit,0) - COALESCE(gl.amount_debit,0)) AS revenue
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa
              ON gl.gl_account_id = coa.account_id
            WHERE LEFT(gl.period,4)::int = CAST(:year AS INT)
              AND coa.account_type = 'Revenue'
        ),
        marketing_raw AS (
            SELECT
                SUM(COALESCE(spend_amount,0)) AS spend
            FROM fpna_landing.crm_marketing_spend
            WHERE LEFT(period,4)::int = CAST(:year AS INT)
        )
        SELECT
            CASE
                WHEN COALESCE(marketing_raw.spend,0) = 0 THEN 0
                ELSE ((COALESCE(revenue_raw.revenue,0) - marketing_raw.spend) / marketing_raw.spend) * 100
            END AS value
        FROM revenue_raw, marketing_raw;
    """)

    cpl_total_query = text("""
        WITH marketing_raw AS (
            SELECT
                SUM(COALESCE(spend_amount,0)) AS spend,
                SUM(COALESCE(leads_generated,0)) AS leads
            FROM fpna_landing.crm_marketing_spend
            WHERE LEFT(period,4)::int = CAST(:year AS INT)
        )
        SELECT
            CASE
                WHEN COALESCE(marketing_raw.leads,0) = 0 THEN 0
                ELSE COALESCE(marketing_raw.spend,0) / COALESCE(marketing_raw.leads,0)
            END AS value
        FROM marketing_raw;
    """)

    rev_emp_val = (await db.execute(revenue_total_query, {"year": year})).mappings().first()
    roi_val = (await db.execute(marketing_total_query, {"year": year})).mappings().first()
    cpl_val = (await db.execute(cpl_total_query, {"year": year})).mappings().first()

    key_metrics[KPIWidgetNameEnum.REVENUM_PER_EMPLOYEE.value] = float(
        f"{float((rev_emp_val or {}).get('value', 0) or 0):.2f}"
    )
    key_metrics[KPIWidgetNameEnum.MARKETING_ROI.value] = float(
        f"{float((roi_val or {}).get('value', 0) or 0):.2f}"
    )
    key_metrics[KPIWidgetNameEnum.COST_PER_LEAD.value] = float(
        f"{float((cpl_val or {}).get('value', 0) or 0):.2f}"
    )

    seg_sum = 0.0
    for r in profit_rows:
        seg_sum += float(r["value"] or 0)

    key_metrics[KPIWidgetNameEnum.PROFITABILITY_BY_SEGMENT.value] = float(f"{seg_sum:.2f}")

    response = {
        "year": year,
        "currency": currency,
        "currency_filter": currency_filter.value,
        "filter": filter.value,
        "key_metrics": key_metrics,
        "metrics": metrics_sorted
    }

    key_metrics_wrapped = {}
    for k, v in (response.get("key_metrics") or {}).items():
        key_metrics_wrapped[k] = {"value": v}

    key_metrics_wrapped = apply_smart_currency(
        response=key_metrics_wrapped,
        currency=currency,
        currency_filter=currency_filter.value,
        money_fields=["value"]
    )

    key_metrics_unwrapped = {}
    for k, obj in (key_metrics_wrapped or {}).items():
        if k == KPIWidgetNameEnum.MARKETING_ROI.value:
            key_metrics_unwrapped[k] = response["key_metrics"].get(k, 0)
        else:
            key_metrics_unwrapped[k] = obj.get("value", 0)

    response["key_metrics"] = key_metrics_unwrapped

    response = apply_smart_currency(
        response=response,
        currency=currency,
        currency_filter=currency_filter.value,
        money_fields=["value"]
    )

    return response



#######################################################################################
####################################################################################
######################################################################################

#PNL Revenue Drilldown


REVENUE_LEVEL_CONFIG = [
    {
        "field": "customer_name",
        "key": "customer_name",
        "child_key": "by_customer_by_product",
        "value_key": "selected_revenue",
        "prev_key": "previous_revenue"
    },
    {
        "field": "product_name",
        "key": "product_name",
        "child_key": "by_customer_by_product_by_entity",
        "value_key": "selected_revenue",
        "prev_key": "previous_revenue"
    },
    {
        "field": "entity_name",
        "key": "entity_name",
        "child_key": "by_customer_by_product_by_entity_by_industry",
        "value_key": "selected_revenue",
        "prev_key": "previous_revenue"
    },
    {
        "field": "industry",
        "key": "industry",
        "child_key": None,
        "value_key": "selected_revenue",
        "prev_key": "previous_revenue"
    }
]


def _get_previous_period(period: str) -> str:
    try:
        dt = datetime.strptime(period, "%Y-%m")
    except Exception:
        raise HTTPException(status_code=400, detail="period must be in YYYY-MM format")
    if dt.month == 1:
        return f"{dt.year - 1}-12"
    return f"{dt.year}-{dt.month - 1:02d}"


def _round_pct_fields(obj: Any, decimals: int = 2) -> Any:
    if isinstance(obj, dict):
        new_obj = {}
        for k, v in obj.items():
            if k in ("pct", "change_pct") and isinstance(v, (int, float)) and v is not None:
                new_obj[k] = round(float(v), decimals)
            else:
                new_obj[k] = _round_pct_fields(v, decimals)
        return new_obj
    if isinstance(obj, list):
        return [_round_pct_fields(x, decimals) for x in obj]
    return obj


def _build_nested_breakdown(
    df_current: pd.DataFrame,
    df_previous: pd.DataFrame,
    level_configs: list
) -> List[Dict[str, Any]]:

    def _build_level(df_curr, df_prev, configs, depth=0):
        if not configs:
            return None

        config = configs[0]
        remaining_configs = configs[1:]

        field = config["field"]
        label_key = config["key"]
        child_key = config["child_key"]
        value_key = config["value_key"]
        prev_key = config["prev_key"]

        curr_agg = df_curr.groupby(field)["amount"].sum()
        prev_agg = df_prev.groupby(field)["amount"].sum()

        all_keys = set(curr_agg.index) | set(prev_agg.index)

        result = []
        for key in all_keys:
            current_val = round(float(curr_agg.get(key, 0)), 2)
            previous_val = round(float(prev_agg.get(key, 0)), 2)
            change = round(current_val - previous_val, 2)
            change_pct = round((change / previous_val * 100), 2) if previous_val else None

            trend = "stable"
            if change_pct is not None:
                if change_pct > 5:
                    trend = "up"
                elif change_pct < -5:
                    trend = "down"

            node = {
                label_key: str(key),
                value_key: current_val,
                prev_key: previous_val,
                "change": change,
                "change_pct": change_pct,
                "trend": trend,
            }

            if child_key and remaining_configs:
                child_curr = df_curr[df_curr[field] == key]
                child_prev = df_prev[df_prev[field] == key]
                node[child_key] = _build_level(child_curr, child_prev, remaining_configs, depth + 1)
            else:
                node["children"] = None

            result.append(node)

        result.sort(key=lambda x: x[value_key], reverse=True)
        return result

    return _build_level(df_current, df_previous, level_configs)


def _calculate_revenue_waterfall(
    df_current: pd.DataFrame,
    df_previous: pd.DataFrame,
    current_period: str,
    previous_period: str
) -> List[Dict[str, Any]]:

    current_by_customer = df_current.groupby("customer_name")["amount"].sum()
    previous_by_customer = df_previous.groupby("customer_name")["amount"].sum()

    new_customers = current_by_customer[~current_by_customer.index.isin(previous_by_customer.index)]
    churned_customers = previous_by_customer[~previous_by_customer.index.isin(current_by_customer.index)]
    existing_curr = current_by_customer[current_by_customer.index.isin(previous_by_customer.index)]
    existing_prev = previous_by_customer[previous_by_customer.index.isin(current_by_customer.index)]

    new_customer_revenue = round(float(new_customers.sum() or 0), 2)
    churned_revenue = round(float(churned_customers.sum() or 0), 2)
    existing_changes = existing_curr - existing_prev
    upsells = round(float(existing_changes[existing_changes > 0].sum() or 0), 2)
    downsells = round(float(existing_changes[existing_changes < 0].sum() or 0), 2)

    previous_total = round(float(df_previous["amount"].sum() or 0), 2)
    current_total = round(float(df_current["amount"].sum() or 0), 2)

    waterfall = [
        {
            "label": f"{previous_period} Revenue",
            "value": previous_total,
            "type": "baseline",
            "key": "baseline_previous",
            "description": "Starting point"
        }
    ]

    if new_customer_revenue > 0:
        waterfall.append({
            "label": "New Customers",
            "value": new_customer_revenue,
            "type": "positive",
            "key": "new_customers",
            "description": f"{len(new_customers)} new customers added"
        })

    if upsells > 0:
        waterfall.append({
            "label": "Existing Customer Growth",
            "value": upsells,
            "type": "positive",
            "key": "existing_growth",
            "description": "Expansion in existing accounts"
        })

    if downsells < 0:
        waterfall.append({
            "label": "Existing Customer Decline",
            "value": downsells,
            "type": "negative",
            "key": "existing_decline",
            "description": "Contraction in existing accounts"
        })

    if churned_revenue > 0:
        waterfall.append({
            "label": "Customer Churn",
            "value": -churned_revenue,
            "type": "negative",
            "key": "churn",
            "description": f"{len(churned_customers)} customers lost"
        })

    calculated_total = previous_total + new_customer_revenue + upsells + downsells - churned_revenue
    residual = round(current_total - calculated_total, 2)
    if abs(residual) > 0.01:
        waterfall.append({
            "label": "Other Adjustments",
            "value": residual,
            "type": "positive" if residual > 0 else "negative",
            "key": "other_adjustments",
            "description": "Rounding and other factors"
        })

    waterfall.append({
        "label": f"{current_period} Revenue",
        "value": current_total,
        "type": "baseline",
        "key": "baseline_current",
        "description": "Ending point"
    })

    return waterfall


def _calculate_waterfall_detail(
    df_current: pd.DataFrame,
    df_previous: pd.DataFrame
) -> Dict[str, Any]:

    current_by_customer = df_current.groupby("customer_name")["amount"].sum()
    previous_by_customer = df_previous.groupby("customer_name")["amount"].sum()

    new_customers_series = current_by_customer[~current_by_customer.index.isin(previous_by_customer.index)]
    churned_customers_series = previous_by_customer[~previous_by_customer.index.isin(current_by_customer.index)]
    existing_curr = current_by_customer[current_by_customer.index.isin(previous_by_customer.index)]
    existing_prev = previous_by_customer[previous_by_customer.index.isin(current_by_customer.index)]
    existing_changes = existing_curr - existing_prev

    new_customers_detail = [
        {"customer_name": str(cname), "revenue": round(float(rev), 2)}
        for cname, rev in new_customers_series.sort_values(ascending=False).items()
    ]

    existing_growth_detail = [
        {"customer_name": str(cname), "revenue": round(float(chg), 2)}
        for cname, chg in existing_changes[existing_changes > 0].sort_values(ascending=False).items()
    ]

    existing_decline_detail = [
        {"customer_name": str(cname), "revenue": round(float(chg), 2)}
        for cname, chg in existing_changes[existing_changes < 0].sort_values(ascending=True).items()
    ]

    churn_detail = [
        {"customer_name": str(cname), "revenue": round(float(-rev), 2)}
        for cname, rev in churned_customers_series.sort_values(ascending=False).items()
    ]

    return {
        "new_customers": new_customers_detail,
        "existing_growth": existing_growth_detail,
        "existing_decline": existing_decline_detail,
        "churn": churn_detail
    }


def _calculate_top_impact(
    df_current: pd.DataFrame,
    df_previous: pd.DataFrame,
    dimension: str
) -> Dict[str, Any]:

    current_by_customer = df_current.groupby(dimension)["amount"].sum().reset_index()
    previous_by_customer = df_previous.groupby(dimension)["amount"].sum().reset_index()

    current_by_customer.columns = [dimension, "current_revenue"]
    previous_by_customer.columns = [dimension, "previous_revenue"]

    merged = pd.merge(
        current_by_customer,
        previous_by_customer,
        on=dimension,
        how="outer"
    ).fillna(0)

    merged["impact"] = merged["current_revenue"] - merged["previous_revenue"]
    merged["change_pct"] = (merged["impact"] / merged["previous_revenue"] * 100).replace([np.inf, -np.inf], None)

    increases = merged[merged["impact"] > 0].sort_values("impact", ascending=False)
    decreases = merged[merged["impact"] < 0].sort_values("impact", ascending=True)

    def _build_rows(subset):
        result = []
        for _, row in subset.iterrows():
            result.append({
                "customer_name": str(row[dimension]) if pd.notna(row[dimension]) else "Unknown",
                "current": round(float(row["current_revenue"]), 2),
                "previous": round(float(row["previous_revenue"]), 2),
                "change": round(float(row["impact"]), 2),
                "change_pct": round(float(row["change_pct"]), 2) if pd.notna(row["change_pct"]) else None,
                "impact": round(float(row["impact"]), 2)
            })
        return result

    return {
        "increases": _build_rows(increases),
        "decreases": _build_rows(decreases)
    }


def _generate_revenue_insights(
    current_total: float,
    previous_total: float,
    variance: float,
    variance_pct: float,
    waterfall: List[Dict],
    breakdowns: List[Dict],
    top_impact: Dict[str, Any],
    drivers_detail: Dict[str, Any]
) -> Dict[str, Any]:

    direction = "increased" if variance > 0 else "decreased"
    main_summary = (
        f"Revenue {direction} by {round(abs(variance), 2)} "
        f"({round(abs(variance_pct), 1)}%) from {round(previous_total, 2)} "
        f"to {round(current_total, 2)}."
    )

    drivers = []
    for item in waterfall:
        if item["type"] in ("positive", "negative") and abs(item["value"]) > abs(variance) * 0.1:
            drivers.append({
                "type": item["type"],
                "key": item["key"],
                "description": item["label"],
                "amount": round(float(item["value"]), 2),
                "detail": item.get("description", "")
            })

    recommendations = []
    if breakdowns:
        best = breakdowns[0]
        cp = best.get("change_pct")
        if cp and cp > 0:
            recommendations.append(f"Focus on {best['customer_name']} segment where revenue grew {cp:.1f}% MoM")

    declining = [p for p in breakdowns if p.get("change_pct") and p["change_pct"] < -5]
    if declining:
        worst = declining[0]
        recommendations.append(f"Investigate {worst['customer_name']} - revenue declined {abs(worst['change_pct']):.1f}%")

    churn = [w for w in waterfall if "churn" in w["label"].lower()]
    if churn and abs(churn[0]["value"]) > abs(variance) * 0.15:
        recommendations.append(f"Address customer churn - lost {round(abs(churn[0]['value']), 2)} in revenue")

    new_cust = [w for w in waterfall if "new customer" in w["label"].lower()]
    if new_cust and new_cust[0]["value"] > abs(variance) * 0.15:
        recommendations.append(f"Capitalize on successful customer acquisition - {new_cust[0].get('description', 'multiple new customers added')}")

    if len(recommendations) < 3:
        recommendations.append("Monitor revenue trends closely for early warning signs")
        recommendations.append("Review pricing strategy to optimize revenue per customer")
        recommendations.append("Enhance customer retention programs to reduce churn")

    all_impacts = top_impact.get("increases", []) + top_impact.get("decreases", [])
    biggest_mover = all_impacts[0]["customer_name"] if all_impacts else None

    return {
        "summary": main_summary,
        "drivers": drivers,
        "drivers_detail": drivers_detail,
        "recommendations": recommendations[:5],
        "key_findings": {
            "top_product": breakdowns[0]["customer_name"] if breakdowns else None,
            "top_region": None,
            "biggest_mover": biggest_mover,
            "trend": "positive" if variance_pct > 0 else "negative"
        }
    }


@financial_router.get(
    "/pnl-revenue-drilldown/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def pnl_revenue_drilldown(
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    period: str = Query(..., description="Format: YYYY-MM"),
    entity_name: Optional[str] = Query(None),
    product_name: Optional[str] = Query(None),
    region: Optional[str] = Query(None),
    category: Optional[str] = Query(None),
    industry: Optional[str] = Query(None),
    db: AsyncSession = Depends(get_db)
):
    current_period = period
    previous_period = _get_previous_period(period)

    filters = []
    params = {"current_period": current_period, "previous_period": previous_period}

    if entity_name:
        filters.append("e.entity_name = :entity_name")
        params["entity_name"] = entity_name
    if product_name:
        filters.append("p.product_name = :product_name")
        params["product_name"] = product_name
    if region:
        filters.append("c.region = :region")
        params["region"] = region
    if category:
        filters.append("d.department_name = :category")
        params["category"] = category
    if industry:
        filters.append("c.industry = :industry")
        params["industry"] = industry

    filter_sql = ("AND " + " AND ".join(filters)) if filters else ""

    sql = text(f"""
    WITH customers_dedup AS (
        SELECT DISTINCT ON (customer_id)
            customer_id,
            customer_name,
            region,
            industry
        FROM fpna_landing.finance_customers
        WHERE customer_id IS NOT NULL
        ORDER BY customer_id
    ),
    entities_dedup AS (
        SELECT DISTINCT ON (entity_id)
            entity_id,
            entity_name
        FROM fpna_landing.finance_entities
        WHERE entity_id IS NOT NULL
        ORDER BY entity_id
    ),
    departments_dedup AS (
        SELECT DISTINCT ON (department_id)
            department_id,
            department_name
        FROM fpna_landing.finance_departments
        WHERE department_id IS NOT NULL
        ORDER BY department_id
    ),
    products_dedup AS (
        SELECT DISTINCT ON (profit_center_id)
            profit_center_id,
            product_name
        FROM fpna_landing.crm_products
        WHERE profit_center_id IS NOT NULL
        ORDER BY profit_center_id
    ),
    period_data AS (
        SELECT
            gl.period,
            COALESCE(c.customer_name, 'Unknown') AS customer_name,
            COALESCE(p.product_name, 'Unknown') AS product_name,
            COALESCE(e.entity_name, 'Unknown') AS entity_name,
            COALESCE(c.region, 'Unknown') AS region,
            COALESCE(d.department_name, 'Unknown') AS category,
            COALESCE(c.industry, 'Unknown') AS industry,
            SUM(gl.amount_credit - gl.amount_debit) AS amount
        FROM fpna_landing.finance_general_ledger_transactions gl
        JOIN fpna_landing.finance_chart_of_accounts coa
            ON gl.gl_account_id = coa.account_id
        LEFT JOIN customers_dedup c
            ON CAST(NULLIF(REGEXP_REPLACE(TRIM(gl.customer_id::text), '\\.0+$', ''), '') AS BIGINT)
             = CAST(NULLIF(REGEXP_REPLACE(TRIM(c.customer_id::text), '\\.0+$', ''), '') AS BIGINT)
        LEFT JOIN entities_dedup e
            ON CAST(NULLIF(REGEXP_REPLACE(TRIM(gl.entity_id::text), '\\.0+$', ''), '') AS BIGINT)
             = CAST(NULLIF(REGEXP_REPLACE(TRIM(e.entity_id::text), '\\.0+$', ''), '') AS BIGINT)
        LEFT JOIN departments_dedup d
            ON CAST(NULLIF(REGEXP_REPLACE(TRIM(gl.department_id::text), '\\.0+$', ''), '') AS BIGINT)
             = CAST(NULLIF(REGEXP_REPLACE(TRIM(d.department_id::text), '\\.0+$', ''), '') AS BIGINT)
        LEFT JOIN products_dedup p
            ON CAST(NULLIF(REGEXP_REPLACE(TRIM(gl.profit_center_id::text), '\\.0+$', ''), '') AS BIGINT)
             = CAST(NULLIF(REGEXP_REPLACE(TRIM(p.profit_center_id::text), '\\.0+$', ''), '') AS BIGINT)
        WHERE coa.account_type = 'Revenue'
          AND gl.period IN (:current_period, :previous_period)
          {filter_sql}
        GROUP BY
            gl.period,
            c.customer_name,
            p.product_name,
            e.entity_name,
            c.region,
            d.department_name,
            c.industry
    )
    SELECT * FROM period_data
    ORDER BY period, amount DESC
    """)

    result = await db.execute(sql, params)
    rows = result.mappings().all()
    df = pd.DataFrame(rows)

    if df.empty:
        return _empty_drilldown_response("revenue", current_period, previous_period)

    df_current = df[df["period"] == current_period].copy()
    df_previous = df[df["period"] == previous_period].copy()

    current_total = round(float(df_current["amount"].sum() or 0), 2)
    previous_total = round(float(df_previous["amount"].sum() or 0), 2)
    variance = round(current_total - previous_total, 2)
    variance_pct = round((variance / previous_total * 100), 2) if previous_total else 0

    waterfall = _calculate_revenue_waterfall(df_current, df_previous, current_period, previous_period)
    waterfall_detail = _calculate_waterfall_detail(df_current, df_previous)
    breakdowns = _build_nested_breakdown(df_current, df_previous, level_configs=REVENUE_LEVEL_CONFIG)
    top_impact = _calculate_top_impact(df_current, df_previous, "customer_name")

    ai_insights = _generate_revenue_insights(
        current_total,
        previous_total,
        variance,
        variance_pct,
        waterfall,
        breakdowns,
        top_impact,
        waterfall_detail
    )

    response = {
        "metric": "revenue",
        "title": "Revenue Analysis",
        "subtitle": f"Why revenue {'increased' if variance > 0 else 'decreased'} ({variance_pct:+.1f}% MoM)",
        "summary": {
            "current_period": current_period,
            "current_value": current_total,
            "previous_period": previous_period,
            "previous_value": previous_total,
            "variance": variance,
            "pct": variance_pct
        },
        "insights": ai_insights,
        "waterfall": waterfall,
        "breakdowns": breakdowns,
        "top_impact": top_impact
    }

    response = apply_smart_currency(
        response,
        currency,
        currency_filter,
        ["current_value", "previous_value", "variance", "value", "amount",
         "impact", "current", "previous", "change", "selected_revenue",
         "previous_revenue", "revenue"]
    )

    response = _round_pct_fields(response, decimals=2)

    return response


#######################################################################################
####################################################################################
######################################################################################

#PNL Cost_of_good_sold Drilldown

EXPENSE_LEVEL_CONFIG = [
    {
        "field": "account_name",
        "key": "account_name",
        "child_key": "by_account_by_entity",
        "value_key": "selected_amount",
        "prev_key": "previous_amount"
    },
    {
        "field": "entity_name",
        "key": "entity_name",
        "child_key": "by_account_by_entity_by_region",
        "value_key": "selected_amount",
        "prev_key": "previous_amount"
    },
    {
        "field": "region",
        "key": "region",
        "child_key": "by_account_by_entity_by_region_by_department",
        "value_key": "selected_amount",
        "prev_key": "previous_amount"
    },
    {
        "field": "category",
        "key": "category",
        "child_key": None,
        "value_key": "selected_amount",
        "prev_key": "previous_amount"
    }
]


def _get_previous_period(period: str) -> str:
    try:
        dt = datetime.strptime(period, "%Y-%m")
    except Exception:
        raise HTTPException(status_code=400, detail="period must be in YYYY-MM format")
    if dt.month == 1:
        return f"{dt.year - 1}-12"
    return f"{dt.year}-{dt.month - 1:02d}"


def _round_pct_fields(obj: Any, decimals: int = 2) -> Any:
    if isinstance(obj, dict):
        new_obj = {}
        for k, v in obj.items():
            if k in ("pct", "change_pct") and isinstance(v, (int, float)) and v is not None:
                new_obj[k] = round(float(v), decimals)
            else:
                new_obj[k] = _round_pct_fields(v, decimals)
        return new_obj
    if isinstance(obj, list):
        return [_round_pct_fields(x, decimals) for x in obj]
    return obj


def _build_nested_breakdown(
    df_current: pd.DataFrame,
    df_previous: pd.DataFrame,
    level_configs: list
) -> List[Dict[str, Any]]:

    def _build_level(df_curr, df_prev, configs, depth=0):
        if not configs:
            return None

        config = configs[0]
        remaining_configs = configs[1:]

        field = config["field"]
        label_key = config["key"]
        child_key = config["child_key"]
        value_key = config["value_key"]
        prev_key = config["prev_key"]

        curr_agg = df_curr.groupby(field)["amount"].sum()
        prev_agg = df_prev.groupby(field)["amount"].sum()

        all_keys = set(curr_agg.index) | set(prev_agg.index)

        result = []
        for key in all_keys:
            current_val = round(float(curr_agg.get(key, 0)), 2)
            previous_val = round(float(prev_agg.get(key, 0)), 2)
            change = round(current_val - previous_val, 2)
            change_pct = round((change / previous_val * 100), 2) if previous_val else None

            trend = "stable"
            if change_pct is not None:
                if change_pct > 5:
                    trend = "up"
                elif change_pct < -5:
                    trend = "down"

            node = {
                label_key: str(key),
                value_key: current_val,
                prev_key: previous_val,
                "change": change,
                "change_pct": change_pct,
                "trend": trend,
            }

            if child_key and remaining_configs:
                child_curr = df_curr[df_curr[field] == key]
                child_prev = df_prev[df_prev[field] == key]
                node[child_key] = _build_level(child_curr, child_prev, remaining_configs, depth + 1)
            else:
                node["children"] = None

            result.append(node)

        result.sort(key=lambda x: abs(x["change"]), reverse=True)
        return result

    return _build_level(df_current, df_previous, level_configs)


def _calculate_expense_waterfall(
    df_current: pd.DataFrame,
    df_previous: pd.DataFrame,
    current_period: str,
    previous_period: str,
    dimension: str
) -> List[Dict[str, Any]]:

    current_by_dim = df_current.groupby(dimension)["amount"].sum()
    previous_by_dim = df_previous.groupby(dimension)["amount"].sum()

    previous_total = round(float(df_previous["amount"].sum() or 0), 2)
    current_total = round(float(df_current["amount"].sum() or 0), 2)

    waterfall = [
        {"label": f"{previous_period} Total", "value": previous_total, "type": "baseline", "description": "Starting point"}
    ]

    all_dimensions = set(current_by_dim.index) | set(previous_by_dim.index)
    changes = []
    for name in all_dimensions:
        change = float(current_by_dim.get(name, 0)) - float(previous_by_dim.get(name, 0))
        if abs(change) > 0.01:
            changes.append({"name": str(name) if name else "Unknown", "change": round(change, 2)})

    changes.sort(key=lambda x: abs(x["change"]), reverse=True)

    for item in changes[:5]:
        if item["change"] > 0:
            waterfall.append({"label": f"{item['name']} Increase", "value": item["change"], "type": "negative", "description": f"Higher spending in {item['name']}"})
        else:
            waterfall.append({"label": f"{item['name']} Decrease", "value": item["change"], "type": "positive", "description": f"Cost savings in {item['name']}"})

    calculated_total = previous_total + sum(item["change"] for item in changes)
    residual = round(current_total - calculated_total, 2)
    if abs(residual) > 0.01:
        waterfall.append({"label": "Other Changes", "value": residual, "type": "positive" if residual < 0 else "negative", "description": "Other expense adjustments"})

    waterfall.append({"label": f"{current_period} Total", "value": current_total, "type": "baseline", "description": "Ending point"})
    return waterfall


def _calculate_top_impact_expense(
    df_current: pd.DataFrame,
    df_previous: pd.DataFrame,
    dimension: str,
    limit: int = 10
) -> List[Dict[str, Any]]:

    group_cols = [dimension, "entity_name", "region"]
    current_by_dim = df_current.groupby(group_cols)["amount"].sum().reset_index()
    previous_by_dim = df_previous.groupby(group_cols)["amount"].sum().reset_index()

    merged = pd.merge(
        current_by_dim, previous_by_dim,
        on=group_cols, how="outer", suffixes=("_current", "_previous")
    ).fillna(0)

    merged["impact"] = merged["amount_current"] - merged["amount_previous"]
    merged["change_pct"] = (merged["impact"] / merged["amount_previous"] * 100).replace([np.inf, -np.inf], None)
    merged["abs_impact"] = merged["impact"].abs()
    merged = merged.sort_values("abs_impact", ascending=False).head(limit)

    result = []
    for _, row in merged.iterrows():
        result.append({
            "account": str(row[dimension]) if pd.notna(row[dimension]) else "Unknown",
            "entity": str(row["entity_name"]) if pd.notna(row["entity_name"]) else "Unknown",
            "region": str(row["region"]) if pd.notna(row["region"]) else "Unknown",
            "current": round(float(row["amount_current"]), 2),
            "previous": round(float(row["amount_previous"]), 2),
            "change": round(float(row["impact"]), 2),
            "change_pct": round(float(row["change_pct"]), 2) if pd.notna(row["change_pct"]) else None,
            "impact": round(float(row["impact"]), 2)
        })
    return result


def _generate_expense_insights(
    expense_type: str,
    current_total: float,
    previous_total: float,
    variance: float,
    variance_pct: float,
    waterfall: List[Dict],
    breakdowns: List[Dict],
    top_impact: List[Dict]
) -> Dict[str, Any]:

    direction = "increased" if variance > 0 else "decreased"
    main_summary = (
        f"{expense_type} {direction} by {round(abs(variance), 2)} "
        f"({round(abs(variance_pct), 1)}%) from {round(previous_total, 2)} "
        f"to {round(current_total, 2)}."
    )

    drivers = []
    for item in waterfall:
        if item["type"] in ("positive", "negative") and abs(item["value"]) > abs(variance) * 0.1:
            drivers.append({
                "type": item["type"],
                "description": item["label"],
                "amount": round(float(item["value"]), 2),
                "detail": item.get("description", "")
            })

    recommendations = []
    increasing = [a for a in breakdowns if a.get("change_pct") and a["change_pct"] > 10]
    if increasing:
        w = increasing[0]
        recommendations.append(f"Review {w['account_name']} expenses - increased {w['change_pct']:.1f}% MoM")

    decreasing = [a for a in breakdowns if a.get("change_pct") and a["change_pct"] < -5]
    if decreasing:
        b = decreasing[0]
        recommendations.append(f"Replicate cost savings approach from {b['account_name']} ({abs(b['change_pct']):.1f}% reduction)")

    if variance > 0:
        recommendations.append("Implement cost control measures to manage expense growth")
        recommendations.append("Review vendor contracts for potential renegotiation opportunities")
    else:
        recommendations.append("Continue current cost optimization strategies")
        recommendations.append("Document successful cost reduction initiatives for replication")

    if len(recommendations) < 3:
        recommendations.append("Conduct quarterly expense reviews by department")
        recommendations.append("Establish expense benchmarks against industry standards")
        recommendations.append("Implement automated expense tracking and alerts")

    return {
        "summary": main_summary,
        "drivers": drivers,
        "recommendations": recommendations[:5],
        "key_findings": {
            "top_account": breakdowns[0]["account_name"] if breakdowns else None,
            "top_region": None,
            "biggest_impact": top_impact[0]["account"] if top_impact else None,
            "trend": "increasing" if variance_pct > 0 else "decreasing"
        }
    }


def _empty_drilldown_response(metric: str, current_period: str, previous_period: str) -> Dict[str, Any]:
    titles = {
        "revenue": "Revenue Analysis",
        "cost_of_goods_sold": "Cost of Goods Sold Analysis",
        "operating_expense": "Operating Expense Analysis"
    }
    return {
        "metric": metric,
        "title": titles.get(metric, f"{metric.upper()} Analysis"),
        "subtitle": "No data available for selected period",
        "summary": {
            "current_period": current_period,
            "current_value": 0,
            "previous_period": previous_period,
            "previous_value": 0,
            "variance": 0,
            "pct": 0
        },
        "insights": {
            "summary": "No data available to generate insights.",
            "drivers": [],
            "recommendations": [],
            "key_findings": {
                "top_product": None,
                "top_region": None,
                "biggest_mover": None,
                "trend": "neutral"
            }
        },
        "waterfall": [],
        "breakdowns": [],
        "top_impact": []
    }


@financial_router.get(
    "/pnl-cost-of-good-sold-drilldown/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def pnl_cost_of_good_sold_drilldown(
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    period: str = Query(..., description="Format: YYYY-MM"),
    entity_name: Optional[str] = Query(None),
    product_name: Optional[str] = Query(None),
    region: Optional[str] = Query(None),
    category: Optional[str] = Query(None),
    account_name: Optional[str] = Query(None),
    db: AsyncSession = Depends(get_db)
):
    current_period = period
    previous_period = _get_previous_period(period)

    filters = []
    params = {"current_period": current_period, "previous_period": previous_period}

    if entity_name:
        filters.append("e.entity_name = :entity_name")
        params["entity_name"] = entity_name
    if product_name:
        filters.append("p.product_name = :product_name")
        params["product_name"] = product_name
    if region:
        filters.append("c.region = :region")
        params["region"] = region
    if category:
        filters.append("d.department_name = :category")
        params["category"] = category
    if account_name:
        filters.append("coa.account_name = :account_name")
        params["account_name"] = account_name

    filter_sql = ("AND " + " AND ".join(filters)) if filters else ""

    sql = text(f"""
    WITH customers_dedup AS (
        SELECT DISTINCT ON (customer_id)
            customer_id,
            customer_name,
            region,
            industry
        FROM fpna_landing.finance_customers
        WHERE customer_id IS NOT NULL
        ORDER BY customer_id
    ),
    entities_dedup AS (
        SELECT DISTINCT ON (entity_id)
            entity_id,
            entity_name,
            region
        FROM fpna_landing.finance_entities
        WHERE entity_id IS NOT NULL
        ORDER BY entity_id
    ),
    departments_dedup AS (
        SELECT DISTINCT ON (department_id)
            department_id,
            department_name
        FROM fpna_landing.finance_departments
        WHERE department_id IS NOT NULL
        ORDER BY department_id
    ),
    products_dedup AS (
        SELECT DISTINCT ON (profit_center_id)
            profit_center_id,
            product_name
        FROM fpna_landing.crm_products
        WHERE profit_center_id IS NOT NULL
        ORDER BY profit_center_id
    ),
    period_data AS (
        SELECT
            gl.period,
            COALESCE(coa.account_name, 'Unknown') AS account_name,
            COALESCE(p.product_name, 'Unknown') AS product_name,
            COALESCE(e.entity_name, 'Unknown') AS entity_name,
            COALESCE(NULLIF(c.region, ''), NULLIF(e.region, ''), 'Unknown') AS region,
            COALESCE(d.department_name, 'Unknown') AS category,
            SUM(COALESCE(gl.amount_debit, 0) - COALESCE(gl.amount_credit, 0)) AS amount
        FROM fpna_landing.finance_general_ledger_transactions gl
        JOIN fpna_landing.finance_chart_of_accounts coa
            ON gl.gl_account_id = coa.account_id
        LEFT JOIN customers_dedup c
            ON NULLIF(TRIM(gl.customer_id::text), '') = NULLIF(TRIM(c.customer_id::text), '')
        LEFT JOIN entities_dedup e
            ON NULLIF(TRIM(gl.entity_id::text), '') = NULLIF(TRIM(e.entity_id::text), '')
        LEFT JOIN departments_dedup d
            ON NULLIF(TRIM(gl.department_id::text), '') = NULLIF(TRIM(d.department_id::text), '')
        LEFT JOIN products_dedup p
            ON NULLIF(TRIM(gl.profit_center_id::text), '') = NULLIF(TRIM(p.profit_center_id::text), '')
        WHERE coa.account_type = 'Expense'
          AND coa.account_name IN ('Inventory', 'Professional Services', 'IT & Software', 'Depreciation Expense')
          AND gl.period IN (:current_period, :previous_period)
          {filter_sql}
        GROUP BY
            gl.period,
            coa.account_name,
            p.product_name,
            e.entity_name,
            c.region,
            e.region,
            d.department_name
    )
    SELECT * FROM period_data
    ORDER BY period, amount DESC
    """)

    result = await db.execute(sql, params)
    rows = result.mappings().all()
    df = pd.DataFrame(rows)

    if df.empty:
        return _empty_drilldown_response("cost_of_goods_sold", current_period, previous_period)

    df_current = df[df["period"] == current_period].copy()
    df_previous = df[df["period"] == previous_period].copy()

    current_total = round(float(df_current["amount"].sum() or 0), 2)
    previous_total = round(float(df_previous["amount"].sum() or 0), 2)
    variance = round(current_total - previous_total, 2)
    variance_pct = round((variance / previous_total * 100), 2) if previous_total else 0

    waterfall = _calculate_expense_waterfall(df_current, df_previous, current_period, previous_period, "account_name")
    breakdowns = _build_nested_breakdown(df_current, df_previous, level_configs=EXPENSE_LEVEL_CONFIG)
    top_impact = _calculate_top_impact_expense(df_current, df_previous, "account_name")
    ai_insights = _generate_expense_insights("cost_of_goods_sold", current_total, previous_total, variance, variance_pct, waterfall, breakdowns, top_impact)

    response = {
        "metric": "cost_of_goods_sold",
        "title": "Cost of Goods Sold Analysis",
        "subtitle": f"Why cost_of_goods_sold {'increased' if variance > 0 else 'decreased'} ({variance_pct:+.1f}% MoM)",
        "summary": {
            "current_period": current_period,
            "current_value": current_total,
            "previous_period": previous_period,
            "previous_value": previous_total,
            "variance": variance,
            "pct": variance_pct
        },
        "insights": ai_insights,
        "waterfall": waterfall,
        "breakdowns": breakdowns,
        "top_impact": top_impact
    }

    response = apply_smart_currency(
        response, currency, currency_filter,
        ["current_value", "previous_value", "variance", "value", "amount",
         "impact", "current", "previous", "change", "selected_amount", "previous_amount"]
    )
    response = _round_pct_fields(response, decimals=2)
    return response


#######################################################################################
####################################################################################
######################################################################################

#PNL Operating Expense Drilldown

EXPENSE_LEVEL_CONFIG = [
    {
        "field": "account_name",
        "key": "account_name",
        "child_key": "by_account_by_entity",
        "value_key": "selected_amount",
        "prev_key": "previous_amount"
    },
    {
        "field": "entity_name",
        "key": "entity_name",
        "child_key": "by_account_by_entity_by_region",
        "value_key": "selected_amount",
        "prev_key": "previous_amount"
    },
    {
        "field": "region",
        "key": "region",
        "child_key": "by_account_by_entity_by_region_by_department",
        "value_key": "selected_amount",
        "prev_key": "previous_amount"
    },
    {
        "field": "category",
        "key": "category",
        "child_key": None,
        "value_key": "selected_amount",
        "prev_key": "previous_amount"
    }
]


def _get_previous_period(period: str) -> str:
    try:
        dt = datetime.strptime(period, "%Y-%m")
    except Exception:
        raise HTTPException(status_code=400, detail="period must be in YYYY-MM format")
    if dt.month == 1:
        return f"{dt.year - 1}-12"
    return f"{dt.year}-{dt.month - 1:02d}"


def _round_pct_fields(obj: Any, decimals: int = 2) -> Any:
    if isinstance(obj, dict):
        new_obj = {}
        for k, v in obj.items():
            if k in ("pct", "change_pct") and isinstance(v, (int, float)) and v is not None:
                new_obj[k] = round(float(v), decimals)
            else:
                new_obj[k] = _round_pct_fields(v, decimals)
        return new_obj
    if isinstance(obj, list):
        return [_round_pct_fields(x, decimals) for x in obj]
    return obj


def _build_nested_breakdown(
    df_current: pd.DataFrame,
    df_previous: pd.DataFrame,
    level_configs: list
) -> List[Dict[str, Any]]:

    def _build_level(df_curr, df_prev, configs, depth=0):
        if not configs:
            return None

        config = configs[0]
        remaining_configs = configs[1:]

        field = config["field"]
        label_key = config["key"]
        child_key = config["child_key"]
        value_key = config["value_key"]
        prev_key = config["prev_key"]

        curr_agg = df_curr.groupby(field)["amount"].sum()
        prev_agg = df_prev.groupby(field)["amount"].sum()

        all_keys = set(curr_agg.index) | set(prev_agg.index)

        result = []
        for key in all_keys:
            current_val = round(float(curr_agg.get(key, 0)), 2)
            previous_val = round(float(prev_agg.get(key, 0)), 2)
            change = round(current_val - previous_val, 2)
            change_pct = round((change / previous_val * 100), 2) if previous_val else None

            trend = "stable"
            if change_pct is not None:
                if change_pct > 5:
                    trend = "up"
                elif change_pct < -5:
                    trend = "down"

            node = {
                label_key: str(key),
                value_key: current_val,
                prev_key: previous_val,
                "change": change,
                "change_pct": change_pct,
                "trend": trend,
            }

            if child_key and remaining_configs:
                child_curr = df_curr[df_curr[field] == key]
                child_prev = df_prev[df_prev[field] == key]
                node[child_key] = _build_level(child_curr, child_prev, remaining_configs, depth + 1)
            else:
                node["children"] = None

            result.append(node)

        result.sort(key=lambda x: abs(x["change"]), reverse=True)
        return result

    return _build_level(df_current, df_previous, level_configs)


def _calculate_expense_waterfall(
    df_current: pd.DataFrame,
    df_previous: pd.DataFrame,
    current_period: str,
    previous_period: str,
    dimension: str
) -> List[Dict[str, Any]]:

    current_by_dim = df_current.groupby(dimension)["amount"].sum()
    previous_by_dim = df_previous.groupby(dimension)["amount"].sum()

    previous_total = round(float(df_previous["amount"].sum() or 0), 2)
    current_total = round(float(df_current["amount"].sum() or 0), 2)

    waterfall = [
        {"label": f"{previous_period} Total", "value": previous_total, "type": "baseline", "description": "Starting point"}
    ]

    all_dimensions = set(current_by_dim.index) | set(previous_by_dim.index)
    changes = []
    for name in all_dimensions:
        change = float(current_by_dim.get(name, 0)) - float(previous_by_dim.get(name, 0))
        if abs(change) > 0.01:
            changes.append({"name": str(name) if name else "Unknown", "change": round(change, 2)})

    changes.sort(key=lambda x: abs(x["change"]), reverse=True)

    for item in changes[:5]:
        if item["change"] > 0:
            waterfall.append({"label": f"{item['name']} Increase", "value": item["change"], "type": "negative", "description": f"Higher spending in {item['name']}"})
        else:
            waterfall.append({"label": f"{item['name']} Decrease", "value": item["change"], "type": "positive", "description": f"Cost savings in {item['name']}"})

    calculated_total = previous_total + sum(item["change"] for item in changes)
    residual = round(current_total - calculated_total, 2)
    if abs(residual) > 0.01:
        waterfall.append({"label": "Other Changes", "value": residual, "type": "positive" if residual < 0 else "negative", "description": "Other expense adjustments"})

    waterfall.append({"label": f"{current_period} Total", "value": current_total, "type": "baseline", "description": "Ending point"})
    return waterfall


def _calculate_top_impact_expense(
    df_current: pd.DataFrame,
    df_previous: pd.DataFrame,
    dimension: str,
    limit: int = 10
) -> List[Dict[str, Any]]:

    group_cols = [dimension, "entity_name", "region"]
    current_by_dim = df_current.groupby(group_cols)["amount"].sum().reset_index()
    previous_by_dim = df_previous.groupby(group_cols)["amount"].sum().reset_index()

    merged = pd.merge(
        current_by_dim, previous_by_dim,
        on=group_cols, how="outer", suffixes=("_current", "_previous")
    ).fillna(0)

    merged["impact"] = merged["amount_current"] - merged["amount_previous"]
    merged["change_pct"] = (merged["impact"] / merged["amount_previous"] * 100).replace([np.inf, -np.inf], None)
    merged["abs_impact"] = merged["impact"].abs()
    merged = merged.sort_values("abs_impact", ascending=False).head(limit)

    result = []
    for _, row in merged.iterrows():
        result.append({
            "account": str(row[dimension]) if pd.notna(row[dimension]) else "Unknown",
            "entity": str(row["entity_name"]) if pd.notna(row["entity_name"]) else "Unknown",
            "region": str(row["region"]) if pd.notna(row["region"]) else "Unknown",
            "current": round(float(row["amount_current"]), 2),
            "previous": round(float(row["amount_previous"]), 2),
            "change": round(float(row["impact"]), 2),
            "change_pct": round(float(row["change_pct"]), 2) if pd.notna(row["change_pct"]) else None,
            "impact": round(float(row["impact"]), 2)
        })
    return result


def _generate_expense_insights(
    expense_type: str,
    current_total: float,
    previous_total: float,
    variance: float,
    variance_pct: float,
    waterfall: List[Dict],
    breakdowns: List[Dict],
    top_impact: List[Dict]
) -> Dict[str, Any]:

    direction = "increased" if variance > 0 else "decreased"
    main_summary = (
        f"{expense_type} {direction} by {round(abs(variance), 2)} "
        f"({round(abs(variance_pct), 1)}%) from {round(previous_total, 2)} "
        f"to {round(current_total, 2)}."
    )

    drivers = []
    for item in waterfall:
        if item["type"] in ("positive", "negative") and abs(item["value"]) > abs(variance) * 0.1:
            drivers.append({
                "type": item["type"],
                "description": item["label"],
                "amount": round(float(item["value"]), 2),
                "detail": item.get("description", "")
            })

    recommendations = []
    increasing = [a for a in breakdowns if a.get("change_pct") and a["change_pct"] > 10]
    if increasing:
        w = increasing[0]
        recommendations.append(f"Review {w['account_name']} expenses - increased {w['change_pct']:.1f}% MoM")

    decreasing = [a for a in breakdowns if a.get("change_pct") and a["change_pct"] < -5]
    if decreasing:
        b = decreasing[0]
        recommendations.append(f"Replicate cost savings approach from {b['account_name']} ({abs(b['change_pct']):.1f}% reduction)")

    if variance > 0:
        recommendations.append("Implement cost control measures to manage expense growth")
        recommendations.append("Review vendor contracts for potential renegotiation opportunities")
    else:
        recommendations.append("Continue current cost optimization strategies")
        recommendations.append("Document successful cost reduction initiatives for replication")

    if len(recommendations) < 3:
        recommendations.append("Conduct quarterly expense reviews by department")
        recommendations.append("Establish expense benchmarks against industry standards")
        recommendations.append("Implement automated expense tracking and alerts")

    return {
        "summary": main_summary,
        "drivers": drivers,
        "recommendations": recommendations[:5],
        "key_findings": {
            "top_account": breakdowns[0]["account_name"] if breakdowns else None,
            "top_region": None,
            "biggest_impact": top_impact[0]["account"] if top_impact else None,
            "trend": "increasing" if variance_pct > 0 else "decreasing"
        }
    }


def _empty_drilldown_response(metric: str, current_period: str, previous_period: str) -> Dict[str, Any]:
    titles = {
        "revenue": "Revenue Analysis",
        "cost_of_goods_sold": "Cost of Goods Sold Analysis",
        "operating_expense": "Operating Expense Analysis"
    }
    return {
        "metric": metric,
        "title": titles.get(metric, f"{metric.upper()} Analysis"),
        "subtitle": "No data available for selected period",
        "summary": {
            "current_period": current_period,
            "current_value": 0,
            "previous_period": previous_period,
            "previous_value": 0,
            "variance": 0,
            "pct": 0
        },
        "insights": {
            "summary": "No data available to generate insights.",
            "drivers": [],
            "recommendations": [],
            "key_findings": {
                "top_product": None,
                "top_region": None,
                "biggest_mover": None,
                "trend": "neutral"
            }
        },
        "waterfall": [],
        "breakdowns": [],
        "top_impact": []
    }


@financial_router.get(
    "/pnl-operating-expense-drilldown/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def pnl_operating_expense_drilldown(
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    period: str = Query(..., description="Format: YYYY-MM"),
    entity_name: Optional[str] = Query(None),
    region: Optional[str] = Query(None),
    category: Optional[str] = Query(None),
    account_name: Optional[str] = Query(None),
    db: AsyncSession = Depends(get_db)
):
    current_period = period
    previous_period = _get_previous_period(period)

    filters = []
    params = {"current_period": current_period, "previous_period": previous_period}

    if entity_name:
        filters.append("e.entity_name = :entity_name")
        params["entity_name"] = entity_name
    if region:
        filters.append("c.region = :region")
        params["region"] = region
    if category:
        filters.append("d.department_name = :category")
        params["category"] = category
    if account_name:
        filters.append("coa.account_name = :account_name")
        params["account_name"] = account_name

    filter_sql = ("AND " + " AND ".join(filters)) if filters else ""

    sql = text(f"""
    WITH customers_dedup AS (
        SELECT DISTINCT ON (customer_id)
            customer_id,
            customer_name,
            region,
            industry
        FROM fpna_landing.finance_customers
        WHERE customer_id IS NOT NULL
        ORDER BY customer_id
    ),
    entities_dedup AS (
        SELECT DISTINCT ON (entity_id)
            entity_id,
            entity_name,
            region
        FROM fpna_landing.finance_entities
        WHERE entity_id IS NOT NULL
        ORDER BY entity_id
    ),
    departments_dedup AS (
        SELECT DISTINCT ON (department_id)
            department_id,
            department_name
        FROM fpna_landing.finance_departments
        WHERE department_id IS NOT NULL
        ORDER BY department_id
    ),
    period_data AS (
        SELECT
            gl.period,
            COALESCE(coa.account_name, 'Unknown') AS account_name,
            COALESCE(e.entity_name, 'Unknown') AS entity_name,
            COALESCE(NULLIF(c.region, ''), NULLIF(e.region, ''), 'Unknown') AS region,
            COALESCE(d.department_name, 'Unknown') AS category,
            SUM(COALESCE(gl.amount_debit, 0) - COALESCE(gl.amount_credit, 0)) AS amount
        FROM fpna_landing.finance_general_ledger_transactions gl
        JOIN fpna_landing.finance_chart_of_accounts coa
            ON gl.gl_account_id = coa.account_id
        LEFT JOIN customers_dedup c
            ON NULLIF(TRIM(gl.customer_id::text), '') = NULLIF(TRIM(c.customer_id::text), '')
        LEFT JOIN entities_dedup e
            ON NULLIF(TRIM(gl.entity_id::text), '') = NULLIF(TRIM(e.entity_id::text), '')
        LEFT JOIN departments_dedup d
            ON NULLIF(TRIM(gl.department_id::text), '') = NULLIF(TRIM(d.department_id::text), '')
        WHERE coa.account_type = 'Expense'
          AND coa.account_name IN (
              'Marketing', 'Salaries', 'Rent', 'Travel',
              'Employee Benefits', 'Utilities', 'Payroll Taxes',
              'Insurance', 'Other Operating Expense'
          )
          AND gl.period IN (:current_period, :previous_period)
          {filter_sql}
        GROUP BY
            gl.period,
            coa.account_name,
            e.entity_name,
            c.region,
            e.region,
            d.department_name
    )
    SELECT * FROM period_data
    ORDER BY period, amount DESC
    """)

    result = await db.execute(sql, params)
    rows = result.mappings().all()
    df = pd.DataFrame(rows)

    if df.empty:
        return _empty_drilldown_response("operating_expense", current_period, previous_period)

    df_current = df[df["period"] == current_period].copy()
    df_previous = df[df["period"] == previous_period].copy()

    current_total = round(float(df_current["amount"].sum() or 0), 2)
    previous_total = round(float(df_previous["amount"].sum() or 0), 2)
    variance = round(current_total - previous_total, 2)
    variance_pct = round((variance / previous_total * 100), 2) if previous_total else 0

    waterfall = _calculate_expense_waterfall(df_current, df_previous, current_period, previous_period, "account_name")
    breakdowns = _build_nested_breakdown(df_current, df_previous, level_configs=EXPENSE_LEVEL_CONFIG)
    top_impact = _calculate_top_impact_expense(df_current, df_previous, "account_name")
    ai_insights = _generate_expense_insights("operating_expense", current_total, previous_total, variance, variance_pct, waterfall, breakdowns, top_impact)

    response = {
        "metric": "operating_expense",
        "title": "Operating Expense Analysis",
        "subtitle": f"Why operating_expense {'increased' if variance > 0 else 'decreased'} ({variance_pct:+.1f}% MoM)",
        "summary": {
            "current_period": current_period,
            "current_value": current_total,
            "previous_period": previous_period,
            "previous_value": previous_total,
            "variance": variance,
            "pct": variance_pct
        },
        "insights": ai_insights,
        "waterfall": waterfall,
        "breakdowns": breakdowns,
        "top_impact": top_impact
    }

    response = apply_smart_currency(
        response, currency, currency_filter,
        ["current_value", "previous_value", "variance", "value", "amount",
         "impact", "current", "previous", "change", "selected_amount", "previous_amount"]
    )
    response = _round_pct_fields(response, decimals=2)
    return response

############################################################################
###############################################################################
################################################################################

#customer_profitability_drilldown

CUSTOMER_PROFITABILITY_LEVEL_CONFIG = [
    {
        "field": "customer_name",
        "key": "customer_name",
        "child_key": "by_customer_by_product",
        "value_key": "selected_profit",
        "prev_key": "next_profit"
    },
    {
        "field": "product_name",
        "key": "product_name",
        "child_key": "by_customer_by_product_by_entity",
        "value_key": "selected_profit",
        "prev_key": "next_profit"
    },
    {
        "field": "entity_name",
        "key": "entity_name",
        "child_key": "by_customer_by_product_by_entity_by_industry",
        "value_key": "selected_profit",
        "prev_key": "next_profit"
    },
    {
        "field": "industry",
        "key": "industry",
        "child_key": None,
        "value_key": "selected_profit",
        "prev_key": "next_profit"
    }
]


def _build_profitability_nested_breakdown(
    selected_df: pd.DataFrame,
    next_df: pd.DataFrame,
    level_configs: list,
    selected_customer_name: str,
    next_customer_name: str
) -> List[Dict[str, Any]]:

    def _build_level(sel_df, nxt_df, configs, depth=0) -> List[Dict[str, Any]]:
        if not configs:
            return None

        config = configs[0]
        remaining_configs = configs[1:]

        field = config["field"]
        label_key = config["key"]
        child_key = config["child_key"]
        value_key = config["value_key"]
        prev_key = config["prev_key"]

        sel_rev = sel_df.groupby(field)["revenue"].sum()
        sel_cost = sel_df.groupby(field)["total_cost"].sum()
        sel_profit = sel_df.groupby(field)["profit"].sum()

        nxt_rev = nxt_df.groupby(field)["revenue"].sum()
        nxt_cost = nxt_df.groupby(field)["total_cost"].sum()
        nxt_profit = nxt_df.groupby(field)["profit"].sum()

        all_keys = set(sel_profit.index) | set(nxt_profit.index)

        result = []
        for key in all_keys:
            s_revenue = round(float(sel_rev.get(key, 0)), 2)
            s_cost = round(float(sel_cost.get(key, 0)), 2)
            s_profit = round(float(sel_profit.get(key, 0)), 2)

            n_revenue = round(float(nxt_rev.get(key, 0)), 2)
            n_cost = round(float(nxt_cost.get(key, 0)), 2)
            n_profit = round(float(nxt_profit.get(key, 0)), 2)

            change = round(s_profit - n_profit, 2)
            change_pct = round((change / n_profit * 100), 2) if n_profit else 0
            if pd.isna(change_pct) or not np.isfinite(change_pct):
                change_pct = 0

            trend = "up" if change > 0 else "down" if change < 0 else "flat"

            child_sel = sel_df[sel_df[field] == key]
            child_nxt = nxt_df[nxt_df[field] == key]

            node = {
                label_key: str(key),
                "selected_customer": selected_customer_name,
                "selected_revenue": s_revenue,
                "selected_cost": s_cost,
                value_key: s_profit,
                "next_customer": next_customer_name,
                "next_revenue": n_revenue,
                "next_cost": n_cost,
                prev_key: n_profit,
                "change": change,
                "change_pct": change_pct,
                "trend": trend
            }

            if child_key and remaining_configs:
                node[child_key] = _build_level(child_sel, child_nxt, remaining_configs, depth + 1)
            else:
                node["children"] = None

            result.append(node)

        result.sort(key=lambda x: abs(x["change"]), reverse=True)
        return result

    return _build_level(selected_df, next_df, level_configs)


@financial_router.get(
    "/customer-profitability-drilldown/{year}/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def customer_profitability_drilldown(
    request: Request,
    year: int = Path(...),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    customer_name: str = Query(...),
    db: AsyncSession = Depends(get_db),
):
    sql = text("""
        WITH base_revenue AS (
            SELECT
                c.customer_name,
                c.industry,
                c.region,
                e.entity_name,
                d.department_name,
                p.product_name,
                gl.period,
                SUM(gl.amount_credit - gl.amount_debit) AS revenue
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa
                ON gl.gl_account_id = coa.account_id
            JOIN fpna_landing.finance_customers c
                ON REGEXP_REPLACE(gl.customer_id::TEXT,'\\.0$','')
                 = REGEXP_REPLACE(c.customer_id::TEXT,'\\.0$','')
            JOIN fpna_landing.finance_entities e
                ON gl.entity_id::TEXT = e.entity_id::TEXT
            JOIN fpna_landing.finance_departments d
                ON gl.department_id::TEXT = d.department_id::TEXT
            LEFT JOIN fpna_landing.crm_products p
                ON NULLIF(TRIM(gl.profit_center_id::TEXT), '') = NULLIF(TRIM(p.profit_center_id::TEXT), '')
            WHERE coa.account_type = 'Revenue'
              AND gl.customer_id IS NOT NULL
              AND gl.period LIKE :year_filter
            GROUP BY
                c.customer_name,
                c.industry,
                c.region,
                e.entity_name,
                d.department_name,
                p.product_name,
                gl.period
        ),
        company_revenue AS (
            SELECT
                gl.period,
                COALESCE(SUM(gl.amount_credit - gl.amount_debit),0) AS total_revenue
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa
                ON gl.gl_account_id = coa.account_id
            WHERE coa.account_type = 'Revenue'
              AND gl.period LIKE :year_filter
            GROUP BY gl.period
        ),
        company_cost AS (
            SELECT
                gl.period,
                COALESCE(SUM(gl.amount_debit),0) AS total_cost
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa
                ON gl.gl_account_id = coa.account_id
            WHERE coa.account_type = 'Expense'
              AND gl.period LIKE :year_filter
            GROUP BY gl.period
        )
        SELECT
            r.customer_name,
            r.industry,
            r.region,
            r.entity_name,
            r.department_name,
            COALESCE(r.product_name, 'Unknown') AS product_name,
            r.period,
            r.revenue,
            (r.revenue / NULLIF(cr.total_revenue,0)) * cc.total_cost AS total_cost,
            r.revenue - ((r.revenue / NULLIF(cr.total_revenue,0)) * cc.total_cost) AS profit
        FROM base_revenue r
        LEFT JOIN company_revenue cr ON r.period = cr.period
        LEFT JOIN company_cost cc ON r.period = cc.period
        ORDER BY profit DESC
    """)

    rows = (await db.execute(sql, {"year_filter": f"{year}%"})).fetchall()

    if not rows:
        return {
            "error": "No data available",
            "message": f"No customer profitability data found for year {year}"
        }

    df = pd.DataFrame(rows, columns=[
        "customer_name", "industry", "region", "entity_name",
        "department_name", "product_name", "period", "revenue", "total_cost", "profit"
    ])

    for c in ["revenue", "total_cost", "profit"]:
        df[c] = df[c].astype(float)

    df = df.replace([np.inf, -np.inf], 0)
    df = df.fillna(0)

    df["pct_margin"] = ((df["profit"] / df["revenue"]) * 100).replace([np.inf, -np.inf], 0).fillna(0).round(2)

    df_grouped = df.groupby("customer_name").agg({
        "revenue": "sum",
        "total_cost": "sum",
        "profit": "sum"
    }).reset_index()

    df_grouped["pct_margin"] = ((df_grouped["profit"] / df_grouped["revenue"]) * 100).replace([np.inf, -np.inf], 0).fillna(0).round(2)

    df_grouped = df_grouped.sort_values("profit", ascending=False).reset_index(drop=True)
    df_grouped["row_index"] = df_grouped.index

    selected_customer_row = df_grouped[df_grouped["customer_name"] == customer_name]

    if selected_customer_row.empty:
        return {
            "error": "Customer not found",
            "message": f"Customer '{customer_name}' not found in the data"
        }

    selected_row_index = int(selected_customer_row["row_index"].iloc[0])

    if selected_row_index + 1 >= len(df_grouped):
        return {
            "error": "No customer below",
            "message": f"{customer_name} is the last customer in the list. No customer to compare below."
        }

    next_customer_row = df_grouped.iloc[selected_row_index + 1]
    next_customer_name = next_customer_row["customer_name"]

    selected_revenue = float(selected_customer_row["revenue"].iloc[0])
    selected_cost = float(selected_customer_row["total_cost"].iloc[0])
    selected_profit = float(selected_customer_row["profit"].iloc[0])
    selected_margin = float(selected_customer_row["pct_margin"].iloc[0])

    next_revenue = float(next_customer_row["revenue"])
    next_cost = float(next_customer_row["total_cost"])
    next_profit = float(next_customer_row["profit"])
    next_margin = float(next_customer_row["pct_margin"])

    revenue_diff = selected_revenue - next_revenue
    revenue_diff_pct = ((revenue_diff / next_revenue) * 100) if next_revenue and next_revenue != 0 else 0

    cost_diff = selected_cost - next_cost
    cost_diff_pct = ((cost_diff / next_cost) * 100) if next_cost and next_cost != 0 else 0

    profit_diff = selected_profit - next_profit
    profit_diff_pct = ((profit_diff / next_profit) * 100) if next_profit and next_profit != 0 else 0

    margin_diff = selected_margin - next_margin

    revenue_diff_pct = 0 if pd.isna(revenue_diff_pct) or not np.isfinite(revenue_diff_pct) else revenue_diff_pct
    cost_diff_pct = 0 if pd.isna(cost_diff_pct) or not np.isfinite(cost_diff_pct) else cost_diff_pct
    profit_diff_pct = 0 if pd.isna(profit_diff_pct) or not np.isfinite(profit_diff_pct) else profit_diff_pct

    selected_customer_details = df[df["customer_name"] == customer_name].copy()
    next_customer_details = df[df["customer_name"] == next_customer_name].copy()

    breakdowns = _build_profitability_nested_breakdown(
        selected_customer_details,
        next_customer_details,
        CUSTOMER_PROFITABILITY_LEVEL_CONFIG,
        customer_name,
        next_customer_name
    )

    by_entity = selected_customer_details.groupby("entity_name").agg({"profit": "sum"}).reset_index()

    performance_flow = []

    performance_flow.append({
        "label": f"{next_customer_name} Profit",
        "value": round(next_profit, 2),
        "type": "base"
    })

    if revenue_diff != 0:
        performance_flow.append({
            "label": "Revenue Impact",
            "value": round(revenue_diff, 2),
            "type": "increase" if revenue_diff > 0 else "decrease"
        })

    if cost_diff != 0:
        performance_flow.append({
            "label": "Cost Impact",
            "value": round(-cost_diff, 2),
            "type": "increase" if cost_diff < 0 else "decrease"
        })

    top_from_breakdowns = sorted(breakdowns, key=lambda x: abs(x["change"]), reverse=True)[:3]
    for item in top_from_breakdowns:
        if abs(item["change"]) > 100:
            performance_flow.append({
                "label": f"Customer: {item['customer_name']}",
                "value": round(item["change"], 2),
                "type": "increase" if item["change"] > 0 else "decrease"
            })

    performance_flow.append({
        "label": f"{customer_name} Profit",
        "value": round(selected_profit, 2),
        "type": "total"
    })

    summary = []

    if abs(revenue_diff_pct) > 5:
        if revenue_diff > 0:
            summary.append({
                "key": "Higher Revenue",
                "impact": round(abs(revenue_diff), 2),
                "impact_pct": round(abs(revenue_diff_pct), 2),
                "explanation": f"{customer_name} generates {abs(revenue_diff_pct):.1f}% more revenue than {next_customer_name}"
            })
        else:
            summary.append({
                "key": "Lower Revenue",
                "impact": round(abs(revenue_diff), 2),
                "impact_pct": round(abs(revenue_diff_pct), 2),
                "explanation": f"{customer_name} generates {abs(revenue_diff_pct):.1f}% less revenue than {next_customer_name}"
            })

    if abs(cost_diff_pct) > 5:
        if cost_diff < 0:
            summary.append({
                "key": "Better Cost Efficiency",
                "impact": round(abs(cost_diff), 2),
                "impact_pct": round(abs(cost_diff_pct), 2),
                "explanation": f"{customer_name} has {abs(cost_diff_pct):.1f}% lower costs, improving profitability"
            })
        else:
            summary.append({
                "key": "Higher Costs",
                "impact": round(abs(cost_diff), 2),
                "impact_pct": round(abs(cost_diff_pct), 2),
                "explanation": f"{customer_name} has {abs(cost_diff_pct):.1f}% higher costs, reducing profitability"
            })

    if abs(margin_diff) > 2:
        if margin_diff > 0:
            summary.append({
                "key": "Superior Margin Efficiency",
                "impact": round(abs(margin_diff), 2),
                "impact_pct": round(abs(margin_diff), 2),
                "explanation": f"{customer_name} operates with {abs(margin_diff):.1f}% better profit margin"
            })
        else:
            summary.append({
                "key": "Lower Margin Efficiency",
                "impact": round(abs(margin_diff), 2),
                "impact_pct": round(abs(margin_diff), 2),
                "explanation": f"{customer_name} operates with {abs(margin_diff):.1f}% lower profit margin"
            })

    if breakdowns:
        top_item = breakdowns[0]
        if abs(top_item["change"]) > 1000:
            summary.append({
                "key": f"Customer: {top_item['customer_name']}",
                "impact": round(abs(top_item["change"]), 2),
                "impact_pct": round(abs(top_item["change_pct"]), 2),
                "explanation": f"Customer segment '{top_item['customer_name']}' shows {abs(top_item['change_pct']):.1f}% difference in profit"
            })
        if len(breakdowns) > 1:
            second_item = breakdowns[1]
            if abs(second_item["change"]) > 1000:
                summary.append({
                    "key": f"Customer: {second_item['customer_name']}",
                    "impact": round(abs(second_item["change"]), 2),
                    "impact_pct": round(abs(second_item["change_pct"]), 2),
                    "explanation": f"Customer segment '{second_item['customer_name']}' shows {abs(second_item['change_pct']):.1f}% difference in profit"
                })

    insights = []

    if profit_diff > 0:
        insights.append(
            f"WHY {customer_name} outperforms {next_customer_name}: {customer_name} achieves ${abs(profit_diff):,.2f} ({abs(profit_diff_pct):.1f}%) higher profit"
        )
    else:
        insights.append(
            f"WHY {customer_name} underperforms {next_customer_name}: {customer_name} achieves ${abs(profit_diff):,.2f} ({abs(profit_diff_pct):.1f}%) lower profit"
        )

    if summary:
        insights.append("Key drivers of the difference:")
        for driver in summary[:3]:
            insights.append(f"• {driver['explanation']}")

    if profit_diff < 0:
        insights.append(f"Recommendation: Analyze why {next_customer_name} has better performance and apply learnings to {customer_name}")
        if margin_diff < 0:
            insights.append(f"Focus area: Improve cost efficiency - current margin gap is {abs(margin_diff):.1f}%")
    else:
        insights.append(f"Recommendation: Replicate {customer_name}'s success factors to improve {next_customer_name}'s performance")

    response = {
        "metric": "customer_profitability_drilldown",
        "title": f"Why {customer_name} {'Outperforms' if profit_diff > 0 else 'Underperforms'} {next_customer_name}",
        "subtitle": f"Root cause analysis: {customer_name} vs {next_customer_name} for year {year}",
        "selected_customer": {
            "customer_name": customer_name,
            "revenue": round(selected_revenue, 2),
            "total_cost": round(selected_cost, 2),
            "profit": round(selected_profit, 2),
            "pct_margin": round(selected_margin, 2)
        },
        "next_customer": {
            "customer_name": next_customer_name,
            "revenue": round(next_revenue, 2),
            "total_cost": round(next_cost, 2),
            "profit": round(next_profit, 2),
            "pct_margin": round(next_margin, 2)
        },
        "comparison": {
            "revenue_diff": round(revenue_diff, 2),
            "revenue_diff_pct": round(revenue_diff_pct, 2),
            "cost_diff": round(cost_diff, 2),
            "cost_diff_pct": round(cost_diff_pct, 2),
            "profit_diff": round(profit_diff, 2),
            "profit_diff_pct": round(profit_diff_pct, 2),
            "margin_diff": round(margin_diff, 2)
        },
        "summary": summary,
        "insights": insights,
        "performance_flow": performance_flow,
        "breakdowns": breakdowns
    }

    response = apply_smart_currency(
        response,
        currency,
        currency_filter,
        [
            "revenue",
            "total_cost",
            "profit",
            "revenue_diff",
            "cost_diff",
            "profit_diff",
            "change",
            "impact",
            "value",
            "selected_revenue",
            "selected_cost",
            "selected_profit",
            "next_revenue",
            "next_cost",
            "next_profit"
        ]
    )

    return response


################################################################
################################################################
###############################################################

#revenue_product_drilldown

PRODUCT_PROFITABILITY_LEVEL_CONFIG = [
    {
        "field": "customer_name",
        "key": "customer_name",
        "child_key": "by_customer_by_product",
        "value_key": "selected_profit",
        "prev_key": "next_profit"
    },
    {
        "field": "product_name",
        "key": "product_name",
        "child_key": "by_customer_by_product_by_entity",
        "value_key": "selected_profit",
        "prev_key": "next_profit"
    },
    {
        "field": "entity_name",
        "key": "entity_name",
        "child_key": "by_customer_by_product_by_entity_by_industry",
        "value_key": "selected_profit",
        "prev_key": "next_profit"
    },
    {
        "field": "region",
        "key": "region",
        "child_key": None,
        "value_key": "selected_profit",
        "prev_key": "next_profit"
    }
]


def _build_product_nested_breakdown(
    selected_df: pd.DataFrame,
    next_df: pd.DataFrame,
    level_configs: list,
    selected_product_name: str,
    next_product_name: str
) -> List[Dict[str, Any]]:

    def _build_level(sel_df, nxt_df, configs, depth=0) -> List[Dict[str, Any]]:
        if not configs:
            return None

        config = configs[0]
        remaining_configs = configs[1:]

        field = config["field"]
        label_key = config["key"]
        child_key = config["child_key"]
        value_key = config["value_key"]
        prev_key = config["prev_key"]

        sel_rev = sel_df.groupby(field)["revenue"].sum()
        sel_cost = sel_df.groupby(field)["total_cost"].sum()
        sel_profit = sel_df.groupby(field)["profit"].sum()

        nxt_rev = nxt_df.groupby(field)["revenue"].sum()
        nxt_cost = nxt_df.groupby(field)["total_cost"].sum()
        nxt_profit = nxt_df.groupby(field)["profit"].sum()

        all_keys = set(sel_profit.index) | set(nxt_profit.index)

        result = []
        for key in all_keys:
            s_revenue = round(float(sel_rev.get(key, 0)), 2)
            s_cost = round(float(sel_cost.get(key, 0)), 2)
            s_profit = round(float(sel_profit.get(key, 0)), 2)

            n_revenue = round(float(nxt_rev.get(key, 0)), 2)
            n_cost = round(float(nxt_cost.get(key, 0)), 2)
            n_profit = round(float(nxt_profit.get(key, 0)), 2)

            change = round(s_profit - n_profit, 2)
            change_pct = round((change / n_profit * 100), 2) if n_profit else 0
            if pd.isna(change_pct) or not np.isfinite(change_pct):
                change_pct = 0

            trend = "up" if change > 0 else "down" if change < 0 else "flat"

            child_sel = sel_df[sel_df[field] == key]
            child_nxt = nxt_df[nxt_df[field] == key]

            node = {
                label_key: str(key),
                "selected_product": selected_product_name,
                "selected_revenue": s_revenue,
                "selected_cost": s_cost,
                value_key: s_profit,
                "next_product": next_product_name,
                "next_revenue": n_revenue,
                "next_cost": n_cost,
                prev_key: n_profit,
                "change": change,
                "change_pct": change_pct,
                "trend": trend
            }

            if child_key and remaining_configs:
                node[child_key] = _build_level(child_sel, child_nxt, remaining_configs, depth + 1)
            else:
                node["children"] = None

            result.append(node)

        result.sort(key=lambda x: abs(x["change"]), reverse=True)
        return result

    return _build_level(selected_df, next_df, level_configs)


@financial_router.get(
    "/revenue_product_drilldown/{year}/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def revenue_product_drilldown(
    request: Request,
    year: int = Path(...),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    product_name: str = Query(...),
    db: AsyncSession = Depends(get_db),
):
    sql = text("""
        WITH product_dim AS (
            SELECT
                profit_center_id,
                MAX(NULLIF(TRIM(product_name), '')) AS product_name
            FROM fpna_landing.crm_products
            GROUP BY profit_center_id
        ),
        base_revenue AS (
            SELECT
                COALESCE(p.product_name, 'UNMAPPED PRODUCT') AS product_name,
                COALESCE(c.customer_name, 'Unknown') AS customer_name,
                COALESCE(c.region, 'Unknown') AS region,
                COALESCE(e.entity_name, 'Unknown') AS entity_name,
                COALESCE(d.department_name, 'Unknown') AS department_name,
                gl.period,
                SUM(gl.amount_credit - gl.amount_debit) AS revenue
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa
                ON gl.gl_account_id = coa.account_id
            LEFT JOIN fpna_landing.finance_customers c
                ON REGEXP_REPLACE(gl.customer_id::TEXT,'\\.0$','')
                 = REGEXP_REPLACE(c.customer_id::TEXT,'\\.0$','')
            LEFT JOIN fpna_landing.finance_entities e
                ON gl.entity_id::TEXT = e.entity_id::TEXT
            LEFT JOIN fpna_landing.finance_departments d
                ON gl.department_id::TEXT = d.department_id::TEXT
            LEFT JOIN product_dim p
                ON gl.profit_center_id::TEXT = p.profit_center_id::TEXT
            WHERE coa.account_type = 'Revenue'
              AND gl.period LIKE :year_filter
            GROUP BY
                p.product_name,
                c.customer_name,
                c.region,
                e.entity_name,
                d.department_name,
                gl.period
        ),
        company_revenue AS (
            SELECT
                gl.period,
                COALESCE(SUM(gl.amount_credit - gl.amount_debit),0) AS total_revenue
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa
                ON gl.gl_account_id = coa.account_id
            WHERE coa.account_type = 'Revenue'
              AND gl.period LIKE :year_filter
            GROUP BY gl.period
        ),
        company_cost AS (
            SELECT
                gl.period,
                COALESCE(SUM(gl.amount_debit),0) AS total_cost
            FROM fpna_landing.finance_general_ledger_transactions gl
            JOIN fpna_landing.finance_chart_of_accounts coa
                ON gl.gl_account_id = coa.account_id
            WHERE coa.account_type = 'Expense'
              AND gl.period LIKE :year_filter
            GROUP BY gl.period
        )
        SELECT
            r.product_name,
            r.customer_name,
            r.region,
            r.entity_name,
            r.department_name,
            r.period,
            r.revenue,
            (r.revenue / NULLIF(cr.total_revenue,0)) * cc.total_cost AS total_cost,
            r.revenue - ((r.revenue / NULLIF(cr.total_revenue,0)) * cc.total_cost) AS profit
        FROM base_revenue r
        LEFT JOIN company_revenue cr ON r.period = cr.period
        LEFT JOIN company_cost cc ON r.period = cc.period
        ORDER BY profit DESC
    """)

    rows = (await db.execute(sql, {"year_filter": f"{year}%"})).fetchall()

    if not rows:
        return {
            "error": "No data available",
            "message": f"No product profitability data found for year {year}"
        }

    df = pd.DataFrame(rows, columns=[
        "product_name", "customer_name", "region", "entity_name",
        "department_name", "period", "revenue", "total_cost", "profit"
    ])

    for c in ["revenue", "total_cost", "profit"]:
        df[c] = df[c].astype(float)

    df = df.replace([np.inf, -np.inf], 0)
    df = df.fillna(0)

    df["pct_margin"] = ((df["profit"] / df["revenue"]) * 100).replace([np.inf, -np.inf], 0).fillna(0).round(2)

    df_grouped = df.groupby("product_name").agg({
        "revenue": "sum",
        "total_cost": "sum",
        "profit": "sum"
    }).reset_index()

    df_grouped["pct_margin"] = ((df_grouped["profit"] / df_grouped["revenue"]) * 100).replace([np.inf, -np.inf], 0).fillna(0).round(2)

    df_grouped = df_grouped.sort_values("profit", ascending=False).reset_index(drop=True)
    df_grouped["row_index"] = df_grouped.index

    selected_product_row = df_grouped[df_grouped["product_name"] == product_name]

    if selected_product_row.empty:
        return {
            "error": "Product not found",
            "message": f"Product '{product_name}' not found in the data"
        }

    selected_row_index = int(selected_product_row["row_index"].iloc[0])

    if selected_row_index + 1 >= len(df_grouped):
        return {
            "error": "No product below",
            "message": f"{product_name} is the last product in the list. No product to compare below."
        }

    next_product_row = df_grouped.iloc[selected_row_index + 1]
    next_product_name = next_product_row["product_name"]

    selected_revenue = float(selected_product_row["revenue"].iloc[0])
    selected_cost = float(selected_product_row["total_cost"].iloc[0])
    selected_profit = float(selected_product_row["profit"].iloc[0])
    selected_margin = float(selected_product_row["pct_margin"].iloc[0])

    next_revenue = float(next_product_row["revenue"])
    next_cost = float(next_product_row["total_cost"])
    next_profit = float(next_product_row["profit"])
    next_margin = float(next_product_row["pct_margin"])

    revenue_diff = selected_revenue - next_revenue
    revenue_diff_pct = ((revenue_diff / next_revenue) * 100) if next_revenue and next_revenue != 0 else 0

    cost_diff = selected_cost - next_cost
    cost_diff_pct = ((cost_diff / next_cost) * 100) if next_cost and next_cost != 0 else 0

    profit_diff = selected_profit - next_profit
    profit_diff_pct = ((profit_diff / next_profit) * 100) if next_profit and next_profit != 0 else 0

    margin_diff = selected_margin - next_margin

    revenue_diff_pct = 0 if pd.isna(revenue_diff_pct) or not np.isfinite(revenue_diff_pct) else revenue_diff_pct
    cost_diff_pct = 0 if pd.isna(cost_diff_pct) or not np.isfinite(cost_diff_pct) else cost_diff_pct
    profit_diff_pct = 0 if pd.isna(profit_diff_pct) or not np.isfinite(profit_diff_pct) else profit_diff_pct

    selected_product_details = df[df["product_name"] == product_name].copy()
    next_product_details = df[df["product_name"] == next_product_name].copy()

    breakdowns = _build_product_nested_breakdown(
        selected_product_details,
        next_product_details,
        PRODUCT_PROFITABILITY_LEVEL_CONFIG,
        product_name,
        next_product_name
    )

    performance_flow = []

    performance_flow.append({
        "label": f"{next_product_name} Profit",
        "value": round(next_profit, 2),
        "type": "base"
    })

    if revenue_diff != 0:
        performance_flow.append({
            "label": "Revenue Impact",
            "value": round(revenue_diff, 2),
            "type": "increase" if revenue_diff > 0 else "decrease"
        })

    if cost_diff != 0:
        performance_flow.append({
            "label": "Cost Impact",
            "value": round(-cost_diff, 2),
            "type": "increase" if cost_diff < 0 else "decrease"
        })

    top_from_breakdowns = sorted(breakdowns, key=lambda x: abs(x["change"]), reverse=True)[:3]
    for item in top_from_breakdowns:
        if abs(item["change"]) > 100:
            performance_flow.append({
                "label": f"Customer: {item['customer_name']}",
                "value": round(item["change"], 2),
                "type": "increase" if item["change"] > 0 else "decrease"
            })

    performance_flow.append({
        "label": f"{product_name} Profit",
        "value": round(selected_profit, 2),
        "type": "total"
    })

    summary = []

    if abs(revenue_diff_pct) > 5:
        if revenue_diff > 0:
            summary.append({
                "key": "Higher Revenue",
                "impact": round(abs(revenue_diff), 2),
                "impact_pct": round(abs(revenue_diff_pct), 2),
                "explanation": f"{product_name} generates {abs(revenue_diff_pct):.1f}% more revenue than {next_product_name}"
            })
        else:
            summary.append({
                "key": "Lower Revenue",
                "impact": round(abs(revenue_diff), 2),
                "impact_pct": round(abs(revenue_diff_pct), 2),
                "explanation": f"{product_name} generates {abs(revenue_diff_pct):.1f}% less revenue than {next_product_name}"
            })

    if abs(cost_diff_pct) > 5:
        if cost_diff < 0:
            summary.append({
                "key": "Better Cost Efficiency",
                "impact": round(abs(cost_diff), 2),
                "impact_pct": round(abs(cost_diff_pct), 2),
                "explanation": f"{product_name} has {abs(cost_diff_pct):.1f}% lower costs, improving profitability"
            })
        else:
            summary.append({
                "key": "Higher Costs",
                "impact": round(abs(cost_diff), 2),
                "impact_pct": round(abs(cost_diff_pct), 2),
                "explanation": f"{product_name} has {abs(cost_diff_pct):.1f}% higher costs, reducing profitability"
            })

    if abs(margin_diff) > 2:
        if margin_diff > 0:
            summary.append({
                "key": "Superior Margin Efficiency",
                "impact": round(abs(margin_diff), 2),
                "impact_pct": round(abs(margin_diff), 2),
                "explanation": f"{product_name} operates with {abs(margin_diff):.1f}% better profit margin"
            })
        else:
            summary.append({
                "key": "Lower Margin Efficiency",
                "impact": round(abs(margin_diff), 2),
                "impact_pct": round(abs(margin_diff), 2),
                "explanation": f"{product_name} operates with {abs(margin_diff):.1f}% lower profit margin"
            })

    if breakdowns:
        top_item = breakdowns[0]
        if abs(top_item["change"]) > 1000:
            summary.append({
                "key": f"Customer: {top_item['customer_name']}",
                "impact": round(abs(top_item["change"]), 2),
                "impact_pct": round(abs(top_item["change_pct"]), 2),
                "explanation": f"Customer '{top_item['customer_name']}' shows {abs(top_item['change_pct']):.1f}% difference in profit"
            })
        if len(breakdowns) > 1:
            second_item = breakdowns[1]
            if abs(second_item["change"]) > 1000:
                summary.append({
                    "key": f"Customer: {second_item['customer_name']}",
                    "impact": round(abs(second_item["change"]), 2),
                    "impact_pct": round(abs(second_item["change_pct"]), 2),
                    "explanation": f"Customer '{second_item['customer_name']}' shows {abs(second_item['change_pct']):.1f}% difference in profit"
                })

    insights = []

    if profit_diff > 0:
        insights.append(
            f"WHY {product_name} outperforms {next_product_name}: {product_name} achieves ${abs(profit_diff):,.2f} ({abs(profit_diff_pct):.1f}%) higher profit"
        )
    else:
        insights.append(
            f"WHY {product_name} underperforms {next_product_name}: {product_name} achieves ${abs(profit_diff):,.2f} ({abs(profit_diff_pct):.1f}%) lower profit"
        )

    if summary:
        insights.append("Key drivers of the difference:")
        for s in summary[:3]:
            insights.append(f"• {s['explanation']}")

    if profit_diff < 0:
        insights.append(f"Recommendation: Analyze why {next_product_name} has better performance and apply learnings to {product_name}")
        if margin_diff < 0:
            insights.append(f"Focus area: Improve cost efficiency - current margin gap is {abs(margin_diff):.1f}%")
    else:
        insights.append(f"Recommendation: Replicate {product_name}'s success factors to improve {next_product_name}'s performance")

    response = {
        "metric": "product_profitability_drilldown",
        "title": f"Why {product_name} {'Outperforms' if profit_diff > 0 else 'Underperforms'} {next_product_name}",
        "subtitle": f"Root cause analysis: {product_name} vs {next_product_name} for year {year}",
        "selected_product": {
            "product_name": product_name,
            "revenue": round(selected_revenue, 2),
            "total_cost": round(selected_cost, 2),
            "profit": round(selected_profit, 2),
            "pct_margin": round(selected_margin, 2)
        },
        "next_product": {
            "product_name": next_product_name,
            "revenue": round(next_revenue, 2),
            "total_cost": round(next_cost, 2),
            "profit": round(next_profit, 2),
            "pct_margin": round(next_margin, 2)
        },
        "comparison": {
            "revenue_diff": round(revenue_diff, 2),
            "revenue_diff_pct": round(revenue_diff_pct, 2),
            "cost_diff": round(cost_diff, 2),
            "cost_diff_pct": round(cost_diff_pct, 2),
            "profit_diff": round(profit_diff, 2),
            "profit_diff_pct": round(profit_diff_pct, 2),
            "margin_diff": round(margin_diff, 2)
        },
        "summary": summary,
        "insights": insights,
        "performance_flow": performance_flow,
        "breakdowns": breakdowns
    }

    response = apply_smart_currency(
        response,
        currency,
        currency_filter,
        [
            "revenue",
            "total_cost",
            "profit",
            "revenue_diff",
            "cost_diff",
            "profit_diff",
            "change",
            "impact",
            "value",
            "selected_revenue",
            "selected_cost",
            "selected_profit",
            "next_revenue",
            "next_cost",
            "next_profit"
        ]
    )

    return response



#############################################################################
#############################################################################
###########################################################################

# Balence sheet dril down

BALANCE_SHEET_LEVEL_CONFIG = [
    {
        "field": "entity_name",
        "key": "entity_name",
        "child_key": "by_entity_by_account_type",
        "value_key": "current_balance",
        "prev_key": "previous_balance"
    },
    {
        "field": "account_type",
        "key": "account_type",
        "child_key": None,
        "value_key": "current_balance",
        "prev_key": "previous_balance"
    }
]


def _build_balance_nested_breakdown(
    df_current: pd.DataFrame,
    df_previous: pd.DataFrame,
    level_configs: list
) -> List[Dict[str, Any]]:

    def _build_level(df_curr, df_prev, configs, depth=0) -> List[Dict[str, Any]]:
        if not configs:
            return None

        config = configs[0]
        remaining_configs = configs[1:]

        field = config["field"]
        label_key = config["key"]
        child_key = config["child_key"]
        value_key = config["value_key"]
        prev_key = config["prev_key"]

        curr_agg = df_curr.groupby(field)["ending_balance"].sum()
        prev_agg = df_prev.groupby(field)["ending_balance"].sum()

        all_keys = set(curr_agg.index) | set(prev_agg.index)

        result = []
        for key in all_keys:
            if depth == 0 and str(key) == "Unknown":
                continue

            current_val = round(float(curr_agg.get(key, 0)), 2)
            previous_val = round(float(prev_agg.get(key, 0)), 2)
            change = round(current_val - previous_val, 2)
            change_pct = round((change / previous_val * 100), 2) if previous_val and previous_val != 0 else None

            trend = "stable"
            if change_pct is not None:
                if change_pct > 5:
                    trend = "up"
                elif change_pct < -5:
                    trend = "down"

            child_curr = df_curr[df_curr[field] == key]
            child_prev = df_prev[df_prev[field] == key]

            node = {
                label_key: str(key),
                value_key: current_val,
                prev_key: previous_val,
                "change": change,
                "change_pct": change_pct,
                "trend": trend
            }

            if child_key and remaining_configs:
                node[child_key] = _build_level(child_curr, child_prev, remaining_configs, depth + 1)
            else:
                node["children"] = None

            result.append(node)

        result.sort(key=lambda x: abs(x["change"]), reverse=True)
        return result

    return _build_level(df_current, df_previous, level_configs)


@financial_router.get(
    "/balance-sheet-drilldown/{account_name}/{currency}",
    dependencies=[Depends(get_current_user)]
)
async def balance_sheet_drilldown(
    account_name: str = Path(...),
    currency: str = Path(...),
    currency_filter: CurrencyFilter = Query(default=CurrencyFilter.actual_value),
    period: Optional[str] = Query(None, description="Format: YYYY-MM"),
    entity: Optional[str] = Query(None),
    db: AsyncSession = Depends(get_db)
):
    if period:
        current_period = period
    else:
        period_query = text("""
            SELECT MAX(period) as max_period
            FROM fpna_landing.finance_trial_balance
            WHERE period IS NOT NULL
        """)
        period_result = await db.execute(period_query)
        max_period_row = period_result.fetchone()

        if max_period_row and max_period_row[0]:
            current_period = max_period_row[0]
        else:
            today = date.today()
            current_period = f"{today.year}-{today.month:02d}"

    previous_period = _get_previous_period(current_period)

    return await _balance_sheet_account_drilldown(
        db, account_name, currency, currency_filter,
        current_period, previous_period, entity
    )


async def _balance_sheet_account_drilldown(
    db: AsyncSession,
    account_name: str,
    currency: str,
    currency_filter: CurrencyFilter,
    current_period: str,
    previous_period: str,
    entity: Optional[str]
) -> Dict[str, Any]:

    filters = []
    params = {
        "account_name": account_name,
        "current_period": current_period,
        "previous_period": previous_period
    }

    if entity:
        filters.append("CAST(tb.entity_id AS TEXT) = :entity")
        params["entity"] = str(entity)

    filter_sql_tb = "AND " + " AND ".join(filters) if filters else ""

    gl_filters = []
    if entity:
        gl_filters.append("CAST(gl.entity_id AS TEXT) = :entity")
    filter_sql_gl = "AND " + " AND ".join(gl_filters) if gl_filters else ""

    sql_period = text(f"""
    WITH entities_dedup AS (
        SELECT DISTINCT ON (entity_id)
            entity_id,
            entity_name
        FROM fpna_landing.finance_entities
        WHERE entity_id IS NOT NULL
        ORDER BY entity_id
    )
    SELECT
        tb.period,
        COALESCE(e.entity_name, 'Unknown') AS entity_name,
        COALESCE(CAST(e.entity_id AS TEXT), 'Unknown') AS entity_id,
        coa.account_code,
        coa.account_name,
        coa.account_type,
        SUM(tb.ending_balance) AS ending_balance,
        SUM(tb.debit_total) AS debit_total,
        SUM(tb.credit_total) AS credit_total
    FROM fpna_landing.finance_trial_balance tb
    JOIN fpna_landing.finance_chart_of_accounts coa
        ON tb.gl_account_id = coa.account_id
    LEFT JOIN entities_dedup e
        ON CAST(tb.entity_id AS TEXT) = CAST(e.entity_id AS TEXT)
    WHERE coa.account_name = :account_name
      AND tb.period IN (:current_period, :previous_period)
      {filter_sql_tb}
    GROUP BY
        tb.period,
        e.entity_name,
        e.entity_id,
        coa.account_code,
        coa.account_name,
        coa.account_type
    ORDER BY tb.period, e.entity_name
    """)

    period_result = await db.execute(sql_period, params)
    period_rows = period_result.mappings().all()
    df_period = pd.DataFrame(period_rows)

    if df_period.empty:
        return _empty_balance_drilldown_response(account_name, current_period, previous_period)

    sql_txn = text(f"""
    WITH entities_dedup AS (
        SELECT DISTINCT ON (entity_id)
            entity_id,
            entity_name
        FROM fpna_landing.finance_entities
        WHERE entity_id IS NOT NULL
        ORDER BY entity_id
    )
    SELECT
        gl.period,
        COALESCE(e.entity_name, 'Unknown') AS entity_name,
        COALESCE(CAST(e.entity_id AS TEXT), 'Unknown') AS entity_id,
        coa.account_name,
        gl.transaction_date,
        COALESCE(gl.amount_debit, 0) AS debit_amount,
        COALESCE(gl.amount_credit, 0) AS credit_amount,
        (COALESCE(gl.amount_debit, 0) - COALESCE(gl.amount_credit, 0)) AS net_amount
    FROM fpna_landing.finance_general_ledger_transactions gl
    JOIN fpna_landing.finance_chart_of_accounts coa
        ON gl.gl_account_id = coa.account_id
    LEFT JOIN entities_dedup e
        ON CAST(gl.entity_id AS TEXT) = CAST(e.entity_id AS TEXT)
    WHERE coa.account_name = :account_name
      AND gl.period IN (:current_period, :previous_period)
      {filter_sql_gl}
    ORDER BY gl.period, e.entity_name, gl.transaction_date
    """)

    txn_result = await db.execute(sql_txn, params)
    txn_rows = txn_result.mappings().all()
    df_txn = pd.DataFrame(txn_rows)

    df_current_period = df_period[df_period["period"] == current_period].copy()
    df_previous_period = df_period[df_period["period"] == previous_period].copy()

    current_balance = round(float(df_current_period["ending_balance"].sum() or 0), 2)
    previous_balance = round(float(df_previous_period["ending_balance"].sum() or 0), 2)
    variance = round(current_balance - previous_balance, 2)
    variance_pct = round((variance / previous_balance * 100), 2) if previous_balance else 0

    account_type = df_period["account_type"].iloc[0] if len(df_period) > 0 else "Unknown"

    performance_flow = _calculate_balance_waterfall(
        df_current_period, df_previous_period, current_period, previous_period
    )

    breakdowns = _build_balance_nested_breakdown(
        df_current_period,
        df_previous_period,
        BALANCE_SHEET_LEVEL_CONFIG
    )

    ai_insights = _generate_balance_insights(
        account_name, account_type, current_balance, previous_balance,
        variance, variance_pct, performance_flow, breakdowns
    )

    response = {
        "metric": "balance_sheet",
        "account_name": account_name,
        "account_type": account_type,
        "title": f"{account_name} Analysis",
        "subtitle": f"Why {account_name} {'increased' if variance > 0 else 'decreased'} ({variance_pct:+.1f}% MoM)",
        "summary": {
            "current_period": current_period,
            "current_balance": current_balance,
            "previous_period": previous_period,
            "previous_balance": previous_balance,
            "variance": variance,
            "pct": variance_pct
        },
        "insights": ai_insights,
        "performance_flow": performance_flow,
        "breakdowns": breakdowns
    }

    response = apply_smart_currency(
        response,
        currency,
        currency_filter,
        [
            "current_balance",
            "previous_balance",
            "variance",
            "value",
            "amount",
            "ending_balance",
            "debit_amount",
            "credit_amount",
            "net_amount",
            "debit_total",
            "credit_total",
            "current_balance",
            "previous_balance",
            "change"
        ]
    )

    response = _round_pct_fields(response, decimals=2)

    return response


def _get_previous_period(period: str) -> str:
    try:
        dt = datetime.strptime(period, "%Y-%m")
    except Exception:
        raise HTTPException(status_code=400, detail="period must be in YYYY-MM format")

    year = dt.year
    month = dt.month

    if month == 1:
        return f"{year - 1}-12"
    return f"{year}-{month - 1:02d}"


def _round_pct_fields(obj: Any, decimals: int = 2) -> Any:
    if isinstance(obj, dict):
        new_obj = {}
        for k, v in obj.items():
            if k in ["pct", "change_pct"] and isinstance(v, (int, float)) and v is not None:
                new_obj[k] = round(float(v), decimals)
            else:
                new_obj[k] = _round_pct_fields(v, decimals)
        return new_obj
    if isinstance(obj, list):
        return [_round_pct_fields(x, decimals) for x in obj]
    return obj


def _calculate_balance_waterfall(
    df_current: pd.DataFrame,
    df_previous: pd.DataFrame,
    current_period: str,
    previous_period: str
) -> List[Dict[str, Any]]:

    previous_balance = round(float(df_previous["ending_balance"].sum() or 0), 2)
    current_balance = round(float(df_current["ending_balance"].sum() or 0), 2)

    current_debits = round(float(df_current["debit_total"].sum() or 0), 2)
    current_credits = round(float(df_current["credit_total"].sum() or 0), 2)
    previous_debits = round(float(df_previous["debit_total"].sum() or 0), 2)
    previous_credits = round(float(df_previous["credit_total"].sum() or 0), 2)

    debit_change = round(current_debits - previous_debits, 2)
    credit_change = round(current_credits - previous_credits, 2)

    flow = [
        {
            "label": f"{previous_period} Balance",
            "value": previous_balance,
            "type": "baseline",
            "description": "Starting balance"
        }
    ]

    if abs(debit_change) > 0.01:
        flow.append({
            "label": "Debit Transactions",
            "value": debit_change,
            "type": "positive" if debit_change > 0 else "negative",
            "description": f"Change in debit transactions: {debit_change:+.2f}"
        })

    if abs(credit_change) > 0.01:
        flow.append({
            "label": "Credit Transactions",
            "value": -credit_change,
            "type": "negative" if credit_change > 0 else "positive",
            "description": f"Change in credit transactions: {credit_change:+.2f}"
        })

    calculated_balance = previous_balance + debit_change - credit_change
    residual = round(current_balance - calculated_balance, 2)

    if abs(residual) > 0.01:
        flow.append({
            "label": "Other Adjustments",
            "value": residual,
            "type": "positive" if residual > 0 else "negative",
            "description": "Rounding and adjustments"
        })

    flow.append({
        "label": f"{current_period} Balance",
        "value": current_balance,
        "type": "baseline",
        "description": "Ending balance"
    })

    return flow


def _generate_balance_insights(
    account_name: str,
    account_type: str,
    current_balance: float,
    previous_balance: float,
    variance: float,
    variance_pct: float,
    performance_flow: List[Dict],
    breakdowns: List[Dict]
) -> Dict[str, Any]:

    direction = "increased" if variance > 0 else "decreased"

    main_summary = (
        f"{account_name} {direction} by {round(abs(variance), 2)} "
        f"({round(abs(variance_pct), 1)}%) from {round(previous_balance, 2)} "
        f"to {round(current_balance, 2)}."
    )

    sumary = []
    for item in performance_flow:
        if item["type"] in ["positive", "negative"] and abs(item["value"]) > abs(variance) * 0.1:
            sumary.append({
                "type": item["type"],
                "description": item["label"],
                "amount": round(float(item["value"]), 2),
                "detail": item.get("description", "")
            })

    recommendations = []

    if account_type == "Asset":
        if variance > 0:
            recommendations.append("Asset growth is positive - consider strategic investments")
            recommendations.append("Monitor asset utilization and returns")
        else:
            recommendations.append("Asset decline detected - investigate cash flow or asset utilization")
            recommendations.append("Review recent asset disposals or write-downs")

    elif account_type == "Liability":
        if variance > 0:
            recommendations.append("Liability increased - ensure debt service capacity")
            recommendations.append("Review terms and conditions of new obligations")
        else:
            recommendations.append("Liability reduction is positive - continue debt management strategy")

    elif account_type == "Equity":
        if variance > 0:
            recommendations.append("Equity strengthened - sign of financial health")
        else:
            recommendations.append("Equity decreased - review profitability and distributions")

    if breakdowns:
        top_item = breakdowns[0]
        change_pct = top_item.get("change_pct")
        if change_pct is not None and abs(change_pct) > 10:
            recommendations.append(
                f"Significant movement of {change_pct:+.1f}% in entity '{top_item['entity_name']}' requires attention"
            )

    if len(recommendations) < 3:
        recommendations.append(f"Monitor {account_name} trends for unusual patterns")
        recommendations.append(f"Reconcile {account_name} regularly for accuracy")
        recommendations.append(f"Review supporting documentation for major transactions")

    return {
        "summary": main_summary,
        "sumary": sumary,
        "recommendations": recommendations[:5],
        "key_findings": {
            "account_type": account_type,
            "trend": "increasing" if variance_pct > 0 else "decreasing"
        }
    }


def _empty_balance_drilldown_response(account_name: str, current_period: str, previous_period: str) -> Dict[str, Any]:
    return {
        "metric": "balance_sheet",
        "account_name": account_name,
        "account_type": "Unknown",
        "title": f"{account_name} Analysis",
        "subtitle": "No data available for selected period",
        "summary": {
            "current_period": current_period,
            "current_balance": 0,
            "previous_period": previous_period,
            "previous_balance": 0,
            "variance": 0,
            "pct": 0
        },
        "insights": {
            "summary": "No data available to generate insights.",
            "sumary": [],
            "recommendations": [],
            "key_findings": {
                "account_type": "Unknown",
                "trend": "neutral"
            }
        },
        "performance_flow": [],
        "breakdowns": []
    }
    
# from fastapi import APIRouter, UploadFile, File, HTTPException
# import os
# import io
# import pandas as pd
# import json

# #financial_router = APIRouter()


# UPLOAD_FOLDER = "uploads"
# os.makedirs(UPLOAD_FOLDER, exist_ok=True)

# FILE_XLSX = os.path.join(UPLOAD_FOLDER, "financial_data.xlsx")
# FILE_CSV = os.path.join(UPLOAD_FOLDER, "financial_data.csv")
# LAST_UPLOADED_TYPE = os.path.join(UPLOAD_FOLDER, "last_type.txt")


# @financial_router.post("/upload-gl-file")
# async def upload_gl_file(file: UploadFile = File(...)):
#     if not file.filename.endswith((".csv", ".xlsx")):
#         raise HTTPException(status_code=400, detail="Only .csv and .xlsx files are allowed")

#     await file.seek(0)
#     contents = await file.read()

#     if len(contents) == 0:
#         raise HTTPException(status_code=400, detail="Uploaded file is empty")

#     is_xlsx = file.filename.endswith(".xlsx")
#     path = FILE_XLSX if is_xlsx else FILE_CSV

#     with open(path, "wb") as f:
#         f.write(contents)

#     with open(LAST_UPLOADED_TYPE, "w") as f:
#         f.write("xlsx" if is_xlsx else "csv")

#     try:
#         if is_xlsx:
#             pd.read_excel(io.BytesIO(contents), engine="openpyxl", nrows=1)
#         else:
#             pd.read_csv(io.BytesIO(contents), nrows=1)
#     except Exception as e:
#         if os.path.exists(path):
#             os.remove(path)
#         raise HTTPException(status_code=400, detail=f"File is not valid: {str(e)}")

#     return {
#         "message": "GL file uploaded successfully",
#         "file_type": "xlsx" if is_xlsx else "csv",
#         "file_path": path,
#         "file_size_kb": round(len(contents) / 1024, 2)
#     }


# @financial_router.get("/customer-profitability2/{year}/{currency}")
# async def customer_profitability2(year: int, currency: str):
#     if not os.path.exists(LAST_UPLOADED_TYPE):
#         raise HTTPException(status_code=400, detail="No GL file uploaded yet")

#     with open(LAST_UPLOADED_TYPE, "r") as f:
#         last_type = f.read().strip()

#     try:
#         if last_type == "xlsx" and os.path.exists(FILE_XLSX):
#             df = pd.read_excel(FILE_XLSX, engine="openpyxl")
#         elif last_type == "csv" and os.path.exists(FILE_CSV):
#             df = pd.read_csv(FILE_CSV)
#         else:
#             raise HTTPException(status_code=400, detail="Uploaded file not found on disk")
#     except HTTPException:
#         raise
#     except Exception as e:
#         raise HTTPException(status_code=500, detail=f"Failed to read file: {str(e)}")

#     # Normalize column names: convert to lowercase and strip whitespace
#     df.columns = df.columns.str.strip().str.lower().str.replace(' ', '_')
    
#     required_cols = [
#         "transaction_id",
#         "gl_account_id",
#         "transaction_date",
#         "amount_debit",
#         "amount_credit",
#         "currency",
#         "department_id",
#         "cost_center_id",
#         "profit_center_id",
#         "entity_id",
#         "customer_id",
#         "vendor_id",
#         "period"
#     ]

#     missing = [col for col in required_cols if col not in df.columns]
#     if missing:
#         raise HTTPException(status_code=400, detail=f"Missing columns: {missing}")

#     df = df[df["currency"] == currency]
#     df["period"] = df["period"].astype(str)
#     df = df[df["period"].str.startswith(str(year))]

#     if df.empty:
#         return {"message": "No data found"}

#     df["revenue"] = df["amount_credit"] - df["amount_debit"]
#     df_grouped = df.groupby(
#         ["customer_id", "entity_id", "department_id", "period"],
#         as_index=False
#     )["revenue"].sum()

#     total_revenue = df_grouped["revenue"].sum()
#     if total_revenue == 0:
#         return {"message": "Total revenue is zero"}

#     total_cost = total_revenue * 0.6
#     df_grouped["allocated_cost"] = (df_grouped["revenue"] / total_revenue) * total_cost
#     df_grouped["profit"] = df_grouped["revenue"] - df_grouped["allocated_cost"]
    
#     # Fix: Handle division by zero and replace NaN values with 0
#     df_grouped["margin_pct"] = df_grouped.apply(
#         lambda row: (row["profit"] / row["revenue"] * 100) if row["revenue"] != 0 else 0.0,
#         axis=1
#     )
#     df_grouped["rank"] = df_grouped["profit"].rank(method="dense", ascending=False)
    
#     # Replace any remaining NaN or inf values
#     df_grouped = df_grouped.replace([float('inf'), float('-inf')], 0.0)
#     df_grouped = df_grouped.fillna(0.0)
    
#     # Convert to dict and ensure all values are JSON serializable
#     records = []
#     for _, row in df_grouped.iterrows():
#         record = row.to_dict()
#         # Ensure all numeric values are proper Python types
#         for key, value in record.items():
#             if pd.isna(value) or (isinstance(value, float) and not (-1e308 < value < 1e308)):
#                 record[key] = 0.0
#             elif isinstance(value, (pd.Int64Dtype, pd.Float64Dtype)):
#                 record[key] = float(value) if pd.notna(value) else 0.0
#         records.append(record)

#     return {
#         "year": year,
#         "currency": currency,
#         "records": records
#     }