/*
 * SPDX-FileCopyrightText: Copyright (c) 2013-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */


/*!
 * @file
 * @brief  Provides KERNEL only functions for OBJBIF
 */

/* ------------------------ Includes ---------------------------------------- */
#include "gpu/bif/kernel_bif.h"
#include "gpu/bus/kern_bus.h"
#include "gpu/gpu.h"
#include "gpu/intr/intr.h"
#include "os/os.h"
#include "platform/chipset/chipset.h"
#include "core/locks.h"
#include "nvrm_registry.h"
#include "diagnostics/tracer.h"
#include "nvpcie.h"
#include "lib/base_utils.h"
#include "gpu/device/device.h"
#include "gpu/subdevice/subdevice.h"

/* ------------------------ Macros ------------------------------------------ */
/* ------------------------ Compile Time Checks ----------------------------- */
/* ------------------------ Static Function Prototypes ---------------------- */
static void       _kbifInitRegistryOverrides(OBJGPU *, KernelBif *);
static void       _kbifCheckIfGpuExists(OBJGPU *, void*);
static NV_STATUS  _kbifSetPcieRelaxedOrdering(OBJGPU *, KernelBif *, NvBool);
static void       _kbifCreateChipsetPayloadStr(OBJGPU *pGpu, KernelBif *pKernelBif, char *pChipsetPayloadStr);
static void       _kbifCreateChipsetGpuPayloadStr(OBJGPU *pGpu, KernelBif *pKernelBif, char *pChipsetPayloadStr, char *pChipsetGpuPayloadStr);
static NV_STATUS  _kbifParsePciePowerControl(OBJGPU *pGpu, KernelBif *pKernelBif, const char *pChipsetPayloadStr, const char *pChipsetGpuPayloadStr, NvU32 *pPciePowerControlValue);

static NV_STATUS _kbifNbsiReadRTD3RegistryDword(OBJGPU *pGpu, KernelBif *pKernelBif, const char *pRegParmStr,NvU32 *pData);

extern const GUID NV_GUID_UEFI_VARIABLE;
/* ------------------------ Public Functions -------------------------------- */

/*!
 * @brief KernelBif Constructor
 *
 * @param[in] pGpu        GPU object pointer
 * @param[in] pKernelBif  BIF object pointer
 * @param[in] engDesc     Engine descriptor
 */
NV_STATUS
kbifConstructEngine_IMPL
(
    OBJGPU        *pGpu,
    KernelBif     *pKernelBif,
    ENGDESCRIPTOR  engDesc
)
{
    // Initialize registry overrides
    _kbifInitRegistryOverrides(pGpu, pKernelBif);

    // Disables P2P on VF
    kbifDisableP2PTransactions_HAL(pGpu, pKernelBif);

    // Cache MNOC interface support
    kbifCacheMnocSupport_HAL(pGpu, pKernelBif);

    // Cache FLR support
    kbifCacheFlrSupport_HAL(pGpu, pKernelBif);

    // Cache 64B BAR0 support
    kbifCache64bBar0Support_HAL(pGpu, pKernelBif);

    // Cache VF info
    kbifCacheVFInfo_HAL(pGpu, pKernelBif);

    // Cache GPU link capabilities
    kbifGetGpuLinkCapabilities(pGpu, pKernelBif);

    // Construct XTL aperture
    kbifConstructXtlAperture_HAL(pGpu, pKernelBif);

    // Used to track when the link has gone into Recovery, which can cause CEs.
    pKernelBif->EnteredRecoverySinceErrorsLastChecked = NV_FALSE;

    // Default scale is 1 and could be overriden by registry
    pKernelBif->flrDevInitTimeoutScale = 1;

    return NV_OK;
}

/*!
 * @brief KernelBif Constructor
 *
 * @param[in] pGpu        GPU object pointer
 * @param[in] pKernelBif  BIF object pointer
 */
NV_STATUS
kbifStateInitLocked_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    OBJSYS    *pSys   = SYS_GET_INSTANCE();
    OBJCL     *pCl    = SYS_GET_CL(pSys);

    kbifInitXveRegMap_HAL(pGpu, pKernelBif, 1);
    kbifInitXveRegMap_HAL(pGpu, pKernelBif, 0);

    // Do the HAL dependent init
    NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, kbifInit_HAL(pGpu, pKernelBif));

    // Return early if GPU is connected to an unsupported chipset
    if (pCl->getProperty(pCl, PDB_PROP_CL_UNSUPPORTED_CHIPSET))
    {
        return NV_ERR_NOT_COMPATIBLE;
    }

    // Initialize OS mapping and core logic
    NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, osInitMapping(pGpu));

    // Initialize BIF static info
    kbifStaticInfoInit(pGpu, pKernelBif);

    // Initialize DMA caps
    kbifInitDmaCaps(pGpu, pKernelBif);

    // Check for OS w/o usable PAT support
    if ((gpuGetBusIntfType_HAL(pGpu) == NV2080_CTRL_BUS_INFO_TYPE_PCI_EXPRESS) &&
        pSys->getProperty(pOS, PDB_PROP_SYS_PAT_UNSUPPORTED))
    {
        NV_PRINTF(LEVEL_INFO,
                  "BIF disabling noncoherent on OS w/o usable PAT support\n");

        pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_SUPPORT_NONCOHERENT, NV_FALSE);
    }

    return NV_OK;
}

/*!
 * @brief KernelBif state load
 *
 * @param[in] pGpu        GPU object pointer
 * @param[in] pKernelBif  BIF object pointer
 * @param[in] flags       GPU state flag
 */
NV_STATUS
kbifStateLoad_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif,
    NvU32      flags
)
{
    NV_PRINTF(LEVEL_INFO, "BIF DMA Caps: %08x\n", kbifGetDmaCaps(pGpu, pKernelBif));

    // Apply C73 chipset WAR
    kbifExecC73War_HAL(pGpu, pKernelBif);

    // Check for stale PCI-E dev ctrl/status errors and AER errors
    kbifClearConfigErrors(pGpu, pKernelBif, NV_TRUE, KBIF_CLEAR_XVE_AER_ALL_MASK);

    //  Cache PCI config registers to be restored during resume
    if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_PM_RESUME_CODEPATH))
    {
        kbifSavePcieConfigRegisters_HAL(pGpu, pKernelBif);
    }

    //
    // A vGPU cannot disappear and these accesses are
    // particularly expensive on vGPUs
    //
    if (pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_CHECK_IF_GPU_EXISTS_DEF) &&
        !IS_VIRTUAL(pGpu))
    {
        osSchedule1HzCallback(pGpu, _kbifCheckIfGpuExists, NULL, NV_OS_1HZ_REPEAT);
    }

    return NV_OK;
}

/*!
 * @brief Helper function to create chipset payload string
 *
 * @param[in]   pGpu               OBJGPU pointer
 * @param[in]   pKernelBif         Kernel BIF Object pointer
 * @param[out]  pChipsetPayloadStr Holds the complete PCIePowerControl chipset payload string
 *                                 (PCIePowerControl_VVVVDDDDddddvvvv)
 */
static void
_kbifCreateChipsetPayloadStr
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif,
    char      *pChipsetPayloadStr
)
{
    char    chipsetFourPartIdStr[17];
    NvU32   offsetFromBase     = 0;
    NvU64   chipsetFourpartIds = 0;
    OBJSYS *pSys               = SYS_GET_INSTANCE();
    OBJCL  *pCl                = SYS_GET_CL(pSys);

    chipsetFourpartIds = (NvU64)  pCl->FHBBusInfo.vendorID << 48 |
                         (NvU64)  pCl->FHBBusInfo.deviceID << 32 |
                         (NvU64)  pCl->FHBBusInfo.subdeviceID << 16 |
                         (NvU64)  pCl->FHBBusInfo.subvendorID;

    // Convert the chipsetIds to string format
    nvU64ToStr(chipsetFourpartIds, chipsetFourPartIdStr, 16);

    portMemSet(pChipsetPayloadStr, '\0', NV_REG_STR_PCIPOWERCONTROL_CHIPSET_LENGTH);

    // Copy the PCIePowerControl suffix
    portMemCopy(pChipsetPayloadStr,
                nvStringLen(NV_REG_STR_ENABLE_PCIPOWERCONTROL_WITHOUT_SUFFIX),
                NV_REG_STR_ENABLE_PCIPOWERCONTROL_WITHOUT_SUFFIX,
                nvStringLen(NV_REG_STR_ENABLE_PCIPOWERCONTROL_WITHOUT_SUFFIX));
    offsetFromBase = nvStringLen(NV_REG_STR_ENABLE_PCIPOWERCONTROL_WITHOUT_SUFFIX);

    portMemCopy(pChipsetPayloadStr + offsetFromBase, 1, "_", 1);
    offsetFromBase += 1;

    // Copy the chipset four part Ids
    portMemCopy(pChipsetPayloadStr + offsetFromBase,
                nvStringLen(chipsetFourPartIdStr),
                chipsetFourPartIdStr,
                nvStringLen(chipsetFourPartIdStr));
}

/*!
 * @brief Helper function to create chipset+GPU payload string.
 *
 * @param[in]   pGpu                  OBJGPU pointer
 * @param[in]   pKernelBif            Kernel BIF Object pointer
 * @param[in]   pChipsetPayloadStr    Chipset payload string
 * @param[out]  pChipsetGpuPayloadStr Holds the complete PCIePowerControl chipset+GPU payload string
 *                                    (PCIePowerControl_VVVVDDDDddddvvvv_VVVVDDDDddddvvvv)
 */
static void
_kbifCreateChipsetGpuPayloadStr
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif,
    char      *pChipsetPayloadStr,
    char      *pChipsetGpuPayloadStr
)
{
    char    gpuFourPartIdStr[17];
    NvU32   offsetFromBase = 0;
    NvU64   gpuFourPartIds = 0;

    gpuFourPartIds = (((NvU64) pGpu->idInfo.PCIDeviceID & 0xffff) << 48) |
                     ((((NvU64) pGpu->idInfo.PCIDeviceID >> 16) & 0xffff) << 32) |
                     ((((NvU64) pGpu->idInfo.PCISubDeviceID >> 16) & 0xffff) << 16) |
                     ((NvU64) pGpu->idInfo.PCISubDeviceID & 0xffff);

    // Convert the gpu four part ids to string format
    nvU64ToStr(gpuFourPartIds, gpuFourPartIdStr, 16);

    portMemSet(pChipsetGpuPayloadStr, '\0', NV_REG_STR_PCIPOWERCONTROL_CHIPSET_GPU_LENGTH);

    // Copy the chipset payload string
    portMemCopy(pChipsetGpuPayloadStr,
                nvStringLen(pChipsetPayloadStr),
                pChipsetPayloadStr,
                nvStringLen(pChipsetPayloadStr));
    offsetFromBase = nvStringLen(pChipsetPayloadStr);

    portMemCopy(pChipsetGpuPayloadStr + offsetFromBase, 1, "_", 1);
    offsetFromBase += 1;

    // Copy the gpu four part Ids
    portMemCopy(pChipsetGpuPayloadStr + offsetFromBase,
                nvStringLen(gpuFourPartIdStr),
                gpuFourPartIdStr,
                nvStringLen(gpuFourPartIdStr));
}

/*!
 * @brief Helper function to parse RTD3 DR Key
 *
 * @param[in]   pGpu               OBJGPU pointer
 * @param[in]   pKernelBif         KernelBif Object pointer
 * @param[in]   pRegParmStr        PCIePowerControl  payload string
 * @param[out]  pData              DR key value
 *
 * @return the NV_OK if DR key was found and data returned in pData
 */
static NV_STATUS
_kbifNbsiReadRTD3RegistryDword
(
    OBJGPU     *pGpu,
    KernelBif  *pKernelBif,
    const char *pRegParmStr,
    NvU32      *pData
)
{
    NBSI_OBJ  *pNbsiObj = getNbsiObject();
    NvU32      nbsiDword;
    NvU16      pathHash;
    NvU8      *pRetBuf;
    NvU32      retSize;
    NvU32      errorCode;
    NvU16      module   = NV2080_CTRL_BIOS_NBSI_MODULE_DISPLAYDRIVER;
    NV_STATUS  status;
    NvU32      elementHashArray[MAX_NBSI_OS];
    NvU8       maxOSndx;

    NV_ASSERT_OR_RETURN(pRegParmStr != NULL, NV_ERR_INVALID_ARGUMENT);
    NV_ASSERT_OR_RETURN(pData != NULL, NV_ERR_INVALID_ARGUMENT);

    if ((pGpu == NULL) || (pNbsiObj == NULL))
    {
        return NV_ERR_INVALID_OBJECT;
    }

    // Check to make sure we are not in the power management code path
    if ((pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_PM_CODEPATH))
        && (!pGpu->getProperty(pGpu, PDB_PROP_GPU_DO_NOT_CHECK_REG_ACCESS_IN_PM_CODEPATH))
       )
    {
        NV_PRINTF(LEVEL_ERROR,
                  "osReadRegistryDword called in Sleep path can cause excessive delays!\n");

        NV_ASSERT(0);
        return NV_ERR_INVALID_REQUEST;
    }

    pathHash = pNbsiObj->nbsiBlankPathHash;
    maxOSndx = pNbsiObj->curMaxNbsiOSes;

    // The RM module uses a blank (relative path) so just do element.
    fnv1Hash20Array((const NvU8 *) pRegParmStr,
                    nvStringLen(pRegParmStr),
                    elementHashArray,
                    maxOSndx);

    retSize = sizeof(nbsiDword);
    pRetBuf = (NvU8 *) &nbsiDword;

    //
    // Read from both NBSI table and os reg tables... then decide what
    // to return.
    //
    status = getNbsiValue(pGpu,
                          module,
                          pathHash,
                          maxOSndx,
                          elementHashArray,
                          pRetBuf,
                          &retSize,
                          &errorCode);
    if (status == NV_OK)
    {
        if (errorCode == NV2080_CTRL_BIOS_GET_NBSI_SUCCESS)
        {
            *pData = nbsiDword;
        }
        else
        {
            //
            // assert if we didn't have enough room to hold the returned data.
            // Since we define the return as a dword here, this could only be
            // because the registry has the key but it's defined longer than a
            // dword. i.e. byte array or extended byte array.
            //
            NV_ASSERT(errorCode != NV2080_CTRL_BIOS_GET_NBSI_INCOMPLETE);
            status = NV_ERR_GENERIC;
        }
    }

    return status;
}

/*!
 * @brief      Helper function to parse unified PCIE power control based on below priority order
 *
 *  1. HKR Key Dual MB+GPU
 *  2. NVGlobal Key Dual MB+GPU
 *  3. UEFI variable Dual MB+GPU
 *  4. DR Key Dual MB+GPU
 *  5. HKR Key MB
 *  6. NVGlobal Key MB
 *  7. UEFI variable MB
 *  8. DR Key MB
 *  9. HKR Key wildcard
 * 10. NVGlobal Key wildcard
 * 11. UEFI variable wildcard
 * 12. DR Key wildcard
 * 13. SBIOS cookie - legacy mechanism (should not be used anymore, here for completeness
 *
 * @param[in]  pGpu                   OBJGPU pointer
 * @param[in]  pKernelBif             KernelBif Object pointer
 * @param[in]  pChipsetPayloadStr     Chipset payload string
 * @param[in]  pChipsetGpuPayloadStr  Chipset+GPU payload string
 * @param[out] pPciePowerControlValue pciePowerControlValue pointer
 *
 * @return     NV_OK on success, otherwise an error code.
 */
static NV_STATUS
_kbifParsePciePowerControl
(
    OBJGPU     *pGpu,
    KernelBif  *pKernelBif,
    const char *pChipsetPayloadStr,
    const char *pChipsetGpuPayloadStr,
    NvU32      *pPciePowerControlValue
)
{
    NV_STATUS   status      = NV_OK;
    NvU32       uefiSize    = sizeof(NvU32);
    NvU32       cookieSize  = sizeof(NvU8);
    NvU8        aspmCookie  = 0;
    const char *wildcardVar = NV_REG_STR_ENABLE_PCIPOWERCONTROL_WITHOUT_SUFFIX;

    // Read the HKR Key with Chipset + GPU 4-part ID
    if (osReadRegistryDword(pGpu, pChipsetGpuPayloadStr,
                            pPciePowerControlValue) == NV_OK)
    {
        pKernelBif->pciePowerControlInfo.identifiedKeyOrder    = PCIEPOWERCONTROL_KEY_ORDER_CHIPSET_GPU_ID;
        pKernelBif->pciePowerControlInfo.identifiedKeyLocation = PCIEPOWERCONTROL_KEY_LOCATION_ADAPTER;
    }
    // Read the NvGlobal Key with Chipset + GPU 4-part ID
    else if (osGetNvGlobalRegistryDword(pGpu, pChipsetGpuPayloadStr,
                                        pPciePowerControlValue) == NV_OK)
    {
        pKernelBif->pciePowerControlInfo.identifiedKeyOrder    = PCIEPOWERCONTROL_KEY_ORDER_CHIPSET_GPU_ID;
        pKernelBif->pciePowerControlInfo.identifiedKeyLocation = PCIEPOWERCONTROL_KEY_LOCATION_GLOBAL;
    }
    // Read the UEFI variable with Chipset + GPU 4-part ID
    else if (osGetUefiVariable(pChipsetGpuPayloadStr,
                               (LPGUID)&NV_GUID_UEFI_VARIABLE,
                               (NvU8 *)pPciePowerControlValue, &uefiSize) == NV_OK)
    {
        pKernelBif->pciePowerControlInfo.identifiedKeyOrder    = PCIEPOWERCONTROL_KEY_ORDER_CHIPSET_GPU_ID;
        pKernelBif->pciePowerControlInfo.identifiedKeyLocation = PCIEPOWERCONTROL_KEY_LOCATION_UEFI;
    }
    // Read the DR Key with Chipset + GPU 4-part ID
    else if (_kbifNbsiReadRTD3RegistryDword(pGpu, pKernelBif, pChipsetGpuPayloadStr,
                                            pPciePowerControlValue) == NV_OK)
    {
        pKernelBif->pciePowerControlInfo.identifiedKeyOrder    = PCIEPOWERCONTROL_KEY_ORDER_CHIPSET_GPU_ID;
        pKernelBif->pciePowerControlInfo.identifiedKeyLocation = PCIEPOWERCONTROL_KEY_LOCATION_DR;
    }
    // Read the HKR Key with Chipset 4-part ID
    else if (osReadRegistryDword(pGpu, pChipsetPayloadStr,
                                 pPciePowerControlValue) == NV_OK)
    {
        pKernelBif->pciePowerControlInfo.identifiedKeyOrder    = PCIEPOWERCONTROL_KEY_ORDER_CHIPSET_ID;
        pKernelBif->pciePowerControlInfo.identifiedKeyLocation = PCIEPOWERCONTROL_KEY_LOCATION_ADAPTER;
    }
    // Read the NvGlobal Key with Chipset 4-part ID
    else if (osGetNvGlobalRegistryDword(pGpu,pChipsetPayloadStr,
                                        pPciePowerControlValue) == NV_OK)
    {
        pKernelBif->pciePowerControlInfo.identifiedKeyOrder    = PCIEPOWERCONTROL_KEY_ORDER_CHIPSET_ID;
        pKernelBif->pciePowerControlInfo.identifiedKeyLocation = PCIEPOWERCONTROL_KEY_LOCATION_GLOBAL;
    }
    // Read the UEFI variable with Chipset 4-part ID
    else if (osGetUefiVariable(pChipsetPayloadStr,
                               (LPGUID)&NV_GUID_UEFI_VARIABLE,
                               (NvU8 *)pPciePowerControlValue, &uefiSize) == NV_OK)
    {
        pKernelBif->pciePowerControlInfo.identifiedKeyOrder    = PCIEPOWERCONTROL_KEY_ORDER_CHIPSET_ID;
        pKernelBif->pciePowerControlInfo.identifiedKeyLocation = PCIEPOWERCONTROL_KEY_LOCATION_UEFI;
    }
    // Read the DR Key with Chipset 4-part ID
    else if (_kbifNbsiReadRTD3RegistryDword(pGpu, pKernelBif, pChipsetPayloadStr,
                                            pPciePowerControlValue) == NV_OK)
    {
        pKernelBif->pciePowerControlInfo.identifiedKeyOrder    = PCIEPOWERCONTROL_KEY_ORDER_CHIPSET_ID;
        pKernelBif->pciePowerControlInfo.identifiedKeyLocation = PCIEPOWERCONTROL_KEY_LOCATION_DR;
    }
    // Read the wildcard HKR Key
    else if (osReadRegistryDword(pGpu, wildcardVar,
                                 pPciePowerControlValue) == NV_OK)
    {
        pKernelBif->pciePowerControlInfo.identifiedKeyOrder    = PCIEPOWERCONTROL_KEY_ORDER_WILDCARD;
        pKernelBif->pciePowerControlInfo.identifiedKeyLocation = PCIEPOWERCONTROL_KEY_LOCATION_ADAPTER;
    }
    // Read the wildcard NvGlobal Key
    else if (osGetNvGlobalRegistryDword(pGpu, wildcardVar,
                                        pPciePowerControlValue) == NV_OK)
    {
        pKernelBif->pciePowerControlInfo.identifiedKeyOrder    = PCIEPOWERCONTROL_KEY_ORDER_WILDCARD;
        pKernelBif->pciePowerControlInfo.identifiedKeyLocation = PCIEPOWERCONTROL_KEY_LOCATION_GLOBAL;
    }
    // Read the wildcard UEFI variable
    else if (osGetUefiVariable(wildcardVar,
                               (LPGUID)&NV_GUID_UEFI_VARIABLE,
                               (NvU8 *)pPciePowerControlValue, &uefiSize) == NV_OK)
    {
        pKernelBif->pciePowerControlInfo.identifiedKeyOrder    = PCIEPOWERCONTROL_KEY_ORDER_WILDCARD;
        pKernelBif->pciePowerControlInfo.identifiedKeyLocation = PCIEPOWERCONTROL_KEY_LOCATION_UEFI;
    }
    // Read the wildcard DR Key
    else if (_kbifNbsiReadRTD3RegistryDword(pGpu, pKernelBif, wildcardVar,
                                            pPciePowerControlValue) == NV_OK)
    {
        pKernelBif->pciePowerControlInfo.identifiedKeyOrder    = PCIEPOWERCONTROL_KEY_ORDER_WILDCARD;
        pKernelBif->pciePowerControlInfo.identifiedKeyLocation = PCIEPOWERCONTROL_KEY_LOCATION_DR;
    }
    // SBIOS cookie legacy mechanisms support
    else if (nbsiReadRegistryString(pGpu, NV_REG_STR_RM_SBIOS_ENABLE_ASPM_DT,
                                    &aspmCookie, &cookieSize) == NV_OK)
    {
        pKernelBif->pciePowerControlInfo.identifiedKeyOrder    = PCIEPOWERCONTROL_KEY_ORDER_WILDCARD;
        pKernelBif->pciePowerControlInfo.identifiedKeyLocation = PCIEPOWERCONTROL_KEY_LOCATION_COOKIE;

        // Left shift by 4 to match the bit field mapping of pciePowerControlValue
        aspmCookie = (aspmCookie << 4);
       *pPciePowerControlValue = (NvU32) aspmCookie;
    }
    else
    {
        status = NV_ERR_NOT_SUPPORTED;
    }

    return status;
}

/*!
 * @brief Cache PCIe Power Control variables
 *
 * @param[in] pGpu                  GPU object pointer
 * @param[in] pKernelBif            Kernel BIF object pointer
 * @param[in] pPciePowerControlValue Pcie Power Control mask
 * 
 * return     NV_OK                 on successfully caching UEFI variables
 */
NV_STATUS
kbifGetPciePowerControlValue_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif,
    NvU32     *pPciePowerControlValue
)
{
    char       chipsetPayloadStr[NV_REG_STR_PCIPOWERCONTROL_CHIPSET_LENGTH];
    char       chipsetGpuPayloadStr[NV_REG_STR_PCIPOWERCONTROL_CHIPSET_GPU_LENGTH];
    NV_STATUS  status = NV_OK;

    // Create chipset payload string (PCIePowerControl_VVVVDDDDddddvvvv)
    _kbifCreateChipsetPayloadStr(pGpu, pKernelBif, chipsetPayloadStr);

    // Create chipset+GPU payload string (PCIePowerControl_VVVVDDDDddddvvvv_VVVVDDDDddddvvvv)
    _kbifCreateChipsetGpuPayloadStr(pGpu, pKernelBif, chipsetPayloadStr, chipsetGpuPayloadStr);

    // Parse PCIePowerControl mask
    status = _kbifParsePciePowerControl(pGpu, pKernelBif,
                                        chipsetPayloadStr,
                                        chipsetGpuPayloadStr,
                                        pPciePowerControlValue);

    return status;
}

/*!
 * @brief: Control call to get RTD3+ASPM override mask
 *
 * @return
 *   NV_OK on Success
 */
NV_STATUS
deviceCtrlCmdBifGetPciePowerControlMask_IMPL
(
    Device *pDevice,
    NV0080_CTRL_CMD_BIF_GET_PCIE_POWER_CONTROL_MASK_PARAMS *pBifPciePowerControlParams
)
{
    NV_STATUS  status               = NV_OK;
    OBJGPU    *pGpu                 = GPU_RES_GET_GPU(pDevice);
    KernelBif *pKernelBif           = GPU_GET_KERNEL_BIF(pGpu);

    pBifPciePowerControlParams->pciePowerControlMask                  = pKernelBif->pciePowerControlInfo.pciePowerControlValue;
    pBifPciePowerControlParams->pciePowerControlIdentifiedKeyOrder    = pKernelBif->pciePowerControlInfo.identifiedKeyOrder;
    pBifPciePowerControlParams->pciePowerControlIdentifiedKeyLocation = pKernelBif->pciePowerControlInfo.identifiedKeyLocation;

    return status;
}

/*!
 * @brief Configure PCIe Relaxed Ordering in BIF
 *
 * @param[in] pGpu        GPU object pointer
 * @param[in] pKernelBif  KBIF object pointer
 * @param[in] enableRo    Enable/disable RO
 */
static NV_STATUS
_kbifSetPcieRelaxedOrdering
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif,
    NvBool    enableRo
)
{
    NV2080_CTRL_INTERNAL_BIF_SET_PCIE_RO_PARAMS pcieRo;
    RM_API    *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
    NV_STATUS  status;

    pcieRo.enableRo = enableRo;

    status = pRmApi->Control(pRmApi, pGpu->hInternalClient, pGpu->hInternalSubdevice,
                             NV2080_CTRL_CMD_INTERNAL_BIF_SET_PCIE_RO,
                             &pcieRo, sizeof(pcieRo));
    if (status != NV_OK) {
        NV_PRINTF(LEVEL_ERROR, "NV2080_CTRL_CMD_INTERNAL_BIF_SET_PCIE_RO failed %s (0x%x)\n",
                  nvstatusToString(status), status);
        return status;
    }

    return NV_OK;
}

/*!
 * @brief KernelBif state post-load
 *
 * @param[in] pGpu        GPU object pointer
 * @param[in] pKernelBif  KBIF object pointer
 * @param[in] flags       GPU state flag
 */
NV_STATUS
kbifStatePostLoad_IMPL
(
    OBJGPU      *pGpu,
    KernelBif   *pKernelBif,
    NvU32       flags
)
{
    NV_STATUS status;

    kbifInitRelaxedOrderingFromEmulatedConfigSpace(pGpu, pKernelBif);
    if (pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_PCIE_RELAXED_ORDERING_SET_IN_EMULATED_CONFIG_SPACE)) {
        //
        // This is done from StatePostLoad() to guarantee that BIF's StateLoad()
        // is already completed for both monolithic RM and GSP RM.
        //
        status = _kbifSetPcieRelaxedOrdering(pGpu, pKernelBif, NV_TRUE);
        if (status != NV_OK)
            return NV_OK;
    }

    return NV_OK;
}

/*!
 * @brief KernelBif state unload
 *
 * @param[in] pGpu        GPU object pointer
 * @param[in] pKernelBif  BIF object pointer
 * @param[in] flags       GPU state flag
 */
NV_STATUS
kbifStateUnload_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif,
    NvU32      flags
)
{

    return NV_OK;
}


/*!
 * @brief Cache L1substates info from chipset or bridge behind the GPUs.
 *
 * @param[in]  pGpu       GPU object pointer
 * @param[in]  pKernelBif Kernel BIF object pointer
 */
void
kbifCacheChipsetL1SubstatesEnable_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    OBJSYS                  *pSys = SYS_GET_INSTANCE();
    OBJCL                   *pCl  = SYS_GET_CL(pSys);
    PexL1SubstateCapability  L1SsCap;
    NvBool                   bChipsetPcipmL12Enabled;
    NvBool                   bChipsetPcipmL11Enabled;
    NvBool                   bChipsetAspmL12Enabled;
    NvBool                   bChipsetAspmL11Enabled;

    if (pGpu->gpuClData.upstreamPort.addr.valid)
    {
        if (pCl && (clPcieReadL1SsCapability(pGpu, pCl, &L1SsCap) == NV_OK))
        {
            pKernelBif->chipsetL1ssEnable = L1SsCap.Control1Reg;

            bChipsetPcipmL12Enabled = FLD_TEST_DRF(2080, _CTRL_BUS_INFO_PCIE_L1_SS_CTRL1, _PCIPM_L1_2_ENABLED, _YES, L1SsCap.Control1Reg);
            bChipsetPcipmL11Enabled = FLD_TEST_DRF(2080, _CTRL_BUS_INFO_PCIE_L1_SS_CTRL1, _PCIPM_L1_1_ENABLED, _YES, L1SsCap.Control1Reg);
            bChipsetAspmL12Enabled  = FLD_TEST_DRF(2080, _CTRL_BUS_INFO_PCIE_L1_SS_CTRL1, _ASPM_L1_2_ENABLED,  _YES, L1SsCap.Control1Reg);
            bChipsetAspmL11Enabled  = FLD_TEST_DRF(2080, _CTRL_BUS_INFO_PCIE_L1_SS_CTRL1, _ASPM_L1_1_ENABLED,  _YES, L1SsCap.Control1Reg);

            //
            // L1 PM substates can be done with ASPM or PCI-PM
            // Its supported if any of L1.1/L1.2 PCI-PM/ASPM susbtates is enabled
            //
            if (bChipsetPcipmL12Enabled || bChipsetPcipmL11Enabled || bChipsetAspmL12Enabled || bChipsetAspmL11Enabled)
            {
                // Chipset Supports L1 PM Substates
                NV_PRINTF(LEVEL_INFO,
                          "Chipset supports L1 PM substates.\n      L1 PM Capabilities Register 0x%x\n      L1 PM Control 1 Register 0x%x\n      L1 PM Control 2 Register 0x%x\n",
                          L1SsCap.Capabilities, L1SsCap.Control1Reg, L1SsCap.Control1Reg);
            }
            else
            {
                NV_PRINTF(LEVEL_INFO,
                          "L1 PM susbstates is not enabled in RootPort. L1 PM Control 1 Register 0x%x \n",
                          L1SsCap.Control1Reg);
            }
        }
        else
        {
            NV_PRINTF(LEVEL_INFO, "Failed to get L1 PM substates capabilites from Root Port\n");
        }
    }
}

/*!
 * @brief Initialize DMA caps
 *
 * @param[in] pGpu        GPU object pointer
 * @param[in] pKernelBif  BIF object pointer
 */
void
kbifInitDmaCaps_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    OBJSYS *pSys = SYS_GET_INSTANCE();
    OBJCL  *pCl  = SYS_GET_CL(pSys);

    pKernelBif->dmaCaps = REF_DEF(BIF_DMA_CAPS_NOSNOOP, _CTXDMA);

    // Set the coherency cap on host RM based on the chipset
    if (IsAMODEL(pGpu) ||
        pCl->getProperty(pCl, PDB_PROP_CL_IS_CHIPSET_IO_COHERENT))
    {
        pKernelBif->dmaCaps |= REF_DEF(BIF_DMA_CAPS_SNOOP, _CTXDMA);
    }
}

NvU32
kbifGetDmaCaps_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    NvU32 retval;

    // Start out with system specific DMA caps
    retval = pKernelBif->dmaCaps;

    // If noncoherent support is disabled, mask out SNOOP caps
    if (!pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_SUPPORT_NONCOHERENT))
    {
        retval &= ~DRF_SHIFTMASK(BIF_DMA_CAPS_NOSNOOP);
    }

    return retval;
}

/*!
 * @brief Initialize BIF static info in Kernel object through RPC
 *
 * @param[in] pGpu        GPU object pointer
 * @param[in] pKernelBif  BIF object pointer
 */
NV_STATUS
kbifStaticInfoInit_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    NV2080_CTRL_INTERNAL_BIF_GET_STATIC_INFO_PARAMS *pStaticInfo;
    RM_API    *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
    NV_STATUS  status = NV_OK;

    // Allocate memory for the command parameter
    pStaticInfo = portMemAllocNonPaged(sizeof(*pStaticInfo));
    if (pStaticInfo == NULL)
    {
        NV_PRINTF(LEVEL_ERROR, "Could not allocate pStaticInfo for KernelBif");
        status = NV_ERR_NO_MEMORY;
        goto kBifStaticInfoInit_IMPL_exit;
    }
    portMemSet(pStaticInfo, 0, sizeof(*pStaticInfo));

    // Send the command
    NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
                        pRmApi->Control(pRmApi, pGpu->hInternalClient, pGpu->hInternalSubdevice,
                                        NV2080_CTRL_CMD_INTERNAL_BIF_GET_STATIC_INFO,
                                        pStaticInfo, sizeof(*pStaticInfo)),
                        kBifStaticInfoInit_IMPL_exit);

    // Initialize Kernel object fields with RPC response
    pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_PCIE_GEN4_CAPABLE,
                            pStaticInfo->bPcieGen4Capable);
    pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP,
                            pStaticInfo->bIsC2CLinkUp);
    pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_DEVICE_IS_MULTIFUNCTION,
                            pStaticInfo->bIsDeviceMultiFunction);
    pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_GCX_PMU_CFG_SPACE_RESTORE,
                            pStaticInfo->bGcxPmuCfgSpaceRestore);

kBifStaticInfoInit_IMPL_exit:
    portMemFree(pStaticInfo);

    return status;
}

/*!
 * @brief Initialize PCI-E config space bits based on chipset and GPU support.
 */
void
kbifInitPcieDeviceControlStatus_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    OBJSYS *pSys = SYS_GET_INSTANCE();
    OBJCL  *pCl  = SYS_GET_CL(pSys);

    kbifEnableExtendedTagSupport_HAL(pGpu, pKernelBif);

    //
    // Bug 382675 and 482867: Many SBIOSes default to disabling relaxed
    // ordering on GPUs, we want to always force it back on unless
    // the upstream root port is known to be broken with respect to this
    // feature.
    //
    if (!pCl->getProperty(pCl, PDB_PROP_CL_RELAXED_ORDERING_NOT_CAPABLE))
    {
        kbifPcieConfigEnableRelaxedOrdering_HAL(pGpu, pKernelBif);
    }
    else
    {
        kbifPcieConfigDisableRelaxedOrdering_HAL(pGpu, pKernelBif);
    }

    //
    // WAR for bug 3661529. All GH100 SKUs will need the NoSnoop WAR.
    // But currently GSP-RM does not detect this correctly,
    //
    if (IsGH100(pGpu))
    {
        pCl->setProperty(pCl, PDB_PROP_CL_ROOTPORT_NEEDS_NOSNOOP_WAR, NV_TRUE);
    }

    if (!pCl->getProperty(pCl, PDB_PROP_CL_NOSNOOP_NOT_CAPABLE) &&
        !pCl->getProperty(pCl, PDB_PROP_CL_ROOTPORT_NEEDS_NOSNOOP_WAR))
    {
        // Bug 393398 - Re-enable DEVICE_CONTROL_STATUS_ENABLE_NO_SNOOP
        kbifEnableNoSnoop_HAL(pGpu, pKernelBif, NV_TRUE);
    }
    else
    {
        //
        // Check for NO_SNOOP P2P bug on specific chipset.  More info in bug 332764.
        // Check for NO_SNOOP enabled by default on specific CPU. Refer bug 1511622.
        //
        kbifEnableNoSnoop_HAL(pGpu, pKernelBif, NV_FALSE);
    }
}

/*!
 * @brief Check and rearm MSI
 *
 * @param[in]   pGpu          GPU object pointer
 * @param[in]   pKernelBif    BIF object pointer
 *
 * @return NV_TRUE   if MSI is enabled
 *         NV_FALSE  if MSI is disabled
 */
void
kbifCheckAndRearmMSI_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    Intr *pIntr = GPU_GET_INTR(pGpu);

    if (kbifIsMSIEnabled(pGpu, pKernelBif))
    {
        if (!IS_VIRTUAL(pGpu) ||
            pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_ALLOW_REARM_MSI_FOR_VF))
        {
            // Send EOI to rearm
            if (pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_USE_CONFIG_SPACE_TO_REARM_MSI))
            {
                kbifRearmMSI_HAL(pGpu, pKernelBif);
            }
            else
            {
                intrRetriggerTopLevel_HAL(pGpu, pIntr);
            }
        }
    }
    else if (kbifIsMSIXEnabled(pGpu, pKernelBif))
    {
        intrRetriggerTopLevel_HAL(pGpu, pIntr);
    }
}

/*!
 * @brief Checks if MSI is enabled. Prefers to check the SW cache, but if
 * uncached, checks HW state and updates the SW cache for future use
 *
 * @param[in]   pGpu          GPU object pointer
 * @param[in]   pKernelBif    BIF object pointer
 *
 * @return NV_TRUE   if MSI is enabled
 *         NV_FALSE  if MSI is disabled
 */
NvBool
kbifIsMSIEnabled_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    //
    // Bug 418883: We shall rely upon value cached at boot, for the value
    // should not change during execution. If however, we must ever change
    // this back to be read at every ISR, we *must* read the value through
    // PCI CFG cycles.
    //
    if (!pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_MSI_CACHED))
    {
        if (kbifIsMSIEnabledInHW_HAL(pGpu, pKernelBif))
        {
            pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_MSI_ENABLED, NV_TRUE);

            if (IS_VIRTUAL(pGpu))
            {
                // vGPU guests want an FYI print that re-arming is not required
                NV_PRINTF(LEVEL_WARNING,
                          "MSI is enabled for vGPU, but no need to re-ARM\n");
            }
        }
        else
        {
            pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_MSI_ENABLED, NV_FALSE);
        }
        pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_MSI_CACHED, NV_TRUE);
    }

    return pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_MSI_ENABLED);
}

/*!
 * @brief Checks if MSI-X is enabled. Prefers to check the SW cache, but if
 * uncached, checks HW state and updates the SW cache for future use
 *
 * @param[in]   pGpu          GPU object pointer
 * @param[in]   pKernelBif    BIF object pointer
 *
 * @return NV_TRUE   if MSI is enabled
 *         NV_FALSE  if MSI is disabled
 */
NvBool
kbifIsMSIXEnabled_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    if (!pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_MSIX_CACHED))
    {
        if (kbifIsMSIXEnabledInHW_HAL(pGpu, pKernelBif))
        {
            pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_MSIX_ENABLED, NV_TRUE);
        }
        else
        {
            pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_MSIX_ENABLED, NV_FALSE);
        }
        pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_MSIX_CACHED, NV_TRUE);
    }
    return pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_MSIX_ENABLED);
}

/*!
 * @brief Clear PCIe HW PCIe config space error counters.
 * All of these should be cleared using config cycles.
 *
 * @param[in]   pGpu          GPU object pointer
 * @param[in]   pKernelBif    BIF object pointer
 */
void
kbifClearConfigErrors_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif,
    NvBool     bClearStatus,
    NvU32      xveAerFlagsMask
)
{
    NvU32 xveStatusFlags = 0;
    NvU32 xveStatus      = 0;
    NvU32 xveAerFlags    = 0;

    if ((bClearStatus) &&
        (kbifGetXveStatusBits_HAL(pGpu, pKernelBif, &xveStatusFlags, &xveStatus) == NV_OK) &&
        (xveStatusFlags != 0))
    {
        NV_PRINTF(LEVEL_WARNING, "PCI-E device status errors pending (%08X):\n",
                  xveStatusFlags);
#ifdef DEBUG
        if ( xveStatusFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_CORR_ERROR )
        {
            NV_PRINTF(LEVEL_WARNING, "     _CORR_ERROR_DETECTED\n");
        }
        if ( xveStatusFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_NON_FATAL_ERROR )
        {
            NV_PRINTF(LEVEL_WARNING, "     _NON_FATAL_ERROR_DETECTED\n");
        }
        if ( xveStatusFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_FATAL_ERROR )
        {
            NV_PRINTF(LEVEL_WARNING, "     _FATAL_ERROR_DETECTED\n");
        }
        if ( xveStatusFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_UNSUPP_REQUEST )
        {
            NV_PRINTF(LEVEL_WARNING, "     _UNSUPP_REQUEST_DETECTED\n");
        }
#endif
        NV_PRINTF(LEVEL_WARNING, "Clearing these errors..\n");
        kbifClearXveStatus_HAL(pGpu, pKernelBif, &xveStatus);
    }

    if ((xveAerFlagsMask) &&
        (kbifGetXveAerBits_HAL(pGpu, pKernelBif, &xveAerFlags) == NV_OK))
    {
        xveAerFlags &= xveAerFlagsMask;

        if (xveAerFlags != 0)
        {
            NV_PRINTF(LEVEL_WARNING,
                      "PCI-E device AER errors pending (%08X):\n",
                      xveAerFlags);
#ifdef DEBUG
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_TRAINING_ERR)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_UNCORR_TRAINING_ERR\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_DLINK_PROTO_ERR)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_UNCORR_DLINK_PROTO_ERR\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_POISONED_TLP)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_UNCORR_POISONED_TLP\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_FC_PROTO_ERR)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_UNCORR_FC_PROTO_ERR\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_CPL_TIMEOUT)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_UNCORR_CPL_TIMEOUT\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_CPL_ABORT)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_UNCORR_CPL_ABORT\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_UNEXP_CPL)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_UNCORR_UNEXP_CPL\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_RCVR_OVERFLOW)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_UNCORR_RCVR_OVERFLOW\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_MALFORMED_TLP)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_UNCORR_MALFORMED_TLP\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_ECRC_ERROR)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_UNCORR_ECRC_ERROR\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_UNSUPPORTED_REQ)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_UNCORR_UNSUPPORTED_REQ\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RCV_ERR)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_CORR_RCV_ERR\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_BAD_TLP)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_CORR_BAD_TLP\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_BAD_DLLP)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_CORR_BAD_DLLP\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RPLY_ROLLOVER)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_CORR_RPLY_ROLLOVER\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RPLY_TIMEOUT)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_CORR_RPLY_TIMEOUT\n");
            }
            if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_ADVISORY_NONFATAL)
            {
                NV_PRINTF(LEVEL_WARNING, "     _AER_CORR_ADVISORY_NONFATAL\n");
            }
#endif
            NV_PRINTF(LEVEL_WARNING, "Clearing these errors..\n");
            kbifClearXveAer_HAL(pGpu, pKernelBif, xveAerFlags);
        }
    }
}

/*!
 * @brief Regkey Overrides for Bif
 *
 * @param[in]   pGpu          GPU object pointer
 * @param[in]   pKernelBif    BIF object pointer
 */
static void
_kbifInitRegistryOverrides
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    NvU32 data32;

    // P2P Override
    pKernelBif->p2pOverride = BIF_P2P_NOT_OVERRIDEN;
    if (osReadRegistryDword(pGpu, NV_REG_STR_CL_FORCE_P2P, &data32) == NV_OK)
    {
        pKernelBif->p2pOverride = data32;
        pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_P2P_READS_DISABLED, FLD_TEST_DRF(_REG_STR, _CL_FORCE_P2P, _READ, _DISABLE, data32));
        pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_P2P_WRITES_DISABLED, FLD_TEST_DRF(_REG_STR, _CL_FORCE_P2P, _WRITE, _DISABLE, data32));
    }

    // P2P force type override
    pKernelBif->forceP2PType = NV_REG_STR_RM_FORCE_P2P_TYPE_DEFAULT;
    if (osReadRegistryDword(pGpu, NV_REG_STR_RM_FORCE_P2P_TYPE, &data32) == NV_OK &&
        (data32 <= NV_REG_STR_RM_FORCE_P2P_TYPE_MAX))
    {
        pKernelBif->forceP2PType = data32;
    }

    pKernelBif->pcieP2PType = NV_REG_STR_RM_PCIEP2P_TYPE_DEFAULT;
    if (osReadRegistryDword(pGpu, NV_REG_STR_RM_PCIEP2P_TYPE, &data32) == NV_OK)
    {
        pKernelBif->pcieP2PType = data32;
    }

    // Peer Mapping override
    pKernelBif->peerMappingOverride = NV_REG_STR_PEERMAPPING_OVERRIDE_DEFAULT;
    if (osReadRegistryDword(pGpu, NV_REG_STR_PEERMAPPING_OVERRIDE, &data32) == NV_OK)
    {
        NV_PRINTF(LEVEL_INFO, "allow peermapping reg key = %d\n", data32);
        pKernelBif->peerMappingOverride = !!data32;
    }

    // Check if the PCIe SBR recovery feature is enabled.
    if (osReadRegistryDword(pGpu, NV_REG_STR_SECONDARY_BUS_RESET_ENABLED, &data32) == NV_OK)
    {
        if (data32)
        {
            // The regkey was set to a non-zero value, enable the feature.
            pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_SECONDARY_BUS_RESET_ENABLED, NV_TRUE);
        }
        else
        {
            // The regkey was set to 0, disable the feature.
            pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_SECONDARY_BUS_RESET_ENABLED, NV_FALSE);
        }
    }

    // Check if saving of the PCIe config space during SBR is enabled.
    if (osReadRegistryDword(pGpu, NV_REG_STR_FORCE_PCIE_CONFIG_SAVE, &data32) == NV_OK)
    {
        if (data32)
        {
            // The regkey was set to a non-zero value, enable config save
            pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_FORCE_PCIE_CONFIG_SAVE, NV_TRUE);
        }
        else
        {
            // The regkey was set to a 0, disable config save
            pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_FORCE_PCIE_CONFIG_SAVE, NV_FALSE);
        }
    }

    // Check if Function Level Reset(FLR) is disabled
    pKernelBif->bForceDisableFLR = NV_REG_STR_RM_PCIE_FLR_POLICY_DEFAULT;
    if (osReadRegistryDword(pGpu,
           NV_REG_STR_RM_PCIE_FLR_POLICY, &data32) == NV_OK)
    {
        NV_PRINTF(LEVEL_INFO, "Pcie FLR Policy reg key = %d\n", data32);

        if (data32 == NV_REG_STR_RM_PCIE_FLR_POLICY_FORCE_DISABLE)
        {
            pKernelBif->bForceDisableFLR = NV_TRUE;
        }
        // There might be a requirement to handle (data32 == NV_REG_STR_RM_PCIE_FLR_POLICY_DEFAULT) case as per platform's policy
        else
        {
            pKernelBif->bForceDisableFLR = NV_FALSE;
        }
    }

    // Check for FLR timeout scale override
    if (osReadRegistryDword(pGpu,
           NV_REG_STR_RM_PCIE_FLR_DEVINIT_TIMEOUT_SCALE, &data32) == NV_OK)
    {
        if ((data32 >= NV_REG_STR_RM_PCIE_FLR_DEVINIT_TIMEOUT_SCALE_MIN_ALLOWED) &&
            (data32 <= NV_REG_STR_RM_PCIE_FLR_DEVINIT_TIMEOUT_SCALE_MAX_ALLOWED))
        {
            pKernelBif->flrDevInitTimeoutScale = data32;
        }
    }

}

/*!
 * Callback function to check if GPU exists
 *
 * @param[in]  pGpu    GPU object pointer
 * @param[in]  rsvd    Reserved  field
 */
static void
_kbifCheckIfGpuExists
(
    OBJGPU *pGpu,
    void   *rsvd
)
{
    if (FULL_GPU_SANITY_CHECK(pGpu))
    {
        if (gpuVerifyExistence_HAL(pGpu) != NV_OK)
        {
            osRemove1HzCallback(pGpu, _kbifCheckIfGpuExists, NULL);
        }
    }
}

/*!
 * @brief Get link capabilities register value
 *
 * @param[in] pGpu       GPU object pointer
 * @param[in] pKernelBif KernelBif object pointer
 *
 * @return    Register value
 */
NvU32
kbifGetGpuLinkCapabilities_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    NvU32 addrLinkCap = 0;
    NvU32 data        = 0;

    if (NV_OK != kbifGetBusOptionsAddr_HAL(pGpu, pKernelBif, BUS_OPTIONS_LINK_CAPABILITIES, &addrLinkCap))
    {
        return 0;
    }

    if (NV_OK != GPU_BUS_CFG_RD32(pGpu, addrLinkCap, &data))
    {
        NV_PRINTF(LEVEL_ERROR, "Unable to read %x\n", addrLinkCap);
        data = 0;
    }

    // Cache link capabilities here to be used by GSP-RM
    pKernelBif->pcieConfigReg.linkCap = data;

    return data;
}

/*!
 * @brief Get link control status register value
 *
 * @param[in] pGpu       GPU object pointer
 * @param[in] pKernelBif KernelBif object pointer
 *
 * @return    Register value
 */
NvU32
kbifGetGpuLinkControlStatus_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    NvU32 addrLinkControlStatus = 0;
    NvU32 data                  = 0;

    if (NV_OK != kbifGetBusOptionsAddr_HAL(pGpu, pKernelBif, BUS_OPTIONS_LINK_CONTROL_STATUS, &addrLinkControlStatus))
    {
        return 0;
    }

    if (NV_OK != GPU_BUS_CFG_RD32(pGpu, addrLinkControlStatus, &data ))
    {
        NV_PRINTF(LEVEL_INFO, "Unable to read %x\n", addrLinkControlStatus);
        return 0;
    }

    return data;
}

/*!
 * @brief Get device control status register value
 *
 * @param[in] pGpu       GPU object pointer
 * @param[in] pKernelBif KernelBif object pointer
 *
 * @return    Register value
 */
NvU32
kbifGetGpuDevControlStatus_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    NvU32 addrDevControlStatus = 0;
    NvU32 data                 = 0;

    if (NV_OK != kbifGetBusOptionsAddr_HAL(pGpu, pKernelBif, BUS_OPTIONS_DEV_CONTROL_STATUS, &addrDevControlStatus))
    {
        return 0;
    }

    if (NV_OK != GPU_BUS_CFG_RD32(pGpu, addrDevControlStatus, &data ))
    {
        NV_PRINTF(LEVEL_INFO, "Unable to read %x\n", addrDevControlStatus);
        return 0;
    }

    return data;
}

/*!
 * @brief Get device control status 2 register value
 *
 * @param[in] pGpu       GPU object pointer
 * @param[in] pKernelBif KernelBif object pointer
 *
 * @return    Register value
 */
NvU32
kbifGetGpuDevControlStatus2_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    NvU32 addrDevControlStatus2 = 0;
    NvU32 data                  = 0;

    if (NV_OK != kbifGetBusOptionsAddr_HAL(pGpu, pKernelBif, BUS_OPTIONS_DEV_CONTROL_STATUS_2, &addrDevControlStatus2))
    {
        return 0;
    }

    if (NV_OK != GPU_BUS_CFG_RD32(pGpu, addrDevControlStatus2, &data ))
    {
        NV_PRINTF(LEVEL_INFO, "Unable to read %x\n", addrDevControlStatus2);
        return 0;
    }

    return data;
}

/*!
 * @brief Get L1 Substates control register value
 *
 * @param[in] pGpu       GPU object pointer
 * @param[in] pKernelBif KernelBif object pointer
 *
 * @return    Register value
 */
NvU32
kbifGetGpuL1PmSubstatesCtrl1_IMPL
(
    OBJGPU    *pGpu,
    KernelBif *pKernelBif
)
{
    NvU32 addrL1PmSubstatesCtrl1 = 0;
    NvU32 data                   = 0;

    if (NV_OK != kbifGetBusOptionsAddr_HAL(pGpu, pKernelBif, BUS_OPTIONS_L1_PM_SUBSTATES_CTRL_1, &addrL1PmSubstatesCtrl1))
    {
        return 0;
    }

    if (NV_OK != GPU_BUS_CFG_RD32(pGpu, addrL1PmSubstatesCtrl1, &data ))
    {
        NV_PRINTF(LEVEL_INFO, "Unable to read %x\n", addrL1PmSubstatesCtrl1);
        return 0;
    }

    return data;
}

static NvBool
_doesBoardHaveMultipleGpusAndSwitch(OBJGPU *pGpu)
{
    if (((gpuIsMultiGpuBoard(pGpu)) ||
        (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_GEMINI)))&&
        ((pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_PLX_PRESENT))  ||
         (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_BR03_PRESENT)) ||
         (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_BR04_PRESENT))))
    {
        return NV_TRUE;
    }
    else
    {
        return NV_FALSE;
    }
}

NV_STATUS
kbifControlGetPCIEInfo_IMPL
(
    OBJGPU               *pGpu,
    KernelBif            *pKernelBif,
    NV2080_CTRL_BUS_INFO *pBusInfo
)
{
    OBJSYS *pSys  = SYS_GET_INSTANCE();
    OBJCL  *pCl   = SYS_GET_CL(pSys);
    NvU32   index = pBusInfo->index;
    NvU32   data  = 0;

    if (gpuGetBusIntfType_HAL(pGpu) != NV2080_CTRL_BUS_INFO_TYPE_PCI_EXPRESS)
    {
        // KMD cannot handle error codes for this ctrl call, hence returning
        // NV_OK, once KMD fixes the bug:3545197, RM can return NV_ERR_NOT_SUPPORTED
        return NV_OK;
    }

    switch (index)
    {
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_CAPS:
        {
            data = kbifGetGpuLinkCapabilities(pGpu, pKernelBif);
            break;
        }
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_UPSTREAM_LINK_CAPS:
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_BOARD_LINK_CAPS:
        {
            if (_doesBoardHaveMultipleGpusAndSwitch(pGpu))
            {
                if (clPcieReadPortConfigReg(pGpu, pCl,
                                            &pGpu->gpuClData.boardUpstreamPort,
                                            CL_PCIE_LINK_CAP, &data) != NV_OK)
                {
                    data = 0;
                }
            }
            else
            {
                data = kbifGetGpuLinkCapabilities(pGpu, pKernelBif);
            }
            break;
        }
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_UPSTREAM_GEN_INFO:
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_BOARD_GEN_INFO:
        {
            NvU32 temp;

            if (_doesBoardHaveMultipleGpusAndSwitch(pGpu))
            {
                if (clPcieReadPortConfigReg(pGpu, pCl,
                                            &pGpu->gpuClData.boardUpstreamPort,
                                            CL_PCIE_LINK_CTRL_STATUS, &temp) != NV_OK)
                {
                    data = 0;
                    break;
                }
                else
                {
                    temp = REF_VAL(NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED, temp);
                    if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_64000MBPS)
                    {
                        data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
                                           _CURR_LEVEL, _GEN6, data);
                    }
                    else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_32000MBPS)
                    {
                        data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
                                           _CURR_LEVEL, _GEN5, data);
                    }
                    else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_16000MBPS)
                    {
                        data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
                                           _CURR_LEVEL, _GEN4, data);
                    }
                    else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_8000MBPS)
                    {
                        data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
                                           _CURR_LEVEL, _GEN3, data);
                    }
                    else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_5000MBPS)
                    {
                        data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
                                           _CURR_LEVEL, _GEN2, data);
                    }
                    else
                    {
                        data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
                                           _CURR_LEVEL, _GEN1, data);
                    }
                }

                if (clPcieReadPortConfigReg(pGpu, pCl,
                                            &pGpu->gpuClData.boardUpstreamPort,
                                            CL_PCIE_LINK_CAP, &temp) != NV_OK)
                {
                    data = 0;
                    break;
                }
                else
                {
                    temp = REF_VAL(NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED, temp);
                    if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_64000MBPS)
                    {
                        data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
                                           _GEN, _GEN6, data);
                    }
                    else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_32000MBPS)
                    {
                        data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
                                           _GEN, _GEN5, data);
                    }
                    else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_16000MBPS)
                    {
                        data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
                                           _GEN, _GEN4, data);
                    }
                    else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_8000MBPS)
                    {
                        data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
                                           _GEN, _GEN3, data);
                    }
                    else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_5000MBPS)
                    {
                        data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
                                           _GEN, _GEN2, data);
                    }
                    else
                    {
                        data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
                                           _GEN, _GEN1, data);
                    }
                }
            }
            else
            {
                if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu))
                {
                    NV2080_CTRL_BUS_INFO busInfo = {0};
                    NV_STATUS rmStatus = NV_OK;

                    busInfo.index = NV2080_CTRL_BUS_INFO_INDEX_PCIE_GEN_INFO;

                    if ((rmStatus = kbusSendBusInfo(pGpu, GPU_GET_KERNEL_BUS(pGpu), &busInfo)) != NV_OK)
                    {
                        NV_PRINTF(LEVEL_INFO, "Squashing rmStatus: %x \n", rmStatus);
                        rmStatus = NV_OK;
                        busInfo.data = 0;
                    }
                    data = busInfo.data;
                }
            }
            break;
        }
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_ROOT_LINK_CAPS:
        {
            if (clPcieReadPortConfigReg(pGpu, pCl,
                                        &pGpu->gpuClData.rootPort,
                                        CL_PCIE_LINK_CAP, &data) != NV_OK)
            {
                data = 0;
            }
            break;
        }
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_DOWNSTREAM_LINK_CAPS:
        {
            if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_BR03_PRESENT) ||
                pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_BR04_PRESENT))
            {
                if (clPcieReadPortConfigReg(pGpu, pCl,
                                            &pGpu->gpuClData.boardDownstreamPort,
                                            CL_PCIE_LINK_CAP, &data) != NV_OK)
                {
                    data = 0;
                }
            }
            else
            {
                // no br03/br04, same as link from RC
                if (clPcieReadPortConfigReg(pGpu, pCl,
                                            &pGpu->gpuClData.rootPort,
                                            CL_PCIE_LINK_CAP, &data) != NV_OK)
                {
                    data = 0;
                }
            }
            break;
        }
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_CTRL_STATUS:
        {
            data = kbifGetGpuLinkControlStatus(pGpu, pKernelBif);
            break;
        }
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_UPSTREAM_LINK_CTRL_STATUS:
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_BOARD_LINK_CTRL_STATUS:
        {
            if (_doesBoardHaveMultipleGpusAndSwitch(pGpu))
            {
                if (clPcieReadPortConfigReg(pGpu, pCl,
                                            &pGpu->gpuClData.boardUpstreamPort,
                                            CL_PCIE_LINK_CTRL_STATUS, &data) != NV_OK)
                {
                    data = 0;
                }
            }
            else
            {
                data = kbifGetGpuLinkControlStatus(pGpu, pKernelBif);
            }
            break;
        }
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_ROOT_LINK_CTRL_STATUS:
        {
            if (clPcieReadPortConfigReg(pGpu, pCl,
                                        &pGpu->gpuClData.rootPort,
                                        CL_PCIE_LINK_CTRL_STATUS,
                                        &data) != NV_OK)
            {
                data = 0;
            }
            break;
        }
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_DOWNSTREAM_LINK_CTRL_STATUS:
        {
            if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_BR03_PRESENT) ||
                pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_BR04_PRESENT))
            {
                if (clPcieReadPortConfigReg(pGpu, pCl,
                                            &pGpu->gpuClData.boardDownstreamPort,
                                            CL_PCIE_LINK_CTRL_STATUS, &data) != NV_OK)
                {
                    data = 0;
                }
            }
            else
            {
                // no br03/br04, same as link from RC
                if (clPcieReadPortConfigReg(pGpu, pCl,
                                            &pGpu->gpuClData.rootPort,
                                            CL_PCIE_LINK_CTRL_STATUS,
                                            &data) != NV_OK)
                {
                    data = 0;
                }
            }
            break;
        }
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_ERRORS:
        {
            NvU32 xveStatus = 0;

            if (pKernelBif != NULL)
            {
                if (kbifGetXveStatusBits_HAL(pGpu, pKernelBif, &data, &xveStatus) != NV_OK)
                {
                    data = 0;
                    break;
                }
                if (kbifClearXveStatus_HAL(pGpu, pKernelBif, &xveStatus) != NV_OK)
                {
                    data = 0;
                }
            }
            break;
        }
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_ROOT_LINK_ERRORS:
        {
            NvU32 clStatus = 0;

            if (clPcieReadDevCtrlStatus(pGpu, pCl, &data, &clStatus) != NV_OK)
            {
                data = 0;
                break;
            }
            if (clPcieClearDevCtrlStatus(pGpu, pCl, &clStatus) != NV_OK)
            {
                data = 0;
            }
            break;
        }
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_AER:
        {
            if (pKernelBif != NULL)
            {
                if (kbifGetXveAerBits_HAL(pGpu, pKernelBif, &data) != NV_OK)
                {
                    data = 0;
                    break;
                }
                if (kbifClearXveAer_HAL(pGpu, pKernelBif, data) != NV_OK)
                {
                    data = 0;
                }
            }
            break;
        }
        case NV2080_CTRL_BUS_INFO_INDEX_MSI_INFO:
        {
            if (kbifIsMSIEnabledInHW_HAL(pGpu, pKernelBif))
            {
                data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_MSI,
                                   _STATUS, _ENABLED, data);
            }
            else
            {
                data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_MSI,
                                   _STATUS, _DISABLED, data);
            }
            break;
        }

        default:
            break;
    }

    pBusInfo->data = data;
    return NV_OK;
}

/*!
 * @brief RM control call handler to return requested data from kernel Bif module or redirect it to physical RM
 *        if the info required is not present in Kernel RM
 *
 * @param[in]    pSubdevice    Subdevice object pointer
 * @param[in]    pParams       Params passed to RM control call
 *
 * @return NV_OK
 */
NV_STATUS
subdeviceCtrlCmdBifGetData_IMPL
(
  Subdevice                         *pSubdevice,
  NV2080_CTRL_BIF_GET_DATA_PARAMS   *pParams
)
{
    OBJGPU     *pGpu       = GPU_RES_GET_GPU(pSubdevice);
    KernelBif  *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
    NV_STATUS   status     = NV_OK;
    RM_API     *pRmApi     = GPU_GET_PHYSICAL_RMAPI(pGpu);

    if (pParams->paramSize > NV_ARRAY_ELEMENTS(pParams->paramId))
    {
        return NV_ERR_INVALID_ARGUMENT;
    }

    switch (pParams->featureId)
    {
        case NV2080_CTRL_BIF_FEATURE_ID_CONFIG_REG:
           status = kbifGetConfigRegData(pGpu, pKernelBif, pParams);
           break;
        default:
           NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
                    pRmApi->Control(pRmApi, pGpu->hInternalClient, pGpu->hInternalSubdevice,
                                    NV2080_CTRL_CMD_INTERNAL_BIF_GET_DATA,
                                    pParams, sizeof(*pParams)),
                    subdeviceCtrlCmdBifGetData_IMPL_exit);
    }

    pParams->flags = (NvU32) status;
    if (status != NV_OK)
    {
        NV_PRINTF(LEVEL_INFO, "Call failed for feature ID = %d.\n", pParams->featureId);
    }

subdeviceCtrlCmdBifGetData_IMPL_exit:
    return status;
}

/*!
 * @brief Get pcie config space registers data
 *
 * @param[in]    pGpu        GPU object pointer
 * @param[in]    pKernelBif  KernelBif object pointer
 * @param[inout] pParams     Params passed to RM control call
 *
 * @return NV_OK
 */
NV_STATUS
kbifGetConfigRegData_IMPL
(
    OBJGPU                           *pGpu,
    KernelBif                        *pKernelBif,
    NV2080_CTRL_BIF_GET_DATA_PARAMS  *pParams
)
{
    NV_STATUS  status     = NV_OK;
    NvU32      paramSize  = pParams->paramSize;
    NvU32      id         = 0;

    for (id = 0; id < paramSize; id++)
    {
        switch (pParams->paramId[id])
        {
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_DEVICE_CTRL_STATUS:
                 pParams->paramVal[id] = kbifGetGpuDevControlStatus(pGpu, pKernelBif);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_LINK_CTRL_STATUS:
                 pParams->paramVal[id] = kbifGetGpuLinkControlStatus(pGpu, pKernelBif);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_DEVICE_CTRL_STATUS_2:
                 pParams->paramVal[id] = kbifGetGpuDevControlStatus2(pGpu, pKernelBif);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_LINK_CAP:
                 pParams->paramVal[id] = kbifGetGpuLinkCapabilities(pGpu, pKernelBif);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_L1_PM_SUBSTATES_CTRL1:
                 pParams->paramVal[id] = kbifGetGpuL1PmSubstatesCtrl1(pGpu, pKernelBif);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_CHECK_FAILED_STATUS:
                 pParams->paramVal[id] = kbifGetIdeInfo_HAL(pGpu, pKernelBif, NV_BIF_PEX_CONFIG_REG_FN0_IDE_CHECK_FAILED_STATUS);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_MISROUTED_TLP_STATUS:
                 pParams->paramVal[id] = kbifGetIdeInfo_HAL(pGpu, pKernelBif, NV_BIF_PEX_CONFIG_REG_FN0_IDE_MISROUTED_TLP_STATUS);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_PCRC_CHECK_FAILED_STATUS:
                 pParams->paramVal[id] = kbifGetIdeInfo_HAL(pGpu, pKernelBif, NV_BIF_PEX_CONFIG_REG_FN0_IDE_PCRC_CHECK_FAILED_STATUS);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_CHECK_FAILED_MASK:
                 pParams->paramVal[id] = kbifGetIdeInfo_HAL(pGpu, pKernelBif, NV_BIF_PEX_CONFIG_REG_FN0_IDE_CHECK_FAILED_MASK);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_MISROUTED_TLP_MASK:
                 pParams->paramVal[id] = kbifGetIdeInfo_HAL(pGpu, pKernelBif, NV_BIF_PEX_CONFIG_REG_FN0_IDE_MISROUTED_TLP_MASK);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_PCRC_CHECK_FAILED_MASK:
                 pParams->paramVal[id] = kbifGetIdeInfo_HAL(pGpu, pKernelBif, NV_BIF_PEX_CONFIG_REG_FN0_IDE_PCRC_CHECK_FAILED_MASK);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_CHECK_FAILED_SEVERITY:
                 pParams->paramVal[id] = kbifGetIdeInfo_HAL(pGpu, pKernelBif, NV_BIF_PEX_CONFIG_REG_FN0_IDE_CHECK_FAILED_SEVERITY);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_MISROUTED_TLP_SEVERITY:
                 pParams->paramVal[id] = kbifGetIdeInfo_HAL(pGpu, pKernelBif, NV_BIF_PEX_CONFIG_REG_FN0_IDE_MISROUTED_TLP_SEVERITY);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_PCRC_CHECK_FAILED_SEVERITY:
                 pParams->paramVal[id] = kbifGetIdeInfo_HAL(pGpu, pKernelBif, NV_BIF_PEX_CONFIG_REG_FN0_IDE_PCRC_CHECK_FAILED_SEVERITY);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_TLP_ENCRYPTION_STATUS:
                 pParams->paramVal[id] = kbifGetIdeInfo_HAL(pGpu, pKernelBif, NV_BIF_PEX_CONFIG_REG_FN0_IDE_TLP_ENCRYPTION_STATUS); 
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_TYPE_LINK_STREAM_ENABLED:
                 pParams->paramVal[id] = kbifGetIdeInfo_HAL(pGpu, pKernelBif, NV_BIF_PEX_CONFIG_REG_FN0_IDE_TYPE_LINK_STREAM_ENABLED);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_TYPE_SELECTIVE_STREAM_ENABLED:
                 pParams->paramVal[id] = kbifGetIdeInfo_HAL(pGpu, pKernelBif, NV_BIF_PEX_CONFIG_REG_FN0_IDE_TYPE_SELECTIVE_STREAM_ENABLED);
                 break;
            case NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_TYPE_FLOW_THROUGH_STREAM_ENABLED:
                 pParams->paramVal[id] = kbifGetIdeInfo_HAL(pGpu, pKernelBif, NV_BIF_PEX_CONFIG_REG_FN0_IDE_TYPE_FLOW_THROUGH_STREAM_ENABLED);
                 break;     
            default:
                 NV_PRINTF(LEVEL_INFO, "Invalid command.\n");
                 status = NV_ERR_INVALID_REQUEST;
        }
    }

    return status;
}

/*!
 * @brief To ensure GPU is back on bus and accessible by polling device ID
 *
 * @param[in]  pGpu        GPU object pointer
 * @param[in]  pKernelBif  Kernel BIF object pointer
 *
 * @returns NV_OK
 * @returns NV_ERR_TIMEOUT
 */
NV_STATUS
kbifPollDeviceOnBus_IMPL
(
    OBJGPU     *pGpu,
    KernelBif  *pKernelBif
)
{
    RMTIMEOUT timeout;

    gpuSetTimeout(pGpu, GPU_TIMEOUT_DEFAULT, &timeout, 0);

    while (osPciInitHandle(gpuGetDomain(pGpu),
                           gpuGetBus(pGpu),
                           gpuGetDevice(pGpu), 0, NULL, NULL) == NULL)
    {
        if (gpuCheckTimeout(pGpu, &timeout) == NV_ERR_TIMEOUT)
        {
            NV_PRINTF(LEVEL_ERROR, "Timeout polling GPU back on bus\n");
            DBG_BREAKPOINT();
            return NV_ERR_TIMEOUT;
        }
        osDelayUs(100);
    }

    return NV_OK;
}

/*!
 * @brief To get PCI link max spead from PCI link Gen info
 *
 * @param[in]  pGpu        GPU object pointer
 * @param[in]  pKernelBif  Kernel BIF object pointer
 * @param[in]  pciLinkGenInfo   PCI link Gen info
 * @param[in]  pciLinkMaxSpeed  pointer to PCI link max spead

 * @returns NV_OK
 * @returns NV_ERR_INVALID_STATE
 */
NV_STATUS
kbifGetPciLinkMaxSpeedByPciGenInfo_IMPL
(
    OBJGPU     *pGpu,
    KernelBif  *pKernelBif,
    NvU32      pciLinkGenInfo,
    NvU32      *pciLinkMaxSpeed
)
{
    NV_STATUS rmStatus = NV_OK;

    switch (pciLinkGenInfo)
    {
        case NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN1:
            *pciLinkMaxSpeed = NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_2500MBPS;
            break;

        case NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN2:
            *pciLinkMaxSpeed = NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_5000MBPS;
            break;

        case NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN3:
            *pciLinkMaxSpeed = NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_8000MBPS;
            break;

        case NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN4:
            *pciLinkMaxSpeed = NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_16000MBPS;
            break;

        case NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN5:
            *pciLinkMaxSpeed = NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_32000MBPS;
            break;

        case NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN6:
            *pciLinkMaxSpeed = NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_64000MBPS;
            break;

        default:
            rmStatus = NV_ERR_INVALID_STATE;
            NV_PRINTF(LEVEL_ERROR, "Unknown PCIe Gen Info\n");
    }

    return rmStatus;
}

NV_STATUS
kbifResetFromTimeoutFullChip_IMPL
(
    OBJGPU *pGpu,
    KernelBif *pKernelBif
)
{

    OBJSYS *pSys = SYS_GET_INSTANCE();
    OBJCL  *pCl = SYS_GET_CL(pSys);
    NV_STATUS status;

    if (pCl == NULL || 
        (!pCl->getProperty(pCl, PDB_PROP_CL_PCIE_CONFIG_ACCESSIBLE)
       ))
    {
        //
        // We can not issue SW_RESET as we are not able to access PCI config
        // space.
        return NV_ERR_NOT_SUPPORTED;
    }

    //
    // Execute any workarounds or changes needed to make sure we can reset
    // properly.
    //
    kbifPrepareForFullChipReset_HAL(pGpu, pKernelBif);

    // Reset the gpu.
    NV_ASSERT_OK(status = kbifDoFullChipReset_HAL(pGpu, pKernelBif));

    //
    // This 100ms wait is only required for legacy FLR where RM itself
    // triggers FLR and it also does save/restore of config space.
    // For hopper and later, HW will keep sending CRS status to OS
    // until FWSEC boot sequence is not complete
    //
    if (!(pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_FLR_HANDLED_BY_OS)))
    {
        // Wait 100ms after GPU is reset to ensure Pre-OS completion.
        osDelay(100);
    }

    return status;
}


NV_STATUS
kbifWaitForConfigAccessAfterReset_IMPL
(
    OBJGPU *pGpu,
    KernelBif *pKernelBif
)
{


    NvU32 domain = gpuGetDomain(pGpu);
    NvU8  bus    = gpuGetBus(pGpu);
    NvU8  device = gpuGetDevice(pGpu);

    NvU32 waitCount = 20; // iterations of the osDelayUs statements
    while (NV_TRUE)
    {
        NvU16 vendorId;
        NvU16 deviceId;
        void *pGpuHandle = osPciInitHandle(domain,
                                           bus,
                                           device,
                                           0 /* Function */,
                                           &vendorId,
                                           &deviceId);

        NvU32 id = osPciReadDword(pGpuHandle, 0 /* Offset */);
        if (id == pGpu->idInfo.PCIDeviceID)
        {
            // Found it.  Config space is accessible.  Break out.
            return NV_OK;
        }

        if (waitCount == 0)
        {
            break;
        }
        waitCount--;

        //
        // Keep trying for at least 1 second.  Note that completion timeouts
        // here can extend the amount of time we are in this loop.  But they are
        // not fatal.
        //
        {
            // Wait 50ms
            osDelayUs(50000);
        }
    }

    return NV_ERR_GENERIC;
}

