/*
 * SPDX-FileCopyrightText: Copyright (c) 2004-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include "rmapi/rmapi.h"
#include "rmapi/control.h"
#include "rmapi/client.h"
#include "rmapi/rs_utils.h"
#include "diagnostics/tracer.h"
#include "core/locks.h"
#include "core/thread_state.h"
#include "gpu/device/device.h"

#include "entry_points.h"
#include "resserv/rs_access_map.h"
#include "gpu/gpu.h"
#include "gpu/subdevice/subdevice.h"

#include "ctrl/ctrl0000/ctrl0000client.h" // NV0000_CTRL_CMD_CLIENT_*
#include "ctrl/ctrl0000/ctrl0000gpu.h" // NV0000_CTRL_CMD_GPU_*
#include "ctrl/ctrl0000/ctrl0000system.h" // NV0000_CTRL_CMD_SYSTEM_*
#include "ctrl/ctrl0000/ctrl0000syncgpuboost.h" // NV0000_CTRL_CMD_SYNC_GPU_BOOST_*
#include "ctrl/ctrl0000/ctrl0000nvd.h" // NV0000_CTRL_CMD_NVD_*
#include "ctrl/ctrl2080/ctrl2080rc.h" // NV2080_CTRL_CMD_RC_READ_VIRTUAL_MEM
#include "ctrl/ctrl0002.h" // N09002_CTRL_CMD_*_CONTEXTDMA
#include "ctrl/ctrl906f.h" // NV906F_CTRL_CMD_GET_MMU_FAULT_INFO
#include "ctrl/ctrlc370/ctrlc370chnc.h" // NVC370_CTRL_CMD_*
#include "ctrl/ctrl9010.h" //NV9010_CTRL_CMD_SET_VBLANK_NOTIFICATION
#include "ctrl/ctrl2080/ctrl2080tmr.h" // NV2080_CTRL_CMD_TIMER_*

static NV_STATUS
releaseDeferRmCtrlBuffer(RmCtrlDeferredCmd* pRmCtrlDeferredCmd)
{
    portMemSet(&pRmCtrlDeferredCmd->paramBuffer, 0, RMCTRL_DEFERRED_MAX_PARAM_SIZE);

    portAtomicSetS32(&pRmCtrlDeferredCmd->pending, RMCTRL_DEFERRED_FREE);

    return NV_OK;
}

//
// This is the rmControl internal handler for deferred calls.
//
//

NV_STATUS
rmControl_Deferred(RmCtrlDeferredCmd* pRmCtrlDeferredCmd)
{
    RmCtrlParams rmCtrlParams;
    RmClient *pClient;
    NvU8 paramBuffer[RMCTRL_DEFERRED_MAX_PARAM_SIZE];
    NV_STATUS status;
    RS_LOCK_INFO lockInfo = {0};
    RS_CONTROL_COOKIE rmCtrlExecuteCookie = {0};

    // init RmCtrlParams
    portMemCopy(&rmCtrlParams, sizeof(RmCtrlParams), &pRmCtrlDeferredCmd->rmCtrlDeferredParams, sizeof(RmCtrlParams));
    rmCtrlParams.hParent    = NV01_NULL_OBJECT;
    rmCtrlParams.pGpu       = NULL;
    rmCtrlParams.pLockInfo = &lockInfo;
    rmCtrlParams.pCookie = &rmCtrlExecuteCookie;

    // Temporary: tell ResServ not to take any locks
    lockInfo.flags = RM_LOCK_FLAGS_NO_GPUS_LOCK |
                     RM_LOCK_FLAGS_NO_CLIENT_LOCK;

    if (rmapiLockIsOwner())
    {
        lockInfo.state = RM_LOCK_STATES_API_LOCK_ACQUIRED;
    }
    else
    {
        lockInfo.flags |= RM_LOCK_FLAGS_NO_API_LOCK;
    }

    // paramsSize not > _MAX already verified in _rmControlDeferred
    if ((rmCtrlParams.pParams != NvP64_NULL) && (rmCtrlParams.paramsSize != 0))
    {
        // copy param to a local buffer so that pRmCtrlDeferredCmd can be released
        portMemSet(paramBuffer, 0, RMCTRL_DEFERRED_MAX_PARAM_SIZE);
        portMemCopy(paramBuffer, rmCtrlParams.paramsSize, rmCtrlParams.pParams, rmCtrlParams.paramsSize);
        rmCtrlParams.pParams = paramBuffer;
    }

    releaseDeferRmCtrlBuffer(pRmCtrlDeferredCmd);

    // client was checked when we came in through rmControl()
    // but check again to make sure it's still good
    if (serverutilGetClientUnderLock(rmCtrlParams.hClient, &pClient) != NV_OK)
    {
        status = NV_ERR_INVALID_CLIENT;
        goto exit;
    }

    status = serverControl(&g_resServ, &rmCtrlParams);

exit:

    if (status != NV_OK)
    {
        NV_PRINTF(LEVEL_WARNING, "deferred rmctrl %x failed %x!\n",
                  rmCtrlParams.cmd, status);
    }

    return status;
}

static NV_STATUS
_rmControlDeferred(RmCtrlParams *pRmCtrlParams, NvP64 pUserParams, NvU32 paramsSize)
{
    // Schedule a deferred rmctrl call
    OBJGPU     *pGpu;
    NvBool      bBcResource;
    NV_STATUS   rmStatus;
    RsClient   *pClient;

    // We can't allocate memory at DIRQL, so use pre-allocated buffer to store any rmctrl param.
    // The size can't be large than DEFERRED_RMCTRL_MAX_PARAM_SIZE (defined in rmctrl.h), otherwise,
    // fail this call.
    if (paramsSize > RMCTRL_DEFERRED_MAX_PARAM_SIZE)
    {
        NV_PRINTF(LEVEL_WARNING,
                  "rmctrl param size (%d) larger than limit (%d).\n",
                  paramsSize, RMCTRL_DEFERRED_MAX_PARAM_SIZE);
        rmStatus = NV_ERR_INSUFFICIENT_RESOURCES;
        goto done;
    }

    rmStatus = serverGetClientUnderLock(&g_resServ, pRmCtrlParams->hClient, &pClient);
    if (rmStatus != NV_OK)
        return rmStatus;

    rmStatus = gpuGetByHandle(pClient, pRmCtrlParams->hObject, &bBcResource, &pGpu);
    if (rmStatus != NV_OK)
        return rmStatus;

    // Set SLI BC state for thread
    gpuSetThreadBcState(pGpu, bBcResource);

    pRmCtrlParams->pGpu = pGpu;
    pRmCtrlParams->pLockInfo = NULL;

    switch (pRmCtrlParams->cmd)
    {
        // we don't have available bit left in RmCtrlParams.cmd to
        // indicate a rmctrl type as deferrable so use cmd list here
        case NV2080_CTRL_CMD_TIMER_SCHEDULE:
        {
            if (pRmCtrlParams->flags & NVOS54_FLAGS_IRQL_RAISED)
            {
                OBJSYS    *pSys = SYS_GET_INSTANCE();
                NvU32 idx;

                for ( idx = 0; idx < MAX_DEFERRED_CMDS; idx++)
                {
                    if (portAtomicCompareAndSwapS32(&pGpu->pRmCtrlDeferredCmd[idx].pending,
                                                    RMCTRL_DEFERRED_ACQUIRED,
                                                    RMCTRL_DEFERRED_FREE))
                    {
                        portMemCopy(&pGpu->pRmCtrlDeferredCmd[idx].rmCtrlDeferredParams,
                                    sizeof(RmCtrlParams), pRmCtrlParams, sizeof(RmCtrlParams));

                        // copyin param to kernel buffer for deferred rmctrl
                        if (paramsSize != 0 && pUserParams != 0)
                        {
                            portMemCopy(pGpu->pRmCtrlDeferredCmd[idx].paramBuffer, paramsSize,
                                        NvP64_VALUE(pUserParams), paramsSize);

                            if (paramsSize < RMCTRL_DEFERRED_MAX_PARAM_SIZE)
                            {
                                portMemSet(pGpu->pRmCtrlDeferredCmd[idx].paramBuffer +
                                           paramsSize,
                                           0, RMCTRL_DEFERRED_MAX_PARAM_SIZE - paramsSize);
                            }

                            pGpu->pRmCtrlDeferredCmd[idx].rmCtrlDeferredParams.pParams =
                                pGpu->pRmCtrlDeferredCmd[idx].paramBuffer;
                        }

                        portAtomicSetS32(&pGpu->pRmCtrlDeferredCmd[idx].pending,
                                         RMCTRL_DEFERRED_READY);

                        // Make sure there's a release call to trigger the deferred rmctrl.
                        // Previous rmctrl that is holding the lock can already
                        // finished (release its lock) during the period before the pending
                        // flag is set and after this rmctrl failed to acquire lock.

                        // LOCK: try to acquire GPUs lock
                        if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_COND_ACQUIRE,
                                              RM_LOCK_MODULES_CLIENT) == NV_OK)
                        {
                            if (osCondAcquireRmSema(pSys->pSema) == NV_OK)
                            {
                                // In case this is called from device interrupt, use pGpu to queue DPC.
                                osReleaseRmSema(pSys->pSema, pGpu);
                            }
                            // In case this is called from device interrupt, use pGpu to queue DPC.
                            // UNLOCK: release GPUs lock
                            rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, pGpu);
                        }

                        rmStatus = NV_OK;
                        goto done;
                    }
                }
            }

            rmStatus = NV_ERR_STATE_IN_USE;
            break;
        }

        default:
            rmStatus = NV_ERR_BUSY_RETRY;
            break;
    }

done:
    return rmStatus;
}

NV_STATUS
serverControlApiCopyIn
(
    RsServer                        *pServer,
    RS_RES_CONTROL_PARAMS_INTERNAL  *pRmCtrlParams,
    RS_CONTROL_COOKIE               *pCookie
)
{
    NV_STATUS rmStatus;
    API_STATE *pParamCopy;
    API_STATE *pEmbeddedParamCopies;
    NvP64 pUserParams;
    NvU32 paramsSize;

    NV_ASSERT_OR_RETURN(pCookie != NULL, NV_ERR_INVALID_ARGUMENT);
    pParamCopy = &pCookie->paramCopy;
    pEmbeddedParamCopies = pCookie->embeddedParamCopies;
    pUserParams = NV_PTR_TO_NvP64(pRmCtrlParams->pParams);
    paramsSize = pRmCtrlParams->paramsSize;

    RMAPI_PARAM_COPY_INIT(*pParamCopy, pRmCtrlParams->pParams, pUserParams, paramsSize, 1);

    rmStatus = rmapiParamsAcquire(pParamCopy, (pRmCtrlParams->secInfo.paramLocation == PARAM_LOCATION_USER));
    if (rmStatus != NV_OK)
        return rmStatus;
    pCookie->bFreeParamCopy = NV_TRUE;

    rmStatus = embeddedParamCopyIn(pEmbeddedParamCopies, pRmCtrlParams);
    if (rmStatus != NV_OK)
        return rmStatus;
    pCookie->bFreeEmbeddedCopy = NV_TRUE;

    return NV_OK;
}

NV_STATUS
serverControlApiCopyOut
(
    RsServer                       *pServer,
    RS_RES_CONTROL_PARAMS_INTERNAL *pRmCtrlParams,
    RS_CONTROL_COOKIE              *pCookie,
    NV_STATUS                       rmStatus
)
{
    NV_STATUS  cpStatus;
    API_STATE *pParamCopy;
    API_STATE *pEmbeddedParamCopies;
    NvP64      pUserParams;
    NvBool     bFreeEmbeddedCopy;
    NvBool     bFreeParamCopy;

    NV_ASSERT_OR_RETURN(pCookie != NULL, NV_ERR_INVALID_ARGUMENT);

    pParamCopy = &pCookie->paramCopy;
    pEmbeddedParamCopies = pCookie->embeddedParamCopies;
    pUserParams = pCookie->paramCopy.pUserParams;
    bFreeParamCopy = pCookie->bFreeParamCopy;
    bFreeEmbeddedCopy = pCookie->bFreeEmbeddedCopy;

    if ((rmStatus != NV_OK) && !(pCookie->ctrlFlags & RMCTRL_FLAGS_COPYOUT_ON_ERROR))
    {
        pParamCopy->flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT;

        if (bFreeEmbeddedCopy)
        {
            pEmbeddedParamCopies[0].flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT;
            pEmbeddedParamCopies[1].flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT;
            pEmbeddedParamCopies[2].flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT;
            pEmbeddedParamCopies[3].flags |= RMAPI_PARAM_COPY_FLAGS_SKIP_COPYOUT;
        }
    }

    if (bFreeEmbeddedCopy)
    {
        cpStatus = embeddedParamCopyOut(pEmbeddedParamCopies, pRmCtrlParams);
        if (rmStatus == NV_OK)
            rmStatus = cpStatus;
        pCookie->bFreeEmbeddedCopy = NV_FALSE;
    }

    if (bFreeParamCopy)
    {
        cpStatus = rmapiParamsRelease(pParamCopy);
        if (rmStatus == NV_OK)
            rmStatus = cpStatus;
        pRmCtrlParams->pParams = NvP64_VALUE(pUserParams);
        pCookie->bFreeParamCopy = NV_FALSE;
    }

    return rmStatus;
}

static NvBool _rmapiRmControlCanBeRaisedIrql(NvU32 cmd)
{
    switch (cmd)
    {
        case NV2080_CTRL_CMD_TIMER_SCHEDULE:
        case NV2080_CTRL_CMD_TIMER_GET_TIME:
        // Below 2 control calls are used for flip canceling (HW Flip Queue)
        // We use TRASH/ABORT mode to discard queued hw commands in the push buffer (bug 200644346)
        case NVC370_CTRL_CMD_SET_ACCL:
        case NVC370_CTRL_CMD_GET_CHANNEL_INFO:
        case NV9010_CTRL_CMD_SET_VBLANK_NOTIFICATION:
            return NV_TRUE;
        default:
            return NV_FALSE;
    }
}

static NvBool _rmapiRmControlCanBeBypassLock(NvU32 cmd)
{
    switch (cmd)
    {
        case NV2080_CTRL_CMD_RC_READ_VIRTUAL_MEM:
        case NV2080_CTRL_CMD_TIMER_GET_TIME:
        case NV906F_CTRL_CMD_GET_MMU_FAULT_INFO:
        // Below 2 control calls are used for flip canceling (HW Flip Queue)
        // We use TRASH/ABORT mode to discard queued hw commands in the push buffer (bug 200644346)
        case NVC370_CTRL_CMD_SET_ACCL:
        case NVC370_CTRL_CMD_GET_CHANNEL_INFO:
        case NV2080_CTRL_CMD_BUS_SYSMEM_ACCESS:
        case NV9010_CTRL_CMD_SET_VBLANK_NOTIFICATION:
        case NV2080_CTRL_CMD_NVD_SET_NOCAT_JOURNAL_DATA:
            return NV_TRUE;
        default:
            return NV_FALSE;
    }
}

static NV_STATUS
_rmapiRmControl(NvHandle hClient, NvHandle hObject, NvU32 cmd, NvP64 pUserParams, NvU32 paramsSize, NvU32 flags, RM_API *pRmApi, API_SECURITY_INFO *pSecInfo)
{
    OBJSYS    *pSys = SYS_GET_INSTANCE();
    RmClient *pClient;
    RmCtrlParams rmCtrlParams;
    RS_CONTROL_COOKIE rmCtrlExecuteCookie = {0};
    NvBool bIsRaisedIrqlCmd;
    NvBool bIsLockBypassCmd;
    NvBool bInternalRequest;
    NV_STATUS  rmStatus = NV_OK;
    RS_LOCK_INFO        lockInfo = {0};

    RMTRACE_RMAPI(_RMCTRL_ENTRY, cmd);

    // Check first for the NULL command.
    // Return NV_OK immediately for NVXXXX_CTRL_CMD_NULL (0x00000000)
    // as well as the per-class NULL cmd ( _CATEGORY==0x00 and _INDEX==0x00 )
    if ((cmd == NVXXXX_CTRL_CMD_NULL) ||
        (FLD_TEST_DRF_NUM(XXXX, _CTRL_CMD, _CATEGORY, 0x00, cmd) &&
         FLD_TEST_DRF_NUM(XXXX, _CTRL_CMD, _INDEX,    0x00, cmd)))
    {
        return NV_OK;
    }

    NV_PRINTF(LEVEL_INFO,
              "rmControl: hClient 0x%x hObject 0x%x cmd 0x%x\n", hClient,
              hObject, cmd);

    NV_PRINTF(LEVEL_INFO, "rmControl: pUserParams 0x%p paramSize 0x%x\n",
              NvP64_VALUE(pUserParams), paramsSize);

    // If we're behind either API lock or GPU lock treat as internal.
    bInternalRequest = pRmApi->bApiLockInternal || pRmApi->bGpuLockInternal;

    // is this a raised IRQL cmd?
    bIsRaisedIrqlCmd = (flags & NVOS54_FLAGS_IRQL_RAISED);

    // is this a lock bypass cmd?
    bIsLockBypassCmd = ((flags & NVOS54_FLAGS_LOCK_BYPASS) || pRmApi->bGpuLockInternal);

    // NVOS54_FLAGS_IRQL_RAISED cmds are only allowed to be called in raised irq level.
    if (bIsRaisedIrqlCmd)
    {
        // Check that we support this control call at raised IRQL
        if (!_rmapiRmControlCanBeRaisedIrql(cmd))
        {
            NV_PRINTF(LEVEL_WARNING,
                      "rmControl:  cmd 0x%x cannot be called at raised irq level\n", cmd);
            rmStatus = NV_ERR_INVALID_ARGUMENT;
            goto done;
        }

        if (!osIsRaisedIRQL())
        {
            NV_PRINTF(LEVEL_WARNING,
                      "rmControl:  raised cmd 0x%x at normal irq level\n", cmd);
            rmStatus = NV_ERR_INVALID_ARGUMENT;
            goto done;
        }
    }

    if (bIsLockBypassCmd)
    {
        flags |= NVOS54_FLAGS_LOCK_BYPASS;

        if (!bInternalRequest)
        {
            // Check that we support bypassing locks with this control call
            if (!_rmapiRmControlCanBeBypassLock(cmd))
            {
                NV_PRINTF(LEVEL_WARNING,
                          "rmControl:  cmd 0x%x cannot bypass locks\n", cmd);
                rmStatus = NV_ERR_INVALID_ARGUMENT;
                goto done;
            }
        }
    }

    // Potential race condition if run lockless?
    if (serverutilGetClientUnderLock(hClient, &pClient) != NV_OK)
    {
        rmStatus = NV_ERR_INVALID_CLIENT;
        goto done;
    }

    // only kernel clients can issue raised IRQL or lock bypass cmds
    // bypass client priv check for internal calls done on behalf of lower priv
    // clients
    if ((bIsRaisedIrqlCmd || bIsLockBypassCmd) && !bInternalRequest)
    {
        if (pSecInfo->privLevel < RS_PRIV_LEVEL_KERNEL)
        {
            rmStatus = NV_ERR_INVALID_CLIENT;
            goto done;
        }
    }

    // error check parameters
    if (((paramsSize != 0) && (pUserParams == (NvP64) 0))   ||
        ((paramsSize == 0) && (pUserParams != (NvP64) 0)))
    {
        NV_PRINTF(LEVEL_WARNING, "bad params: ptr " NvP64_fmt " size: 0x%x\n",
                  pUserParams, paramsSize);
        rmStatus = NV_ERR_INVALID_ARGUMENT;
        goto done;
    }

    // init RmCtrlParams
    portMemSet(&rmCtrlParams, 0, sizeof(rmCtrlParams));
    rmCtrlParams.hClient = hClient;
    rmCtrlParams.hObject = hObject;
    rmCtrlParams.cmd = cmd;
    rmCtrlParams.flags = flags;
    rmCtrlParams.pParams = NvP64_VALUE(pUserParams);
    rmCtrlParams.paramsSize = paramsSize;
    rmCtrlParams.hParent = NV01_NULL_OBJECT;
    rmCtrlParams.pGpu = NULL;
    rmCtrlParams.pResourceRef = NULL;
    rmCtrlParams.secInfo = *pSecInfo;
    rmCtrlParams.pLockInfo = &lockInfo;
    rmCtrlParams.pCookie = &rmCtrlExecuteCookie;
    rmCtrlParams.bInternal = bInternalRequest;

    if (pRmApi->bApiLockInternal)
    {
        lockInfo.state |= RM_LOCK_STATES_API_LOCK_ACQUIRED;
        lockInfo.flags |= RM_LOCK_FLAGS_NO_API_LOCK;
    }

    //
    // Three separate rmctrl command modes:
    //
    //  mode#1: lock bypass rmctrl request
    //  mode#2: raised-irql rmctrl request
    //  mode#3: normal rmctrl request
    //
    if (bIsLockBypassCmd)
    {
        lockInfo.state |= RM_LOCK_STATES_API_LOCK_ACQUIRED;
        lockInfo.flags |= RM_LOCK_FLAGS_NO_API_LOCK |
                          RM_LOCK_FLAGS_NO_GPUS_LOCK |
                          RM_LOCK_FLAGS_NO_CLIENT_LOCK;

        //
        // Lock bypass rmctrl request.
        //
        rmStatus = serverControl(&g_resServ, &rmCtrlParams);
    }
    else if (bIsRaisedIrqlCmd)
    {
        //
        // Raised IRQL rmctrl request.
        //
        // Try to get lock; if we cannot get it then place on deferred queue.
        //

        // LOCK: try to acquire GPUs lock
        if (osCondAcquireRmSema(pSys->pSema) == NV_OK)
        {
            if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_COND_ACQUIRE, RM_LOCK_MODULES_CLIENT) == NV_OK)
            {
                lockInfo.state |= RM_LOCK_STATES_GPUS_LOCK_ACQUIRED;
                lockInfo.flags |= RM_LOCK_FLAGS_NO_API_LOCK |
                                  RM_LOCK_FLAGS_NO_GPUS_LOCK |
                                  RM_LOCK_FLAGS_NO_CLIENT_LOCK;
                rmStatus = serverControl(&g_resServ, &rmCtrlParams);

                // UNLOCK: release GPUs lock
                rmGpuLocksRelease(GPUS_LOCK_FLAGS_COND_ACQUIRE, osIsISR() ? rmCtrlParams.pGpu : NULL);
            }
            else
            {
                rmStatus = _rmControlDeferred(&rmCtrlParams, pUserParams, paramsSize);
            }
            // we must have a pGpu here for queuing of a DPC.
            NV_ASSERT(!osIsISR() || rmCtrlParams.pGpu);
            osReleaseRmSema(pSys->pSema, osIsISR() ? rmCtrlParams.pGpu : NULL);
        }
        else
        {
            rmStatus = _rmControlDeferred(&rmCtrlParams, pUserParams, paramsSize);
        }
    }
    else
    {
        //
        // Normal rmctrl request.
        //

        RM_API_CONTEXT rmApiContext = {0};
        rmStatus = rmapiPrologue(pRmApi, &rmApiContext);
        if (rmStatus != NV_OK)
            goto done;

        lockInfo.flags |= RM_LOCK_FLAGS_RM_SEMA;
        rmStatus = serverControl(&g_resServ, &rmCtrlParams);
        rmapiEpilogue(pRmApi, &rmApiContext);
    }
done:

    RMTRACE_RMAPI(_RMCTRL_EXIT, cmd);
    return rmStatus;
}

// validate rmctrl flags
NV_STATUS serverControl_ValidateCookie
(
    RS_RES_CONTROL_PARAMS_INTERNAL *pRmCtrlParams,
    RS_CONTROL_COOKIE *pRmCtrlExecuteCookie
)
{
    NV_STATUS           status;

    if (g_resServ.bRsAccessEnabled)
    {
        if (pRmCtrlParams->pResourceRef != NULL)
        {
            //
            // Check that the invoking client has appropriate access rights
            // For control calls, the invoking client is the owner of the ref
            //
            status = rsAccessCheckRights(pRmCtrlParams->pResourceRef,
                                         pRmCtrlParams->pResourceRef->pClient,
                                         &pRmCtrlExecuteCookie->rightsRequired);
            if (status != NV_OK)
                return status;
        }
        else
        {
            // pResourceRef can be NULL when rmControlCmdExecute is manually
            // invoked from the deferred API path (see class5080DeferredApiV2).
            // For now, we skip performing any access right checks in this case.
        }
    }
    else
    {
        //
        // When access rights are disabled, any control calls that have the
        // *_IF_RS_ACCESS_DISABLED flags should be treated as if they were declared
        // with the corresponding flags
        //
        if ((pRmCtrlExecuteCookie->ctrlFlags & RMCTRL_FLAGS_PRIVILEGED_IF_RS_ACCESS_DISABLED) != 0)
        {
            pRmCtrlExecuteCookie->ctrlFlags |= RMCTRL_FLAGS_PRIVILEGED;
        }
    }

    if ((pRmCtrlExecuteCookie->ctrlFlags & RMCTRL_FLAGS_INTERNAL))
    {
        NvBool bInternalCall = pRmCtrlParams->bInternal;
        if (!bInternalCall)
            return NV_ERR_NOT_SUPPORTED;
    }

    if (pRmCtrlExecuteCookie->ctrlFlags & RMCTRL_FLAGS_PRIVILEGED)
    {
        //
        // Calls originating from usermode require admin perms while calls
        // originating from other kernel drivers are always allowed.
        //
        if ((pRmCtrlParams->secInfo.privLevel < RS_PRIV_LEVEL_USER_ROOT)
           )
        {
            NV_PRINTF(LEVEL_WARNING,
                      "hClient: 0x%08x, hObject 0x%08x, cmd 0x%08x: non-privileged context issued privileged cmd\n",
                      pRmCtrlParams->hClient, pRmCtrlParams->hObject,
                      pRmCtrlParams->cmd);
            return NV_ERR_INSUFFICIENT_PERMISSIONS;
        }
    }

    // permissions check for KERNEL_PRIVILEGED (default) unless NON_PRIVILEGED, PRIVILEGED or INTERNAL is specified
    if ( !(pRmCtrlExecuteCookie->ctrlFlags & (RMCTRL_FLAGS_NON_PRIVILEGED | RMCTRL_FLAGS_PRIVILEGED | RMCTRL_FLAGS_INTERNAL)))
    {
        if ((pRmCtrlParams->secInfo.privLevel < RS_PRIV_LEVEL_KERNEL)
           )
        {
            NV_PRINTF(LEVEL_WARNING,
                      "hClient: 0x%08x, hObject 0x%08x, cmd 0x%08x: non-kernel client issued kernel-only cmd\n",
                      pRmCtrlParams->hClient, pRmCtrlParams->hObject,
                      pRmCtrlParams->cmd);
            return NV_ERR_INSUFFICIENT_PERMISSIONS;
        }
    }

    // fail if GPU isn't ready
    if ((!(pRmCtrlExecuteCookie->ctrlFlags & RMCTRL_FLAGS_NO_GPUS_ACCESS)) && (pRmCtrlParams->pGpu != NULL))
    {
        API_GPU_FULL_POWER_SANITY_CHECK(pRmCtrlParams->pGpu, NV_FALSE,
            pRmCtrlExecuteCookie->ctrlFlags & RMCTRL_FLAGS_ALLOW_WITHOUT_SYSMEM_ACCESS);

        if ( ! API_GPU_ATTACHED_SANITY_CHECK(pRmCtrlParams->pGpu))
            return NV_ERR_GPU_IS_LOST;
    }

    if ((pRmCtrlParams->flags & NVOS54_FLAGS_IRQL_RAISED) &&
        (pRmCtrlParams->secInfo.paramLocation != PARAM_LOCATION_KERNEL))
    {
        return NV_ERR_INVALID_PARAMETER;
    }

    return NV_OK;
}

NV_STATUS
serverControlLookupLockFlags
(
    RsServer *pServer,
    RS_LOCK_ENUM lock,
    RmCtrlParams *pRmCtrlParams,
    RmCtrlExecuteCookie *pRmCtrlExecuteCookie,
    LOCK_ACCESS_TYPE *pAccess
)
{
    if (pAccess == NULL)
        return NV_ERR_INVALID_ARGUMENT;

    *pAccess = LOCK_ACCESS_WRITE;

    if (lock == RS_LOCK_TOP)
    {
        if (!serverSupportsReadOnlyLock(&g_resServ, RS_LOCK_TOP, RS_API_CTRL))
        {
            *pAccess = LOCK_ACCESS_WRITE;
            return NV_OK;
        }

        if (pRmCtrlExecuteCookie->ctrlFlags & RMCTRL_FLAGS_API_LOCK_READONLY)
            *pAccess = LOCK_ACCESS_READ;

        return NV_OK;
    }

    if (lock == RS_LOCK_RESOURCE)
    {
        RS_LOCK_INFO *pLockInfo = pRmCtrlParams->pLockInfo;

        //
        // Do not acquire the GPU lock if we were explicitly told
        // not to or if this is an Internal Call meaning that
        // we already own the GPUs Lock.
        //
        if  ((pLockInfo->state & RM_LOCK_STATES_GPUS_LOCK_ACQUIRED) ||
             (pRmCtrlExecuteCookie->ctrlFlags & RMCTRL_FLAGS_NO_GPUS_LOCK) ||
             (pRmCtrlParams->flags & NVOS54_FLAGS_IRQL_RAISED) ||
             (pRmCtrlParams->flags & NVOS54_FLAGS_LOCK_BYPASS))
        {
            pLockInfo->flags |= RM_LOCK_FLAGS_NO_GPUS_LOCK;
            pLockInfo->flags &= ~RM_LOCK_FLAGS_GPU_GROUP_LOCK;
        }
        else
        {
            if (pRmCtrlExecuteCookie->ctrlFlags & RMCTRL_FLAGS_GPU_LOCK_DEVICE_ONLY)
            {
                pLockInfo->flags |= RM_LOCK_FLAGS_NO_GPUS_LOCK;
                pLockInfo->flags |= RM_LOCK_FLAGS_GPU_GROUP_LOCK;
            }
            else
            {
                pLockInfo->flags &= ~RM_LOCK_FLAGS_NO_GPUS_LOCK;
                pLockInfo->flags &= ~RM_LOCK_FLAGS_GPU_GROUP_LOCK;
            }

            if (pRmCtrlExecuteCookie->ctrlFlags & RMCTRL_FLAGS_GPU_LOCK_READONLY)
                *pAccess = LOCK_ACCESS_READ;
        }

        return NV_OK;
    }

    return NV_ERR_NOT_SUPPORTED;
}

NV_STATUS
rmapiControl
(
    RM_API   *pRmApi,
    NvHandle  hClient,
    NvHandle  hObject,
    NvU32     cmd,
    void     *pParams,
    NvU32     paramsSize
)
{
    if (!pRmApi->bHasDefaultSecInfo)
        return NV_ERR_NOT_SUPPORTED;

    return pRmApi->ControlWithSecInfo(pRmApi, hClient, hObject, cmd, NV_PTR_TO_NvP64(pParams),
                                      paramsSize, 0, &pRmApi->defaultSecInfo);
}

NV_STATUS
rmapiControlWithSecInfo
(
    RM_API            *pRmApi,
    NvHandle           hClient,
    NvHandle           hObject,
    NvU32              cmd,
    NvP64              pParams,
    NvU32              paramsSize,
    NvU32              flags,
    API_SECURITY_INFO *pSecInfo
)
{
    NV_STATUS status;

    NV_PRINTF(LEVEL_INFO,
              "Nv04Control: hClient:0x%x hObject:0x%x cmd:0x%x params:" NvP64_fmt " paramSize:0x%x flags:0x%x\n",
              hClient, hObject, cmd, pParams, paramsSize, flags);

    NVRM_TRACE_API('CTRL', hClient, hObject, cmd);

    status = _rmapiRmControl(hClient, hObject, cmd, pParams, paramsSize, flags, pRmApi, pSecInfo);

    if (status == NV_OK)
    {
        NV_PRINTF(LEVEL_INFO, "Nv04Control: control complete\n");
        NVRM_TRACE('ctrl');
    }
    else
    {
        NV_PRINTF(LEVEL_INFO,
                  "Nv04Control: control failed; status: %s (0x%08x)\n",
                  nvstatusToString(status), status);
        NV_PRINTF(LEVEL_INFO,
                  "Nv04Control:  hClient:0x%x hObject:0x%x cmd:0x%x params:" NvP64_fmt " paramSize:0x%x flags:0x%x\n",
                  hClient, hObject, cmd, pParams, paramsSize, flags);
        NVRM_TRACE_ERROR('ctrl', status);
    }

    return status;
}


//
// Called at DIRQL, where we can't do memory allocations
// Do not inline that function to save stack space
//
static NV_NOINLINE NV_STATUS
_rmapiControlWithSecInfoTlsIRQL
(
    RM_API* pRmApi,
    NvHandle           hClient,
    NvHandle           hObject,
    NvU32              cmd,
    NvP64              pParams,
    NvU32              paramsSize,
    NvU32              flags,
    API_SECURITY_INFO* pSecInfo
)
{
    NV_STATUS           status;
    THREAD_STATE_NODE   threadState;

    NvU8                stackAllocator[TLS_ISR_ALLOCATOR_SIZE];
    PORT_MEM_ALLOCATOR* pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator));
    tlsIsrInit(pIsrAllocator);

    //
    // SMP synchronization for Nv04Control is handled lower in the
    // call sequence to accommodate the various operation-specific
    // lock requirements (e.g. some operations can run locklessly).
    //
    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);

    status = rmapiControlWithSecInfo(pRmApi, hClient, hObject, cmd, pParams, paramsSize, flags, pSecInfo);

    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);

    tlsIsrDestroy(pIsrAllocator);
    portMemAllocatorRelease(pIsrAllocator);

    return status;
}


NV_STATUS
rmapiControlWithSecInfoTls
(
    RM_API            *pRmApi,
    NvHandle           hClient,
    NvHandle           hObject,
    NvU32              cmd,
    NvP64              pParams,
    NvU32              paramsSize,
    NvU32              flags,
    API_SECURITY_INFO *pSecInfo
)
{
    NV_STATUS           status;
    THREAD_STATE_NODE   threadState;

    if (!portMemExSafeForNonPagedAlloc())
    {
        return _rmapiControlWithSecInfoTlsIRQL(pRmApi, hClient, hObject, cmd, pParams, paramsSize, flags, pSecInfo);
    }

    //
    // SMP synchronization for Nv04Control is handled lower in the
    // call sequence to accommodate the various operation-specific
    // lock requirements (e.g. some operations can run locklessly).
    //
    threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);

    status = rmapiControlWithSecInfo(pRmApi, hClient, hObject, cmd, pParams, paramsSize, flags, pSecInfo);

    threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);

    return status;
}

