/* _NVRM_COPYRIGHT_BEGIN_
 *
 * Copyright 1999-2017 by NVIDIA Corporation.  All rights reserved.  All
 * information contained herein is proprietary and confidential to NVIDIA
 * Corporation.  Any use, reproduction, or disclosure without the written
 * permission of NVIDIA Corporation is prohibited.
 *
 * _NVRM_COPYRIGHT_END_
 */

#define  __NO_VERSION__
#include "nv-misc.h"

#include "os-interface.h"
#include "nv-linux.h"
#include "nv-reg.h"
#include "nv-pat.h"

int nv_pat_mode = NV_PAT_MODE_DISABLED;

#if defined(NV_ENABLE_PAT_SUPPORT)
/*
 * Private PAT support for use by the NVIDIA driver. This is used on
 * kernels that do not modify the PAT to include a write-combining
 * entry.
 */
static unsigned long orig_pat1, orig_pat2;

#define NV_READ_PAT_ENTRIES(pat1, pat2)   rdmsr(0x277, (pat1), (pat2))
#define NV_WRITE_PAT_ENTRIES(pat1, pat2)  wrmsr(0x277, (pat1), (pat2))
#define NV_PAT_ENTRY(pat, index) \
    (((pat) & (0xff << ((index)*8))) >> ((index)*8))

static inline void nv_disable_caches(unsigned long *cr4)
{
    unsigned long cr0 = read_cr0();
    write_cr0(((cr0 & (0xdfffffff)) | 0x40000000));
    wbinvd();
    *cr4 = NV_READ_CR4();
    if (*cr4 & 0x80) NV_WRITE_CR4(*cr4 & ~0x80);
    __flush_tlb();
}

static inline void nv_enable_caches(unsigned long cr4)
{
    unsigned long cr0 = read_cr0();
    wbinvd();
    __flush_tlb();
    write_cr0((cr0 & 0x9fffffff));
    if (cr4 & 0x80) NV_WRITE_CR4(cr4);
}

static int nv_determine_pat_mode(void)
{
    unsigned int pat1, pat2, i;
    NvU8 PAT_WC_index;

    if (!test_bit(X86_FEATURE_PAT,
            (volatile unsigned long *)&boot_cpu_data.x86_capability))
    {
        if ((boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) ||
                (boot_cpu_data.cpuid_level < 1) ||
                ((cpuid_edx(1) & (1 << 16)) == 0) ||
                (boot_cpu_data.x86 != 6) || (boot_cpu_data.x86_model >= 15))
        {
            nv_printf(NV_DBG_ERRORS,
                "NVRM: CPU does not support the PAT.\n");
            return NV_PAT_MODE_DISABLED;
        }
    }

    NV_READ_PAT_ENTRIES(pat1, pat2);
    PAT_WC_index = 0xf;

    for (i = 0; i < 4; i++)
    {
        if (NV_PAT_ENTRY(pat1, i) == 0x01)
        {
            PAT_WC_index = i;
            break;
        }

        if (NV_PAT_ENTRY(pat2, i) == 0x01)
        {
            PAT_WC_index = (i + 4);
            break;
        }
    }

    if (PAT_WC_index == 1)
        return NV_PAT_MODE_KERNEL;
    else if (PAT_WC_index != 0xf)
    {
        nv_printf(NV_DBG_ERRORS,
            "NVRM: PAT configuration unsupported.\n");
        return NV_PAT_MODE_DISABLED;
    }
    else
        return NV_PAT_MODE_BUILTIN;
}

static void nv_setup_pat_entries(void *info)
{
    unsigned long pat1, pat2, cr4;
    unsigned long eflags;

#if defined(NV_ENABLE_HOTPLUG_CPU)
    int cpu = (NvUPtr)info;
    if ((cpu != 0) && (cpu != (int)smp_processor_id()))
        return;
#endif

    NV_SAVE_FLAGS(eflags);
    NV_CLI();
    nv_disable_caches(&cr4);

    NV_READ_PAT_ENTRIES(pat1, pat2);

    pat1 &= 0xffff00ff;
    pat1 |= 0x00000100;

    NV_WRITE_PAT_ENTRIES(pat1, pat2);

    nv_enable_caches(cr4);
    NV_RESTORE_FLAGS(eflags);
}

static void nv_restore_pat_entries(void *info)
{
    unsigned long cr4;
    unsigned long eflags;

#if defined(NV_ENABLE_HOTPLUG_CPU)
    int cpu = (NvUPtr)info;
    if ((cpu != 0) && (cpu != (int)smp_processor_id()))
        return;
#endif

    NV_SAVE_FLAGS(eflags);
    NV_CLI();
    nv_disable_caches(&cr4);

    NV_WRITE_PAT_ENTRIES(orig_pat1, orig_pat2);

    nv_enable_caches(cr4);
    NV_RESTORE_FLAGS(eflags);
}

/*
 * NOTE 1:
 * Functions register_cpu_notifier(), unregister_cpu_notifier(),
 * macros register_hotcpu_notifier, register_hotcpu_notifier, 
 * and CPU states CPU_DOWN_FAILED, CPU_DOWN_PREPARE
 * were removed by the following commit:
 *   2016 Dec 25: b272f732f888d4cf43c943a40c9aaa836f9b7431
 * 
 * NV_REGISTER_CPU_NOTIFIER_PRESENT is true when
 * register_cpu_notifier() is present. 
 * 
 * The functions cpuhp_setup_state() and cpuhp_remove_state() should be
 * used as an alternative to register_cpu_notifier() and
 * unregister_cpu_notifier() functions. The following
 * commit introduced these functions as well as the enum cpuhp_state.
 *   2016 Feb 26: 5b7aa87e0482be768486e0c2277aa4122487eb9d 
 *
 * NV_CPUHP_CPUHP_STATE_PRESENT is true when cpuhp_setup_state() is present.
 *
 * For kernels where both cpuhp_setup_state() and register_cpu_notifier()
 * are present, we still use register_cpu_notifier().
 */

static int
nvidia_cpu_teardown(unsigned int cpu)
{
#if defined(NV_ENABLE_HOTPLUG_CPU)
    unsigned int this_cpu = get_cpu();

    if (this_cpu == cpu)
        nv_restore_pat_entries(NULL);
    else
        NV_SMP_CALL_FUNCTION(nv_restore_pat_entries, &cpu, 1);

    put_cpu();
#endif
    return 0;
}

static int
nvidia_cpu_online(unsigned int cpu)
{
#if defined(NV_ENABLE_HOTPLUG_CPU)
    unsigned int this_cpu = get_cpu();

    if (this_cpu == cpu)
        nv_setup_pat_entries(NULL);
    else
        NV_SMP_CALL_FUNCTION(nv_setup_pat_entries, &cpu, 1);

    put_cpu();
#endif
    return 0;
}


int nv_enable_pat_support(void)
{
    unsigned long pat1, pat2;

    if (nv_pat_mode != NV_PAT_MODE_DISABLED)
        return 1;

    nv_pat_mode = nv_determine_pat_mode();

    switch (nv_pat_mode)
    {
        case NV_PAT_MODE_DISABLED:
            /* avoid the PAT if unavailable/unusable */
            return 0;
        case NV_PAT_MODE_KERNEL:
            /* inherit the kernel's PAT layout */
            return 1;
        case NV_PAT_MODE_BUILTIN:
            /* use builtin code to modify the PAT layout */
            break;
    }

    NV_READ_PAT_ENTRIES(orig_pat1, orig_pat2);
    nv_printf(NV_DBG_SETUP, "saved orig pats as 0x%lx 0x%lx\n", orig_pat1, orig_pat2);

    if (nv_execute_on_all_cpus(nv_setup_pat_entries, NULL) != 0)
    {
        nv_execute_on_all_cpus(nv_restore_pat_entries, NULL);
        return 0;
    }

    NV_READ_PAT_ENTRIES(pat1, pat2);
    nv_printf(NV_DBG_SETUP, "changed pats to 0x%lx 0x%lx\n", pat1, pat2);
    return 1;
}

void nv_disable_pat_support(void)
{
    unsigned long pat1, pat2;

    if (nv_pat_mode != NV_PAT_MODE_BUILTIN)
        return;

    if (nv_execute_on_all_cpus(nv_restore_pat_entries, NULL) != 0)
        return;

    nv_pat_mode = NV_PAT_MODE_DISABLED;

    NV_READ_PAT_ENTRIES(pat1, pat2);
    nv_printf(NV_DBG_SETUP, "restored orig pats as 0x%lx 0x%lx\n", pat1, pat2);
}

static int
nvidia_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
{
/* CPU_DOWN_FAILED was added by the following commit
 *   2004 Oct 18: 71da3667be80d30121df3972caa0bf5684228379
 *
 * CPU_DOWN_PREPARE was added by the following commit
 *   2004 Oct 18: d13d28de21d913aacd3c91e76e307fa2eb7835d8
 *
 * We use one ifdef for both macros since they were added on the same day.
 */
#if defined(CPU_DOWN_FAILED)
    switch (action)
    {
        case CPU_DOWN_FAILED:
        case CPU_ONLINE:
            nvidia_cpu_online((NvUPtr)hcpu);
            break;
        case CPU_DOWN_PREPARE:
            nvidia_cpu_teardown((NvUPtr)hcpu);
            break;
    }
#endif
    return NOTIFY_OK;
}

/*
 * See NOTE 1. 
 * In order to avoid warnings for unused variable when compiling against
 * kernel versions which include changes of commit id 
 * b272f732f888d4cf43c943a40c9aaa836f9b7431, we have to protect declaration
 * of nv_hotcpu_nfb with #if.
 * 
 * NV_REGISTER_CPU_NOTIFIER_PRESENT is checked before
 * NV_CPUHP_SETUP_STATE_PRESENT to avoid compilation warnings for unused
 * variable nvidia_pat_online for kernels where both 
 * NV_REGISTER_CPU_NOTIFIER_PRESENT and NV_CPUHP_SETUP_STATE_PRESENT
 * are true.
 */
#if defined(NV_REGISTER_CPU_NOTIFIER_PRESENT)
static struct notifier_block nv_hotcpu_nfb = {
    .notifier_call = nvidia_cpu_callback,
    .priority = 0
};
#elif defined(NV_CPUHP_SETUP_STATE_PRESENT)
static enum cpuhp_state nvidia_pat_online;
#endif

static int
nvidia_register_cpu_hotplug_notifier(void)
{
    int ret;
/* See NOTE 1 */
#if defined(NV_REGISTER_CPU_NOTIFIER_PRESENT)
    if ((ret = register_hotcpu_notifier(&nv_hotcpu_nfb)) != 0)
        goto error;
#elif defined(NV_CPUHP_SETUP_STATE_PRESENT)
    if ((ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
                                        "nvidia/pat:online",
                                        nvidia_cpu_online,
                                        nvidia_cpu_teardown)) < 0)
    {
        /* 
         * If cpuhp_setup_state() fails, the cpuhp_remove_state()
         * should never be called. If it gets called, we might remove
         * some other state. Hence, explicitly set
         * nvidia_pat_online to zero. This will trigger a BUG()
         * in cpuhp_remove_state().
         */
        nvidia_pat_online = 0;
        goto error; 
    }
    nvidia_pat_online = ret;
#else
    goto error;
#endif
    return 0;

error:
    nv_disable_pat_support();
    nv_printf(NV_DBG_ERRORS,
        "NVRM: CPU hotplug notifier registration failed!\n");
    return -EIO;
}

static void
nvidia_unregister_cpu_hotplug_notifier(void)
{
/* See NOTE 1 */
#if defined(NV_REGISTER_CPU_NOTIFIER_PRESENT)
    unregister_hotcpu_notifier(&nv_hotcpu_nfb);
#elif defined(NV_CPUHP_SETUP_STATE_PRESENT)
    cpuhp_remove_state(nvidia_pat_online);
#endif
    return; 
}

int nv_init_pat_support(nv_stack_t *sp)
{
    RM_STATUS status;
    NvU32 data;
    int disable_pat = 0;
    int ret = 0;

    status = rm_read_registry_dword(sp, NULL,
            "NVreg", NV_USE_PAGE_ATTRIBUTE_TABLE, &data);
    if ((status == RM_OK) && ((int)data != ~0))
    {
        disable_pat = (data == 0);
    }

    if (!disable_pat)
    {
        nv_enable_pat_support();
        if (nv_pat_mode == NV_PAT_MODE_BUILTIN)
        {
             ret = nvidia_register_cpu_hotplug_notifier();
             return ret;
        }
    }
    else
    {
        nv_printf(NV_DBG_ERRORS,
            "NVRM: builtin PAT support disabled.\n");
    }

    return 0;
}

void nv_teardown_pat_support(void)
{
    if (nv_pat_mode == NV_PAT_MODE_BUILTIN)
    {
        nv_disable_pat_support();
        nvidia_unregister_cpu_hotplug_notifier();
    }
}
#endif /* end of NV_ENABLE_PAT_SUPPORT */
