1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
  24  * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
  25  * Copyright 2014 Josef "Jeff" Sipek <jeffpc@josefsipek.net>
  26  */
  27 /*
  28  * Copyright (c) 2010, Intel Corporation.
  29  * All rights reserved.
  30  */
  31 /*
  32  * Portions Copyright 2009 Advanced Micro Devices, Inc.
  33  */
  34 /*
  35  * Copyright 2018 Joyent, Inc.
  36  */
  37 /*
  38  * Various routines to handle identification
  39  * and classification of x86 processors.
  40  */
  41 
  42 #include <sys/types.h>
  43 #include <sys/archsystm.h>
  44 #include <sys/x86_archext.h>
  45 #include <sys/kmem.h>
  46 #include <sys/systm.h>
  47 #include <sys/cmn_err.h>
  48 #include <sys/sunddi.h>
  49 #include <sys/sunndi.h>
  50 #include <sys/cpuvar.h>
  51 #include <sys/processor.h>
  52 #include <sys/sysmacros.h>
  53 #include <sys/pg.h>
  54 #include <sys/fp.h>
  55 #include <sys/controlregs.h>
  56 #include <sys/bitmap.h>
  57 #include <sys/auxv_386.h>
  58 #include <sys/memnode.h>
  59 #include <sys/pci_cfgspace.h>
  60 #include <sys/comm_page.h>
  61 #include <sys/mach_mmu.h>
  62 #include <sys/tsc.h>
  63 
  64 #ifdef __xpv
  65 #include <sys/hypervisor.h>
  66 #else
  67 #include <sys/ontrap.h>
  68 #endif
  69 
  70 /*
  71  * Pass 0 of cpuid feature analysis happens in locore. It contains special code
  72  * to recognize Cyrix processors that are not cpuid-compliant, and to deal with
  73  * them accordingly. For most modern processors, feature detection occurs here
  74  * in pass 1.
  75  *
  76  * Pass 1 of cpuid feature analysis happens just at the beginning of mlsetup()
  77  * for the boot CPU and does the basic analysis that the early kernel needs.
  78  * x86_featureset is set based on the return value of cpuid_pass1() of the boot
  79  * CPU.
  80  *
  81  * Pass 1 includes:
  82  *
  83  *      o Determining vendor/model/family/stepping and setting x86_type and
  84  *        x86_vendor accordingly.
  85  *      o Processing the feature flags returned by the cpuid instruction while
  86  *        applying any workarounds or tricks for the specific processor.
  87  *      o Mapping the feature flags into illumos feature bits (X86_*).
  88  *      o Processing extended feature flags if supported by the processor,
  89  *        again while applying specific processor knowledge.
  90  *      o Determining the CMT characteristics of the system.
  91  *
  92  * Pass 1 is done on non-boot CPUs during their initialization and the results
  93  * are used only as a meager attempt at ensuring that all processors within the
  94  * system support the same features.
  95  *
  96  * Pass 2 of cpuid feature analysis happens just at the beginning
  97  * of startup().  It just copies in and corrects the remainder
  98  * of the cpuid data we depend on: standard cpuid functions that we didn't
  99  * need for pass1 feature analysis, and extended cpuid functions beyond the
 100  * simple feature processing done in pass1.
 101  *
 102  * Pass 3 of cpuid analysis is invoked after basic kernel services; in
 103  * particular kernel memory allocation has been made available. It creates a
 104  * readable brand string based on the data collected in the first two passes.
 105  *
 106  * Pass 4 of cpuid analysis is invoked after post_startup() when all
 107  * the support infrastructure for various hardware features has been
 108  * initialized. It determines which processor features will be reported
 109  * to userland via the aux vector.
 110  *
 111  * All passes are executed on all CPUs, but only the boot CPU determines what
 112  * features the kernel will use.
 113  *
 114  * Much of the worst junk in this file is for the support of processors
 115  * that didn't really implement the cpuid instruction properly.
 116  *
 117  * NOTE: The accessor functions (cpuid_get*) are aware of, and ASSERT upon,
 118  * the pass numbers.  Accordingly, changes to the pass code may require changes
 119  * to the accessor code.
 120  */
 121 
 122 uint_t x86_vendor = X86_VENDOR_IntelClone;
 123 uint_t x86_type = X86_TYPE_OTHER;
 124 uint_t x86_clflush_size = 0;
 125 
 126 uint_t x86_use_pcid = 1;
 127 uint_t x86_use_invpcid = 1;
 128 
 129 uint_t pentiumpro_bug4046376;
 130 
 131 uchar_t x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
 132 
 133 static char *x86_feature_names[NUM_X86_FEATURES] = {
 134         "lgpg",
 135         "tsc",
 136         "msr",
 137         "mtrr",
 138         "pge",
 139         "de",
 140         "cmov",
 141         "mmx",
 142         "mca",
 143         "pae",
 144         "cv8",
 145         "pat",
 146         "sep",
 147         "sse",
 148         "sse2",
 149         "htt",
 150         "asysc",
 151         "nx",
 152         "sse3",
 153         "cx16",
 154         "cmp",
 155         "tscp",
 156         "mwait",
 157         "sse4a",
 158         "cpuid",
 159         "ssse3",
 160         "sse4_1",
 161         "sse4_2",
 162         "1gpg",
 163         "clfsh",
 164         "64",
 165         "aes",
 166         "pclmulqdq",
 167         "xsave",
 168         "avx",
 169         "vmx",
 170         "svm",
 171         "topoext",
 172         "f16c",
 173         "rdrand",
 174         "x2apic",
 175         "avx2",
 176         "bmi1",
 177         "bmi2",
 178         "fma",
 179         "smep",
 180         "smap",
 181         "adx",
 182         "rdseed",
 183         "mpx",
 184         "avx512f",
 185         "avx512dq",
 186         "avx512pf",
 187         "avx512er",
 188         "avx512cd",
 189         "avx512bw",
 190         "avx512vl",
 191         "avx512fma",
 192         "avx512vbmi",
 193         "avx512_vpopcntdq",
 194         "avx512_4vnniw",
 195         "avx512_4fmaps",
 196         "xsaveopt",
 197         "xsavec",
 198         "xsaves",
 199         "sha",
 200         "umip",
 201         "pku",
 202         "ospke",
 203         "pcid",
 204         "invpcid",
 205 };
 206 
 207 boolean_t
 208 is_x86_feature(void *featureset, uint_t feature)
 209 {
 210         ASSERT(feature < NUM_X86_FEATURES);
 211         return (BT_TEST((ulong_t *)featureset, feature));
 212 }
 213 
 214 void
 215 add_x86_feature(void *featureset, uint_t feature)
 216 {
 217         ASSERT(feature < NUM_X86_FEATURES);
 218         BT_SET((ulong_t *)featureset, feature);
 219 }
 220 
 221 void
 222 remove_x86_feature(void *featureset, uint_t feature)
 223 {
 224         ASSERT(feature < NUM_X86_FEATURES);
 225         BT_CLEAR((ulong_t *)featureset, feature);
 226 }
 227 
 228 boolean_t
 229 compare_x86_featureset(void *setA, void *setB)
 230 {
 231         /*
 232          * We assume that the unused bits of the bitmap are always zero.
 233          */
 234         if (memcmp(setA, setB, BT_SIZEOFMAP(NUM_X86_FEATURES)) == 0) {
 235                 return (B_TRUE);
 236         } else {
 237                 return (B_FALSE);
 238         }
 239 }
 240 
 241 void
 242 print_x86_featureset(void *featureset)
 243 {
 244         uint_t i;
 245 
 246         for (i = 0; i < NUM_X86_FEATURES; i++) {
 247                 if (is_x86_feature(featureset, i)) {
 248                         cmn_err(CE_CONT, "?x86_feature: %s\n",
 249                             x86_feature_names[i]);
 250                 }
 251         }
 252 }
 253 
 254 /* Note: This is the maximum size for the CPU, not the size of the structure. */
 255 static size_t xsave_state_size = 0;
 256 uint64_t xsave_bv_all = (XFEATURE_LEGACY_FP | XFEATURE_SSE);
 257 boolean_t xsave_force_disable = B_FALSE;
 258 extern int disable_smap;
 259 
 260 /*
 261  * This is set to platform type we are running on.
 262  */
 263 static int platform_type = -1;
 264 
 265 #if !defined(__xpv)
 266 /*
 267  * Variable to patch if hypervisor platform detection needs to be
 268  * disabled (e.g. platform_type will always be HW_NATIVE if this is 0).
 269  */
 270 int enable_platform_detection = 1;
 271 #endif
 272 
 273 /*
 274  * monitor/mwait info.
 275  *
 276  * size_actual and buf_actual are the real address and size allocated to get
 277  * proper mwait_buf alignement.  buf_actual and size_actual should be passed
 278  * to kmem_free().  Currently kmem_alloc() and mwait happen to both use
 279  * processor cache-line alignment, but this is not guarantied in the furture.
 280  */
 281 struct mwait_info {
 282         size_t          mon_min;        /* min size to avoid missed wakeups */
 283         size_t          mon_max;        /* size to avoid false wakeups */
 284         size_t          size_actual;    /* size actually allocated */
 285         void            *buf_actual;    /* memory actually allocated */
 286         uint32_t        support;        /* processor support of monitor/mwait */
 287 };
 288 
 289 /*
 290  * xsave/xrestor info.
 291  *
 292  * This structure contains HW feature bits and the size of the xsave save area.
 293  * Note: the kernel declares a fixed size (AVX_XSAVE_SIZE) structure
 294  * (xsave_state) to describe the xsave layout. However, at runtime the
 295  * per-lwp xsave area is dynamically allocated based on xsav_max_size. The
 296  * xsave_state structure simply represents the legacy layout of the beginning
 297  * of the xsave area.
 298  */
 299 struct xsave_info {
 300         uint32_t        xsav_hw_features_low;   /* Supported HW features */
 301         uint32_t        xsav_hw_features_high;  /* Supported HW features */
 302         size_t          xsav_max_size;  /* max size save area for HW features */
 303         size_t          ymm_size;       /* AVX: size of ymm save area */
 304         size_t          ymm_offset;     /* AVX: offset for ymm save area */
 305         size_t          bndregs_size;   /* MPX: size of bndregs save area */
 306         size_t          bndregs_offset; /* MPX: offset for bndregs save area */
 307         size_t          bndcsr_size;    /* MPX: size of bndcsr save area */
 308         size_t          bndcsr_offset;  /* MPX: offset for bndcsr save area */
 309         size_t          opmask_size;    /* AVX512: size of opmask save */
 310         size_t          opmask_offset;  /* AVX512: offset for opmask save */
 311         size_t          zmmlo_size;     /* AVX512: size of zmm 256 save */
 312         size_t          zmmlo_offset;   /* AVX512: offset for zmm 256 save */
 313         size_t          zmmhi_size;     /* AVX512: size of zmm hi reg save */
 314         size_t          zmmhi_offset;   /* AVX512: offset for zmm hi reg save */
 315 };
 316 
 317 
 318 /*
 319  * These constants determine how many of the elements of the
 320  * cpuid we cache in the cpuid_info data structure; the
 321  * remaining elements are accessible via the cpuid instruction.
 322  */
 323 
 324 #define NMAX_CPI_STD    8               /* eax = 0 .. 7 */
 325 #define NMAX_CPI_EXTD   0x1f            /* eax = 0x80000000 .. 0x8000001e */
 326 
 327 /*
 328  * Some terminology needs to be explained:
 329  *  - Socket: Something that can be plugged into a motherboard.
 330  *  - Package: Same as socket
 331  *  - Chip: Same as socket. Note that AMD's documentation uses term "chip"
 332  *    differently: there, chip is the same as processor node (below)
 333  *  - Processor node: Some AMD processors have more than one
 334  *    "subprocessor" embedded in a package. These subprocessors (nodes)
 335  *    are fully-functional processors themselves with cores, caches,
 336  *    memory controllers, PCI configuration spaces. They are connected
 337  *    inside the package with Hypertransport links. On single-node
 338  *    processors, processor node is equivalent to chip/socket/package.
 339  *  - Compute Unit: Some AMD processors pair cores in "compute units" that
 340  *    share the FPU and the I$ and L2 caches.
 341  */
 342 
 343 struct cpuid_info {
 344         uint_t cpi_pass;                /* last pass completed */
 345         /*
 346          * standard function information
 347          */
 348         uint_t cpi_maxeax;              /* fn 0: %eax */
 349         char cpi_vendorstr[13];         /* fn 0: %ebx:%ecx:%edx */
 350         uint_t cpi_vendor;              /* enum of cpi_vendorstr */
 351 
 352         uint_t cpi_family;              /* fn 1: extended family */
 353         uint_t cpi_model;               /* fn 1: extended model */
 354         uint_t cpi_step;                /* fn 1: stepping */
 355         chipid_t cpi_chipid;            /* fn 1: %ebx:  Intel: chip # */
 356                                         /*              AMD: package/socket # */
 357         uint_t cpi_brandid;             /* fn 1: %ebx: brand ID */
 358         int cpi_clogid;                 /* fn 1: %ebx: thread # */
 359         uint_t cpi_ncpu_per_chip;       /* fn 1: %ebx: logical cpu count */
 360         uint8_t cpi_cacheinfo[16];      /* fn 2: intel-style cache desc */
 361         uint_t cpi_ncache;              /* fn 2: number of elements */
 362         uint_t cpi_ncpu_shr_last_cache; /* fn 4: %eax: ncpus sharing cache */
 363         id_t cpi_last_lvl_cacheid;      /* fn 4: %eax: derived cache id */
 364         uint_t cpi_std_4_size;          /* fn 4: number of fn 4 elements */
 365         struct cpuid_regs **cpi_std_4;  /* fn 4: %ecx == 0 .. fn4_size */
 366         struct cpuid_regs cpi_std[NMAX_CPI_STD];        /* 0 .. 7 */
 367         /*
 368          * extended function information
 369          */
 370         uint_t cpi_xmaxeax;             /* fn 0x80000000: %eax */
 371         char cpi_brandstr[49];          /* fn 0x8000000[234] */
 372         uint8_t cpi_pabits;             /* fn 0x80000006: %eax */
 373         uint8_t cpi_vabits;             /* fn 0x80000006: %eax */
 374         uint8_t cpi_fp_amd_save;        /* AMD: FP error pointer save rqd. */
 375         struct  cpuid_regs cpi_extd[NMAX_CPI_EXTD];     /* 0x800000XX */
 376 
 377         id_t cpi_coreid;                /* same coreid => strands share core */
 378         int cpi_pkgcoreid;              /* core number within single package */
 379         uint_t cpi_ncore_per_chip;      /* AMD: fn 0x80000008: %ecx[7-0] */
 380                                         /* Intel: fn 4: %eax[31-26] */
 381         /*
 382          * supported feature information
 383          */
 384         uint32_t cpi_support[6];
 385 #define STD_EDX_FEATURES        0
 386 #define AMD_EDX_FEATURES        1
 387 #define TM_EDX_FEATURES         2
 388 #define STD_ECX_FEATURES        3
 389 #define AMD_ECX_FEATURES        4
 390 #define STD_EBX_FEATURES        5
 391         /*
 392          * Synthesized information, where known.
 393          */
 394         uint32_t cpi_chiprev;           /* See X86_CHIPREV_* in x86_archext.h */
 395         const char *cpi_chiprevstr;     /* May be NULL if chiprev unknown */
 396         uint32_t cpi_socket;            /* Chip package/socket type */
 397 
 398         struct mwait_info cpi_mwait;    /* fn 5: monitor/mwait info */
 399         uint32_t cpi_apicid;
 400         uint_t cpi_procnodeid;          /* AMD: nodeID on HT, Intel: chipid */
 401         uint_t cpi_procnodes_per_pkg;   /* AMD: # of nodes in the package */
 402                                         /* Intel: 1 */
 403         uint_t cpi_compunitid;          /* AMD: ComputeUnit ID, Intel: coreid */
 404         uint_t cpi_cores_per_compunit;  /* AMD: # of cores in the ComputeUnit */
 405 
 406         struct xsave_info cpi_xsave;    /* fn D: xsave/xrestor info */
 407 };
 408 
 409 
 410 static struct cpuid_info cpuid_info0;
 411 
 412 /*
 413  * These bit fields are defined by the Intel Application Note AP-485
 414  * "Intel Processor Identification and the CPUID Instruction"
 415  */
 416 #define CPI_FAMILY_XTD(cpi)     BITX((cpi)->cpi_std[1].cp_eax, 27, 20)
 417 #define CPI_MODEL_XTD(cpi)      BITX((cpi)->cpi_std[1].cp_eax, 19, 16)
 418 #define CPI_TYPE(cpi)           BITX((cpi)->cpi_std[1].cp_eax, 13, 12)
 419 #define CPI_FAMILY(cpi)         BITX((cpi)->cpi_std[1].cp_eax, 11, 8)
 420 #define CPI_STEP(cpi)           BITX((cpi)->cpi_std[1].cp_eax, 3, 0)
 421 #define CPI_MODEL(cpi)          BITX((cpi)->cpi_std[1].cp_eax, 7, 4)
 422 
 423 #define CPI_FEATURES_EDX(cpi)           ((cpi)->cpi_std[1].cp_edx)
 424 #define CPI_FEATURES_ECX(cpi)           ((cpi)->cpi_std[1].cp_ecx)
 425 #define CPI_FEATURES_XTD_EDX(cpi)       ((cpi)->cpi_extd[1].cp_edx)
 426 #define CPI_FEATURES_XTD_ECX(cpi)       ((cpi)->cpi_extd[1].cp_ecx)
 427 #define CPI_FEATURES_7_0_EBX(cpi)       ((cpi)->cpi_std[7].cp_ebx)
 428 #define CPI_FEATURES_7_0_ECX(cpi)       ((cpi)->cpi_std[7].cp_ecx)
 429 #define CPI_FEATURES_7_0_EDX(cpi)       ((cpi)->cpi_std[7].cp_edx)
 430 
 431 #define CPI_BRANDID(cpi)        BITX((cpi)->cpi_std[1].cp_ebx, 7, 0)
 432 #define CPI_CHUNKS(cpi)         BITX((cpi)->cpi_std[1].cp_ebx, 15, 7)
 433 #define CPI_CPU_COUNT(cpi)      BITX((cpi)->cpi_std[1].cp_ebx, 23, 16)
 434 #define CPI_APIC_ID(cpi)        BITX((cpi)->cpi_std[1].cp_ebx, 31, 24)
 435 
 436 #define CPI_MAXEAX_MAX          0x100           /* sanity control */
 437 #define CPI_XMAXEAX_MAX         0x80000100
 438 #define CPI_FN4_ECX_MAX         0x20            /* sanity: max fn 4 levels */
 439 #define CPI_FNB_ECX_MAX         0x20            /* sanity: max fn B levels */
 440 
 441 /*
 442  * Function 4 (Deterministic Cache Parameters) macros
 443  * Defined by Intel Application Note AP-485
 444  */
 445 #define CPI_NUM_CORES(regs)             BITX((regs)->cp_eax, 31, 26)
 446 #define CPI_NTHR_SHR_CACHE(regs)        BITX((regs)->cp_eax, 25, 14)
 447 #define CPI_FULL_ASSOC_CACHE(regs)      BITX((regs)->cp_eax, 9, 9)
 448 #define CPI_SELF_INIT_CACHE(regs)       BITX((regs)->cp_eax, 8, 8)
 449 #define CPI_CACHE_LVL(regs)             BITX((regs)->cp_eax, 7, 5)
 450 #define CPI_CACHE_TYPE(regs)            BITX((regs)->cp_eax, 4, 0)
 451 #define CPI_CPU_LEVEL_TYPE(regs)        BITX((regs)->cp_ecx, 15, 8)
 452 
 453 #define CPI_CACHE_WAYS(regs)            BITX((regs)->cp_ebx, 31, 22)
 454 #define CPI_CACHE_PARTS(regs)           BITX((regs)->cp_ebx, 21, 12)
 455 #define CPI_CACHE_COH_LN_SZ(regs)       BITX((regs)->cp_ebx, 11, 0)
 456 
 457 #define CPI_CACHE_SETS(regs)            BITX((regs)->cp_ecx, 31, 0)
 458 
 459 #define CPI_PREFCH_STRIDE(regs)         BITX((regs)->cp_edx, 9, 0)
 460 
 461 
 462 /*
 463  * A couple of shorthand macros to identify "later" P6-family chips
 464  * like the Pentium M and Core.  First, the "older" P6-based stuff
 465  * (loosely defined as "pre-Pentium-4"):
 466  * P6, PII, Mobile PII, PII Xeon, PIII, Mobile PIII, PIII Xeon
 467  */
 468 #define IS_LEGACY_P6(cpi) (                     \
 469         cpi->cpi_family == 6 &&                      \
 470                 (cpi->cpi_model == 1 ||              \
 471                 cpi->cpi_model == 3 ||               \
 472                 cpi->cpi_model == 5 ||               \
 473                 cpi->cpi_model == 6 ||               \
 474                 cpi->cpi_model == 7 ||               \
 475                 cpi->cpi_model == 8 ||               \
 476                 cpi->cpi_model == 0xA ||     \
 477                 cpi->cpi_model == 0xB)               \
 478 )
 479 
 480 /* A "new F6" is everything with family 6 that's not the above */
 481 #define IS_NEW_F6(cpi) ((cpi->cpi_family == 6) && !IS_LEGACY_P6(cpi))
 482 
 483 /* Extended family/model support */
 484 #define IS_EXTENDED_MODEL_INTEL(cpi) (cpi->cpi_family == 0x6 || \
 485         cpi->cpi_family >= 0xf)
 486 
 487 /*
 488  * Info for monitor/mwait idle loop.
 489  *
 490  * See cpuid section of "Intel 64 and IA-32 Architectures Software Developer's
 491  * Manual Volume 2A: Instruction Set Reference, A-M" #25366-022US, November
 492  * 2006.
 493  * See MONITOR/MWAIT section of "AMD64 Architecture Programmer's Manual
 494  * Documentation Updates" #33633, Rev 2.05, December 2006.
 495  */
 496 #define MWAIT_SUPPORT           (0x00000001)    /* mwait supported */
 497 #define MWAIT_EXTENSIONS        (0x00000002)    /* extenstion supported */
 498 #define MWAIT_ECX_INT_ENABLE    (0x00000004)    /* ecx 1 extension supported */
 499 #define MWAIT_SUPPORTED(cpi)    ((cpi)->cpi_std[1].cp_ecx & CPUID_INTC_ECX_MON)
 500 #define MWAIT_INT_ENABLE(cpi)   ((cpi)->cpi_std[5].cp_ecx & 0x2)
 501 #define MWAIT_EXTENSION(cpi)    ((cpi)->cpi_std[5].cp_ecx & 0x1)
 502 #define MWAIT_SIZE_MIN(cpi)     BITX((cpi)->cpi_std[5].cp_eax, 15, 0)
 503 #define MWAIT_SIZE_MAX(cpi)     BITX((cpi)->cpi_std[5].cp_ebx, 15, 0)
 504 /*
 505  * Number of sub-cstates for a given c-state.
 506  */
 507 #define MWAIT_NUM_SUBC_STATES(cpi, c_state)                     \
 508         BITX((cpi)->cpi_std[5].cp_edx, c_state + 3, c_state)
 509 
 510 /*
 511  * XSAVE leaf 0xD enumeration
 512  */
 513 #define CPUID_LEAFD_2_YMM_OFFSET        576
 514 #define CPUID_LEAFD_2_YMM_SIZE          256
 515 
 516 /*
 517  * Functions we consune from cpuid_subr.c;  don't publish these in a header
 518  * file to try and keep people using the expected cpuid_* interfaces.
 519  */
 520 extern uint32_t _cpuid_skt(uint_t, uint_t, uint_t, uint_t);
 521 extern const char *_cpuid_sktstr(uint_t, uint_t, uint_t, uint_t);
 522 extern uint32_t _cpuid_chiprev(uint_t, uint_t, uint_t, uint_t);
 523 extern const char *_cpuid_chiprevstr(uint_t, uint_t, uint_t, uint_t);
 524 extern uint_t _cpuid_vendorstr_to_vendorcode(char *);
 525 
 526 /*
 527  * Apply up various platform-dependent restrictions where the
 528  * underlying platform restrictions mean the CPU can be marked
 529  * as less capable than its cpuid instruction would imply.
 530  */
 531 #if defined(__xpv)
 532 static void
 533 platform_cpuid_mangle(uint_t vendor, uint32_t eax, struct cpuid_regs *cp)
 534 {
 535         switch (eax) {
 536         case 1: {
 537                 uint32_t mcamask = DOMAIN_IS_INITDOMAIN(xen_info) ?
 538                     0 : CPUID_INTC_EDX_MCA;
 539                 cp->cp_edx &=
 540                     ~(mcamask |
 541                     CPUID_INTC_EDX_PSE |
 542                     CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
 543                     CPUID_INTC_EDX_SEP | CPUID_INTC_EDX_MTRR |
 544                     CPUID_INTC_EDX_PGE | CPUID_INTC_EDX_PAT |
 545                     CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
 546                     CPUID_INTC_EDX_PSE36 | CPUID_INTC_EDX_HTT);
 547                 break;
 548         }
 549 
 550         case 0x80000001:
 551                 cp->cp_edx &=
 552                     ~(CPUID_AMD_EDX_PSE |
 553                     CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
 554                     CPUID_AMD_EDX_MTRR | CPUID_AMD_EDX_PGE |
 555                     CPUID_AMD_EDX_PAT | CPUID_AMD_EDX_PSE36 |
 556                     CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
 557                     CPUID_AMD_EDX_TSCP);
 558                 cp->cp_ecx &= ~CPUID_AMD_ECX_CMP_LGCY;
 559                 break;
 560         default:
 561                 break;
 562         }
 563 
 564         switch (vendor) {
 565         case X86_VENDOR_Intel:
 566                 switch (eax) {
 567                 case 4:
 568                         /*
 569                          * Zero out the (ncores-per-chip - 1) field
 570                          */
 571                         cp->cp_eax &= 0x03fffffff;
 572                         break;
 573                 default:
 574                         break;
 575                 }
 576                 break;
 577         case X86_VENDOR_AMD:
 578                 switch (eax) {
 579 
 580                 case 0x80000001:
 581                         cp->cp_ecx &= ~CPUID_AMD_ECX_CR8D;
 582                         break;
 583 
 584                 case 0x80000008:
 585                         /*
 586                          * Zero out the (ncores-per-chip - 1) field
 587                          */
 588                         cp->cp_ecx &= 0xffffff00;
 589                         break;
 590                 default:
 591                         break;
 592                 }
 593                 break;
 594         default:
 595                 break;
 596         }
 597 }
 598 #else
 599 #define platform_cpuid_mangle(vendor, eax, cp)  /* nothing */
 600 #endif
 601 
 602 /*
 603  *  Some undocumented ways of patching the results of the cpuid
 604  *  instruction to permit running Solaris 10 on future cpus that
 605  *  we don't currently support.  Could be set to non-zero values
 606  *  via settings in eeprom.
 607  */
 608 
 609 uint32_t cpuid_feature_ecx_include;
 610 uint32_t cpuid_feature_ecx_exclude;
 611 uint32_t cpuid_feature_edx_include;
 612 uint32_t cpuid_feature_edx_exclude;
 613 
 614 /*
 615  * Allocate space for mcpu_cpi in the machcpu structure for all non-boot CPUs.
 616  */
 617 void
 618 cpuid_alloc_space(cpu_t *cpu)
 619 {
 620         /*
 621          * By convention, cpu0 is the boot cpu, which is set up
 622          * before memory allocation is available.  All other cpus get
 623          * their cpuid_info struct allocated here.
 624          */
 625         ASSERT(cpu->cpu_id != 0);
 626         ASSERT(cpu->cpu_m.mcpu_cpi == NULL);
 627         cpu->cpu_m.mcpu_cpi =
 628             kmem_zalloc(sizeof (*cpu->cpu_m.mcpu_cpi), KM_SLEEP);
 629 }
 630 
 631 void
 632 cpuid_free_space(cpu_t *cpu)
 633 {
 634         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
 635         int i;
 636 
 637         ASSERT(cpi != NULL);
 638         ASSERT(cpi != &cpuid_info0);
 639 
 640         /*
 641          * Free up any function 4 related dynamic storage
 642          */
 643         for (i = 1; i < cpi->cpi_std_4_size; i++)
 644                 kmem_free(cpi->cpi_std_4[i], sizeof (struct cpuid_regs));
 645         if (cpi->cpi_std_4_size > 0)
 646                 kmem_free(cpi->cpi_std_4,
 647                     cpi->cpi_std_4_size * sizeof (struct cpuid_regs *));
 648 
 649         kmem_free(cpi, sizeof (*cpi));
 650         cpu->cpu_m.mcpu_cpi = NULL;
 651 }
 652 
 653 #if !defined(__xpv)
 654 /*
 655  * Determine the type of the underlying platform. This is used to customize
 656  * initialization of various subsystems (e.g. TSC). determine_platform() must
 657  * only ever be called once to prevent two processors from seeing different
 658  * values of platform_type. Must be called before cpuid_pass1(), the earliest
 659  * consumer to execute (uses _cpuid_chiprev --> synth_amd_info --> get_hwenv).
 660  */
 661 void
 662 determine_platform(void)
 663 {
 664         struct cpuid_regs cp;
 665         uint32_t base;
 666         uint32_t regs[4];
 667         char *hvstr = (char *)regs;
 668 
 669         ASSERT(platform_type == -1);
 670 
 671         platform_type = HW_NATIVE;
 672 
 673         if (!enable_platform_detection)
 674                 return;
 675 
 676         /*
 677          * If Hypervisor CPUID bit is set, try to determine hypervisor
 678          * vendor signature, and set platform type accordingly.
 679          *
 680          * References:
 681          * http://lkml.org/lkml/2008/10/1/246
 682          * http://kb.vmware.com/kb/1009458
 683          */
 684         cp.cp_eax = 0x1;
 685         (void) __cpuid_insn(&cp);
 686         if ((cp.cp_ecx & CPUID_INTC_ECX_HV) != 0) {
 687                 cp.cp_eax = 0x40000000;
 688                 (void) __cpuid_insn(&cp);
 689                 regs[0] = cp.cp_ebx;
 690                 regs[1] = cp.cp_ecx;
 691                 regs[2] = cp.cp_edx;
 692                 regs[3] = 0;
 693                 if (strcmp(hvstr, HVSIG_XEN_HVM) == 0) {
 694                         platform_type = HW_XEN_HVM;
 695                         return;
 696                 }
 697                 if (strcmp(hvstr, HVSIG_VMWARE) == 0) {
 698                         platform_type = HW_VMWARE;
 699                         return;
 700                 }
 701                 if (strcmp(hvstr, HVSIG_KVM) == 0) {
 702                         platform_type = HW_KVM;
 703                         return;
 704                 }
 705                 if (strcmp(hvstr, HVSIG_BHYVE) == 0) {
 706                         platform_type = HW_BHYVE;
 707                         return;
 708                 }
 709                 if (strcmp(hvstr, HVSIG_MICROSOFT) == 0)
 710                         platform_type = HW_MICROSOFT;
 711         } else {
 712                 /*
 713                  * Check older VMware hardware versions. VMware hypervisor is
 714                  * detected by performing an IN operation to VMware hypervisor
 715                  * port and checking that value returned in %ebx is VMware
 716                  * hypervisor magic value.
 717                  *
 718                  * References: http://kb.vmware.com/kb/1009458
 719                  */
 720                 vmware_port(VMWARE_HVCMD_GETVERSION, regs);
 721                 if (regs[1] == VMWARE_HVMAGIC) {
 722                         platform_type = HW_VMWARE;
 723                         return;
 724                 }
 725         }
 726 
 727         /*
 728          * Check Xen hypervisor. In a fully virtualized domain,
 729          * Xen's pseudo-cpuid function returns a string representing the
 730          * Xen signature in %ebx, %ecx, and %edx. %eax contains the maximum
 731          * supported cpuid function. We need at least a (base + 2) leaf value
 732          * to do what we want to do. Try different base values, since the
 733          * hypervisor might use a different one depending on whether Hyper-V
 734          * emulation is switched on by default or not.
 735          */
 736         for (base = 0x40000000; base < 0x40010000; base += 0x100) {
 737                 cp.cp_eax = base;
 738                 (void) __cpuid_insn(&cp);
 739                 regs[0] = cp.cp_ebx;
 740                 regs[1] = cp.cp_ecx;
 741                 regs[2] = cp.cp_edx;
 742                 regs[3] = 0;
 743                 if (strcmp(hvstr, HVSIG_XEN_HVM) == 0 &&
 744                     cp.cp_eax >= (base + 2)) {
 745                         platform_type &= ~HW_NATIVE;
 746                         platform_type |= HW_XEN_HVM;
 747                         return;
 748                 }
 749         }
 750 }
 751 
 752 int
 753 get_hwenv(void)
 754 {
 755         ASSERT(platform_type != -1);
 756         return (platform_type);
 757 }
 758 
 759 int
 760 is_controldom(void)
 761 {
 762         return (0);
 763 }
 764 
 765 #else
 766 
 767 int
 768 get_hwenv(void)
 769 {
 770         return (HW_XEN_PV);
 771 }
 772 
 773 int
 774 is_controldom(void)
 775 {
 776         return (DOMAIN_IS_INITDOMAIN(xen_info));
 777 }
 778 
 779 #endif  /* __xpv */
 780 
 781 static void
 782 cpuid_intel_getids(cpu_t *cpu, void *feature)
 783 {
 784         uint_t i;
 785         uint_t chipid_shift = 0;
 786         uint_t coreid_shift = 0;
 787         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
 788 
 789         for (i = 1; i < cpi->cpi_ncpu_per_chip; i <<= 1)
 790                 chipid_shift++;
 791 
 792         cpi->cpi_chipid = cpi->cpi_apicid >> chipid_shift;
 793         cpi->cpi_clogid = cpi->cpi_apicid & ((1 << chipid_shift) - 1);
 794 
 795         if (is_x86_feature(feature, X86FSET_CMP)) {
 796                 /*
 797                  * Multi-core (and possibly multi-threaded)
 798                  * processors.
 799                  */
 800                 uint_t ncpu_per_core;
 801                 if (cpi->cpi_ncore_per_chip == 1)
 802                         ncpu_per_core = cpi->cpi_ncpu_per_chip;
 803                 else if (cpi->cpi_ncore_per_chip > 1)
 804                         ncpu_per_core = cpi->cpi_ncpu_per_chip /
 805                             cpi->cpi_ncore_per_chip;
 806                 /*
 807                  * 8bit APIC IDs on dual core Pentiums
 808                  * look like this:
 809                  *
 810                  * +-----------------------+------+------+
 811                  * | Physical Package ID   |  MC  |  HT  |
 812                  * +-----------------------+------+------+
 813                  * <------- chipid -------->
 814                  * <------- coreid --------------->
 815                  *                         <--- clogid -->
 816                  *                         <------>
 817                  *                         pkgcoreid
 818                  *
 819                  * Where the number of bits necessary to
 820                  * represent MC and HT fields together equals
 821                  * to the minimum number of bits necessary to
 822                  * store the value of cpi->cpi_ncpu_per_chip.
 823                  * Of those bits, the MC part uses the number
 824                  * of bits necessary to store the value of
 825                  * cpi->cpi_ncore_per_chip.
 826                  */
 827                 for (i = 1; i < ncpu_per_core; i <<= 1)
 828                         coreid_shift++;
 829                 cpi->cpi_coreid = cpi->cpi_apicid >> coreid_shift;
 830                 cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
 831         } else if (is_x86_feature(feature, X86FSET_HTT)) {
 832                 /*
 833                  * Single-core multi-threaded processors.
 834                  */
 835                 cpi->cpi_coreid = cpi->cpi_chipid;
 836                 cpi->cpi_pkgcoreid = 0;
 837         }
 838         cpi->cpi_procnodeid = cpi->cpi_chipid;
 839         cpi->cpi_compunitid = cpi->cpi_coreid;
 840 }
 841 
 842 static void
 843 cpuid_amd_getids(cpu_t *cpu)
 844 {
 845         int i, first_half, coreidsz;
 846         uint32_t nb_caps_reg;
 847         uint_t node2_1;
 848         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
 849         struct cpuid_regs *cp;
 850 
 851         /*
 852          * AMD CMP chips currently have a single thread per core.
 853          *
 854          * Since no two cpus share a core we must assign a distinct coreid
 855          * per cpu, and we do this by using the cpu_id.  This scheme does not,
 856          * however, guarantee that sibling cores of a chip will have sequential
 857          * coreids starting at a multiple of the number of cores per chip -
 858          * that is usually the case, but if the ACPI MADT table is presented
 859          * in a different order then we need to perform a few more gymnastics
 860          * for the pkgcoreid.
 861          *
 862          * All processors in the system have the same number of enabled
 863          * cores. Cores within a processor are always numbered sequentially
 864          * from 0 regardless of how many or which are disabled, and there
 865          * is no way for operating system to discover the real core id when some
 866          * are disabled.
 867          *
 868          * In family 0x15, the cores come in pairs called compute units. They
 869          * share I$ and L2 caches and the FPU. Enumeration of this feature is
 870          * simplified by the new topology extensions CPUID leaf, indicated by
 871          * the X86 feature X86FSET_TOPOEXT.
 872          */
 873 
 874         cpi->cpi_coreid = cpu->cpu_id;
 875         cpi->cpi_compunitid = cpu->cpu_id;
 876 
 877         if (cpi->cpi_xmaxeax >= 0x80000008) {
 878 
 879                 coreidsz = BITX((cpi)->cpi_extd[8].cp_ecx, 15, 12);
 880 
 881                 /*
 882                  * In AMD parlance chip is really a node while Solaris
 883                  * sees chip as equivalent to socket/package.
 884                  */
 885                 cpi->cpi_ncore_per_chip =
 886                     BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
 887                 if (coreidsz == 0) {
 888                         /* Use legacy method */
 889                         for (i = 1; i < cpi->cpi_ncore_per_chip; i <<= 1)
 890                                 coreidsz++;
 891                         if (coreidsz == 0)
 892                                 coreidsz = 1;
 893                 }
 894         } else {
 895                 /* Assume single-core part */
 896                 cpi->cpi_ncore_per_chip = 1;
 897                 coreidsz = 1;
 898         }
 899 
 900         cpi->cpi_clogid = cpi->cpi_pkgcoreid =
 901             cpi->cpi_apicid & ((1<<coreidsz) - 1);
 902         cpi->cpi_ncpu_per_chip = cpi->cpi_ncore_per_chip;
 903 
 904         /* Get node ID, compute unit ID */
 905         if (is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
 906             cpi->cpi_xmaxeax >= 0x8000001e) {
 907                 cp = &cpi->cpi_extd[0x1e];
 908                 cp->cp_eax = 0x8000001e;
 909                 (void) __cpuid_insn(cp);
 910 
 911                 cpi->cpi_procnodes_per_pkg = BITX(cp->cp_ecx, 10, 8) + 1;
 912                 cpi->cpi_procnodeid = BITX(cp->cp_ecx, 7, 0);
 913                 cpi->cpi_cores_per_compunit = BITX(cp->cp_ebx, 15, 8) + 1;
 914                 cpi->cpi_compunitid = BITX(cp->cp_ebx, 7, 0)
 915                     + (cpi->cpi_ncore_per_chip / cpi->cpi_cores_per_compunit)
 916                     * (cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg);
 917         } else if (cpi->cpi_family == 0xf || cpi->cpi_family >= 0x11) {
 918                 cpi->cpi_procnodeid = (cpi->cpi_apicid >> coreidsz) & 7;
 919         } else if (cpi->cpi_family == 0x10) {
 920                 /*
 921                  * See if we are a multi-node processor.
 922                  * All processors in the system have the same number of nodes
 923                  */
 924                 nb_caps_reg =  pci_getl_func(0, 24, 3, 0xe8);
 925                 if ((cpi->cpi_model < 8) || BITX(nb_caps_reg, 29, 29) == 0) {
 926                         /* Single-node */
 927                         cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 5,
 928                             coreidsz);
 929                 } else {
 930 
 931                         /*
 932                          * Multi-node revision D (2 nodes per package
 933                          * are supported)
 934                          */
 935                         cpi->cpi_procnodes_per_pkg = 2;
 936 
 937                         first_half = (cpi->cpi_pkgcoreid <=
 938                             (cpi->cpi_ncore_per_chip/2 - 1));
 939 
 940                         if (cpi->cpi_apicid == cpi->cpi_pkgcoreid) {
 941                                 /* We are BSP */
 942                                 cpi->cpi_procnodeid = (first_half ? 0 : 1);
 943                         } else {
 944 
 945                                 /* We are AP */
 946                                 /* NodeId[2:1] bits to use for reading F3xe8 */
 947                                 node2_1 = BITX(cpi->cpi_apicid, 5, 4) << 1;
 948 
 949                                 nb_caps_reg =
 950                                     pci_getl_func(0, 24 + node2_1, 3, 0xe8);
 951 
 952                                 /*
 953                                  * Check IntNodeNum bit (31:30, but bit 31 is
 954                                  * always 0 on dual-node processors)
 955                                  */
 956                                 if (BITX(nb_caps_reg, 30, 30) == 0)
 957                                         cpi->cpi_procnodeid = node2_1 +
 958                                             !first_half;
 959                                 else
 960                                         cpi->cpi_procnodeid = node2_1 +
 961                                             first_half;
 962                         }
 963                 }
 964         } else {
 965                 cpi->cpi_procnodeid = 0;
 966         }
 967 
 968         cpi->cpi_chipid =
 969             cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg;
 970 }
 971 
 972 /*
 973  * Setup XFeature_Enabled_Mask register. Required by xsave feature.
 974  */
 975 void
 976 setup_xfem(void)
 977 {
 978         uint64_t flags = XFEATURE_LEGACY_FP;
 979 
 980         ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
 981 
 982         if (is_x86_feature(x86_featureset, X86FSET_SSE))
 983                 flags |= XFEATURE_SSE;
 984 
 985         if (is_x86_feature(x86_featureset, X86FSET_AVX))
 986                 flags |= XFEATURE_AVX;
 987 
 988         if (is_x86_feature(x86_featureset, X86FSET_AVX512F))
 989                 flags |= XFEATURE_AVX512;
 990 
 991         set_xcr(XFEATURE_ENABLED_MASK, flags);
 992 
 993         xsave_bv_all = flags;
 994 }
 995 
 996 void
 997 cpuid_pass1(cpu_t *cpu, uchar_t *featureset)
 998 {
 999         uint32_t mask_ecx, mask_edx;
1000         struct cpuid_info *cpi;
1001         struct cpuid_regs *cp;
1002         int xcpuid;
1003 #if !defined(__xpv)
1004         extern int idle_cpu_prefer_mwait;
1005 #endif
1006 
1007         /*
1008          * Space statically allocated for BSP, ensure pointer is set
1009          */
1010         if (cpu->cpu_id == 0) {
1011                 if (cpu->cpu_m.mcpu_cpi == NULL)
1012                         cpu->cpu_m.mcpu_cpi = &cpuid_info0;
1013         }
1014 
1015         add_x86_feature(featureset, X86FSET_CPUID);
1016 
1017         cpi = cpu->cpu_m.mcpu_cpi;
1018         ASSERT(cpi != NULL);
1019         cp = &cpi->cpi_std[0];
1020         cp->cp_eax = 0;
1021         cpi->cpi_maxeax = __cpuid_insn(cp);
1022         {
1023                 uint32_t *iptr = (uint32_t *)cpi->cpi_vendorstr;
1024                 *iptr++ = cp->cp_ebx;
1025                 *iptr++ = cp->cp_edx;
1026                 *iptr++ = cp->cp_ecx;
1027                 *(char *)&cpi->cpi_vendorstr[12] = '\0';
1028         }
1029 
1030         cpi->cpi_vendor = _cpuid_vendorstr_to_vendorcode(cpi->cpi_vendorstr);
1031         x86_vendor = cpi->cpi_vendor; /* for compatibility */
1032 
1033         /*
1034          * Limit the range in case of weird hardware
1035          */
1036         if (cpi->cpi_maxeax > CPI_MAXEAX_MAX)
1037                 cpi->cpi_maxeax = CPI_MAXEAX_MAX;
1038         if (cpi->cpi_maxeax < 1)
1039                 goto pass1_done;
1040 
1041         cp = &cpi->cpi_std[1];
1042         cp->cp_eax = 1;
1043         (void) __cpuid_insn(cp);
1044 
1045         /*
1046          * Extract identifying constants for easy access.
1047          */
1048         cpi->cpi_model = CPI_MODEL(cpi);
1049         cpi->cpi_family = CPI_FAMILY(cpi);
1050 
1051         if (cpi->cpi_family == 0xf)
1052                 cpi->cpi_family += CPI_FAMILY_XTD(cpi);
1053 
1054         /*
1055          * Beware: AMD uses "extended model" iff base *FAMILY* == 0xf.
1056          * Intel, and presumably everyone else, uses model == 0xf, as
1057          * one would expect (max value means possible overflow).  Sigh.
1058          */
1059 
1060         switch (cpi->cpi_vendor) {
1061         case X86_VENDOR_Intel:
1062                 if (IS_EXTENDED_MODEL_INTEL(cpi))
1063                         cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1064                 break;
1065         case X86_VENDOR_AMD:
1066                 if (CPI_FAMILY(cpi) == 0xf)
1067                         cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1068                 break;
1069         default:
1070                 if (cpi->cpi_model == 0xf)
1071                         cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1072                 break;
1073         }
1074 
1075         cpi->cpi_step = CPI_STEP(cpi);
1076         cpi->cpi_brandid = CPI_BRANDID(cpi);
1077 
1078         /*
1079          * *default* assumptions:
1080          * - believe %edx feature word
1081          * - ignore %ecx feature word
1082          * - 32-bit virtual and physical addressing
1083          */
1084         mask_edx = 0xffffffff;
1085         mask_ecx = 0;
1086 
1087         cpi->cpi_pabits = cpi->cpi_vabits = 32;
1088 
1089         switch (cpi->cpi_vendor) {
1090         case X86_VENDOR_Intel:
1091                 if (cpi->cpi_family == 5)
1092                         x86_type = X86_TYPE_P5;
1093                 else if (IS_LEGACY_P6(cpi)) {
1094                         x86_type = X86_TYPE_P6;
1095                         pentiumpro_bug4046376 = 1;
1096                         /*
1097                          * Clear the SEP bit when it was set erroneously
1098                          */
1099                         if (cpi->cpi_model < 3 && cpi->cpi_step < 3)
1100                                 cp->cp_edx &= ~CPUID_INTC_EDX_SEP;
1101                 } else if (IS_NEW_F6(cpi) || cpi->cpi_family == 0xf) {
1102                         x86_type = X86_TYPE_P4;
1103                         /*
1104                          * We don't currently depend on any of the %ecx
1105                          * features until Prescott, so we'll only check
1106                          * this from P4 onwards.  We might want to revisit
1107                          * that idea later.
1108                          */
1109                         mask_ecx = 0xffffffff;
1110                 } else if (cpi->cpi_family > 0xf)
1111                         mask_ecx = 0xffffffff;
1112                 /*
1113                  * We don't support MONITOR/MWAIT if leaf 5 is not available
1114                  * to obtain the monitor linesize.
1115                  */
1116                 if (cpi->cpi_maxeax < 5)
1117                         mask_ecx &= ~CPUID_INTC_ECX_MON;
1118                 break;
1119         case X86_VENDOR_IntelClone:
1120         default:
1121                 break;
1122         case X86_VENDOR_AMD:
1123 #if defined(OPTERON_ERRATUM_108)
1124                 if (cpi->cpi_family == 0xf && cpi->cpi_model == 0xe) {
1125                         cp->cp_eax = (0xf0f & cp->cp_eax) | 0xc0;
1126                         cpi->cpi_model = 0xc;
1127                 } else
1128 #endif
1129                 if (cpi->cpi_family == 5) {
1130                         /*
1131                          * AMD K5 and K6
1132                          *
1133                          * These CPUs have an incomplete implementation
1134                          * of MCA/MCE which we mask away.
1135                          */
1136                         mask_edx &= ~(CPUID_INTC_EDX_MCE | CPUID_INTC_EDX_MCA);
1137 
1138                         /*
1139                          * Model 0 uses the wrong (APIC) bit
1140                          * to indicate PGE.  Fix it here.
1141                          */
1142                         if (cpi->cpi_model == 0) {
1143                                 if (cp->cp_edx & 0x200) {
1144                                         cp->cp_edx &= ~0x200;
1145                                         cp->cp_edx |= CPUID_INTC_EDX_PGE;
1146                                 }
1147                         }
1148 
1149                         /*
1150                          * Early models had problems w/ MMX; disable.
1151                          */
1152                         if (cpi->cpi_model < 6)
1153                                 mask_edx &= ~CPUID_INTC_EDX_MMX;
1154                 }
1155 
1156                 /*
1157                  * For newer families, SSE3 and CX16, at least, are valid;
1158                  * enable all
1159                  */
1160                 if (cpi->cpi_family >= 0xf)
1161                         mask_ecx = 0xffffffff;
1162                 /*
1163                  * We don't support MONITOR/MWAIT if leaf 5 is not available
1164                  * to obtain the monitor linesize.
1165                  */
1166                 if (cpi->cpi_maxeax < 5)
1167                         mask_ecx &= ~CPUID_INTC_ECX_MON;
1168 
1169 #if !defined(__xpv)
1170                 /*
1171                  * Do not use MONITOR/MWAIT to halt in the idle loop on any AMD
1172                  * processors.  AMD does not intend MWAIT to be used in the cpu
1173                  * idle loop on current and future processors.  10h and future
1174                  * AMD processors use more power in MWAIT than HLT.
1175                  * Pre-family-10h Opterons do not have the MWAIT instruction.
1176                  */
1177                 idle_cpu_prefer_mwait = 0;
1178 #endif
1179 
1180                 break;
1181         case X86_VENDOR_TM:
1182                 /*
1183                  * workaround the NT workaround in CMS 4.1
1184                  */
1185                 if (cpi->cpi_family == 5 && cpi->cpi_model == 4 &&
1186                     (cpi->cpi_step == 2 || cpi->cpi_step == 3))
1187                         cp->cp_edx |= CPUID_INTC_EDX_CX8;
1188                 break;
1189         case X86_VENDOR_Centaur:
1190                 /*
1191                  * workaround the NT workarounds again
1192                  */
1193                 if (cpi->cpi_family == 6)
1194                         cp->cp_edx |= CPUID_INTC_EDX_CX8;
1195                 break;
1196         case X86_VENDOR_Cyrix:
1197                 /*
1198                  * We rely heavily on the probing in locore
1199                  * to actually figure out what parts, if any,
1200                  * of the Cyrix cpuid instruction to believe.
1201                  */
1202                 switch (x86_type) {
1203                 case X86_TYPE_CYRIX_486:
1204                         mask_edx = 0;
1205                         break;
1206                 case X86_TYPE_CYRIX_6x86:
1207                         mask_edx = 0;
1208                         break;
1209                 case X86_TYPE_CYRIX_6x86L:
1210                         mask_edx =
1211                             CPUID_INTC_EDX_DE |
1212                             CPUID_INTC_EDX_CX8;
1213                         break;
1214                 case X86_TYPE_CYRIX_6x86MX:
1215                         mask_edx =
1216                             CPUID_INTC_EDX_DE |
1217                             CPUID_INTC_EDX_MSR |
1218                             CPUID_INTC_EDX_CX8 |
1219                             CPUID_INTC_EDX_PGE |
1220                             CPUID_INTC_EDX_CMOV |
1221                             CPUID_INTC_EDX_MMX;
1222                         break;
1223                 case X86_TYPE_CYRIX_GXm:
1224                         mask_edx =
1225                             CPUID_INTC_EDX_MSR |
1226                             CPUID_INTC_EDX_CX8 |
1227                             CPUID_INTC_EDX_CMOV |
1228                             CPUID_INTC_EDX_MMX;
1229                         break;
1230                 case X86_TYPE_CYRIX_MediaGX:
1231                         break;
1232                 case X86_TYPE_CYRIX_MII:
1233                 case X86_TYPE_VIA_CYRIX_III:
1234                         mask_edx =
1235                             CPUID_INTC_EDX_DE |
1236                             CPUID_INTC_EDX_TSC |
1237                             CPUID_INTC_EDX_MSR |
1238                             CPUID_INTC_EDX_CX8 |
1239                             CPUID_INTC_EDX_PGE |
1240                             CPUID_INTC_EDX_CMOV |
1241                             CPUID_INTC_EDX_MMX;
1242                         break;
1243                 default:
1244                         break;
1245                 }
1246                 break;
1247         }
1248 
1249 #if defined(__xpv)
1250         /*
1251          * Do not support MONITOR/MWAIT under a hypervisor
1252          */
1253         mask_ecx &= ~CPUID_INTC_ECX_MON;
1254         /*
1255          * Do not support XSAVE under a hypervisor for now
1256          */
1257         xsave_force_disable = B_TRUE;
1258 
1259 #endif  /* __xpv */
1260 
1261         if (xsave_force_disable) {
1262                 mask_ecx &= ~CPUID_INTC_ECX_XSAVE;
1263                 mask_ecx &= ~CPUID_INTC_ECX_AVX;
1264                 mask_ecx &= ~CPUID_INTC_ECX_F16C;
1265                 mask_ecx &= ~CPUID_INTC_ECX_FMA;
1266         }
1267 
1268         /*
1269          * Now we've figured out the masks that determine
1270          * which bits we choose to believe, apply the masks
1271          * to the feature words, then map the kernel's view
1272          * of these feature words into its feature word.
1273          */
1274         cp->cp_edx &= mask_edx;
1275         cp->cp_ecx &= mask_ecx;
1276 
1277         /*
1278          * apply any platform restrictions (we don't call this
1279          * immediately after __cpuid_insn here, because we need the
1280          * workarounds applied above first)
1281          */
1282         platform_cpuid_mangle(cpi->cpi_vendor, 1, cp);
1283 
1284         /*
1285          * In addition to ecx and edx, Intel is storing a bunch of instruction
1286          * set extensions in leaf 7's ebx, ecx, and edx.
1287          */
1288         if (cpi->cpi_vendor == X86_VENDOR_Intel && cpi->cpi_maxeax >= 7) {
1289                 struct cpuid_regs *ecp;
1290                 ecp = &cpi->cpi_std[7];
1291                 ecp->cp_eax = 7;
1292                 ecp->cp_ecx = 0;
1293                 (void) __cpuid_insn(ecp);
1294                 /*
1295                  * If XSAVE has been disabled, just ignore all of the
1296                  * extended-save-area dependent flags here.
1297                  */
1298                 if (xsave_force_disable) {
1299                         ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
1300                         ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
1301                         ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
1302                         ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_MPX;
1303                         ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_ALL_AVX512;
1304                         ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_ALL_AVX512;
1305                         ecp->cp_edx &= ~CPUID_INTC_EDX_7_0_ALL_AVX512;
1306                 }
1307 
1308                 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMEP)
1309                         add_x86_feature(featureset, X86FSET_SMEP);
1310 
1311                 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_INVPCID) {
1312                         add_x86_feature(featureset, X86FSET_INVPCID);
1313                 }
1314 
1315                 /*
1316                  * We check disable_smap here in addition to in startup_smap()
1317                  * to ensure CPUs that aren't the boot CPU don't accidentally
1318                  * include it in the feature set and thus generate a mismatched
1319                  * x86 feature set across CPUs. Note that at this time we only
1320                  * enable SMAP for the 64-bit kernel.
1321                  */
1322 #if defined(__amd64)
1323                 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMAP &&
1324                     disable_smap == 0)
1325                         add_x86_feature(featureset, X86FSET_SMAP);
1326 #endif
1327                 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_MPX)
1328                         add_x86_feature(featureset, X86FSET_MPX);
1329 
1330                 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_RDSEED)
1331                         add_x86_feature(featureset, X86FSET_RDSEED);
1332 
1333                 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_ADX)
1334                         add_x86_feature(featureset, X86FSET_ADX);
1335         }
1336 
1337         /*
1338          * fold in overrides from the "eeprom" mechanism
1339          */
1340         cp->cp_edx |= cpuid_feature_edx_include;
1341         cp->cp_edx &= ~cpuid_feature_edx_exclude;
1342 
1343         cp->cp_ecx |= cpuid_feature_ecx_include;
1344         cp->cp_ecx &= ~cpuid_feature_ecx_exclude;
1345 
1346         if (cp->cp_edx & CPUID_INTC_EDX_PSE) {
1347                 add_x86_feature(featureset, X86FSET_LARGEPAGE);
1348         }
1349         if (cp->cp_edx & CPUID_INTC_EDX_TSC) {
1350                 add_x86_feature(featureset, X86FSET_TSC);
1351         }
1352         if (cp->cp_edx & CPUID_INTC_EDX_MSR) {
1353                 add_x86_feature(featureset, X86FSET_MSR);
1354         }
1355         if (cp->cp_edx & CPUID_INTC_EDX_MTRR) {
1356                 add_x86_feature(featureset, X86FSET_MTRR);
1357         }
1358         if (cp->cp_edx & CPUID_INTC_EDX_PGE) {
1359                 add_x86_feature(featureset, X86FSET_PGE);
1360         }
1361         if (cp->cp_edx & CPUID_INTC_EDX_CMOV) {
1362                 add_x86_feature(featureset, X86FSET_CMOV);
1363         }
1364         if (cp->cp_edx & CPUID_INTC_EDX_MMX) {
1365                 add_x86_feature(featureset, X86FSET_MMX);
1366         }
1367         if ((cp->cp_edx & CPUID_INTC_EDX_MCE) != 0 &&
1368             (cp->cp_edx & CPUID_INTC_EDX_MCA) != 0) {
1369                 add_x86_feature(featureset, X86FSET_MCA);
1370         }
1371         if (cp->cp_edx & CPUID_INTC_EDX_PAE) {
1372                 add_x86_feature(featureset, X86FSET_PAE);
1373         }
1374         if (cp->cp_edx & CPUID_INTC_EDX_CX8) {
1375                 add_x86_feature(featureset, X86FSET_CX8);
1376         }
1377         if (cp->cp_ecx & CPUID_INTC_ECX_CX16) {
1378                 add_x86_feature(featureset, X86FSET_CX16);
1379         }
1380         if (cp->cp_edx & CPUID_INTC_EDX_PAT) {
1381                 add_x86_feature(featureset, X86FSET_PAT);
1382         }
1383         if (cp->cp_edx & CPUID_INTC_EDX_SEP) {
1384                 add_x86_feature(featureset, X86FSET_SEP);
1385         }
1386         if (cp->cp_edx & CPUID_INTC_EDX_FXSR) {
1387                 /*
1388                  * In our implementation, fxsave/fxrstor
1389                  * are prerequisites before we'll even
1390                  * try and do SSE things.
1391                  */
1392                 if (cp->cp_edx & CPUID_INTC_EDX_SSE) {
1393                         add_x86_feature(featureset, X86FSET_SSE);
1394                 }
1395                 if (cp->cp_edx & CPUID_INTC_EDX_SSE2) {
1396                         add_x86_feature(featureset, X86FSET_SSE2);
1397                 }
1398                 if (cp->cp_ecx & CPUID_INTC_ECX_SSE3) {
1399                         add_x86_feature(featureset, X86FSET_SSE3);
1400                 }
1401                 if (cp->cp_ecx & CPUID_INTC_ECX_SSSE3) {
1402                         add_x86_feature(featureset, X86FSET_SSSE3);
1403                 }
1404                 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_1) {
1405                         add_x86_feature(featureset, X86FSET_SSE4_1);
1406                 }
1407                 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_2) {
1408                         add_x86_feature(featureset, X86FSET_SSE4_2);
1409                 }
1410                 if (cp->cp_ecx & CPUID_INTC_ECX_AES) {
1411                         add_x86_feature(featureset, X86FSET_AES);
1412                 }
1413                 if (cp->cp_ecx & CPUID_INTC_ECX_PCLMULQDQ) {
1414                         add_x86_feature(featureset, X86FSET_PCLMULQDQ);
1415                 }
1416 
1417                 if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_SHA)
1418                         add_x86_feature(featureset, X86FSET_SHA);
1419 
1420                 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_UMIP)
1421                         add_x86_feature(featureset, X86FSET_UMIP);
1422                 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_PKU)
1423                         add_x86_feature(featureset, X86FSET_PKU);
1424                 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_OSPKE)
1425                         add_x86_feature(featureset, X86FSET_OSPKE);
1426 
1427                 if (cp->cp_ecx & CPUID_INTC_ECX_XSAVE) {
1428                         add_x86_feature(featureset, X86FSET_XSAVE);
1429 
1430                         /* We only test AVX & AVX512 when there is XSAVE */
1431 
1432                         if (cp->cp_ecx & CPUID_INTC_ECX_AVX) {
1433                                 add_x86_feature(featureset,
1434                                     X86FSET_AVX);
1435 
1436                                 /*
1437                                  * Intel says we can't check these without also
1438                                  * checking AVX.
1439                                  */
1440                                 if (cp->cp_ecx & CPUID_INTC_ECX_F16C)
1441                                         add_x86_feature(featureset,
1442                                             X86FSET_F16C);
1443 
1444                                 if (cp->cp_ecx & CPUID_INTC_ECX_FMA)
1445                                         add_x86_feature(featureset,
1446                                             X86FSET_FMA);
1447 
1448                                 if (cpi->cpi_std[7].cp_ebx &
1449                                     CPUID_INTC_EBX_7_0_BMI1)
1450                                         add_x86_feature(featureset,
1451                                             X86FSET_BMI1);
1452 
1453                                 if (cpi->cpi_std[7].cp_ebx &
1454                                     CPUID_INTC_EBX_7_0_BMI2)
1455                                         add_x86_feature(featureset,
1456                                             X86FSET_BMI2);
1457 
1458                                 if (cpi->cpi_std[7].cp_ebx &
1459                                     CPUID_INTC_EBX_7_0_AVX2)
1460                                         add_x86_feature(featureset,
1461                                             X86FSET_AVX2);
1462                         }
1463 
1464                         if (cpi->cpi_std[7].cp_ebx &
1465                             CPUID_INTC_EBX_7_0_AVX512F) {
1466                                 add_x86_feature(featureset, X86FSET_AVX512F);
1467 
1468                                 if (cpi->cpi_std[7].cp_ebx &
1469                                     CPUID_INTC_EBX_7_0_AVX512DQ)
1470                                         add_x86_feature(featureset,
1471                                             X86FSET_AVX512DQ);
1472                                 if (cpi->cpi_std[7].cp_ebx &
1473                                     CPUID_INTC_EBX_7_0_AVX512IFMA)
1474                                         add_x86_feature(featureset,
1475                                             X86FSET_AVX512FMA);
1476                                 if (cpi->cpi_std[7].cp_ebx &
1477                                     CPUID_INTC_EBX_7_0_AVX512PF)
1478                                         add_x86_feature(featureset,
1479                                             X86FSET_AVX512PF);
1480                                 if (cpi->cpi_std[7].cp_ebx &
1481                                     CPUID_INTC_EBX_7_0_AVX512ER)
1482                                         add_x86_feature(featureset,
1483                                             X86FSET_AVX512ER);
1484                                 if (cpi->cpi_std[7].cp_ebx &
1485                                     CPUID_INTC_EBX_7_0_AVX512CD)
1486                                         add_x86_feature(featureset,
1487                                             X86FSET_AVX512CD);
1488                                 if (cpi->cpi_std[7].cp_ebx &
1489                                     CPUID_INTC_EBX_7_0_AVX512BW)
1490                                         add_x86_feature(featureset,
1491                                             X86FSET_AVX512BW);
1492                                 if (cpi->cpi_std[7].cp_ebx &
1493                                     CPUID_INTC_EBX_7_0_AVX512VL)
1494                                         add_x86_feature(featureset,
1495                                             X86FSET_AVX512VL);
1496 
1497                                 if (cpi->cpi_std[7].cp_ecx &
1498                                     CPUID_INTC_ECX_7_0_AVX512VBMI)
1499                                         add_x86_feature(featureset,
1500                                             X86FSET_AVX512VBMI);
1501                                 if (cpi->cpi_std[7].cp_ecx &
1502                                     CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
1503                                         add_x86_feature(featureset,
1504                                             X86FSET_AVX512VPOPCDQ);
1505 
1506                                 if (cpi->cpi_std[7].cp_edx &
1507                                     CPUID_INTC_EDX_7_0_AVX5124NNIW)
1508                                         add_x86_feature(featureset,
1509                                             X86FSET_AVX512NNIW);
1510                                 if (cpi->cpi_std[7].cp_edx &
1511                                     CPUID_INTC_EDX_7_0_AVX5124FMAPS)
1512                                         add_x86_feature(featureset,
1513                                             X86FSET_AVX512FMAPS);
1514                         }
1515                 }
1516         }
1517 
1518         if (cpi->cpi_vendor == X86_VENDOR_Intel) {
1519                 if (cp->cp_ecx & CPUID_INTC_ECX_PCID) {
1520                         add_x86_feature(featureset, X86FSET_PCID);
1521                 }
1522         }
1523 
1524         if (cp->cp_ecx & CPUID_INTC_ECX_X2APIC) {
1525                 add_x86_feature(featureset, X86FSET_X2APIC);
1526         }
1527         if (cp->cp_edx & CPUID_INTC_EDX_DE) {
1528                 add_x86_feature(featureset, X86FSET_DE);
1529         }
1530 #if !defined(__xpv)
1531         if (cp->cp_ecx & CPUID_INTC_ECX_MON) {
1532 
1533                 /*
1534                  * We require the CLFLUSH instruction for erratum workaround
1535                  * to use MONITOR/MWAIT.
1536                  */
1537                 if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1538                         cpi->cpi_mwait.support |= MWAIT_SUPPORT;
1539                         add_x86_feature(featureset, X86FSET_MWAIT);
1540                 } else {
1541                         extern int idle_cpu_assert_cflush_monitor;
1542 
1543                         /*
1544                          * All processors we are aware of which have
1545                          * MONITOR/MWAIT also have CLFLUSH.
1546                          */
1547                         if (idle_cpu_assert_cflush_monitor) {
1548                                 ASSERT((cp->cp_ecx & CPUID_INTC_ECX_MON) &&
1549                                     (cp->cp_edx & CPUID_INTC_EDX_CLFSH));
1550                         }
1551                 }
1552         }
1553 #endif  /* __xpv */
1554 
1555         if (cp->cp_ecx & CPUID_INTC_ECX_VMX) {
1556                 add_x86_feature(featureset, X86FSET_VMX);
1557         }
1558 
1559         if (cp->cp_ecx & CPUID_INTC_ECX_RDRAND)
1560                 add_x86_feature(featureset, X86FSET_RDRAND);
1561 
1562         /*
1563          * Only need it first time, rest of the cpus would follow suit.
1564          * we only capture this for the bootcpu.
1565          */
1566         if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1567                 add_x86_feature(featureset, X86FSET_CLFSH);
1568                 x86_clflush_size = (BITX(cp->cp_ebx, 15, 8) * 8);
1569         }
1570         if (is_x86_feature(featureset, X86FSET_PAE))
1571                 cpi->cpi_pabits = 36;
1572 
1573         /*
1574          * Hyperthreading configuration is slightly tricky on Intel
1575          * and pure clones, and even trickier on AMD.
1576          *
1577          * (AMD chose to set the HTT bit on their CMP processors,
1578          * even though they're not actually hyperthreaded.  Thus it
1579          * takes a bit more work to figure out what's really going
1580          * on ... see the handling of the CMP_LGCY bit below)
1581          */
1582         if (cp->cp_edx & CPUID_INTC_EDX_HTT) {
1583                 cpi->cpi_ncpu_per_chip = CPI_CPU_COUNT(cpi);
1584                 if (cpi->cpi_ncpu_per_chip > 1)
1585                         add_x86_feature(featureset, X86FSET_HTT);
1586         } else {
1587                 cpi->cpi_ncpu_per_chip = 1;
1588         }
1589 
1590         if (cpi->cpi_vendor == X86_VENDOR_Intel && cpi->cpi_maxeax >= 0xD &&
1591             !xsave_force_disable) {
1592                 struct cpuid_regs r, *ecp;
1593 
1594                 ecp = &r;
1595                 ecp->cp_eax = 0xD;
1596                 ecp->cp_ecx = 1;
1597                 ecp->cp_edx = ecp->cp_ebx = 0;
1598                 (void) __cpuid_insn(ecp);
1599 
1600                 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEOPT)
1601                         add_x86_feature(featureset, X86FSET_XSAVEOPT);
1602                 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEC)
1603                         add_x86_feature(featureset, X86FSET_XSAVEC);
1604                 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVES)
1605                         add_x86_feature(featureset, X86FSET_XSAVES);
1606         }
1607 
1608         /*
1609          * Work on the "extended" feature information, doing
1610          * some basic initialization for cpuid_pass2()
1611          */
1612         xcpuid = 0;
1613         switch (cpi->cpi_vendor) {
1614         case X86_VENDOR_Intel:
1615                 /*
1616                  * On KVM we know we will have proper support for extended
1617                  * cpuid.
1618                  */
1619                 if (IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf ||
1620                     (get_hwenv() == HW_KVM && cpi->cpi_family == 6 &&
1621                     (cpi->cpi_model == 6 || cpi->cpi_model == 2)))
1622                         xcpuid++;
1623                 break;
1624         case X86_VENDOR_AMD:
1625                 if (cpi->cpi_family > 5 ||
1626                     (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
1627                         xcpuid++;
1628                 break;
1629         case X86_VENDOR_Cyrix:
1630                 /*
1631                  * Only these Cyrix CPUs are -known- to support
1632                  * extended cpuid operations.
1633                  */
1634                 if (x86_type == X86_TYPE_VIA_CYRIX_III ||
1635                     x86_type == X86_TYPE_CYRIX_GXm)
1636                         xcpuid++;
1637                 break;
1638         case X86_VENDOR_Centaur:
1639         case X86_VENDOR_TM:
1640         default:
1641                 xcpuid++;
1642                 break;
1643         }
1644 
1645         if (xcpuid) {
1646                 cp = &cpi->cpi_extd[0];
1647                 cp->cp_eax = 0x80000000;
1648                 cpi->cpi_xmaxeax = __cpuid_insn(cp);
1649         }
1650 
1651         if (cpi->cpi_xmaxeax & 0x80000000) {
1652 
1653                 if (cpi->cpi_xmaxeax > CPI_XMAXEAX_MAX)
1654                         cpi->cpi_xmaxeax = CPI_XMAXEAX_MAX;
1655 
1656                 switch (cpi->cpi_vendor) {
1657                 case X86_VENDOR_Intel:
1658                 case X86_VENDOR_AMD:
1659                         if (cpi->cpi_xmaxeax < 0x80000001)
1660                                 break;
1661                         cp = &cpi->cpi_extd[1];
1662                         cp->cp_eax = 0x80000001;
1663                         (void) __cpuid_insn(cp);
1664 
1665                         if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1666                             cpi->cpi_family == 5 &&
1667                             cpi->cpi_model == 6 &&
1668                             cpi->cpi_step == 6) {
1669                                 /*
1670                                  * K6 model 6 uses bit 10 to indicate SYSC
1671                                  * Later models use bit 11. Fix it here.
1672                                  */
1673                                 if (cp->cp_edx & 0x400) {
1674                                         cp->cp_edx &= ~0x400;
1675                                         cp->cp_edx |= CPUID_AMD_EDX_SYSC;
1676                                 }
1677                         }
1678 
1679                         platform_cpuid_mangle(cpi->cpi_vendor, 0x80000001, cp);
1680 
1681                         /*
1682                          * Compute the additions to the kernel's feature word.
1683                          */
1684                         if (cp->cp_edx & CPUID_AMD_EDX_NX) {
1685                                 add_x86_feature(featureset, X86FSET_NX);
1686                         }
1687 
1688                         /*
1689                          * Regardless whether or not we boot 64-bit,
1690                          * we should have a way to identify whether
1691                          * the CPU is capable of running 64-bit.
1692                          */
1693                         if (cp->cp_edx & CPUID_AMD_EDX_LM) {
1694                                 add_x86_feature(featureset, X86FSET_64);
1695                         }
1696 
1697 #if defined(__amd64)
1698                         /* 1 GB large page - enable only for 64 bit kernel */
1699                         if (cp->cp_edx & CPUID_AMD_EDX_1GPG) {
1700                                 add_x86_feature(featureset, X86FSET_1GPG);
1701                         }
1702 #endif
1703 
1704                         if ((cpi->cpi_vendor == X86_VENDOR_AMD) &&
1705                             (cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_FXSR) &&
1706                             (cp->cp_ecx & CPUID_AMD_ECX_SSE4A)) {
1707                                 add_x86_feature(featureset, X86FSET_SSE4A);
1708                         }
1709 
1710                         /*
1711                          * If both the HTT and CMP_LGCY bits are set,
1712                          * then we're not actually HyperThreaded.  Read
1713                          * "AMD CPUID Specification" for more details.
1714                          */
1715                         if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1716                             is_x86_feature(featureset, X86FSET_HTT) &&
1717                             (cp->cp_ecx & CPUID_AMD_ECX_CMP_LGCY)) {
1718                                 remove_x86_feature(featureset, X86FSET_HTT);
1719                                 add_x86_feature(featureset, X86FSET_CMP);
1720                         }
1721 #if defined(__amd64)
1722                         /*
1723                          * It's really tricky to support syscall/sysret in
1724                          * the i386 kernel; we rely on sysenter/sysexit
1725                          * instead.  In the amd64 kernel, things are -way-
1726                          * better.
1727                          */
1728                         if (cp->cp_edx & CPUID_AMD_EDX_SYSC) {
1729                                 add_x86_feature(featureset, X86FSET_ASYSC);
1730                         }
1731 
1732                         /*
1733                          * While we're thinking about system calls, note
1734                          * that AMD processors don't support sysenter
1735                          * in long mode at all, so don't try to program them.
1736                          */
1737                         if (x86_vendor == X86_VENDOR_AMD) {
1738                                 remove_x86_feature(featureset, X86FSET_SEP);
1739                         }
1740 #endif
1741                         if (cp->cp_edx & CPUID_AMD_EDX_TSCP) {
1742                                 add_x86_feature(featureset, X86FSET_TSCP);
1743                         }
1744 
1745                         if (cp->cp_ecx & CPUID_AMD_ECX_SVM) {
1746                                 add_x86_feature(featureset, X86FSET_SVM);
1747                         }
1748 
1749                         if (cp->cp_ecx & CPUID_AMD_ECX_TOPOEXT) {
1750                                 add_x86_feature(featureset, X86FSET_TOPOEXT);
1751                         }
1752                         break;
1753                 default:
1754                         break;
1755                 }
1756 
1757                 /*
1758                  * Get CPUID data about processor cores and hyperthreads.
1759                  */
1760                 switch (cpi->cpi_vendor) {
1761                 case X86_VENDOR_Intel:
1762                         if (cpi->cpi_maxeax >= 4) {
1763                                 cp = &cpi->cpi_std[4];
1764                                 cp->cp_eax = 4;
1765                                 cp->cp_ecx = 0;
1766                                 (void) __cpuid_insn(cp);
1767                                 platform_cpuid_mangle(cpi->cpi_vendor, 4, cp);
1768                         }
1769                         /*FALLTHROUGH*/
1770                 case X86_VENDOR_AMD:
1771                         if (cpi->cpi_xmaxeax < 0x80000008)
1772                                 break;
1773                         cp = &cpi->cpi_extd[8];
1774                         cp->cp_eax = 0x80000008;
1775                         (void) __cpuid_insn(cp);
1776                         platform_cpuid_mangle(cpi->cpi_vendor, 0x80000008, cp);
1777 
1778                         /*
1779                          * Virtual and physical address limits from
1780                          * cpuid override previously guessed values.
1781                          */
1782                         cpi->cpi_pabits = BITX(cp->cp_eax, 7, 0);
1783                         cpi->cpi_vabits = BITX(cp->cp_eax, 15, 8);
1784                         break;
1785                 default:
1786                         break;
1787                 }
1788 
1789                 /*
1790                  * Derive the number of cores per chip
1791                  */
1792                 switch (cpi->cpi_vendor) {
1793                 case X86_VENDOR_Intel:
1794                         if (cpi->cpi_maxeax < 4) {
1795                                 cpi->cpi_ncore_per_chip = 1;
1796                                 break;
1797                         } else {
1798                                 cpi->cpi_ncore_per_chip =
1799                                     BITX((cpi)->cpi_std[4].cp_eax, 31, 26) + 1;
1800                         }
1801                         break;
1802                 case X86_VENDOR_AMD:
1803                         if (cpi->cpi_xmaxeax < 0x80000008) {
1804                                 cpi->cpi_ncore_per_chip = 1;
1805                                 break;
1806                         } else {
1807                                 /*
1808                                  * On family 0xf cpuid fn 2 ECX[7:0] "NC" is
1809                                  * 1 less than the number of physical cores on
1810                                  * the chip.  In family 0x10 this value can
1811                                  * be affected by "downcoring" - it reflects
1812                                  * 1 less than the number of cores actually
1813                                  * enabled on this node.
1814                                  */
1815                                 cpi->cpi_ncore_per_chip =
1816                                     BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
1817                         }
1818                         break;
1819                 default:
1820                         cpi->cpi_ncore_per_chip = 1;
1821                         break;
1822                 }
1823 
1824                 /*
1825                  * Get CPUID data about TSC Invariance in Deep C-State.
1826                  */
1827                 switch (cpi->cpi_vendor) {
1828                 case X86_VENDOR_Intel:
1829                         if (cpi->cpi_maxeax >= 7) {
1830                                 cp = &cpi->cpi_extd[7];
1831                                 cp->cp_eax = 0x80000007;
1832                                 cp->cp_ecx = 0;
1833                                 (void) __cpuid_insn(cp);
1834                         }
1835                         break;
1836                 default:
1837                         break;
1838                 }
1839         } else {
1840                 cpi->cpi_ncore_per_chip = 1;
1841         }
1842 
1843         /*
1844          * If more than one core, then this processor is CMP.
1845          */
1846         if (cpi->cpi_ncore_per_chip > 1) {
1847                 add_x86_feature(featureset, X86FSET_CMP);
1848         }
1849 
1850         /*
1851          * If the number of cores is the same as the number
1852          * of CPUs, then we cannot have HyperThreading.
1853          */
1854         if (cpi->cpi_ncpu_per_chip == cpi->cpi_ncore_per_chip) {
1855                 remove_x86_feature(featureset, X86FSET_HTT);
1856         }
1857 
1858         cpi->cpi_apicid = CPI_APIC_ID(cpi);
1859         cpi->cpi_procnodes_per_pkg = 1;
1860         cpi->cpi_cores_per_compunit = 1;
1861         if (is_x86_feature(featureset, X86FSET_HTT) == B_FALSE &&
1862             is_x86_feature(featureset, X86FSET_CMP) == B_FALSE) {
1863                 /*
1864                  * Single-core single-threaded processors.
1865                  */
1866                 cpi->cpi_chipid = -1;
1867                 cpi->cpi_clogid = 0;
1868                 cpi->cpi_coreid = cpu->cpu_id;
1869                 cpi->cpi_pkgcoreid = 0;
1870                 if (cpi->cpi_vendor == X86_VENDOR_AMD)
1871                         cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 3, 0);
1872                 else
1873                         cpi->cpi_procnodeid = cpi->cpi_chipid;
1874         } else if (cpi->cpi_ncpu_per_chip > 1) {
1875                 if (cpi->cpi_vendor == X86_VENDOR_Intel)
1876                         cpuid_intel_getids(cpu, featureset);
1877                 else if (cpi->cpi_vendor == X86_VENDOR_AMD)
1878                         cpuid_amd_getids(cpu);
1879                 else {
1880                         /*
1881                          * All other processors are currently
1882                          * assumed to have single cores.
1883                          */
1884                         cpi->cpi_coreid = cpi->cpi_chipid;
1885                         cpi->cpi_pkgcoreid = 0;
1886                         cpi->cpi_procnodeid = cpi->cpi_chipid;
1887                         cpi->cpi_compunitid = cpi->cpi_chipid;
1888                 }
1889         }
1890 
1891         /*
1892          * Synthesize chip "revision" and socket type
1893          */
1894         cpi->cpi_chiprev = _cpuid_chiprev(cpi->cpi_vendor, cpi->cpi_family,
1895             cpi->cpi_model, cpi->cpi_step);
1896         cpi->cpi_chiprevstr = _cpuid_chiprevstr(cpi->cpi_vendor,
1897             cpi->cpi_family, cpi->cpi_model, cpi->cpi_step);
1898         cpi->cpi_socket = _cpuid_skt(cpi->cpi_vendor, cpi->cpi_family,
1899             cpi->cpi_model, cpi->cpi_step);
1900 
1901         /*
1902          * While we're here, check for the AMD "Error Pointer Zero/Restore"
1903          * feature. This can be used to setup the FP save handlers
1904          * appropriately.
1905          */
1906         if (cpi->cpi_vendor == X86_VENDOR_AMD) {
1907                 if (cpi->cpi_xmaxeax >= 0x80000008 &&
1908                     cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_ERR_PTR_ZERO) {
1909                         /* Special handling for AMD FP not necessary. */
1910                         cpi->cpi_fp_amd_save = 0;
1911                 } else {
1912                         cpi->cpi_fp_amd_save = 1;
1913                 }
1914         }
1915 
1916 pass1_done:
1917         cpi->cpi_pass = 1;
1918 }
1919 
1920 /*
1921  * Make copies of the cpuid table entries we depend on, in
1922  * part for ease of parsing now, in part so that we have only
1923  * one place to correct any of it, in part for ease of
1924  * later export to userland, and in part so we can look at
1925  * this stuff in a crash dump.
1926  */
1927 
1928 /*ARGSUSED*/
1929 void
1930 cpuid_pass2(cpu_t *cpu)
1931 {
1932         uint_t n, nmax;
1933         int i;
1934         struct cpuid_regs *cp;
1935         uint8_t *dp;
1936         uint32_t *iptr;
1937         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
1938 
1939         ASSERT(cpi->cpi_pass == 1);
1940 
1941         if (cpi->cpi_maxeax < 1)
1942                 goto pass2_done;
1943 
1944         if ((nmax = cpi->cpi_maxeax + 1) > NMAX_CPI_STD)
1945                 nmax = NMAX_CPI_STD;
1946         /*
1947          * (We already handled n == 0 and n == 1 in pass 1)
1948          */
1949         for (n = 2, cp = &cpi->cpi_std[2]; n < nmax; n++, cp++) {
1950                 cp->cp_eax = n;
1951 
1952                 /*
1953                  * CPUID function 4 expects %ecx to be initialized
1954                  * with an index which indicates which cache to return
1955                  * information about. The OS is expected to call function 4
1956                  * with %ecx set to 0, 1, 2, ... until it returns with
1957                  * EAX[4:0] set to 0, which indicates there are no more
1958                  * caches.
1959                  *
1960                  * Here, populate cpi_std[4] with the information returned by
1961                  * function 4 when %ecx == 0, and do the rest in cpuid_pass3()
1962                  * when dynamic memory allocation becomes available.
1963                  *
1964                  * Note: we need to explicitly initialize %ecx here, since
1965                  * function 4 may have been previously invoked.
1966                  *
1967                  * The same is all true for CPUID function 7.
1968                  */
1969                 if (n == 4 || n == 7)
1970                         cp->cp_ecx = 0;
1971 
1972                 (void) __cpuid_insn(cp);
1973                 platform_cpuid_mangle(cpi->cpi_vendor, n, cp);
1974                 switch (n) {
1975                 case 2:
1976                         /*
1977                          * "the lower 8 bits of the %eax register
1978                          * contain a value that identifies the number
1979                          * of times the cpuid [instruction] has to be
1980                          * executed to obtain a complete image of the
1981                          * processor's caching systems."
1982                          *
1983                          * How *do* they make this stuff up?
1984                          */
1985                         cpi->cpi_ncache = sizeof (*cp) *
1986                             BITX(cp->cp_eax, 7, 0);
1987                         if (cpi->cpi_ncache == 0)
1988                                 break;
1989                         cpi->cpi_ncache--;   /* skip count byte */
1990 
1991                         /*
1992                          * Well, for now, rather than attempt to implement
1993                          * this slightly dubious algorithm, we just look
1994                          * at the first 15 ..
1995                          */
1996                         if (cpi->cpi_ncache > (sizeof (*cp) - 1))
1997                                 cpi->cpi_ncache = sizeof (*cp) - 1;
1998 
1999                         dp = cpi->cpi_cacheinfo;
2000                         if (BITX(cp->cp_eax, 31, 31) == 0) {
2001                                 uint8_t *p = (void *)&cp->cp_eax;
2002                                 for (i = 1; i < 4; i++)
2003                                         if (p[i] != 0)
2004                                                 *dp++ = p[i];
2005                         }
2006                         if (BITX(cp->cp_ebx, 31, 31) == 0) {
2007                                 uint8_t *p = (void *)&cp->cp_ebx;
2008                                 for (i = 0; i < 4; i++)
2009                                         if (p[i] != 0)
2010                                                 *dp++ = p[i];
2011                         }
2012                         if (BITX(cp->cp_ecx, 31, 31) == 0) {
2013                                 uint8_t *p = (void *)&cp->cp_ecx;
2014                                 for (i = 0; i < 4; i++)
2015                                         if (p[i] != 0)
2016                                                 *dp++ = p[i];
2017                         }
2018                         if (BITX(cp->cp_edx, 31, 31) == 0) {
2019                                 uint8_t *p = (void *)&cp->cp_edx;
2020                                 for (i = 0; i < 4; i++)
2021                                         if (p[i] != 0)
2022                                                 *dp++ = p[i];
2023                         }
2024                         break;
2025 
2026                 case 3: /* Processor serial number, if PSN supported */
2027                         break;
2028 
2029                 case 4: /* Deterministic cache parameters */
2030                         break;
2031 
2032                 case 5: /* Monitor/Mwait parameters */
2033                 {
2034                         size_t mwait_size;
2035 
2036                         /*
2037                          * check cpi_mwait.support which was set in cpuid_pass1
2038                          */
2039                         if (!(cpi->cpi_mwait.support & MWAIT_SUPPORT))
2040                                 break;
2041 
2042                         /*
2043                          * Protect ourself from insane mwait line size.
2044                          * Workaround for incomplete hardware emulator(s).
2045                          */
2046                         mwait_size = (size_t)MWAIT_SIZE_MAX(cpi);
2047                         if (mwait_size < sizeof (uint32_t) ||
2048                             !ISP2(mwait_size)) {
2049 #if DEBUG
2050                                 cmn_err(CE_NOTE, "Cannot handle cpu %d mwait "
2051                                     "size %ld", cpu->cpu_id, (long)mwait_size);
2052 #endif
2053                                 break;
2054                         }
2055 
2056                         cpi->cpi_mwait.mon_min = (size_t)MWAIT_SIZE_MIN(cpi);
2057                         cpi->cpi_mwait.mon_max = mwait_size;
2058                         if (MWAIT_EXTENSION(cpi)) {
2059                                 cpi->cpi_mwait.support |= MWAIT_EXTENSIONS;
2060                                 if (MWAIT_INT_ENABLE(cpi))
2061                                         cpi->cpi_mwait.support |=
2062                                             MWAIT_ECX_INT_ENABLE;
2063                         }
2064                         break;
2065                 }
2066                 default:
2067                         break;
2068                 }
2069         }
2070 
2071         if (cpi->cpi_maxeax >= 0xB && cpi->cpi_vendor == X86_VENDOR_Intel) {
2072                 struct cpuid_regs regs;
2073 
2074                 cp = &regs;
2075                 cp->cp_eax = 0xB;
2076                 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
2077 
2078                 (void) __cpuid_insn(cp);
2079 
2080                 /*
2081                  * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
2082                  * indicates that the extended topology enumeration leaf is
2083                  * available.
2084                  */
2085                 if (cp->cp_ebx) {
2086                         uint32_t x2apic_id;
2087                         uint_t coreid_shift = 0;
2088                         uint_t ncpu_per_core = 1;
2089                         uint_t chipid_shift = 0;
2090                         uint_t ncpu_per_chip = 1;
2091                         uint_t i;
2092                         uint_t level;
2093 
2094                         for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
2095                                 cp->cp_eax = 0xB;
2096                                 cp->cp_ecx = i;
2097 
2098                                 (void) __cpuid_insn(cp);
2099                                 level = CPI_CPU_LEVEL_TYPE(cp);
2100 
2101                                 if (level == 1) {
2102                                         x2apic_id = cp->cp_edx;
2103                                         coreid_shift = BITX(cp->cp_eax, 4, 0);
2104                                         ncpu_per_core = BITX(cp->cp_ebx, 15, 0);
2105                                 } else if (level == 2) {
2106                                         x2apic_id = cp->cp_edx;
2107                                         chipid_shift = BITX(cp->cp_eax, 4, 0);
2108                                         ncpu_per_chip = BITX(cp->cp_ebx, 15, 0);
2109                                 }
2110                         }
2111 
2112                         cpi->cpi_apicid = x2apic_id;
2113                         cpi->cpi_ncpu_per_chip = ncpu_per_chip;
2114                         cpi->cpi_ncore_per_chip = ncpu_per_chip /
2115                             ncpu_per_core;
2116                         cpi->cpi_chipid = x2apic_id >> chipid_shift;
2117                         cpi->cpi_clogid = x2apic_id & ((1 << chipid_shift) - 1);
2118                         cpi->cpi_coreid = x2apic_id >> coreid_shift;
2119                         cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
2120                 }
2121 
2122                 /* Make cp NULL so that we don't stumble on others */
2123                 cp = NULL;
2124         }
2125 
2126         /*
2127          * XSAVE enumeration
2128          */
2129         if (cpi->cpi_maxeax >= 0xD) {
2130                 struct cpuid_regs regs;
2131                 boolean_t cpuid_d_valid = B_TRUE;
2132 
2133                 cp = &regs;
2134                 cp->cp_eax = 0xD;
2135                 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
2136 
2137                 (void) __cpuid_insn(cp);
2138 
2139                 /*
2140                  * Sanity checks for debug
2141                  */
2142                 if ((cp->cp_eax & XFEATURE_LEGACY_FP) == 0 ||
2143                     (cp->cp_eax & XFEATURE_SSE) == 0) {
2144                         cpuid_d_valid = B_FALSE;
2145                 }
2146 
2147                 cpi->cpi_xsave.xsav_hw_features_low = cp->cp_eax;
2148                 cpi->cpi_xsave.xsav_hw_features_high = cp->cp_edx;
2149                 cpi->cpi_xsave.xsav_max_size = cp->cp_ecx;
2150 
2151                 /*
2152                  * If the hw supports AVX, get the size and offset in the save
2153                  * area for the ymm state.
2154                  */
2155                 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX) {
2156                         cp->cp_eax = 0xD;
2157                         cp->cp_ecx = 2;
2158                         cp->cp_edx = cp->cp_ebx = 0;
2159 
2160                         (void) __cpuid_insn(cp);
2161 
2162                         if (cp->cp_ebx != CPUID_LEAFD_2_YMM_OFFSET ||
2163                             cp->cp_eax != CPUID_LEAFD_2_YMM_SIZE) {
2164                                 cpuid_d_valid = B_FALSE;
2165                         }
2166 
2167                         cpi->cpi_xsave.ymm_size = cp->cp_eax;
2168                         cpi->cpi_xsave.ymm_offset = cp->cp_ebx;
2169                 }
2170 
2171                 /*
2172                  * If the hw supports MPX, get the size and offset in the
2173                  * save area for BNDREGS and BNDCSR.
2174                  */
2175                 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_MPX) {
2176                         cp->cp_eax = 0xD;
2177                         cp->cp_ecx = 3;
2178                         cp->cp_edx = cp->cp_ebx = 0;
2179 
2180                         (void) __cpuid_insn(cp);
2181 
2182                         cpi->cpi_xsave.bndregs_size = cp->cp_eax;
2183                         cpi->cpi_xsave.bndregs_offset = cp->cp_ebx;
2184 
2185                         cp->cp_eax = 0xD;
2186                         cp->cp_ecx = 4;
2187                         cp->cp_edx = cp->cp_ebx = 0;
2188 
2189                         (void) __cpuid_insn(cp);
2190 
2191                         cpi->cpi_xsave.bndcsr_size = cp->cp_eax;
2192                         cpi->cpi_xsave.bndcsr_offset = cp->cp_ebx;
2193                 }
2194 
2195                 /*
2196                  * If the hw supports AVX512, get the size and offset in the
2197                  * save area for the opmask registers and zmm state.
2198                  */
2199                 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX512) {
2200                         cp->cp_eax = 0xD;
2201                         cp->cp_ecx = 5;
2202                         cp->cp_edx = cp->cp_ebx = 0;
2203 
2204                         (void) __cpuid_insn(cp);
2205 
2206                         cpi->cpi_xsave.opmask_size = cp->cp_eax;
2207                         cpi->cpi_xsave.opmask_offset = cp->cp_ebx;
2208 
2209                         cp->cp_eax = 0xD;
2210                         cp->cp_ecx = 6;
2211                         cp->cp_edx = cp->cp_ebx = 0;
2212 
2213                         (void) __cpuid_insn(cp);
2214 
2215                         cpi->cpi_xsave.zmmlo_size = cp->cp_eax;
2216                         cpi->cpi_xsave.zmmlo_offset = cp->cp_ebx;
2217 
2218                         cp->cp_eax = 0xD;
2219                         cp->cp_ecx = 7;
2220                         cp->cp_edx = cp->cp_ebx = 0;
2221 
2222                         (void) __cpuid_insn(cp);
2223 
2224                         cpi->cpi_xsave.zmmhi_size = cp->cp_eax;
2225                         cpi->cpi_xsave.zmmhi_offset = cp->cp_ebx;
2226                 }
2227 
2228                 if (is_x86_feature(x86_featureset, X86FSET_XSAVE)) {
2229                         xsave_state_size = 0;
2230                 } else if (cpuid_d_valid) {
2231                         xsave_state_size = cpi->cpi_xsave.xsav_max_size;
2232                 } else {
2233                         /* Broken CPUID 0xD, probably in HVM */
2234                         cmn_err(CE_WARN, "cpu%d: CPUID.0xD returns invalid "
2235                             "value: hw_low = %d, hw_high = %d, xsave_size = %d"
2236                             ", ymm_size = %d, ymm_offset = %d\n",
2237                             cpu->cpu_id, cpi->cpi_xsave.xsav_hw_features_low,
2238                             cpi->cpi_xsave.xsav_hw_features_high,
2239                             (int)cpi->cpi_xsave.xsav_max_size,
2240                             (int)cpi->cpi_xsave.ymm_size,
2241                             (int)cpi->cpi_xsave.ymm_offset);
2242 
2243                         if (xsave_state_size != 0) {
2244                                 /*
2245                                  * This must be a non-boot CPU. We cannot
2246                                  * continue, because boot cpu has already
2247                                  * enabled XSAVE.
2248                                  */
2249                                 ASSERT(cpu->cpu_id != 0);
2250                                 cmn_err(CE_PANIC, "cpu%d: we have already "
2251                                     "enabled XSAVE on boot cpu, cannot "
2252                                     "continue.", cpu->cpu_id);
2253                         } else {
2254                                 /*
2255                                  * If we reached here on the boot CPU, it's also
2256                                  * almost certain that we'll reach here on the
2257                                  * non-boot CPUs. When we're here on a boot CPU
2258                                  * we should disable the feature, on a non-boot
2259                                  * CPU we need to confirm that we have.
2260                                  */
2261                                 if (cpu->cpu_id == 0) {
2262                                         remove_x86_feature(x86_featureset,
2263                                             X86FSET_XSAVE);
2264                                         remove_x86_feature(x86_featureset,
2265                                             X86FSET_AVX);
2266                                         remove_x86_feature(x86_featureset,
2267                                             X86FSET_F16C);
2268                                         remove_x86_feature(x86_featureset,
2269                                             X86FSET_BMI1);
2270                                         remove_x86_feature(x86_featureset,
2271                                             X86FSET_BMI2);
2272                                         remove_x86_feature(x86_featureset,
2273                                             X86FSET_FMA);
2274                                         remove_x86_feature(x86_featureset,
2275                                             X86FSET_AVX2);
2276                                         remove_x86_feature(x86_featureset,
2277                                             X86FSET_MPX);
2278                                         remove_x86_feature(x86_featureset,
2279                                             X86FSET_AVX512F);
2280                                         remove_x86_feature(x86_featureset,
2281                                             X86FSET_AVX512DQ);
2282                                         remove_x86_feature(x86_featureset,
2283                                             X86FSET_AVX512PF);
2284                                         remove_x86_feature(x86_featureset,
2285                                             X86FSET_AVX512ER);
2286                                         remove_x86_feature(x86_featureset,
2287                                             X86FSET_AVX512CD);
2288                                         remove_x86_feature(x86_featureset,
2289                                             X86FSET_AVX512BW);
2290                                         remove_x86_feature(x86_featureset,
2291                                             X86FSET_AVX512VL);
2292                                         remove_x86_feature(x86_featureset,
2293                                             X86FSET_AVX512FMA);
2294                                         remove_x86_feature(x86_featureset,
2295                                             X86FSET_AVX512VBMI);
2296                                         remove_x86_feature(x86_featureset,
2297                                             X86FSET_AVX512VPOPCDQ);
2298                                         remove_x86_feature(x86_featureset,
2299                                             X86FSET_AVX512NNIW);
2300                                         remove_x86_feature(x86_featureset,
2301                                             X86FSET_AVX512FMAPS);
2302 
2303                                         CPI_FEATURES_ECX(cpi) &=
2304                                             ~CPUID_INTC_ECX_XSAVE;
2305                                         CPI_FEATURES_ECX(cpi) &=
2306                                             ~CPUID_INTC_ECX_AVX;
2307                                         CPI_FEATURES_ECX(cpi) &=
2308                                             ~CPUID_INTC_ECX_F16C;
2309                                         CPI_FEATURES_ECX(cpi) &=
2310                                             ~CPUID_INTC_ECX_FMA;
2311                                         CPI_FEATURES_7_0_EBX(cpi) &=
2312                                             ~CPUID_INTC_EBX_7_0_BMI1;
2313                                         CPI_FEATURES_7_0_EBX(cpi) &=
2314                                             ~CPUID_INTC_EBX_7_0_BMI2;
2315                                         CPI_FEATURES_7_0_EBX(cpi) &=
2316                                             ~CPUID_INTC_EBX_7_0_AVX2;
2317                                         CPI_FEATURES_7_0_EBX(cpi) &=
2318                                             ~CPUID_INTC_EBX_7_0_MPX;
2319                                         CPI_FEATURES_7_0_EBX(cpi) &=
2320                                             ~CPUID_INTC_EBX_7_0_ALL_AVX512;
2321 
2322                                         CPI_FEATURES_7_0_ECX(cpi) &=
2323                                             ~CPUID_INTC_ECX_7_0_ALL_AVX512;
2324 
2325                                         CPI_FEATURES_7_0_EDX(cpi) &=
2326                                             ~CPUID_INTC_EDX_7_0_ALL_AVX512;
2327 
2328                                         xsave_force_disable = B_TRUE;
2329                                 } else {
2330                                         VERIFY(is_x86_feature(x86_featureset,
2331                                             X86FSET_XSAVE) == B_FALSE);
2332                                 }
2333                         }
2334                 }
2335         }
2336 
2337 
2338         if ((cpi->cpi_xmaxeax & 0x80000000) == 0)
2339                 goto pass2_done;
2340 
2341         if ((nmax = cpi->cpi_xmaxeax - 0x80000000 + 1) > NMAX_CPI_EXTD)
2342                 nmax = NMAX_CPI_EXTD;
2343         /*
2344          * Copy the extended properties, fixing them as we go.
2345          * (We already handled n == 0 and n == 1 in pass 1)
2346          */
2347         iptr = (void *)cpi->cpi_brandstr;
2348         for (n = 2, cp = &cpi->cpi_extd[2]; n < nmax; cp++, n++) {
2349                 cp->cp_eax = 0x80000000 + n;
2350                 (void) __cpuid_insn(cp);
2351                 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000000 + n, cp);
2352                 switch (n) {
2353                 case 2:
2354                 case 3:
2355                 case 4:
2356                         /*
2357                          * Extract the brand string
2358                          */
2359                         *iptr++ = cp->cp_eax;
2360                         *iptr++ = cp->cp_ebx;
2361                         *iptr++ = cp->cp_ecx;
2362                         *iptr++ = cp->cp_edx;
2363                         break;
2364                 case 5:
2365                         switch (cpi->cpi_vendor) {
2366                         case X86_VENDOR_AMD:
2367                                 /*
2368                                  * The Athlon and Duron were the first
2369                                  * parts to report the sizes of the
2370                                  * TLB for large pages. Before then,
2371                                  * we don't trust the data.
2372                                  */
2373                                 if (cpi->cpi_family < 6 ||
2374                                     (cpi->cpi_family == 6 &&
2375                                     cpi->cpi_model < 1))
2376                                         cp->cp_eax = 0;
2377                                 break;
2378                         default:
2379                                 break;
2380                         }
2381                         break;
2382                 case 6:
2383                         switch (cpi->cpi_vendor) {
2384                         case X86_VENDOR_AMD:
2385                                 /*
2386                                  * The Athlon and Duron were the first
2387                                  * AMD parts with L2 TLB's.
2388                                  * Before then, don't trust the data.
2389                                  */
2390                                 if (cpi->cpi_family < 6 ||
2391                                     cpi->cpi_family == 6 &&
2392                                     cpi->cpi_model < 1)
2393                                         cp->cp_eax = cp->cp_ebx = 0;
2394                                 /*
2395                                  * AMD Duron rev A0 reports L2
2396                                  * cache size incorrectly as 1K
2397                                  * when it is really 64K
2398                                  */
2399                                 if (cpi->cpi_family == 6 &&
2400                                     cpi->cpi_model == 3 &&
2401                                     cpi->cpi_step == 0) {
2402                                         cp->cp_ecx &= 0xffff;
2403                                         cp->cp_ecx |= 0x400000;
2404                                 }
2405                                 break;
2406                         case X86_VENDOR_Cyrix:  /* VIA C3 */
2407                                 /*
2408                                  * VIA C3 processors are a bit messed
2409                                  * up w.r.t. encoding cache sizes in %ecx
2410                                  */
2411                                 if (cpi->cpi_family != 6)
2412                                         break;
2413                                 /*
2414                                  * model 7 and 8 were incorrectly encoded
2415                                  *
2416                                  * xxx is model 8 really broken?
2417                                  */
2418                                 if (cpi->cpi_model == 7 ||
2419                                     cpi->cpi_model == 8)
2420                                         cp->cp_ecx =
2421                                             BITX(cp->cp_ecx, 31, 24) << 16 |
2422                                             BITX(cp->cp_ecx, 23, 16) << 12 |
2423                                             BITX(cp->cp_ecx, 15, 8) << 8 |
2424                                             BITX(cp->cp_ecx, 7, 0);
2425                                 /*
2426                                  * model 9 stepping 1 has wrong associativity
2427                                  */
2428                                 if (cpi->cpi_model == 9 && cpi->cpi_step == 1)
2429                                         cp->cp_ecx |= 8 << 12;
2430                                 break;
2431                         case X86_VENDOR_Intel:
2432                                 /*
2433                                  * Extended L2 Cache features function.
2434                                  * First appeared on Prescott.
2435                                  */
2436                         default:
2437                                 break;
2438                         }
2439                         break;
2440                 default:
2441                         break;
2442                 }
2443         }
2444 
2445 pass2_done:
2446         cpi->cpi_pass = 2;
2447 }
2448 
2449 static const char *
2450 intel_cpubrand(const struct cpuid_info *cpi)
2451 {
2452         int i;
2453 
2454         if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2455             cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2456                 return ("i486");
2457 
2458         switch (cpi->cpi_family) {
2459         case 5:
2460                 return ("Intel Pentium(r)");
2461         case 6:
2462                 switch (cpi->cpi_model) {
2463                         uint_t celeron, xeon;
2464                         const struct cpuid_regs *cp;
2465                 case 0:
2466                 case 1:
2467                 case 2:
2468                         return ("Intel Pentium(r) Pro");
2469                 case 3:
2470                 case 4:
2471                         return ("Intel Pentium(r) II");
2472                 case 6:
2473                         return ("Intel Celeron(r)");
2474                 case 5:
2475                 case 7:
2476                         celeron = xeon = 0;
2477                         cp = &cpi->cpi_std[2];   /* cache info */
2478 
2479                         for (i = 1; i < 4; i++) {
2480                                 uint_t tmp;
2481 
2482                                 tmp = (cp->cp_eax >> (8 * i)) & 0xff;
2483                                 if (tmp == 0x40)
2484                                         celeron++;
2485                                 if (tmp >= 0x44 && tmp <= 0x45)
2486                                         xeon++;
2487                         }
2488 
2489                         for (i = 0; i < 2; i++) {
2490                                 uint_t tmp;
2491 
2492                                 tmp = (cp->cp_ebx >> (8 * i)) & 0xff;
2493                                 if (tmp == 0x40)
2494                                         celeron++;
2495                                 else if (tmp >= 0x44 && tmp <= 0x45)
2496                                         xeon++;
2497                         }
2498 
2499                         for (i = 0; i < 4; i++) {
2500                                 uint_t tmp;
2501 
2502                                 tmp = (cp->cp_ecx >> (8 * i)) & 0xff;
2503                                 if (tmp == 0x40)
2504                                         celeron++;
2505                                 else if (tmp >= 0x44 && tmp <= 0x45)
2506                                         xeon++;
2507                         }
2508 
2509                         for (i = 0; i < 4; i++) {
2510                                 uint_t tmp;
2511 
2512                                 tmp = (cp->cp_edx >> (8 * i)) & 0xff;
2513                                 if (tmp == 0x40)
2514                                         celeron++;
2515                                 else if (tmp >= 0x44 && tmp <= 0x45)
2516                                         xeon++;
2517                         }
2518 
2519                         if (celeron)
2520                                 return ("Intel Celeron(r)");
2521                         if (xeon)
2522                                 return (cpi->cpi_model == 5 ?
2523                                     "Intel Pentium(r) II Xeon(tm)" :
2524                                     "Intel Pentium(r) III Xeon(tm)");
2525                         return (cpi->cpi_model == 5 ?
2526                             "Intel Pentium(r) II or Pentium(r) II Xeon(tm)" :
2527                             "Intel Pentium(r) III or Pentium(r) III Xeon(tm)");
2528                 default:
2529                         break;
2530                 }
2531         default:
2532                 break;
2533         }
2534 
2535         /* BrandID is present if the field is nonzero */
2536         if (cpi->cpi_brandid != 0) {
2537                 static const struct {
2538                         uint_t bt_bid;
2539                         const char *bt_str;
2540                 } brand_tbl[] = {
2541                         { 0x1,  "Intel(r) Celeron(r)" },
2542                         { 0x2,  "Intel(r) Pentium(r) III" },
2543                         { 0x3,  "Intel(r) Pentium(r) III Xeon(tm)" },
2544                         { 0x4,  "Intel(r) Pentium(r) III" },
2545                         { 0x6,  "Mobile Intel(r) Pentium(r) III" },
2546                         { 0x7,  "Mobile Intel(r) Celeron(r)" },
2547                         { 0x8,  "Intel(r) Pentium(r) 4" },
2548                         { 0x9,  "Intel(r) Pentium(r) 4" },
2549                         { 0xa,  "Intel(r) Celeron(r)" },
2550                         { 0xb,  "Intel(r) Xeon(tm)" },
2551                         { 0xc,  "Intel(r) Xeon(tm) MP" },
2552                         { 0xe,  "Mobile Intel(r) Pentium(r) 4" },
2553                         { 0xf,  "Mobile Intel(r) Celeron(r)" },
2554                         { 0x11, "Mobile Genuine Intel(r)" },
2555                         { 0x12, "Intel(r) Celeron(r) M" },
2556                         { 0x13, "Mobile Intel(r) Celeron(r)" },
2557                         { 0x14, "Intel(r) Celeron(r)" },
2558                         { 0x15, "Mobile Genuine Intel(r)" },
2559                         { 0x16, "Intel(r) Pentium(r) M" },
2560                         { 0x17, "Mobile Intel(r) Celeron(r)" }
2561                 };
2562                 uint_t btblmax = sizeof (brand_tbl) / sizeof (brand_tbl[0]);
2563                 uint_t sgn;
2564 
2565                 sgn = (cpi->cpi_family << 8) |
2566                     (cpi->cpi_model << 4) | cpi->cpi_step;
2567 
2568                 for (i = 0; i < btblmax; i++)
2569                         if (brand_tbl[i].bt_bid == cpi->cpi_brandid)
2570                                 break;
2571                 if (i < btblmax) {
2572                         if (sgn == 0x6b1 && cpi->cpi_brandid == 3)
2573                                 return ("Intel(r) Celeron(r)");
2574                         if (sgn < 0xf13 && cpi->cpi_brandid == 0xb)
2575                                 return ("Intel(r) Xeon(tm) MP");
2576                         if (sgn < 0xf13 && cpi->cpi_brandid == 0xe)
2577                                 return ("Intel(r) Xeon(tm)");
2578                         return (brand_tbl[i].bt_str);
2579                 }
2580         }
2581 
2582         return (NULL);
2583 }
2584 
2585 static const char *
2586 amd_cpubrand(const struct cpuid_info *cpi)
2587 {
2588         if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2589             cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2590                 return ("i486 compatible");
2591 
2592         switch (cpi->cpi_family) {
2593         case 5:
2594                 switch (cpi->cpi_model) {
2595                 case 0:
2596                 case 1:
2597                 case 2:
2598                 case 3:
2599                 case 4:
2600                 case 5:
2601                         return ("AMD-K5(r)");
2602                 case 6:
2603                 case 7:
2604                         return ("AMD-K6(r)");
2605                 case 8:
2606                         return ("AMD-K6(r)-2");
2607                 case 9:
2608                         return ("AMD-K6(r)-III");
2609                 default:
2610                         return ("AMD (family 5)");
2611                 }
2612         case 6:
2613                 switch (cpi->cpi_model) {
2614                 case 1:
2615                         return ("AMD-K7(tm)");
2616                 case 0:
2617                 case 2:
2618                 case 4:
2619                         return ("AMD Athlon(tm)");
2620                 case 3:
2621                 case 7:
2622                         return ("AMD Duron(tm)");
2623                 case 6:
2624                 case 8:
2625                 case 10:
2626                         /*
2627                          * Use the L2 cache size to distinguish
2628                          */
2629                         return ((cpi->cpi_extd[6].cp_ecx >> 16) >= 256 ?
2630                             "AMD Athlon(tm)" : "AMD Duron(tm)");
2631                 default:
2632                         return ("AMD (family 6)");
2633                 }
2634         default:
2635                 break;
2636         }
2637 
2638         if (cpi->cpi_family == 0xf && cpi->cpi_model == 5 &&
2639             cpi->cpi_brandid != 0) {
2640                 switch (BITX(cpi->cpi_brandid, 7, 5)) {
2641                 case 3:
2642                         return ("AMD Opteron(tm) UP 1xx");
2643                 case 4:
2644                         return ("AMD Opteron(tm) DP 2xx");
2645                 case 5:
2646                         return ("AMD Opteron(tm) MP 8xx");
2647                 default:
2648                         return ("AMD Opteron(tm)");
2649                 }
2650         }
2651 
2652         return (NULL);
2653 }
2654 
2655 static const char *
2656 cyrix_cpubrand(struct cpuid_info *cpi, uint_t type)
2657 {
2658         if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2659             cpi->cpi_maxeax < 1 || cpi->cpi_family < 5 ||
2660             type == X86_TYPE_CYRIX_486)
2661                 return ("i486 compatible");
2662 
2663         switch (type) {
2664         case X86_TYPE_CYRIX_6x86:
2665                 return ("Cyrix 6x86");
2666         case X86_TYPE_CYRIX_6x86L:
2667                 return ("Cyrix 6x86L");
2668         case X86_TYPE_CYRIX_6x86MX:
2669                 return ("Cyrix 6x86MX");
2670         case X86_TYPE_CYRIX_GXm:
2671                 return ("Cyrix GXm");
2672         case X86_TYPE_CYRIX_MediaGX:
2673                 return ("Cyrix MediaGX");
2674         case X86_TYPE_CYRIX_MII:
2675                 return ("Cyrix M2");
2676         case X86_TYPE_VIA_CYRIX_III:
2677                 return ("VIA Cyrix M3");
2678         default:
2679                 /*
2680                  * Have another wild guess ..
2681                  */
2682                 if (cpi->cpi_family == 4 && cpi->cpi_model == 9)
2683                         return ("Cyrix 5x86");
2684                 else if (cpi->cpi_family == 5) {
2685                         switch (cpi->cpi_model) {
2686                         case 2:
2687                                 return ("Cyrix 6x86");  /* Cyrix M1 */
2688                         case 4:
2689                                 return ("Cyrix MediaGX");
2690                         default:
2691                                 break;
2692                         }
2693                 } else if (cpi->cpi_family == 6) {
2694                         switch (cpi->cpi_model) {
2695                         case 0:
2696                                 return ("Cyrix 6x86MX"); /* Cyrix M2? */
2697                         case 5:
2698                         case 6:
2699                         case 7:
2700                         case 8:
2701                         case 9:
2702                                 return ("VIA C3");
2703                         default:
2704                                 break;
2705                         }
2706                 }
2707                 break;
2708         }
2709         return (NULL);
2710 }
2711 
2712 /*
2713  * This only gets called in the case that the CPU extended
2714  * feature brand string (0x80000002, 0x80000003, 0x80000004)
2715  * aren't available, or contain null bytes for some reason.
2716  */
2717 static void
2718 fabricate_brandstr(struct cpuid_info *cpi)
2719 {
2720         const char *brand = NULL;
2721 
2722         switch (cpi->cpi_vendor) {
2723         case X86_VENDOR_Intel:
2724                 brand = intel_cpubrand(cpi);
2725                 break;
2726         case X86_VENDOR_AMD:
2727                 brand = amd_cpubrand(cpi);
2728                 break;
2729         case X86_VENDOR_Cyrix:
2730                 brand = cyrix_cpubrand(cpi, x86_type);
2731                 break;
2732         case X86_VENDOR_NexGen:
2733                 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2734                         brand = "NexGen Nx586";
2735                 break;
2736         case X86_VENDOR_Centaur:
2737                 if (cpi->cpi_family == 5)
2738                         switch (cpi->cpi_model) {
2739                         case 4:
2740                                 brand = "Centaur C6";
2741                                 break;
2742                         case 8:
2743                                 brand = "Centaur C2";
2744                                 break;
2745                         case 9:
2746                                 brand = "Centaur C3";
2747                                 break;
2748                         default:
2749                                 break;
2750                         }
2751                 break;
2752         case X86_VENDOR_Rise:
2753                 if (cpi->cpi_family == 5 &&
2754                     (cpi->cpi_model == 0 || cpi->cpi_model == 2))
2755                         brand = "Rise mP6";
2756                 break;
2757         case X86_VENDOR_SiS:
2758                 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2759                         brand = "SiS 55x";
2760                 break;
2761         case X86_VENDOR_TM:
2762                 if (cpi->cpi_family == 5 && cpi->cpi_model == 4)
2763                         brand = "Transmeta Crusoe TM3x00 or TM5x00";
2764                 break;
2765         case X86_VENDOR_NSC:
2766         case X86_VENDOR_UMC:
2767         default:
2768                 break;
2769         }
2770         if (brand) {
2771                 (void) strcpy((char *)cpi->cpi_brandstr, brand);
2772                 return;
2773         }
2774 
2775         /*
2776          * If all else fails ...
2777          */
2778         (void) snprintf(cpi->cpi_brandstr, sizeof (cpi->cpi_brandstr),
2779             "%s %d.%d.%d", cpi->cpi_vendorstr, cpi->cpi_family,
2780             cpi->cpi_model, cpi->cpi_step);
2781 }
2782 
2783 /*
2784  * This routine is called just after kernel memory allocation
2785  * becomes available on cpu0, and as part of mp_startup() on
2786  * the other cpus.
2787  *
2788  * Fixup the brand string, and collect any information from cpuid
2789  * that requires dynamically allocated storage to represent.
2790  */
2791 /*ARGSUSED*/
2792 void
2793 cpuid_pass3(cpu_t *cpu)
2794 {
2795         int     i, max, shft, level, size;
2796         struct cpuid_regs regs;
2797         struct cpuid_regs *cp;
2798         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2799 
2800         ASSERT(cpi->cpi_pass == 2);
2801 
2802         /*
2803          * Function 4: Deterministic cache parameters
2804          *
2805          * Take this opportunity to detect the number of threads
2806          * sharing the last level cache, and construct a corresponding
2807          * cache id. The respective cpuid_info members are initialized
2808          * to the default case of "no last level cache sharing".
2809          */
2810         cpi->cpi_ncpu_shr_last_cache = 1;
2811         cpi->cpi_last_lvl_cacheid = cpu->cpu_id;
2812 
2813         if (cpi->cpi_maxeax >= 4 && cpi->cpi_vendor == X86_VENDOR_Intel) {
2814 
2815                 /*
2816                  * Find the # of elements (size) returned by fn 4, and along
2817                  * the way detect last level cache sharing details.
2818                  */
2819                 bzero(&regs, sizeof (regs));
2820                 cp = &regs;
2821                 for (i = 0, max = 0; i < CPI_FN4_ECX_MAX; i++) {
2822                         cp->cp_eax = 4;
2823                         cp->cp_ecx = i;
2824 
2825                         (void) __cpuid_insn(cp);
2826 
2827                         if (CPI_CACHE_TYPE(cp) == 0)
2828                                 break;
2829                         level = CPI_CACHE_LVL(cp);
2830                         if (level > max) {
2831                                 max = level;
2832                                 cpi->cpi_ncpu_shr_last_cache =
2833                                     CPI_NTHR_SHR_CACHE(cp) + 1;
2834                         }
2835                 }
2836                 cpi->cpi_std_4_size = size = i;
2837 
2838                 /*
2839                  * Allocate the cpi_std_4 array. The first element
2840                  * references the regs for fn 4, %ecx == 0, which
2841                  * cpuid_pass2() stashed in cpi->cpi_std[4].
2842                  */
2843                 if (size > 0) {
2844                         cpi->cpi_std_4 =
2845                             kmem_alloc(size * sizeof (cp), KM_SLEEP);
2846                         cpi->cpi_std_4[0] = &cpi->cpi_std[4];
2847 
2848                         /*
2849                          * Allocate storage to hold the additional regs
2850                          * for function 4, %ecx == 1 .. cpi_std_4_size.
2851                          *
2852                          * The regs for fn 4, %ecx == 0 has already
2853                          * been allocated as indicated above.
2854                          */
2855                         for (i = 1; i < size; i++) {
2856                                 cp = cpi->cpi_std_4[i] =
2857                                     kmem_zalloc(sizeof (regs), KM_SLEEP);
2858                                 cp->cp_eax = 4;
2859                                 cp->cp_ecx = i;
2860 
2861                                 (void) __cpuid_insn(cp);
2862                         }
2863                 }
2864                 /*
2865                  * Determine the number of bits needed to represent
2866                  * the number of CPUs sharing the last level cache.
2867                  *
2868                  * Shift off that number of bits from the APIC id to
2869                  * derive the cache id.
2870                  */
2871                 shft = 0;
2872                 for (i = 1; i < cpi->cpi_ncpu_shr_last_cache; i <<= 1)
2873                         shft++;
2874                 cpi->cpi_last_lvl_cacheid = cpi->cpi_apicid >> shft;
2875         }
2876 
2877         /*
2878          * Now fixup the brand string
2879          */
2880         if ((cpi->cpi_xmaxeax & 0x80000000) == 0) {
2881                 fabricate_brandstr(cpi);
2882         } else {
2883 
2884                 /*
2885                  * If we successfully extracted a brand string from the cpuid
2886                  * instruction, clean it up by removing leading spaces and
2887                  * similar junk.
2888                  */
2889                 if (cpi->cpi_brandstr[0]) {
2890                         size_t maxlen = sizeof (cpi->cpi_brandstr);
2891                         char *src, *dst;
2892 
2893                         dst = src = (char *)cpi->cpi_brandstr;
2894                         src[maxlen - 1] = '\0';
2895                         /*
2896                          * strip leading spaces
2897                          */
2898                         while (*src == ' ')
2899                                 src++;
2900                         /*
2901                          * Remove any 'Genuine' or "Authentic" prefixes
2902                          */
2903                         if (strncmp(src, "Genuine ", 8) == 0)
2904                                 src += 8;
2905                         if (strncmp(src, "Authentic ", 10) == 0)
2906                                 src += 10;
2907 
2908                         /*
2909                          * Now do an in-place copy.
2910                          * Map (R) to (r) and (TM) to (tm).
2911                          * The era of teletypes is long gone, and there's
2912                          * -really- no need to shout.
2913                          */
2914                         while (*src != '\0') {
2915                                 if (src[0] == '(') {
2916                                         if (strncmp(src + 1, "R)", 2) == 0) {
2917                                                 (void) strncpy(dst, "(r)", 3);
2918                                                 src += 3;
2919                                                 dst += 3;
2920                                                 continue;
2921                                         }
2922                                         if (strncmp(src + 1, "TM)", 3) == 0) {
2923                                                 (void) strncpy(dst, "(tm)", 4);
2924                                                 src += 4;
2925                                                 dst += 4;
2926                                                 continue;
2927                                         }
2928                                 }
2929                                 *dst++ = *src++;
2930                         }
2931                         *dst = '\0';
2932 
2933                         /*
2934                          * Finally, remove any trailing spaces
2935                          */
2936                         while (--dst > cpi->cpi_brandstr)
2937                                 if (*dst == ' ')
2938                                         *dst = '\0';
2939                                 else
2940                                         break;
2941                 } else
2942                         fabricate_brandstr(cpi);
2943         }
2944         cpi->cpi_pass = 3;
2945 }
2946 
2947 /*
2948  * This routine is called out of bind_hwcap() much later in the life
2949  * of the kernel (post_startup()).  The job of this routine is to resolve
2950  * the hardware feature support and kernel support for those features into
2951  * what we're actually going to tell applications via the aux vector.
2952  */
2953 void
2954 cpuid_pass4(cpu_t *cpu, uint_t *hwcap_out)
2955 {
2956         struct cpuid_info *cpi;
2957         uint_t hwcap_flags = 0, hwcap_flags_2 = 0;
2958 
2959         if (cpu == NULL)
2960                 cpu = CPU;
2961         cpi = cpu->cpu_m.mcpu_cpi;
2962 
2963         ASSERT(cpi->cpi_pass == 3);
2964 
2965         if (cpi->cpi_maxeax >= 1) {
2966                 uint32_t *edx = &cpi->cpi_support[STD_EDX_FEATURES];
2967                 uint32_t *ecx = &cpi->cpi_support[STD_ECX_FEATURES];
2968                 uint32_t *ebx = &cpi->cpi_support[STD_EBX_FEATURES];
2969 
2970                 *edx = CPI_FEATURES_EDX(cpi);
2971                 *ecx = CPI_FEATURES_ECX(cpi);
2972                 *ebx = CPI_FEATURES_7_0_EBX(cpi);
2973 
2974                 /*
2975                  * [these require explicit kernel support]
2976                  */
2977                 if (!is_x86_feature(x86_featureset, X86FSET_SEP))
2978                         *edx &= ~CPUID_INTC_EDX_SEP;
2979 
2980                 if (!is_x86_feature(x86_featureset, X86FSET_SSE))
2981                         *edx &= ~(CPUID_INTC_EDX_FXSR|CPUID_INTC_EDX_SSE);
2982                 if (!is_x86_feature(x86_featureset, X86FSET_SSE2))
2983                         *edx &= ~CPUID_INTC_EDX_SSE2;
2984 
2985                 if (!is_x86_feature(x86_featureset, X86FSET_HTT))
2986                         *edx &= ~CPUID_INTC_EDX_HTT;
2987 
2988                 if (!is_x86_feature(x86_featureset, X86FSET_SSE3))
2989                         *ecx &= ~CPUID_INTC_ECX_SSE3;
2990 
2991                 if (!is_x86_feature(x86_featureset, X86FSET_SSSE3))
2992                         *ecx &= ~CPUID_INTC_ECX_SSSE3;
2993                 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_1))
2994                         *ecx &= ~CPUID_INTC_ECX_SSE4_1;
2995                 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_2))
2996                         *ecx &= ~CPUID_INTC_ECX_SSE4_2;
2997                 if (!is_x86_feature(x86_featureset, X86FSET_AES))
2998                         *ecx &= ~CPUID_INTC_ECX_AES;
2999                 if (!is_x86_feature(x86_featureset, X86FSET_PCLMULQDQ))
3000                         *ecx &= ~CPUID_INTC_ECX_PCLMULQDQ;
3001                 if (!is_x86_feature(x86_featureset, X86FSET_XSAVE))
3002                         *ecx &= ~(CPUID_INTC_ECX_XSAVE |
3003                             CPUID_INTC_ECX_OSXSAVE);
3004                 if (!is_x86_feature(x86_featureset, X86FSET_AVX))
3005                         *ecx &= ~CPUID_INTC_ECX_AVX;
3006                 if (!is_x86_feature(x86_featureset, X86FSET_F16C))
3007                         *ecx &= ~CPUID_INTC_ECX_F16C;
3008                 if (!is_x86_feature(x86_featureset, X86FSET_FMA))
3009                         *ecx &= ~CPUID_INTC_ECX_FMA;
3010                 if (!is_x86_feature(x86_featureset, X86FSET_BMI1))
3011                         *ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
3012                 if (!is_x86_feature(x86_featureset, X86FSET_BMI2))
3013                         *ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
3014                 if (!is_x86_feature(x86_featureset, X86FSET_AVX2))
3015                         *ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
3016                 if (!is_x86_feature(x86_featureset, X86FSET_RDSEED))
3017                         *ebx &= ~CPUID_INTC_EBX_7_0_RDSEED;
3018                 if (!is_x86_feature(x86_featureset, X86FSET_ADX))
3019                         *ebx &= ~CPUID_INTC_EBX_7_0_ADX;
3020 
3021                 /*
3022                  * [no explicit support required beyond x87 fp context]
3023                  */
3024                 if (!fpu_exists)
3025                         *edx &= ~(CPUID_INTC_EDX_FPU | CPUID_INTC_EDX_MMX);
3026 
3027                 /*
3028                  * Now map the supported feature vector to things that we
3029                  * think userland will care about.
3030                  */
3031                 if (*edx & CPUID_INTC_EDX_SEP)
3032                         hwcap_flags |= AV_386_SEP;
3033                 if (*edx & CPUID_INTC_EDX_SSE)
3034                         hwcap_flags |= AV_386_FXSR | AV_386_SSE;
3035                 if (*edx & CPUID_INTC_EDX_SSE2)
3036                         hwcap_flags |= AV_386_SSE2;
3037                 if (*ecx & CPUID_INTC_ECX_SSE3)
3038                         hwcap_flags |= AV_386_SSE3;
3039                 if (*ecx & CPUID_INTC_ECX_SSSE3)
3040                         hwcap_flags |= AV_386_SSSE3;
3041                 if (*ecx & CPUID_INTC_ECX_SSE4_1)
3042                         hwcap_flags |= AV_386_SSE4_1;
3043                 if (*ecx & CPUID_INTC_ECX_SSE4_2)
3044                         hwcap_flags |= AV_386_SSE4_2;
3045                 if (*ecx & CPUID_INTC_ECX_MOVBE)
3046                         hwcap_flags |= AV_386_MOVBE;
3047                 if (*ecx & CPUID_INTC_ECX_AES)
3048                         hwcap_flags |= AV_386_AES;
3049                 if (*ecx & CPUID_INTC_ECX_PCLMULQDQ)
3050                         hwcap_flags |= AV_386_PCLMULQDQ;
3051                 if ((*ecx & CPUID_INTC_ECX_XSAVE) &&
3052                     (*ecx & CPUID_INTC_ECX_OSXSAVE)) {
3053                         hwcap_flags |= AV_386_XSAVE;
3054 
3055                         if (*ecx & CPUID_INTC_ECX_AVX) {
3056                                 uint32_t *ecx_7 = &CPI_FEATURES_7_0_ECX(cpi);
3057                                 uint32_t *edx_7 = &CPI_FEATURES_7_0_EDX(cpi);
3058 
3059                                 hwcap_flags |= AV_386_AVX;
3060                                 if (*ecx & CPUID_INTC_ECX_F16C)
3061                                         hwcap_flags_2 |= AV_386_2_F16C;
3062                                 if (*ecx & CPUID_INTC_ECX_FMA)
3063                                         hwcap_flags_2 |= AV_386_2_FMA;
3064 
3065                                 if (*ebx & CPUID_INTC_EBX_7_0_BMI1)
3066                                         hwcap_flags_2 |= AV_386_2_BMI1;
3067                                 if (*ebx & CPUID_INTC_EBX_7_0_BMI2)
3068                                         hwcap_flags_2 |= AV_386_2_BMI2;
3069                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX2)
3070                                         hwcap_flags_2 |= AV_386_2_AVX2;
3071                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512F)
3072                                         hwcap_flags_2 |= AV_386_2_AVX512F;
3073                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512DQ)
3074                                         hwcap_flags_2 |= AV_386_2_AVX512DQ;
3075                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512IFMA)
3076                                         hwcap_flags_2 |= AV_386_2_AVX512IFMA;
3077                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512PF)
3078                                         hwcap_flags_2 |= AV_386_2_AVX512PF;
3079                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512ER)
3080                                         hwcap_flags_2 |= AV_386_2_AVX512ER;
3081                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512CD)
3082                                         hwcap_flags_2 |= AV_386_2_AVX512CD;
3083                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512BW)
3084                                         hwcap_flags_2 |= AV_386_2_AVX512BW;
3085                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512VL)
3086                                         hwcap_flags_2 |= AV_386_2_AVX512VL;
3087 
3088                                 if (*ecx_7 & CPUID_INTC_ECX_7_0_AVX512VBMI)
3089                                         hwcap_flags_2 |= AV_386_2_AVX512VBMI;
3090                                 if (*ecx_7 & CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
3091                                         hwcap_flags_2 |= AV_386_2_AVX512VPOPCDQ;
3092 
3093                                 if (*edx_7 & CPUID_INTC_EDX_7_0_AVX5124NNIW)
3094                                         hwcap_flags_2 |= AV_386_2_AVX512_4NNIW;
3095                                 if (*edx_7 & CPUID_INTC_EDX_7_0_AVX5124FMAPS)
3096                                         hwcap_flags_2 |= AV_386_2_AVX512_4FMAPS;
3097                         }
3098                 }
3099                 if (*ecx & CPUID_INTC_ECX_VMX)
3100                         hwcap_flags |= AV_386_VMX;
3101                 if (*ecx & CPUID_INTC_ECX_POPCNT)
3102                         hwcap_flags |= AV_386_POPCNT;
3103                 if (*edx & CPUID_INTC_EDX_FPU)
3104                         hwcap_flags |= AV_386_FPU;
3105                 if (*edx & CPUID_INTC_EDX_MMX)
3106                         hwcap_flags |= AV_386_MMX;
3107 
3108                 if (*edx & CPUID_INTC_EDX_TSC)
3109                         hwcap_flags |= AV_386_TSC;
3110                 if (*edx & CPUID_INTC_EDX_CX8)
3111                         hwcap_flags |= AV_386_CX8;
3112                 if (*edx & CPUID_INTC_EDX_CMOV)
3113                         hwcap_flags |= AV_386_CMOV;
3114                 if (*ecx & CPUID_INTC_ECX_CX16)
3115                         hwcap_flags |= AV_386_CX16;
3116 
3117                 if (*ecx & CPUID_INTC_ECX_RDRAND)
3118                         hwcap_flags_2 |= AV_386_2_RDRAND;
3119                 if (*ebx & CPUID_INTC_EBX_7_0_ADX)
3120                         hwcap_flags_2 |= AV_386_2_ADX;
3121                 if (*ebx & CPUID_INTC_EBX_7_0_RDSEED)
3122                         hwcap_flags_2 |= AV_386_2_RDSEED;
3123 
3124         }
3125 
3126         /* Detect systems with a potential CPUID limit  */
3127         if (cpi->cpi_vendor == X86_VENDOR_Intel && cpi->cpi_maxeax < 4) {
3128                 cmn_err(CE_NOTE, "CPUID limit detected, "
3129                     "see the CPUID(7D) man page for details\n");
3130         }
3131 
3132 
3133         if (cpi->cpi_xmaxeax < 0x80000001)
3134                 goto pass4_done;
3135 
3136         switch (cpi->cpi_vendor) {
3137                 struct cpuid_regs cp;
3138                 uint32_t *edx, *ecx;
3139 
3140         case X86_VENDOR_Intel:
3141                 /*
3142                  * Seems like Intel duplicated what we necessary
3143                  * here to make the initial crop of 64-bit OS's work.
3144                  * Hopefully, those are the only "extended" bits
3145                  * they'll add.
3146                  */
3147                 /*FALLTHROUGH*/
3148 
3149         case X86_VENDOR_AMD:
3150                 edx = &cpi->cpi_support[AMD_EDX_FEATURES];
3151                 ecx = &cpi->cpi_support[AMD_ECX_FEATURES];
3152 
3153                 *edx = CPI_FEATURES_XTD_EDX(cpi);
3154                 *ecx = CPI_FEATURES_XTD_ECX(cpi);
3155 
3156                 /*
3157                  * [these features require explicit kernel support]
3158                  */
3159                 switch (cpi->cpi_vendor) {
3160                 case X86_VENDOR_Intel:
3161                         if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
3162                                 *edx &= ~CPUID_AMD_EDX_TSCP;
3163                         break;
3164 
3165                 case X86_VENDOR_AMD:
3166                         if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
3167                                 *edx &= ~CPUID_AMD_EDX_TSCP;
3168                         if (!is_x86_feature(x86_featureset, X86FSET_SSE4A))
3169                                 *ecx &= ~CPUID_AMD_ECX_SSE4A;
3170                         break;
3171 
3172                 default:
3173                         break;
3174                 }
3175 
3176                 /*
3177                  * [no explicit support required beyond
3178                  * x87 fp context and exception handlers]
3179                  */
3180                 if (!fpu_exists)
3181                         *edx &= ~(CPUID_AMD_EDX_MMXamd |
3182                             CPUID_AMD_EDX_3DNow | CPUID_AMD_EDX_3DNowx);
3183 
3184                 if (!is_x86_feature(x86_featureset, X86FSET_NX))
3185                         *edx &= ~CPUID_AMD_EDX_NX;
3186 #if !defined(__amd64)
3187                 *edx &= ~CPUID_AMD_EDX_LM;
3188 #endif
3189                 /*
3190                  * Now map the supported feature vector to
3191                  * things that we think userland will care about.
3192                  */
3193 #if defined(__amd64)
3194                 if (*edx & CPUID_AMD_EDX_SYSC)
3195                         hwcap_flags |= AV_386_AMD_SYSC;
3196 #endif
3197                 if (*edx & CPUID_AMD_EDX_MMXamd)
3198                         hwcap_flags |= AV_386_AMD_MMX;
3199                 if (*edx & CPUID_AMD_EDX_3DNow)
3200                         hwcap_flags |= AV_386_AMD_3DNow;
3201                 if (*edx & CPUID_AMD_EDX_3DNowx)
3202                         hwcap_flags |= AV_386_AMD_3DNowx;
3203                 if (*ecx & CPUID_AMD_ECX_SVM)
3204                         hwcap_flags |= AV_386_AMD_SVM;
3205 
3206                 switch (cpi->cpi_vendor) {
3207                 case X86_VENDOR_AMD:
3208                         if (*edx & CPUID_AMD_EDX_TSCP)
3209                                 hwcap_flags |= AV_386_TSCP;
3210                         if (*ecx & CPUID_AMD_ECX_AHF64)
3211                                 hwcap_flags |= AV_386_AHF;
3212                         if (*ecx & CPUID_AMD_ECX_SSE4A)
3213                                 hwcap_flags |= AV_386_AMD_SSE4A;
3214                         if (*ecx & CPUID_AMD_ECX_LZCNT)
3215                                 hwcap_flags |= AV_386_AMD_LZCNT;
3216                         break;
3217 
3218                 case X86_VENDOR_Intel:
3219                         if (*edx & CPUID_AMD_EDX_TSCP)
3220                                 hwcap_flags |= AV_386_TSCP;
3221                         /*
3222                          * Aarrgh.
3223                          * Intel uses a different bit in the same word.
3224                          */
3225                         if (*ecx & CPUID_INTC_ECX_AHF64)
3226                                 hwcap_flags |= AV_386_AHF;
3227                         break;
3228 
3229                 default:
3230                         break;
3231                 }
3232                 break;
3233 
3234         case X86_VENDOR_TM:
3235                 cp.cp_eax = 0x80860001;
3236                 (void) __cpuid_insn(&cp);
3237                 cpi->cpi_support[TM_EDX_FEATURES] = cp.cp_edx;
3238                 break;
3239 
3240         default:
3241                 break;
3242         }
3243 
3244 pass4_done:
3245         cpi->cpi_pass = 4;
3246         if (hwcap_out != NULL) {
3247                 hwcap_out[0] = hwcap_flags;
3248                 hwcap_out[1] = hwcap_flags_2;
3249         }
3250 }
3251 
3252 
3253 /*
3254  * Simulate the cpuid instruction using the data we previously
3255  * captured about this CPU.  We try our best to return the truth
3256  * about the hardware, independently of kernel support.
3257  */
3258 uint32_t
3259 cpuid_insn(cpu_t *cpu, struct cpuid_regs *cp)
3260 {
3261         struct cpuid_info *cpi;
3262         struct cpuid_regs *xcp;
3263 
3264         if (cpu == NULL)
3265                 cpu = CPU;
3266         cpi = cpu->cpu_m.mcpu_cpi;
3267 
3268         ASSERT(cpuid_checkpass(cpu, 3));
3269 
3270         /*
3271          * CPUID data is cached in two separate places: cpi_std for standard
3272          * CPUID functions, and cpi_extd for extended CPUID functions.
3273          */
3274         if (cp->cp_eax <= cpi->cpi_maxeax && cp->cp_eax < NMAX_CPI_STD)
3275                 xcp = &cpi->cpi_std[cp->cp_eax];
3276         else if (cp->cp_eax >= 0x80000000 && cp->cp_eax <= cpi->cpi_xmaxeax &&
3277             cp->cp_eax < 0x80000000 + NMAX_CPI_EXTD)
3278                 xcp = &cpi->cpi_extd[cp->cp_eax - 0x80000000];
3279         else
3280                 /*
3281                  * The caller is asking for data from an input parameter which
3282                  * the kernel has not cached.  In this case we go fetch from
3283                  * the hardware and return the data directly to the user.
3284                  */
3285                 return (__cpuid_insn(cp));
3286 
3287         cp->cp_eax = xcp->cp_eax;
3288         cp->cp_ebx = xcp->cp_ebx;
3289         cp->cp_ecx = xcp->cp_ecx;
3290         cp->cp_edx = xcp->cp_edx;
3291         return (cp->cp_eax);
3292 }
3293 
3294 int
3295 cpuid_checkpass(cpu_t *cpu, int pass)
3296 {
3297         return (cpu != NULL && cpu->cpu_m.mcpu_cpi != NULL &&
3298             cpu->cpu_m.mcpu_cpi->cpi_pass >= pass);
3299 }
3300 
3301 int
3302 cpuid_getbrandstr(cpu_t *cpu, char *s, size_t n)
3303 {
3304         ASSERT(cpuid_checkpass(cpu, 3));
3305 
3306         return (snprintf(s, n, "%s", cpu->cpu_m.mcpu_cpi->cpi_brandstr));
3307 }
3308 
3309 int
3310 cpuid_is_cmt(cpu_t *cpu)
3311 {
3312         if (cpu == NULL)
3313                 cpu = CPU;
3314 
3315         ASSERT(cpuid_checkpass(cpu, 1));
3316 
3317         return (cpu->cpu_m.mcpu_cpi->cpi_chipid >= 0);
3318 }
3319 
3320 /*
3321  * AMD and Intel both implement the 64-bit variant of the syscall
3322  * instruction (syscallq), so if there's -any- support for syscall,
3323  * cpuid currently says "yes, we support this".
3324  *
3325  * However, Intel decided to -not- implement the 32-bit variant of the
3326  * syscall instruction, so we provide a predicate to allow our caller
3327  * to test that subtlety here.
3328  *
3329  * XXPV Currently, 32-bit syscall instructions don't work via the hypervisor,
3330  *      even in the case where the hardware would in fact support it.
3331  */
3332 /*ARGSUSED*/
3333 int
3334 cpuid_syscall32_insn(cpu_t *cpu)
3335 {
3336         ASSERT(cpuid_checkpass((cpu == NULL ? CPU : cpu), 1));
3337 
3338 #if !defined(__xpv)
3339         if (cpu == NULL)
3340                 cpu = CPU;
3341 
3342         /*CSTYLED*/
3343         {
3344                 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3345 
3346                 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
3347                     cpi->cpi_xmaxeax >= 0x80000001 &&
3348                     (CPI_FEATURES_XTD_EDX(cpi) & CPUID_AMD_EDX_SYSC))
3349                         return (1);
3350         }
3351 #endif
3352         return (0);
3353 }
3354 
3355 int
3356 cpuid_getidstr(cpu_t *cpu, char *s, size_t n)
3357 {
3358         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3359 
3360         static const char fmt[] =
3361             "x86 (%s %X family %d model %d step %d clock %d MHz)";
3362         static const char fmt_ht[] =
3363             "x86 (chipid 0x%x %s %X family %d model %d step %d clock %d MHz)";
3364 
3365         ASSERT(cpuid_checkpass(cpu, 1));
3366 
3367         if (cpuid_is_cmt(cpu))
3368                 return (snprintf(s, n, fmt_ht, cpi->cpi_chipid,
3369                     cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
3370                     cpi->cpi_family, cpi->cpi_model,
3371                     cpi->cpi_step, cpu->cpu_type_info.pi_clock));
3372         return (snprintf(s, n, fmt,
3373             cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
3374             cpi->cpi_family, cpi->cpi_model,
3375             cpi->cpi_step, cpu->cpu_type_info.pi_clock));
3376 }
3377 
3378 const char *
3379 cpuid_getvendorstr(cpu_t *cpu)
3380 {
3381         ASSERT(cpuid_checkpass(cpu, 1));
3382         return ((const char *)cpu->cpu_m.mcpu_cpi->cpi_vendorstr);
3383 }
3384 
3385 uint_t
3386 cpuid_getvendor(cpu_t *cpu)
3387 {
3388         ASSERT(cpuid_checkpass(cpu, 1));
3389         return (cpu->cpu_m.mcpu_cpi->cpi_vendor);
3390 }
3391 
3392 uint_t
3393 cpuid_getfamily(cpu_t *cpu)
3394 {
3395         ASSERT(cpuid_checkpass(cpu, 1));
3396         return (cpu->cpu_m.mcpu_cpi->cpi_family);
3397 }
3398 
3399 uint_t
3400 cpuid_getmodel(cpu_t *cpu)
3401 {
3402         ASSERT(cpuid_checkpass(cpu, 1));
3403         return (cpu->cpu_m.mcpu_cpi->cpi_model);
3404 }
3405 
3406 uint_t
3407 cpuid_get_ncpu_per_chip(cpu_t *cpu)
3408 {
3409         ASSERT(cpuid_checkpass(cpu, 1));
3410         return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_per_chip);
3411 }
3412 
3413 uint_t
3414 cpuid_get_ncore_per_chip(cpu_t *cpu)
3415 {
3416         ASSERT(cpuid_checkpass(cpu, 1));
3417         return (cpu->cpu_m.mcpu_cpi->cpi_ncore_per_chip);
3418 }
3419 
3420 uint_t
3421 cpuid_get_ncpu_sharing_last_cache(cpu_t *cpu)
3422 {
3423         ASSERT(cpuid_checkpass(cpu, 2));
3424         return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_shr_last_cache);
3425 }
3426 
3427 id_t
3428 cpuid_get_last_lvl_cacheid(cpu_t *cpu)
3429 {
3430         ASSERT(cpuid_checkpass(cpu, 2));
3431         return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
3432 }
3433 
3434 uint_t
3435 cpuid_getstep(cpu_t *cpu)
3436 {
3437         ASSERT(cpuid_checkpass(cpu, 1));
3438         return (cpu->cpu_m.mcpu_cpi->cpi_step);
3439 }
3440 
3441 uint_t
3442 cpuid_getsig(struct cpu *cpu)
3443 {
3444         ASSERT(cpuid_checkpass(cpu, 1));
3445         return (cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_eax);
3446 }
3447 
3448 uint32_t
3449 cpuid_getchiprev(struct cpu *cpu)
3450 {
3451         ASSERT(cpuid_checkpass(cpu, 1));
3452         return (cpu->cpu_m.mcpu_cpi->cpi_chiprev);
3453 }
3454 
3455 const char *
3456 cpuid_getchiprevstr(struct cpu *cpu)
3457 {
3458         ASSERT(cpuid_checkpass(cpu, 1));
3459         return (cpu->cpu_m.mcpu_cpi->cpi_chiprevstr);
3460 }
3461 
3462 uint32_t
3463 cpuid_getsockettype(struct cpu *cpu)
3464 {
3465         ASSERT(cpuid_checkpass(cpu, 1));
3466         return (cpu->cpu_m.mcpu_cpi->cpi_socket);
3467 }
3468 
3469 const char *
3470 cpuid_getsocketstr(cpu_t *cpu)
3471 {
3472         static const char *socketstr = NULL;
3473         struct cpuid_info *cpi;
3474 
3475         ASSERT(cpuid_checkpass(cpu, 1));
3476         cpi = cpu->cpu_m.mcpu_cpi;
3477 
3478         /* Assume that socket types are the same across the system */
3479         if (socketstr == NULL)
3480                 socketstr = _cpuid_sktstr(cpi->cpi_vendor, cpi->cpi_family,
3481                     cpi->cpi_model, cpi->cpi_step);
3482 
3483 
3484         return (socketstr);
3485 }
3486 
3487 int
3488 cpuid_get_chipid(cpu_t *cpu)
3489 {
3490         ASSERT(cpuid_checkpass(cpu, 1));
3491 
3492         if (cpuid_is_cmt(cpu))
3493                 return (cpu->cpu_m.mcpu_cpi->cpi_chipid);
3494         return (cpu->cpu_id);
3495 }
3496 
3497 id_t
3498 cpuid_get_coreid(cpu_t *cpu)
3499 {
3500         ASSERT(cpuid_checkpass(cpu, 1));
3501         return (cpu->cpu_m.mcpu_cpi->cpi_coreid);
3502 }
3503 
3504 int
3505 cpuid_get_pkgcoreid(cpu_t *cpu)
3506 {
3507         ASSERT(cpuid_checkpass(cpu, 1));
3508         return (cpu->cpu_m.mcpu_cpi->cpi_pkgcoreid);
3509 }
3510 
3511 int
3512 cpuid_get_clogid(cpu_t *cpu)
3513 {
3514         ASSERT(cpuid_checkpass(cpu, 1));
3515         return (cpu->cpu_m.mcpu_cpi->cpi_clogid);
3516 }
3517 
3518 int
3519 cpuid_get_cacheid(cpu_t *cpu)
3520 {
3521         ASSERT(cpuid_checkpass(cpu, 1));
3522         return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
3523 }
3524 
3525 uint_t
3526 cpuid_get_procnodeid(cpu_t *cpu)
3527 {
3528         ASSERT(cpuid_checkpass(cpu, 1));
3529         return (cpu->cpu_m.mcpu_cpi->cpi_procnodeid);
3530 }
3531 
3532 uint_t
3533 cpuid_get_procnodes_per_pkg(cpu_t *cpu)
3534 {
3535         ASSERT(cpuid_checkpass(cpu, 1));
3536         return (cpu->cpu_m.mcpu_cpi->cpi_procnodes_per_pkg);
3537 }
3538 
3539 uint_t
3540 cpuid_get_compunitid(cpu_t *cpu)
3541 {
3542         ASSERT(cpuid_checkpass(cpu, 1));
3543         return (cpu->cpu_m.mcpu_cpi->cpi_compunitid);
3544 }
3545 
3546 uint_t
3547 cpuid_get_cores_per_compunit(cpu_t *cpu)
3548 {
3549         ASSERT(cpuid_checkpass(cpu, 1));
3550         return (cpu->cpu_m.mcpu_cpi->cpi_cores_per_compunit);
3551 }
3552 
3553 /*ARGSUSED*/
3554 int
3555 cpuid_have_cr8access(cpu_t *cpu)
3556 {
3557 #if defined(__amd64)
3558         return (1);
3559 #else
3560         struct cpuid_info *cpi;
3561 
3562         ASSERT(cpu != NULL);
3563         cpi = cpu->cpu_m.mcpu_cpi;
3564         if (cpi->cpi_vendor == X86_VENDOR_AMD && cpi->cpi_maxeax >= 1 &&
3565             (CPI_FEATURES_XTD_ECX(cpi) & CPUID_AMD_ECX_CR8D) != 0)
3566                 return (1);
3567         return (0);
3568 #endif
3569 }
3570 
3571 uint32_t
3572 cpuid_get_apicid(cpu_t *cpu)
3573 {
3574         ASSERT(cpuid_checkpass(cpu, 1));
3575         if (cpu->cpu_m.mcpu_cpi->cpi_maxeax < 1) {
3576                 return (UINT32_MAX);
3577         } else {
3578                 return (cpu->cpu_m.mcpu_cpi->cpi_apicid);
3579         }
3580 }
3581 
3582 void
3583 cpuid_get_addrsize(cpu_t *cpu, uint_t *pabits, uint_t *vabits)
3584 {
3585         struct cpuid_info *cpi;
3586 
3587         if (cpu == NULL)
3588                 cpu = CPU;
3589         cpi = cpu->cpu_m.mcpu_cpi;
3590 
3591         ASSERT(cpuid_checkpass(cpu, 1));
3592 
3593         if (pabits)
3594                 *pabits = cpi->cpi_pabits;
3595         if (vabits)
3596                 *vabits = cpi->cpi_vabits;
3597 }
3598 
3599 size_t
3600 cpuid_get_xsave_size()
3601 {
3602         return (MAX(cpuid_info0.cpi_xsave.xsav_max_size,
3603             sizeof (struct xsave_state)));
3604 }
3605 
3606 /*
3607  * Return true if the CPUs on this system require 'pointer clearing' for the
3608  * floating point error pointer exception handling. In the past, this has been
3609  * true for all AMD K7 & K8 CPUs, although newer AMD CPUs have been changed to
3610  * behave the same as Intel. This is checked via the CPUID_AMD_EBX_ERR_PTR_ZERO
3611  * feature bit and is reflected in the cpi_fp_amd_save member. Once this has
3612  * been confirmed on hardware which supports that feature, this test should be
3613  * narrowed. In the meantime, we always follow the existing behavior on any AMD
3614  * CPU.
3615  */
3616 boolean_t
3617 cpuid_need_fp_excp_handling()
3618 {
3619         return (cpuid_info0.cpi_vendor == X86_VENDOR_AMD);
3620 }
3621 
3622 /*
3623  * Returns the number of data TLB entries for a corresponding
3624  * pagesize.  If it can't be computed, or isn't known, the
3625  * routine returns zero.  If you ask about an architecturally
3626  * impossible pagesize, the routine will panic (so that the
3627  * hat implementor knows that things are inconsistent.)
3628  */
3629 uint_t
3630 cpuid_get_dtlb_nent(cpu_t *cpu, size_t pagesize)
3631 {
3632         struct cpuid_info *cpi;
3633         uint_t dtlb_nent = 0;
3634 
3635         if (cpu == NULL)
3636                 cpu = CPU;
3637         cpi = cpu->cpu_m.mcpu_cpi;
3638 
3639         ASSERT(cpuid_checkpass(cpu, 1));
3640 
3641         /*
3642          * Check the L2 TLB info
3643          */
3644         if (cpi->cpi_xmaxeax >= 0x80000006) {
3645                 struct cpuid_regs *cp = &cpi->cpi_extd[6];
3646 
3647                 switch (pagesize) {
3648 
3649                 case 4 * 1024:
3650                         /*
3651                          * All zero in the top 16 bits of the register
3652                          * indicates a unified TLB. Size is in low 16 bits.
3653                          */
3654                         if ((cp->cp_ebx & 0xffff0000) == 0)
3655                                 dtlb_nent = cp->cp_ebx & 0x0000ffff;
3656                         else
3657                                 dtlb_nent = BITX(cp->cp_ebx, 27, 16);
3658                         break;
3659 
3660                 case 2 * 1024 * 1024:
3661                         if ((cp->cp_eax & 0xffff0000) == 0)
3662                                 dtlb_nent = cp->cp_eax & 0x0000ffff;
3663                         else
3664                                 dtlb_nent = BITX(cp->cp_eax, 27, 16);
3665                         break;
3666 
3667                 default:
3668                         panic("unknown L2 pagesize");
3669                         /*NOTREACHED*/
3670                 }
3671         }
3672 
3673         if (dtlb_nent != 0)
3674                 return (dtlb_nent);
3675 
3676         /*
3677          * No L2 TLB support for this size, try L1.
3678          */
3679         if (cpi->cpi_xmaxeax >= 0x80000005) {
3680                 struct cpuid_regs *cp = &cpi->cpi_extd[5];
3681 
3682                 switch (pagesize) {
3683                 case 4 * 1024:
3684                         dtlb_nent = BITX(cp->cp_ebx, 23, 16);
3685                         break;
3686                 case 2 * 1024 * 1024:
3687                         dtlb_nent = BITX(cp->cp_eax, 23, 16);
3688                         break;
3689                 default:
3690                         panic("unknown L1 d-TLB pagesize");
3691                         /*NOTREACHED*/
3692                 }
3693         }
3694 
3695         return (dtlb_nent);
3696 }
3697 
3698 /*
3699  * Return 0 if the erratum is not present or not applicable, positive
3700  * if it is, and negative if the status of the erratum is unknown.
3701  *
3702  * See "Revision Guide for AMD Athlon(tm) 64 and AMD Opteron(tm)
3703  * Processors" #25759, Rev 3.57, August 2005
3704  */
3705 int
3706 cpuid_opteron_erratum(cpu_t *cpu, uint_t erratum)
3707 {
3708         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3709         uint_t eax;
3710 
3711         /*
3712          * Bail out if this CPU isn't an AMD CPU, or if it's
3713          * a legacy (32-bit) AMD CPU.
3714          */
3715         if (cpi->cpi_vendor != X86_VENDOR_AMD ||
3716             cpi->cpi_family == 4 || cpi->cpi_family == 5 ||
3717             cpi->cpi_family == 6)
3718 
3719                 return (0);
3720 
3721         eax = cpi->cpi_std[1].cp_eax;
3722 
3723 #define SH_B0(eax)      (eax == 0xf40 || eax == 0xf50)
3724 #define SH_B3(eax)      (eax == 0xf51)
3725 #define B(eax)          (SH_B0(eax) || SH_B3(eax))
3726 
3727 #define SH_C0(eax)      (eax == 0xf48 || eax == 0xf58)
3728 
3729 #define SH_CG(eax)      (eax == 0xf4a || eax == 0xf5a || eax == 0xf7a)
3730 #define DH_CG(eax)      (eax == 0xfc0 || eax == 0xfe0 || eax == 0xff0)
3731 #define CH_CG(eax)      (eax == 0xf82 || eax == 0xfb2)
3732 #define CG(eax)         (SH_CG(eax) || DH_CG(eax) || CH_CG(eax))
3733 
3734 #define SH_D0(eax)      (eax == 0x10f40 || eax == 0x10f50 || eax == 0x10f70)
3735 #define DH_D0(eax)      (eax == 0x10fc0 || eax == 0x10ff0)
3736 #define CH_D0(eax)      (eax == 0x10f80 || eax == 0x10fb0)
3737 #define D0(eax)         (SH_D0(eax) || DH_D0(eax) || CH_D0(eax))
3738 
3739 #define SH_E0(eax)      (eax == 0x20f50 || eax == 0x20f40 || eax == 0x20f70)
3740 #define JH_E1(eax)      (eax == 0x20f10)        /* JH8_E0 had 0x20f30 */
3741 #define DH_E3(eax)      (eax == 0x20fc0 || eax == 0x20ff0)
3742 #define SH_E4(eax)      (eax == 0x20f51 || eax == 0x20f71)
3743 #define BH_E4(eax)      (eax == 0x20fb1)
3744 #define SH_E5(eax)      (eax == 0x20f42)
3745 #define DH_E6(eax)      (eax == 0x20ff2 || eax == 0x20fc2)
3746 #define JH_E6(eax)      (eax == 0x20f12 || eax == 0x20f32)
3747 #define EX(eax)         (SH_E0(eax) || JH_E1(eax) || DH_E3(eax) || \
3748                             SH_E4(eax) || BH_E4(eax) || SH_E5(eax) || \
3749                             DH_E6(eax) || JH_E6(eax))
3750 
3751 #define DR_AX(eax)      (eax == 0x100f00 || eax == 0x100f01 || eax == 0x100f02)
3752 #define DR_B0(eax)      (eax == 0x100f20)
3753 #define DR_B1(eax)      (eax == 0x100f21)
3754 #define DR_BA(eax)      (eax == 0x100f2a)
3755 #define DR_B2(eax)      (eax == 0x100f22)
3756 #define DR_B3(eax)      (eax == 0x100f23)
3757 #define RB_C0(eax)      (eax == 0x100f40)
3758 
3759         switch (erratum) {
3760         case 1:
3761                 return (cpi->cpi_family < 0x10);
3762         case 51:        /* what does the asterisk mean? */
3763                 return (B(eax) || SH_C0(eax) || CG(eax));
3764         case 52:
3765                 return (B(eax));
3766         case 57:
3767                 return (cpi->cpi_family <= 0x11);
3768         case 58:
3769                 return (B(eax));
3770         case 60:
3771                 return (cpi->cpi_family <= 0x11);
3772         case 61:
3773         case 62:
3774         case 63:
3775         case 64:
3776         case 65:
3777         case 66:
3778         case 68:
3779         case 69:
3780         case 70:
3781         case 71:
3782                 return (B(eax));
3783         case 72:
3784                 return (SH_B0(eax));
3785         case 74:
3786                 return (B(eax));
3787         case 75:
3788                 return (cpi->cpi_family < 0x10);
3789         case 76:
3790                 return (B(eax));
3791         case 77:
3792                 return (cpi->cpi_family <= 0x11);
3793         case 78:
3794                 return (B(eax) || SH_C0(eax));
3795         case 79:
3796                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3797         case 80:
3798         case 81:
3799         case 82:
3800                 return (B(eax));
3801         case 83:
3802                 return (B(eax) || SH_C0(eax) || CG(eax));
3803         case 85:
3804                 return (cpi->cpi_family < 0x10);
3805         case 86:
3806                 return (SH_C0(eax) || CG(eax));
3807         case 88:
3808 #if !defined(__amd64)
3809                 return (0);
3810 #else
3811                 return (B(eax) || SH_C0(eax));
3812 #endif
3813         case 89:
3814                 return (cpi->cpi_family < 0x10);
3815         case 90:
3816                 return (B(eax) || SH_C0(eax) || CG(eax));
3817         case 91:
3818         case 92:
3819                 return (B(eax) || SH_C0(eax));
3820         case 93:
3821                 return (SH_C0(eax));
3822         case 94:
3823                 return (B(eax) || SH_C0(eax) || CG(eax));
3824         case 95:
3825 #if !defined(__amd64)
3826                 return (0);
3827 #else
3828                 return (B(eax) || SH_C0(eax));
3829 #endif
3830         case 96:
3831                 return (B(eax) || SH_C0(eax) || CG(eax));
3832         case 97:
3833         case 98:
3834                 return (SH_C0(eax) || CG(eax));
3835         case 99:
3836                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3837         case 100:
3838                 return (B(eax) || SH_C0(eax));
3839         case 101:
3840         case 103:
3841                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3842         case 104:
3843                 return (SH_C0(eax) || CG(eax) || D0(eax));
3844         case 105:
3845         case 106:
3846         case 107:
3847                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3848         case 108:
3849                 return (DH_CG(eax));
3850         case 109:
3851                 return (SH_C0(eax) || CG(eax) || D0(eax));
3852         case 110:
3853                 return (D0(eax) || EX(eax));
3854         case 111:
3855                 return (CG(eax));
3856         case 112:
3857                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3858         case 113:
3859                 return (eax == 0x20fc0);
3860         case 114:
3861                 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3862         case 115:
3863                 return (SH_E0(eax) || JH_E1(eax));
3864         case 116:
3865                 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3866         case 117:
3867                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3868         case 118:
3869                 return (SH_E0(eax) || JH_E1(eax) || SH_E4(eax) || BH_E4(eax) ||
3870                     JH_E6(eax));
3871         case 121:
3872                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3873         case 122:
3874                 return (cpi->cpi_family < 0x10 || cpi->cpi_family == 0x11);
3875         case 123:
3876                 return (JH_E1(eax) || BH_E4(eax) || JH_E6(eax));
3877         case 131:
3878                 return (cpi->cpi_family < 0x10);
3879         case 6336786:
3880                 /*
3881                  * Test for AdvPowerMgmtInfo.TscPStateInvariant
3882                  * if this is a K8 family or newer processor
3883                  */
3884                 if (CPI_FAMILY(cpi) == 0xf) {
3885                         struct cpuid_regs regs;
3886                         regs.cp_eax = 0x80000007;
3887                         (void) __cpuid_insn(&regs);
3888                         return (!(regs.cp_edx & 0x100));
3889                 }
3890                 return (0);
3891         case 6323525:
3892                 return (((((eax >> 12) & 0xff00) + (eax & 0xf00)) |
3893                     (((eax >> 4) & 0xf) | ((eax >> 12) & 0xf0))) < 0xf40);
3894 
3895         case 6671130:
3896                 /*
3897                  * check for processors (pre-Shanghai) that do not provide
3898                  * optimal management of 1gb ptes in its tlb.
3899                  */
3900                 return (cpi->cpi_family == 0x10 && cpi->cpi_model < 4);
3901 
3902         case 298:
3903                 return (DR_AX(eax) || DR_B0(eax) || DR_B1(eax) || DR_BA(eax) ||
3904                     DR_B2(eax) || RB_C0(eax));
3905 
3906         case 721:
3907 #if defined(__amd64)
3908                 return (cpi->cpi_family == 0x10 || cpi->cpi_family == 0x12);
3909 #else
3910                 return (0);
3911 #endif
3912 
3913         default:
3914                 return (-1);
3915 
3916         }
3917 }
3918 
3919 /*
3920  * Determine if specified erratum is present via OSVW (OS Visible Workaround).
3921  * Return 1 if erratum is present, 0 if not present and -1 if indeterminate.
3922  */
3923 int
3924 osvw_opteron_erratum(cpu_t *cpu, uint_t erratum)
3925 {
3926         struct cpuid_info       *cpi;
3927         uint_t                  osvwid;
3928         static int              osvwfeature = -1;
3929         uint64_t                osvwlength;
3930 
3931 
3932         cpi = cpu->cpu_m.mcpu_cpi;
3933 
3934         /* confirm OSVW supported */
3935         if (osvwfeature == -1) {
3936                 osvwfeature = cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW;
3937         } else {
3938                 /* assert that osvw feature setting is consistent on all cpus */
3939                 ASSERT(osvwfeature ==
3940                     (cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW));
3941         }
3942         if (!osvwfeature)
3943                 return (-1);
3944 
3945         osvwlength = rdmsr(MSR_AMD_OSVW_ID_LEN) & OSVW_ID_LEN_MASK;
3946 
3947         switch (erratum) {
3948         case 298:       /* osvwid is 0 */
3949                 osvwid = 0;
3950                 if (osvwlength <= (uint64_t)osvwid) {
3951                         /* osvwid 0 is unknown */
3952                         return (-1);
3953                 }
3954 
3955                 /*
3956                  * Check the OSVW STATUS MSR to determine the state
3957                  * of the erratum where:
3958                  *   0 - fixed by HW
3959                  *   1 - BIOS has applied the workaround when BIOS
3960                  *   workaround is available. (Or for other errata,
3961                  *   OS workaround is required.)
3962                  * For a value of 1, caller will confirm that the
3963                  * erratum 298 workaround has indeed been applied by BIOS.
3964                  *
3965                  * A 1 may be set in cpus that have a HW fix
3966                  * in a mixed cpu system. Regarding erratum 298:
3967                  *   In a multiprocessor platform, the workaround above
3968                  *   should be applied to all processors regardless of
3969                  *   silicon revision when an affected processor is
3970                  *   present.
3971                  */
3972 
3973                 return (rdmsr(MSR_AMD_OSVW_STATUS +
3974                     (osvwid / OSVW_ID_CNT_PER_MSR)) &
3975                     (1ULL << (osvwid % OSVW_ID_CNT_PER_MSR)));
3976 
3977         default:
3978                 return (-1);
3979         }
3980 }
3981 
3982 static const char assoc_str[] = "associativity";
3983 static const char line_str[] = "line-size";
3984 static const char size_str[] = "size";
3985 
3986 static void
3987 add_cache_prop(dev_info_t *devi, const char *label, const char *type,
3988     uint32_t val)
3989 {
3990         char buf[128];
3991 
3992         /*
3993          * ndi_prop_update_int() is used because it is desirable for
3994          * DDI_PROP_HW_DEF and DDI_PROP_DONTSLEEP to be set.
3995          */
3996         if (snprintf(buf, sizeof (buf), "%s-%s", label, type) < sizeof (buf))
3997                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, devi, buf, val);
3998 }
3999 
4000 /*
4001  * Intel-style cache/tlb description
4002  *
4003  * Standard cpuid level 2 gives a randomly ordered
4004  * selection of tags that index into a table that describes
4005  * cache and tlb properties.
4006  */
4007 
4008 static const char l1_icache_str[] = "l1-icache";
4009 static const char l1_dcache_str[] = "l1-dcache";
4010 static const char l2_cache_str[] = "l2-cache";
4011 static const char l3_cache_str[] = "l3-cache";
4012 static const char itlb4k_str[] = "itlb-4K";
4013 static const char dtlb4k_str[] = "dtlb-4K";
4014 static const char itlb2M_str[] = "itlb-2M";
4015 static const char itlb4M_str[] = "itlb-4M";
4016 static const char dtlb4M_str[] = "dtlb-4M";
4017 static const char dtlb24_str[] = "dtlb0-2M-4M";
4018 static const char itlb424_str[] = "itlb-4K-2M-4M";
4019 static const char itlb24_str[] = "itlb-2M-4M";
4020 static const char dtlb44_str[] = "dtlb-4K-4M";
4021 static const char sl1_dcache_str[] = "sectored-l1-dcache";
4022 static const char sl2_cache_str[] = "sectored-l2-cache";
4023 static const char itrace_str[] = "itrace-cache";
4024 static const char sl3_cache_str[] = "sectored-l3-cache";
4025 static const char sh_l2_tlb4k_str[] = "shared-l2-tlb-4k";
4026 
4027 static const struct cachetab {
4028         uint8_t         ct_code;
4029         uint8_t         ct_assoc;
4030         uint16_t        ct_line_size;
4031         size_t          ct_size;
4032         const char      *ct_label;
4033 } intel_ctab[] = {
4034         /*
4035          * maintain descending order!
4036          *
4037          * Codes ignored - Reason
4038          * ----------------------
4039          * 40H - intel_cpuid_4_cache_info() disambiguates l2/l3 cache
4040          * f0H/f1H - Currently we do not interpret prefetch size by design
4041          */
4042         { 0xe4, 16, 64, 8*1024*1024, l3_cache_str},
4043         { 0xe3, 16, 64, 4*1024*1024, l3_cache_str},
4044         { 0xe2, 16, 64, 2*1024*1024, l3_cache_str},
4045         { 0xde, 12, 64, 6*1024*1024, l3_cache_str},
4046         { 0xdd, 12, 64, 3*1024*1024, l3_cache_str},
4047         { 0xdc, 12, 64, ((1*1024*1024)+(512*1024)), l3_cache_str},
4048         { 0xd8, 8, 64, 4*1024*1024, l3_cache_str},
4049         { 0xd7, 8, 64, 2*1024*1024, l3_cache_str},
4050         { 0xd6, 8, 64, 1*1024*1024, l3_cache_str},
4051         { 0xd2, 4, 64, 2*1024*1024, l3_cache_str},
4052         { 0xd1, 4, 64, 1*1024*1024, l3_cache_str},
4053         { 0xd0, 4, 64, 512*1024, l3_cache_str},
4054         { 0xca, 4, 0, 512, sh_l2_tlb4k_str},
4055         { 0xc0, 4, 0, 8, dtlb44_str },
4056         { 0xba, 4, 0, 64, dtlb4k_str },
4057         { 0xb4, 4, 0, 256, dtlb4k_str },
4058         { 0xb3, 4, 0, 128, dtlb4k_str },
4059         { 0xb2, 4, 0, 64, itlb4k_str },
4060         { 0xb0, 4, 0, 128, itlb4k_str },
4061         { 0x87, 8, 64, 1024*1024, l2_cache_str},
4062         { 0x86, 4, 64, 512*1024, l2_cache_str},
4063         { 0x85, 8, 32, 2*1024*1024, l2_cache_str},
4064         { 0x84, 8, 32, 1024*1024, l2_cache_str},
4065         { 0x83, 8, 32, 512*1024, l2_cache_str},
4066         { 0x82, 8, 32, 256*1024, l2_cache_str},
4067         { 0x80, 8, 64, 512*1024, l2_cache_str},
4068         { 0x7f, 2, 64, 512*1024, l2_cache_str},
4069         { 0x7d, 8, 64, 2*1024*1024, sl2_cache_str},
4070         { 0x7c, 8, 64, 1024*1024, sl2_cache_str},
4071         { 0x7b, 8, 64, 512*1024, sl2_cache_str},
4072         { 0x7a, 8, 64, 256*1024, sl2_cache_str},
4073         { 0x79, 8, 64, 128*1024, sl2_cache_str},
4074         { 0x78, 8, 64, 1024*1024, l2_cache_str},
4075         { 0x73, 8, 0, 64*1024, itrace_str},
4076         { 0x72, 8, 0, 32*1024, itrace_str},
4077         { 0x71, 8, 0, 16*1024, itrace_str},
4078         { 0x70, 8, 0, 12*1024, itrace_str},
4079         { 0x68, 4, 64, 32*1024, sl1_dcache_str},
4080         { 0x67, 4, 64, 16*1024, sl1_dcache_str},
4081         { 0x66, 4, 64, 8*1024, sl1_dcache_str},
4082         { 0x60, 8, 64, 16*1024, sl1_dcache_str},
4083         { 0x5d, 0, 0, 256, dtlb44_str},
4084         { 0x5c, 0, 0, 128, dtlb44_str},
4085         { 0x5b, 0, 0, 64, dtlb44_str},
4086         { 0x5a, 4, 0, 32, dtlb24_str},
4087         { 0x59, 0, 0, 16, dtlb4k_str},
4088         { 0x57, 4, 0, 16, dtlb4k_str},
4089         { 0x56, 4, 0, 16, dtlb4M_str},
4090         { 0x55, 0, 0, 7, itlb24_str},
4091         { 0x52, 0, 0, 256, itlb424_str},
4092         { 0x51, 0, 0, 128, itlb424_str},
4093         { 0x50, 0, 0, 64, itlb424_str},
4094         { 0x4f, 0, 0, 32, itlb4k_str},
4095         { 0x4e, 24, 64, 6*1024*1024, l2_cache_str},
4096         { 0x4d, 16, 64, 16*1024*1024, l3_cache_str},
4097         { 0x4c, 12, 64, 12*1024*1024, l3_cache_str},
4098         { 0x4b, 16, 64, 8*1024*1024, l3_cache_str},
4099         { 0x4a, 12, 64, 6*1024*1024, l3_cache_str},
4100         { 0x49, 16, 64, 4*1024*1024, l3_cache_str},
4101         { 0x48, 12, 64, 3*1024*1024, l2_cache_str},
4102         { 0x47, 8, 64, 8*1024*1024, l3_cache_str},
4103         { 0x46, 4, 64, 4*1024*1024, l3_cache_str},
4104         { 0x45, 4, 32, 2*1024*1024, l2_cache_str},
4105         { 0x44, 4, 32, 1024*1024, l2_cache_str},
4106         { 0x43, 4, 32, 512*1024, l2_cache_str},
4107         { 0x42, 4, 32, 256*1024, l2_cache_str},
4108         { 0x41, 4, 32, 128*1024, l2_cache_str},
4109         { 0x3e, 4, 64, 512*1024, sl2_cache_str},
4110         { 0x3d, 6, 64, 384*1024, sl2_cache_str},
4111         { 0x3c, 4, 64, 256*1024, sl2_cache_str},
4112         { 0x3b, 2, 64, 128*1024, sl2_cache_str},
4113         { 0x3a, 6, 64, 192*1024, sl2_cache_str},
4114         { 0x39, 4, 64, 128*1024, sl2_cache_str},
4115         { 0x30, 8, 64, 32*1024, l1_icache_str},
4116         { 0x2c, 8, 64, 32*1024, l1_dcache_str},
4117         { 0x29, 8, 64, 4096*1024, sl3_cache_str},
4118         { 0x25, 8, 64, 2048*1024, sl3_cache_str},
4119         { 0x23, 8, 64, 1024*1024, sl3_cache_str},
4120         { 0x22, 4, 64, 512*1024, sl3_cache_str},
4121         { 0x0e, 6, 64, 24*1024, l1_dcache_str},
4122         { 0x0d, 4, 32, 16*1024, l1_dcache_str},
4123         { 0x0c, 4, 32, 16*1024, l1_dcache_str},
4124         { 0x0b, 4, 0, 4, itlb4M_str},
4125         { 0x0a, 2, 32, 8*1024, l1_dcache_str},
4126         { 0x08, 4, 32, 16*1024, l1_icache_str},
4127         { 0x06, 4, 32, 8*1024, l1_icache_str},
4128         { 0x05, 4, 0, 32, dtlb4M_str},
4129         { 0x04, 4, 0, 8, dtlb4M_str},
4130         { 0x03, 4, 0, 64, dtlb4k_str},
4131         { 0x02, 4, 0, 2, itlb4M_str},
4132         { 0x01, 4, 0, 32, itlb4k_str},
4133         { 0 }
4134 };
4135 
4136 static const struct cachetab cyrix_ctab[] = {
4137         { 0x70, 4, 0, 32, "tlb-4K" },
4138         { 0x80, 4, 16, 16*1024, "l1-cache" },
4139         { 0 }
4140 };
4141 
4142 /*
4143  * Search a cache table for a matching entry
4144  */
4145 static const struct cachetab *
4146 find_cacheent(const struct cachetab *ct, uint_t code)
4147 {
4148         if (code != 0) {
4149                 for (; ct->ct_code != 0; ct++)
4150                         if (ct->ct_code <= code)
4151                                 break;
4152                 if (ct->ct_code == code)
4153                         return (ct);
4154         }
4155         return (NULL);
4156 }
4157 
4158 /*
4159  * Populate cachetab entry with L2 or L3 cache-information using
4160  * cpuid function 4. This function is called from intel_walk_cacheinfo()
4161  * when descriptor 0x49 is encountered. It returns 0 if no such cache
4162  * information is found.
4163  */
4164 static int
4165 intel_cpuid_4_cache_info(struct cachetab *ct, struct cpuid_info *cpi)
4166 {
4167         uint32_t level, i;
4168         int ret = 0;
4169 
4170         for (i = 0; i < cpi->cpi_std_4_size; i++) {
4171                 level = CPI_CACHE_LVL(cpi->cpi_std_4[i]);
4172 
4173                 if (level == 2 || level == 3) {
4174                         ct->ct_assoc = CPI_CACHE_WAYS(cpi->cpi_std_4[i]) + 1;
4175                         ct->ct_line_size =
4176                             CPI_CACHE_COH_LN_SZ(cpi->cpi_std_4[i]) + 1;
4177                         ct->ct_size = ct->ct_assoc *
4178                             (CPI_CACHE_PARTS(cpi->cpi_std_4[i]) + 1) *
4179                             ct->ct_line_size *
4180                             (cpi->cpi_std_4[i]->cp_ecx + 1);
4181 
4182                         if (level == 2) {
4183                                 ct->ct_label = l2_cache_str;
4184                         } else if (level == 3) {
4185                                 ct->ct_label = l3_cache_str;
4186                         }
4187                         ret = 1;
4188                 }
4189         }
4190 
4191         return (ret);
4192 }
4193 
4194 /*
4195  * Walk the cacheinfo descriptor, applying 'func' to every valid element
4196  * The walk is terminated if the walker returns non-zero.
4197  */
4198 static void
4199 intel_walk_cacheinfo(struct cpuid_info *cpi,
4200     void *arg, int (*func)(void *, const struct cachetab *))
4201 {
4202         const struct cachetab *ct;
4203         struct cachetab des_49_ct, des_b1_ct;
4204         uint8_t *dp;
4205         int i;
4206 
4207         if ((dp = cpi->cpi_cacheinfo) == NULL)
4208                 return;
4209         for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
4210                 /*
4211                  * For overloaded descriptor 0x49 we use cpuid function 4
4212                  * if supported by the current processor, to create
4213                  * cache information.
4214                  * For overloaded descriptor 0xb1 we use X86_PAE flag
4215                  * to disambiguate the cache information.
4216                  */
4217                 if (*dp == 0x49 && cpi->cpi_maxeax >= 0x4 &&
4218                     intel_cpuid_4_cache_info(&des_49_ct, cpi) == 1) {
4219                                 ct = &des_49_ct;
4220                 } else if (*dp == 0xb1) {
4221                         des_b1_ct.ct_code = 0xb1;
4222                         des_b1_ct.ct_assoc = 4;
4223                         des_b1_ct.ct_line_size = 0;
4224                         if (is_x86_feature(x86_featureset, X86FSET_PAE)) {
4225                                 des_b1_ct.ct_size = 8;
4226                                 des_b1_ct.ct_label = itlb2M_str;
4227                         } else {
4228                                 des_b1_ct.ct_size = 4;
4229                                 des_b1_ct.ct_label = itlb4M_str;
4230                         }
4231                         ct = &des_b1_ct;
4232                 } else {
4233                         if ((ct = find_cacheent(intel_ctab, *dp)) == NULL) {
4234                                 continue;
4235                         }
4236                 }
4237 
4238                 if (func(arg, ct) != 0) {
4239                         break;
4240                 }
4241         }
4242 }
4243 
4244 /*
4245  * (Like the Intel one, except for Cyrix CPUs)
4246  */
4247 static void
4248 cyrix_walk_cacheinfo(struct cpuid_info *cpi,
4249     void *arg, int (*func)(void *, const struct cachetab *))
4250 {
4251         const struct cachetab *ct;
4252         uint8_t *dp;
4253         int i;
4254 
4255         if ((dp = cpi->cpi_cacheinfo) == NULL)
4256                 return;
4257         for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
4258                 /*
4259                  * Search Cyrix-specific descriptor table first ..
4260                  */
4261                 if ((ct = find_cacheent(cyrix_ctab, *dp)) != NULL) {
4262                         if (func(arg, ct) != 0)
4263                                 break;
4264                         continue;
4265                 }
4266                 /*
4267                  * .. else fall back to the Intel one
4268                  */
4269                 if ((ct = find_cacheent(intel_ctab, *dp)) != NULL) {
4270                         if (func(arg, ct) != 0)
4271                                 break;
4272                         continue;
4273                 }
4274         }
4275 }
4276 
4277 /*
4278  * A cacheinfo walker that adds associativity, line-size, and size properties
4279  * to the devinfo node it is passed as an argument.
4280  */
4281 static int
4282 add_cacheent_props(void *arg, const struct cachetab *ct)
4283 {
4284         dev_info_t *devi = arg;
4285 
4286         add_cache_prop(devi, ct->ct_label, assoc_str, ct->ct_assoc);
4287         if (ct->ct_line_size != 0)
4288                 add_cache_prop(devi, ct->ct_label, line_str,
4289                     ct->ct_line_size);
4290         add_cache_prop(devi, ct->ct_label, size_str, ct->ct_size);
4291         return (0);
4292 }
4293 
4294 
4295 static const char fully_assoc[] = "fully-associative?";
4296 
4297 /*
4298  * AMD style cache/tlb description
4299  *
4300  * Extended functions 5 and 6 directly describe properties of
4301  * tlbs and various cache levels.
4302  */
4303 static void
4304 add_amd_assoc(dev_info_t *devi, const char *label, uint_t assoc)
4305 {
4306         switch (assoc) {
4307         case 0: /* reserved; ignore */
4308                 break;
4309         default:
4310                 add_cache_prop(devi, label, assoc_str, assoc);
4311                 break;
4312         case 0xff:
4313                 add_cache_prop(devi, label, fully_assoc, 1);
4314                 break;
4315         }
4316 }
4317 
4318 static void
4319 add_amd_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
4320 {
4321         if (size == 0)
4322                 return;
4323         add_cache_prop(devi, label, size_str, size);
4324         add_amd_assoc(devi, label, assoc);
4325 }
4326 
4327 static void
4328 add_amd_cache(dev_info_t *devi, const char *label,
4329     uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
4330 {
4331         if (size == 0 || line_size == 0)
4332                 return;
4333         add_amd_assoc(devi, label, assoc);
4334         /*
4335          * Most AMD parts have a sectored cache. Multiple cache lines are
4336          * associated with each tag. A sector consists of all cache lines
4337          * associated with a tag. For example, the AMD K6-III has a sector
4338          * size of 2 cache lines per tag.
4339          */
4340         if (lines_per_tag != 0)
4341                 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
4342         add_cache_prop(devi, label, line_str, line_size);
4343         add_cache_prop(devi, label, size_str, size * 1024);
4344 }
4345 
4346 static void
4347 add_amd_l2_assoc(dev_info_t *devi, const char *label, uint_t assoc)
4348 {
4349         switch (assoc) {
4350         case 0: /* off */
4351                 break;
4352         case 1:
4353         case 2:
4354         case 4:
4355                 add_cache_prop(devi, label, assoc_str, assoc);
4356                 break;
4357         case 6:
4358                 add_cache_prop(devi, label, assoc_str, 8);
4359                 break;
4360         case 8:
4361                 add_cache_prop(devi, label, assoc_str, 16);
4362                 break;
4363         case 0xf:
4364                 add_cache_prop(devi, label, fully_assoc, 1);
4365                 break;
4366         default: /* reserved; ignore */
4367                 break;
4368         }
4369 }
4370 
4371 static void
4372 add_amd_l2_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
4373 {
4374         if (size == 0 || assoc == 0)
4375                 return;
4376         add_amd_l2_assoc(devi, label, assoc);
4377         add_cache_prop(devi, label, size_str, size);
4378 }
4379 
4380 static void
4381 add_amd_l2_cache(dev_info_t *devi, const char *label,
4382     uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
4383 {
4384         if (size == 0 || assoc == 0 || line_size == 0)
4385                 return;
4386         add_amd_l2_assoc(devi, label, assoc);
4387         if (lines_per_tag != 0)
4388                 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
4389         add_cache_prop(devi, label, line_str, line_size);
4390         add_cache_prop(devi, label, size_str, size * 1024);
4391 }
4392 
4393 static void
4394 amd_cache_info(struct cpuid_info *cpi, dev_info_t *devi)
4395 {
4396         struct cpuid_regs *cp;
4397 
4398         if (cpi->cpi_xmaxeax < 0x80000005)
4399                 return;
4400         cp = &cpi->cpi_extd[5];
4401 
4402         /*
4403          * 4M/2M L1 TLB configuration
4404          *
4405          * We report the size for 2M pages because AMD uses two
4406          * TLB entries for one 4M page.
4407          */
4408         add_amd_tlb(devi, "dtlb-2M",
4409             BITX(cp->cp_eax, 31, 24), BITX(cp->cp_eax, 23, 16));
4410         add_amd_tlb(devi, "itlb-2M",
4411             BITX(cp->cp_eax, 15, 8), BITX(cp->cp_eax, 7, 0));
4412 
4413         /*
4414          * 4K L1 TLB configuration
4415          */
4416 
4417         switch (cpi->cpi_vendor) {
4418                 uint_t nentries;
4419         case X86_VENDOR_TM:
4420                 if (cpi->cpi_family >= 5) {
4421                         /*
4422                          * Crusoe processors have 256 TLB entries, but
4423                          * cpuid data format constrains them to only
4424                          * reporting 255 of them.
4425                          */
4426                         if ((nentries = BITX(cp->cp_ebx, 23, 16)) == 255)
4427                                 nentries = 256;
4428                         /*
4429                          * Crusoe processors also have a unified TLB
4430                          */
4431                         add_amd_tlb(devi, "tlb-4K", BITX(cp->cp_ebx, 31, 24),
4432                             nentries);
4433                         break;
4434                 }
4435                 /*FALLTHROUGH*/
4436         default:
4437                 add_amd_tlb(devi, itlb4k_str,
4438                     BITX(cp->cp_ebx, 31, 24), BITX(cp->cp_ebx, 23, 16));
4439                 add_amd_tlb(devi, dtlb4k_str,
4440                     BITX(cp->cp_ebx, 15, 8), BITX(cp->cp_ebx, 7, 0));
4441                 break;
4442         }
4443 
4444         /*
4445          * data L1 cache configuration
4446          */
4447 
4448         add_amd_cache(devi, l1_dcache_str,
4449             BITX(cp->cp_ecx, 31, 24), BITX(cp->cp_ecx, 23, 16),
4450             BITX(cp->cp_ecx, 15, 8), BITX(cp->cp_ecx, 7, 0));
4451 
4452         /*
4453          * code L1 cache configuration
4454          */
4455 
4456         add_amd_cache(devi, l1_icache_str,
4457             BITX(cp->cp_edx, 31, 24), BITX(cp->cp_edx, 23, 16),
4458             BITX(cp->cp_edx, 15, 8), BITX(cp->cp_edx, 7, 0));
4459 
4460         if (cpi->cpi_xmaxeax < 0x80000006)
4461                 return;
4462         cp = &cpi->cpi_extd[6];
4463 
4464         /* Check for a unified L2 TLB for large pages */
4465 
4466         if (BITX(cp->cp_eax, 31, 16) == 0)
4467                 add_amd_l2_tlb(devi, "l2-tlb-2M",
4468                     BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4469         else {
4470                 add_amd_l2_tlb(devi, "l2-dtlb-2M",
4471                     BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
4472                 add_amd_l2_tlb(devi, "l2-itlb-2M",
4473                     BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4474         }
4475 
4476         /* Check for a unified L2 TLB for 4K pages */
4477 
4478         if (BITX(cp->cp_ebx, 31, 16) == 0) {
4479                 add_amd_l2_tlb(devi, "l2-tlb-4K",
4480                     BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4481         } else {
4482                 add_amd_l2_tlb(devi, "l2-dtlb-4K",
4483                     BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
4484                 add_amd_l2_tlb(devi, "l2-itlb-4K",
4485                     BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4486         }
4487 
4488         add_amd_l2_cache(devi, l2_cache_str,
4489             BITX(cp->cp_ecx, 31, 16), BITX(cp->cp_ecx, 15, 12),
4490             BITX(cp->cp_ecx, 11, 8), BITX(cp->cp_ecx, 7, 0));
4491 }
4492 
4493 /*
4494  * There are two basic ways that the x86 world describes it cache
4495  * and tlb architecture - Intel's way and AMD's way.
4496  *
4497  * Return which flavor of cache architecture we should use
4498  */
4499 static int
4500 x86_which_cacheinfo(struct cpuid_info *cpi)
4501 {
4502         switch (cpi->cpi_vendor) {
4503         case X86_VENDOR_Intel:
4504                 if (cpi->cpi_maxeax >= 2)
4505                         return (X86_VENDOR_Intel);
4506                 break;
4507         case X86_VENDOR_AMD:
4508                 /*
4509                  * The K5 model 1 was the first part from AMD that reported
4510                  * cache sizes via extended cpuid functions.
4511                  */
4512                 if (cpi->cpi_family > 5 ||
4513                     (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
4514                         return (X86_VENDOR_AMD);
4515                 break;
4516         case X86_VENDOR_TM:
4517                 if (cpi->cpi_family >= 5)
4518                         return (X86_VENDOR_AMD);
4519                 /*FALLTHROUGH*/
4520         default:
4521                 /*
4522                  * If they have extended CPU data for 0x80000005
4523                  * then we assume they have AMD-format cache
4524                  * information.
4525                  *
4526                  * If not, and the vendor happens to be Cyrix,
4527                  * then try our-Cyrix specific handler.
4528                  *
4529                  * If we're not Cyrix, then assume we're using Intel's
4530                  * table-driven format instead.
4531                  */
4532                 if (cpi->cpi_xmaxeax >= 0x80000005)
4533                         return (X86_VENDOR_AMD);
4534                 else if (cpi->cpi_vendor == X86_VENDOR_Cyrix)
4535                         return (X86_VENDOR_Cyrix);
4536                 else if (cpi->cpi_maxeax >= 2)
4537                         return (X86_VENDOR_Intel);
4538                 break;
4539         }
4540         return (-1);
4541 }
4542 
4543 void
4544 cpuid_set_cpu_properties(void *dip, processorid_t cpu_id,
4545     struct cpuid_info *cpi)
4546 {
4547         dev_info_t *cpu_devi;
4548         int create;
4549 
4550         cpu_devi = (dev_info_t *)dip;
4551 
4552         /* device_type */
4553         (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4554             "device_type", "cpu");
4555 
4556         /* reg */
4557         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4558             "reg", cpu_id);
4559 
4560         /* cpu-mhz, and clock-frequency */
4561         if (cpu_freq > 0) {
4562                 long long mul;
4563 
4564                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4565                     "cpu-mhz", cpu_freq);
4566                 if ((mul = cpu_freq * 1000000LL) <= INT_MAX)
4567                         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4568                             "clock-frequency", (int)mul);
4569         }
4570 
4571         if (!is_x86_feature(x86_featureset, X86FSET_CPUID)) {
4572                 return;
4573         }
4574 
4575         /* vendor-id */
4576         (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4577             "vendor-id", cpi->cpi_vendorstr);
4578 
4579         if (cpi->cpi_maxeax == 0) {
4580                 return;
4581         }
4582 
4583         /*
4584          * family, model, and step
4585          */
4586         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4587             "family", CPI_FAMILY(cpi));
4588         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4589             "cpu-model", CPI_MODEL(cpi));
4590         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4591             "stepping-id", CPI_STEP(cpi));
4592 
4593         /* type */
4594         switch (cpi->cpi_vendor) {
4595         case X86_VENDOR_Intel:
4596                 create = 1;
4597                 break;
4598         default:
4599                 create = 0;
4600                 break;
4601         }
4602         if (create)
4603                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4604                     "type", CPI_TYPE(cpi));
4605 
4606         /* ext-family */
4607         switch (cpi->cpi_vendor) {
4608         case X86_VENDOR_Intel:
4609         case X86_VENDOR_AMD:
4610                 create = cpi->cpi_family >= 0xf;
4611                 break;
4612         default:
4613                 create = 0;
4614                 break;
4615         }
4616         if (create)
4617                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4618                     "ext-family", CPI_FAMILY_XTD(cpi));
4619 
4620         /* ext-model */
4621         switch (cpi->cpi_vendor) {
4622         case X86_VENDOR_Intel:
4623                 create = IS_EXTENDED_MODEL_INTEL(cpi);
4624                 break;
4625         case X86_VENDOR_AMD:
4626                 create = CPI_FAMILY(cpi) == 0xf;
4627                 break;
4628         default:
4629                 create = 0;
4630                 break;
4631         }
4632         if (create)
4633                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4634                     "ext-model", CPI_MODEL_XTD(cpi));
4635 
4636         /* generation */
4637         switch (cpi->cpi_vendor) {
4638         case X86_VENDOR_AMD:
4639                 /*
4640                  * AMD K5 model 1 was the first part to support this
4641                  */
4642                 create = cpi->cpi_xmaxeax >= 0x80000001;
4643                 break;
4644         default:
4645                 create = 0;
4646                 break;
4647         }
4648         if (create)
4649                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4650                     "generation", BITX((cpi)->cpi_extd[1].cp_eax, 11, 8));
4651 
4652         /* brand-id */
4653         switch (cpi->cpi_vendor) {
4654         case X86_VENDOR_Intel:
4655                 /*
4656                  * brand id first appeared on Pentium III Xeon model 8,
4657                  * and Celeron model 8 processors and Opteron
4658                  */
4659                 create = cpi->cpi_family > 6 ||
4660                     (cpi->cpi_family == 6 && cpi->cpi_model >= 8);
4661                 break;
4662         case X86_VENDOR_AMD:
4663                 create = cpi->cpi_family >= 0xf;
4664                 break;
4665         default:
4666                 create = 0;
4667                 break;
4668         }
4669         if (create && cpi->cpi_brandid != 0) {
4670                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4671                     "brand-id", cpi->cpi_brandid);
4672         }
4673 
4674         /* chunks, and apic-id */
4675         switch (cpi->cpi_vendor) {
4676                 /*
4677                  * first available on Pentium IV and Opteron (K8)
4678                  */
4679         case X86_VENDOR_Intel:
4680                 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4681                 break;
4682         case X86_VENDOR_AMD:
4683                 create = cpi->cpi_family >= 0xf;
4684                 break;
4685         default:
4686                 create = 0;
4687                 break;
4688         }
4689         if (create) {
4690                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4691                     "chunks", CPI_CHUNKS(cpi));
4692                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4693                     "apic-id", cpi->cpi_apicid);
4694                 if (cpi->cpi_chipid >= 0) {
4695                         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4696                             "chip#", cpi->cpi_chipid);
4697                         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4698                             "clog#", cpi->cpi_clogid);
4699                 }
4700         }
4701 
4702         /* cpuid-features */
4703         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4704             "cpuid-features", CPI_FEATURES_EDX(cpi));
4705 
4706 
4707         /* cpuid-features-ecx */
4708         switch (cpi->cpi_vendor) {
4709         case X86_VENDOR_Intel:
4710                 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4711                 break;
4712         case X86_VENDOR_AMD:
4713                 create = cpi->cpi_family >= 0xf;
4714                 break;
4715         default:
4716                 create = 0;
4717                 break;
4718         }
4719         if (create)
4720                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4721                     "cpuid-features-ecx", CPI_FEATURES_ECX(cpi));
4722 
4723         /* ext-cpuid-features */
4724         switch (cpi->cpi_vendor) {
4725         case X86_VENDOR_Intel:
4726         case X86_VENDOR_AMD:
4727         case X86_VENDOR_Cyrix:
4728         case X86_VENDOR_TM:
4729         case X86_VENDOR_Centaur:
4730                 create = cpi->cpi_xmaxeax >= 0x80000001;
4731                 break;
4732         default:
4733                 create = 0;
4734                 break;
4735         }
4736         if (create) {
4737                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4738                     "ext-cpuid-features", CPI_FEATURES_XTD_EDX(cpi));
4739                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4740                     "ext-cpuid-features-ecx", CPI_FEATURES_XTD_ECX(cpi));
4741         }
4742 
4743         /*
4744          * Brand String first appeared in Intel Pentium IV, AMD K5
4745          * model 1, and Cyrix GXm.  On earlier models we try and
4746          * simulate something similar .. so this string should always
4747          * same -something- about the processor, however lame.
4748          */
4749         (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4750             "brand-string", cpi->cpi_brandstr);
4751 
4752         /*
4753          * Finally, cache and tlb information
4754          */
4755         switch (x86_which_cacheinfo(cpi)) {
4756         case X86_VENDOR_Intel:
4757                 intel_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4758                 break;
4759         case X86_VENDOR_Cyrix:
4760                 cyrix_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4761                 break;
4762         case X86_VENDOR_AMD:
4763                 amd_cache_info(cpi, cpu_devi);
4764                 break;
4765         default:
4766                 break;
4767         }
4768 }
4769 
4770 struct l2info {
4771         int *l2i_csz;
4772         int *l2i_lsz;
4773         int *l2i_assoc;
4774         int l2i_ret;
4775 };
4776 
4777 /*
4778  * A cacheinfo walker that fetches the size, line-size and associativity
4779  * of the L2 cache
4780  */
4781 static int
4782 intel_l2cinfo(void *arg, const struct cachetab *ct)
4783 {
4784         struct l2info *l2i = arg;
4785         int *ip;
4786 
4787         if (ct->ct_label != l2_cache_str &&
4788             ct->ct_label != sl2_cache_str)
4789                 return (0);     /* not an L2 -- keep walking */
4790 
4791         if ((ip = l2i->l2i_csz) != NULL)
4792                 *ip = ct->ct_size;
4793         if ((ip = l2i->l2i_lsz) != NULL)
4794                 *ip = ct->ct_line_size;
4795         if ((ip = l2i->l2i_assoc) != NULL)
4796                 *ip = ct->ct_assoc;
4797         l2i->l2i_ret = ct->ct_size;
4798         return (1);             /* was an L2 -- terminate walk */
4799 }
4800 
4801 /*
4802  * AMD L2/L3 Cache and TLB Associativity Field Definition:
4803  *
4804  *      Unlike the associativity for the L1 cache and tlb where the 8 bit
4805  *      value is the associativity, the associativity for the L2 cache and
4806  *      tlb is encoded in the following table. The 4 bit L2 value serves as
4807  *      an index into the amd_afd[] array to determine the associativity.
4808  *      -1 is undefined. 0 is fully associative.
4809  */
4810 
4811 static int amd_afd[] =
4812         {-1, 1, 2, -1, 4, -1, 8, -1, 16, -1, 32, 48, 64, 96, 128, 0};
4813 
4814 static void
4815 amd_l2cacheinfo(struct cpuid_info *cpi, struct l2info *l2i)
4816 {
4817         struct cpuid_regs *cp;
4818         uint_t size, assoc;
4819         int i;
4820         int *ip;
4821 
4822         if (cpi->cpi_xmaxeax < 0x80000006)
4823                 return;
4824         cp = &cpi->cpi_extd[6];
4825 
4826         if ((i = BITX(cp->cp_ecx, 15, 12)) != 0 &&
4827             (size = BITX(cp->cp_ecx, 31, 16)) != 0) {
4828                 uint_t cachesz = size * 1024;
4829                 assoc = amd_afd[i];
4830 
4831                 ASSERT(assoc != -1);
4832 
4833                 if ((ip = l2i->l2i_csz) != NULL)
4834                         *ip = cachesz;
4835                 if ((ip = l2i->l2i_lsz) != NULL)
4836                         *ip = BITX(cp->cp_ecx, 7, 0);
4837                 if ((ip = l2i->l2i_assoc) != NULL)
4838                         *ip = assoc;
4839                 l2i->l2i_ret = cachesz;
4840         }
4841 }
4842 
4843 int
4844 getl2cacheinfo(cpu_t *cpu, int *csz, int *lsz, int *assoc)
4845 {
4846         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
4847         struct l2info __l2info, *l2i = &__l2info;
4848 
4849         l2i->l2i_csz = csz;
4850         l2i->l2i_lsz = lsz;
4851         l2i->l2i_assoc = assoc;
4852         l2i->l2i_ret = -1;
4853 
4854         switch (x86_which_cacheinfo(cpi)) {
4855         case X86_VENDOR_Intel:
4856                 intel_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4857                 break;
4858         case X86_VENDOR_Cyrix:
4859                 cyrix_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4860                 break;
4861         case X86_VENDOR_AMD:
4862                 amd_l2cacheinfo(cpi, l2i);
4863                 break;
4864         default:
4865                 break;
4866         }
4867         return (l2i->l2i_ret);
4868 }
4869 
4870 #if !defined(__xpv)
4871 
4872 uint32_t *
4873 cpuid_mwait_alloc(cpu_t *cpu)
4874 {
4875         uint32_t        *ret;
4876         size_t          mwait_size;
4877 
4878         ASSERT(cpuid_checkpass(CPU, 2));
4879 
4880         mwait_size = CPU->cpu_m.mcpu_cpi->cpi_mwait.mon_max;
4881         if (mwait_size == 0)
4882                 return (NULL);
4883 
4884         /*
4885          * kmem_alloc() returns cache line size aligned data for mwait_size
4886          * allocations.  mwait_size is currently cache line sized.  Neither
4887          * of these implementation details are guarantied to be true in the
4888          * future.
4889          *
4890          * First try allocating mwait_size as kmem_alloc() currently returns
4891          * correctly aligned memory.  If kmem_alloc() does not return
4892          * mwait_size aligned memory, then use mwait_size ROUNDUP.
4893          *
4894          * Set cpi_mwait.buf_actual and cpi_mwait.size_actual in case we
4895          * decide to free this memory.
4896          */
4897         ret = kmem_zalloc(mwait_size, KM_SLEEP);
4898         if (ret == (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size)) {
4899                 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
4900                 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size;
4901                 *ret = MWAIT_RUNNING;
4902                 return (ret);
4903         } else {
4904                 kmem_free(ret, mwait_size);
4905                 ret = kmem_zalloc(mwait_size * 2, KM_SLEEP);
4906                 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
4907                 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size * 2;
4908                 ret = (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size);
4909                 *ret = MWAIT_RUNNING;
4910                 return (ret);
4911         }
4912 }
4913 
4914 void
4915 cpuid_mwait_free(cpu_t *cpu)
4916 {
4917         if (cpu->cpu_m.mcpu_cpi == NULL) {
4918                 return;
4919         }
4920 
4921         if (cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual != NULL &&
4922             cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual > 0) {
4923                 kmem_free(cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual,
4924                     cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual);
4925         }
4926 
4927         cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = NULL;
4928         cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = 0;
4929 }
4930 
4931 void
4932 patch_tsc_read(int flag)
4933 {
4934         size_t cnt;
4935 
4936         switch (flag) {
4937         case TSC_NONE:
4938                 cnt = &_no_rdtsc_end - &_no_rdtsc_start;
4939                 (void) memcpy((void *)tsc_read, (void *)&_no_rdtsc_start, cnt);
4940                 break;
4941         case TSC_RDTSC_MFENCE:
4942                 cnt = &_tsc_mfence_end - &_tsc_mfence_start;
4943                 (void) memcpy((void *)tsc_read,
4944                     (void *)&_tsc_mfence_start, cnt);
4945                 break;
4946         case TSC_RDTSC_LFENCE:
4947                 cnt = &_tsc_lfence_end - &_tsc_lfence_start;
4948                 (void) memcpy((void *)tsc_read,
4949                     (void *)&_tsc_lfence_start, cnt);
4950                 break;
4951         case TSC_TSCP:
4952                 cnt = &_tscp_end - &_tscp_start;
4953                 (void) memcpy((void *)tsc_read, (void *)&_tscp_start, cnt);
4954                 break;
4955         default:
4956                 /* Bail for unexpected TSC types. (TSC_NONE covers 0) */
4957                 cmn_err(CE_PANIC, "Unrecogized TSC type: %d", flag);
4958                 break;
4959         }
4960         tsc_type = flag;
4961 }
4962 
4963 int
4964 cpuid_deep_cstates_supported(void)
4965 {
4966         struct cpuid_info *cpi;
4967         struct cpuid_regs regs;
4968 
4969         ASSERT(cpuid_checkpass(CPU, 1));
4970 
4971         cpi = CPU->cpu_m.mcpu_cpi;
4972 
4973         if (!is_x86_feature(x86_featureset, X86FSET_CPUID))
4974                 return (0);
4975 
4976         switch (cpi->cpi_vendor) {
4977         case X86_VENDOR_Intel:
4978                 if (cpi->cpi_xmaxeax < 0x80000007)
4979                         return (0);
4980 
4981                 /*
4982                  * TSC run at a constant rate in all ACPI C-states?
4983                  */
4984                 regs.cp_eax = 0x80000007;
4985                 (void) __cpuid_insn(&regs);
4986                 return (regs.cp_edx & CPUID_TSC_CSTATE_INVARIANCE);
4987 
4988         default:
4989                 return (0);
4990         }
4991 }
4992 
4993 #endif  /* !__xpv */
4994 
4995 void
4996 post_startup_cpu_fixups(void)
4997 {
4998 #ifndef __xpv
4999         /*
5000          * Some AMD processors support C1E state. Entering this state will
5001          * cause the local APIC timer to stop, which we can't deal with at
5002          * this time.
5003          */
5004         if (cpuid_getvendor(CPU) == X86_VENDOR_AMD) {
5005                 on_trap_data_t otd;
5006                 uint64_t reg;
5007 
5008                 if (!on_trap(&otd, OT_DATA_ACCESS)) {
5009                         reg = rdmsr(MSR_AMD_INT_PENDING_CMP_HALT);
5010                         /* Disable C1E state if it is enabled by BIOS */
5011                         if ((reg >> AMD_ACTONCMPHALT_SHIFT) &
5012                             AMD_ACTONCMPHALT_MASK) {
5013                                 reg &= ~(AMD_ACTONCMPHALT_MASK <<
5014                                     AMD_ACTONCMPHALT_SHIFT);
5015                                 wrmsr(MSR_AMD_INT_PENDING_CMP_HALT, reg);
5016                         }
5017                 }
5018                 no_trap();
5019         }
5020 #endif  /* !__xpv */
5021 }
5022 
5023 void
5024 enable_pcid(void)
5025 {
5026 #if !defined(__xpv)
5027         if (!x86_use_pcid || !is_x86_feature(x86_featureset, X86FSET_PCID))
5028                 return;
5029 
5030         /*
5031          * Intel say that on setting PCIDE, it immediately starts using the PCID
5032          * bits; better make sure there's nothing there.
5033          */
5034         ASSERT((getcr3() & MMU_PAGEOFFSET) == PCID_NONE);
5035 
5036         setcr4(getcr4() | CR4_PCIDE);
5037 #endif
5038 }
5039 
5040 /*
5041  * Setup necessary registers to enable XSAVE feature on this processor.
5042  * This function needs to be called early enough, so that no xsave/xrstor
5043  * ops will execute on the processor before the MSRs are properly set up.
5044  *
5045  * Current implementation has the following assumption:
5046  * - cpuid_pass1() is done, so that X86 features are known.
5047  * - fpu_probe() is done, so that fp_save_mech is chosen.
5048  */
5049 void
5050 xsave_setup_msr(cpu_t *cpu)
5051 {
5052         ASSERT(fp_save_mech == FP_XSAVE);
5053         ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
5054 
5055         /* Enable OSXSAVE in CR4. */
5056         setcr4(getcr4() | CR4_OSXSAVE);
5057         /*
5058          * Update SW copy of ECX, so that /dev/cpu/self/cpuid will report
5059          * correct value.
5060          */
5061         cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_ecx |= CPUID_INTC_ECX_OSXSAVE;
5062         setup_xfem();
5063 }
5064 
5065 /*
5066  * Starting with the Westmere processor the local
5067  * APIC timer will continue running in all C-states,
5068  * including the deepest C-states.
5069  */
5070 int
5071 cpuid_arat_supported(void)
5072 {
5073         struct cpuid_info *cpi;
5074         struct cpuid_regs regs;
5075 
5076         ASSERT(cpuid_checkpass(CPU, 1));
5077         ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
5078 
5079         cpi = CPU->cpu_m.mcpu_cpi;
5080 
5081         switch (cpi->cpi_vendor) {
5082         case X86_VENDOR_Intel:
5083                 /*
5084                  * Always-running Local APIC Timer is
5085                  * indicated by CPUID.6.EAX[2].
5086                  */
5087                 if (cpi->cpi_maxeax >= 6) {
5088                         regs.cp_eax = 6;
5089                         (void) cpuid_insn(NULL, &regs);
5090                         return (regs.cp_eax & CPUID_CSTATE_ARAT);
5091                 } else {
5092                         return (0);
5093                 }
5094         default:
5095                 return (0);
5096         }
5097 }
5098 
5099 /*
5100  * Check support for Intel ENERGY_PERF_BIAS feature
5101  */
5102 int
5103 cpuid_iepb_supported(struct cpu *cp)
5104 {
5105         struct cpuid_info *cpi = cp->cpu_m.mcpu_cpi;
5106         struct cpuid_regs regs;
5107 
5108         ASSERT(cpuid_checkpass(cp, 1));
5109 
5110         if (!(is_x86_feature(x86_featureset, X86FSET_CPUID)) ||
5111             !(is_x86_feature(x86_featureset, X86FSET_MSR))) {
5112                 return (0);
5113         }
5114 
5115         /*
5116          * Intel ENERGY_PERF_BIAS MSR is indicated by
5117          * capability bit CPUID.6.ECX.3
5118          */
5119         if ((cpi->cpi_vendor != X86_VENDOR_Intel) || (cpi->cpi_maxeax < 6))
5120                 return (0);
5121 
5122         regs.cp_eax = 0x6;
5123         (void) cpuid_insn(NULL, &regs);
5124         return (regs.cp_ecx & CPUID_EPB_SUPPORT);
5125 }
5126 
5127 /*
5128  * Check support for TSC deadline timer
5129  *
5130  * TSC deadline timer provides a superior software programming
5131  * model over local APIC timer that eliminates "time drifts".
5132  * Instead of specifying a relative time, software specifies an
5133  * absolute time as the target at which the processor should
5134  * generate a timer event.
5135  */
5136 int
5137 cpuid_deadline_tsc_supported(void)
5138 {
5139         struct cpuid_info *cpi = CPU->cpu_m.mcpu_cpi;
5140         struct cpuid_regs regs;
5141 
5142         ASSERT(cpuid_checkpass(CPU, 1));
5143         ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
5144 
5145         switch (cpi->cpi_vendor) {
5146         case X86_VENDOR_Intel:
5147                 if (cpi->cpi_maxeax >= 1) {
5148                         regs.cp_eax = 1;
5149                         (void) cpuid_insn(NULL, &regs);
5150                         return (regs.cp_ecx & CPUID_DEADLINE_TSC);
5151                 } else {
5152                         return (0);
5153                 }
5154         default:
5155                 return (0);
5156         }
5157 }
5158 
5159 #if defined(__amd64) && !defined(__xpv)
5160 /*
5161  * Patch in versions of bcopy for high performance Intel Nhm processors
5162  * and later...
5163  */
5164 void
5165 patch_memops(uint_t vendor)
5166 {
5167         size_t cnt, i;
5168         caddr_t to, from;
5169 
5170         if ((vendor == X86_VENDOR_Intel) &&
5171             is_x86_feature(x86_featureset, X86FSET_SSE4_2)) {
5172                 cnt = &bcopy_patch_end - &bcopy_patch_start;
5173                 to = &bcopy_ck_size;
5174                 from = &bcopy_patch_start;
5175                 for (i = 0; i < cnt; i++) {
5176                         *to++ = *from++;
5177                 }
5178         }
5179 }
5180 #endif  /* __amd64 && !__xpv */
5181 
5182 /*
5183  * This function finds the number of bits to represent the number of cores per
5184  * chip and the number of strands per core for the Intel platforms.
5185  * It re-uses the x2APIC cpuid code of the cpuid_pass2().
5186  */
5187 void
5188 cpuid_get_ext_topo(uint_t vendor, uint_t *core_nbits, uint_t *strand_nbits)
5189 {
5190         struct cpuid_regs regs;
5191         struct cpuid_regs *cp = &regs;
5192 
5193         if (vendor != X86_VENDOR_Intel) {
5194                 return;
5195         }
5196 
5197         /* if the cpuid level is 0xB, extended topo is available. */
5198         cp->cp_eax = 0;
5199         if (__cpuid_insn(cp) >= 0xB) {
5200 
5201                 cp->cp_eax = 0xB;
5202                 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
5203                 (void) __cpuid_insn(cp);
5204 
5205                 /*
5206                  * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
5207                  * indicates that the extended topology enumeration leaf is
5208                  * available.
5209                  */
5210                 if (cp->cp_ebx) {
5211                         uint_t coreid_shift = 0;
5212                         uint_t chipid_shift = 0;
5213                         uint_t i;
5214                         uint_t level;
5215 
5216                         for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
5217                                 cp->cp_eax = 0xB;
5218                                 cp->cp_ecx = i;
5219 
5220                                 (void) __cpuid_insn(cp);
5221                                 level = CPI_CPU_LEVEL_TYPE(cp);
5222 
5223                                 if (level == 1) {
5224                                         /*
5225                                          * Thread level processor topology
5226                                          * Number of bits shift right APIC ID
5227                                          * to get the coreid.
5228                                          */
5229                                         coreid_shift = BITX(cp->cp_eax, 4, 0);
5230                                 } else if (level == 2) {
5231                                         /*
5232                                          * Core level processor topology
5233                                          * Number of bits shift right APIC ID
5234                                          * to get the chipid.
5235                                          */
5236                                         chipid_shift = BITX(cp->cp_eax, 4, 0);
5237                                 }
5238                         }
5239 
5240                         if (coreid_shift > 0 && chipid_shift > coreid_shift) {
5241                                 *strand_nbits = coreid_shift;
5242                                 *core_nbits = chipid_shift - coreid_shift;
5243                         }
5244                 }
5245         }
5246 }