1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 /*
  25  * Copyright (c) 2010, Intel Corporation.
  26  * All rights reserved.
  27  */
  28         
  29 #include <sys/asm_linkage.h>
  30 #include <sys/asm_misc.h>
  31 #include <sys/regset.h>
  32 #include <sys/privregs.h>
  33 #include <sys/x86_archext.h>
  34 
  35 #if !defined(__lint)
  36 #include <sys/segments.h>
  37 #include "assym.h"
  38 #endif
  39 
  40 /*
  41  *      Our assumptions:
  42  *              - We are running in real mode.
  43  *              - Interrupts are disabled.
  44  *              - Selectors are equal (cs == ds == ss) for all real mode code
  45  *              - The GDT, IDT, ktss and page directory has been built for us
  46  *
  47  *      Our actions:
  48  *      Start CPU:
  49  *              - We start using our GDT by loading correct values in the
  50  *                selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL,
  51  *                gs=KGS_SEL).
  52  *              - We change over to using our IDT.
  53  *              - We load the default LDT into the hardware LDT register.
  54  *              - We load the default TSS into the hardware task register.
  55  *              - call mp_startup(void) indirectly through the T_PC
  56  *      Stop CPU:
  57  *              - Put CPU into halted state with interrupts disabled
  58  *
  59  */
  60 
  61 #if defined(__lint)
  62 
  63 void
  64 real_mode_start_cpu(void)
  65 {}
  66 
  67 void
  68 real_mode_stop_cpu_stage1(void)
  69 {}
  70 
  71 void
  72 real_mode_stop_cpu_stage2(void)
  73 {}
  74 
  75 #else   /* __lint */
  76 
  77 #if defined(__amd64)
  78 
  79         ENTRY_NP(real_mode_start_cpu)
  80 
  81         /*
  82          * NOTE:  The GNU assembler automatically does the right thing to
  83          *        generate data size operand prefixes based on the code size
  84          *        generation mode (e.g. .code16, .code32, .code64) and as such
  85          *        prefixes need not be used on instructions EXCEPT in the case
  86          *        of address prefixes for code for which the reference is not
  87          *        automatically of the default operand size.
  88          */      
  89         .code16
  90         cli
  91         movw            %cs, %ax
  92         movw            %ax, %ds        /* load cs into ds */
  93         movw            %ax, %ss        /* and into ss */
  94 
  95         /*
  96          * Helps in debugging by giving us the fault address.
  97          *
  98          * Remember to patch a hlt (0xf4) at cmntrap to get a good stack.
  99          */
 100         movl            $0xffc, %esp
 101         movl            %cr0, %eax
 102 
 103         /*
 104          * Enable protected-mode, write protect, and alignment mask
 105          */
 106         orl             $(CR0_PE|CR0_WP|CR0_AM), %eax
 107         movl            %eax, %cr0
 108 
 109         /*
 110          * Do a jmp immediately after writing to cr0 when enabling protected
 111          * mode to clear the real mode prefetch queue (per Intel's docs)
 112          */
 113         jmp             pestart
 114 
 115 pestart:
 116         /*
 117          * 16-bit protected mode is now active, so prepare to turn on long
 118          * mode.
 119          *
 120          * Note that we currently assume that if we're attempting to run a
 121          * kernel compiled with (__amd64) #defined, the target CPU has long
 122          * mode support.
 123          */
 124 
 125 #if 0
 126         /*
 127          * If there's a chance this might not be true, the following test should
 128          * be done, with the no_long_mode branch then doing something
 129          * appropriate:
 130          */
 131 
 132         movl            $0x80000000, %eax       /* get largest extended CPUID */
 133         cpuid
 134         cmpl            $0x80000000, %eax       /* check if > 0x80000000 */
 135         jbe             no_long_mode            /* nope, no long mode */
 136         movl            $0x80000001, %eax       
 137         cpuid                                   /* get extended feature flags */
 138         btl             $29, %edx               /* check for long mode */
 139         jnc             no_long_mode            /* long mode not supported */
 140 #endif
 141 
 142         /*
 143          * Add any initial cr4 bits
 144          */
 145         movl            %cr4, %eax
 146         addr32 orl      CR4OFF, %eax
 147 
 148         /*
 149          * Enable PAE mode (CR4.PAE)
 150          */
 151         orl             $CR4_PAE, %eax
 152         movl            %eax, %cr4
 153 
 154         /*
 155          * Point cr3 to the 64-bit long mode page tables.
 156          *
 157          * Note that these MUST exist in 32-bit space, as we don't have
 158          * a way to load %cr3 with a 64-bit base address for the page tables
 159          * until the CPU is actually executing in 64-bit long mode.
 160          */
 161         addr32 movl     CR3OFF, %eax
 162         movl            %eax, %cr3
 163 
 164         /*
 165          * Set long mode enable in EFER (EFER.LME = 1)
 166          */
 167         movl    $MSR_AMD_EFER, %ecx
 168         rdmsr
 169         orl     $AMD_EFER_LME, %eax
 170         wrmsr
 171 
 172         /*
 173          * Finally, turn on paging (CR0.PG = 1) to activate long mode.
 174          */
 175         movl    %cr0, %eax
 176         orl     $CR0_PG, %eax
 177         movl    %eax, %cr0
 178 
 179         /*
 180          * The instruction after enabling paging in CR0 MUST be a branch.
 181          */
 182         jmp     long_mode_active
 183 
 184 long_mode_active:
 185         /*
 186          * Long mode is now active but since we're still running with the
 187          * original 16-bit CS we're actually in 16-bit compatability mode.
 188          *
 189          * We have to load an intermediate GDT and IDT here that we know are
 190          * in 32-bit space before we can use the kernel's GDT and IDT, which
 191          * may be in the 64-bit address space, and since we're in compatability
 192          * mode, we only have access to 16 and 32-bit instructions at the
 193          * moment.
 194          */
 195         addr32 lgdtl    TEMPGDTOFF      /* load temporary GDT */
 196         addr32 lidtl    TEMPIDTOFF      /* load temporary IDT */
 197 
 198         /*
 199          * Do a far transfer to 64-bit mode.  Set the CS selector to a 64-bit
 200          * long mode selector (CS.L=1) in the temporary 32-bit GDT and jump
 201          * to the real mode platter address of long_mode 64 as until the 64-bit
 202          * CS is in place we don't have access to 64-bit instructions and thus
 203          * can't reference a 64-bit %rip.
 204          */
 205         pushl           $TEMP_CS64_SEL
 206         addr32 pushl    LM64OFF
 207         lretl
 208 
 209         .globl  long_mode_64
 210 long_mode_64:
 211         .code64
 212         /*
 213          * We are now running in long mode with a 64-bit CS (EFER.LMA=1,
 214          * CS.L=1) so we now have access to 64-bit instructions.
 215          *
 216          * First, set the 64-bit GDT base.
 217          */
 218         .globl  rm_platter_pa
 219         movl    rm_platter_pa, %eax
 220         lgdtq   GDTROFF(%rax)           /* load 64-bit GDT */
 221 
 222         /*
 223          * Save the CPU number in %r11; get the value here since it's saved in
 224          * the real mode platter.
 225          */
 226         movl    CPUNOFF(%rax), %r11d
 227 
 228         /*
 229          * Add rm_platter_pa to %rsp to point it to the same location as seen
 230          * from 64-bit mode.
 231          */
 232         addq    %rax, %rsp
 233 
 234         /*
 235          * Now do an lretq to load CS with the appropriate selector for the
 236          * kernel's 64-bit GDT and to start executing 64-bit setup code at the
 237          * virtual address where boot originally loaded this code rather than
 238          * the copy in the real mode platter's rm_code array as we've been
 239          * doing so far.
 240          */
 241         pushq   $KCS_SEL
 242         pushq   $kernel_cs_code
 243         lretq
 244         .globl real_mode_start_cpu_end
 245 real_mode_start_cpu_end:
 246         nop
 247 
 248 kernel_cs_code:
 249         /*
 250          * Complete the balance of the setup we need to before executing
 251          * 64-bit kernel code (namely init rsp, TSS, LGDT, FS and GS).
 252          */
 253         .globl  rm_platter_va
 254         movq    rm_platter_va, %rax
 255         lidtq   IDTROFF(%rax)
 256 
 257         movw    $KDS_SEL, %ax
 258         movw    %ax, %ds
 259         movw    %ax, %es
 260         movw    %ax, %ss
 261 
 262         movw    $KTSS_SEL, %ax          /* setup kernel TSS */
 263         ltr     %ax
 264 
 265         xorw    %ax, %ax                /* clear LDTR */
 266         lldt    %ax
 267 
 268         /*
 269          * Set GS to the address of the per-cpu structure as contained in
 270          * cpu[cpu_number].
 271          *
 272          * Unfortunately there's no way to set the 64-bit gsbase with a mov,
 273          * so we have to stuff the low 32 bits in %eax and the high 32 bits in
 274          * %edx, then call wrmsr.
 275          */
 276         leaq    cpu(%rip), %rdi
 277         movl    (%rdi, %r11, 8), %eax
 278         movl    4(%rdi, %r11, 8), %edx
 279         movl    $MSR_AMD_GSBASE, %ecx
 280         wrmsr
 281 
 282         /*
 283          * Init FS and KernelGSBase.
 284          *
 285          * Based on code in mlsetup(), set them both to 8G (which shouldn't be
 286          * valid until some 64-bit processes run); this will then cause an
 287          * exception in any code that tries to index off them before they are
 288          * properly setup.
 289          */
 290         xorl    %eax, %eax              /* low 32 bits = 0 */
 291         movl    $2, %edx                /* high 32 bits = 2 */
 292         movl    $MSR_AMD_FSBASE, %ecx
 293         wrmsr
 294 
 295         movl    $MSR_AMD_KGSBASE, %ecx
 296         wrmsr
 297 
 298         /*
 299          * Init %rsp to the exception stack set in tss_ist1 and create a legal
 300          * AMD64 ABI stack frame
 301          */
 302         movq    %gs:CPU_TSS, %rax
 303         movq    TSS_IST1(%rax), %rsp
 304         pushq   $0              /* null return address */
 305         pushq   $0              /* null frame pointer terminates stack trace */
 306         movq    %rsp, %rbp      /* stack aligned on 16-byte boundary */
 307 
 308         movq    %cr0, %rax
 309         andq    $~(CR0_TS|CR0_EM), %rax /* clear emulate math chip bit */
 310         orq     $(CR0_MP|CR0_NE), %rax
 311         movq    %rax, %cr0              /* set machine status word */
 312 
 313         /*
 314          * Before going any further, enable usage of page table NX bit if 
 315          * that's how our page tables are set up.
 316          */
 317         bt      $X86FSET_NX, x86_featureset(%rip)
 318         jnc     1f
 319         movl    $MSR_AMD_EFER, %ecx
 320         rdmsr
 321         orl     $AMD_EFER_NXE, %eax
 322         wrmsr
 323 1:
 324 
 325         /*
 326          * Complete the rest of the setup and call mp_startup().
 327          */
 328         movq    %gs:CPU_THREAD, %rax    /* get thread ptr */
 329         call    *T_PC(%rax)             /* call mp_startup_boot */
 330         /* not reached */
 331         int     $20                     /* whoops, returned somehow! */
 332 
 333         SET_SIZE(real_mode_start_cpu)
 334 
 335 #elif defined(__i386)
 336 
 337         ENTRY_NP(real_mode_start_cpu)
 338 
 339 #if !defined(__GNUC_AS__)
 340 
 341         cli
 342         D16 movw        %cs, %eax
 343         movw            %eax, %ds       /* load cs into ds */
 344         movw            %eax, %ss       /* and into ss */
 345 
 346         /*
 347          * Helps in debugging by giving us the fault address.
 348          *
 349          * Remember to patch a hlt (0xf4) at cmntrap to get a good stack.
 350          */
 351         D16 movl        $0xffc, %esp
 352 
 353         D16 A16 lgdt    %cs:GDTROFF
 354         D16 A16 lidt    %cs:IDTROFF
 355         D16 A16 movl    %cs:CR4OFF, %eax        /* set up CR4, if desired */
 356         D16 andl        %eax, %eax
 357         D16 A16 je      no_cr4
 358 
 359         D16 movl        %eax, %ecx
 360         D16 movl        %cr4, %eax
 361         D16 orl         %ecx, %eax
 362         D16 movl        %eax, %cr4
 363 no_cr4:
 364         D16 A16 movl    %cs:CR3OFF, %eax
 365         A16 movl        %eax, %cr3
 366         movl            %cr0, %eax
 367 
 368         /*
 369          * Enable protected-mode, paging, write protect, and alignment mask
 370          */
 371         D16 orl         $[CR0_PG|CR0_PE|CR0_WP|CR0_AM], %eax
 372         movl            %eax, %cr0
 373         jmp             pestart
 374 
 375 pestart:
 376         D16 pushl       $KCS_SEL
 377         D16 pushl       $kernel_cs_code
 378         D16 lret
 379         .globl real_mode_start_cpu_end
 380 real_mode_start_cpu_end:
 381         nop
 382 
 383         .globl  kernel_cs_code
 384 kernel_cs_code:
 385         /*
 386          * At this point we are with kernel's cs and proper eip.
 387          *
 388          * We will be executing not from the copy in real mode platter,
 389          * but from the original code where boot loaded us.
 390          *
 391          * By this time GDT and IDT are loaded as is cr3.
 392          */
 393         movw    $KFS_SEL,%eax
 394         movw    %eax,%fs
 395         movw    $KGS_SEL,%eax
 396         movw    %eax,%gs
 397         movw    $KDS_SEL,%eax
 398         movw    %eax,%ds
 399         movw    %eax,%es
 400         movl    %gs:CPU_TSS,%esi
 401         movw    %eax,%ss
 402         movl    TSS_ESP0(%esi),%esp
 403         movw    $KTSS_SEL,%ax
 404         ltr     %ax
 405         xorw    %ax, %ax                /* clear LDTR */
 406         lldt    %ax
 407         movl    %cr0,%edx
 408         andl    $-1![CR0_TS|CR0_EM],%edx  /* clear emulate math chip bit */
 409         orl     $[CR0_MP|CR0_NE],%edx
 410         movl    %edx,%cr0                 /* set machine status word */
 411 
 412         /*
 413          * Before going any further, enable usage of page table NX bit if 
 414          * that's how our page tables are set up.
 415          */
 416         bt      $X86FSET_NX, x86_featureset
 417         jnc     1f
 418         movl    %cr4, %ecx
 419         andl    $CR4_PAE, %ecx
 420         jz      1f
 421         movl    $MSR_AMD_EFER, %ecx
 422         rdmsr
 423         orl     $AMD_EFER_NXE, %eax
 424         wrmsr
 425 1:
 426         movl    %gs:CPU_THREAD, %eax    /* get thread ptr */
 427         call    *T_PC(%eax)             /* call mp_startup */
 428         /* not reached */
 429         int     $20                     /* whoops, returned somehow! */
 430 
 431 #else
 432 
 433         cli
 434         mov             %cs, %ax
 435         mov             %eax, %ds       /* load cs into ds */
 436         mov             %eax, %ss       /* and into ss */
 437 
 438         /*
 439          * Helps in debugging by giving us the fault address.
 440          *
 441          * Remember to patch a hlt (0xf4) at cmntrap to get a good stack.
 442          */
 443         D16 mov         $0xffc, %esp
 444 
 445         D16 A16 lgdtl   %cs:GDTROFF
 446         D16 A16 lidtl   %cs:IDTROFF
 447         D16 A16 mov     %cs:CR4OFF, %eax        /* set up CR4, if desired */
 448         D16 and         %eax, %eax
 449         D16 A16 je      no_cr4
 450 
 451         D16 mov         %eax, %ecx
 452         D16 mov         %cr4, %eax
 453         D16 or          %ecx, %eax
 454         D16 mov         %eax, %cr4
 455 no_cr4:
 456         D16 A16 mov     %cs:CR3OFF, %eax
 457         A16 mov         %eax, %cr3
 458         mov             %cr0, %eax
 459 
 460         /*
 461          * Enable protected-mode, paging, write protect, and alignment mask
 462          */
 463         D16 or          $(CR0_PG|CR0_PE|CR0_WP|CR0_AM), %eax
 464         mov             %eax, %cr0
 465         jmp             pestart
 466 
 467 pestart:
 468         D16 pushl       $KCS_SEL
 469         D16 pushl       $kernel_cs_code
 470         D16 lret
 471         .globl real_mode_start_cpu_end
 472 real_mode_start_cpu_end:
 473         nop
 474         .globl  kernel_cs_code
 475 kernel_cs_code:
 476         /*
 477          * At this point we are with kernel's cs and proper eip.
 478          *
 479          * We will be executing not from the copy in real mode platter,
 480          * but from the original code where boot loaded us.
 481          *
 482          * By this time GDT and IDT are loaded as is cr3.
 483          */
 484         mov     $KFS_SEL, %ax
 485         mov     %eax, %fs
 486         mov     $KGS_SEL, %ax
 487         mov     %eax, %gs
 488         mov     $KDS_SEL, %ax
 489         mov     %eax, %ds
 490         mov     %eax, %es
 491         mov     %gs:CPU_TSS, %esi
 492         mov     %eax, %ss
 493         mov     TSS_ESP0(%esi), %esp
 494         mov     $(KTSS_SEL), %ax
 495         ltr     %ax
 496         xorw    %ax, %ax                /* clear LDTR */
 497         lldt    %ax
 498         mov     %cr0, %edx
 499         and     $~(CR0_TS|CR0_EM), %edx /* clear emulate math chip bit */
 500         or      $(CR0_MP|CR0_NE), %edx
 501         mov     %edx, %cr0              /* set machine status word */
 502 
 503         /*
 504          * Before going any farther, enable usage of page table NX bit if 
 505          * that's how our page tables are set up.  (PCIDE is enabled later on).
 506          */
 507         bt      $X86FSET_NX, x86_featureset
 508         jnc     1f
 509         movl    %cr4, %ecx
 510         andl    $CR4_PAE, %ecx
 511         jz      1f
 512         movl    $MSR_AMD_EFER, %ecx
 513         rdmsr
 514         orl     $AMD_EFER_NXE, %eax
 515         wrmsr
 516 1:
 517         mov     %gs:CPU_THREAD, %eax    /* get thread ptr */
 518         call    *T_PC(%eax)             /* call mp_startup */
 519         /* not reached */
 520         int     $20                     /* whoops, returned somehow! */
 521 #endif
 522 
 523         SET_SIZE(real_mode_start_cpu)
 524 
 525 #endif  /* __amd64 */
 526 
 527 #if defined(__amd64)
 528 
 529         ENTRY_NP(real_mode_stop_cpu_stage1)
 530 
 531 #if !defined(__GNUC_AS__)
 532 
 533         /*
 534          * For vulcan as we need to do a .code32 and mentally invert the
 535          * meaning of the addr16 and data16 prefixes to get 32-bit access when
 536          * generating code to be executed in 16-bit mode (sigh...)
 537          */
 538         .code32
 539         cli
 540         movw            %cs, %ax
 541         movw            %ax, %ds        /* load cs into ds */
 542         movw            %ax, %ss        /* and into ss */
 543 
 544         /*
 545          * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
 546          */
 547         movw            $CPUHALTCODEOFF, %ax
 548         .byte           0xff, 0xe0      /* jmp *%ax */
 549 
 550 #else   /* __GNUC_AS__ */
 551 
 552         /*
 553          * NOTE:  The GNU assembler automatically does the right thing to
 554          *        generate data size operand prefixes based on the code size
 555          *        generation mode (e.g. .code16, .code32, .code64) and as such
 556          *        prefixes need not be used on instructions EXCEPT in the case
 557          *        of address prefixes for code for which the reference is not
 558          *        automatically of the default operand size.
 559          */      
 560         .code16
 561         cli
 562         movw            %cs, %ax
 563         movw            %ax, %ds        /* load cs into ds */
 564         movw            %ax, %ss        /* and into ss */
 565 
 566         /*
 567          * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
 568          */
 569         movw            $CPUHALTCODEOFF, %ax
 570         jmp             *%ax
 571 
 572 #endif  /* !__GNUC_AS__ */
 573 
 574         .globl real_mode_stop_cpu_stage1_end
 575 real_mode_stop_cpu_stage1_end:
 576         nop
 577 
 578         SET_SIZE(real_mode_stop_cpu_stage1)
 579 
 580 #elif defined(__i386)
 581 
 582         ENTRY_NP(real_mode_stop_cpu_stage1)
 583 
 584 #if !defined(__GNUC_AS__)
 585 
 586         cli
 587         D16 movw        %cs, %eax
 588         movw            %eax, %ds       /* load cs into ds */
 589         movw            %eax, %ss       /* and into ss */
 590 
 591         /*
 592          * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
 593          */
 594         movw            $CPUHALTCODEOFF, %ax
 595         .byte           0xff, 0xe0      /* jmp *%ax */
 596 
 597 #else   /* __GNUC_AS__ */
 598 
 599         cli
 600         mov             %cs, %ax
 601         mov             %eax, %ds       /* load cs into ds */
 602         mov             %eax, %ss       /* and into ss */
 603 
 604         /*
 605          * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
 606          */
 607         movw            $CPUHALTCODEOFF, %ax
 608         jmp             *%ax
 609 
 610 #endif  /* !__GNUC_AS__ */
 611 
 612         .globl real_mode_stop_cpu_stage1_end
 613 real_mode_stop_cpu_stage1_end:
 614         nop
 615 
 616         SET_SIZE(real_mode_stop_cpu_stage1)
 617 
 618 #endif  /* __amd64 */
 619 
 620         ENTRY_NP(real_mode_stop_cpu_stage2)
 621 
 622         movw            $0xdead, %ax
 623         movw            %ax, CPUHALTEDOFF
 624 
 625 real_mode_stop_cpu_loop:
 626         /*
 627          * Put CPU into halted state.
 628          * Only INIT, SMI, NMI could break the loop.
 629          */
 630         hlt
 631         jmp             real_mode_stop_cpu_loop
 632 
 633         .globl real_mode_stop_cpu_stage2_end
 634 real_mode_stop_cpu_stage2_end:
 635         nop
 636 
 637         SET_SIZE(real_mode_stop_cpu_stage2)
 638 
 639 #endif  /* __lint */