1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  *
  25  * Copyright 2018 Joyent, Inc.
  26  */
  27 
  28 #include <sys/t_lock.h>
  29 #include <sys/memlist.h>
  30 #include <sys/cpuvar.h>
  31 #include <sys/vmem.h>
  32 #include <sys/mman.h>
  33 #include <sys/vm.h>
  34 #include <sys/kmem.h>
  35 #include <sys/cmn_err.h>
  36 #include <sys/debug.h>
  37 #include <sys/vm_machparam.h>
  38 #include <sys/tss.h>
  39 #include <sys/vnode.h>
  40 #include <vm/hat.h>
  41 #include <vm/anon.h>
  42 #include <vm/as.h>
  43 #include <vm/page.h>
  44 #include <vm/seg.h>
  45 #include <vm/seg_kmem.h>
  46 #include <vm/seg_map.h>
  47 #include <vm/hat_i86.h>
  48 #include <sys/promif.h>
  49 #include <sys/x86_archext.h>
  50 #include <sys/systm.h>
  51 #include <sys/archsystm.h>
  52 #include <sys/sunddi.h>
  53 #include <sys/ddidmareq.h>
  54 #include <sys/controlregs.h>
  55 #include <sys/reboot.h>
  56 #include <sys/kdi.h>
  57 #include <sys/bootconf.h>
  58 #include <sys/bootsvcs.h>
  59 #include <sys/bootinfo.h>
  60 #include <vm/kboot_mmu.h>
  61 
  62 #ifdef __xpv
  63 #include <sys/hypervisor.h>
  64 #endif
  65 
  66 #define ON_USER_HAT(cpu) \
  67         ((cpu)->cpu_m.mcpu_current_hat != NULL && \
  68         (cpu)->cpu_m.mcpu_current_hat != kas.a_hat)
  69 
  70 /*
  71  * Flag is not set early in boot. Once it is set we are no longer
  72  * using boot's page tables.
  73  */
  74 uint_t khat_running = 0;
  75 
  76 /*
  77  * This procedure is callable only while the boot loader is in charge of the
  78  * MMU. It assumes that PA == VA for page table pointers.  It doesn't live in
  79  * kboot_mmu.c since it's used from common code.
  80  */
  81 pfn_t
  82 va_to_pfn(void *vaddr)
  83 {
  84         uintptr_t       des_va = ALIGN2PAGE(vaddr);
  85         uintptr_t       va = des_va;
  86         size_t          len;
  87         uint_t          prot;
  88         pfn_t           pfn;
  89 
  90         if (khat_running)
  91                 panic("va_to_pfn(): called too late\n");
  92 
  93         if (kbm_probe(&va, &len, &pfn, &prot) == 0)
  94                 return (PFN_INVALID);
  95         if (va > des_va)
  96                 return (PFN_INVALID);
  97         if (va < des_va)
  98                 pfn += mmu_btop(des_va - va);
  99         return (pfn);
 100 }
 101 
 102 /*
 103  * Initialize a special area in the kernel that always holds some PTEs for
 104  * faster performance. This always holds segmap's PTEs.
 105  * In the 32 bit kernel this maps the kernel heap too.
 106  */
 107 void
 108 hat_kmap_init(uintptr_t base, size_t len)
 109 {
 110         uintptr_t map_addr;     /* base rounded down to large page size */
 111         uintptr_t map_eaddr;    /* base + len rounded up */
 112         size_t map_len;
 113         caddr_t ptes;           /* mapping area in kernel for kmap ptes */
 114         size_t window_size;     /* size of mapping area for ptes */
 115         ulong_t htable_cnt;     /* # of page tables to cover map_len */
 116         ulong_t i;
 117         htable_t *ht;
 118         uintptr_t va;
 119 
 120         /*
 121          * We have to map in an area that matches an entire page table.
 122          * The PTEs are large page aligned to avoid spurious pagefaults
 123          * on the hypervisor.
 124          */
 125         map_addr = base & LEVEL_MASK(1);
 126         map_eaddr = (base + len + LEVEL_SIZE(1) - 1) & LEVEL_MASK(1);
 127         map_len = map_eaddr - map_addr;
 128         window_size = mmu_btop(map_len) * mmu.pte_size;
 129         window_size = (window_size + LEVEL_SIZE(1)) & LEVEL_MASK(1);
 130         htable_cnt = map_len >> LEVEL_SHIFT(1);
 131 
 132         /*
 133          * allocate vmem for the kmap_ptes
 134          */
 135         ptes = vmem_xalloc(heap_arena, window_size, LEVEL_SIZE(1), 0,
 136             0, NULL, NULL, VM_SLEEP);
 137         mmu.kmap_htables =
 138             kmem_alloc(htable_cnt * sizeof (htable_t *), KM_SLEEP);
 139 
 140         /*
 141          * Map the page tables that cover kmap into the allocated range.
 142          * Note we don't ever htable_release() the kmap page tables - they
 143          * can't ever be stolen, freed, etc.
 144          */
 145         for (va = map_addr, i = 0; i < htable_cnt; va += LEVEL_SIZE(1), ++i) {
 146                 ht = htable_create(kas.a_hat, va, 0, NULL);
 147                 if (ht == NULL)
 148                         panic("hat_kmap_init: ht == NULL");
 149                 mmu.kmap_htables[i] = ht;
 150 
 151                 hat_devload(kas.a_hat, ptes + i * MMU_PAGESIZE,
 152                     MMU_PAGESIZE, ht->ht_pfn,
 153 #ifdef __xpv
 154                     PROT_READ | HAT_NOSYNC | HAT_UNORDERED_OK,
 155 #else
 156                     PROT_READ | PROT_WRITE | HAT_NOSYNC | HAT_UNORDERED_OK,
 157 #endif
 158                     HAT_LOAD | HAT_LOAD_NOCONSIST);
 159         }
 160 
 161         /*
 162          * set information in mmu to activate handling of kmap
 163          */
 164         mmu.kmap_addr = map_addr;
 165         mmu.kmap_eaddr = map_eaddr;
 166         mmu.kmap_ptes = (x86pte_t *)ptes;
 167 }
 168 
 169 extern caddr_t  kpm_vbase;
 170 extern size_t   kpm_size;
 171 
 172 #ifdef __xpv
 173 /*
 174  * Create the initial segkpm mappings for the hypervisor. To avoid having
 175  * to deal with page tables being read only, we make all mappings
 176  * read only at first.
 177  */
 178 static void
 179 xen_kpm_create(paddr_t paddr, level_t lvl)
 180 {
 181         ulong_t pg_off;
 182 
 183         for (pg_off = 0; pg_off < LEVEL_SIZE(lvl); pg_off += MMU_PAGESIZE) {
 184                 kbm_map((uintptr_t)kpm_vbase + paddr, (paddr_t)0, 0, 1);
 185                 kbm_read_only((uintptr_t)kpm_vbase + paddr + pg_off,
 186                     paddr + pg_off);
 187         }
 188 }
 189 
 190 /*
 191  * Try to make all kpm mappings writable. Failures are ok, as those
 192  * are just pagetable, GDT, etc. pages.
 193  */
 194 static void
 195 xen_kpm_finish_init(void)
 196 {
 197         pfn_t gdtpfn = mmu_btop(CPU->cpu_m.mcpu_gdtpa);
 198         pfn_t pfn;
 199         page_t *pp;
 200 
 201         for (pfn = 0; pfn < mfn_count; ++pfn) {
 202                 /*
 203                  * skip gdt
 204                  */
 205                 if (pfn == gdtpfn)
 206                         continue;
 207 
 208                 /*
 209                  * p_index is a hint that this is a pagetable
 210                  */
 211                 pp = page_numtopp_nolock(pfn);
 212                 if (pp && pp->p_index) {
 213                         pp->p_index = 0;
 214                         continue;
 215                 }
 216                 (void) xen_kpm_page(pfn, PT_VALID | PT_WRITABLE);
 217         }
 218 }
 219 #endif
 220 
 221 /*
 222  * Routine to pre-allocate data structures for hat_kern_setup(). It computes
 223  * how many pagetables it needs by walking the boot loader's page tables.
 224  */
 225 /*ARGSUSED*/
 226 void
 227 hat_kern_alloc(
 228         caddr_t segmap_base,
 229         size_t  segmap_size,
 230         caddr_t ekernelheap)
 231 {
 232         uintptr_t       last_va = (uintptr_t)-1;        /* catch 1st time */
 233         uintptr_t       va = 0;
 234         size_t          size;
 235         pfn_t           pfn;
 236         uint_t          prot;
 237         uint_t          table_cnt = 1;
 238         uint_t          mapping_cnt;
 239         level_t         start_level;
 240         level_t         l;
 241         struct memlist  *pmem;
 242         level_t         lpagel = mmu.max_page_level;
 243         uint64_t        paddr;
 244         int64_t         psize;
 245         int             nwindows;
 246 
 247         if (kpm_size > 0) {
 248                 /*
 249                  * Create the kpm page tables.  When running on the
 250                  * hypervisor these are made read/only at first.
 251                  * Later we'll add write permission where possible.
 252                  */
 253                 for (pmem = phys_install; pmem; pmem = pmem->ml_next) {
 254                         paddr = pmem->ml_address;
 255                         psize = pmem->ml_size;
 256                         while (psize >= MMU_PAGESIZE) {
 257                                 /* find the largest page size */
 258                                 for (l = lpagel; l > 0; l--) {
 259                                         if ((paddr & LEVEL_OFFSET(l)) == 0 &&
 260                                             psize > LEVEL_SIZE(l))
 261                                                 break;
 262                                 }
 263 
 264 #if defined(__xpv)
 265                                 /*
 266                                  * Create read/only mappings to avoid
 267                                  * conflicting with pagetable usage
 268                                  */
 269                                 xen_kpm_create(paddr, l);
 270 #else
 271                                 kbm_map((uintptr_t)kpm_vbase + paddr, paddr,
 272                                     l, 1);
 273 #endif
 274                                 paddr += LEVEL_SIZE(l);
 275                                 psize -= LEVEL_SIZE(l);
 276                         }
 277                 }
 278         }
 279 
 280         /*
 281          * If this machine doesn't have a kpm segment, we need to allocate
 282          * a small number of 'windows' which can be used to map pagetables.
 283          */
 284         nwindows = (kpm_size == 0) ? 2 * NCPU : 0;
 285 
 286 #if defined(__xpv)
 287         /*
 288          * On a hypervisor, these windows are also used by the xpv_panic
 289          * code, where we need one window for each level of the pagetable
 290          * hierarchy.
 291          */
 292         nwindows = MAX(nwindows, mmu.max_level);
 293 #endif
 294 
 295         if (nwindows != 0) {
 296                 /*
 297                  * Create the page windows and 1 page of VA in
 298                  * which we map the PTEs of those windows.
 299                  */
 300                 mmu.pwin_base = vmem_xalloc(heap_arena, nwindows * MMU_PAGESIZE,
 301                     LEVEL_SIZE(1), 0, 0, NULL, NULL, VM_SLEEP);
 302                 ASSERT(nwindows <= MMU_PAGESIZE / mmu.pte_size);
 303                 mmu.pwin_pte_va = vmem_xalloc(heap_arena, MMU_PAGESIZE,
 304                     MMU_PAGESIZE, 0, 0, NULL, NULL, VM_SLEEP);
 305 
 306                 /*
 307                  * Find/Create the page table window mappings.
 308                  */
 309                 paddr = 0;
 310                 (void) find_pte((uintptr_t)mmu.pwin_base, &paddr, 0, 0);
 311                 ASSERT(paddr != 0);
 312                 ASSERT((paddr & MMU_PAGEOFFSET) == 0);
 313                 mmu.pwin_pte_pa = paddr;
 314 #ifdef __xpv
 315                 (void) find_pte((uintptr_t)mmu.pwin_pte_va, NULL, 0, 0);
 316                 kbm_read_only((uintptr_t)mmu.pwin_pte_va, mmu.pwin_pte_pa);
 317 #else
 318                 kbm_map((uintptr_t)mmu.pwin_pte_va, mmu.pwin_pte_pa, 0, 1);
 319 #endif
 320         }
 321 
 322         /*
 323          * Walk the boot loader's page tables and figure out
 324          * how many tables and page mappings there will be.
 325          */
 326         while (kbm_probe(&va, &size, &pfn, &prot) != 0) {
 327                 /*
 328                  * At each level, if the last_va falls into a new htable,
 329                  * increment table_cnt. We can stop at the 1st level where
 330                  * they are in the same htable.
 331                  */
 332                 start_level = 0;
 333                 while (start_level <= mmu.max_page_level) {
 334                         if (size == LEVEL_SIZE(start_level))
 335                                 break;
 336                         start_level++;
 337                 }
 338 
 339                 for (l = start_level; l < mmu.max_level; ++l) {
 340                         if (va >> LEVEL_SHIFT(l + 1) ==
 341                             last_va >> LEVEL_SHIFT(l + 1))
 342                                 break;
 343                         ++table_cnt;
 344                 }
 345                 last_va = va;
 346                 l = (start_level == 0) ? 1 : start_level;
 347                 va = (va & LEVEL_MASK(l)) + LEVEL_SIZE(l);
 348         }
 349 
 350         /*
 351          * Besides the boot loader mappings, we're going to fill in
 352          * the entire top level page table for the kernel. Make sure there's
 353          * enough reserve for that too.
 354          */
 355         table_cnt += mmu.top_level_count - ((kernelbase >>
 356             LEVEL_SHIFT(mmu.max_level)) & (mmu.top_level_count - 1));
 357 
 358         /*
 359          * Add 1/4 more into table_cnt for extra slop.  The unused
 360          * slop is freed back when we htable_adjust_reserve() later.
 361          */
 362         table_cnt += table_cnt >> 2;
 363 
 364         /*
 365          * We only need mapping entries (hments) for shared pages.
 366          * This should be far, far fewer than the total possible,
 367          * We'll allocate enough for 1/16 of all possible PTEs.
 368          */
 369         mapping_cnt = (table_cnt * mmu.ptes_per_table) >> 4;
 370 
 371         /*
 372          * Now create the initial htable/hment reserves
 373          */
 374         htable_initial_reserve(table_cnt);
 375         hment_reserve(mapping_cnt);
 376         x86pte_cpu_init(CPU);
 377 }
 378 
 379 
 380 /*
 381  * This routine handles the work of creating the kernel's initial mappings
 382  * by deciphering the mappings in the page tables created by the boot program.
 383  *
 384  * We maintain large page mappings, but only to a level 1 pagesize.
 385  * The boot loader can only add new mappings once this function starts.
 386  * In particular it can not change the pagesize used for any existing
 387  * mappings or this code breaks!
 388  */
 389 
 390 void
 391 hat_kern_setup(void)
 392 {
 393         /*
 394          * Attach htables to the existing pagetables
 395          */
 396         /* BEGIN CSTYLED */
 397         htable_attach(kas.a_hat, 0, mmu.max_level, NULL,
 398 #ifdef __xpv
 399             mmu_btop(xen_info->pt_base - ONE_GIG));
 400 #else
 401             mmu_btop(getcr3_pa()));
 402 #endif
 403         /* END CSTYLED */
 404 
 405 #if defined(__xpv)
 406         /*
 407          * Try to make the kpm mappings r/w. Failures here are OK, as
 408          * it's probably just a pagetable
 409          */
 410         xen_kpm_finish_init();
 411 #endif
 412 
 413         /*
 414          * The kernel HAT is now officially open for business.
 415          */
 416         khat_running = 1;
 417 
 418         CPUSET_ATOMIC_ADD(kas.a_hat->hat_cpus, CPU->cpu_id);
 419         CPU->cpu_current_hat = kas.a_hat;
 420 }
 421 
 422 #ifndef __xpv
 423 
 424 /*
 425  * Note that the INVPCID_ALL* variants can be used even in the !PCIDE case, but
 426  * INVPCID_ADDR isn't.
 427  */
 428 static void
 429 invpcid(uint64_t type, uint64_t pcid, uintptr_t addr)
 430 {
 431         ulong_t cr4;
 432 
 433         if (x86_use_invpcid &&
 434             is_x86_feature(x86_featureset, X86FSET_INVPCID)) {
 435                 invpcid_insn(type, pcid, addr);
 436                 return;
 437         }
 438 
 439         cr4 = getcr4();
 440 
 441         switch (type) {
 442         case INVPCID_ALL_GLOBAL:
 443                 setcr4(cr4 & ~(ulong_t)CR4_PGE);
 444                 setcr4(cr4 | CR4_PGE);
 445                 break;
 446 
 447         case INVPCID_ALL_NONGLOBAL:
 448                 if (!(cr4 & CR4_PCIDE)) {
 449                         reload_cr3();
 450                 } else {
 451                         setcr4(cr4 & ~(ulong_t)CR4_PGE);
 452                         setcr4(cr4 | CR4_PGE);
 453                 }
 454                 break;
 455 
 456         case INVPCID_ADDR:
 457                 if (pcid == PCID_USER) {
 458                         ASSERT(addr < kernelbase);
 459                         ASSERT(ON_USER_HAT(CPU));
 460                         ASSERT(CPU->cpu_m.mcpu_kpti.kf_user_cr3 != 0);
 461                         tr_mmu_flush_user_range(addr, MMU_PAGESIZE,
 462                             MMU_PAGESIZE, CPU->cpu_m.mcpu_kpti.kf_user_cr3);
 463                 } else {
 464                         mmu_invlpg((caddr_t)addr);
 465                 }
 466                 break;
 467 
 468         default:
 469                 panic("unsupported invpcid(%lu)", type);
 470                 break;
 471         }
 472 }
 473 
 474 /*
 475  * Flush one kernel mapping.
 476  *
 477  * We want to assert on kernel space here mainly for reasoning about the PCIDE
 478  * case: namely, this flush should never need to flush a non-current PCID
 479  * mapping.  This presumes we never have reason to flush the kernel regions
 480  * available to PCID_USER (the trampolines and so on).  It also relies on
 481  * PCID_KERNEL == PCID_NONE.
 482  */
 483 void
 484 mmu_flush_tlb_kpage(uintptr_t va)
 485 {
 486         ASSERT(va >= kernelbase);
 487         ASSERT(getpcid() == PCID_KERNEL);
 488         mmu_invlpg((caddr_t)va);
 489 }
 490 
 491 /*
 492  * Flush one mapping: local CPU version of hat_tlb_inval().
 493  *
 494  * If this is a userspace address in the PCIDE case, we need two invalidations,
 495  * one for any potentially stale PCID_USER mapping, as well as any established
 496  * while in the kernel.
 497  */
 498 void
 499 mmu_flush_tlb_page(uintptr_t va)
 500 {
 501         ASSERT(getpcid() == PCID_KERNEL);
 502 
 503         if (va >= kernelbase) {
 504                 mmu_flush_tlb_kpage(va);
 505                 return;
 506         }
 507 
 508         if (!(getcr4() & CR4_PCIDE)) {
 509                 mmu_invlpg((caddr_t)va);
 510                 return;
 511         }
 512 
 513         /*
 514          * Yes, kas will need to flush below kernelspace, at least during boot.
 515          * But there's no PCID_USER context.
 516          */
 517         if (ON_USER_HAT(CPU))
 518                 invpcid(INVPCID_ADDR, PCID_USER, va);
 519         invpcid(INVPCID_ADDR, PCID_KERNEL, va);
 520 }
 521 
 522 static void
 523 mmu_flush_tlb_range(uintptr_t addr, size_t len, size_t pgsz)
 524 {
 525         EQUIV(addr < kernelbase, (addr + len - 1) < kernelbase);
 526         ASSERT(len > 0);
 527         ASSERT(pgsz != 0);
 528 
 529         if (!(getcr4() & CR4_PCIDE) || (is_x86_feature(x86_featureset,
 530             X86FSET_INVPCID) && x86_use_invpcid)) {
 531                 for (uintptr_t va = addr; va < (addr + len); va += pgsz)
 532                         mmu_flush_tlb_page(va);
 533                 return;
 534         }
 535 
 536         /*
 537          * As an emulated invpcid() in the PCIDE case requires jumping
 538          * cr3s, we batch the invalidations.  We should only need to flush the
 539          * user range if we're on a user-space HAT.
 540          */
 541         if (addr < kernelbase && ON_USER_HAT(CPU)) {
 542                 ASSERT(CPU->cpu_m.mcpu_kpti.kf_user_cr3 != 0);
 543                 tr_mmu_flush_user_range(addr, len, pgsz,
 544                     CPU->cpu_m.mcpu_kpti.kf_user_cr3);
 545         }
 546 
 547         for (uintptr_t va = addr; va < (addr + len); va += pgsz)
 548                 mmu_invlpg((caddr_t)va);
 549 }
 550 
 551 /*
 552  * MMU TLB (and PT cache) flushing on this CPU.
 553  *
 554  * FLUSH_TLB_ALL: invalidate everything, all PCIDs, all PT_GLOBAL.
 555  * FLUSH_TLB_NONGLOBAL: invalidate all PCIDs, excluding PT_GLOBAL
 556  * FLUSH_TLB_RANGE: invalidate the given range, including PCID_USER
 557  * mappings as appropriate.  If using invpcid, PT_GLOBAL mappings are not
 558  * invalidated.
 559  */
 560 void
 561 mmu_flush_tlb(flush_tlb_type_t type, tlb_range_t *range)
 562 {
 563         ASSERT(getpcid() == PCID_KERNEL);
 564 
 565         switch (type) {
 566         case FLUSH_TLB_ALL:
 567                 ASSERT(range == NULL);
 568                 invpcid(INVPCID_ALL_GLOBAL, 0, 0);
 569                 break;
 570 
 571         case FLUSH_TLB_NONGLOBAL:
 572                 ASSERT(range == NULL);
 573                 invpcid(INVPCID_ALL_NONGLOBAL, 0, 0);
 574                 break;
 575 
 576         case FLUSH_TLB_RANGE: {
 577                 mmu_flush_tlb_range(range->tr_va, TLB_RANGE_LEN(range),
 578                     LEVEL_SIZE(range->tr_level));
 579                 break;
 580         }
 581 
 582         default:
 583                 panic("invalid call mmu_flush_tlb(%d)", type);
 584                 break;
 585         }
 586 }
 587 
 588 #endif /* ! __xpv */