1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2014 by Delphix. All rights reserved.
  25  * Copyright 2018 Joyent, Inc.
  26  */
  27 
  28 #include <sys/types.h>
  29 #include <sys/sysmacros.h>
  30 #include <sys/kmem.h>
  31 #include <sys/atomic.h>
  32 #include <sys/bitmap.h>
  33 #include <sys/machparam.h>
  34 #include <sys/machsystm.h>
  35 #include <sys/mman.h>
  36 #include <sys/systm.h>
  37 #include <sys/cpuvar.h>
  38 #include <sys/thread.h>
  39 #include <sys/proc.h>
  40 #include <sys/cpu.h>
  41 #include <sys/kmem.h>
  42 #include <sys/disp.h>
  43 #include <sys/vmem.h>
  44 #include <sys/vmsystm.h>
  45 #include <sys/promif.h>
  46 #include <sys/var.h>
  47 #include <sys/x86_archext.h>
  48 #include <sys/archsystm.h>
  49 #include <sys/bootconf.h>
  50 #include <sys/dumphdr.h>
  51 #include <vm/seg_kmem.h>
  52 #include <vm/seg_kpm.h>
  53 #include <vm/hat.h>
  54 #include <vm/hat_i86.h>
  55 #include <sys/cmn_err.h>
  56 #include <sys/panic.h>
  57 
  58 #ifdef __xpv
  59 #include <sys/hypervisor.h>
  60 #include <sys/xpv_panic.h>
  61 #endif
  62 
  63 #include <sys/bootinfo.h>
  64 #include <vm/kboot_mmu.h>
  65 
  66 static void x86pte_zero(htable_t *dest, uint_t entry, uint_t count);
  67 
  68 kmem_cache_t *htable_cache;
  69 
  70 /*
  71  * The variable htable_reserve_amount, rather than HTABLE_RESERVE_AMOUNT,
  72  * is used in order to facilitate testing of the htable_steal() code.
  73  * By resetting htable_reserve_amount to a lower value, we can force
  74  * stealing to occur.  The reserve amount is a guess to get us through boot.
  75  */
  76 #define HTABLE_RESERVE_AMOUNT   (200)
  77 uint_t htable_reserve_amount = HTABLE_RESERVE_AMOUNT;
  78 kmutex_t htable_reserve_mutex;
  79 uint_t htable_reserve_cnt;
  80 htable_t *htable_reserve_pool;
  81 
  82 /*
  83  * Used to hand test htable_steal().
  84  */
  85 #ifdef DEBUG
  86 ulong_t force_steal = 0;
  87 ulong_t ptable_cnt = 0;
  88 #endif
  89 
  90 /*
  91  * This variable is so that we can tune this via /etc/system
  92  * Any value works, but a power of two <= mmu.ptes_per_table is best.
  93  */
  94 uint_t htable_steal_passes = 8;
  95 
  96 /*
  97  * mutex stuff for access to htable hash
  98  */
  99 #define NUM_HTABLE_MUTEX 128
 100 kmutex_t htable_mutex[NUM_HTABLE_MUTEX];
 101 #define HTABLE_MUTEX_HASH(h) ((h) & (NUM_HTABLE_MUTEX - 1))
 102 
 103 #define HTABLE_ENTER(h) mutex_enter(&htable_mutex[HTABLE_MUTEX_HASH(h)]);
 104 #define HTABLE_EXIT(h)  mutex_exit(&htable_mutex[HTABLE_MUTEX_HASH(h)]);
 105 
 106 /*
 107  * forward declarations
 108  */
 109 static void link_ptp(htable_t *higher, htable_t *new, uintptr_t vaddr);
 110 static void unlink_ptp(htable_t *higher, htable_t *old, uintptr_t vaddr);
 111 static void htable_free(htable_t *ht);
 112 static x86pte_t *x86pte_access_pagetable(htable_t *ht, uint_t index);
 113 static void x86pte_release_pagetable(htable_t *ht);
 114 static x86pte_t x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old,
 115         x86pte_t new);
 116 
 117 /*
 118  * A counter to track if we are stealing or reaping htables. When non-zero
 119  * htable_free() will directly free htables (either to the reserve or kmem)
 120  * instead of putting them in a hat's htable cache.
 121  */
 122 uint32_t htable_dont_cache = 0;
 123 
 124 /*
 125  * Track the number of active pagetables, so we can know how many to reap
 126  */
 127 static uint32_t active_ptables = 0;
 128 
 129 #ifdef __xpv
 130 /*
 131  * Deal with hypervisor complications.
 132  */
 133 void
 134 xen_flush_va(caddr_t va)
 135 {
 136         struct mmuext_op t;
 137         uint_t count;
 138 
 139         if (IN_XPV_PANIC()) {
 140                 mmu_tlbflush_entry((caddr_t)va);
 141         } else {
 142                 t.cmd = MMUEXT_INVLPG_LOCAL;
 143                 t.arg1.linear_addr = (uintptr_t)va;
 144                 if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
 145                         panic("HYPERVISOR_mmuext_op() failed");
 146                 ASSERT(count == 1);
 147         }
 148 }
 149 
 150 void
 151 xen_gflush_va(caddr_t va, cpuset_t cpus)
 152 {
 153         struct mmuext_op t;
 154         uint_t count;
 155 
 156         if (IN_XPV_PANIC()) {
 157                 mmu_tlbflush_entry((caddr_t)va);
 158                 return;
 159         }
 160 
 161         t.cmd = MMUEXT_INVLPG_MULTI;
 162         t.arg1.linear_addr = (uintptr_t)va;
 163         /*LINTED: constant in conditional context*/
 164         set_xen_guest_handle(t.arg2.vcpumask, &cpus);
 165         if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
 166                 panic("HYPERVISOR_mmuext_op() failed");
 167         ASSERT(count == 1);
 168 }
 169 
 170 void
 171 xen_flush_tlb()
 172 {
 173         struct mmuext_op t;
 174         uint_t count;
 175 
 176         if (IN_XPV_PANIC()) {
 177                 xpv_panic_reload_cr3();
 178         } else {
 179                 t.cmd = MMUEXT_TLB_FLUSH_LOCAL;
 180                 if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
 181                         panic("HYPERVISOR_mmuext_op() failed");
 182                 ASSERT(count == 1);
 183         }
 184 }
 185 
 186 void
 187 xen_gflush_tlb(cpuset_t cpus)
 188 {
 189         struct mmuext_op t;
 190         uint_t count;
 191 
 192         ASSERT(!IN_XPV_PANIC());
 193         t.cmd = MMUEXT_TLB_FLUSH_MULTI;
 194         /*LINTED: constant in conditional context*/
 195         set_xen_guest_handle(t.arg2.vcpumask, &cpus);
 196         if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
 197                 panic("HYPERVISOR_mmuext_op() failed");
 198         ASSERT(count == 1);
 199 }
 200 
 201 /*
 202  * Install/Adjust a kpm mapping under the hypervisor.
 203  * Value of "how" should be:
 204  *      PT_WRITABLE | PT_VALID - regular kpm mapping
 205  *      PT_VALID - make mapping read-only
 206  *      0       - remove mapping
 207  *
 208  * returns 0 on success. non-zero for failure.
 209  */
 210 int
 211 xen_kpm_page(pfn_t pfn, uint_t how)
 212 {
 213         paddr_t pa = mmu_ptob((paddr_t)pfn);
 214         x86pte_t pte = PT_NOCONSIST | PT_REF | PT_MOD;
 215 
 216         if (kpm_vbase == NULL)
 217                 return (0);
 218 
 219         if (how)
 220                 pte |= pa_to_ma(pa) | how;
 221         else
 222                 pte = 0;
 223         return (HYPERVISOR_update_va_mapping((uintptr_t)kpm_vbase + pa,
 224             pte, UVMF_INVLPG | UVMF_ALL));
 225 }
 226 
 227 void
 228 xen_pin(pfn_t pfn, level_t lvl)
 229 {
 230         struct mmuext_op t;
 231         uint_t count;
 232 
 233         t.cmd = MMUEXT_PIN_L1_TABLE + lvl;
 234         t.arg1.mfn = pfn_to_mfn(pfn);
 235         if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
 236                 panic("HYPERVISOR_mmuext_op() failed");
 237         ASSERT(count == 1);
 238 }
 239 
 240 void
 241 xen_unpin(pfn_t pfn)
 242 {
 243         struct mmuext_op t;
 244         uint_t count;
 245 
 246         t.cmd = MMUEXT_UNPIN_TABLE;
 247         t.arg1.mfn = pfn_to_mfn(pfn);
 248         if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
 249                 panic("HYPERVISOR_mmuext_op() failed");
 250         ASSERT(count == 1);
 251 }
 252 
 253 static void
 254 xen_map(uint64_t pte, caddr_t va)
 255 {
 256         if (HYPERVISOR_update_va_mapping((uintptr_t)va, pte,
 257             UVMF_INVLPG | UVMF_LOCAL))
 258                 panic("HYPERVISOR_update_va_mapping() failed");
 259 }
 260 #endif /* __xpv */
 261 
 262 /*
 263  * Allocate a memory page for a hardware page table.
 264  *
 265  * A wrapper around page_get_physical(), with some extra checks.
 266  */
 267 static pfn_t
 268 ptable_alloc(uintptr_t seed)
 269 {
 270         pfn_t pfn;
 271         page_t *pp;
 272 
 273         pfn = PFN_INVALID;
 274 
 275         /*
 276          * The first check is to see if there is memory in the system. If we
 277          * drop to throttlefree, then fail the ptable_alloc() and let the
 278          * stealing code kick in. Note that we have to do this test here,
 279          * since the test in page_create_throttle() would let the NOSLEEP
 280          * allocation go through and deplete the page reserves.
 281          *
 282          * The !NOMEMWAIT() lets pageout, fsflush, etc. skip this check.
 283          */
 284         if (!NOMEMWAIT() && freemem <= throttlefree + 1)
 285                 return (PFN_INVALID);
 286 
 287 #ifdef DEBUG
 288         /*
 289          * This code makes htable_steal() easier to test. By setting
 290          * force_steal we force pagetable allocations to fall
 291          * into the stealing code. Roughly 1 in ever "force_steal"
 292          * page table allocations will fail.
 293          */
 294         if (proc_pageout != NULL && force_steal > 1 &&
 295             ++ptable_cnt > force_steal) {
 296                 ptable_cnt = 0;
 297                 return (PFN_INVALID);
 298         }
 299 #endif /* DEBUG */
 300 
 301         pp = page_get_physical(seed);
 302         if (pp == NULL)
 303                 return (PFN_INVALID);
 304         ASSERT(PAGE_SHARED(pp));
 305         pfn = pp->p_pagenum;
 306         if (pfn == PFN_INVALID)
 307                 panic("ptable_alloc(): Invalid PFN!!");
 308         atomic_inc_32(&active_ptables);
 309         HATSTAT_INC(hs_ptable_allocs);
 310         return (pfn);
 311 }
 312 
 313 /*
 314  * Free an htable's associated page table page.  See the comments
 315  * for ptable_alloc().
 316  */
 317 static void
 318 ptable_free(pfn_t pfn)
 319 {
 320         page_t *pp = page_numtopp_nolock(pfn);
 321 
 322         /*
 323          * need to destroy the page used for the pagetable
 324          */
 325         ASSERT(pfn != PFN_INVALID);
 326         HATSTAT_INC(hs_ptable_frees);
 327         atomic_dec_32(&active_ptables);
 328         if (pp == NULL)
 329                 panic("ptable_free(): no page for pfn!");
 330         ASSERT(PAGE_SHARED(pp));
 331         ASSERT(pfn == pp->p_pagenum);
 332         ASSERT(!IN_XPV_PANIC());
 333 
 334         /*
 335          * Get an exclusive lock, might have to wait for a kmem reader.
 336          */
 337         if (!page_tryupgrade(pp)) {
 338                 u_offset_t off = pp->p_offset;
 339                 page_unlock(pp);
 340                 pp = page_lookup(&kvp, off, SE_EXCL);
 341                 if (pp == NULL)
 342                         panic("page not found");
 343         }
 344 #ifdef __xpv
 345         if (kpm_vbase && xen_kpm_page(pfn, PT_VALID | PT_WRITABLE) < 0)
 346                 panic("failure making kpm r/w pfn=0x%lx", pfn);
 347 #endif
 348         page_hashout(pp, NULL);
 349         page_free(pp, 1);
 350         page_unresv(1);
 351 }
 352 
 353 /*
 354  * Put one htable on the reserve list.
 355  */
 356 static void
 357 htable_put_reserve(htable_t *ht)
 358 {
 359         ht->ht_hat = NULL;           /* no longer tied to a hat */
 360         ASSERT(ht->ht_pfn == PFN_INVALID);
 361         HATSTAT_INC(hs_htable_rputs);
 362         mutex_enter(&htable_reserve_mutex);
 363         ht->ht_next = htable_reserve_pool;
 364         htable_reserve_pool = ht;
 365         ++htable_reserve_cnt;
 366         mutex_exit(&htable_reserve_mutex);
 367 }
 368 
 369 /*
 370  * Take one htable from the reserve.
 371  */
 372 static htable_t *
 373 htable_get_reserve(void)
 374 {
 375         htable_t *ht = NULL;
 376 
 377         mutex_enter(&htable_reserve_mutex);
 378         if (htable_reserve_cnt != 0) {
 379                 ht = htable_reserve_pool;
 380                 ASSERT(ht != NULL);
 381                 ASSERT(ht->ht_pfn == PFN_INVALID);
 382                 htable_reserve_pool = ht->ht_next;
 383                 --htable_reserve_cnt;
 384                 HATSTAT_INC(hs_htable_rgets);
 385         }
 386         mutex_exit(&htable_reserve_mutex);
 387         return (ht);
 388 }
 389 
 390 /*
 391  * Allocate initial htables and put them on the reserve list
 392  */
 393 void
 394 htable_initial_reserve(uint_t count)
 395 {
 396         htable_t *ht;
 397 
 398         count += HTABLE_RESERVE_AMOUNT;
 399         while (count > 0) {
 400                 ht = kmem_cache_alloc(htable_cache, KM_NOSLEEP);
 401                 ASSERT(ht != NULL);
 402 
 403                 ASSERT(use_boot_reserve);
 404                 ht->ht_pfn = PFN_INVALID;
 405                 htable_put_reserve(ht);
 406                 --count;
 407         }
 408 }
 409 
 410 /*
 411  * Readjust the reserves after a thread finishes using them.
 412  */
 413 void
 414 htable_adjust_reserve()
 415 {
 416         htable_t *ht;
 417 
 418         /*
 419          * Free any excess htables in the reserve list
 420          */
 421         while (htable_reserve_cnt > htable_reserve_amount &&
 422             !USE_HAT_RESERVES()) {
 423                 ht = htable_get_reserve();
 424                 if (ht == NULL)
 425                         return;
 426                 ASSERT(ht->ht_pfn == PFN_INVALID);
 427                 kmem_cache_free(htable_cache, ht);
 428         }
 429 }
 430 
 431 /*
 432  * Search the active htables for one to steal. Start at a different hash
 433  * bucket every time to help spread the pain of stealing
 434  */
 435 static void
 436 htable_steal_active(hat_t *hat, uint_t cnt, uint_t threshold,
 437     uint_t *stolen, htable_t **list)
 438 {
 439         static uint_t   h_seed = 0;
 440         htable_t        *higher, *ht;
 441         uint_t          h, e, h_start;
 442         uintptr_t       va;
 443         x86pte_t        pte;
 444 
 445         h = h_start = h_seed++ % hat->hat_num_hash;
 446         do {
 447                 higher = NULL;
 448                 HTABLE_ENTER(h);
 449                 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
 450 
 451                         /*
 452                          * Can we rule out reaping?
 453                          */
 454                         if (ht->ht_busy != 0 ||
 455                             (ht->ht_flags & HTABLE_SHARED_PFN) ||
 456                             ht->ht_level > 0 || ht->ht_valid_cnt > threshold ||
 457                             ht->ht_lock_cnt != 0)
 458                                 continue;
 459 
 460                         /*
 461                          * Increment busy so the htable can't disappear. We
 462                          * drop the htable mutex to avoid deadlocks with
 463                          * hat_pageunload() and the hment mutex while we
 464                          * call hat_pte_unmap()
 465                          */
 466                         ++ht->ht_busy;
 467                         HTABLE_EXIT(h);
 468 
 469                         /*
 470                          * Try stealing.
 471                          * - unload and invalidate all PTEs
 472                          */
 473                         for (e = 0, va = ht->ht_vaddr;
 474                             e < HTABLE_NUM_PTES(ht) && ht->ht_valid_cnt > 0 &&
 475                             ht->ht_busy == 1 && ht->ht_lock_cnt == 0;
 476                             ++e, va += MMU_PAGESIZE) {
 477                                 pte = x86pte_get(ht, e);
 478                                 if (!PTE_ISVALID(pte))
 479                                         continue;
 480                                 hat_pte_unmap(ht, e, HAT_UNLOAD, pte, NULL,
 481                                     B_TRUE);
 482                         }
 483 
 484                         /*
 485                          * Reacquire htable lock. If we didn't remove all
 486                          * mappings in the table, or another thread added a new
 487                          * mapping behind us, give up on this table.
 488                          */
 489                         HTABLE_ENTER(h);
 490                         if (ht->ht_busy != 1 || ht->ht_valid_cnt != 0 ||
 491                             ht->ht_lock_cnt != 0) {
 492                                 --ht->ht_busy;
 493                                 continue;
 494                         }
 495 
 496                         /*
 497                          * Steal it and unlink the page table.
 498                          */
 499                         higher = ht->ht_parent;
 500                         unlink_ptp(higher, ht, ht->ht_vaddr);
 501 
 502                         /*
 503                          * remove from the hash list
 504                          */
 505                         if (ht->ht_next)
 506                                 ht->ht_next->ht_prev = ht->ht_prev;
 507 
 508                         if (ht->ht_prev) {
 509                                 ht->ht_prev->ht_next = ht->ht_next;
 510                         } else {
 511                                 ASSERT(hat->hat_ht_hash[h] == ht);
 512                                 hat->hat_ht_hash[h] = ht->ht_next;
 513                         }
 514 
 515                         /*
 516                          * Break to outer loop to release the
 517                          * higher (ht_parent) pagetable. This
 518                          * spreads out the pain caused by
 519                          * pagefaults.
 520                          */
 521                         ht->ht_next = *list;
 522                         *list = ht;
 523                         ++*stolen;
 524                         break;
 525                 }
 526                 HTABLE_EXIT(h);
 527                 if (higher != NULL)
 528                         htable_release(higher);
 529                 if (++h == hat->hat_num_hash)
 530                         h = 0;
 531         } while (*stolen < cnt && h != h_start);
 532 }
 533 
 534 /*
 535  * Move hat to the end of the kas list
 536  */
 537 static void
 538 move_victim(hat_t *hat)
 539 {
 540         ASSERT(MUTEX_HELD(&hat_list_lock));
 541 
 542         /* unlink victim hat */
 543         if (hat->hat_prev)
 544                 hat->hat_prev->hat_next = hat->hat_next;
 545         else
 546                 kas.a_hat->hat_next = hat->hat_next;
 547 
 548         if (hat->hat_next)
 549                 hat->hat_next->hat_prev = hat->hat_prev;
 550         else
 551                 kas.a_hat->hat_prev = hat->hat_prev;
 552         /* relink at end of hat list */
 553         hat->hat_next = NULL;
 554         hat->hat_prev = kas.a_hat->hat_prev;
 555         if (hat->hat_prev)
 556                 hat->hat_prev->hat_next = hat;
 557         else
 558                 kas.a_hat->hat_next = hat;
 559 
 560         kas.a_hat->hat_prev = hat;
 561 }
 562 
 563 /*
 564  * This routine steals htables from user processes.  Called by htable_reap
 565  * (reap=TRUE) or htable_alloc (reap=FALSE).
 566  */
 567 static htable_t *
 568 htable_steal(uint_t cnt, boolean_t reap)
 569 {
 570         hat_t           *hat = kas.a_hat;       /* list starts with khat */
 571         htable_t        *list = NULL;
 572         htable_t        *ht;
 573         uint_t          stolen = 0;
 574         uint_t          pass, passes;
 575         uint_t          threshold;
 576 
 577         /*
 578          * Limit htable_steal_passes to something reasonable
 579          */
 580         if (htable_steal_passes == 0)
 581                 htable_steal_passes = 1;
 582         if (htable_steal_passes > mmu.ptes_per_table)
 583                 htable_steal_passes = mmu.ptes_per_table;
 584 
 585         /*
 586          * If we're stealing merely as part of kmem reaping (versus stealing
 587          * to assure forward progress), we don't want to actually steal any
 588          * active htables.  (Stealing active htables merely to give memory
 589          * back to the system can inadvertently kick off an htable crime wave
 590          * as active processes repeatedly steal htables from one another,
 591          * plummeting the system into a kind of HAT lawlessness that can
 592          * become so violent as to impede the one thing that can end it:  the
 593          * freeing of memory via ARC reclaim and other means.)  So if we're
 594          * reaping, we limit ourselves to the first pass that steals cached
 595          * htables that aren't in use -- which gives memory back, but averts
 596          * the entire breakdown of social order.
 597          */
 598         passes = reap ? 0 : htable_steal_passes;
 599 
 600         /*
 601          * Loop through all user hats. The 1st pass takes cached htables that
 602          * aren't in use. The later passes steal by removing mappings, too.
 603          */
 604         atomic_inc_32(&htable_dont_cache);
 605         for (pass = 0; pass <= passes && stolen < cnt; ++pass) {
 606                 threshold = pass * mmu.ptes_per_table / htable_steal_passes;
 607 
 608                 mutex_enter(&hat_list_lock);
 609 
 610                 /* skip the first hat (kernel) */
 611                 hat = kas.a_hat->hat_next;
 612                 for (;;) {
 613                         /*
 614                          * Skip any hat that is already being stolen from.
 615                          *
 616                          * We skip SHARED hats, as these are dummy
 617                          * hats that host ISM shared page tables.
 618                          *
 619                          * We also skip if HAT_FREEING because hat_pte_unmap()
 620                          * won't zero out the PTE's. That would lead to hitting
 621                          * stale PTEs either here or under hat_unload() when we
 622                          * steal and unload the same page table in competing
 623                          * threads.
 624                          *
 625                          * We skip HATs that belong to CPUs, to make our lives
 626                          * simpler.
 627                          */
 628                         while (hat != NULL && (hat->hat_flags &
 629                             (HAT_VICTIM | HAT_SHARED | HAT_FREEING |
 630                             HAT_PCP)) != 0) {
 631                                 hat = hat->hat_next;
 632                         }
 633 
 634                         if (hat == NULL)
 635                                 break;
 636 
 637                         /*
 638                          * Mark the HAT as a stealing victim so that it is
 639                          * not freed from under us, e.g. in as_free()
 640                          */
 641                         hat->hat_flags |= HAT_VICTIM;
 642                         mutex_exit(&hat_list_lock);
 643 
 644                         /*
 645                          * Take any htables from the hat's cached "free" list.
 646                          */
 647                         hat_enter(hat);
 648                         while ((ht = hat->hat_ht_cached) != NULL &&
 649                             stolen < cnt) {
 650                                 hat->hat_ht_cached = ht->ht_next;
 651                                 ht->ht_next = list;
 652                                 list = ht;
 653                                 ++stolen;
 654                         }
 655                         hat_exit(hat);
 656 
 657                         /*
 658                          * Don't steal active htables on first pass.
 659                          */
 660                         if (pass != 0 && (stolen < cnt))
 661                                 htable_steal_active(hat, cnt, threshold,
 662                                     &stolen, &list);
 663 
 664                         /*
 665                          * do synchronous teardown for the reap case so that
 666                          * we can forget hat; at this time, hat is
 667                          * guaranteed to be around because HAT_VICTIM is set
 668                          * (see htable_free() for similar code)
 669                          */
 670                         for (ht = list; (ht) && (reap); ht = ht->ht_next) {
 671                                 if (ht->ht_hat == NULL)
 672                                         continue;
 673                                 ASSERT(ht->ht_hat == hat);
 674 #if defined(__xpv) && defined(__amd64)
 675                                 ASSERT(!(ht->ht_flags & HTABLE_COPIED));
 676                                 if (ht->ht_level == mmu.max_level) {
 677                                         ptable_free(hat->hat_user_ptable);
 678                                         hat->hat_user_ptable = PFN_INVALID;
 679                                 }
 680 #endif
 681                                 /*
 682                                  * forget the hat
 683                                  */
 684                                 ht->ht_hat = NULL;
 685                         }
 686 
 687                         mutex_enter(&hat_list_lock);
 688 
 689                         /*
 690                          * Are we finished?
 691                          */
 692                         if (stolen == cnt) {
 693                                 /*
 694                                  * Try to spread the pain of stealing,
 695                                  * move victim HAT to the end of the HAT list.
 696                                  */
 697                                 if (pass >= 1 && cnt == 1 &&
 698                                     kas.a_hat->hat_prev != hat)
 699                                         move_victim(hat);
 700                                 /*
 701                                  * We are finished
 702                                  */
 703                         }
 704 
 705                         /*
 706                          * Clear the victim flag, hat can go away now (once
 707                          * the lock is dropped)
 708                          */
 709                         if (hat->hat_flags & HAT_VICTIM) {
 710                                 ASSERT(hat != kas.a_hat);
 711                                 hat->hat_flags &= ~HAT_VICTIM;
 712                                 cv_broadcast(&hat_list_cv);
 713                         }
 714 
 715                         /* move on to the next hat */
 716                         hat = hat->hat_next;
 717                 }
 718 
 719                 mutex_exit(&hat_list_lock);
 720 
 721         }
 722         ASSERT(!MUTEX_HELD(&hat_list_lock));
 723 
 724         atomic_dec_32(&htable_dont_cache);
 725         return (list);
 726 }
 727 
 728 /*
 729  * This is invoked from kmem when the system is low on memory.  We try
 730  * to free hments, htables, and ptables to improve the memory situation.
 731  */
 732 /*ARGSUSED*/
 733 static void
 734 htable_reap(void *handle)
 735 {
 736         uint_t          reap_cnt;
 737         htable_t        *list;
 738         htable_t        *ht;
 739 
 740         HATSTAT_INC(hs_reap_attempts);
 741         if (!can_steal_post_boot)
 742                 return;
 743 
 744         /*
 745          * Try to reap 5% of the page tables bounded by a maximum of
 746          * 5% of physmem and a minimum of 10.
 747          */
 748         reap_cnt = MAX(MIN(physmem / 20, active_ptables / 20), 10);
 749 
 750         /*
 751          * Note: htable_dont_cache should be set at the time of
 752          * invoking htable_free()
 753          */
 754         atomic_inc_32(&htable_dont_cache);
 755         /*
 756          * Let htable_steal() do the work, we just call htable_free()
 757          */
 758         XPV_DISALLOW_MIGRATE();
 759         list = htable_steal(reap_cnt, B_TRUE);
 760         XPV_ALLOW_MIGRATE();
 761         while ((ht = list) != NULL) {
 762                 list = ht->ht_next;
 763                 HATSTAT_INC(hs_reaped);
 764                 htable_free(ht);
 765         }
 766         atomic_dec_32(&htable_dont_cache);
 767 
 768         /*
 769          * Free up excess reserves
 770          */
 771         htable_adjust_reserve();
 772         hment_adjust_reserve();
 773 }
 774 
 775 /*
 776  * Allocate an htable, stealing one or using the reserve if necessary
 777  */
 778 static htable_t *
 779 htable_alloc(
 780         hat_t           *hat,
 781         uintptr_t       vaddr,
 782         level_t         level,
 783         htable_t        *shared)
 784 {
 785         htable_t        *ht = NULL;
 786         uint_t          is_copied;
 787         uint_t          is_bare = 0;
 788         uint_t          need_to_zero = 1;
 789         int             kmflags = (can_steal_post_boot ? KM_NOSLEEP : KM_SLEEP);
 790 
 791         if (level < 0 || level > TOP_LEVEL(hat))
 792                 panic("htable_alloc(): level %d out of range\n", level);
 793 
 794         is_copied = (hat->hat_flags & HAT_COPIED) &&
 795             level == hat->hat_max_level;
 796         if (is_copied || shared != NULL)
 797                 is_bare = 1;
 798 
 799         /*
 800          * First reuse a cached htable from the hat_ht_cached field, this
 801          * avoids unnecessary trips through kmem/page allocators.
 802          */
 803         if (hat->hat_ht_cached != NULL && !is_bare) {
 804                 hat_enter(hat);
 805                 ht = hat->hat_ht_cached;
 806                 if (ht != NULL) {
 807                         hat->hat_ht_cached = ht->ht_next;
 808                         need_to_zero = 0;
 809                         /* XX64 ASSERT() they're all zero somehow */
 810                         ASSERT(ht->ht_pfn != PFN_INVALID);
 811                 }
 812                 hat_exit(hat);
 813         }
 814 
 815         if (ht == NULL) {
 816                 /*
 817                  * Allocate an htable, possibly refilling the reserves.
 818                  */
 819                 if (USE_HAT_RESERVES()) {
 820                         ht = htable_get_reserve();
 821                 } else {
 822                         /*
 823                          * Donate successful htable allocations to the reserve.
 824                          */
 825                         for (;;) {
 826                                 ht = kmem_cache_alloc(htable_cache, kmflags);
 827                                 if (ht == NULL)
 828                                         break;
 829                                 ht->ht_pfn = PFN_INVALID;
 830                                 if (USE_HAT_RESERVES() ||
 831                                     htable_reserve_cnt >= htable_reserve_amount)
 832                                         break;
 833                                 htable_put_reserve(ht);
 834                         }
 835                 }
 836 
 837                 /*
 838                  * allocate a page for the hardware page table if needed
 839                  */
 840                 if (ht != NULL && !is_bare) {
 841                         ht->ht_hat = hat;
 842                         ht->ht_pfn = ptable_alloc((uintptr_t)ht);
 843                         if (ht->ht_pfn == PFN_INVALID) {
 844                                 if (USE_HAT_RESERVES())
 845                                         htable_put_reserve(ht);
 846                                 else
 847                                         kmem_cache_free(htable_cache, ht);
 848                                 ht = NULL;
 849                         }
 850                 }
 851         }
 852 
 853         /*
 854          * If allocations failed, kick off a kmem_reap() and resort to
 855          * htable steal(). We may spin here if the system is very low on
 856          * memory. If the kernel itself has consumed all memory and kmem_reap()
 857          * can't free up anything, then we'll really get stuck here.
 858          * That should only happen in a system where the administrator has
 859          * misconfigured VM parameters via /etc/system.
 860          */
 861         while (ht == NULL && can_steal_post_boot) {
 862                 kmem_reap();
 863                 ht = htable_steal(1, B_FALSE);
 864                 HATSTAT_INC(hs_steals);
 865 
 866                 /*
 867                  * If we stole for a bare htable, release the pagetable page.
 868                  */
 869                 if (ht != NULL) {
 870                         if (is_bare) {
 871                                 ptable_free(ht->ht_pfn);
 872                                 ht->ht_pfn = PFN_INVALID;
 873 #if defined(__xpv) && defined(__amd64)
 874                         /*
 875                          * make stolen page table writable again in kpm
 876                          */
 877                         } else if (kpm_vbase && xen_kpm_page(ht->ht_pfn,
 878                             PT_VALID | PT_WRITABLE) < 0) {
 879                                 panic("failure making kpm r/w pfn=0x%lx",
 880                                     ht->ht_pfn);
 881 #endif
 882                         }
 883                 }
 884         }
 885 
 886         /*
 887          * All attempts to allocate or steal failed. This should only happen
 888          * if we run out of memory during boot, due perhaps to a huge
 889          * boot_archive. At this point there's no way to continue.
 890          */
 891         if (ht == NULL)
 892                 panic("htable_alloc(): couldn't steal\n");
 893 
 894 #if defined(__amd64) && defined(__xpv)
 895         /*
 896          * Under the 64-bit hypervisor, we have 2 top level page tables.
 897          * If this allocation fails, we'll resort to stealing.
 898          * We use the stolen page indirectly, by freeing the
 899          * stolen htable first.
 900          */
 901         if (level == mmu.max_level) {
 902                 for (;;) {
 903                         htable_t *stolen;
 904 
 905                         hat->hat_user_ptable = ptable_alloc((uintptr_t)ht + 1);
 906                         if (hat->hat_user_ptable != PFN_INVALID)
 907                                 break;
 908                         stolen = htable_steal(1, B_FALSE);
 909                         if (stolen == NULL)
 910                                 panic("2nd steal ptable failed\n");
 911                         htable_free(stolen);
 912                 }
 913                 block_zero_no_xmm(kpm_vbase + pfn_to_pa(hat->hat_user_ptable),
 914                     MMU_PAGESIZE);
 915         }
 916 #endif
 917 
 918         /*
 919          * Shared page tables have all entries locked and entries may not
 920          * be added or deleted.
 921          */
 922         ht->ht_flags = 0;
 923         if (shared != NULL) {
 924                 ASSERT(shared->ht_valid_cnt > 0);
 925                 ht->ht_flags |= HTABLE_SHARED_PFN;
 926                 ht->ht_pfn = shared->ht_pfn;
 927                 ht->ht_lock_cnt = 0;
 928                 ht->ht_valid_cnt = 0;                /* updated in hat_share() */
 929                 ht->ht_shares = shared;
 930                 need_to_zero = 0;
 931         } else {
 932                 ht->ht_shares = NULL;
 933                 ht->ht_lock_cnt = 0;
 934                 ht->ht_valid_cnt = 0;
 935         }
 936 
 937         /*
 938          * setup flags, etc. for copied page tables.
 939          */
 940         if (is_copied) {
 941                 ht->ht_flags |= HTABLE_COPIED;
 942                 ASSERT(ht->ht_pfn == PFN_INVALID);
 943                 need_to_zero = 0;
 944         }
 945 
 946         /*
 947          * fill in the htable
 948          */
 949         ht->ht_hat = hat;
 950         ht->ht_parent = NULL;
 951         ht->ht_vaddr = vaddr;
 952         ht->ht_level = level;
 953         ht->ht_busy = 1;
 954         ht->ht_next = NULL;
 955         ht->ht_prev = NULL;
 956 
 957         /*
 958          * Zero out any freshly allocated page table
 959          */
 960         if (need_to_zero)
 961                 x86pte_zero(ht, 0, mmu.ptes_per_table);
 962 
 963 #if defined(__amd64) && defined(__xpv)
 964         if (!is_bare && kpm_vbase) {
 965                 (void) xen_kpm_page(ht->ht_pfn, PT_VALID);
 966                 if (level == mmu.max_level)
 967                         (void) xen_kpm_page(hat->hat_user_ptable, PT_VALID);
 968         }
 969 #endif
 970 
 971         return (ht);
 972 }
 973 
 974 /*
 975  * Free up an htable, either to a hat's cached list, the reserves or
 976  * back to kmem.
 977  */
 978 static void
 979 htable_free(htable_t *ht)
 980 {
 981         hat_t *hat = ht->ht_hat;
 982 
 983         /*
 984          * If the process isn't exiting, cache the free htable in the hat
 985          * structure. We always do this for the boot time reserve. We don't
 986          * do this if the hat is exiting or we are stealing/reaping htables.
 987          */
 988         if (hat != NULL &&
 989             !(ht->ht_flags & HTABLE_SHARED_PFN) &&
 990             (use_boot_reserve ||
 991             (!(hat->hat_flags & HAT_FREEING) && !htable_dont_cache))) {
 992                 ASSERT((ht->ht_flags & HTABLE_COPIED) == 0);
 993                 ASSERT(ht->ht_pfn != PFN_INVALID);
 994                 hat_enter(hat);
 995                 ht->ht_next = hat->hat_ht_cached;
 996                 hat->hat_ht_cached = ht;
 997                 hat_exit(hat);
 998                 return;
 999         }
1000 
1001         /*
1002          * If we have a hardware page table, free it.
1003          * We don't free page tables that are accessed by sharing.
1004          */
1005         if (ht->ht_flags & HTABLE_SHARED_PFN) {
1006                 ASSERT(ht->ht_pfn != PFN_INVALID);
1007         } else if (!(ht->ht_flags & HTABLE_COPIED)) {
1008                 ptable_free(ht->ht_pfn);
1009 #if defined(__amd64) && defined(__xpv)
1010                 if (ht->ht_level == mmu.max_level && hat != NULL) {
1011                         ptable_free(hat->hat_user_ptable);
1012                         hat->hat_user_ptable = PFN_INVALID;
1013                 }
1014 #endif
1015         }
1016         ht->ht_pfn = PFN_INVALID;
1017 
1018         /*
1019          * Free it or put into reserves.
1020          */
1021         if (USE_HAT_RESERVES() || htable_reserve_cnt < htable_reserve_amount) {
1022                 htable_put_reserve(ht);
1023         } else {
1024                 kmem_cache_free(htable_cache, ht);
1025                 htable_adjust_reserve();
1026         }
1027 }
1028 
1029 
1030 /*
1031  * This is called when a hat is being destroyed or swapped out. We reap all
1032  * the remaining htables in the hat cache. If destroying all left over
1033  * htables are also destroyed.
1034  *
1035  * We also don't need to invalidate any of the PTPs nor do any demapping.
1036  */
1037 void
1038 htable_purge_hat(hat_t *hat)
1039 {
1040         htable_t *ht;
1041         int h;
1042 
1043         /*
1044          * Purge the htable cache if just reaping.
1045          */
1046         if (!(hat->hat_flags & HAT_FREEING)) {
1047                 atomic_inc_32(&htable_dont_cache);
1048                 for (;;) {
1049                         hat_enter(hat);
1050                         ht = hat->hat_ht_cached;
1051                         if (ht == NULL) {
1052                                 hat_exit(hat);
1053                                 break;
1054                         }
1055                         hat->hat_ht_cached = ht->ht_next;
1056                         hat_exit(hat);
1057                         htable_free(ht);
1058                 }
1059                 atomic_dec_32(&htable_dont_cache);
1060                 return;
1061         }
1062 
1063         /*
1064          * if freeing, no locking is needed
1065          */
1066         while ((ht = hat->hat_ht_cached) != NULL) {
1067                 hat->hat_ht_cached = ht->ht_next;
1068                 htable_free(ht);
1069         }
1070 
1071         /*
1072          * walk thru the htable hash table and free all the htables in it.
1073          */
1074         for (h = 0; h < hat->hat_num_hash; ++h) {
1075                 while ((ht = hat->hat_ht_hash[h]) != NULL) {
1076                         if (ht->ht_next)
1077                                 ht->ht_next->ht_prev = ht->ht_prev;
1078 
1079                         if (ht->ht_prev) {
1080                                 ht->ht_prev->ht_next = ht->ht_next;
1081                         } else {
1082                                 ASSERT(hat->hat_ht_hash[h] == ht);
1083                                 hat->hat_ht_hash[h] = ht->ht_next;
1084                         }
1085                         htable_free(ht);
1086                 }
1087         }
1088 }
1089 
1090 /*
1091  * Unlink an entry for a table at vaddr and level out of the existing table
1092  * one level higher. We are always holding the HASH_ENTER() when doing this.
1093  */
1094 static void
1095 unlink_ptp(htable_t *higher, htable_t *old, uintptr_t vaddr)
1096 {
1097         uint_t          entry = htable_va2entry(vaddr, higher);
1098         x86pte_t        expect = MAKEPTP(old->ht_pfn, old->ht_level);
1099         x86pte_t        found;
1100         hat_t           *hat = old->ht_hat;
1101 
1102         ASSERT(higher->ht_busy > 0);
1103         ASSERT(higher->ht_valid_cnt > 0);
1104         ASSERT(old->ht_valid_cnt == 0);
1105         found = x86pte_cas(higher, entry, expect, 0);
1106 #ifdef __xpv
1107         /*
1108          * This is weird, but Xen apparently automatically unlinks empty
1109          * pagetables from the upper page table. So allow PTP to be 0 already.
1110          */
1111         if (found != expect && found != 0)
1112 #else
1113         if (found != expect)
1114 #endif
1115                 panic("Bad PTP found=" FMT_PTE ", expected=" FMT_PTE,
1116                     found, expect);
1117 
1118         /*
1119          * When a top level PTE changes for a copied htable, we must trigger a
1120          * hat_pcp_update() on all HAT CPUs.
1121          *
1122          * If we don't need do do that, then we still have to INVLPG against an
1123          * address covered by the inner page table, as the latest processors
1124          * have TLB-like caches for non-leaf page table entries.
1125          */
1126         if (!(hat->hat_flags & HAT_FREEING)) {
1127                 hat_tlb_inval(hat, (higher->ht_flags & HTABLE_COPIED) ?
1128                     DEMAP_ALL_ADDR : old->ht_vaddr);
1129         }
1130 
1131         HTABLE_DEC(higher->ht_valid_cnt);
1132 }
1133 
1134 /*
1135  * Link an entry for a new table at vaddr and level into the existing table
1136  * one level higher. We are always holding the HASH_ENTER() when doing this.
1137  */
1138 static void
1139 link_ptp(htable_t *higher, htable_t *new, uintptr_t vaddr)
1140 {
1141         uint_t          entry = htable_va2entry(vaddr, higher);
1142         x86pte_t        newptp = MAKEPTP(new->ht_pfn, new->ht_level);
1143         x86pte_t        found;
1144 
1145         ASSERT(higher->ht_busy > 0);
1146 
1147         ASSERT(new->ht_level != mmu.max_level);
1148 
1149         HTABLE_INC(higher->ht_valid_cnt);
1150 
1151         found = x86pte_cas(higher, entry, 0, newptp);
1152         if ((found & ~PT_REF) != 0)
1153                 panic("HAT: ptp not 0, found=" FMT_PTE, found);
1154 
1155         /*
1156          * When a top level PTE changes for a copied htable, we must trigger a
1157          * hat_pcp_update() on all HAT CPUs.
1158          *
1159          * We also need to do this for the kernel hat on PAE 32 bit kernel.
1160          */
1161         if (
1162 #ifdef __i386
1163             (higher->ht_hat == kas.a_hat &&
1164             higher->ht_level == higher->ht_hat->hat_max_level) ||
1165 #endif
1166             (higher->ht_flags & HTABLE_COPIED))
1167                 hat_tlb_inval(higher->ht_hat, DEMAP_ALL_ADDR);
1168 }
1169 
1170 /*
1171  * Release of hold on an htable. If this is the last use and the pagetable
1172  * is empty we may want to free it, then recursively look at the pagetable
1173  * above it. The recursion is handled by the outer while() loop.
1174  *
1175  * On the metal, during process exit, we don't bother unlinking the tables from
1176  * upper level pagetables. They are instead handled in bulk by hat_free_end().
1177  * We can't do this on the hypervisor as we need the page table to be
1178  * implicitly unpinnned before it goes to the free page lists. This can't
1179  * happen unless we fully unlink it from the page table hierarchy.
1180  */
1181 void
1182 htable_release(htable_t *ht)
1183 {
1184         uint_t          hashval;
1185         htable_t        *shared;
1186         htable_t        *higher;
1187         hat_t           *hat;
1188         uintptr_t       va;
1189         level_t         level;
1190 
1191         while (ht != NULL) {
1192                 shared = NULL;
1193                 for (;;) {
1194                         hat = ht->ht_hat;
1195                         va = ht->ht_vaddr;
1196                         level = ht->ht_level;
1197                         hashval = HTABLE_HASH(hat, va, level);
1198 
1199                         /*
1200                          * The common case is that this isn't the last use of
1201                          * an htable so we don't want to free the htable.
1202                          */
1203                         HTABLE_ENTER(hashval);
1204                         ASSERT(ht->ht_valid_cnt >= 0);
1205                         ASSERT(ht->ht_busy > 0);
1206                         if (ht->ht_valid_cnt > 0)
1207                                 break;
1208                         if (ht->ht_busy > 1)
1209                                 break;
1210                         ASSERT(ht->ht_lock_cnt == 0);
1211 
1212 #if !defined(__xpv)
1213                         /*
1214                          * we always release empty shared htables
1215                          */
1216                         if (!(ht->ht_flags & HTABLE_SHARED_PFN)) {
1217 
1218                                 /*
1219                                  * don't release if in address space tear down
1220                                  */
1221                                 if (hat->hat_flags & HAT_FREEING)
1222                                         break;
1223 
1224                                 /*
1225                                  * At and above max_page_level, free if it's for
1226                                  * a boot-time kernel mapping below kernelbase.
1227                                  */
1228                                 if (level >= mmu.max_page_level &&
1229                                     (hat != kas.a_hat || va >= kernelbase))
1230                                         break;
1231                         }
1232 #endif /* __xpv */
1233 
1234                         /*
1235                          * Remember if we destroy an htable that shares its PFN
1236                          * from elsewhere.
1237                          */
1238                         if (ht->ht_flags & HTABLE_SHARED_PFN) {
1239                                 ASSERT(shared == NULL);
1240                                 shared = ht->ht_shares;
1241                                 HATSTAT_INC(hs_htable_unshared);
1242                         }
1243 
1244                         /*
1245                          * Handle release of a table and freeing the htable_t.
1246                          * Unlink it from the table higher (ie. ht_parent).
1247                          */
1248                         higher = ht->ht_parent;
1249                         ASSERT(higher != NULL);
1250 
1251                         /*
1252                          * Unlink the pagetable.
1253                          */
1254                         unlink_ptp(higher, ht, va);
1255 
1256                         /*
1257                          * remove this htable from its hash list
1258                          */
1259                         if (ht->ht_next)
1260                                 ht->ht_next->ht_prev = ht->ht_prev;
1261 
1262                         if (ht->ht_prev) {
1263                                 ht->ht_prev->ht_next = ht->ht_next;
1264                         } else {
1265                                 ASSERT(hat->hat_ht_hash[hashval] == ht);
1266                                 hat->hat_ht_hash[hashval] = ht->ht_next;
1267                         }
1268                         HTABLE_EXIT(hashval);
1269                         htable_free(ht);
1270                         ht = higher;
1271                 }
1272 
1273                 ASSERT(ht->ht_busy >= 1);
1274                 --ht->ht_busy;
1275                 HTABLE_EXIT(hashval);
1276 
1277                 /*
1278                  * If we released a shared htable, do a release on the htable
1279                  * from which it shared
1280                  */
1281                 ht = shared;
1282         }
1283 }
1284 
1285 /*
1286  * Find the htable for the pagetable at the given level for the given address.
1287  * If found acquires a hold that eventually needs to be htable_release()d
1288  */
1289 htable_t *
1290 htable_lookup(hat_t *hat, uintptr_t vaddr, level_t level)
1291 {
1292         uintptr_t       base;
1293         uint_t          hashval;
1294         htable_t        *ht = NULL;
1295 
1296         ASSERT(level >= 0);
1297         ASSERT(level <= TOP_LEVEL(hat));
1298 
1299         if (level == TOP_LEVEL(hat)) {
1300 #if defined(__amd64)
1301                 /*
1302                  * 32 bit address spaces on 64 bit kernels need to check
1303                  * for overflow of the 32 bit address space
1304                  */
1305                 if ((hat->hat_flags & HAT_COPIED_32) &&
1306                     vaddr >= ((uint64_t)1 << 32))
1307                         return (NULL);
1308 #endif
1309                 base = 0;
1310         } else {
1311                 base = vaddr & LEVEL_MASK(level + 1);
1312         }
1313 
1314         hashval = HTABLE_HASH(hat, base, level);
1315         HTABLE_ENTER(hashval);
1316         for (ht = hat->hat_ht_hash[hashval]; ht; ht = ht->ht_next) {
1317                 if (ht->ht_hat == hat &&
1318                     ht->ht_vaddr == base &&
1319                     ht->ht_level == level)
1320                         break;
1321         }
1322         if (ht)
1323                 ++ht->ht_busy;
1324 
1325         HTABLE_EXIT(hashval);
1326         return (ht);
1327 }
1328 
1329 /*
1330  * Acquires a hold on a known htable (from a locked hment entry).
1331  */
1332 void
1333 htable_acquire(htable_t *ht)
1334 {
1335         hat_t           *hat = ht->ht_hat;
1336         level_t         level = ht->ht_level;
1337         uintptr_t       base = ht->ht_vaddr;
1338         uint_t          hashval = HTABLE_HASH(hat, base, level);
1339 
1340         HTABLE_ENTER(hashval);
1341 #ifdef DEBUG
1342         /*
1343          * make sure the htable is there
1344          */
1345         {
1346                 htable_t        *h;
1347 
1348                 for (h = hat->hat_ht_hash[hashval];
1349                     h && h != ht;
1350                     h = h->ht_next)
1351                         ;
1352                 ASSERT(h == ht);
1353         }
1354 #endif /* DEBUG */
1355         ++ht->ht_busy;
1356         HTABLE_EXIT(hashval);
1357 }
1358 
1359 /*
1360  * Find the htable for the pagetable at the given level for the given address.
1361  * If found acquires a hold that eventually needs to be htable_release()d
1362  * If not found the table is created.
1363  *
1364  * Since we can't hold a hash table mutex during allocation, we have to
1365  * drop it and redo the search on a create. Then we may have to free the newly
1366  * allocated htable if another thread raced in and created it ahead of us.
1367  */
1368 htable_t *
1369 htable_create(
1370         hat_t           *hat,
1371         uintptr_t       vaddr,
1372         level_t         level,
1373         htable_t        *shared)
1374 {
1375         uint_t          h;
1376         level_t         l;
1377         uintptr_t       base;
1378         htable_t        *ht;
1379         htable_t        *higher = NULL;
1380         htable_t        *new = NULL;
1381 
1382         if (level < 0 || level > TOP_LEVEL(hat))
1383                 panic("htable_create(): level %d out of range\n", level);
1384 
1385         /*
1386          * Create the page tables in top down order.
1387          */
1388         for (l = TOP_LEVEL(hat); l >= level; --l) {
1389                 new = NULL;
1390                 if (l == TOP_LEVEL(hat))
1391                         base = 0;
1392                 else
1393                         base = vaddr & LEVEL_MASK(l + 1);
1394 
1395                 h = HTABLE_HASH(hat, base, l);
1396 try_again:
1397                 /*
1398                  * look up the htable at this level
1399                  */
1400                 HTABLE_ENTER(h);
1401                 if (l == TOP_LEVEL(hat)) {
1402                         ht = hat->hat_htable;
1403                 } else {
1404                         for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
1405                                 ASSERT(ht->ht_hat == hat);
1406                                 if (ht->ht_vaddr == base &&
1407                                     ht->ht_level == l)
1408                                         break;
1409                         }
1410                 }
1411 
1412                 /*
1413                  * if we found the htable, increment its busy cnt
1414                  * and if we had allocated a new htable, free it.
1415                  */
1416                 if (ht != NULL) {
1417                         /*
1418                          * If we find a pre-existing shared table, it must
1419                          * share from the same place.
1420                          */
1421                         if (l == level && shared && ht->ht_shares &&
1422                             ht->ht_shares != shared) {
1423                                 panic("htable shared from wrong place "
1424                                     "found htable=%p shared=%p",
1425                                     (void *)ht, (void *)shared);
1426                         }
1427                         ++ht->ht_busy;
1428                         HTABLE_EXIT(h);
1429                         if (new)
1430                                 htable_free(new);
1431                         if (higher != NULL)
1432                                 htable_release(higher);
1433                         higher = ht;
1434 
1435                 /*
1436                  * if we didn't find it on the first search
1437                  * allocate a new one and search again
1438                  */
1439                 } else if (new == NULL) {
1440                         HTABLE_EXIT(h);
1441                         new = htable_alloc(hat, base, l,
1442                             l == level ? shared : NULL);
1443                         goto try_again;
1444 
1445                 /*
1446                  * 2nd search and still not there, use "new" table
1447                  * Link new table into higher, when not at top level.
1448                  */
1449                 } else {
1450                         ht = new;
1451                         if (higher != NULL) {
1452                                 link_ptp(higher, ht, base);
1453                                 ht->ht_parent = higher;
1454                         }
1455                         ht->ht_next = hat->hat_ht_hash[h];
1456                         ASSERT(ht->ht_prev == NULL);
1457                         if (hat->hat_ht_hash[h])
1458                                 hat->hat_ht_hash[h]->ht_prev = ht;
1459                         hat->hat_ht_hash[h] = ht;
1460                         HTABLE_EXIT(h);
1461 
1462                         /*
1463                          * Note we don't do htable_release(higher).
1464                          * That happens recursively when "new" is removed by
1465                          * htable_release() or htable_steal().
1466                          */
1467                         higher = ht;
1468 
1469                         /*
1470                          * If we just created a new shared page table we
1471                          * increment the shared htable's busy count, so that
1472                          * it can't be the victim of a steal even if it's empty.
1473                          */
1474                         if (l == level && shared) {
1475                                 (void) htable_lookup(shared->ht_hat,
1476                                     shared->ht_vaddr, shared->ht_level);
1477                                 HATSTAT_INC(hs_htable_shared);
1478                         }
1479                 }
1480         }
1481 
1482         return (ht);
1483 }
1484 
1485 /*
1486  * Inherit initial pagetables from the boot program. On the 64-bit
1487  * hypervisor we also temporarily mark the p_index field of page table
1488  * pages, so we know not to try making them writable in seg_kpm.
1489  */
1490 void
1491 htable_attach(
1492         hat_t *hat,
1493         uintptr_t base,
1494         level_t level,
1495         htable_t *parent,
1496         pfn_t pfn)
1497 {
1498         htable_t        *ht;
1499         uint_t          h;
1500         uint_t          i;
1501         x86pte_t        pte;
1502         x86pte_t        *ptep;
1503         page_t          *pp;
1504         extern page_t   *boot_claim_page(pfn_t);
1505 
1506         ht = htable_get_reserve();
1507         if (level == mmu.max_level)
1508                 kas.a_hat->hat_htable = ht;
1509         ht->ht_hat = hat;
1510         ht->ht_parent = parent;
1511         ht->ht_vaddr = base;
1512         ht->ht_level = level;
1513         ht->ht_busy = 1;
1514         ht->ht_next = NULL;
1515         ht->ht_prev = NULL;
1516         ht->ht_flags = 0;
1517         ht->ht_pfn = pfn;
1518         ht->ht_lock_cnt = 0;
1519         ht->ht_valid_cnt = 0;
1520         if (parent != NULL)
1521                 ++parent->ht_busy;
1522 
1523         h = HTABLE_HASH(hat, base, level);
1524         HTABLE_ENTER(h);
1525         ht->ht_next = hat->hat_ht_hash[h];
1526         ASSERT(ht->ht_prev == NULL);
1527         if (hat->hat_ht_hash[h])
1528                 hat->hat_ht_hash[h]->ht_prev = ht;
1529         hat->hat_ht_hash[h] = ht;
1530         HTABLE_EXIT(h);
1531 
1532         /*
1533          * make sure the page table physical page is not FREE
1534          */
1535         if (page_resv(1, KM_NOSLEEP) == 0)
1536                 panic("page_resv() failed in ptable alloc");
1537 
1538         pp = boot_claim_page(pfn);
1539         ASSERT(pp != NULL);
1540 
1541         /*
1542          * Page table pages that were allocated by dboot or
1543          * in very early startup didn't go through boot_mapin()
1544          * and so won't have vnode/offsets. Fix that here.
1545          */
1546         if (pp->p_vnode == NULL) {
1547                 /* match offset calculation in page_get_physical() */
1548                 u_offset_t offset = (uintptr_t)ht;
1549                 if (offset > kernelbase)
1550                         offset -= kernelbase;
1551                 offset <<= MMU_PAGESHIFT;
1552 #if defined(__amd64)
1553                 offset += mmu.hole_start;       /* something in VA hole */
1554 #else
1555                 offset += 1ULL << 40;             /* something > 4 Gig */
1556 #endif
1557                 ASSERT(page_exists(&kvp, offset) == NULL);
1558                 (void) page_hashin(pp, &kvp, offset, NULL);
1559         }
1560         page_downgrade(pp);
1561 #if defined(__xpv) && defined(__amd64)
1562         /*
1563          * Record in the page_t that is a pagetable for segkpm setup.
1564          */
1565         if (kpm_vbase)
1566                 pp->p_index = 1;
1567 #endif
1568 
1569         /*
1570          * Count valid mappings and recursively attach lower level pagetables.
1571          */
1572         ptep = kbm_remap_window(pfn_to_pa(pfn), 0);
1573         for (i = 0; i < HTABLE_NUM_PTES(ht); ++i) {
1574                 if (mmu.pae_hat)
1575                         pte = ptep[i];
1576                 else
1577                         pte = ((x86pte32_t *)ptep)[i];
1578                 if (!IN_HYPERVISOR_VA(base) && PTE_ISVALID(pte)) {
1579                         ++ht->ht_valid_cnt;
1580                         if (!PTE_ISPAGE(pte, level)) {
1581                                 htable_attach(hat, base, level - 1,
1582                                     ht, PTE2PFN(pte, level));
1583                                 ptep = kbm_remap_window(pfn_to_pa(pfn), 0);
1584                         }
1585                 }
1586                 base += LEVEL_SIZE(level);
1587                 if (base == mmu.hole_start)
1588                         base = (mmu.hole_end + MMU_PAGEOFFSET) & MMU_PAGEMASK;
1589         }
1590 
1591         /*
1592          * As long as all the mappings we had were below kernel base
1593          * we can release the htable.
1594          */
1595         if (base < kernelbase)
1596                 htable_release(ht);
1597 }
1598 
1599 /*
1600  * Walk through a given htable looking for the first valid entry.  This
1601  * routine takes both a starting and ending address.  The starting address
1602  * is required to be within the htable provided by the caller, but there is
1603  * no such restriction on the ending address.
1604  *
1605  * If the routine finds a valid entry in the htable (at or beyond the
1606  * starting address), the PTE (and its address) will be returned.
1607  * This PTE may correspond to either a page or a pagetable - it is the
1608  * caller's responsibility to determine which.  If no valid entry is
1609  * found, 0 (and invalid PTE) and the next unexamined address will be
1610  * returned.
1611  *
1612  * The loop has been carefully coded for optimization.
1613  */
1614 static x86pte_t
1615 htable_scan(htable_t *ht, uintptr_t *vap, uintptr_t eaddr)
1616 {
1617         uint_t e;
1618         x86pte_t found_pte = (x86pte_t)0;
1619         caddr_t pte_ptr;
1620         caddr_t end_pte_ptr;
1621         int l = ht->ht_level;
1622         uintptr_t va = *vap & LEVEL_MASK(l);
1623         size_t pgsize = LEVEL_SIZE(l);
1624 
1625         ASSERT(va >= ht->ht_vaddr);
1626         ASSERT(va <= HTABLE_LAST_PAGE(ht));
1627 
1628         /*
1629          * Compute the starting index and ending virtual address
1630          */
1631         e = htable_va2entry(va, ht);
1632 
1633         /*
1634          * The following page table scan code knows that the valid
1635          * bit of a PTE is in the lowest byte AND that x86 is little endian!!
1636          */
1637         pte_ptr = (caddr_t)x86pte_access_pagetable(ht, 0);
1638         end_pte_ptr = (caddr_t)PT_INDEX_PTR(pte_ptr, HTABLE_NUM_PTES(ht));
1639         pte_ptr = (caddr_t)PT_INDEX_PTR((x86pte_t *)pte_ptr, e);
1640         while (!PTE_ISVALID(*pte_ptr)) {
1641                 va += pgsize;
1642                 if (va >= eaddr)
1643                         break;
1644                 pte_ptr += mmu.pte_size;
1645                 ASSERT(pte_ptr <= end_pte_ptr);
1646                 if (pte_ptr == end_pte_ptr)
1647                         break;
1648         }
1649 
1650         /*
1651          * if we found a valid PTE, load the entire PTE
1652          */
1653         if (va < eaddr && pte_ptr != end_pte_ptr)
1654                 found_pte = GET_PTE((x86pte_t *)pte_ptr);
1655         x86pte_release_pagetable(ht);
1656 
1657 #if defined(__amd64)
1658         /*
1659          * deal with VA hole on amd64
1660          */
1661         if (l == mmu.max_level && va >= mmu.hole_start && va <= mmu.hole_end)
1662                 va = mmu.hole_end + va - mmu.hole_start;
1663 #endif /* __amd64 */
1664 
1665         *vap = va;
1666         return (found_pte);
1667 }
1668 
1669 /*
1670  * Find the address and htable for the first populated translation at or
1671  * above the given virtual address.  The caller may also specify an upper
1672  * limit to the address range to search.  Uses level information to quickly
1673  * skip unpopulated sections of virtual address spaces.
1674  *
1675  * If not found returns NULL. When found, returns the htable and virt addr
1676  * and has a hold on the htable.
1677  */
1678 x86pte_t
1679 htable_walk(
1680         struct hat *hat,
1681         htable_t **htp,
1682         uintptr_t *vaddr,
1683         uintptr_t eaddr)
1684 {
1685         uintptr_t va = *vaddr;
1686         htable_t *ht;
1687         htable_t *prev = *htp;
1688         level_t l;
1689         level_t max_mapped_level;
1690         x86pte_t pte;
1691 
1692         ASSERT(eaddr > va);
1693 
1694         /*
1695          * If this is a user address, then we know we need not look beyond
1696          * kernelbase.
1697          */
1698         ASSERT(hat == kas.a_hat || eaddr <= kernelbase ||
1699             eaddr == HTABLE_WALK_TO_END);
1700         if (hat != kas.a_hat && eaddr == HTABLE_WALK_TO_END)
1701                 eaddr = kernelbase;
1702 
1703         /*
1704          * If we're coming in with a previous page table, search it first
1705          * without doing an htable_lookup(), this should be frequent.
1706          */
1707         if (prev) {
1708                 ASSERT(prev->ht_busy > 0);
1709                 ASSERT(prev->ht_vaddr <= va);
1710                 l = prev->ht_level;
1711                 if (va <= HTABLE_LAST_PAGE(prev)) {
1712                         pte = htable_scan(prev, &va, eaddr);
1713 
1714                         if (PTE_ISPAGE(pte, l)) {
1715                                 *vaddr = va;
1716                                 *htp = prev;
1717                                 return (pte);
1718                         }
1719                 }
1720 
1721                 /*
1722                  * We found nothing in the htable provided by the caller,
1723                  * so fall through and do the full search
1724                  */
1725                 htable_release(prev);
1726         }
1727 
1728         /*
1729          * Find the level of the largest pagesize used by this HAT.
1730          */
1731         if (hat->hat_ism_pgcnt > 0) {
1732                 max_mapped_level = mmu.umax_page_level;
1733         } else {
1734                 max_mapped_level = 0;
1735                 for (l = 1; l <= mmu.max_page_level; ++l)
1736                         if (hat->hat_pages_mapped[l] != 0)
1737                                 max_mapped_level = l;
1738         }
1739 
1740         while (va < eaddr && va >= *vaddr) {
1741                 /*
1742                  *  Find lowest table with any entry for given address.
1743                  */
1744                 for (l = 0; l <= TOP_LEVEL(hat); ++l) {
1745                         ht = htable_lookup(hat, va, l);
1746                         if (ht != NULL) {
1747                                 pte = htable_scan(ht, &va, eaddr);
1748                                 if (PTE_ISPAGE(pte, l)) {
1749                                         VERIFY(!IN_VA_HOLE(va));
1750                                         *vaddr = va;
1751                                         *htp = ht;
1752                                         return (pte);
1753                                 }
1754                                 htable_release(ht);
1755                                 break;
1756                         }
1757 
1758                         /*
1759                          * No htable at this level for the address. If there
1760                          * is no larger page size that could cover it, we can
1761                          * skip right to the start of the next page table.
1762                          */
1763                         ASSERT(l < TOP_LEVEL(hat));
1764                         if (l >= max_mapped_level) {
1765                                 va = NEXT_ENTRY_VA(va, l + 1);
1766                                 if (va >= eaddr)
1767                                         break;
1768                         }
1769                 }
1770         }
1771 
1772         *vaddr = 0;
1773         *htp = NULL;
1774         return (0);
1775 }
1776 
1777 /*
1778  * Find the htable and page table entry index of the given virtual address
1779  * with pagesize at or below given level.
1780  * If not found returns NULL. When found, returns the htable, sets
1781  * entry, and has a hold on the htable.
1782  */
1783 htable_t *
1784 htable_getpte(
1785         struct hat *hat,
1786         uintptr_t vaddr,
1787         uint_t *entry,
1788         x86pte_t *pte,
1789         level_t level)
1790 {
1791         htable_t        *ht;
1792         level_t         l;
1793         uint_t          e;
1794 
1795         ASSERT(level <= mmu.max_page_level);
1796 
1797         for (l = 0; l <= level; ++l) {
1798                 ht = htable_lookup(hat, vaddr, l);
1799                 if (ht == NULL)
1800                         continue;
1801                 e = htable_va2entry(vaddr, ht);
1802                 if (entry != NULL)
1803                         *entry = e;
1804                 if (pte != NULL)
1805                         *pte = x86pte_get(ht, e);
1806                 return (ht);
1807         }
1808         return (NULL);
1809 }
1810 
1811 /*
1812  * Find the htable and page table entry index of the given virtual address.
1813  * There must be a valid page mapped at the given address.
1814  * If not found returns NULL. When found, returns the htable, sets
1815  * entry, and has a hold on the htable.
1816  */
1817 htable_t *
1818 htable_getpage(struct hat *hat, uintptr_t vaddr, uint_t *entry)
1819 {
1820         htable_t        *ht;
1821         uint_t          e;
1822         x86pte_t        pte;
1823 
1824         ht = htable_getpte(hat, vaddr, &e, &pte, mmu.max_page_level);
1825         if (ht == NULL)
1826                 return (NULL);
1827 
1828         if (entry)
1829                 *entry = e;
1830 
1831         if (PTE_ISPAGE(pte, ht->ht_level))
1832                 return (ht);
1833         htable_release(ht);
1834         return (NULL);
1835 }
1836 
1837 
1838 void
1839 htable_init()
1840 {
1841         /*
1842          * To save on kernel VA usage, we avoid debug information in 32 bit
1843          * kernels.
1844          */
1845 #if defined(__amd64)
1846         int     kmem_flags = KMC_NOHASH;
1847 #elif defined(__i386)
1848         int     kmem_flags = KMC_NOHASH | KMC_NODEBUG;
1849 #endif
1850 
1851         /*
1852          * initialize kmem caches
1853          */
1854         htable_cache = kmem_cache_create("htable_t",
1855             sizeof (htable_t), 0, NULL, NULL,
1856             htable_reap, NULL, hat_memload_arena, kmem_flags);
1857 }
1858 
1859 /*
1860  * get the pte index for the virtual address in the given htable's pagetable
1861  */
1862 uint_t
1863 htable_va2entry(uintptr_t va, htable_t *ht)
1864 {
1865         level_t l = ht->ht_level;
1866 
1867         ASSERT(va >= ht->ht_vaddr);
1868         ASSERT(va <= HTABLE_LAST_PAGE(ht));
1869         return ((va >> LEVEL_SHIFT(l)) & (HTABLE_NUM_PTES(ht) - 1));
1870 }
1871 
1872 /*
1873  * Given an htable and the index of a pte in it, return the virtual address
1874  * of the page.
1875  */
1876 uintptr_t
1877 htable_e2va(htable_t *ht, uint_t entry)
1878 {
1879         level_t l = ht->ht_level;
1880         uintptr_t va;
1881 
1882         ASSERT(entry < HTABLE_NUM_PTES(ht));
1883         va = ht->ht_vaddr + ((uintptr_t)entry << LEVEL_SHIFT(l));
1884 
1885         /*
1886          * Need to skip over any VA hole in top level table
1887          */
1888 #if defined(__amd64)
1889         if (ht->ht_level == mmu.max_level && va >= mmu.hole_start)
1890                 va += ((mmu.hole_end - mmu.hole_start) + 1);
1891 #endif
1892 
1893         return (va);
1894 }
1895 
1896 /*
1897  * The code uses compare and swap instructions to read/write PTE's to
1898  * avoid atomicity problems, since PTEs can be 8 bytes on 32 bit systems.
1899  * will naturally be atomic.
1900  *
1901  * The combination of using kpreempt_disable()/_enable() and the hci_mutex
1902  * are used to ensure that an interrupt won't overwrite a temporary mapping
1903  * while it's in use. If an interrupt thread tries to access a PTE, it will
1904  * yield briefly back to the pinned thread which holds the cpu's hci_mutex.
1905  */
1906 void
1907 x86pte_cpu_init(cpu_t *cpu)
1908 {
1909         struct hat_cpu_info *hci;
1910 
1911         hci = kmem_zalloc(sizeof (*hci), KM_SLEEP);
1912         mutex_init(&hci->hci_mutex, NULL, MUTEX_DEFAULT, NULL);
1913         cpu->cpu_hat_info = hci;
1914 }
1915 
1916 void
1917 x86pte_cpu_fini(cpu_t *cpu)
1918 {
1919         struct hat_cpu_info *hci = cpu->cpu_hat_info;
1920 
1921         kmem_free(hci, sizeof (*hci));
1922         cpu->cpu_hat_info = NULL;
1923 }
1924 
1925 #ifdef __i386
1926 /*
1927  * On 32 bit kernels, loading a 64 bit PTE is a little tricky
1928  */
1929 x86pte_t
1930 get_pte64(x86pte_t *ptr)
1931 {
1932         volatile uint32_t *p = (uint32_t *)ptr;
1933         x86pte_t t;
1934 
1935         ASSERT(mmu.pae_hat != 0);
1936         for (;;) {
1937                 t = p[0];
1938                 t |= (uint64_t)p[1] << 32;
1939                 if ((t & 0xffffffff) == p[0])
1940                         return (t);
1941         }
1942 }
1943 #endif /* __i386 */
1944 
1945 /*
1946  * Disable preemption and establish a mapping to the pagetable with the
1947  * given pfn. This is optimized for there case where it's the same
1948  * pfn as we last used referenced from this CPU.
1949  */
1950 static x86pte_t *
1951 x86pte_access_pagetable(htable_t *ht, uint_t index)
1952 {
1953         /*
1954          * HTABLE_COPIED pagetables are contained in the hat_t
1955          */
1956         if (ht->ht_flags & HTABLE_COPIED) {
1957                 ASSERT3U(index, <, ht->ht_hat->hat_num_copied);
1958                 return (PT_INDEX_PTR(ht->ht_hat->hat_copied_ptes, index));
1959         }
1960         return (x86pte_mapin(ht->ht_pfn, index, ht));
1961 }
1962 
1963 /*
1964  * map the given pfn into the page table window.
1965  */
1966 /*ARGSUSED*/
1967 x86pte_t *
1968 x86pte_mapin(pfn_t pfn, uint_t index, htable_t *ht)
1969 {
1970         x86pte_t *pteptr;
1971         x86pte_t pte = 0;
1972         x86pte_t newpte;
1973         int x;
1974 
1975         ASSERT(pfn != PFN_INVALID);
1976 
1977         if (!khat_running) {
1978                 caddr_t va = kbm_remap_window(pfn_to_pa(pfn), 1);
1979                 return (PT_INDEX_PTR(va, index));
1980         }
1981 
1982         /*
1983          * If kpm is available, use it.
1984          */
1985         if (kpm_vbase)
1986                 return (PT_INDEX_PTR(hat_kpm_pfn2va(pfn), index));
1987 
1988         /*
1989          * Disable preemption and grab the CPU's hci_mutex
1990          */
1991         kpreempt_disable();
1992         ASSERT(CPU->cpu_hat_info != NULL);
1993         mutex_enter(&CPU->cpu_hat_info->hci_mutex);
1994         x = PWIN_TABLE(CPU->cpu_id);
1995         pteptr = (x86pte_t *)PWIN_PTE_VA(x);
1996 #ifndef __xpv
1997         if (mmu.pae_hat)
1998                 pte = *pteptr;
1999         else
2000                 pte = *(x86pte32_t *)pteptr;
2001 #endif
2002 
2003         newpte = MAKEPTE(pfn, 0) | mmu.pt_global | mmu.pt_nx;
2004 
2005         /*
2006          * For hardware we can use a writable mapping.
2007          */
2008 #ifdef __xpv
2009         if (IN_XPV_PANIC())
2010 #endif
2011                 newpte |= PT_WRITABLE;
2012 
2013         if (!PTE_EQUIV(newpte, pte)) {
2014 
2015 #ifdef __xpv
2016                 if (!IN_XPV_PANIC()) {
2017                         xen_map(newpte, PWIN_VA(x));
2018                 } else
2019 #endif
2020                 {
2021                         XPV_ALLOW_PAGETABLE_UPDATES();
2022                         if (mmu.pae_hat)
2023                                 *pteptr = newpte;
2024                         else
2025                                 *(x86pte32_t *)pteptr = newpte;
2026                         XPV_DISALLOW_PAGETABLE_UPDATES();
2027                         mmu_tlbflush_entry((caddr_t)(PWIN_VA(x)));
2028                 }
2029         }
2030         return (PT_INDEX_PTR(PWIN_VA(x), index));
2031 }
2032 
2033 /*
2034  * Release access to a page table.
2035  */
2036 static void
2037 x86pte_release_pagetable(htable_t *ht)
2038 {
2039         if (ht->ht_flags & HTABLE_COPIED)
2040                 return;
2041 
2042         x86pte_mapout();
2043 }
2044 
2045 void
2046 x86pte_mapout(void)
2047 {
2048         if (kpm_vbase != NULL || !khat_running)
2049                 return;
2050 
2051         /*
2052          * Drop the CPU's hci_mutex and restore preemption.
2053          */
2054 #ifdef __xpv
2055         if (!IN_XPV_PANIC()) {
2056                 uintptr_t va;
2057 
2058                 /*
2059                  * We need to always clear the mapping in case a page
2060                  * that was once a page table page is ballooned out.
2061                  */
2062                 va = (uintptr_t)PWIN_VA(PWIN_TABLE(CPU->cpu_id));
2063                 (void) HYPERVISOR_update_va_mapping(va, 0,
2064                     UVMF_INVLPG | UVMF_LOCAL);
2065         }
2066 #endif
2067         mutex_exit(&CPU->cpu_hat_info->hci_mutex);
2068         kpreempt_enable();
2069 }
2070 
2071 /*
2072  * Atomic retrieval of a pagetable entry
2073  */
2074 x86pte_t
2075 x86pte_get(htable_t *ht, uint_t entry)
2076 {
2077         x86pte_t        pte;
2078         x86pte_t        *ptep;
2079 
2080         /*
2081          * Be careful that loading PAE entries in 32 bit kernel is atomic.
2082          */
2083         ASSERT(entry < mmu.ptes_per_table);
2084         ptep = x86pte_access_pagetable(ht, entry);
2085         pte = GET_PTE(ptep);
2086         x86pte_release_pagetable(ht);
2087         return (pte);
2088 }
2089 
2090 /*
2091  * Atomic unconditional set of a page table entry, it returns the previous
2092  * value. For pre-existing mappings if the PFN changes, then we don't care
2093  * about the old pte's REF / MOD bits. If the PFN remains the same, we leave
2094  * the MOD/REF bits unchanged.
2095  *
2096  * If asked to overwrite a link to a lower page table with a large page
2097  * mapping, this routine returns the special value of LPAGE_ERROR. This
2098  * allows the upper HAT layers to retry with a smaller mapping size.
2099  */
2100 x86pte_t
2101 x86pte_set(htable_t *ht, uint_t entry, x86pte_t new, void *ptr)
2102 {
2103         x86pte_t        old;
2104         x86pte_t        prev;
2105         x86pte_t        *ptep;
2106         level_t         l = ht->ht_level;
2107         x86pte_t        pfn_mask = (l != 0) ? PT_PADDR_LGPG : PT_PADDR;
2108         x86pte_t        n;
2109         uintptr_t       addr = htable_e2va(ht, entry);
2110         hat_t           *hat = ht->ht_hat;
2111 
2112         ASSERT(new != 0); /* don't use to invalidate a PTE, see x86pte_update */
2113         ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
2114         if (ptr == NULL)
2115                 ptep = x86pte_access_pagetable(ht, entry);
2116         else
2117                 ptep = ptr;
2118 
2119         /*
2120          * Install the new PTE. If remapping the same PFN, then
2121          * copy existing REF/MOD bits to new mapping.
2122          */
2123         do {
2124                 prev = GET_PTE(ptep);
2125                 n = new;
2126                 if (PTE_ISVALID(n) && (prev & pfn_mask) == (new & pfn_mask))
2127                         n |= prev & (PT_REF | PT_MOD);
2128 
2129                 /*
2130                  * Another thread may have installed this mapping already,
2131                  * flush the local TLB and be done.
2132                  */
2133                 if (prev == n) {
2134                         old = new;
2135 #ifdef __xpv
2136                         if (!IN_XPV_PANIC())
2137                                 xen_flush_va((caddr_t)addr);
2138                         else
2139 #endif
2140                                 mmu_tlbflush_entry((caddr_t)addr);
2141                         goto done;
2142                 }
2143 
2144                 /*
2145                  * Detect if we have a collision of installing a large
2146                  * page mapping where there already is a lower page table.
2147                  */
2148                 if (l > 0 && (prev & PT_VALID) && !(prev & PT_PAGESIZE)) {
2149                         old = LPAGE_ERROR;
2150                         goto done;
2151                 }
2152 
2153                 XPV_ALLOW_PAGETABLE_UPDATES();
2154                 old = CAS_PTE(ptep, prev, n);
2155                 XPV_DISALLOW_PAGETABLE_UPDATES();
2156         } while (old != prev);
2157 
2158         /*
2159          * Do a TLB demap if needed, ie. the old pte was valid.
2160          *
2161          * Note that a stale TLB writeback to the PTE here either can't happen
2162          * or doesn't matter. The PFN can only change for NOSYNC|NOCONSIST
2163          * mappings, but they were created with REF and MOD already set, so
2164          * no stale writeback will happen.
2165          *
2166          * Segmap is the only place where remaps happen on the same pfn and for
2167          * that we want to preserve the stale REF/MOD bits.
2168          */
2169         if (old & PT_REF)
2170                 hat_tlb_inval(hat, addr);
2171 
2172 done:
2173         if (ptr == NULL)
2174                 x86pte_release_pagetable(ht);
2175         return (old);
2176 }
2177 
2178 /*
2179  * Atomic compare and swap of a page table entry. No TLB invalidates are done.
2180  * This is used for links between pagetables of different levels.
2181  * Note we always create these links with dirty/access set, so they should
2182  * never change.
2183  */
2184 x86pte_t
2185 x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old, x86pte_t new)
2186 {
2187         x86pte_t        pte;
2188         x86pte_t        *ptep;
2189 #ifdef __xpv
2190         /*
2191          * We can't use writable pagetables for upper level tables, so fake it.
2192          */
2193         mmu_update_t t[2];
2194         int cnt = 1;
2195         int count;
2196         maddr_t ma;
2197 
2198         if (!IN_XPV_PANIC()) {
2199                 ASSERT(!(ht->ht_flags & HTABLE_COPIED));
2200                 ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry));
2201                 t[0].ptr = ma | MMU_NORMAL_PT_UPDATE;
2202                 t[0].val = new;
2203 
2204 #if defined(__amd64)
2205                 /*
2206                  * On the 64-bit hypervisor we need to maintain the user mode
2207                  * top page table too.
2208                  */
2209                 if (ht->ht_level == mmu.max_level && ht->ht_hat != kas.a_hat) {
2210                         ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(
2211                             ht->ht_hat->hat_user_ptable), entry));
2212                         t[1].ptr = ma | MMU_NORMAL_PT_UPDATE;
2213                         t[1].val = new;
2214                         ++cnt;
2215                 }
2216 #endif  /* __amd64 */
2217 
2218                 if (HYPERVISOR_mmu_update(t, cnt, &count, DOMID_SELF))
2219                         panic("HYPERVISOR_mmu_update() failed");
2220                 ASSERT(count == cnt);
2221                 return (old);
2222         }
2223 #endif
2224         ptep = x86pte_access_pagetable(ht, entry);
2225         XPV_ALLOW_PAGETABLE_UPDATES();
2226         pte = CAS_PTE(ptep, old, new);
2227         XPV_DISALLOW_PAGETABLE_UPDATES();
2228         x86pte_release_pagetable(ht);
2229         return (pte);
2230 }
2231 
2232 /*
2233  * Invalidate a page table entry as long as it currently maps something that
2234  * matches the value determined by expect.
2235  *
2236  * If tlb is set, also invalidates any TLB entries.
2237  *
2238  * Returns the previous value of the PTE.
2239  */
2240 x86pte_t
2241 x86pte_inval(
2242         htable_t *ht,
2243         uint_t entry,
2244         x86pte_t expect,
2245         x86pte_t *pte_ptr,
2246         boolean_t tlb)
2247 {
2248         x86pte_t        *ptep;
2249         x86pte_t        oldpte;
2250         x86pte_t        found;
2251 
2252         ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
2253         ASSERT(ht->ht_level <= mmu.max_page_level);
2254 
2255         if (pte_ptr != NULL)
2256                 ptep = pte_ptr;
2257         else
2258                 ptep = x86pte_access_pagetable(ht, entry);
2259 
2260 #if defined(__xpv)
2261         /*
2262          * If exit()ing just use HYPERVISOR_mmu_update(), as we can't be racing
2263          * with anything else.
2264          */
2265         if ((ht->ht_hat->hat_flags & HAT_FREEING) && !IN_XPV_PANIC()) {
2266                 int count;
2267                 mmu_update_t t[1];
2268                 maddr_t ma;
2269 
2270                 oldpte = GET_PTE(ptep);
2271                 if (expect != 0 && (oldpte & PT_PADDR) != (expect & PT_PADDR))
2272                         goto done;
2273                 ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry));
2274                 t[0].ptr = ma | MMU_NORMAL_PT_UPDATE;
2275                 t[0].val = 0;
2276                 if (HYPERVISOR_mmu_update(t, 1, &count, DOMID_SELF))
2277                         panic("HYPERVISOR_mmu_update() failed");
2278                 ASSERT(count == 1);
2279                 goto done;
2280         }
2281 #endif /* __xpv */
2282 
2283         /*
2284          * Note that the loop is needed to handle changes due to h/w updating
2285          * of PT_MOD/PT_REF.
2286          */
2287         do {
2288                 oldpte = GET_PTE(ptep);
2289                 if (expect != 0 && (oldpte & PT_PADDR) != (expect & PT_PADDR))
2290                         goto done;
2291                 XPV_ALLOW_PAGETABLE_UPDATES();
2292                 found = CAS_PTE(ptep, oldpte, 0);
2293                 XPV_DISALLOW_PAGETABLE_UPDATES();
2294         } while (found != oldpte);
2295         if (tlb && (oldpte & (PT_REF | PT_MOD)))
2296                 hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry));
2297 
2298 done:
2299         if (pte_ptr == NULL)
2300                 x86pte_release_pagetable(ht);
2301         return (oldpte);
2302 }
2303 
2304 /*
2305  * Change a page table entry af it currently matches the value in expect.
2306  */
2307 x86pte_t
2308 x86pte_update(
2309         htable_t *ht,
2310         uint_t entry,
2311         x86pte_t expect,
2312         x86pte_t new)
2313 {
2314         x86pte_t        *ptep;
2315         x86pte_t        found;
2316 
2317         ASSERT(new != 0);
2318         ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
2319         ASSERT(ht->ht_level <= mmu.max_page_level);
2320 
2321         ptep = x86pte_access_pagetable(ht, entry);
2322         XPV_ALLOW_PAGETABLE_UPDATES();
2323         found = CAS_PTE(ptep, expect, new);
2324         XPV_DISALLOW_PAGETABLE_UPDATES();
2325         if (found == expect) {
2326                 hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry));
2327 
2328                 /*
2329                  * When removing write permission *and* clearing the
2330                  * MOD bit, check if a write happened via a stale
2331                  * TLB entry before the TLB shootdown finished.
2332                  *
2333                  * If it did happen, simply re-enable write permission and
2334                  * act like the original CAS failed.
2335                  */
2336                 if ((expect & (PT_WRITABLE | PT_MOD)) == PT_WRITABLE &&
2337                     (new & (PT_WRITABLE | PT_MOD)) == 0 &&
2338                     (GET_PTE(ptep) & PT_MOD) != 0) {
2339                         do {
2340                                 found = GET_PTE(ptep);
2341                                 XPV_ALLOW_PAGETABLE_UPDATES();
2342                                 found =
2343                                     CAS_PTE(ptep, found, found | PT_WRITABLE);
2344                                 XPV_DISALLOW_PAGETABLE_UPDATES();
2345                         } while ((found & PT_WRITABLE) == 0);
2346                 }
2347         }
2348         x86pte_release_pagetable(ht);
2349         return (found);
2350 }
2351 
2352 #ifndef __xpv
2353 /*
2354  * Copy page tables - this is just a little more complicated than the
2355  * previous routines. Note that it's also not atomic! It also is never
2356  * used for HTABLE_COPIED pagetables.
2357  */
2358 void
2359 x86pte_copy(htable_t *src, htable_t *dest, uint_t entry, uint_t count)
2360 {
2361         caddr_t src_va;
2362         caddr_t dst_va;
2363         size_t size;
2364         x86pte_t *pteptr;
2365         x86pte_t pte;
2366 
2367         ASSERT(khat_running);
2368         ASSERT(!(dest->ht_flags & HTABLE_COPIED));
2369         ASSERT(!(src->ht_flags & HTABLE_COPIED));
2370         ASSERT(!(src->ht_flags & HTABLE_SHARED_PFN));
2371         ASSERT(!(dest->ht_flags & HTABLE_SHARED_PFN));
2372 
2373         /*
2374          * Acquire access to the CPU pagetable windows for the dest and source.
2375          */
2376         dst_va = (caddr_t)x86pte_access_pagetable(dest, entry);
2377         if (kpm_vbase) {
2378                 src_va = (caddr_t)
2379                     PT_INDEX_PTR(hat_kpm_pfn2va(src->ht_pfn), entry);
2380         } else {
2381                 uint_t x = PWIN_SRC(CPU->cpu_id);
2382 
2383                 /*
2384                  * Finish defining the src pagetable mapping
2385                  */
2386                 src_va = (caddr_t)PT_INDEX_PTR(PWIN_VA(x), entry);
2387                 pte = MAKEPTE(src->ht_pfn, 0) | mmu.pt_global | mmu.pt_nx;
2388                 pteptr = (x86pte_t *)PWIN_PTE_VA(x);
2389                 if (mmu.pae_hat)
2390                         *pteptr = pte;
2391                 else
2392                         *(x86pte32_t *)pteptr = pte;
2393                 mmu_tlbflush_entry((caddr_t)(PWIN_VA(x)));
2394         }
2395 
2396         /*
2397          * now do the copy
2398          */
2399         size = count << mmu.pte_size_shift;
2400         bcopy(src_va, dst_va, size);
2401 
2402         x86pte_release_pagetable(dest);
2403 }
2404 
2405 #else /* __xpv */
2406 
2407 /*
2408  * The hypervisor only supports writable pagetables at level 0, so we have
2409  * to install these 1 by 1 the slow way.
2410  */
2411 void
2412 x86pte_copy(htable_t *src, htable_t *dest, uint_t entry, uint_t count)
2413 {
2414         caddr_t src_va;
2415         x86pte_t pte;
2416 
2417         ASSERT(!IN_XPV_PANIC());
2418         src_va = (caddr_t)x86pte_access_pagetable(src, entry);
2419         while (count) {
2420                 if (mmu.pae_hat)
2421                         pte = *(x86pte_t *)src_va;
2422                 else
2423                         pte = *(x86pte32_t *)src_va;
2424                 if (pte != 0) {
2425                         set_pteval(pfn_to_pa(dest->ht_pfn), entry,
2426                             dest->ht_level, pte);
2427 #ifdef __amd64
2428                         if (dest->ht_level == mmu.max_level &&
2429                             htable_e2va(dest, entry) < HYPERVISOR_VIRT_END)
2430                                 set_pteval(
2431                                     pfn_to_pa(dest->ht_hat->hat_user_ptable),
2432                                     entry, dest->ht_level, pte);
2433 #endif
2434                 }
2435                 --count;
2436                 ++entry;
2437                 src_va += mmu.pte_size;
2438         }
2439         x86pte_release_pagetable(src);
2440 }
2441 #endif /* __xpv */
2442 
2443 /*
2444  * Zero page table entries - Note this doesn't use atomic stores!
2445  */
2446 static void
2447 x86pte_zero(htable_t *dest, uint_t entry, uint_t count)
2448 {
2449         caddr_t dst_va;
2450         size_t size;
2451 #ifdef __xpv
2452         int x;
2453         x86pte_t newpte;
2454 #endif
2455 
2456         /*
2457          * Map in the page table to be zeroed.
2458          */
2459         ASSERT(!(dest->ht_flags & HTABLE_SHARED_PFN));
2460         ASSERT(!(dest->ht_flags & HTABLE_COPIED));
2461 
2462         /*
2463          * On the hypervisor we don't use x86pte_access_pagetable() since
2464          * in this case the page is not pinned yet.
2465          */
2466 #ifdef __xpv
2467         if (kpm_vbase == NULL) {
2468                 kpreempt_disable();
2469                 ASSERT(CPU->cpu_hat_info != NULL);
2470                 mutex_enter(&CPU->cpu_hat_info->hci_mutex);
2471                 x = PWIN_TABLE(CPU->cpu_id);
2472                 newpte = MAKEPTE(dest->ht_pfn, 0) | PT_WRITABLE;
2473                 xen_map(newpte, PWIN_VA(x));
2474                 dst_va = (caddr_t)PT_INDEX_PTR(PWIN_VA(x), entry);
2475         } else
2476 #endif
2477                 dst_va = (caddr_t)x86pte_access_pagetable(dest, entry);
2478 
2479         size = count << mmu.pte_size_shift;
2480         ASSERT(size > BLOCKZEROALIGN);
2481 #ifdef __i386
2482         if (!is_x86_feature(x86_featureset, X86FSET_SSE2))
2483                 bzero(dst_va, size);
2484         else
2485 #endif
2486                 block_zero_no_xmm(dst_va, size);
2487 
2488 #ifdef __xpv
2489         if (kpm_vbase == NULL) {
2490                 xen_map(0, PWIN_VA(x));
2491                 mutex_exit(&CPU->cpu_hat_info->hci_mutex);
2492                 kpreempt_enable();
2493         } else
2494 #endif
2495                 x86pte_release_pagetable(dest);
2496 }
2497 
2498 /*
2499  * Called to ensure that all pagetables are in the system dump
2500  */
2501 void
2502 hat_dump(void)
2503 {
2504         hat_t *hat;
2505         uint_t h;
2506         htable_t *ht;
2507 
2508         /*
2509          * Dump all page tables
2510          */
2511         for (hat = kas.a_hat; hat != NULL; hat = hat->hat_next) {
2512                 for (h = 0; h < hat->hat_num_hash; ++h) {
2513                         for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
2514                                 if ((ht->ht_flags & HTABLE_COPIED) == 0)
2515                                         dump_page(ht->ht_pfn);
2516                         }
2517                 }
2518         }
2519 }