972 lines
27 KiB
C
972 lines
27 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* KVM guest address space mapping code
|
|
*
|
|
* Copyright IBM Corp. 2024, 2025
|
|
* Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com>
|
|
*/
|
|
|
|
#ifndef __KVM_S390_DAT_H
|
|
#define __KVM_S390_DAT_H
|
|
|
|
#include <linux/radix-tree.h>
|
|
#include <linux/refcount.h>
|
|
#include <linux/io.h>
|
|
#include <linux/kvm_types.h>
|
|
#include <linux/pgalloc.h>
|
|
#include <asm/tlbflush.h>
|
|
#include <asm/dat-bits.h>
|
|
|
|
/*
|
|
* Base address and length must be sent at the start of each block, therefore
|
|
* it's cheaper to send some clean data, as long as it's less than the size of
|
|
* two longs.
|
|
*/
|
|
#define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
|
|
/* For consistency */
|
|
#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
|
|
|
|
#define _ASCE(x) ((union asce) { .val = (x), })
|
|
#define NULL_ASCE _ASCE(0)
|
|
|
|
enum {
|
|
_DAT_TOKEN_NONE = 0,
|
|
_DAT_TOKEN_PIC,
|
|
};
|
|
|
|
#define _CRSTE_TOK(l, t, p) ((union crste) { \
|
|
.tok.i = 1, \
|
|
.tok.tt = (l), \
|
|
.tok.type = (t), \
|
|
.tok.par = (p) \
|
|
})
|
|
#define _CRSTE_PIC(l, p) _CRSTE_TOK(l, _DAT_TOKEN_PIC, p)
|
|
|
|
#define _CRSTE_HOLE(l) _CRSTE_PIC(l, PGM_ADDRESSING)
|
|
#define _CRSTE_EMPTY(l) _CRSTE_TOK(l, _DAT_TOKEN_NONE, 0)
|
|
|
|
#define _PMD_EMPTY _CRSTE_EMPTY(TABLE_TYPE_SEGMENT)
|
|
|
|
#define _PTE_TOK(t, p) ((union pte) { .tok.i = 1, .tok.type = (t), .tok.par = (p) })
|
|
#define _PTE_EMPTY _PTE_TOK(_DAT_TOKEN_NONE, 0)
|
|
|
|
/* This fake table type is used for page table walks (both for normal page tables and vSIE) */
|
|
#define TABLE_TYPE_PAGE_TABLE -1
|
|
|
|
enum dat_walk_flags {
|
|
DAT_WALK_USES_SKEYS = 0x40,
|
|
DAT_WALK_CONTINUE = 0x20,
|
|
DAT_WALK_IGN_HOLES = 0x10,
|
|
DAT_WALK_SPLIT = 0x08,
|
|
DAT_WALK_ALLOC = 0x04,
|
|
DAT_WALK_ANY = 0x02,
|
|
DAT_WALK_LEAF = 0x01,
|
|
DAT_WALK_DEFAULT = 0
|
|
};
|
|
|
|
#define DAT_WALK_SPLIT_ALLOC (DAT_WALK_SPLIT | DAT_WALK_ALLOC)
|
|
#define DAT_WALK_ALLOC_CONTINUE (DAT_WALK_CONTINUE | DAT_WALK_ALLOC)
|
|
#define DAT_WALK_LEAF_ALLOC (DAT_WALK_LEAF | DAT_WALK_ALLOC)
|
|
|
|
union pte {
|
|
unsigned long val;
|
|
union page_table_entry h;
|
|
struct {
|
|
unsigned long :56; /* Hardware bits */
|
|
unsigned long u : 1; /* Page unused */
|
|
unsigned long s : 1; /* Special */
|
|
unsigned long w : 1; /* Writable */
|
|
unsigned long r : 1; /* Readable */
|
|
unsigned long d : 1; /* Dirty */
|
|
unsigned long y : 1; /* Young */
|
|
unsigned long sd: 1; /* Soft dirty */
|
|
unsigned long pr: 1; /* Present */
|
|
} s;
|
|
struct {
|
|
unsigned char hwbytes[7];
|
|
unsigned char swbyte;
|
|
};
|
|
union {
|
|
struct {
|
|
unsigned long type :16; /* Token type */
|
|
unsigned long par :16; /* Token parameter */
|
|
unsigned long :20;
|
|
unsigned long : 1; /* Must be 0 */
|
|
unsigned long i : 1; /* Must be 1 */
|
|
unsigned long : 2;
|
|
unsigned long : 7;
|
|
unsigned long pr : 1; /* Must be 0 */
|
|
};
|
|
struct {
|
|
unsigned long token:32; /* Token and parameter */
|
|
unsigned long :32;
|
|
};
|
|
} tok;
|
|
};
|
|
|
|
/* Soft dirty, needed as macro for atomic operations on ptes */
|
|
#define _PAGE_SD 0x002
|
|
|
|
/* Needed as macro to perform atomic operations */
|
|
#define PGSTE_PCL_BIT 0x0080000000000000UL /* PCL lock, HW bit */
|
|
#define PGSTE_CMMA_D_BIT 0x0000000000008000UL /* CMMA dirty soft-bit */
|
|
|
|
enum pgste_gps_usage {
|
|
PGSTE_GPS_USAGE_STABLE = 0,
|
|
PGSTE_GPS_USAGE_UNUSED,
|
|
PGSTE_GPS_USAGE_POT_VOLATILE,
|
|
PGSTE_GPS_USAGE_VOLATILE,
|
|
};
|
|
|
|
union pgste {
|
|
unsigned long val;
|
|
struct {
|
|
unsigned long acc : 4;
|
|
unsigned long fp : 1;
|
|
unsigned long : 3;
|
|
unsigned long pcl : 1;
|
|
unsigned long hr : 1;
|
|
unsigned long hc : 1;
|
|
unsigned long : 2;
|
|
unsigned long gr : 1;
|
|
unsigned long gc : 1;
|
|
unsigned long : 1;
|
|
unsigned long :16; /* val16 */
|
|
unsigned long zero : 1;
|
|
unsigned long nodat : 1;
|
|
unsigned long : 4;
|
|
unsigned long usage : 2;
|
|
unsigned long : 8;
|
|
unsigned long cmma_d : 1; /* Dirty flag for CMMA bits */
|
|
unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
|
|
unsigned long vsie_notif : 1; /* Referenced in a shadow table */
|
|
unsigned long : 5;
|
|
unsigned long : 8;
|
|
};
|
|
struct {
|
|
unsigned short hwbytes0;
|
|
unsigned short val16; /* Used to store chunked values, see dat_{s,g}et_ptval() */
|
|
unsigned short hwbytes4;
|
|
unsigned char flags; /* Maps to the software bits */
|
|
unsigned char hwbyte7;
|
|
} __packed;
|
|
};
|
|
|
|
union pmd {
|
|
unsigned long val;
|
|
union segment_table_entry h;
|
|
struct {
|
|
struct {
|
|
unsigned long :44; /* HW */
|
|
unsigned long : 3; /* Unused */
|
|
unsigned long : 1; /* HW */
|
|
unsigned long s : 1; /* Special */
|
|
unsigned long w : 1; /* Writable soft-bit */
|
|
unsigned long r : 1; /* Readable soft-bit */
|
|
unsigned long d : 1; /* Dirty */
|
|
unsigned long y : 1; /* Young */
|
|
unsigned long : 3; /* HW */
|
|
unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
|
|
unsigned long vsie_notif : 1; /* Referenced in a shadow table */
|
|
unsigned long : 4; /* HW */
|
|
unsigned long sd : 1; /* Soft-Dirty */
|
|
unsigned long pr : 1; /* Present */
|
|
} fc1;
|
|
} s;
|
|
};
|
|
|
|
union pud {
|
|
unsigned long val;
|
|
union region3_table_entry h;
|
|
struct {
|
|
struct {
|
|
unsigned long :33; /* HW */
|
|
unsigned long :14; /* Unused */
|
|
unsigned long : 1; /* HW */
|
|
unsigned long s : 1; /* Special */
|
|
unsigned long w : 1; /* Writable soft-bit */
|
|
unsigned long r : 1; /* Readable soft-bit */
|
|
unsigned long d : 1; /* Dirty */
|
|
unsigned long y : 1; /* Young */
|
|
unsigned long : 3; /* HW */
|
|
unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
|
|
unsigned long vsie_notif : 1; /* Referenced in a shadow table */
|
|
unsigned long : 4; /* HW */
|
|
unsigned long sd : 1; /* Soft-Dirty */
|
|
unsigned long pr : 1; /* Present */
|
|
} fc1;
|
|
} s;
|
|
};
|
|
|
|
union p4d {
|
|
unsigned long val;
|
|
union region2_table_entry h;
|
|
};
|
|
|
|
union pgd {
|
|
unsigned long val;
|
|
union region1_table_entry h;
|
|
};
|
|
|
|
union crste {
|
|
unsigned long val;
|
|
union {
|
|
struct {
|
|
unsigned long :52;
|
|
unsigned long : 1;
|
|
unsigned long fc: 1;
|
|
unsigned long p : 1;
|
|
unsigned long : 1;
|
|
unsigned long : 2;
|
|
unsigned long i : 1;
|
|
unsigned long : 1;
|
|
unsigned long tt: 2;
|
|
unsigned long : 2;
|
|
};
|
|
struct {
|
|
unsigned long to:52;
|
|
unsigned long : 1;
|
|
unsigned long fc: 1;
|
|
unsigned long p : 1;
|
|
unsigned long : 1;
|
|
unsigned long tf: 2;
|
|
unsigned long i : 1;
|
|
unsigned long : 1;
|
|
unsigned long tt: 2;
|
|
unsigned long tl: 2;
|
|
} fc0;
|
|
struct {
|
|
unsigned long :47;
|
|
unsigned long av : 1; /* ACCF-Validity Control */
|
|
unsigned long acc: 4; /* Access-Control Bits */
|
|
unsigned long f : 1; /* Fetch-Protection Bit */
|
|
unsigned long fc : 1; /* Format-Control */
|
|
unsigned long p : 1; /* DAT-Protection Bit */
|
|
unsigned long iep: 1; /* Instruction-Execution-Protection */
|
|
unsigned long : 2;
|
|
unsigned long i : 1; /* Segment-Invalid Bit */
|
|
unsigned long cs : 1; /* Common-Segment Bit */
|
|
unsigned long tt : 2; /* Table-Type Bits */
|
|
unsigned long : 2;
|
|
} fc1;
|
|
} h;
|
|
struct {
|
|
struct {
|
|
unsigned long :47;
|
|
unsigned long : 1; /* HW (should be 0) */
|
|
unsigned long s : 1; /* Special */
|
|
unsigned long w : 1; /* Writable */
|
|
unsigned long r : 1; /* Readable */
|
|
unsigned long d : 1; /* Dirty */
|
|
unsigned long y : 1; /* Young */
|
|
unsigned long : 3; /* HW */
|
|
unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
|
|
unsigned long vsie_notif : 1; /* Referenced in a shadow table */
|
|
unsigned long : 4; /* HW */
|
|
unsigned long sd : 1; /* Soft-Dirty */
|
|
unsigned long pr : 1; /* Present */
|
|
} fc1;
|
|
} s;
|
|
union {
|
|
struct {
|
|
unsigned long type :16; /* Token type */
|
|
unsigned long par :16; /* Token parameter */
|
|
unsigned long :26;
|
|
unsigned long i : 1; /* Must be 1 */
|
|
unsigned long : 1;
|
|
unsigned long tt : 2;
|
|
unsigned long : 1;
|
|
unsigned long pr : 1; /* Must be 0 */
|
|
};
|
|
struct {
|
|
unsigned long token:32; /* Token and parameter */
|
|
unsigned long :32;
|
|
};
|
|
} tok;
|
|
union pmd pmd;
|
|
union pud pud;
|
|
union p4d p4d;
|
|
union pgd pgd;
|
|
};
|
|
|
|
union skey {
|
|
unsigned char skey;
|
|
struct {
|
|
unsigned char acc :4;
|
|
unsigned char fp :1;
|
|
unsigned char r :1;
|
|
unsigned char c :1;
|
|
unsigned char zero:1;
|
|
};
|
|
};
|
|
|
|
static_assert(sizeof(union pgste) == sizeof(unsigned long));
|
|
static_assert(sizeof(union pte) == sizeof(unsigned long));
|
|
static_assert(sizeof(union pmd) == sizeof(unsigned long));
|
|
static_assert(sizeof(union pud) == sizeof(unsigned long));
|
|
static_assert(sizeof(union p4d) == sizeof(unsigned long));
|
|
static_assert(sizeof(union pgd) == sizeof(unsigned long));
|
|
static_assert(sizeof(union crste) == sizeof(unsigned long));
|
|
static_assert(sizeof(union skey) == sizeof(char));
|
|
|
|
struct segment_table {
|
|
union pmd pmds[_CRST_ENTRIES];
|
|
};
|
|
|
|
struct region3_table {
|
|
union pud puds[_CRST_ENTRIES];
|
|
};
|
|
|
|
struct region2_table {
|
|
union p4d p4ds[_CRST_ENTRIES];
|
|
};
|
|
|
|
struct region1_table {
|
|
union pgd pgds[_CRST_ENTRIES];
|
|
};
|
|
|
|
struct crst_table {
|
|
union {
|
|
union crste crstes[_CRST_ENTRIES];
|
|
struct segment_table segment;
|
|
struct region3_table region3;
|
|
struct region2_table region2;
|
|
struct region1_table region1;
|
|
};
|
|
};
|
|
|
|
struct page_table {
|
|
union pte ptes[_PAGE_ENTRIES];
|
|
union pgste pgstes[_PAGE_ENTRIES];
|
|
};
|
|
|
|
static_assert(sizeof(struct crst_table) == _CRST_TABLE_SIZE);
|
|
static_assert(sizeof(struct page_table) == PAGE_SIZE);
|
|
|
|
struct dat_walk;
|
|
|
|
typedef long (*dat_walk_op)(union crste *crste, gfn_t gfn, gfn_t next, struct dat_walk *w);
|
|
|
|
struct dat_walk_ops {
|
|
union {
|
|
dat_walk_op crste_ops[4];
|
|
struct {
|
|
dat_walk_op pmd_entry;
|
|
dat_walk_op pud_entry;
|
|
dat_walk_op p4d_entry;
|
|
dat_walk_op pgd_entry;
|
|
};
|
|
};
|
|
long (*pte_entry)(union pte *pte, gfn_t gfn, gfn_t next, struct dat_walk *w);
|
|
};
|
|
|
|
struct dat_walk {
|
|
const struct dat_walk_ops *ops;
|
|
union crste *last;
|
|
union pte *last_pte;
|
|
union asce asce;
|
|
gfn_t start;
|
|
gfn_t end;
|
|
int flags;
|
|
void *priv;
|
|
};
|
|
|
|
struct ptval_param {
|
|
unsigned char offset : 6;
|
|
unsigned char len : 2;
|
|
};
|
|
|
|
/**
|
|
* _pte() - Useful constructor for union pte
|
|
* @pfn: the pfn this pte should point to.
|
|
* @writable: whether the pte should be writable.
|
|
* @dirty: whether the pte should be dirty.
|
|
* @special: whether the pte should be marked as special
|
|
*
|
|
* The pte is also marked as young and present. If the pte is marked as dirty,
|
|
* it gets marked as soft-dirty too. If the pte is not dirty, the hardware
|
|
* protect bit is set (independently of the write softbit); this way proper
|
|
* dirty tracking can be performed.
|
|
*
|
|
* Return: a union pte value.
|
|
*/
|
|
static inline union pte _pte(kvm_pfn_t pfn, bool writable, bool dirty, bool special)
|
|
{
|
|
union pte res = { .val = PFN_PHYS(pfn) };
|
|
|
|
res.h.p = !dirty;
|
|
res.s.y = 1;
|
|
res.s.pr = 1;
|
|
res.s.w = writable;
|
|
res.s.d = dirty;
|
|
res.s.sd = dirty;
|
|
res.s.s = special;
|
|
return res;
|
|
}
|
|
|
|
static inline union crste _crste_fc0(kvm_pfn_t pfn, int tt)
|
|
{
|
|
union crste res = { .val = PFN_PHYS(pfn) };
|
|
|
|
res.h.tt = tt;
|
|
res.h.fc0.tl = _REGION_ENTRY_LENGTH;
|
|
res.h.fc0.tf = 0;
|
|
return res;
|
|
}
|
|
|
|
/**
|
|
* _crste() - Useful constructor for union crste with FC=1
|
|
* @pfn: the pfn this pte should point to.
|
|
* @tt: the table type
|
|
* @writable: whether the pte should be writable.
|
|
* @dirty: whether the pte should be dirty.
|
|
*
|
|
* The crste is also marked as young and present. If the crste is marked as
|
|
* dirty, it gets marked as soft-dirty too. If the crste is not dirty, the
|
|
* hardware protect bit is set (independently of the write softbit); this way
|
|
* proper dirty tracking can be performed.
|
|
*
|
|
* Return: a union crste value.
|
|
*/
|
|
static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool writable, bool dirty)
|
|
{
|
|
union crste res = { .val = PFN_PHYS(pfn) & _SEGMENT_MASK };
|
|
|
|
res.h.tt = tt;
|
|
res.h.p = !dirty;
|
|
res.h.fc = 1;
|
|
res.s.fc1.y = 1;
|
|
res.s.fc1.pr = 1;
|
|
res.s.fc1.w = writable;
|
|
res.s.fc1.d = dirty;
|
|
res.s.fc1.sd = dirty;
|
|
return res;
|
|
}
|
|
|
|
union essa_state {
|
|
unsigned char val;
|
|
struct {
|
|
unsigned char : 2;
|
|
unsigned char nodat : 1;
|
|
unsigned char exception : 1;
|
|
unsigned char usage : 2;
|
|
unsigned char content : 2;
|
|
};
|
|
};
|
|
|
|
/**
|
|
* struct vsie_rmap - reverse mapping for shadow page table entries
|
|
* @next: pointer to next rmap in the list
|
|
* @r_gfn: virtual rmap address in the shadow guest address space
|
|
*/
|
|
struct vsie_rmap {
|
|
struct vsie_rmap *next;
|
|
union {
|
|
unsigned long val;
|
|
struct {
|
|
long level: 8;
|
|
unsigned long : 4;
|
|
unsigned long r_gfn:52;
|
|
};
|
|
};
|
|
};
|
|
|
|
static_assert(sizeof(struct vsie_rmap) == 2 * sizeof(long));
|
|
|
|
#define KVM_S390_MMU_CACHE_N_CRSTS 6
|
|
#define KVM_S390_MMU_CACHE_N_PTS 2
|
|
#define KVM_S390_MMU_CACHE_N_RMAPS 16
|
|
struct kvm_s390_mmu_cache {
|
|
void *crsts[KVM_S390_MMU_CACHE_N_CRSTS];
|
|
void *pts[KVM_S390_MMU_CACHE_N_PTS];
|
|
void *rmaps[KVM_S390_MMU_CACHE_N_RMAPS];
|
|
short int n_crsts;
|
|
short int n_pts;
|
|
short int n_rmaps;
|
|
};
|
|
|
|
struct guest_fault {
|
|
gfn_t gfn; /* Guest frame */
|
|
kvm_pfn_t pfn; /* Host PFN */
|
|
struct page *page; /* Host page */
|
|
union pte *ptep; /* Used to resolve the fault, or NULL */
|
|
union crste *crstep; /* Used to resolve the fault, or NULL */
|
|
bool writable; /* Mapping is writable */
|
|
bool write_attempt; /* Write access attempted */
|
|
bool attempt_pfault; /* Attempt a pfault first */
|
|
bool valid; /* This entry contains valid data */
|
|
void (*callback)(struct guest_fault *f);
|
|
void *priv;
|
|
};
|
|
|
|
/*
|
|
* 0 1 2 3 4 5 6 7
|
|
* +-------+-------+-------+-------+-------+-------+-------+-------+
|
|
* 0 | | PGT_ADDR |
|
|
* 8 | VMADDR | |
|
|
* 16 | |
|
|
* 24 | |
|
|
*/
|
|
#define MKPTVAL(o, l) ((struct ptval_param) { .offset = (o), .len = ((l) + 1) / 2 - 1})
|
|
#define PTVAL_PGT_ADDR MKPTVAL(4, 8)
|
|
#define PTVAL_VMADDR MKPTVAL(8, 6)
|
|
|
|
union pgste __must_check __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new,
|
|
gfn_t gfn, union asce asce, bool uses_skeys);
|
|
bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
|
|
union asce asce);
|
|
void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce);
|
|
|
|
long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce,
|
|
const struct dat_walk_ops *ops, int flags, void *priv);
|
|
|
|
int dat_entry_walk(struct kvm_s390_mmu_cache *mc, gfn_t gfn, union asce asce, int flags,
|
|
int walk_level, union crste **last, union pte **ptepp);
|
|
void dat_free_level(struct crst_table *table, bool owns_ptes);
|
|
struct crst_table *dat_alloc_crst_sleepable(unsigned long init);
|
|
int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newtype);
|
|
int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey);
|
|
int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn,
|
|
union skey skey, bool nq);
|
|
int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gfn_t gfn,
|
|
union skey skey, union skey *oldkey, bool nq, bool mr, bool mc);
|
|
int dat_reset_reference_bit(union asce asce, gfn_t gfn);
|
|
long dat_reset_skeys(union asce asce, gfn_t start);
|
|
|
|
unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param);
|
|
void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val);
|
|
|
|
int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end,
|
|
u16 type, u16 param);
|
|
int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn);
|
|
bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end);
|
|
|
|
int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty);
|
|
long dat_reset_cmma(union asce asce, gfn_t start_gfn);
|
|
int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values);
|
|
int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem);
|
|
int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn,
|
|
unsigned long count, unsigned long mask, const uint8_t *bits);
|
|
|
|
int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc);
|
|
|
|
#define GFP_KVM_S390_MMU_CACHE (GFP_ATOMIC | __GFP_ACCOUNT | __GFP_NOWARN)
|
|
|
|
static inline struct page_table *kvm_s390_mmu_cache_alloc_pt(struct kvm_s390_mmu_cache *mc)
|
|
{
|
|
if (mc->n_pts)
|
|
return mc->pts[--mc->n_pts];
|
|
return (void *)__get_free_page(GFP_KVM_S390_MMU_CACHE);
|
|
}
|
|
|
|
static inline struct crst_table *kvm_s390_mmu_cache_alloc_crst(struct kvm_s390_mmu_cache *mc)
|
|
{
|
|
if (mc->n_crsts)
|
|
return mc->crsts[--mc->n_crsts];
|
|
return (void *)__get_free_pages(GFP_KVM_S390_MMU_CACHE | __GFP_COMP, CRST_ALLOC_ORDER);
|
|
}
|
|
|
|
static inline struct vsie_rmap *kvm_s390_mmu_cache_alloc_rmap(struct kvm_s390_mmu_cache *mc)
|
|
{
|
|
if (mc->n_rmaps)
|
|
return mc->rmaps[--mc->n_rmaps];
|
|
return kzalloc_obj(struct vsie_rmap, GFP_KVM_S390_MMU_CACHE);
|
|
}
|
|
|
|
static inline struct crst_table *crste_table_start(union crste *crstep)
|
|
{
|
|
return (struct crst_table *)ALIGN_DOWN((unsigned long)crstep, _CRST_TABLE_SIZE);
|
|
}
|
|
|
|
static inline struct page_table *pte_table_start(union pte *ptep)
|
|
{
|
|
return (struct page_table *)ALIGN_DOWN((unsigned long)ptep, _PAGE_TABLE_SIZE);
|
|
}
|
|
|
|
static inline bool crdte_crste(union crste *crstep, union crste old, union crste new, gfn_t gfn,
|
|
union asce asce)
|
|
{
|
|
unsigned long dtt = 0x10 | new.h.tt << 2;
|
|
void *table = crste_table_start(crstep);
|
|
|
|
return crdte(old.val, new.val, table, dtt, gfn_to_gpa(gfn), asce.val);
|
|
}
|
|
|
|
/**
|
|
* idte_crste() - invalidate a crste entry using idte
|
|
* @crstep: pointer to the crste to be invalidated
|
|
* @gfn: a gfn mapped by the crste
|
|
* @opt: options for the idte instruction
|
|
* @asce: the asce
|
|
* @local: whether the operation is cpu-local
|
|
*/
|
|
static __always_inline void idte_crste(union crste *crstep, gfn_t gfn, unsigned long opt,
|
|
union asce asce, int local)
|
|
{
|
|
unsigned long table_origin = __pa(crste_table_start(crstep));
|
|
unsigned long gaddr = gfn_to_gpa(gfn) & HPAGE_MASK;
|
|
|
|
if (__builtin_constant_p(opt) && opt == 0) {
|
|
/* flush without guest asce */
|
|
asm volatile("idte %[table_origin],0,%[gaddr],%[local]"
|
|
: "+m" (*crstep)
|
|
: [table_origin] "a" (table_origin), [gaddr] "a" (gaddr),
|
|
[local] "i" (local)
|
|
: "cc");
|
|
} else {
|
|
/* flush with guest asce */
|
|
asm volatile("idte %[table_origin],%[asce],%[gaddr_opt],%[local]"
|
|
: "+m" (*crstep)
|
|
: [table_origin] "a" (table_origin), [gaddr_opt] "a" (gaddr | opt),
|
|
[asce] "a" (asce.val), [local] "i" (local)
|
|
: "cc");
|
|
}
|
|
}
|
|
|
|
static inline void dat_init_pgstes(struct page_table *pt, unsigned long val)
|
|
{
|
|
memset64((void *)pt->pgstes, val, PTRS_PER_PTE);
|
|
}
|
|
|
|
static inline void dat_init_page_table(struct page_table *pt, unsigned long ptes,
|
|
unsigned long pgstes)
|
|
{
|
|
memset64((void *)pt->ptes, ptes, PTRS_PER_PTE);
|
|
dat_init_pgstes(pt, pgstes);
|
|
}
|
|
|
|
static inline gfn_t asce_end(union asce asce)
|
|
{
|
|
return 1ULL << ((asce.dt + 1) * 11 + _SEGMENT_SHIFT - PAGE_SHIFT);
|
|
}
|
|
|
|
#define _CRSTE(x) ((union crste) { .val = _Generic((x), \
|
|
union pgd : (x).val, \
|
|
union p4d : (x).val, \
|
|
union pud : (x).val, \
|
|
union pmd : (x).val, \
|
|
union crste : (x).val)})
|
|
|
|
#define _CRSTEP(x) ((union crste *)_Generic((*(x)), \
|
|
union pgd : (x), \
|
|
union p4d : (x), \
|
|
union pud : (x), \
|
|
union pmd : (x), \
|
|
union crste : (x)))
|
|
|
|
#define _CRSTP(x) ((struct crst_table *)_Generic((*(x)), \
|
|
struct crst_table : (x), \
|
|
struct segment_table : (x), \
|
|
struct region3_table : (x), \
|
|
struct region2_table : (x), \
|
|
struct region1_table : (x)))
|
|
|
|
static inline bool asce_contains_gfn(union asce asce, gfn_t gfn)
|
|
{
|
|
return gfn < asce_end(asce);
|
|
}
|
|
|
|
static inline bool is_pmd(union crste crste)
|
|
{
|
|
return crste.h.tt == TABLE_TYPE_SEGMENT;
|
|
}
|
|
|
|
static inline bool is_pud(union crste crste)
|
|
{
|
|
return crste.h.tt == TABLE_TYPE_REGION3;
|
|
}
|
|
|
|
static inline bool is_p4d(union crste crste)
|
|
{
|
|
return crste.h.tt == TABLE_TYPE_REGION2;
|
|
}
|
|
|
|
static inline bool is_pgd(union crste crste)
|
|
{
|
|
return crste.h.tt == TABLE_TYPE_REGION1;
|
|
}
|
|
|
|
static inline phys_addr_t pmd_origin_large(union pmd pmd)
|
|
{
|
|
return pmd.val & _SEGMENT_ENTRY_ORIGIN_LARGE;
|
|
}
|
|
|
|
static inline phys_addr_t pud_origin_large(union pud pud)
|
|
{
|
|
return pud.val & _REGION3_ENTRY_ORIGIN_LARGE;
|
|
}
|
|
|
|
/**
|
|
* crste_origin_large() - Return the large frame origin of a large crste
|
|
* @crste: The crste whose origin is to be returned. Should be either a
|
|
* region-3 table entry or a segment table entry, in both cases with
|
|
* FC set to 1 (large pages).
|
|
*
|
|
* Return: The origin of the large frame pointed to by @crste, or -1 if the
|
|
* crste was not large (wrong table type, or FC==0)
|
|
*/
|
|
static inline phys_addr_t crste_origin_large(union crste crste)
|
|
{
|
|
if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3))
|
|
return -1;
|
|
if (is_pmd(crste))
|
|
return pmd_origin_large(crste.pmd);
|
|
return pud_origin_large(crste.pud);
|
|
}
|
|
|
|
#define crste_origin(x) (_Generic((x), \
|
|
union pmd : (x).val & _SEGMENT_ENTRY_ORIGIN, \
|
|
union pud : (x).val & _REGION_ENTRY_ORIGIN, \
|
|
union p4d : (x).val & _REGION_ENTRY_ORIGIN, \
|
|
union pgd : (x).val & _REGION_ENTRY_ORIGIN))
|
|
|
|
static inline unsigned long pte_origin(union pte pte)
|
|
{
|
|
return pte.val & PAGE_MASK;
|
|
}
|
|
|
|
static inline bool pmd_prefix(union pmd pmd)
|
|
{
|
|
return pmd.h.fc && pmd.s.fc1.prefix_notif;
|
|
}
|
|
|
|
static inline bool pud_prefix(union pud pud)
|
|
{
|
|
return pud.h.fc && pud.s.fc1.prefix_notif;
|
|
}
|
|
|
|
static inline bool crste_leaf(union crste crste)
|
|
{
|
|
return (crste.h.tt <= TABLE_TYPE_REGION3) && crste.h.fc;
|
|
}
|
|
|
|
static inline bool crste_prefix(union crste crste)
|
|
{
|
|
return crste_leaf(crste) && crste.s.fc1.prefix_notif;
|
|
}
|
|
|
|
static inline bool crste_dirty(union crste crste)
|
|
{
|
|
return crste_leaf(crste) && crste.s.fc1.d;
|
|
}
|
|
|
|
static inline union pgste *pgste_of(union pte *pte)
|
|
{
|
|
return (union pgste *)(pte + _PAGE_ENTRIES);
|
|
}
|
|
|
|
static inline bool pte_hole(union pte pte)
|
|
{
|
|
return pte.h.i && !pte.tok.pr && pte.tok.type != _DAT_TOKEN_NONE;
|
|
}
|
|
|
|
static inline bool _crste_hole(union crste crste)
|
|
{
|
|
return crste.h.i && !crste.tok.pr && crste.tok.type != _DAT_TOKEN_NONE;
|
|
}
|
|
|
|
#define crste_hole(x) _crste_hole(_CRSTE(x))
|
|
|
|
static inline bool _crste_none(union crste crste)
|
|
{
|
|
return crste.h.i && !crste.tok.pr && crste.tok.type == _DAT_TOKEN_NONE;
|
|
}
|
|
|
|
#define crste_none(x) _crste_none(_CRSTE(x))
|
|
|
|
static inline phys_addr_t large_pud_to_phys(union pud pud, gfn_t gfn)
|
|
{
|
|
return pud_origin_large(pud) | (gfn_to_gpa(gfn) & ~_REGION3_MASK);
|
|
}
|
|
|
|
static inline phys_addr_t large_pmd_to_phys(union pmd pmd, gfn_t gfn)
|
|
{
|
|
return pmd_origin_large(pmd) | (gfn_to_gpa(gfn) & ~_SEGMENT_MASK);
|
|
}
|
|
|
|
static inline phys_addr_t large_crste_to_phys(union crste crste, gfn_t gfn)
|
|
{
|
|
if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3))
|
|
return -1;
|
|
if (is_pmd(crste))
|
|
return large_pmd_to_phys(crste.pmd, gfn);
|
|
return large_pud_to_phys(crste.pud, gfn);
|
|
}
|
|
|
|
static inline bool cspg_crste(union crste *crstep, union crste old, union crste new)
|
|
{
|
|
return cspg(&crstep->val, old.val, new.val);
|
|
}
|
|
|
|
static inline struct page_table *dereference_pmd(union pmd pmd)
|
|
{
|
|
return phys_to_virt(crste_origin(pmd));
|
|
}
|
|
|
|
static inline struct segment_table *dereference_pud(union pud pud)
|
|
{
|
|
return phys_to_virt(crste_origin(pud));
|
|
}
|
|
|
|
static inline struct region3_table *dereference_p4d(union p4d p4d)
|
|
{
|
|
return phys_to_virt(crste_origin(p4d));
|
|
}
|
|
|
|
static inline struct region2_table *dereference_pgd(union pgd pgd)
|
|
{
|
|
return phys_to_virt(crste_origin(pgd));
|
|
}
|
|
|
|
static inline struct crst_table *_dereference_crste(union crste crste)
|
|
{
|
|
if (unlikely(is_pmd(crste)))
|
|
return NULL;
|
|
return phys_to_virt(crste_origin(crste.pud));
|
|
}
|
|
|
|
#define dereference_crste(x) (_Generic((x), \
|
|
union pud : _dereference_crste(_CRSTE(x)), \
|
|
union p4d : _dereference_crste(_CRSTE(x)), \
|
|
union pgd : _dereference_crste(_CRSTE(x)), \
|
|
union crste : _dereference_crste(_CRSTE(x))))
|
|
|
|
static inline struct crst_table *dereference_asce(union asce asce)
|
|
{
|
|
return phys_to_virt(asce.val & _ASCE_ORIGIN);
|
|
}
|
|
|
|
static inline void asce_flush_tlb(union asce asce)
|
|
{
|
|
__tlb_flush_idte(asce.val);
|
|
}
|
|
|
|
static inline bool pgste_get_trylock(union pte *ptep, union pgste *res)
|
|
{
|
|
union pgste *pgstep = pgste_of(ptep);
|
|
union pgste old_pgste;
|
|
|
|
if (READ_ONCE(pgstep->val) & PGSTE_PCL_BIT)
|
|
return false;
|
|
old_pgste.val = __atomic64_or_barrier(PGSTE_PCL_BIT, &pgstep->val);
|
|
if (old_pgste.pcl)
|
|
return false;
|
|
old_pgste.pcl = 1;
|
|
*res = old_pgste;
|
|
return true;
|
|
}
|
|
|
|
static inline union pgste pgste_get_lock(union pte *ptep)
|
|
{
|
|
union pgste res;
|
|
|
|
while (!pgste_get_trylock(ptep, &res))
|
|
cpu_relax();
|
|
return res;
|
|
}
|
|
|
|
static inline void pgste_set_unlock(union pte *ptep, union pgste pgste)
|
|
{
|
|
pgste.pcl = 0;
|
|
barrier();
|
|
WRITE_ONCE(*pgste_of(ptep), pgste);
|
|
}
|
|
|
|
static inline void dat_ptep_xchg(union pte *ptep, union pte new, gfn_t gfn, union asce asce,
|
|
bool has_skeys)
|
|
{
|
|
union pgste pgste;
|
|
|
|
pgste = pgste_get_lock(ptep);
|
|
pgste = __dat_ptep_xchg(ptep, pgste, new, gfn, asce, has_skeys);
|
|
pgste_set_unlock(ptep, pgste);
|
|
}
|
|
|
|
static inline void dat_ptep_clear(union pte *ptep, gfn_t gfn, union asce asce, bool has_skeys)
|
|
{
|
|
dat_ptep_xchg(ptep, _PTE_EMPTY, gfn, asce, has_skeys);
|
|
}
|
|
|
|
static inline void dat_free_pt(struct page_table *pt)
|
|
{
|
|
free_page((unsigned long)pt);
|
|
}
|
|
|
|
static inline void _dat_free_crst(struct crst_table *table)
|
|
{
|
|
free_pages((unsigned long)table, CRST_ALLOC_ORDER);
|
|
}
|
|
|
|
#define dat_free_crst(x) _dat_free_crst(_CRSTP(x))
|
|
|
|
static inline void kvm_s390_free_mmu_cache(struct kvm_s390_mmu_cache *mc)
|
|
{
|
|
if (!mc)
|
|
return;
|
|
while (mc->n_pts)
|
|
dat_free_pt(mc->pts[--mc->n_pts]);
|
|
while (mc->n_crsts)
|
|
_dat_free_crst(mc->crsts[--mc->n_crsts]);
|
|
while (mc->n_rmaps)
|
|
kfree(mc->rmaps[--mc->n_rmaps]);
|
|
kfree(mc);
|
|
}
|
|
|
|
DEFINE_FREE(kvm_s390_mmu_cache, struct kvm_s390_mmu_cache *, if (_T) kvm_s390_free_mmu_cache(_T))
|
|
|
|
static inline struct kvm_s390_mmu_cache *kvm_s390_new_mmu_cache(void)
|
|
{
|
|
struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
|
|
|
|
mc = kzalloc_obj(*mc, GFP_KERNEL_ACCOUNT);
|
|
if (mc && !kvm_s390_mmu_cache_topup(mc))
|
|
return_ptr(mc);
|
|
return NULL;
|
|
}
|
|
|
|
static inline bool dat_pmdp_xchg_atomic(union pmd *pmdp, union pmd old, union pmd new,
|
|
gfn_t gfn, union asce asce)
|
|
{
|
|
return dat_crstep_xchg_atomic(_CRSTEP(pmdp), _CRSTE(old), _CRSTE(new), gfn, asce);
|
|
}
|
|
|
|
static inline bool dat_pudp_xchg_atomic(union pud *pudp, union pud old, union pud new,
|
|
gfn_t gfn, union asce asce)
|
|
{
|
|
return dat_crstep_xchg_atomic(_CRSTEP(pudp), _CRSTE(old), _CRSTE(new), gfn, asce);
|
|
}
|
|
|
|
static inline union crste dat_crstep_clear_atomic(union crste *crstep, gfn_t gfn, union asce asce)
|
|
{
|
|
union crste oldcrste, empty = _CRSTE_EMPTY(crstep->h.tt);
|
|
|
|
do {
|
|
oldcrste = READ_ONCE(*crstep);
|
|
} while (!dat_crstep_xchg_atomic(crstep, oldcrste, empty, gfn, asce));
|
|
return oldcrste;
|
|
}
|
|
|
|
static inline int get_level(union crste *crstep, union pte *ptep)
|
|
{
|
|
return ptep ? TABLE_TYPE_PAGE_TABLE : crstep->h.tt;
|
|
}
|
|
|
|
static inline int dat_delete_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start,
|
|
unsigned long npages)
|
|
{
|
|
return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_PIC, PGM_ADDRESSING);
|
|
}
|
|
|
|
static inline int dat_create_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start,
|
|
unsigned long npages)
|
|
{
|
|
return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_NONE, 0);
|
|
}
|
|
|
|
static inline bool crste_is_ucas(union crste crste)
|
|
{
|
|
return is_pmd(crste) && crste.h.i && crste.h.fc0.tl == 1 && crste.h.fc == 0;
|
|
}
|
|
|
|
#endif /* __KVM_S390_DAT_H */
|