/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_TLB_H
#define _ASM_X86_TLB_H

#define tlb_flush tlb_flush
static inline void tlb_flush(struct mmu_gather *tlb);

#include <asm-generic/tlb.h>
#include <linux/kernel.h>
#include <vdso/bits.h>
#include <vdso/page.h>

static inline void tlb_flush(struct mmu_gather *tlb)
{
	unsigned long start = 0UL, end = TLB_FLUSH_ALL;
	unsigned int stride_shift = tlb_get_unmap_shift(tlb);

	if (!tlb->fullmm && !tlb->need_flush_all) {
		start = tlb->start;
		end = tlb->end;
	}

	flush_tlb_mm_range(tlb->mm, start, end, stride_shift, tlb->freed_tables);
}

static inline void invlpg(unsigned long addr)
{
	asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
}

enum addr_stride {
	PTE_STRIDE = 0,
	PMD_STRIDE = 1
};


#define INVLPGB_FLAG_VA			BIT(0)
#define INVLPGB_FLAG_PCID		BIT(1)
#define INVLPGB_FLAG_ASID		BIT(2)
#define INVLPGB_FLAG_INCLUDE_GLOBAL	BIT(3)
#define INVLPGB_FLAG_FINAL_ONLY		BIT(4)
#define INVLPGB_FLAG_INCLUDE_NESTED	BIT(5)


#define INVLPGB_MODE_ALL_NONGLOBALS	0UL

#ifdef CONFIG_BROADCAST_TLB_FLUSH

static inline void __invlpgb(unsigned long asid, unsigned long pcid,
			     unsigned long addr, u16 nr_pages,
			     enum addr_stride stride, u8 flags)
{
	u64 rax = addr | flags | INVLPGB_FLAG_ASID;
	u32 ecx = (stride << 31) | (nr_pages - 1);
	u32 edx = (pcid << 16) | asid;

	
	VM_WARN_ON_ONCE(addr & ~PAGE_MASK);

	
	asm volatile(".byte 0x0f, 0x01, 0xfe" :: "a" (rax), "c" (ecx), "d" (edx));
}

static inline void __invlpgb_all(unsigned long asid, unsigned long pcid, u8 flags)
{
	__invlpgb(asid, pcid, 0, 1, 0, flags);
}

static inline void __tlbsync(void)
{
	
	cant_migrate();

	
	asm volatile(".byte 0x0f, 0x01, 0xff" ::: "memory");
}
#else

static inline void __invlpgb(unsigned long asid, unsigned long pcid,
			     unsigned long addr, u16 nr_pages,
			     enum addr_stride s, u8 flags) { }
static inline void __invlpgb_all(unsigned long asid, unsigned long pcid, u8 flags) { }
static inline void __tlbsync(void) { }
#endif

static inline void invlpgb_flush_user_nr_nosync(unsigned long pcid,
						unsigned long addr,
						u16 nr, bool stride)
{
	enum addr_stride str = stride ? PMD_STRIDE : PTE_STRIDE;
	u8 flags = INVLPGB_FLAG_PCID | INVLPGB_FLAG_VA;

	__invlpgb(0, pcid, addr, nr, str, flags);
}


static inline void invlpgb_flush_single_pcid_nosync(unsigned long pcid)
{
	__invlpgb_all(0, pcid, INVLPGB_FLAG_PCID);
}


static inline void invlpgb_flush_all(void)
{
	
	guard(preempt)();
	__invlpgb_all(0, 0, INVLPGB_FLAG_INCLUDE_GLOBAL);
	__tlbsync();
}


static inline void invlpgb_flush_addr_nosync(unsigned long addr, u16 nr)
{
	__invlpgb(0, 0, addr, nr, PTE_STRIDE, INVLPGB_FLAG_INCLUDE_GLOBAL);
}


static inline void invlpgb_flush_all_nonglobals(void)
{
	guard(preempt)();
	__invlpgb_all(0, 0, INVLPGB_MODE_ALL_NONGLOBALS);
	__tlbsync();
}
#endif 
