/* SPDX-License-Identifier: GPL-2.0 */

#ifndef _LINUX_LEAFOPS_H
#define _LINUX_LEAFOPS_H

#include <linux/mm_types.h>
#include <linux/swapops.h>
#include <linux/swap.h>

#ifdef CONFIG_MMU


#define LEAF_TYPE_SHIFT SWP_TYPE_SHIFT

enum softleaf_type {
	
	SOFTLEAF_NONE,
	SOFTLEAF_SWAP,
	
	SOFTLEAF_MIGRATION_READ,
	SOFTLEAF_MIGRATION_READ_EXCLUSIVE,
	SOFTLEAF_MIGRATION_WRITE,
	
	SOFTLEAF_DEVICE_PRIVATE_READ,
	SOFTLEAF_DEVICE_PRIVATE_WRITE,
	SOFTLEAF_DEVICE_EXCLUSIVE,
	
	SOFTLEAF_HWPOISON,
	
	SOFTLEAF_MARKER,
};


static inline softleaf_t softleaf_mk_none(void)
{
	return ((softleaf_t) { 0 });
}


static inline softleaf_t softleaf_from_pte(pte_t pte)
{
	softleaf_t arch_entry;

	if (pte_present(pte) || pte_none(pte))
		return softleaf_mk_none();

	pte = pte_swp_clear_flags(pte);
	arch_entry = __pte_to_swp_entry(pte);

	
	return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
}


static inline pte_t softleaf_to_pte(softleaf_t entry)
{
	
	return swp_entry_to_pte(entry);
}

#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION

static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
{
	softleaf_t arch_entry;

	if (pmd_present(pmd) || pmd_none(pmd))
		return softleaf_mk_none();

	if (pmd_swp_soft_dirty(pmd))
		pmd = pmd_swp_clear_soft_dirty(pmd);
	if (pmd_swp_uffd_wp(pmd))
		pmd = pmd_swp_clear_uffd_wp(pmd);
	arch_entry = __pmd_to_swp_entry(pmd);

	
	return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
}

#else

static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
{
	return softleaf_mk_none();
}

#endif


static inline bool softleaf_is_none(softleaf_t entry)
{
	return entry.val == 0;
}


static inline enum softleaf_type softleaf_type(softleaf_t entry)
{
	unsigned int type_num;

	if (softleaf_is_none(entry))
		return SOFTLEAF_NONE;

	type_num = entry.val >> LEAF_TYPE_SHIFT;

	if (type_num < MAX_SWAPFILES)
		return SOFTLEAF_SWAP;

	switch (type_num) {
#ifdef CONFIG_MIGRATION
	case SWP_MIGRATION_READ:
		return SOFTLEAF_MIGRATION_READ;
	case SWP_MIGRATION_READ_EXCLUSIVE:
		return SOFTLEAF_MIGRATION_READ_EXCLUSIVE;
	case SWP_MIGRATION_WRITE:
		return SOFTLEAF_MIGRATION_WRITE;
#endif
#ifdef CONFIG_DEVICE_PRIVATE
	case SWP_DEVICE_WRITE:
		return SOFTLEAF_DEVICE_PRIVATE_WRITE;
	case SWP_DEVICE_READ:
		return SOFTLEAF_DEVICE_PRIVATE_READ;
	case SWP_DEVICE_EXCLUSIVE:
		return SOFTLEAF_DEVICE_EXCLUSIVE;
#endif
#ifdef CONFIG_MEMORY_FAILURE
	case SWP_HWPOISON:
		return SOFTLEAF_HWPOISON;
#endif
	case SWP_PTE_MARKER:
		return SOFTLEAF_MARKER;
	}

	
	VM_WARN_ON_ONCE(1);
	return SOFTLEAF_NONE;
}


static inline bool softleaf_is_swap(softleaf_t entry)
{
	return softleaf_type(entry) == SOFTLEAF_SWAP;
}


static inline bool softleaf_is_migration_write(softleaf_t entry)
{
	return softleaf_type(entry) == SOFTLEAF_MIGRATION_WRITE;
}


static inline bool softleaf_is_migration_read(softleaf_t entry)
{
	return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ;
}


static inline bool softleaf_is_migration_read_exclusive(softleaf_t entry)
{
	return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ_EXCLUSIVE;
}


static inline bool softleaf_is_migration(softleaf_t entry)
{
	switch (softleaf_type(entry)) {
	case SOFTLEAF_MIGRATION_READ:
	case SOFTLEAF_MIGRATION_READ_EXCLUSIVE:
	case SOFTLEAF_MIGRATION_WRITE:
		return true;
	default:
		return false;
	}
}


static inline bool softleaf_is_device_private_write(softleaf_t entry)
{
	return softleaf_type(entry) == SOFTLEAF_DEVICE_PRIVATE_WRITE;
}


static inline bool softleaf_is_device_private(softleaf_t entry)
{
	switch (softleaf_type(entry)) {
	case SOFTLEAF_DEVICE_PRIVATE_WRITE:
	case SOFTLEAF_DEVICE_PRIVATE_READ:
		return true;
	default:
		return false;
	}
}


static inline bool softleaf_is_device_exclusive(softleaf_t entry)
{
	return softleaf_type(entry) == SOFTLEAF_DEVICE_EXCLUSIVE;
}


static inline bool softleaf_is_hwpoison(softleaf_t entry)
{
	return softleaf_type(entry) == SOFTLEAF_HWPOISON;
}


static inline bool softleaf_is_marker(softleaf_t entry)
{
	return softleaf_type(entry) == SOFTLEAF_MARKER;
}


static inline pte_marker softleaf_to_marker(softleaf_t entry)
{
	VM_WARN_ON_ONCE(!softleaf_is_marker(entry));

	return swp_offset(entry) & PTE_MARKER_MASK;
}


static inline bool softleaf_has_pfn(softleaf_t entry)
{
	
	BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS);

	if (softleaf_is_migration(entry))
		return true;
	if (softleaf_is_device_private(entry))
		return true;
	if (softleaf_is_device_exclusive(entry))
		return true;
	if (softleaf_is_hwpoison(entry))
		return true;

	return false;
}


static inline unsigned long softleaf_to_pfn(softleaf_t entry)
{
	VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));

	
	return swp_offset(entry) & SWP_PFN_MASK;
}

static inline void softleaf_migration_sync(softleaf_t entry,
		struct folio *folio)
{
	
	smp_rmb();

	
	VM_WARN_ON_ONCE(!folio_test_locked(folio));
}


static inline struct page *softleaf_to_page(softleaf_t entry)
{
	struct page *page = pfn_to_page(softleaf_to_pfn(entry));

	VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
	if (softleaf_is_migration(entry))
		softleaf_migration_sync(entry, page_folio(page));

	return page;
}


static inline struct folio *softleaf_to_folio(softleaf_t entry)
{
	struct folio *folio = pfn_folio(softleaf_to_pfn(entry));

	VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
	if (softleaf_is_migration(entry))
		softleaf_migration_sync(entry, folio);

	return folio;
}


static inline bool softleaf_is_poison_marker(softleaf_t entry)
{
	if (!softleaf_is_marker(entry))
		return false;

	return softleaf_to_marker(entry) & PTE_MARKER_POISONED;
}


static inline bool softleaf_is_guard_marker(softleaf_t entry)
{
	if (!softleaf_is_marker(entry))
		return false;

	return softleaf_to_marker(entry) & PTE_MARKER_GUARD;
}


static inline bool softleaf_is_uffd_wp_marker(softleaf_t entry)
{
	if (!softleaf_is_marker(entry))
		return false;

	return softleaf_to_marker(entry) & PTE_MARKER_UFFD_WP;
}

#ifdef CONFIG_MIGRATION


static inline bool softleaf_is_migration_young(softleaf_t entry)
{
	VM_WARN_ON_ONCE(!softleaf_is_migration(entry));

	if (migration_entry_supports_ad())
		return swp_offset(entry) & SWP_MIG_YOUNG;
	
	return false;
}


static inline bool softleaf_is_migration_dirty(softleaf_t entry)
{
	VM_WARN_ON_ONCE(!softleaf_is_migration(entry));

	if (migration_entry_supports_ad())
		return swp_offset(entry) & SWP_MIG_DIRTY;
	
	return false;
}

#else 

static inline bool softleaf_is_migration_young(softleaf_t entry)
{
	return false;
}

static inline bool softleaf_is_migration_dirty(softleaf_t entry)
{
	return false;
}
#endif 


static inline bool pte_is_marker(pte_t pte)
{
	return softleaf_is_marker(softleaf_from_pte(pte));
}


static inline bool pte_is_uffd_wp_marker(pte_t pte)
{
	const softleaf_t entry = softleaf_from_pte(pte);

	return softleaf_is_uffd_wp_marker(entry);
}


static inline bool pte_is_uffd_marker(pte_t pte)
{
	const softleaf_t entry = softleaf_from_pte(pte);

	if (!softleaf_is_marker(entry))
		return false;

	
	if (softleaf_is_uffd_wp_marker(entry))
		return true;
	if (softleaf_is_poison_marker(entry))
		return true;

	return false;
}

#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION)


static inline bool pmd_is_device_private_entry(pmd_t pmd)
{
	return softleaf_is_device_private(softleaf_from_pmd(pmd));
}

#else  

static inline bool pmd_is_device_private_entry(pmd_t pmd)
{
	return false;
}

#endif 


static inline bool pmd_is_migration_entry(pmd_t pmd)
{
	return softleaf_is_migration(softleaf_from_pmd(pmd));
}


static inline bool pmd_is_valid_softleaf(pmd_t pmd)
{
	const softleaf_t entry = softleaf_from_pmd(pmd);

	
	return softleaf_is_device_private(entry) ||
		softleaf_is_migration(entry);
}

#endif  
#endif  
