/* SPDX-License-Identifier: GPL-2.0-only OR MIT */


#ifndef __DRM_GPUSVM_H__
#define __DRM_GPUSVM_H__

#include <linux/kref.h>
#include <linux/interval_tree.h>
#include <linux/mmu_notifier.h>

struct dev_pagemap_ops;
struct drm_device;
struct drm_gpusvm;
struct drm_gpusvm_notifier;
struct drm_gpusvm_ops;
struct drm_gpusvm_range;
struct drm_pagemap;
struct drm_pagemap_addr;


struct drm_gpusvm_ops {
	
	struct drm_gpusvm_notifier *(*notifier_alloc)(void);

	
	void (*notifier_free)(struct drm_gpusvm_notifier *notifier);

	
	struct drm_gpusvm_range *(*range_alloc)(struct drm_gpusvm *gpusvm);

	
	void (*range_free)(struct drm_gpusvm_range *range);

	
	void (*invalidate)(struct drm_gpusvm *gpusvm,
			   struct drm_gpusvm_notifier *notifier,
			   const struct mmu_notifier_range *mmu_range);
};


struct drm_gpusvm_notifier {
	struct drm_gpusvm *gpusvm;
	struct mmu_interval_notifier notifier;
	struct interval_tree_node itree;
	struct list_head entry;
	struct rb_root_cached root;
	struct list_head range_list;
	struct {
		u32 removed : 1;
	} flags;
};


struct drm_gpusvm_pages_flags {
	union {
		struct {
			
			u16 migrate_devmem : 1;
			
			u16 unmapped : 1;
			u16 partial_unmap : 1;
			u16 has_devmem_pages : 1;
			u16 has_dma_mapping : 1;
		};
		u16 __flags;
	};
};


struct drm_gpusvm_pages {
	struct drm_pagemap_addr *dma_addr;
	struct drm_pagemap *dpagemap;
	unsigned long notifier_seq;
	struct drm_gpusvm_pages_flags flags;
};


struct drm_gpusvm_range {
	struct drm_gpusvm *gpusvm;
	struct drm_gpusvm_notifier *notifier;
	struct kref refcount;
	struct interval_tree_node itree;
	struct list_head entry;
	struct drm_gpusvm_pages pages;
};


struct drm_gpusvm {
	const char *name;
	struct drm_device *drm;
	struct mm_struct *mm;
	unsigned long mm_start;
	unsigned long mm_range;
	unsigned long notifier_size;
	const struct drm_gpusvm_ops *ops;
	const unsigned long *chunk_sizes;
	int num_chunks;
	struct rw_semaphore notifier_lock;
	struct rb_root_cached root;
	struct list_head notifier_list;
#ifdef CONFIG_LOCKDEP
	
	struct lockdep_map *lock_dep_map;
#endif
};


struct drm_gpusvm_ctx {
	void *device_private_page_owner;
	unsigned long check_pages_threshold;
	unsigned long timeslice_ms;
	unsigned int in_notifier :1;
	unsigned int read_only :1;
	unsigned int devmem_possible :1;
	unsigned int devmem_only :1;
	unsigned int allow_mixed :1;
};

int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
		    const char *name, struct drm_device *drm,
		    struct mm_struct *mm,
		    unsigned long mm_start, unsigned long mm_range,
		    unsigned long notifier_size,
		    const struct drm_gpusvm_ops *ops,
		    const unsigned long *chunk_sizes, int num_chunks);

void drm_gpusvm_fini(struct drm_gpusvm *gpusvm);

void drm_gpusvm_free(struct drm_gpusvm *gpusvm);

unsigned long
drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm,
			  unsigned long start,
			  unsigned long end);

struct drm_gpusvm_range *
drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm,
				unsigned long fault_addr,
				unsigned long gpuva_start,
				unsigned long gpuva_end,
				const struct drm_gpusvm_ctx *ctx);

void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm,
			     struct drm_gpusvm_range *range);

int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm,
			   struct drm_gpusvm_range *range);

struct drm_gpusvm_range *
drm_gpusvm_range_get(struct drm_gpusvm_range *range);

void drm_gpusvm_range_put(struct drm_gpusvm_range *range);

bool drm_gpusvm_range_pages_valid(struct drm_gpusvm *gpusvm,
				  struct drm_gpusvm_range *range);

int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm,
			       struct drm_gpusvm_range *range,
			       const struct drm_gpusvm_ctx *ctx);

void drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm,
				  struct drm_gpusvm_range *range,
				  const struct drm_gpusvm_ctx *ctx);

bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start,
			    unsigned long end);

struct drm_gpusvm_notifier *
drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start,
			 unsigned long end);

struct drm_gpusvm_range *
drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,
		      unsigned long end);

void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range,
				   const struct mmu_notifier_range *mmu_range);

int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
			 struct drm_gpusvm_pages *svm_pages,
			 struct mm_struct *mm,
			 struct mmu_interval_notifier *notifier,
			 unsigned long pages_start, unsigned long pages_end,
			 const struct drm_gpusvm_ctx *ctx);

void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
			    struct drm_gpusvm_pages *svm_pages,
			    unsigned long npages,
			    const struct drm_gpusvm_ctx *ctx);

void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
			   struct drm_gpusvm_pages *svm_pages,
			   unsigned long npages);


enum drm_gpusvm_scan_result {
	DRM_GPUSVM_SCAN_UNPOPULATED,
	DRM_GPUSVM_SCAN_EQUAL,
	DRM_GPUSVM_SCAN_OTHER,
	DRM_GPUSVM_SCAN_SYSTEM,
	DRM_GPUSVM_SCAN_MIXED_DEVICE,
	DRM_GPUSVM_SCAN_MIXED,
};

enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range,
					       void *dev_private_owner,
					       const struct dev_pagemap *pagemap);

#ifdef CONFIG_LOCKDEP

#define drm_gpusvm_driver_set_lock(gpusvm, lock) \
	do { \
		if (!WARN((gpusvm)->lock_dep_map, \
			  "GPUSVM range lock should be set only once."))\
			(gpusvm)->lock_dep_map = &(lock)->dep_map;	\
	} while (0)
#else
#define drm_gpusvm_driver_set_lock(gpusvm, lock) do {} while (0)
#endif


#define drm_gpusvm_notifier_lock(gpusvm__)	\
	down_read(&(gpusvm__)->notifier_lock)


#define drm_gpusvm_notifier_unlock(gpusvm__)	\
	up_read(&(gpusvm__)->notifier_lock)


static inline unsigned long
drm_gpusvm_range_start(struct drm_gpusvm_range *range)
{
	return range->itree.start;
}


static inline unsigned long
drm_gpusvm_range_end(struct drm_gpusvm_range *range)
{
	return range->itree.last + 1;
}


static inline unsigned long
drm_gpusvm_range_size(struct drm_gpusvm_range *range)
{
	return drm_gpusvm_range_end(range) - drm_gpusvm_range_start(range);
}


static inline unsigned long
drm_gpusvm_notifier_start(struct drm_gpusvm_notifier *notifier)
{
	return notifier->itree.start;
}


static inline unsigned long
drm_gpusvm_notifier_end(struct drm_gpusvm_notifier *notifier)
{
	return notifier->itree.last + 1;
}


static inline unsigned long
drm_gpusvm_notifier_size(struct drm_gpusvm_notifier *notifier)
{
	return drm_gpusvm_notifier_end(notifier) -
		drm_gpusvm_notifier_start(notifier);
}


static inline struct drm_gpusvm_range *
__drm_gpusvm_range_next(struct drm_gpusvm_range *range)
{
	if (range && !list_is_last(&range->entry,
				   &range->notifier->range_list))
		return list_next_entry(range, entry);

	return NULL;
}


#define drm_gpusvm_for_each_range(range__, notifier__, start__, end__)	\
	for ((range__) = (range__) ?:					\
	     drm_gpusvm_range_find((notifier__), (start__), (end__));	\
	     (range__) && (drm_gpusvm_range_start(range__) < (end__));	\
	     (range__) = __drm_gpusvm_range_next(range__))


#define drm_gpusvm_for_each_range_safe(range__, next__, notifier__, start__, end__)	\
	for ((range__) = drm_gpusvm_range_find((notifier__), (start__), (end__)),	\
	     (next__) = __drm_gpusvm_range_next(range__);				\
	     (range__) && (drm_gpusvm_range_start(range__) < (end__));			\
	     (range__) = (next__), (next__) = __drm_gpusvm_range_next(range__))


static inline struct drm_gpusvm_notifier *
__drm_gpusvm_notifier_next(struct drm_gpusvm_notifier *notifier)
{
	if (notifier && !list_is_last(&notifier->entry,
				      &notifier->gpusvm->notifier_list))
		return list_next_entry(notifier, entry);

	return NULL;
}


#define drm_gpusvm_for_each_notifier(notifier__, gpusvm__, start__, end__)		\
	for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__));	\
	     (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__));		\
	     (notifier__) = __drm_gpusvm_notifier_next(notifier__))


#define drm_gpusvm_for_each_notifier_safe(notifier__, next__, gpusvm__, start__, end__)	\
	for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__)),	\
	     (next__) = __drm_gpusvm_notifier_next(notifier__);				\
	     (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__));		\
	     (notifier__) = (next__), (next__) = __drm_gpusvm_notifier_next(notifier__))

#endif 
