/* SPDX-License-Identifier: GPL-2.0 */


#ifndef _DAMON_H_
#define _DAMON_H_

#include <linux/memcontrol.h>
#include <linux/mutex.h>
#include <linux/time64.h>
#include <linux/types.h>
#include <linux/random.h>


#define DAMON_MIN_REGION_SZ	PAGE_SIZE

#define DAMOS_MAX_SCORE		(99)


static inline unsigned long damon_rand(unsigned long l, unsigned long r)
{
	return l + get_random_u32_below(r - l);
}


struct damon_addr_range {
	unsigned long start;
	unsigned long end;
};


struct damon_size_range {
	unsigned long min;
	unsigned long max;
};


struct damon_region {
	struct damon_addr_range ar;
	unsigned long sampling_addr;
	unsigned int nr_accesses;
	unsigned int nr_accesses_bp;
	struct list_head list;

	unsigned int age;

	unsigned int last_nr_accesses;
};


struct damon_target {
	struct pid *pid;
	unsigned int nr_regions;
	struct list_head regions_list;
	struct list_head list;
	bool obsolete;
};


enum damos_action {
	DAMOS_WILLNEED,
	DAMOS_COLD,
	DAMOS_PAGEOUT,
	DAMOS_HUGEPAGE,
	DAMOS_NOHUGEPAGE,
	DAMOS_LRU_PRIO,
	DAMOS_LRU_DEPRIO,
	DAMOS_MIGRATE_HOT,
	DAMOS_MIGRATE_COLD,
	DAMOS_STAT,		
	NR_DAMOS_ACTIONS,
};


enum damos_quota_goal_metric {
	DAMOS_QUOTA_USER_INPUT,
	DAMOS_QUOTA_SOME_MEM_PSI_US,
	DAMOS_QUOTA_NODE_MEM_USED_BP,
	DAMOS_QUOTA_NODE_MEM_FREE_BP,
	DAMOS_QUOTA_NODE_MEMCG_USED_BP,
	DAMOS_QUOTA_NODE_MEMCG_FREE_BP,
	DAMOS_QUOTA_ACTIVE_MEM_BP,
	DAMOS_QUOTA_INACTIVE_MEM_BP,
	NR_DAMOS_QUOTA_GOAL_METRICS,
};


struct damos_quota_goal {
	enum damos_quota_goal_metric metric;
	unsigned long target_value;
	unsigned long current_value;
	
	union {
		u64 last_psi_total;
		struct {
			int nid;
			u64 memcg_id;
		};
	};
	struct list_head list;
};


struct damos_quota {
	unsigned long reset_interval;
	unsigned long ms;
	unsigned long sz;
	struct list_head goals;
	unsigned long esz;

	unsigned int weight_sz;
	unsigned int weight_nr_accesses;
	unsigned int weight_age;


	
	unsigned long total_charged_sz;
	unsigned long total_charged_ns;

	
	unsigned long charged_sz;
	unsigned long charged_from;
	struct damon_target *charge_target_from;
	unsigned long charge_addr_from;

	
	unsigned int min_score;

	
	unsigned long esz_bp;
};


enum damos_wmark_metric {
	DAMOS_WMARK_NONE,
	DAMOS_WMARK_FREE_MEM_RATE,
	NR_DAMOS_WMARK_METRICS,
};


struct damos_watermarks {
	enum damos_wmark_metric metric;
	unsigned long interval;
	unsigned long high;
	unsigned long mid;
	unsigned long low;


	bool activated;
};


struct damos_stat {
	unsigned long nr_tried;
	unsigned long sz_tried;
	unsigned long nr_applied;
	unsigned long sz_applied;
	unsigned long sz_ops_filter_passed;
	unsigned long qt_exceeds;
	unsigned long nr_snapshots;
};


enum damos_filter_type {
	DAMOS_FILTER_TYPE_ANON,
	DAMOS_FILTER_TYPE_ACTIVE,
	DAMOS_FILTER_TYPE_MEMCG,
	DAMOS_FILTER_TYPE_YOUNG,
	DAMOS_FILTER_TYPE_HUGEPAGE_SIZE,
	DAMOS_FILTER_TYPE_UNMAPPED,
	DAMOS_FILTER_TYPE_ADDR,
	DAMOS_FILTER_TYPE_TARGET,
	NR_DAMOS_FILTER_TYPES,
};


struct damos_filter {
	enum damos_filter_type type;
	bool matching;
	bool allow;
	union {
		u64 memcg_id;
		struct damon_addr_range addr_range;
		int target_idx;
		struct damon_size_range sz_range;
	};
	struct list_head list;
};

struct damon_ctx;
struct damos;


struct damos_walk_control {
	void (*walk_fn)(void *data, struct damon_ctx *ctx,
			struct damon_target *t, struct damon_region *r,
			struct damos *s, unsigned long sz_filter_passed);
	void *data;

	
	struct completion completion;
	
	bool canceled;
};


struct damos_access_pattern {
	unsigned long min_sz_region;
	unsigned long max_sz_region;
	unsigned int min_nr_accesses;
	unsigned int max_nr_accesses;
	unsigned int min_age_region;
	unsigned int max_age_region;
};


struct damos_migrate_dests {
	unsigned int *node_id_arr;
	unsigned int *weight_arr;
	size_t nr_dests;
};


struct damos {
	struct damos_access_pattern pattern;
	enum damos_action action;
	unsigned long apply_interval_us;

	
	unsigned long next_apply_sis;
	
	bool walk_completed;
	
	bool core_filters_allowed;
	
	bool core_filters_default_reject;
	bool ops_filters_default_reject;

	struct damos_quota quota;
	struct damos_watermarks wmarks;
	union {
		struct {
			int target_nid;
			struct damos_migrate_dests migrate_dests;
		};
	};
	struct list_head core_filters;
	struct list_head ops_filters;
	void *last_applied;
	struct damos_stat stat;
	unsigned long max_nr_snapshots;
	struct list_head list;
};


enum damon_ops_id {
	DAMON_OPS_VADDR,
	DAMON_OPS_FVADDR,
	DAMON_OPS_PADDR,
	NR_DAMON_OPS,
};


struct damon_operations {
	enum damon_ops_id id;
	void (*init)(struct damon_ctx *context);
	void (*update)(struct damon_ctx *context);
	void (*prepare_access_checks)(struct damon_ctx *context);
	unsigned int (*check_accesses)(struct damon_ctx *context);
	int (*get_scheme_score)(struct damon_ctx *context,
			struct damon_target *t, struct damon_region *r,
			struct damos *scheme);
	unsigned long (*apply_scheme)(struct damon_ctx *context,
			struct damon_target *t, struct damon_region *r,
			struct damos *scheme, unsigned long *sz_filter_passed);
	bool (*target_valid)(struct damon_target *t);
	void (*cleanup_target)(struct damon_target *t);
};


struct damon_call_control {
	int (*fn)(void *data);
	void *data;
	bool repeat;
	int return_code;
	bool dealloc_on_cancel;

	
	struct completion completion;
	
	bool canceled;
	
	struct list_head list;
};


struct damon_intervals_goal {
	unsigned long access_bp;
	unsigned long aggrs;
	unsigned long min_sample_us;
	unsigned long max_sample_us;
};


struct damon_attrs {
	unsigned long sample_interval;
	unsigned long aggr_interval;
	unsigned long ops_update_interval;
	struct damon_intervals_goal intervals_goal;
	unsigned long min_nr_regions;
	unsigned long max_nr_regions;

	
	unsigned long aggr_samples;
};


struct damon_ctx {
	struct damon_attrs attrs;


	
	unsigned long passed_sample_intervals;
	
	unsigned long next_aggregation_sis;
	
	unsigned long next_ops_update_sis;
	
	unsigned long next_intervals_tune_sis;
	
	struct completion kdamond_started;
	
	unsigned long *regions_score_histogram;

	
	struct list_head call_controls;
	bool call_controls_obsolete;
	struct mutex call_controls_lock;

	struct damos_walk_control *walk_control;
	bool walk_control_obsolete;
	struct mutex walk_control_lock;

	
	bool maybe_corrupted;

	
	struct task_struct *kdamond;
	
	struct mutex kdamond_lock;


	struct damon_operations ops;
	unsigned long addr_unit;
	unsigned long min_region_sz;

	struct list_head adaptive_targets;
	struct list_head schemes;
};

static inline struct damon_region *damon_next_region(struct damon_region *r)
{
	return container_of(r->list.next, struct damon_region, list);
}

static inline struct damon_region *damon_prev_region(struct damon_region *r)
{
	return container_of(r->list.prev, struct damon_region, list);
}

static inline struct damon_region *damon_last_region(struct damon_target *t)
{
	return list_last_entry(&t->regions_list, struct damon_region, list);
}

static inline struct damon_region *damon_first_region(struct damon_target *t)
{
	return list_first_entry(&t->regions_list, struct damon_region, list);
}

static inline unsigned long damon_sz_region(struct damon_region *r)
{
	return r->ar.end - r->ar.start;
}


#define damon_for_each_region(r, t) \
	list_for_each_entry(r, &t->regions_list, list)

#define damon_for_each_region_from(r, t) \
	list_for_each_entry_from(r, &t->regions_list, list)

#define damon_for_each_region_safe(r, next, t) \
	list_for_each_entry_safe(r, next, &t->regions_list, list)

#define damon_for_each_target(t, ctx) \
	list_for_each_entry(t, &(ctx)->adaptive_targets, list)

#define damon_for_each_target_safe(t, next, ctx)	\
	list_for_each_entry_safe(t, next, &(ctx)->adaptive_targets, list)

#define damon_for_each_scheme(s, ctx) \
	list_for_each_entry(s, &(ctx)->schemes, list)

#define damon_for_each_scheme_safe(s, next, ctx) \
	list_for_each_entry_safe(s, next, &(ctx)->schemes, list)

#define damos_for_each_quota_goal(goal, quota) \
	list_for_each_entry(goal, &quota->goals, list)

#define damos_for_each_quota_goal_safe(goal, next, quota) \
	list_for_each_entry_safe(goal, next, &(quota)->goals, list)

#define damos_for_each_core_filter(f, scheme) \
	list_for_each_entry(f, &(scheme)->core_filters, list)

#define damos_for_each_core_filter_safe(f, next, scheme) \
	list_for_each_entry_safe(f, next, &(scheme)->core_filters, list)

#define damos_for_each_ops_filter(f, scheme) \
	list_for_each_entry(f, &(scheme)->ops_filters, list)

#define damos_for_each_ops_filter_safe(f, next, scheme) \
	list_for_each_entry_safe(f, next, &(scheme)->ops_filters, list)

#ifdef CONFIG_DAMON

struct damon_region *damon_new_region(unsigned long start, unsigned long end);


static inline void damon_insert_region(struct damon_region *r,
		struct damon_region *prev, struct damon_region *next,
		struct damon_target *t)
{
	__list_add(&r->list, &prev->list, &next->list);
	t->nr_regions++;
}

void damon_add_region(struct damon_region *r, struct damon_target *t);
void damon_destroy_region(struct damon_region *r, struct damon_target *t);
int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges,
		unsigned int nr_ranges, unsigned long min_region_sz);
void damon_update_region_access_rate(struct damon_region *r, bool accessed,
		struct damon_attrs *attrs);

struct damos_filter *damos_new_filter(enum damos_filter_type type,
		bool matching, bool allow);
void damos_add_filter(struct damos *s, struct damos_filter *f);
bool damos_filter_for_ops(enum damos_filter_type type);
void damos_destroy_filter(struct damos_filter *f);

struct damos_quota_goal *damos_new_quota_goal(
		enum damos_quota_goal_metric metric,
		unsigned long target_value);
void damos_add_quota_goal(struct damos_quota *q, struct damos_quota_goal *g);
void damos_destroy_quota_goal(struct damos_quota_goal *goal);

struct damos *damon_new_scheme(struct damos_access_pattern *pattern,
			enum damos_action action,
			unsigned long apply_interval_us,
			struct damos_quota *quota,
			struct damos_watermarks *wmarks,
			int target_nid);
void damon_add_scheme(struct damon_ctx *ctx, struct damos *s);
void damon_destroy_scheme(struct damos *s);
int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src);

struct damon_target *damon_new_target(void);
void damon_add_target(struct damon_ctx *ctx, struct damon_target *t);
bool damon_targets_empty(struct damon_ctx *ctx);
void damon_free_target(struct damon_target *t);
void damon_destroy_target(struct damon_target *t, struct damon_ctx *ctx);
unsigned int damon_nr_regions(struct damon_target *t);

struct damon_ctx *damon_new_ctx(void);
void damon_destroy_ctx(struct damon_ctx *ctx);
int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs);
void damon_set_schemes(struct damon_ctx *ctx,
			struct damos **schemes, ssize_t nr_schemes);
int damon_commit_ctx(struct damon_ctx *old_ctx, struct damon_ctx *new_ctx);
int damon_nr_running_ctxs(void);
bool damon_is_registered_ops(enum damon_ops_id id);
int damon_register_ops(struct damon_operations *ops);
int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id);

static inline bool damon_target_has_pid(const struct damon_ctx *ctx)
{
	return ctx->ops.id == DAMON_OPS_VADDR || ctx->ops.id == DAMON_OPS_FVADDR;
}

static inline unsigned int damon_max_nr_accesses(const struct damon_attrs *attrs)
{
	
	return min(attrs->aggr_interval / attrs->sample_interval,
			(unsigned long)UINT_MAX);
}


bool damon_initialized(void);
int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive);
int damon_stop(struct damon_ctx **ctxs, int nr_ctxs);
bool damon_is_running(struct damon_ctx *ctx);
int damon_kdamond_pid(struct damon_ctx *ctx);

int damon_call(struct damon_ctx *ctx, struct damon_call_control *control);
int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control);

int damon_set_region_biggest_system_ram_default(struct damon_target *t,
				unsigned long *start, unsigned long *end,
				unsigned long min_region_sz);

#endif	

#endif	
