/* SPDX-License-Identifier: GPL-2.0+ */


#ifndef _LINUX_SRCU_H
#define _LINUX_SRCU_H

#include <linux/mutex.h>
#include <linux/rcupdate.h>
#include <linux/workqueue.h>
#include <linux/rcu_segcblist.h>

context_lock_struct(srcu_struct, __reentrant_ctx_lock);

#ifdef CONFIG_DEBUG_LOCK_ALLOC

int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key);
#ifndef CONFIG_TINY_SRCU
int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key);
int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name,
				   struct lock_class_key *key);
#endif // #ifndef CONFIG_TINY_SRCU

#define init_srcu_struct(ssp) \
({ \
	static struct lock_class_key __srcu_key; \
	\
	__init_srcu_struct((ssp), #ssp, &__srcu_key); \
})

#define init_srcu_struct_fast(ssp) \
({ \
	static struct lock_class_key __srcu_key; \
	\
	__init_srcu_struct_fast((ssp), #ssp, &__srcu_key); \
})

#define init_srcu_struct_fast_updown(ssp) \
({ \
	static struct lock_class_key __srcu_key; \
	\
	__init_srcu_struct_fast_updown((ssp), #ssp, &__srcu_key); \
})

#define __SRCU_DEP_MAP_INIT(srcu_name)	.dep_map = { .name = #srcu_name },
#else 

int init_srcu_struct(struct srcu_struct *ssp);
#ifndef CONFIG_TINY_SRCU
int init_srcu_struct_fast(struct srcu_struct *ssp);
int init_srcu_struct_fast_updown(struct srcu_struct *ssp);
#endif // #ifndef CONFIG_TINY_SRCU

#define __SRCU_DEP_MAP_INIT(srcu_name)
#endif 


#define SRCU_READ_FLAVOR_NORMAL		0x1		// srcu_read_lock().
#define SRCU_READ_FLAVOR_NMI		0x2		// srcu_read_lock_nmisafe().
//					0x4		// SRCU-lite is no longer with us.
#define SRCU_READ_FLAVOR_FAST		0x4		// srcu_read_lock_fast().
#define SRCU_READ_FLAVOR_FAST_UPDOWN	0x8		// srcu_read_lock_fast().
#define SRCU_READ_FLAVOR_ALL		(SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \
					 SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN)
						// All of the above.
#define SRCU_READ_FLAVOR_SLOWGP		(SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN)
						// Flavors requiring synchronize_rcu()
						// instead of smp_mb().
void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases_shared(ssp);

#ifdef CONFIG_TINY_SRCU
#include <linux/srcutiny.h>
#elif defined(CONFIG_TREE_SRCU)
#include <linux/srcutree.h>
#else
#error "Unknown SRCU implementation specified to kernel configuration"
#endif

void call_srcu(struct srcu_struct *ssp, struct rcu_head *head,
		void (*func)(struct rcu_head *head));
void cleanup_srcu_struct(struct srcu_struct *ssp);
void synchronize_srcu(struct srcu_struct *ssp);

#define SRCU_GET_STATE_COMPLETED 0x1


static inline unsigned long get_completed_synchronize_srcu(void)
{
	return SRCU_GET_STATE_COMPLETED;
}

unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp);
unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp);
bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie);

// Maximum number of unsigned long values corresponding to
// not-yet-completed SRCU grace periods.
#define NUM_ACTIVE_SRCU_POLL_OLDSTATE 2


static inline bool same_state_synchronize_srcu(unsigned long oldstate1, unsigned long oldstate2)
{
	return oldstate1 == oldstate2;
}

#ifdef CONFIG_NEED_SRCU_NMI_SAFE
int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires_shared(ssp);
void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases_shared(ssp);
#else
static inline int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)
	__acquires_shared(ssp)
{
	return __srcu_read_lock(ssp);
}
static inline void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
	__releases_shared(ssp)
{
	__srcu_read_unlock(ssp, idx);
}
#endif 

void srcu_init(void);

#ifdef CONFIG_DEBUG_LOCK_ALLOC


static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
{
	if (!debug_lockdep_rcu_enabled())
		return 1;
	return lock_is_held(&ssp->dep_map);
}




static inline void srcu_lock_acquire(struct lockdep_map *map)
{
	lock_map_acquire_read(map);
}


static inline void srcu_lock_release(struct lockdep_map *map)
{
	lock_map_release(map);
}


static inline void srcu_lock_sync(struct lockdep_map *map)
{
	lock_map_sync(map);
}

#else 

static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
{
	return 1;
}

#define srcu_lock_acquire(m) do { } while (0)
#define srcu_lock_release(m) do { } while (0)
#define srcu_lock_sync(m) do { } while (0)

#endif 


static inline void __srcu_read_lock_must_hold(const struct srcu_struct *ssp) __must_hold_shared(ssp) { }


#define srcu_dereference_check(p, ssp, c)					\
({										\
	__srcu_read_lock_must_hold(ssp);					\
	__acquire_shared_ctx_lock(RCU);					\
	__auto_type __v = __rcu_dereference_check((p), __UNIQUE_ID(rcu),	\
				(c) || srcu_read_lock_held(ssp), __rcu);	\
	__release_shared_ctx_lock(RCU);					\
	__v;									\
})


#define srcu_dereference(p, ssp) srcu_dereference_check((p), (ssp), 0)


#define srcu_dereference_notrace(p, ssp) srcu_dereference_check((p), (ssp), 1)


static inline int srcu_read_lock(struct srcu_struct *ssp)
	__acquires_shared(ssp)
{
	int retval;

	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
	retval = __srcu_read_lock(ssp);
	srcu_lock_acquire(&ssp->dep_map);
	return retval;
}


static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *ssp) __acquires_shared(ssp)
	__acquires_shared(ssp)
{
	struct srcu_ctr __percpu *retval;

	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast().");
	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
	retval = __srcu_read_lock_fast(ssp);
	rcu_try_lock_acquire(&ssp->dep_map);
	return retval;
}


static inline struct srcu_ctr __percpu *srcu_read_lock_fast_updown(struct srcu_struct *ssp)
	__acquires_shared(ssp)
{
	struct srcu_ctr __percpu *retval;

	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast_updown().");
	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN);
	retval = __srcu_read_lock_fast_updown(ssp);
	rcu_try_lock_acquire(&ssp->dep_map);
	return retval;
}


static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_struct *ssp)
	__acquires_shared(ssp)
{
	struct srcu_ctr __percpu *retval;

	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
	retval = __srcu_read_lock_fast(ssp);
	return retval;
}


static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires_shared(ssp)
{
	WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_down_read_fast().");
	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN);
	return __srcu_read_lock_fast_updown(ssp);
}


static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp)
	__acquires_shared(ssp)
{
	int retval;

	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
	retval = __srcu_read_lock_nmisafe(ssp);
	rcu_try_lock_acquire(&ssp->dep_map);
	return retval;
}


static inline notrace int
srcu_read_lock_notrace(struct srcu_struct *ssp)
	__acquires_shared(ssp)
{
	int retval;

	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
	retval = __srcu_read_lock(ssp);
	return retval;
}


static inline int srcu_down_read(struct srcu_struct *ssp)
	__acquires_shared(ssp)
{
	WARN_ON_ONCE(in_nmi());
	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
	return __srcu_read_lock(ssp);
}


static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)
	__releases_shared(ssp)
{
	WARN_ON_ONCE(idx & ~0x1);
	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
	srcu_lock_release(&ssp->dep_map);
	__srcu_read_unlock(ssp, idx);
}


static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
	__releases_shared(ssp)
{
	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
	srcu_lock_release(&ssp->dep_map);
	__srcu_read_unlock_fast(ssp, scp);
	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_fast().");
}


static inline void
srcu_read_unlock_fast_updown(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) __releases_shared(ssp)
{
	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN);
	srcu_lock_release(&ssp->dep_map);
	__srcu_read_unlock_fast_updown(ssp, scp);
	RCU_LOCKDEP_WARN(!rcu_is_watching(),
			 "RCU must be watching srcu_read_unlock_fast_updown().");
}


static inline void srcu_read_unlock_fast_notrace(struct srcu_struct *ssp,
						 struct srcu_ctr __percpu *scp) __releases_shared(ssp)
{
	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
	__srcu_read_unlock_fast(ssp, scp);
}


static inline void srcu_up_read_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
	__releases_shared(ssp)
{
	WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN);
	__srcu_read_unlock_fast_updown(ssp, scp);
	RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_up_read_fast_updown().");
}


static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
	__releases_shared(ssp)
{
	WARN_ON_ONCE(idx & ~0x1);
	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
	rcu_lock_release(&ssp->dep_map);
	__srcu_read_unlock_nmisafe(ssp, idx);
}


static inline notrace void
srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases_shared(ssp)
{
	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
	__srcu_read_unlock(ssp, idx);
}


static inline void srcu_up_read(struct srcu_struct *ssp, int idx)
	__releases_shared(ssp)
{
	WARN_ON_ONCE(idx & ~0x1);
	WARN_ON_ONCE(in_nmi());
	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
	__srcu_read_unlock(ssp, idx);
}


static inline void smp_mb__after_srcu_read_unlock(void)
{
	
}


static inline void smp_mb__after_srcu_read_lock(void)
{
	
}

DEFINE_LOCK_GUARD_1(srcu, struct srcu_struct,
		    _T->idx = srcu_read_lock(_T->lock),
		    srcu_read_unlock(_T->lock, _T->idx),
		    int idx)
DECLARE_LOCK_GUARD_1_ATTRS(srcu, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T))
#define class_srcu_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu, _T)

DEFINE_LOCK_GUARD_1(srcu_fast, struct srcu_struct,
		    _T->scp = srcu_read_lock_fast(_T->lock),
		    srcu_read_unlock_fast(_T->lock, _T->scp),
		    struct srcu_ctr __percpu *scp)
DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T))
#define class_srcu_fast_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast, _T)

DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct,
		    _T->scp = srcu_read_lock_fast_notrace(_T->lock),
		    srcu_read_unlock_fast_notrace(_T->lock, _T->scp),
		    struct srcu_ctr __percpu *scp)
DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T))
#define class_srcu_fast_notrace_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, _T)

#endif
