/* SPDX-License-Identifier: GPL-2.0+ */


#ifndef __LINUX_RCUPDATE_TRACE_H
#define __LINUX_RCUPDATE_TRACE_H

#include <linux/sched.h>
#include <linux/rcupdate.h>
#include <linux/cleanup.h>

#ifdef CONFIG_TASKS_TRACE_RCU
extern struct srcu_struct rcu_tasks_trace_srcu_struct;
#endif // #ifdef CONFIG_TASKS_TRACE_RCU

#if defined(CONFIG_DEBUG_LOCK_ALLOC) && defined(CONFIG_TASKS_TRACE_RCU)

static inline int rcu_read_lock_trace_held(void)
{
	return srcu_read_lock_held(&rcu_tasks_trace_srcu_struct);
}

#else // #if defined(CONFIG_DEBUG_LOCK_ALLOC) && defined(CONFIG_TASKS_TRACE_RCU)

static inline int rcu_read_lock_trace_held(void)
{
	return 1;
}

#endif // #else // #if defined(CONFIG_DEBUG_LOCK_ALLOC) && defined(CONFIG_TASKS_TRACE_RCU)

#ifdef CONFIG_TASKS_TRACE_RCU


static inline struct srcu_ctr __percpu *rcu_read_lock_tasks_trace(void)
{
	struct srcu_ctr __percpu *ret = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct);

	rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map);
	if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
		smp_mb(); // Provide ordering on noinstr-incomplete architectures.
	return ret;
}


static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp)
{
	if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
		smp_mb(); // Provide ordering on noinstr-incomplete architectures.
	__srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp);
	srcu_lock_release(&rcu_tasks_trace_srcu_struct.dep_map);
}


static inline void rcu_read_lock_trace(void)
{
	struct task_struct *t = current;

	rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map);
	if (t->trc_reader_nesting++) {
		// In case we interrupted a Tasks Trace RCU reader.
		return;
	}
	barrier();  // nesting before scp to protect against interrupt handler.
	t->trc_reader_scp = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct);
	if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
		smp_mb(); // Placeholder for more selective ordering
}


static inline void rcu_read_unlock_trace(void)
{
	struct srcu_ctr __percpu *scp;
	struct task_struct *t = current;

	scp = t->trc_reader_scp;
	barrier();  // scp before nesting to protect against interrupt handler.
	if (!--t->trc_reader_nesting) {
		if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
			smp_mb(); // Placeholder for more selective ordering
		__srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp);
	}
	srcu_lock_release(&rcu_tasks_trace_srcu_struct.dep_map);
}


static inline void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func)
{
	call_srcu(&rcu_tasks_trace_srcu_struct, rhp, func);
}


static inline void synchronize_rcu_tasks_trace(void)
{
	synchronize_srcu(&rcu_tasks_trace_srcu_struct);
}


static inline void rcu_barrier_tasks_trace(void)
{
	srcu_barrier(&rcu_tasks_trace_srcu_struct);
}


static inline void rcu_tasks_trace_expedite_current(void)
{
	srcu_expedite_current(&rcu_tasks_trace_srcu_struct);
}

// Placeholders to enable stepwise transition.
void __init rcu_tasks_trace_suppress_unused(void);

#else

static inline void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func) { BUG(); }
static inline void rcu_read_lock_trace(void) { BUG(); }
static inline void rcu_read_unlock_trace(void) { BUG(); }
#endif 

DEFINE_LOCK_GUARD_0(rcu_tasks_trace,
	rcu_read_lock_trace(),
	rcu_read_unlock_trace())

#endif 
