sched_ext机制研究

  1/*
  2 * This is the main, per-CPU runqueue data structure.
  3 *
  4 * Locking rule: those places that want to lock multiple runqueues
  5 * (such as the load balancing or the thread migration code), lock
  6 * acquire operations must be ordered by ascending &runqueue.
  7 */
  8struct rq {
  9	/* runqueue lock: */
 10	raw_spinlock_t		__lock;
 11
 12	/*
 13	 * nr_running and cpu_load should be in the same cacheline because
 14	 * remote CPUs use both these fields when doing load calculation.
 15	 */
 16	unsigned int		nr_running;
 17#ifdef CONFIG_NUMA_BALANCING
 18	unsigned int		nr_numa_running;
 19	unsigned int		nr_preferred_running;
 20	unsigned int		numa_migrate_on;
 21#endif
 22#ifdef CONFIG_NO_HZ_COMMON
 23#ifdef CONFIG_SMP
 24	unsigned long		last_blocked_load_update_tick;
 25	unsigned int		has_blocked_load;
 26	call_single_data_t	nohz_csd;
 27#endif /* CONFIG_SMP */
 28	unsigned int		nohz_tick_stopped;
 29	atomic_t		nohz_flags;
 30#endif /* CONFIG_NO_HZ_COMMON */
 31
 32#ifdef CONFIG_SMP
 33	unsigned int		ttwu_pending;
 34#endif
 35	u64			nr_switches;
 36
 37#ifdef CONFIG_UCLAMP_TASK
 38	/* Utilization clamp values based on CPU's RUNNABLE tasks */
 39	struct uclamp_rq	uclamp[UCLAMP_CNT] ____cacheline_aligned;
 40	unsigned int		uclamp_flags;
 41#define UCLAMP_FLAG_IDLE 0x01
 42#endif
 43
 44	struct cfs_rq		cfs;
 45	struct rt_rq		rt;
 46	struct dl_rq		dl;
 47#ifdef CONFIG_SCHED_CLASS_EXT
 48	struct scx_rq		scx;
 49#endif
 50
 51#ifdef CONFIG_FAIR_GROUP_SCHED
 52	/* list of leaf cfs_rq on this CPU: */
 53	struct list_head	leaf_cfs_rq_list;
 54	struct list_head	*tmp_alone_branch;
 55#endif /* CONFIG_FAIR_GROUP_SCHED */
 56
 57	/*
 58	 * This is part of a global counter where only the total sum
 59	 * over all CPUs matters. A task can increase this counter on
 60	 * one CPU and if it got migrated afterwards it may decrease
 61	 * it on another CPU. Always updated under the runqueue lock:
 62	 */
 63	unsigned int		nr_uninterruptible;
 64
 65	struct task_struct __rcu	*curr;
 66	struct task_struct	*idle;
 67	struct task_struct	*stop;
 68	unsigned long		next_balance;
 69	struct mm_struct	*prev_mm;
 70
 71	unsigned int		clock_update_flags;
 72	u64			clock;
 73	/* Ensure that all clocks are in the same cache line */
 74	u64			clock_task ____cacheline_aligned;
 75	u64			clock_pelt;
 76	unsigned long		lost_idle_time;
 77	u64			clock_pelt_idle;
 78	u64			clock_idle;
 79#ifndef CONFIG_64BIT
 80	u64			clock_pelt_idle_copy;
 81	u64			clock_idle_copy;
 82#endif
 83
 84	atomic_t		nr_iowait;
 85
 86#ifdef CONFIG_SCHED_DEBUG
 87	u64 last_seen_need_resched_ns;
 88	int ticks_without_resched;
 89#endif
 90
 91#ifdef CONFIG_MEMBARRIER
 92	int membarrier_state;
 93#endif
 94
 95#ifdef CONFIG_SMP
 96	struct root_domain		*rd;
 97	struct sched_domain __rcu	*sd;
 98
 99	unsigned long		cpu_capacity;
100	unsigned long		cpu_capacity_orig;
101
102	struct balance_callback *balance_callback;
103
104	unsigned char		nohz_idle_balance;
105	unsigned char		idle_balance;
106
107	unsigned long		misfit_task_load;
108
109	/* For active balancing */
110	int			active_balance;
111	int			push_cpu;
112	struct cpu_stop_work	active_balance_work;
113
114	/* CPU of this runqueue: */
115	int			cpu;
116	int			online;
117
118	struct list_head cfs_tasks;
119
120	struct sched_avg	avg_rt;
121	struct sched_avg	avg_dl;
122#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
123	struct sched_avg	avg_irq;
124#endif
125#ifdef CONFIG_SCHED_THERMAL_PRESSURE
126	struct sched_avg	avg_thermal;
127#endif
128	u64			idle_stamp;
129	u64			avg_idle;
130
131	unsigned long		wake_stamp;
132	u64			wake_avg_idle;
133
134	/* This is used to determine avg_idle's max value */
135	u64			max_idle_balance_cost;
136
137#ifdef CONFIG_HOTPLUG_CPU
138	struct rcuwait		hotplug_wait;
139#endif
140#endif /* CONFIG_SMP */
141
142#ifdef CONFIG_IRQ_TIME_ACCOUNTING
143	u64			prev_irq_time;
144#endif
145#ifdef CONFIG_PARAVIRT
146	u64			prev_steal_time;
147#endif
148#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
149	u64			prev_steal_time_rq;
150#endif
151
152	/* calc_load related fields */
153	unsigned long		calc_load_update;
154	long			calc_load_active;
155
156#ifdef CONFIG_SCHED_HRTICK
157#ifdef CONFIG_SMP
158	call_single_data_t	hrtick_csd;
159#endif
160	struct hrtimer		hrtick_timer;
161	ktime_t 		hrtick_time;
162#endif
163
164#ifdef CONFIG_SCHEDSTATS
165	/* latency stats */
166	struct sched_info	rq_sched_info;
167	unsigned long long	rq_cpu_time;
168	/* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
169
170	/* sys_sched_yield() stats */
171	unsigned int		yld_count;
172
173	/* schedule() stats */
174	unsigned int		sched_count;
175	unsigned int		sched_goidle;
176
177	/* try_to_wake_up() stats */
178	unsigned int		ttwu_count;
179	unsigned int		ttwu_local;
180#endif
181
182#ifdef CONFIG_CPU_IDLE
183	/* Must be inspected within a rcu lock section */
184	struct cpuidle_state	*idle_state;
185#endif
186
187#ifdef CONFIG_SMP
188	unsigned int		nr_pinned;
189#endif
190	unsigned int		push_busy;
191	struct cpu_stop_work	push_work;
192
193#ifdef CONFIG_SCHED_CORE
194	/* per rq */
195	struct rq		*core;
196	struct task_struct	*core_pick;
197	unsigned int		core_enabled;
198	unsigned int		core_sched_seq;
199	struct rb_root		core_tree;
200
201	/* shared state -- careful with sched_core_cpu_deactivate() */
202	unsigned int		core_task_seq;
203	unsigned int		core_pick_seq;
204	unsigned long		core_cookie;
205	unsigned int		core_forceidle_count;
206	unsigned int		core_forceidle_seq;
207	unsigned int		core_forceidle_occupation;
208	u64			core_forceidle_start;
209#endif
210};

 1#ifdef CONFIG_SCHED_CLASS_EXT
 2/* scx_rq->flags, protected by the rq lock */
 3enum scx_rq_flags {
 4	SCX_RQ_CAN_STOP_TICK	= 1 << 0,
 5};
 6
 7struct scx_rq {
 8	struct scx_dispatch_q	local_dsq;
 9	struct list_head	watchdog_list;
10	u64			ops_qseq;
11	u32			nr_running;
12	u32			flags;
13	bool			cpu_released;
14#ifdef CONFIG_SMP
15	cpumask_var_t		cpus_to_kick;
16	cpumask_var_t		cpus_to_preempt;
17	cpumask_var_t		cpus_to_wait;
18	u64			pnt_seq;
19	struct irq_work		kick_cpus_irq_work;
20#endif
21};
22#endif /* CONFIG_SCHED_CLASS_EXT */

  1/**
  2 * struct sched_ext_ops - Operation table for BPF scheduler implementation
  3 *
  4 * Userland can implement an arbitrary scheduling policy by implementing and
  5 * loading operations in this table.
  6 */
  7struct sched_ext_ops {
  8	/**
  9	 * select_cpu - Pick the target CPU for a task which is being woken up
 10	 * @p: task being woken up
 11	 * @prev_cpu: the cpu @p was on before sleeping
 12	 * @wake_flags: SCX_WAKE_*
 13	 *
 14	 * Decision made here isn't final. @p may be moved to any CPU while it
 15	 * is getting dispatched for execution later. However, as @p is not on
 16	 * the rq at this point, getting the eventual execution CPU right here
 17	 * saves a small bit of overhead down the line.
 18	 *
 19	 * If an idle CPU is returned, the CPU is kicked and will try to
 20	 * dispatch. While an explicit custom mechanism can be added,
 21	 * select_cpu() serves as the default way to wake up idle CPUs.
 22	 */
 23	s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags);
 24
 25	/**
 26	 * enqueue - Enqueue a task on the BPF scheduler
 27	 * @p: task being enqueued
 28	 * @enq_flags: %SCX_ENQ_*
 29	 *
 30	 * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch()
 31	 * or enqueue on the BPF scheduler. If not directly dispatched, the bpf
 32	 * scheduler owns @p and if it fails to dispatch @p, the task will
 33	 * stall.
 34	 */
 35	void (*enqueue)(struct task_struct *p, u64 enq_flags);
 36
 37	/**
 38	 * dequeue - Remove a task from the BPF scheduler
 39	 * @p: task being dequeued
 40	 * @deq_flags: %SCX_DEQ_*
 41	 *
 42	 * Remove @p from the BPF scheduler. This is usually called to isolate
 43	 * the task while updating its scheduling properties (e.g. priority).
 44	 *
 45	 * The ext core keeps track of whether the BPF side owns a given task or
 46	 * not and can gracefully ignore spurious dispatches from BPF side,
 47	 * which makes it safe to not implement this method. However, depending
 48	 * on the scheduling logic, this can lead to confusing behaviors - e.g.
 49	 * scheduling position not being updated across a priority change.
 50	 */
 51	void (*dequeue)(struct task_struct *p, u64 deq_flags);
 52
 53	/**
 54	 * dispatch - Dispatch tasks from the BPF scheduler into dsq's
 55	 * @cpu: CPU to dispatch tasks for
 56	 * @prev: previous task being switched out
 57	 *
 58	 * Called when a CPU can't find a task to execute after ops.consume().
 59	 * The operation should dispatch one or more tasks from the BPF
 60	 * scheduler to the dsq's using scx_bpf_dispatch(). The maximum number
 61	 * of tasks which can be dispatched in a single call is specified by the
 62	 * @dispatch_max_batch field of this struct.
 63	 */
 64	void (*dispatch)(s32 cpu, struct task_struct *prev);
 65
 66	/**
 67	 * consume - Consume tasks from the dsq's to the local dsq for execution
 68	 * @cpu: CPU to consume tasks for
 69	 *
 70	 * Called when a CPU's local dsq is empty. The operation should transfer
 71	 * one or more tasks from the dsq's to the CPU's local dsq using
 72	 * scx_bpf_consume(). If this function fails to fill the local dsq,
 73	 * ops.dispatch() will be called.
 74	 *
 75	 * This operation is unnecessary if the BPF scheduler always dispatches
 76	 * either to one of the local dsq's or the global dsq. If implemented,
 77	 * this operation is also responsible for consuming the global_dsq.
 78	 */
 79	void (*consume)(s32 cpu);
 80
 81	/**
 82	 * consume_final - Final consume call before going idle
 83	 * @cpu: CPU to consume tasks for
 84	 *
 85	 * After ops.consume() and .dispatch(), @cpu still doesn't have a task
 86	 * to execute and is about to go idle. This operation can be used to
 87	 * implement more aggressive consumption strategies. Otherwise
 88	 * equivalent to ops.consume().
 89	 */
 90	void (*consume_final)(s32 cpu);
 91
 92	/**
 93	 * runnable - A task is becoming runnable on its associated CPU
 94	 * @p: task becoming runnable
 95	 * @enq_flags: %SCX_ENQ_*
 96	 *
 97	 * This and the following three functions can be used to track a task's
 98	 * execution state transitions. A task becomes ->runnable() on a CPU,
 99	 * and then goes through one or more ->running() and ->stopping() pairs
100	 * as it runs on the CPU, and eventually becomes ->quiescent() when it's
101	 * done running on the CPU.
102	 *
103	 * @p is becoming runnable on the CPU because it's
104	 *
105	 * - waking up (%SCX_ENQ_WAKEUP)
106	 * - being moved from another CPU
107	 * - being restored after temporarily taken off the queue for an
108	 *   attribute change.
109	 *
110	 * This and ->enqueue() are related but not coupled. This operation
111	 * notifies @p's state transition and may not be followed by ->enqueue()
112	 * e.g. when @p is being dispatched to a remote CPU. Likewise, a task
113	 * may be ->enqueue()'d without being preceded by this operation e.g.
114	 * after exhausting its slice.
115	 */
116	void (*runnable)(struct task_struct *p, u64 enq_flags);
117
118	/**
119	 * running - A task is starting to run on its associated CPU
120	 * @p: task starting to run
121	 *
122	 * See ->runnable() for explanation on the task state notifiers.
123	 */
124	void (*running)(struct task_struct *p);
125
126	/**
127	 * stopping - A task is stopping execution
128	 * @p: task stopping to run
129	 * @runnable: is task @p still runnable?
130	 *
131	 * See ->runnable() for explanation on the task state notifiers. If
132	 * !@runnable, ->quiescent() will be invoked after this operation
133	 * returns.
134	 */
135	void (*stopping)(struct task_struct *p, bool runnable);
136
137	/**
138	 * quiescent - A task is becoming not runnable on its associated CPU
139	 * @p: task becoming not runnable
140	 * @deq_flags: %SCX_DEQ_*
141	 *
142	 * See ->runnable() for explanation on the task state notifiers.
143	 *
144	 * @p is becoming quiescent on the CPU because it's
145	 *
146	 * - sleeping (%SCX_DEQ_SLEEP)
147	 * - being moved to another CPU
148	 * - being temporarily taken off the queue for an attribute change
149	 *   (%SCX_DEQ_SAVE)
150	 *
151	 * This and ->dequeue() are related but not coupled. This operation
152	 * notifies @p's state transition and may not be preceded by ->dequeue()
153	 * e.g. when @p is being dispatched to a remote CPU.
154	 */
155	void (*quiescent)(struct task_struct *p, u64 deq_flags);
156
157	/**
158	 * yield - Yield CPU
159	 * @from: yielding task
160	 * @to: optional yield target task
161	 *
162	 * If @to is NULL, @from is yielding the CPU to other runnable tasks.
163	 * The BPF scheduler should ensure that other available tasks are
164	 * dispatched before the yielding task. Return value is ignored in this
165	 * case.
166	 *
167	 * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf
168	 * scheduler can implement the request, return %true; otherwise, %false.
169	 */
170	bool (*yield)(struct task_struct *from, struct task_struct *to);
171
172	/**
173	 * set_cpumask - Set CPU affinity
174	 * @p: task to set CPU affinity for
175	 * @cpumask: cpumask of cpus that @p can run on
176	 *
177	 * Update @p's CPU affinity to @cpumask.
178	 */
179	void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask);
180
181	/**
182	 * update_idle - Update the idle state of a CPU
183	 * @cpu: CPU to udpate the idle state for
184	 * @idle: whether entering or exiting the idle state
185	 *
186	 * This operation is called when @rq's CPU goes or leaves the idle
187	 * state. By default, implementing this operation disables the built-in
188	 * idle CPU tracking and the following helpers become unavailable:
189	 *
190	 * - scx_bpf_select_cpu_dfl()
191	 * - scx_bpf_test_and_clear_cpu_idle()
192	 * - scx_bpf_pick_idle_cpu()
193	 * - scx_bpf_any_idle_cpu()
194	 *
195	 * The user also must implement ops.select_cpu() as the default
196	 * implementation relies on scx_bpf_select_cpu_dfl().
197	 *
198	 * If you keep the built-in idle tracking, specify the
199	 * %SCX_OPS_KEEP_BUILTIN_IDLE flag.
200	 */
201	void (*update_idle)(s32 cpu, bool idle);
202
203	/**
204	 * cpu_acquire - A CPU is becoming available to the BPF scheduler
205	 * @cpu: The CPU being acquired by the BPF scheduler.
206	 * @args: Acquire arguments, see the struct definition.
207	 *
208	 * A CPU that was previously released from the BPF scheduler is now once
209	 * again under its control.
210	 */
211	void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args);
212
213	/**
214	 * cpu_release - A CPU is taken away from the BPF scheduler
215	 * @cpu: The CPU being released by the BPF scheduler.
216	 * @args: Release arguments, see the struct definition.
217	 *
218	 * The specified CPU is no longer under the control of the BPF
219	 * scheduler. This could be because it was preempted by a higher
220	 * priority sched_class, though there may be other reasons as well. The
221	 * caller should consult @args->reason to determine the cause.
222	 */
223	void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args);
224
225	/**
226	 * cpu_online - A CPU became online
227	 * @cpu: CPU which just came up
228	 *
229	 * @cpu just came online. @cpu doesn't call ->enqueue() or consume tasks
230	 * associated with other CPUs beforehand.
231	 */
232	void (*cpu_online)(s32 cpu);
233
234	/**
235	 * cpu_offline - A CPU is going offline
236	 * @cpu: CPU which is going offline
237	 *
238	 * @cpu is going offline. @cpu doesn't call ->enqueue() or consume tasks
239	 * associated with other CPUs afterwards.
240	 */
241	void (*cpu_offline)(s32 cpu);
242
243	/**
244	 * prep_enable - Prepare to enable BPF scheduling for a task
245	 * @p: task to prepare BPF scheduling for
246	 * @args: enable arguments, see the struct definition
247	 *
248	 * Either we're loading a BPF scheduler or a new task is being forked.
249	 * Prepare BPF scheduling for @p. This operation may block and can be
250	 * used for allocations.
251	 *
252	 * Return 0 for success, -errno for failure. An error return while
253	 * loading will abort loading of the BPF scheduler. During a fork, will
254	 * abort the specific fork.
255	 */
256	s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args);
257
258	/**
259	 * enable - Enable BPF scheduling for a task
260	 * @p: task to enable BPF scheduling for
261	 * @args: enable arguments, see the struct definition
262	 *
263	 * Enable @p for BPF scheduling. @p is now in the cgroup specified for
264	 * the preceding prep_enable() and will start running soon.
265	 */
266	void (*enable)(struct task_struct *p, struct scx_enable_args *args);
267
268	/**
269	 * cancel_enable - Cancel prep_enable()
270	 * @p: task being canceled
271	 * @args: enable arguments, see the struct definition
272	 *
273	 * @p was prep_enable()'d but failed before reaching enable(). Undo the
274	 * preparation.
275	 */
276	void (*cancel_enable)(struct task_struct *p,
277			      struct scx_enable_args *args);
278
279	/**
280	 * disable - Disable BPF scheduling for a task
281	 * @p: task to disable BPF scheduling for
282	 *
283	 * @p is exiting, leaving SCX or the BPF scheduler is being unloaded.
284	 * Disable BPF scheduling for @p.
285	 */
286	void (*disable)(struct task_struct *p);
287
288	/**
289	 * cgroup_init - Initialize a cgroup
290	 * @cgrp: cgroup being initialized
291	 * @args: init arguments, see the struct definition
292	 *
293	 * Either the BPF scheduler is being loaded or @cgrp created, initialize
294	 * @cgrp for sched_ext. This operation may block.
295	 *
296	 * Return 0 for success, -errno for failure. An error return while
297	 * loading will abort loading of the BPF scheduler. During cgroup
298	 * creation, it will abort the specific cgroup creation.
299	 */
300	s32 (*cgroup_init)(struct cgroup *cgrp,
301			   struct scx_cgroup_init_args *args);
302
303	/**
304	 * cgroup_exit - Exit a cgroup
305	 * @cgrp: cgroup being exited
306	 *
307	 * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit
308	 * @cgrp for sched_ext. This operation my block.
309	 */
310	void (*cgroup_exit)(struct cgroup *cgrp);
311
312	/**
313	 * cgroup_prep_move - Prepare a task to be moved to a different cgroup
314	 * @p: task being moved
315	 * @from: cgroup @p is being moved from
316	 * @to: cgroup @p is being moved to
317	 *
318	 * Prepare @p for move from cgroup @from to @to. This operation may
319	 * block and can be used for allocations.
320	 *
321	 * Return 0 for success, -errno for failure. An error return aborts the
322	 * migration.
323	 */
324	s32 (*cgroup_prep_move)(struct task_struct *p,
325				struct cgroup *from, struct cgroup *to);
326
327	/**
328	 * cgroup_move - Commit cgroup move
329	 * @p: task being moved
330	 * @from: cgroup @p is being moved from
331	 * @to: cgroup @p is being moved to
332	 *
333	 * Commit the move. @p is dequeued during this operation.
334	 */
335	void (*cgroup_move)(struct task_struct *p,
336			    struct cgroup *from, struct cgroup *to);
337
338	/**
339	 * cgroup_cancel_move - Cancel cgroup move
340	 * @p: task whose cgroup move is being canceled
341	 * @from: cgroup @p was being moved from
342	 * @to: cgroup @p was being moved to
343	 *
344	 * @p was cgroup_prep_move()'d but failed before reaching cgroup_move().
345	 * Undo the preparation.
346	 */
347	void (*cgroup_cancel_move)(struct task_struct *p,
348				   struct cgroup *from, struct cgroup *to);
349
350	/**
351	 * cgroup_set_weight - A cgroup's weight is being changed
352	 * @cgrp: cgroup whose weight is being updated
353	 * @weight: new weight [1..10000]
354	 *
355	 * Update @tg's weight to @weight.
356	 */
357	void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight);
358
359	/*
360	 * All online ops must come before ops.init().
361	 */
362
363	/**
364	 * init - Initialize the BPF scheduler
365	 */
366	s32 (*init)(void);
367
368	/**
369	 * exit - Clean up after the BPF scheduler
370	 * @info: Exit info
371	 */
372	void (*exit)(struct scx_exit_info *info);
373
374	/**
375	 * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch
376	 */
377	u32 dispatch_max_batch;
378
379	/**
380	 * flags - %SCX_OPS_* flags
381	 */
382	u64 flags;
383
384	/**
385	 * timeout_ms - The maximum amount of time, in milliseconds, that a
386	 * runnable task should be able to wait before being scheduled. The
387	 * maximum timeout may not exceed the default timeout of 30 seconds.
388	 *
389	 * Defaults to the maximum allowed timeout value of 30 seconds.
390	 */
391	u32 timeout_ms;
392
393	/**
394	 * name - BPF scheduler's name
395	 *
396	 * Must be a non-zero valid BPF object name including only isalnum(),
397	 * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the
398	 * BPF scheduler is enabled.
399	 */
400	char name[SCX_OPS_NAME_LEN];
401};

 1/*
 2 * DSQ (dispatch queue) IDs are 64bit of the format:
 3 *
 4 *   Bits: [63] [62 ..  0]
 5 *         [ B] [   ID   ]
 6 *
 7 *    B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs
 8 *   ID: 63 bit ID
 9 *
10 * Built-in IDs:
11 *
12 *   Bits: [63] [62] [61..32] [31 ..  0]
13 *         [ 1] [ L] [   R  ] [    V   ]
14 *
15 *    1: 1 for built-in DSQs.
16 *    L: 1 for LOCAL_ON DSQ IDs, 0 for others
17 *    V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value.
18 */
19enum scx_dsq_id_flags {
20	SCX_DSQ_FLAG_BUILTIN	= 1LLU << 63,
21	SCX_DSQ_FLAG_LOCAL_ON	= 1LLU << 62,
22
23	SCX_DSQ_INVALID		= SCX_DSQ_FLAG_BUILTIN | 0,
24	SCX_DSQ_GLOBAL		= SCX_DSQ_FLAG_BUILTIN | 1,
25	SCX_DSQ_LOCAL		= SCX_DSQ_FLAG_BUILTIN | 2,
26	SCX_DSQ_LOCAL_ON	= SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON,
27	SCX_DSQ_LOCAL_CPU_MASK	= 0xffffffffLLU,
28};

 1enum scx_enq_flags {
 2	/* expose select ENQUEUE_* flags as enums */
 3	SCX_ENQ_WAKEUP		= ENQUEUE_WAKEUP,
 4	SCX_ENQ_HEAD		= ENQUEUE_HEAD,
 5
 6	/* high 32bits are ext specific flags */
 7
 8	/*
 9	 * Set the following to trigger preemption when calling
10	 * scx_bpf_dispatch() with a local dsq as the target. The slice of the
11	 * current task is cleared to zero and the CPU is kicked into the
12	 * scheduling path. Implies %SCX_ENQ_HEAD.
13	 */
14	SCX_ENQ_PREEMPT		= 1LLU << 32,
15
16	/*
17	 * The task being enqueued was previously enqueued on the current CPU's
18	 * %SCX_DSQ_LOCAL, but was removed from it in a call to the
19	 * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was
20	 * invoked in a ->cpu_release() callback, and the task is again
21	 * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the
22	 * task will not be scheduled on the CPU until at least the next invocation
23	 * of the ->cpu_acquire() callback.
24	 */
25	SCX_ENQ_REENQ		= 1LLU << 40,
26
27	/*
28	 * The task being enqueued is the only task available for the cpu. By
29	 * default, ext core keeps executing such tasks but when
30	 * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with
31	 * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set.
32	 *
33	 * If the BPF scheduler wants to continue executing the task,
34	 * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately.
35	 * If the task gets queued on a different dsq or the BPF side, the BPF
36	 * scheduler is responsible for triggering a follow-up scheduling event.
37	 * Otherwise, Execution may stall.
38	 */
39	SCX_ENQ_LAST		= 1LLU << 41,
40
41	/*
42	 * A hint indicating that it's advisable to enqueue the task on the
43	 * local dsq of the currently selected CPU. Currently used by
44	 * select_cpu_dfl() and together with %SCX_ENQ_LAST.
45	 */
46	SCX_ENQ_LOCAL		= 1LLU << 42,
47
48	/* high 8 bits are internal */
49	__SCX_ENQ_INTERNAL_MASK	= 0xffLLU << 56,
50
51	SCX_ENQ_CLEAR_OPSS	= 1LLU << 56,
52};

 1/*
 2 * The following is embedded in task_struct and contains all fields necessary
 3 * for a task to be scheduled by SCX.
 4 */
 5struct sched_ext_entity {
 6	struct scx_dispatch_q	*dsq;
 7	struct list_head	dsq_node;
 8	struct list_head	watchdog_node;
 9	u32			flags;		/* protected by rq lock */
10	u32			weight;
11	s32			sticky_cpu;
12	s32			holding_cpu;
13	atomic64_t		ops_state;
14	unsigned long		runnable_at;
15
16	/* BPF scheduler modifiable fields */
17
18	/*
19	 * Runtime budget in nsecs. This is usually set through
20	 * scx_bpf_dispatch() but can also be modified directly by the BPF
21	 * scheduler. Automatically decreased by SCX as the task executes. On
22	 * depletion, a scheduling event is triggered.
23	 *
24	 * This value is cleared to zero if the task is preempted by
25	 * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the
26	 * task ran. Use p->se.sum_exec_runtime instead.
27	 */
28	u64			slice;
29
30	/*
31	 * If set, reject future sched_setscheduler(2) calls updating the policy
32	 * to %SCHED_EXT with -%EACCES.
33	 *
34	 * If set from ops.prep_enable() and the task's policy is already
35	 * %SCHED_EXT, which can happen while the BPF scheduler is being loaded
36	 * or by inhering the parent's policy during fork, the task's policy is
37	 * rejected and forcefully reverted to %SCHED_NORMAL. The number of such
38	 * events are reported through /sys/kernel/debug/sched_ext::nr_rejected.
39	 */
40	bool			disallow;	/* reject switching into SCX */
41
42	/* cold fields */
43	struct list_head	tasks_node;
44#ifdef CONFIG_EXT_GROUP_SCHED
45	struct cgroup		*cgrp_moving_from;
46#endif
47};

 1/**
 2 * scx_bpf_dispatch - Dispatch a task to a dsq
 3 * @p: task_struct to dispatch
 4 * @dsq_id: dsq to dispatch to
 5 * @slice: duration @p can run for in nsecs
 6 * @enq_flags: SCX_ENQ_*
 7 *
 8 * Dispatch @p to the dsq identified by @dsq_id. It is safe to call this
 9 * function spuriously. Can be called from ops.enqueue() and ops.dispatch().
10 *
11 * When called from ops.enqueue(), it's for direct dispatch and @p must match
12 * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the
13 * local dsq of a CPU other than the enqueueing one. Use ops.select_cpu() to be
14 * on the target CPU in the first place.
15 *
16 * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id
17 * and this function can be called upto ops.dispatch_max_batch times to dispatch
18 * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the
19 * remaining slots.
20 *
21 * @p is allowed to run for @slice. The scheduling path is triggered on slice
22 * exhaustion. If zero, the current residual slice is maintained. If
23 * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with
24 * scx_bpf_kick_cpu() to trigger scheduling.
25 */
26void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice,
27		      u64 enq_flags)
28{
29	struct task_struct *ddsp_task;
30	int idx;
31
32	lockdep_assert_irqs_disabled();
33
34	if (unlikely(!p)) {
35		scx_ops_error("called with NULL task");
36		return;
37	}
38
39	if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) {
40		scx_ops_error("invalid enq_flags 0x%llx", enq_flags);
41		return;
42	}
43
44	if (slice)
45		p->scx.slice = slice;
46	else
47		p->scx.slice = p->scx.slice ?: 1;
48
49	ddsp_task = __this_cpu_read(direct_dispatch_task);
50	if (ddsp_task) {
51		direct_dispatch(ddsp_task, p, dsq_id, enq_flags);
52		return;
53	}
54
55	idx = __this_cpu_read(dispatch_buf_cursor);
56	if (unlikely(idx >= dispatch_max_batch)) {
57		scx_ops_error("dispatch buffer overflow");
58		return;
59	}
60
61	this_cpu_ptr(dispatch_buf)[idx] = (struct dispatch_buf_ent){
62		.task = p,
63		.qseq = atomic64_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK,
64		.dsq_id = dsq_id,
65		.enq_flags = enq_flags,
66	};
67	__this_cpu_inc(dispatch_buf_cursor);
68}

 1static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p,
 2			    u64 dsq_id, u64 enq_flags)
 3{
 4	struct scx_dispatch_q *dsq;
 5
 6	/* @p must match the task which is being enqueued */
 7	if (unlikely(p != ddsp_task)) {
 8		if (IS_ERR(ddsp_task))
 9			scx_ops_error("%s[%d] already direct-dispatched",
10				      p->comm, p->pid);
11		else
12			scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]",
13				      ddsp_task->comm, ddsp_task->pid,
14				      p->comm, p->pid);
15		return;
16	}
17
18	/*
19	 * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because
20	 * dispatching to the local dsq of a different CPU requires unlocking
21	 * the current rq which isn't allowed in the enqueue path. Use
22	 * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL.
23	 */
24	if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) {
25		scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch");
26		return;
27	}
28
29	dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p);
30	dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
31
32	/*
33	 * Mark that dispatch already happened by spoiling direct_dispatch_task
34	 * with a non-NULL value which can never match a valid task pointer.
35	 */
36	__this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH));
37}

 1static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id,
 2						    struct task_struct *p)
 3{
 4	struct scx_dispatch_q *dsq;
 5
 6	if (dsq_id == SCX_DSQ_LOCAL)
 7		return &rq->scx.local_dsq;
 8
 9	dsq = find_non_local_dsq(dsq_id);
10	if (unlikely(!dsq)) {
11		scx_ops_error("non-existent dsq 0x%llx for %s[%d]",
12			      dsq_id, p->comm, p->pid);
13		return &scx_dsq_global;
14	}
15
16	return dsq;
17}

 1/**
 2 * rhashtable_lookup_fast - search hash table, without RCU read lock
 3 * @ht:		hash table
 4 * @key:	the pointer to the key
 5 * @params:	hash table parameters
 6 *
 7 * Computes the hash value for the key and traverses the bucket chain looking
 8 * for a entry with an identical key. The first matching entry is returned.
 9 *
10 * Only use this function when you have other mechanisms guaranteeing
11 * that the object won't go away after the RCU read lock is released.
12 *
13 * Returns the first entry on which the compare function returned true.
14 */
15static inline void *rhashtable_lookup_fast(
16	struct rhashtable *ht, const void *key,
17	const struct rhashtable_params params)
18{
19	void *obj;
20
21	rcu_read_lock();
22	obj = rhashtable_lookup(ht, key, params);
23	rcu_read_unlock();
24
25	return obj;
26}

 1static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p,
 2			     u64 enq_flags)
 3{
 4	bool is_local = dsq->id == SCX_DSQ_LOCAL;
 5
 6	WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_node));
 7
 8	if (!is_local) {
 9		raw_spin_lock(&dsq->lock);
10		if (unlikely(dsq->id == SCX_DSQ_INVALID)) {
11			scx_ops_error("attempting to dispatch to a destroyed dsq");
12			/* fall back to the global dsq */
13			raw_spin_unlock(&dsq->lock);
14			dsq = &scx_dsq_global;
15			raw_spin_lock(&dsq->lock);
16		}
17	}
18
19	if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT))
20		list_add(&p->scx.dsq_node, &dsq->fifo);
21	else
22		list_add_tail(&p->scx.dsq_node, &dsq->fifo);
23	dsq->nr++;
24	p->scx.dsq = dsq;
25
26	/*
27	 * We're transitioning out of QUEUEING or DISPATCHING. store_release to
28	 * match waiters' load_acquire.
29	 */
30	if (enq_flags & SCX_ENQ_CLEAR_OPSS)
31		atomic64_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
32
33	if (is_local) {
34		struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
35		bool preempt = false;
36
37		if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
38		    rq->curr->sched_class == &ext_sched_class) {
39			rq->curr->scx.slice = 0;
40			preempt = true;
41		}
42
43		if (preempt || sched_class_above(&ext_sched_class,
44						 rq->curr->sched_class))
45			resched_curr(rq);
46	} else {
47		raw_spin_unlock(&dsq->lock);
48	}
49}

 1/**
 2 * scx_bpf_consume - Transfer a task from a dsq to the current CPU's local dsq
 3 * @dsq_id: dsq to consume
 4 *
 5 * Consume a task from the dsq identified by @dsq_id and transfer it to the
 6 * current CPU's local dsq for execution. Can only be called from
 7 * ops.consume[_final]().
 8 *
 9 * Returns %true if a task has been consumed, %false if there isn't any task to
10 * consume.
11 */
12bool scx_bpf_consume(u64 dsq_id)
13{
14	struct consume_ctx *cctx = this_cpu_ptr(&consume_ctx);
15	struct scx_dispatch_q *dsq;
16
17	dsq = find_non_local_dsq(dsq_id);
18	if (unlikely(!dsq)) {
19		scx_ops_error("invalid dsq_id 0x%016llx", dsq_id);
20		return false;
21	}
22
23	return consume_dispatch_q(cctx->rq, cctx->rf, dsq);
24}

 1static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf,
 2			       struct scx_dispatch_q *dsq)
 3{
 4	struct scx_rq *scx_rq = &rq->scx;
 5	struct task_struct *p;
 6	struct rq *task_rq;
 7	bool moved = false;
 8retry:
 9	if (list_empty(&dsq->fifo))
10		return false;
11
12	raw_spin_lock(&dsq->lock);
13	list_for_each_entry(p, &dsq->fifo, scx.dsq_node) {
14		task_rq = task_rq(p);
15		if (rq == task_rq)
16			goto this_rq;
17		if (likely(rq->online) && !is_migration_disabled(p) &&
18		    cpumask_test_cpu(cpu_of(rq), p->cpus_ptr))
19			goto remote_rq;
20	}
21	raw_spin_unlock(&dsq->lock);
22	return false;
23
24this_rq:
25	/* @dsq is locked and @p is on this rq */
26	WARN_ON_ONCE(p->scx.holding_cpu >= 0);
27	list_move_tail(&p->scx.dsq_node, &scx_rq->local_dsq.fifo);
28	dsq->nr--;
29	scx_rq->local_dsq.nr++;
30	p->scx.dsq = &scx_rq->local_dsq;
31	raw_spin_unlock(&dsq->lock);
32	return true;
33
34remote_rq:
35#ifdef CONFIG_SMP
36	/*
37	 * @dsq is locked and @p is on a remote rq. @p is currently protected by
38	 * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab
39	 * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the
40	 * rq lock or fail, do a little dancing from our side. See
41	 * move_task_to_local_dsq().
42	 */
43	WARN_ON_ONCE(p->scx.holding_cpu >= 0);
44	list_del_init(&p->scx.dsq_node);
45	dsq->nr--;
46	p->scx.holding_cpu = raw_smp_processor_id();
47	raw_spin_unlock(&dsq->lock);
48
49	rq_unpin_lock(rq, rf);
50	double_lock_balance(rq, task_rq);
51	rq_repin_lock(rq, rf);
52
53	moved = move_task_to_local_dsq(rq, p);
54
55	double_unlock_balance(rq, task_rq);
56#endif /* CONFIG_SMP */
57	if (likely(moved))
58		return true;
59	goto retry;
60}

 1/**
 2 * scx_bpf_kick_cpu - Trigger reschedule on a CPU
 3 * @cpu: cpu to kick
 4 * @flags: SCX_KICK_* flags
 5 *
 6 * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or
 7 * trigger rescheduling on a busy CPU. This can be called from any online
 8 * scx_ops operation and the actual kicking is performed asynchronously through
 9 * an irq work.
10 */
11void scx_bpf_kick_cpu(s32 cpu, u64 flags)
12{
13	if (!ops_cpu_valid(cpu)) {
14		scx_ops_error("invalid cpu %d", cpu);
15		return;
16	}
17#ifdef CONFIG_SMP
18	{
19		struct rq *rq;
20
21		preempt_disable();
22		rq = this_rq();
23
24		/*
25		 * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid
26		 * nesting rq locks. We can probably be smarter and avoid
27		 * bouncing if called from ops which don't hold a rq lock.
28		 */
29		cpumask_set_cpu(cpu, rq->scx.cpus_to_kick);
30		if (flags & SCX_KICK_PREEMPT)
31			cpumask_set_cpu(cpu, rq->scx.cpus_to_preempt);
32		if (flags & SCX_KICK_WAIT)
33			cpumask_set_cpu(cpu, rq->scx.cpus_to_wait);
34
35		irq_work_queue(&rq->scx.kick_cpus_irq_work);
36		preempt_enable();
37	}
38#endif
39}

 1/**
 2 * scx_bpf_dsq_nr_queued - Return the number of queued tasks
 3 * @dsq_id: id of the dsq
 4 *
 5 * Return the number of tasks in the dsq matching @dsq_id. If not found,
 6 * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops
 7 * operations.
 8 */
 9s32 scx_bpf_dsq_nr_queued(u64 dsq_id)
10{
11	struct scx_dispatch_q *dsq;
12
13	lockdep_assert(rcu_read_lock_any_held());
14
15	if (dsq_id == SCX_DSQ_LOCAL) {
16		return this_rq()->scx.local_dsq.nr;
17	} else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
18		s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK;
19
20		if (ops_cpu_valid(cpu))
21			return cpu_rq(cpu)->scx.local_dsq.nr;
22	} else {
23		dsq = find_non_local_dsq(dsq_id);
24		if (dsq)
25			return dsq->nr;
26	}
27	return -ENOENT;
28}

 1struct bpf_struct_ops {
 2	const struct bpf_verifier_ops *verifier_ops;
 3	int (*init)(struct btf *btf);
 4	int (*check_member)(const struct btf_type *t,
 5			    const struct btf_member *member,
 6			    struct bpf_prog *prog);
 7	int (*init_member)(const struct btf_type *t,
 8			   const struct btf_member *member,
 9			   void *kdata, const void *udata);
10	int (*reg)(void *kdata);
11	void (*unreg)(void *kdata);
12	const struct btf_type *type;
13	const struct btf_type *value_type;
14	const char *name;
15	struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
16	u32 type_id;
17	u32 value_id;
18};

 1enum scx_exit_type {
 2	SCX_EXIT_NONE,
 3	SCX_EXIT_DONE,
 4
 5	SCX_EXIT_UNREG = 64,	/* BPF unregistration */
 6	SCX_EXIT_SYSRQ,		/* requested by 'S' sysrq */
 7
 8	SCX_EXIT_ERROR = 1024,	/* runtime error, error msg contains details */
 9	SCX_EXIT_ERROR_BPF,	/* ERROR but triggered through scx_bpf_error() */
10	SCX_EXIT_ERROR_STALL,	/* watchdog detected stalled runnable tasks */
11};
12
13/*
14 * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is
15 * being disabled.
16 */
17struct scx_exit_info {
18	/* %SCX_EXIT_* - broad category of the exit reason */
19	enum scx_exit_type	type;
20	/* textual representation of the above */
21	char			reason[SCX_EXIT_REASON_LEN];
22	/* number of entries in the backtrace */
23	u32			bt_len;
24	/* backtrace if exiting due to an error */
25	unsigned long		bt[SCX_EXIT_BT_LEN];
26	/* extra message */
27	char			msg[SCX_EXIT_MSG_LEN];
28};
29
30
31static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE);
32static struct scx_exit_info scx_exit_info;

  1static int scx_ops_enable(struct sched_ext_ops *ops)
  2{
  3	struct scx_task_iter sti;
  4	struct task_struct *p;
  5	int i, ret;
  6
  7	mutex_lock(&scx_ops_enable_mutex);
  8
  9	if (!scx_ops_helper) {
 10		WRITE_ONCE(scx_ops_helper,
 11			   scx_create_rt_helper("sched_ext_ops_helper"));
 12		if (!scx_ops_helper) {
 13			ret = -ENOMEM;
 14			goto err_unlock;
 15		}
 16	}
 17
 18	if (scx_ops_enable_state() != SCX_OPS_DISABLED) {
 19		ret = -EBUSY;
 20		goto err_unlock;
 21	}
 22
 23	ret = rhashtable_init(&dsq_hash, &dsq_hash_params);
 24	if (ret)
 25		goto err_unlock;
 26
 27	/*
 28	 * Set scx_ops, transition to PREPPING and clear exit info to arm the
 29	 * disable path. Failure triggers full disabling from here on.
 30	 */
 31	scx_ops = *ops;
 32
 33	WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) !=
 34		     SCX_OPS_DISABLED);
 35
 36	memset(&scx_exit_info, 0, sizeof(scx_exit_info));
 37	atomic_set(&scx_exit_type, SCX_EXIT_NONE);
 38	warned_zero_slice = false;
 39
 40	atomic64_set(&scx_nr_rejected, 0);
 41
 42	/*
 43	 * Keep CPUs stable during enable so that the BPF scheduler can track
 44	 * online CPUs by watching ->on/offline_cpu() after ->init().
 45	 */
 46	cpus_read_lock();
 47
 48	scx_switch_all_req = false;
 49	if (scx_ops.init) {
 50		ret = scx_ops.init();
 51
 52		if (ret) {
 53			ret = ops_sanitize_err("init", ret);
 54			goto err_disable;
 55		}
 56
 57		/*
 58		 * Exit early if ops.init() triggered scx_bpf_error(). Not
 59		 * strictly necessary as we'll fail transitioning into ENABLING
 60		 * later but that'd be after calling ops.prep_enable() on all
 61		 * tasks and with -EBUSY which isn't very intuitive. Let's exit
 62		 * early with success so that the condition is notified through
 63		 * ops.exit() like other scx_bpf_error() invocations.
 64		 */
 65		if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE)
 66			goto err_disable;
 67	}
 68
 69	WARN_ON_ONCE(dispatch_buf);
 70	dispatch_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH;
 71	dispatch_buf = __alloc_percpu(sizeof(dispatch_buf[0]) * dispatch_max_batch,
 72				      __alignof__(dispatch_buf[0]));
 73	if (!dispatch_buf) {
 74		ret = -ENOMEM;
 75		goto err_disable;
 76	}
 77
 78	task_runnable_timeout_ms = SCX_MAX_RUNNABLE_TIMEOUT;
 79	if (ops->timeout_ms) {
 80		/*
 81		 * Excessively large timeouts should have been rejected in
 82		 * bpf_scx_init_member().
 83		 */
 84		WARN_ON_ONCE(ops->timeout_ms > SCX_MAX_RUNNABLE_TIMEOUT);
 85		task_runnable_timeout_ms = ops->timeout_ms;
 86	}
 87
 88	last_timeout_check = jiffies;
 89	queue_delayed_work(system_unbound_wq, &check_timeout_work,
 90			   task_runnable_timeout_ms / 2);
 91
 92	/*
 93	 * Lock out forks, cgroup on/offlining and moves before opening the
 94	 * floodgate so that they don't wander into the operations prematurely.
 95	 */
 96	percpu_down_write(&scx_fork_rwsem);
 97	scx_cgroup_lock();
 98
 99	for (i = 0; i < SCX_NR_ONLINE_OPS; i++)
100		if (((void (**)(void))ops)[i])
101			static_branch_enable_cpuslocked(&scx_has_op[i]);
102
103	if (ops->flags & SCX_OPS_ENQ_LAST)
104		static_branch_enable_cpuslocked(&scx_ops_enq_last);
105
106	if (ops->flags & SCX_OPS_ENQ_EXITING)
107		static_branch_enable_cpuslocked(&scx_ops_enq_exiting);
108	if (scx_ops.cpu_acquire || scx_ops.cpu_release)
109		static_branch_enable_cpuslocked(&scx_ops_cpu_preempt);
110
111	if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) {
112		reset_idle_masks();
113		static_branch_enable_cpuslocked(&scx_builtin_idle_enabled);
114	} else {
115		static_branch_disable_cpuslocked(&scx_builtin_idle_enabled);
116	}
117
118	/*
119	 * All cgroups should be initialized before letting in tasks. cgroup
120	 * on/offlining and task migrations are already locked out.
121	 */
122	ret = scx_cgroup_init();
123	if (ret)
124		goto err_disable_unlock;
125
126	static_branch_enable_cpuslocked(&__scx_ops_enabled);
127
128	/*
129	 * Enable ops for every task. Fork is excluded by scx_fork_rwsem
130	 * preventing new tasks from being added. No need to exclude tasks
131	 * leaving as sched_ext_free() can handle both prepped and enabled
132	 * tasks. Prep all tasks first and then enable them with preemption
133	 * disabled.
134	 */
135	spin_lock_irq(&scx_tasks_lock);
136
137	scx_task_iter_init(&sti);
138	while ((p = scx_task_iter_next_filtered(&sti))) {
139		get_task_struct(p);
140		spin_unlock_irq(&scx_tasks_lock);
141
142		ret = scx_ops_prepare_task(p, task_group(p));
143		if (ret) {
144			put_task_struct(p);
145			spin_lock_irq(&scx_tasks_lock);
146			scx_task_iter_exit(&sti);
147			spin_unlock_irq(&scx_tasks_lock);
148			pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n",
149			       ret, p->comm, p->pid);
150			goto err_disable_unlock;
151		}
152
153		put_task_struct(p);
154		spin_lock_irq(&scx_tasks_lock);
155	}
156	scx_task_iter_exit(&sti);
157
158	/*
159	 * All tasks are prepped but are still ops-disabled. Ensure that
160	 * %current can't be scheduled out and switch everyone.
161	 * preempt_disable() is necessary because we can't guarantee that
162	 * %current won't be starved if scheduled out while switching.
163	 */
164	preempt_disable();
165
166	/*
167	 * From here on, the disable path must assume that tasks have ops
168	 * enabled and need to be recovered.
169	 */
170	if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) {
171		preempt_enable();
172		spin_unlock_irq(&scx_tasks_lock);
173		ret = -EBUSY;
174		goto err_disable_unlock;
175	}
176
177	/*
178	 * We're fully committed and can't fail. The PREPPED -> ENABLED
179	 * transitions here are synchronized against sched_ext_free() through
180	 * scx_tasks_lock.
181	 */
182	WRITE_ONCE(scx_switching_all, scx_switch_all_req);
183
184	scx_task_iter_init(&sti);
185	while ((p = scx_task_iter_next_filtered_locked(&sti))) {
186		if (READ_ONCE(p->__state) != TASK_DEAD) {
187			const struct sched_class *old_class = p->sched_class;
188			struct sched_enq_and_set_ctx ctx;
189
190			sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE,
191					       &ctx);
192			scx_ops_enable_task(p);
193
194			__setscheduler_prio(p, p->prio);
195			check_class_changing(task_rq(p), p, old_class);
196
197			sched_enq_and_set_task(&ctx);
198
199			check_class_changed(task_rq(p), p, old_class, p->prio);
200		} else {
201			scx_ops_disable_task(p);
202		}
203	}
204	scx_task_iter_exit(&sti);
205
206	spin_unlock_irq(&scx_tasks_lock);
207	preempt_enable();
208	scx_cgroup_unlock();
209	percpu_up_write(&scx_fork_rwsem);
210
211	if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) {
212		ret = -EBUSY;
213		goto err_disable_unlock;
214	}
215
216	if (scx_switch_all_req)
217		static_branch_enable_cpuslocked(&__scx_switched_all);
218
219	cpus_read_unlock();
220	mutex_unlock(&scx_ops_enable_mutex);
221
222	scx_cgroup_config_knobs();
223
224	return 0;
225
226err_unlock:
227	mutex_unlock(&scx_ops_enable_mutex);
228	return ret;
229
230err_disable_unlock:
231	scx_cgroup_unlock();
232	percpu_up_write(&scx_fork_rwsem);
233err_disable:
234	cpus_read_unlock();
235	mutex_unlock(&scx_ops_enable_mutex);
236	/* must be fully disabled before returning */
237	scx_ops_disable(SCX_EXIT_ERROR);
238	kthread_flush_work(&scx_ops_disable_work);
239	return ret;
240}

 1/**
 2 * scx_task_iter_init - Initialize a task iterator
 3 * @iter: iterator to init
 4 *
 5 * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized,
 6 * @iter must eventually be exited with scx_task_iter_exit().
 7 *
 8 * scx_tasks_lock may be released between this and the first next() call or
 9 * between any two next() calls. If scx_tasks_lock is released between two
10 * next() calls, the caller is responsible for ensuring that the task being
11 * iterated remains accessible either through RCU read lock or obtaining a
12 * reference count.
13 *
14 * All tasks which existed when the iteration started are guaranteed to be
15 * visited as long as they still exist.
16 */
17static void scx_task_iter_init(struct scx_task_iter *iter)
18{
19	lockdep_assert_held(&scx_tasks_lock);
20
21	iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR };
22	list_add(&iter->cursor.tasks_node, &scx_tasks);
23	iter->locked = NULL;
24}