diff --git a/android/ACK_SHA b/android/ACK_SHA index c58c1160dd520..f63432363acb6 100644 --- a/android/ACK_SHA +++ b/android/ACK_SHA @@ -1,2 +1,2 @@ -f580df859bb06948e26f249d348a74348c237271 -android14-6.1-2023-08_r3 +d96859ba8012783ffcc0a1e60c04d073a9c4ed3a +android14-6.1-2023-08_r5 diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index a725534e2b17e..23edac82100ca 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -311039,6 +311039,13 @@ function { parameter_id: 0x17047654 parameter_id: 0xc9082b19 } +function { + id: 0x9b6602ad + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x33756485 + parameter_id: 0x11b57133 +} function { id: 0x9b660b2c return_type_id: 0x6720d32f @@ -313122,6 +313129,14 @@ function { parameter_id: 0x18bd6530 parameter_id: 0x141e84b4 } +function { + id: 0x9bddb3a2 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x1d19a9d5 + parameter_id: 0x18a16691 + parameter_id: 0x11cfee5a +} function { id: 0x9bddb929 return_type_id: 0x6720d32f @@ -327121,6 +327136,15 @@ elf_symbol { type_id: 0x9bc25990 full_name: "__traceiter_android_rvh_can_migrate_task" } +elf_symbol { + id: 0xbbd04d50 + name: "__traceiter_android_rvh_cgroup_force_kthread_migration" + is_defined: true + symbol_type: FUNCTION + crc: 0x94b3e68f + type_id: 0x9bddb3a2 + full_name: "__traceiter_android_rvh_cgroup_force_kthread_migration" +} elf_symbol { id: 0xc93c7d6d name: "__traceiter_android_rvh_check_preempt_tick" @@ -329227,6 +329251,15 @@ elf_symbol { type_id: 0x9bd7019d full_name: "__traceiter_android_vh_record_pcpu_rwsem_starttime" } +elf_symbol { + id: 0x1a91ec8c + name: "__traceiter_android_vh_record_pcpu_rwsem_time_early" + is_defined: true + symbol_type: FUNCTION + crc: 0xeeef021b + type_id: 0x9b6602ad + full_name: "__traceiter_android_vh_record_pcpu_rwsem_time_early" +} elf_symbol { id: 0x92518ec5 name: "__traceiter_android_vh_record_rtmutex_lock_starttime" @@ -330379,6 +330412,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_rvh_can_migrate_task" } +elf_symbol { + id: 0xb208306e + name: "__tracepoint_android_rvh_cgroup_force_kthread_migration" + is_defined: true + symbol_type: OBJECT + crc: 0x0c180285 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_cgroup_force_kthread_migration" +} elf_symbol { id: 0x60b5a917 name: "__tracepoint_android_rvh_check_preempt_tick" @@ -332485,6 +332527,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_record_pcpu_rwsem_starttime" } +elf_symbol { + id: 0x158c4cfa + name: "__tracepoint_android_vh_record_pcpu_rwsem_time_early" + is_defined: true + symbol_type: OBJECT + crc: 0xfb2f7ea7 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_record_pcpu_rwsem_time_early" +} elf_symbol { id: 0x4568ff8f name: "__tracepoint_android_vh_record_rtmutex_lock_starttime" @@ -337584,6 +337635,15 @@ elf_symbol { type_id: 0x751b5661 full_name: "cgroup_taskset_next" } +elf_symbol { + id: 0xb7533de5 + name: "cgroup_threadgroup_rwsem" + is_defined: true + symbol_type: OBJECT + crc: 0x4e1c73cb + type_id: 0x6c952252 + full_name: "cgroup_threadgroup_rwsem" +} elf_symbol { id: 0x7a871d1c name: "check_move_unevictable_pages" @@ -385632,6 +385692,7 @@ interface { symbol_id: 0x144db0a1 symbol_id: 0x192bbbd5 symbol_id: 0xadc13d20 + symbol_id: 0xbbd04d50 symbol_id: 0xc93c7d6d symbol_id: 0x9d00b8f0 symbol_id: 0x5e9397c4 @@ -385866,6 +385927,7 @@ interface { symbol_id: 0x93303c51 symbol_id: 0x7d069e91 symbol_id: 0x0fa39b81 + symbol_id: 0x1a91ec8c symbol_id: 0x92518ec5 symbol_id: 0x9792c22e symbol_id: 0xe2d75052 @@ -385994,6 +386056,7 @@ interface { symbol_id: 0xd7757253 symbol_id: 0x1e8a7e23 symbol_id: 0xfe3875f6 + symbol_id: 0xb208306e symbol_id: 0x60b5a917 symbol_id: 0x1dd402e6 symbol_id: 0x070d1e2a @@ -386228,6 +386291,7 @@ interface { symbol_id: 0xb0c197a3 symbol_id: 0x761f292f symbol_id: 0xef7ad117 + symbol_id: 0x158c4cfa symbol_id: 0x4568ff8f symbol_id: 0xe918e2ec symbol_id: 0x13b2fb38 @@ -386795,6 +386859,7 @@ interface { symbol_id: 0x4ce62869 symbol_id: 0x6d77f512 symbol_id: 0xb3cbf3c8 + symbol_id: 0xb7533de5 symbol_id: 0x7a871d1c symbol_id: 0x91718d34 symbol_id: 0x65e5fa26 diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom index 637e7ef90f207..e0a2b29a42b5f 100644 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -3286,6 +3286,7 @@ __traceiter_android_rvh_before_do_sched_yield __traceiter_android_rvh_build_perf_domains __traceiter_android_rvh_can_migrate_task + __traceiter_android_rvh_cgroup_force_kthread_migration __traceiter_android_rvh_check_preempt_tick __traceiter_android_rvh_check_preempt_wakeup __traceiter_android_rvh_check_preempt_wakeup_ignore @@ -3430,6 +3431,7 @@ __tracepoint_android_rvh_before_do_sched_yield __tracepoint_android_rvh_build_perf_domains __tracepoint_android_rvh_can_migrate_task + __tracepoint_android_rvh_cgroup_force_kthread_migration __tracepoint_android_rvh_check_preempt_tick __tracepoint_android_rvh_check_preempt_wakeup __tracepoint_android_rvh_check_preempt_wakeup_ignore diff --git a/android/abi_gki_aarch64_xiaomi b/android/abi_gki_aarch64_xiaomi index 5a0852cf19f6d..5c5426861df60 100644 --- a/android/abi_gki_aarch64_xiaomi +++ b/android/abi_gki_aarch64_xiaomi @@ -332,3 +332,8 @@ #required by xm_ispv4_pcie.ko pci_ioremap_bar pci_disable_pcie_error_reporting + +#required by lock_optimization module + __traceiter_android_vh_record_pcpu_rwsem_time_early + __tracepoint_android_vh_record_pcpu_rwsem_time_early + cgroup_threadgroup_rwsem diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 229f0e712f93d..a17eabcb75947 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -136,6 +136,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_update_sysfs); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_send_command); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_compl_command); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cgroup_set_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cgroup_force_kthread_migration); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_syscall_prctl_finished); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_send_uic_command); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_send_tm_command); @@ -186,6 +187,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_module_permit_before_init); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_module_permit_after_init); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_is_initialized); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_shmem_get_folio); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_pcpu_rwsem_time_early); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_mmap_file); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_file_open); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_bpf_syscall); diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h index ebd81e03f3c1e..7734ef9050115 100644 --- a/include/linux/percpu-rwsem.h +++ b/include/linux/percpu-rwsem.h @@ -23,6 +23,9 @@ struct percpu_rw_semaphore { #endif }; +void _trace_android_vh_record_pcpu_rwsem_time_early( + unsigned long settime, struct percpu_rw_semaphore *sem); + #ifdef CONFIG_DEBUG_LOCK_ALLOC #define __PERCPU_RWSEM_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname }, #else @@ -54,6 +57,8 @@ static inline void percpu_down_read(struct percpu_rw_semaphore *sem) rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); preempt_disable(); + _trace_android_vh_record_pcpu_rwsem_time_early(jiffies, sem); + /* * We are in an RCU-sched read-side critical section, so the writer * cannot both change sem->state from readers_fast and start checking @@ -93,6 +98,7 @@ static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem) */ if (ret) { + _trace_android_vh_record_pcpu_rwsem_time_early(jiffies, sem); _trace_android_vh_record_pcpu_rwsem_starttime(current, jiffies); rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_); } @@ -124,6 +130,7 @@ static inline void percpu_up_read(struct percpu_rw_semaphore *sem) this_cpu_dec(*sem->read_count); rcuwait_wake_up(&sem->writer); } + _trace_android_vh_record_pcpu_rwsem_time_early(0, sem); _trace_android_vh_record_pcpu_rwsem_starttime(current, 0); preempt_enable(); } diff --git a/include/trace/hooks/cgroup.h b/include/trace/hooks/cgroup.h index a50e6abc55ee7..b824197118f9a 100644 --- a/include/trace/hooks/cgroup.h +++ b/include/trace/hooks/cgroup.h @@ -23,6 +23,10 @@ DECLARE_HOOK(android_vh_cgroup_attach, TP_PROTO(struct cgroup_subsys *ss, struct cgroup_taskset *tset), TP_ARGS(ss, tset)); +DECLARE_RESTRICTED_HOOK(android_rvh_cgroup_force_kthread_migration, + TP_PROTO(struct task_struct *tsk, struct cgroup *dst_cgrp, bool *force_migration), + TP_ARGS(tsk, dst_cgrp, force_migration), 1); + DECLARE_RESTRICTED_HOOK(android_rvh_cpuset_fork, TP_PROTO(struct task_struct *p, bool *inherit_cpus), TP_ARGS(p, inherit_cpus), 1); diff --git a/include/trace/hooks/dtask.h b/include/trace/hooks/dtask.h index 1552b71c17925..f5cdfe9b04f90 100644 --- a/include/trace/hooks/dtask.h +++ b/include/trace/hooks/dtask.h @@ -15,6 +15,7 @@ struct mutex; struct rt_mutex_base; struct rw_semaphore; struct task_struct; +struct percpu_rw_semaphore; DECLARE_HOOK(android_vh_mutex_wait_start, TP_PROTO(struct mutex *lock), @@ -80,6 +81,9 @@ DECLARE_HOOK(android_vh_record_rwsem_lock_starttime, DECLARE_HOOK(android_vh_record_pcpu_rwsem_starttime, TP_PROTO(struct task_struct *tsk, unsigned long settime_jiffies), TP_ARGS(tsk, settime_jiffies)); +DECLARE_HOOK(android_vh_record_pcpu_rwsem_time_early, + TP_PROTO(unsigned long settime_jiffies, struct percpu_rw_semaphore *sem), + TP_ARGS(settime_jiffies, sem)); struct mutex_waiter; DECLARE_HOOK(android_vh_alter_mutex_list_add, diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h index 367b0a42ada90..892b770067b5b 100644 --- a/kernel/cgroup/cgroup-internal.h +++ b/kernel/cgroup/cgroup-internal.h @@ -251,7 +251,8 @@ int cgroup_attach_task(struct cgroup *dst_cgrp, struct task_struct *leader, void cgroup_attach_lock(bool lock_threadgroup); void cgroup_attach_unlock(bool lock_threadgroup); struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup, - bool *locked) + bool *locked, + struct cgroup *dst_cgrp); __acquires(&cgroup_threadgroup_rwsem); void cgroup_procs_write_finish(struct task_struct *task, bool locked) __releases(&cgroup_threadgroup_rwsem); diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c index c6822bf6c918d..fed5bf717ce19 100644 --- a/kernel/cgroup/cgroup-v1.c +++ b/kernel/cgroup/cgroup-v1.c @@ -501,7 +501,7 @@ static ssize_t __cgroup1_procs_write(struct kernfs_open_file *of, if (!cgrp) return -ENODEV; - task = cgroup_procs_write_start(buf, threadgroup, &locked); + task = cgroup_procs_write_start(buf, threadgroup, &locked, cgrp); ret = PTR_ERR_OR_ZERO(task); if (ret) goto out_unlock; diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index c827832ff7a2a..4f5074da14481 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -114,6 +114,7 @@ static DEFINE_SPINLOCK(cgroup_idr_lock); static DEFINE_SPINLOCK(cgroup_file_kn_lock); DEFINE_PERCPU_RWSEM(cgroup_threadgroup_rwsem); +EXPORT_SYMBOL_GPL(cgroup_threadgroup_rwsem); #define cgroup_assert_mutex_or_rcu_locked() \ RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \ @@ -2920,10 +2921,12 @@ int cgroup_attach_task(struct cgroup *dst_cgrp, struct task_struct *leader, } struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup, - bool *threadgroup_locked) + bool *threadgroup_locked, + struct cgroup *dst_cgrp) { struct task_struct *tsk; pid_t pid; + bool force_migration = false; if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0) return ERR_PTR(-EINVAL); @@ -2954,13 +2957,16 @@ struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup, if (threadgroup) tsk = tsk->group_leader; + if (tsk->flags & PF_KTHREAD) + trace_android_rvh_cgroup_force_kthread_migration(tsk, dst_cgrp, &force_migration); + /* * kthreads may acquire PF_NO_SETAFFINITY during initialization. * If userland migrates such a kthread to a non-root cgroup, it can * become trapped in a cpuset, or RT kthread may be born in a * cgroup with no rt_runtime allocated. Just say no. */ - if (tsk->no_cgroup_migration || (tsk->flags & PF_NO_SETAFFINITY)) { + if (!force_migration && (tsk->no_cgroup_migration || (tsk->flags & PF_NO_SETAFFINITY))) { tsk = ERR_PTR(-EINVAL); goto out_unlock_threadgroup; } @@ -5147,7 +5153,7 @@ static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf, if (!dst_cgrp) return -ENODEV; - task = cgroup_procs_write_start(buf, threadgroup, &threadgroup_locked); + task = cgroup_procs_write_start(buf, threadgroup, &threadgroup_locked, dst_cgrp); ret = PTR_ERR_OR_ZERO(task); if (ret) goto out_unlock; diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index 084aedde3d0fb..8b64155ebb1f6 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -26,6 +26,20 @@ void _trace_android_vh_record_pcpu_rwsem_starttime(struct task_struct *tsk, } EXPORT_SYMBOL_GPL(_trace_android_vh_record_pcpu_rwsem_starttime); +/* + * trace_android_vh_record_pcpu_rwsem_time_early is called in + * include/linux/percpu-rwsem.h by including include/hooks/dtask.h, which + * will result to build-err. So we create + * func: _trace_android_vh_record_pcpu_rwsem_time_early for percpu-rwsem.h to call. +*/ + +void _trace_android_vh_record_pcpu_rwsem_time_early( + unsigned long settime, struct percpu_rw_semaphore *sem) +{ + trace_android_vh_record_pcpu_rwsem_time_early(settime, sem); +} +EXPORT_SYMBOL_GPL(_trace_android_vh_record_pcpu_rwsem_time_early); + int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, const char *name, struct lock_class_key *key) { @@ -242,6 +256,8 @@ void __sched percpu_down_write(struct percpu_rw_semaphore *sem) rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); trace_contention_begin(sem, LCB_F_PERCPU | LCB_F_WRITE); + trace_android_vh_record_pcpu_rwsem_time_early(jiffies, sem); + /* Notify readers to take the slow path. */ rcu_sync_enter(&sem->rss); @@ -294,6 +310,7 @@ void percpu_up_write(struct percpu_rw_semaphore *sem) * exclusive write lock because its counting. */ rcu_sync_exit(&sem->rss); + trace_android_vh_record_pcpu_rwsem_time_early(0, sem); trace_android_vh_record_pcpu_rwsem_starttime(current, 0); } EXPORT_SYMBOL_GPL(percpu_up_write);