diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index 0b25b839ecef..4879dc1e2762 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -292,8 +292,9 @@ static inline struct ll_schedule_domain *domain_init struct ll_schedule_domain *domain; #ifdef CONFIG_SOF_USERSPACE_LL - domain = sof_heap_alloc(zephyr_ll_user_heap(), SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*domain), sizeof(void *)); + /* created at boot by kernel code and never freed */ + domain = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW, SOF_MEM_FLAG_COHERENT, + sizeof(*domain), 0); if (domain) memset(domain, 0, sizeof(*domain)); #else @@ -318,10 +319,8 @@ static inline struct ll_schedule_domain *domain_init #ifdef CONFIG_SOF_USERSPACE_LL /* Allocate mutex dynamically for userspace access */ domain->lock = k_object_alloc(K_OBJ_MUTEX); - if (!domain->lock) { - sof_heap_free(zephyr_ll_user_heap(), domain); + if (!domain->lock) return NULL; - } k_mutex_init(domain->lock); #else k_spinlock_init(&domain->lock); diff --git a/src/include/sof/schedule/schedule.h b/src/include/sof/schedule/schedule.h index 362d15fe948d..25c1e101177e 100644 --- a/src/include/sof/schedule/schedule.h +++ b/src/include/sof/schedule/schedule.h @@ -221,10 +221,20 @@ struct schedulers **arch_schedulers_get(void); * Invokes cpu_get_id(), so this function must only be called from kernel context. * User-space callers should use @ref arch_user_schedulers_get_for_core instead. * + * Returns the kernel-only copy of the list head. After changing it, call + * arch_user_schedulers_publish() to make it visible to user threads. + * * @return Pointer to the per-core user schedulers list head pointer, or NULL if user LL is disabled. */ struct schedulers **arch_user_schedulers_get(void); +/** + * @brief Make the user scheduler list of the current core visible to user threads. + * + * Kernel context only. + */ +void arch_user_schedulers_publish(void); + /** * @brief Retrieve registered schedulers for user-space threads on a specific core. * diff --git a/src/include/sof/schedule/zephyr_ll.h b/src/include/sof/schedule/zephyr_ll.h index 14a7b75c1821..1da2a2f64bc3 100644 --- a/src/include/sof/schedule/zephyr_ll.h +++ b/src/include/sof/schedule/zephyr_ll.h @@ -12,6 +12,7 @@ #ifndef __SOF_SCHEDULE_ZEPHYR_LL_H__ #define __SOF_SCHEDULE_ZEPHYR_LL_H__ +#include #include #include #include @@ -29,7 +30,6 @@ struct zephyr_ll { #if CONFIG_SOF_USERSPACE_LL struct k_mutex *lock; /* mutex for userspace */ #endif - struct k_heap *heap; }; /* per-task scheduler data */ @@ -42,6 +42,37 @@ struct zephyr_ll_pdata { #endif }; +#if CONFIG_SOF_USERSPACE_LL +/* + * The scheduler state and the tasks that kernel threads create at boot are + * in the kernel RW area of the LL memory region, see + * zephyr_ll_karea_alloc(). They are never freed. Other tasks and their + * data are on the LL user heap. + */ +static inline bool zephyr_ll_in_karea(const void *ptr, size_t size) +{ + /* + * The area bounds are kernel-only. Boot tasks are only created and + * freed by kernel code, user threads only handle heap tasks. + */ + if (k_is_user_context()) + return false; + + return zephyr_ll_karea_contains(ZEPHYR_LL_KAREA_RW, ptr, size); +} + +static inline void zephyr_ll_free(void *ptr) +{ + if (!zephyr_ll_in_karea(ptr, 1)) + sof_heap_free(zephyr_ll_user_heap(), ptr); +} +#else +static inline void zephyr_ll_free(void *ptr) +{ + sof_heap_free(sof_sys_heap_get(), ptr); +} +#endif + static inline void zephyr_ll_assert_core(const struct zephyr_ll *sch) { assert(CONFIG_CORE_COUNT == 1 || IS_ENABLED(CONFIG_SOF_USERSPACE_LL) || diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md index f73bc1b53e12..045e6c9552fb 100644 --- a/src/include/sof/userspace/README.md +++ b/src/include/sof/userspace/README.md @@ -171,9 +171,23 @@ live until the firmware is reset. The Xtensa MMU has one write permission for bo the RO area is also read-only to kernel code running in the LL memory domain, such as system calls from LL threads: only kernel threads can write objects there. +Objects in the kernel areas: + +- RO: the user scheduler list heads and descriptors (`struct schedule_data`, including the + scheduler ops), and the LL allocation context `ipc->ll_alloc`. Kernel code finds the user + scheduler lists through a kernel-only copy of the per-core list heads. +- RW: `struct ipc_user`, the IPC data buffer `ipc->comp_data`, the LL scheduler state + (`struct zephyr_ll`), the LL scheduling domain and its Zephyr domain data, the DP scheduler + data, and the LL tasks that kernel threads create at boot together with their task data. + +The LL timer interrupt and the LL thread setup and teardown use kernel-only copies of the +domain data pointer, the per-core semaphores and the LL threads, not the user-writable domain +data. + The LL user heap keeps its `sys_heap` metadata in user-writable memory. Kernel code is being moved off it; `CONFIG_SOF_ZEPHYR_LL_HEAP_KERNEL_TRACE` logs the remaining uses by kernel -threads. +threads. Kernel code still follows some pointers stored in user-writable memory, for example +`task->sch` and the domain `ops`. ## Memory-partition markers diff --git a/src/kernel/ipc/ipc-common-kernel.c b/src/kernel/ipc/ipc-common-kernel.c index 82cfb9d6049c..40d8f07c52c2 100644 --- a/src/kernel/ipc/ipc-common-kernel.c +++ b/src/kernel/ipc/ipc-common-kernel.c @@ -614,9 +614,9 @@ __cold static void ipc_user_init(void) { struct ipc *ipc = ipc_get(); struct k_thread *audio_thread; - struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*ipc_user), 0); + struct ipc_user *ipc_user = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW, + SOF_MEM_FLAG_COHERENT, + sizeof(*ipc_user), 0); int ret, core; if (!ipc_user) { @@ -624,7 +624,7 @@ __cold static void ipc_user_init(void) sof_panic(SOF_IPC_PANIC_IPC); } - /* sof_heap_alloc() does not zero, start with clean state (e.g. ppl_phase) */ + /* the allocation is not zeroed, start with clean state (e.g. ppl_phase) */ memset(ipc_user, 0, sizeof(*ipc_user)); assert_can_be_cold(); @@ -749,8 +749,9 @@ __cold int ipc_init(struct sof *sof) ipc = ipc_get(); memset(ipc, 0, sizeof(*ipc)); - ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*ipc->ll_alloc), 0); + /* user threads use but must not change it, see the LL region README */ + ipc->ll_alloc = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RO, SOF_MEM_FLAG_COHERENT, + sizeof(*ipc->ll_alloc), 0); if (!ipc->ll_alloc) { tr_err(&ipc_tr, "Unable to allocate IPC ll_alloc"); sof_panic(SOF_IPC_PANIC_IPC); @@ -769,8 +770,13 @@ __cold int ipc_init(struct sof *sof) sof->ipc = ipc; #endif /* CONFIG_SOF_USERSPACE_LL */ +#ifdef CONFIG_SOF_USERSPACE_LL + ipc->comp_data = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW, SOF_MEM_FLAG_COHERENT, + SOF_IPC_MSG_MAX_SIZE, 0); +#else ipc->comp_data = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, SOF_IPC_MSG_MAX_SIZE, 0); +#endif if (!ipc->comp_data) { tr_err(&ipc_tr, "Unable to allocate IPC component data"); sof_panic(SOF_IPC_PANIC_IPC); diff --git a/src/kernel/schedule/schedule-kernel.c b/src/kernel/schedule/schedule-kernel.c index bdefa7f35567..af9dd1ac99f3 100644 --- a/src/kernel/schedule/schedule-kernel.c +++ b/src/kernel/schedule/schedule-kernel.c @@ -12,6 +12,9 @@ #include #include #include +#if CONFIG_SOF_USERSPACE_LL +#include +#endif #include #include #include @@ -26,6 +29,21 @@ extern struct tr_ctx sch_tr; * src/schedule/schedule.c */ +/* + * The scheduler descriptors of user schedulers, including their ops, are + * read by LL user threads but must not be written by them: put them in the + * kernel RO area of the LL memory region. They are created by kernel threads + * at boot and at secondary core bring-up and never freed. + */ +__cold static void *scheduler_alloc(struct k_heap *heap, size_t size) +{ +#if CONFIG_SOF_USERSPACE_LL + if (heap) + return zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RO, 0, size, 0); +#endif + return sof_heap_alloc(heap, SOF_MEM_FLAG_KERNEL, size, 0); +} + __cold static void scheduler_register(struct schedule_data *scheduler) { struct schedulers **sch = arch_schedulers_get(); @@ -40,7 +58,7 @@ __cold static void scheduler_register(struct schedule_data *scheduler) if (!*sch) { /* init schedulers list */ - *sch = sof_heap_alloc(heap, 0, sizeof(**sch), 0); + *sch = scheduler_alloc(heap, sizeof(**sch)); if (!*sch) { tr_err(&sch_tr, "allocation failed"); return; @@ -50,6 +68,9 @@ __cold static void scheduler_register(struct schedule_data *scheduler) } list_item_append(&scheduler->list, &(*sch)->list); + + if (heap) + arch_user_schedulers_publish(); } __cold void scheduler_init(int type, const struct scheduler_ops *ops, void *data) @@ -66,7 +87,7 @@ __cold void scheduler_init(int type, const struct scheduler_ops *ops, void *data !ops->schedule_task_free) return; - sch = sof_heap_alloc(heap, SOF_MEM_FLAG_KERNEL, sizeof(*sch), 0); + sch = scheduler_alloc(heap, sizeof(*sch)); if (!sch) { tr_err(&sch_tr, "allocation failed"); sof_panic(SOF_IPC_PANIC_IPC); diff --git a/src/kernel/schedule/zephyr_domain-kernel.c b/src/kernel/schedule/zephyr_domain-kernel.c index 6acbb5e00904..e1c7bd6f1156 100644 --- a/src/kernel/schedule/zephyr_domain-kernel.c +++ b/src/kernel/schedule/zephyr_domain-kernel.c @@ -52,10 +52,27 @@ K_THREAD_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_S K_KERNEL_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); #endif +#ifdef CONFIG_SOF_USERSPACE_LL +/* + * The Zephyr domain data is writable by LL user threads, which also have + * access to the timer and can set its user data. The timer IRQ therefore + * uses kernel-only copies of the domain pointer and of the per-core + * semaphores, set up by kernel code. + */ +static struct zephyr_domain *zephyr_domain_kptr; +static struct k_sem *ll_thread_ksem[CONFIG_CORE_COUNT]; +#endif + /* Timer callback: runs in timer IRQ context */ static void zephyr_domain_timer_fn(struct k_timer *timer) { +#ifdef CONFIG_SOF_USERSPACE_LL + /* only used as an "is registered" flag, see zephyr_domain_kptr */ + struct zephyr_domain *zephyr_domain = k_timer_user_data_get(timer) ? + zephyr_domain_kptr : NULL; +#else struct zephyr_domain *zephyr_domain = k_timer_user_data_get(timer); +#endif int core; /* @@ -74,8 +91,14 @@ static void zephyr_domain_timer_fn(struct k_timer *timer) for (core = 0; core < CONFIG_CORE_COUNT; core++) { struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; +#ifdef CONFIG_SOF_USERSPACE_LL + /* a forged handler only causes an extra or a missed tick */ + if (dt->handler && ll_thread_ksem[core]) + k_sem_give(ll_thread_ksem[core]); +#else if (dt->handler) k_sem_give(dt->sem); +#endif } } @@ -203,9 +226,11 @@ static struct k_thread *ll_thread_tid[CONFIG_CORE_COUNT]; static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, struct task *task) { - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + /* kernel context: do not take the domain data from user memory */ + struct zephyr_domain *zephyr_domain = zephyr_domain_kptr; struct zephyr_domain_thread *dt; char thread_name[] = "ll_thread0"; + struct k_sem *sem; k_tid_t thread; int core = task->core; @@ -223,14 +248,16 @@ static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, dt->handler = NULL; dt->arg = NULL; - dt->sem = k_object_alloc(K_OBJ_SEM); - if (!dt->sem) { + /* keep the handle in a local, dt is user-writable */ + sem = k_object_alloc(K_OBJ_SEM); + if (!sem) { tr_err(&ll_tr, "Failed to allocate semaphore for core %d", core); return -ENOMEM; } /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ - k_sem_init(dt->sem, 0, 10); + k_sem_init(sem, 0, 10); + dt->sem = sem; thread_name[sizeof(thread_name) - 2] = '0' + core; @@ -242,7 +269,7 @@ static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, thread = k_object_alloc(K_OBJ_THREAD); if (!thread) { tr_err(&ll_tr, "Failed to allocate thread object for core %d", core); - k_object_free(dt->sem); + k_object_free(sem); dt->sem = NULL; return -ENOMEM; } @@ -252,8 +279,9 @@ static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, INT_TO_POINTER(core), NULL, CONFIG_LL_THREAD_PRIORITY, K_USER, K_FOREVER); - /* record in the kernel-only table for syscall-context lookups */ + /* record in the kernel-only tables for syscall-context lookups */ ll_thread_tid[core] = thread; + ll_thread_ksem[core] = sem; sof_sysuser_thread_set(SOF_SYSUSER_LL, core, thread); dt->ll_thread = thread; @@ -263,7 +291,7 @@ static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, k_thread_name_set(thread, thread_name); k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); - k_thread_access_grant(thread, dt->sem, domain->lock, zephyr_domain->timer); + k_thread_access_grant(thread, sem, domain->lock, zephyr_domain->timer); user_grant_dai_access_all(thread); user_grant_dma_access_all(thread); user_grant_uaol_access_all(thread); @@ -300,7 +328,8 @@ static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, static void zephyr_domain_thread_free(struct ll_schedule_domain *domain, uint32_t num_tasks) { - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + /* kernel context: do not take the domain data from user memory */ + struct zephyr_domain *zephyr_domain = zephyr_domain_kptr; int core = cpu_get_id(); struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; @@ -322,16 +351,22 @@ static void zephyr_domain_thread_free(struct ll_schedule_domain *domain, k_mutex_unlock(domain->lock); - if (dt->ll_thread) { + /* free the kernel-recorded objects, not the user-writable dt copies */ + if (ll_thread_tid[core]) { + struct k_thread *thread = ll_thread_tid[core]; + sof_sysuser_thread_set(SOF_SYSUSER_LL, core, NULL); ll_thread_tid[core] = NULL; - k_thread_abort(dt->ll_thread); - k_object_free(dt->ll_thread); + k_thread_abort(thread); + k_object_free(thread); dt->ll_thread = NULL; } - if (dt->sem) { - k_object_free(dt->sem); + if (ll_thread_ksem[core]) { + struct k_sem *sem = ll_thread_ksem[core]; + + ll_thread_ksem[core] = NULL; + k_object_free(sem); dt->sem = NULL; } @@ -409,9 +444,8 @@ struct ll_schedule_domain *zephyr_domain_init(int clk) } #if CONFIG_SOF_USERSPACE_LL - zephyr_domain = sof_heap_alloc(zephyr_ll_user_heap(), - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*zephyr_domain), sizeof(void *)); + zephyr_domain = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW, SOF_MEM_FLAG_COHERENT, + sizeof(*zephyr_domain), 0); if (zephyr_domain) memset(zephyr_domain, 0, sizeof(*zephyr_domain)); #else @@ -446,6 +480,9 @@ struct ll_schedule_domain *zephyr_domain_init(int clk) #endif ll_sch_domain_set_pdata(domain, zephyr_domain); +#ifdef CONFIG_SOF_USERSPACE_LL + zephyr_domain_kptr = zephyr_domain; +#endif for (core = 0; core < CONFIG_CORE_COUNT; core++) { #ifndef CONFIG_SOF_USERSPACE_LL diff --git a/src/kernel/schedule/zephyr_dp_schedule-kernel.c b/src/kernel/schedule/zephyr_dp_schedule-kernel.c index 7f0d7c285d62..ede504665ca1 100644 --- a/src/kernel/schedule/zephyr_dp_schedule-kernel.c +++ b/src/kernel/schedule/zephyr_dp_schedule-kernel.c @@ -280,8 +280,14 @@ __cold int scheduler_dp_init(void) { assert_can_be_cold(); +#if CONFIG_SOF_USERSPACE_LL + /* used by LL user threads, created at boot and never freed */ + struct scheduler_dp_data *dp_sch = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW, 0, + sizeof(*dp_sch), 0); +#else struct scheduler_dp_data *dp_sch = sof_heap_alloc(sof_sys_user_heap_get(), SOF_MEM_FLAG_KERNEL, sizeof(*dp_sch), 0); +#endif if (!dp_sch) return -ENOMEM; diff --git a/src/kernel/schedule/zephyr_ll-kernel.c b/src/kernel/schedule/zephyr_ll-kernel.c index c0a8398b3dc8..028cae362b55 100644 --- a/src/kernel/schedule/zephyr_ll-kernel.c +++ b/src/kernel/schedule/zephyr_ll-kernel.c @@ -241,7 +241,7 @@ static void zephyr_ll_scheduler_free(void *data, uint32_t flags) #if CONFIG_SOF_USERSPACE_LL domain_thread_free(sch->ll_domain, sch->n_tasks); #endif - sof_heap_free(sch->heap, sch); + zephyr_ll_free(sch); } #endif /* CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY */ @@ -265,19 +265,18 @@ __cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) { struct zephyr_ll *sch; int core = cpu_get_id(); - struct k_heap *heap = sof_sys_heap_get(); - int flags = SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT; assert_can_be_cold(); -#if CONFIG_SOF_USERSPACE_LL - heap = zephyr_ll_user_heap(); - flags = SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT; -#endif tr_dbg(&ll_tr, "init on core %d", core); /* initialize per-core scheduler private data */ - sch = sof_heap_alloc(heap, flags, sizeof(*sch), 0); +#if CONFIG_SOF_USERSPACE_LL + sch = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW, SOF_MEM_FLAG_COHERENT, sizeof(*sch), 0); +#else + sch = sof_heap_alloc(sof_sys_heap_get(), SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + sizeof(*sch), 0); +#endif if (!sch) { tr_err(&ll_tr, "allocation failed"); return -ENOMEM; @@ -289,14 +288,13 @@ __cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) sch->ll_domain = domain; sch->core = core; sch->n_tasks = 0; - sch->heap = heap; #if CONFIG_SOF_USERSPACE_LL /* Allocate mutex dynamically for userspace access */ sch->lock = k_object_alloc(K_OBJ_MUTEX); if (!sch->lock) { tr_err(&ll_tr, "mutex allocation failed"); - sof_heap_free(sch->heap, sch); + zephyr_ll_free(sch); return -ENOMEM; } assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] == NULL); diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 7daa415b8fd1..10747f996c56 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -492,7 +492,7 @@ int zephyr_ll_task_free(void *data, struct task *task) zephyr_ll_task_sem_free(task); #endif task->priv_data = NULL; - sof_heap_free(sch->heap, pdata); + zephyr_ll_free(pdata); zephyr_ll_unlock(sch, &flags); return 0; @@ -529,10 +529,11 @@ int zephyr_ll_task_cancel(void *data, struct task *task) } #if CONFIG_SOF_USERSPACE_LL +/* For tasks that kernel threads create at boot, which are never freed */ struct task *zephyr_ll_task_alloc(void) { - struct task *task = sof_heap_alloc(zephyr_ll_user_heap(), SOF_MEM_FLAG_USER, - sizeof(*task), sizeof(void *)); + struct task *task = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW, 0, sizeof(*task), + sizeof(void *)); if (task) /* At least .priv_data must be NULL for zephyr_ll_task_init() */ @@ -620,8 +621,14 @@ int zephyr_ll_task_init(struct task *task, #if CONFIG_SOF_USERSPACE_LL heap = zephyr_ll_user_heap(); alloc_flags = SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT; + + /* the data of a boot task lives as long as the task */ + if (zephyr_ll_in_karea(task, sizeof(*task))) + pdata = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW, SOF_MEM_FLAG_COHERENT, + sizeof(*pdata), 0); + else #endif - pdata = sof_heap_alloc(heap, alloc_flags, sizeof(*pdata), 0); + pdata = sof_heap_alloc(heap, alloc_flags, sizeof(*pdata), 0); if (!pdata) { tr_err(&ll_tr, "alloc failed"); return -ENOMEM; @@ -634,7 +641,7 @@ int zephyr_ll_task_init(struct task *task, #if CONFIG_DYNAMIC_OBJECTS ret = zephyr_ll_task_sem_alloc(task); if (ret < 0) { - sof_heap_free(heap, pdata); + zephyr_ll_free(pdata); task->priv_data = NULL; return ret; } diff --git a/zephyr/lib/alloc.c b/zephyr/lib/alloc.c index 332635770c50..a58934f363df 100644 --- a/zephyr/lib/alloc.c +++ b/zephyr/lib/alloc.c @@ -910,6 +910,19 @@ void z_impl_sof_heap_free(struct k_heap *heap, void *addr) { ll_heap_kernel_trace("free", heap, addr, 0, __builtin_return_address(0)); +#ifdef CONFIG_SOF_USERSPACE_LL + /* + * The kernel-managed LL areas are inside a kernel heap block and are + * never freed: passing such memory to rfree() would corrupt the + * kernel heap. + */ + if (addr && (zephyr_ll_karea_contains(ZEPHYR_LL_KAREA_RW, addr, 1) || + zephyr_ll_karea_contains(ZEPHYR_LL_KAREA_RO, addr, 1))) { + tr_err(&zephyr_tr, "%p is in a kernel LL area, not freed", addr); + return; + } +#endif + if (heap && addr && is_heap_pointer(heap, addr)) heap_free(heap, addr); else diff --git a/zephyr/schedule.c b/zephyr/schedule.c index 3874b28ff1f6..dea2bd10ab5f 100644 --- a/zephyr/schedule.c +++ b/zephyr/schedule.c @@ -25,6 +25,12 @@ static struct schedulers *_k_schedulers[CONFIG_CORE_COUNT]; * types that are managed in user-space */ static APP_SYSUSER_BSS struct schedulers *_u_schedulers[CONFIG_CORE_COUNT]; + +/* + * Kernel-only copy of _u_schedulers: user threads can write _u_schedulers, + * so kernel code uses this copy to find the user scheduler lists. + */ +static struct schedulers *_ku_schedulers[CONFIG_CORE_COUNT]; #endif /** @@ -53,12 +59,28 @@ struct schedulers **arch_user_schedulers_get(void) { #ifdef CONFIG_SOF_USERSPACE_LL assert(!k_is_user_context()); - return _u_schedulers + cpu_get_id(); + return _ku_schedulers + cpu_get_id(); #else return NULL; #endif } +/** + * Makes the user scheduler list of the current core visible to user threads. + * + * Kernel context only, call after changing the list head returned by + * arch_user_schedulers_get(). + */ +void arch_user_schedulers_publish(void) +{ +#ifdef CONFIG_SOF_USERSPACE_LL + unsigned int core = cpu_get_id(); + + assert(!k_is_user_context()); + _u_schedulers[core] = _ku_schedulers[core]; +#endif +} + /** * Retrieves registered user schedulers for the given core. * @@ -70,6 +92,9 @@ struct schedulers **arch_user_schedulers_get(void) struct schedulers **arch_user_schedulers_get_for_core(int core) { #ifdef CONFIG_SOF_USERSPACE_LL + if (!k_is_user_context()) + return _ku_schedulers + core; + return _u_schedulers + core; #else return NULL; diff --git a/zephyr/test/userspace/test_ll_karea.c b/zephyr/test/userspace/test_ll_karea.c index 3fe36bd4458d..f7a3221381a4 100644 --- a/zephyr/test/userspace/test_ll_karea.c +++ b/zephyr/test/userspace/test_ll_karea.c @@ -6,11 +6,14 @@ /* * Tests for the kernel-managed areas of the LL memory region: LL user * threads can write objects in the RW area, only read objects in the RO - * area, and the kernel allocator state is not in user memory. + * area, such as the user scheduler descriptors, and the kernel allocator + * state is not in user memory. */ #include +#include #include +#include #include #include @@ -38,6 +41,8 @@ enum karea_test_case { KAREA_READ_RO, /* write an object in the RO area */ KAREA_WRITE_RO, + /* write the user scheduler list of the current core */ + KAREA_WRITE_SCHEDULERS, }; static void user_function(void *p1, void *p2, void *p3) @@ -60,6 +65,12 @@ static void user_function(void *p1, void *p2, void *p3) case KAREA_WRITE_RO: obj[0] = 0; break; + case KAREA_WRITE_SCHEDULERS: + /* user threads find the list through their own copy of the head */ + obj = (uint32_t *)*arch_user_schedulers_get_for_core((int)(uintptr_t)p3); + zassert_not_null(obj); + obj[0] = 0; + break; default: zassert_unreachable("unknown test case"); } @@ -72,8 +83,9 @@ static void run_case(enum karea_test_case test_case, uint32_t *obj, bool expect_ int ret; k_thread_create(&user_thread, user_stack, USER_STACKSIZE, - user_function, (void *)(uintptr_t)test_case, obj, NULL, - -1, K_USER, K_FOREVER); + user_function, (void *)(uintptr_t)test_case, obj, + (void *)(uintptr_t)cpu_get_id(), -1, K_USER, K_FOREVER); + k_thread_cpu_pin(&user_thread, cpu_get_id()); k_mem_domain_add_thread(zephyr_ll_mem_domain(), &user_thread); if (expect_fault) @@ -127,6 +139,24 @@ ZTEST(userspace_ll_karea, ro_area_user_write_faults) zassert_equal(ro_obj[0], TEST_PATTERN, "RO object was written"); } +ZTEST(userspace_ll_karea, schedulers_read_only) +{ + struct schedulers *sch = *arch_user_schedulers_get(); + struct list_item *slist; + + zassert_not_null(sch, "no user schedulers"); + zassert_true(zephyr_ll_karea_contains(ZEPHYR_LL_KAREA_RO, sch, sizeof(*sch))); + + list_for_item(slist, &sch->list) { + struct schedule_data *sd = container_of(slist, struct schedule_data, list); + + zassert_true(zephyr_ll_karea_contains(ZEPHYR_LL_KAREA_RO, sd, sizeof(*sd)), + "scheduler %d data not in the RO area", sd->type); + } + + run_case(KAREA_WRITE_SCHEDULERS, NULL, true); +} + ZTEST(userspace_ll_karea, contains_and_limits) { size_t sz = TEST_WORDS * sizeof(uint32_t);