Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions src/include/sof/schedule/ll_schedule_domain.h
Original file line number Diff line number Diff line change
Expand Up @@ -292,8 +292,9 @@ static inline struct ll_schedule_domain *domain_init
struct ll_schedule_domain *domain;

#ifdef CONFIG_SOF_USERSPACE_LL
domain = sof_heap_alloc(zephyr_ll_user_heap(), SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
sizeof(*domain), sizeof(void *));
/* created at boot by kernel code and never freed */
domain = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW, SOF_MEM_FLAG_COHERENT,
sizeof(*domain), 0);
if (domain)
memset(domain, 0, sizeof(*domain));
#else
Expand All @@ -318,10 +319,8 @@ static inline struct ll_schedule_domain *domain_init
#ifdef CONFIG_SOF_USERSPACE_LL
/* Allocate mutex dynamically for userspace access */
domain->lock = k_object_alloc(K_OBJ_MUTEX);
if (!domain->lock) {
sof_heap_free(zephyr_ll_user_heap(), domain);
if (!domain->lock)
return NULL;
}
k_mutex_init(domain->lock);
#else
k_spinlock_init(&domain->lock);
Expand Down
10 changes: 10 additions & 0 deletions src/include/sof/schedule/schedule.h
Original file line number Diff line number Diff line change
Expand Up @@ -221,10 +221,20 @@ struct schedulers **arch_schedulers_get(void);
* Invokes cpu_get_id(), so this function must only be called from kernel context.
* User-space callers should use @ref arch_user_schedulers_get_for_core instead.
*
* Returns the kernel-only copy of the list head. After changing it, call
* arch_user_schedulers_publish() to make it visible to user threads.
*
* @return Pointer to the per-core user schedulers list head pointer, or NULL if user LL is disabled.
*/
struct schedulers **arch_user_schedulers_get(void);

/**
* @brief Make the user scheduler list of the current core visible to user threads.
*
* Kernel context only.
*/
void arch_user_schedulers_publish(void);

/**
* @brief Retrieve registered schedulers for user-space threads on a specific core.
*
Expand Down
33 changes: 32 additions & 1 deletion src/include/sof/schedule/zephyr_ll.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#ifndef __SOF_SCHEDULE_ZEPHYR_LL_H__
#define __SOF_SCHEDULE_ZEPHYR_LL_H__

#include <rtos/alloc.h>
#include <sof/lib/cpu.h>
#include <sof/list.h>
#include <sof/schedule/ll_schedule_domain.h>
Expand All @@ -29,7 +30,6 @@ struct zephyr_ll {
#if CONFIG_SOF_USERSPACE_LL
struct k_mutex *lock; /* mutex for userspace */
#endif
struct k_heap *heap;
};

/* per-task scheduler data */
Expand All @@ -42,6 +42,37 @@ struct zephyr_ll_pdata {
#endif
};

#if CONFIG_SOF_USERSPACE_LL
/*
* The scheduler state and the tasks that kernel threads create at boot are
* in the kernel RW area of the LL memory region, see
* zephyr_ll_karea_alloc(). They are never freed. Other tasks and their
* data are on the LL user heap.
*/
static inline bool zephyr_ll_in_karea(const void *ptr, size_t size)
{
/*
* The area bounds are kernel-only. Boot tasks are only created and
* freed by kernel code, user threads only handle heap tasks.
*/
if (k_is_user_context())
return false;

return zephyr_ll_karea_contains(ZEPHYR_LL_KAREA_RW, ptr, size);
}

static inline void zephyr_ll_free(void *ptr)
{
if (!zephyr_ll_in_karea(ptr, 1))
sof_heap_free(zephyr_ll_user_heap(), ptr);
}
#else
static inline void zephyr_ll_free(void *ptr)
{
sof_heap_free(sof_sys_heap_get(), ptr);
}
#endif

static inline void zephyr_ll_assert_core(const struct zephyr_ll *sch)
{
assert(CONFIG_CORE_COUNT == 1 || IS_ENABLED(CONFIG_SOF_USERSPACE_LL) ||
Expand Down
16 changes: 15 additions & 1 deletion src/include/sof/userspace/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -171,9 +171,23 @@ live until the firmware is reset. The Xtensa MMU has one write permission for bo
the RO area is also read-only to kernel code running in the LL memory domain, such as system
calls from LL threads: only kernel threads can write objects there.

Objects in the kernel areas:

- RO: the user scheduler list heads and descriptors (`struct schedule_data`, including the
scheduler ops), and the LL allocation context `ipc->ll_alloc`. Kernel code finds the user
scheduler lists through a kernel-only copy of the per-core list heads.
- RW: `struct ipc_user`, the IPC data buffer `ipc->comp_data`, the LL scheduler state
(`struct zephyr_ll`), the LL scheduling domain and its Zephyr domain data, the DP scheduler
data, and the LL tasks that kernel threads create at boot together with their task data.

The LL timer interrupt and the LL thread setup and teardown use kernel-only copies of the
domain data pointer, the per-core semaphores and the LL threads, not the user-writable domain
data.

The LL user heap keeps its `sys_heap` metadata in user-writable memory. Kernel code is being
moved off it; `CONFIG_SOF_ZEPHYR_LL_HEAP_KERNEL_TRACE` logs the remaining uses by kernel
threads.
threads. Kernel code still follows some pointers stored in user-writable memory, for example
`task->sch` and the domain `ops`.

## Memory-partition markers

Expand Down
18 changes: 12 additions & 6 deletions src/kernel/ipc/ipc-common-kernel.c
Original file line number Diff line number Diff line change
Expand Up @@ -614,17 +614,17 @@ __cold static void ipc_user_init(void)
{
struct ipc *ipc = ipc_get();
struct k_thread *audio_thread;
struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(),
SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
sizeof(*ipc_user), 0);
struct ipc_user *ipc_user = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW,
SOF_MEM_FLAG_COHERENT,
sizeof(*ipc_user), 0);
int ret, core;

if (!ipc_user) {
LOG_ERR("user IPC pdata alloc failed");
sof_panic(SOF_IPC_PANIC_IPC);
}

/* sof_heap_alloc() does not zero, start with clean state (e.g. ppl_phase) */
/* the allocation is not zeroed, start with clean state (e.g. ppl_phase) */
memset(ipc_user, 0, sizeof(*ipc_user));

assert_can_be_cold();
Expand Down Expand Up @@ -749,8 +749,9 @@ __cold int ipc_init(struct sof *sof)

ipc = ipc_get();
memset(ipc, 0, sizeof(*ipc));
ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
sizeof(*ipc->ll_alloc), 0);
/* user threads use but must not change it, see the LL region README */
ipc->ll_alloc = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RO, SOF_MEM_FLAG_COHERENT,
sizeof(*ipc->ll_alloc), 0);
if (!ipc->ll_alloc) {
tr_err(&ipc_tr, "Unable to allocate IPC ll_alloc");
sof_panic(SOF_IPC_PANIC_IPC);
Expand All @@ -769,8 +770,13 @@ __cold int ipc_init(struct sof *sof)
sof->ipc = ipc;
#endif /* CONFIG_SOF_USERSPACE_LL */

#ifdef CONFIG_SOF_USERSPACE_LL
ipc->comp_data = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW, SOF_MEM_FLAG_COHERENT,
SOF_IPC_MSG_MAX_SIZE, 0);
#else
ipc->comp_data = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
SOF_IPC_MSG_MAX_SIZE, 0);
#endif
if (!ipc->comp_data) {
tr_err(&ipc_tr, "Unable to allocate IPC component data");
sof_panic(SOF_IPC_PANIC_IPC);
Expand Down
25 changes: 23 additions & 2 deletions src/kernel/schedule/schedule-kernel.c
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
#include <sof/lib/uuid.h>
#include <sof/list.h>
#include <sof/schedule/schedule.h>
#if CONFIG_SOF_USERSPACE_LL
#include <sof/schedule/ll_schedule_domain.h>
#endif
#include <rtos/task.h>
#include <ipc/topology.h>
#include <errno.h>
Expand All @@ -26,6 +29,21 @@ extern struct tr_ctx sch_tr;
* src/schedule/schedule.c
*/

/*
* The scheduler descriptors of user schedulers, including their ops, are
* read by LL user threads but must not be written by them: put them in the
* kernel RO area of the LL memory region. They are created by kernel threads
* at boot and at secondary core bring-up and never freed.
*/
__cold static void *scheduler_alloc(struct k_heap *heap, size_t size)
{
#if CONFIG_SOF_USERSPACE_LL
if (heap)
return zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RO, 0, size, 0);
#endif
return sof_heap_alloc(heap, SOF_MEM_FLAG_KERNEL, size, 0);
}

__cold static void scheduler_register(struct schedule_data *scheduler)
{
struct schedulers **sch = arch_schedulers_get();
Expand All @@ -40,7 +58,7 @@ __cold static void scheduler_register(struct schedule_data *scheduler)

if (!*sch) {
/* init schedulers list */
*sch = sof_heap_alloc(heap, 0, sizeof(**sch), 0);
*sch = scheduler_alloc(heap, sizeof(**sch));
if (!*sch) {
tr_err(&sch_tr, "allocation failed");
return;
Expand All @@ -50,6 +68,9 @@ __cold static void scheduler_register(struct schedule_data *scheduler)
}

list_item_append(&scheduler->list, &(*sch)->list);

if (heap)
arch_user_schedulers_publish();
}

__cold void scheduler_init(int type, const struct scheduler_ops *ops, void *data)
Expand All @@ -66,7 +87,7 @@ __cold void scheduler_init(int type, const struct scheduler_ops *ops, void *data
!ops->schedule_task_free)
return;

sch = sof_heap_alloc(heap, SOF_MEM_FLAG_KERNEL, sizeof(*sch), 0);
sch = scheduler_alloc(heap, sizeof(*sch));
if (!sch) {
tr_err(&sch_tr, "allocation failed");
sof_panic(SOF_IPC_PANIC_IPC);
Expand Down
69 changes: 53 additions & 16 deletions src/kernel/schedule/zephyr_domain-kernel.c
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,27 @@ K_THREAD_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_S
K_KERNEL_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE);
#endif

#ifdef CONFIG_SOF_USERSPACE_LL
/*
* The Zephyr domain data is writable by LL user threads, which also have
* access to the timer and can set its user data. The timer IRQ therefore
* uses kernel-only copies of the domain pointer and of the per-core
* semaphores, set up by kernel code.
*/
static struct zephyr_domain *zephyr_domain_kptr;
static struct k_sem *ll_thread_ksem[CONFIG_CORE_COUNT];
#endif

/* Timer callback: runs in timer IRQ context */
static void zephyr_domain_timer_fn(struct k_timer *timer)
{
#ifdef CONFIG_SOF_USERSPACE_LL
/* only used as an "is registered" flag, see zephyr_domain_kptr */
struct zephyr_domain *zephyr_domain = k_timer_user_data_get(timer) ?
zephyr_domain_kptr : NULL;
#else
struct zephyr_domain *zephyr_domain = k_timer_user_data_get(timer);
#endif
int core;

/*
Expand All @@ -74,8 +91,14 @@ static void zephyr_domain_timer_fn(struct k_timer *timer)
for (core = 0; core < CONFIG_CORE_COUNT; core++) {
struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core;

#ifdef CONFIG_SOF_USERSPACE_LL
/* a forged handler only causes an extra or a missed tick */
if (dt->handler && ll_thread_ksem[core])
k_sem_give(ll_thread_ksem[core]);
#else
if (dt->handler)
k_sem_give(dt->sem);
#endif
}
}

Expand Down Expand Up @@ -203,9 +226,11 @@ static struct k_thread *ll_thread_tid[CONFIG_CORE_COUNT];
static int zephyr_domain_thread_init(struct ll_schedule_domain *domain,
struct task *task)
{
struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain);
/* kernel context: do not take the domain data from user memory */
struct zephyr_domain *zephyr_domain = zephyr_domain_kptr;
struct zephyr_domain_thread *dt;
char thread_name[] = "ll_thread0";
struct k_sem *sem;
k_tid_t thread;
int core = task->core;

Expand All @@ -223,14 +248,16 @@ static int zephyr_domain_thread_init(struct ll_schedule_domain *domain,
dt->handler = NULL;
dt->arg = NULL;

dt->sem = k_object_alloc(K_OBJ_SEM);
if (!dt->sem) {
/* keep the handle in a local, dt is user-writable */
sem = k_object_alloc(K_OBJ_SEM);
if (!sem) {
tr_err(&ll_tr, "Failed to allocate semaphore for core %d", core);
return -ENOMEM;
}

/* 10 is rather random, we better not accumulate 10 missed timer interrupts */
k_sem_init(dt->sem, 0, 10);
k_sem_init(sem, 0, 10);
dt->sem = sem;

thread_name[sizeof(thread_name) - 2] = '0' + core;

Expand All @@ -242,7 +269,7 @@ static int zephyr_domain_thread_init(struct ll_schedule_domain *domain,
thread = k_object_alloc(K_OBJ_THREAD);
if (!thread) {
tr_err(&ll_tr, "Failed to allocate thread object for core %d", core);
k_object_free(dt->sem);
k_object_free(sem);
dt->sem = NULL;
return -ENOMEM;
}
Expand All @@ -252,8 +279,9 @@ static int zephyr_domain_thread_init(struct ll_schedule_domain *domain,
INT_TO_POINTER(core), NULL, CONFIG_LL_THREAD_PRIORITY,
K_USER, K_FOREVER);

/* record in the kernel-only table for syscall-context lookups */
/* record in the kernel-only tables for syscall-context lookups */
ll_thread_tid[core] = thread;
ll_thread_ksem[core] = sem;
sof_sysuser_thread_set(SOF_SYSUSER_LL, core, thread);
dt->ll_thread = thread;

Expand All @@ -263,7 +291,7 @@ static int zephyr_domain_thread_init(struct ll_schedule_domain *domain,
k_thread_name_set(thread, thread_name);

k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread);
k_thread_access_grant(thread, dt->sem, domain->lock, zephyr_domain->timer);
k_thread_access_grant(thread, sem, domain->lock, zephyr_domain->timer);
user_grant_dai_access_all(thread);
user_grant_dma_access_all(thread);
user_grant_uaol_access_all(thread);
Expand Down Expand Up @@ -300,7 +328,8 @@ static int zephyr_domain_thread_init(struct ll_schedule_domain *domain,
static void zephyr_domain_thread_free(struct ll_schedule_domain *domain,
uint32_t num_tasks)
{
struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain);
/* kernel context: do not take the domain data from user memory */
struct zephyr_domain *zephyr_domain = zephyr_domain_kptr;
int core = cpu_get_id();
struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core;

Expand All @@ -322,16 +351,22 @@ static void zephyr_domain_thread_free(struct ll_schedule_domain *domain,

k_mutex_unlock(domain->lock);

if (dt->ll_thread) {
/* free the kernel-recorded objects, not the user-writable dt copies */
if (ll_thread_tid[core]) {
struct k_thread *thread = ll_thread_tid[core];

sof_sysuser_thread_set(SOF_SYSUSER_LL, core, NULL);
ll_thread_tid[core] = NULL;
k_thread_abort(dt->ll_thread);
k_object_free(dt->ll_thread);
k_thread_abort(thread);
k_object_free(thread);
dt->ll_thread = NULL;
}

if (dt->sem) {
k_object_free(dt->sem);
if (ll_thread_ksem[core]) {
struct k_sem *sem = ll_thread_ksem[core];

ll_thread_ksem[core] = NULL;
k_object_free(sem);
dt->sem = NULL;
}

Expand Down Expand Up @@ -409,9 +444,8 @@ struct ll_schedule_domain *zephyr_domain_init(int clk)
}

#if CONFIG_SOF_USERSPACE_LL
zephyr_domain = sof_heap_alloc(zephyr_ll_user_heap(),
SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
sizeof(*zephyr_domain), sizeof(void *));
zephyr_domain = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW, SOF_MEM_FLAG_COHERENT,
sizeof(*zephyr_domain), 0);
if (zephyr_domain)
memset(zephyr_domain, 0, sizeof(*zephyr_domain));
#else
Expand Down Expand Up @@ -446,6 +480,9 @@ struct ll_schedule_domain *zephyr_domain_init(int clk)
#endif

ll_sch_domain_set_pdata(domain, zephyr_domain);
#ifdef CONFIG_SOF_USERSPACE_LL
zephyr_domain_kptr = zephyr_domain;
#endif

for (core = 0; core < CONFIG_CORE_COUNT; core++) {
#ifndef CONFIG_SOF_USERSPACE_LL
Expand Down
Loading
Loading