From 96b199867f66137be905f68e50d085d1aca661c4 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 14 Nov 2026 18:15:53 +0000 Subject: [PATCH 1/4] lib: split the kernel part of dma.c into src/kernel/lib dma.c mixes the DMA controller lookup with DMA buffer helpers that the host, DAI and copier components use, also in the user-space LL thread. Move the kernel part to src/kernel/lib/dma-kernel.c: - the kernel half of the sof_dma_get() and sof_dma_put() system calls, which pick a DMA controller, keep its reference count and allocate its channel array, with the accessors of the kernel-only DMA state; - dma_get() and dma_put() of the legacy (non-native) DMA drivers; - the DMA log module, UUID and trace context, which only that code uses. dma.c keeps the scatter-gather list helpers and the buffer copy functions. The kernel part is built wherever dma.c is, including the host library. The generated code is unchanged, apart from the line numbers that the DMA buffer copy functions pass to the assertion handler in builds with assertions (qemu_xtensa). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/kernel/lib/CMakeLists.txt | 7 +- src/kernel/lib/dma-kernel.c | 377 ++++++++++++++++++++++++++++++++++ src/lib/dma.c | 345 +------------------------------ 3 files changed, 385 insertions(+), 344 deletions(-) create mode 100644 src/kernel/lib/dma-kernel.c diff --git a/src/kernel/lib/CMakeLists.txt b/src/kernel/lib/CMakeLists.txt index efe6347d02e1..8d1e08b8c8e3 100644 --- a/src/kernel/lib/CMakeLists.txt +++ b/src/kernel/lib/CMakeLists.txt @@ -1,10 +1,15 @@ # SPDX-License-Identifier: BSD-3-Clause +# The kernel parts of src/lib/dma.c and dai.c, built with them +set(common_files dma-kernel.c) + if(CONFIG_LIBRARY) - add_local_sources(sof agent.c) + add_local_sources(sof agent.c ${common_files}) return() endif() +add_local_sources(sof ${common_files}) + if(CONFIG_HAVE_AGENT) add_local_sources(sof agent.c) endif() diff --git a/src/kernel/lib/dma-kernel.c b/src/kernel/lib/dma-kernel.c new file mode 100644 index 000000000000..422c89f20d2c --- /dev/null +++ b/src/kernel/lib/dma-kernel.c @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2016 Intel Corporation. All rights reserved. +// +// Author: Ranjani Sridharan + +#include +#include +#include +#include +#include +#if CONFIG_INTEL_ADSP_MIC_PRIVACY +#include +#endif +#include +#include +#include +#include +#include +#include /* zephyr_ll_user_heap_verify() */ +#include +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_REGISTER(dma, CONFIG_SOF_LOG_LEVEL); + +SOF_DEFINE_REG_UUID(dma); + +/* unused with Zephyr, generates no output */ +DECLARE_TR_CTX(dma_tr, SOF_UUID(dma_uuid), LOG_LEVEL_INFO); + +/* Kernel part of the DMA library: the DMA controller lookup, reference counting and + * channel allocation, i.e. the kernel half of the sof_dma_get() and sof_dma_put() + * system calls, and the getters of the legacy DMA drivers. The DMA buffer helpers + * are in src/lib/dma.c + */ + +#if CONFIG_ZEPHYR_NATIVE_DRIVERS +static int dma_init(struct sof_dma *dma); + +#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA +/* + * dma[] is writable by user threads, so the reference count, the lock and + * the channel array to free are kept in the kernel-only struct + * sof_dma_kinfo. dma->sref is still updated for diagnostics, but never + * trusted. + */ +static int *dma_sref(struct sof_dma *dma) +{ + return &sof_dma_kinfo_get(dma)->sref; +} + +/* + * dma->lock is user-writable too: a user thread could release it while the + * kernel holds it, letting two cores update the reference count at once. + */ +static struct k_spinlock *dma_lock(struct sof_dma *dma) +{ + return &sof_dma_kinfo_get(dma)->lock; +} +#else +static int *dma_sref(struct sof_dma *dma) +{ + return &dma->sref; +} + +static struct k_spinlock *dma_lock(struct sof_dma *dma) +{ + return &dma->lock; +} +#endif + +__cold struct sof_dma *z_impl_sof_dma_get(uint32_t dir, uint32_t cap, uint32_t dev, uint32_t flags) +{ + const struct dma_info *info = dma_info_get(); + int users, ret = 0; + int min_users = INT32_MAX; + struct sof_dma *d = NULL, *dmin = NULL; + k_spinlock_key_t key; + + assert_can_be_cold(); + + if (!info->num_dmas) { + tr_err(&dma_tr, "dma_get(): No DMACs installed"); + return NULL; + } + + /* find DMAC with free channels that matches request */ + for (d = info->dma_array; d < info->dma_array + info->num_dmas; + d++) { + /* skip if this DMAC does not support the requested dir */ + if (dir && (d->plat_data.dir & dir) == 0) + continue; + + /* skip if this DMAC does not support the requested caps */ + if (cap && (d->plat_data.caps & cap) == 0) + continue; + + /* skip if this DMAC does not support the requested dev */ + if (dev && (d->plat_data.devs & dev) == 0) + continue; + + /* if exclusive access is requested */ + if (flags & SOF_DMA_ACCESS_EXCLUSIVE) { + /* ret DMA with no users */ + if (!d->sref) { + dmin = d; + break; + } + } else { + /* get number of users for this DMAC*/ + users = d->sref; + + /* pick DMAC with the least num of users */ + if (users < min_users) { + dmin = d; + min_users = users; + } + } + } + + if (!dmin) { + tr_err(&dma_tr, "No DMAC dir %d caps 0x%x dev 0x%x flags 0x%x", + dir, cap, dev, flags); + + for (d = info->dma_array; + d < info->dma_array + info->num_dmas; + d++) { + tr_err(&dma_tr, " DMAC ID %d users %d busy channels %ld", + d->plat_data.id, d->sref, + atomic_read(&d->num_channels_busy)); + tr_err(&dma_tr, " caps 0x%x dev 0x%x", + d->plat_data.caps, d->plat_data.devs); + } + + return NULL; + } + + /* return DMAC */ + tr_dbg(&dma_tr, "dma_get(), dma-probe id = %d", + dmin->plat_data.id); + + /* Shared DMA controllers with multiple channels + * may be requested many times, let the probe() + * do on-first-use initialization. + */ + key = k_spin_lock(dma_lock(dmin)); + + if (!*dma_sref(dmin)) { + ret = dma_init(dmin); + if (ret < 0) { + tr_err(&dma_tr, "dma_get(): dma-probe failed id = %d, ret = %d", + dmin->plat_data.id, ret); + goto out; + } + } + + ++*dma_sref(dmin); + /* user-visible copy for diagnostics and DMAC selection, never trusted */ + dmin->sref = *dma_sref(dmin); + + tr_info(&dma_tr, "dma_get() ID %d sref = %d busy channels %ld", + dmin->plat_data.id, dmin->sref, + atomic_read(&dmin->num_channels_busy)); +out: + k_spin_unlock(dma_lock(dmin), key); + return !ret ? dmin : NULL; +} + +__cold void z_impl_sof_dma_put(struct sof_dma *dma) +{ + k_spinlock_key_t key; + + assert_can_be_cold(); + + key = k_spin_lock(dma_lock(dma)); + if (*dma_sref(dma) <= 0) { + tr_err(&dma_tr, "dma_put(), dma = %p not in use", dma); + k_spin_unlock(dma_lock(dma), key); + return; + } + + --*dma_sref(dma); + dma->sref = *dma_sref(dma); + if (!*dma_sref(dma)) { +#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA + struct sof_dma_kinfo *kinfo = sof_dma_kinfo_get(dma); + + sof_heap_free(kinfo->heap, kinfo->chan); + kinfo->chan = NULL; +#else + sof_heap_free(dma->heap, dma->chan); +#endif + dma->chan = NULL; + } + + tr_info(&dma_tr, "dma_put(), dma = %p, sref = %d", + dma, dma->sref); + k_spin_unlock(dma_lock(dma), key); +} + +__cold static int dma_init(struct sof_dma *dma) +{ + struct dma_chan_data *chans, *chan; + struct k_heap *heap = sof_sys_user_heap_get(); +#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA + /* dma->plat_data is user-writable, use the kernel snapshot */ + const uint32_t channels = sof_dma_kinfo_get(dma)->channels; +#else + const uint32_t channels = dma->plat_data.channels; +#endif + uint32_t i; + + assert_can_be_cold(); + +#if CONFIG_SOF_USERSPACE_INTERFACE_DMA && CONFIG_SOF_USERSPACE_LL + /* + * Called in supervisor mode from the sof_dma_get() system call. The + * heap pointer comes from user-writable memory, so check that it is + * the LL user heap before running the allocator on it. + */ + if (!zephyr_ll_user_heap_verify(heap)) { + tr_err(&dma_tr, "dma %d: invalid channel heap %p", dma->plat_data.id, heap); + return -EINVAL; + } +#endif + + /* + * allocate dma channels, use a local pointer until the array is + * initialized: dma->chan is user-writable and must not be read back + */ + chans = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(struct dma_chan_data) * channels, 0); + if (!chans) { + tr_err(&dma_tr, "dma %d allocation of channels failed", + dma->plat_data.id); + return -ENOMEM; + } + + memset(chans, 0, sizeof(struct dma_chan_data) * channels); + /* init work */ + for (i = 0, chan = chans; i < channels; i++, chan++) { + chan->dma = dma; + chan->index = i; + } + +#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA + sof_dma_kinfo_get(dma)->heap = heap; + sof_dma_kinfo_get(dma)->chan = chans; +#endif + dma->heap = heap; + dma->chan = chans; + + return 0; +} +#else +struct dma *dma_get(uint32_t dir, uint32_t cap, uint32_t dev, uint32_t flags) +{ + const struct dma_info *info = dma_info_get(); + int users, ret; + int min_users = INT32_MAX; + struct dma *d = NULL, *dmin = NULL; + k_spinlock_key_t key; + + if (!info->num_dmas) { + tr_err(&dma_tr, "dma_get(): No DMACs installed"); + return NULL; + } + + /* find DMAC with free channels that matches request */ + for (d = info->dma_array; d < info->dma_array + info->num_dmas; + d++) { + /* skip if this DMAC does not support the requested dir */ + if (dir && (d->plat_data.dir & dir) == 0) + continue; + + /* skip if this DMAC does not support the requested caps */ + if (cap && (d->plat_data.caps & cap) == 0) + continue; + + /* skip if this DMAC does not support the requested dev */ + if (dev && (d->plat_data.devs & dev) == 0) + continue; + + /* skip if this DMAC has 1 user per avail channel */ + /* TODO: this should be fixed in dai.c to allow more users */ + if (d->sref >= d->plat_data.channels) + continue; + + /* if exclusive access is requested */ + if (flags & DMA_ACCESS_EXCLUSIVE) { + /* ret DMA with no users */ + if (!d->sref) { + dmin = d; + break; + } + } else { + /* get number of users for this DMAC*/ + users = d->sref; + + /* pick DMAC with the least num of users */ + if (users < min_users) { + dmin = d; + min_users = users; + } + } + } + + if (!dmin) { + tr_err(&dma_tr, "No DMAC dir %d caps 0x%x dev 0x%x flags 0x%x", + dir, cap, dev, flags); + + for (d = info->dma_array; + d < info->dma_array + info->num_dmas; + d++) { + tr_err(&dma_tr, " DMAC ID %d users %d busy channels %ld", + d->plat_data.id, d->sref, + atomic_read(&d->num_channels_busy)); + tr_err(&dma_tr, " caps 0x%x dev 0x%x", + d->plat_data.caps, d->plat_data.devs); + } + + return NULL; + } + + /* return DMAC */ + tr_dbg(&dma_tr, "dma_get(), dma-probe id = %d", + dmin->plat_data.id); + + /* Shared DMA controllers with multiple channels + * may be requested many times, let the probe() + * do on-first-use initialization. + */ + key = k_spin_lock(&dmin->lock); + + ret = 0; + if (!dmin->sref) { + ret = dma_probe_legacy(dmin); + if (ret < 0) { + tr_err(&dma_tr, "dma_get(): dma-probe failed id = %d, ret = %d", + dmin->plat_data.id, ret); + } + } + if (!ret) + dmin->sref++; + + tr_info(&dma_tr, "dma_get() ID %d sref = %d busy channels %ld", + dmin->plat_data.id, dmin->sref, + atomic_read(&dmin->num_channels_busy)); + + k_spin_unlock(&dmin->lock, key); + return !ret ? dmin : NULL; +} + +void dma_put(struct dma *dma) +{ + k_spinlock_key_t key; + int ret; + + key = k_spin_lock(&dma->lock); + if (--dma->sref == 0) { + ret = dma_remove_legacy(dma); + if (ret < 0) { + tr_err(&dma_tr, "dma_remove() failed id = %d, ret = %d", + dma->plat_data.id, ret); + } + } + tr_info(&dma_tr, "dma = %p, sref = %d", + dma, dma->sref); + k_spin_unlock(&dma->lock, key); +} +#endif diff --git a/src/lib/dma.c b/src/lib/dma.c index 8a5195212fb7..4c454d8d928c 100644 --- a/src/lib/dma.c +++ b/src/lib/dma.c @@ -17,8 +17,6 @@ #include #include #include -#include /* zephyr_ll_user_heap_verify() */ -#include #include #include #include @@ -28,348 +26,9 @@ #include #include "../audio/copier/copier.h" -LOG_MODULE_REGISTER(dma, CONFIG_SOF_LOG_LEVEL); - -SOF_DEFINE_REG_UUID(dma); - -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(dma_tr, SOF_UUID(dma_uuid), LOG_LEVEL_INFO); - -#if CONFIG_ZEPHYR_NATIVE_DRIVERS -static int dma_init(struct sof_dma *dma); - -#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA -/* - * dma[] is writable by user threads, so the reference count, the lock and - * the channel array to free are kept in the kernel-only struct - * sof_dma_kinfo. dma->sref is still updated for diagnostics, but never - * trusted. - */ -static int *dma_sref(struct sof_dma *dma) -{ - return &sof_dma_kinfo_get(dma)->sref; -} - -/* - * dma->lock is user-writable too: a user thread could release it while the - * kernel holds it, letting two cores update the reference count at once. +/* The DMA controller lookup and its system calls, which run in kernel mode, are in + * src/kernel/lib/dma-kernel.c */ -static struct k_spinlock *dma_lock(struct sof_dma *dma) -{ - return &sof_dma_kinfo_get(dma)->lock; -} -#else -static int *dma_sref(struct sof_dma *dma) -{ - return &dma->sref; -} - -static struct k_spinlock *dma_lock(struct sof_dma *dma) -{ - return &dma->lock; -} -#endif - -__cold struct sof_dma *z_impl_sof_dma_get(uint32_t dir, uint32_t cap, uint32_t dev, uint32_t flags) -{ - const struct dma_info *info = dma_info_get(); - int users, ret = 0; - int min_users = INT32_MAX; - struct sof_dma *d = NULL, *dmin = NULL; - k_spinlock_key_t key; - - assert_can_be_cold(); - - if (!info->num_dmas) { - tr_err(&dma_tr, "dma_get(): No DMACs installed"); - return NULL; - } - - /* find DMAC with free channels that matches request */ - for (d = info->dma_array; d < info->dma_array + info->num_dmas; - d++) { - /* skip if this DMAC does not support the requested dir */ - if (dir && (d->plat_data.dir & dir) == 0) - continue; - - /* skip if this DMAC does not support the requested caps */ - if (cap && (d->plat_data.caps & cap) == 0) - continue; - - /* skip if this DMAC does not support the requested dev */ - if (dev && (d->plat_data.devs & dev) == 0) - continue; - - /* if exclusive access is requested */ - if (flags & SOF_DMA_ACCESS_EXCLUSIVE) { - /* ret DMA with no users */ - if (!d->sref) { - dmin = d; - break; - } - } else { - /* get number of users for this DMAC*/ - users = d->sref; - - /* pick DMAC with the least num of users */ - if (users < min_users) { - dmin = d; - min_users = users; - } - } - } - - if (!dmin) { - tr_err(&dma_tr, "No DMAC dir %d caps 0x%x dev 0x%x flags 0x%x", - dir, cap, dev, flags); - - for (d = info->dma_array; - d < info->dma_array + info->num_dmas; - d++) { - tr_err(&dma_tr, " DMAC ID %d users %d busy channels %ld", - d->plat_data.id, d->sref, - atomic_read(&d->num_channels_busy)); - tr_err(&dma_tr, " caps 0x%x dev 0x%x", - d->plat_data.caps, d->plat_data.devs); - } - - return NULL; - } - - /* return DMAC */ - tr_dbg(&dma_tr, "dma_get(), dma-probe id = %d", - dmin->plat_data.id); - - /* Shared DMA controllers with multiple channels - * may be requested many times, let the probe() - * do on-first-use initialization. - */ - key = k_spin_lock(dma_lock(dmin)); - - if (!*dma_sref(dmin)) { - ret = dma_init(dmin); - if (ret < 0) { - tr_err(&dma_tr, "dma_get(): dma-probe failed id = %d, ret = %d", - dmin->plat_data.id, ret); - goto out; - } - } - - ++*dma_sref(dmin); - /* user-visible copy for diagnostics and DMAC selection, never trusted */ - dmin->sref = *dma_sref(dmin); - - tr_info(&dma_tr, "dma_get() ID %d sref = %d busy channels %ld", - dmin->plat_data.id, dmin->sref, - atomic_read(&dmin->num_channels_busy)); -out: - k_spin_unlock(dma_lock(dmin), key); - return !ret ? dmin : NULL; -} - -__cold void z_impl_sof_dma_put(struct sof_dma *dma) -{ - k_spinlock_key_t key; - - assert_can_be_cold(); - - key = k_spin_lock(dma_lock(dma)); - if (*dma_sref(dma) <= 0) { - tr_err(&dma_tr, "dma_put(), dma = %p not in use", dma); - k_spin_unlock(dma_lock(dma), key); - return; - } - - --*dma_sref(dma); - dma->sref = *dma_sref(dma); - if (!*dma_sref(dma)) { -#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA - struct sof_dma_kinfo *kinfo = sof_dma_kinfo_get(dma); - - sof_heap_free(kinfo->heap, kinfo->chan); - kinfo->chan = NULL; -#else - sof_heap_free(dma->heap, dma->chan); -#endif - dma->chan = NULL; - } - - tr_info(&dma_tr, "dma_put(), dma = %p, sref = %d", - dma, dma->sref); - k_spin_unlock(dma_lock(dma), key); -} - -__cold static int dma_init(struct sof_dma *dma) -{ - struct dma_chan_data *chans, *chan; - struct k_heap *heap = sof_sys_user_heap_get(); -#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA - /* dma->plat_data is user-writable, use the kernel snapshot */ - const uint32_t channels = sof_dma_kinfo_get(dma)->channels; -#else - const uint32_t channels = dma->plat_data.channels; -#endif - uint32_t i; - - assert_can_be_cold(); - -#if CONFIG_SOF_USERSPACE_INTERFACE_DMA && CONFIG_SOF_USERSPACE_LL - /* - * Called in supervisor mode from the sof_dma_get() system call. The - * heap pointer comes from user-writable memory, so check that it is - * the LL user heap before running the allocator on it. - */ - if (!zephyr_ll_user_heap_verify(heap)) { - tr_err(&dma_tr, "dma %d: invalid channel heap %p", dma->plat_data.id, heap); - return -EINVAL; - } -#endif - - /* - * allocate dma channels, use a local pointer until the array is - * initialized: dma->chan is user-writable and must not be read back - */ - chans = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(struct dma_chan_data) * channels, 0); - if (!chans) { - tr_err(&dma_tr, "dma %d allocation of channels failed", - dma->plat_data.id); - return -ENOMEM; - } - - memset(chans, 0, sizeof(struct dma_chan_data) * channels); - /* init work */ - for (i = 0, chan = chans; i < channels; i++, chan++) { - chan->dma = dma; - chan->index = i; - } - -#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA - sof_dma_kinfo_get(dma)->heap = heap; - sof_dma_kinfo_get(dma)->chan = chans; -#endif - dma->heap = heap; - dma->chan = chans; - - return 0; -} -#else -struct dma *dma_get(uint32_t dir, uint32_t cap, uint32_t dev, uint32_t flags) -{ - const struct dma_info *info = dma_info_get(); - int users, ret; - int min_users = INT32_MAX; - struct dma *d = NULL, *dmin = NULL; - k_spinlock_key_t key; - - if (!info->num_dmas) { - tr_err(&dma_tr, "dma_get(): No DMACs installed"); - return NULL; - } - - /* find DMAC with free channels that matches request */ - for (d = info->dma_array; d < info->dma_array + info->num_dmas; - d++) { - /* skip if this DMAC does not support the requested dir */ - if (dir && (d->plat_data.dir & dir) == 0) - continue; - - /* skip if this DMAC does not support the requested caps */ - if (cap && (d->plat_data.caps & cap) == 0) - continue; - - /* skip if this DMAC does not support the requested dev */ - if (dev && (d->plat_data.devs & dev) == 0) - continue; - - /* skip if this DMAC has 1 user per avail channel */ - /* TODO: this should be fixed in dai.c to allow more users */ - if (d->sref >= d->plat_data.channels) - continue; - - /* if exclusive access is requested */ - if (flags & DMA_ACCESS_EXCLUSIVE) { - /* ret DMA with no users */ - if (!d->sref) { - dmin = d; - break; - } - } else { - /* get number of users for this DMAC*/ - users = d->sref; - - /* pick DMAC with the least num of users */ - if (users < min_users) { - dmin = d; - min_users = users; - } - } - } - - if (!dmin) { - tr_err(&dma_tr, "No DMAC dir %d caps 0x%x dev 0x%x flags 0x%x", - dir, cap, dev, flags); - - for (d = info->dma_array; - d < info->dma_array + info->num_dmas; - d++) { - tr_err(&dma_tr, " DMAC ID %d users %d busy channels %ld", - d->plat_data.id, d->sref, - atomic_read(&d->num_channels_busy)); - tr_err(&dma_tr, " caps 0x%x dev 0x%x", - d->plat_data.caps, d->plat_data.devs); - } - - return NULL; - } - - /* return DMAC */ - tr_dbg(&dma_tr, "dma_get(), dma-probe id = %d", - dmin->plat_data.id); - - /* Shared DMA controllers with multiple channels - * may be requested many times, let the probe() - * do on-first-use initialization. - */ - key = k_spin_lock(&dmin->lock); - - ret = 0; - if (!dmin->sref) { - ret = dma_probe_legacy(dmin); - if (ret < 0) { - tr_err(&dma_tr, "dma_get(): dma-probe failed id = %d, ret = %d", - dmin->plat_data.id, ret); - } - } - if (!ret) - dmin->sref++; - - tr_info(&dma_tr, "dma_get() ID %d sref = %d busy channels %ld", - dmin->plat_data.id, dmin->sref, - atomic_read(&dmin->num_channels_busy)); - - k_spin_unlock(&dmin->lock, key); - return !ret ? dmin : NULL; -} - -void dma_put(struct dma *dma) -{ - k_spinlock_key_t key; - int ret; - - key = k_spin_lock(&dma->lock); - if (--dma->sref == 0) { - ret = dma_remove_legacy(dma); - if (ret < 0) { - tr_err(&dma_tr, "dma_remove() failed id = %d, ret = %d", - dma->plat_data.id, ret); - } - } - tr_info(&dma_tr, "dma = %p, sref = %d", - dma, dma->sref); - k_spin_unlock(&dma->lock, key); -} -#endif int dma_sg_alloc(struct k_heap *heap, struct dma_sg_elem_array *elem_array, From a5bfec2abe2007abcc788f8a02a99e186aee4973 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 14 Nov 2026 18:16:29 +0000 Subject: [PATCH 2/4] lib: split the kernel part of dai.c into src/kernel/lib dai.c mixes the DAI device lookup and the DAI getters with the DAI groups, which the DAI component uses, also in the user-space LL thread. Move the kernel part to src/kernel/lib/dai-kernel.c: - the Zephyr DAI device table and dai_get_device_list(), which the user-space access grants use; - the kernel half of the dai_get_device() system call; - dai_get() and dai_put() of the native and legacy DAI drivers, which probe and remove the DAI drivers and are called by the IPC3 handler and the platform code. dai.c keeps the DAI groups and the DAI log module and trace context, which both parts use. It no longer includes the Zephyr DAI driver and LL scheduler domain headers. The kernel part is built wherever dai.c is, including the host library. The generated code is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/kernel/lib/CMakeLists.txt | 2 +- src/kernel/lib/dai-kernel.c | 347 ++++++++++++++++++++++++++++++++++ src/lib/dai.c | 323 +------------------------------ 3 files changed, 352 insertions(+), 320 deletions(-) create mode 100644 src/kernel/lib/dai-kernel.c diff --git a/src/kernel/lib/CMakeLists.txt b/src/kernel/lib/CMakeLists.txt index 8d1e08b8c8e3..0747994fbe3d 100644 --- a/src/kernel/lib/CMakeLists.txt +++ b/src/kernel/lib/CMakeLists.txt @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # The kernel parts of src/lib/dma.c and dai.c, built with them -set(common_files dma-kernel.c) +set(common_files dma-kernel.c dai-kernel.c) if(CONFIG_LIBRARY) add_local_sources(sof agent.c ${common_files}) diff --git a/src/kernel/lib/dai-kernel.c b/src/kernel/lib/dai-kernel.c new file mode 100644 index 000000000000..c12bfeafa24e --- /dev/null +++ b/src/kernel/lib/dai-kernel.c @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2018 Intel Corporation. All rights reserved. +// +// Author: Marcin Maka + +#include +#include +#include +#include +#include +#include +#include +#include /* for zephyr_ll_user_heap() */ +#include +#include +#include +#include +#include +#include + +#if CONFIG_ZEPHYR_NATIVE_DRIVERS +#include +#include +#endif +LOG_MODULE_DECLARE(dai, CONFIG_SOF_LOG_LEVEL); + +extern struct tr_ctx dai_tr; + +/* Kernel part of the DAI library: the Zephyr DAI device table, the kernel half of the + * dai_get_device() system call and the DAI getters, which probe and remove the DAI + * drivers. The DAI groups are in src/lib/dai.c + */ + +#if CONFIG_ZEPHYR_NATIVE_DRIVERS + +#define GET_DEVICE_LIST(node) DEVICE_DT_GET(node), + +const struct device *zephyr_dev[] = { +#if CONFIG_DAI_INTEL_SSP + DT_FOREACH_STATUS_OKAY(intel_ssp_dai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_INTEL_DMIC + DT_FOREACH_STATUS_OKAY(intel_dai_dmic, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_INTEL_ALH + DT_FOREACH_STATUS_OKAY(intel_alh_dai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_INTEL_HDA + DT_FOREACH_STATUS_OKAY(intel_hda_dai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_NXP_SAI + DT_FOREACH_STATUS_OKAY(nxp_dai_sai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_NXP_ESAI + DT_FOREACH_STATUS_OKAY(nxp_dai_esai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_NXP_MICFIL + DT_FOREACH_STATUS_OKAY(nxp_dai_micfil, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_AMD_DMIC + DT_FOREACH_STATUS_OKAY(amd_dmic_dai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_AMD_SDW + DT_FOREACH_STATUS_OKAY(amd_acp_sdw_dai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_AMD_TDM + DT_FOREACH_STATUS_OKAY(amd_tdm_dai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_INTEL_UAOL + DT_FOREACH_STATUS_OKAY(intel_uaol_dai, GET_DEVICE_LIST) +#endif +#if DT_HAS_COMPAT_STATUS_OKAY(mediatek_afe) + DT_FOREACH_STATUS_OKAY(mediatek_afe, GET_DEVICE_LIST) +#endif +}; + +const struct device **dai_get_device_list(size_t *count) +{ + __ASSERT_NO_MSG(count); + *count = ARRAY_SIZE(zephyr_dev); + + return zephyr_dev; +} + +/* convert sof_ipc_dai_type to Zephyr dai_type */ +static int sof_dai_type_to_zephyr(uint32_t type) +{ + switch (type) { + case SOF_DAI_INTEL_SSP: + return DAI_INTEL_SSP; + case SOF_DAI_INTEL_DMIC: + return DAI_INTEL_DMIC; + case SOF_DAI_INTEL_HDA: + return DAI_INTEL_HDA; + case SOF_DAI_INTEL_ALH: + return DAI_INTEL_ALH; + case SOF_DAI_INTEL_UAOL: + return DAI_INTEL_UAOL; + case SOF_DAI_IMX_SAI: + return DAI_IMX_SAI; + case SOF_DAI_IMX_ESAI: + return DAI_IMX_ESAI; + case SOF_DAI_AMD_DMIC: + return DAI_AMD_DMIC; + case SOF_DAI_MEDIATEK_AFE: + return DAI_MEDIATEK_AFE; + case SOF_DAI_IMX_MICFIL: + return DAI_IMX_MICFIL; + case SOF_DAI_AMD_SDW: + return DAI_AMD_SDW; + case SOF_DAI_AMD_HS: + case SOF_DAI_AMD_HS_VIRTUAL: + case SOF_DAI_AMD_SP: + case SOF_DAI_AMD_SP_VIRTUAL: + case SOF_DAI_AMD_BT: + case SOF_DAI_AMD_TDM: + return DAI_AMD_TDM; + default: + return -EINVAL; + } +} + +__cold const struct device *z_impl_dai_get_device(enum sof_ipc_dai_type type, uint32_t index) +{ + struct dai_config cfg; + int z_type; + int dir; + int i; + + assert_can_be_cold(); + + dir = (type == SOF_DAI_INTEL_DMIC) ? DAI_DIR_RX : DAI_DIR_BOTH; + + z_type = sof_dai_type_to_zephyr(type); + if (z_type < 0) { + tr_err(&dai_tr, "dai_get_device: no matching zephyr DAI type for %d ret = %d", + type, z_type); + return NULL; + } + + for (i = 0; i < ARRAY_SIZE(zephyr_dev); i++) { + if (dai_config_get(zephyr_dev[i], &cfg, dir)) + continue; + if (cfg.type == z_type && cfg.dai_index == index) + return zephyr_dev[i]; + } + + return NULL; +} + +static void dai_set_device_params(struct dai *d) +{ + switch (d->type) { + case SOF_DAI_INTEL_SSP: + d->dma_dev = SOF_DMA_DEV_SSP; +#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA + d->dma_caps = SOF_DMA_CAP_GP_LP | SOF_DMA_CAP_GP_HP; +#else + d->dma_caps = SOF_DMA_CAP_HDA; +#endif + break; + case SOF_DAI_INTEL_DMIC: + d->dma_dev = SOF_DMA_DEV_DMIC; +#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA + d->dma_caps = SOF_DMA_CAP_GP_LP | SOF_DMA_CAP_GP_HP; +#else + d->dma_caps = SOF_DMA_CAP_HDA; +#endif + break; + case SOF_DAI_INTEL_ALH: + d->dma_dev = SOF_DMA_DEV_ALH; +#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA + d->dma_caps = SOF_DMA_CAP_GP_LP | SOF_DMA_CAP_GP_HP; +#else + d->dma_caps = SOF_DMA_CAP_HDA; +#endif + break; + case SOF_DAI_INTEL_HDA: + case SOF_DAI_INTEL_UAOL: + d->dma_dev = SOF_DMA_DEV_HDA; + d->dma_caps = SOF_DMA_CAP_HDA; + break; + case SOF_DAI_AMD_SDW: + d->dma_dev = SOF_DMA_DEV_SW; + d->dma_caps = SOF_DMA_CAP_SW; + break; + /* All TDM-capable AMD DAIs share acp_tdm_dma. */ + case SOF_DAI_AMD_HS: + case SOF_DAI_AMD_HS_VIRTUAL: + case SOF_DAI_AMD_SP: + case SOF_DAI_AMD_SP_VIRTUAL: + case SOF_DAI_AMD_BT: + d->dma_dev = SOF_DMA_DEV_HS | SOF_DMA_DEV_SP | SOF_DMA_DEV_BT; + d->dma_caps = SOF_DMA_CAP_HS | SOF_DMA_CAP_SP | SOF_DMA_CAP_BT; + break; + case SOF_DAI_AMD_DMIC: + d->dma_dev = SOF_DMA_DEV_DMIC; + d->dma_caps = SOF_DMA_CAP_DMIC; + break; + case SOF_DAI_AMD_TDM: + d->dma_dev = SOF_DMA_DEV_TDM; + d->dma_caps = SOF_DMA_CAP_TDM; + break; + case SOF_DAI_MEDIATEK_AFE: + d->dma_dev = SOF_DMA_DEV_AFE_MEMIF; + break; + default: + break; + } +} + +/* called from kernel/ipc/ipc3/handler.c and some platform.c files */ +__cold struct dai *dai_get(uint32_t type, uint32_t index, uint32_t flags) +{ + const struct device *dev; + struct dai *d; + struct k_heap *heap = NULL; + + assert_can_be_cold(); + +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); +#endif + + dev = dai_get_device(type, index); + if (!dev) { + tr_err(&dai_tr, "dai_get: failed to get dai with index %d type %d", + index, type); + return NULL; + } + + d = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(struct dai), 0); + if (!d) + return NULL; + + memset(d, 0, sizeof(struct dai)); + + d->index = index; + d->type = type; + d->dev = dev; + + dai_set_device_params(d); + + if (dai_probe(d->dev)) { + tr_err(&dai_tr, "dai_get: failed to probe dai with index %d type %d", + index, type); + sof_heap_free(heap, d); + return NULL; + } + + return d; +} + +/* called from src/kernel/ipc/ipc3/handler.c */ +__cold void dai_put(struct dai *dai) +{ + int ret; + struct k_heap *heap = NULL; + + assert_can_be_cold(); + +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); +#endif + + ret = dai_remove(dai->dev); + if (ret < 0) { + tr_err(&dai_tr, "index %d failed ret = %d", + dai->index, ret); + } + + sof_heap_free(heap, dai); +} +#else +static inline const struct dai_type_info *dai_find_type(uint32_t type) +{ + const struct dai_info *info = dai_info_get(); + const struct dai_type_info *dti; + + for (dti = info->dai_type_array; + dti < info->dai_type_array + info->num_dai_types; dti++) { + if (dti->type == type) + return dti; + } + return NULL; +} + +__cold struct dai *dai_get(uint32_t type, uint32_t index, uint32_t flags) +{ + int ret = 0; + const struct dai_type_info *dti; + struct dai *d; + k_spinlock_key_t key; + + assert_can_be_cold(); + + dti = dai_find_type(type); + if (!dti) + return NULL; /* type not found */ + + for (d = dti->dai_array; d < dti->dai_array + dti->num_dais; d++) { + if (d->index != index) { + continue; + } + /* device created? */ + key = k_spin_lock(&d->lock); + if (d->sref == 0) { + if (flags & DAI_CREAT) + ret = dai_probe(d); + else + ret = -ENODEV; + } + if (!ret) + d->sref++; + + tr_info(&dai_tr, "dai_get type %d index %d new sref %d", + type, index, d->sref); + + k_spin_unlock(&d->lock, key); + + return !ret ? d : NULL; + } + tr_err(&dai_tr, "dai_get: type %d index %d not found", type, index); + return NULL; +} + +__cold void dai_put(struct dai *dai) +{ + int ret; + k_spinlock_key_t key; + + assert_can_be_cold(); + + key = k_spin_lock(&dai->lock); + if (--dai->sref == 0) { + ret = dai_remove(dai); + if (ret < 0) { + tr_err(&dai_tr, "type %d index %d dai_remove() failed ret = %d", + dai->drv->type, dai->index, ret); + } + } + tr_info(&dai_tr, "type %d index %d new sref %d", + dai->drv->type, dai->index, dai->sref); + k_spin_unlock(&dai->lock, key); +} +#endif diff --git a/src/lib/dai.c b/src/lib/dai.c index 214dcf979711..4bab1fbdc24e 100644 --- a/src/lib/dai.c +++ b/src/lib/dai.c @@ -11,7 +11,6 @@ #include #include #include -#include /* for zephyr_ll_user_heap() */ #include #include #include @@ -19,10 +18,6 @@ #include #include -#if CONFIG_ZEPHYR_NATIVE_DRIVERS -#include -#include -#endif LOG_MODULE_REGISTER(dai, CONFIG_SOF_LOG_LEVEL); SOF_DEFINE_REG_UUID(dai_lib); @@ -30,6 +25,10 @@ SOF_DEFINE_REG_UUID(dai_lib); /* unused with Zephyr, generates no output */ DECLARE_TR_CTX(dai_tr, SOF_UUID(dai_lib_uuid), LOG_LEVEL_INFO); +/* The DAI device lookup and the DAI getters, which run in kernel mode, are in + * src/kernel/lib/dai-kernel.c + */ + struct dai_group_list { struct list_item list; } __aligned(PLATFORM_DCACHE_ALIGN); @@ -160,317 +159,3 @@ __cold void dai_group_put(struct dai_group *group) if (!group->num_dais) group->group_id = 0; } - -#if CONFIG_ZEPHYR_NATIVE_DRIVERS - -#define GET_DEVICE_LIST(node) DEVICE_DT_GET(node), - -const struct device *zephyr_dev[] = { -#if CONFIG_DAI_INTEL_SSP - DT_FOREACH_STATUS_OKAY(intel_ssp_dai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_INTEL_DMIC - DT_FOREACH_STATUS_OKAY(intel_dai_dmic, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_INTEL_ALH - DT_FOREACH_STATUS_OKAY(intel_alh_dai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_INTEL_HDA - DT_FOREACH_STATUS_OKAY(intel_hda_dai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_NXP_SAI - DT_FOREACH_STATUS_OKAY(nxp_dai_sai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_NXP_ESAI - DT_FOREACH_STATUS_OKAY(nxp_dai_esai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_NXP_MICFIL - DT_FOREACH_STATUS_OKAY(nxp_dai_micfil, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_AMD_DMIC - DT_FOREACH_STATUS_OKAY(amd_dmic_dai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_AMD_SDW - DT_FOREACH_STATUS_OKAY(amd_acp_sdw_dai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_AMD_TDM - DT_FOREACH_STATUS_OKAY(amd_tdm_dai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_INTEL_UAOL - DT_FOREACH_STATUS_OKAY(intel_uaol_dai, GET_DEVICE_LIST) -#endif -#if DT_HAS_COMPAT_STATUS_OKAY(mediatek_afe) - DT_FOREACH_STATUS_OKAY(mediatek_afe, GET_DEVICE_LIST) -#endif -}; - -const struct device **dai_get_device_list(size_t *count) -{ - __ASSERT_NO_MSG(count); - *count = ARRAY_SIZE(zephyr_dev); - - return zephyr_dev; -} - -/* convert sof_ipc_dai_type to Zephyr dai_type */ -static int sof_dai_type_to_zephyr(uint32_t type) -{ - switch (type) { - case SOF_DAI_INTEL_SSP: - return DAI_INTEL_SSP; - case SOF_DAI_INTEL_DMIC: - return DAI_INTEL_DMIC; - case SOF_DAI_INTEL_HDA: - return DAI_INTEL_HDA; - case SOF_DAI_INTEL_ALH: - return DAI_INTEL_ALH; - case SOF_DAI_INTEL_UAOL: - return DAI_INTEL_UAOL; - case SOF_DAI_IMX_SAI: - return DAI_IMX_SAI; - case SOF_DAI_IMX_ESAI: - return DAI_IMX_ESAI; - case SOF_DAI_AMD_DMIC: - return DAI_AMD_DMIC; - case SOF_DAI_MEDIATEK_AFE: - return DAI_MEDIATEK_AFE; - case SOF_DAI_IMX_MICFIL: - return DAI_IMX_MICFIL; - case SOF_DAI_AMD_SDW: - return DAI_AMD_SDW; - case SOF_DAI_AMD_HS: - case SOF_DAI_AMD_HS_VIRTUAL: - case SOF_DAI_AMD_SP: - case SOF_DAI_AMD_SP_VIRTUAL: - case SOF_DAI_AMD_BT: - case SOF_DAI_AMD_TDM: - return DAI_AMD_TDM; - default: - return -EINVAL; - } -} - -__cold const struct device *z_impl_dai_get_device(enum sof_ipc_dai_type type, uint32_t index) -{ - struct dai_config cfg; - int z_type; - int dir; - int i; - - assert_can_be_cold(); - - dir = (type == SOF_DAI_INTEL_DMIC) ? DAI_DIR_RX : DAI_DIR_BOTH; - - z_type = sof_dai_type_to_zephyr(type); - if (z_type < 0) { - tr_err(&dai_tr, "dai_get_device: no matching zephyr DAI type for %d ret = %d", - type, z_type); - return NULL; - } - - for (i = 0; i < ARRAY_SIZE(zephyr_dev); i++) { - if (dai_config_get(zephyr_dev[i], &cfg, dir)) - continue; - if (cfg.type == z_type && cfg.dai_index == index) - return zephyr_dev[i]; - } - - return NULL; -} - -static void dai_set_device_params(struct dai *d) -{ - switch (d->type) { - case SOF_DAI_INTEL_SSP: - d->dma_dev = SOF_DMA_DEV_SSP; -#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA - d->dma_caps = SOF_DMA_CAP_GP_LP | SOF_DMA_CAP_GP_HP; -#else - d->dma_caps = SOF_DMA_CAP_HDA; -#endif - break; - case SOF_DAI_INTEL_DMIC: - d->dma_dev = SOF_DMA_DEV_DMIC; -#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA - d->dma_caps = SOF_DMA_CAP_GP_LP | SOF_DMA_CAP_GP_HP; -#else - d->dma_caps = SOF_DMA_CAP_HDA; -#endif - break; - case SOF_DAI_INTEL_ALH: - d->dma_dev = SOF_DMA_DEV_ALH; -#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA - d->dma_caps = SOF_DMA_CAP_GP_LP | SOF_DMA_CAP_GP_HP; -#else - d->dma_caps = SOF_DMA_CAP_HDA; -#endif - break; - case SOF_DAI_INTEL_HDA: - case SOF_DAI_INTEL_UAOL: - d->dma_dev = SOF_DMA_DEV_HDA; - d->dma_caps = SOF_DMA_CAP_HDA; - break; - case SOF_DAI_AMD_SDW: - d->dma_dev = SOF_DMA_DEV_SW; - d->dma_caps = SOF_DMA_CAP_SW; - break; - /* All TDM-capable AMD DAIs share acp_tdm_dma. */ - case SOF_DAI_AMD_HS: - case SOF_DAI_AMD_HS_VIRTUAL: - case SOF_DAI_AMD_SP: - case SOF_DAI_AMD_SP_VIRTUAL: - case SOF_DAI_AMD_BT: - d->dma_dev = SOF_DMA_DEV_HS | SOF_DMA_DEV_SP | SOF_DMA_DEV_BT; - d->dma_caps = SOF_DMA_CAP_HS | SOF_DMA_CAP_SP | SOF_DMA_CAP_BT; - break; - case SOF_DAI_AMD_DMIC: - d->dma_dev = SOF_DMA_DEV_DMIC; - d->dma_caps = SOF_DMA_CAP_DMIC; - break; - case SOF_DAI_AMD_TDM: - d->dma_dev = SOF_DMA_DEV_TDM; - d->dma_caps = SOF_DMA_CAP_TDM; - break; - case SOF_DAI_MEDIATEK_AFE: - d->dma_dev = SOF_DMA_DEV_AFE_MEMIF; - break; - default: - break; - } -} - -/* called from kernel/ipc/ipc3/handler.c and some platform.c files */ -__cold struct dai *dai_get(uint32_t type, uint32_t index, uint32_t flags) -{ - const struct device *dev; - struct dai *d; - struct k_heap *heap = NULL; - - assert_can_be_cold(); - -#ifdef CONFIG_SOF_USERSPACE_LL - heap = zephyr_ll_user_heap(); -#endif - - dev = dai_get_device(type, index); - if (!dev) { - tr_err(&dai_tr, "dai_get: failed to get dai with index %d type %d", - index, type); - return NULL; - } - - d = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(struct dai), 0); - if (!d) - return NULL; - - memset(d, 0, sizeof(struct dai)); - - d->index = index; - d->type = type; - d->dev = dev; - - dai_set_device_params(d); - - if (dai_probe(d->dev)) { - tr_err(&dai_tr, "dai_get: failed to probe dai with index %d type %d", - index, type); - sof_heap_free(heap, d); - return NULL; - } - - return d; -} - -/* called from src/kernel/ipc/ipc3/handler.c */ -__cold void dai_put(struct dai *dai) -{ - int ret; - struct k_heap *heap = NULL; - - assert_can_be_cold(); - -#ifdef CONFIG_SOF_USERSPACE_LL - heap = zephyr_ll_user_heap(); -#endif - - ret = dai_remove(dai->dev); - if (ret < 0) { - tr_err(&dai_tr, "index %d failed ret = %d", - dai->index, ret); - } - - sof_heap_free(heap, dai); -} -#else -static inline const struct dai_type_info *dai_find_type(uint32_t type) -{ - const struct dai_info *info = dai_info_get(); - const struct dai_type_info *dti; - - for (dti = info->dai_type_array; - dti < info->dai_type_array + info->num_dai_types; dti++) { - if (dti->type == type) - return dti; - } - return NULL; -} - -__cold struct dai *dai_get(uint32_t type, uint32_t index, uint32_t flags) -{ - int ret = 0; - const struct dai_type_info *dti; - struct dai *d; - k_spinlock_key_t key; - - assert_can_be_cold(); - - dti = dai_find_type(type); - if (!dti) - return NULL; /* type not found */ - - for (d = dti->dai_array; d < dti->dai_array + dti->num_dais; d++) { - if (d->index != index) { - continue; - } - /* device created? */ - key = k_spin_lock(&d->lock); - if (d->sref == 0) { - if (flags & DAI_CREAT) - ret = dai_probe(d); - else - ret = -ENODEV; - } - if (!ret) - d->sref++; - - tr_info(&dai_tr, "dai_get type %d index %d new sref %d", - type, index, d->sref); - - k_spin_unlock(&d->lock, key); - - return !ret ? d : NULL; - } - tr_err(&dai_tr, "dai_get: type %d index %d not found", type, index); - return NULL; -} - -__cold void dai_put(struct dai *dai) -{ - int ret; - k_spinlock_key_t key; - - assert_can_be_cold(); - - key = k_spin_lock(&dai->lock); - if (--dai->sref == 0) { - ret = dai_remove(dai); - if (ret < 0) { - tr_err(&dai_tr, "type %d index %d dai_remove() failed ret = %d", - dai->drv->type, dai->index, ret); - } - } - tr_info(&dai_tr, "type %d index %d new sref %d", - dai->drv->type, dai->index, dai->sref); - k_spin_unlock(&dai->lock, key); -} -#endif From 928dd0b657c4b849ee571323eee4e4037b1d4d54 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 14 Nov 2026 18:17:36 +0000 Subject: [PATCH 3/4] audio: split the kernel part of intel_uaol.c into src/kernel/audio The UAOL feedback support made intel_uaol.c mixed: the DAI component calls the UAOL feedback, rate adjustment and feedback DMA functions from the user-space LL thread, while the base firmware queries the UAOL capabilities and stream mapping in kernel mode. Move the kernel part, tlv_value_set_uaol_caps() and uaol_stream_id_to_hda_link_stream_id(), with the IPC4 UAOL capabilities structures, to src/kernel/audio/intel_uaol-kernel.c. intel_uaol.c keeps the code that runs in the LL thread, including the UAOL device table and its lookup, get_uaol_zdevice(). The table is in rodata, readable from user space. It is no longer static: it is declared in sof/audio/intel_uaol.h, sized from the devicetree, so the kernel queries keep using ARRAY_SIZE() on it. intel_uaol.c is user code now, so drop it from the kernel sources listed under src/audio in the kernel include checker's manifest. The kernel part is built with the same condition as intel_uaol.c. The code is unchanged, but the two kernel queries now load the device from the table: with a single UAOL device, the compiler had reduced the static table to a constant. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- scripts/kernel_user_partition.txt | 1 - src/audio/intel_uaol.c | 66 ++--------------------- src/include/sof/audio/intel_uaol.h | 6 +++ src/kernel/audio/CMakeLists.txt | 2 + src/kernel/audio/intel_uaol-kernel.c | 79 ++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 63 deletions(-) create mode 100644 src/kernel/audio/intel_uaol-kernel.c diff --git a/scripts/kernel_user_partition.txt b/scripts/kernel_user_partition.txt index 39d5e1191366..95b4bc0ae99f 100644 --- a/scripts/kernel_user_partition.txt +++ b/scripts/kernel_user_partition.txt @@ -30,7 +30,6 @@ src/lib/objpool.c [not-user-sources] src/audio/dai-legacy.c src/audio/host-legacy.c -src/audio/intel_uaol.c # SOF headers that only kernel code may include. # Not listed on purpose: rtos/init.h, which only provides the diff --git a/src/audio/intel_uaol.c b/src/audio/intel_uaol.c index ba676a9aea06..ebe77b487698 100644 --- a/src/audio/intel_uaol.c +++ b/src/audio/intel_uaol.c @@ -14,74 +14,16 @@ LOG_MODULE_REGISTER(uaol, CONFIG_SOF_LOG_LEVEL); -struct ipc4_uaol_link_capabilities { - uint32_t input_streams_supported : 4; - uint32_t output_streams_supported : 4; - uint32_t bidirectional_streams_supported : 5; - uint32_t rsvd : 19; - uint32_t max_tx_fifo_size; - uint32_t max_rx_fifo_size; -} __packed __aligned(4); - -struct ipc4_uaol_capabilities { - uint32_t link_count; - struct ipc4_uaol_link_capabilities link_caps[]; -} __packed __aligned(4); +/* The UAOL capabilities and stream mapping queries of the base firmware, which run in + * kernel mode, are in src/kernel/audio/intel_uaol-kernel.c + */ #define DEV_AND_COMMA(node) DEVICE_DT_GET(node), /* const pointers keep the array in rodata, readable by the user-space LL thread */ -static const struct device *const uaol_devs[] = { +const struct device *const uaol_devs[] = { DT_FOREACH_STATUS_OKAY(intel_adsp_uaol, DEV_AND_COMMA) }; -#if !CONFIG_SOF_OS_LINUX_COMPAT_PRIORITY -__cold void tlv_value_set_uaol_caps(struct sof_tlv *tuple, uint32_t type) -{ - const size_t dev_count = ARRAY_SIZE(uaol_devs); - struct uaol_capabilities dev_cap; - struct ipc4_uaol_capabilities *caps = (struct ipc4_uaol_capabilities *)tuple->value; - size_t caps_size = offsetof(struct ipc4_uaol_capabilities, link_caps[dev_count]); - size_t i; - int ret; - - assert_can_be_cold(); - - memset(caps, 0, caps_size); - - caps->link_count = dev_count; - for (i = 0; i < dev_count; i++) { - ret = uaol_get_capabilities(uaol_devs[i], &dev_cap); - if (ret) - continue; - - caps->link_caps[i].input_streams_supported = dev_cap.input_streams; - caps->link_caps[i].output_streams_supported = dev_cap.output_streams; - caps->link_caps[i].bidirectional_streams_supported = dev_cap.bidirectional_streams; - caps->link_caps[i].max_tx_fifo_size = dev_cap.max_tx_fifo_size; - caps->link_caps[i].max_rx_fifo_size = dev_cap.max_rx_fifo_size; - } - - tlv_value_set(tuple, type, caps_size, caps); -} -#endif /* !CONFIG_SOF_OS_LINUX_COMPAT_PRIORITY */ - -__cold int uaol_stream_id_to_hda_link_stream_id(int uaol_stream_id) -{ - size_t dev_count = ARRAY_SIZE(uaol_devs); - size_t i; - - assert_can_be_cold(); - - for (i = 0; i < dev_count; i++) { - int hda_link_stream_id = uaol_get_mapped_hda_link_stream_id(uaol_devs[i], - uaol_stream_id); - if (hda_link_stream_id >= 0) - return hda_link_stream_id; - } - - return -1; -} - const struct device *get_uaol_zdevice(int uaol_link_id) { /* uaol_link_id is just an index for the device tree device */ diff --git a/src/include/sof/audio/intel_uaol.h b/src/include/sof/audio/intel_uaol.h index d36335baf068..8b7cee072670 100644 --- a/src/include/sof/audio/intel_uaol.h +++ b/src/include/sof/audio/intel_uaol.h @@ -8,6 +8,7 @@ #include #include +#include struct sof_tlv; struct device; @@ -20,6 +21,11 @@ __cold int uaol_stream_id_to_hda_link_stream_id(int uaol_stream_id); const struct device *get_uaol_zdevice(int uaol_link_id); +#if CONFIG_UAOL_INTEL_ADSP +/* UAOL devices, defined in intel_uaol.c. In rodata, readable from user space. */ +extern const struct device *const uaol_devs[DT_NUM_INST_STATUS_OKAY(intel_adsp_uaol)]; +#endif + /* these are called from dai-zephyr.c */ struct dai; diff --git a/src/kernel/audio/CMakeLists.txt b/src/kernel/audio/CMakeLists.txt index e310aa2585a3..ddd239e57420 100644 --- a/src/kernel/audio/CMakeLists.txt +++ b/src/kernel/audio/CMakeLists.txt @@ -4,6 +4,8 @@ if(CONFIG_COMP_BASEFW_IPC4 AND NOT CONFIG_LIBRARY) add_local_sources(sof base_fw.c) endif() add_local_sources_ifdef(CONFIG_IPC4_BASE_FW_INTEL sof base_fw_intel.c) +# The kernel part of src/audio/intel_uaol.c, built with it +add_local_sources_ifdef(CONFIG_UAOL_INTEL_ADSP sof intel_uaol-kernel.c) if(CONFIG_INTEL_ADSP_MIC_PRIVACY) add_subdirectory(mic_privacy_manager) endif() diff --git a/src/kernel/audio/intel_uaol-kernel.c b/src/kernel/audio/intel_uaol-kernel.c new file mode 100644 index 000000000000..788bf49893ec --- /dev/null +++ b/src/kernel/audio/intel_uaol-kernel.c @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright 2026 Intel Corporation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +/* Kernel part of the UAOL support: the UAOL capabilities and stream mapping queries of + * the base firmware. The UAOL device table, its lookup and the UAOL feedback and rate + * adjustment, which run in the LL thread, are in src/audio/intel_uaol.c + */ + +struct ipc4_uaol_link_capabilities { + uint32_t input_streams_supported : 4; + uint32_t output_streams_supported : 4; + uint32_t bidirectional_streams_supported : 5; + uint32_t rsvd : 19; + uint32_t max_tx_fifo_size; + uint32_t max_rx_fifo_size; +} __packed __aligned(4); + +struct ipc4_uaol_capabilities { + uint32_t link_count; + struct ipc4_uaol_link_capabilities link_caps[]; +} __packed __aligned(4); + +#if !CONFIG_SOF_OS_LINUX_COMPAT_PRIORITY +__cold void tlv_value_set_uaol_caps(struct sof_tlv *tuple, uint32_t type) +{ + const size_t dev_count = ARRAY_SIZE(uaol_devs); + struct uaol_capabilities dev_cap; + struct ipc4_uaol_capabilities *caps = (struct ipc4_uaol_capabilities *)tuple->value; + size_t caps_size = offsetof(struct ipc4_uaol_capabilities, link_caps[dev_count]); + size_t i; + int ret; + + assert_can_be_cold(); + + memset(caps, 0, caps_size); + + caps->link_count = dev_count; + for (i = 0; i < dev_count; i++) { + ret = uaol_get_capabilities(uaol_devs[i], &dev_cap); + if (ret) + continue; + + caps->link_caps[i].input_streams_supported = dev_cap.input_streams; + caps->link_caps[i].output_streams_supported = dev_cap.output_streams; + caps->link_caps[i].bidirectional_streams_supported = dev_cap.bidirectional_streams; + caps->link_caps[i].max_tx_fifo_size = dev_cap.max_tx_fifo_size; + caps->link_caps[i].max_rx_fifo_size = dev_cap.max_rx_fifo_size; + } + + tlv_value_set(tuple, type, caps_size, caps); +} +#endif /* !CONFIG_SOF_OS_LINUX_COMPAT_PRIORITY */ + +__cold int uaol_stream_id_to_hda_link_stream_id(int uaol_stream_id) +{ + size_t dev_count = ARRAY_SIZE(uaol_devs); + size_t i; + + assert_can_be_cold(); + + for (i = 0; i < dev_count; i++) { + int hda_link_stream_id = uaol_get_mapped_hda_link_stream_id(uaol_devs[i], + uaol_stream_id); + if (hda_link_stream_id >= 0) + return hda_link_stream_id; + } + + return -1; +} From 1d75ad585596336c4da7b4b17464238e502e87f9 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 14 Nov 2026 18:17:52 +0000 Subject: [PATCH 4/4] docs: lib: describe the kernel parts of dma.c and dai.c Update the src/lib and user-space READMEs for the split of dma.c and dai.c: their kernel parts, with the kernel halves of the sof_dma_get(), sof_dma_put() and dai_get_device() system calls, are now in src/kernel/lib. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/userspace/README.md | 12 ++++++------ src/lib/README.md | 18 ++++++++++-------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md index 1328248eac2b..8ee63ea87515 100644 --- a/src/include/sof/userspace/README.md +++ b/src/include/sof/userspace/README.md @@ -92,13 +92,13 @@ Classification is grep-verifiable — see [Verifying the taxonomy](#verifying-th > collects the validating half of several syscalls in one place — `zephyr/syscall/alloc.c`, > `cpu.c`, `vregion.c`, `dai.c`, `sof_dma.c` each hold the `z_vrfy_*` wrappers for their > subsystem, physically separated from the `z_impl_*` logic (in `zephyr/lib/*.c` and -> `src/lib/*.c`). This is exactly the boundary-tier separation this document promotes; the +> `src/kernel/lib/*.c`). This is exactly the boundary-tier separation this document promotes; the > migration opportunity is to move the *still-scattered* validators (IPC, module management, > library manager, scheduling, debug) toward the same convention. -> Borderline cases (e.g. parts of `lib/` that host syscall *implementations* such as `dma.c` -> and `dai.c`) behave as boundary code even though the directory is mostly shared-library. When -> a single directory genuinely mixes tiers, its README states the exception explicitly. +> Mixed files are split: the syscall *implementations* of `lib/dma.c` and `lib/dai.c`, for +> example, are in `kernel/lib/dma-kernel.c` and `dai-kernel.c`. When a single directory +> genuinely mixes tiers, its README states the exception explicitly. ## The boundary is the syscall set @@ -119,8 +119,8 @@ syscall by subsystem: | IPC | `ipc_msg_send`, `ipc_msg_list_remove`, `ipc_msg_reply`, `ipc_compound_pre_start`, `ipc_compound_post_start`, `ipc_wait_for_compound_msg`, `send_resource_notif` | `sof/ipc/msg.h`, `sof/ipc/ipc_reply.h`, `ipc4/handler.h`, `ipc4/notification.h` | scattered: `ipc/ipc-common.c`, `kernel/ipc/ipc3/helper.c`, `kernel/ipc/ipc4/handler-kernel.c`, `kernel/ipc/ipc4/notification.c` | | Heap alloc | `sof_heap_alloc`, `sof_heap_free` | `rtos/alloc.h` | `zephyr/syscall/alloc.c` / `zephyr/lib/alloc.c` | | Virtual regions | `vregion_create_map`, `vregion_set_interim`, `vregion_get`, `vregion_put`, `vregion_alloc_align`, `vregion_alloc_coherent_align`, `vregion_free` | `sof/lib/vregion.h` | `zephyr/syscall/vregion.c` / `zephyr/lib/vregion.c` | -| DMA | `sof_dma_get`, `sof_dma_put`, `sof_dma_get_attribute`, `sof_dma_request_channel`, `sof_dma_release_channel`, `sof_dma_config`, `sof_dma_start`, `sof_dma_stop`, `sof_dma_get_status`, `sof_dma_reload`, `sof_dma_suspend`, `sof_dma_resume` | `sof/lib/sof_dma.h` | `zephyr/syscall/sof_dma.c` / `lib/dma.c` (get/put); remaining 10 are `static inline z_impl_*` in `sof/lib/sof_dma.h` | -| DAI | `dai_get_device` | `sof/lib/dai-zephyr.h` | `zephyr/syscall/dai.c` / `lib/dai.c` | +| DMA | `sof_dma_get`, `sof_dma_put`, `sof_dma_get_attribute`, `sof_dma_request_channel`, `sof_dma_release_channel`, `sof_dma_config`, `sof_dma_start`, `sof_dma_stop`, `sof_dma_get_status`, `sof_dma_reload`, `sof_dma_suspend`, `sof_dma_resume` | `sof/lib/sof_dma.h` | `zephyr/syscall/sof_dma.c` / `kernel/lib/dma-kernel.c` (get/put); remaining 10 are `static inline z_impl_*` in `sof/lib/sof_dma.h` | +| DAI | `dai_get_device` | `sof/lib/dai-zephyr.h` | `zephyr/syscall/dai.c` / `kernel/lib/dai-kernel.c` | | CPU | `cpu_get_id` | `sof/lib/cpu.h` | `zephyr/syscall/cpu.c` / `zephyr/lib/cpu.c` | | Module management | `mod_balloc_align`, `mod_alloc_ext`, `mod_free`, `mod_free_all`, `mod_data_blob_handler_new`, `mod_data_blob_set`, `mod_fast_get` | `sof/audio/module_adapter/module/generic.h` | scattered: `audio/module_adapter/module/generic.c` | | Library manager | `lib_manager_free_module` | `sof/lib_manager.h` | scattered: `kernel/library_manager/lib_manager.c` | diff --git a/src/lib/README.md b/src/lib/README.md index 6c986bdffafc..5cb01f0945f3 100644 --- a/src/lib/README.md +++ b/src/lib/README.md @@ -1,15 +1,17 @@ # Common Library (`src/lib`) -> **Runs in:** Mixed — this directory contains code in more than one tier. `dma.c` and `dai.c` -> host the kernel-side implementations (`z_impl_sof_dma_get/put`, `z_impl_dai_get_device`) of -> syscalls whose validators live in `zephyr/syscall/`; treat them as boundary/kernel code. -> `notifier.c` takes Zephyr spinlocks and is similarly restricted to kernel context. The clock, -> KCPS and AMS code (`clk.c`, `cpu-clk-manager.c`, `ams.c`) is kernel-only and lives in -> `src/kernel/lib`. Truly context-agnostic helpers (`lib.c`, `objpool.c`) can safely be linked -> into user builds. See [User/kernel split](../include/sof/userspace/README.md). +> **Runs in:** Mixed — this directory contains code in more than one tier. `dma.c` (the DMA +> buffer helpers) and `dai.c` (the DAI groups) also run in the user-space LL thread. Their +> kernel parts, including the kernel-side implementations (`z_impl_sof_dma_get/put`, +> `z_impl_dai_get_device`) of syscalls whose validators live in `zephyr/syscall/`, are in +> `src/kernel/lib/dma-kernel.c` and `dai-kernel.c`. `notifier.c` takes Zephyr spinlocks and is +> restricted to kernel context. The clock, KCPS and AMS code (`clk.c`, `cpu-clk-manager.c`, +> `ams.c`) is kernel-only and lives in `src/kernel/lib`. Truly context-agnostic helpers +> (`lib.c`, `objpool.c`) can safely be linked into user builds. See [User/kernel +> split](../include/sof/userspace/README.md). This directory collects cross-cutting helpers that don't belong to a single subsystem. When adding code here, decide first which tier it belongs to: a pure helper (no locks, no privileged ops), a privileged-kernel helper (spinlocks, platform callbacks), or a syscall implementation (kernel-only, reached from user space through a `z_vrfy_` validator). Keep the -tiers in separate files as `dma.c`/`dai.c` and `lib.c`/`objpool.c` already demonstrate. +tiers in separate files: kernel code goes to `src/kernel/lib`, as for `dma.c` and `dai.c`.