diff --git a/scripts/kernel_user_partition.txt b/scripts/kernel_user_partition.txt index 39d5e1191366..95b4bc0ae99f 100644 --- a/scripts/kernel_user_partition.txt +++ b/scripts/kernel_user_partition.txt @@ -30,7 +30,6 @@ src/lib/objpool.c [not-user-sources] src/audio/dai-legacy.c src/audio/host-legacy.c -src/audio/intel_uaol.c # SOF headers that only kernel code may include. # Not listed on purpose: rtos/init.h, which only provides the diff --git a/src/audio/intel_uaol.c b/src/audio/intel_uaol.c index ba676a9aea06..ebe77b487698 100644 --- a/src/audio/intel_uaol.c +++ b/src/audio/intel_uaol.c @@ -14,74 +14,16 @@ LOG_MODULE_REGISTER(uaol, CONFIG_SOF_LOG_LEVEL); -struct ipc4_uaol_link_capabilities { - uint32_t input_streams_supported : 4; - uint32_t output_streams_supported : 4; - uint32_t bidirectional_streams_supported : 5; - uint32_t rsvd : 19; - uint32_t max_tx_fifo_size; - uint32_t max_rx_fifo_size; -} __packed __aligned(4); - -struct ipc4_uaol_capabilities { - uint32_t link_count; - struct ipc4_uaol_link_capabilities link_caps[]; -} __packed __aligned(4); +/* The UAOL capabilities and stream mapping queries of the base firmware, which run in + * kernel mode, are in src/kernel/audio/intel_uaol-kernel.c + */ #define DEV_AND_COMMA(node) DEVICE_DT_GET(node), /* const pointers keep the array in rodata, readable by the user-space LL thread */ -static const struct device *const uaol_devs[] = { +const struct device *const uaol_devs[] = { DT_FOREACH_STATUS_OKAY(intel_adsp_uaol, DEV_AND_COMMA) }; -#if !CONFIG_SOF_OS_LINUX_COMPAT_PRIORITY -__cold void tlv_value_set_uaol_caps(struct sof_tlv *tuple, uint32_t type) -{ - const size_t dev_count = ARRAY_SIZE(uaol_devs); - struct uaol_capabilities dev_cap; - struct ipc4_uaol_capabilities *caps = (struct ipc4_uaol_capabilities *)tuple->value; - size_t caps_size = offsetof(struct ipc4_uaol_capabilities, link_caps[dev_count]); - size_t i; - int ret; - - assert_can_be_cold(); - - memset(caps, 0, caps_size); - - caps->link_count = dev_count; - for (i = 0; i < dev_count; i++) { - ret = uaol_get_capabilities(uaol_devs[i], &dev_cap); - if (ret) - continue; - - caps->link_caps[i].input_streams_supported = dev_cap.input_streams; - caps->link_caps[i].output_streams_supported = dev_cap.output_streams; - caps->link_caps[i].bidirectional_streams_supported = dev_cap.bidirectional_streams; - caps->link_caps[i].max_tx_fifo_size = dev_cap.max_tx_fifo_size; - caps->link_caps[i].max_rx_fifo_size = dev_cap.max_rx_fifo_size; - } - - tlv_value_set(tuple, type, caps_size, caps); -} -#endif /* !CONFIG_SOF_OS_LINUX_COMPAT_PRIORITY */ - -__cold int uaol_stream_id_to_hda_link_stream_id(int uaol_stream_id) -{ - size_t dev_count = ARRAY_SIZE(uaol_devs); - size_t i; - - assert_can_be_cold(); - - for (i = 0; i < dev_count; i++) { - int hda_link_stream_id = uaol_get_mapped_hda_link_stream_id(uaol_devs[i], - uaol_stream_id); - if (hda_link_stream_id >= 0) - return hda_link_stream_id; - } - - return -1; -} - const struct device *get_uaol_zdevice(int uaol_link_id) { /* uaol_link_id is just an index for the device tree device */ diff --git a/src/include/sof/audio/intel_uaol.h b/src/include/sof/audio/intel_uaol.h index d36335baf068..8b7cee072670 100644 --- a/src/include/sof/audio/intel_uaol.h +++ b/src/include/sof/audio/intel_uaol.h @@ -8,6 +8,7 @@ #include #include +#include struct sof_tlv; struct device; @@ -20,6 +21,11 @@ __cold int uaol_stream_id_to_hda_link_stream_id(int uaol_stream_id); const struct device *get_uaol_zdevice(int uaol_link_id); +#if CONFIG_UAOL_INTEL_ADSP +/* UAOL devices, defined in intel_uaol.c. In rodata, readable from user space. */ +extern const struct device *const uaol_devs[DT_NUM_INST_STATUS_OKAY(intel_adsp_uaol)]; +#endif + /* these are called from dai-zephyr.c */ struct dai; diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md index 1328248eac2b..8ee63ea87515 100644 --- a/src/include/sof/userspace/README.md +++ b/src/include/sof/userspace/README.md @@ -92,13 +92,13 @@ Classification is grep-verifiable — see [Verifying the taxonomy](#verifying-th > collects the validating half of several syscalls in one place — `zephyr/syscall/alloc.c`, > `cpu.c`, `vregion.c`, `dai.c`, `sof_dma.c` each hold the `z_vrfy_*` wrappers for their > subsystem, physically separated from the `z_impl_*` logic (in `zephyr/lib/*.c` and -> `src/lib/*.c`). This is exactly the boundary-tier separation this document promotes; the +> `src/kernel/lib/*.c`). This is exactly the boundary-tier separation this document promotes; the > migration opportunity is to move the *still-scattered* validators (IPC, module management, > library manager, scheduling, debug) toward the same convention. -> Borderline cases (e.g. parts of `lib/` that host syscall *implementations* such as `dma.c` -> and `dai.c`) behave as boundary code even though the directory is mostly shared-library. When -> a single directory genuinely mixes tiers, its README states the exception explicitly. +> Mixed files are split: the syscall *implementations* of `lib/dma.c` and `lib/dai.c`, for +> example, are in `kernel/lib/dma-kernel.c` and `dai-kernel.c`. When a single directory +> genuinely mixes tiers, its README states the exception explicitly. ## The boundary is the syscall set @@ -119,8 +119,8 @@ syscall by subsystem: | IPC | `ipc_msg_send`, `ipc_msg_list_remove`, `ipc_msg_reply`, `ipc_compound_pre_start`, `ipc_compound_post_start`, `ipc_wait_for_compound_msg`, `send_resource_notif` | `sof/ipc/msg.h`, `sof/ipc/ipc_reply.h`, `ipc4/handler.h`, `ipc4/notification.h` | scattered: `ipc/ipc-common.c`, `kernel/ipc/ipc3/helper.c`, `kernel/ipc/ipc4/handler-kernel.c`, `kernel/ipc/ipc4/notification.c` | | Heap alloc | `sof_heap_alloc`, `sof_heap_free` | `rtos/alloc.h` | `zephyr/syscall/alloc.c` / `zephyr/lib/alloc.c` | | Virtual regions | `vregion_create_map`, `vregion_set_interim`, `vregion_get`, `vregion_put`, `vregion_alloc_align`, `vregion_alloc_coherent_align`, `vregion_free` | `sof/lib/vregion.h` | `zephyr/syscall/vregion.c` / `zephyr/lib/vregion.c` | -| DMA | `sof_dma_get`, `sof_dma_put`, `sof_dma_get_attribute`, `sof_dma_request_channel`, `sof_dma_release_channel`, `sof_dma_config`, `sof_dma_start`, `sof_dma_stop`, `sof_dma_get_status`, `sof_dma_reload`, `sof_dma_suspend`, `sof_dma_resume` | `sof/lib/sof_dma.h` | `zephyr/syscall/sof_dma.c` / `lib/dma.c` (get/put); remaining 10 are `static inline z_impl_*` in `sof/lib/sof_dma.h` | -| DAI | `dai_get_device` | `sof/lib/dai-zephyr.h` | `zephyr/syscall/dai.c` / `lib/dai.c` | +| DMA | `sof_dma_get`, `sof_dma_put`, `sof_dma_get_attribute`, `sof_dma_request_channel`, `sof_dma_release_channel`, `sof_dma_config`, `sof_dma_start`, `sof_dma_stop`, `sof_dma_get_status`, `sof_dma_reload`, `sof_dma_suspend`, `sof_dma_resume` | `sof/lib/sof_dma.h` | `zephyr/syscall/sof_dma.c` / `kernel/lib/dma-kernel.c` (get/put); remaining 10 are `static inline z_impl_*` in `sof/lib/sof_dma.h` | +| DAI | `dai_get_device` | `sof/lib/dai-zephyr.h` | `zephyr/syscall/dai.c` / `kernel/lib/dai-kernel.c` | | CPU | `cpu_get_id` | `sof/lib/cpu.h` | `zephyr/syscall/cpu.c` / `zephyr/lib/cpu.c` | | Module management | `mod_balloc_align`, `mod_alloc_ext`, `mod_free`, `mod_free_all`, `mod_data_blob_handler_new`, `mod_data_blob_set`, `mod_fast_get` | `sof/audio/module_adapter/module/generic.h` | scattered: `audio/module_adapter/module/generic.c` | | Library manager | `lib_manager_free_module` | `sof/lib_manager.h` | scattered: `kernel/library_manager/lib_manager.c` | diff --git a/src/kernel/audio/CMakeLists.txt b/src/kernel/audio/CMakeLists.txt index e310aa2585a3..ddd239e57420 100644 --- a/src/kernel/audio/CMakeLists.txt +++ b/src/kernel/audio/CMakeLists.txt @@ -4,6 +4,8 @@ if(CONFIG_COMP_BASEFW_IPC4 AND NOT CONFIG_LIBRARY) add_local_sources(sof base_fw.c) endif() add_local_sources_ifdef(CONFIG_IPC4_BASE_FW_INTEL sof base_fw_intel.c) +# The kernel part of src/audio/intel_uaol.c, built with it +add_local_sources_ifdef(CONFIG_UAOL_INTEL_ADSP sof intel_uaol-kernel.c) if(CONFIG_INTEL_ADSP_MIC_PRIVACY) add_subdirectory(mic_privacy_manager) endif() diff --git a/src/kernel/audio/intel_uaol-kernel.c b/src/kernel/audio/intel_uaol-kernel.c new file mode 100644 index 000000000000..788bf49893ec --- /dev/null +++ b/src/kernel/audio/intel_uaol-kernel.c @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright 2026 Intel Corporation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +/* Kernel part of the UAOL support: the UAOL capabilities and stream mapping queries of + * the base firmware. The UAOL device table, its lookup and the UAOL feedback and rate + * adjustment, which run in the LL thread, are in src/audio/intel_uaol.c + */ + +struct ipc4_uaol_link_capabilities { + uint32_t input_streams_supported : 4; + uint32_t output_streams_supported : 4; + uint32_t bidirectional_streams_supported : 5; + uint32_t rsvd : 19; + uint32_t max_tx_fifo_size; + uint32_t max_rx_fifo_size; +} __packed __aligned(4); + +struct ipc4_uaol_capabilities { + uint32_t link_count; + struct ipc4_uaol_link_capabilities link_caps[]; +} __packed __aligned(4); + +#if !CONFIG_SOF_OS_LINUX_COMPAT_PRIORITY +__cold void tlv_value_set_uaol_caps(struct sof_tlv *tuple, uint32_t type) +{ + const size_t dev_count = ARRAY_SIZE(uaol_devs); + struct uaol_capabilities dev_cap; + struct ipc4_uaol_capabilities *caps = (struct ipc4_uaol_capabilities *)tuple->value; + size_t caps_size = offsetof(struct ipc4_uaol_capabilities, link_caps[dev_count]); + size_t i; + int ret; + + assert_can_be_cold(); + + memset(caps, 0, caps_size); + + caps->link_count = dev_count; + for (i = 0; i < dev_count; i++) { + ret = uaol_get_capabilities(uaol_devs[i], &dev_cap); + if (ret) + continue; + + caps->link_caps[i].input_streams_supported = dev_cap.input_streams; + caps->link_caps[i].output_streams_supported = dev_cap.output_streams; + caps->link_caps[i].bidirectional_streams_supported = dev_cap.bidirectional_streams; + caps->link_caps[i].max_tx_fifo_size = dev_cap.max_tx_fifo_size; + caps->link_caps[i].max_rx_fifo_size = dev_cap.max_rx_fifo_size; + } + + tlv_value_set(tuple, type, caps_size, caps); +} +#endif /* !CONFIG_SOF_OS_LINUX_COMPAT_PRIORITY */ + +__cold int uaol_stream_id_to_hda_link_stream_id(int uaol_stream_id) +{ + size_t dev_count = ARRAY_SIZE(uaol_devs); + size_t i; + + assert_can_be_cold(); + + for (i = 0; i < dev_count; i++) { + int hda_link_stream_id = uaol_get_mapped_hda_link_stream_id(uaol_devs[i], + uaol_stream_id); + if (hda_link_stream_id >= 0) + return hda_link_stream_id; + } + + return -1; +} diff --git a/src/kernel/lib/CMakeLists.txt b/src/kernel/lib/CMakeLists.txt index efe6347d02e1..0747994fbe3d 100644 --- a/src/kernel/lib/CMakeLists.txt +++ b/src/kernel/lib/CMakeLists.txt @@ -1,10 +1,15 @@ # SPDX-License-Identifier: BSD-3-Clause +# The kernel parts of src/lib/dma.c and dai.c, built with them +set(common_files dma-kernel.c dai-kernel.c) + if(CONFIG_LIBRARY) - add_local_sources(sof agent.c) + add_local_sources(sof agent.c ${common_files}) return() endif() +add_local_sources(sof ${common_files}) + if(CONFIG_HAVE_AGENT) add_local_sources(sof agent.c) endif() diff --git a/src/kernel/lib/dai-kernel.c b/src/kernel/lib/dai-kernel.c new file mode 100644 index 000000000000..c12bfeafa24e --- /dev/null +++ b/src/kernel/lib/dai-kernel.c @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2018 Intel Corporation. All rights reserved. +// +// Author: Marcin Maka + +#include +#include +#include +#include +#include +#include +#include +#include /* for zephyr_ll_user_heap() */ +#include +#include +#include +#include +#include +#include + +#if CONFIG_ZEPHYR_NATIVE_DRIVERS +#include +#include +#endif +LOG_MODULE_DECLARE(dai, CONFIG_SOF_LOG_LEVEL); + +extern struct tr_ctx dai_tr; + +/* Kernel part of the DAI library: the Zephyr DAI device table, the kernel half of the + * dai_get_device() system call and the DAI getters, which probe and remove the DAI + * drivers. The DAI groups are in src/lib/dai.c + */ + +#if CONFIG_ZEPHYR_NATIVE_DRIVERS + +#define GET_DEVICE_LIST(node) DEVICE_DT_GET(node), + +const struct device *zephyr_dev[] = { +#if CONFIG_DAI_INTEL_SSP + DT_FOREACH_STATUS_OKAY(intel_ssp_dai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_INTEL_DMIC + DT_FOREACH_STATUS_OKAY(intel_dai_dmic, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_INTEL_ALH + DT_FOREACH_STATUS_OKAY(intel_alh_dai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_INTEL_HDA + DT_FOREACH_STATUS_OKAY(intel_hda_dai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_NXP_SAI + DT_FOREACH_STATUS_OKAY(nxp_dai_sai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_NXP_ESAI + DT_FOREACH_STATUS_OKAY(nxp_dai_esai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_NXP_MICFIL + DT_FOREACH_STATUS_OKAY(nxp_dai_micfil, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_AMD_DMIC + DT_FOREACH_STATUS_OKAY(amd_dmic_dai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_AMD_SDW + DT_FOREACH_STATUS_OKAY(amd_acp_sdw_dai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_AMD_TDM + DT_FOREACH_STATUS_OKAY(amd_tdm_dai, GET_DEVICE_LIST) +#endif +#if CONFIG_DAI_INTEL_UAOL + DT_FOREACH_STATUS_OKAY(intel_uaol_dai, GET_DEVICE_LIST) +#endif +#if DT_HAS_COMPAT_STATUS_OKAY(mediatek_afe) + DT_FOREACH_STATUS_OKAY(mediatek_afe, GET_DEVICE_LIST) +#endif +}; + +const struct device **dai_get_device_list(size_t *count) +{ + __ASSERT_NO_MSG(count); + *count = ARRAY_SIZE(zephyr_dev); + + return zephyr_dev; +} + +/* convert sof_ipc_dai_type to Zephyr dai_type */ +static int sof_dai_type_to_zephyr(uint32_t type) +{ + switch (type) { + case SOF_DAI_INTEL_SSP: + return DAI_INTEL_SSP; + case SOF_DAI_INTEL_DMIC: + return DAI_INTEL_DMIC; + case SOF_DAI_INTEL_HDA: + return DAI_INTEL_HDA; + case SOF_DAI_INTEL_ALH: + return DAI_INTEL_ALH; + case SOF_DAI_INTEL_UAOL: + return DAI_INTEL_UAOL; + case SOF_DAI_IMX_SAI: + return DAI_IMX_SAI; + case SOF_DAI_IMX_ESAI: + return DAI_IMX_ESAI; + case SOF_DAI_AMD_DMIC: + return DAI_AMD_DMIC; + case SOF_DAI_MEDIATEK_AFE: + return DAI_MEDIATEK_AFE; + case SOF_DAI_IMX_MICFIL: + return DAI_IMX_MICFIL; + case SOF_DAI_AMD_SDW: + return DAI_AMD_SDW; + case SOF_DAI_AMD_HS: + case SOF_DAI_AMD_HS_VIRTUAL: + case SOF_DAI_AMD_SP: + case SOF_DAI_AMD_SP_VIRTUAL: + case SOF_DAI_AMD_BT: + case SOF_DAI_AMD_TDM: + return DAI_AMD_TDM; + default: + return -EINVAL; + } +} + +__cold const struct device *z_impl_dai_get_device(enum sof_ipc_dai_type type, uint32_t index) +{ + struct dai_config cfg; + int z_type; + int dir; + int i; + + assert_can_be_cold(); + + dir = (type == SOF_DAI_INTEL_DMIC) ? DAI_DIR_RX : DAI_DIR_BOTH; + + z_type = sof_dai_type_to_zephyr(type); + if (z_type < 0) { + tr_err(&dai_tr, "dai_get_device: no matching zephyr DAI type for %d ret = %d", + type, z_type); + return NULL; + } + + for (i = 0; i < ARRAY_SIZE(zephyr_dev); i++) { + if (dai_config_get(zephyr_dev[i], &cfg, dir)) + continue; + if (cfg.type == z_type && cfg.dai_index == index) + return zephyr_dev[i]; + } + + return NULL; +} + +static void dai_set_device_params(struct dai *d) +{ + switch (d->type) { + case SOF_DAI_INTEL_SSP: + d->dma_dev = SOF_DMA_DEV_SSP; +#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA + d->dma_caps = SOF_DMA_CAP_GP_LP | SOF_DMA_CAP_GP_HP; +#else + d->dma_caps = SOF_DMA_CAP_HDA; +#endif + break; + case SOF_DAI_INTEL_DMIC: + d->dma_dev = SOF_DMA_DEV_DMIC; +#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA + d->dma_caps = SOF_DMA_CAP_GP_LP | SOF_DMA_CAP_GP_HP; +#else + d->dma_caps = SOF_DMA_CAP_HDA; +#endif + break; + case SOF_DAI_INTEL_ALH: + d->dma_dev = SOF_DMA_DEV_ALH; +#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA + d->dma_caps = SOF_DMA_CAP_GP_LP | SOF_DMA_CAP_GP_HP; +#else + d->dma_caps = SOF_DMA_CAP_HDA; +#endif + break; + case SOF_DAI_INTEL_HDA: + case SOF_DAI_INTEL_UAOL: + d->dma_dev = SOF_DMA_DEV_HDA; + d->dma_caps = SOF_DMA_CAP_HDA; + break; + case SOF_DAI_AMD_SDW: + d->dma_dev = SOF_DMA_DEV_SW; + d->dma_caps = SOF_DMA_CAP_SW; + break; + /* All TDM-capable AMD DAIs share acp_tdm_dma. */ + case SOF_DAI_AMD_HS: + case SOF_DAI_AMD_HS_VIRTUAL: + case SOF_DAI_AMD_SP: + case SOF_DAI_AMD_SP_VIRTUAL: + case SOF_DAI_AMD_BT: + d->dma_dev = SOF_DMA_DEV_HS | SOF_DMA_DEV_SP | SOF_DMA_DEV_BT; + d->dma_caps = SOF_DMA_CAP_HS | SOF_DMA_CAP_SP | SOF_DMA_CAP_BT; + break; + case SOF_DAI_AMD_DMIC: + d->dma_dev = SOF_DMA_DEV_DMIC; + d->dma_caps = SOF_DMA_CAP_DMIC; + break; + case SOF_DAI_AMD_TDM: + d->dma_dev = SOF_DMA_DEV_TDM; + d->dma_caps = SOF_DMA_CAP_TDM; + break; + case SOF_DAI_MEDIATEK_AFE: + d->dma_dev = SOF_DMA_DEV_AFE_MEMIF; + break; + default: + break; + } +} + +/* called from kernel/ipc/ipc3/handler.c and some platform.c files */ +__cold struct dai *dai_get(uint32_t type, uint32_t index, uint32_t flags) +{ + const struct device *dev; + struct dai *d; + struct k_heap *heap = NULL; + + assert_can_be_cold(); + +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); +#endif + + dev = dai_get_device(type, index); + if (!dev) { + tr_err(&dai_tr, "dai_get: failed to get dai with index %d type %d", + index, type); + return NULL; + } + + d = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(struct dai), 0); + if (!d) + return NULL; + + memset(d, 0, sizeof(struct dai)); + + d->index = index; + d->type = type; + d->dev = dev; + + dai_set_device_params(d); + + if (dai_probe(d->dev)) { + tr_err(&dai_tr, "dai_get: failed to probe dai with index %d type %d", + index, type); + sof_heap_free(heap, d); + return NULL; + } + + return d; +} + +/* called from src/kernel/ipc/ipc3/handler.c */ +__cold void dai_put(struct dai *dai) +{ + int ret; + struct k_heap *heap = NULL; + + assert_can_be_cold(); + +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); +#endif + + ret = dai_remove(dai->dev); + if (ret < 0) { + tr_err(&dai_tr, "index %d failed ret = %d", + dai->index, ret); + } + + sof_heap_free(heap, dai); +} +#else +static inline const struct dai_type_info *dai_find_type(uint32_t type) +{ + const struct dai_info *info = dai_info_get(); + const struct dai_type_info *dti; + + for (dti = info->dai_type_array; + dti < info->dai_type_array + info->num_dai_types; dti++) { + if (dti->type == type) + return dti; + } + return NULL; +} + +__cold struct dai *dai_get(uint32_t type, uint32_t index, uint32_t flags) +{ + int ret = 0; + const struct dai_type_info *dti; + struct dai *d; + k_spinlock_key_t key; + + assert_can_be_cold(); + + dti = dai_find_type(type); + if (!dti) + return NULL; /* type not found */ + + for (d = dti->dai_array; d < dti->dai_array + dti->num_dais; d++) { + if (d->index != index) { + continue; + } + /* device created? */ + key = k_spin_lock(&d->lock); + if (d->sref == 0) { + if (flags & DAI_CREAT) + ret = dai_probe(d); + else + ret = -ENODEV; + } + if (!ret) + d->sref++; + + tr_info(&dai_tr, "dai_get type %d index %d new sref %d", + type, index, d->sref); + + k_spin_unlock(&d->lock, key); + + return !ret ? d : NULL; + } + tr_err(&dai_tr, "dai_get: type %d index %d not found", type, index); + return NULL; +} + +__cold void dai_put(struct dai *dai) +{ + int ret; + k_spinlock_key_t key; + + assert_can_be_cold(); + + key = k_spin_lock(&dai->lock); + if (--dai->sref == 0) { + ret = dai_remove(dai); + if (ret < 0) { + tr_err(&dai_tr, "type %d index %d dai_remove() failed ret = %d", + dai->drv->type, dai->index, ret); + } + } + tr_info(&dai_tr, "type %d index %d new sref %d", + dai->drv->type, dai->index, dai->sref); + k_spin_unlock(&dai->lock, key); +} +#endif diff --git a/src/kernel/lib/dma-kernel.c b/src/kernel/lib/dma-kernel.c new file mode 100644 index 000000000000..422c89f20d2c --- /dev/null +++ b/src/kernel/lib/dma-kernel.c @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2016 Intel Corporation. All rights reserved. +// +// Author: Ranjani Sridharan + +#include +#include +#include +#include +#include +#if CONFIG_INTEL_ADSP_MIC_PRIVACY +#include +#endif +#include +#include +#include +#include +#include +#include /* zephyr_ll_user_heap_verify() */ +#include +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_REGISTER(dma, CONFIG_SOF_LOG_LEVEL); + +SOF_DEFINE_REG_UUID(dma); + +/* unused with Zephyr, generates no output */ +DECLARE_TR_CTX(dma_tr, SOF_UUID(dma_uuid), LOG_LEVEL_INFO); + +/* Kernel part of the DMA library: the DMA controller lookup, reference counting and + * channel allocation, i.e. the kernel half of the sof_dma_get() and sof_dma_put() + * system calls, and the getters of the legacy DMA drivers. The DMA buffer helpers + * are in src/lib/dma.c + */ + +#if CONFIG_ZEPHYR_NATIVE_DRIVERS +static int dma_init(struct sof_dma *dma); + +#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA +/* + * dma[] is writable by user threads, so the reference count, the lock and + * the channel array to free are kept in the kernel-only struct + * sof_dma_kinfo. dma->sref is still updated for diagnostics, but never + * trusted. + */ +static int *dma_sref(struct sof_dma *dma) +{ + return &sof_dma_kinfo_get(dma)->sref; +} + +/* + * dma->lock is user-writable too: a user thread could release it while the + * kernel holds it, letting two cores update the reference count at once. + */ +static struct k_spinlock *dma_lock(struct sof_dma *dma) +{ + return &sof_dma_kinfo_get(dma)->lock; +} +#else +static int *dma_sref(struct sof_dma *dma) +{ + return &dma->sref; +} + +static struct k_spinlock *dma_lock(struct sof_dma *dma) +{ + return &dma->lock; +} +#endif + +__cold struct sof_dma *z_impl_sof_dma_get(uint32_t dir, uint32_t cap, uint32_t dev, uint32_t flags) +{ + const struct dma_info *info = dma_info_get(); + int users, ret = 0; + int min_users = INT32_MAX; + struct sof_dma *d = NULL, *dmin = NULL; + k_spinlock_key_t key; + + assert_can_be_cold(); + + if (!info->num_dmas) { + tr_err(&dma_tr, "dma_get(): No DMACs installed"); + return NULL; + } + + /* find DMAC with free channels that matches request */ + for (d = info->dma_array; d < info->dma_array + info->num_dmas; + d++) { + /* skip if this DMAC does not support the requested dir */ + if (dir && (d->plat_data.dir & dir) == 0) + continue; + + /* skip if this DMAC does not support the requested caps */ + if (cap && (d->plat_data.caps & cap) == 0) + continue; + + /* skip if this DMAC does not support the requested dev */ + if (dev && (d->plat_data.devs & dev) == 0) + continue; + + /* if exclusive access is requested */ + if (flags & SOF_DMA_ACCESS_EXCLUSIVE) { + /* ret DMA with no users */ + if (!d->sref) { + dmin = d; + break; + } + } else { + /* get number of users for this DMAC*/ + users = d->sref; + + /* pick DMAC with the least num of users */ + if (users < min_users) { + dmin = d; + min_users = users; + } + } + } + + if (!dmin) { + tr_err(&dma_tr, "No DMAC dir %d caps 0x%x dev 0x%x flags 0x%x", + dir, cap, dev, flags); + + for (d = info->dma_array; + d < info->dma_array + info->num_dmas; + d++) { + tr_err(&dma_tr, " DMAC ID %d users %d busy channels %ld", + d->plat_data.id, d->sref, + atomic_read(&d->num_channels_busy)); + tr_err(&dma_tr, " caps 0x%x dev 0x%x", + d->plat_data.caps, d->plat_data.devs); + } + + return NULL; + } + + /* return DMAC */ + tr_dbg(&dma_tr, "dma_get(), dma-probe id = %d", + dmin->plat_data.id); + + /* Shared DMA controllers with multiple channels + * may be requested many times, let the probe() + * do on-first-use initialization. + */ + key = k_spin_lock(dma_lock(dmin)); + + if (!*dma_sref(dmin)) { + ret = dma_init(dmin); + if (ret < 0) { + tr_err(&dma_tr, "dma_get(): dma-probe failed id = %d, ret = %d", + dmin->plat_data.id, ret); + goto out; + } + } + + ++*dma_sref(dmin); + /* user-visible copy for diagnostics and DMAC selection, never trusted */ + dmin->sref = *dma_sref(dmin); + + tr_info(&dma_tr, "dma_get() ID %d sref = %d busy channels %ld", + dmin->plat_data.id, dmin->sref, + atomic_read(&dmin->num_channels_busy)); +out: + k_spin_unlock(dma_lock(dmin), key); + return !ret ? dmin : NULL; +} + +__cold void z_impl_sof_dma_put(struct sof_dma *dma) +{ + k_spinlock_key_t key; + + assert_can_be_cold(); + + key = k_spin_lock(dma_lock(dma)); + if (*dma_sref(dma) <= 0) { + tr_err(&dma_tr, "dma_put(), dma = %p not in use", dma); + k_spin_unlock(dma_lock(dma), key); + return; + } + + --*dma_sref(dma); + dma->sref = *dma_sref(dma); + if (!*dma_sref(dma)) { +#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA + struct sof_dma_kinfo *kinfo = sof_dma_kinfo_get(dma); + + sof_heap_free(kinfo->heap, kinfo->chan); + kinfo->chan = NULL; +#else + sof_heap_free(dma->heap, dma->chan); +#endif + dma->chan = NULL; + } + + tr_info(&dma_tr, "dma_put(), dma = %p, sref = %d", + dma, dma->sref); + k_spin_unlock(dma_lock(dma), key); +} + +__cold static int dma_init(struct sof_dma *dma) +{ + struct dma_chan_data *chans, *chan; + struct k_heap *heap = sof_sys_user_heap_get(); +#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA + /* dma->plat_data is user-writable, use the kernel snapshot */ + const uint32_t channels = sof_dma_kinfo_get(dma)->channels; +#else + const uint32_t channels = dma->plat_data.channels; +#endif + uint32_t i; + + assert_can_be_cold(); + +#if CONFIG_SOF_USERSPACE_INTERFACE_DMA && CONFIG_SOF_USERSPACE_LL + /* + * Called in supervisor mode from the sof_dma_get() system call. The + * heap pointer comes from user-writable memory, so check that it is + * the LL user heap before running the allocator on it. + */ + if (!zephyr_ll_user_heap_verify(heap)) { + tr_err(&dma_tr, "dma %d: invalid channel heap %p", dma->plat_data.id, heap); + return -EINVAL; + } +#endif + + /* + * allocate dma channels, use a local pointer until the array is + * initialized: dma->chan is user-writable and must not be read back + */ + chans = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(struct dma_chan_data) * channels, 0); + if (!chans) { + tr_err(&dma_tr, "dma %d allocation of channels failed", + dma->plat_data.id); + return -ENOMEM; + } + + memset(chans, 0, sizeof(struct dma_chan_data) * channels); + /* init work */ + for (i = 0, chan = chans; i < channels; i++, chan++) { + chan->dma = dma; + chan->index = i; + } + +#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA + sof_dma_kinfo_get(dma)->heap = heap; + sof_dma_kinfo_get(dma)->chan = chans; +#endif + dma->heap = heap; + dma->chan = chans; + + return 0; +} +#else +struct dma *dma_get(uint32_t dir, uint32_t cap, uint32_t dev, uint32_t flags) +{ + const struct dma_info *info = dma_info_get(); + int users, ret; + int min_users = INT32_MAX; + struct dma *d = NULL, *dmin = NULL; + k_spinlock_key_t key; + + if (!info->num_dmas) { + tr_err(&dma_tr, "dma_get(): No DMACs installed"); + return NULL; + } + + /* find DMAC with free channels that matches request */ + for (d = info->dma_array; d < info->dma_array + info->num_dmas; + d++) { + /* skip if this DMAC does not support the requested dir */ + if (dir && (d->plat_data.dir & dir) == 0) + continue; + + /* skip if this DMAC does not support the requested caps */ + if (cap && (d->plat_data.caps & cap) == 0) + continue; + + /* skip if this DMAC does not support the requested dev */ + if (dev && (d->plat_data.devs & dev) == 0) + continue; + + /* skip if this DMAC has 1 user per avail channel */ + /* TODO: this should be fixed in dai.c to allow more users */ + if (d->sref >= d->plat_data.channels) + continue; + + /* if exclusive access is requested */ + if (flags & DMA_ACCESS_EXCLUSIVE) { + /* ret DMA with no users */ + if (!d->sref) { + dmin = d; + break; + } + } else { + /* get number of users for this DMAC*/ + users = d->sref; + + /* pick DMAC with the least num of users */ + if (users < min_users) { + dmin = d; + min_users = users; + } + } + } + + if (!dmin) { + tr_err(&dma_tr, "No DMAC dir %d caps 0x%x dev 0x%x flags 0x%x", + dir, cap, dev, flags); + + for (d = info->dma_array; + d < info->dma_array + info->num_dmas; + d++) { + tr_err(&dma_tr, " DMAC ID %d users %d busy channels %ld", + d->plat_data.id, d->sref, + atomic_read(&d->num_channels_busy)); + tr_err(&dma_tr, " caps 0x%x dev 0x%x", + d->plat_data.caps, d->plat_data.devs); + } + + return NULL; + } + + /* return DMAC */ + tr_dbg(&dma_tr, "dma_get(), dma-probe id = %d", + dmin->plat_data.id); + + /* Shared DMA controllers with multiple channels + * may be requested many times, let the probe() + * do on-first-use initialization. + */ + key = k_spin_lock(&dmin->lock); + + ret = 0; + if (!dmin->sref) { + ret = dma_probe_legacy(dmin); + if (ret < 0) { + tr_err(&dma_tr, "dma_get(): dma-probe failed id = %d, ret = %d", + dmin->plat_data.id, ret); + } + } + if (!ret) + dmin->sref++; + + tr_info(&dma_tr, "dma_get() ID %d sref = %d busy channels %ld", + dmin->plat_data.id, dmin->sref, + atomic_read(&dmin->num_channels_busy)); + + k_spin_unlock(&dmin->lock, key); + return !ret ? dmin : NULL; +} + +void dma_put(struct dma *dma) +{ + k_spinlock_key_t key; + int ret; + + key = k_spin_lock(&dma->lock); + if (--dma->sref == 0) { + ret = dma_remove_legacy(dma); + if (ret < 0) { + tr_err(&dma_tr, "dma_remove() failed id = %d, ret = %d", + dma->plat_data.id, ret); + } + } + tr_info(&dma_tr, "dma = %p, sref = %d", + dma, dma->sref); + k_spin_unlock(&dma->lock, key); +} +#endif diff --git a/src/lib/README.md b/src/lib/README.md index 6c986bdffafc..5cb01f0945f3 100644 --- a/src/lib/README.md +++ b/src/lib/README.md @@ -1,15 +1,17 @@ # Common Library (`src/lib`) -> **Runs in:** Mixed — this directory contains code in more than one tier. `dma.c` and `dai.c` -> host the kernel-side implementations (`z_impl_sof_dma_get/put`, `z_impl_dai_get_device`) of -> syscalls whose validators live in `zephyr/syscall/`; treat them as boundary/kernel code. -> `notifier.c` takes Zephyr spinlocks and is similarly restricted to kernel context. The clock, -> KCPS and AMS code (`clk.c`, `cpu-clk-manager.c`, `ams.c`) is kernel-only and lives in -> `src/kernel/lib`. Truly context-agnostic helpers (`lib.c`, `objpool.c`) can safely be linked -> into user builds. See [User/kernel split](../include/sof/userspace/README.md). +> **Runs in:** Mixed — this directory contains code in more than one tier. `dma.c` (the DMA +> buffer helpers) and `dai.c` (the DAI groups) also run in the user-space LL thread. Their +> kernel parts, including the kernel-side implementations (`z_impl_sof_dma_get/put`, +> `z_impl_dai_get_device`) of syscalls whose validators live in `zephyr/syscall/`, are in +> `src/kernel/lib/dma-kernel.c` and `dai-kernel.c`. `notifier.c` takes Zephyr spinlocks and is +> restricted to kernel context. The clock, KCPS and AMS code (`clk.c`, `cpu-clk-manager.c`, +> `ams.c`) is kernel-only and lives in `src/kernel/lib`. Truly context-agnostic helpers +> (`lib.c`, `objpool.c`) can safely be linked into user builds. See [User/kernel +> split](../include/sof/userspace/README.md). This directory collects cross-cutting helpers that don't belong to a single subsystem. When adding code here, decide first which tier it belongs to: a pure helper (no locks, no privileged ops), a privileged-kernel helper (spinlocks, platform callbacks), or a syscall implementation (kernel-only, reached from user space through a `z_vrfy_` validator). Keep the -tiers in separate files as `dma.c`/`dai.c` and `lib.c`/`objpool.c` already demonstrate. +tiers in separate files: kernel code goes to `src/kernel/lib`, as for `dma.c` and `dai.c`. diff --git a/src/lib/dai.c b/src/lib/dai.c index 214dcf979711..4bab1fbdc24e 100644 --- a/src/lib/dai.c +++ b/src/lib/dai.c @@ -11,7 +11,6 @@ #include #include #include -#include /* for zephyr_ll_user_heap() */ #include #include #include @@ -19,10 +18,6 @@ #include #include -#if CONFIG_ZEPHYR_NATIVE_DRIVERS -#include -#include -#endif LOG_MODULE_REGISTER(dai, CONFIG_SOF_LOG_LEVEL); SOF_DEFINE_REG_UUID(dai_lib); @@ -30,6 +25,10 @@ SOF_DEFINE_REG_UUID(dai_lib); /* unused with Zephyr, generates no output */ DECLARE_TR_CTX(dai_tr, SOF_UUID(dai_lib_uuid), LOG_LEVEL_INFO); +/* The DAI device lookup and the DAI getters, which run in kernel mode, are in + * src/kernel/lib/dai-kernel.c + */ + struct dai_group_list { struct list_item list; } __aligned(PLATFORM_DCACHE_ALIGN); @@ -160,317 +159,3 @@ __cold void dai_group_put(struct dai_group *group) if (!group->num_dais) group->group_id = 0; } - -#if CONFIG_ZEPHYR_NATIVE_DRIVERS - -#define GET_DEVICE_LIST(node) DEVICE_DT_GET(node), - -const struct device *zephyr_dev[] = { -#if CONFIG_DAI_INTEL_SSP - DT_FOREACH_STATUS_OKAY(intel_ssp_dai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_INTEL_DMIC - DT_FOREACH_STATUS_OKAY(intel_dai_dmic, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_INTEL_ALH - DT_FOREACH_STATUS_OKAY(intel_alh_dai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_INTEL_HDA - DT_FOREACH_STATUS_OKAY(intel_hda_dai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_NXP_SAI - DT_FOREACH_STATUS_OKAY(nxp_dai_sai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_NXP_ESAI - DT_FOREACH_STATUS_OKAY(nxp_dai_esai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_NXP_MICFIL - DT_FOREACH_STATUS_OKAY(nxp_dai_micfil, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_AMD_DMIC - DT_FOREACH_STATUS_OKAY(amd_dmic_dai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_AMD_SDW - DT_FOREACH_STATUS_OKAY(amd_acp_sdw_dai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_AMD_TDM - DT_FOREACH_STATUS_OKAY(amd_tdm_dai, GET_DEVICE_LIST) -#endif -#if CONFIG_DAI_INTEL_UAOL - DT_FOREACH_STATUS_OKAY(intel_uaol_dai, GET_DEVICE_LIST) -#endif -#if DT_HAS_COMPAT_STATUS_OKAY(mediatek_afe) - DT_FOREACH_STATUS_OKAY(mediatek_afe, GET_DEVICE_LIST) -#endif -}; - -const struct device **dai_get_device_list(size_t *count) -{ - __ASSERT_NO_MSG(count); - *count = ARRAY_SIZE(zephyr_dev); - - return zephyr_dev; -} - -/* convert sof_ipc_dai_type to Zephyr dai_type */ -static int sof_dai_type_to_zephyr(uint32_t type) -{ - switch (type) { - case SOF_DAI_INTEL_SSP: - return DAI_INTEL_SSP; - case SOF_DAI_INTEL_DMIC: - return DAI_INTEL_DMIC; - case SOF_DAI_INTEL_HDA: - return DAI_INTEL_HDA; - case SOF_DAI_INTEL_ALH: - return DAI_INTEL_ALH; - case SOF_DAI_INTEL_UAOL: - return DAI_INTEL_UAOL; - case SOF_DAI_IMX_SAI: - return DAI_IMX_SAI; - case SOF_DAI_IMX_ESAI: - return DAI_IMX_ESAI; - case SOF_DAI_AMD_DMIC: - return DAI_AMD_DMIC; - case SOF_DAI_MEDIATEK_AFE: - return DAI_MEDIATEK_AFE; - case SOF_DAI_IMX_MICFIL: - return DAI_IMX_MICFIL; - case SOF_DAI_AMD_SDW: - return DAI_AMD_SDW; - case SOF_DAI_AMD_HS: - case SOF_DAI_AMD_HS_VIRTUAL: - case SOF_DAI_AMD_SP: - case SOF_DAI_AMD_SP_VIRTUAL: - case SOF_DAI_AMD_BT: - case SOF_DAI_AMD_TDM: - return DAI_AMD_TDM; - default: - return -EINVAL; - } -} - -__cold const struct device *z_impl_dai_get_device(enum sof_ipc_dai_type type, uint32_t index) -{ - struct dai_config cfg; - int z_type; - int dir; - int i; - - assert_can_be_cold(); - - dir = (type == SOF_DAI_INTEL_DMIC) ? DAI_DIR_RX : DAI_DIR_BOTH; - - z_type = sof_dai_type_to_zephyr(type); - if (z_type < 0) { - tr_err(&dai_tr, "dai_get_device: no matching zephyr DAI type for %d ret = %d", - type, z_type); - return NULL; - } - - for (i = 0; i < ARRAY_SIZE(zephyr_dev); i++) { - if (dai_config_get(zephyr_dev[i], &cfg, dir)) - continue; - if (cfg.type == z_type && cfg.dai_index == index) - return zephyr_dev[i]; - } - - return NULL; -} - -static void dai_set_device_params(struct dai *d) -{ - switch (d->type) { - case SOF_DAI_INTEL_SSP: - d->dma_dev = SOF_DMA_DEV_SSP; -#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA - d->dma_caps = SOF_DMA_CAP_GP_LP | SOF_DMA_CAP_GP_HP; -#else - d->dma_caps = SOF_DMA_CAP_HDA; -#endif - break; - case SOF_DAI_INTEL_DMIC: - d->dma_dev = SOF_DMA_DEV_DMIC; -#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA - d->dma_caps = SOF_DMA_CAP_GP_LP | SOF_DMA_CAP_GP_HP; -#else - d->dma_caps = SOF_DMA_CAP_HDA; -#endif - break; - case SOF_DAI_INTEL_ALH: - d->dma_dev = SOF_DMA_DEV_ALH; -#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA - d->dma_caps = SOF_DMA_CAP_GP_LP | SOF_DMA_CAP_GP_HP; -#else - d->dma_caps = SOF_DMA_CAP_HDA; -#endif - break; - case SOF_DAI_INTEL_HDA: - case SOF_DAI_INTEL_UAOL: - d->dma_dev = SOF_DMA_DEV_HDA; - d->dma_caps = SOF_DMA_CAP_HDA; - break; - case SOF_DAI_AMD_SDW: - d->dma_dev = SOF_DMA_DEV_SW; - d->dma_caps = SOF_DMA_CAP_SW; - break; - /* All TDM-capable AMD DAIs share acp_tdm_dma. */ - case SOF_DAI_AMD_HS: - case SOF_DAI_AMD_HS_VIRTUAL: - case SOF_DAI_AMD_SP: - case SOF_DAI_AMD_SP_VIRTUAL: - case SOF_DAI_AMD_BT: - d->dma_dev = SOF_DMA_DEV_HS | SOF_DMA_DEV_SP | SOF_DMA_DEV_BT; - d->dma_caps = SOF_DMA_CAP_HS | SOF_DMA_CAP_SP | SOF_DMA_CAP_BT; - break; - case SOF_DAI_AMD_DMIC: - d->dma_dev = SOF_DMA_DEV_DMIC; - d->dma_caps = SOF_DMA_CAP_DMIC; - break; - case SOF_DAI_AMD_TDM: - d->dma_dev = SOF_DMA_DEV_TDM; - d->dma_caps = SOF_DMA_CAP_TDM; - break; - case SOF_DAI_MEDIATEK_AFE: - d->dma_dev = SOF_DMA_DEV_AFE_MEMIF; - break; - default: - break; - } -} - -/* called from kernel/ipc/ipc3/handler.c and some platform.c files */ -__cold struct dai *dai_get(uint32_t type, uint32_t index, uint32_t flags) -{ - const struct device *dev; - struct dai *d; - struct k_heap *heap = NULL; - - assert_can_be_cold(); - -#ifdef CONFIG_SOF_USERSPACE_LL - heap = zephyr_ll_user_heap(); -#endif - - dev = dai_get_device(type, index); - if (!dev) { - tr_err(&dai_tr, "dai_get: failed to get dai with index %d type %d", - index, type); - return NULL; - } - - d = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(struct dai), 0); - if (!d) - return NULL; - - memset(d, 0, sizeof(struct dai)); - - d->index = index; - d->type = type; - d->dev = dev; - - dai_set_device_params(d); - - if (dai_probe(d->dev)) { - tr_err(&dai_tr, "dai_get: failed to probe dai with index %d type %d", - index, type); - sof_heap_free(heap, d); - return NULL; - } - - return d; -} - -/* called from src/kernel/ipc/ipc3/handler.c */ -__cold void dai_put(struct dai *dai) -{ - int ret; - struct k_heap *heap = NULL; - - assert_can_be_cold(); - -#ifdef CONFIG_SOF_USERSPACE_LL - heap = zephyr_ll_user_heap(); -#endif - - ret = dai_remove(dai->dev); - if (ret < 0) { - tr_err(&dai_tr, "index %d failed ret = %d", - dai->index, ret); - } - - sof_heap_free(heap, dai); -} -#else -static inline const struct dai_type_info *dai_find_type(uint32_t type) -{ - const struct dai_info *info = dai_info_get(); - const struct dai_type_info *dti; - - for (dti = info->dai_type_array; - dti < info->dai_type_array + info->num_dai_types; dti++) { - if (dti->type == type) - return dti; - } - return NULL; -} - -__cold struct dai *dai_get(uint32_t type, uint32_t index, uint32_t flags) -{ - int ret = 0; - const struct dai_type_info *dti; - struct dai *d; - k_spinlock_key_t key; - - assert_can_be_cold(); - - dti = dai_find_type(type); - if (!dti) - return NULL; /* type not found */ - - for (d = dti->dai_array; d < dti->dai_array + dti->num_dais; d++) { - if (d->index != index) { - continue; - } - /* device created? */ - key = k_spin_lock(&d->lock); - if (d->sref == 0) { - if (flags & DAI_CREAT) - ret = dai_probe(d); - else - ret = -ENODEV; - } - if (!ret) - d->sref++; - - tr_info(&dai_tr, "dai_get type %d index %d new sref %d", - type, index, d->sref); - - k_spin_unlock(&d->lock, key); - - return !ret ? d : NULL; - } - tr_err(&dai_tr, "dai_get: type %d index %d not found", type, index); - return NULL; -} - -__cold void dai_put(struct dai *dai) -{ - int ret; - k_spinlock_key_t key; - - assert_can_be_cold(); - - key = k_spin_lock(&dai->lock); - if (--dai->sref == 0) { - ret = dai_remove(dai); - if (ret < 0) { - tr_err(&dai_tr, "type %d index %d dai_remove() failed ret = %d", - dai->drv->type, dai->index, ret); - } - } - tr_info(&dai_tr, "type %d index %d new sref %d", - dai->drv->type, dai->index, dai->sref); - k_spin_unlock(&dai->lock, key); -} -#endif diff --git a/src/lib/dma.c b/src/lib/dma.c index 8a5195212fb7..4c454d8d928c 100644 --- a/src/lib/dma.c +++ b/src/lib/dma.c @@ -17,8 +17,6 @@ #include #include #include -#include /* zephyr_ll_user_heap_verify() */ -#include #include #include #include @@ -28,348 +26,9 @@ #include #include "../audio/copier/copier.h" -LOG_MODULE_REGISTER(dma, CONFIG_SOF_LOG_LEVEL); - -SOF_DEFINE_REG_UUID(dma); - -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(dma_tr, SOF_UUID(dma_uuid), LOG_LEVEL_INFO); - -#if CONFIG_ZEPHYR_NATIVE_DRIVERS -static int dma_init(struct sof_dma *dma); - -#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA -/* - * dma[] is writable by user threads, so the reference count, the lock and - * the channel array to free are kept in the kernel-only struct - * sof_dma_kinfo. dma->sref is still updated for diagnostics, but never - * trusted. - */ -static int *dma_sref(struct sof_dma *dma) -{ - return &sof_dma_kinfo_get(dma)->sref; -} - -/* - * dma->lock is user-writable too: a user thread could release it while the - * kernel holds it, letting two cores update the reference count at once. +/* The DMA controller lookup and its system calls, which run in kernel mode, are in + * src/kernel/lib/dma-kernel.c */ -static struct k_spinlock *dma_lock(struct sof_dma *dma) -{ - return &sof_dma_kinfo_get(dma)->lock; -} -#else -static int *dma_sref(struct sof_dma *dma) -{ - return &dma->sref; -} - -static struct k_spinlock *dma_lock(struct sof_dma *dma) -{ - return &dma->lock; -} -#endif - -__cold struct sof_dma *z_impl_sof_dma_get(uint32_t dir, uint32_t cap, uint32_t dev, uint32_t flags) -{ - const struct dma_info *info = dma_info_get(); - int users, ret = 0; - int min_users = INT32_MAX; - struct sof_dma *d = NULL, *dmin = NULL; - k_spinlock_key_t key; - - assert_can_be_cold(); - - if (!info->num_dmas) { - tr_err(&dma_tr, "dma_get(): No DMACs installed"); - return NULL; - } - - /* find DMAC with free channels that matches request */ - for (d = info->dma_array; d < info->dma_array + info->num_dmas; - d++) { - /* skip if this DMAC does not support the requested dir */ - if (dir && (d->plat_data.dir & dir) == 0) - continue; - - /* skip if this DMAC does not support the requested caps */ - if (cap && (d->plat_data.caps & cap) == 0) - continue; - - /* skip if this DMAC does not support the requested dev */ - if (dev && (d->plat_data.devs & dev) == 0) - continue; - - /* if exclusive access is requested */ - if (flags & SOF_DMA_ACCESS_EXCLUSIVE) { - /* ret DMA with no users */ - if (!d->sref) { - dmin = d; - break; - } - } else { - /* get number of users for this DMAC*/ - users = d->sref; - - /* pick DMAC with the least num of users */ - if (users < min_users) { - dmin = d; - min_users = users; - } - } - } - - if (!dmin) { - tr_err(&dma_tr, "No DMAC dir %d caps 0x%x dev 0x%x flags 0x%x", - dir, cap, dev, flags); - - for (d = info->dma_array; - d < info->dma_array + info->num_dmas; - d++) { - tr_err(&dma_tr, " DMAC ID %d users %d busy channels %ld", - d->plat_data.id, d->sref, - atomic_read(&d->num_channels_busy)); - tr_err(&dma_tr, " caps 0x%x dev 0x%x", - d->plat_data.caps, d->plat_data.devs); - } - - return NULL; - } - - /* return DMAC */ - tr_dbg(&dma_tr, "dma_get(), dma-probe id = %d", - dmin->plat_data.id); - - /* Shared DMA controllers with multiple channels - * may be requested many times, let the probe() - * do on-first-use initialization. - */ - key = k_spin_lock(dma_lock(dmin)); - - if (!*dma_sref(dmin)) { - ret = dma_init(dmin); - if (ret < 0) { - tr_err(&dma_tr, "dma_get(): dma-probe failed id = %d, ret = %d", - dmin->plat_data.id, ret); - goto out; - } - } - - ++*dma_sref(dmin); - /* user-visible copy for diagnostics and DMAC selection, never trusted */ - dmin->sref = *dma_sref(dmin); - - tr_info(&dma_tr, "dma_get() ID %d sref = %d busy channels %ld", - dmin->plat_data.id, dmin->sref, - atomic_read(&dmin->num_channels_busy)); -out: - k_spin_unlock(dma_lock(dmin), key); - return !ret ? dmin : NULL; -} - -__cold void z_impl_sof_dma_put(struct sof_dma *dma) -{ - k_spinlock_key_t key; - - assert_can_be_cold(); - - key = k_spin_lock(dma_lock(dma)); - if (*dma_sref(dma) <= 0) { - tr_err(&dma_tr, "dma_put(), dma = %p not in use", dma); - k_spin_unlock(dma_lock(dma), key); - return; - } - - --*dma_sref(dma); - dma->sref = *dma_sref(dma); - if (!*dma_sref(dma)) { -#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA - struct sof_dma_kinfo *kinfo = sof_dma_kinfo_get(dma); - - sof_heap_free(kinfo->heap, kinfo->chan); - kinfo->chan = NULL; -#else - sof_heap_free(dma->heap, dma->chan); -#endif - dma->chan = NULL; - } - - tr_info(&dma_tr, "dma_put(), dma = %p, sref = %d", - dma, dma->sref); - k_spin_unlock(dma_lock(dma), key); -} - -__cold static int dma_init(struct sof_dma *dma) -{ - struct dma_chan_data *chans, *chan; - struct k_heap *heap = sof_sys_user_heap_get(); -#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA - /* dma->plat_data is user-writable, use the kernel snapshot */ - const uint32_t channels = sof_dma_kinfo_get(dma)->channels; -#else - const uint32_t channels = dma->plat_data.channels; -#endif - uint32_t i; - - assert_can_be_cold(); - -#if CONFIG_SOF_USERSPACE_INTERFACE_DMA && CONFIG_SOF_USERSPACE_LL - /* - * Called in supervisor mode from the sof_dma_get() system call. The - * heap pointer comes from user-writable memory, so check that it is - * the LL user heap before running the allocator on it. - */ - if (!zephyr_ll_user_heap_verify(heap)) { - tr_err(&dma_tr, "dma %d: invalid channel heap %p", dma->plat_data.id, heap); - return -EINVAL; - } -#endif - - /* - * allocate dma channels, use a local pointer until the array is - * initialized: dma->chan is user-writable and must not be read back - */ - chans = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(struct dma_chan_data) * channels, 0); - if (!chans) { - tr_err(&dma_tr, "dma %d allocation of channels failed", - dma->plat_data.id); - return -ENOMEM; - } - - memset(chans, 0, sizeof(struct dma_chan_data) * channels); - /* init work */ - for (i = 0, chan = chans; i < channels; i++, chan++) { - chan->dma = dma; - chan->index = i; - } - -#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA - sof_dma_kinfo_get(dma)->heap = heap; - sof_dma_kinfo_get(dma)->chan = chans; -#endif - dma->heap = heap; - dma->chan = chans; - - return 0; -} -#else -struct dma *dma_get(uint32_t dir, uint32_t cap, uint32_t dev, uint32_t flags) -{ - const struct dma_info *info = dma_info_get(); - int users, ret; - int min_users = INT32_MAX; - struct dma *d = NULL, *dmin = NULL; - k_spinlock_key_t key; - - if (!info->num_dmas) { - tr_err(&dma_tr, "dma_get(): No DMACs installed"); - return NULL; - } - - /* find DMAC with free channels that matches request */ - for (d = info->dma_array; d < info->dma_array + info->num_dmas; - d++) { - /* skip if this DMAC does not support the requested dir */ - if (dir && (d->plat_data.dir & dir) == 0) - continue; - - /* skip if this DMAC does not support the requested caps */ - if (cap && (d->plat_data.caps & cap) == 0) - continue; - - /* skip if this DMAC does not support the requested dev */ - if (dev && (d->plat_data.devs & dev) == 0) - continue; - - /* skip if this DMAC has 1 user per avail channel */ - /* TODO: this should be fixed in dai.c to allow more users */ - if (d->sref >= d->plat_data.channels) - continue; - - /* if exclusive access is requested */ - if (flags & DMA_ACCESS_EXCLUSIVE) { - /* ret DMA with no users */ - if (!d->sref) { - dmin = d; - break; - } - } else { - /* get number of users for this DMAC*/ - users = d->sref; - - /* pick DMAC with the least num of users */ - if (users < min_users) { - dmin = d; - min_users = users; - } - } - } - - if (!dmin) { - tr_err(&dma_tr, "No DMAC dir %d caps 0x%x dev 0x%x flags 0x%x", - dir, cap, dev, flags); - - for (d = info->dma_array; - d < info->dma_array + info->num_dmas; - d++) { - tr_err(&dma_tr, " DMAC ID %d users %d busy channels %ld", - d->plat_data.id, d->sref, - atomic_read(&d->num_channels_busy)); - tr_err(&dma_tr, " caps 0x%x dev 0x%x", - d->plat_data.caps, d->plat_data.devs); - } - - return NULL; - } - - /* return DMAC */ - tr_dbg(&dma_tr, "dma_get(), dma-probe id = %d", - dmin->plat_data.id); - - /* Shared DMA controllers with multiple channels - * may be requested many times, let the probe() - * do on-first-use initialization. - */ - key = k_spin_lock(&dmin->lock); - - ret = 0; - if (!dmin->sref) { - ret = dma_probe_legacy(dmin); - if (ret < 0) { - tr_err(&dma_tr, "dma_get(): dma-probe failed id = %d, ret = %d", - dmin->plat_data.id, ret); - } - } - if (!ret) - dmin->sref++; - - tr_info(&dma_tr, "dma_get() ID %d sref = %d busy channels %ld", - dmin->plat_data.id, dmin->sref, - atomic_read(&dmin->num_channels_busy)); - - k_spin_unlock(&dmin->lock, key); - return !ret ? dmin : NULL; -} - -void dma_put(struct dma *dma) -{ - k_spinlock_key_t key; - int ret; - - key = k_spin_lock(&dma->lock); - if (--dma->sref == 0) { - ret = dma_remove_legacy(dma); - if (ret < 0) { - tr_err(&dma_tr, "dma_remove() failed id = %d, ret = %d", - dma->plat_data.id, ret); - } - } - tr_info(&dma_tr, "dma = %p, sref = %d", - dma, dma->sref); - k_spin_unlock(&dma->lock, key); -} -#endif int dma_sg_alloc(struct k_heap *heap, struct dma_sg_elem_array *elem_array,