From 7f14334d055da433521c32c622682f89fe2dd1c5 Mon Sep 17 00:00:00 2001 From: Ashok Sidipotu Date: Thu, 24 Feb 2022 18:02:48 +0530 Subject: [PATCH 11/12] Revert "module-echo-cancel: Move backends to dynamic libaries" This reverts commit 9386c70b3a2cc2df6aabfd7b6a6bc1d7ec873bd1. Upstream-Status: Inappropriate[meson version dependent] --- spa/include/meson.build | 1 - spa/include/spa/utils/names.h | 2 - spa/plugins/aec/aec-null.c | 185 ----------- spa/plugins/aec/aec-webrtc.cpp | 286 ------------------ spa/plugins/aec/meson.build | 16 - spa/plugins/meson.build | 2 - src/modules/meson.build | 9 +- src/modules/module-echo-cancel.c | 83 ++--- .../modules/module-echo-cancel/aec-null.c | 56 ++-- src/modules/module-echo-cancel/aec-webrtc.cpp | 163 ++++++++++ src/modules/module-echo-cancel/echo-cancel.h | 20 +- 11 files changed, 247 insertions(+), 576 deletions(-) delete mode 100644 spa/plugins/aec/aec-null.c delete mode 100644 spa/plugins/aec/aec-webrtc.cpp delete mode 100644 spa/plugins/aec/meson.build rename spa/include/spa/interfaces/audio/aec.h => src/modules/module-echo-cancel/aec-null.c (58%) create mode 100644 src/modules/module-echo-cancel/aec-webrtc.cpp diff --git a/spa/include/meson.build b/spa/include/meson.build index 443db7dca..0566b025d 100644 --- a/spa/include/meson.build +++ b/spa/include/meson.build @@ -3,7 +3,6 @@ spa_sections = [ 'control', 'debug', 'graph', - 'interfaces', 'monitor', 'node', 'param', diff --git a/spa/include/spa/utils/names.h b/spa/include/spa/utils/names.h index 3ece5a861..1e570d098 100644 --- a/spa/include/spa/utils/names.h +++ b/spa/include/spa/utils/names.h @@ -82,8 +82,6 @@ extern "C" { #define SPA_NAME_AUDIO_ADAPT "audio.adapt" /**< combination of a node and an * audio.convert. Does clock slaving */ -#define SPA_NAME_AEC "audio.aec" /**< Echo canceling */ - /** video processing */ #define SPA_NAME_VIDEO_PROCESS_FORMAT "video.process.format" /**< processes raw video from one format * to another */ diff --git a/spa/plugins/aec/aec-null.c b/spa/plugins/aec/aec-null.c deleted file mode 100644 index 3168a6b36..000000000 --- a/spa/plugins/aec/aec-null.c +++ /dev/null @@ -1,185 +0,0 @@ -/* PipeWire - * - * Copyright © 2021 Wim Taymans - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include -#include -#include - -struct impl { - struct spa_handle handle; - struct spa_log *log; - uint32_t channels; -}; - -static struct spa_log_topic log_topic = SPA_LOG_TOPIC(0, "spa.aec.null"); -#undef SPA_LOG_TOPIC_DEFAULT -#define SPA_LOG_TOPIC_DEFAULT &log_topic - -static int null_create(struct spa_handle *handle, const struct spa_dict *args, const struct spa_audio_info_raw *info) -{ - struct impl *impl; - impl = (struct impl *) handle; - impl->channels = info->channels; - - return 0; -} - -static int null_run(struct spa_handle *handle, const float *rec[], const float *play[], float *out[], uint32_t n_samples) -{ - struct impl *impl = (struct impl *) handle; - uint32_t i; - for (i = 0; i < impl->channels; i++) - memcpy(out[i], rec[i], n_samples * sizeof(float)); - return 0; -} - -struct spa_dict *null_get_properties(SPA_UNUSED struct spa_handle *handle) -{ - /* Not supported */ - return NULL; -} - -int null_set_properties(SPA_UNUSED struct spa_handle *handle, SPA_UNUSED const struct spa_dict *args) -{ - /* Not supported */ - return -1; -} - -static struct echo_cancel_info echo_cancel_null_impl = { - .name = "null", - .info = SPA_DICT_INIT(NULL, 0), - .latency = NULL, - .create = null_create, - .run = null_run, - .get_properties = null_get_properties, - .set_properties = null_set_properties, -}; - -static int impl_get_interface(struct spa_handle *handle, const char *type, void **interface) -{ - - spa_return_val_if_fail(handle != NULL, -EINVAL); - spa_return_val_if_fail(interface != NULL, -EINVAL); - - if (spa_streq(type, SPA_TYPE_INTERFACE_AEC)) - *interface = &echo_cancel_null_impl; - else - return -ENOENT; - - return 0; -} - -static int impl_clear(struct spa_handle *handle) -{ - spa_return_val_if_fail(handle != NULL, -EINVAL); - - return 0; -} - -static size_t -impl_get_size(const struct spa_handle_factory *factory, - const struct spa_dict *params) -{ - return sizeof(struct impl); -} - -static int -impl_init(const struct spa_handle_factory *factory, - struct spa_handle *handle, - const struct spa_dict *info, - const struct spa_support *support, - uint32_t n_support) -{ - struct impl *impl; - - spa_return_val_if_fail(factory != NULL, -EINVAL); - spa_return_val_if_fail(handle != NULL, -EINVAL); - - echo_cancel_null_impl.iface = SPA_INTERFACE_INIT( - SPA_TYPE_INTERFACE_AEC, - SPA_VERSION_AUDIO_AEC, - NULL, - NULL); - - handle->get_interface = impl_get_interface; - handle->clear = impl_clear; - impl = (struct impl *) handle; - impl->log = (struct spa_log*)spa_support_find(support, n_support, SPA_TYPE_INTERFACE_Log); - spa_log_topic_init(impl->log, &log_topic); - - return 0; -} - -static const struct spa_interface_info impl_interfaces[] = { - {SPA_TYPE_INTERFACE_AEC,}, -}; - -static int -impl_enum_interface_info(const struct spa_handle_factory *factory, - const struct spa_interface_info **info, - uint32_t *index) -{ - spa_return_val_if_fail(factory != NULL, -EINVAL); - spa_return_val_if_fail(info != NULL, -EINVAL); - spa_return_val_if_fail(index != NULL, -EINVAL); - - switch (*index) { - case 0: - *info = &impl_interfaces[*index]; - break; - default: - return 0; - } - (*index)++; - return 1; -} - -const struct spa_handle_factory spa_aec_exaudio_factory = { - SPA_VERSION_HANDLE_FACTORY, - SPA_NAME_AEC, - NULL, - impl_get_size, - impl_init, - impl_enum_interface_info, -}; - - -SPA_EXPORT -int spa_handle_factory_enum(const struct spa_handle_factory **factory, uint32_t *index) -{ - spa_return_val_if_fail(factory != NULL, -EINVAL); - spa_return_val_if_fail(index != NULL, -EINVAL); - - switch (*index) { - case 0: - *factory = &spa_aec_exaudio_factory; - break; - default: - return 0; - } - (*index)++; - return 1; -} diff --git a/spa/plugins/aec/aec-webrtc.cpp b/spa/plugins/aec/aec-webrtc.cpp deleted file mode 100644 index d44fa6e30..000000000 --- a/spa/plugins/aec/aec-webrtc.cpp +++ /dev/null @@ -1,286 +0,0 @@ -/* PipeWire - * - * Copyright © 2021 Wim Taymans - * © 2021 Arun Raghavan - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -#include -#include -#include -#include -#include - -#include -#include -#include - -struct impl_data { - struct spa_handle handle; - struct spa_log *log; - std::unique_ptr apm; - spa_audio_info_raw info; - std::unique_ptr play_buffer, rec_buffer, out_buffer; -}; - -static struct spa_log_topic log_topic = SPA_LOG_TOPIC(0, "spa.eac.webrtc"); -#undef SPA_LOG_TOPIC_DEFAULT -#define SPA_LOG_TOPIC_DEFAULT &log_topic - -static bool webrtc_get_spa_bool(const struct spa_dict *args, const char *key, bool default_value) { - const char *str_val; - bool value = default_value; - str_val = spa_dict_lookup(args, key); - if (str_val != NULL) - value =spa_atob(str_val); - - return value; -} - -static int webrtc_create(struct spa_handle *handle, const struct spa_dict *args, const struct spa_audio_info_raw *info) -{ - auto impl = reinterpret_cast(handle); - - bool extended_filter = webrtc_get_spa_bool(args, "webrtc.extended_filter", true); - bool delay_agnostic = webrtc_get_spa_bool(args, "webrtc.delay_agnostic", true); - bool high_pass_filter = webrtc_get_spa_bool(args, "webrtc.high_pass_filter", true); - bool noise_suppression = webrtc_get_spa_bool(args, "webrtc.noise_suppression", true); - bool voice_detection = webrtc_get_spa_bool(args, "webrtc.voice_detection", true); - - // Note: AGC seems to mess up with Agnostic Delay Detection, especially with speech, - // result in very poor performance, disable by default - bool gain_control = webrtc_get_spa_bool(args, "webrtc.gain_control", false); - - // Disable experimental flags by default - bool experimental_agc = webrtc_get_spa_bool(args, "webrtc.experimental_agc", false); - bool experimental_ns = webrtc_get_spa_bool(args, "webrtc.experimental_ns", false); - - // FIXME: Intelligibility enhancer is not currently supported - // This filter will modify playback buffer (when calling ProcessReverseStream), but now - // playback buffer modifications are discarded. - - webrtc::Config config; - config.Set(new webrtc::ExtendedFilter(extended_filter)); - config.Set(new webrtc::DelayAgnostic(delay_agnostic)); - config.Set(new webrtc::ExperimentalAgc(experimental_agc)); - config.Set(new webrtc::ExperimentalNs(experimental_ns)); - - webrtc::ProcessingConfig pconfig = {{ - webrtc::StreamConfig(info->rate, info->channels, false), /* input stream */ - webrtc::StreamConfig(info->rate, info->channels, false), /* output stream */ - webrtc::StreamConfig(info->rate, info->channels, false), /* reverse input stream */ - webrtc::StreamConfig(info->rate, info->channels, false), /* reverse output stream */ - }}; - - auto apm = std::unique_ptr(webrtc::AudioProcessing::Create(config)); - if (apm->Initialize(pconfig) != webrtc::AudioProcessing::kNoError) { - spa_log_error(impl->log, "Error initialising webrtc audio processing module"); - return -1; - } - - apm->high_pass_filter()->Enable(high_pass_filter); - // Always disable drift compensation since it requires drift sampling - apm->echo_cancellation()->enable_drift_compensation(false); - apm->echo_cancellation()->Enable(true); - // TODO: wire up supression levels to args - apm->echo_cancellation()->set_suppression_level(webrtc::EchoCancellation::kHighSuppression); - apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kHigh); - apm->noise_suppression()->Enable(noise_suppression); - apm->voice_detection()->Enable(voice_detection); - // TODO: wire up AGC parameters to args - apm->gain_control()->set_analog_level_limits(0, 255); - apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveDigital); - apm->gain_control()->Enable(gain_control); - impl->apm = std::move(apm); - impl->info = *info; - impl->play_buffer = std::make_unique(info->channels); - impl->rec_buffer = std::make_unique(info->channels); - impl->out_buffer = std::make_unique(info->channels); - return 0; -} - -static int webrtc_run(struct spa_handle *handle, const float *rec[], const float *play[], float *out[], uint32_t n_samples) -{ - auto impl = reinterpret_cast(handle); - webrtc::StreamConfig config = - webrtc::StreamConfig(impl->info.rate, impl->info.channels, false); - unsigned int num_blocks = n_samples * 1000 / impl->info.rate / 10; - - if (n_samples * 1000 / impl->info.rate % 10 != 0) { - spa_log_error(impl->log, "Buffers must be multiples of 10ms in length (currently %u samples)", n_samples); - return -1; - } - - for (size_t i = 0; i < num_blocks; i ++) { - for (size_t j = 0; j < impl->info.channels; j++) { - impl->play_buffer[j] = const_cast(play[j]) + config.num_frames() * i; - impl->rec_buffer[j] = const_cast(rec[j]) + config.num_frames() * i; - impl->out_buffer[j] = out[j] + config.num_frames() * i; - } - /* FIXME: ProcessReverseStream may change the playback buffer, in which - * case we should use that, if we ever expose the intelligibility - * enhancer */ - if (impl->apm->ProcessReverseStream(impl->play_buffer.get(), config, config, impl->play_buffer.get()) != - webrtc::AudioProcessing::kNoError) { - spa_log_error(impl->log, "Processing reverse stream failed"); - } - - // Extra delay introduced by multiple frames - impl->apm->set_stream_delay_ms((num_blocks - 1) * 10); - - if (impl->apm->ProcessStream(impl->rec_buffer.get(), config, config, impl->out_buffer.get()) != - webrtc::AudioProcessing::kNoError) { - spa_log_error(impl->log, "Processing stream failed"); - } - } - - return 0; -} - -struct spa_dict *webrtc_get_properties(SPA_UNUSED struct spa_handle *handle) -{ - /* Not supported */ - return NULL; -} - -int webrtc_set_properties(SPA_UNUSED struct spa_handle *handle, SPA_UNUSED const struct spa_dict *args) -{ - /* Not supported */ - return -1; -} - -static struct echo_cancel_info echo_cancel_webrtc_impl = { - .name = "webrtc", - .info = SPA_DICT_INIT(NULL, 0), - .latency = "480/48000", - .create = webrtc_create, - .run = webrtc_run, - .get_properties = webrtc_get_properties, - .set_properties = webrtc_set_properties, -}; - -static int impl_get_interface(struct spa_handle *handle, const char *type, void **interface) -{ - - spa_return_val_if_fail(handle != NULL, -EINVAL); - spa_return_val_if_fail(interface != NULL, -EINVAL); - - if (spa_streq(type, SPA_TYPE_INTERFACE_AEC)) - *interface = &echo_cancel_webrtc_impl; - else - return -ENOENT; - - return 0; -} - -static int impl_clear(struct spa_handle *handle) -{ - spa_return_val_if_fail(handle != NULL, -EINVAL); - auto impl = reinterpret_cast(handle); - impl->~impl_data(); - return 0; -} - -static size_t -impl_get_size(const struct spa_handle_factory *factory, - const struct spa_dict *params) -{ - return sizeof(struct impl_data); -} - -static int -impl_init(const struct spa_handle_factory *factory, - struct spa_handle *handle, - const struct spa_dict *info, - const struct spa_support *support, - uint32_t n_support) -{ - spa_return_val_if_fail(factory != NULL, -EINVAL); - spa_return_val_if_fail(handle != NULL, -EINVAL); - - echo_cancel_webrtc_impl.iface = SPA_INTERFACE_INIT( - SPA_TYPE_INTERFACE_AEC, - SPA_VERSION_AUDIO_AEC, - NULL, - NULL); - - auto impl = new (handle) impl_data(); - impl->handle.get_interface = impl_get_interface; - impl->handle.clear = impl_clear; - impl->log = (struct spa_log*)spa_support_find(support, n_support, SPA_TYPE_INTERFACE_Log); - spa_log_topic_init(impl->log, &log_topic); - - return 0; -} - -static const struct spa_interface_info impl_interfaces[] = { - {SPA_TYPE_INTERFACE_AEC,}, -}; - -static int -impl_enum_interface_info(const struct spa_handle_factory *factory, - const struct spa_interface_info **info, - uint32_t *index) -{ - spa_return_val_if_fail(factory != NULL, -EINVAL); - spa_return_val_if_fail(info != NULL, -EINVAL); - spa_return_val_if_fail(index != NULL, -EINVAL); - - switch (*index) { - case 0: - *info = &impl_interfaces[*index]; - break; - default: - return 0; - } - (*index)++; - return 1; -} - -const struct spa_handle_factory spa_aec_exaudio_factory = { - SPA_VERSION_HANDLE_FACTORY, - SPA_NAME_AEC, - NULL, - impl_get_size, - impl_init, - impl_enum_interface_info, -}; - - -SPA_EXPORT -int spa_handle_factory_enum(const struct spa_handle_factory **factory, uint32_t *index) -{ - spa_return_val_if_fail(factory != NULL, -EINVAL); - spa_return_val_if_fail(index != NULL, -EINVAL); - - switch (*index) { - case 0: - *factory = &spa_aec_exaudio_factory; - break; - default: - return 0; - } - (*index)++; - return 1; -} diff --git a/spa/plugins/aec/meson.build b/spa/plugins/aec/meson.build deleted file mode 100644 index 2b1a2c05a..000000000 --- a/spa/plugins/aec/meson.build +++ /dev/null @@ -1,16 +0,0 @@ -aec_null = shared_library('spa-aec-null', - [ 'aec-null.c' ], - include_directories : [ configinc ], - dependencies : [ spa_dep ], - install : true, - install_dir : spa_plugindir / 'aec') - -if webrtc_dep.found() - aec_webrtc = shared_library('spa-aec-webrtc', - [ 'aec-webrtc.cpp' ], - include_directories : [ configinc ], - dependencies : [ spa_dep, webrtc_dep ], - install : true, - install_dir : spa_plugindir / 'aec') -endif - diff --git a/spa/plugins/meson.build b/spa/plugins/meson.build index 0b581b29b..fd229d460 100644 --- a/spa/plugins/meson.build +++ b/spa/plugins/meson.build @@ -51,5 +51,3 @@ endif if libcamera_dep.found() subdir('libcamera') endif - -subdir('aec') \ No newline at end of file diff --git a/src/modules/meson.build b/src/modules/meson.build index bd7d3f711..568b32c50 100644 --- a/src/modules/meson.build +++ b/src/modules/meson.build @@ -110,15 +110,22 @@ pipewire_module_filter_chain = shared_library('pipewire-module-filter-chain', pipewire_module_echo_cancel_sources = [ 'module-echo-cancel.c', + 'module-echo-cancel/aec-null.c', ] +if webrtc_dep.found() + pipewire_module_echo_cancel_sources += [ + 'module-echo-cancel/aec-webrtc.cpp' + ] +endif + pipewire_module_echo_cancel = shared_library('pipewire-module-echo-cancel', pipewire_module_echo_cancel_sources, include_directories : [configinc, spa_inc], install : true, install_dir : modules_install_dir, install_rpath: modules_install_dir, - dependencies : [mathlib, dl_lib, pipewire_dep], + dependencies : [mathlib, dl_lib, pipewire_dep, webrtc_dep], ) pipewire_module_profiler = shared_library('pipewire-module-profiler', diff --git a/src/modules/module-echo-cancel.c b/src/modules/module-echo-cancel.c index 728325938..00adbbfc5 100644 --- a/src/modules/module-echo-cancel.c +++ b/src/modules/module-echo-cancel.c @@ -24,7 +24,6 @@ */ #include "config.h" -#include "module-echo-cancel/echo-cancel.h" #include #include @@ -44,14 +43,10 @@ #include #include #include -#include #include -#include #include #include #include -#include -#include #include #include @@ -59,6 +54,8 @@ #include +#include "module-echo-cancel/echo-cancel.h" + /** \page page_module_echo_cancel PipeWire Module: Echo Cancel * * The `echo-cancel` module performs echo cancellation. The module creates @@ -71,8 +68,8 @@ * * - `source.props = {}`: properties to be passed to the source stream * - `sink.props = {}`: properties to be passed to the sink stream - * - `library.name = `: the echo cancellation library Currently supported: - * `aec/libspa-aec-exaudio`. Leave unset to use the default method (`aec/libspa-aec-exaudio`). + * - `aec.method = `: the echo cancellation method. Currently supported: + * `webrtc`. Leave unset to use the default method (`webrtc`). * - `aec.args = `: arguments to pass to the echo cancellation method * * ## General options @@ -97,7 +94,7 @@ * context.modules = [ * { name = libpipewire-module-echo-cancel * args = { - * # library.name = aec/libspa-aec-exaudio + * # aec.method = webrtc * # node.latency = 1024/48000 * source.props = { * node.name = "Echo Cancellation Source" @@ -141,7 +138,7 @@ static const struct spa_dict_item module_props[] = { "[ audio.position= ] " "[ buffer.max_size= ] " "[ buffer.play_delay= ] " - "[ library.name = ] " + "[ aec.method= ] " "[ aec.args= ] " "[ source.props= ] " "[ sink.props= ] " }, @@ -189,6 +186,7 @@ struct impl { struct spa_ringbuffer out_ring; const struct echo_cancel_info *aec_info; + void *aec; uint32_t aec_blocksize; unsigned int capture_ready:1; @@ -199,9 +197,6 @@ struct impl { uint32_t max_buffer_size; uint32_t buffer_delay; - - struct spa_handle *spa_handle; - struct spa_plugin_loader *loader; }; static void do_unload_module(void *obj, void *data, int res, uint32_t id) @@ -288,7 +283,7 @@ static void process(struct impl *impl) pw_stream_queue_buffer(impl->playback, pout); /* Now run the canceller */ - echo_cancel_run(impl->aec_info, impl->spa_handle, rec, play_delayed, out, size / sizeof(float)); + echo_cancel_run(impl->aec_info, impl->aec, rec, play_delayed, out, size / sizeof(float)); /* Next, copy over the output to the output ringbuffer */ avail = spa_ringbuffer_get_write_index(&impl->out_ring, &oindex); @@ -808,8 +803,8 @@ static void impl_destroy(struct impl *impl) pw_stream_destroy(impl->sink); if (impl->core && impl->do_disconnect) pw_core_disconnect(impl->core); - if (impl->spa_handle) - spa_plugin_loader_unload(impl->loader, impl->spa_handle); + if (impl->aec) + echo_cancel_destroy(impl->aec_info, impl->aec); pw_properties_free(impl->source_props); pw_properties_free(impl->sink_props); @@ -898,10 +893,7 @@ int pipewire__module_init(struct pw_impl_module *module, const char *args) struct impl *impl; uint32_t id = pw_global_get_id(pw_impl_module_get_global(module)); const char *str; - const char *path; - int res = 0; - struct spa_handle *handle = NULL; - void *iface; + int res; PW_LOG_TOPIC_INIT(mod_topic); @@ -975,57 +967,22 @@ int pipewire__module_init(struct pw_impl_module *module, const char *args) if (pw_properties_get(impl->sink_props, PW_KEY_MEDIA_CLASS) == NULL) pw_properties_set(impl->sink_props, PW_KEY_MEDIA_CLASS, "Audio/Sink"); - if ((str = pw_properties_get(props, "aec.method")) != NULL) - pw_log_warn("aec.method is not supported anymore use library.name"); - - /* Use webrtc as default */ - if ((path = pw_properties_get(props, "library.name")) == NULL) - path = "aec/libspa-aec-webrtc"; - - struct spa_dict_item info_items[] = { - { SPA_KEY_LIBRARY_NAME, path }, - }; - struct spa_dict info = SPA_DICT_INIT_ARRAY(info_items); - - impl->loader = spa_support_find(context->support, context->n_support, SPA_TYPE_INTERFACE_PluginLoader); - if (impl->loader == NULL) { - pw_log_error("a plugin loader is needed"); - return -EINVAL; - } - - handle = spa_plugin_loader_load(impl->loader, SPA_NAME_AEC, &info); - if (handle == NULL) { - pw_log_error("AEC codec plugin %s not available library.name %s", SPA_NAME_AEC, path); - return -ENOENT; - } - - if ((res = spa_handle_get_interface(handle, SPA_TYPE_INTERFACE_AEC, &iface)) < 0) { - pw_log_error("can't get %s interface %d", SPA_TYPE_INTERFACE_AEC, res); - return res; - } - impl->aec_info = iface; - impl->spa_handle = handle; - if (impl->aec_info->iface.version != SPA_VERSION_AUDIO_AEC) { - pw_log_error("codec plugin %s has incompatible ABI version (%d != %d)", - SPA_NAME_AEC, impl->aec_info->iface.version, SPA_VERSION_AUDIO_AEC); - res = -ENOENT; - goto error; - } - - (void)SPA_SUPPORT_INIT(SPA_TYPE_INTERFACE_AEC, (struct echo_cancel_info *)impl->aec_info); + if ((str = pw_properties_get(props, "aec.method")) == NULL) + str = "webrtc"; - pw_log_info("Using plugin AEC %s", impl->aec_info->name); +#ifdef HAVE_WEBRTC + if (spa_streq(str, "webrtc")) + impl->aec_info = echo_cancel_webrtc; + else +#endif + impl->aec_info = echo_cancel_null; if ((str = pw_properties_get(props, "aec.args")) != NULL) aec_props = pw_properties_new_string(str); else aec_props = pw_properties_new(NULL, NULL); - if (echo_cancel_create(impl->aec_info, impl->spa_handle, &aec_props->dict, &impl->info)) { - pw_log_error("codec plugin %s create failed", impl->aec_info->name); - res = -ENOENT; - goto error; - } + impl->aec = echo_cancel_create(impl->aec_info, aec_props, &impl->info); pw_properties_free(aec_props); diff --git a/spa/include/spa/interfaces/audio/aec.h b/src/modules/module-echo-cancel/aec-null.c similarity index 58% rename from spa/include/spa/interfaces/audio/aec.h rename to src/modules/module-echo-cancel/aec-null.c index e1fcda563..4ee9ac6d7 100644 --- a/spa/include/spa/interfaces/audio/aec.h +++ b/src/modules/module-echo-cancel/aec-null.c @@ -22,23 +22,43 @@ * DEALINGS IN THE SOFTWARE. */ +#include "echo-cancel.h" -#include -#include -#include -#include -#include - -#define SPA_TYPE_INTERFACE_AEC SPA_TYPE_INFO_INTERFACE_BASE "AEC" -#define SPA_VERSION_AUDIO_AEC 1 - -struct echo_cancel_info { - struct spa_interface iface; - const char *name; - const struct spa_dict info; - const char *latency; - int (*create) (struct spa_handle *handle, const struct spa_dict *args, const struct spa_audio_info_raw *info); - int (*run) (struct spa_handle *handle, const float *rec[], const float *play[], float *out[], uint32_t n_samples); - struct spa_dict *(*get_properties) (struct spa_handle *handle); - int (*set_properties) (struct spa_handle *handle, const struct spa_dict *args); +struct impl { + uint32_t channels; }; + +static void *null_create(const struct pw_properties *args, const struct spa_audio_info_raw *info) +{ + struct impl *impl; + impl = calloc(1, sizeof(struct impl)); + impl->channels = info->channels; + return impl; +} + +static void null_destroy(void *ec) +{ + free(ec); +} + +static int null_run(void *ec, const float *rec[], const float *play[], float *out[], uint32_t n_samples) +{ + struct impl *impl = ec; + uint32_t i; + for (i = 0; i < impl->channels; i++) + memcpy(out[i], rec[i], n_samples * sizeof(float)); + return 0; +} + +static const struct echo_cancel_info echo_cancel_null_impl = { + .name = "null", + .info = SPA_DICT_INIT(NULL, 0), + .latency = NULL, + + .create = null_create, + .destroy = null_destroy, + + .run = null_run, +}; + +const struct echo_cancel_info *echo_cancel_null = &echo_cancel_null_impl; diff --git a/src/modules/module-echo-cancel/aec-webrtc.cpp b/src/modules/module-echo-cancel/aec-webrtc.cpp new file mode 100644 index 000000000..77b569a22 --- /dev/null +++ b/src/modules/module-echo-cancel/aec-webrtc.cpp @@ -0,0 +1,163 @@ +/* PipeWire + * + * Copyright © 2021 Wim Taymans + * © 2021 Arun Raghavan + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +#include "echo-cancel.h" + +#include + +#include +#include +#include + +struct impl { + std::unique_ptr apm; + spa_audio_info_raw info; + std::unique_ptr play_buffer, rec_buffer, out_buffer; + + impl(std::unique_ptr apm, const spa_audio_info_raw& info) + : apm(std::move(apm)), + info(info), + play_buffer(std::make_unique(info.channels)), + rec_buffer(std::make_unique(info.channels)), + out_buffer(std::make_unique(info.channels)) + { } +}; + +static void *webrtc_create(const struct pw_properties *args, const spa_audio_info_raw *info) +{ + bool extended_filter = pw_properties_get_bool(args, "webrtc.extended_filter", true); + bool delay_agnostic = pw_properties_get_bool(args, "webrtc.delay_agnostic", true); + bool high_pass_filter = pw_properties_get_bool(args, "webrtc.high_pass_filter", true); + bool noise_suppression = pw_properties_get_bool(args, "webrtc.noise_suppression", true); + bool voice_detection = pw_properties_get_bool(args, "webrtc.voice_detection", true); + + // Note: AGC seems to mess up with Agnostic Delay Detection, especially with speech, + // result in very poor performance, disable by default + bool gain_control = pw_properties_get_bool(args, "webrtc.gain_control", false); + + // Disable experimental flags by default + bool experimental_agc = pw_properties_get_bool(args, "webrtc.experimental_agc", false); + bool experimental_ns = pw_properties_get_bool(args, "webrtc.experimental_ns", false); + + // FIXME: Intelligibility enhancer is not currently supported + // This filter will modify playback buffer (when calling ProcessReverseStream), but now + // playback buffer modifications are discarded. + + webrtc::Config config; + config.Set(new webrtc::ExtendedFilter(extended_filter)); + config.Set(new webrtc::DelayAgnostic(delay_agnostic)); + config.Set(new webrtc::ExperimentalAgc(experimental_agc)); + config.Set(new webrtc::ExperimentalNs(experimental_ns)); + + webrtc::ProcessingConfig pconfig = {{ + webrtc::StreamConfig(info->rate, info->channels, false), /* input stream */ + webrtc::StreamConfig(info->rate, info->channels, false), /* output stream */ + webrtc::StreamConfig(info->rate, info->channels, false), /* reverse input stream */ + webrtc::StreamConfig(info->rate, info->channels, false), /* reverse output stream */ + }}; + + auto apm = std::unique_ptr(webrtc::AudioProcessing::Create(config)); + if (apm->Initialize(pconfig) != webrtc::AudioProcessing::kNoError) { + pw_log_error("Error initialising webrtc audio processing module"); + return nullptr; + } + + apm->high_pass_filter()->Enable(high_pass_filter); + // Always disable drift compensation since it requires drift sampling + apm->echo_cancellation()->enable_drift_compensation(false); + apm->echo_cancellation()->Enable(true); + // TODO: wire up supression levels to args + apm->echo_cancellation()->set_suppression_level(webrtc::EchoCancellation::kHighSuppression); + apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kHigh); + apm->noise_suppression()->Enable(noise_suppression); + apm->voice_detection()->Enable(voice_detection); + // TODO: wire up AGC parameters to args + apm->gain_control()->set_analog_level_limits(0, 255); + apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveDigital); + apm->gain_control()->Enable(gain_control); + + return new impl(std::move(apm), *info); +} + +static void webrtc_destroy(void *ec) +{ + auto impl = static_cast(ec); + + delete impl; +} + +static int webrtc_run(void *ec, const float *rec[], const float *play[], float *out[], uint32_t n_samples) +{ + auto impl = static_cast(ec); + webrtc::StreamConfig config = + webrtc::StreamConfig(impl->info.rate, impl->info.channels, false); + unsigned int num_blocks = n_samples * 1000 / impl->info.rate / 10; + + if (n_samples * 1000 / impl->info.rate % 10 != 0) { + pw_log_error("Buffers must be multiples of 10ms in length (currently %u samples)", n_samples); + return -1; + } + + for (size_t i = 0; i < num_blocks; i ++) { + for (size_t j = 0; j < impl->info.channels; j++) { + impl->play_buffer[j] = const_cast(play[j]) + config.num_frames() * i; + impl->rec_buffer[j] = const_cast(rec[j]) + config.num_frames() * i; + impl->out_buffer[j] = out[j] + config.num_frames() * i; + } + /* FIXME: ProcessReverseStream may change the playback buffer, in which + * case we should use that, if we ever expose the intelligibility + * enhancer */ + if (impl->apm->ProcessReverseStream(impl->play_buffer.get(), config, config, impl->play_buffer.get()) != + webrtc::AudioProcessing::kNoError) { + pw_log_error("Processing reverse stream failed"); + } + + // Extra delay introduced by multiple frames + impl->apm->set_stream_delay_ms((num_blocks - 1) * 10); + + if (impl->apm->ProcessStream(impl->rec_buffer.get(), config, config, impl->out_buffer.get()) != + webrtc::AudioProcessing::kNoError) { + pw_log_error("Processing stream failed"); + } + } + + return 0; +} + +static const struct echo_cancel_info echo_cancel_webrtc_impl = { + .name = "webrtc", + .info = SPA_DICT_INIT(NULL, 0), + .latency = "480/48000", + + .create = webrtc_create, + .destroy = webrtc_destroy, + + .run = webrtc_run, +}; + +const struct echo_cancel_info *echo_cancel_webrtc = &echo_cancel_webrtc_impl; diff --git a/src/modules/module-echo-cancel/echo-cancel.h b/src/modules/module-echo-cancel/echo-cancel.h index ac83f70e4..fe011b962 100644 --- a/src/modules/module-echo-cancel/echo-cancel.h +++ b/src/modules/module-echo-cancel/echo-cancel.h @@ -22,13 +22,29 @@ * DEALINGS IN THE SOFTWARE. */ +#include "config.h" #include -#include #include -#include #include +struct echo_cancel_info { + const char *name; + const struct spa_dict info; + const char *latency; + + void *(*create) (const struct pw_properties *args, const struct spa_audio_info_raw *info); + void (*destroy) (void *ec); + + int (*run) (void *ec, const float *rec[], const float *play[], float *out[], uint32_t n_samples); +}; + #define echo_cancel_create(i,...) (i)->create(__VA_ARGS__) +#define echo_cancel_destroy(i,...) (i)->destroy(__VA_ARGS__) #define echo_cancel_run(i,...) (i)->run(__VA_ARGS__) + +#ifdef HAVE_WEBRTC +extern const struct echo_cancel_info *echo_cancel_webrtc; +#endif +extern const struct echo_cancel_info *echo_cancel_null; -- 2.35.1