1 From 7f14334d055da433521c32c622682f89fe2dd1c5 Mon Sep 17 00:00:00 2001
2 From: Ashok Sidipotu <ashok.sidipotu@collabora.com>
3 Date: Thu, 24 Feb 2022 18:02:48 +0530
4 Subject: [PATCH 11/12] Revert "module-echo-cancel: Move backends to dynamic
7 This reverts commit 9386c70b3a2cc2df6aabfd7b6a6bc1d7ec873bd1.
8 Upstream-Status: Inappropriate[meson version dependent]
10 spa/include/meson.build | 1 -
11 spa/include/spa/utils/names.h | 2 -
12 spa/plugins/aec/aec-null.c | 185 -----------
13 spa/plugins/aec/aec-webrtc.cpp | 286 ------------------
14 spa/plugins/aec/meson.build | 16 -
15 spa/plugins/meson.build | 2 -
16 src/modules/meson.build | 9 +-
17 src/modules/module-echo-cancel.c | 83 ++---
18 .../modules/module-echo-cancel/aec-null.c | 56 ++--
19 src/modules/module-echo-cancel/aec-webrtc.cpp | 163 ++++++++++
20 src/modules/module-echo-cancel/echo-cancel.h | 20 +-
21 11 files changed, 247 insertions(+), 576 deletions(-)
22 delete mode 100644 spa/plugins/aec/aec-null.c
23 delete mode 100644 spa/plugins/aec/aec-webrtc.cpp
24 delete mode 100644 spa/plugins/aec/meson.build
25 rename spa/include/spa/interfaces/audio/aec.h => src/modules/module-echo-cancel/aec-null.c (58%)
26 create mode 100644 src/modules/module-echo-cancel/aec-webrtc.cpp
28 diff --git a/spa/include/meson.build b/spa/include/meson.build
29 index 443db7dca..0566b025d 100644
30 --- a/spa/include/meson.build
31 +++ b/spa/include/meson.build
32 @@ -3,7 +3,6 @@ spa_sections = [
40 diff --git a/spa/include/spa/utils/names.h b/spa/include/spa/utils/names.h
41 index 3ece5a861..1e570d098 100644
42 --- a/spa/include/spa/utils/names.h
43 +++ b/spa/include/spa/utils/names.h
44 @@ -82,8 +82,6 @@ extern "C" {
45 #define SPA_NAME_AUDIO_ADAPT "audio.adapt" /**< combination of a node and an
46 * audio.convert. Does clock slaving */
48 -#define SPA_NAME_AEC "audio.aec" /**< Echo canceling */
50 /** video processing */
51 #define SPA_NAME_VIDEO_PROCESS_FORMAT "video.process.format" /**< processes raw video from one format
53 diff --git a/spa/plugins/aec/aec-null.c b/spa/plugins/aec/aec-null.c
54 deleted file mode 100644
55 index 3168a6b36..000000000
56 --- a/spa/plugins/aec/aec-null.c
61 - * Copyright © 2021 Wim Taymans <wim.taymans@gmail.com>
63 - * Permission is hereby granted, free of charge, to any person obtaining a
64 - * copy of this software and associated documentation files (the "Software"),
65 - * to deal in the Software without restriction, including without limitation
66 - * the rights to use, copy, modify, merge, publish, distribute, sublicense,
67 - * and/or sell copies of the Software, and to permit persons to whom the
68 - * Software is furnished to do so, subject to the following conditions:
70 - * The above copyright notice and this permission notice (including the next
71 - * paragraph) shall be included in all copies or substantial portions of the
74 - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
75 - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
76 - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
77 - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
78 - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
79 - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
80 - * DEALINGS IN THE SOFTWARE.
83 -#include <spa/interfaces/audio/aec.h>
84 -#include <spa/support/log.h>
85 -#include <spa/utils/string.h>
86 -#include <spa/utils/names.h>
87 -#include <spa/support/plugin.h>
90 - struct spa_handle handle;
91 - struct spa_log *log;
95 -static struct spa_log_topic log_topic = SPA_LOG_TOPIC(0, "spa.aec.null");
96 -#undef SPA_LOG_TOPIC_DEFAULT
97 -#define SPA_LOG_TOPIC_DEFAULT &log_topic
99 -static int null_create(struct spa_handle *handle, const struct spa_dict *args, const struct spa_audio_info_raw *info)
102 - impl = (struct impl *) handle;
103 - impl->channels = info->channels;
108 -static int null_run(struct spa_handle *handle, const float *rec[], const float *play[], float *out[], uint32_t n_samples)
110 - struct impl *impl = (struct impl *) handle;
112 - for (i = 0; i < impl->channels; i++)
113 - memcpy(out[i], rec[i], n_samples * sizeof(float));
117 -struct spa_dict *null_get_properties(SPA_UNUSED struct spa_handle *handle)
119 - /* Not supported */
123 -int null_set_properties(SPA_UNUSED struct spa_handle *handle, SPA_UNUSED const struct spa_dict *args)
125 - /* Not supported */
129 -static struct echo_cancel_info echo_cancel_null_impl = {
131 - .info = SPA_DICT_INIT(NULL, 0),
133 - .create = null_create,
135 - .get_properties = null_get_properties,
136 - .set_properties = null_set_properties,
139 -static int impl_get_interface(struct spa_handle *handle, const char *type, void **interface)
142 - spa_return_val_if_fail(handle != NULL, -EINVAL);
143 - spa_return_val_if_fail(interface != NULL, -EINVAL);
145 - if (spa_streq(type, SPA_TYPE_INTERFACE_AEC))
146 - *interface = &echo_cancel_null_impl;
153 -static int impl_clear(struct spa_handle *handle)
155 - spa_return_val_if_fail(handle != NULL, -EINVAL);
161 -impl_get_size(const struct spa_handle_factory *factory,
162 - const struct spa_dict *params)
164 - return sizeof(struct impl);
168 -impl_init(const struct spa_handle_factory *factory,
169 - struct spa_handle *handle,
170 - const struct spa_dict *info,
171 - const struct spa_support *support,
172 - uint32_t n_support)
176 - spa_return_val_if_fail(factory != NULL, -EINVAL);
177 - spa_return_val_if_fail(handle != NULL, -EINVAL);
179 - echo_cancel_null_impl.iface = SPA_INTERFACE_INIT(
180 - SPA_TYPE_INTERFACE_AEC,
181 - SPA_VERSION_AUDIO_AEC,
185 - handle->get_interface = impl_get_interface;
186 - handle->clear = impl_clear;
187 - impl = (struct impl *) handle;
188 - impl->log = (struct spa_log*)spa_support_find(support, n_support, SPA_TYPE_INTERFACE_Log);
189 - spa_log_topic_init(impl->log, &log_topic);
194 -static const struct spa_interface_info impl_interfaces[] = {
195 - {SPA_TYPE_INTERFACE_AEC,},
199 -impl_enum_interface_info(const struct spa_handle_factory *factory,
200 - const struct spa_interface_info **info,
203 - spa_return_val_if_fail(factory != NULL, -EINVAL);
204 - spa_return_val_if_fail(info != NULL, -EINVAL);
205 - spa_return_val_if_fail(index != NULL, -EINVAL);
209 - *info = &impl_interfaces[*index];
218 -const struct spa_handle_factory spa_aec_exaudio_factory = {
219 - SPA_VERSION_HANDLE_FACTORY,
224 - impl_enum_interface_info,
229 -int spa_handle_factory_enum(const struct spa_handle_factory **factory, uint32_t *index)
231 - spa_return_val_if_fail(factory != NULL, -EINVAL);
232 - spa_return_val_if_fail(index != NULL, -EINVAL);
236 - *factory = &spa_aec_exaudio_factory;
244 diff --git a/spa/plugins/aec/aec-webrtc.cpp b/spa/plugins/aec/aec-webrtc.cpp
245 deleted file mode 100644
246 index d44fa6e30..000000000
247 --- a/spa/plugins/aec/aec-webrtc.cpp
252 - * Copyright © 2021 Wim Taymans <wim.taymans@gmail.com>
253 - * © 2021 Arun Raghavan <arun@asymptotic.io>
255 - * Permission is hereby granted, free of charge, to any person obtaining a
256 - * copy of this software and associated documentation files (the "Software"),
257 - * to deal in the Software without restriction, including without limitation
258 - * the rights to use, copy, modify, merge, publish, distribute, sublicense,
259 - * and/or sell copies of the Software, and to permit persons to whom the
260 - * Software is furnished to do so, subject to the following conditions:
262 - * The above copyright notice and this permission notice (including the next
263 - * paragraph) shall be included in all copies or substantial portions of the
266 - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
267 - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
268 - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
269 - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
270 - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
271 - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
272 - * DEALINGS IN THE SOFTWARE.
278 -#include <spa/interfaces/audio/aec.h>
279 -#include <spa/support/log.h>
280 -#include <spa/utils/string.h>
281 -#include <spa/utils/names.h>
282 -#include <spa/support/plugin.h>
284 -#include <webrtc/modules/audio_processing/include/audio_processing.h>
285 -#include <webrtc/modules/interface/module_common_types.h>
286 -#include <webrtc/system_wrappers/include/trace.h>
289 - struct spa_handle handle;
290 - struct spa_log *log;
291 - std::unique_ptr<webrtc::AudioProcessing> apm;
292 - spa_audio_info_raw info;
293 - std::unique_ptr<float *[]> play_buffer, rec_buffer, out_buffer;
296 -static struct spa_log_topic log_topic = SPA_LOG_TOPIC(0, "spa.eac.webrtc");
297 -#undef SPA_LOG_TOPIC_DEFAULT
298 -#define SPA_LOG_TOPIC_DEFAULT &log_topic
300 -static bool webrtc_get_spa_bool(const struct spa_dict *args, const char *key, bool default_value) {
301 - const char *str_val;
302 - bool value = default_value;
303 - str_val = spa_dict_lookup(args, key);
304 - if (str_val != NULL)
305 - value =spa_atob(str_val);
310 -static int webrtc_create(struct spa_handle *handle, const struct spa_dict *args, const struct spa_audio_info_raw *info)
312 - auto impl = reinterpret_cast<struct impl_data*>(handle);
314 - bool extended_filter = webrtc_get_spa_bool(args, "webrtc.extended_filter", true);
315 - bool delay_agnostic = webrtc_get_spa_bool(args, "webrtc.delay_agnostic", true);
316 - bool high_pass_filter = webrtc_get_spa_bool(args, "webrtc.high_pass_filter", true);
317 - bool noise_suppression = webrtc_get_spa_bool(args, "webrtc.noise_suppression", true);
318 - bool voice_detection = webrtc_get_spa_bool(args, "webrtc.voice_detection", true);
320 - // Note: AGC seems to mess up with Agnostic Delay Detection, especially with speech,
321 - // result in very poor performance, disable by default
322 - bool gain_control = webrtc_get_spa_bool(args, "webrtc.gain_control", false);
324 - // Disable experimental flags by default
325 - bool experimental_agc = webrtc_get_spa_bool(args, "webrtc.experimental_agc", false);
326 - bool experimental_ns = webrtc_get_spa_bool(args, "webrtc.experimental_ns", false);
328 - // FIXME: Intelligibility enhancer is not currently supported
329 - // This filter will modify playback buffer (when calling ProcessReverseStream), but now
330 - // playback buffer modifications are discarded.
332 - webrtc::Config config;
333 - config.Set<webrtc::ExtendedFilter>(new webrtc::ExtendedFilter(extended_filter));
334 - config.Set<webrtc::DelayAgnostic>(new webrtc::DelayAgnostic(delay_agnostic));
335 - config.Set<webrtc::ExperimentalAgc>(new webrtc::ExperimentalAgc(experimental_agc));
336 - config.Set<webrtc::ExperimentalNs>(new webrtc::ExperimentalNs(experimental_ns));
338 - webrtc::ProcessingConfig pconfig = {{
339 - webrtc::StreamConfig(info->rate, info->channels, false), /* input stream */
340 - webrtc::StreamConfig(info->rate, info->channels, false), /* output stream */
341 - webrtc::StreamConfig(info->rate, info->channels, false), /* reverse input stream */
342 - webrtc::StreamConfig(info->rate, info->channels, false), /* reverse output stream */
345 - auto apm = std::unique_ptr<webrtc::AudioProcessing>(webrtc::AudioProcessing::Create(config));
346 - if (apm->Initialize(pconfig) != webrtc::AudioProcessing::kNoError) {
347 - spa_log_error(impl->log, "Error initialising webrtc audio processing module");
351 - apm->high_pass_filter()->Enable(high_pass_filter);
352 - // Always disable drift compensation since it requires drift sampling
353 - apm->echo_cancellation()->enable_drift_compensation(false);
354 - apm->echo_cancellation()->Enable(true);
355 - // TODO: wire up supression levels to args
356 - apm->echo_cancellation()->set_suppression_level(webrtc::EchoCancellation::kHighSuppression);
357 - apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kHigh);
358 - apm->noise_suppression()->Enable(noise_suppression);
359 - apm->voice_detection()->Enable(voice_detection);
360 - // TODO: wire up AGC parameters to args
361 - apm->gain_control()->set_analog_level_limits(0, 255);
362 - apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveDigital);
363 - apm->gain_control()->Enable(gain_control);
364 - impl->apm = std::move(apm);
365 - impl->info = *info;
366 - impl->play_buffer = std::make_unique<float *[]>(info->channels);
367 - impl->rec_buffer = std::make_unique<float *[]>(info->channels);
368 - impl->out_buffer = std::make_unique<float *[]>(info->channels);
372 -static int webrtc_run(struct spa_handle *handle, const float *rec[], const float *play[], float *out[], uint32_t n_samples)
374 - auto impl = reinterpret_cast<struct impl_data*>(handle);
375 - webrtc::StreamConfig config =
376 - webrtc::StreamConfig(impl->info.rate, impl->info.channels, false);
377 - unsigned int num_blocks = n_samples * 1000 / impl->info.rate / 10;
379 - if (n_samples * 1000 / impl->info.rate % 10 != 0) {
380 - spa_log_error(impl->log, "Buffers must be multiples of 10ms in length (currently %u samples)", n_samples);
384 - for (size_t i = 0; i < num_blocks; i ++) {
385 - for (size_t j = 0; j < impl->info.channels; j++) {
386 - impl->play_buffer[j] = const_cast<float *>(play[j]) + config.num_frames() * i;
387 - impl->rec_buffer[j] = const_cast<float *>(rec[j]) + config.num_frames() * i;
388 - impl->out_buffer[j] = out[j] + config.num_frames() * i;
390 - /* FIXME: ProcessReverseStream may change the playback buffer, in which
391 - * case we should use that, if we ever expose the intelligibility
393 - if (impl->apm->ProcessReverseStream(impl->play_buffer.get(), config, config, impl->play_buffer.get()) !=
394 - webrtc::AudioProcessing::kNoError) {
395 - spa_log_error(impl->log, "Processing reverse stream failed");
398 - // Extra delay introduced by multiple frames
399 - impl->apm->set_stream_delay_ms((num_blocks - 1) * 10);
401 - if (impl->apm->ProcessStream(impl->rec_buffer.get(), config, config, impl->out_buffer.get()) !=
402 - webrtc::AudioProcessing::kNoError) {
403 - spa_log_error(impl->log, "Processing stream failed");
410 -struct spa_dict *webrtc_get_properties(SPA_UNUSED struct spa_handle *handle)
412 - /* Not supported */
416 -int webrtc_set_properties(SPA_UNUSED struct spa_handle *handle, SPA_UNUSED const struct spa_dict *args)
418 - /* Not supported */
422 -static struct echo_cancel_info echo_cancel_webrtc_impl = {
424 - .info = SPA_DICT_INIT(NULL, 0),
425 - .latency = "480/48000",
426 - .create = webrtc_create,
428 - .get_properties = webrtc_get_properties,
429 - .set_properties = webrtc_set_properties,
432 -static int impl_get_interface(struct spa_handle *handle, const char *type, void **interface)
435 - spa_return_val_if_fail(handle != NULL, -EINVAL);
436 - spa_return_val_if_fail(interface != NULL, -EINVAL);
438 - if (spa_streq(type, SPA_TYPE_INTERFACE_AEC))
439 - *interface = &echo_cancel_webrtc_impl;
446 -static int impl_clear(struct spa_handle *handle)
448 - spa_return_val_if_fail(handle != NULL, -EINVAL);
449 - auto impl = reinterpret_cast<struct impl_data*>(handle);
450 - impl->~impl_data();
455 -impl_get_size(const struct spa_handle_factory *factory,
456 - const struct spa_dict *params)
458 - return sizeof(struct impl_data);
462 -impl_init(const struct spa_handle_factory *factory,
463 - struct spa_handle *handle,
464 - const struct spa_dict *info,
465 - const struct spa_support *support,
466 - uint32_t n_support)
468 - spa_return_val_if_fail(factory != NULL, -EINVAL);
469 - spa_return_val_if_fail(handle != NULL, -EINVAL);
471 - echo_cancel_webrtc_impl.iface = SPA_INTERFACE_INIT(
472 - SPA_TYPE_INTERFACE_AEC,
473 - SPA_VERSION_AUDIO_AEC,
477 - auto impl = new (handle) impl_data();
478 - impl->handle.get_interface = impl_get_interface;
479 - impl->handle.clear = impl_clear;
480 - impl->log = (struct spa_log*)spa_support_find(support, n_support, SPA_TYPE_INTERFACE_Log);
481 - spa_log_topic_init(impl->log, &log_topic);
486 -static const struct spa_interface_info impl_interfaces[] = {
487 - {SPA_TYPE_INTERFACE_AEC,},
491 -impl_enum_interface_info(const struct spa_handle_factory *factory,
492 - const struct spa_interface_info **info,
495 - spa_return_val_if_fail(factory != NULL, -EINVAL);
496 - spa_return_val_if_fail(info != NULL, -EINVAL);
497 - spa_return_val_if_fail(index != NULL, -EINVAL);
501 - *info = &impl_interfaces[*index];
510 -const struct spa_handle_factory spa_aec_exaudio_factory = {
511 - SPA_VERSION_HANDLE_FACTORY,
516 - impl_enum_interface_info,
521 -int spa_handle_factory_enum(const struct spa_handle_factory **factory, uint32_t *index)
523 - spa_return_val_if_fail(factory != NULL, -EINVAL);
524 - spa_return_val_if_fail(index != NULL, -EINVAL);
528 - *factory = &spa_aec_exaudio_factory;
536 diff --git a/spa/plugins/aec/meson.build b/spa/plugins/aec/meson.build
537 deleted file mode 100644
538 index 2b1a2c05a..000000000
539 --- a/spa/plugins/aec/meson.build
542 -aec_null = shared_library('spa-aec-null',
544 - include_directories : [ configinc ],
545 - dependencies : [ spa_dep ],
547 - install_dir : spa_plugindir / 'aec')
549 -if webrtc_dep.found()
550 - aec_webrtc = shared_library('spa-aec-webrtc',
551 - [ 'aec-webrtc.cpp' ],
552 - include_directories : [ configinc ],
553 - dependencies : [ spa_dep, webrtc_dep ],
555 - install_dir : spa_plugindir / 'aec')
558 diff --git a/spa/plugins/meson.build b/spa/plugins/meson.build
559 index 0b581b29b..fd229d460 100644
560 --- a/spa/plugins/meson.build
561 +++ b/spa/plugins/meson.build
562 @@ -51,5 +51,3 @@ endif
563 if libcamera_dep.found()
568 \ No newline at end of file
569 diff --git a/src/modules/meson.build b/src/modules/meson.build
570 index bd7d3f711..568b32c50 100644
571 --- a/src/modules/meson.build
572 +++ b/src/modules/meson.build
573 @@ -110,15 +110,22 @@ pipewire_module_filter_chain = shared_library('pipewire-module-filter-chain',
575 pipewire_module_echo_cancel_sources = [
576 'module-echo-cancel.c',
577 + 'module-echo-cancel/aec-null.c',
580 +if webrtc_dep.found()
581 + pipewire_module_echo_cancel_sources += [
582 + 'module-echo-cancel/aec-webrtc.cpp'
586 pipewire_module_echo_cancel = shared_library('pipewire-module-echo-cancel',
587 pipewire_module_echo_cancel_sources,
588 include_directories : [configinc, spa_inc],
590 install_dir : modules_install_dir,
591 install_rpath: modules_install_dir,
592 - dependencies : [mathlib, dl_lib, pipewire_dep],
593 + dependencies : [mathlib, dl_lib, pipewire_dep, webrtc_dep],
596 pipewire_module_profiler = shared_library('pipewire-module-profiler',
597 diff --git a/src/modules/module-echo-cancel.c b/src/modules/module-echo-cancel.c
598 index 728325938..00adbbfc5 100644
599 --- a/src/modules/module-echo-cancel.c
600 +++ b/src/modules/module-echo-cancel.c
605 -#include "module-echo-cancel/echo-cancel.h"
610 #include <spa/param/audio/raw.h>
611 #include <spa/param/profiler.h>
612 #include <spa/pod/builder.h>
613 -#include <spa/support/plugin.h>
614 #include <spa/utils/json.h>
615 -#include <spa/utils/names.h>
616 #include <spa/utils/result.h>
617 #include <spa/utils/ringbuffer.h>
618 #include <spa/utils/string.h>
619 -#include <spa/support/plugin-loader.h>
620 -#include <spa/interfaces/audio/aec.h>
622 #include <pipewire/private.h>
623 #include <pipewire/impl.h>
626 #include <pipewire/extensions/profiler.h>
628 +#include "module-echo-cancel/echo-cancel.h"
630 /** \page page_module_echo_cancel PipeWire Module: Echo Cancel
632 * The `echo-cancel` module performs echo cancellation. The module creates
635 * - `source.props = {}`: properties to be passed to the source stream
636 * - `sink.props = {}`: properties to be passed to the sink stream
637 - * - `library.name = <str>`: the echo cancellation library Currently supported:
638 - * `aec/libspa-aec-exaudio`. Leave unset to use the default method (`aec/libspa-aec-exaudio`).
639 + * - `aec.method = <str>`: the echo cancellation method. Currently supported:
640 + * `webrtc`. Leave unset to use the default method (`webrtc`).
641 * - `aec.args = <str>`: arguments to pass to the echo cancellation method
645 * context.modules = [
646 * { name = libpipewire-module-echo-cancel
648 - * # library.name = aec/libspa-aec-exaudio
649 + * # aec.method = webrtc
650 * # node.latency = 1024/48000
652 * node.name = "Echo Cancellation Source"
653 @@ -141,7 +138,7 @@ static const struct spa_dict_item module_props[] = {
654 "[ audio.position=<channel map> ] "
655 "[ buffer.max_size=<max buffer size in ms> ] "
656 "[ buffer.play_delay=<play delay in ms> ] "
657 - "[ library.name =<library name> ] "
658 + "[ aec.method=<aec method> ] "
659 "[ aec.args=<aec arguments> ] "
660 "[ source.props=<properties> ] "
661 "[ sink.props=<properties> ] " },
662 @@ -189,6 +186,7 @@ struct impl {
663 struct spa_ringbuffer out_ring;
665 const struct echo_cancel_info *aec_info;
667 uint32_t aec_blocksize;
669 unsigned int capture_ready:1;
670 @@ -199,9 +197,6 @@ struct impl {
672 uint32_t max_buffer_size;
673 uint32_t buffer_delay;
675 - struct spa_handle *spa_handle;
676 - struct spa_plugin_loader *loader;
679 static void do_unload_module(void *obj, void *data, int res, uint32_t id)
680 @@ -288,7 +283,7 @@ static void process(struct impl *impl)
681 pw_stream_queue_buffer(impl->playback, pout);
683 /* Now run the canceller */
684 - echo_cancel_run(impl->aec_info, impl->spa_handle, rec, play_delayed, out, size / sizeof(float));
685 + echo_cancel_run(impl->aec_info, impl->aec, rec, play_delayed, out, size / sizeof(float));
687 /* Next, copy over the output to the output ringbuffer */
688 avail = spa_ringbuffer_get_write_index(&impl->out_ring, &oindex);
689 @@ -808,8 +803,8 @@ static void impl_destroy(struct impl *impl)
690 pw_stream_destroy(impl->sink);
691 if (impl->core && impl->do_disconnect)
692 pw_core_disconnect(impl->core);
693 - if (impl->spa_handle)
694 - spa_plugin_loader_unload(impl->loader, impl->spa_handle);
696 + echo_cancel_destroy(impl->aec_info, impl->aec);
697 pw_properties_free(impl->source_props);
698 pw_properties_free(impl->sink_props);
700 @@ -898,10 +893,7 @@ int pipewire__module_init(struct pw_impl_module *module, const char *args)
702 uint32_t id = pw_global_get_id(pw_impl_module_get_global(module));
706 - struct spa_handle *handle = NULL;
710 PW_LOG_TOPIC_INIT(mod_topic);
712 @@ -975,57 +967,22 @@ int pipewire__module_init(struct pw_impl_module *module, const char *args)
713 if (pw_properties_get(impl->sink_props, PW_KEY_MEDIA_CLASS) == NULL)
714 pw_properties_set(impl->sink_props, PW_KEY_MEDIA_CLASS, "Audio/Sink");
716 - if ((str = pw_properties_get(props, "aec.method")) != NULL)
717 - pw_log_warn("aec.method is not supported anymore use library.name");
719 - /* Use webrtc as default */
720 - if ((path = pw_properties_get(props, "library.name")) == NULL)
721 - path = "aec/libspa-aec-webrtc";
723 - struct spa_dict_item info_items[] = {
724 - { SPA_KEY_LIBRARY_NAME, path },
726 - struct spa_dict info = SPA_DICT_INIT_ARRAY(info_items);
728 - impl->loader = spa_support_find(context->support, context->n_support, SPA_TYPE_INTERFACE_PluginLoader);
729 - if (impl->loader == NULL) {
730 - pw_log_error("a plugin loader is needed");
734 - handle = spa_plugin_loader_load(impl->loader, SPA_NAME_AEC, &info);
735 - if (handle == NULL) {
736 - pw_log_error("AEC codec plugin %s not available library.name %s", SPA_NAME_AEC, path);
740 - if ((res = spa_handle_get_interface(handle, SPA_TYPE_INTERFACE_AEC, &iface)) < 0) {
741 - pw_log_error("can't get %s interface %d", SPA_TYPE_INTERFACE_AEC, res);
744 - impl->aec_info = iface;
745 - impl->spa_handle = handle;
746 - if (impl->aec_info->iface.version != SPA_VERSION_AUDIO_AEC) {
747 - pw_log_error("codec plugin %s has incompatible ABI version (%d != %d)",
748 - SPA_NAME_AEC, impl->aec_info->iface.version, SPA_VERSION_AUDIO_AEC);
753 - (void)SPA_SUPPORT_INIT(SPA_TYPE_INTERFACE_AEC, (struct echo_cancel_info *)impl->aec_info);
754 + if ((str = pw_properties_get(props, "aec.method")) == NULL)
757 - pw_log_info("Using plugin AEC %s", impl->aec_info->name);
759 + if (spa_streq(str, "webrtc"))
760 + impl->aec_info = echo_cancel_webrtc;
763 + impl->aec_info = echo_cancel_null;
765 if ((str = pw_properties_get(props, "aec.args")) != NULL)
766 aec_props = pw_properties_new_string(str);
768 aec_props = pw_properties_new(NULL, NULL);
770 - if (echo_cancel_create(impl->aec_info, impl->spa_handle, &aec_props->dict, &impl->info)) {
771 - pw_log_error("codec plugin %s create failed", impl->aec_info->name);
775 + impl->aec = echo_cancel_create(impl->aec_info, aec_props, &impl->info);
777 pw_properties_free(aec_props);
779 diff --git a/spa/include/spa/interfaces/audio/aec.h b/src/modules/module-echo-cancel/aec-null.c
781 rename from spa/include/spa/interfaces/audio/aec.h
782 rename to src/modules/module-echo-cancel/aec-null.c
783 index e1fcda563..4ee9ac6d7 100644
784 --- a/spa/include/spa/interfaces/audio/aec.h
785 +++ b/src/modules/module-echo-cancel/aec-null.c
787 * DEALINGS IN THE SOFTWARE.
790 +#include "echo-cancel.h"
792 -#include <spa/utils/dict.h>
793 -#include <spa/utils/hook.h>
794 -#include <spa/pod/pod.h>
795 -#include <spa/param/audio/raw.h>
796 -#include <spa/support/plugin.h>
798 -#define SPA_TYPE_INTERFACE_AEC SPA_TYPE_INFO_INTERFACE_BASE "AEC"
799 -#define SPA_VERSION_AUDIO_AEC 1
801 -struct echo_cancel_info {
802 - struct spa_interface iface;
804 - const struct spa_dict info;
805 - const char *latency;
806 - int (*create) (struct spa_handle *handle, const struct spa_dict *args, const struct spa_audio_info_raw *info);
807 - int (*run) (struct spa_handle *handle, const float *rec[], const float *play[], float *out[], uint32_t n_samples);
808 - struct spa_dict *(*get_properties) (struct spa_handle *handle);
809 - int (*set_properties) (struct spa_handle *handle, const struct spa_dict *args);
814 +static void *null_create(const struct pw_properties *args, const struct spa_audio_info_raw *info)
817 + impl = calloc(1, sizeof(struct impl));
818 + impl->channels = info->channels;
822 +static void null_destroy(void *ec)
827 +static int null_run(void *ec, const float *rec[], const float *play[], float *out[], uint32_t n_samples)
829 + struct impl *impl = ec;
831 + for (i = 0; i < impl->channels; i++)
832 + memcpy(out[i], rec[i], n_samples * sizeof(float));
836 +static const struct echo_cancel_info echo_cancel_null_impl = {
838 + .info = SPA_DICT_INIT(NULL, 0),
841 + .create = null_create,
842 + .destroy = null_destroy,
847 +const struct echo_cancel_info *echo_cancel_null = &echo_cancel_null_impl;
848 diff --git a/src/modules/module-echo-cancel/aec-webrtc.cpp b/src/modules/module-echo-cancel/aec-webrtc.cpp
850 index 000000000..77b569a22
852 +++ b/src/modules/module-echo-cancel/aec-webrtc.cpp
856 + * Copyright © 2021 Wim Taymans <wim.taymans@gmail.com>
857 + * © 2021 Arun Raghavan <arun@asymptotic.io>
859 + * Permission is hereby granted, free of charge, to any person obtaining a
860 + * copy of this software and associated documentation files (the "Software"),
861 + * to deal in the Software without restriction, including without limitation
862 + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
863 + * and/or sell copies of the Software, and to permit persons to whom the
864 + * Software is furnished to do so, subject to the following conditions:
866 + * The above copyright notice and this permission notice (including the next
867 + * paragraph) shall be included in all copies or substantial portions of the
870 + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
871 + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
872 + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
873 + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
874 + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
875 + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
876 + * DEALINGS IN THE SOFTWARE.
882 +#include "echo-cancel.h"
884 +#include <pipewire/pipewire.h>
886 +#include <webrtc/modules/audio_processing/include/audio_processing.h>
887 +#include <webrtc/modules/interface/module_common_types.h>
888 +#include <webrtc/system_wrappers/include/trace.h>
891 + std::unique_ptr<webrtc::AudioProcessing> apm;
892 + spa_audio_info_raw info;
893 + std::unique_ptr<float *[]> play_buffer, rec_buffer, out_buffer;
895 + impl(std::unique_ptr<webrtc::AudioProcessing> apm, const spa_audio_info_raw& info)
896 + : apm(std::move(apm)),
898 + play_buffer(std::make_unique<float *[]>(info.channels)),
899 + rec_buffer(std::make_unique<float *[]>(info.channels)),
900 + out_buffer(std::make_unique<float *[]>(info.channels))
904 +static void *webrtc_create(const struct pw_properties *args, const spa_audio_info_raw *info)
906 + bool extended_filter = pw_properties_get_bool(args, "webrtc.extended_filter", true);
907 + bool delay_agnostic = pw_properties_get_bool(args, "webrtc.delay_agnostic", true);
908 + bool high_pass_filter = pw_properties_get_bool(args, "webrtc.high_pass_filter", true);
909 + bool noise_suppression = pw_properties_get_bool(args, "webrtc.noise_suppression", true);
910 + bool voice_detection = pw_properties_get_bool(args, "webrtc.voice_detection", true);
912 + // Note: AGC seems to mess up with Agnostic Delay Detection, especially with speech,
913 + // result in very poor performance, disable by default
914 + bool gain_control = pw_properties_get_bool(args, "webrtc.gain_control", false);
916 + // Disable experimental flags by default
917 + bool experimental_agc = pw_properties_get_bool(args, "webrtc.experimental_agc", false);
918 + bool experimental_ns = pw_properties_get_bool(args, "webrtc.experimental_ns", false);
920 + // FIXME: Intelligibility enhancer is not currently supported
921 + // This filter will modify playback buffer (when calling ProcessReverseStream), but now
922 + // playback buffer modifications are discarded.
924 + webrtc::Config config;
925 + config.Set<webrtc::ExtendedFilter>(new webrtc::ExtendedFilter(extended_filter));
926 + config.Set<webrtc::DelayAgnostic>(new webrtc::DelayAgnostic(delay_agnostic));
927 + config.Set<webrtc::ExperimentalAgc>(new webrtc::ExperimentalAgc(experimental_agc));
928 + config.Set<webrtc::ExperimentalNs>(new webrtc::ExperimentalNs(experimental_ns));
930 + webrtc::ProcessingConfig pconfig = {{
931 + webrtc::StreamConfig(info->rate, info->channels, false), /* input stream */
932 + webrtc::StreamConfig(info->rate, info->channels, false), /* output stream */
933 + webrtc::StreamConfig(info->rate, info->channels, false), /* reverse input stream */
934 + webrtc::StreamConfig(info->rate, info->channels, false), /* reverse output stream */
937 + auto apm = std::unique_ptr<webrtc::AudioProcessing>(webrtc::AudioProcessing::Create(config));
938 + if (apm->Initialize(pconfig) != webrtc::AudioProcessing::kNoError) {
939 + pw_log_error("Error initialising webrtc audio processing module");
943 + apm->high_pass_filter()->Enable(high_pass_filter);
944 + // Always disable drift compensation since it requires drift sampling
945 + apm->echo_cancellation()->enable_drift_compensation(false);
946 + apm->echo_cancellation()->Enable(true);
947 + // TODO: wire up supression levels to args
948 + apm->echo_cancellation()->set_suppression_level(webrtc::EchoCancellation::kHighSuppression);
949 + apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kHigh);
950 + apm->noise_suppression()->Enable(noise_suppression);
951 + apm->voice_detection()->Enable(voice_detection);
952 + // TODO: wire up AGC parameters to args
953 + apm->gain_control()->set_analog_level_limits(0, 255);
954 + apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveDigital);
955 + apm->gain_control()->Enable(gain_control);
957 + return new impl(std::move(apm), *info);
960 +static void webrtc_destroy(void *ec)
962 + auto impl = static_cast<struct impl *>(ec);
967 +static int webrtc_run(void *ec, const float *rec[], const float *play[], float *out[], uint32_t n_samples)
969 + auto impl = static_cast<struct impl *>(ec);
970 + webrtc::StreamConfig config =
971 + webrtc::StreamConfig(impl->info.rate, impl->info.channels, false);
972 + unsigned int num_blocks = n_samples * 1000 / impl->info.rate / 10;
974 + if (n_samples * 1000 / impl->info.rate % 10 != 0) {
975 + pw_log_error("Buffers must be multiples of 10ms in length (currently %u samples)", n_samples);
979 + for (size_t i = 0; i < num_blocks; i ++) {
980 + for (size_t j = 0; j < impl->info.channels; j++) {
981 + impl->play_buffer[j] = const_cast<float *>(play[j]) + config.num_frames() * i;
982 + impl->rec_buffer[j] = const_cast<float *>(rec[j]) + config.num_frames() * i;
983 + impl->out_buffer[j] = out[j] + config.num_frames() * i;
985 + /* FIXME: ProcessReverseStream may change the playback buffer, in which
986 + * case we should use that, if we ever expose the intelligibility
988 + if (impl->apm->ProcessReverseStream(impl->play_buffer.get(), config, config, impl->play_buffer.get()) !=
989 + webrtc::AudioProcessing::kNoError) {
990 + pw_log_error("Processing reverse stream failed");
993 + // Extra delay introduced by multiple frames
994 + impl->apm->set_stream_delay_ms((num_blocks - 1) * 10);
996 + if (impl->apm->ProcessStream(impl->rec_buffer.get(), config, config, impl->out_buffer.get()) !=
997 + webrtc::AudioProcessing::kNoError) {
998 + pw_log_error("Processing stream failed");
1005 +static const struct echo_cancel_info echo_cancel_webrtc_impl = {
1007 + .info = SPA_DICT_INIT(NULL, 0),
1008 + .latency = "480/48000",
1010 + .create = webrtc_create,
1011 + .destroy = webrtc_destroy,
1013 + .run = webrtc_run,
1016 +const struct echo_cancel_info *echo_cancel_webrtc = &echo_cancel_webrtc_impl;
1017 diff --git a/src/modules/module-echo-cancel/echo-cancel.h b/src/modules/module-echo-cancel/echo-cancel.h
1018 index ac83f70e4..fe011b962 100644
1019 --- a/src/modules/module-echo-cancel/echo-cancel.h
1020 +++ b/src/modules/module-echo-cancel/echo-cancel.h
1022 * DEALINGS IN THE SOFTWARE.
1025 +#include "config.h"
1027 #include <spa/utils/dict.h>
1028 -#include <spa/utils/hook.h>
1029 #include <spa/param/audio/raw.h>
1030 -#include <spa/support/plugin.h>
1032 #include <pipewire/properties.h>
1034 +struct echo_cancel_info {
1036 + const struct spa_dict info;
1037 + const char *latency;
1039 + void *(*create) (const struct pw_properties *args, const struct spa_audio_info_raw *info);
1040 + void (*destroy) (void *ec);
1042 + int (*run) (void *ec, const float *rec[], const float *play[], float *out[], uint32_t n_samples);
1045 #define echo_cancel_create(i,...) (i)->create(__VA_ARGS__)
1046 +#define echo_cancel_destroy(i,...) (i)->destroy(__VA_ARGS__)
1047 #define echo_cancel_run(i,...) (i)->run(__VA_ARGS__)
1050 +extern const struct echo_cancel_info *echo_cancel_webrtc;
1052 +extern const struct echo_cancel_info *echo_cancel_null;