add webrtc aec code.

.vscode/c_cpp_properties.json

@@ -13,7 +13,9 @@
                 "${workspaceFolder}/build/_deps/kylin-src/Universal",
                 "${workspaceFolder}/build/_deps/kylin-src/HttpProxy"
             ],
-            "defines": [],
+            "defines": [
+                "WEBRTC_POSIX"
+            ],
             "compilerPath": "/usr/bin/gcc",
             "cStandard": "c17",
             "cppStandard": "gnu++17",

CMakeLists.txt

@@ -29,6 +29,7 @@ FetchContent_Declare(Kylin
 )
 FetchContent_MakeAvailable(Kylin)
 
+add_subdirectory(VocieProcess)
 add_subdirectory(GatePass)
 add_subdirectory(Record)
 add_subdirectory(VoucherVerifyServer)

Record/CMakeLists.txt

@@ -9,6 +9,7 @@ add_executable(Record main.cpp
     Recorder.cpp
     SpeexDsp.h SpeexDsp.cpp
     Utility.h Utility.cpp
+    WebRtcAecm.h WebRtcAecm.cpp
     WebRTCPublisher.h WebRTCPublisher.cpp
 )
 
@@ -35,6 +36,7 @@ target_link_directories(Record
 )
 
 target_link_libraries(Record
+    PRIVATE VocieProcess
     PRIVATE asound
     PRIVATE easymedia
     PRIVATE drm

Record/EchoRecord.cpp

@@ -1,4 +1,7 @@
 #include "BoostLog.h"
+#include "SpeexDsp.h"
+#include "Utility.h"
+#include "WebRtcAecm.h"
 #include "main.h"
 #include <memory>
 
@@ -14,18 +17,54 @@ void EchoRecordTask::setChannels(int channels) {
     }
 }
 
+// underrun occurred pcm播放饥饿
+// 回采信号提前于mic信号，时间差<80ms
+// ./Record --echo --vqe=false --channels=2
+// ./Record --echo --vqe=true --channels=2
+// ./Record --echo --vqe=false --channels=1
 void EchoRecordTask::run() {
+
     RkAudio::Format format;
     format.channels = m_channels;
-    format.period = 64;
+    format.period = 10;
+
+    m_speex = std::make_shared<SpeexDsp>();
+    m_speex->start(format.sampleRate, m_channels, format.period);
+
+    m_farendBuffer.resize(m_channels * sizeof(int16_t) * format.sampleRate / 1000 * format.period);
+    m_nearendBuffer.resize(m_channels * sizeof(int16_t) * format.sampleRate / 1000 * format.period);
+    m_webRtcAecm = std::make_shared<WebRtcAecm>();
+    m_webRtcAecm->start(format.sampleRate, format.channels, format.period);
+
     m_output = std::make_shared<RkAudio::Output>();
-    if (!m_output->open(sizeof(uint16_t), format.sampleRate, format.channels, format.period, m_vqeEnabled)) {
+    if (!m_output->open(sizeof(uint16_t), format.sampleRate, 2, format.period, m_vqeEnabled)) {
         LOG(error) << "audio output open failed.";
         return;
     }
 
+    m_outBuffer.resize(m_channels * sizeof(int16_t) * format.sampleRate / 1000 * format.period);
+
     m_input = std::make_shared<RkAudio::Input>();
-    m_input->setDataCallback([this](const RkAudio::Frame &frame) { m_output->write(frame.data, frame.byteSize); });
+    m_input->setDataCallback([this](const RkAudio::Frame &frame) {
+        memcpy(m_nearendBuffer.data(), frame.data, frame.byteSize);
+        m_webRtcAecm->echoPlayback(reinterpret_cast<const int16_t *>(m_farendBuffer.data()), m_farendBuffer.size() / 2);
+
+        m_webRtcAecm->echoCancellation(reinterpret_cast<int16_t *>(frame.data), reinterpret_cast<int16_t *>(m_nearendBuffer.data()),
+                                       reinterpret_cast<int16_t *>(m_outBuffer.data()), frame.frameSize);
+        // m_speex->echoPlayback(reinterpret_cast<const int16_t *>(frame.data));
+        // m_speex->echoPlayback(reinterpret_cast<const int16_t *>(m_buffer.data()));
+
+        // m_speex->echoCapture(reinterpret_cast<const int16_t *>(frame.data), reinterpret_cast<int16_t *>(m_buffer.data()));
+
+        if (m_channels == 2) {
+            m_output->write(frame.data, frame.byteSize);
+        } else if (m_channels == 1) {
+            auto filledData = duplicate(m_outBuffer.data(), m_outBuffer.size());
+            m_output->write(filledData.data(), filledData.size());
+        }
+        memcpy(m_farendBuffer.data(), m_outBuffer.data(), m_outBuffer.size());
+        // m_output->write(reinterpret_cast<const uint8_t *>(m_buffer.data()), m_buffer.size());
+    });
 
     m_input->open(format, m_vqeEnabled);
 }

Record/WebRtcAecm.cpp

@@ -0,0 +1,50 @@
+#include "WebRtcAecm.h"
+#include "BoostLog.h"
+#include "modules/audio_processing/aecm/echo_control_mobile.h"
+
+WebRtcAecm::~WebRtcAecm() {
+    close();
+}
+
+void WebRtcAecm::close() {
+    if (m_hanlde != nullptr) {
+        webrtc::WebRtcAecm_Free(m_hanlde);
+        m_hanlde = nullptr;
+    }
+}
+
+void WebRtcAecm::echoPlayback(const int16_t *play, int samples) {
+    if (webrtc::WebRtcAecm_BufferFarend(m_hanlde, play, samples) != 0) {
+        LOG(warning) << "failed to buffer farend signal.";
+    }
+}
+
+void WebRtcAecm::echoCancellation(const int16_t *nearendNoisy, const int16_t *nearendClean, int16_t *out, int samples) {
+    if (webrtc::WebRtcAecm_Process(m_hanlde, nearendNoisy, nearendClean, out, samples, 60) != 0) {
+        LOG(warning) << "WebRtcAecm_Process() failed.";
+    }
+}
+
+bool WebRtcAecm::start(int sampleRate, int channels, int period) {
+    close();
+    m_hanlde = webrtc::WebRtcAecm_Create();
+    if (m_hanlde == nullptr) {
+        LOG(error) << "WebRtcAecm_Create() failed.";
+        return false;
+    }
+    if (webrtc::WebRtcAecm_Init(m_hanlde, sampleRate) != 0) {
+        LOG(error) << "WebRtcAecm_Init() failed.";
+        close();
+        return false;
+    }
+
+    webrtc::AecmConfig config;
+    config.cngMode = webrtc::AecmTrue; // 启用舒适噪声生成
+    config.echoMode = 3;               // 回声消除模式，范围是 0-4
+    if (WebRtcAecm_set_config(m_hanlde, config) != 0) {
+        LOG(error) << "WebRtcAecm_set_config() failed.";
+        close();
+        return false;
+    }
+    return true;
+}

Record/WebRtcAecm.h

@@ -0,0 +1,18 @@
+#ifndef __WEBRTCAECM_H__
+#define __WEBRTCAECM_H__
+
+#include <cstdint>
+
+class WebRtcAecm {
+public:
+    ~WebRtcAecm();
+    bool start(int sampleRate, int channels, int period);
+    void close();
+    void echoPlayback(const int16_t *play, int samples);
+    void echoCancellation(const int16_t *nearendNoisy, const int16_t *nearendClean, int16_t *out, int samples);
+
+private:
+    void *m_hanlde = nullptr;
+};
+
+#endif // __WEBRTCAECM_H__

Record/main.h

@@ -6,7 +6,7 @@
 #include <memory>
 
 class SpeexDsp;
-
+class WebRtcAecm;
 class Task {
 public:
     virtual void run() = 0;
@@ -50,7 +50,11 @@ private:
     std::shared_ptr<RkAudio::Output> m_output;
     std::shared_ptr<RkAudio::Input> m_input;
     std::shared_ptr<SpeexDsp> m_speex;
-    std::vector<uint8_t> m_buffer;
+    std::shared_ptr<WebRtcAecm> m_webRtcAecm;
+    std::vector<uint8_t> m_outBuffer;
+
+    std::vector<uint8_t> m_nearendBuffer;
+    std::vector<uint8_t> m_farendBuffer;
 };
 
 #endif // __MAIN_H__