diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 3809836..68fa7ff 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -4,8 +4,9 @@ "name": "Linux", "includePath": [ "${workspaceFolder}/**", - "/opt/Libraries/boost_1_84_0/include", - "${workspaceFolder}/3rdparty/arm-linux-gnueabihf/mpp/inc", + "/opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/boost_1_86_0/include", + "/opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/speexdsp-1.2.1/include", + "/opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/rockchip_mpp/include", "/opt/toolchains/Libraries/opus-1.4/include", "ThirdParty/libjpegTurbo/inc", "ThirdParty/libalsa/inc", diff --git a/CMakeLists.txt b/CMakeLists.txt index f8c76b2..f3b68f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,8 +4,10 @@ cmake_minimum_required(VERSION 3.27) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(OPENSSL_ROOT /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/openssl-1.0.2q) +set(OPENSSL_ROOT /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/openssl-3.3.1) set(OPENSSL_INCLUDE_DIR ${OPENSSL_ROOT}/include) +set(OPENSSL_LIBRARY_DIRS ${OPENSSL_ROOT}/lib) +set(OPENSSL_LIBRARIES ssl crypto) set(ALSA_ROOT /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/libalsa-1.1.5) set(ALSA_INCLUDE_DIR ${ALSA_ROOT}/include) diff --git a/Readme.md b/Readme.md index 35b4dd6..17080d4 100644 --- a/Readme.md +++ b/Readme.md @@ -1,5 +1,6 @@ ```shell +aplay -t raw -c 1 -f S16_LE -r 16000 /system/audio/pleaseFaceTheCamera.wav amixer sset 'Master' 50% # 设置音量 mount -o remount rw /system/ # 读写挂载 keydata get mac # 操作mac diff --git a/Record/CMakeLists.txt b/Record/CMakeLists.txt index 3080c87..1a6fdb6 100644 --- a/Record/CMakeLists.txt +++ b/Record/CMakeLists.txt @@ -1,8 +1,14 @@ +find_package(Boost REQUIRED COMPONENTS json) + add_executable(Record main.cpp RkAudio.h RkAudio.cpp OpusCodec.h OpusCodec.cpp FFmpegResample.h FFmpegResample.cpp - RkRecorder.cpp + EchoRecord.cpp + Player.cpp + Recorder.cpp + SpeexDsp.h SpeexDsp.cpp + WebRTCPublisher.h WebRTCPublisher.cpp ) target_include_directories(Record @@ -10,6 +16,8 @@ target_include_directories(Record PRIVATE ${MPP_INCLUDE_DIR} PRIVATE ${MPP_INCLUDE_DIR}/rkmedia PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/opus-1.4/include + PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/amazon-kinesis-video-streams-webrtc-sdk-c/include + PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/speexdsp-1.2.1/include PRIVATE ${FFMPEG_INCLUDE_DIR} ) @@ -18,7 +26,11 @@ target_link_directories(Record PRIVATE ${MPP_LIBRARY_DIRS} PRIVATE ${3rdparty_ROOT}/rkap_3a/lib PRIVATE ${FFMPEG_LIBRARY_DIRS} + PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/speexdsp-1.2.1/lib PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/opus-1.4/lib + PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/amazon-kinesis-video-streams-webrtc-sdk-c/lib + PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/usrsctp-0.9.5.0/lib + PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/libsrtp-2.6.0/lib ) target_link_libraries(Record @@ -38,7 +50,18 @@ target_link_libraries(Record PRIVATE glib-2.0 PRIVATE pcre PRIVATE opus + PRIVATE speexdsp + PRIVATE Boost::json + PRIVATE kvsCommonLws + PRIVATE kvsWebrtcClient + PRIVATE kvsWebrtcSignalingClient + PRIVATE kvspic + PRIVATE kvspicUtils + PRIVATE kvspicClient + PRIVATE usrsctp + PRIVATE srtp2 PRIVATE Universal + PRIVATE HttpProxy PRIVATE stdc++fs PRIVATE RKAP_ANR PRIVATE RKAP_Common @@ -46,36 +69,4 @@ target_link_libraries(Record PRIVATE dl PRIVATE z PRIVATE ${FFMPEG_LIBRARY} -) - -add_executable(rkmedia_audio_test rkmedia_audio_test.c) - -target_include_directories(rkmedia_audio_test - PRIVATE ${MPP_INCLUDE_DIR}/rkmedia -) - -target_link_directories(rkmedia_audio_test - PRIVATE ${MPP_LIBRARY_DIRS} - PRIVATE ${3rdparty_ROOT}/rkap_3a/lib - PRIVATE ${ALSA_LIBRARY_DIRS} -) - -target_link_libraries(rkmedia_audio_test - PRIVATE asound - PRIVATE easymedia - PRIVATE drm - PRIVATE rkaiq - PRIVATE rockchip_mpp - PRIVATE v4l2 - PRIVATE v4lconvert - PRIVATE jpeg - PRIVATE rga - PRIVATE glib-2.0 - PRIVATE pcre - # PRIVATE opus - # PRIVATE Universal - # PRIVATE stdc++fs - PRIVATE RKAP_ANR - PRIVATE RKAP_Common - PRIVATE dl ) \ No newline at end of file diff --git a/Record/EchoRecord.cpp b/Record/EchoRecord.cpp new file mode 100644 index 0000000..40454af --- /dev/null +++ b/Record/EchoRecord.cpp @@ -0,0 +1,31 @@ +#include "BoostLog.h" +#include "main.h" +#include + +void EchoRecordTask::setVqeEnabled(bool enabled) { + if (m_vqeEnabled != enabled) { + m_vqeEnabled = enabled; + } +} + +void EchoRecordTask::setChannels(int channels) { + if (m_channels != channels) { + m_channels = channels; + } +} + +void EchoRecordTask::run() { + RkAudio::Format format; + format.channels = m_channels; + format.period = 64; + m_output = std::make_shared(); + if (!m_output->open(sizeof(uint16_t), format.sampleRate, format.channels, format.period, m_vqeEnabled)) { + LOG(error) << "audio output open failed."; + return; + } + + m_input = std::make_shared(); + m_input->setDataCallback([this](const RkAudio::Frame &frame) { m_output->write(frame.data, frame.byteSize); }); + + m_input->open(format, m_vqeEnabled); +} \ No newline at end of file diff --git a/Record/Player.cpp b/Record/Player.cpp new file mode 100644 index 0000000..6527982 --- /dev/null +++ b/Record/Player.cpp @@ -0,0 +1,56 @@ +#include "IoContext.h" +#include "main.h" +#include +#include + +void PlayerTask::setPath(const std::string &path) { + if (m_path != path) { + m_path = path; + } + if (std::filesystem::exists(path)) { + m_ifs = std::make_shared(path, std::ifstream::binary); + } +} + +/* +ffmpeg将一个1通道mp3文件，转换为 2通道,16bit,16000采样率的pcm文件,但是左通道数据为静音 +ffmpeg -i 20200316_1900.mp3 -ac 2 -ar 16000 -filter_complex "[0:a]pan=stereo|c0=0*c0|c1=1*c0[a]" -map "[a]" -f s16le -acodec pcm_s16le 20200316_1900.pcm + +ffmpeg将一个1通道mp3文件，转换为 2通道,16bit,16000采样率的pcm文件,但是右通道数据为静音 +ffmpeg -i 20200316_1900.mp3 -ac 2 -ar 16000 -filter_complex "[0:a]pan=stereo|c0=1*c0|c1=0*c0[a]" -map "[a]" -f s16le -acodec pcm_s16le 20200316_1900.pcm + + + + +*/ + + +// ./Record --play --path=/sdcard/data/20240904160913.pcm +void PlayerTask::run() { + using namespace Amass; + + RkAudio::Format format; + format.channels = 2; + format.period = 64; + m_output = std::make_shared(); + if (!m_output->open(sizeof(uint16_t), format.sampleRate, format.channels, format.period, false)) { + LOG(error) << "audio output open failed."; + return; + } + play(); +} + +void PlayerTask::play() { + using namespace Amass; + auto ioConext = Singleton::instance(); + if (!m_ifs || !(*m_ifs)) { + LOG(info) << "play finished"; + return; + } + char buffer[2 * sizeof(int16_t) * 16 * 64]; + m_ifs->read(buffer, sizeof(buffer)); + auto readedSize = m_ifs->gcount(); + m_output->write(reinterpret_cast(buffer), readedSize); + + boost::asio::post(*ioConext->ioContext(), [this]() { play(); }); +} \ No newline at end of file diff --git a/Record/Recorder.cpp b/Record/Recorder.cpp new file mode 100644 index 0000000..6c5981e --- /dev/null +++ b/Record/Recorder.cpp @@ -0,0 +1,29 @@ +#include "BoostLog.h" +#include "DateTime.h" +#include "main.h" +#include +#include + +void RecorderTask::run() { + constexpr auto path = "/sdcard/data"; + if (!std::filesystem::exists(path)) { + std::filesystem::create_directory(path); + } + std::ostringstream oss; + oss << path << "/" << DateTime::currentDateTime().toString("%Y%m%d%H%M%S") << ".pcm"; + auto filePath = oss.str(); + m_ofs = std::make_shared(filePath, std::ofstream::binary); + + RkAudio::Format format; + format.channels = 2; + format.period = 64; + m_input = std::make_shared(); + m_input->setDataCallback( + [this](const RkAudio::Frame &frame) { m_ofs->write(reinterpret_cast(frame.data), frame.byteSize); }); + + if (m_input->open(format, false)) { + LOG(info) << "open record succeed, path: " << filePath; + } else { + LOG(info) << "open record failed."; + } +} \ No newline at end of file diff --git a/Record/RkAudio.cpp b/Record/RkAudio.cpp index 98955e2..bc6a2f6 100644 --- a/Record/RkAudio.cpp +++ b/Record/RkAudio.cpp @@ -5,6 +5,10 @@ namespace RkAudio { +constexpr RK_U32 VqeFrameSample = 256; // 16ms; +constexpr auto AudioNode = "hw:0,0"; +constexpr auto ParamFilePath = "/system/etc/RKAP_3A_Para.bin"; + static SAMPLE_FORMAT_E rkAiFormat(Format::SampleType sampleType) { SAMPLE_FORMAT_E ret = RK_SAMPLE_FMT_NONE; switch (sampleType) { @@ -36,53 +40,51 @@ Input::~Input() { stop(); } } -bool Input::open(const Format &format) { +bool Input::open(const Format &format, bool enableVqe) { bool ret = false; - // RK_MPI_SYS_DumpChn(RK_ID_AI); m_channel = 0; AI_CHN_ATTR_S parameter = {0}; - parameter.pcAudioNode = "default"; + parameter.pcAudioNode = (RK_CHAR *)AudioNode; parameter.enAiLayout = AI_LAYOUT_MIC_REF; // remove ref channel, and output mic mono parameter.enSampleFormat = rkAiFormat(format.sampleType); parameter.u32Channels = format.channels; parameter.u32SampleRate = format.sampleRate; parameter.u32NbSamples = format.sampleRate / 1000 * format.period; - int status = RK_MPI_AI_SetChnAttr(m_channel, ¶meter); if (status) { LOG(error) << "RK_MPI_AI_SetChnAttr() failed, status: " << status; return ret; } - + status = RK_MPI_AI_EnableChn(m_channel); if (status) { LOG(error) << "RK_MPI_AI_EnableChn() failed, status: " << status; return ret; } - AI_TALKVQE_CONFIG_S config = {0}; - status = RK_MPI_AI_GetTalkVqeAttr(m_channel, &config); - if (status) { - LOG(error) << "RK_MPI_AI_GetTalkVqeAttr() failed, status: " << status; - return ret; - } - // LOG(info) << "param file: " << config.aParamFilePath; - config.s32WorkSampleRate = format.sampleRate; - config.s32FrameSample = format.sampleRate / 1000 * format.period; - config.u32OpenMask = AI_TALKVQE_MASK_AEC | AI_TALKVQE_MASK_ANR | AI_TALKVQE_MASK_AGC; - strncpy(config.aParamFilePath, ParamFilePath, sizeof(config.aParamFilePath)); - - RK_MPI_AI_SetTalkVqeAttr(m_channel, &config); - fprintf(stderr, "end\n"); - if (status) { - LOG(error) << "RK_MPI_AI_SetTalkVqeAttr() failed, status: " << status; - return ret; - } - status = RK_MPI_AI_EnableVqe(m_channel); - if (status) { - LOG(error) << "RK_MPI_AI_EnableVqe() failed, status: " << status; - return ret; + if (enableVqe) { + AI_TALKVQE_CONFIG_S config = {0}; + status = RK_MPI_AI_GetTalkVqeAttr(m_channel, &config); + if (status) { + LOG(error) << "RK_MPI_AI_GetTalkVqeAttr() failed, status: " << status; + return ret; + } + LOG(info) << "param file: " << config.aParamFilePath; + config.s32WorkSampleRate = format.sampleRate; + config.s32FrameSample = VqeFrameSample; + config.u32OpenMask = AI_TALKVQE_MASK_AEC | AI_TALKVQE_MASK_ANR | AI_TALKVQE_MASK_AGC; + strncpy(config.aParamFilePath, ParamFilePath, sizeof(config.aParamFilePath)); + RK_MPI_AI_SetTalkVqeAttr(m_channel, &config); + if (status) { + LOG(error) << "RK_MPI_AI_SetTalkVqeAttr() failed, status: " << status; + return ret; + } + status = RK_MPI_AI_EnableVqe(m_channel); + if (status) { + LOG(error) << "RK_MPI_AI_EnableVqe() failed, status: " << status; + return ret; + } } status = RK_MPI_AI_StartStream(0); @@ -138,27 +140,29 @@ Output::~Output() { close(); } -bool Output::open(uint32_t sampleSize, uint32_t sampleRate, uint32_t channels) { +bool Output::open(uint32_t sampleSize, uint32_t sampleRate, uint32_t channels, uint32_t period, bool enableVqe) { m_channel = 0; AO_CHN_ATTR_S parameter = {0}; - parameter.pcAudioNode = "default"; + parameter.pcAudioNode = (RK_CHAR *)AudioNode; parameter.enSampleFormat = RK_SAMPLE_FMT_S16; - parameter.u32NbSamples = sampleRate / 1000 * 20; + parameter.u32NbSamples = sampleRate / 1000 * period; parameter.u32SampleRate = sampleRate; parameter.u32Channels = channels; RK_MPI_AO_SetChnAttr(m_channel, ¶meter); RK_MPI_AO_EnableChn(m_channel); - AO_VQE_CONFIG_S config = {0}; - config.s32WorkSampleRate = sampleRate; - config.s32FrameSample = parameter.u32NbSamples; - config.u32OpenMask = AO_VQE_MASK_ANR | AO_VQE_MASK_AGC; - strncpy(config.aParamFilePath, ParamFilePath, sizeof(config.aParamFilePath)); + if (enableVqe) { + AO_VQE_CONFIG_S config = {0}; + config.s32WorkSampleRate = sampleRate; + config.s32FrameSample = VqeFrameSample; + config.u32OpenMask = AO_VQE_MASK_ANR | AO_VQE_MASK_AGC; + strncpy(config.aParamFilePath, ParamFilePath, sizeof(config.aParamFilePath)); - RK_MPI_AO_SetVqeAttr(m_channel, &config); - RK_MPI_AO_EnableVqe(m_channel); + RK_MPI_AO_SetVqeAttr(m_channel, &config); + RK_MPI_AO_EnableVqe(m_channel); + } return true; } @@ -185,3 +189,76 @@ void Output::write(const uint8_t *data, uint32_t byteSize) { } } // namespace RkAudio + +PcmStreamBuffer::PcmStreamBuffer(uint32_t sampleRate, uint32_t channels, RkAudio::Format::SampleType sampleType, uint32_t popDuration, + uint32_t capacity) + : m_sampleRate(sampleRate), m_channels(channels), m_buffer(capacity), m_capacity(capacity) { + if (sampleType == RkAudio::Format::SignedInt16) { + m_pointByteSize = 2; + } + + uint32_t frameSize = sampleRate * channels * m_pointByteSize * popDuration / 1000; + m_popBuffer = std::vector(frameSize); + m_popFrame.data = m_popBuffer.data(); +} + +bool PcmStreamBuffer::push(const RkAudio::Frame &frame) { + std::lock_guard locker(m_mutex); + uint32_t byteSize = availableByteSize(); + uint32_t freeSize = m_capacity - byteSize; + if (freeSize < frame.byteSize) { + LOG_FORMAT(warning, "buffer is full, capacity: %d, free size: %d, need size: %d", m_capacity, freeSize, frame.byteSize); + return false; + } + if ((m_tail + frame.byteSize) > m_capacity) { + + uint32_t size1 = m_capacity - m_tail; + memcpy(m_buffer.data() + m_tail, frame.data, size1); + + uint32_t size2 = frame.byteSize - size1; + memcpy(m_buffer.data(), frame.data + size1, size2); + } else { + memcpy(m_buffer.data() + m_tail, frame.data, frame.byteSize); + } + m_tail = (m_tail + frame.byteSize) % m_capacity; + m_full = (m_tail == m_head); + return true; +} + +std::chrono::milliseconds PcmStreamBuffer::availableDuration() const { + auto byteSize = availableByteSize(); + return std::chrono::milliseconds(1000 * byteSize / (m_sampleRate * m_channels * m_pointByteSize)); +} + +const RkAudio::Frame *PcmStreamBuffer::pop() { + // return nullptr; + std::lock_guard locker(m_mutex); + auto byteSize = availableByteSize(); + + if (byteSize < m_popBuffer.size()) return nullptr; + + if ((m_head + m_popBuffer.size()) > m_capacity) { + uint32_t size1 = m_capacity - m_head; + memcpy(m_popBuffer.data(), m_buffer.data() + m_head, size1); + uint32_t size2 = m_popBuffer.size() - size1; + memcpy(m_popBuffer.data() + size1, m_buffer.data(), size2); + } else { + memcpy(m_popBuffer.data(), m_buffer.data() + m_head, m_popBuffer.size()); + } + m_head = (m_head + m_popBuffer.size()) % m_capacity; + + m_popFrame.byteSize = m_popBuffer.size(); + m_popFrame.frameSize = m_popFrame.byteSize / m_channels / m_pointByteSize; + m_full = false; + return &m_popFrame; +} + +uint32_t PcmStreamBuffer::availableByteSize() const { + if (m_full) { + return m_capacity; + } else if (m_tail >= m_head) { + return m_tail - m_head; + } else { + return m_capacity + m_tail - m_head; + } +} diff --git a/Record/RkAudio.h b/Record/RkAudio.h index aeb3e95..140f3c2 100644 --- a/Record/RkAudio.h +++ b/Record/RkAudio.h @@ -1,12 +1,13 @@ #ifndef __RKAUDIO_H__ #define __RKAUDIO_H__ +#include #include +#include #include +#include namespace RkAudio { -// constexpr auto ParamFilePath="/data/sdcard/rw_3a.bin"; -constexpr auto ParamFilePath = "/data/sdcard/RKAP_3A_Para.bin"; class Format { public: @@ -23,9 +24,9 @@ public: }; SampleType sampleType = SignedInt16; Endian byteOrder = LittleEndian; - uint32_t sampleRate = 48000; - uint32_t channels = 2; - uint32_t period = 60; + uint32_t sampleRate = 16000; + uint32_t channels = 1; + uint32_t period = 16; }; class Frame { @@ -42,7 +43,7 @@ public: Input(); ~Input(); void setDataCallback(const ReadCallback &callback); - bool open(const Format &format); + bool open(const Format &format,bool enableVqe); void stop(); protected: @@ -60,7 +61,7 @@ class Output { public: Output(); ~Output(); - bool open(uint32_t sampleSize, uint32_t sampleRate, uint32_t channels); + bool open(uint32_t sampleSize, uint32_t sampleRate, uint32_t channels, uint32_t period,bool enableVqe); void close(); void write(const uint8_t *data, uint32_t byteSize); @@ -70,4 +71,33 @@ private: } // namespace RkAudio +/** + * @brief 目前系统取录音数据,只能64ms。而opus编码需要20ms一周期 + * + */ +class PcmStreamBuffer { +public: + PcmStreamBuffer(uint32_t sampleRate, uint32_t channels, RkAudio::Format::SampleType sampleType, uint32_t popDuration, + uint32_t capacity); + bool push(const RkAudio::Frame &frame); + std::chrono::milliseconds availableDuration() const; + uint32_t availableByteSize() const; + const RkAudio::Frame *pop(); + +private: + uint32_t m_sampleRate; + uint32_t m_channels; + uint32_t m_pointByteSize = 0; + + std::vector m_buffer; + uint32_t m_capacity; + uint32_t m_head = 0; + uint32_t m_tail = 0; + bool m_full = false; + + RkAudio::Frame m_popFrame; + std::vector m_popBuffer; + std::mutex m_mutex; +}; + #endif // __RKAUDIO_H__ \ No newline at end of file diff --git a/Record/RkRecorder.cpp b/Record/RkRecorder.cpp deleted file mode 100644 index 0c58541..0000000 --- a/Record/RkRecorder.cpp +++ /dev/null @@ -1,226 +0,0 @@ -#include "BoostLog.h" -#include -#include -#include -#include - -#define VQEFILE "/data/sdcard/RKAP_3A_Para.bin" -#define ALSA_PATH "default" // get from "arecord -L" - -static bool quit = false; -static void sigterm_handler(int sig) { - fprintf(stderr, "signal %d\n", sig); - quit = true; -} - -std::shared_ptr ofs; - -void run() { - ofs = std::make_shared("/data/sdcard/test.pcm", std::ofstream::binary); - while (!quit) { - auto mediaBuffer = RK_MPI_SYS_GetMediaBuffer(RK_ID_AI, 0, -1); - if (!mediaBuffer) { - LOG(error) << "RK_MPI_SYS_GetMediaBuffer() failed."; - continue; - } - LOG(info) << "get frame, timestamp: " << RK_MPI_MB_GetTimestamp(mediaBuffer) - << ", size: " << RK_MPI_MB_GetSize(mediaBuffer); - ofs->write((const char *)RK_MPI_MB_GetPtr(mediaBuffer), RK_MPI_MB_GetSize(mediaBuffer)); - RK_MPI_MB_ReleaseBuffer(mediaBuffer); - } - ofs.reset(); -} - -void rkDemo() { - AI_CHN_ATTR_S ai_attr; - ai_attr.pcAudioNode = "default"; - ai_attr.enSampleFormat = RK_SAMPLE_FMT_S16; - ai_attr.u32NbSamples = 16 * 20; - ai_attr.u32SampleRate = 16000; - ai_attr.u32Channels = 2; - ai_attr.enAiLayout = AI_LAYOUT_REF_MIC; - int status = RK_MPI_AI_SetChnAttr(0, &ai_attr); - status |= RK_MPI_AI_EnableChn(0); - if (status) { - LOG(error) << "enable AI[0] failed, status = " << status; - return; - } - - AI_TALKVQE_CONFIG_S config = {0}; - strcpy(config.aParamFilePath, VQEFILE); - config.s32WorkSampleRate = 16000; - config.s32FrameSample = 16 * 20; - config.u32OpenMask = AI_TALKVQE_MASK_AEC | AI_TALKVQE_MASK_ANR | AI_TALKVQE_MASK_AGC; - status = RK_MPI_AI_SetTalkVqeAttr(0, &config); - if (status) { - LOG(error) << "RK_MPI_AI_SetTalkVqeAttr() failed, status: " << status; - return; - } - - status = RK_MPI_AI_EnableVqe(0); - if (status) { - LOG(error) << "RK_MPI_AI_EnableVqe() failed, status: " << status; - return; - } - - std::thread thread(&run); - - status = RK_MPI_AI_StartStream(0); - if (status) { - LOG(info) << "start AI failed, status: " << status; - return; - } - signal(SIGINT, sigterm_handler); - while (!quit) { - usleep(500000); - } - - if (thread.joinable()) { - thread.join(); - } - - RK_MPI_AI_DisableChn(0); -} - -static RK_U32 g_enWorkSampleRate = 8000; -static RK_U32 g_s32VqeFrameSample = 320; // 20ms; - -// #define VQEFILE "/data/sdcard/3a.bin" -int AI_VqeProcess_AO() { - AI_TALKVQE_CONFIG_S stAiVqeTalkAttr; - AI_RECORDVQE_CONFIG_S stAiVqeRecordAttr; - AO_VQE_CONFIG_S stAoVqeAttr; - MPP_CHN_S mpp_chn_ai, mpp_chn_ao; - - memset(&stAiVqeTalkAttr, 0, sizeof(AI_TALKVQE_CONFIG_S)); - stAiVqeTalkAttr.s32WorkSampleRate = g_enWorkSampleRate; - stAiVqeTalkAttr.s32FrameSample = g_s32VqeFrameSample; - strcpy(stAiVqeTalkAttr.aParamFilePath, VQEFILE); - stAiVqeTalkAttr.u32OpenMask = AI_TALKVQE_MASK_AEC | AI_TALKVQE_MASK_ANR | AI_TALKVQE_MASK_AGC; - // stAiVqeTalkAttr.u32OpenMask = AI_TALKVQE_MASK_AEC ; - - memset(&stAoVqeAttr, 0, sizeof(AO_VQE_CONFIG_S)); - stAoVqeAttr.s32WorkSampleRate = g_enWorkSampleRate; - stAoVqeAttr.s32FrameSample = g_s32VqeFrameSample; - strcpy(stAoVqeAttr.aParamFilePath, VQEFILE); - stAoVqeAttr.u32OpenMask = AO_VQE_MASK_ANR | AO_VQE_MASK_AGC; - - mpp_chn_ai.enModId = RK_ID_AI; - mpp_chn_ai.s32ChnId = 0; - mpp_chn_ao.enModId = RK_ID_AO; - mpp_chn_ao.s32ChnId = 0; - - AI_CHN_ATTR_S ai_attr; - ai_attr.pcAudioNode = "default"; - ai_attr.enSampleFormat = RK_SAMPLE_FMT_S16; - ai_attr.u32NbSamples = 320; - ai_attr.u32SampleRate = g_enWorkSampleRate; - ai_attr.u32Channels = 1; - ai_attr.enAiLayout = AI_LAYOUT_MIC_REF; // remove ref channel, and output mic mono - - AO_CHN_ATTR_S ao_attr; - ao_attr.pcAudioNode = "default"; - ao_attr.enSampleFormat = RK_SAMPLE_FMT_S16; - ao_attr.u32NbSamples = 320; - ao_attr.u32SampleRate = g_enWorkSampleRate; - ao_attr.u32Channels = 1; - - // 1. create AI - RK_MPI_AI_SetChnAttr(mpp_chn_ai.s32ChnId, &ai_attr); - RK_MPI_AI_EnableChn(mpp_chn_ai.s32ChnId); - - RK_MPI_AI_SetTalkVqeAttr(mpp_chn_ai.s32ChnId, &stAiVqeTalkAttr); - RK_MPI_AI_EnableVqe(mpp_chn_ai.s32ChnId); - - // 2. create AO - RK_MPI_AO_SetChnAttr(mpp_chn_ao.s32ChnId, &ao_attr); - RK_MPI_AO_EnableChn(mpp_chn_ao.s32ChnId); - - RK_MPI_AO_SetVqeAttr(mpp_chn_ao.s32ChnId, &stAoVqeAttr); - RK_MPI_AO_EnableVqe(mpp_chn_ao.s32ChnId); - - // 3. bind AI-AO - RK_MPI_SYS_Bind(&mpp_chn_ai, &mpp_chn_ao); - - printf("%s initial finish\n", __func__); - signal(SIGINT, sigterm_handler); - while (!quit) { - usleep(500000); - } - - printf("%s exit!\n", __func__); - RK_MPI_SYS_UnBind(&mpp_chn_ai, &mpp_chn_ao); - RK_MPI_AI_DisableChn(mpp_chn_ai.s32ChnId); - RK_MPI_AO_DisableChn(mpp_chn_ao.s32ChnId); - - return RK_SUCCESS; -} - -int AI_VqeProcess_AO1() { - AI_TALKVQE_CONFIG_S stAiVqeTalkAttr; - AI_RECORDVQE_CONFIG_S stAiVqeRecordAttr; - AO_VQE_CONFIG_S stAoVqeAttr; - MPP_CHN_S mpp_chn_ai, mpp_chn_ao; - - memset(&stAiVqeTalkAttr, 0, sizeof(AI_TALKVQE_CONFIG_S)); - stAiVqeTalkAttr.s32WorkSampleRate = g_enWorkSampleRate; - stAiVqeTalkAttr.s32FrameSample = g_s32VqeFrameSample; - strcpy(stAiVqeTalkAttr.aParamFilePath, VQEFILE); - stAiVqeTalkAttr.u32OpenMask = AI_TALKVQE_MASK_AEC | AI_TALKVQE_MASK_ANR | AI_TALKVQE_MASK_AGC; - - memset(&stAoVqeAttr, 0, sizeof(AO_VQE_CONFIG_S)); - stAoVqeAttr.s32WorkSampleRate = g_enWorkSampleRate; - stAoVqeAttr.s32FrameSample = g_s32VqeFrameSample; - strcpy(stAoVqeAttr.aParamFilePath, VQEFILE); - stAoVqeAttr.u32OpenMask = AO_VQE_MASK_ANR | AO_VQE_MASK_AGC; - - RK_MPI_SYS_Init(); - mpp_chn_ai.enModId = RK_ID_AI; - mpp_chn_ai.s32ChnId = 0; - mpp_chn_ao.enModId = RK_ID_AO; - mpp_chn_ao.s32ChnId = 0; - - AI_CHN_ATTR_S ai_attr; - ai_attr.pcAudioNode = ALSA_PATH; - ai_attr.enSampleFormat = RK_SAMPLE_FMT_S16; - ai_attr.u32NbSamples = 640; - ai_attr.u32SampleRate = g_enWorkSampleRate; - ai_attr.u32Channels = 1; - ai_attr.enAiLayout = AI_LAYOUT_MIC_REF; // remove ref channel, and output mic mono - - AO_CHN_ATTR_S ao_attr; - ao_attr.pcAudioNode = ALSA_PATH; - ao_attr.enSampleFormat = RK_SAMPLE_FMT_S16; - ao_attr.u32NbSamples = 640; - ao_attr.u32SampleRate = g_enWorkSampleRate; - ao_attr.u32Channels = 1; - - // 1. create AI - RK_MPI_AI_SetChnAttr(mpp_chn_ai.s32ChnId, &ai_attr); - RK_MPI_AI_EnableChn(mpp_chn_ai.s32ChnId); - - RK_MPI_AI_SetTalkVqeAttr(mpp_chn_ai.s32ChnId, &stAiVqeTalkAttr); - RK_MPI_AI_EnableVqe(mpp_chn_ai.s32ChnId); - - // 2. create AO - RK_MPI_AO_SetChnAttr(mpp_chn_ao.s32ChnId, &ao_attr); - RK_MPI_AO_EnableChn(mpp_chn_ao.s32ChnId); - RK_MPI_AO_SetVqeAttr(mpp_chn_ao.s32ChnId, &stAoVqeAttr); - RK_MPI_AO_EnableVqe(mpp_chn_ao.s32ChnId); - - // 3. bind AI-AO - RK_MPI_SYS_Bind(&mpp_chn_ai, &mpp_chn_ao); - - printf("%s initial finish\n", __func__); - signal(SIGINT, sigterm_handler); - while (!quit) { - usleep(500000); - } - - printf("%s exit!\n", __func__); - RK_MPI_SYS_UnBind(&mpp_chn_ai, &mpp_chn_ao); - RK_MPI_AI_DisableChn(mpp_chn_ai.s32ChnId); - RK_MPI_AO_DisableChn(mpp_chn_ao.s32ChnId); - - return RK_SUCCESS; -} \ No newline at end of file diff --git a/Record/SpeexDsp.cpp b/Record/SpeexDsp.cpp new file mode 100644 index 0000000..1e399f9 --- /dev/null +++ b/Record/SpeexDsp.cpp @@ -0,0 +1,15 @@ +#include "SpeexDsp.h" +#include +#include + +void SpeexDsp::reset() { + if (m_state != nullptr) { + speex_echo_state_reset(m_state); + } +} + +void SpeexDsp::preprocess(int16_t *pcm) { + if (m_preprocessState != nullptr) { + speex_preprocess_run(m_preprocessState, pcm); + } +} diff --git a/Record/SpeexDsp.h b/Record/SpeexDsp.h new file mode 100644 index 0000000..a18d21c --- /dev/null +++ b/Record/SpeexDsp.h @@ -0,0 +1,19 @@ +#ifndef __SPEEXDSP_H__ +#define __SPEEXDSP_H__ + +#include + +typedef struct SpeexEchoState_ SpeexEchoState; +typedef struct SpeexPreprocessState_ SpeexPreprocessState; + +class SpeexDsp { +public: +void preprocess(int16_t *pcm); + void reset(); + +private: + SpeexEchoState *m_state = nullptr; + SpeexPreprocessState *m_preprocessState=nullptr; +}; + +#endif // __SPEEXDSP_H__ \ No newline at end of file diff --git a/Record/WebRTCPublisher.cpp b/Record/WebRTCPublisher.cpp new file mode 100644 index 0000000..03e178c --- /dev/null +++ b/Record/WebRTCPublisher.cpp @@ -0,0 +1,221 @@ +#include "WebRTCPublisher.h" +#include "BoostLog.h" +#include "IoContext.h" +#include "NetworkUtility.h" +#include +#include +#include +#include + +class WebRTCPublisherPrivate { +public: + WebRTCPublisherPrivate() { + m_exit = false; + m_thread = std::thread(&WebRTCPublisherPrivate::run, this); + } + ~WebRTCPublisherPrivate() { + m_exit = true; + if (m_thread.joinable()) { + m_thread.join(); + } + } + void run() { + int i = 0; + int opusIndex = 0; + int opusT = 0; + int t = 0; + while (!m_exit) { + auto begin = std::chrono::system_clock::now(); + + if (i > 1500) i = 0; + if (opusIndex > 618) opusIndex = 0; + if (!m_connected) continue; + + { + std::ostringstream oss; + oss << "/data/sdcard/lib/h264SampleFrames/frame-" << std::setfill('0') << std::setw(4) << i++ << ".h264"; + + std::ifstream ifs(oss.str(), std::ifstream::binary); + auto buffer = std::vector((std::istreambuf_iterator(ifs)), std::istreambuf_iterator()); + + Frame frame; + t += 40 * 1000 * 10; + frame.presentationTs = t; + frame.decodingTs = t; + frame.frameData = buffer.data(); + frame.size = buffer.size(); + STATUS retStatus = writeFrame(videoReceiver, &frame); + } + + { + std::ostringstream oss; + oss << "/data/sdcard/lib/opusSampleFrames/sample-" << std::setfill('0') << std::setw(3) << opusIndex << ".opus"; + std::ifstream ifs(oss.str(), std::ifstream::binary); + auto buffer = std::vector((std::istreambuf_iterator(ifs)), std::istreambuf_iterator()); + Frame frame; + frame.index = opusT; + opusT += 20 * HUNDREDS_OF_NANOS_IN_A_MILLISECOND; + frame.presentationTs = opusT; + frame.decodingTs = opusT; + frame.frameData = buffer.data(); + frame.size = buffer.size(); + STATUS retStatus = writeFrame(audioReceiver, &frame); + LOG(info) << "send opus " << opusIndex; + opusIndex++; + } + while (true) { + auto ele = std::chrono::duration_cast(std::chrono::system_clock::now() - begin); + auto s = std::chrono::milliseconds(20) - ele; + if (s.count() > 0) { + continue; + } else { + break; + } + } + } + } + RtcPeerConnection *peer = nullptr; + RtcRtpTransceiver *videoReceiver = nullptr; + RtcRtpTransceiver *audioReceiver = nullptr; + RtcSessionDescriptionInit offer; + static void onConnectionStateChange(UINT64 userData, RTC_PEER_CONNECTION_STATE state) { + auto self = reinterpret_cast(userData); + LOG(info) << "connection state: " << state; + if (state == RTC_PEER_CONNECTION_STATE_CONNECTED) { + self->m_connected = true; + } + } + + std::thread m_thread; + bool m_exit = true; + bool m_connected = false; +}; + +WebRTCPublisher::WebRTCPublisher(bool videoEnabled, bool audioEnabled) + : m_d(new WebRTCPublisherPrivate()), m_videoEnabled(videoEnabled), m_audioEnabled(audioEnabled) { +} + +WebRTCPublisher::~WebRTCPublisher() { + stop(); + if (m_d != nullptr) { + delete m_d; + } +} + +bool WebRTCPublisher::start(const std::string &address, const std::string &port, const std::string &url) { + using namespace Amass; + stop(); + + RtcConfiguration configuration{}; + memset(&configuration, 0, sizeof(configuration)); + STATUS status = createPeerConnection(&configuration, &m_d->peer); + if (status != STATUS_SUCCESS) { + LOG(error) << "createPeerConnection() failed, status: " << status; + return false; + } + + if (m_videoEnabled) { + RtcMediaStreamTrack track{}; + track.codec = RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE; + track.kind = MEDIA_STREAM_TRACK_KIND_VIDEO; + strncpy(track.streamId, "0", sizeof(track.streamId)); + strncpy(track.trackId, "0", sizeof(track.trackId)); + + RtcRtpTransceiverInit init; + init.direction = RTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY; + status = addTransceiver(m_d->peer, &track, &init, &m_d->videoReceiver); + if (status != STATUS_SUCCESS) { + LOG(error) << "addTransceiver() failed, status: " << status; + return false; + } + } + + if (m_audioEnabled) { + RtcMediaStreamTrack track{}; + track.codec = RTC_CODEC_OPUS; + track.kind = MEDIA_STREAM_TRACK_KIND_AUDIO; + strncpy(track.streamId, "1", sizeof(track.streamId)); + strncpy(track.trackId, "1", sizeof(track.trackId)); + + RtcRtpTransceiverInit init; + init.direction = RTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY; + status = addTransceiver(m_d->peer, &track, &init, &m_d->audioReceiver); + if (status != STATUS_SUCCESS) { + LOG(error) << "addTransceiver() failed, status: " << status; + return false; + } + } + + status = createOffer(m_d->peer, &m_d->offer); + if (status != STATUS_SUCCESS) { + LOG(error) << "createOffer() failed, status: " << status; + + return false; + } + + status = setLocalDescription(m_d->peer, &m_d->offer); + if (status != STATUS_SUCCESS) { + LOG(error) << "setLocalDescription() failed, status: " << status; + return false; + } + // LOG(info) << "sdp: \n" << m_d->offer.sdp; + std::string sdp; + + boost::system::error_code error; + auto ioContext = Singleton::instance(); + Http::Client client(*ioContext->ioContext(), Http::SSL); + client.loadRootCertificates(error); + auto reply = client.post(address, port, url, m_d->offer.sdp, error); + if (error) { + LOG(error) << "post error: " << error.message(); + return false; + } + auto replyValue = boost::json::parse(reply, error); + if (error) { + LOG(info) << reply; + LOG(error) << error.message(); + } + auto replyObject = replyValue.as_object(); + sdp = std::string(replyObject.at("sdp").as_string()); + + // replyObject.at("sdp").as_string().c_str(); + // LOG(info) << reply; + LOG(info) << sdp; + RtcSessionDescriptionInit answer{}; + answer.type = SDP_TYPE_ANSWER; + strncpy(answer.sdp, sdp.c_str(), sizeof(answer.sdp)); + status = setRemoteDescription(m_d->peer, &answer); + if (status != STATUS_SUCCESS) { + LOG(error) << "setRemoteDescription() failed, status: " << status; + return false; + } + + status = + peerConnectionOnConnectionStateChange(m_d->peer, reinterpret_cast(m_d), &WebRTCPublisherPrivate::onConnectionStateChange); + if (status != STATUS_SUCCESS) { + LOG(error) << "peerConnectionOnConnectionStateChange() failed, status: " << status; + return false; + } + + return true; +} + +void WebRTCPublisher::stop() { + if (m_d->peer != nullptr) { + closePeerConnection(m_d->peer); + if (m_d->videoReceiver != nullptr) { + freeTransceiver(&m_d->videoReceiver); + m_d->videoReceiver = nullptr; + } + if (m_d->audioReceiver != nullptr) { + freeTransceiver(&m_d->audioReceiver); + m_d->audioReceiver = nullptr; + } + + freePeerConnection(&m_d->peer); + m_d->peer = nullptr; + } +} + +void WebRTCPublisher::exchangeSdp() { +} diff --git a/Record/WebRTCPublisher.h b/Record/WebRTCPublisher.h new file mode 100644 index 0000000..a282971 --- /dev/null +++ b/Record/WebRTCPublisher.h @@ -0,0 +1,26 @@ +#ifndef WEBRTCPUBLISHER_H +#define WEBRTCPUBLISHER_H + +#include + +class WebRTCPublisherPrivate; + +class WebRTCPublisher +{ +public: + WebRTCPublisher(bool videoEnabled, bool audioEnabled); + ~WebRTCPublisher(); + bool start(const std::string &address, const std::string &port, const std::string &url); + + void stop(); + +protected: + void exchangeSdp(); + +private: + WebRTCPublisherPrivate *m_d = nullptr; + bool m_videoEnabled = false; + bool m_audioEnabled = false; +}; + +#endif // WEBRTCPUBLISHER_H diff --git a/Record/main.cpp b/Record/main.cpp index 130856a..5e7884f 100644 --- a/Record/main.cpp +++ b/Record/main.cpp @@ -1,61 +1,110 @@ +#include "main.h" +#include "BoostLog.h" +#include "DateTime.h" #include "FFmpegResample.h" +#include "IoContext.h" #include "OpusCodec.h" +#include "RkAudio.h" +#include "WebRTCPublisher.h" +#include +// #include +#include +#include +#include +#include #include #include -extern int recorder_demo(); extern void rkDemo(); extern int AI_VqeProcess_AO(); extern int AI_VqeProcess_AO1(); extern void AecTest(); extern int opus_test(); +void signal_handler(const boost::system::error_code &error, int signal_number) { + if (!error) { + LOG(info) << "Caught signal: " << signal_number << std::endl; + + LOG(info) << "task finished."; + std::exit(0); + } +} + int main(int argc, char **argv) { - RK_MPI_SYS_Init(); - // std::ifstream ifs("/data/sdcard/test2.pcm", std::ifstream::binary); - // AlsaPcmPlayer player; - // player.open(2, 48000, 2); + using namespace Amass; + using namespace std::placeholders; + boost::program_options::options_description optionsDescription("Allowed options"); + // clang-format off + optionsDescription.add_options() + ("help,h", "produce help message") + ("echo", "Self-recording and self-play test") + ("record", "Record to file.") + ("play", "Play pcm file.") + ("vqe", boost::program_options::value(), "Enable rk 3a.") + ("channels", boost::program_options::value(), "set audio channles") + ("path", boost::program_options::value(), "file path") + ; + // clang-format on - // char buffer[2 * 2 * 48 * 60]; - // while (ifs.read(buffer, sizeof(buffer))) { - // int size = ifs.gcount(); - // player.write((const uint8_t *)buffer, size); - // } - // recorder_demo(); - // rkDemo(); - // AI_VqeProcess_AO(); - // AI_VqeProcess_AO1(); - // AecTest(); + boost::program_options::variables_map variablesMap; + boost::program_options::store(boost::program_options::parse_command_line(argc, argv, optionsDescription), variablesMap); + boost::program_options::notify(variablesMap); - { - FFmpegResample resample; - resample.initialize(16000, 1, 48000, 2, 20); - - std::ifstream ifs("/sdcard/input.pcm", std::ifstream::binary); - std::ofstream ofs("/sdcard/my_48kz.pcm", std::ifstream::binary); - char buffer[16 * 20 * 2]; - while (ifs.read(buffer, sizeof(buffer))) { - auto frame = resample.resample((uint8_t *)buffer, sizeof(buffer)); - if (frame.data != nullptr) { - ofs.write((char *)frame.data, frame.byteSize); - } - } + if (variablesMap.count("help")) { + std::cout << optionsDescription << std::endl; + return 1; } - { - FFmpegResample resample1; - resample1.initialize(48000, 2, 16000, 1, 20); - - std::ifstream ifs1("/sdcard/my_48kz.pcm", std::ifstream::binary); - std::ofstream ofs1("/sdcard/my_16kz.pcm", std::ifstream::binary); - char buffer1[48 * 20 * 2 * 2]; - while (ifs1.read(buffer1, sizeof(buffer1))) { - auto frame = resample1.resample((uint8_t *)buffer1, sizeof(buffer1)); - if (frame.data != nullptr) { - ofs1.write((char *)frame.data, frame.byteSize); - } + std::shared_ptr task; + if (variablesMap.count("echo")) { + bool vqe = false; + if (variablesMap.count("vqe")) { + vqe = variablesMap["vqe"].as(); } + + int channels = 2; + if (variablesMap.count("channels")) { + channels = variablesMap["channels"].as(); + } + + auto t = std::make_shared(); + t->setVqeEnabled(vqe); + t->setChannels(channels); + task = std::dynamic_pointer_cast(t); + } else if (variablesMap.count("record")) { + task = std::make_shared(); + } else if (variablesMap.count("play")) { + std::string path; + if (variablesMap.count("path")) { + path = variablesMap["path"].as(); + } + auto t = std::make_shared(); + t->setPath(path); + task = std::dynamic_pointer_cast(t); } + if (!task) { + std::cout << "there is not task." << std::endl << std::endl; + std::cout << optionsDescription << std::endl; + return 2; + } + + try { + LOG(info) << "app start."; + RK_MPI_SYS_Init(); + initKvsWebRtc(); + auto ioConext = Singleton::instance(); + boost::asio::signal_set signals(*ioConext->ioContext(), SIGINT, SIGTERM); + signals.async_wait(std::bind(&signal_handler, _1, _2)); + + task->run(); + + ioConext->run(); + } catch (std::exception &e) { + LOG(error) << "Exception: " << e.what() << std::endl; + } + + // WebRTCPublisher publisher(true, true); + // publisher.start("172.16.103.68", "443", "/index/api/webrtc?app=live&stream=test&type=push"); return 0; } \ No newline at end of file diff --git a/Record/main.h b/Record/main.h new file mode 100644 index 0000000..6289dec --- /dev/null +++ b/Record/main.h @@ -0,0 +1,49 @@ +#ifndef __MAIN_H__ +#define __MAIN_H__ + +#include "RkAudio.h" +#include +#include + +class Task { +public: + virtual void run() = 0; +}; + +class RecorderTask : public Task { +public: + void run() final; + +private: + std::shared_ptr m_input; + std::shared_ptr m_ofs; +}; + +class PlayerTask : public Task { +public: + void setPath(const std::string &path); + void run() final; + +protected: + void play(); + +private: + std::string m_path; + std::shared_ptr m_ifs; + std::shared_ptr m_output; +}; + +class EchoRecordTask : public Task { +public: + void setVqeEnabled(bool enabled); + void setChannels(int channels); + void run() final; + +private: + int m_channels = 2; + bool m_vqeEnabled = false; + std::shared_ptr m_output; + std::shared_ptr m_input; +}; + +#endif // __MAIN_H__ \ No newline at end of file diff --git a/Record/rkmedia_audio_test.c b/Record/rkmedia_audio_test.c deleted file mode 100644 index 51d7bbc..0000000 --- a/Record/rkmedia_audio_test.c +++ /dev/null @@ -1,411 +0,0 @@ -// Copyright 2020 Fuzhou Rockchip Electronics Co., Ltd. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#define MP3_NB_SAMPLES 1024 -#define MP2_NB_SAMPLES 1152 -//#define ALSA_PATH "default:CARD=rockchiprk809co" // get from "arecord -L" -#define ALSA_PATH "hw:0,0" // get from "arecord -L" -#define VQEFILE "/sdcard/RKAP_3A_Para.bin" - -static bool quit = false; -static void sigterm_handler(int sig) { - fprintf(stderr, "signal %d\n", sig); - quit = true; -} - -FILE *fp = NULL; -static RK_U32 g_enWorkSampleRate = 16000; -static RK_U32 g_s32VqeFrameSample = 256; // 20ms; -static RK_U32 g_s32AiLayout = AI_LAYOUT_MIC_REF; - -static void audio_packet_cb(MEDIA_BUFFER mb) { - printf("Get Audio Encoded packet:ptr:%p, fd:%d, size:%zu, mode:%d\n", - RK_MPI_MB_GetPtr(mb), RK_MPI_MB_GetFD(mb), RK_MPI_MB_GetSize(mb), - RK_MPI_MB_GetModeID(mb)); - fwrite(RK_MPI_MB_GetPtr(mb), 1, RK_MPI_MB_GetSize(mb), fp); - RK_MPI_MB_ReleaseBuffer(mb); -} - -static RK_VOID AI_AO() { - RK_MPI_SYS_Init(); - MPP_CHN_S mpp_chn_ai, mpp_chn_ao; - mpp_chn_ai.enModId = RK_ID_AI; - mpp_chn_ai.s32ChnId = 0; - mpp_chn_ao.enModId = RK_ID_AO; - mpp_chn_ao.s32ChnId = 0; - - AI_CHN_ATTR_S ai_attr; - ai_attr.pcAudioNode = ALSA_PATH; - ai_attr.enSampleFormat = RK_SAMPLE_FMT_S16; - ai_attr.u32NbSamples = 1152; - ai_attr.u32SampleRate = g_enWorkSampleRate; - ai_attr.u32Channels = 1; - ai_attr.enAiLayout = g_s32AiLayout; // chanel layout: [ref:mic]; remove - // ref, output mic mono - - AO_CHN_ATTR_S ao_attr; - ao_attr.pcAudioNode = ALSA_PATH; - ao_attr.enSampleFormat = RK_SAMPLE_FMT_S16; - ao_attr.u32NbSamples = 1152; - ao_attr.u32SampleRate = g_enWorkSampleRate; - ao_attr.u32Channels = 1; - - // 1. create AI - RK_MPI_AI_SetChnAttr(mpp_chn_ai.s32ChnId, &ai_attr); - RK_MPI_AI_EnableChn(mpp_chn_ai.s32ChnId); - RK_MPI_AI_SetVolume(mpp_chn_ai.s32ChnId, 100); - - // 2. create AO - RK_MPI_AO_SetChnAttr(mpp_chn_ao.s32ChnId, &ao_attr); - RK_MPI_AO_EnableChn(mpp_chn_ao.s32ChnId); - RK_MPI_AO_SetVolume(mpp_chn_ao.s32ChnId, 100); - - // 3. bind AI-AO - RK_MPI_SYS_Bind(&mpp_chn_ai, &mpp_chn_ao); - - printf("%s initial finish\n", __func__); - signal(SIGINT, sigterm_handler); - while (!quit) { - usleep(500000); - } - - printf("%s exit!\n", __func__); - RK_MPI_SYS_UnBind(&mpp_chn_ai, &mpp_chn_ao); - RK_MPI_AI_DisableChn(mpp_chn_ai.s32ChnId); - RK_MPI_AO_DisableChn(mpp_chn_ao.s32ChnId); -} - -static RK_VOID AI_AENC_FILE(char *file_path) { - fp = fopen(file_path, "w+"); - RK_MPI_SYS_Init(); - MPP_CHN_S mpp_chn_ai, mpp_chn_aenc; - mpp_chn_ai.enModId = RK_ID_AI; - mpp_chn_ai.s32ChnId = 0; - mpp_chn_aenc.enModId = RK_ID_AENC; - mpp_chn_aenc.s32ChnId = 0; - - AI_CHN_ATTR_S ai_attr; - ai_attr.pcAudioNode = ALSA_PATH; - ai_attr.enSampleFormat = RK_SAMPLE_FMT_S16; - ai_attr.u32NbSamples = MP3_NB_SAMPLES; - ai_attr.u32SampleRate = g_enWorkSampleRate; - ai_attr.u32Channels = 1; - ai_attr.enAiLayout = g_s32AiLayout; // chanel layout: [ref:mic]; remove - // ref, output mic mono - - AENC_CHN_ATTR_S aenc_attr; - aenc_attr.enCodecType = RK_CODEC_TYPE_MP3; - aenc_attr.u32Bitrate = 64000; - aenc_attr.u32Quality = 1; - aenc_attr.stAencMP3.u32Channels = 1; - aenc_attr.stAencMP3.u32SampleRate = g_enWorkSampleRate; - - // 1. create AI - RK_MPI_AI_SetChnAttr(mpp_chn_ai.s32ChnId, &ai_attr); - RK_MPI_AI_EnableChn(mpp_chn_ai.s32ChnId); - - // 2. create AENC - RK_MPI_AENC_CreateChn(mpp_chn_aenc.s32ChnId, &aenc_attr); - RK_U32 ret = RK_MPI_SYS_RegisterOutCb(&mpp_chn_aenc, audio_packet_cb); - printf("ret = %d.\n", ret); - - // 3. bind AI-AENC - RK_MPI_SYS_Bind(&mpp_chn_ai, &mpp_chn_aenc); - - printf("%s initial finish\n", __func__); - signal(SIGINT, sigterm_handler); - while (!quit) { - usleep(500000); - } - - RK_MPI_SYS_UnBind(&mpp_chn_ai, &mpp_chn_aenc); - RK_MPI_AI_DisableChn(mpp_chn_ai.s32ChnId); - RK_MPI_AENC_DestroyChn(mpp_chn_aenc.s32ChnId); - fclose(fp); -} - -static RK_VOID FILE_ADEC_AO(char *file_path) { - CODEC_TYPE_E codec_type = RK_CODEC_TYPE_MP3; - RK_U32 channels = 2; - RK_U32 sample_rate = g_enWorkSampleRate; - ADEC_CHN_ATTR_S stAdecAttr; - AO_CHN_ATTR_S stAoAttr; - - stAdecAttr.enCodecType = codec_type; - MPP_CHN_S mpp_chn_ao, mpp_chn_adec; - mpp_chn_ao.enModId = RK_ID_AO; - mpp_chn_ao.s32ChnId = 0; - mpp_chn_adec.enModId = RK_ID_ADEC; - mpp_chn_adec.s32ChnId = 0; - - stAoAttr.u32Channels = channels; - stAoAttr.u32SampleRate = sample_rate; - stAoAttr.u32NbSamples = 1024; - stAoAttr.pcAudioNode = ALSA_PATH; - - switch (codec_type) { - case RK_CODEC_TYPE_MP3: - stAoAttr.enSampleFormat = RK_SAMPLE_FMT_S16; - stAoAttr.u32NbSamples = 1024; - break; - case RK_CODEC_TYPE_MP2: - stAoAttr.enSampleFormat = RK_SAMPLE_FMT_S16; - stAoAttr.u32NbSamples = 1152; - break; - case RK_CODEC_TYPE_G711A: - stAdecAttr.stAdecG711A.u32Channels = channels; - stAdecAttr.stAdecG711A.u32SampleRate = sample_rate; - stAoAttr.enSampleFormat = RK_SAMPLE_FMT_S16; - break; - case RK_CODEC_TYPE_G711U: - stAdecAttr.stAdecG711U.u32Channels = channels; - stAdecAttr.stAdecG711U.u32SampleRate = sample_rate; - stAoAttr.enSampleFormat = RK_SAMPLE_FMT_S16; - break; - case RK_CODEC_TYPE_G726: - stAoAttr.enSampleFormat = RK_SAMPLE_FMT_S16; - break; - default: - printf("audio codec type error.\n"); - return; - } - // init MPI - RK_MPI_SYS_Init(); - // create ADEC - RK_MPI_ADEC_CreateChn(mpp_chn_adec.s32ChnId, &stAdecAttr); - // create AO - RK_MPI_AO_SetChnAttr(mpp_chn_ao.s32ChnId, &stAoAttr); - RK_MPI_AO_EnableChn(mpp_chn_ao.s32ChnId); - - RK_MPI_SYS_Bind(&mpp_chn_adec, &mpp_chn_ao); - - RK_S32 buffer_size = 20480; - FILE *read_file = fopen(file_path, "r"); - if (!read_file) { - printf("ERROR: open %s failed!\n", file_path); - exit(0); - } - quit = true; - while (quit) { - MEDIA_BUFFER mb = RK_MPI_MB_CreateAudioBuffer(buffer_size, RK_FALSE); - if (!mb) { - printf("ERROR: no space left!\n"); - break; - } - - RK_S32 s32ReadSize = fread(RK_MPI_MB_GetPtr(mb), 1, buffer_size, read_file); - - RK_MPI_MB_SetSize(mb, s32ReadSize); - RK_MPI_SYS_SendMediaBuffer(RK_ID_ADEC, mpp_chn_adec.s32ChnId, mb); - RK_MPI_MB_ReleaseBuffer(mb); - if (s32ReadSize != buffer_size) { - printf("Get end of file!\n"); - break; - } - } - sleep(2); - { - // flush decoder - printf("start flush decoder.\n"); - MEDIA_BUFFER mb = RK_MPI_MB_CreateAudioBuffer(buffer_size, RK_FALSE); - RK_MPI_MB_SetSize(mb, 0); - RK_MPI_SYS_SendMediaBuffer(RK_ID_ADEC, mpp_chn_adec.s32ChnId, mb); - RK_MPI_MB_ReleaseBuffer(mb); - printf("end flush decoder.\n"); - } - - sleep(10); -} - -/* 0: close, 1: talk, 2: record */ -static RK_U32 u32AiVqeType = 1; -/* 0: close, 1: open */ -static RK_U32 u32AoVqeType = 1; - -/****************************************************************************** - * function : Ai ->VqeProcess-> Ao - ******************************************************************************/ -RK_S32 AI_VqeProcess_AO(RK_VOID) { - AI_TALKVQE_CONFIG_S stAiVqeTalkAttr; - AI_RECORDVQE_CONFIG_S stAiVqeRecordAttr; - AO_VQE_CONFIG_S stAoVqeAttr; - MPP_CHN_S mpp_chn_ai, mpp_chn_ao; - - if (1 == u32AiVqeType) { - memset(&stAiVqeTalkAttr, 0, sizeof(AI_TALKVQE_CONFIG_S)); - stAiVqeTalkAttr.s32WorkSampleRate = g_enWorkSampleRate; - stAiVqeTalkAttr.s32FrameSample = g_s32VqeFrameSample; - strcpy(stAiVqeTalkAttr.aParamFilePath, VQEFILE); - stAiVqeTalkAttr.u32OpenMask = - AI_TALKVQE_MASK_AEC | AI_TALKVQE_MASK_ANR | AI_TALKVQE_MASK_AGC; - } else if (2 == u32AiVqeType) { - memset(&stAiVqeRecordAttr, 0, sizeof(AI_RECORDVQE_CONFIG_S)); - stAiVqeRecordAttr.s32WorkSampleRate = g_enWorkSampleRate; - stAiVqeRecordAttr.s32FrameSample = g_s32VqeFrameSample; - stAiVqeRecordAttr.stAnrConfig.fPostAddGain = 0; - stAiVqeRecordAttr.stAnrConfig.fGmin = -30; - stAiVqeRecordAttr.stAnrConfig.fNoiseFactor = 0.98; - stAiVqeRecordAttr.u32OpenMask = AI_RECORDVQE_MASK_ANR; - } - - if (1 == u32AoVqeType) { - memset(&stAoVqeAttr, 0, sizeof(AO_VQE_CONFIG_S)); - stAoVqeAttr.s32WorkSampleRate = g_enWorkSampleRate; - stAoVqeAttr.s32FrameSample = g_s32VqeFrameSample; - strcpy(stAoVqeAttr.aParamFilePath, VQEFILE); - stAoVqeAttr.u32OpenMask = AO_VQE_MASK_ANR | AO_VQE_MASK_AGC; - } - - RK_MPI_SYS_Init(); - mpp_chn_ai.enModId = RK_ID_AI; - mpp_chn_ai.s32ChnId = 0; - mpp_chn_ao.enModId = RK_ID_AO; - mpp_chn_ao.s32ChnId = 0; - - AI_CHN_ATTR_S ai_attr; - ai_attr.pcAudioNode = ALSA_PATH; - ai_attr.enSampleFormat = RK_SAMPLE_FMT_S16; - ai_attr.u32NbSamples = 1024; - ai_attr.u32SampleRate = g_enWorkSampleRate; - ai_attr.u32Channels = 2; - ai_attr.enAiLayout = g_s32AiLayout; // remove ref channel, and output mic mono - - AO_CHN_ATTR_S ao_attr; - ao_attr.pcAudioNode = ALSA_PATH; - ao_attr.enSampleFormat = RK_SAMPLE_FMT_S16; - ao_attr.u32NbSamples = 1024; - ao_attr.u32SampleRate = g_enWorkSampleRate; - ao_attr.u32Channels = 2; - - // 1. create AI - RK_MPI_AI_SetChnAttr(mpp_chn_ai.s32ChnId, &ai_attr); - RK_MPI_AI_EnableChn(mpp_chn_ai.s32ChnId); - //RK_MPI_AI_SetVolume(mpp_chn_ai.s32ChnId, 100); - if (1 == u32AiVqeType) { - RK_MPI_AI_SetTalkVqeAttr(mpp_chn_ai.s32ChnId, &stAiVqeTalkAttr); - RK_MPI_AI_EnableVqe(mpp_chn_ai.s32ChnId); - } else if (2 == u32AiVqeType) { - RK_MPI_AI_SetRecordVqeAttr(mpp_chn_ai.s32ChnId, &stAiVqeRecordAttr); - RK_MPI_AI_EnableVqe(mpp_chn_ai.s32ChnId); - } - // 2. create AO - RK_MPI_AO_SetChnAttr(mpp_chn_ao.s32ChnId, &ao_attr); - RK_MPI_AO_EnableChn(mpp_chn_ao.s32ChnId); - //RK_MPI_AO_SetVolume(mpp_chn_ao.s32ChnId, 100); - if (1 == u32AoVqeType) { - RK_MPI_AO_SetVqeAttr(mpp_chn_ao.s32ChnId, &stAoVqeAttr); - RK_MPI_AO_EnableVqe(mpp_chn_ao.s32ChnId); - } - // 3. bind AI-AO - RK_MPI_SYS_Bind(&mpp_chn_ai, &mpp_chn_ao); - - printf("%s initial finish\n", __func__); - signal(SIGINT, sigterm_handler); - while (!quit) { - usleep(500000); - } - - printf("%s exit!\n", __func__); - RK_MPI_SYS_UnBind(&mpp_chn_ai, &mpp_chn_ao); - RK_MPI_AI_DisableChn(mpp_chn_ai.s32ChnId); - RK_MPI_AO_DisableChn(mpp_chn_ao.s32ChnId); - - return RK_SUCCESS; -} - -static RK_VOID RKMEDIA_AUDIO_Usage() { - printf("\n\n/Usage:./rkmdia_audio [filePath]/ " - "[nbsamples] [ailayout]\n"); - printf("\tindex and its function list below\n"); - printf("\t0: start AI to AO loop\n"); - printf("\t1: send audio frame to AENC channel from AI, save them\n"); - printf("\t2: read audio stream from file, decode and send AO\n"); - printf("\t3: start AI(VQE process), then send to AO\n"); - // printf("\t4: start AI to Extern Resampler\n"); - printf("\n"); - printf("\tsampleRate list:\n"); - printf("\t0 16000 22050 24000 32000 44100 48000\n"); - printf("\n"); - printf("\tfilePath represents the path of audio file to be decoded, only for " - "sample 2.\n"); - printf("\tdefault filePath: /userdata/out.mp2\n"); - printf("\n"); - printf("\tnbsamples, for example: 160 is 10ms at 16kHz\n"); - printf("\n"); - printf("\tailayout:\n"); - printf("\t0: AI_LAYOUT_NORMAL\n"); - printf("\t1: AI_LAYOUT_MIC_REF\n"); - printf("\t2: AI_LAYOUT_REF_MIC\n"); - printf("\t3: AI_LAYOUT_2MIC_REF_NONE\n"); - printf("\t4: AI_LAYOUT_2MIC_NONE_REF\n"); - printf("\t5: AI_LAYOUT_2MIC_2REF\n"); - printf("\t6: AI_LAYOUT_BUTT\n"); - printf("\n"); - printf("\texample: ./rkmdia_audio 0 48000 480 1 /tmp/out_aiao.mp2\n"); -} - -int main(int argc, char *argv[]) { - - RK_U32 u32Index; - RK_CHAR *pFilePath = RK_NULL; - - if (!(argc >= 3 && argc <= 6)) { - RKMEDIA_AUDIO_Usage(); - return -1; - } - - if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "-?") == 0) { - RKMEDIA_AUDIO_Usage(); - return -1; - } - - u32Index = atoi(argv[1]); - g_enWorkSampleRate = atoi(argv[2]); - - switch (argc) { - case 6: - pFilePath = argv[5]; - g_s32AiLayout = atoi(argv[4]); - g_s32VqeFrameSample = atoi(argv[3]); - break; - case 5: - g_s32AiLayout = atoi(argv[4]); - g_s32VqeFrameSample = atoi(argv[3]); - break; - case 4: - g_s32VqeFrameSample = atoi(argv[3]); - break; - default: - pFilePath = (char *)"/tmp/out.mp2"; - break; - } - - switch (u32Index) { - case 0: - AI_AO(); - break; - case 1: - AI_AENC_FILE(pFilePath); - break; - case 2: - FILE_ADEC_AO(pFilePath); - break; - case 3: - AI_VqeProcess_AO(); - default: - break; - } - return 0; -} diff --git a/resources/build.sh b/resources/build.sh index 1b22961..543177e 100755 --- a/resources/build.sh +++ b/resources/build.sh @@ -49,6 +49,7 @@ function qtmoc() { "src/qt/recoUi/recoUiCallConsole.h:src/qt/recoUi/moc_recoUiCallConsole.cpp" "src/qt/recoUi/recoUiCallDial.h:src/qt/recoUi/moc_recoUiCallDial.cpp" "src/qt/utility/DndModeCountDownItem.h:src/qt/utility/moc_DndModeCountDownItem.cpp" + "src/qt/mainUi/UvcView.h:src/qt/mainUi/moc_UvcView.cpp" ) for file in "${files[@]}"; do @@ -93,19 +94,18 @@ function deploy() { fi echo "deploy to target $TARGET_IP, path: ${TARGET_PATH} ..." - ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "mount -o remount rw /system/" + ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "mount -o remount rw /system/; mount -o remount rw /" ssh -i ~/Projects/ssh_host_rsa_key_ok root@${TARGET_IP} "killall start-app.sh; pgrep -f GateFace | xargs kill -s 9" ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "pgrep -f VoucherVerifyServer | xargs kill -s 9" ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "pgrep -f Record | xargs kill -s 9" - scp -i ~/Projects/ssh_host_rsa_key_ok ${build_path}/src/gate_face/GateFace root@${TARGET_IP}:${TARGET_PATH} - # scp -i ~/Projects/ssh_host_rsa_key_ok ${build_path}/ThirdParty/librwSrvProtocol.so root@${TARGET_IP}:/system/lib/ - scp -i resources/ssh_host_rsa_key_ok ${build_path}/Tools/Record/Record root@${TARGET_IP}:/sdcard/ - scp -i resources/ssh_host_rsa_key_ok ${build_path}/Tools/Record/rkmedia_audio_test root@${TARGET_IP}:/sdcard/ - scp -i resources/ssh_host_rsa_key_ok ./3rdparty/arm-linux-gnueabihf/mpp/rk-libs/libeasymedia.so.1.0.1 root@${TARGET_IP}:/system/lib/ - + scp -i resources/ssh_host_rsa_key_ok ${build_path}/Record/Record root@${TARGET_IP}:/sdcard/ ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "sync" + # ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "echo 116 > /sys/class/gpio/export" + # ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "echo out > /sys/class/gpio/gpio116/direction" + # ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "echo 1 > /sys/class/gpio/gpio116/value" + if [ $debug_deploy != true ]; then echo "reboot remote device." ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "reboot" @@ -146,6 +146,11 @@ function copy_ssh() { } function build_old() { + # start-app.sh 以下就可以让应用不开机自启 + # d_state=0 + # l_state=0 + # n_state=1 + # i_state=1 if [ -n "$1" ]; then TARGET_IP=$1 fi @@ -175,6 +180,7 @@ function build_old() { } function deploy_old() { + # killall start-app.sh; ps | grep GateFace | grep -v "grep" | awk '{print $1}' | xargs kill -s 9 if [ -n "$1" ]; then TARGET_IP=$1 fi @@ -186,9 +192,9 @@ function deploy_old() { ssh -i ~/Projects/ssh_host_rsa_key_ok root@${TARGET_IP} "killall start-app.sh; killall GateFace" ssh -i ~/Projects/ssh_host_rsa_key_ok root@${TARGET_IP} "killall start-app.sh; killall GateFace" else - ssh -i ~/Projects/ssh_host_rsa_key_ok root@${TARGET_IP} "killall start-app.sh; killall GateFace; killall netconfig" + ssh -i ~/Projects/ssh_host_rsa_key_ok root@${TARGET_IP} "killall start-app.sh; killall GateFace;" fi - scp -r -i ~/Projects/ssh_host_rsa_key_ok src/web/php/*.php root@${TARGET_IP}:/system/www/web/ + # scp -r -i ~/Projects/ssh_host_rsa_key_ok src/web/php/*.php root@${TARGET_IP}:/system/www/web/ # scp -r -i ~/Projects/ssh_host_rsa_key_ok resources/audio/*.wav root@${TARGET_IP}:/system/audio scp -i ~/Projects/ssh_host_rsa_key_ok ./build/GateFace root@${TARGET_IP}:$TARGET_PATH # scp -i ~/Projects/ssh_host_rsa_key_ok resources/language/FaceTick_EN.qm root@${TARGET_IP}:/system/language diff --git a/resources/cmake/toolchain.cmake b/resources/cmake/toolchain.cmake index d785f88..1a696be 100644 --- a/resources/cmake/toolchain.cmake +++ b/resources/cmake/toolchain.cmake @@ -19,4 +19,4 @@ set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) string(APPEND CMAKE_CXX_FLAGS " -mfloat-abi=hard -mfpu=neon") -set(BOOST_ROOT /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/boost_1_85_0) \ No newline at end of file +set(BOOST_ROOT /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/boost_1_86_0) \ No newline at end of file