amass/FaceAccess
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update code.

Browse Source
This commit is contained in:
luocai 2024-09-04 17:57:23 +08:00
parent 6c9632d659
commit e8a3532cee
19 changed files with 733 additions and 767 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.vscode/c_cpp_properties.json vendored
View File

@ -4,8 +4,9 @@
"name": "Linux",
"includePath": [
"${workspaceFolder}/**",
"/opt/Libraries/boost_1_84_0/include",
"${workspaceFolder}/3rdparty/arm-linux-gnueabihf/mpp/inc",
"/opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/boost_1_86_0/include",
"/opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/speexdsp-1.2.1/include",
"/opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/rockchip_mpp/include",
"/opt/toolchains/Libraries/opus-1.4/include",
"ThirdParty/libjpegTurbo/inc",
"ThirdParty/libalsa/inc",

4
CMakeLists.txt
View File

@ -4,8 +4,10 @@ cmake_minimum_required(VERSION 3.27)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(OPENSSL_ROOT /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/openssl-1.0.2q)
set(OPENSSL_ROOT /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/openssl-3.3.1)
set(OPENSSL_INCLUDE_DIR ${OPENSSL_ROOT}/include)
set(OPENSSL_LIBRARY_DIRS ${OPENSSL_ROOT}/lib)
set(OPENSSL_LIBRARIES ssl crypto)
set(ALSA_ROOT /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/libalsa-1.1.5)
set(ALSA_INCLUDE_DIR ${ALSA_ROOT}/include)

1
Readme.md
View File

@ -1,5 +1,6 @@
```shell
aplay -t raw -c 1 -f S16_LE -r 16000 /system/audio/pleaseFaceTheCamera.wav
amixer sset 'Master' 50% # 设置音量
mount -o remount rw /system/ # 读写挂载
keydata get mac # 操作mac

57
Record/CMakeLists.txt
View File

@ -1,8 +1,14 @@
find_package(Boost REQUIRED COMPONENTS json)
add_executable(Record main.cpp
RkAudio.h RkAudio.cpp
OpusCodec.h OpusCodec.cpp
FFmpegResample.h FFmpegResample.cpp
RkRecorder.cpp
EchoRecord.cpp
Player.cpp
Recorder.cpp
SpeexDsp.h SpeexDsp.cpp
WebRTCPublisher.h WebRTCPublisher.cpp
)
target_include_directories(Record
@ -10,6 +16,8 @@ target_include_directories(Record
PRIVATE ${MPP_INCLUDE_DIR}
PRIVATE ${MPP_INCLUDE_DIR}/rkmedia
PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/opus-1.4/include
PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/amazon-kinesis-video-streams-webrtc-sdk-c/include
PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/speexdsp-1.2.1/include
PRIVATE ${FFMPEG_INCLUDE_DIR}
)
@ -18,7 +26,11 @@ target_link_directories(Record
PRIVATE ${MPP_LIBRARY_DIRS}
PRIVATE ${3rdparty_ROOT}/rkap_3a/lib
PRIVATE ${FFMPEG_LIBRARY_DIRS}
PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/speexdsp-1.2.1/lib
PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/opus-1.4/lib
PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/amazon-kinesis-video-streams-webrtc-sdk-c/lib
PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/usrsctp-0.9.5.0/lib
PRIVATE /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/libsrtp-2.6.0/lib
)
target_link_libraries(Record
@ -38,7 +50,18 @@ target_link_libraries(Record
PRIVATE glib-2.0
PRIVATE pcre
PRIVATE opus
PRIVATE speexdsp
PRIVATE Boost::json
PRIVATE kvsCommonLws
PRIVATE kvsWebrtcClient
PRIVATE kvsWebrtcSignalingClient
PRIVATE kvspic
PRIVATE kvspicUtils
PRIVATE kvspicClient
PRIVATE usrsctp
PRIVATE srtp2
PRIVATE Universal
PRIVATE HttpProxy
PRIVATE stdc++fs
PRIVATE RKAP_ANR
PRIVATE RKAP_Common
@ -46,36 +69,4 @@ target_link_libraries(Record
PRIVATE dl
PRIVATE z
PRIVATE ${FFMPEG_LIBRARY}
)
add_executable(rkmedia_audio_test rkmedia_audio_test.c)
target_include_directories(rkmedia_audio_test
PRIVATE ${MPP_INCLUDE_DIR}/rkmedia
)
target_link_directories(rkmedia_audio_test
PRIVATE ${MPP_LIBRARY_DIRS}
PRIVATE ${3rdparty_ROOT}/rkap_3a/lib
PRIVATE ${ALSA_LIBRARY_DIRS}
)
target_link_libraries(rkmedia_audio_test
PRIVATE asound
PRIVATE easymedia
PRIVATE drm
PRIVATE rkaiq
PRIVATE rockchip_mpp
PRIVATE v4l2
PRIVATE v4lconvert
PRIVATE jpeg
PRIVATE rga
PRIVATE glib-2.0
PRIVATE pcre
# PRIVATE opus
# PRIVATE Universal
# PRIVATE stdc++fs
PRIVATE RKAP_ANR
PRIVATE RKAP_Common
PRIVATE dl
)

31
Record/EchoRecord.cpp Normal file
View File

@ -0,0 +1,31 @@
#include "BoostLog.h"
#include "main.h"
#include <memory>
void EchoRecordTask::setVqeEnabled(bool enabled) {
if (m_vqeEnabled != enabled) {
m_vqeEnabled = enabled;
}
}
void EchoRecordTask::setChannels(int channels) {
if (m_channels != channels) {
m_channels = channels;
}
}
void EchoRecordTask::run() {
RkAudio::Format format;
format.channels = m_channels;
format.period = 64;
m_output = std::make_shared<RkAudio::Output>();
if (!m_output->open(sizeof(uint16_t), format.sampleRate, format.channels, format.period, m_vqeEnabled)) {
LOG(error) << "audio output open failed.";
return;
}
m_input = std::make_shared<RkAudio::Input>();
m_input->setDataCallback([this](const RkAudio::Frame &frame) { m_output->write(frame.data, frame.byteSize); });
m_input->open(format, m_vqeEnabled);
}

56
Record/Player.cpp Normal file
View File

@ -0,0 +1,56 @@
#include "IoContext.h"
#include "main.h"
#include <boost/asio/post.hpp>
#include <filesystem>
void PlayerTask::setPath(const std::string &path) {
if (m_path != path) {
m_path = path;
}
if (std::filesystem::exists(path)) {
m_ifs = std::make_shared<std::ifstream>(path, std::ifstream::binary);
}
}
/*
ffmpeg将一个1通道mp3文件，转换为 2通道,16bit,16000采样率的pcm文件,但是左通道数据为静音
ffmpeg -i 20200316_1900.mp3 -ac 2 -ar 16000 -filter_complex "[0:a]pan=stereo|c0=0*c0|c1=1*c0[a]" -map "[a]" -f s16le -acodec pcm_s16le 20200316_1900.pcm
ffmpeg将一个1通道mp3文件，转换为 2通道,16bit,16000采样率的pcm文件,但是右通道数据为静音
ffmpeg -i 20200316_1900.mp3 -ac 2 -ar 16000 -filter_complex "[0:a]pan=stereo|c0=1*c0|c1=0*c0[a]" -map "[a]" -f s16le -acodec pcm_s16le 20200316_1900.pcm
*/
// ./Record --play --path=/sdcard/data/20240904160913.pcm
void PlayerTask::run() {
using namespace Amass;
RkAudio::Format format;
format.channels = 2;
format.period = 64;
m_output = std::make_shared<RkAudio::Output>();
if (!m_output->open(sizeof(uint16_t), format.sampleRate, format.channels, format.period, false)) {
LOG(error) << "audio output open failed.";
return;
}
play();
}
void PlayerTask::play() {
using namespace Amass;
auto ioConext = Singleton<IoContext>::instance();
if (!m_ifs || !(*m_ifs)) {
LOG(info) << "play finished";
return;
}
char buffer[2 * sizeof(int16_t) * 16 * 64];
m_ifs->read(buffer, sizeof(buffer));
auto readedSize = m_ifs->gcount();
m_output->write(reinterpret_cast<const uint8_t *>(buffer), readedSize);
boost::asio::post(*ioConext->ioContext(), [this]() { play(); });
}

29
Record/Recorder.cpp Normal file
View File

@ -0,0 +1,29 @@
#include "BoostLog.h"
#include "DateTime.h"
#include "main.h"
#include <filesystem>
#include <sstream>
void RecorderTask::run() {
constexpr auto path = "/sdcard/data";
if (!std::filesystem::exists(path)) {
std::filesystem::create_directory(path);
}
std::ostringstream oss;
oss << path << "/" << DateTime::currentDateTime().toString("%Y%m%d%H%M%S") << ".pcm";
auto filePath = oss.str();
m_ofs = std::make_shared<std::ofstream>(filePath, std::ofstream::binary);
RkAudio::Format format;
format.channels = 2;
format.period = 64;
m_input = std::make_shared<RkAudio::Input>();
m_input->setDataCallback(
[this](const RkAudio::Frame &frame) { m_ofs->write(reinterpret_cast<const char *>(frame.data), frame.byteSize); });
if (m_input->open(format, false)) {
LOG(info) << "open record succeed, path: " << filePath;
} else {
LOG(info) << "open record failed.";
}
}

151
Record/RkAudio.cpp
View File

@ -5,6 +5,10 @@
namespace RkAudio {
constexpr RK_U32 VqeFrameSample = 256; // 16ms;
constexpr auto AudioNode = "hw:0,0";
constexpr auto ParamFilePath = "/system/etc/RKAP_3A_Para.bin";
static SAMPLE_FORMAT_E rkAiFormat(Format::SampleType sampleType) {
SAMPLE_FORMAT_E ret = RK_SAMPLE_FMT_NONE;
switch (sampleType) {
@ -36,53 +40,51 @@ Input::~Input() {
stop();
}
}
bool Input::open(const Format &format) {
bool Input::open(const Format &format, bool enableVqe) {
bool ret = false;
// RK_MPI_SYS_DumpChn(RK_ID_AI);
m_channel = 0;
AI_CHN_ATTR_S parameter = {0};
parameter.pcAudioNode = "default";
parameter.pcAudioNode = (RK_CHAR *)AudioNode;
parameter.enAiLayout = AI_LAYOUT_MIC_REF; // remove ref channel, and output mic mono
parameter.enSampleFormat = rkAiFormat(format.sampleType);
parameter.u32Channels = format.channels;
parameter.u32SampleRate = format.sampleRate;
parameter.u32NbSamples = format.sampleRate / 1000 * format.period;
int status = RK_MPI_AI_SetChnAttr(m_channel, &parameter);
if (status) {
LOG(error) << "RK_MPI_AI_SetChnAttr() failed, status: " << status;
return ret;
}
status = RK_MPI_AI_EnableChn(m_channel);
if (status) {
LOG(error) << "RK_MPI_AI_EnableChn() failed, status: " << status;
return ret;
}
AI_TALKVQE_CONFIG_S config = {0};
status = RK_MPI_AI_GetTalkVqeAttr(m_channel, &config);
if (status) {
LOG(error) << "RK_MPI_AI_GetTalkVqeAttr() failed, status: " << status;
return ret;
}
// LOG(info) << "param file: " << config.aParamFilePath;
config.s32WorkSampleRate = format.sampleRate;
config.s32FrameSample = format.sampleRate / 1000 * format.period;
config.u32OpenMask = AI_TALKVQE_MASK_AEC | AI_TALKVQE_MASK_ANR | AI_TALKVQE_MASK_AGC;
strncpy(config.aParamFilePath, ParamFilePath, sizeof(config.aParamFilePath));
RK_MPI_AI_SetTalkVqeAttr(m_channel, &config);
fprintf(stderr, "end\n");
if (status) {
LOG(error) << "RK_MPI_AI_SetTalkVqeAttr() failed, status: " << status;
return ret;
}
status = RK_MPI_AI_EnableVqe(m_channel);
if (status) {
LOG(error) << "RK_MPI_AI_EnableVqe() failed, status: " << status;
return ret;
if (enableVqe) {
AI_TALKVQE_CONFIG_S config = {0};
status = RK_MPI_AI_GetTalkVqeAttr(m_channel, &config);
if (status) {
LOG(error) << "RK_MPI_AI_GetTalkVqeAttr() failed, status: " << status;
return ret;
}
LOG(info) << "param file: " << config.aParamFilePath;
config.s32WorkSampleRate = format.sampleRate;
config.s32FrameSample = VqeFrameSample;
config.u32OpenMask = AI_TALKVQE_MASK_AEC | AI_TALKVQE_MASK_ANR | AI_TALKVQE_MASK_AGC;
strncpy(config.aParamFilePath, ParamFilePath, sizeof(config.aParamFilePath));
RK_MPI_AI_SetTalkVqeAttr(m_channel, &config);
if (status) {
LOG(error) << "RK_MPI_AI_SetTalkVqeAttr() failed, status: " << status;
return ret;
}
status = RK_MPI_AI_EnableVqe(m_channel);
if (status) {
LOG(error) << "RK_MPI_AI_EnableVqe() failed, status: " << status;
return ret;
}
}
status = RK_MPI_AI_StartStream(0);
@ -138,27 +140,29 @@ Output::~Output() {
close();
}
bool Output::open(uint32_t sampleSize, uint32_t sampleRate, uint32_t channels) {
bool Output::open(uint32_t sampleSize, uint32_t sampleRate, uint32_t channels, uint32_t period, bool enableVqe) {
m_channel = 0;
AO_CHN_ATTR_S parameter = {0};
parameter.pcAudioNode = "default";
parameter.pcAudioNode = (RK_CHAR *)AudioNode;
parameter.enSampleFormat = RK_SAMPLE_FMT_S16;
parameter.u32NbSamples = sampleRate / 1000 * 20;
parameter.u32NbSamples = sampleRate / 1000 * period;
parameter.u32SampleRate = sampleRate;
parameter.u32Channels = channels;
RK_MPI_AO_SetChnAttr(m_channel, &parameter);
RK_MPI_AO_EnableChn(m_channel);
AO_VQE_CONFIG_S config = {0};
config.s32WorkSampleRate = sampleRate;
config.s32FrameSample = parameter.u32NbSamples;
config.u32OpenMask = AO_VQE_MASK_ANR | AO_VQE_MASK_AGC;
strncpy(config.aParamFilePath, ParamFilePath, sizeof(config.aParamFilePath));
if (enableVqe) {
AO_VQE_CONFIG_S config = {0};
config.s32WorkSampleRate = sampleRate;
config.s32FrameSample = VqeFrameSample;
config.u32OpenMask = AO_VQE_MASK_ANR | AO_VQE_MASK_AGC;
strncpy(config.aParamFilePath, ParamFilePath, sizeof(config.aParamFilePath));
RK_MPI_AO_SetVqeAttr(m_channel, &config);
RK_MPI_AO_EnableVqe(m_channel);
RK_MPI_AO_SetVqeAttr(m_channel, &config);
RK_MPI_AO_EnableVqe(m_channel);
}
return true;
}
@ -185,3 +189,76 @@ void Output::write(const uint8_t *data, uint32_t byteSize) {
}
} // namespace RkAudio
PcmStreamBuffer::PcmStreamBuffer(uint32_t sampleRate, uint32_t channels, RkAudio::Format::SampleType sampleType, uint32_t popDuration,
uint32_t capacity)
: m_sampleRate(sampleRate), m_channels(channels), m_buffer(capacity), m_capacity(capacity) {
if (sampleType == RkAudio::Format::SignedInt16) {
m_pointByteSize = 2;
}
uint32_t frameSize = sampleRate * channels * m_pointByteSize * popDuration / 1000;
m_popBuffer = std::vector<uint8_t>(frameSize);
m_popFrame.data = m_popBuffer.data();
}
bool PcmStreamBuffer::push(const RkAudio::Frame &frame) {
std::lock_guard<std::mutex> locker(m_mutex);
uint32_t byteSize = availableByteSize();
uint32_t freeSize = m_capacity - byteSize;
if (freeSize < frame.byteSize) {
LOG_FORMAT(warning, "buffer is full, capacity: %d, free size: %d, need size: %d", m_capacity, freeSize, frame.byteSize);
return false;
}
if ((m_tail + frame.byteSize) > m_capacity) {
uint32_t size1 = m_capacity - m_tail;
memcpy(m_buffer.data() + m_tail, frame.data, size1);
uint32_t size2 = frame.byteSize - size1;
memcpy(m_buffer.data(), frame.data + size1, size2);
} else {
memcpy(m_buffer.data() + m_tail, frame.data, frame.byteSize);
}
m_tail = (m_tail + frame.byteSize) % m_capacity;
m_full = (m_tail == m_head);
return true;
}
std::chrono::milliseconds PcmStreamBuffer::availableDuration() const {
auto byteSize = availableByteSize();
return std::chrono::milliseconds(1000 * byteSize / (m_sampleRate * m_channels * m_pointByteSize));
}
const RkAudio::Frame *PcmStreamBuffer::pop() {
// return nullptr;
std::lock_guard<std::mutex> locker(m_mutex);
auto byteSize = availableByteSize();
if (byteSize < m_popBuffer.size()) return nullptr;
if ((m_head + m_popBuffer.size()) > m_capacity) {
uint32_t size1 = m_capacity - m_head;
memcpy(m_popBuffer.data(), m_buffer.data() + m_head, size1);
uint32_t size2 = m_popBuffer.size() - size1;
memcpy(m_popBuffer.data() + size1, m_buffer.data(), size2);
} else {
memcpy(m_popBuffer.data(), m_buffer.data() + m_head, m_popBuffer.size());
}
m_head = (m_head + m_popBuffer.size()) % m_capacity;
m_popFrame.byteSize = m_popBuffer.size();
m_popFrame.frameSize = m_popFrame.byteSize / m_channels / m_pointByteSize;
m_full = false;
return &m_popFrame;
}
uint32_t PcmStreamBuffer::availableByteSize() const {
if (m_full) {
return m_capacity;
} else if (m_tail >= m_head) {
return m_tail - m_head;
} else {
return m_capacity + m_tail - m_head;
}
}

44
Record/RkAudio.h
View File

@ -1,12 +1,13 @@
#ifndef __RKAUDIO_H__
#define __RKAUDIO_H__
#include <chrono>
#include <functional>
#include <mutex>
#include <thread>
#include <vector>
namespace RkAudio {
// constexpr auto ParamFilePath="/data/sdcard/rw_3a.bin";
constexpr auto ParamFilePath = "/data/sdcard/RKAP_3A_Para.bin";
class Format {
public:
@ -23,9 +24,9 @@ public:
};
SampleType sampleType = SignedInt16;
Endian byteOrder = LittleEndian;
uint32_t sampleRate = 48000;
uint32_t channels = 2;
uint32_t period = 60;
uint32_t sampleRate = 16000;
uint32_t channels = 1;
uint32_t period = 16;
};
class Frame {
@ -42,7 +43,7 @@ public:
Input();
~Input();
void setDataCallback(const ReadCallback &callback);
bool open(const Format &format);
bool open(const Format &format,bool enableVqe);
void stop();
protected:
@ -60,7 +61,7 @@ class Output {
public:
Output();
~Output();
bool open(uint32_t sampleSize, uint32_t sampleRate, uint32_t channels);
bool open(uint32_t sampleSize, uint32_t sampleRate, uint32_t channels, uint32_t period,bool enableVqe);
void close();
void write(const uint8_t *data, uint32_t byteSize);
@ -70,4 +71,33 @@ private:
} // namespace RkAudio
/**
* @brief 目前系统取录音数据,只能64ms。而opus编码需要20ms一周期
*
*/
class PcmStreamBuffer {
public:
PcmStreamBuffer(uint32_t sampleRate, uint32_t channels, RkAudio::Format::SampleType sampleType, uint32_t popDuration,
uint32_t capacity);
bool push(const RkAudio::Frame &frame);
std::chrono::milliseconds availableDuration() const;
uint32_t availableByteSize() const;
const RkAudio::Frame *pop();
private:
uint32_t m_sampleRate;
uint32_t m_channels;
uint32_t m_pointByteSize = 0;
std::vector<uint8_t> m_buffer;
uint32_t m_capacity;
uint32_t m_head = 0;
uint32_t m_tail = 0;
bool m_full = false;
RkAudio::Frame m_popFrame;
std::vector<uint8_t> m_popBuffer;
std::mutex m_mutex;
};
#endif // __RKAUDIO_H__

226
Record/RkRecorder.cpp
View File

@ -1,226 +0,0 @@
#include "BoostLog.h"
#include <fstream>
#include <rkmedia/rkmedia_api.h>
#include <signal.h>
#include <thread>
#define VQEFILE "/data/sdcard/RKAP_3A_Para.bin"
#define ALSA_PATH "default" // get from "arecord -L"
static bool quit = false;
static void sigterm_handler(int sig) {
fprintf(stderr, "signal %d\n", sig);
quit = true;
}
std::shared_ptr<std::ofstream> ofs;
void run() {
ofs = std::make_shared<std::ofstream>("/data/sdcard/test.pcm", std::ofstream::binary);
while (!quit) {
auto mediaBuffer = RK_MPI_SYS_GetMediaBuffer(RK_ID_AI, 0, -1);
if (!mediaBuffer) {
LOG(error) << "RK_MPI_SYS_GetMediaBuffer() failed.";
continue;
}
LOG(info) << "get frame, timestamp: " << RK_MPI_MB_GetTimestamp(mediaBuffer)
<< ", size: " << RK_MPI_MB_GetSize(mediaBuffer);
ofs->write((const char *)RK_MPI_MB_GetPtr(mediaBuffer), RK_MPI_MB_GetSize(mediaBuffer));
RK_MPI_MB_ReleaseBuffer(mediaBuffer);
}
ofs.reset();
}
void rkDemo() {
AI_CHN_ATTR_S ai_attr;
ai_attr.pcAudioNode = "default";
ai_attr.enSampleFormat = RK_SAMPLE_FMT_S16;
ai_attr.u32NbSamples = 16 * 20;
ai_attr.u32SampleRate = 16000;
ai_attr.u32Channels = 2;
ai_attr.enAiLayout = AI_LAYOUT_REF_MIC;
int status = RK_MPI_AI_SetChnAttr(0, &ai_attr);
status |= RK_MPI_AI_EnableChn(0);
if (status) {
LOG(error) << "enable AI[0] failed, status = " << status;
return;
}
AI_TALKVQE_CONFIG_S config = {0};
strcpy(config.aParamFilePath, VQEFILE);
config.s32WorkSampleRate = 16000;
config.s32FrameSample = 16 * 20;
config.u32OpenMask = AI_TALKVQE_MASK_AEC | AI_TALKVQE_MASK_ANR | AI_TALKVQE_MASK_AGC;
status = RK_MPI_AI_SetTalkVqeAttr(0, &config);
if (status) {
LOG(error) << "RK_MPI_AI_SetTalkVqeAttr() failed, status: " << status;
return;
}
status = RK_MPI_AI_EnableVqe(0);
if (status) {
LOG(error) << "RK_MPI_AI_EnableVqe() failed, status: " << status;
return;
}
std::thread thread(&run);
status = RK_MPI_AI_StartStream(0);
if (status) {
LOG(info) << "start AI failed, status: " << status;
return;
}
signal(SIGINT, sigterm_handler);
while (!quit) {
usleep(500000);
}
if (thread.joinable()) {
thread.join();
}
RK_MPI_AI_DisableChn(0);
}
static RK_U32 g_enWorkSampleRate = 8000;
static RK_U32 g_s32VqeFrameSample = 320; // 20ms;
// #define VQEFILE "/data/sdcard/3a.bin"
int AI_VqeProcess_AO() {
AI_TALKVQE_CONFIG_S stAiVqeTalkAttr;
AI_RECORDVQE_CONFIG_S stAiVqeRecordAttr;
AO_VQE_CONFIG_S stAoVqeAttr;
MPP_CHN_S mpp_chn_ai, mpp_chn_ao;
memset(&stAiVqeTalkAttr, 0, sizeof(AI_TALKVQE_CONFIG_S));
stAiVqeTalkAttr.s32WorkSampleRate = g_enWorkSampleRate;
stAiVqeTalkAttr.s32FrameSample = g_s32VqeFrameSample;
strcpy(stAiVqeTalkAttr.aParamFilePath, VQEFILE);
stAiVqeTalkAttr.u32OpenMask = AI_TALKVQE_MASK_AEC | AI_TALKVQE_MASK_ANR | AI_TALKVQE_MASK_AGC;
// stAiVqeTalkAttr.u32OpenMask = AI_TALKVQE_MASK_AEC ;
memset(&stAoVqeAttr, 0, sizeof(AO_VQE_CONFIG_S));
stAoVqeAttr.s32WorkSampleRate = g_enWorkSampleRate;
stAoVqeAttr.s32FrameSample = g_s32VqeFrameSample;
strcpy(stAoVqeAttr.aParamFilePath, VQEFILE);
stAoVqeAttr.u32OpenMask = AO_VQE_MASK_ANR | AO_VQE_MASK_AGC;
mpp_chn_ai.enModId = RK_ID_AI;
mpp_chn_ai.s32ChnId = 0;
mpp_chn_ao.enModId = RK_ID_AO;
mpp_chn_ao.s32ChnId = 0;
AI_CHN_ATTR_S ai_attr;
ai_attr.pcAudioNode = "default";
ai_attr.enSampleFormat = RK_SAMPLE_FMT_S16;
ai_attr.u32NbSamples = 320;
ai_attr.u32SampleRate = g_enWorkSampleRate;
ai_attr.u32Channels = 1;
ai_attr.enAiLayout = AI_LAYOUT_MIC_REF; // remove ref channel, and output mic mono
AO_CHN_ATTR_S ao_attr;
ao_attr.pcAudioNode = "default";
ao_attr.enSampleFormat = RK_SAMPLE_FMT_S16;
ao_attr.u32NbSamples = 320;
ao_attr.u32SampleRate = g_enWorkSampleRate;
ao_attr.u32Channels = 1;
// 1. create AI
RK_MPI_AI_SetChnAttr(mpp_chn_ai.s32ChnId, &ai_attr);
RK_MPI_AI_EnableChn(mpp_chn_ai.s32ChnId);
RK_MPI_AI_SetTalkVqeAttr(mpp_chn_ai.s32ChnId, &stAiVqeTalkAttr);
RK_MPI_AI_EnableVqe(mpp_chn_ai.s32ChnId);
// 2. create AO
RK_MPI_AO_SetChnAttr(mpp_chn_ao.s32ChnId, &ao_attr);
RK_MPI_AO_EnableChn(mpp_chn_ao.s32ChnId);
RK_MPI_AO_SetVqeAttr(mpp_chn_ao.s32ChnId, &stAoVqeAttr);
RK_MPI_AO_EnableVqe(mpp_chn_ao.s32ChnId);
// 3. bind AI-AO
RK_MPI_SYS_Bind(&mpp_chn_ai, &mpp_chn_ao);
printf("%s initial finish\n", __func__);
signal(SIGINT, sigterm_handler);
while (!quit) {
usleep(500000);
}
printf("%s exit!\n", __func__);
RK_MPI_SYS_UnBind(&mpp_chn_ai, &mpp_chn_ao);
RK_MPI_AI_DisableChn(mpp_chn_ai.s32ChnId);
RK_MPI_AO_DisableChn(mpp_chn_ao.s32ChnId);
return RK_SUCCESS;
}
int AI_VqeProcess_AO1() {
AI_TALKVQE_CONFIG_S stAiVqeTalkAttr;
AI_RECORDVQE_CONFIG_S stAiVqeRecordAttr;
AO_VQE_CONFIG_S stAoVqeAttr;
MPP_CHN_S mpp_chn_ai, mpp_chn_ao;
memset(&stAiVqeTalkAttr, 0, sizeof(AI_TALKVQE_CONFIG_S));
stAiVqeTalkAttr.s32WorkSampleRate = g_enWorkSampleRate;
stAiVqeTalkAttr.s32FrameSample = g_s32VqeFrameSample;
strcpy(stAiVqeTalkAttr.aParamFilePath, VQEFILE);
stAiVqeTalkAttr.u32OpenMask = AI_TALKVQE_MASK_AEC | AI_TALKVQE_MASK_ANR | AI_TALKVQE_MASK_AGC;
memset(&stAoVqeAttr, 0, sizeof(AO_VQE_CONFIG_S));
stAoVqeAttr.s32WorkSampleRate = g_enWorkSampleRate;
stAoVqeAttr.s32FrameSample = g_s32VqeFrameSample;
strcpy(stAoVqeAttr.aParamFilePath, VQEFILE);
stAoVqeAttr.u32OpenMask = AO_VQE_MASK_ANR | AO_VQE_MASK_AGC;
RK_MPI_SYS_Init();
mpp_chn_ai.enModId = RK_ID_AI;
mpp_chn_ai.s32ChnId = 0;
mpp_chn_ao.enModId = RK_ID_AO;
mpp_chn_ao.s32ChnId = 0;
AI_CHN_ATTR_S ai_attr;
ai_attr.pcAudioNode = ALSA_PATH;
ai_attr.enSampleFormat = RK_SAMPLE_FMT_S16;
ai_attr.u32NbSamples = 640;
ai_attr.u32SampleRate = g_enWorkSampleRate;
ai_attr.u32Channels = 1;
ai_attr.enAiLayout = AI_LAYOUT_MIC_REF; // remove ref channel, and output mic mono
AO_CHN_ATTR_S ao_attr;
ao_attr.pcAudioNode = ALSA_PATH;
ao_attr.enSampleFormat = RK_SAMPLE_FMT_S16;
ao_attr.u32NbSamples = 640;
ao_attr.u32SampleRate = g_enWorkSampleRate;
ao_attr.u32Channels = 1;
// 1. create AI
RK_MPI_AI_SetChnAttr(mpp_chn_ai.s32ChnId, &ai_attr);
RK_MPI_AI_EnableChn(mpp_chn_ai.s32ChnId);
RK_MPI_AI_SetTalkVqeAttr(mpp_chn_ai.s32ChnId, &stAiVqeTalkAttr);
RK_MPI_AI_EnableVqe(mpp_chn_ai.s32ChnId);
// 2. create AO
RK_MPI_AO_SetChnAttr(mpp_chn_ao.s32ChnId, &ao_attr);
RK_MPI_AO_EnableChn(mpp_chn_ao.s32ChnId);
RK_MPI_AO_SetVqeAttr(mpp_chn_ao.s32ChnId, &stAoVqeAttr);
RK_MPI_AO_EnableVqe(mpp_chn_ao.s32ChnId);
// 3. bind AI-AO
RK_MPI_SYS_Bind(&mpp_chn_ai, &mpp_chn_ao);
printf("%s initial finish\n", __func__);
signal(SIGINT, sigterm_handler);
while (!quit) {
usleep(500000);
}
printf("%s exit!\n", __func__);
RK_MPI_SYS_UnBind(&mpp_chn_ai, &mpp_chn_ao);
RK_MPI_AI_DisableChn(mpp_chn_ai.s32ChnId);
RK_MPI_AO_DisableChn(mpp_chn_ao.s32ChnId);
return RK_SUCCESS;
}

15
Record/SpeexDsp.cpp Normal file
View File

@ -0,0 +1,15 @@
#include "SpeexDsp.h"
#include <speex/speex_echo.h>
#include <speex/speex_preprocess.h>
void SpeexDsp::reset() {
if (m_state != nullptr) {
speex_echo_state_reset(m_state);
}
}
void SpeexDsp::preprocess(int16_t *pcm) {
if (m_preprocessState != nullptr) {
speex_preprocess_run(m_preprocessState, pcm);
}
}

19
Record/SpeexDsp.h Normal file
View File

@ -0,0 +1,19 @@
#ifndef __SPEEXDSP_H__
#define __SPEEXDSP_H__
#include <cstdint>
typedef struct SpeexEchoState_ SpeexEchoState;
typedef struct SpeexPreprocessState_ SpeexPreprocessState;
class SpeexDsp {
public:
void preprocess(int16_t *pcm);
void reset();
private:
SpeexEchoState *m_state = nullptr;
SpeexPreprocessState *m_preprocessState=nullptr;
};
#endif // __SPEEXDSP_H__

221
Record/WebRTCPublisher.cpp Normal file
View File

@ -0,0 +1,221 @@
#include "WebRTCPublisher.h"
#include "BoostLog.h"
#include "IoContext.h"
#include "NetworkUtility.h"
#include <boost/json/object.hpp>
#include <boost/json/parse.hpp>
#include <com/amazonaws/kinesis/video/webrtcclient/Include.h>
#include <fstream>
class WebRTCPublisherPrivate {
public:
WebRTCPublisherPrivate() {
m_exit = false;
m_thread = std::thread(&WebRTCPublisherPrivate::run, this);
}
~WebRTCPublisherPrivate() {
m_exit = true;
if (m_thread.joinable()) {
m_thread.join();
}
}
void run() {
int i = 0;
int opusIndex = 0;
int opusT = 0;
int t = 0;
while (!m_exit) {
auto begin = std::chrono::system_clock::now();
if (i > 1500) i = 0;
if (opusIndex > 618) opusIndex = 0;
if (!m_connected) continue;
{
std::ostringstream oss;
oss << "/data/sdcard/lib/h264SampleFrames/frame-" << std::setfill('0') << std::setw(4) << i++ << ".h264";
std::ifstream ifs(oss.str(), std::ifstream::binary);
auto buffer = std::vector<uint8_t>((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());
Frame frame;
t += 40 * 1000 * 10;
frame.presentationTs = t;
frame.decodingTs = t;
frame.frameData = buffer.data();
frame.size = buffer.size();
STATUS retStatus = writeFrame(videoReceiver, &frame);
}
{
std::ostringstream oss;
oss << "/data/sdcard/lib/opusSampleFrames/sample-" << std::setfill('0') << std::setw(3) << opusIndex << ".opus";
std::ifstream ifs(oss.str(), std::ifstream::binary);
auto buffer = std::vector<uint8_t>((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());
Frame frame;
frame.index = opusT;
opusT += 20 * HUNDREDS_OF_NANOS_IN_A_MILLISECOND;
frame.presentationTs = opusT;
frame.decodingTs = opusT;
frame.frameData = buffer.data();
frame.size = buffer.size();
STATUS retStatus = writeFrame(audioReceiver, &frame);
LOG(info) << "send opus " << opusIndex;
opusIndex++;
}
while (true) {
auto ele = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - begin);
auto s = std::chrono::milliseconds(20) - ele;
if (s.count() > 0) {
continue;
} else {
break;
}
}
}
}
RtcPeerConnection *peer = nullptr;
RtcRtpTransceiver *videoReceiver = nullptr;
RtcRtpTransceiver *audioReceiver = nullptr;
RtcSessionDescriptionInit offer;
static void onConnectionStateChange(UINT64 userData, RTC_PEER_CONNECTION_STATE state) {
auto self = reinterpret_cast<WebRTCPublisherPrivate *>(userData);
LOG(info) << "connection state: " << state;
if (state == RTC_PEER_CONNECTION_STATE_CONNECTED) {
self->m_connected = true;
}
}
std::thread m_thread;
bool m_exit = true;
bool m_connected = false;
};
WebRTCPublisher::WebRTCPublisher(bool videoEnabled, bool audioEnabled)
: m_d(new WebRTCPublisherPrivate()), m_videoEnabled(videoEnabled), m_audioEnabled(audioEnabled) {
}
WebRTCPublisher::~WebRTCPublisher() {
stop();
if (m_d != nullptr) {
delete m_d;
}
}
bool WebRTCPublisher::start(const std::string &address, const std::string &port, const std::string &url) {
using namespace Amass;
stop();
RtcConfiguration configuration{};
memset(&configuration, 0, sizeof(configuration));
STATUS status = createPeerConnection(&configuration, &m_d->peer);
if (status != STATUS_SUCCESS) {
LOG(error) << "createPeerConnection() failed, status: " << status;
return false;
}
if (m_videoEnabled) {
RtcMediaStreamTrack track{};
track.codec = RTC_CODEC_H264_PROFILE_42E01F_LEVEL_ASYMMETRY_ALLOWED_PACKETIZATION_MODE;
track.kind = MEDIA_STREAM_TRACK_KIND_VIDEO;
strncpy(track.streamId, "0", sizeof(track.streamId));
strncpy(track.trackId, "0", sizeof(track.trackId));
RtcRtpTransceiverInit init;
init.direction = RTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY;
status = addTransceiver(m_d->peer, &track, &init, &m_d->videoReceiver);
if (status != STATUS_SUCCESS) {
LOG(error) << "addTransceiver() failed, status: " << status;
return false;
}
}
if (m_audioEnabled) {
RtcMediaStreamTrack track{};
track.codec = RTC_CODEC_OPUS;
track.kind = MEDIA_STREAM_TRACK_KIND_AUDIO;
strncpy(track.streamId, "1", sizeof(track.streamId));
strncpy(track.trackId, "1", sizeof(track.trackId));
RtcRtpTransceiverInit init;
init.direction = RTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY;
status = addTransceiver(m_d->peer, &track, &init, &m_d->audioReceiver);
if (status != STATUS_SUCCESS) {
LOG(error) << "addTransceiver() failed, status: " << status;
return false;
}
}
status = createOffer(m_d->peer, &m_d->offer);
if (status != STATUS_SUCCESS) {
LOG(error) << "createOffer() failed, status: " << status;
return false;
}
status = setLocalDescription(m_d->peer, &m_d->offer);
if (status != STATUS_SUCCESS) {
LOG(error) << "setLocalDescription() failed, status: " << status;
return false;
}
// LOG(info) << "sdp: \n" << m_d->offer.sdp;
std::string sdp;
boost::system::error_code error;
auto ioContext = Singleton<IoContext>::instance();
Http::Client client(*ioContext->ioContext(), Http::SSL);
client.loadRootCertificates(error);
auto reply = client.post(address, port, url, m_d->offer.sdp, error);
if (error) {
LOG(error) << "post error: " << error.message();
return false;
}
auto replyValue = boost::json::parse(reply, error);
if (error) {
LOG(info) << reply;
LOG(error) << error.message();
}
auto replyObject = replyValue.as_object();
sdp = std::string(replyObject.at("sdp").as_string());
// replyObject.at("sdp").as_string().c_str();
// LOG(info) << reply;
LOG(info) << sdp;
RtcSessionDescriptionInit answer{};
answer.type = SDP_TYPE_ANSWER;
strncpy(answer.sdp, sdp.c_str(), sizeof(answer.sdp));
status = setRemoteDescription(m_d->peer, &answer);
if (status != STATUS_SUCCESS) {
LOG(error) << "setRemoteDescription() failed, status: " << status;
return false;
}
status =
peerConnectionOnConnectionStateChange(m_d->peer, reinterpret_cast<UINT64>(m_d), &WebRTCPublisherPrivate::onConnectionStateChange);
if (status != STATUS_SUCCESS) {
LOG(error) << "peerConnectionOnConnectionStateChange() failed, status: " << status;
return false;
}
return true;
}
void WebRTCPublisher::stop() {
if (m_d->peer != nullptr) {
closePeerConnection(m_d->peer);
if (m_d->videoReceiver != nullptr) {
freeTransceiver(&m_d->videoReceiver);
m_d->videoReceiver = nullptr;
}
if (m_d->audioReceiver != nullptr) {
freeTransceiver(&m_d->audioReceiver);
m_d->audioReceiver = nullptr;
}
freePeerConnection(&m_d->peer);
m_d->peer = nullptr;
}
}
void WebRTCPublisher::exchangeSdp() {
}

26
Record/WebRTCPublisher.h Normal file
View File

@ -0,0 +1,26 @@
#ifndef WEBRTCPUBLISHER_H
#define WEBRTCPUBLISHER_H
#include <string>
class WebRTCPublisherPrivate;
class WebRTCPublisher
{
public:
WebRTCPublisher(bool videoEnabled, bool audioEnabled);
~WebRTCPublisher();
bool start(const std::string &address, const std::string &port, const std::string &url);
void stop();
protected:
void exchangeSdp();
private:
WebRTCPublisherPrivate *m_d = nullptr;
bool m_videoEnabled = false;
bool m_audioEnabled = false;
};
#endif // WEBRTCPUBLISHER_H

129
Record/main.cpp
View File

@ -1,61 +1,110 @@
#include "main.h"
#include "BoostLog.h"
#include "DateTime.h"
#include "FFmpegResample.h"
#include "IoContext.h"
#include "OpusCodec.h"
#include "RkAudio.h"
#include "WebRTCPublisher.h"
#include <boost/asio/signal_set.hpp>
// #include <boost/program_options.hpp>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/variables_map.hpp>
#include <com/amazonaws/kinesis/video/webrtcclient/Include.h>
#include <fstream>
#include <rkmedia/rkmedia_api.h>
extern int recorder_demo();
extern void rkDemo();
extern int AI_VqeProcess_AO();
extern int AI_VqeProcess_AO1();
extern void AecTest();
extern int opus_test();
void signal_handler(const boost::system::error_code &error, int signal_number) {
if (!error) {
LOG(info) << "Caught signal: " << signal_number << std::endl;
LOG(info) << "task finished.";
std::exit(0);
}
}
int main(int argc, char **argv) {
RK_MPI_SYS_Init();
// std::ifstream ifs("/data/sdcard/test2.pcm", std::ifstream::binary);
// AlsaPcmPlayer player;
// player.open(2, 48000, 2);
using namespace Amass;
using namespace std::placeholders;
boost::program_options::options_description optionsDescription("Allowed options");
// clang-format off
optionsDescription.add_options()
("help,h", "produce help message")
("echo", "Self-recording and self-play test")
("record", "Record to file.")
("play", "Play pcm file.")
("vqe", boost::program_options::value<bool>(), "Enable rk 3a.")
("channels", boost::program_options::value<int>(), "set audio channles")
("path", boost::program_options::value<std::string>(), "file path")
;
// clang-format on
// char buffer[2 * 2 * 48 * 60];
// while (ifs.read(buffer, sizeof(buffer))) {
// int size = ifs.gcount();
// player.write((const uint8_t *)buffer, size);
// }
// recorder_demo();
// rkDemo();
// AI_VqeProcess_AO();
// AI_VqeProcess_AO1();
// AecTest();
boost::program_options::variables_map variablesMap;
boost::program_options::store(boost::program_options::parse_command_line(argc, argv, optionsDescription), variablesMap);
boost::program_options::notify(variablesMap);
{
FFmpegResample resample;
resample.initialize(16000, 1, 48000, 2, 20);
std::ifstream ifs("/sdcard/input.pcm", std::ifstream::binary);
std::ofstream ofs("/sdcard/my_48kz.pcm", std::ifstream::binary);
char buffer[16 * 20 * 2];
while (ifs.read(buffer, sizeof(buffer))) {
auto frame = resample.resample((uint8_t *)buffer, sizeof(buffer));
if (frame.data != nullptr) {
ofs.write((char *)frame.data, frame.byteSize);
}
}
if (variablesMap.count("help")) {
std::cout << optionsDescription << std::endl;
return 1;
}
{
FFmpegResample resample1;
resample1.initialize(48000, 2, 16000, 1, 20);
std::ifstream ifs1("/sdcard/my_48kz.pcm", std::ifstream::binary);
std::ofstream ofs1("/sdcard/my_16kz.pcm", std::ifstream::binary);
char buffer1[48 * 20 * 2 * 2];
while (ifs1.read(buffer1, sizeof(buffer1))) {
auto frame = resample1.resample((uint8_t *)buffer1, sizeof(buffer1));
if (frame.data != nullptr) {
ofs1.write((char *)frame.data, frame.byteSize);
}
std::shared_ptr<Task> task;
if (variablesMap.count("echo")) {
bool vqe = false;
if (variablesMap.count("vqe")) {
vqe = variablesMap["vqe"].as<bool>();
}
int channels = 2;
if (variablesMap.count("channels")) {
channels = variablesMap["channels"].as<int>();
}
auto t = std::make_shared<EchoRecordTask>();
t->setVqeEnabled(vqe);
t->setChannels(channels);
task = std::dynamic_pointer_cast<Task>(t);
} else if (variablesMap.count("record")) {
task = std::make_shared<RecorderTask>();
} else if (variablesMap.count("play")) {
std::string path;
if (variablesMap.count("path")) {
path = variablesMap["path"].as<std::string>();
}
auto t = std::make_shared<PlayerTask>();
t->setPath(path);
task = std::dynamic_pointer_cast<Task>(t);
}
if (!task) {
std::cout << "there is not task." << std::endl << std::endl;
std::cout << optionsDescription << std::endl;
return 2;
}
try {
LOG(info) << "app start.";
RK_MPI_SYS_Init();
initKvsWebRtc();
auto ioConext = Singleton<IoContext>::instance<Construct>();
boost::asio::signal_set signals(*ioConext->ioContext(), SIGINT, SIGTERM);
signals.async_wait(std::bind(&signal_handler, _1, _2));
task->run();
ioConext->run<IoContext::Mode::Synchronous>();
} catch (std::exception &e) {
LOG(error) << "Exception: " << e.what() << std::endl;
}
// WebRTCPublisher publisher(true, true);
// publisher.start("172.16.103.68", "443", "/index/api/webrtc?app=live&stream=test&type=push");
return 0;
}

49
Record/main.h Normal file
View File

@ -0,0 +1,49 @@
#ifndef __MAIN_H__
#define __MAIN_H__
#include "RkAudio.h"
#include <fstream>
#include <memory>
class Task {
public:
virtual void run() = 0;
};
class RecorderTask : public Task {
public:
void run() final;
private:
std::shared_ptr<RkAudio::Input> m_input;
std::shared_ptr<std::ofstream> m_ofs;
};
class PlayerTask : public Task {
public:
void setPath(const std::string &path);
void run() final;
protected:
void play();
private:
std::string m_path;
std::shared_ptr<std::ifstream> m_ifs;
std::shared_ptr<RkAudio::Output> m_output;
};
class EchoRecordTask : public Task {
public:
void setVqeEnabled(bool enabled);
void setChannels(int channels);
void run() final;
private:
int m_channels = 2;
bool m_vqeEnabled = false;
std::shared_ptr<RkAudio::Output> m_output;
std::shared_ptr<RkAudio::Input> m_input;
};
#endif // __MAIN_H__

411
Record/rkmedia_audio_test.c
View File

@ -1,411 +0,0 @@
// Copyright 2020 Fuzhou Rockchip Electronics Co., Ltd. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <assert.h>
#include <fcntl.h>
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <rkmedia_api.h>
#define MP3_NB_SAMPLES 1024
#define MP2_NB_SAMPLES 1152
//#define ALSA_PATH "default:CARD=rockchiprk809co" // get from "arecord -L"
#define ALSA_PATH "hw:0,0" // get from "arecord -L"
#define VQEFILE "/sdcard/RKAP_3A_Para.bin"
static bool quit = false;
static void sigterm_handler(int sig) {
fprintf(stderr, "signal %d\n", sig);
quit = true;
}
FILE *fp = NULL;
static RK_U32 g_enWorkSampleRate = 16000;
static RK_U32 g_s32VqeFrameSample = 256; // 20ms;
static RK_U32 g_s32AiLayout = AI_LAYOUT_MIC_REF;
static void audio_packet_cb(MEDIA_BUFFER mb) {
printf("Get Audio Encoded packet:ptr:%p, fd:%d, size:%zu, mode:%d\n",
RK_MPI_MB_GetPtr(mb), RK_MPI_MB_GetFD(mb), RK_MPI_MB_GetSize(mb),
RK_MPI_MB_GetModeID(mb));
fwrite(RK_MPI_MB_GetPtr(mb), 1, RK_MPI_MB_GetSize(mb), fp);
RK_MPI_MB_ReleaseBuffer(mb);
}
static RK_VOID AI_AO() {
RK_MPI_SYS_Init();
MPP_CHN_S mpp_chn_ai, mpp_chn_ao;
mpp_chn_ai.enModId = RK_ID_AI;
mpp_chn_ai.s32ChnId = 0;
mpp_chn_ao.enModId = RK_ID_AO;
mpp_chn_ao.s32ChnId = 0;
AI_CHN_ATTR_S ai_attr;
ai_attr.pcAudioNode = ALSA_PATH;
ai_attr.enSampleFormat = RK_SAMPLE_FMT_S16;
ai_attr.u32NbSamples = 1152;
ai_attr.u32SampleRate = g_enWorkSampleRate;
ai_attr.u32Channels = 1;
ai_attr.enAiLayout = g_s32AiLayout; // chanel layout: [ref:mic]; remove
// ref, output mic mono
AO_CHN_ATTR_S ao_attr;
ao_attr.pcAudioNode = ALSA_PATH;
ao_attr.enSampleFormat = RK_SAMPLE_FMT_S16;
ao_attr.u32NbSamples = 1152;
ao_attr.u32SampleRate = g_enWorkSampleRate;
ao_attr.u32Channels = 1;
// 1. create AI
RK_MPI_AI_SetChnAttr(mpp_chn_ai.s32ChnId, &ai_attr);
RK_MPI_AI_EnableChn(mpp_chn_ai.s32ChnId);
RK_MPI_AI_SetVolume(mpp_chn_ai.s32ChnId, 100);
// 2. create AO
RK_MPI_AO_SetChnAttr(mpp_chn_ao.s32ChnId, &ao_attr);
RK_MPI_AO_EnableChn(mpp_chn_ao.s32ChnId);
RK_MPI_AO_SetVolume(mpp_chn_ao.s32ChnId, 100);
// 3. bind AI-AO
RK_MPI_SYS_Bind(&mpp_chn_ai, &mpp_chn_ao);
printf("%s initial finish\n", __func__);
signal(SIGINT, sigterm_handler);
while (!quit) {
usleep(500000);
}
printf("%s exit!\n", __func__);
RK_MPI_SYS_UnBind(&mpp_chn_ai, &mpp_chn_ao);
RK_MPI_AI_DisableChn(mpp_chn_ai.s32ChnId);
RK_MPI_AO_DisableChn(mpp_chn_ao.s32ChnId);
}
static RK_VOID AI_AENC_FILE(char *file_path) {
fp = fopen(file_path, "w+");
RK_MPI_SYS_Init();
MPP_CHN_S mpp_chn_ai, mpp_chn_aenc;
mpp_chn_ai.enModId = RK_ID_AI;
mpp_chn_ai.s32ChnId = 0;
mpp_chn_aenc.enModId = RK_ID_AENC;
mpp_chn_aenc.s32ChnId = 0;
AI_CHN_ATTR_S ai_attr;
ai_attr.pcAudioNode = ALSA_PATH;
ai_attr.enSampleFormat = RK_SAMPLE_FMT_S16;
ai_attr.u32NbSamples = MP3_NB_SAMPLES;
ai_attr.u32SampleRate = g_enWorkSampleRate;
ai_attr.u32Channels = 1;
ai_attr.enAiLayout = g_s32AiLayout; // chanel layout: [ref:mic]; remove
// ref, output mic mono
AENC_CHN_ATTR_S aenc_attr;
aenc_attr.enCodecType = RK_CODEC_TYPE_MP3;
aenc_attr.u32Bitrate = 64000;
aenc_attr.u32Quality = 1;
aenc_attr.stAencMP3.u32Channels = 1;
aenc_attr.stAencMP3.u32SampleRate = g_enWorkSampleRate;
// 1. create AI
RK_MPI_AI_SetChnAttr(mpp_chn_ai.s32ChnId, &ai_attr);
RK_MPI_AI_EnableChn(mpp_chn_ai.s32ChnId);
// 2. create AENC
RK_MPI_AENC_CreateChn(mpp_chn_aenc.s32ChnId, &aenc_attr);
RK_U32 ret = RK_MPI_SYS_RegisterOutCb(&mpp_chn_aenc, audio_packet_cb);
printf("ret = %d.\n", ret);
// 3. bind AI-AENC
RK_MPI_SYS_Bind(&mpp_chn_ai, &mpp_chn_aenc);
printf("%s initial finish\n", __func__);
signal(SIGINT, sigterm_handler);
while (!quit) {
usleep(500000);
}
RK_MPI_SYS_UnBind(&mpp_chn_ai, &mpp_chn_aenc);
RK_MPI_AI_DisableChn(mpp_chn_ai.s32ChnId);
RK_MPI_AENC_DestroyChn(mpp_chn_aenc.s32ChnId);
fclose(fp);
}
static RK_VOID FILE_ADEC_AO(char *file_path) {
CODEC_TYPE_E codec_type = RK_CODEC_TYPE_MP3;
RK_U32 channels = 2;
RK_U32 sample_rate = g_enWorkSampleRate;
ADEC_CHN_ATTR_S stAdecAttr;
AO_CHN_ATTR_S stAoAttr;
stAdecAttr.enCodecType = codec_type;
MPP_CHN_S mpp_chn_ao, mpp_chn_adec;
mpp_chn_ao.enModId = RK_ID_AO;
mpp_chn_ao.s32ChnId = 0;
mpp_chn_adec.enModId = RK_ID_ADEC;
mpp_chn_adec.s32ChnId = 0;
stAoAttr.u32Channels = channels;
stAoAttr.u32SampleRate = sample_rate;
stAoAttr.u32NbSamples = 1024;
stAoAttr.pcAudioNode = ALSA_PATH;
switch (codec_type) {
case RK_CODEC_TYPE_MP3:
stAoAttr.enSampleFormat = RK_SAMPLE_FMT_S16;
stAoAttr.u32NbSamples = 1024;
break;
case RK_CODEC_TYPE_MP2:
stAoAttr.enSampleFormat = RK_SAMPLE_FMT_S16;
stAoAttr.u32NbSamples = 1152;
break;
case RK_CODEC_TYPE_G711A:
stAdecAttr.stAdecG711A.u32Channels = channels;
stAdecAttr.stAdecG711A.u32SampleRate = sample_rate;
stAoAttr.enSampleFormat = RK_SAMPLE_FMT_S16;
break;
case RK_CODEC_TYPE_G711U:
stAdecAttr.stAdecG711U.u32Channels = channels;
stAdecAttr.stAdecG711U.u32SampleRate = sample_rate;
stAoAttr.enSampleFormat = RK_SAMPLE_FMT_S16;
break;
case RK_CODEC_TYPE_G726:
stAoAttr.enSampleFormat = RK_SAMPLE_FMT_S16;
break;
default:
printf("audio codec type error.\n");
return;
}
// init MPI
RK_MPI_SYS_Init();
// create ADEC
RK_MPI_ADEC_CreateChn(mpp_chn_adec.s32ChnId, &stAdecAttr);
// create AO
RK_MPI_AO_SetChnAttr(mpp_chn_ao.s32ChnId, &stAoAttr);
RK_MPI_AO_EnableChn(mpp_chn_ao.s32ChnId);
RK_MPI_SYS_Bind(&mpp_chn_adec, &mpp_chn_ao);
RK_S32 buffer_size = 20480;
FILE *read_file = fopen(file_path, "r");
if (!read_file) {
printf("ERROR: open %s failed!\n", file_path);
exit(0);
}
quit = true;
while (quit) {
MEDIA_BUFFER mb = RK_MPI_MB_CreateAudioBuffer(buffer_size, RK_FALSE);
if (!mb) {
printf("ERROR: no space left!\n");
break;
}
RK_S32 s32ReadSize = fread(RK_MPI_MB_GetPtr(mb), 1, buffer_size, read_file);
RK_MPI_MB_SetSize(mb, s32ReadSize);
RK_MPI_SYS_SendMediaBuffer(RK_ID_ADEC, mpp_chn_adec.s32ChnId, mb);
RK_MPI_MB_ReleaseBuffer(mb);
if (s32ReadSize != buffer_size) {
printf("Get end of file!\n");
break;
}
}
sleep(2);
{
// flush decoder
printf("start flush decoder.\n");
MEDIA_BUFFER mb = RK_MPI_MB_CreateAudioBuffer(buffer_size, RK_FALSE);
RK_MPI_MB_SetSize(mb, 0);
RK_MPI_SYS_SendMediaBuffer(RK_ID_ADEC, mpp_chn_adec.s32ChnId, mb);
RK_MPI_MB_ReleaseBuffer(mb);
printf("end flush decoder.\n");
}
sleep(10);
}
/* 0: close, 1: talk, 2: record */
static RK_U32 u32AiVqeType = 1;
/* 0: close, 1: open */
static RK_U32 u32AoVqeType = 1;
/******************************************************************************
* function : Ai ->VqeProcess-> Ao
******************************************************************************/
RK_S32 AI_VqeProcess_AO(RK_VOID) {
AI_TALKVQE_CONFIG_S stAiVqeTalkAttr;
AI_RECORDVQE_CONFIG_S stAiVqeRecordAttr;
AO_VQE_CONFIG_S stAoVqeAttr;
MPP_CHN_S mpp_chn_ai, mpp_chn_ao;
if (1 == u32AiVqeType) {
memset(&stAiVqeTalkAttr, 0, sizeof(AI_TALKVQE_CONFIG_S));
stAiVqeTalkAttr.s32WorkSampleRate = g_enWorkSampleRate;
stAiVqeTalkAttr.s32FrameSample = g_s32VqeFrameSample;
strcpy(stAiVqeTalkAttr.aParamFilePath, VQEFILE);
stAiVqeTalkAttr.u32OpenMask =
AI_TALKVQE_MASK_AEC | AI_TALKVQE_MASK_ANR | AI_TALKVQE_MASK_AGC;
} else if (2 == u32AiVqeType) {
memset(&stAiVqeRecordAttr, 0, sizeof(AI_RECORDVQE_CONFIG_S));
stAiVqeRecordAttr.s32WorkSampleRate = g_enWorkSampleRate;
stAiVqeRecordAttr.s32FrameSample = g_s32VqeFrameSample;
stAiVqeRecordAttr.stAnrConfig.fPostAddGain = 0;
stAiVqeRecordAttr.stAnrConfig.fGmin = -30;
stAiVqeRecordAttr.stAnrConfig.fNoiseFactor = 0.98;
stAiVqeRecordAttr.u32OpenMask = AI_RECORDVQE_MASK_ANR;
}
if (1 == u32AoVqeType) {
memset(&stAoVqeAttr, 0, sizeof(AO_VQE_CONFIG_S));
stAoVqeAttr.s32WorkSampleRate = g_enWorkSampleRate;
stAoVqeAttr.s32FrameSample = g_s32VqeFrameSample;
strcpy(stAoVqeAttr.aParamFilePath, VQEFILE);
stAoVqeAttr.u32OpenMask = AO_VQE_MASK_ANR | AO_VQE_MASK_AGC;
}
RK_MPI_SYS_Init();
mpp_chn_ai.enModId = RK_ID_AI;
mpp_chn_ai.s32ChnId = 0;
mpp_chn_ao.enModId = RK_ID_AO;
mpp_chn_ao.s32ChnId = 0;
AI_CHN_ATTR_S ai_attr;
ai_attr.pcAudioNode = ALSA_PATH;
ai_attr.enSampleFormat = RK_SAMPLE_FMT_S16;
ai_attr.u32NbSamples = 1024;
ai_attr.u32SampleRate = g_enWorkSampleRate;
ai_attr.u32Channels = 2;
ai_attr.enAiLayout = g_s32AiLayout; // remove ref channel, and output mic mono
AO_CHN_ATTR_S ao_attr;
ao_attr.pcAudioNode = ALSA_PATH;
ao_attr.enSampleFormat = RK_SAMPLE_FMT_S16;
ao_attr.u32NbSamples = 1024;
ao_attr.u32SampleRate = g_enWorkSampleRate;
ao_attr.u32Channels = 2;
// 1. create AI
RK_MPI_AI_SetChnAttr(mpp_chn_ai.s32ChnId, &ai_attr);
RK_MPI_AI_EnableChn(mpp_chn_ai.s32ChnId);
//RK_MPI_AI_SetVolume(mpp_chn_ai.s32ChnId, 100);
if (1 == u32AiVqeType) {
RK_MPI_AI_SetTalkVqeAttr(mpp_chn_ai.s32ChnId, &stAiVqeTalkAttr);
RK_MPI_AI_EnableVqe(mpp_chn_ai.s32ChnId);
} else if (2 == u32AiVqeType) {
RK_MPI_AI_SetRecordVqeAttr(mpp_chn_ai.s32ChnId, &stAiVqeRecordAttr);
RK_MPI_AI_EnableVqe(mpp_chn_ai.s32ChnId);
}
// 2. create AO
RK_MPI_AO_SetChnAttr(mpp_chn_ao.s32ChnId, &ao_attr);
RK_MPI_AO_EnableChn(mpp_chn_ao.s32ChnId);
//RK_MPI_AO_SetVolume(mpp_chn_ao.s32ChnId, 100);
if (1 == u32AoVqeType) {
RK_MPI_AO_SetVqeAttr(mpp_chn_ao.s32ChnId, &stAoVqeAttr);
RK_MPI_AO_EnableVqe(mpp_chn_ao.s32ChnId);
}
// 3. bind AI-AO
RK_MPI_SYS_Bind(&mpp_chn_ai, &mpp_chn_ao);
printf("%s initial finish\n", __func__);
signal(SIGINT, sigterm_handler);
while (!quit) {
usleep(500000);
}
printf("%s exit!\n", __func__);
RK_MPI_SYS_UnBind(&mpp_chn_ai, &mpp_chn_ao);
RK_MPI_AI_DisableChn(mpp_chn_ai.s32ChnId);
RK_MPI_AO_DisableChn(mpp_chn_ao.s32ChnId);
return RK_SUCCESS;
}
static RK_VOID RKMEDIA_AUDIO_Usage() {
printf("\n\n/Usage:./rkmdia_audio <index> <sampleRate> [filePath]/ "
"[nbsamples] [ailayout]\n");
printf("\tindex and its function list below\n");
printf("\t0: start AI to AO loop\n");
printf("\t1: send audio frame to AENC channel from AI, save them\n");
printf("\t2: read audio stream from file, decode and send AO\n");
printf("\t3: start AI(VQE process), then send to AO\n");
// printf("\t4: start AI to Extern Resampler\n");
printf("\n");
printf("\tsampleRate list:\n");
printf("\t0 16000 22050 24000 32000 44100 48000\n");
printf("\n");
printf("\tfilePath represents the path of audio file to be decoded, only for "
"sample 2.\n");
printf("\tdefault filePath: /userdata/out.mp2\n");
printf("\n");
printf("\tnbsamples, for example: 160 is 10ms at 16kHz\n");
printf("\n");
printf("\tailayout:\n");
printf("\t0: AI_LAYOUT_NORMAL\n");
printf("\t1: AI_LAYOUT_MIC_REF\n");
printf("\t2: AI_LAYOUT_REF_MIC\n");
printf("\t3: AI_LAYOUT_2MIC_REF_NONE\n");
printf("\t4: AI_LAYOUT_2MIC_NONE_REF\n");
printf("\t5: AI_LAYOUT_2MIC_2REF\n");
printf("\t6: AI_LAYOUT_BUTT\n");
printf("\n");
printf("\texample: ./rkmdia_audio 0 48000 480 1 /tmp/out_aiao.mp2\n");
}
int main(int argc, char *argv[]) {
RK_U32 u32Index;
RK_CHAR *pFilePath = RK_NULL;
if (!(argc >= 3 && argc <= 6)) {
RKMEDIA_AUDIO_Usage();
return -1;
}
if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "-?") == 0) {
RKMEDIA_AUDIO_Usage();
return -1;
}
u32Index = atoi(argv[1]);
g_enWorkSampleRate = atoi(argv[2]);
switch (argc) {
case 6:
pFilePath = argv[5];
g_s32AiLayout = atoi(argv[4]);
g_s32VqeFrameSample = atoi(argv[3]);
break;
case 5:
g_s32AiLayout = atoi(argv[4]);
g_s32VqeFrameSample = atoi(argv[3]);
break;
case 4:
g_s32VqeFrameSample = atoi(argv[3]);
break;
default:
pFilePath = (char *)"/tmp/out.mp2";
break;
}
switch (u32Index) {
case 0:
AI_AO();
break;
case 1:
AI_AENC_FILE(pFilePath);
break;
case 2:
FILE_ADEC_AO(pFilePath);
break;
case 3:
AI_VqeProcess_AO();
default:
break;
}
return 0;
}

24
resources/build.sh
View File

@ -49,6 +49,7 @@ function qtmoc() {
"src/qt/recoUi/recoUiCallConsole.h:src/qt/recoUi/moc_recoUiCallConsole.cpp"
"src/qt/recoUi/recoUiCallDial.h:src/qt/recoUi/moc_recoUiCallDial.cpp"
"src/qt/utility/DndModeCountDownItem.h:src/qt/utility/moc_DndModeCountDownItem.cpp"
"src/qt/mainUi/UvcView.h:src/qt/mainUi/moc_UvcView.cpp"
)
for file in "${files[@]}"; do
@ -93,19 +94,18 @@ function deploy() {
fi
echo "deploy to target $TARGET_IP, path: ${TARGET_PATH} ..."
ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "mount -o remount rw /system/"
ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "mount -o remount rw /system/; mount -o remount rw /"
ssh -i ~/Projects/ssh_host_rsa_key_ok root@${TARGET_IP} "killall start-app.sh; pgrep -f GateFace | xargs kill -s 9"
ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "pgrep -f VoucherVerifyServer | xargs kill -s 9"
ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "pgrep -f Record | xargs kill -s 9"
scp -i ~/Projects/ssh_host_rsa_key_ok ${build_path}/src/gate_face/GateFace root@${TARGET_IP}:${TARGET_PATH}
# scp -i ~/Projects/ssh_host_rsa_key_ok ${build_path}/ThirdParty/librwSrvProtocol.so root@${TARGET_IP}:/system/lib/
scp -i resources/ssh_host_rsa_key_ok ${build_path}/Tools/Record/Record root@${TARGET_IP}:/sdcard/
scp -i resources/ssh_host_rsa_key_ok ${build_path}/Tools/Record/rkmedia_audio_test root@${TARGET_IP}:/sdcard/
scp -i resources/ssh_host_rsa_key_ok ./3rdparty/arm-linux-gnueabihf/mpp/rk-libs/libeasymedia.so.1.0.1 root@${TARGET_IP}:/system/lib/
scp -i resources/ssh_host_rsa_key_ok ${build_path}/Record/Record root@${TARGET_IP}:/sdcard/
ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "sync"
# ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "echo 116 > /sys/class/gpio/export"
# ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "echo out > /sys/class/gpio/gpio116/direction"
# ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "echo 1 > /sys/class/gpio/gpio116/value"
if [ $debug_deploy != true ]; then
echo "reboot remote device."
ssh -i resources/ssh_host_rsa_key_ok root@${TARGET_IP} "reboot"
@ -146,6 +146,11 @@ function copy_ssh() {
}
function build_old() {
# start-app.sh 以下就可以让应用不开机自启
# d_state=0
# l_state=0
# n_state=1
# i_state=1
if [ -n "$1" ]; then
TARGET_IP=$1
fi
@ -175,6 +180,7 @@ function build_old() {
}
function deploy_old() {
# killall start-app.sh; ps | grep GateFace | grep -v "grep" | awk '{print $1}' | xargs kill -s 9
if [ -n "$1" ]; then
TARGET_IP=$1
fi
@ -186,9 +192,9 @@ function deploy_old() {
ssh -i ~/Projects/ssh_host_rsa_key_ok root@${TARGET_IP} "killall start-app.sh; killall GateFace"
ssh -i ~/Projects/ssh_host_rsa_key_ok root@${TARGET_IP} "killall start-app.sh; killall GateFace"
else
ssh -i ~/Projects/ssh_host_rsa_key_ok root@${TARGET_IP} "killall start-app.sh; killall GateFace; killall netconfig"
ssh -i ~/Projects/ssh_host_rsa_key_ok root@${TARGET_IP} "killall start-app.sh; killall GateFace;"
fi
scp -r -i ~/Projects/ssh_host_rsa_key_ok src/web/php/*.php root@${TARGET_IP}:/system/www/web/
# scp -r -i ~/Projects/ssh_host_rsa_key_ok src/web/php/*.php root@${TARGET_IP}:/system/www/web/
# scp -r -i ~/Projects/ssh_host_rsa_key_ok resources/audio/*.wav root@${TARGET_IP}:/system/audio
scp -i ~/Projects/ssh_host_rsa_key_ok ./build/GateFace root@${TARGET_IP}:$TARGET_PATH
# scp -i ~/Projects/ssh_host_rsa_key_ok resources/language/FaceTick_EN.qm root@${TARGET_IP}:/system/language

2
resources/cmake/toolchain.cmake
View File

@ -19,4 +19,4 @@ set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
string(APPEND CMAKE_CXX_FLAGS " -mfloat-abi=hard -mfpu=neon")
set(BOOST_ROOT /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/boost_1_85_0)
set(BOOST_ROOT /opt/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/lib/boost_1_86_0)