From 2bed1dacf213742dff37bb8bf4ed12da5d373754 Mon Sep 17 00:00:00 2001 From: luocai Date: Fri, 6 Sep 2024 18:26:45 +0800 Subject: [PATCH] add vad code. --- VocieProcess/CMakeLists.txt | 43 +- VocieProcess/api/audio/audio_frame.cc | 235 +++++ VocieProcess/api/audio/audio_frame.h | 230 +++++ VocieProcess/api/function_view.h | 131 +++ VocieProcess/api/make_ref_counted.h | 170 ++++ VocieProcess/api/ref_counted_base.h | 106 ++ VocieProcess/api/rtp_headers.cc | 58 ++ VocieProcess/api/rtp_headers.h | 208 ++++ VocieProcess/api/rtp_packet_info.cc | 63 ++ VocieProcess/api/rtp_packet_info.h | 117 +++ VocieProcess/api/rtp_packet_infos.h | 130 +++ VocieProcess/api/sequence_checker.h | 141 +++ VocieProcess/api/video/color_space.cc | 269 +++++ VocieProcess/api/video/color_space.h | 181 ++++ VocieProcess/api/video/hdr_metadata.cc | 21 + VocieProcess/api/video/hdr_metadata.h | 105 ++ VocieProcess/api/video/video_content_type.cc | 45 + VocieProcess/api/video/video_content_type.h | 36 + VocieProcess/api/video/video_rotation.h | 26 + VocieProcess/api/video/video_timing.cc | 124 +++ VocieProcess/api/video/video_timing.h | 150 +++ VocieProcess/common_audio/audio_converter.cc | 219 ++++ VocieProcess/common_audio/audio_converter.h | 72 ++ .../resampler/include/push_resampler.h | 58 ++ .../resampler/include/resampler.h | 99 ++ .../common_audio/vad/include/webrtc_vad.h | 87 ++ .../audio_coding/codecs/isac/bandwidth_info.h | 24 + .../isac/main/source/filter_functions.c | 195 ++++ .../isac/main/source/filter_functions.h | 25 + .../codecs/isac/main/source/isac_vad.c | 409 ++++++++ .../codecs/isac/main/source/isac_vad.h | 45 + .../isac/main/source/os_specific_inline.h | 42 + .../codecs/isac/main/source/pitch_estimator.c | 695 +++++++++++++ .../codecs/isac/main/source/pitch_estimator.h | 32 + .../codecs/isac/main/source/pitch_filter.c | 388 ++++++++ .../codecs/isac/main/source/pitch_filter.h | 42 + .../codecs/isac/main/source/settings.h | 196 ++++ .../codecs/isac/main/source/structs.h | 448 +++++++++ .../aec_dump/aec_dump_factory.h | 48 + .../audio_samples_scaler.cc | 92 ++ .../audio_samples_scaler.h | 46 + .../capture_levels_adjuster.cc | 96 ++ .../capture_levels_adjuster.h | 88 ++ .../echo_control_mobile_impl.cc | 287 ++++++ .../echo_control_mobile_impl.h | 86 ++ .../audio_processing/include/aec_dump.cc | 41 + .../audio_processing/include/aec_dump.h | 116 +++ .../include/audio_frame_proxies.cc | 66 ++ .../include/audio_frame_proxies.h | 41 + .../include/audio_frame_view.h | 66 ++ .../render_queue_item_verifier.h | 36 + .../modules/audio_processing/rms_level.cc | 138 +++ .../modules/audio_processing/rms_level.h | 77 ++ .../modules/audio_processing/vad/common.h | 29 + .../modules/audio_processing/vad/gmm.cc | 61 ++ .../modules/audio_processing/vad/gmm.h | 45 + .../audio_processing/vad/noise_gmm_tables.h | 82 ++ .../audio_processing/vad/pitch_based_vad.cc | 120 +++ .../audio_processing/vad/pitch_based_vad.h | 57 ++ .../audio_processing/vad/pitch_internal.cc | 55 + .../audio_processing/vad/pitch_internal.h | 30 + .../audio_processing/vad/pole_zero_filter.cc | 107 ++ .../audio_processing/vad/pole_zero_filter.h | 51 + .../audio_processing/vad/standalone_vad.cc | 91 ++ .../audio_processing/vad/standalone_vad.h | 69 ++ .../audio_processing/vad/vad_audio_proc.cc | 275 +++++ .../audio_processing/vad/vad_audio_proc.h | 88 ++ .../vad/vad_audio_proc_internal.h | 81 ++ .../vad/vad_circular_buffer.cc | 135 +++ .../vad/vad_circular_buffer.h | 69 ++ .../vad/voice_activity_detector.cc | 85 ++ .../vad/voice_activity_detector.h | 74 ++ .../audio_processing/vad/voice_gmm_tables.h | 77 ++ VocieProcess/modules/third_party/fft/fft.c | 942 ++++++++++++++++++ VocieProcess/modules/third_party/fft/fft.h | 58 ++ VocieProcess/rtc_base/event.cc | 210 ++++ VocieProcess/rtc_base/event.h | 137 +++ VocieProcess/rtc_base/event_tracer.cc | 449 +++++++++ VocieProcess/rtc_base/event_tracer.h | 88 ++ VocieProcess/rtc_base/platform_thread.cc | 213 ++++ VocieProcess/rtc_base/platform_thread.h | 120 +++ VocieProcess/rtc_base/ref_count.h | 29 + VocieProcess/rtc_base/ref_counted_object.h | 142 +++ VocieProcess/rtc_base/ref_counter.h | 75 ++ .../sequence_checker_internal.cc | 86 ++ .../sequence_checker_internal.h | 90 ++ .../rtc_base/synchronization/yield_policy.cc | 82 ++ .../rtc_base/synchronization/yield_policy.h | 38 + .../rtc_base/system/ignore_warnings.h | 29 + .../warn_current_thread_is_deadlocked.cc | 19 + .../warn_current_thread_is_deadlocked.h | 24 + VocieProcess/rtc_base/trace_event.h | 807 +++++++++++++++ .../include/denormal_disabler.h | 56 ++ 93 files changed, 12362 insertions(+), 2 deletions(-) create mode 100644 VocieProcess/api/audio/audio_frame.cc create mode 100644 VocieProcess/api/audio/audio_frame.h create mode 100644 VocieProcess/api/function_view.h create mode 100644 VocieProcess/api/make_ref_counted.h create mode 100644 VocieProcess/api/ref_counted_base.h create mode 100644 VocieProcess/api/rtp_headers.cc create mode 100644 VocieProcess/api/rtp_headers.h create mode 100644 VocieProcess/api/rtp_packet_info.cc create mode 100644 VocieProcess/api/rtp_packet_info.h create mode 100644 VocieProcess/api/rtp_packet_infos.h create mode 100644 VocieProcess/api/sequence_checker.h create mode 100644 VocieProcess/api/video/color_space.cc create mode 100644 VocieProcess/api/video/color_space.h create mode 100644 VocieProcess/api/video/hdr_metadata.cc create mode 100644 VocieProcess/api/video/hdr_metadata.h create mode 100644 VocieProcess/api/video/video_content_type.cc create mode 100644 VocieProcess/api/video/video_content_type.h create mode 100644 VocieProcess/api/video/video_rotation.h create mode 100644 VocieProcess/api/video/video_timing.cc create mode 100644 VocieProcess/api/video/video_timing.h create mode 100644 VocieProcess/common_audio/audio_converter.cc create mode 100644 VocieProcess/common_audio/audio_converter.h create mode 100644 VocieProcess/common_audio/resampler/include/push_resampler.h create mode 100644 VocieProcess/common_audio/resampler/include/resampler.h create mode 100644 VocieProcess/common_audio/vad/include/webrtc_vad.h create mode 100644 VocieProcess/modules/audio_coding/codecs/isac/bandwidth_info.h create mode 100644 VocieProcess/modules/audio_coding/codecs/isac/main/source/filter_functions.c create mode 100644 VocieProcess/modules/audio_coding/codecs/isac/main/source/filter_functions.h create mode 100644 VocieProcess/modules/audio_coding/codecs/isac/main/source/isac_vad.c create mode 100644 VocieProcess/modules/audio_coding/codecs/isac/main/source/isac_vad.h create mode 100644 VocieProcess/modules/audio_coding/codecs/isac/main/source/os_specific_inline.h create mode 100644 VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c create mode 100644 VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_estimator.h create mode 100644 VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_filter.c create mode 100644 VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_filter.h create mode 100644 VocieProcess/modules/audio_coding/codecs/isac/main/source/settings.h create mode 100644 VocieProcess/modules/audio_coding/codecs/isac/main/source/structs.h create mode 100644 VocieProcess/modules/audio_processing/aec_dump/aec_dump_factory.h create mode 100644 VocieProcess/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.cc create mode 100644 VocieProcess/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.h create mode 100644 VocieProcess/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.cc create mode 100644 VocieProcess/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h create mode 100644 VocieProcess/modules/audio_processing/echo_control_mobile_impl.cc create mode 100644 VocieProcess/modules/audio_processing/echo_control_mobile_impl.h create mode 100644 VocieProcess/modules/audio_processing/include/aec_dump.cc create mode 100644 VocieProcess/modules/audio_processing/include/aec_dump.h create mode 100644 VocieProcess/modules/audio_processing/include/audio_frame_proxies.cc create mode 100644 VocieProcess/modules/audio_processing/include/audio_frame_proxies.h create mode 100644 VocieProcess/modules/audio_processing/include/audio_frame_view.h create mode 100644 VocieProcess/modules/audio_processing/render_queue_item_verifier.h create mode 100644 VocieProcess/modules/audio_processing/rms_level.cc create mode 100644 VocieProcess/modules/audio_processing/rms_level.h create mode 100644 VocieProcess/modules/audio_processing/vad/common.h create mode 100644 VocieProcess/modules/audio_processing/vad/gmm.cc create mode 100644 VocieProcess/modules/audio_processing/vad/gmm.h create mode 100644 VocieProcess/modules/audio_processing/vad/noise_gmm_tables.h create mode 100644 VocieProcess/modules/audio_processing/vad/pitch_based_vad.cc create mode 100644 VocieProcess/modules/audio_processing/vad/pitch_based_vad.h create mode 100644 VocieProcess/modules/audio_processing/vad/pitch_internal.cc create mode 100644 VocieProcess/modules/audio_processing/vad/pitch_internal.h create mode 100644 VocieProcess/modules/audio_processing/vad/pole_zero_filter.cc create mode 100644 VocieProcess/modules/audio_processing/vad/pole_zero_filter.h create mode 100644 VocieProcess/modules/audio_processing/vad/standalone_vad.cc create mode 100644 VocieProcess/modules/audio_processing/vad/standalone_vad.h create mode 100644 VocieProcess/modules/audio_processing/vad/vad_audio_proc.cc create mode 100644 VocieProcess/modules/audio_processing/vad/vad_audio_proc.h create mode 100644 VocieProcess/modules/audio_processing/vad/vad_audio_proc_internal.h create mode 100644 VocieProcess/modules/audio_processing/vad/vad_circular_buffer.cc create mode 100644 VocieProcess/modules/audio_processing/vad/vad_circular_buffer.h create mode 100644 VocieProcess/modules/audio_processing/vad/voice_activity_detector.cc create mode 100644 VocieProcess/modules/audio_processing/vad/voice_activity_detector.h create mode 100644 VocieProcess/modules/audio_processing/vad/voice_gmm_tables.h create mode 100644 VocieProcess/modules/third_party/fft/fft.c create mode 100644 VocieProcess/modules/third_party/fft/fft.h create mode 100644 VocieProcess/rtc_base/event.cc create mode 100644 VocieProcess/rtc_base/event.h create mode 100644 VocieProcess/rtc_base/event_tracer.cc create mode 100644 VocieProcess/rtc_base/event_tracer.h create mode 100644 VocieProcess/rtc_base/platform_thread.cc create mode 100644 VocieProcess/rtc_base/platform_thread.h create mode 100644 VocieProcess/rtc_base/ref_count.h create mode 100644 VocieProcess/rtc_base/ref_counted_object.h create mode 100644 VocieProcess/rtc_base/ref_counter.h create mode 100644 VocieProcess/rtc_base/synchronization/sequence_checker_internal.cc create mode 100644 VocieProcess/rtc_base/synchronization/sequence_checker_internal.h create mode 100644 VocieProcess/rtc_base/synchronization/yield_policy.cc create mode 100644 VocieProcess/rtc_base/synchronization/yield_policy.h create mode 100644 VocieProcess/rtc_base/system/ignore_warnings.h create mode 100644 VocieProcess/rtc_base/system/warn_current_thread_is_deadlocked.cc create mode 100644 VocieProcess/rtc_base/system/warn_current_thread_is_deadlocked.h create mode 100644 VocieProcess/rtc_base/trace_event.h create mode 100644 VocieProcess/system_wrappers/include/denormal_disabler.h diff --git a/VocieProcess/CMakeLists.txt b/VocieProcess/CMakeLists.txt index d13b3a9..a3906f7 100644 --- a/VocieProcess/CMakeLists.txt +++ b/VocieProcess/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.29) - project(VocieProcess) set(CMAKE_CXX_STANDARD 17) @@ -15,6 +13,10 @@ FetchContent_MakeAvailable(absl) add_library(VocieProcess + api/rtp_headers.h api/rtp_headers.cc + api/rtp_packet_info.h api/rtp_packet_info.cc + + api/audio/audio_frame.h api/audio/audio_frame.cc api/audio/audio_processing_statistics.h api/audio/audio_processing_statistics.cc api/audio/audio_processing.h api/audio/audio_processing.cc api/audio/channel_layout.h api/audio/channel_layout.cc @@ -26,6 +28,12 @@ add_library(VocieProcess api/units/time_delta.h api/units/time_delta.cc api/units/timestamp.h api/units/timestamp.cc + api/video/color_space.h api/video/color_space.cc + api/video/hdr_metadata.h api/video/hdr_metadata.cc + api/video/video_content_type.h api/video/video_content_type.cc + api/video/video_timing.h api/video/video_timing.cc + + common_audio/audio_converter.h common_audio/audio_converter.cc common_audio/audio_util.cc common_audio/channel_buffer.h common_audio/channel_buffer.cc common_audio/fir_filter_neon.h common_audio/fir_filter_neon.cc @@ -59,8 +67,11 @@ add_library(VocieProcess common_audio/third_party/spl_sqrt_floor/spl_sqrt_floor.h common_audio/third_party/spl_sqrt_floor/spl_sqrt_floor.c rtc_base/checks.h rtc_base/checks.cc + rtc_base/event_tracer.h rtc_base/event_tracer.cc + rtc_base/event.h rtc_base/event.cc rtc_base/logging.h rtc_base/logging.cc rtc_base/platform_thread_types.h rtc_base/platform_thread_types.cc + rtc_base/platform_thread.h rtc_base/platform_thread.cc rtc_base/race_checker.h rtc_base/race_checker.cc rtc_base/string_encode.h rtc_base/string_encode.cc rtc_base/string_to_number.h rtc_base/string_to_number.cc @@ -77,13 +88,27 @@ add_library(VocieProcess rtc_base/strings/string_builder.h rtc_base/strings/string_builder.cc + rtc_base/synchronization/sequence_checker_internal.h rtc_base/synchronization/sequence_checker_internal.cc + rtc_base/synchronization/yield_policy.h rtc_base/synchronization/yield_policy.cc + rtc_base/system/file_wrapper.h rtc_base/system/file_wrapper.cc + rtc_base/system/warn_current_thread_is_deadlocked.h rtc_base/system/warn_current_thread_is_deadlocked.cc + + modules/audio_coding/codecs/isac/main/source/filter_functions.h modules/audio_coding/codecs/isac/main/source/filter_functions.c + modules/audio_coding/codecs/isac/main/source/isac_vad.h modules/audio_coding/codecs/isac/main/source/isac_vad.c + modules/audio_coding/codecs/isac/main/source/pitch_estimator.h modules/audio_coding/codecs/isac/main/source/pitch_estimator.c + modules/audio_coding/codecs/isac/main/source/pitch_filter.h modules/audio_coding/codecs/isac/main/source/pitch_filter.c modules/audio_processing/audio_buffer.h modules/audio_processing/audio_buffer.cc + modules/audio_processing/echo_control_mobile_impl.h modules/audio_processing/echo_control_mobile_impl.cc modules/audio_processing/high_pass_filter.h modules/audio_processing/high_pass_filter.cc + modules/audio_processing/rms_level.h modules/audio_processing/rms_level.cc modules/audio_processing/splitting_filter.h modules/audio_processing/splitting_filter.cc modules/audio_processing/three_band_filter_bank.h modules/audio_processing/three_band_filter_bank.cc + modules/audio_processing/include/aec_dump.h modules/audio_processing/include/aec_dump.cc + modules/audio_processing/include/audio_frame_proxies.h modules/audio_processing/include/audio_frame_proxies.cc + modules/audio_processing/aec3/adaptive_fir_filter_erl.h modules/audio_processing/aec3/adaptive_fir_filter_erl.cc modules/audio_processing/aec3/adaptive_fir_filter.h modules/audio_processing/aec3/adaptive_fir_filter.cc modules/audio_processing/aec3/aec_state.h modules/audio_processing/aec3/aec_state.cc @@ -146,6 +171,9 @@ add_library(VocieProcess modules/audio_processing/aecm/aecm_core_neon.cc modules/audio_processing/aecm/echo_control_mobile.h modules/audio_processing/aecm/echo_control_mobile.cc + modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.h modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.cc + modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.cc + modules/audio_processing/logging/apm_data_dumper.h modules/audio_processing/logging/apm_data_dumper.cc modules/audio_processing/ns/fast_math.h modules/audio_processing/ns/fast_math.cc @@ -166,6 +194,17 @@ add_library(VocieProcess modules/audio_processing/utility/delay_estimator_wrapper.h modules/audio_processing/utility/delay_estimator_wrapper.cc modules/audio_processing/utility/delay_estimator.h modules/audio_processing/utility/delay_estimator.cc + modules/audio_processing/vad/gmm.h modules/audio_processing/vad/gmm.cc + modules/audio_processing/vad/pitch_based_vad.h modules/audio_processing/vad/pitch_based_vad.cc + modules/audio_processing/vad/pitch_internal.h modules/audio_processing/vad/pitch_internal.cc + modules/audio_processing/vad/pole_zero_filter.h modules/audio_processing/vad/pole_zero_filter.cc + modules/audio_processing/vad/standalone_vad.h modules/audio_processing/vad/standalone_vad.cc + modules/audio_processing/vad/vad_audio_proc.h modules/audio_processing/vad/vad_audio_proc.cc + modules/audio_processing/vad/vad_circular_buffer.h modules/audio_processing/vad/vad_circular_buffer.cc + modules/audio_processing/vad/voice_activity_detector.h modules/audio_processing/vad/voice_activity_detector.cc + + modules/third_party/fft/fft.h modules/third_party/fft/fft.c + system_wrappers/source/field_trial.cc system_wrappers/source/metrics.cc ) diff --git a/VocieProcess/api/audio/audio_frame.cc b/VocieProcess/api/audio/audio_frame.cc new file mode 100644 index 0000000..a2aa774 --- /dev/null +++ b/VocieProcess/api/audio/audio_frame.cc @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/audio/audio_frame.h" + +#include + +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/audio/audio_view.h" +#include "api/audio/channel_layout.h" +#include "api/rtp_packet_infos.h" +#include "rtc_base/checks.h" +#include "rtc_base/time_utils.h" + +namespace webrtc { + +AudioFrame::AudioFrame() { + // Visual Studio doesn't like this in the class definition. + static_assert(sizeof(data_) == kMaxDataSizeBytes, "kMaxDataSizeBytes"); +} + +AudioFrame::AudioFrame(int sample_rate_hz, + size_t num_channels, + ChannelLayout layout /*= CHANNEL_LAYOUT_UNSUPPORTED*/) + : samples_per_channel_(SampleRateToDefaultChannelSize(sample_rate_hz)), + sample_rate_hz_(sample_rate_hz), + num_channels_(num_channels), + channel_layout_(layout == CHANNEL_LAYOUT_UNSUPPORTED + ? GuessChannelLayout(num_channels) + : layout) { + RTC_DCHECK_LE(num_channels_, kMaxConcurrentChannels); + RTC_DCHECK_GT(sample_rate_hz_, 0); + RTC_DCHECK_GT(samples_per_channel_, 0u); +} + +void AudioFrame::Reset() { + ResetWithoutMuting(); + muted_ = true; +} + +void AudioFrame::ResetWithoutMuting() { + // TODO(wu): Zero is a valid value for `timestamp_`. We should initialize + // to an invalid value, or add a new member to indicate invalidity. + timestamp_ = 0; + elapsed_time_ms_ = -1; + ntp_time_ms_ = -1; + samples_per_channel_ = 0; + sample_rate_hz_ = 0; + num_channels_ = 0; + channel_layout_ = CHANNEL_LAYOUT_NONE; + speech_type_ = kUndefined; + vad_activity_ = kVadUnknown; + profile_timestamp_ms_ = 0; + packet_infos_ = RtpPacketInfos(); + absolute_capture_timestamp_ms_ = absl::nullopt; +} + +void AudioFrame::UpdateFrame(uint32_t timestamp, + const int16_t* data, + size_t samples_per_channel, + int sample_rate_hz, + SpeechType speech_type, + VADActivity vad_activity, + size_t num_channels) { + RTC_CHECK_LE(num_channels, kMaxConcurrentChannels); + timestamp_ = timestamp; + samples_per_channel_ = samples_per_channel; + sample_rate_hz_ = sample_rate_hz; + speech_type_ = speech_type; + vad_activity_ = vad_activity; + num_channels_ = num_channels; + channel_layout_ = GuessChannelLayout(num_channels); + if (channel_layout_ != CHANNEL_LAYOUT_UNSUPPORTED) { + RTC_DCHECK_EQ(num_channels, ChannelLayoutToChannelCount(channel_layout_)); + } + + const size_t length = samples_per_channel * num_channels; + RTC_CHECK_LE(length, data_.size()); + if (data != nullptr) { + memcpy(data_.data(), data, sizeof(int16_t) * length); + muted_ = false; + } else { + muted_ = true; + } +} + +void AudioFrame::CopyFrom(const AudioFrame& src) { + if (this == &src) + return; + + if (muted_ && !src.muted()) { + // TODO: bugs.webrtc.org/5647 - Since the default value for `muted_` is + // false and `data_` may still be uninitialized (because we don't initialize + // data_ as part of construction), we clear the full buffer here before + // copying over new values. If we don't, msan might complain in some tests. + // Consider locking down construction, avoiding the default constructor and + // prefering construction that initializes all state. + ClearSamples(data_); + } + + timestamp_ = src.timestamp_; + elapsed_time_ms_ = src.elapsed_time_ms_; + ntp_time_ms_ = src.ntp_time_ms_; + packet_infos_ = src.packet_infos_; + muted_ = src.muted(); + samples_per_channel_ = src.samples_per_channel_; + sample_rate_hz_ = src.sample_rate_hz_; + speech_type_ = src.speech_type_; + vad_activity_ = src.vad_activity_; + num_channels_ = src.num_channels_; + channel_layout_ = src.channel_layout_; + absolute_capture_timestamp_ms_ = src.absolute_capture_timestamp_ms(); + + auto data = src.data_view(); + RTC_CHECK_LE(data.size(), data_.size()); + if (!muted_ && !data.empty()) { + memcpy(&data_[0], &data[0], sizeof(int16_t) * data.size()); + } +} + +void AudioFrame::UpdateProfileTimeStamp() { + profile_timestamp_ms_ = rtc::TimeMillis(); +} + +int64_t AudioFrame::ElapsedProfileTimeMs() const { + if (profile_timestamp_ms_ == 0) { + // Profiling has not been activated. + return -1; + } + return rtc::TimeSince(profile_timestamp_ms_); +} + +const int16_t* AudioFrame::data() const { + return muted_ ? zeroed_data().begin() : data_.data(); +} + +InterleavedView AudioFrame::data_view() const { + // If you get a nullptr from `data_view()`, it's likely because the + // samples_per_channel_ and/or num_channels_ members haven't been properly + // set. Since `data_view()` returns an InterleavedView<> (which internally + // uses rtc::ArrayView<>), we inherit the behavior in InterleavedView when the + // view size is 0 that ArrayView<>::data() returns nullptr. So, even when an + // AudioFrame is muted and we want to return `zeroed_data()`, if + // samples_per_channel_ or num_channels_ is 0, the view will point to + // nullptr. + return InterleavedView(muted_ ? &zeroed_data()[0] : &data_[0], + samples_per_channel_, num_channels_); +} + +int16_t* AudioFrame::mutable_data() { + // TODO: bugs.webrtc.org/5647 - Can we skip zeroing the buffer? + // Consider instead if we should rather zero the buffer when `muted_` is set + // to `true`. + if (muted_) { + ClearSamples(data_); + muted_ = false; + } + return &data_[0]; +} + +InterleavedView AudioFrame::mutable_data(size_t samples_per_channel, + size_t num_channels) { + const size_t total_samples = samples_per_channel * num_channels; + RTC_CHECK_LE(total_samples, data_.size()); + RTC_CHECK_LE(num_channels, kMaxConcurrentChannels); + // Sanity check for valid argument values during development. + // If `samples_per_channel` is < `num_channels` but larger than 0, + // then chances are the order of arguments is incorrect. + RTC_DCHECK((samples_per_channel == 0 && num_channels == 0) || + num_channels <= samples_per_channel) + << "samples_per_channel=" << samples_per_channel + << "num_channels=" << num_channels; + + // TODO: bugs.webrtc.org/5647 - Can we skip zeroing the buffer? + // Consider instead if we should rather zero the whole buffer when `muted_` is + // set to `true`. + if (muted_) { + ClearSamples(data_, total_samples); + muted_ = false; + } + samples_per_channel_ = samples_per_channel; + num_channels_ = num_channels; + return InterleavedView(&data_[0], samples_per_channel, num_channels); +} + +void AudioFrame::Mute() { + muted_ = true; +} + +bool AudioFrame::muted() const { + return muted_; +} + +void AudioFrame::SetLayoutAndNumChannels(ChannelLayout layout, + size_t num_channels) { + channel_layout_ = layout; + num_channels_ = num_channels; +#if RTC_DCHECK_IS_ON + // Do a sanity check that the layout and num_channels match. + // If this lookup yield 0u, then the layout is likely CHANNEL_LAYOUT_DISCRETE. + auto expected_num_channels = ChannelLayoutToChannelCount(layout); + if (expected_num_channels) { // If expected_num_channels is 0 + RTC_DCHECK_EQ(expected_num_channels, num_channels_); + } +#endif + RTC_CHECK_LE(samples_per_channel_ * num_channels_, data_.size()); +} + +void AudioFrame::SetSampleRateAndChannelSize(int sample_rate) { + sample_rate_hz_ = sample_rate; + // We could call `AudioProcessing::GetFrameSize()` here, but that requires + // adding a dependency on the ":audio_processing" build target, which can + // complicate the dependency tree. Some refactoring is probably in order to + // get some consistency around this since there are many places across the + // code that assume this default buffer size. + samples_per_channel_ = SampleRateToDefaultChannelSize(sample_rate_hz_); +} + +// static +rtc::ArrayView AudioFrame::zeroed_data() { + static int16_t* null_data = new int16_t[kMaxDataSizeSamples](); + return rtc::ArrayView(null_data, kMaxDataSizeSamples); +} + +} // namespace webrtc diff --git a/VocieProcess/api/audio/audio_frame.h b/VocieProcess/api/audio/audio_frame.h new file mode 100644 index 0000000..5683e8b --- /dev/null +++ b/VocieProcess/api/audio/audio_frame.h @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_AUDIO_AUDIO_FRAME_H_ +#define API_AUDIO_AUDIO_FRAME_H_ + +#include +#include + +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/audio/audio_view.h" +#include "api/audio/channel_layout.h" +#include "api/rtp_packet_infos.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +// Default webrtc buffer size in milliseconds. +constexpr size_t kDefaultAudioBufferLengthMs = 10u; + +// Default total number of audio buffers per second based on the default length. +constexpr size_t kDefaultAudioBuffersPerSec = + 1000u / kDefaultAudioBufferLengthMs; + +// Returns the number of samples a buffer needs to hold for ~10ms of a single +// audio channel at a given sample rate. +// See also `AudioProcessing::GetFrameSize()`. +inline size_t SampleRateToDefaultChannelSize(size_t sample_rate) { + // Basic sanity check. 192kHz is the highest supported input sample rate. + RTC_DCHECK_LE(sample_rate, 192000); + return sample_rate / kDefaultAudioBuffersPerSec; +} +///////////////////////////////////////////////////////////////////// + +/* This class holds up to 120 ms of super-wideband (32 kHz) stereo audio. It + * allows for adding and subtracting frames while keeping track of the resulting + * states. + * + * Notes + * - This is a de-facto api, not designed for external use. The AudioFrame class + * is in need of overhaul or even replacement, and anyone depending on it + * should be prepared for that. + * - The total number of samples is samples_per_channel_ * num_channels_. + * - Stereo data is interleaved starting with the left channel. + */ +class AudioFrame { + public: + // Using constexpr here causes linker errors unless the variable also has an + // out-of-class definition, which is impractical in this header-only class. + // (This makes no sense because it compiles as an enum value, which we most + // certainly cannot take the address of, just fine.) C++17 introduces inline + // variables which should allow us to switch to constexpr and keep this a + // header-only class. + enum : size_t { + // Stereo, 32 kHz, 120 ms (2 * 32 * 120) + // Stereo, 192 kHz, 20 ms (2 * 192 * 20) + kMaxDataSizeSamples = 7680, + kMaxDataSizeBytes = kMaxDataSizeSamples * sizeof(int16_t), + }; + + enum VADActivity { kVadActive = 0, kVadPassive = 1, kVadUnknown = 2 }; + enum SpeechType { + kNormalSpeech = 0, + kPLC = 1, + kCNG = 2, + kPLCCNG = 3, + kCodecPLC = 5, + kUndefined = 4 + }; + + AudioFrame(); + + // Construct an audio frame with frame length properties and channel + // information. `samples_per_channel()` will be initialized to a 10ms buffer + // size and if `layout` is not specified (default value of + // CHANNEL_LAYOUT_UNSUPPORTED is set), then the channel layout is derived + // (guessed) from `num_channels`. + AudioFrame(int sample_rate_hz, + size_t num_channels, + ChannelLayout layout = CHANNEL_LAYOUT_UNSUPPORTED); + + AudioFrame(const AudioFrame&) = delete; + AudioFrame& operator=(const AudioFrame&) = delete; + + // Resets all members to their default state. + void Reset(); + // Same as Reset(), but leaves mute state unchanged. Muting a frame requires + // the buffer to be zeroed on the next call to mutable_data(). Callers + // intending to write to the buffer immediately after Reset() can instead use + // ResetWithoutMuting() to skip this wasteful zeroing. + void ResetWithoutMuting(); + + // TODO: b/335805780 - Accept InterleavedView. + void UpdateFrame(uint32_t timestamp, + const int16_t* data, + size_t samples_per_channel, + int sample_rate_hz, + SpeechType speech_type, + VADActivity vad_activity, + size_t num_channels = 1); + + void CopyFrom(const AudioFrame& src); + + // Sets a wall-time clock timestamp in milliseconds to be used for profiling + // of time between two points in the audio chain. + // Example: + // t0: UpdateProfileTimeStamp() + // t1: ElapsedProfileTimeMs() => t1 - t0 [msec] + void UpdateProfileTimeStamp(); + // Returns the time difference between now and when UpdateProfileTimeStamp() + // was last called. Returns -1 if UpdateProfileTimeStamp() has not yet been + // called. + int64_t ElapsedProfileTimeMs() const; + + // data() returns a zeroed static buffer if the frame is muted. + // TODO: b/335805780 - Return InterleavedView. + const int16_t* data() const; + + // Returns a read-only view of all the valid samples held by the AudioFrame. + // For a muted AudioFrame, the samples will all be 0. + InterleavedView data_view() const; + + // mutable_frame() always returns a non-static buffer; the first call to + // mutable_frame() zeros the buffer and marks the frame as unmuted. + // TODO: b/335805780 - Return an InterleavedView. + int16_t* mutable_data(); + + // Grants write access to the audio buffer. The size of the returned writable + // view is determined by the `samples_per_channel` and `num_channels` + // dimensions which the function checks for correctness and stores in the + // internal member variables; `samples_per_channel()` and `num_channels()` + // respectively. + // If the state is currently muted, the returned view will be zeroed out. + InterleavedView mutable_data(size_t samples_per_channel, + size_t num_channels); + + // Prefer to mute frames using AudioFrameOperations::Mute. + void Mute(); + // Frame is muted by default. + bool muted() const; + + size_t max_16bit_samples() const { return data_.size(); } + size_t samples_per_channel() const { return samples_per_channel_; } + size_t num_channels() const { return num_channels_; } + + ChannelLayout channel_layout() const { return channel_layout_; } + // Sets the `channel_layout` property as well as `num_channels`. + void SetLayoutAndNumChannels(ChannelLayout layout, size_t num_channels); + + int sample_rate_hz() const { return sample_rate_hz_; } + + void set_absolute_capture_timestamp_ms( + int64_t absolute_capture_time_stamp_ms) { + absolute_capture_timestamp_ms_ = absolute_capture_time_stamp_ms; + } + + absl::optional absolute_capture_timestamp_ms() const { + return absolute_capture_timestamp_ms_; + } + + // Sets the sample_rate_hz and samples_per_channel properties based on a + // given sample rate and calculates a default 10ms samples_per_channel value. + void SetSampleRateAndChannelSize(int sample_rate); + + // RTP timestamp of the first sample in the AudioFrame. + uint32_t timestamp_ = 0; + // Time since the first frame in milliseconds. + // -1 represents an uninitialized value. + int64_t elapsed_time_ms_ = -1; + // NTP time of the estimated capture time in local timebase in milliseconds. + // -1 represents an uninitialized value. + int64_t ntp_time_ms_ = -1; + size_t samples_per_channel_ = 0; + int sample_rate_hz_ = 0; + size_t num_channels_ = 0; + SpeechType speech_type_ = kUndefined; + VADActivity vad_activity_ = kVadUnknown; + // Monotonically increasing timestamp intended for profiling of audio frames. + // Typically used for measuring elapsed time between two different points in + // the audio path. No lock is used to save resources and we are thread safe + // by design. + // TODO(nisse@webrtc.org): consider using absl::optional. + int64_t profile_timestamp_ms_ = 0; + + // Information about packets used to assemble this audio frame. This is needed + // by `SourceTracker` when the frame is delivered to the RTCRtpReceiver's + // MediaStreamTrack, in order to implement getContributingSources(). See: + // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources + // + // TODO(bugs.webrtc.org/10757): + // Note that this information might not be fully accurate since we currently + // don't have a proper way to track it across the audio sync buffer. The + // sync buffer is the small sample-holding buffer located after the audio + // decoder and before where samples are assembled into output frames. + // + // `RtpPacketInfos` may also be empty if the audio samples did not come from + // RTP packets. E.g. if the audio were locally generated by packet loss + // concealment, comfort noise generation, etc. + RtpPacketInfos packet_infos_; + + private: + // A permanently zeroed out buffer to represent muted frames. This is a + // header-only class, so the only way to avoid creating a separate zeroed + // buffer per translation unit is to wrap a static in an inline function. + static rtc::ArrayView zeroed_data(); + + std::array data_; + bool muted_ = true; + ChannelLayout channel_layout_ = CHANNEL_LAYOUT_NONE; + + // Absolute capture timestamp when this audio frame was originally captured. + // This is only valid for audio frames captured on this machine. The absolute + // capture timestamp of a received frame is found in `packet_infos_`. + // This timestamp MUST be based on the same clock as rtc::TimeMillis(). + absl::optional absolute_capture_timestamp_ms_; +}; + +} // namespace webrtc + +#endif // API_AUDIO_AUDIO_FRAME_H_ diff --git a/VocieProcess/api/function_view.h b/VocieProcess/api/function_view.h new file mode 100644 index 0000000..b191a05 --- /dev/null +++ b/VocieProcess/api/function_view.h @@ -0,0 +1,131 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_FUNCTION_VIEW_H_ +#define API_FUNCTION_VIEW_H_ + +#include +#include +#include + +#include "rtc_base/checks.h" + +// Just like std::function, FunctionView will wrap any callable and hide its +// actual type, exposing only its signature. But unlike std::function, +// FunctionView doesn't own its callable---it just points to it. Thus, it's a +// good choice mainly as a function argument when the callable argument will +// not be called again once the function has returned. +// +// Its constructors are implicit, so that callers won't have to convert lambdas +// and other callables to FunctionView explicitly. This is +// safe because FunctionView is only a reference to the real callable. +// +// Example use: +// +// void SomeFunction(rtc::FunctionView index_transform); +// ... +// SomeFunction([](int i) { return 2 * i + 1; }); +// +// Note: FunctionView is tiny (essentially just two pointers) and trivially +// copyable, so it's probably cheaper to pass it by value than by const +// reference. + +namespace rtc { + +template +class FunctionView; // Undefined. + +template +class FunctionView final { + public: + // Constructor for lambdas and other callables; it accepts every type of + // argument except those noted in its enable_if call. + template < + typename F, + typename std::enable_if< + // Not for function pointers; we have another constructor for that + // below. + !std::is_function::type>::type>::value && + + // Not for nullptr; we have another constructor for that below. + !std::is_same::type>::value && + + // Not for FunctionView objects; we have another constructor for that + // (the implicitly declared copy constructor). + !std::is_same::type>::type>::value>::type* = nullptr> + FunctionView(F&& f) + : call_(CallVoidPtr::type>) { + f_.void_ptr = &f; + } + + // Constructor that accepts function pointers. If the argument is null, the + // result is an empty FunctionView. + template < + typename F, + typename std::enable_if::type>::type>::value>::type* = + nullptr> + FunctionView(F&& f) + : call_(f ? CallFunPtr::type> : nullptr) { + f_.fun_ptr = reinterpret_cast(f); + } + + // Constructor that accepts nullptr. It creates an empty FunctionView. + template ::type>::value>::type* = nullptr> + FunctionView(F&& f) : call_(nullptr) {} + + // Default constructor. Creates an empty FunctionView. + FunctionView() : call_(nullptr) {} + + RetT operator()(ArgT... args) const { + RTC_DCHECK(call_); + return call_(f_, std::forward(args)...); + } + + // Returns true if we have a function, false if we don't (i.e., we're null). + explicit operator bool() const { return !!call_; } + + private: + union VoidUnion { + void* void_ptr; + void (*fun_ptr)(); + }; + + template + static RetT CallVoidPtr(VoidUnion vu, ArgT... args) { + return (*static_cast(vu.void_ptr))(std::forward(args)...); + } + template + static RetT CallFunPtr(VoidUnion vu, ArgT... args) { + return (reinterpret_cast::type>(vu.fun_ptr))( + std::forward(args)...); + } + + // A pointer to the callable thing, with type information erased. It's a + // union because we have to use separate types depending on if the callable + // thing is a function pointer or something else. + VoidUnion f_; + + // Pointer to a dispatch function that knows the type of the callable thing + // that's stored in f_, and how to call it. A FunctionView object is empty + // (null) iff call_ is null. + RetT (*call_)(VoidUnion, ArgT...); +}; + +} // namespace rtc + +#endif // API_FUNCTION_VIEW_H_ diff --git a/VocieProcess/api/make_ref_counted.h b/VocieProcess/api/make_ref_counted.h new file mode 100644 index 0000000..40eb3ad --- /dev/null +++ b/VocieProcess/api/make_ref_counted.h @@ -0,0 +1,170 @@ +/* + * Copyright 2022 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_MAKE_REF_COUNTED_H_ +#define API_MAKE_REF_COUNTED_H_ + +#include +#include + +#include "rtc_base/ref_counted_object.h" + +namespace webrtc { + +namespace webrtc_make_ref_counted_internal { +// Determines if the given class has AddRef and Release methods. +template +class HasAddRefAndRelease { + private: + template ().AddRef())* = nullptr, + decltype(std::declval().Release())* = nullptr> + static int Test(int); + template + static char Test(...); + + public: + static constexpr bool value = std::is_same_v(0)), int>; +}; +} // namespace webrtc_make_ref_counted_internal + +// General utilities for constructing a reference counted class and the +// appropriate reference count implementation for that class. +// +// These utilities select either the `RefCountedObject` implementation or +// `FinalRefCountedObject` depending on whether the to-be-shared class is +// derived from the RefCountInterface interface or not (respectively). + +// `make_ref_counted`: +// +// Use this when you want to construct a reference counted object of type T and +// get a `scoped_refptr<>` back. Example: +// +// auto p = make_ref_counted("bar", 123); +// +// For a class that inherits from RefCountInterface, this is equivalent to: +// +// auto p = scoped_refptr(new RefCountedObject("bar", 123)); +// +// If the class does not inherit from RefCountInterface, but does have +// AddRef/Release methods (so a T* is convertible to rtc::scoped_refptr), this +// is equivalent to just +// +// auto p = scoped_refptr(new Foo("bar", 123)); +// +// Otherwise, the example is equivalent to: +// +// auto p = scoped_refptr>( +// new FinalRefCountedObject("bar", 123)); +// +// In these cases, `make_ref_counted` reduces the amount of boilerplate code but +// also helps with the most commonly intended usage of RefCountedObject whereby +// methods for reference counting, are virtual and designed to satisfy the need +// of an interface. When such a need does not exist, it is more efficient to use +// the `FinalRefCountedObject` template, which does not add the vtable overhead. +// +// Note that in some cases, using RefCountedObject directly may still be what's +// needed. + +// `make_ref_counted` for abstract classes that are convertible to +// RefCountInterface. The is_abstract requirement rejects classes that inherit +// both RefCountInterface and RefCounted object, which is a a discouraged +// pattern, and would result in double inheritance of RefCountedObject if this +// template was applied. +template < + typename T, + typename... Args, + typename std::enable_if && + std::is_abstract_v, + T>::type* = nullptr> +scoped_refptr make_ref_counted(Args&&... args) { + return scoped_refptr(new RefCountedObject(std::forward(args)...)); +} + +// `make_ref_counted` for complete classes that are not convertible to +// RefCountInterface and already carry a ref count. +template < + typename T, + typename... Args, + typename std::enable_if< + !std::is_convertible_v && + webrtc_make_ref_counted_internal::HasAddRefAndRelease::value, + T>::type* = nullptr> +scoped_refptr make_ref_counted(Args&&... args) { + return scoped_refptr(new T(std::forward(args)...)); +} + +// `make_ref_counted` for complete classes that are not convertible to +// RefCountInterface and have no ref count of their own. +template < + typename T, + typename... Args, + typename std::enable_if< + !std::is_convertible_v && + !webrtc_make_ref_counted_internal::HasAddRefAndRelease::value, + + T>::type* = nullptr> +scoped_refptr> make_ref_counted(Args&&... args) { + return scoped_refptr>( + new FinalRefCountedObject(std::forward(args)...)); +} + +} // namespace webrtc + +// Backwards compatibe aliases. +// TODO: https://issues.webrtc.org/42225969 - deprecate and remove. +namespace rtc { +// This doesn't work: +// template +// using make_ref_counted(Args&&... args) = +// webrtc::make_ref_counted(Args&&... args); +// Instead, reproduce the templates. +template && + std::is_abstract_v, + T>::type* = nullptr> +scoped_refptr make_ref_counted(Args&&... args) { + return webrtc::scoped_refptr( + new webrtc::RefCountedObject(std::forward(args)...)); +} + +// `make_ref_counted` for complete classes that are not convertible to +// RefCountInterface and already carry a ref count. +template && + webrtc::webrtc_make_ref_counted_internal::HasAddRefAndRelease< + T>::value, + T>::type* = nullptr> +scoped_refptr make_ref_counted(Args&&... args) { + return webrtc::scoped_refptr(new T(std::forward(args)...)); +} + +// `make_ref_counted` for complete classes that are not convertible to +// RefCountInterface and have no ref count of their own. +template && + !webrtc::webrtc_make_ref_counted_internal:: + HasAddRefAndRelease::value, + + T>::type* = nullptr> +scoped_refptr> make_ref_counted( + Args&&... args) { + return webrtc::scoped_refptr>( + new webrtc::FinalRefCountedObject(std::forward(args)...)); +} + +} // namespace rtc + +#endif // API_MAKE_REF_COUNTED_H_ diff --git a/VocieProcess/api/ref_counted_base.h b/VocieProcess/api/ref_counted_base.h new file mode 100644 index 0000000..2638ac1 --- /dev/null +++ b/VocieProcess/api/ref_counted_base.h @@ -0,0 +1,106 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_REF_COUNTED_BASE_H_ +#define API_REF_COUNTED_BASE_H_ + +#include + +#include "rtc_base/ref_counter.h" + +namespace webrtc { + +class RefCountedBase { + public: + RefCountedBase() = default; + + RefCountedBase(const RefCountedBase&) = delete; + RefCountedBase& operator=(const RefCountedBase&) = delete; + + void AddRef() const { ref_count_.IncRef(); } + RefCountReleaseStatus Release() const { + const auto status = ref_count_.DecRef(); + if (status == RefCountReleaseStatus::kDroppedLastRef) { + delete this; + } + return status; + } + + protected: + // Provided for internal webrtc subclasses for corner cases where it's + // necessary to know whether or not a reference is exclusively held. + bool HasOneRef() const { return ref_count_.HasOneRef(); } + + virtual ~RefCountedBase() = default; + + private: + mutable webrtc::webrtc_impl::RefCounter ref_count_{0}; +}; + +// Template based version of `RefCountedBase` for simple implementations that do +// not need (or want) destruction via virtual destructor or the overhead of a +// vtable. +// +// To use: +// struct MyInt : public rtc::RefCountedNonVirtual { +// int foo_ = 0; +// }; +// +// rtc::scoped_refptr my_int(new MyInt()); +// +// sizeof(MyInt) on a 32 bit system would then be 8, int + refcount and no +// vtable generated. +template +class RefCountedNonVirtual { + public: + RefCountedNonVirtual() = default; + + RefCountedNonVirtual(const RefCountedNonVirtual&) = delete; + RefCountedNonVirtual& operator=(const RefCountedNonVirtual&) = delete; + + void AddRef() const { ref_count_.IncRef(); } + RefCountReleaseStatus Release() const { + // If you run into this assert, T has virtual methods. There are two + // options: + // 1) The class doesn't actually need virtual methods, the type is complete + // so the virtual attribute(s) can be removed. + // 2) The virtual methods are a part of the design of the class. In this + // case you can consider using `RefCountedBase` instead or alternatively + // use `rtc::RefCountedObject`. + static_assert(!std::is_polymorphic::value, + "T has virtual methods. RefCountedBase is a better fit."); + const auto status = ref_count_.DecRef(); + if (status == RefCountReleaseStatus::kDroppedLastRef) { + delete static_cast(this); + } + return status; + } + + protected: + // Provided for internal webrtc subclasses for corner cases where it's + // necessary to know whether or not a reference is exclusively held. + bool HasOneRef() const { return ref_count_.HasOneRef(); } + + ~RefCountedNonVirtual() = default; + + private: + mutable webrtc::webrtc_impl::RefCounter ref_count_{0}; +}; + +} // namespace webrtc + +// Backwards compatibe aliases. +// TODO: https://issues.webrtc.org/42225969 - deprecate and remove. +namespace rtc { +using RefCountedBase = webrtc::RefCountedBase; +template +using RefCountedNonVirtual = webrtc::RefCountedNonVirtual; +} // namespace rtc + +#endif // API_REF_COUNTED_BASE_H_ diff --git a/VocieProcess/api/rtp_headers.cc b/VocieProcess/api/rtp_headers.cc new file mode 100644 index 0000000..8813e0f --- /dev/null +++ b/VocieProcess/api/rtp_headers.cc @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/rtp_headers.h" + +namespace webrtc { + +AudioLevel::AudioLevel() : voice_activity_(false), audio_level_(0) {} + +AudioLevel::AudioLevel(bool voice_activity, int audio_level) + : voice_activity_(voice_activity), audio_level_(audio_level) { + RTC_CHECK_GE(audio_level, 0); + RTC_CHECK_LE(audio_level, 127); +} + +RTPHeaderExtension::RTPHeaderExtension() + : hasTransmissionTimeOffset(false), + transmissionTimeOffset(0), + hasAbsoluteSendTime(false), + absoluteSendTime(0), + hasTransportSequenceNumber(false), + transportSequenceNumber(0), + hasVideoRotation(false), + videoRotation(kVideoRotation_0), + hasVideoContentType(false), + videoContentType(VideoContentType::UNSPECIFIED), + has_video_timing(false) {} + +RTPHeaderExtension::RTPHeaderExtension(const RTPHeaderExtension& other) = + default; + +RTPHeaderExtension& RTPHeaderExtension::operator=( + const RTPHeaderExtension& other) = default; + +RTPHeader::RTPHeader() + : markerBit(false), + payloadType(0), + sequenceNumber(0), + timestamp(0), + ssrc(0), + numCSRCs(0), + arrOfCSRCs(), + paddingLength(0), + headerLength(0), + extension() {} + +RTPHeader::RTPHeader(const RTPHeader& other) = default; + +RTPHeader& RTPHeader::operator=(const RTPHeader& other) = default; + +} // namespace webrtc diff --git a/VocieProcess/api/rtp_headers.h b/VocieProcess/api/rtp_headers.h new file mode 100644 index 0000000..129ab5f --- /dev/null +++ b/VocieProcess/api/rtp_headers.h @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_RTP_HEADERS_H_ +#define API_RTP_HEADERS_H_ + +#include +#include + +#include + +#include "absl/types/optional.h" +#include "api/units/timestamp.h" +#include "api/video/color_space.h" +#include "api/video/video_content_type.h" +#include "api/video/video_rotation.h" +#include "api/video/video_timing.h" +#include "rtc_base/checks.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +struct FeedbackRequest { + // Determines whether the recv delta as specified in + // https://tools.ietf.org/html/draft-holmer-rmcat-transport-wide-cc-extensions-01 + // should be included. + bool include_timestamps; + // Include feedback of received packets in the range [sequence_number - + // sequence_count + 1, sequence_number]. That is, no feedback will be sent if + // sequence_count is zero. + int sequence_count; +}; + +// The Absolute Capture Time extension is used to stamp RTP packets with a NTP +// timestamp showing when the first audio or video frame in a packet was +// originally captured. The intent of this extension is to provide a way to +// accomplish audio-to-video synchronization when RTCP-terminating intermediate +// systems (e.g. mixers) are involved. See: +// http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time +struct AbsoluteCaptureTime { + // Absolute capture timestamp is the NTP timestamp of when the first frame in + // a packet was originally captured. This timestamp MUST be based on the same + // clock as the clock used to generate NTP timestamps for RTCP sender reports + // on the capture system. + // + // It’s not always possible to do an NTP clock readout at the exact moment of + // when a media frame is captured. A capture system MAY postpone the readout + // until a more convenient time. A capture system SHOULD have known delays + // (e.g. from hardware buffers) subtracted from the readout to make the final + // timestamp as close to the actual capture time as possible. + // + // This field is encoded as a 64-bit unsigned fixed-point number with the high + // 32 bits for the timestamp in seconds and low 32 bits for the fractional + // part. This is also known as the UQ32.32 format and is what the RTP + // specification defines as the canonical format to represent NTP timestamps. + uint64_t absolute_capture_timestamp; + + // Estimated capture clock offset is the sender’s estimate of the offset + // between its own NTP clock and the capture system’s NTP clock. The sender is + // here defined as the system that owns the NTP clock used to generate the NTP + // timestamps for the RTCP sender reports on this stream. The sender system is + // typically either the capture system or a mixer. + // + // This field is encoded as a 64-bit two’s complement signed fixed-point + // number with the high 32 bits for the seconds and low 32 bits for the + // fractional part. It’s intended to make it easy for a receiver, that knows + // how to estimate the sender system’s NTP clock, to also estimate the capture + // system’s NTP clock: + // + // Capture NTP Clock = Sender NTP Clock + Capture Clock Offset + absl::optional estimated_capture_clock_offset; +}; + +// The audio level extension is used to indicate the voice activity and the +// audio level of the payload in the RTP stream. See: +// https://tools.ietf.org/html/rfc6464#section-3. +class AudioLevel { + public: + AudioLevel(); + AudioLevel(bool voice_activity, int audio_level); + AudioLevel(const AudioLevel& other) = default; + AudioLevel& operator=(const AudioLevel& other) = default; + + // Flag indicating whether the encoder believes the audio packet contains + // voice activity. + bool voice_activity() const { return voice_activity_; } + + // Audio level in -dBov. Values range from 0 to 127, representing 0 to -127 + // dBov. 127 represents digital silence. + int level() const { return audio_level_; } + + private: + bool voice_activity_; + int audio_level_; +}; + +inline bool operator==(const AbsoluteCaptureTime& lhs, + const AbsoluteCaptureTime& rhs) { + return (lhs.absolute_capture_timestamp == rhs.absolute_capture_timestamp) && + (lhs.estimated_capture_clock_offset == + rhs.estimated_capture_clock_offset); +} + +inline bool operator!=(const AbsoluteCaptureTime& lhs, + const AbsoluteCaptureTime& rhs) { + return !(lhs == rhs); +} + +struct RTPHeaderExtension { + RTPHeaderExtension(); + RTPHeaderExtension(const RTPHeaderExtension& other); + RTPHeaderExtension& operator=(const RTPHeaderExtension& other); + + static constexpr int kAbsSendTimeFraction = 18; + + Timestamp GetAbsoluteSendTimestamp() const { + RTC_DCHECK(hasAbsoluteSendTime); + RTC_DCHECK(absoluteSendTime < (1ul << 24)); + return Timestamp::Micros((absoluteSendTime * 1000000ll) / + (1 << kAbsSendTimeFraction)); + } + + bool hasTransmissionTimeOffset; + int32_t transmissionTimeOffset; + bool hasAbsoluteSendTime; + uint32_t absoluteSendTime; + absl::optional absolute_capture_time; + bool hasTransportSequenceNumber; + uint16_t transportSequenceNumber; + absl::optional feedback_request; + + // Audio Level includes both level in dBov and voiced/unvoiced bit. See: + // https://tools.ietf.org/html/rfc6464#section-3 + absl::optional audio_level() const { return audio_level_; } + + void set_audio_level(absl::optional audio_level) { + audio_level_ = audio_level; + } + + // For Coordination of Video Orientation. See + // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ + // ts_126114v120700p.pdf + bool hasVideoRotation; + VideoRotation videoRotation; + + // TODO(ilnik): Refactor this and one above to be absl::optional() and remove + // a corresponding bool flag. + bool hasVideoContentType; + VideoContentType videoContentType; + + bool has_video_timing; + VideoSendTiming video_timing; + + VideoPlayoutDelay playout_delay; + + // For identification of a stream when ssrc is not signaled. See + // https://tools.ietf.org/html/rfc8852 + std::string stream_id; + std::string repaired_stream_id; + + // For identifying the media section used to interpret this RTP packet. See + // https://tools.ietf.org/html/rfc8843 + std::string mid; + + absl::optional color_space; + + private: + absl::optional audio_level_; +}; + +enum { kRtpCsrcSize = 15 }; // RFC 3550 page 13 + +struct RTC_EXPORT RTPHeader { + RTPHeader(); + RTPHeader(const RTPHeader& other); + RTPHeader& operator=(const RTPHeader& other); + + bool markerBit; + uint8_t payloadType; + uint16_t sequenceNumber; + uint32_t timestamp; + uint32_t ssrc; + uint8_t numCSRCs; + uint32_t arrOfCSRCs[kRtpCsrcSize]; + size_t paddingLength; + size_t headerLength; + RTPHeaderExtension extension; +}; + +// RTCP mode to use. Compound mode is described by RFC 4585 and reduced-size +// RTCP mode is described by RFC 5506. +enum class RtcpMode { kOff, kCompound, kReducedSize }; + +enum NetworkState { + kNetworkUp, + kNetworkDown, +}; + +} // namespace webrtc + +#endif // API_RTP_HEADERS_H_ diff --git a/VocieProcess/api/rtp_packet_info.cc b/VocieProcess/api/rtp_packet_info.cc new file mode 100644 index 0000000..90cd275 --- /dev/null +++ b/VocieProcess/api/rtp_packet_info.cc @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/rtp_packet_info.h" + +#include + +#include +#include +#include +#include + +#include "api/rtp_headers.h" +#include "api/units/timestamp.h" + +namespace webrtc { + +RtpPacketInfo::RtpPacketInfo() + : ssrc_(0), rtp_timestamp_(0), receive_time_(Timestamp::MinusInfinity()) {} + +RtpPacketInfo::RtpPacketInfo(uint32_t ssrc, + std::vector csrcs, + uint32_t rtp_timestamp, + Timestamp receive_time) + : ssrc_(ssrc), + csrcs_(std::move(csrcs)), + rtp_timestamp_(rtp_timestamp), + receive_time_(receive_time) {} + +RtpPacketInfo::RtpPacketInfo(const RTPHeader& rtp_header, + Timestamp receive_time) + : ssrc_(rtp_header.ssrc), + rtp_timestamp_(rtp_header.timestamp), + receive_time_(receive_time) { + const auto& extension = rtp_header.extension; + const auto csrcs_count = std::min(rtp_header.numCSRCs, kRtpCsrcSize); + + csrcs_.assign(&rtp_header.arrOfCSRCs[0], &rtp_header.arrOfCSRCs[csrcs_count]); + + if (extension.audio_level()) { + audio_level_ = extension.audio_level()->level(); + } + + absolute_capture_time_ = extension.absolute_capture_time; +} + +bool operator==(const RtpPacketInfo& lhs, const RtpPacketInfo& rhs) { + return (lhs.ssrc() == rhs.ssrc()) && (lhs.csrcs() == rhs.csrcs()) && + (lhs.rtp_timestamp() == rhs.rtp_timestamp()) && + (lhs.receive_time() == rhs.receive_time()) && + (lhs.audio_level() == rhs.audio_level()) && + (lhs.absolute_capture_time() == rhs.absolute_capture_time()) && + (lhs.local_capture_clock_offset() == rhs.local_capture_clock_offset()); +} + +} // namespace webrtc diff --git a/VocieProcess/api/rtp_packet_info.h b/VocieProcess/api/rtp_packet_info.h new file mode 100644 index 0000000..8df12a3 --- /dev/null +++ b/VocieProcess/api/rtp_packet_info.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_RTP_PACKET_INFO_H_ +#define API_RTP_PACKET_INFO_H_ + +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/rtp_headers.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// +// Structure to hold information about a received `RtpPacket`. It is primarily +// used to carry per-packet information from when a packet is received until +// the information is passed to `SourceTracker`. +// +class RTC_EXPORT RtpPacketInfo { + public: + RtpPacketInfo(); + + RtpPacketInfo(uint32_t ssrc, + std::vector csrcs, + uint32_t rtp_timestamp, + Timestamp receive_time); + + RtpPacketInfo(const RTPHeader& rtp_header, Timestamp receive_time); + + RtpPacketInfo(const RtpPacketInfo& other) = default; + RtpPacketInfo(RtpPacketInfo&& other) = default; + RtpPacketInfo& operator=(const RtpPacketInfo& other) = default; + RtpPacketInfo& operator=(RtpPacketInfo&& other) = default; + + uint32_t ssrc() const { return ssrc_; } + void set_ssrc(uint32_t value) { ssrc_ = value; } + + const std::vector& csrcs() const { return csrcs_; } + void set_csrcs(std::vector value) { csrcs_ = std::move(value); } + + uint32_t rtp_timestamp() const { return rtp_timestamp_; } + void set_rtp_timestamp(uint32_t value) { rtp_timestamp_ = value; } + + Timestamp receive_time() const { return receive_time_; } + void set_receive_time(Timestamp value) { receive_time_ = value; } + + absl::optional audio_level() const { return audio_level_; } + RtpPacketInfo& set_audio_level(absl::optional value) { + audio_level_ = value; + return *this; + } + + const absl::optional& absolute_capture_time() const { + return absolute_capture_time_; + } + RtpPacketInfo& set_absolute_capture_time( + const absl::optional& value) { + absolute_capture_time_ = value; + return *this; + } + + const absl::optional& local_capture_clock_offset() const { + return local_capture_clock_offset_; + } + RtpPacketInfo& set_local_capture_clock_offset( + absl::optional value) { + local_capture_clock_offset_ = value; + return *this; + } + + private: + // Fields from the RTP header: + // https://tools.ietf.org/html/rfc3550#section-5.1 + uint32_t ssrc_; + std::vector csrcs_; + uint32_t rtp_timestamp_; + + // Local `webrtc::Clock`-based timestamp of when the packet was received. + Timestamp receive_time_; + + // Fields from the Audio Level header extension: + // https://tools.ietf.org/html/rfc6464#section-3 + absl::optional audio_level_; + + // Fields from the Absolute Capture Time header extension: + // http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time + absl::optional absolute_capture_time_; + + // Clock offset between the local clock and the capturer's clock. + // Do not confuse with `AbsoluteCaptureTime::estimated_capture_clock_offset` + // which instead represents the clock offset between a remote sender and the + // capturer. The following holds: + // Capture's NTP Clock = Local NTP Clock + Local-Capture Clock Offset + absl::optional local_capture_clock_offset_; +}; + +bool operator==(const RtpPacketInfo& lhs, const RtpPacketInfo& rhs); + +inline bool operator!=(const RtpPacketInfo& lhs, const RtpPacketInfo& rhs) { + return !(lhs == rhs); +} + +} // namespace webrtc + +#endif // API_RTP_PACKET_INFO_H_ diff --git a/VocieProcess/api/rtp_packet_infos.h b/VocieProcess/api/rtp_packet_infos.h new file mode 100644 index 0000000..dfb4e1e --- /dev/null +++ b/VocieProcess/api/rtp_packet_infos.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_RTP_PACKET_INFOS_H_ +#define API_RTP_PACKET_INFOS_H_ + +#include +#include + +#include "api/make_ref_counted.h" +#include "api/ref_counted_base.h" +#include "api/rtp_packet_info.h" +#include "api/scoped_refptr.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// Semi-immutable structure to hold information about packets used to assemble +// an audio or video frame. Uses internal reference counting to make it very +// cheap to copy. +// +// We should ideally just use `std::vector` and have it +// `std::move()`-ed as the per-packet information is transferred from one object +// to another. But moving the info, instead of copying it, is not easily done +// for the current video code. +class RTC_EXPORT RtpPacketInfos { + public: + using vector_type = std::vector; + + using value_type = vector_type::value_type; + using size_type = vector_type::size_type; + using difference_type = vector_type::difference_type; + using const_reference = vector_type::const_reference; + using const_pointer = vector_type::const_pointer; + using const_iterator = vector_type::const_iterator; + using const_reverse_iterator = vector_type::const_reverse_iterator; + + using reference = const_reference; + using pointer = const_pointer; + using iterator = const_iterator; + using reverse_iterator = const_reverse_iterator; + + RtpPacketInfos() {} + explicit RtpPacketInfos(const vector_type& entries) + : data_(Data::Create(entries)) {} + + explicit RtpPacketInfos(vector_type&& entries) + : data_(Data::Create(std::move(entries))) {} + + RtpPacketInfos(const RtpPacketInfos& other) = default; + RtpPacketInfos(RtpPacketInfos&& other) = default; + RtpPacketInfos& operator=(const RtpPacketInfos& other) = default; + RtpPacketInfos& operator=(RtpPacketInfos&& other) = default; + + const_reference operator[](size_type pos) const { return entries()[pos]; } + + const_reference at(size_type pos) const { return entries().at(pos); } + const_reference front() const { return entries().front(); } + const_reference back() const { return entries().back(); } + + const_iterator begin() const { return entries().begin(); } + const_iterator end() const { return entries().end(); } + const_reverse_iterator rbegin() const { return entries().rbegin(); } + const_reverse_iterator rend() const { return entries().rend(); } + + const_iterator cbegin() const { return entries().cbegin(); } + const_iterator cend() const { return entries().cend(); } + const_reverse_iterator crbegin() const { return entries().crbegin(); } + const_reverse_iterator crend() const { return entries().crend(); } + + bool empty() const { return entries().empty(); } + size_type size() const { return entries().size(); } + + private: + class Data final : public rtc::RefCountedNonVirtual { + public: + static rtc::scoped_refptr Create(const vector_type& entries) { + // Performance optimization for the empty case. + if (entries.empty()) { + return nullptr; + } + + return rtc::make_ref_counted(entries); + } + + static rtc::scoped_refptr Create(vector_type&& entries) { + // Performance optimization for the empty case. + if (entries.empty()) { + return nullptr; + } + + return rtc::make_ref_counted(std::move(entries)); + } + + const vector_type& entries() const { return entries_; } + + explicit Data(const vector_type& entries) : entries_(entries) {} + explicit Data(vector_type&& entries) : entries_(std::move(entries)) {} + ~Data() = default; + + private: + const vector_type entries_; + }; + + static const vector_type& empty_entries() { + static const vector_type& value = *new vector_type(); + return value; + } + + const vector_type& entries() const { + if (data_ != nullptr) { + return data_->entries(); + } else { + return empty_entries(); + } + } + + rtc::scoped_refptr data_; +}; + +} // namespace webrtc + +#endif // API_RTP_PACKET_INFOS_H_ diff --git a/VocieProcess/api/sequence_checker.h b/VocieProcess/api/sequence_checker.h new file mode 100644 index 0000000..2c352f7 --- /dev/null +++ b/VocieProcess/api/sequence_checker.h @@ -0,0 +1,141 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_SEQUENCE_CHECKER_H_ +#define API_SEQUENCE_CHECKER_H_ + +#include "api/task_queue/task_queue_base.h" +#include "rtc_base/checks.h" +#include "rtc_base/synchronization/sequence_checker_internal.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +// SequenceChecker is a helper class used to help verify that some methods +// of a class are called on the same task queue or thread. A +// SequenceChecker is bound to a a task queue if the object is +// created on a task queue, or a thread otherwise. +// +// +// Example: +// class MyClass { +// public: +// void Foo() { +// RTC_DCHECK_RUN_ON(&sequence_checker_); +// ... (do stuff) ... +// } +// +// private: +// SequenceChecker sequence_checker_; +// } +// +// In Release mode, IsCurrent will always return true. +class RTC_LOCKABLE SequenceChecker +#if RTC_DCHECK_IS_ON + : public webrtc_sequence_checker_internal::SequenceCheckerImpl { + using Impl = webrtc_sequence_checker_internal::SequenceCheckerImpl; +#else + : public webrtc_sequence_checker_internal::SequenceCheckerDoNothing { + using Impl = webrtc_sequence_checker_internal::SequenceCheckerDoNothing; +#endif + public: + enum InitialState : bool { kDetached = false, kAttached = true }; + + // TODO(tommi): We could maybe join these two ctors and have fewer factory + // functions. At the moment they're separate to minimize code changes when + // we added the second ctor as well as avoiding to have unnecessary code at + // the SequenceChecker which much only run for the SequenceCheckerImpl + // implementation. + // In theory we could have something like: + // + // SequenceChecker(InitialState initial_state = kAttached, + // TaskQueueBase* attached_queue = TaskQueueBase::Current()); + // + // But the problem with that is having the call to `Current()` exist for + // `SequenceCheckerDoNothing`. + explicit SequenceChecker(InitialState initial_state = kAttached) + : Impl(initial_state) {} + explicit SequenceChecker(TaskQueueBase* attached_queue) + : Impl(attached_queue) {} + + // Returns true if sequence checker is attached to the current sequence. + bool IsCurrent() const { return Impl::IsCurrent(); } + // Detaches checker from sequence to which it is attached. Next attempt + // to do a check with this checker will result in attaching this checker + // to the sequence on which check was performed. + void Detach() { Impl::Detach(); } +}; + +} // namespace webrtc + +// RTC_RUN_ON/RTC_GUARDED_BY/RTC_DCHECK_RUN_ON macros allows to annotate +// variables are accessed from same thread/task queue. +// Using tools designed to check mutexes, it checks at compile time everywhere +// variable is access, there is a run-time dcheck thread/task queue is correct. +// +// class SequenceCheckerExample { +// public: +// int CalledFromPacer() RTC_RUN_ON(pacer_sequence_checker_) { +// return var2_; +// } +// +// void CallMeFromPacer() { +// RTC_DCHECK_RUN_ON(&pacer_sequence_checker_) +// << "Should be called from pacer"; +// CalledFromPacer(); +// } +// +// private: +// int pacer_var_ RTC_GUARDED_BY(pacer_sequence_checker_); +// SequenceChecker pacer_sequence_checker_; +// }; +// +// class TaskQueueExample { +// public: +// class Encoder { +// public: +// rtc::TaskQueueBase& Queue() { return encoder_queue_; } +// void Encode() { +// RTC_DCHECK_RUN_ON(&encoder_queue_); +// DoSomething(var_); +// } +// +// private: +// rtc::TaskQueueBase& encoder_queue_; +// Frame var_ RTC_GUARDED_BY(encoder_queue_); +// }; +// +// void Encode() { +// // Will fail at runtime when DCHECK is enabled: +// // encoder_->Encode(); +// // Will work: +// rtc::scoped_refptr encoder = encoder_; +// encoder_->Queue().PostTask([encoder] { encoder->Encode(); }); +// } +// +// private: +// rtc::scoped_refptr encoder_; +// } + +// Document if a function expected to be called from same thread/task queue. +#define RTC_RUN_ON(x) \ + RTC_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(x)) + +// Checks current code is running on the desired sequence. +// +// First statement validates it is running on the sequence `x`. +// Second statement annotates for the thread safety analyzer the check was done. +// Such annotation has to be attached to a function, and that function has to be +// called. Thus current implementation creates a noop lambda and calls it. +#define RTC_DCHECK_RUN_ON(x) \ + RTC_DCHECK((x)->IsCurrent()) \ + << webrtc::webrtc_sequence_checker_internal::ExpectationToString(x); \ + []() RTC_ASSERT_EXCLUSIVE_LOCK(x) {}() + +#endif // API_SEQUENCE_CHECKER_H_ diff --git a/VocieProcess/api/video/color_space.cc b/VocieProcess/api/video/color_space.cc new file mode 100644 index 0000000..05c0135 --- /dev/null +++ b/VocieProcess/api/video/color_space.cc @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/color_space.h" + +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/video/hdr_metadata.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +namespace { +// Try to convert `enum_value` into the enum class T. `enum_bitmask` is created +// by the funciton below. Returns true if conversion was successful, false +// otherwise. +template +bool SetFromUint8(uint8_t enum_value, uint64_t enum_bitmask, T* out) { + if ((enum_value < 64) && ((enum_bitmask >> enum_value) & 1)) { + *out = static_cast(enum_value); + return true; + } + return false; +} + +// This function serves as an assert for the constexpr function below. It's on +// purpose not declared as constexpr so that it causes a build problem if enum +// values of 64 or above are used. The bitmask and the code generating it would +// have to be extended if the standard is updated to include enum values >= 64. +int EnumMustBeLessThan64() { + return -1; +} + +template +constexpr int MakeMask(const int index, const int length, T (&values)[N]) { + return length > 1 + ? (MakeMask(index, 1, values) + + MakeMask(index + 1, length - 1, values)) + : (static_cast(values[index]) < 64 + ? (uint64_t{1} << static_cast(values[index])) + : EnumMustBeLessThan64()); +} + +// Create a bitmask where each bit corresponds to one potential enum value. +// `values` should be an array listing all possible enum values. The bit is set +// to one if the corresponding enum exists. Only works for enums with values +// less than 64. +template +constexpr uint64_t CreateEnumBitmask(T (&values)[N]) { + return MakeMask(0, N, values); +} + +bool SetChromaSitingFromUint8(uint8_t enum_value, + ColorSpace::ChromaSiting* chroma_siting) { + constexpr ColorSpace::ChromaSiting kChromaSitings[] = { + ColorSpace::ChromaSiting::kUnspecified, + ColorSpace::ChromaSiting::kCollocated, ColorSpace::ChromaSiting::kHalf}; + constexpr uint64_t enum_bitmask = CreateEnumBitmask(kChromaSitings); + + return SetFromUint8(enum_value, enum_bitmask, chroma_siting); +} + +} // namespace + +ColorSpace::ColorSpace() = default; +ColorSpace::ColorSpace(const ColorSpace& other) = default; +ColorSpace::ColorSpace(ColorSpace&& other) = default; +ColorSpace& ColorSpace::operator=(const ColorSpace& other) = default; + +ColorSpace::ColorSpace(PrimaryID primaries, + TransferID transfer, + MatrixID matrix, + RangeID range) + : ColorSpace(primaries, + transfer, + matrix, + range, + ChromaSiting::kUnspecified, + ChromaSiting::kUnspecified, + nullptr) {} + +ColorSpace::ColorSpace(PrimaryID primaries, + TransferID transfer, + MatrixID matrix, + RangeID range, + ChromaSiting chroma_siting_horz, + ChromaSiting chroma_siting_vert, + const HdrMetadata* hdr_metadata) + : primaries_(primaries), + transfer_(transfer), + matrix_(matrix), + range_(range), + chroma_siting_horizontal_(chroma_siting_horz), + chroma_siting_vertical_(chroma_siting_vert), + hdr_metadata_(hdr_metadata ? absl::make_optional(*hdr_metadata) + : absl::nullopt) {} + +ColorSpace::PrimaryID ColorSpace::primaries() const { + return primaries_; +} + +ColorSpace::TransferID ColorSpace::transfer() const { + return transfer_; +} + +ColorSpace::MatrixID ColorSpace::matrix() const { + return matrix_; +} + +ColorSpace::RangeID ColorSpace::range() const { + return range_; +} + +ColorSpace::ChromaSiting ColorSpace::chroma_siting_horizontal() const { + return chroma_siting_horizontal_; +} + +ColorSpace::ChromaSiting ColorSpace::chroma_siting_vertical() const { + return chroma_siting_vertical_; +} + +const HdrMetadata* ColorSpace::hdr_metadata() const { + return hdr_metadata_ ? &*hdr_metadata_ : nullptr; +} + +#define PRINT_ENUM_CASE(TYPE, NAME) \ + case TYPE::NAME: \ + ss << #NAME; \ + break; + +std::string ColorSpace::AsString() const { + char buf[1024]; + rtc::SimpleStringBuilder ss(buf); + ss << "{primaries:"; + switch (primaries_) { + PRINT_ENUM_CASE(PrimaryID, kBT709) + PRINT_ENUM_CASE(PrimaryID, kUnspecified) + PRINT_ENUM_CASE(PrimaryID, kBT470M) + PRINT_ENUM_CASE(PrimaryID, kBT470BG) + PRINT_ENUM_CASE(PrimaryID, kSMPTE170M) + PRINT_ENUM_CASE(PrimaryID, kSMPTE240M) + PRINT_ENUM_CASE(PrimaryID, kFILM) + PRINT_ENUM_CASE(PrimaryID, kBT2020) + PRINT_ENUM_CASE(PrimaryID, kSMPTEST428) + PRINT_ENUM_CASE(PrimaryID, kSMPTEST431) + PRINT_ENUM_CASE(PrimaryID, kSMPTEST432) + PRINT_ENUM_CASE(PrimaryID, kJEDECP22) + } + ss << ", transfer:"; + switch (transfer_) { + PRINT_ENUM_CASE(TransferID, kBT709) + PRINT_ENUM_CASE(TransferID, kUnspecified) + PRINT_ENUM_CASE(TransferID, kGAMMA22) + PRINT_ENUM_CASE(TransferID, kGAMMA28) + PRINT_ENUM_CASE(TransferID, kSMPTE170M) + PRINT_ENUM_CASE(TransferID, kSMPTE240M) + PRINT_ENUM_CASE(TransferID, kLINEAR) + PRINT_ENUM_CASE(TransferID, kLOG) + PRINT_ENUM_CASE(TransferID, kLOG_SQRT) + PRINT_ENUM_CASE(TransferID, kIEC61966_2_4) + PRINT_ENUM_CASE(TransferID, kBT1361_ECG) + PRINT_ENUM_CASE(TransferID, kIEC61966_2_1) + PRINT_ENUM_CASE(TransferID, kBT2020_10) + PRINT_ENUM_CASE(TransferID, kBT2020_12) + PRINT_ENUM_CASE(TransferID, kSMPTEST2084) + PRINT_ENUM_CASE(TransferID, kSMPTEST428) + PRINT_ENUM_CASE(TransferID, kARIB_STD_B67) + } + ss << ", matrix:"; + switch (matrix_) { + PRINT_ENUM_CASE(MatrixID, kRGB) + PRINT_ENUM_CASE(MatrixID, kBT709) + PRINT_ENUM_CASE(MatrixID, kUnspecified) + PRINT_ENUM_CASE(MatrixID, kFCC) + PRINT_ENUM_CASE(MatrixID, kBT470BG) + PRINT_ENUM_CASE(MatrixID, kSMPTE170M) + PRINT_ENUM_CASE(MatrixID, kSMPTE240M) + PRINT_ENUM_CASE(MatrixID, kYCOCG) + PRINT_ENUM_CASE(MatrixID, kBT2020_NCL) + PRINT_ENUM_CASE(MatrixID, kBT2020_CL) + PRINT_ENUM_CASE(MatrixID, kSMPTE2085) + PRINT_ENUM_CASE(MatrixID, kCDNCLS) + PRINT_ENUM_CASE(MatrixID, kCDCLS) + PRINT_ENUM_CASE(MatrixID, kBT2100_ICTCP) + } + + ss << ", range:"; + switch (range_) { + PRINT_ENUM_CASE(RangeID, kInvalid) + PRINT_ENUM_CASE(RangeID, kLimited) + PRINT_ENUM_CASE(RangeID, kFull) + PRINT_ENUM_CASE(RangeID, kDerived) + } + ss << "}"; + return ss.str(); +} + +#undef PRINT_ENUM_CASE + +bool ColorSpace::set_primaries_from_uint8(uint8_t enum_value) { + constexpr PrimaryID kPrimaryIds[] = { + PrimaryID::kBT709, PrimaryID::kUnspecified, PrimaryID::kBT470M, + PrimaryID::kBT470BG, PrimaryID::kSMPTE170M, PrimaryID::kSMPTE240M, + PrimaryID::kFILM, PrimaryID::kBT2020, PrimaryID::kSMPTEST428, + PrimaryID::kSMPTEST431, PrimaryID::kSMPTEST432, PrimaryID::kJEDECP22}; + constexpr uint64_t enum_bitmask = CreateEnumBitmask(kPrimaryIds); + + return SetFromUint8(enum_value, enum_bitmask, &primaries_); +} + +bool ColorSpace::set_transfer_from_uint8(uint8_t enum_value) { + constexpr TransferID kTransferIds[] = { + TransferID::kBT709, TransferID::kUnspecified, + TransferID::kGAMMA22, TransferID::kGAMMA28, + TransferID::kSMPTE170M, TransferID::kSMPTE240M, + TransferID::kLINEAR, TransferID::kLOG, + TransferID::kLOG_SQRT, TransferID::kIEC61966_2_4, + TransferID::kBT1361_ECG, TransferID::kIEC61966_2_1, + TransferID::kBT2020_10, TransferID::kBT2020_12, + TransferID::kSMPTEST2084, TransferID::kSMPTEST428, + TransferID::kARIB_STD_B67}; + constexpr uint64_t enum_bitmask = CreateEnumBitmask(kTransferIds); + + return SetFromUint8(enum_value, enum_bitmask, &transfer_); +} + +bool ColorSpace::set_matrix_from_uint8(uint8_t enum_value) { + constexpr MatrixID kMatrixIds[] = { + MatrixID::kRGB, MatrixID::kBT709, MatrixID::kUnspecified, + MatrixID::kFCC, MatrixID::kBT470BG, MatrixID::kSMPTE170M, + MatrixID::kSMPTE240M, MatrixID::kYCOCG, MatrixID::kBT2020_NCL, + MatrixID::kBT2020_CL, MatrixID::kSMPTE2085, MatrixID::kCDNCLS, + MatrixID::kCDCLS, MatrixID::kBT2100_ICTCP}; + constexpr uint64_t enum_bitmask = CreateEnumBitmask(kMatrixIds); + + return SetFromUint8(enum_value, enum_bitmask, &matrix_); +} + +bool ColorSpace::set_range_from_uint8(uint8_t enum_value) { + constexpr RangeID kRangeIds[] = {RangeID::kInvalid, RangeID::kLimited, + RangeID::kFull, RangeID::kDerived}; + constexpr uint64_t enum_bitmask = CreateEnumBitmask(kRangeIds); + + return SetFromUint8(enum_value, enum_bitmask, &range_); +} + +bool ColorSpace::set_chroma_siting_horizontal_from_uint8(uint8_t enum_value) { + return SetChromaSitingFromUint8(enum_value, &chroma_siting_horizontal_); +} + +bool ColorSpace::set_chroma_siting_vertical_from_uint8(uint8_t enum_value) { + return SetChromaSitingFromUint8(enum_value, &chroma_siting_vertical_); +} + +void ColorSpace::set_hdr_metadata(const HdrMetadata* hdr_metadata) { + hdr_metadata_ = + hdr_metadata ? absl::make_optional(*hdr_metadata) : absl::nullopt; +} + +} // namespace webrtc diff --git a/VocieProcess/api/video/color_space.h b/VocieProcess/api/video/color_space.h new file mode 100644 index 0000000..31963a1 --- /dev/null +++ b/VocieProcess/api/video/color_space.h @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_COLOR_SPACE_H_ +#define API_VIDEO_COLOR_SPACE_H_ + +#include + +#include + +#include "absl/types/optional.h" +#include "api/video/hdr_metadata.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// This class represents color information as specified in T-REC H.273, +// available from https://www.itu.int/rec/T-REC-H.273. +// +// WebRTC's supported codecs: +// - VP9 supports color profiles, see VP9 Bitstream & Decoding Process +// Specification Version 0.6 Section 7.2.2 "Color config semantics" available +// from https://www.webmproject.org. +// - VP8 only supports BT.601, see +// https://tools.ietf.org/html/rfc6386#section-9.2 +// - H264 uses the exact same representation as T-REC H.273. See T-REC-H.264 +// E.2.1, "VUI parameters semantics", available from +// https://www.itu.int/rec/T-REC-H.264. + +class RTC_EXPORT ColorSpace { + public: + enum class PrimaryID : uint8_t { + // The indices are equal to the values specified in T-REC H.273 Table 2. + kBT709 = 1, + kUnspecified = 2, + kBT470M = 4, + kBT470BG = 5, + kSMPTE170M = 6, // Identical to BT601 + kSMPTE240M = 7, + kFILM = 8, + kBT2020 = 9, + kSMPTEST428 = 10, + kSMPTEST431 = 11, + kSMPTEST432 = 12, + kJEDECP22 = 22, // Identical to EBU3213-E + // When adding/removing entries here, please make sure to do the + // corresponding change to kPrimaryIds. + }; + + enum class TransferID : uint8_t { + // The indices are equal to the values specified in T-REC H.273 Table 3. + kBT709 = 1, + kUnspecified = 2, + kGAMMA22 = 4, + kGAMMA28 = 5, + kSMPTE170M = 6, + kSMPTE240M = 7, + kLINEAR = 8, + kLOG = 9, + kLOG_SQRT = 10, + kIEC61966_2_4 = 11, + kBT1361_ECG = 12, + kIEC61966_2_1 = 13, + kBT2020_10 = 14, + kBT2020_12 = 15, + kSMPTEST2084 = 16, + kSMPTEST428 = 17, + kARIB_STD_B67 = 18, + // When adding/removing entries here, please make sure to do the + // corresponding change to kTransferIds. + }; + + enum class MatrixID : uint8_t { + // The indices are equal to the values specified in T-REC H.273 Table 4. + kRGB = 0, + kBT709 = 1, + kUnspecified = 2, + kFCC = 4, + kBT470BG = 5, + kSMPTE170M = 6, + kSMPTE240M = 7, + kYCOCG = 8, + kBT2020_NCL = 9, + kBT2020_CL = 10, + kSMPTE2085 = 11, + kCDNCLS = 12, + kCDCLS = 13, + kBT2100_ICTCP = 14, + // When adding/removing entries here, please make sure to do the + // corresponding change to kMatrixIds. + }; + + enum class RangeID { + // The indices are equal to the values specified at + // https://www.webmproject.org/docs/container/#colour for the element Range. + kInvalid = 0, + // Limited Rec. 709 color range with RGB values ranging from 16 to 235. + kLimited = 1, + // Full RGB color range with RGB values from 0 to 255. + kFull = 2, + // Range is defined by MatrixCoefficients/TransferCharacteristics. + kDerived = 3, + // When adding/removing entries here, please make sure to do the + // corresponding change to kRangeIds. + }; + + enum class ChromaSiting { + // Chroma siting specifies how chroma is subsampled relative to the luma + // samples in a YUV video frame. + // The indices are equal to the values specified at + // https://www.webmproject.org/docs/container/#colour for the element + // ChromaSitingVert and ChromaSitingHorz. + kUnspecified = 0, + kCollocated = 1, + kHalf = 2, + // When adding/removing entries here, please make sure to do the + // corresponding change to kChromaSitings. + }; + + ColorSpace(); + ColorSpace(const ColorSpace& other); + ColorSpace(ColorSpace&& other); + ColorSpace& operator=(const ColorSpace& other); + ColorSpace(PrimaryID primaries, + TransferID transfer, + MatrixID matrix, + RangeID range); + ColorSpace(PrimaryID primaries, + TransferID transfer, + MatrixID matrix, + RangeID range, + ChromaSiting chroma_siting_horizontal, + ChromaSiting chroma_siting_vertical, + const HdrMetadata* hdr_metadata); + friend bool operator==(const ColorSpace& lhs, const ColorSpace& rhs) { + return lhs.primaries_ == rhs.primaries_ && lhs.transfer_ == rhs.transfer_ && + lhs.matrix_ == rhs.matrix_ && lhs.range_ == rhs.range_ && + lhs.chroma_siting_horizontal_ == rhs.chroma_siting_horizontal_ && + lhs.chroma_siting_vertical_ == rhs.chroma_siting_vertical_ && + lhs.hdr_metadata_ == rhs.hdr_metadata_; + } + friend bool operator!=(const ColorSpace& lhs, const ColorSpace& rhs) { + return !(lhs == rhs); + } + + PrimaryID primaries() const; + TransferID transfer() const; + MatrixID matrix() const; + RangeID range() const; + ChromaSiting chroma_siting_horizontal() const; + ChromaSiting chroma_siting_vertical() const; + const HdrMetadata* hdr_metadata() const; + std::string AsString() const; + + bool set_primaries_from_uint8(uint8_t enum_value); + bool set_transfer_from_uint8(uint8_t enum_value); + bool set_matrix_from_uint8(uint8_t enum_value); + bool set_range_from_uint8(uint8_t enum_value); + bool set_chroma_siting_horizontal_from_uint8(uint8_t enum_value); + bool set_chroma_siting_vertical_from_uint8(uint8_t enum_value); + void set_hdr_metadata(const HdrMetadata* hdr_metadata); + + private: + PrimaryID primaries_ = PrimaryID::kUnspecified; + TransferID transfer_ = TransferID::kUnspecified; + MatrixID matrix_ = MatrixID::kUnspecified; + RangeID range_ = RangeID::kInvalid; + ChromaSiting chroma_siting_horizontal_ = ChromaSiting::kUnspecified; + ChromaSiting chroma_siting_vertical_ = ChromaSiting::kUnspecified; + absl::optional hdr_metadata_; +}; + +} // namespace webrtc +#endif // API_VIDEO_COLOR_SPACE_H_ diff --git a/VocieProcess/api/video/hdr_metadata.cc b/VocieProcess/api/video/hdr_metadata.cc new file mode 100644 index 0000000..e2a669c --- /dev/null +++ b/VocieProcess/api/video/hdr_metadata.cc @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/hdr_metadata.h" + +namespace webrtc { + +HdrMasteringMetadata::Chromaticity::Chromaticity() = default; + +HdrMasteringMetadata::HdrMasteringMetadata() = default; + +HdrMetadata::HdrMetadata() = default; + +} // namespace webrtc diff --git a/VocieProcess/api/video/hdr_metadata.h b/VocieProcess/api/video/hdr_metadata.h new file mode 100644 index 0000000..e9001a2 --- /dev/null +++ b/VocieProcess/api/video/hdr_metadata.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_HDR_METADATA_H_ +#define API_VIDEO_HDR_METADATA_H_ + +namespace webrtc { + +// SMPTE ST 2086 mastering metadata, +// see https://ieeexplore.ieee.org/document/8353899. +struct HdrMasteringMetadata { + struct Chromaticity { + Chromaticity(); + + bool operator==(const Chromaticity& rhs) const { + return x == rhs.x && y == rhs.y; + } + + bool Validate() const { + return x >= 0.0 && x <= 1.0 && y >= 0.0 && y <= 1.0; + } + + // xy chromaticity coordinates must be calculated as specified in ISO + // 11664-3:2012 Section 7, and must be specified with four decimal places. + // The x coordinate should be in the range [0.0001, 0.7400] and the y + // coordinate should be in the range [0.0001, 0.8400]. Valid range [0.0000, + // 1.0000]. + float x = 0.0f; + float y = 0.0f; + }; + + HdrMasteringMetadata(); + + bool operator==(const HdrMasteringMetadata& rhs) const { + return ((primary_r == rhs.primary_r) && (primary_g == rhs.primary_g) && + (primary_b == rhs.primary_b) && (white_point == rhs.white_point) && + (luminance_max == rhs.luminance_max) && + (luminance_min == rhs.luminance_min)); + } + + bool Validate() const { + return luminance_max >= 0.0 && luminance_max <= 20000.0 && + luminance_min >= 0.0 && luminance_min <= 5.0 && + primary_r.Validate() && primary_g.Validate() && + primary_b.Validate() && white_point.Validate(); + } + + // The nominal primaries of the mastering display. + Chromaticity primary_r; + Chromaticity primary_g; + Chromaticity primary_b; + + // The nominal chromaticity of the white point of the mastering display. + Chromaticity white_point; + + // The nominal maximum display luminance of the mastering display. Specified + // in the unit candela/m2. The value should be in the range [5, 10000] with + // zero decimal places. Valid range [0, 20000]. + float luminance_max = 0.0f; + + // The nominal minimum display luminance of the mastering display. Specified + // in the unit candela/m2. The value should be in the range [0.0001, 5.0000] + // with four decimal places. Valid range [0.0000, 5.0000]. + float luminance_min = 0.0f; +}; + +// High dynamic range (HDR) metadata common for HDR10 and WebM/VP9-based HDR +// formats. This struct replicates the HDRMetadata struct defined in +// https://cs.chromium.org/chromium/src/media/base/hdr_metadata.h +struct HdrMetadata { + HdrMetadata(); + + bool operator==(const HdrMetadata& rhs) const { + return ( + (max_content_light_level == rhs.max_content_light_level) && + (max_frame_average_light_level == rhs.max_frame_average_light_level) && + (mastering_metadata == rhs.mastering_metadata)); + } + + bool Validate() const { + return max_content_light_level >= 0 && max_content_light_level <= 20000 && + max_frame_average_light_level >= 0 && + max_frame_average_light_level <= 20000 && + mastering_metadata.Validate(); + } + + HdrMasteringMetadata mastering_metadata; + // Max content light level (CLL), i.e. maximum brightness level present in the + // stream, in nits. 1 nit = 1 candela/m2. Valid range [0, 20000]. + int max_content_light_level = 0; + // Max frame-average light level (FALL), i.e. maximum average brightness of + // the brightest frame in the stream, in nits. Valid range [0, 20000]. + int max_frame_average_light_level = 0; +}; + +} // namespace webrtc + +#endif // API_VIDEO_HDR_METADATA_H_ diff --git a/VocieProcess/api/video/video_content_type.cc b/VocieProcess/api/video/video_content_type.cc new file mode 100644 index 0000000..75beb5c --- /dev/null +++ b/VocieProcess/api/video/video_content_type.cc @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/video_content_type.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace videocontenttypehelpers { + +namespace { +static constexpr uint8_t kScreenshareBitsSize = 1; +static constexpr uint8_t kScreenshareBitsMask = + (1u << kScreenshareBitsSize) - 1; +} // namespace + +bool IsScreenshare(const VideoContentType& content_type) { + // Ensure no bits apart from the screenshare bit is set. + // This CHECK is a temporary measure to detect code that introduces + // values according to old versions. + RTC_CHECK((static_cast(content_type) & !kScreenshareBitsMask) == 0); + return (static_cast(content_type) & kScreenshareBitsMask) > 0; +} + +bool IsValidContentType(uint8_t value) { + // Only the screenshare bit is allowed. + // However, due to previous usage of the next 5 bits, we allow + // the lower 6 bits to be set. + return value < (1 << 6); +} + +const char* ToString(const VideoContentType& content_type) { + return IsScreenshare(content_type) ? "screen" : "realtime"; +} +} // namespace videocontenttypehelpers +} // namespace webrtc diff --git a/VocieProcess/api/video/video_content_type.h b/VocieProcess/api/video/video_content_type.h new file mode 100644 index 0000000..b574201 --- /dev/null +++ b/VocieProcess/api/video/video_content_type.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_VIDEO_CONTENT_TYPE_H_ +#define API_VIDEO_VIDEO_CONTENT_TYPE_H_ + +#include + +namespace webrtc { + +// VideoContentType stored as a single byte, which is sent over the network +// in the rtp-hdrext/video-content-type extension. +// Only the lowest bit is used, per the enum. +enum class VideoContentType : uint8_t { + UNSPECIFIED = 0, + SCREENSHARE = 1, +}; + +namespace videocontenttypehelpers { +bool IsScreenshare(const VideoContentType& content_type); + +bool IsValidContentType(uint8_t value); + +const char* ToString(const VideoContentType& content_type); +} // namespace videocontenttypehelpers + +} // namespace webrtc + +#endif // API_VIDEO_VIDEO_CONTENT_TYPE_H_ diff --git a/VocieProcess/api/video/video_rotation.h b/VocieProcess/api/video/video_rotation.h new file mode 100644 index 0000000..6a29588 --- /dev/null +++ b/VocieProcess/api/video/video_rotation.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_VIDEO_ROTATION_H_ +#define API_VIDEO_VIDEO_ROTATION_H_ + +namespace webrtc { + +// enum for clockwise rotation. +enum VideoRotation { + kVideoRotation_0 = 0, + kVideoRotation_90 = 90, + kVideoRotation_180 = 180, + kVideoRotation_270 = 270 +}; + +} // namespace webrtc + +#endif // API_VIDEO_VIDEO_ROTATION_H_ diff --git a/VocieProcess/api/video/video_timing.cc b/VocieProcess/api/video/video_timing.cc new file mode 100644 index 0000000..8e9e9f0 --- /dev/null +++ b/VocieProcess/api/video/video_timing.cc @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/video_timing.h" + +#include +#include +#include + +#include "api/array_view.h" +#include "api/units/time_delta.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { + +uint16_t VideoSendTiming::GetDeltaCappedMs(int64_t base_ms, int64_t time_ms) { + if (time_ms < base_ms) { + RTC_DLOG(LS_ERROR) << "Delta " << (time_ms - base_ms) + << "ms expected to be positive"; + } + return rtc::saturated_cast(time_ms - base_ms); +} + +uint16_t VideoSendTiming::GetDeltaCappedMs(TimeDelta delta) { + if (delta < TimeDelta::Zero()) { + RTC_DLOG(LS_ERROR) << "Delta " << delta.ms() + << "ms expected to be positive"; + } + return rtc::saturated_cast(delta.ms()); +} + +TimingFrameInfo::TimingFrameInfo() + : rtp_timestamp(0), + capture_time_ms(-1), + encode_start_ms(-1), + encode_finish_ms(-1), + packetization_finish_ms(-1), + pacer_exit_ms(-1), + network_timestamp_ms(-1), + network2_timestamp_ms(-1), + receive_start_ms(-1), + receive_finish_ms(-1), + decode_start_ms(-1), + decode_finish_ms(-1), + render_time_ms(-1), + flags(VideoSendTiming::kNotTriggered) {} + +int64_t TimingFrameInfo::EndToEndDelay() const { + return capture_time_ms >= 0 ? decode_finish_ms - capture_time_ms : -1; +} + +bool TimingFrameInfo::IsLongerThan(const TimingFrameInfo& other) const { + int64_t other_delay = other.EndToEndDelay(); + return other_delay == -1 || EndToEndDelay() > other_delay; +} + +bool TimingFrameInfo::operator<(const TimingFrameInfo& other) const { + return other.IsLongerThan(*this); +} + +bool TimingFrameInfo::operator<=(const TimingFrameInfo& other) const { + return !IsLongerThan(other); +} + +bool TimingFrameInfo::IsOutlier() const { + return !IsInvalid() && (flags & VideoSendTiming::kTriggeredBySize); +} + +bool TimingFrameInfo::IsTimerTriggered() const { + return !IsInvalid() && (flags & VideoSendTiming::kTriggeredByTimer); +} + +bool TimingFrameInfo::IsInvalid() const { + return flags == VideoSendTiming::kInvalid; +} + +std::string TimingFrameInfo::ToString() const { + if (IsInvalid()) { + return ""; + } + + char buf[1024]; + rtc::SimpleStringBuilder sb(buf); + + sb << rtp_timestamp << ',' << capture_time_ms << ',' << encode_start_ms << ',' + << encode_finish_ms << ',' << packetization_finish_ms << ',' + << pacer_exit_ms << ',' << network_timestamp_ms << ',' + << network2_timestamp_ms << ',' << receive_start_ms << ',' + << receive_finish_ms << ',' << decode_start_ms << ',' << decode_finish_ms + << ',' << render_time_ms << ',' << IsOutlier() << ',' + << IsTimerTriggered(); + + return sb.str(); +} + +VideoPlayoutDelay::VideoPlayoutDelay(TimeDelta min, TimeDelta max) + : min_(std::clamp(min, TimeDelta::Zero(), kMax)), + max_(std::clamp(max, min_, kMax)) { + if (!(TimeDelta::Zero() <= min && min <= max && max <= kMax)) { + RTC_LOG(LS_ERROR) << "Invalid video playout delay: [" << min << "," << max + << "]. Clamped to [" << this->min() << "," << this->max() + << "]"; + } +} + +bool VideoPlayoutDelay::Set(TimeDelta min, TimeDelta max) { + if (TimeDelta::Zero() <= min && min <= max && max <= kMax) { + min_ = min; + max_ = max; + return true; + } + return false; +} + +} // namespace webrtc diff --git a/VocieProcess/api/video/video_timing.h b/VocieProcess/api/video/video_timing.h new file mode 100644 index 0000000..0a450cd --- /dev/null +++ b/VocieProcess/api/video/video_timing.h @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_VIDEO_TIMING_H_ +#define API_VIDEO_VIDEO_TIMING_H_ + +#include + +#include +#include + +#include "api/units/time_delta.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// Video timing timestamps in ms counted from capture_time_ms of a frame. +// This structure represents data sent in video-timing RTP header extension. +struct RTC_EXPORT VideoSendTiming { + enum TimingFrameFlags : uint8_t { + kNotTriggered = 0, // Timing info valid, but not to be transmitted. + // Used on send-side only. + kTriggeredByTimer = 1 << 0, // Frame marked for tracing by periodic timer. + kTriggeredBySize = 1 << 1, // Frame marked for tracing due to size. + kInvalid = std::numeric_limits::max() // Invalid, ignore! + }; + + // Returns |time_ms - base_ms| capped at max 16-bit value. + // Used to fill this data structure as per + // https://webrtc.org/experiments/rtp-hdrext/video-timing/ extension stores + // 16-bit deltas of timestamps from packet capture time. + static uint16_t GetDeltaCappedMs(int64_t base_ms, int64_t time_ms); + static uint16_t GetDeltaCappedMs(TimeDelta delta); + + uint16_t encode_start_delta_ms; + uint16_t encode_finish_delta_ms; + uint16_t packetization_finish_delta_ms; + uint16_t pacer_exit_delta_ms; + uint16_t network_timestamp_delta_ms; + uint16_t network2_timestamp_delta_ms; + uint8_t flags = TimingFrameFlags::kInvalid; +}; + +// Used to report precise timings of a 'timing frames'. Contains all important +// timestamps for a lifetime of that specific frame. Reported as a string via +// GetStats(). Only frame which took the longest between two GetStats calls is +// reported. +struct RTC_EXPORT TimingFrameInfo { + TimingFrameInfo(); + + // Returns end-to-end delay of a frame, if sender and receiver timestamps are + // synchronized, -1 otherwise. + int64_t EndToEndDelay() const; + + // Returns true if current frame took longer to process than `other` frame. + // If other frame's clocks are not synchronized, current frame is always + // preferred. + bool IsLongerThan(const TimingFrameInfo& other) const; + + // Returns true if flags are set to indicate this frame was marked for tracing + // due to the size being outside some limit. + bool IsOutlier() const; + + // Returns true if flags are set to indicate this frame was marked fro tracing + // due to cyclic timer. + bool IsTimerTriggered() const; + + // Returns true if the timing data is marked as invalid, in which case it + // should be ignored. + bool IsInvalid() const; + + std::string ToString() const; + + bool operator<(const TimingFrameInfo& other) const; + + bool operator<=(const TimingFrameInfo& other) const; + + uint32_t rtp_timestamp; // Identifier of a frame. + // All timestamps below are in local monotonous clock of a receiver. + // If sender clock is not yet estimated, sender timestamps + // (capture_time_ms ... pacer_exit_ms) are negative values, still + // relatively correct. + int64_t capture_time_ms; // Captrue time of a frame. + int64_t encode_start_ms; // Encode start time. + int64_t encode_finish_ms; // Encode completion time. + int64_t packetization_finish_ms; // Time when frame was passed to pacer. + int64_t pacer_exit_ms; // Time when last packet was pushed out of pacer. + // Two in-network RTP processor timestamps: meaning is application specific. + int64_t network_timestamp_ms; + int64_t network2_timestamp_ms; + int64_t receive_start_ms; // First received packet time. + int64_t receive_finish_ms; // Last received packet time. + int64_t decode_start_ms; // Decode start time. + int64_t decode_finish_ms; // Decode completion time. + int64_t render_time_ms; // Proposed render time to insure smooth playback. + + uint8_t flags; // Flags indicating validity and/or why tracing was triggered. +}; + +// Minimum and maximum playout delay values from capture to render. +// These are best effort values. +// +// min = max = 0 indicates that the receiver should try and render +// frame as soon as possible. +// +// min = x, max = y indicates that the receiver is free to adapt +// in the range (x, y) based on network jitter. +// This class ensures invariant 0 <= min <= max <= kMax. +class RTC_EXPORT VideoPlayoutDelay { + public: + // Maximum supported value for the delay limit. + static constexpr TimeDelta kMax = TimeDelta::Millis(10) * 0xFFF; + + // Creates delay limits that indicates receiver should try to render frame + // as soon as possible. + static VideoPlayoutDelay Minimal() { + return VideoPlayoutDelay(TimeDelta::Zero(), TimeDelta::Zero()); + } + + // Creates valid, but unspecified limits. + VideoPlayoutDelay() = default; + VideoPlayoutDelay(const VideoPlayoutDelay&) = default; + VideoPlayoutDelay& operator=(const VideoPlayoutDelay&) = default; + VideoPlayoutDelay(TimeDelta min, TimeDelta max); + + bool Set(TimeDelta min, TimeDelta max); + + TimeDelta min() const { return min_; } + TimeDelta max() const { return max_; } + + friend bool operator==(const VideoPlayoutDelay& lhs, + const VideoPlayoutDelay& rhs) { + return lhs.min_ == rhs.min_ && lhs.max_ == rhs.max_; + } + + private: + TimeDelta min_ = TimeDelta::Zero(); + TimeDelta max_ = kMax; +}; + +} // namespace webrtc + +#endif // API_VIDEO_VIDEO_TIMING_H_ diff --git a/VocieProcess/common_audio/audio_converter.cc b/VocieProcess/common_audio/audio_converter.cc new file mode 100644 index 0000000..485ec80 --- /dev/null +++ b/VocieProcess/common_audio/audio_converter.cc @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "common_audio/audio_converter.h" + +#include +#include +#include +#include + +#include "common_audio/channel_buffer.h" +#include "common_audio/resampler/push_sinc_resampler.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +class CopyConverter : public AudioConverter { + public: + CopyConverter(size_t src_channels, + size_t src_frames, + size_t dst_channels, + size_t dst_frames) + : AudioConverter(src_channels, src_frames, dst_channels, dst_frames) {} + ~CopyConverter() override {} + + void Convert(const float* const* src, + size_t src_size, + float* const* dst, + size_t dst_capacity) override { + CheckSizes(src_size, dst_capacity); + if (src != dst) { + for (size_t i = 0; i < src_channels(); ++i) + std::memcpy(dst[i], src[i], dst_frames() * sizeof(*dst[i])); + } + } +}; + +class UpmixConverter : public AudioConverter { + public: + UpmixConverter(size_t src_channels, + size_t src_frames, + size_t dst_channels, + size_t dst_frames) + : AudioConverter(src_channels, src_frames, dst_channels, dst_frames) {} + ~UpmixConverter() override {} + + void Convert(const float* const* src, + size_t src_size, + float* const* dst, + size_t dst_capacity) override { + CheckSizes(src_size, dst_capacity); + for (size_t i = 0; i < dst_frames(); ++i) { + const float value = src[0][i]; + for (size_t j = 0; j < dst_channels(); ++j) + dst[j][i] = value; + } + } +}; + +class DownmixConverter : public AudioConverter { + public: + DownmixConverter(size_t src_channels, + size_t src_frames, + size_t dst_channels, + size_t dst_frames) + : AudioConverter(src_channels, src_frames, dst_channels, dst_frames) {} + ~DownmixConverter() override {} + + void Convert(const float* const* src, + size_t src_size, + float* const* dst, + size_t dst_capacity) override { + CheckSizes(src_size, dst_capacity); + float* dst_mono = dst[0]; + for (size_t i = 0; i < src_frames(); ++i) { + float sum = 0; + for (size_t j = 0; j < src_channels(); ++j) + sum += src[j][i]; + dst_mono[i] = sum / src_channels(); + } + } +}; + +class ResampleConverter : public AudioConverter { + public: + ResampleConverter(size_t src_channels, + size_t src_frames, + size_t dst_channels, + size_t dst_frames) + : AudioConverter(src_channels, src_frames, dst_channels, dst_frames) { + resamplers_.reserve(src_channels); + for (size_t i = 0; i < src_channels; ++i) + resamplers_.push_back(std::unique_ptr( + new PushSincResampler(src_frames, dst_frames))); + } + ~ResampleConverter() override {} + + void Convert(const float* const* src, + size_t src_size, + float* const* dst, + size_t dst_capacity) override { + CheckSizes(src_size, dst_capacity); + for (size_t i = 0; i < resamplers_.size(); ++i) + resamplers_[i]->Resample(src[i], src_frames(), dst[i], dst_frames()); + } + + private: + std::vector> resamplers_; +}; + +// Apply a vector of converters in serial, in the order given. At least two +// converters must be provided. +class CompositionConverter : public AudioConverter { + public: + explicit CompositionConverter( + std::vector> converters) + : converters_(std::move(converters)) { + RTC_CHECK_GE(converters_.size(), 2); + // We need an intermediate buffer after every converter. + for (auto it = converters_.begin(); it != converters_.end() - 1; ++it) + buffers_.push_back( + std::unique_ptr>(new ChannelBuffer( + (*it)->dst_frames(), (*it)->dst_channels()))); + } + ~CompositionConverter() override {} + + void Convert(const float* const* src, + size_t src_size, + float* const* dst, + size_t dst_capacity) override { + converters_.front()->Convert(src, src_size, buffers_.front()->channels(), + buffers_.front()->size()); + for (size_t i = 2; i < converters_.size(); ++i) { + auto& src_buffer = buffers_[i - 2]; + auto& dst_buffer = buffers_[i - 1]; + converters_[i]->Convert(src_buffer->channels(), src_buffer->size(), + dst_buffer->channels(), dst_buffer->size()); + } + converters_.back()->Convert(buffers_.back()->channels(), + buffers_.back()->size(), dst, dst_capacity); + } + + private: + std::vector> converters_; + std::vector>> buffers_; +}; + +std::unique_ptr AudioConverter::Create(size_t src_channels, + size_t src_frames, + size_t dst_channels, + size_t dst_frames) { + std::unique_ptr sp; + if (src_channels > dst_channels) { + if (src_frames != dst_frames) { + std::vector> converters; + converters.push_back(std::unique_ptr(new DownmixConverter( + src_channels, src_frames, dst_channels, src_frames))); + converters.push_back( + std::unique_ptr(new ResampleConverter( + dst_channels, src_frames, dst_channels, dst_frames))); + sp.reset(new CompositionConverter(std::move(converters))); + } else { + sp.reset(new DownmixConverter(src_channels, src_frames, dst_channels, + dst_frames)); + } + } else if (src_channels < dst_channels) { + if (src_frames != dst_frames) { + std::vector> converters; + converters.push_back( + std::unique_ptr(new ResampleConverter( + src_channels, src_frames, src_channels, dst_frames))); + converters.push_back(std::unique_ptr(new UpmixConverter( + src_channels, dst_frames, dst_channels, dst_frames))); + sp.reset(new CompositionConverter(std::move(converters))); + } else { + sp.reset(new UpmixConverter(src_channels, src_frames, dst_channels, + dst_frames)); + } + } else if (src_frames != dst_frames) { + sp.reset(new ResampleConverter(src_channels, src_frames, dst_channels, + dst_frames)); + } else { + sp.reset( + new CopyConverter(src_channels, src_frames, dst_channels, dst_frames)); + } + + return sp; +} + +// For CompositionConverter. +AudioConverter::AudioConverter() + : src_channels_(0), src_frames_(0), dst_channels_(0), dst_frames_(0) {} + +AudioConverter::AudioConverter(size_t src_channels, + size_t src_frames, + size_t dst_channels, + size_t dst_frames) + : src_channels_(src_channels), + src_frames_(src_frames), + dst_channels_(dst_channels), + dst_frames_(dst_frames) { + RTC_CHECK(dst_channels == src_channels || dst_channels == 1 || + src_channels == 1); +} + +void AudioConverter::CheckSizes(size_t src_size, size_t dst_capacity) const { + RTC_CHECK_EQ(src_size, src_channels() * src_frames()); + RTC_CHECK_GE(dst_capacity, dst_channels() * dst_frames()); +} + +} // namespace webrtc diff --git a/VocieProcess/common_audio/audio_converter.h b/VocieProcess/common_audio/audio_converter.h new file mode 100644 index 0000000..4afbb6d --- /dev/null +++ b/VocieProcess/common_audio/audio_converter.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef COMMON_AUDIO_AUDIO_CONVERTER_H_ +#define COMMON_AUDIO_AUDIO_CONVERTER_H_ + +#include + +#include + +namespace webrtc { + +// Format conversion (remixing and resampling) for audio. Only simple remixing +// conversions are supported: downmix to mono (i.e. `dst_channels` == 1) or +// upmix from mono (i.e. |src_channels == 1|). +// +// The source and destination chunks have the same duration in time; specifying +// the number of frames is equivalent to specifying the sample rates. +class AudioConverter { + public: + // Returns a new AudioConverter, which will use the supplied format for its + // lifetime. Caller is responsible for the memory. + static std::unique_ptr Create(size_t src_channels, + size_t src_frames, + size_t dst_channels, + size_t dst_frames); + virtual ~AudioConverter() {} + + AudioConverter(const AudioConverter&) = delete; + AudioConverter& operator=(const AudioConverter&) = delete; + + // Convert `src`, containing `src_size` samples, to `dst`, having a sample + // capacity of `dst_capacity`. Both point to a series of buffers containing + // the samples for each channel. The sizes must correspond to the format + // passed to Create(). + virtual void Convert(const float* const* src, + size_t src_size, + float* const* dst, + size_t dst_capacity) = 0; + + size_t src_channels() const { return src_channels_; } + size_t src_frames() const { return src_frames_; } + size_t dst_channels() const { return dst_channels_; } + size_t dst_frames() const { return dst_frames_; } + + protected: + AudioConverter(); + AudioConverter(size_t src_channels, + size_t src_frames, + size_t dst_channels, + size_t dst_frames); + + // Helper to RTC_CHECK that inputs are correctly sized. + void CheckSizes(size_t src_size, size_t dst_capacity) const; + + private: + const size_t src_channels_; + const size_t src_frames_; + const size_t dst_channels_; + const size_t dst_frames_; +}; + +} // namespace webrtc + +#endif // COMMON_AUDIO_AUDIO_CONVERTER_H_ diff --git a/VocieProcess/common_audio/resampler/include/push_resampler.h b/VocieProcess/common_audio/resampler/include/push_resampler.h new file mode 100644 index 0000000..394e96b --- /dev/null +++ b/VocieProcess/common_audio/resampler/include/push_resampler.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef COMMON_AUDIO_RESAMPLER_INCLUDE_PUSH_RESAMPLER_H_ +#define COMMON_AUDIO_RESAMPLER_INCLUDE_PUSH_RESAMPLER_H_ + +#include +#include + +#include "api/audio/audio_view.h" + +namespace webrtc { + +class PushSincResampler; + +// Wraps PushSincResampler to provide stereo support. +// Note: This implementation assumes 10ms buffer sizes throughout. +template +class PushResampler final { + public: + PushResampler(); + PushResampler(size_t src_samples_per_channel, + size_t dst_samples_per_channel, + size_t num_channels); + ~PushResampler(); + + // Returns the total number of samples provided in destination (e.g. 32 kHz, + // 2 channel audio gives 640 samples). + int Resample(InterleavedView src, InterleavedView dst); + // For when a deinterleaved/mono channel already exists and we can skip the + // deinterleaved operation. + int Resample(MonoView src, MonoView dst); + + private: + // Ensures that source and destination buffers for deinterleaving are + // correctly configured prior to resampling that requires deinterleaving. + void EnsureInitialized(size_t src_samples_per_channel, + size_t dst_samples_per_channel, + size_t num_channels); + + // Buffers used for when a deinterleaving step is necessary. + std::unique_ptr source_; + std::unique_ptr destination_; + DeinterleavedView source_view_; + DeinterleavedView destination_view_; + + std::vector> resamplers_; +}; +} // namespace webrtc + +#endif // COMMON_AUDIO_RESAMPLER_INCLUDE_PUSH_RESAMPLER_H_ diff --git a/VocieProcess/common_audio/resampler/include/resampler.h b/VocieProcess/common_audio/resampler/include/resampler.h new file mode 100644 index 0000000..41940f9 --- /dev/null +++ b/VocieProcess/common_audio/resampler/include/resampler.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * A wrapper for resampling a numerous amount of sampling combinations. + */ + +#ifndef COMMON_AUDIO_RESAMPLER_INCLUDE_RESAMPLER_H_ +#define COMMON_AUDIO_RESAMPLER_INCLUDE_RESAMPLER_H_ + +#include +#include + +namespace webrtc { + +// All methods return 0 on success and -1 on failure. +class Resampler { + public: + Resampler(); + Resampler(int inFreq, int outFreq, size_t num_channels); + ~Resampler(); + + // Reset all states + int Reset(int inFreq, int outFreq, size_t num_channels); + + // Reset all states if any parameter has changed + int ResetIfNeeded(int inFreq, int outFreq, size_t num_channels); + + // Resample samplesIn to samplesOut. + int Push(const int16_t* samplesIn, + size_t lengthIn, + int16_t* samplesOut, + size_t maxLen, + size_t& outLen); // NOLINT: to avoid changing APIs + + private: + enum ResamplerMode { + kResamplerMode1To1, + kResamplerMode1To2, + kResamplerMode1To3, + kResamplerMode1To4, + kResamplerMode1To6, + kResamplerMode1To12, + kResamplerMode2To3, + kResamplerMode2To11, + kResamplerMode4To11, + kResamplerMode8To11, + kResamplerMode11To16, + kResamplerMode11To32, + kResamplerMode2To1, + kResamplerMode3To1, + kResamplerMode4To1, + kResamplerMode6To1, + kResamplerMode12To1, + kResamplerMode3To2, + kResamplerMode11To2, + kResamplerMode11To4, + kResamplerMode11To8 + }; + + // Computes the resampler mode for a given sampling frequency pair. + // Returns -1 for unsupported frequency pairs. + static int ComputeResamplerMode(int in_freq_hz, + int out_freq_hz, + ResamplerMode* mode); + + // Generic pointers since we don't know what states we'll need + void* state1_; + void* state2_; + void* state3_; + + // Storage if needed + int16_t* in_buffer_; + int16_t* out_buffer_; + size_t in_buffer_size_; + size_t out_buffer_size_; + size_t in_buffer_size_max_; + size_t out_buffer_size_max_; + + int my_in_frequency_khz_; + int my_out_frequency_khz_; + ResamplerMode my_mode_; + size_t num_channels_; + + // Extra instance for stereo + Resampler* helper_left_; + Resampler* helper_right_; +}; + +} // namespace webrtc + +#endif // COMMON_AUDIO_RESAMPLER_INCLUDE_RESAMPLER_H_ diff --git a/VocieProcess/common_audio/vad/include/webrtc_vad.h b/VocieProcess/common_audio/vad/include/webrtc_vad.h new file mode 100644 index 0000000..31e628f --- /dev/null +++ b/VocieProcess/common_audio/vad/include/webrtc_vad.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * This header file includes the VAD API calls. Specific function calls are + * given below. + */ + +#ifndef COMMON_AUDIO_VAD_INCLUDE_WEBRTC_VAD_H_ // NOLINT +#define COMMON_AUDIO_VAD_INCLUDE_WEBRTC_VAD_H_ + +#include +#include + +typedef struct WebRtcVadInst VadInst; + +#ifdef __cplusplus +extern "C" { +#endif + +// Creates an instance to the VAD structure. +VadInst* WebRtcVad_Create(void); + +// Frees the dynamic memory of a specified VAD instance. +// +// - handle [i] : Pointer to VAD instance that should be freed. +void WebRtcVad_Free(VadInst* handle); + +// Initializes a VAD instance. +// +// - handle [i/o] : Instance that should be initialized. +// +// returns : 0 - (OK), +// -1 - (null pointer or Default mode could not be set). +int WebRtcVad_Init(VadInst* handle); + +// Sets the VAD operating mode. A more aggressive (higher mode) VAD is more +// restrictive in reporting speech. Put in other words the probability of being +// speech when the VAD returns 1 is increased with increasing mode. As a +// consequence also the missed detection rate goes up. +// +// - handle [i/o] : VAD instance. +// - mode [i] : Aggressiveness mode (0, 1, 2, or 3). +// +// returns : 0 - (OK), +// -1 - (null pointer, mode could not be set or the VAD instance +// has not been initialized). +int WebRtcVad_set_mode(VadInst* handle, int mode); + +// Calculates a VAD decision for the `audio_frame`. For valid sampling rates +// frame lengths, see the description of WebRtcVad_ValidRatesAndFrameLengths(). +// +// - handle [i/o] : VAD Instance. Needs to be initialized by +// WebRtcVad_Init() before call. +// - fs [i] : Sampling frequency (Hz): 8000, 16000, or 32000 +// - audio_frame [i] : Audio frame buffer. +// - frame_length [i] : Length of audio frame buffer in number of samples. +// +// returns : 1 - (Active Voice), +// 0 - (Non-active Voice), +// -1 - (Error) +int WebRtcVad_Process(VadInst* handle, + int fs, + const int16_t* audio_frame, + size_t frame_length); + +// Checks for valid combinations of `rate` and `frame_length`. We support 10, +// 20 and 30 ms frames and the rates 8000, 16000 and 32000 Hz. +// +// - rate [i] : Sampling frequency (Hz). +// - frame_length [i] : Speech frame buffer length in number of samples. +// +// returns : 0 - (valid combination), -1 - (invalid combination) +int WebRtcVad_ValidRateAndFrameLength(int rate, size_t frame_length); + +#ifdef __cplusplus +} +#endif + +#endif // COMMON_AUDIO_VAD_INCLUDE_WEBRTC_VAD_H_ // NOLINT diff --git a/VocieProcess/modules/audio_coding/codecs/isac/bandwidth_info.h b/VocieProcess/modules/audio_coding/codecs/isac/bandwidth_info.h new file mode 100644 index 0000000..c3830a5 --- /dev/null +++ b/VocieProcess/modules/audio_coding/codecs/isac/bandwidth_info.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_BANDWIDTH_INFO_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_BANDWIDTH_INFO_H_ + +#include + +typedef struct { + int in_use; + int32_t send_bw_avg; + int32_t send_max_delay_avg; + int16_t bottleneck_idx; + int16_t jitter_info; +} IsacBandwidthInfo; + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_BANDWIDTH_INFO_H_ diff --git a/VocieProcess/modules/audio_coding/codecs/isac/main/source/filter_functions.c b/VocieProcess/modules/audio_coding/codecs/isac/main/source/filter_functions.c new file mode 100644 index 0000000..a4f297c --- /dev/null +++ b/VocieProcess/modules/audio_coding/codecs/isac/main/source/filter_functions.c @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#ifdef WEBRTC_ANDROID +#include +#endif + +#include "modules/audio_coding/codecs/isac/main/source/pitch_estimator.h" +#include "modules/audio_coding/codecs/isac/main/source/isac_vad.h" + +static void WebRtcIsac_AllPoleFilter(double* InOut, + double* Coef, + size_t lengthInOut, + int orderCoef) { + /* the state of filter is assumed to be in InOut[-1] to InOut[-orderCoef] */ + double scal; + double sum; + size_t n; + int k; + + //if (fabs(Coef[0]-1.0)<0.001) { + if ( (Coef[0] > 0.9999) && (Coef[0] < 1.0001) ) + { + for(n = 0; n < lengthInOut; n++) + { + sum = Coef[1] * InOut[-1]; + for(k = 2; k <= orderCoef; k++){ + sum += Coef[k] * InOut[-k]; + } + *InOut++ -= sum; + } + } + else + { + scal = 1.0 / Coef[0]; + for(n=0;nbuffer, sizeof(double) * PITCH_WLPCBUFLEN); + memcpy(tmpbuffer+PITCH_WLPCBUFLEN, in, sizeof(double) * PITCH_FRAME_LEN); + memcpy(wfdata->buffer, tmpbuffer+PITCH_FRAME_LEN, sizeof(double) * PITCH_WLPCBUFLEN); + + dp=weoutbuf; + dp2=whoutbuf; + for (k=0;kweostate[k]; + *dp2++ = wfdata->whostate[k]; + opol[k]=0.0; + } + opol[0]=1.0; + opol[PITCH_WLPCORDER]=0.0; + weo=dp; + who=dp2; + + endpos=PITCH_WLPCBUFLEN + PITCH_SUBFRAME_LEN; + inp=tmpbuffer + PITCH_WLPCBUFLEN; + + for (n=0; nwindow[k]*tmpbuffer[start+k]; + } + + /* Get LPC polynomial */ + WebRtcIsac_AutoCorr(corr, ext, PITCH_WLPCWINLEN, PITCH_WLPCORDER); + corr[0]=1.01*corr[0]+1.0; /* White noise correction */ + WebRtcIsac_LevDurb(apol, rc, corr, PITCH_WLPCORDER); + WebRtcIsac_BwExpand(apolr, apol, rho, PITCH_WLPCORDER+1); + + /* Filtering */ + WebRtcIsac_ZeroPoleFilter(inp, apol, apolr, PITCH_SUBFRAME_LEN, PITCH_WLPCORDER, weo); + WebRtcIsac_ZeroPoleFilter(inp, apolr, opol, PITCH_SUBFRAME_LEN, PITCH_WLPCORDER, who); + + inp+=PITCH_SUBFRAME_LEN; + endpos+=PITCH_SUBFRAME_LEN; + weo+=PITCH_SUBFRAME_LEN; + who+=PITCH_SUBFRAME_LEN; + } + + /* Export filter states */ + for (k=0;kweostate[k]=weoutbuf[PITCH_FRAME_LEN+k]; + wfdata->whostate[k]=whoutbuf[PITCH_FRAME_LEN+k]; + } + + /* Export output data */ + memcpy(weiout, weoutbuf+PITCH_WLPCORDER, sizeof(double) * PITCH_FRAME_LEN); + memcpy(whiout, whoutbuf+PITCH_WLPCORDER, sizeof(double) * PITCH_FRAME_LEN); +} diff --git a/VocieProcess/modules/audio_coding/codecs/isac/main/source/filter_functions.h b/VocieProcess/modules/audio_coding/codecs/isac/main/source/filter_functions.h new file mode 100644 index 0000000..a747a7f --- /dev/null +++ b/VocieProcess/modules/audio_coding/codecs/isac/main/source/filter_functions.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_FILTER_FUNCTIONS_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_FILTER_FUNCTIONS_H_ + +#include + +#include "modules/audio_coding/codecs/isac/main/source/structs.h" + +void WebRtcIsac_AutoCorr(double* r, const double* x, size_t N, size_t order); + +void WebRtcIsac_WeightingFilter(const double* in, + double* weiout, + double* whiout, + WeightFiltstr* wfdata); + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_FILTER_FUNCTIONS_H_ diff --git a/VocieProcess/modules/audio_coding/codecs/isac/main/source/isac_vad.c b/VocieProcess/modules/audio_coding/codecs/isac/main/source/isac_vad.c new file mode 100644 index 0000000..57cf0c3 --- /dev/null +++ b/VocieProcess/modules/audio_coding/codecs/isac/main/source/isac_vad.c @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/main/source/isac_vad.h" + +#include + +void WebRtcIsac_InitPitchFilter(PitchFiltstr* pitchfiltdata) { + int k; + + for (k = 0; k < PITCH_BUFFSIZE; k++) { + pitchfiltdata->ubuf[k] = 0.0; + } + pitchfiltdata->ystate[0] = 0.0; + for (k = 1; k < (PITCH_DAMPORDER); k++) { + pitchfiltdata->ystate[k] = 0.0; + } + pitchfiltdata->oldlagp[0] = 50.0; + pitchfiltdata->oldgainp[0] = 0.0; +} + +static void WebRtcIsac_InitWeightingFilter(WeightFiltstr* wfdata) { + int k; + double t, dtmp, dtmp2, denum, denum2; + + for (k = 0; k < PITCH_WLPCBUFLEN; k++) + wfdata->buffer[k] = 0.0; + + for (k = 0; k < PITCH_WLPCORDER; k++) { + wfdata->istate[k] = 0.0; + wfdata->weostate[k] = 0.0; + wfdata->whostate[k] = 0.0; + } + + /* next part should be in Matlab, writing to a global table */ + t = 0.5; + denum = 1.0 / ((double)PITCH_WLPCWINLEN); + denum2 = denum * denum; + for (k = 0; k < PITCH_WLPCWINLEN; k++) { + dtmp = PITCH_WLPCASYM * t * denum + (1 - PITCH_WLPCASYM) * t * t * denum2; + dtmp *= 3.14159265; + dtmp2 = sin(dtmp); + wfdata->window[k] = dtmp2 * dtmp2; + t++; + } +} + +void WebRtcIsac_InitPitchAnalysis(PitchAnalysisStruct* State) { + int k; + + for (k = 0; k < PITCH_CORR_LEN2 + PITCH_CORR_STEP2 + PITCH_MAX_LAG / 2 - + PITCH_FRAME_LEN / 2 + 2; + k++) + State->dec_buffer[k] = 0.0; + for (k = 0; k < 2 * ALLPASSSECTIONS + 1; k++) + State->decimator_state[k] = 0.0; + for (k = 0; k < 2; k++) + State->hp_state[k] = 0.0; + for (k = 0; k < QLOOKAHEAD; k++) + State->whitened_buf[k] = 0.0; + for (k = 0; k < QLOOKAHEAD; k++) + State->inbuf[k] = 0.0; + + WebRtcIsac_InitPitchFilter(&(State->PFstr_wght)); + + WebRtcIsac_InitPitchFilter(&(State->PFstr)); + + WebRtcIsac_InitWeightingFilter(&(State->Wghtstr)); +} + +void WebRtcIsac_InitPreFilterbank(PreFiltBankstr* prefiltdata) { + int k; + + for (k = 0; k < QLOOKAHEAD; k++) { + prefiltdata->INLABUF1[k] = 0; + prefiltdata->INLABUF2[k] = 0; + + prefiltdata->INLABUF1_float[k] = 0; + prefiltdata->INLABUF2_float[k] = 0; + } + for (k = 0; k < 2 * (QORDER - 1); k++) { + prefiltdata->INSTAT1[k] = 0; + prefiltdata->INSTAT2[k] = 0; + prefiltdata->INSTATLA1[k] = 0; + prefiltdata->INSTATLA2[k] = 0; + + prefiltdata->INSTAT1_float[k] = 0; + prefiltdata->INSTAT2_float[k] = 0; + prefiltdata->INSTATLA1_float[k] = 0; + prefiltdata->INSTATLA2_float[k] = 0; + } + + /* High pass filter states */ + prefiltdata->HPstates[0] = 0.0; + prefiltdata->HPstates[1] = 0.0; + + prefiltdata->HPstates_float[0] = 0.0f; + prefiltdata->HPstates_float[1] = 0.0f; + + return; +} + +double WebRtcIsac_LevDurb(double* a, double* k, double* r, size_t order) { + const double LEVINSON_EPS = 1.0e-10; + + double sum, alpha; + size_t m, m_h, i; + alpha = 0; // warning -DH + a[0] = 1.0; + if (r[0] < LEVINSON_EPS) { /* if r[0] <= 0, set LPC coeff. to zero */ + for (i = 0; i < order; i++) { + k[i] = 0; + a[i + 1] = 0; + } + } else { + a[1] = k[0] = -r[1] / r[0]; + alpha = r[0] + r[1] * k[0]; + for (m = 1; m < order; m++) { + sum = r[m + 1]; + for (i = 0; i < m; i++) { + sum += a[i + 1] * r[m - i]; + } + k[m] = -sum / alpha; + alpha += k[m] * sum; + m_h = (m + 1) >> 1; + for (i = 0; i < m_h; i++) { + sum = a[i + 1] + k[m] * a[m - i]; + a[m - i] += k[m] * a[i + 1]; + a[i + 1] = sum; + } + a[m + 1] = k[m]; + } + } + return alpha; +} + +/* The upper channel all-pass filter factors */ +const float WebRtcIsac_kUpperApFactorsFloat[2] = {0.03470000000000f, + 0.38260000000000f}; + +/* The lower channel all-pass filter factors */ +const float WebRtcIsac_kLowerApFactorsFloat[2] = {0.15440000000000f, + 0.74400000000000f}; + +/* This function performs all-pass filtering--a series of first order all-pass + * sections are used to filter the input in a cascade manner. + * The input is overwritten!! + */ +void WebRtcIsac_AllPassFilter2Float(float* InOut, + const float* APSectionFactors, + int lengthInOut, + int NumberOfSections, + float* FilterState) { + int n, j; + float temp; + for (j = 0; j < NumberOfSections; j++) { + for (n = 0; n < lengthInOut; n++) { + temp = FilterState[j] + APSectionFactors[j] * InOut[n]; + FilterState[j] = -APSectionFactors[j] * temp + InOut[n]; + InOut[n] = temp; + } + } +} + +/* The number of composite all-pass filter factors */ +#define NUMBEROFCOMPOSITEAPSECTIONS 4 + +/* Function WebRtcIsac_SplitAndFilter + * This function creates low-pass and high-pass decimated versions of part of + the input signal, and part of the signal in the input 'lookahead buffer'. + + INPUTS: + in: a length FRAMESAMPLES array of input samples + prefiltdata: input data structure containing the filterbank states + and lookahead samples from the previous encoding + iteration. + OUTPUTS: + LP: a FRAMESAMPLES_HALF array of low-pass filtered samples that + have been phase equalized. The first QLOOKAHEAD samples are + based on the samples in the two prefiltdata->INLABUFx arrays + each of length QLOOKAHEAD. + The remaining FRAMESAMPLES_HALF-QLOOKAHEAD samples are based + on the first FRAMESAMPLES_HALF-QLOOKAHEAD samples of the input + array in[]. + HP: a FRAMESAMPLES_HALF array of high-pass filtered samples that + have been phase equalized. The first QLOOKAHEAD samples are + based on the samples in the two prefiltdata->INLABUFx arrays + each of length QLOOKAHEAD. + The remaining FRAMESAMPLES_HALF-QLOOKAHEAD samples are based + on the first FRAMESAMPLES_HALF-QLOOKAHEAD samples of the input + array in[]. + + LP_la: a FRAMESAMPLES_HALF array of low-pass filtered samples. + These samples are not phase equalized. They are computed + from the samples in the in[] array. + HP_la: a FRAMESAMPLES_HALF array of high-pass filtered samples + that are not phase equalized. They are computed from + the in[] vector. + prefiltdata: this input data structure's filterbank state and + lookahead sample buffers are updated for the next + encoding iteration. +*/ +void WebRtcIsac_SplitAndFilterFloat(float* pin, + float* LP, + float* HP, + double* LP_la, + double* HP_la, + PreFiltBankstr* prefiltdata) { + int k, n; + float CompositeAPFilterState[NUMBEROFCOMPOSITEAPSECTIONS]; + float ForTransform_CompositeAPFilterState[NUMBEROFCOMPOSITEAPSECTIONS]; + float ForTransform_CompositeAPFilterState2[NUMBEROFCOMPOSITEAPSECTIONS]; + float tempinoutvec[FRAMESAMPLES + MAX_AR_MODEL_ORDER]; + float tempin_ch1[FRAMESAMPLES + MAX_AR_MODEL_ORDER]; + float tempin_ch2[FRAMESAMPLES + MAX_AR_MODEL_ORDER]; + float in[FRAMESAMPLES]; + float ftmp; + + /* HPstcoeff_in = {a1, a2, b1 - b0 * a1, b2 - b0 * a2}; */ + static const float kHpStCoefInFloat[4] = { + -1.94895953203325f, 0.94984516000000f, -0.05101826139794f, + 0.05015484000000f}; + + /* The composite all-pass filter factors */ + static const float WebRtcIsac_kCompositeApFactorsFloat[4] = { + 0.03470000000000f, 0.15440000000000f, 0.38260000000000f, + 0.74400000000000f}; + + // The matrix for transforming the backward composite state to upper channel + // state. + static const float WebRtcIsac_kTransform1Float[8] = { + -0.00158678506084f, 0.00127157815343f, -0.00104805672709f, + 0.00084837248079f, 0.00134467983258f, -0.00107756549387f, + 0.00088814793277f, -0.00071893072525f}; + + // The matrix for transforming the backward composite state to lower channel + // state. + static const float WebRtcIsac_kTransform2Float[8] = { + -0.00170686041697f, 0.00136780109829f, -0.00112736532350f, + 0.00091257055385f, 0.00103094281812f, -0.00082615076557f, + 0.00068092756088f, -0.00055119165484f}; + + /* High pass filter */ + + for (k = 0; k < FRAMESAMPLES; k++) { + in[k] = pin[k] + kHpStCoefInFloat[2] * prefiltdata->HPstates_float[0] + + kHpStCoefInFloat[3] * prefiltdata->HPstates_float[1]; + ftmp = pin[k] - kHpStCoefInFloat[0] * prefiltdata->HPstates_float[0] - + kHpStCoefInFloat[1] * prefiltdata->HPstates_float[1]; + prefiltdata->HPstates_float[1] = prefiltdata->HPstates_float[0]; + prefiltdata->HPstates_float[0] = ftmp; + } + + /* First Channel */ + + /*initial state of composite filter is zero */ + for (k = 0; k < NUMBEROFCOMPOSITEAPSECTIONS; k++) { + CompositeAPFilterState[k] = 0.0; + } + /* put every other sample of input into a temporary vector in reverse + * (backward) order*/ + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + tempinoutvec[k] = in[FRAMESAMPLES - 1 - 2 * k]; + } + + /* now all-pass filter the backwards vector. Output values overwrite the + * input vector. */ + WebRtcIsac_AllPassFilter2Float( + tempinoutvec, WebRtcIsac_kCompositeApFactorsFloat, FRAMESAMPLES_HALF, + NUMBEROFCOMPOSITEAPSECTIONS, CompositeAPFilterState); + + /* save the backwards filtered output for later forward filtering, + but write it in forward order*/ + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + tempin_ch1[FRAMESAMPLES_HALF + QLOOKAHEAD - 1 - k] = tempinoutvec[k]; + } + + /* save the backwards filter state becaue it will be transformed + later into a forward state */ + for (k = 0; k < NUMBEROFCOMPOSITEAPSECTIONS; k++) { + ForTransform_CompositeAPFilterState[k] = CompositeAPFilterState[k]; + } + + /* now backwards filter the samples in the lookahead buffer. The samples were + placed there in the encoding of the previous frame. The output samples + overwrite the input samples */ + WebRtcIsac_AllPassFilter2Float( + prefiltdata->INLABUF1_float, WebRtcIsac_kCompositeApFactorsFloat, + QLOOKAHEAD, NUMBEROFCOMPOSITEAPSECTIONS, CompositeAPFilterState); + + /* save the output, but write it in forward order */ + /* write the lookahead samples for the next encoding iteration. Every other + sample at the end of the input frame is written in reverse order for the + lookahead length. Exported in the prefiltdata structure. */ + for (k = 0; k < QLOOKAHEAD; k++) { + tempin_ch1[QLOOKAHEAD - 1 - k] = prefiltdata->INLABUF1_float[k]; + prefiltdata->INLABUF1_float[k] = in[FRAMESAMPLES - 1 - 2 * k]; + } + + /* Second Channel. This is exactly like the first channel, except that the + even samples are now filtered instead (lower channel). */ + for (k = 0; k < NUMBEROFCOMPOSITEAPSECTIONS; k++) { + CompositeAPFilterState[k] = 0.0; + } + + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + tempinoutvec[k] = in[FRAMESAMPLES - 2 - 2 * k]; + } + + WebRtcIsac_AllPassFilter2Float( + tempinoutvec, WebRtcIsac_kCompositeApFactorsFloat, FRAMESAMPLES_HALF, + NUMBEROFCOMPOSITEAPSECTIONS, CompositeAPFilterState); + + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + tempin_ch2[FRAMESAMPLES_HALF + QLOOKAHEAD - 1 - k] = tempinoutvec[k]; + } + + for (k = 0; k < NUMBEROFCOMPOSITEAPSECTIONS; k++) { + ForTransform_CompositeAPFilterState2[k] = CompositeAPFilterState[k]; + } + + WebRtcIsac_AllPassFilter2Float( + prefiltdata->INLABUF2_float, WebRtcIsac_kCompositeApFactorsFloat, + QLOOKAHEAD, NUMBEROFCOMPOSITEAPSECTIONS, CompositeAPFilterState); + + for (k = 0; k < QLOOKAHEAD; k++) { + tempin_ch2[QLOOKAHEAD - 1 - k] = prefiltdata->INLABUF2_float[k]; + prefiltdata->INLABUF2_float[k] = in[FRAMESAMPLES - 2 - 2 * k]; + } + + /* Transform filter states from backward to forward */ + /*At this point, each of the states of the backwards composite filters for the + two channels are transformed into forward filtering states for the + corresponding forward channel filters. Each channel's forward filtering + state from the previous + encoding iteration is added to the transformed state to get a proper forward + state */ + + /* So the existing NUMBEROFCOMPOSITEAPSECTIONS x 1 (4x1) state vector is + multiplied by a NUMBEROFCHANNELAPSECTIONSxNUMBEROFCOMPOSITEAPSECTIONS (2x4) + transform matrix to get the new state that is added to the previous 2x1 + input state */ + + for (k = 0; k < NUMBEROFCHANNELAPSECTIONS; k++) { /* k is row variable */ + for (n = 0; n < NUMBEROFCOMPOSITEAPSECTIONS; + n++) { /* n is column variable */ + prefiltdata->INSTAT1_float[k] += + ForTransform_CompositeAPFilterState[n] * + WebRtcIsac_kTransform1Float[k * NUMBEROFCHANNELAPSECTIONS + n]; + prefiltdata->INSTAT2_float[k] += + ForTransform_CompositeAPFilterState2[n] * + WebRtcIsac_kTransform2Float[k * NUMBEROFCHANNELAPSECTIONS + n]; + } + } + + /*obtain polyphase components by forward all-pass filtering through each + * channel */ + /* the backward filtered samples are now forward filtered with the + * corresponding channel filters */ + /* The all pass filtering automatically updates the filter states which are + exported in the prefiltdata structure */ + WebRtcIsac_AllPassFilter2Float(tempin_ch1, WebRtcIsac_kUpperApFactorsFloat, + FRAMESAMPLES_HALF, NUMBEROFCHANNELAPSECTIONS, + prefiltdata->INSTAT1_float); + WebRtcIsac_AllPassFilter2Float(tempin_ch2, WebRtcIsac_kLowerApFactorsFloat, + FRAMESAMPLES_HALF, NUMBEROFCHANNELAPSECTIONS, + prefiltdata->INSTAT2_float); + + /* Now Construct low-pass and high-pass signals as combinations of polyphase + * components */ + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + LP[k] = 0.5f * (tempin_ch1[k] + tempin_ch2[k]); /* low pass signal*/ + HP[k] = 0.5f * (tempin_ch1[k] - tempin_ch2[k]); /* high pass signal*/ + } + + /* Lookahead LP and HP signals */ + /* now create low pass and high pass signals of the input vector. However, no + backwards filtering is performed, and hence no phase equalization is + involved. Also, the input contains some samples that are lookahead samples. + The high pass and low pass signals that are created are used outside this + function for analysis (not encoding) purposes */ + + /* set up input */ + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + tempin_ch1[k] = in[2 * k + 1]; + tempin_ch2[k] = in[2 * k]; + } + + /* the input filter states are passed in and updated by the all-pass filtering + routine and exported in the prefiltdata structure*/ + WebRtcIsac_AllPassFilter2Float(tempin_ch1, WebRtcIsac_kUpperApFactorsFloat, + FRAMESAMPLES_HALF, NUMBEROFCHANNELAPSECTIONS, + prefiltdata->INSTATLA1_float); + WebRtcIsac_AllPassFilter2Float(tempin_ch2, WebRtcIsac_kLowerApFactorsFloat, + FRAMESAMPLES_HALF, NUMBEROFCHANNELAPSECTIONS, + prefiltdata->INSTATLA2_float); + + for (k = 0; k < FRAMESAMPLES_HALF; k++) { + LP_la[k] = (float)(0.5f * (tempin_ch1[k] + tempin_ch2[k])); /*low pass */ + HP_la[k] = (double)(0.5f * (tempin_ch1[k] - tempin_ch2[k])); /* high pass */ + } +} diff --git a/VocieProcess/modules/audio_coding/codecs/isac/main/source/isac_vad.h b/VocieProcess/modules/audio_coding/codecs/isac/main/source/isac_vad.h new file mode 100644 index 0000000..1aecfc4 --- /dev/null +++ b/VocieProcess/modules/audio_coding/codecs/isac/main/source/isac_vad.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ISAC_VAD_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ISAC_VAD_H_ + +#include + +#include "modules/audio_coding/codecs/isac/main/source/structs.h" + +void WebRtcIsac_InitPitchFilter(PitchFiltstr* pitchfiltdata); +void WebRtcIsac_InitPitchAnalysis(PitchAnalysisStruct* state); +void WebRtcIsac_InitPreFilterbank(PreFiltBankstr* prefiltdata); + +double WebRtcIsac_LevDurb(double* a, double* k, double* r, size_t order); + +/* The number of all-pass filter factors in an upper or lower channel*/ +#define NUMBEROFCHANNELAPSECTIONS 2 + +/* The upper channel all-pass filter factors */ +extern const float WebRtcIsac_kUpperApFactorsFloat[2]; + +/* The lower channel all-pass filter factors */ +extern const float WebRtcIsac_kLowerApFactorsFloat[2]; + +void WebRtcIsac_AllPassFilter2Float(float* InOut, + const float* APSectionFactors, + int lengthInOut, + int NumberOfSections, + float* FilterState); +void WebRtcIsac_SplitAndFilterFloat(float* in, + float* LP, + float* HP, + double* LP_la, + double* HP_la, + PreFiltBankstr* prefiltdata); + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_ISAC_VAD_H_ diff --git a/VocieProcess/modules/audio_coding/codecs/isac/main/source/os_specific_inline.h b/VocieProcess/modules/audio_coding/codecs/isac/main/source/os_specific_inline.h new file mode 100644 index 0000000..fe9afa4 --- /dev/null +++ b/VocieProcess/modules/audio_coding/codecs/isac/main/source/os_specific_inline.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_OS_SPECIFIC_INLINE_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_OS_SPECIFIC_INLINE_H_ + +#include + +#include "rtc_base/system/arch.h" + +#if defined(WEBRTC_POSIX) +#define WebRtcIsac_lrint lrint +#elif (defined(WEBRTC_ARCH_X86) && defined(WIN32)) +static __inline long int WebRtcIsac_lrint(double x_dbl) { + long int x_int; + + __asm { + fld x_dbl + fistp x_int + } + ; + + return x_int; +} +#else // Do a slow but correct implementation of lrint + +static __inline long int WebRtcIsac_lrint(double x_dbl) { + long int x_int; + x_int = (long int)floor(x_dbl + 0.499999999999); + return x_int; +} + +#endif + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_OS_SPECIFIC_INLINE_H_ diff --git a/VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c b/VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c new file mode 100644 index 0000000..8a19ac1 --- /dev/null +++ b/VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_estimator.c @@ -0,0 +1,695 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/codecs/isac/main/source/pitch_estimator.h" + +#include +#include +#include +#ifdef WEBRTC_ANDROID +#include +#endif + +#include "modules/audio_coding/codecs/isac/main/source/filter_functions.h" +#include "modules/audio_coding/codecs/isac/main/source/pitch_filter.h" +#include "rtc_base/system/ignore_warnings.h" + +static const double kInterpolWin[8] = {-0.00067556028640, 0.02184247643159, -0.12203175715679, 0.60086484101160, + 0.60086484101160, -0.12203175715679, 0.02184247643159, -0.00067556028640}; + +/* interpolation filter */ +__inline static void IntrepolFilter(double *data_ptr, double *intrp) +{ + *intrp = kInterpolWin[0] * data_ptr[-3]; + *intrp += kInterpolWin[1] * data_ptr[-2]; + *intrp += kInterpolWin[2] * data_ptr[-1]; + *intrp += kInterpolWin[3] * data_ptr[0]; + *intrp += kInterpolWin[4] * data_ptr[1]; + *intrp += kInterpolWin[5] * data_ptr[2]; + *intrp += kInterpolWin[6] * data_ptr[3]; + *intrp += kInterpolWin[7] * data_ptr[4]; +} + + +/* 2D parabolic interpolation */ +/* probably some 0.5 factors can be eliminated, and the square-roots can be removed from the Cholesky fact. */ +__inline static void Intrpol2D(double T[3][3], double *x, double *y, double *peak_val) +{ + double c, b[2], A[2][2]; + double t1, t2, d; + double delta1, delta2; + + + // double T[3][3] = {{-1.25, -.25,-.25}, {-.25, .75, .75}, {-.25, .75, .75}}; + // should result in: delta1 = 0.5; delta2 = 0.0; peak_val = 1.0 + + c = T[1][1]; + b[0] = 0.5 * (T[1][2] + T[2][1] - T[0][1] - T[1][0]); + b[1] = 0.5 * (T[1][0] + T[2][1] - T[0][1] - T[1][2]); + A[0][1] = -0.5 * (T[0][1] + T[2][1] - T[1][0] - T[1][2]); + t1 = 0.5 * (T[0][0] + T[2][2]) - c; + t2 = 0.5 * (T[2][0] + T[0][2]) - c; + d = (T[0][1] + T[1][2] + T[1][0] + T[2][1]) - 4.0 * c - t1 - t2; + A[0][0] = -t1 - 0.5 * d; + A[1][1] = -t2 - 0.5 * d; + + /* deal with singularities or ill-conditioned cases */ + if ( (A[0][0] < 1e-7) || ((A[0][0] * A[1][1] - A[0][1] * A[0][1]) < 1e-7) ) { + *peak_val = T[1][1]; + return; + } + + /* Cholesky decomposition: replace A by upper-triangular factor */ + A[0][0] = sqrt(A[0][0]); + A[0][1] = A[0][1] / A[0][0]; + A[1][1] = sqrt(A[1][1] - A[0][1] * A[0][1]); + + /* compute [x; y] = -0.5 * inv(A) * b */ + t1 = b[0] / A[0][0]; + t2 = (b[1] - t1 * A[0][1]) / A[1][1]; + delta2 = t2 / A[1][1]; + delta1 = 0.5 * (t1 - delta2 * A[0][1]) / A[0][0]; + delta2 *= 0.5; + + /* limit norm */ + t1 = delta1 * delta1 + delta2 * delta2; + if (t1 > 1.0) { + delta1 /= t1; + delta2 /= t1; + } + + *peak_val = 0.5 * (b[0] * delta1 + b[1] * delta2) + c; + + *x += delta1; + *y += delta2; +} + + +static void PCorr(const double *in, double *outcorr) +{ + double sum, ysum, prod; + const double *x, *inptr; + int k, n; + + //ysum = 1e-6; /* use this with float (i.s.o. double)! */ + ysum = 1e-13; + sum = 0.0; + x = in + PITCH_MAX_LAG/2 + 2; + for (n = 0; n < PITCH_CORR_LEN2; n++) { + ysum += in[n] * in[n]; + sum += x[n] * in[n]; + } + + outcorr += PITCH_LAG_SPAN2 - 1; /* index of last element in array */ + *outcorr = sum / sqrt(ysum); + + for (k = 1; k < PITCH_LAG_SPAN2; k++) { + ysum -= in[k-1] * in[k-1]; + ysum += in[PITCH_CORR_LEN2 + k - 1] * in[PITCH_CORR_LEN2 + k - 1]; + sum = 0.0; + inptr = &in[k]; + prod = x[0] * inptr[0]; + for (n = 1; n < PITCH_CORR_LEN2; n++) { + sum += prod; + prod = x[n] * inptr[n]; + } + sum += prod; + outcorr--; + *outcorr = sum / sqrt(ysum); + } +} + +static void WebRtcIsac_AllpassFilterForDec(double* InOut, + const double* APSectionFactors, + size_t lengthInOut, + double* FilterState) { + // This performs all-pass filtering--a series of first order all-pass + // sections are used to filter the input in a cascade manner. + size_t n, j; + double temp; + for (j = 0; j < ALLPASSSECTIONS; j++) { + for (n = 0; n < lengthInOut; n += 2) { + temp = InOut[n]; // store input + InOut[n] = FilterState[j] + APSectionFactors[j] * temp; + FilterState[j] = -APSectionFactors[j] * InOut[n] + temp; + } + } +} + +static void WebRtcIsac_DecimateAllpass( + const double* in, + double* state_in, // array of size: 2*ALLPASSSECTIONS+1 + size_t N, // number of input samples + double* out) { // array of size N/2 + + static const double APupper[ALLPASSSECTIONS] = {0.0347, 0.3826}; + static const double APlower[ALLPASSSECTIONS] = {0.1544, 0.744}; + + size_t n; + double data_vec[PITCH_FRAME_LEN]; + + /* copy input */ + memcpy(data_vec + 1, in, sizeof(double) * (N - 1)); + + data_vec[0] = state_in[2 * ALLPASSSECTIONS]; // the z^(-1) state + state_in[2 * ALLPASSSECTIONS] = in[N - 1]; + + WebRtcIsac_AllpassFilterForDec(data_vec + 1, APupper, N, state_in); + WebRtcIsac_AllpassFilterForDec(data_vec, APlower, N, + state_in + ALLPASSSECTIONS); + + for (n = 0; n < N / 2; n++) + out[n] = data_vec[2 * n] + data_vec[2 * n + 1]; +} + +RTC_PUSH_IGNORING_WFRAME_LARGER_THAN() + +static void WebRtcIsac_InitializePitch(const double* in, + const double old_lag, + const double old_gain, + PitchAnalysisStruct* State, + double* lags) { + double buf_dec[PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2+2]; + double ratio, log_lag, gain_bias; + double bias; + double corrvec1[PITCH_LAG_SPAN2]; + double corrvec2[PITCH_LAG_SPAN2]; + int m, k; + // Allocating 10 extra entries at the begining of the CorrSurf + double corrSurfBuff[10 + (2*PITCH_BW+3)*(PITCH_LAG_SPAN2+4)]; + double* CorrSurf[2*PITCH_BW+3]; + double *CorrSurfPtr1, *CorrSurfPtr2; + double LagWin[3] = {0.2, 0.5, 0.98}; + int ind1, ind2, peaks_ind, peak, max_ind; + int peaks[PITCH_MAX_NUM_PEAKS]; + double adj, gain_tmp; + double corr, corr_max; + double intrp_a, intrp_b, intrp_c, intrp_d; + double peak_vals[PITCH_MAX_NUM_PEAKS]; + double lags1[PITCH_MAX_NUM_PEAKS]; + double lags2[PITCH_MAX_NUM_PEAKS]; + double T[3][3]; + int row; + + for(k = 0; k < 2*PITCH_BW+3; k++) + { + CorrSurf[k] = &corrSurfBuff[10 + k * (PITCH_LAG_SPAN2+4)]; + } + /* reset CorrSurf matrix */ + memset(corrSurfBuff, 0, sizeof(double) * (10 + (2*PITCH_BW+3) * (PITCH_LAG_SPAN2+4))); + + //warnings -DH + max_ind = 0; + peak = 0; + + /* copy old values from state buffer */ + memcpy(buf_dec, State->dec_buffer, sizeof(double) * (PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2)); + + /* decimation; put result after the old values */ + WebRtcIsac_DecimateAllpass(in, State->decimator_state, PITCH_FRAME_LEN, + &buf_dec[PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2]); + + /* low-pass filtering */ + for (k = PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2; k < PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2+2; k++) + buf_dec[k] += 0.75 * buf_dec[k-1] - 0.25 * buf_dec[k-2]; + + /* copy end part back into state buffer */ + memcpy(State->dec_buffer, buf_dec+PITCH_FRAME_LEN/2, sizeof(double) * (PITCH_CORR_LEN2+PITCH_CORR_STEP2+PITCH_MAX_LAG/2-PITCH_FRAME_LEN/2+2)); + + /* compute correlation for first and second half of the frame */ + PCorr(buf_dec, corrvec1); + PCorr(buf_dec + PITCH_CORR_STEP2, corrvec2); + + /* bias towards pitch lag of previous frame */ + log_lag = log(0.5 * old_lag); + gain_bias = 4.0 * old_gain * old_gain; + if (gain_bias > 0.8) gain_bias = 0.8; + for (k = 0; k < PITCH_LAG_SPAN2; k++) + { + ratio = log((double) (k + (PITCH_MIN_LAG/2-2))) - log_lag; + bias = 1.0 + gain_bias * exp(-5.0 * ratio * ratio); + corrvec1[k] *= bias; + } + + /* taper correlation functions */ + for (k = 0; k < 3; k++) { + gain_tmp = LagWin[k]; + corrvec1[k] *= gain_tmp; + corrvec2[k] *= gain_tmp; + corrvec1[PITCH_LAG_SPAN2-1-k] *= gain_tmp; + corrvec2[PITCH_LAG_SPAN2-1-k] *= gain_tmp; + } + + corr_max = 0.0; + /* fill middle row of correlation surface */ + ind1 = 0; + ind2 = 0; + CorrSurfPtr1 = &CorrSurf[PITCH_BW][2]; + for (k = 0; k < PITCH_LAG_SPAN2; k++) { + corr = corrvec1[ind1++] + corrvec2[ind2++]; + CorrSurfPtr1[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + } + } + /* fill first and last rows of correlation surface */ + ind1 = 0; + ind2 = PITCH_BW; + CorrSurfPtr1 = &CorrSurf[0][2]; + CorrSurfPtr2 = &CorrSurf[2*PITCH_BW][PITCH_BW+2]; + for (k = 0; k < PITCH_LAG_SPAN2-PITCH_BW; k++) { + ratio = ((double) (ind1 + 12)) / ((double) (ind2 + 12)); + adj = 0.2 * ratio * (2.0 - ratio); /* adjustment factor; inverse parabola as a function of ratio */ + corr = adj * (corrvec1[ind1] + corrvec2[ind2]); + CorrSurfPtr1[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + } + corr = adj * (corrvec1[ind2++] + corrvec2[ind1++]); + CorrSurfPtr2[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr2[k] - &CorrSurf[0][0]); + } + } + /* fill second and next to last rows of correlation surface */ + ind1 = 0; + ind2 = PITCH_BW-1; + CorrSurfPtr1 = &CorrSurf[1][2]; + CorrSurfPtr2 = &CorrSurf[2*PITCH_BW-1][PITCH_BW+1]; + for (k = 0; k < PITCH_LAG_SPAN2-PITCH_BW+1; k++) { + ratio = ((double) (ind1 + 12)) / ((double) (ind2 + 12)); + adj = 0.9 * ratio * (2.0 - ratio); /* adjustment factor; inverse parabola as a function of ratio */ + corr = adj * (corrvec1[ind1] + corrvec2[ind2]); + CorrSurfPtr1[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + } + corr = adj * (corrvec1[ind2++] + corrvec2[ind1++]); + CorrSurfPtr2[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr2[k] - &CorrSurf[0][0]); + } + } + /* fill remainder of correlation surface */ + for (m = 2; m < PITCH_BW; m++) { + ind1 = 0; + ind2 = PITCH_BW - m; /* always larger than ind1 */ + CorrSurfPtr1 = &CorrSurf[m][2]; + CorrSurfPtr2 = &CorrSurf[2*PITCH_BW-m][PITCH_BW+2-m]; + for (k = 0; k < PITCH_LAG_SPAN2-PITCH_BW+m; k++) { + ratio = ((double) (ind1 + 12)) / ((double) (ind2 + 12)); + adj = ratio * (2.0 - ratio); /* adjustment factor; inverse parabola as a function of ratio */ + corr = adj * (corrvec1[ind1] + corrvec2[ind2]); + CorrSurfPtr1[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + } + corr = adj * (corrvec1[ind2++] + corrvec2[ind1++]); + CorrSurfPtr2[k] = corr; + if (corr > corr_max) { + corr_max = corr; /* update maximum */ + max_ind = (int)(&CorrSurfPtr2[k] - &CorrSurf[0][0]); + } + } + } + + /* threshold value to qualify as a peak */ + corr_max *= 0.6; + + peaks_ind = 0; + /* find peaks */ + for (m = 1; m < PITCH_BW+1; m++) { + if (peaks_ind == PITCH_MAX_NUM_PEAKS) break; + CorrSurfPtr1 = &CorrSurf[m][2]; + for (k = 2; k < PITCH_LAG_SPAN2-PITCH_BW-2+m; k++) { + corr = CorrSurfPtr1[k]; + if (corr > corr_max) { + if ( (corr > CorrSurfPtr1[k - (PITCH_LAG_SPAN2+5)]) && (corr > CorrSurfPtr1[k - (PITCH_LAG_SPAN2+4)]) ) { + if ( (corr > CorrSurfPtr1[k + (PITCH_LAG_SPAN2+4)]) && (corr > CorrSurfPtr1[k + (PITCH_LAG_SPAN2+5)]) ) { + /* found a peak; store index into matrix */ + peaks[peaks_ind++] = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + if (peaks_ind == PITCH_MAX_NUM_PEAKS) break; + } + } + } + } + } + for (m = PITCH_BW+1; m < 2*PITCH_BW; m++) { + if (peaks_ind == PITCH_MAX_NUM_PEAKS) break; + CorrSurfPtr1 = &CorrSurf[m][2]; + for (k = 2+m-PITCH_BW; k < PITCH_LAG_SPAN2-2; k++) { + corr = CorrSurfPtr1[k]; + if (corr > corr_max) { + if ( (corr > CorrSurfPtr1[k - (PITCH_LAG_SPAN2+5)]) && (corr > CorrSurfPtr1[k - (PITCH_LAG_SPAN2+4)]) ) { + if ( (corr > CorrSurfPtr1[k + (PITCH_LAG_SPAN2+4)]) && (corr > CorrSurfPtr1[k + (PITCH_LAG_SPAN2+5)]) ) { + /* found a peak; store index into matrix */ + peaks[peaks_ind++] = (int)(&CorrSurfPtr1[k] - &CorrSurf[0][0]); + if (peaks_ind == PITCH_MAX_NUM_PEAKS) break; + } + } + } + } + } + + if (peaks_ind > 0) { + /* examine each peak */ + CorrSurfPtr1 = &CorrSurf[0][0]; + for (k = 0; k < peaks_ind; k++) { + peak = peaks[k]; + + /* compute four interpolated values around current peak */ + IntrepolFilter(&CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+5)], &intrp_a); + IntrepolFilter(&CorrSurfPtr1[peak - 1 ], &intrp_b); + IntrepolFilter(&CorrSurfPtr1[peak ], &intrp_c); + IntrepolFilter(&CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+4)], &intrp_d); + + /* determine maximum of the interpolated values */ + corr = CorrSurfPtr1[peak]; + corr_max = intrp_a; + if (intrp_b > corr_max) corr_max = intrp_b; + if (intrp_c > corr_max) corr_max = intrp_c; + if (intrp_d > corr_max) corr_max = intrp_d; + + /* determine where the peak sits and fill a 3x3 matrix around it */ + row = peak / (PITCH_LAG_SPAN2+4); + lags1[k] = (double) ((peak - row * (PITCH_LAG_SPAN2+4)) + PITCH_MIN_LAG/2 - 4); + lags2[k] = (double) (lags1[k] + PITCH_BW - row); + if ( corr > corr_max ) { + T[0][0] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+5)]; + T[2][0] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+4)]; + T[1][1] = corr; + T[0][2] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+4)]; + T[2][2] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+5)]; + T[1][0] = intrp_a; + T[0][1] = intrp_b; + T[2][1] = intrp_c; + T[1][2] = intrp_d; + } else { + if (intrp_a == corr_max) { + lags1[k] -= 0.5; + lags2[k] += 0.5; + IntrepolFilter(&CorrSurfPtr1[peak - 2*(PITCH_LAG_SPAN2+5)], &T[0][0]); + IntrepolFilter(&CorrSurfPtr1[peak - (2*PITCH_LAG_SPAN2+9)], &T[2][0]); + T[1][1] = intrp_a; + T[0][2] = intrp_b; + T[2][2] = intrp_c; + T[1][0] = CorrSurfPtr1[peak - (2*PITCH_LAG_SPAN2+9)]; + T[0][1] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+5)]; + T[2][1] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+4)]; + T[1][2] = corr; + } else if (intrp_b == corr_max) { + lags1[k] -= 0.5; + lags2[k] -= 0.5; + IntrepolFilter(&CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+6)], &T[0][0]); + T[2][0] = intrp_a; + T[1][1] = intrp_b; + IntrepolFilter(&CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+3)], &T[0][2]); + T[2][2] = intrp_d; + T[1][0] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+5)]; + T[0][1] = CorrSurfPtr1[peak - 1]; + T[2][1] = corr; + T[1][2] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+4)]; + } else if (intrp_c == corr_max) { + lags1[k] += 0.5; + lags2[k] += 0.5; + T[0][0] = intrp_a; + IntrepolFilter(&CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+4)], &T[2][0]); + T[1][1] = intrp_c; + T[0][2] = intrp_d; + IntrepolFilter(&CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+5)], &T[2][2]); + T[1][0] = CorrSurfPtr1[peak - (PITCH_LAG_SPAN2+4)]; + T[0][1] = corr; + T[2][1] = CorrSurfPtr1[peak + 1]; + T[1][2] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+5)]; + } else { + lags1[k] += 0.5; + lags2[k] -= 0.5; + T[0][0] = intrp_b; + T[2][0] = intrp_c; + T[1][1] = intrp_d; + IntrepolFilter(&CorrSurfPtr1[peak + 2*(PITCH_LAG_SPAN2+4)], &T[0][2]); + IntrepolFilter(&CorrSurfPtr1[peak + (2*PITCH_LAG_SPAN2+9)], &T[2][2]); + T[1][0] = corr; + T[0][1] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+4)]; + T[2][1] = CorrSurfPtr1[peak + (PITCH_LAG_SPAN2+5)]; + T[1][2] = CorrSurfPtr1[peak + (2*PITCH_LAG_SPAN2+9)]; + } + } + + /* 2D parabolic interpolation gives more accurate lags and peak value */ + Intrpol2D(T, &lags1[k], &lags2[k], &peak_vals[k]); + } + + /* determine the highest peak, after applying a bias towards short lags */ + corr_max = 0.0; + for (k = 0; k < peaks_ind; k++) { + corr = peak_vals[k] * pow(PITCH_PEAK_DECAY, log(lags1[k] + lags2[k])); + if (corr > corr_max) { + corr_max = corr; + peak = k; + } + } + + lags1[peak] *= 2.0; + lags2[peak] *= 2.0; + + if (lags1[peak] < (double) PITCH_MIN_LAG) lags1[peak] = (double) PITCH_MIN_LAG; + if (lags2[peak] < (double) PITCH_MIN_LAG) lags2[peak] = (double) PITCH_MIN_LAG; + if (lags1[peak] > (double) PITCH_MAX_LAG) lags1[peak] = (double) PITCH_MAX_LAG; + if (lags2[peak] > (double) PITCH_MAX_LAG) lags2[peak] = (double) PITCH_MAX_LAG; + + /* store lags of highest peak in output array */ + lags[0] = lags1[peak]; + lags[1] = lags1[peak]; + lags[2] = lags2[peak]; + lags[3] = lags2[peak]; + } + else + { + row = max_ind / (PITCH_LAG_SPAN2+4); + lags1[0] = (double) ((max_ind - row * (PITCH_LAG_SPAN2+4)) + PITCH_MIN_LAG/2 - 4); + lags2[0] = (double) (lags1[0] + PITCH_BW - row); + + if (lags1[0] < (double) PITCH_MIN_LAG) lags1[0] = (double) PITCH_MIN_LAG; + if (lags2[0] < (double) PITCH_MIN_LAG) lags2[0] = (double) PITCH_MIN_LAG; + if (lags1[0] > (double) PITCH_MAX_LAG) lags1[0] = (double) PITCH_MAX_LAG; + if (lags2[0] > (double) PITCH_MAX_LAG) lags2[0] = (double) PITCH_MAX_LAG; + + /* store lags of highest peak in output array */ + lags[0] = lags1[0]; + lags[1] = lags1[0]; + lags[2] = lags2[0]; + lags[3] = lags2[0]; + } +} + +RTC_POP_IGNORING_WFRAME_LARGER_THAN() + +/* create weighting matrix by orthogonalizing a basis of polynomials of increasing order + * t = (0:4)'; + * A = [t.^0, t.^1, t.^2, t.^3, t.^4]; + * [Q, dummy] = qr(A); + * P.Weight = Q * diag([0, .1, .5, 1, 1]) * Q'; */ +static const double kWeight[5][5] = { + { 0.29714285714286, -0.30857142857143, -0.05714285714286, 0.05142857142857, 0.01714285714286}, + {-0.30857142857143, 0.67428571428571, -0.27142857142857, -0.14571428571429, 0.05142857142857}, + {-0.05714285714286, -0.27142857142857, 0.65714285714286, -0.27142857142857, -0.05714285714286}, + { 0.05142857142857, -0.14571428571429, -0.27142857142857, 0.67428571428571, -0.30857142857143}, + { 0.01714285714286, 0.05142857142857, -0.05714285714286, -0.30857142857143, 0.29714285714286} +}; + +/* second order high-pass filter */ +static void WebRtcIsac_Highpass(const double* in, + double* out, + double* state, + size_t N) { + /* create high-pass filter ocefficients + * z = 0.998 * exp(j*2*pi*35/8000); + * p = 0.94 * exp(j*2*pi*140/8000); + * HP_b = [1, -2*real(z), abs(z)^2]; + * HP_a = [1, -2*real(p), abs(p)^2]; */ + static const double a_coef[2] = { 1.86864659625574, -0.88360000000000}; + static const double b_coef[2] = {-1.99524591718270, 0.99600400000000}; + + size_t k; + + for (k=0; khp_state, PITCH_FRAME_LEN); + + /* copy from state into buffer */ + memcpy(Whitened, State->whitened_buf, sizeof(double) * QLOOKAHEAD); + + /* compute weighted and whitened signals */ + WebRtcIsac_WeightingFilter(HPin, &Weighted[0], &Whitened[QLOOKAHEAD], &(State->Wghtstr)); + + /* copy from buffer into state */ + memcpy(State->whitened_buf, Whitened+PITCH_FRAME_LEN, sizeof(double) * QLOOKAHEAD); + + old_lag = State->PFstr_wght.oldlagp[0]; + old_gain = State->PFstr_wght.oldgainp[0]; + + /* inital pitch estimate */ + WebRtcIsac_InitializePitch(Weighted, old_lag, old_gain, State, lags); + + + /* Iterative optimization of lags - to be done */ + + /* compute energy of whitened signal */ + nrg_wht = 0.0; + for (k = 0; k < PITCH_FRAME_LEN + QLOOKAHEAD; k++) + nrg_wht += Whitened[k] * Whitened[k]; + + + /* Iterative optimization of gains */ + + /* set weights for energy, gain fluctiation, and spectral gain penalty functions */ + Wnrg = 1.0 / nrg_wht; + Wgain = 0.005; + Wfluct = 3.0; + + /* set initial gains */ + for (k = 0; k < 4; k++) + gains[k] = PITCH_MAX_GAIN_06; + + /* two iterations should be enough */ + for (iter = 0; iter < 2; iter++) { + /* compute Jacobian of pre-filter output towards gains */ + WebRtcIsac_PitchfilterPre_gains(Whitened, out_G, out_dG, &(State->PFstr_wght), lags, gains); + + /* gradient and approximate Hessian (lower triangle) for minimizing the filter's output power */ + for (k = 0; k < 4; k++) { + tmp = 0.0; + for (n = 0; n < PITCH_FRAME_LEN + QLOOKAHEAD; n++) + tmp += out_G[n] * out_dG[k][n]; + grad[k] = tmp * Wnrg; + } + for (k = 0; k < 4; k++) { + for (m = 0; m <= k; m++) { + tmp = 0.0; + for (n = 0; n < PITCH_FRAME_LEN + QLOOKAHEAD; n++) + tmp += out_dG[m][n] * out_dG[k][n]; + H[k][m] = tmp * Wnrg; + } + } + + /* add gradient and Hessian (lower triangle) for dampening fast gain changes */ + for (k = 0; k < 4; k++) { + tmp = kWeight[k+1][0] * old_gain; + for (m = 0; m < 4; m++) + tmp += kWeight[k+1][m+1] * gains[m]; + grad[k] += tmp * Wfluct; + } + for (k = 0; k < 4; k++) { + for (m = 0; m <= k; m++) { + H[k][m] += kWeight[k+1][m+1] * Wfluct; + } + } + + /* add gradient and Hessian for dampening gain */ + for (k = 0; k < 3; k++) { + tmp = 1.0 / (1 - gains[k]); + grad[k] += tmp * tmp * Wgain; + H[k][k] += 2.0 * tmp * (tmp * tmp * Wgain); + } + tmp = 1.0 / (1 - gains[3]); + grad[3] += 1.33 * (tmp * tmp * Wgain); + H[3][3] += 2.66 * tmp * (tmp * tmp * Wgain); + + + /* compute Cholesky factorization of Hessian + * by overwritting the upper triangle; scale factors on diagonal + * (for non pc-platforms store the inverse of the diagonals seperately to minimize divisions) */ + H[0][1] = H[1][0] / H[0][0]; + H[0][2] = H[2][0] / H[0][0]; + H[0][3] = H[3][0] / H[0][0]; + H[1][1] -= H[0][0] * H[0][1] * H[0][1]; + H[1][2] = (H[2][1] - H[0][1] * H[2][0]) / H[1][1]; + H[1][3] = (H[3][1] - H[0][1] * H[3][0]) / H[1][1]; + H[2][2] -= H[0][0] * H[0][2] * H[0][2] + H[1][1] * H[1][2] * H[1][2]; + H[2][3] = (H[3][2] - H[0][2] * H[3][0] - H[1][2] * H[1][1] * H[1][3]) / H[2][2]; + H[3][3] -= H[0][0] * H[0][3] * H[0][3] + H[1][1] * H[1][3] * H[1][3] + H[2][2] * H[2][3] * H[2][3]; + + /* Compute update as delta_gains = -inv(H) * grad */ + /* copy and negate */ + for (k = 0; k < 4; k++) + dG[k] = -grad[k]; + /* back substitution */ + dG[1] -= dG[0] * H[0][1]; + dG[2] -= dG[0] * H[0][2] + dG[1] * H[1][2]; + dG[3] -= dG[0] * H[0][3] + dG[1] * H[1][3] + dG[2] * H[2][3]; + /* scale */ + for (k = 0; k < 4; k++) + dG[k] /= H[k][k]; + /* back substitution */ + dG[2] -= dG[3] * H[2][3]; + dG[1] -= dG[3] * H[1][3] + dG[2] * H[1][2]; + dG[0] -= dG[3] * H[0][3] + dG[2] * H[0][2] + dG[1] * H[0][1]; + + /* update gains and check range */ + for (k = 0; k < 4; k++) { + gains[k] += dG[k]; + if (gains[k] > PITCH_MAX_GAIN) + gains[k] = PITCH_MAX_GAIN; + else if (gains[k] < 0.0) + gains[k] = 0.0; + } + } + + /* update state for next frame */ + WebRtcIsac_PitchfilterPre(Whitened, out, &(State->PFstr_wght), lags, gains); + + /* concatenate previous input's end and current input */ + memcpy(inbuf, State->inbuf, sizeof(double) * QLOOKAHEAD); + memcpy(inbuf+QLOOKAHEAD, in, sizeof(double) * PITCH_FRAME_LEN); + + /* lookahead pitch filtering for masking analysis */ + WebRtcIsac_PitchfilterPre_la(inbuf, out, &(State->PFstr), lags, gains); + + /* store last part of input */ + for (k = 0; k < QLOOKAHEAD; k++) + State->inbuf[k] = inbuf[k + PITCH_FRAME_LEN]; +} + +RTC_POP_IGNORING_WFRAME_LARGER_THAN() diff --git a/VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_estimator.h b/VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_estimator.h new file mode 100644 index 0000000..4ab78c2 --- /dev/null +++ b/VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_estimator.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * pitch_estimator.h + * + * Pitch functions + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_ESTIMATOR_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_ESTIMATOR_H_ + +#include + +#include "modules/audio_coding/codecs/isac/main/source/structs.h" + +void WebRtcIsac_PitchAnalysis( + const double* in, /* PITCH_FRAME_LEN samples */ + double* out, /* PITCH_FRAME_LEN+QLOOKAHEAD samples */ + PitchAnalysisStruct* State, + double* lags, + double* gains); + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_ESTIMATOR_H_ */ diff --git a/VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_filter.c b/VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_filter.c new file mode 100644 index 0000000..bf03dff --- /dev/null +++ b/VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_filter.c @@ -0,0 +1,388 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include "modules/audio_coding/codecs/isac/main/source/pitch_estimator.h" +#include "modules/audio_coding/codecs/isac/main/source/os_specific_inline.h" +#include "rtc_base/compile_assert_c.h" + +/* + * We are implementing the following filters; + * + * Pre-filtering: + * y(z) = x(z) + damper(z) * gain * (x(z) + y(z)) * z ^ (-lag); + * + * Post-filtering: + * y(z) = x(z) - damper(z) * gain * (x(z) + y(z)) * z ^ (-lag); + * + * Note that `lag` is a floating number so we perform an interpolation to + * obtain the correct `lag`. + * + */ + +static const double kDampFilter[PITCH_DAMPORDER] = {-0.07, 0.25, 0.64, 0.25, + -0.07}; + +/* interpolation coefficients; generated by design_pitch_filter.m */ +static const double kIntrpCoef[PITCH_FRACS][PITCH_FRACORDER] = { + {-0.02239172458614, 0.06653315052934, -0.16515880017569, 0.60701333734125, + 0.64671399919202, -0.20249000396417, 0.09926548334755, -0.04765933793109, + 0.01754159521746}, + {-0.01985640750434, 0.05816126837866, -0.13991265473714, 0.44560418147643, + 0.79117042386876, -0.20266133815188, 0.09585268418555, -0.04533310458084, + 0.01654127246314}, + {-0.01463300534216, 0.04229888475060, -0.09897034715253, 0.28284326017787, + 0.90385267956632, -0.16976950138649, 0.07704272393639, -0.03584218578311, + 0.01295781500709}, + {-0.00764851320885, 0.02184035544377, -0.04985561057281, 0.13083306574393, + 0.97545011664662, -0.10177807997561, 0.04400901776474, -0.02010737175166, + 0.00719783432422}, + {-0.00000000000000, 0.00000000000000, -0.00000000000001, 0.00000000000001, + 0.99999999999999, 0.00000000000001, -0.00000000000001, 0.00000000000000, + -0.00000000000000}, + {0.00719783432422, -0.02010737175166, 0.04400901776474, -0.10177807997562, + 0.97545011664663, 0.13083306574393, -0.04985561057280, 0.02184035544377, + -0.00764851320885}, + {0.01295781500710, -0.03584218578312, 0.07704272393640, -0.16976950138650, + 0.90385267956634, 0.28284326017785, -0.09897034715252, 0.04229888475059, + -0.01463300534216}, + {0.01654127246315, -0.04533310458085, 0.09585268418557, -0.20266133815190, + 0.79117042386878, 0.44560418147640, -0.13991265473712, 0.05816126837865, + -0.01985640750433} +}; + +/* + * Enumerating the operation of the filter. + * iSAC has 4 different pitch-filter which are very similar in their structure. + * + * kPitchFilterPre : In this mode the filter is operating as pitch + * pre-filter. This is used at the encoder. + * kPitchFilterPost : In this mode the filter is operating as pitch + * post-filter. This is the inverse of pre-filter and used + * in the decoder. + * kPitchFilterPreLa : This is, in structure, similar to pre-filtering but + * utilizing 3 millisecond lookahead. It is used to + * obtain the signal for LPC analysis. + * kPitchFilterPreGain : This is, in structure, similar to pre-filtering but + * differential changes in gain is considered. This is + * used to find the optimal gain. + */ +typedef enum { + kPitchFilterPre, kPitchFilterPost, kPitchFilterPreLa, kPitchFilterPreGain +} PitchFilterOperation; + +/* + * Structure with parameters used for pitch-filtering. + * buffer : a buffer where the sum of previous inputs and outputs + * are stored. + * damper_state : the state of the damping filter. The filter is defined by + * `kDampFilter`. + * interpol_coeff : pointer to a set of coefficient which are used to utilize + * fractional pitch by interpolation. + * gain : pitch-gain to be applied to the current segment of input. + * lag : pitch-lag for the current segment of input. + * lag_offset : the offset of lag w.r.t. current sample. + * sub_frame : sub-frame index, there are 4 pitch sub-frames in an iSAC + * frame. + * This specifies the usage of the filter. See + * 'PitchFilterOperation' for operational modes. + * num_samples : number of samples to be processed in each segment. + * index : index of the input and output sample. + * damper_state_dg : state of damping filter for different trial gains. + * gain_mult : differential changes to gain. + */ +typedef struct { + double buffer[PITCH_INTBUFFSIZE + QLOOKAHEAD]; + double damper_state[PITCH_DAMPORDER]; + const double *interpol_coeff; + double gain; + double lag; + int lag_offset; + + int sub_frame; + PitchFilterOperation mode; + int num_samples; + int index; + + double damper_state_dg[4][PITCH_DAMPORDER]; + double gain_mult[4]; +} PitchFilterParam; + +/********************************************************************** + * FilterSegment() + * Filter one segment, a quarter of a frame. + * + * Inputs + * in_data : pointer to the input signal of 30 ms at 8 kHz sample-rate. + * filter_param : pitch filter parameters. + * + * Outputs + * out_data : pointer to a buffer where the filtered signal is written to. + * out_dg : [only used in kPitchFilterPreGain] pointer to a buffer + * where the output of different gain values (differential + * change to gain) is written. + */ +static void FilterSegment(const double* in_data, PitchFilterParam* parameters, + double* out_data, + double out_dg[][PITCH_FRAME_LEN + QLOOKAHEAD]) { + int n; + int m; + int j; + double sum; + double sum2; + /* Index of `parameters->buffer` where the output is written to. */ + int pos = parameters->index + PITCH_BUFFSIZE; + /* Index of `parameters->buffer` where samples are read for fractional-lag + * computation. */ + int pos_lag = pos - parameters->lag_offset; + + for (n = 0; n < parameters->num_samples; ++n) { + /* Shift low pass filter states. */ + for (m = PITCH_DAMPORDER - 1; m > 0; --m) { + parameters->damper_state[m] = parameters->damper_state[m - 1]; + } + /* Filter to get fractional pitch. */ + sum = 0.0; + for (m = 0; m < PITCH_FRACORDER; ++m) { + sum += parameters->buffer[pos_lag + m] * parameters->interpol_coeff[m]; + } + /* Multiply with gain. */ + parameters->damper_state[0] = parameters->gain * sum; + + if (parameters->mode == kPitchFilterPreGain) { + int lag_index = parameters->index - parameters->lag_offset; + int m_tmp = (lag_index < 0) ? -lag_index : 0; + /* Update the damper state for the new sample. */ + for (m = PITCH_DAMPORDER - 1; m > 0; --m) { + for (j = 0; j < 4; ++j) { + parameters->damper_state_dg[j][m] = + parameters->damper_state_dg[j][m - 1]; + } + } + + for (j = 0; j < parameters->sub_frame + 1; ++j) { + /* Filter for fractional pitch. */ + sum2 = 0.0; + for (m = PITCH_FRACORDER-1; m >= m_tmp; --m) { + /* `lag_index + m` is always larger than or equal to zero, see how + * m_tmp is computed. This is equivalent to assume samples outside + * `out_dg[j]` are zero. */ + sum2 += out_dg[j][lag_index + m] * parameters->interpol_coeff[m]; + } + /* Add the contribution of differential gain change. */ + parameters->damper_state_dg[j][0] = parameters->gain_mult[j] * sum + + parameters->gain * sum2; + } + + /* Filter with damping filter, and store the results. */ + for (j = 0; j < parameters->sub_frame + 1; ++j) { + sum = 0.0; + for (m = 0; m < PITCH_DAMPORDER; ++m) { + sum -= parameters->damper_state_dg[j][m] * kDampFilter[m]; + } + out_dg[j][parameters->index] = sum; + } + } + /* Filter with damping filter. */ + sum = 0.0; + for (m = 0; m < PITCH_DAMPORDER; ++m) { + sum += parameters->damper_state[m] * kDampFilter[m]; + } + + /* Subtract from input and update buffer. */ + out_data[parameters->index] = in_data[parameters->index] - sum; + parameters->buffer[pos] = in_data[parameters->index] + + out_data[parameters->index]; + + ++parameters->index; + ++pos; + ++pos_lag; + } + return; +} + +/* Update filter parameters based on the pitch-gains and pitch-lags. */ +static void Update(PitchFilterParam* parameters) { + double fraction; + int fraction_index; + /* Compute integer lag-offset. */ + parameters->lag_offset = WebRtcIsac_lrint(parameters->lag + PITCH_FILTDELAY + + 0.5); + /* Find correct set of coefficients for computing fractional pitch. */ + fraction = parameters->lag_offset - (parameters->lag + PITCH_FILTDELAY); + fraction_index = WebRtcIsac_lrint(PITCH_FRACS * fraction - 0.5); + parameters->interpol_coeff = kIntrpCoef[fraction_index]; + + if (parameters->mode == kPitchFilterPreGain) { + /* If in this mode make a differential change to pitch gain. */ + parameters->gain_mult[parameters->sub_frame] += 0.2; + if (parameters->gain_mult[parameters->sub_frame] > 1.0) { + parameters->gain_mult[parameters->sub_frame] = 1.0; + } + if (parameters->sub_frame > 0) { + parameters->gain_mult[parameters->sub_frame - 1] -= 0.2; + } + } +} + +/****************************************************************************** + * FilterFrame() + * Filter a frame of 30 millisecond, given pitch-lags and pitch-gains. + * + * Inputs + * in_data : pointer to the input signal of 30 ms at 8 kHz sample-rate. + * lags : pointer to pitch-lags, 4 lags per frame. + * gains : pointer to pitch-gians, 4 gains per frame. + * mode : defining the functionality of the filter. It takes the + * following values. + * kPitchFilterPre: Pitch pre-filter, used at encoder. + * kPitchFilterPost: Pitch post-filter, used at decoder. + * kPitchFilterPreLa: Pitch pre-filter with lookahead. + * kPitchFilterPreGain: Pitch pre-filter used to otain optimal + * pitch-gains. + * + * Outputs + * out_data : pointer to a buffer where the filtered signal is written to. + * out_dg : [only used in kPitchFilterPreGain] pointer to a buffer + * where the output of different gain values (differential + * change to gain) is written. + */ +static void FilterFrame(const double* in_data, PitchFiltstr* filter_state, + double* lags, double* gains, PitchFilterOperation mode, + double* out_data, + double out_dg[][PITCH_FRAME_LEN + QLOOKAHEAD]) { + PitchFilterParam filter_parameters; + double gain_delta, lag_delta; + double old_lag, old_gain; + int n; + int m; + const double kEnhancer = 1.3; + + /* Set up buffer and states. */ + filter_parameters.index = 0; + filter_parameters.lag_offset = 0; + filter_parameters.mode = mode; + /* Copy states to local variables. */ + memcpy(filter_parameters.buffer, filter_state->ubuf, + sizeof(filter_state->ubuf)); + RTC_COMPILE_ASSERT(sizeof(filter_parameters.buffer) >= + sizeof(filter_state->ubuf)); + memset(filter_parameters.buffer + + sizeof(filter_state->ubuf) / sizeof(filter_state->ubuf[0]), + 0, sizeof(filter_parameters.buffer) - sizeof(filter_state->ubuf)); + memcpy(filter_parameters.damper_state, filter_state->ystate, + sizeof(filter_state->ystate)); + + if (mode == kPitchFilterPreGain) { + /* Clear buffers. */ + memset(filter_parameters.gain_mult, 0, sizeof(filter_parameters.gain_mult)); + memset(filter_parameters.damper_state_dg, 0, + sizeof(filter_parameters.damper_state_dg)); + for (n = 0; n < PITCH_SUBFRAMES; ++n) { + //memset(out_dg[n], 0, sizeof(double) * (PITCH_FRAME_LEN + QLOOKAHEAD)); + memset(out_dg[n], 0, sizeof(out_dg[n])); + } + } else if (mode == kPitchFilterPost) { + /* Make output more periodic. Negative sign is to change the structure + * of the filter. */ + for (n = 0; n < PITCH_SUBFRAMES; ++n) { + gains[n] *= -kEnhancer; + } + } + + old_lag = *filter_state->oldlagp; + old_gain = *filter_state->oldgainp; + + /* No interpolation if pitch lag step is big. */ + if ((lags[0] > (PITCH_UPSTEP * old_lag)) || + (lags[0] < (PITCH_DOWNSTEP * old_lag))) { + old_lag = lags[0]; + old_gain = gains[0]; + + if (mode == kPitchFilterPreGain) { + filter_parameters.gain_mult[0] = 1.0; + } + } + + filter_parameters.num_samples = PITCH_UPDATE; + for (m = 0; m < PITCH_SUBFRAMES; ++m) { + /* Set the sub-frame value. */ + filter_parameters.sub_frame = m; + /* Calculate interpolation steps for pitch-lag and pitch-gain. */ + lag_delta = (lags[m] - old_lag) / PITCH_GRAN_PER_SUBFRAME; + filter_parameters.lag = old_lag; + gain_delta = (gains[m] - old_gain) / PITCH_GRAN_PER_SUBFRAME; + filter_parameters.gain = old_gain; + /* Store for the next sub-frame. */ + old_lag = lags[m]; + old_gain = gains[m]; + + for (n = 0; n < PITCH_GRAN_PER_SUBFRAME; ++n) { + /* Step-wise interpolation of pitch gains and lags. As pitch-lag changes, + * some parameters of filter need to be update. */ + filter_parameters.gain += gain_delta; + filter_parameters.lag += lag_delta; + /* Update parameters according to new lag value. */ + Update(&filter_parameters); + /* Filter a segment of input. */ + FilterSegment(in_data, &filter_parameters, out_data, out_dg); + } + } + + if (mode != kPitchFilterPreGain) { + /* Export buffer and states. */ + memcpy(filter_state->ubuf, &filter_parameters.buffer[PITCH_FRAME_LEN], + sizeof(filter_state->ubuf)); + memcpy(filter_state->ystate, filter_parameters.damper_state, + sizeof(filter_state->ystate)); + + /* Store for the next frame. */ + *filter_state->oldlagp = old_lag; + *filter_state->oldgainp = old_gain; + } + + if ((mode == kPitchFilterPreGain) || (mode == kPitchFilterPreLa)) { + /* Filter the lookahead segment, this is treated as the last sub-frame. So + * set `pf_param` to last sub-frame. */ + filter_parameters.sub_frame = PITCH_SUBFRAMES - 1; + filter_parameters.num_samples = QLOOKAHEAD; + FilterSegment(in_data, &filter_parameters, out_data, out_dg); + } +} + +void WebRtcIsac_PitchfilterPre(double* in_data, double* out_data, + PitchFiltstr* pf_state, double* lags, + double* gains) { + FilterFrame(in_data, pf_state, lags, gains, kPitchFilterPre, out_data, NULL); +} + +void WebRtcIsac_PitchfilterPre_la(double* in_data, double* out_data, + PitchFiltstr* pf_state, double* lags, + double* gains) { + FilterFrame(in_data, pf_state, lags, gains, kPitchFilterPreLa, out_data, + NULL); +} + +void WebRtcIsac_PitchfilterPre_gains( + double* in_data, double* out_data, + double out_dg[][PITCH_FRAME_LEN + QLOOKAHEAD], PitchFiltstr *pf_state, + double* lags, double* gains) { + FilterFrame(in_data, pf_state, lags, gains, kPitchFilterPreGain, out_data, + out_dg); +} + +void WebRtcIsac_PitchfilterPost(double* in_data, double* out_data, + PitchFiltstr* pf_state, double* lags, + double* gains) { + FilterFrame(in_data, pf_state, lags, gains, kPitchFilterPost, out_data, NULL); +} diff --git a/VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_filter.h b/VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_filter.h new file mode 100644 index 0000000..9a232de --- /dev/null +++ b/VocieProcess/modules/audio_coding/codecs/isac/main/source/pitch_filter.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_FILTER_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_FILTER_H_ + +#include "modules/audio_coding/codecs/isac/main/source/structs.h" + +void WebRtcIsac_PitchfilterPre(double* indat, + double* outdat, + PitchFiltstr* pfp, + double* lags, + double* gains); + +void WebRtcIsac_PitchfilterPost(double* indat, + double* outdat, + PitchFiltstr* pfp, + double* lags, + double* gains); + +void WebRtcIsac_PitchfilterPre_la(double* indat, + double* outdat, + PitchFiltstr* pfp, + double* lags, + double* gains); + +void WebRtcIsac_PitchfilterPre_gains( + double* indat, + double* outdat, + double out_dG[][PITCH_FRAME_LEN + QLOOKAHEAD], + PitchFiltstr* pfp, + double* lags, + double* gains); + +#endif // MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_PITCH_FILTER_H_ diff --git a/VocieProcess/modules/audio_coding/codecs/isac/main/source/settings.h b/VocieProcess/modules/audio_coding/codecs/isac/main/source/settings.h new file mode 100644 index 0000000..abce90c --- /dev/null +++ b/VocieProcess/modules/audio_coding/codecs/isac/main/source/settings.h @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * settings.h + * + * Declaration of #defines used in the iSAC codec + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_SETTINGS_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_SETTINGS_H_ + +/* sampling frequency (Hz) */ +#define FS 16000 + +/* number of samples per frame (either 320 (20ms), 480 (30ms) or 960 (60ms)) */ +#define INITIAL_FRAMESAMPLES 960 + +/* do not modify the following; this will have to be modified if we + * have a 20ms framesize option */ +/**********************************************************************/ +/* miliseconds */ +#define FRAMESIZE 30 +/* number of samples per frame processed in the encoder, 480 */ +#define FRAMESAMPLES 480 /* ((FRAMESIZE*FS)/1000) */ +#define FRAMESAMPLES_HALF 240 +#define FRAMESAMPLES_QUARTER 120 +/**********************************************************************/ + +/* max number of samples per frame (= 60 ms frame) */ +#define MAX_FRAMESAMPLES 960 +#define MAX_SWBFRAMESAMPLES (MAX_FRAMESAMPLES * 2) +/* number of samples per 10ms frame */ +#define FRAMESAMPLES_10ms ((10 * FS) / 1000) +#define SWBFRAMESAMPLES_10ms (FRAMESAMPLES_10ms * 2) +/* number of samples in 30 ms frame */ +#define FRAMESAMPLES_30ms 480 +/* number of subframes */ +#define SUBFRAMES 6 +/* length of a subframe */ +#define UPDATE 80 +/* length of half a subframe (low/high band) */ +#define HALF_SUBFRAMELEN (UPDATE / 2) +/* samples of look ahead (in a half-band, so actually + * half the samples of look ahead @ FS) */ +#define QLOOKAHEAD 24 /* 3 ms */ +/* order of AR model in spectral entropy coder */ +#define AR_ORDER 6 +/* order of LP model in spectral entropy coder */ +#define LP_ORDER 0 + +/* window length (masking analysis) */ +#define WINLEN 256 +/* order of low-band pole filter used to approximate masking curve */ +#define ORDERLO 12 +/* order of hi-band pole filter used to approximate masking curve */ +#define ORDERHI 6 + +#define UB_LPC_ORDER 4 +#define UB_LPC_VEC_PER_FRAME 2 +#define UB16_LPC_VEC_PER_FRAME 4 +#define UB_ACTIVE_SUBFRAMES 2 +#define UB_MAX_LPC_ORDER 6 +#define UB_INTERPOL_SEGMENTS 1 +#define UB16_INTERPOL_SEGMENTS 3 +#define LB_TOTAL_DELAY_SAMPLES 48 +enum ISACBandwidth { isac8kHz = 8, isac12kHz = 12, isac16kHz = 16 }; +enum ISACBand { + kIsacLowerBand = 0, + kIsacUpperBand12 = 1, + kIsacUpperBand16 = 2 +}; +enum IsacSamplingRate { kIsacWideband = 16, kIsacSuperWideband = 32 }; +#define UB_LPC_GAIN_DIM SUBFRAMES +#define FB_STATE_SIZE_WORD32 6 + +/* order for post_filter_bank */ +#define POSTQORDER 3 +/* order for pre-filterbank */ +#define QORDER 3 +/* another order */ +#define QORDER_ALL (POSTQORDER + QORDER - 1) +/* for decimator */ +#define ALLPASSSECTIONS 2 + +/* array size for byte stream in number of bytes. */ +/* The old maximum size still needed for the decoding */ +#define STREAM_SIZE_MAX 600 +#define STREAM_SIZE_MAX_30 200 /* 200 bytes=53.4 kbps @ 30 ms.framelength */ +#define STREAM_SIZE_MAX_60 400 /* 400 bytes=53.4 kbps @ 60 ms.framelength */ + +/* storage size for bit counts */ +#define BIT_COUNTER_SIZE 30 +/* maximum order of any AR model or filter */ +#define MAX_AR_MODEL_ORDER 12 // 50 + +/* For pitch analysis */ +#define PITCH_FRAME_LEN (FRAMESAMPLES_HALF) /* 30 ms */ +#define PITCH_MAX_LAG 140 /* 57 Hz */ +#define PITCH_MIN_LAG 20 /* 400 Hz */ +#define PITCH_MAX_GAIN 0.45 +#define PITCH_MAX_GAIN_06 0.27 /* PITCH_MAX_GAIN*0.6 */ +#define PITCH_MAX_GAIN_Q12 1843 +#define PITCH_LAG_SPAN2 (PITCH_MAX_LAG / 2 - PITCH_MIN_LAG / 2 + 5) +#define PITCH_CORR_LEN2 60 /* 15 ms */ +#define PITCH_CORR_STEP2 (PITCH_FRAME_LEN / 4) +#define PITCH_BW 11 /* half the band width of correlation surface */ +#define PITCH_SUBFRAMES 4 +#define PITCH_GRAN_PER_SUBFRAME 5 +#define PITCH_SUBFRAME_LEN (PITCH_FRAME_LEN / PITCH_SUBFRAMES) +#define PITCH_UPDATE (PITCH_SUBFRAME_LEN / PITCH_GRAN_PER_SUBFRAME) +/* maximum number of peaks to be examined in correlation surface */ +#define PITCH_MAX_NUM_PEAKS 10 +#define PITCH_PEAK_DECAY 0.85 +/* For weighting filter */ +#define PITCH_WLPCORDER 6 +#define PITCH_WLPCWINLEN PITCH_FRAME_LEN +#define PITCH_WLPCASYM 0.3 /* asymmetry parameter */ +#define PITCH_WLPCBUFLEN PITCH_WLPCWINLEN +/* For pitch filter */ +/* Extra 50 for fraction and LP filters */ +#define PITCH_BUFFSIZE (PITCH_MAX_LAG + 50) +#define PITCH_INTBUFFSIZE (PITCH_FRAME_LEN + PITCH_BUFFSIZE) +/* Max rel. step for interpolation */ +#define PITCH_UPSTEP 1.5 +/* Max rel. step for interpolation */ +#define PITCH_DOWNSTEP 0.67 +#define PITCH_FRACS 8 +#define PITCH_FRACORDER 9 +#define PITCH_DAMPORDER 5 +#define PITCH_FILTDELAY 1.5f +/* stepsize for quantization of the pitch Gain */ +#define PITCH_GAIN_STEPSIZE 0.125 + +/* Order of high pass filter */ +#define HPORDER 2 + +/* some mathematical constants */ +/* log2(exp) */ +#define LOG2EXP 1.44269504088896 +#define PI 3.14159265358979 + +/* Maximum number of iterations allowed to limit payload size */ +#define MAX_PAYLOAD_LIMIT_ITERATION 5 + +/* Redundant Coding */ +#define RCU_BOTTLENECK_BPS 16000 +#define RCU_TRANSCODING_SCALE 0.40f +#define RCU_TRANSCODING_SCALE_INVERSE 2.5f + +#define RCU_TRANSCODING_SCALE_UB 0.50f +#define RCU_TRANSCODING_SCALE_UB_INVERSE 2.0f + +/* Define Error codes */ +/* 6000 General */ +#define ISAC_MEMORY_ALLOCATION_FAILED 6010 +#define ISAC_MODE_MISMATCH 6020 +#define ISAC_DISALLOWED_BOTTLENECK 6030 +#define ISAC_DISALLOWED_FRAME_LENGTH 6040 +#define ISAC_UNSUPPORTED_SAMPLING_FREQUENCY 6050 + +/* 6200 Bandwidth estimator */ +#define ISAC_RANGE_ERROR_BW_ESTIMATOR 6240 +/* 6400 Encoder */ +#define ISAC_ENCODER_NOT_INITIATED 6410 +#define ISAC_DISALLOWED_CODING_MODE 6420 +#define ISAC_DISALLOWED_FRAME_MODE_ENCODER 6430 +#define ISAC_DISALLOWED_BITSTREAM_LENGTH 6440 +#define ISAC_PAYLOAD_LARGER_THAN_LIMIT 6450 +#define ISAC_DISALLOWED_ENCODER_BANDWIDTH 6460 +/* 6600 Decoder */ +#define ISAC_DECODER_NOT_INITIATED 6610 +#define ISAC_EMPTY_PACKET 6620 +#define ISAC_DISALLOWED_FRAME_MODE_DECODER 6630 +#define ISAC_RANGE_ERROR_DECODE_FRAME_LENGTH 6640 +#define ISAC_RANGE_ERROR_DECODE_BANDWIDTH 6650 +#define ISAC_RANGE_ERROR_DECODE_PITCH_GAIN 6660 +#define ISAC_RANGE_ERROR_DECODE_PITCH_LAG 6670 +#define ISAC_RANGE_ERROR_DECODE_LPC 6680 +#define ISAC_RANGE_ERROR_DECODE_SPECTRUM 6690 +#define ISAC_LENGTH_MISMATCH 6730 +#define ISAC_RANGE_ERROR_DECODE_BANDWITH 6740 +#define ISAC_DISALLOWED_BANDWIDTH_MODE_DECODER 6750 +#define ISAC_DISALLOWED_LPC_MODEL 6760 +/* 6800 Call setup formats */ +#define ISAC_INCOMPATIBLE_FORMATS 6810 + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_SETTINGS_H_ */ diff --git a/VocieProcess/modules/audio_coding/codecs/isac/main/source/structs.h b/VocieProcess/modules/audio_coding/codecs/isac/main/source/structs.h new file mode 100644 index 0000000..6861ca4 --- /dev/null +++ b/VocieProcess/modules/audio_coding/codecs/isac/main/source/structs.h @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * structs.h + * + * This header file contains all the structs used in the ISAC codec + * + */ + +#ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_STRUCTS_H_ +#define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_STRUCTS_H_ + +#include "modules/audio_coding/codecs/isac/bandwidth_info.h" +#include "modules/audio_coding/codecs/isac/main/source/settings.h" +#include "modules/third_party/fft/fft.h" + +typedef struct Bitstreamstruct { + uint8_t stream[STREAM_SIZE_MAX]; + uint32_t W_upper; + uint32_t streamval; + uint32_t stream_index; + +} Bitstr; + +typedef struct { + double DataBufferLo[WINLEN]; + double DataBufferHi[WINLEN]; + + double CorrBufLo[ORDERLO + 1]; + double CorrBufHi[ORDERHI + 1]; + + float PreStateLoF[ORDERLO + 1]; + float PreStateLoG[ORDERLO + 1]; + float PreStateHiF[ORDERHI + 1]; + float PreStateHiG[ORDERHI + 1]; + float PostStateLoF[ORDERLO + 1]; + float PostStateLoG[ORDERLO + 1]; + float PostStateHiF[ORDERHI + 1]; + float PostStateHiG[ORDERHI + 1]; + + double OldEnergy; + +} MaskFiltstr; + +typedef struct { + // state vectors for each of the two analysis filters + double INSTAT1[2 * (QORDER - 1)]; + double INSTAT2[2 * (QORDER - 1)]; + double INSTATLA1[2 * (QORDER - 1)]; + double INSTATLA2[2 * (QORDER - 1)]; + double INLABUF1[QLOOKAHEAD]; + double INLABUF2[QLOOKAHEAD]; + + float INSTAT1_float[2 * (QORDER - 1)]; + float INSTAT2_float[2 * (QORDER - 1)]; + float INSTATLA1_float[2 * (QORDER - 1)]; + float INSTATLA2_float[2 * (QORDER - 1)]; + float INLABUF1_float[QLOOKAHEAD]; + float INLABUF2_float[QLOOKAHEAD]; + + /* High pass filter */ + double HPstates[HPORDER]; + float HPstates_float[HPORDER]; + +} PreFiltBankstr; + +typedef struct { + // state vectors for each of the two analysis filters + double STATE_0_LOWER[2 * POSTQORDER]; + double STATE_0_UPPER[2 * POSTQORDER]; + + /* High pass filter */ + double HPstates1[HPORDER]; + double HPstates2[HPORDER]; + + float STATE_0_LOWER_float[2 * POSTQORDER]; + float STATE_0_UPPER_float[2 * POSTQORDER]; + + float HPstates1_float[HPORDER]; + float HPstates2_float[HPORDER]; + +} PostFiltBankstr; + +typedef struct { + // data buffer for pitch filter + double ubuf[PITCH_BUFFSIZE]; + + // low pass state vector + double ystate[PITCH_DAMPORDER]; + + // old lag and gain + double oldlagp[1]; + double oldgainp[1]; + +} PitchFiltstr; + +typedef struct { + // data buffer + double buffer[PITCH_WLPCBUFLEN]; + + // state vectors + double istate[PITCH_WLPCORDER]; + double weostate[PITCH_WLPCORDER]; + double whostate[PITCH_WLPCORDER]; + + // LPC window -> should be a global array because constant + double window[PITCH_WLPCWINLEN]; + +} WeightFiltstr; + +typedef struct { + // for inital estimator + double dec_buffer[PITCH_CORR_LEN2 + PITCH_CORR_STEP2 + PITCH_MAX_LAG / 2 - + PITCH_FRAME_LEN / 2 + 2]; + double decimator_state[2 * ALLPASSSECTIONS + 1]; + double hp_state[2]; + + double whitened_buf[QLOOKAHEAD]; + + double inbuf[QLOOKAHEAD]; + + PitchFiltstr PFstr_wght; + PitchFiltstr PFstr; + WeightFiltstr Wghtstr; + +} PitchAnalysisStruct; + +/* Have instance of struct together with other iSAC structs */ +typedef struct { + /* Previous frame length (in ms) */ + int32_t prev_frame_length; + + /* Previous RTP timestamp from received + packet (in samples relative beginning) */ + int32_t prev_rec_rtp_number; + + /* Send timestamp for previous packet (in ms using timeGetTime()) */ + uint32_t prev_rec_send_ts; + + /* Arrival time for previous packet (in ms using timeGetTime()) */ + uint32_t prev_rec_arr_ts; + + /* rate of previous packet, derived from RTP timestamps (in bits/s) */ + float prev_rec_rtp_rate; + + /* Time sinse the last update of the BN estimate (in ms) */ + uint32_t last_update_ts; + + /* Time sinse the last reduction (in ms) */ + uint32_t last_reduction_ts; + + /* How many times the estimate was update in the beginning */ + int32_t count_tot_updates_rec; + + /* The estimated bottle neck rate from there to here (in bits/s) */ + int32_t rec_bw; + float rec_bw_inv; + float rec_bw_avg; + float rec_bw_avg_Q; + + /* The estimated mean absolute jitter value, + as seen on this side (in ms) */ + float rec_jitter; + float rec_jitter_short_term; + float rec_jitter_short_term_abs; + float rec_max_delay; + float rec_max_delay_avg_Q; + + /* (assumed) bitrate for headers (bps) */ + float rec_header_rate; + + /* The estimated bottle neck rate from here to there (in bits/s) */ + float send_bw_avg; + + /* The estimated mean absolute jitter value, as seen on + the other siee (in ms) */ + float send_max_delay_avg; + + // number of packets received since last update + int num_pkts_rec; + + int num_consec_rec_pkts_over_30k; + + // flag for marking that a high speed network has been + // detected downstream + int hsn_detect_rec; + + int num_consec_snt_pkts_over_30k; + + // flag for marking that a high speed network has + // been detected upstream + int hsn_detect_snd; + + uint32_t start_wait_period; + + int in_wait_period; + + int change_to_WB; + + uint32_t senderTimestamp; + uint32_t receiverTimestamp; + // enum IsacSamplingRate incomingStreamSampFreq; + uint16_t numConsecLatePkts; + float consecLatency; + int16_t inWaitLatePkts; + + IsacBandwidthInfo external_bw_info; +} BwEstimatorstr; + +typedef struct { + /* boolean, flags if previous packet exceeded B.N. */ + int PrevExceed; + /* ms */ + int ExceedAgo; + /* packets left to send in current burst */ + int BurstCounter; + /* packets */ + int InitCounter; + /* ms remaining in buffer when next packet will be sent */ + double StillBuffered; + +} RateModel; + +/* The following strutc is used to store data from encoding, to make it + fast and easy to construct a new bitstream with a different Bandwidth + estimate. All values (except framelength and minBytes) is double size to + handle 60 ms of data. +*/ +typedef struct { + /* Used to keep track of if it is first or second part of 60 msec packet */ + int startIdx; + + /* Frame length in samples */ + int16_t framelength; + + /* Pitch Gain */ + int pitchGain_index[2]; + + /* Pitch Lag */ + double meanGain[2]; + int pitchIndex[PITCH_SUBFRAMES * 2]; + + /* LPC */ + int LPCindex_s[108 * 2]; /* KLT_ORDER_SHAPE = 108 */ + int LPCindex_g[12 * 2]; /* KLT_ORDER_GAIN = 12 */ + double LPCcoeffs_lo[(ORDERLO + 1) * SUBFRAMES * 2]; + double LPCcoeffs_hi[(ORDERHI + 1) * SUBFRAMES * 2]; + + /* Encode Spec */ + int16_t fre[FRAMESAMPLES]; + int16_t fim[FRAMESAMPLES]; + int16_t AvgPitchGain[2]; + + /* Used in adaptive mode only */ + int minBytes; + +} IsacSaveEncoderData; + +typedef struct { + int indexLPCShape[UB_LPC_ORDER * UB16_LPC_VEC_PER_FRAME]; + double lpcGain[SUBFRAMES << 1]; + int lpcGainIndex[SUBFRAMES << 1]; + + Bitstr bitStreamObj; + + int16_t realFFT[FRAMESAMPLES_HALF]; + int16_t imagFFT[FRAMESAMPLES_HALF]; +} ISACUBSaveEncDataStruct; + +typedef struct { + Bitstr bitstr_obj; + MaskFiltstr maskfiltstr_obj; + PreFiltBankstr prefiltbankstr_obj; + PitchFiltstr pitchfiltstr_obj; + PitchAnalysisStruct pitchanalysisstr_obj; + FFTstr fftstr_obj; + IsacSaveEncoderData SaveEnc_obj; + + int buffer_index; + int16_t current_framesamples; + + float data_buffer_float[FRAMESAMPLES_30ms]; + + int frame_nb; + double bottleneck; + int16_t new_framelength; + double s2nr; + + /* Maximum allowed number of bits for a 30 msec packet */ + int16_t payloadLimitBytes30; + /* Maximum allowed number of bits for a 30 msec packet */ + int16_t payloadLimitBytes60; + /* Maximum allowed number of bits for both 30 and 60 msec packet */ + int16_t maxPayloadBytes; + /* Maximum allowed rate in bytes per 30 msec packet */ + int16_t maxRateInBytes; + + /*--- + If set to 1 iSAC will not adapt the frame-size, if used in + channel-adaptive mode. The initial value will be used for all rates. + ---*/ + int16_t enforceFrameSize; + + /*----- + This records the BWE index the encoder injected into the bit-stream. + It will be used in RCU. The same BWE index of main payload will be in + the redundant payload. We can not retrieve it from BWE because it is + a recursive procedure (WebRtcIsac_GetDownlinkBwJitIndexImpl) and has to be + called only once per each encode. + -----*/ + int16_t lastBWIdx; +} ISACLBEncStruct; + +typedef struct { + Bitstr bitstr_obj; + MaskFiltstr maskfiltstr_obj; + PreFiltBankstr prefiltbankstr_obj; + FFTstr fftstr_obj; + ISACUBSaveEncDataStruct SaveEnc_obj; + + int buffer_index; + float data_buffer_float[MAX_FRAMESAMPLES + LB_TOTAL_DELAY_SAMPLES]; + double bottleneck; + /* Maximum allowed number of bits for a 30 msec packet */ + // int16_t payloadLimitBytes30; + /* Maximum allowed number of bits for both 30 and 60 msec packet */ + // int16_t maxPayloadBytes; + int16_t maxPayloadSizeBytes; + + double lastLPCVec[UB_LPC_ORDER]; + int16_t numBytesUsed; + int16_t lastJitterInfo; +} ISACUBEncStruct; + +typedef struct { + Bitstr bitstr_obj; + MaskFiltstr maskfiltstr_obj; + PostFiltBankstr postfiltbankstr_obj; + PitchFiltstr pitchfiltstr_obj; + FFTstr fftstr_obj; + +} ISACLBDecStruct; + +typedef struct { + Bitstr bitstr_obj; + MaskFiltstr maskfiltstr_obj; + PostFiltBankstr postfiltbankstr_obj; + FFTstr fftstr_obj; + +} ISACUBDecStruct; + +typedef struct { + ISACLBEncStruct ISACencLB_obj; + ISACLBDecStruct ISACdecLB_obj; +} ISACLBStruct; + +typedef struct { + ISACUBEncStruct ISACencUB_obj; + ISACUBDecStruct ISACdecUB_obj; +} ISACUBStruct; + +/* + This struct is used to take a snapshot of the entropy coder and LPC gains + right before encoding LPC gains. This allows us to go back to that state + if we like to limit the payload size. +*/ +typedef struct { + /* 6 lower-band & 6 upper-band */ + double loFiltGain[SUBFRAMES]; + double hiFiltGain[SUBFRAMES]; + /* Upper boundary of interval W */ + uint32_t W_upper; + uint32_t streamval; + /* Index to the current position in bytestream */ + uint32_t stream_index; + uint8_t stream[3]; +} transcode_obj; + +typedef struct { + // TODO(kwiberg): The size of these tables could be reduced by storing floats + // instead of doubles, and by making use of the identity cos(x) = + // sin(x+pi/2). They could also be made global constants that we fill in at + // compile time. + double costab1[FRAMESAMPLES_HALF]; + double sintab1[FRAMESAMPLES_HALF]; + double costab2[FRAMESAMPLES_QUARTER]; + double sintab2[FRAMESAMPLES_QUARTER]; +} TransformTables; + +typedef struct { + // lower-band codec instance + ISACLBStruct instLB; + // upper-band codec instance + ISACUBStruct instUB; + + // Bandwidth Estimator and model for the rate. + BwEstimatorstr bwestimator_obj; + RateModel rate_data_obj; + double MaxDelay; + + /* 0 = adaptive; 1 = instantaneous */ + int16_t codingMode; + + // overall bottleneck of the codec + int32_t bottleneck; + + // QMF Filter state + int32_t analysisFBState1[FB_STATE_SIZE_WORD32]; + int32_t analysisFBState2[FB_STATE_SIZE_WORD32]; + int32_t synthesisFBState1[FB_STATE_SIZE_WORD32]; + int32_t synthesisFBState2[FB_STATE_SIZE_WORD32]; + + // Error Code + int16_t errorCode; + + // bandwidth of the encoded audio 8, 12 or 16 kHz + enum ISACBandwidth bandwidthKHz; + // Sampling rate of audio, encoder and decode, 8 or 16 kHz + enum IsacSamplingRate encoderSamplingRateKHz; + enum IsacSamplingRate decoderSamplingRateKHz; + // Flag to keep track of initializations, lower & upper-band + // encoder and decoder. + int16_t initFlag; + + // Flag to to indicate signal bandwidth switch + int16_t resetFlag_8kHz; + + // Maximum allowed rate, measured in Bytes per 30 ms. + int16_t maxRateBytesPer30Ms; + // Maximum allowed payload-size, measured in Bytes. + int16_t maxPayloadSizeBytes; + /* The expected sampling rate of the input signal. Valid values are 16000 + * and 32000. This is not the operation sampling rate of the codec. */ + uint16_t in_sample_rate_hz; + + // Trig tables for WebRtcIsac_Time2Spec and WebRtcIsac_Spec2time. + TransformTables transform_tables; +} ISACMainStruct; + +#endif /* MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_STRUCTS_H_ */ diff --git a/VocieProcess/modules/audio_processing/aec_dump/aec_dump_factory.h b/VocieProcess/modules/audio_processing/aec_dump/aec_dump_factory.h new file mode 100644 index 0000000..0d258a9 --- /dev/null +++ b/VocieProcess/modules/audio_processing/aec_dump/aec_dump_factory.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC_DUMP_AEC_DUMP_FACTORY_H_ +#define MODULES_AUDIO_PROCESSING_AEC_DUMP_AEC_DUMP_FACTORY_H_ + +#include + +#include "absl/base/nullability.h" +#include "absl/strings/string_view.h" +#include "api/task_queue/task_queue_base.h" +#include "modules/audio_processing/include/aec_dump.h" +#include "rtc_base/system/file_wrapper.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +class RTC_EXPORT AecDumpFactory { + public: + // The `worker_queue` must outlive the created AecDump instance. + // `max_log_size_bytes == -1` means the log size will be unlimited. + // The AecDump takes responsibility for `handle` and closes it in the + // destructor. A non-null return value indicates that the file has been + // sucessfully opened. + static absl::Nullable> Create( + FileWrapper file, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue); + static absl::Nullable> Create( + absl::string_view file_name, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue); + static absl::Nullable> Create( + absl::Nonnull handle, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC_DUMP_AEC_DUMP_FACTORY_H_ diff --git a/VocieProcess/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.cc b/VocieProcess/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.cc new file mode 100644 index 0000000..cb2336b --- /dev/null +++ b/VocieProcess/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.cc @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.h" + +#include + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { + +AudioSamplesScaler::AudioSamplesScaler(float initial_gain) + : previous_gain_(initial_gain), target_gain_(initial_gain) {} + +void AudioSamplesScaler::Process(AudioBuffer& audio_buffer) { + if (static_cast(audio_buffer.num_frames()) != samples_per_channel_) { + // Update the members depending on audio-buffer length if needed. + RTC_DCHECK_GT(audio_buffer.num_frames(), 0); + samples_per_channel_ = static_cast(audio_buffer.num_frames()); + one_by_samples_per_channel_ = 1.f / samples_per_channel_; + } + + if (target_gain_ == 1.f && previous_gain_ == target_gain_) { + // If only a gain of 1 is to be applied, do an early return without applying + // any gain. + return; + } + + float gain = previous_gain_; + if (previous_gain_ == target_gain_) { + // Apply a non-changing gain. + for (size_t channel = 0; channel < audio_buffer.num_channels(); ++channel) { + rtc::ArrayView channel_view(audio_buffer.channels()[channel], + samples_per_channel_); + for (float& sample : channel_view) { + sample *= gain; + } + } + } else { + const float increment = + (target_gain_ - previous_gain_) * one_by_samples_per_channel_; + + if (increment > 0.f) { + // Apply an increasing gain. + for (size_t channel = 0; channel < audio_buffer.num_channels(); + ++channel) { + gain = previous_gain_; + rtc::ArrayView channel_view(audio_buffer.channels()[channel], + samples_per_channel_); + for (float& sample : channel_view) { + gain = std::min(gain + increment, target_gain_); + sample *= gain; + } + } + } else { + // Apply a decreasing gain. + for (size_t channel = 0; channel < audio_buffer.num_channels(); + ++channel) { + gain = previous_gain_; + rtc::ArrayView channel_view(audio_buffer.channels()[channel], + samples_per_channel_); + for (float& sample : channel_view) { + gain = std::max(gain + increment, target_gain_); + sample *= gain; + } + } + } + } + previous_gain_ = target_gain_; + + // Saturate the samples to be in the S16 range. + for (size_t channel = 0; channel < audio_buffer.num_channels(); ++channel) { + rtc::ArrayView channel_view(audio_buffer.channels()[channel], + samples_per_channel_); + for (float& sample : channel_view) { + constexpr float kMinFloatS16Value = -32768.f; + constexpr float kMaxFloatS16Value = 32767.f; + sample = rtc::SafeClamp(sample, kMinFloatS16Value, kMaxFloatS16Value); + } + } +} + +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.h b/VocieProcess/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.h new file mode 100644 index 0000000..2ae8533 --- /dev/null +++ b/VocieProcess/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_CAPTURE_LEVELS_ADJUSTER_AUDIO_SAMPLES_SCALER_H_ +#define MODULES_AUDIO_PROCESSING_CAPTURE_LEVELS_ADJUSTER_AUDIO_SAMPLES_SCALER_H_ + +#include + +#include "modules/audio_processing/audio_buffer.h" + +namespace webrtc { + +// Handles and applies a gain to the samples in an audio buffer. +// The gain is applied for each sample and any changes in the gain take effect +// gradually (in a linear manner) over one frame. +class AudioSamplesScaler { + public: + // C-tor. The supplied `initial_gain` is used immediately at the first call to + // Process(), i.e., in contrast to the gain supplied by SetGain(...) there is + // no gradual change to the `initial_gain`. + explicit AudioSamplesScaler(float initial_gain); + AudioSamplesScaler(const AudioSamplesScaler&) = delete; + AudioSamplesScaler& operator=(const AudioSamplesScaler&) = delete; + + // Applies the specified gain to the audio in `audio_buffer`. + void Process(AudioBuffer& audio_buffer); + + // Sets the gain to apply to each sample. + void SetGain(float gain) { target_gain_ = gain; } + + private: + float previous_gain_ = 1.f; + float target_gain_ = 1.f; + int samples_per_channel_ = -1; + float one_by_samples_per_channel_ = -1.f; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_CAPTURE_LEVELS_ADJUSTER_AUDIO_SAMPLES_SCALER_H_ diff --git a/VocieProcess/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.cc b/VocieProcess/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.cc new file mode 100644 index 0000000..dfda582 --- /dev/null +++ b/VocieProcess/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h" + +#include "modules/audio_processing/audio_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { + +namespace { + +constexpr int kMinAnalogMicGainLevel = 0; +constexpr int kMaxAnalogMicGainLevel = 255; + +float ComputeLevelBasedGain(int emulated_analog_mic_gain_level) { + static_assert( + kMinAnalogMicGainLevel == 0, + "The minimum gain level must be 0 for the maths below to work."); + static_assert(kMaxAnalogMicGainLevel > 0, + "The minimum gain level must be larger than 0 for the maths " + "below to work."); + constexpr float kGainToLevelMultiplier = 1.f / kMaxAnalogMicGainLevel; + + RTC_DCHECK_GE(emulated_analog_mic_gain_level, kMinAnalogMicGainLevel); + RTC_DCHECK_LE(emulated_analog_mic_gain_level, kMaxAnalogMicGainLevel); + return kGainToLevelMultiplier * emulated_analog_mic_gain_level; +} + +float ComputePreGain(float pre_gain, + int emulated_analog_mic_gain_level, + bool emulated_analog_mic_gain_enabled) { + return emulated_analog_mic_gain_enabled + ? pre_gain * ComputeLevelBasedGain(emulated_analog_mic_gain_level) + : pre_gain; +} + +} // namespace + +CaptureLevelsAdjuster::CaptureLevelsAdjuster( + bool emulated_analog_mic_gain_enabled, + int emulated_analog_mic_gain_level, + float pre_gain, + float post_gain) + : emulated_analog_mic_gain_enabled_(emulated_analog_mic_gain_enabled), + emulated_analog_mic_gain_level_(emulated_analog_mic_gain_level), + pre_gain_(pre_gain), + pre_adjustment_gain_(ComputePreGain(pre_gain_, + emulated_analog_mic_gain_level_, + emulated_analog_mic_gain_enabled_)), + pre_scaler_(pre_adjustment_gain_), + post_scaler_(post_gain) {} + +void CaptureLevelsAdjuster::ApplyPreLevelAdjustment(AudioBuffer& audio_buffer) { + pre_scaler_.Process(audio_buffer); +} + +void CaptureLevelsAdjuster::ApplyPostLevelAdjustment( + AudioBuffer& audio_buffer) { + post_scaler_.Process(audio_buffer); +} + +void CaptureLevelsAdjuster::SetPreGain(float pre_gain) { + pre_gain_ = pre_gain; + UpdatePreAdjustmentGain(); +} + +void CaptureLevelsAdjuster::SetPostGain(float post_gain) { + post_scaler_.SetGain(post_gain); +} + +void CaptureLevelsAdjuster::SetAnalogMicGainLevel(int level) { + RTC_DCHECK_GE(level, kMinAnalogMicGainLevel); + RTC_DCHECK_LE(level, kMaxAnalogMicGainLevel); + int clamped_level = + rtc::SafeClamp(level, kMinAnalogMicGainLevel, kMaxAnalogMicGainLevel); + + emulated_analog_mic_gain_level_ = clamped_level; + UpdatePreAdjustmentGain(); +} + +void CaptureLevelsAdjuster::UpdatePreAdjustmentGain() { + pre_adjustment_gain_ = + ComputePreGain(pre_gain_, emulated_analog_mic_gain_level_, + emulated_analog_mic_gain_enabled_); + pre_scaler_.SetGain(pre_adjustment_gain_); +} + +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h b/VocieProcess/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h new file mode 100644 index 0000000..38b68ad --- /dev/null +++ b/VocieProcess/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_AUDIO_PROCESSING_CAPTURE_LEVELS_ADJUSTER_CAPTURE_LEVELS_ADJUSTER_H_ +#define MODULES_AUDIO_PROCESSING_CAPTURE_LEVELS_ADJUSTER_CAPTURE_LEVELS_ADJUSTER_H_ + +#include + +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.h" + +namespace webrtc { + +// Adjusts the level of the capture signal before and after all capture-side +// processing is done using a combination of explicitly specified gains +// and an emulated analog gain functionality where a specified analog level +// results in an additional gain. The pre-adjustment is achieved by combining +// the gain value `pre_gain` and the level `emulated_analog_mic_gain_level` to +// form a combined gain of `pre_gain`*`emulated_analog_mic_gain_level`/255 which +// is multiplied to each sample. The intention of the +// `emulated_analog_mic_gain_level` is to be controlled by the analog AGC +// functionality and to produce an emulated analog mic gain equal to +// `emulated_analog_mic_gain_level`/255. The post level adjustment is achieved +// by multiplying each sample with the value of `post_gain`. Any changes in the +// gains take are done smoothly over one frame and the scaled samples are +// clamped to fit into the allowed S16 sample range. +class CaptureLevelsAdjuster { + public: + // C-tor. The values for the level and the gains must fulfill + // 0 <= emulated_analog_mic_gain_level <= 255. + // 0.f <= pre_gain. + // 0.f <= post_gain. + CaptureLevelsAdjuster(bool emulated_analog_mic_gain_enabled, + int emulated_analog_mic_gain_level, + float pre_gain, + float post_gain); + CaptureLevelsAdjuster(const CaptureLevelsAdjuster&) = delete; + CaptureLevelsAdjuster& operator=(const CaptureLevelsAdjuster&) = delete; + + // Adjusts the level of the signal. This should be called before any of the + // other processing is performed. + void ApplyPreLevelAdjustment(AudioBuffer& audio_buffer); + + // Adjusts the level of the signal. This should be called after all of the + // other processing have been performed. + void ApplyPostLevelAdjustment(AudioBuffer& audio_buffer); + + // Sets the gain to apply to each sample before any of the other processing is + // performed. + void SetPreGain(float pre_gain); + + // Returns the total pre-adjustment gain applied, comprising both the pre_gain + // as well as the gain from the emulated analog mic, to each sample before any + // of the other processing is performed. + float GetPreAdjustmentGain() const { return pre_adjustment_gain_; } + + // Sets the gain to apply to each sample after all of the other processing + // have been performed. + void SetPostGain(float post_gain); + + // Sets the analog gain level to use for the emulated analog gain. + // `level` must be in the range [0...255]. + void SetAnalogMicGainLevel(int level); + + // Returns the current analog gain level used for the emulated analog gain. + int GetAnalogMicGainLevel() const { return emulated_analog_mic_gain_level_; } + + private: + // Updates the value of `pre_adjustment_gain_` based on the supplied values + // for `pre_gain` and `emulated_analog_mic_gain_level_`. + void UpdatePreAdjustmentGain(); + + const bool emulated_analog_mic_gain_enabled_; + int emulated_analog_mic_gain_level_; + float pre_gain_; + float pre_adjustment_gain_; + AudioSamplesScaler pre_scaler_; + AudioSamplesScaler post_scaler_; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_CAPTURE_LEVELS_ADJUSTER_CAPTURE_LEVELS_ADJUSTER_H_ diff --git a/VocieProcess/modules/audio_processing/echo_control_mobile_impl.cc b/VocieProcess/modules/audio_processing/echo_control_mobile_impl.cc new file mode 100644 index 0000000..1dc66ab --- /dev/null +++ b/VocieProcess/modules/audio_processing/echo_control_mobile_impl.cc @@ -0,0 +1,287 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/echo_control_mobile_impl.h" + +#include + +#include + +#include "api/audio/audio_processing.h" +#include "modules/audio_processing/aecm/echo_control_mobile.h" +#include "modules/audio_processing/audio_buffer.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace { +int16_t MapSetting(EchoControlMobileImpl::RoutingMode mode) { + switch (mode) { + case EchoControlMobileImpl::kQuietEarpieceOrHeadset: + return 0; + case EchoControlMobileImpl::kEarpiece: + return 1; + case EchoControlMobileImpl::kLoudEarpiece: + return 2; + case EchoControlMobileImpl::kSpeakerphone: + return 3; + case EchoControlMobileImpl::kLoudSpeakerphone: + return 4; + } + RTC_DCHECK_NOTREACHED(); + return -1; +} + +AudioProcessing::Error MapError(int err) { + switch (err) { + case AECM_UNSUPPORTED_FUNCTION_ERROR: + return AudioProcessing::kUnsupportedFunctionError; + case AECM_NULL_POINTER_ERROR: + return AudioProcessing::kNullPointerError; + case AECM_BAD_PARAMETER_ERROR: + return AudioProcessing::kBadParameterError; + case AECM_BAD_PARAMETER_WARNING: + return AudioProcessing::kBadStreamParameterWarning; + default: + // AECM_UNSPECIFIED_ERROR + // AECM_UNINITIALIZED_ERROR + return AudioProcessing::kUnspecifiedError; + } +} + +} // namespace + +struct EchoControlMobileImpl::StreamProperties { + StreamProperties() = delete; + StreamProperties(int sample_rate_hz, + size_t num_reverse_channels, + size_t num_output_channels) + : sample_rate_hz(sample_rate_hz), + num_reverse_channels(num_reverse_channels), + num_output_channels(num_output_channels) {} + + int sample_rate_hz; + size_t num_reverse_channels; + size_t num_output_channels; +}; + +class EchoControlMobileImpl::Canceller { + public: + Canceller() { + state_ = WebRtcAecm_Create(); + RTC_CHECK(state_); + } + + ~Canceller() { + RTC_DCHECK(state_); + WebRtcAecm_Free(state_); + } + + Canceller(const Canceller&) = delete; + Canceller& operator=(const Canceller&) = delete; + + void* state() { + RTC_DCHECK(state_); + return state_; + } + + void Initialize(int sample_rate_hz) { + RTC_DCHECK(state_); + int error = WebRtcAecm_Init(state_, sample_rate_hz); + RTC_DCHECK_EQ(AudioProcessing::kNoError, error); + } + + private: + void* state_; +}; + +EchoControlMobileImpl::EchoControlMobileImpl() + : routing_mode_(kSpeakerphone), comfort_noise_enabled_(false) {} + +EchoControlMobileImpl::~EchoControlMobileImpl() {} + +void EchoControlMobileImpl::ProcessRenderAudio( + rtc::ArrayView packed_render_audio) { + RTC_DCHECK(stream_properties_); + + size_t buffer_index = 0; + size_t num_frames_per_band = + packed_render_audio.size() / (stream_properties_->num_output_channels * + stream_properties_->num_reverse_channels); + + for (auto& canceller : cancellers_) { + WebRtcAecm_BufferFarend(canceller->state(), + &packed_render_audio[buffer_index], + num_frames_per_band); + + buffer_index += num_frames_per_band; + } +} + +void EchoControlMobileImpl::PackRenderAudioBuffer( + const AudioBuffer* audio, + size_t num_output_channels, + size_t num_channels, + std::vector* packed_buffer) { + RTC_DCHECK_GE(AudioBuffer::kMaxSplitFrameLength, + audio->num_frames_per_band()); + RTC_DCHECK_EQ(num_channels, audio->num_channels()); + + // The ordering convention must be followed to pass to the correct AECM. + packed_buffer->clear(); + int render_channel = 0; + for (size_t i = 0; i < num_output_channels; i++) { + for (size_t j = 0; j < audio->num_channels(); j++) { + std::array data_to_buffer; + FloatS16ToS16(audio->split_bands_const(render_channel)[kBand0To8kHz], + audio->num_frames_per_band(), data_to_buffer.data()); + + // Buffer the samples in the render queue. + packed_buffer->insert( + packed_buffer->end(), data_to_buffer.data(), + data_to_buffer.data() + audio->num_frames_per_band()); + render_channel = (render_channel + 1) % audio->num_channels(); + } + } +} + +size_t EchoControlMobileImpl::NumCancellersRequired( + size_t num_output_channels, + size_t num_reverse_channels) { + return num_output_channels * num_reverse_channels; +} + +int EchoControlMobileImpl::ProcessCaptureAudio(AudioBuffer* audio, + int stream_delay_ms) { + RTC_DCHECK(stream_properties_); + RTC_DCHECK_GE(160, audio->num_frames_per_band()); + RTC_DCHECK_EQ(audio->num_channels(), stream_properties_->num_output_channels); + RTC_DCHECK_GE(cancellers_.size(), stream_properties_->num_reverse_channels * + audio->num_channels()); + + int err = AudioProcessing::kNoError; + + // The ordering convention must be followed to pass to the correct AECM. + size_t handle_index = 0; + for (size_t capture = 0; capture < audio->num_channels(); ++capture) { + // TODO(ajm): improve how this works, possibly inside AECM. + // This is kind of hacked up. + RTC_DCHECK_LT(capture, low_pass_reference_.size()); + const int16_t* noisy = + reference_copied_ ? low_pass_reference_[capture].data() : nullptr; + + RTC_DCHECK_GE(AudioBuffer::kMaxSplitFrameLength, + audio->num_frames_per_band()); + + std::array split_bands_data; + int16_t* split_bands = split_bands_data.data(); + const int16_t* clean = split_bands_data.data(); + if (audio->split_bands(capture)[kBand0To8kHz]) { + FloatS16ToS16(audio->split_bands(capture)[kBand0To8kHz], + audio->num_frames_per_band(), split_bands_data.data()); + } else { + clean = nullptr; + split_bands = nullptr; + } + + if (noisy == NULL) { + noisy = clean; + clean = NULL; + } + for (size_t render = 0; render < stream_properties_->num_reverse_channels; + ++render) { + err = WebRtcAecm_Process(cancellers_[handle_index]->state(), noisy, clean, + split_bands, audio->num_frames_per_band(), + stream_delay_ms); + + if (split_bands) { + S16ToFloatS16(split_bands, audio->num_frames_per_band(), + audio->split_bands(capture)[kBand0To8kHz]); + } + + if (err != AudioProcessing::kNoError) { + return MapError(err); + } + + ++handle_index; + } + for (size_t band = 1u; band < audio->num_bands(); ++band) { + memset(audio->split_bands_f(capture)[band], 0, + audio->num_frames_per_band() * + sizeof(audio->split_bands_f(capture)[band][0])); + } + } + return AudioProcessing::kNoError; +} + +int EchoControlMobileImpl::set_routing_mode(RoutingMode mode) { + if (MapSetting(mode) == -1) { + return AudioProcessing::kBadParameterError; + } + routing_mode_ = mode; + return Configure(); +} + +EchoControlMobileImpl::RoutingMode EchoControlMobileImpl::routing_mode() const { + return routing_mode_; +} + +int EchoControlMobileImpl::enable_comfort_noise(bool enable) { + comfort_noise_enabled_ = enable; + return Configure(); +} + +bool EchoControlMobileImpl::is_comfort_noise_enabled() const { + return comfort_noise_enabled_; +} + +void EchoControlMobileImpl::Initialize(int sample_rate_hz, + size_t num_reverse_channels, + size_t num_output_channels) { + low_pass_reference_.resize(num_output_channels); + for (auto& reference : low_pass_reference_) { + reference.fill(0); + } + + stream_properties_.reset(new StreamProperties( + sample_rate_hz, num_reverse_channels, num_output_channels)); + + // AECM only supports 16 kHz or lower sample rates. + RTC_DCHECK_LE(stream_properties_->sample_rate_hz, + AudioProcessing::kSampleRate16kHz); + + cancellers_.resize( + NumCancellersRequired(stream_properties_->num_output_channels, + stream_properties_->num_reverse_channels)); + + for (auto& canceller : cancellers_) { + if (!canceller) { + canceller.reset(new Canceller()); + } + canceller->Initialize(sample_rate_hz); + } + Configure(); +} + +int EchoControlMobileImpl::Configure() { + AecmConfig config; + config.cngMode = comfort_noise_enabled_; + config.echoMode = MapSetting(routing_mode_); + int error = AudioProcessing::kNoError; + for (auto& canceller : cancellers_) { + int handle_error = WebRtcAecm_set_config(canceller->state(), config); + if (handle_error != AudioProcessing::kNoError) { + error = handle_error; + } + } + return error; +} + +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/echo_control_mobile_impl.h b/VocieProcess/modules/audio_processing/echo_control_mobile_impl.h new file mode 100644 index 0000000..f7f2626 --- /dev/null +++ b/VocieProcess/modules/audio_processing/echo_control_mobile_impl.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_ECHO_CONTROL_MOBILE_IMPL_H_ +#define MODULES_AUDIO_PROCESSING_ECHO_CONTROL_MOBILE_IMPL_H_ + +#include +#include + +#include +#include + +#include "api/array_view.h" + +namespace webrtc { + +class AudioBuffer; + +// The acoustic echo control for mobile (AECM) component is a low complexity +// robust option intended for use on mobile devices. +class EchoControlMobileImpl { + public: + EchoControlMobileImpl(); + + ~EchoControlMobileImpl(); + + // Recommended settings for particular audio routes. In general, the louder + // the echo is expected to be, the higher this value should be set. The + // preferred setting may vary from device to device. + enum RoutingMode { + kQuietEarpieceOrHeadset, + kEarpiece, + kLoudEarpiece, + kSpeakerphone, + kLoudSpeakerphone + }; + + // Sets echo control appropriate for the audio routing `mode` on the device. + // It can and should be updated during a call if the audio routing changes. + int set_routing_mode(RoutingMode mode); + RoutingMode routing_mode() const; + + // Comfort noise replaces suppressed background noise to maintain a + // consistent signal level. + int enable_comfort_noise(bool enable); + bool is_comfort_noise_enabled() const; + + void ProcessRenderAudio(rtc::ArrayView packed_render_audio); + int ProcessCaptureAudio(AudioBuffer* audio, int stream_delay_ms); + + void Initialize(int sample_rate_hz, + size_t num_reverse_channels, + size_t num_output_channels); + + static void PackRenderAudioBuffer(const AudioBuffer* audio, + size_t num_output_channels, + size_t num_channels, + std::vector* packed_buffer); + + static size_t NumCancellersRequired(size_t num_output_channels, + size_t num_reverse_channels); + + private: + class Canceller; + struct StreamProperties; + + int Configure(); + + RoutingMode routing_mode_; + bool comfort_noise_enabled_; + + std::vector> cancellers_; + std::unique_ptr stream_properties_; + std::vector> low_pass_reference_; + bool reference_copied_ = false; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_ECHO_CONTROL_MOBILE_IMPL_H_ diff --git a/VocieProcess/modules/audio_processing/include/aec_dump.cc b/VocieProcess/modules/audio_processing/include/aec_dump.cc new file mode 100644 index 0000000..8f788cb --- /dev/null +++ b/VocieProcess/modules/audio_processing/include/aec_dump.cc @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/include/aec_dump.h" + +namespace webrtc { +InternalAPMConfig::InternalAPMConfig() = default; +InternalAPMConfig::InternalAPMConfig(const InternalAPMConfig&) = default; +InternalAPMConfig::InternalAPMConfig(InternalAPMConfig&&) = default; +InternalAPMConfig& InternalAPMConfig::operator=(const InternalAPMConfig&) = + default; + +bool InternalAPMConfig::operator==(const InternalAPMConfig& other) const { + return aec_enabled == other.aec_enabled && + aec_delay_agnostic_enabled == other.aec_delay_agnostic_enabled && + aec_drift_compensation_enabled == + other.aec_drift_compensation_enabled && + aec_extended_filter_enabled == other.aec_extended_filter_enabled && + aec_suppression_level == other.aec_suppression_level && + aecm_enabled == other.aecm_enabled && + aecm_comfort_noise_enabled == other.aecm_comfort_noise_enabled && + aecm_routing_mode == other.aecm_routing_mode && + agc_enabled == other.agc_enabled && agc_mode == other.agc_mode && + agc_limiter_enabled == other.agc_limiter_enabled && + hpf_enabled == other.hpf_enabled && ns_enabled == other.ns_enabled && + ns_level == other.ns_level && + transient_suppression_enabled == other.transient_suppression_enabled && + noise_robust_agc_enabled == other.noise_robust_agc_enabled && + pre_amplifier_enabled == other.pre_amplifier_enabled && + pre_amplifier_fixed_gain_factor == + other.pre_amplifier_fixed_gain_factor && + experiments_description == other.experiments_description; +} +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/include/aec_dump.h b/VocieProcess/modules/audio_processing/include/aec_dump.h new file mode 100644 index 0000000..f07b911 --- /dev/null +++ b/VocieProcess/modules/audio_processing/include/aec_dump.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AEC_DUMP_H_ +#define MODULES_AUDIO_PROCESSING_INCLUDE_AEC_DUMP_H_ + +#include + +#include + +#include "absl/base/attributes.h" +#include "absl/types/optional.h" +#include "api/audio/audio_processing.h" +#include "modules/audio_processing/include/audio_frame_view.h" + +namespace webrtc { + +// Struct for passing current config from APM without having to +// include protobuf headers. +struct InternalAPMConfig { + InternalAPMConfig(); + InternalAPMConfig(const InternalAPMConfig&); + InternalAPMConfig(InternalAPMConfig&&); + + InternalAPMConfig& operator=(const InternalAPMConfig&); + InternalAPMConfig& operator=(InternalAPMConfig&&) = delete; + + bool operator==(const InternalAPMConfig& other) const; + + bool aec_enabled = false; + bool aec_delay_agnostic_enabled = false; + bool aec_drift_compensation_enabled = false; + bool aec_extended_filter_enabled = false; + int aec_suppression_level = 0; + bool aecm_enabled = false; + bool aecm_comfort_noise_enabled = false; + int aecm_routing_mode = 0; + bool agc_enabled = false; + int agc_mode = 0; + bool agc_limiter_enabled = false; + bool hpf_enabled = false; + bool ns_enabled = false; + int ns_level = 0; + bool transient_suppression_enabled = false; + bool noise_robust_agc_enabled = false; + bool pre_amplifier_enabled = false; + float pre_amplifier_fixed_gain_factor = 1.f; + std::string experiments_description = ""; +}; + +// An interface for recording configuration and input/output streams +// of the Audio Processing Module. The recordings are called +// 'aec-dumps' and are stored in a protobuf format defined in +// debug.proto. +// The Write* methods are always safe to call concurrently or +// otherwise for all implementing subclasses. The intended mode of +// operation is to create a protobuf object from the input, and send +// it away to be written to file asynchronously. +class AecDump { + public: + struct AudioProcessingState { + int delay; + int drift; + absl::optional applied_input_volume; + bool keypress; + }; + + virtual ~AecDump() = default; + + // Logs Event::Type INIT message. + virtual void WriteInitMessage(const ProcessingConfig& api_format, + int64_t time_now_ms) = 0; + ABSL_DEPRECATED("") + void WriteInitMessage(const ProcessingConfig& api_format) { + WriteInitMessage(api_format, 0); + } + + // Logs Event::Type STREAM message. To log an input/output pair, + // call the AddCapture* and AddAudioProcessingState methods followed + // by a WriteCaptureStreamMessage call. + virtual void AddCaptureStreamInput( + const AudioFrameView& src) = 0; + virtual void AddCaptureStreamOutput( + const AudioFrameView& src) = 0; + virtual void AddCaptureStreamInput(const int16_t* const data, + int num_channels, + int samples_per_channel) = 0; + virtual void AddCaptureStreamOutput(const int16_t* const data, + int num_channels, + int samples_per_channel) = 0; + virtual void AddAudioProcessingState(const AudioProcessingState& state) = 0; + virtual void WriteCaptureStreamMessage() = 0; + + // Logs Event::Type REVERSE_STREAM message. + virtual void WriteRenderStreamMessage(const int16_t* const data, + int num_channels, + int samples_per_channel) = 0; + virtual void WriteRenderStreamMessage( + const AudioFrameView& src) = 0; + + virtual void WriteRuntimeSetting( + const AudioProcessing::RuntimeSetting& runtime_setting) = 0; + + // Logs Event::Type CONFIG message. + virtual void WriteConfig(const InternalAPMConfig& config) = 0; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AEC_DUMP_H_ diff --git a/VocieProcess/modules/audio_processing/include/audio_frame_proxies.cc b/VocieProcess/modules/audio_processing/include/audio_frame_proxies.cc new file mode 100644 index 0000000..e37645e --- /dev/null +++ b/VocieProcess/modules/audio_processing/include/audio_frame_proxies.cc @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/include/audio_frame_proxies.h" + +#include "api/audio/audio_frame.h" +#include "api/audio/audio_processing.h" + +namespace webrtc { + +int ProcessAudioFrame(AudioProcessing* ap, AudioFrame* frame) { + if (!frame || !ap) { + return AudioProcessing::Error::kNullPointerError; + } + + StreamConfig input_config(frame->sample_rate_hz_, frame->num_channels_); + StreamConfig output_config(frame->sample_rate_hz_, frame->num_channels_); + RTC_DCHECK_EQ(frame->samples_per_channel(), input_config.num_frames()); + + int result = ap->ProcessStream(frame->data(), input_config, output_config, + frame->mutable_data()); + + AudioProcessingStats stats = ap->GetStatistics(); + + if (stats.voice_detected) { + frame->vad_activity_ = *stats.voice_detected + ? AudioFrame::VADActivity::kVadActive + : AudioFrame::VADActivity::kVadPassive; + } + + return result; +} + +int ProcessReverseAudioFrame(AudioProcessing* ap, AudioFrame* frame) { + if (!frame || !ap) { + return AudioProcessing::Error::kNullPointerError; + } + + // Must be a native rate. + if (frame->sample_rate_hz_ != AudioProcessing::NativeRate::kSampleRate8kHz && + frame->sample_rate_hz_ != AudioProcessing::NativeRate::kSampleRate16kHz && + frame->sample_rate_hz_ != AudioProcessing::NativeRate::kSampleRate32kHz && + frame->sample_rate_hz_ != AudioProcessing::NativeRate::kSampleRate48kHz) { + return AudioProcessing::Error::kBadSampleRateError; + } + + if (frame->num_channels_ <= 0) { + return AudioProcessing::Error::kBadNumberChannelsError; + } + + StreamConfig input_config(frame->sample_rate_hz_, frame->num_channels_); + StreamConfig output_config(frame->sample_rate_hz_, frame->num_channels_); + + int result = ap->ProcessReverseStream(frame->data(), input_config, + output_config, frame->mutable_data()); + return result; +} + +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/include/audio_frame_proxies.h b/VocieProcess/modules/audio_processing/include/audio_frame_proxies.h new file mode 100644 index 0000000..5dd111c --- /dev/null +++ b/VocieProcess/modules/audio_processing/include/audio_frame_proxies.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_PROXIES_H_ +#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_PROXIES_H_ + +namespace webrtc { + +class AudioFrame; +class AudioProcessing; + +// Processes a 10 ms `frame` of the primary audio stream using the provided +// AudioProcessing object. On the client-side, this is the near-end (or +// captured) audio. The `sample_rate_hz_`, `num_channels_`, and +// `samples_per_channel_` members of `frame` must be valid. If changed from the +// previous call to this function, it will trigger an initialization of the +// provided AudioProcessing object. +// The function returns any error codes passed from the AudioProcessing +// ProcessStream method. +int ProcessAudioFrame(AudioProcessing* ap, AudioFrame* frame); + +// Processes a 10 ms `frame` of the reverse direction audio stream using the +// provided AudioProcessing object. The frame may be modified. On the +// client-side, this is the far-end (or to be rendered) audio. The +// `sample_rate_hz_`, `num_channels_`, and `samples_per_channel_` members of +// `frame` must be valid. If changed from the previous call to this function, it +// will trigger an initialization of the provided AudioProcessing object. +// The function returns any error codes passed from the AudioProcessing +// ProcessReverseStream method. +int ProcessReverseAudioFrame(AudioProcessing* ap, AudioFrame* frame); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_PROXIES_H_ diff --git a/VocieProcess/modules/audio_processing/include/audio_frame_view.h b/VocieProcess/modules/audio_processing/include/audio_frame_view.h new file mode 100644 index 0000000..27e2009 --- /dev/null +++ b/VocieProcess/modules/audio_processing/include/audio_frame_view.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_VIEW_H_ +#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_VIEW_H_ + +#include "api/audio/audio_view.h" + +namespace webrtc { + +// Class to pass audio data in T** format, where T is a numeric type. +template +class AudioFrameView { + public: + // `num_channels` and `channel_size` describe the T** + // `audio_samples`. `audio_samples` is assumed to point to a + // two-dimensional |num_channels * channel_size| array of floats. + // + // Note: The implementation now only requires the first channel pointer. + // The previous implementation retained a pointer to externally owned array + // of channel pointers, but since the channel size and count are provided + // and the array is assumed to be a single two-dimensional array, the other + // channel pointers can be calculated based on that (which is what the class + // now uses `DeinterleavedView<>` internally for). + AudioFrameView(T* const* audio_samples, int num_channels, int channel_size) + : view_(num_channels && channel_size ? audio_samples[0] : nullptr, + channel_size, + num_channels) { + RTC_DCHECK_GE(view_.num_channels(), 0); + RTC_DCHECK_GE(view_.samples_per_channel(), 0); + } + + // Implicit cast to allow converting AudioFrameView to + // AudioFrameView. + template + AudioFrameView(AudioFrameView other) : view_(other.view()) {} + + // Allow constructing AudioFrameView from a DeinterleavedView. + template + explicit AudioFrameView(DeinterleavedView view) : view_(view) {} + + AudioFrameView() = delete; + + int num_channels() const { return view_.num_channels(); } + int samples_per_channel() const { return view_.samples_per_channel(); } + MonoView channel(int idx) { return view_[idx]; } + MonoView channel(int idx) const { return view_[idx]; } + MonoView operator[](int idx) { return view_[idx]; } + MonoView operator[](int idx) const { return view_[idx]; } + + DeinterleavedView view() { return view_; } + DeinterleavedView view() const { return view_; } + + private: + DeinterleavedView view_; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_VIEW_H_ diff --git a/VocieProcess/modules/audio_processing/render_queue_item_verifier.h b/VocieProcess/modules/audio_processing/render_queue_item_verifier.h new file mode 100644 index 0000000..b8aff4a --- /dev/null +++ b/VocieProcess/modules/audio_processing/render_queue_item_verifier.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_RENDER_QUEUE_ITEM_VERIFIER_H_ +#define MODULES_AUDIO_PROCESSING_RENDER_QUEUE_ITEM_VERIFIER_H_ + +#include + +namespace webrtc { + +// Functor to use when supplying a verifier function for the queue item +// verifcation. +template +class RenderQueueItemVerifier { + public: + explicit RenderQueueItemVerifier(size_t minimum_capacity) + : minimum_capacity_(minimum_capacity) {} + + bool operator()(const std::vector& v) const { + return v.capacity() >= minimum_capacity_; + } + + private: + size_t minimum_capacity_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_RENDER_QUEUE_ITEM_VERIFIER_H__ diff --git a/VocieProcess/modules/audio_processing/rms_level.cc b/VocieProcess/modules/audio_processing/rms_level.cc new file mode 100644 index 0000000..b0a45cb --- /dev/null +++ b/VocieProcess/modules/audio_processing/rms_level.cc @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/rms_level.h" + +#include +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { +static constexpr float kMaxSquaredLevel = 32768 * 32768; +// kMinLevel is the level corresponding to kMinLevelDb, that is 10^(-127/10). +static constexpr float kMinLevel = 1.995262314968883e-13f; + +// Calculates the normalized RMS value from a mean square value. The input +// should be the sum of squared samples divided by the number of samples. The +// value will be normalized to full range before computing the RMS, wich is +// returned as a negated dBfs. That is, 0 is full amplitude while 127 is very +// faint. +int ComputeRms(float mean_square) { + if (mean_square <= kMinLevel * kMaxSquaredLevel) { + // Very faint; simply return the minimum value. + return RmsLevel::kMinLevelDb; + } + // Normalize by the max level. + const float mean_square_norm = mean_square / kMaxSquaredLevel; + RTC_DCHECK_GT(mean_square_norm, kMinLevel); + // 20log_10(x^0.5) = 10log_10(x) + const float rms = 10.f * std::log10(mean_square_norm); + RTC_DCHECK_LE(rms, 0.f); + RTC_DCHECK_GT(rms, -RmsLevel::kMinLevelDb); + // Return the negated value. + return static_cast(-rms + 0.5f); +} +} // namespace + +RmsLevel::RmsLevel() { + Reset(); +} + +RmsLevel::~RmsLevel() = default; + +void RmsLevel::Reset() { + sum_square_ = 0.f; + sample_count_ = 0; + max_sum_square_ = 0.f; + block_size_ = absl::nullopt; +} + +void RmsLevel::Analyze(rtc::ArrayView data) { + if (data.empty()) { + return; + } + + CheckBlockSize(data.size()); + + const float sum_square = + std::accumulate(data.begin(), data.end(), 0.f, + [](float a, int16_t b) { return a + b * b; }); + RTC_DCHECK_GE(sum_square, 0.f); + sum_square_ += sum_square; + sample_count_ += data.size(); + + max_sum_square_ = std::max(max_sum_square_, sum_square); +} + +void RmsLevel::Analyze(rtc::ArrayView data) { + if (data.empty()) { + return; + } + + CheckBlockSize(data.size()); + + float sum_square = 0.f; + + for (float data_k : data) { + int16_t tmp = + static_cast(std::min(std::max(data_k, -32768.f), 32767.f)); + sum_square += tmp * tmp; + } + RTC_DCHECK_GE(sum_square, 0.f); + sum_square_ += sum_square; + sample_count_ += data.size(); + + max_sum_square_ = std::max(max_sum_square_, sum_square); +} + +void RmsLevel::AnalyzeMuted(size_t length) { + CheckBlockSize(length); + sample_count_ += length; +} + +int RmsLevel::Average() { + const bool have_samples = (sample_count_ != 0); + int rms = have_samples ? ComputeRms(sum_square_ / sample_count_) + : RmsLevel::kMinLevelDb; + + // To ensure that kMinLevelDb represents digital silence (muted audio + // sources) we'll check here if the sum_square is actually 0. If it's not + // we'll bump up the return value to `kInaudibleButNotMuted`. + // https://datatracker.ietf.org/doc/html/rfc6464 + if (have_samples && rms == RmsLevel::kMinLevelDb && sum_square_ != 0.0f) { + rms = kInaudibleButNotMuted; + } + + Reset(); + return rms; +} + +RmsLevel::Levels RmsLevel::AverageAndPeak() { + // Note that block_size_ should by design always be non-empty when + // sample_count_ != 0. Also, the * operator of absl::optional enforces this + // with a DCHECK. + Levels levels = (sample_count_ == 0) + ? Levels{RmsLevel::kMinLevelDb, RmsLevel::kMinLevelDb} + : Levels{ComputeRms(sum_square_ / sample_count_), + ComputeRms(max_sum_square_ / *block_size_)}; + Reset(); + return levels; +} + +void RmsLevel::CheckBlockSize(size_t block_size) { + if (block_size_ != block_size) { + Reset(); + block_size_ = block_size; + } +} +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/rms_level.h b/VocieProcess/modules/audio_processing/rms_level.h new file mode 100644 index 0000000..fbece19 --- /dev/null +++ b/VocieProcess/modules/audio_processing/rms_level.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_RMS_LEVEL_H_ +#define MODULES_AUDIO_PROCESSING_RMS_LEVEL_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" + +namespace webrtc { + +// Computes the root mean square (RMS) level in dBFs (decibels from digital +// full-scale) of audio data. The computation follows RFC 6465: +// https://tools.ietf.org/html/rfc6465 +// with the intent that it can provide the RTP audio level indication. +// +// The expected approach is to provide constant-sized chunks of audio to +// Analyze(). When enough chunks have been accumulated to form a packet, call +// Average() to get the audio level indicator for the RTP header. +class RmsLevel { + public: + struct Levels { + int average; + int peak; + }; + + enum : int { kMinLevelDb = 127, kInaudibleButNotMuted = 126 }; + + RmsLevel(); + ~RmsLevel(); + + // Can be called to reset internal states, but is not required during normal + // operation. + void Reset(); + + // Pass each chunk of audio to Analyze() to accumulate the level. + void Analyze(rtc::ArrayView data); + void Analyze(rtc::ArrayView data); + + // If all samples with the given `length` have a magnitude of zero, this is + // a shortcut to avoid some computation. + void AnalyzeMuted(size_t length); + + // Computes the RMS level over all data passed to Analyze() since the last + // call to Average(). The returned value is positive but should be interpreted + // as negative as per the RFC. It is constrained to [0, 127]. Resets the + // internal state to start a new measurement period. + int Average(); + + // Like Average() above, but also returns the RMS peak value. Resets the + // internal state to start a new measurement period. + Levels AverageAndPeak(); + + private: + // Compares `block_size` with `block_size_`. If they are different, calls + // Reset() and stores the new size. + void CheckBlockSize(size_t block_size); + + float sum_square_; + size_t sample_count_; + float max_sum_square_; + absl::optional block_size_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_RMS_LEVEL_H_ diff --git a/VocieProcess/modules/audio_processing/vad/common.h b/VocieProcess/modules/audio_processing/vad/common.h new file mode 100644 index 0000000..b5a5fb3 --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/common.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_COMMON_H_ +#define MODULES_AUDIO_PROCESSING_VAD_COMMON_H_ + +#include + +static const int kSampleRateHz = 16000; +static const size_t kLength10Ms = kSampleRateHz / 100; +static const size_t kMaxNumFrames = 4; + +struct AudioFeatures { + double log_pitch_gain[kMaxNumFrames]; + double pitch_lag_hz[kMaxNumFrames]; + double spectral_peak[kMaxNumFrames]; + double rms[kMaxNumFrames]; + size_t num_frames; + bool silence; +}; + +#endif // MODULES_AUDIO_PROCESSING_VAD_COMMON_H_ diff --git a/VocieProcess/modules/audio_processing/vad/gmm.cc b/VocieProcess/modules/audio_processing/vad/gmm.cc new file mode 100644 index 0000000..3b8764c --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/gmm.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/gmm.h" + +#include + +namespace webrtc { + +static const int kMaxDimension = 10; + +static void RemoveMean(const double* in, + const double* mean_vec, + int dimension, + double* out) { + for (int n = 0; n < dimension; ++n) + out[n] = in[n] - mean_vec[n]; +} + +static double ComputeExponent(const double* in, + const double* covar_inv, + int dimension) { + double q = 0; + for (int i = 0; i < dimension; ++i) { + double v = 0; + for (int j = 0; j < dimension; j++) + v += (*covar_inv++) * in[j]; + q += v * in[i]; + } + q *= -0.5; + return q; +} + +double EvaluateGmm(const double* x, const GmmParameters& gmm_parameters) { + if (gmm_parameters.dimension > kMaxDimension) { + return -1; // This is invalid pdf so the caller can check this. + } + double f = 0; + double v[kMaxDimension]; + const double* mean_vec = gmm_parameters.mean; + const double* covar_inv = gmm_parameters.covar_inverse; + + for (int n = 0; n < gmm_parameters.num_mixtures; n++) { + RemoveMean(x, mean_vec, gmm_parameters.dimension, v); + double q = ComputeExponent(v, covar_inv, gmm_parameters.dimension) + + gmm_parameters.weight[n]; + f += exp(q); + mean_vec += gmm_parameters.dimension; + covar_inv += gmm_parameters.dimension * gmm_parameters.dimension; + } + return f; +} + +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/vad/gmm.h b/VocieProcess/modules/audio_processing/vad/gmm.h new file mode 100644 index 0000000..d9d68ec --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/gmm.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_GMM_H_ +#define MODULES_AUDIO_PROCESSING_VAD_GMM_H_ + +namespace webrtc { + +// A structure that specifies a GMM. +// A GMM is formulated as +// f(x) = w[0] * mixture[0] + w[1] * mixture[1] + ... + +// w[num_mixtures - 1] * mixture[num_mixtures - 1]; +// Where a 'mixture' is a Gaussian density. + +struct GmmParameters { + // weight[n] = log(w[n]) - `dimension`/2 * log(2*pi) - 1/2 * log(det(cov[n])); + // where cov[n] is the covariance matrix of mixture n; + const double* weight; + // pointer to the first element of a `num_mixtures`x`dimension` matrix + // where kth row is the mean of the kth mixture. + const double* mean; + // pointer to the first element of a `num_mixtures`x`dimension`x`dimension` + // 3D-matrix, where the kth 2D-matrix is the inverse of the covariance + // matrix of the kth mixture. + const double* covar_inverse; + // Dimensionality of the mixtures. + int dimension; + // number of the mixtures. + int num_mixtures; +}; + +// Evaluate the given GMM, according to `gmm_parameters`, at the given point +// `x`. If the dimensionality of the given GMM is larger that the maximum +// acceptable dimension by the following function -1 is returned. +double EvaluateGmm(const double* x, const GmmParameters& gmm_parameters); + +} // namespace webrtc +#endif // MODULES_AUDIO_PROCESSING_VAD_GMM_H_ diff --git a/VocieProcess/modules/audio_processing/vad/noise_gmm_tables.h b/VocieProcess/modules/audio_processing/vad/noise_gmm_tables.h new file mode 100644 index 0000000..944a540 --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/noise_gmm_tables.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// GMM tables for inactive segments. Generated by MakeGmmTables.m. + +#ifndef MODULES_AUDIO_PROCESSING_VAD_NOISE_GMM_TABLES_H_ +#define MODULES_AUDIO_PROCESSING_VAD_NOISE_GMM_TABLES_H_ + +namespace webrtc { + +static const int kNoiseGmmNumMixtures = 12; +static const int kNoiseGmmDim = 3; + +static const double + kNoiseGmmCovarInverse[kNoiseGmmNumMixtures][kNoiseGmmDim][kNoiseGmmDim] = { + {{7.36219567592941e+00, 4.83060785179861e-03, 1.23335151497610e-02}, + {4.83060785179861e-03, 1.65289507047817e-04, -2.41490588169997e-04}, + {1.23335151497610e-02, -2.41490588169997e-04, 6.59472060689382e-03}}, + {{8.70265239309140e+00, -5.30636201431086e-04, 5.44014966585347e-03}, + {-5.30636201431086e-04, 3.11095453521008e-04, -1.86287206836035e-04}, + {5.44014966585347e-03, -1.86287206836035e-04, 6.29493388790744e-04}}, + {{4.53467851955055e+00, -3.92977536695197e-03, -2.46521420693317e-03}, + {-3.92977536695197e-03, 4.94650752632750e-05, -1.08587438501826e-05}, + {-2.46521420693317e-03, -1.08587438501826e-05, 9.28793975422261e-05}}, + {{9.26817997114275e-01, -4.03976069276753e-04, -3.56441427392165e-03}, + {-4.03976069276753e-04, 2.51976251631430e-06, 1.46914206734572e-07}, + {-3.56441427392165e-03, 1.46914206734572e-07, 8.19914567685373e-05}}, + {{7.61715986787441e+00, -1.54889041216888e-04, 2.41756280071656e-02}, + {-1.54889041216888e-04, 3.50282550461672e-07, -6.27251196972490e-06}, + {2.41756280071656e-02, -6.27251196972490e-06, 1.45061847649872e-02}}, + {{8.31193642663158e+00, -3.84070508164323e-04, -3.09750630821876e-02}, + {-3.84070508164323e-04, 3.80433432277336e-07, -1.14321142836636e-06}, + {-3.09750630821876e-02, -1.14321142836636e-06, 8.35091486289997e-04}}, + {{9.67283151270894e-01, 5.82465812445039e-05, -3.18350798617053e-03}, + {5.82465812445039e-05, 2.23762672000318e-07, -7.74196587408623e-07}, + {-3.18350798617053e-03, -7.74196587408623e-07, 3.85120938338325e-04}}, + {{8.28066236985388e+00, 5.87634508319763e-05, 6.99303090891743e-03}, + {5.87634508319763e-05, 2.93746018618058e-07, 3.40843332882272e-07}, + {6.99303090891743e-03, 3.40843332882272e-07, 1.99379171190344e-04}}, + {{6.07488998675646e+00, -1.11494526618473e-02, 5.10013111123381e-03}, + {-1.11494526618473e-02, 6.99238879921751e-04, 5.36718550370870e-05}, + {5.10013111123381e-03, 5.36718550370870e-05, 5.26909853276753e-04}}, + {{6.90492021419175e+00, 4.20639355257863e-04, -2.38612752336481e-03}, + {4.20639355257863e-04, 3.31246767338153e-06, -2.42052288150859e-08}, + {-2.38612752336481e-03, -2.42052288150859e-08, 4.46608368363412e-04}}, + {{1.31069150869715e+01, -1.73718583865670e-04, -1.97591814508578e-02}, + {-1.73718583865670e-04, 2.80451716300124e-07, 9.96570755379865e-07}, + {-1.97591814508578e-02, 9.96570755379865e-07, 2.41361900868847e-03}}, + {{4.69566344239814e+00, -2.61077567563690e-04, 5.26359000761433e-03}, + {-2.61077567563690e-04, 1.82420859823767e-06, -7.83645887541601e-07}, + {5.26359000761433e-03, -7.83645887541601e-07, 1.33586288288802e-02}}}; + +static const double kNoiseGmmMean[kNoiseGmmNumMixtures][kNoiseGmmDim] = { + {-2.01386094766163e+00, 1.69702162045397e+02, 7.41715804872181e+01}, + {-1.94684591777290e+00, 1.42398396732668e+02, 1.64186321157831e+02}, + {-2.29319297562437e+00, 3.86415425589868e+02, 2.13452215267125e+02}, + {-3.25487177070268e+00, 1.08668712553616e+03, 2.33119949467419e+02}, + {-2.13159632447467e+00, 4.83821702557717e+03, 6.86786166673740e+01}, + {-2.26171410780526e+00, 4.79420193982422e+03, 1.53222513286450e+02}, + {-3.32166740703185e+00, 4.35161135834358e+03, 1.33206448431316e+02}, + {-2.19290322814343e+00, 3.98325506609408e+03, 2.13249167359934e+02}, + {-2.02898459255404e+00, 7.37039893155007e+03, 1.12518527491926e+02}, + {-2.26150236399500e+00, 1.54896745196145e+03, 1.49717357868579e+02}, + {-2.00417668301790e+00, 3.82434760310304e+03, 1.07438913004312e+02}, + {-2.30193040814533e+00, 1.43953696546439e+03, 7.04085275122649e+01}}; + +static const double kNoiseGmmWeights[kNoiseGmmNumMixtures] = { + -1.09422832086193e+01, -1.10847897513425e+01, -1.36767587732187e+01, + -1.79789356118641e+01, -1.42830169160894e+01, -1.56500228061379e+01, + -1.83124990950113e+01, -1.69979436177477e+01, -1.12329424387828e+01, + -1.41311785780639e+01, -1.47171861448585e+01, -1.35963362781839e+01}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_VAD_NOISE_GMM_TABLES_H_ diff --git a/VocieProcess/modules/audio_processing/vad/pitch_based_vad.cc b/VocieProcess/modules/audio_processing/vad/pitch_based_vad.cc new file mode 100644 index 0000000..68e60dc --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/pitch_based_vad.cc @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/pitch_based_vad.h" + +#include + +#include "modules/audio_processing/vad/common.h" +#include "modules/audio_processing/vad/noise_gmm_tables.h" +#include "modules/audio_processing/vad/vad_circular_buffer.h" +#include "modules/audio_processing/vad/voice_gmm_tables.h" + +namespace webrtc { + +static_assert(kNoiseGmmDim == kVoiceGmmDim, + "noise and voice gmm dimension not equal"); + +// These values should match MATLAB counterparts for unit-tests to pass. +static const int kPosteriorHistorySize = 500; // 5 sec of 10 ms frames. +static const double kInitialPriorProbability = 0.3; +static const int kTransientWidthThreshold = 7; +static const double kLowProbabilityThreshold = 0.2; + +static double LimitProbability(double p) { + const double kLimHigh = 0.99; + const double kLimLow = 0.01; + + if (p > kLimHigh) + p = kLimHigh; + else if (p < kLimLow) + p = kLimLow; + return p; +} + +PitchBasedVad::PitchBasedVad() + : p_prior_(kInitialPriorProbability), + circular_buffer_(VadCircularBuffer::Create(kPosteriorHistorySize)) { + // Setup noise GMM. + noise_gmm_.dimension = kNoiseGmmDim; + noise_gmm_.num_mixtures = kNoiseGmmNumMixtures; + noise_gmm_.weight = kNoiseGmmWeights; + noise_gmm_.mean = &kNoiseGmmMean[0][0]; + noise_gmm_.covar_inverse = &kNoiseGmmCovarInverse[0][0][0]; + + // Setup voice GMM. + voice_gmm_.dimension = kVoiceGmmDim; + voice_gmm_.num_mixtures = kVoiceGmmNumMixtures; + voice_gmm_.weight = kVoiceGmmWeights; + voice_gmm_.mean = &kVoiceGmmMean[0][0]; + voice_gmm_.covar_inverse = &kVoiceGmmCovarInverse[0][0][0]; +} + +PitchBasedVad::~PitchBasedVad() {} + +int PitchBasedVad::VoicingProbability(const AudioFeatures& features, + double* p_combined) { + double p; + double gmm_features[3]; + double pdf_features_given_voice; + double pdf_features_given_noise; + // These limits are the same in matlab implementation 'VoicingProbGMM().' + const double kLimLowLogPitchGain = -2.0; + const double kLimHighLogPitchGain = -0.9; + const double kLimLowSpectralPeak = 200; + const double kLimHighSpectralPeak = 2000; + const double kEps = 1e-12; + for (size_t n = 0; n < features.num_frames; n++) { + gmm_features[0] = features.log_pitch_gain[n]; + gmm_features[1] = features.spectral_peak[n]; + gmm_features[2] = features.pitch_lag_hz[n]; + + pdf_features_given_voice = EvaluateGmm(gmm_features, voice_gmm_); + pdf_features_given_noise = EvaluateGmm(gmm_features, noise_gmm_); + + if (features.spectral_peak[n] < kLimLowSpectralPeak || + features.spectral_peak[n] > kLimHighSpectralPeak || + features.log_pitch_gain[n] < kLimLowLogPitchGain) { + pdf_features_given_voice = kEps * pdf_features_given_noise; + } else if (features.log_pitch_gain[n] > kLimHighLogPitchGain) { + pdf_features_given_noise = kEps * pdf_features_given_voice; + } + + p = p_prior_ * pdf_features_given_voice / + (pdf_features_given_voice * p_prior_ + + pdf_features_given_noise * (1 - p_prior_)); + + p = LimitProbability(p); + + // Combine pitch-based probability with standalone probability, before + // updating prior probabilities. + double prod_active = p * p_combined[n]; + double prod_inactive = (1 - p) * (1 - p_combined[n]); + p_combined[n] = prod_active / (prod_active + prod_inactive); + + if (UpdatePrior(p_combined[n]) < 0) + return -1; + // Limit prior probability. With a zero prior probability the posterior + // probability is always zero. + p_prior_ = LimitProbability(p_prior_); + } + return 0; +} + +int PitchBasedVad::UpdatePrior(double p) { + circular_buffer_->Insert(p); + if (circular_buffer_->RemoveTransient(kTransientWidthThreshold, + kLowProbabilityThreshold) < 0) + return -1; + p_prior_ = circular_buffer_->Mean(); + return 0; +} + +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/vad/pitch_based_vad.h b/VocieProcess/modules/audio_processing/vad/pitch_based_vad.h new file mode 100644 index 0000000..fa3abc2 --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/pitch_based_vad.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_PITCH_BASED_VAD_H_ +#define MODULES_AUDIO_PROCESSING_VAD_PITCH_BASED_VAD_H_ + +#include + +#include "modules/audio_processing/vad/common.h" +#include "modules/audio_processing/vad/gmm.h" + +namespace webrtc { + +class VadCircularBuffer; + +// Computes the probability of the input audio frame to be active given +// the corresponding pitch-gain and lag of the frame. +class PitchBasedVad { + public: + PitchBasedVad(); + ~PitchBasedVad(); + + // Compute pitch-based voicing probability, given the features. + // features: a structure containing features required for computing voicing + // probabilities. + // + // p_combined: an array which contains the combined activity probabilities + // computed prior to the call of this function. The method, + // then, computes the voicing probabilities and combine them + // with the given values. The result are returned in `p`. + int VoicingProbability(const AudioFeatures& features, double* p_combined); + + private: + int UpdatePrior(double p); + + // TODO(turajs): maybe defining this at a higher level (maybe enum) so that + // all the code recognize it as "no-error." + static const int kNoError = 0; + + GmmParameters noise_gmm_; + GmmParameters voice_gmm_; + + double p_prior_; + + std::unique_ptr circular_buffer_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_VAD_PITCH_BASED_VAD_H_ diff --git a/VocieProcess/modules/audio_processing/vad/pitch_internal.cc b/VocieProcess/modules/audio_processing/vad/pitch_internal.cc new file mode 100644 index 0000000..8f86918 --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/pitch_internal.cc @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/pitch_internal.h" + +#include + +namespace webrtc { + +// A 4-to-3 linear interpolation. +// The interpolation constants are derived as following: +// Input pitch parameters are updated every 7.5 ms. Within a 30-ms interval +// we are interested in pitch parameters of 0-5 ms, 10-15ms and 20-25ms. This is +// like interpolating 4-to-6 and keep the odd samples. +// The reason behind this is that LPC coefficients are computed for the first +// half of each 10ms interval. +static void PitchInterpolation(double old_val, const double* in, double* out) { + out[0] = 1. / 6. * old_val + 5. / 6. * in[0]; + out[1] = 5. / 6. * in[1] + 1. / 6. * in[2]; + out[2] = 0.5 * in[2] + 0.5 * in[3]; +} + +void GetSubframesPitchParameters(int sampling_rate_hz, + double* gains, + double* lags, + int num_in_frames, + int num_out_frames, + double* log_old_gain, + double* old_lag, + double* log_pitch_gain, + double* pitch_lag_hz) { + // Gain interpolation is in log-domain, also returned in log-domain. + for (int n = 0; n < num_in_frames; n++) + gains[n] = log(gains[n] + 1e-12); + + // Interpolate lags and gains. + PitchInterpolation(*log_old_gain, gains, log_pitch_gain); + *log_old_gain = gains[num_in_frames - 1]; + PitchInterpolation(*old_lag, lags, pitch_lag_hz); + *old_lag = lags[num_in_frames - 1]; + + // Convert pitch-lags to Hertz. + for (int n = 0; n < num_out_frames; n++) { + pitch_lag_hz[n] = (sampling_rate_hz) / (pitch_lag_hz[n]); + } +} + +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/vad/pitch_internal.h b/VocieProcess/modules/audio_processing/vad/pitch_internal.h new file mode 100644 index 0000000..e382c1f --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/pitch_internal.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_PITCH_INTERNAL_H_ +#define MODULES_AUDIO_PROCESSING_VAD_PITCH_INTERNAL_H_ + +namespace webrtc { + +// TODO(turajs): Write a description of this function. Also be consistent with +// usage of `sampling_rate_hz` vs `kSamplingFreqHz`. +void GetSubframesPitchParameters(int sampling_rate_hz, + double* gains, + double* lags, + int num_in_frames, + int num_out_frames, + double* log_old_gain, + double* old_lag, + double* log_pitch_gain, + double* pitch_lag_hz); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_VAD_PITCH_INTERNAL_H_ diff --git a/VocieProcess/modules/audio_processing/vad/pole_zero_filter.cc b/VocieProcess/modules/audio_processing/vad/pole_zero_filter.cc new file mode 100644 index 0000000..e7a6113 --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/pole_zero_filter.cc @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/pole_zero_filter.h" + +#include + +#include + +namespace webrtc { + +PoleZeroFilter* PoleZeroFilter::Create(const float* numerator_coefficients, + size_t order_numerator, + const float* denominator_coefficients, + size_t order_denominator) { + if (order_numerator > kMaxFilterOrder || + order_denominator > kMaxFilterOrder || denominator_coefficients[0] == 0 || + numerator_coefficients == NULL || denominator_coefficients == NULL) + return NULL; + return new PoleZeroFilter(numerator_coefficients, order_numerator, + denominator_coefficients, order_denominator); +} + +PoleZeroFilter::PoleZeroFilter(const float* numerator_coefficients, + size_t order_numerator, + const float* denominator_coefficients, + size_t order_denominator) + : past_input_(), + past_output_(), + numerator_coefficients_(), + denominator_coefficients_(), + order_numerator_(order_numerator), + order_denominator_(order_denominator), + highest_order_(std::max(order_denominator, order_numerator)) { + memcpy(numerator_coefficients_, numerator_coefficients, + sizeof(numerator_coefficients_[0]) * (order_numerator_ + 1)); + memcpy(denominator_coefficients_, denominator_coefficients, + sizeof(denominator_coefficients_[0]) * (order_denominator_ + 1)); + + if (denominator_coefficients_[0] != 1) { + for (size_t n = 0; n <= order_numerator_; n++) + numerator_coefficients_[n] /= denominator_coefficients_[0]; + for (size_t n = 0; n <= order_denominator_; n++) + denominator_coefficients_[n] /= denominator_coefficients_[0]; + } +} + +template +static float FilterArPast(const T* past, + size_t order, + const float* coefficients) { + float sum = 0.0f; + size_t past_index = order - 1; + for (size_t k = 1; k <= order; k++, past_index--) + sum += coefficients[k] * past[past_index]; + return sum; +} + +int PoleZeroFilter::Filter(const int16_t* in, + size_t num_input_samples, + float* output) { + if (in == NULL || output == NULL) + return -1; + // This is the typical case, just a memcpy. + const size_t k = std::min(num_input_samples, highest_order_); + size_t n; + for (n = 0; n < k; n++) { + output[n] = in[n] * numerator_coefficients_[0]; + output[n] += FilterArPast(&past_input_[n], order_numerator_, + numerator_coefficients_); + output[n] -= FilterArPast(&past_output_[n], order_denominator_, + denominator_coefficients_); + + past_input_[n + order_numerator_] = in[n]; + past_output_[n + order_denominator_] = output[n]; + } + if (highest_order_ < num_input_samples) { + for (size_t m = 0; n < num_input_samples; n++, m++) { + output[n] = in[n] * numerator_coefficients_[0]; + output[n] += + FilterArPast(&in[m], order_numerator_, numerator_coefficients_); + output[n] -= FilterArPast(&output[m], order_denominator_, + denominator_coefficients_); + } + // Record into the past signal. + memcpy(past_input_, &in[num_input_samples - order_numerator_], + sizeof(in[0]) * order_numerator_); + memcpy(past_output_, &output[num_input_samples - order_denominator_], + sizeof(output[0]) * order_denominator_); + } else { + // Odd case that the length of the input is shorter that filter order. + memmove(past_input_, &past_input_[num_input_samples], + order_numerator_ * sizeof(past_input_[0])); + memmove(past_output_, &past_output_[num_input_samples], + order_denominator_ * sizeof(past_output_[0])); + } + return 0; +} + +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/vad/pole_zero_filter.h b/VocieProcess/modules/audio_processing/vad/pole_zero_filter.h new file mode 100644 index 0000000..11a0511 --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/pole_zero_filter.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_POLE_ZERO_FILTER_H_ +#define MODULES_AUDIO_PROCESSING_VAD_POLE_ZERO_FILTER_H_ + +#include +#include + +namespace webrtc { + +class PoleZeroFilter { + public: + ~PoleZeroFilter() {} + + static PoleZeroFilter* Create(const float* numerator_coefficients, + size_t order_numerator, + const float* denominator_coefficients, + size_t order_denominator); + + int Filter(const int16_t* in, size_t num_input_samples, float* output); + + private: + PoleZeroFilter(const float* numerator_coefficients, + size_t order_numerator, + const float* denominator_coefficients, + size_t order_denominator); + + static const int kMaxFilterOrder = 24; + + int16_t past_input_[kMaxFilterOrder * 2]; + float past_output_[kMaxFilterOrder * 2]; + + float numerator_coefficients_[kMaxFilterOrder + 1]; + float denominator_coefficients_[kMaxFilterOrder + 1]; + + size_t order_numerator_; + size_t order_denominator_; + size_t highest_order_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_VAD_POLE_ZERO_FILTER_H_ diff --git a/VocieProcess/modules/audio_processing/vad/standalone_vad.cc b/VocieProcess/modules/audio_processing/vad/standalone_vad.cc new file mode 100644 index 0000000..1397668 --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/standalone_vad.cc @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/standalone_vad.h" + +#include + +#include "common_audio/vad/include/webrtc_vad.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +static const int kDefaultStandaloneVadMode = 3; + +StandaloneVad::StandaloneVad(VadInst* vad) + : vad_(vad), buffer_(), index_(0), mode_(kDefaultStandaloneVadMode) {} + +StandaloneVad::~StandaloneVad() { + WebRtcVad_Free(vad_); +} + +StandaloneVad* StandaloneVad::Create() { + VadInst* vad = WebRtcVad_Create(); + if (!vad) + return nullptr; + + int err = WebRtcVad_Init(vad); + err |= WebRtcVad_set_mode(vad, kDefaultStandaloneVadMode); + if (err != 0) { + WebRtcVad_Free(vad); + return nullptr; + } + return new StandaloneVad(vad); +} + +int StandaloneVad::AddAudio(const int16_t* data, size_t length) { + if (length != kLength10Ms) + return -1; + + if (index_ + length > kLength10Ms * kMaxNum10msFrames) + // Reset the buffer if it's full. + // TODO(ajm): Instead, consider just processing every 10 ms frame. Then we + // can forgo the buffering. + index_ = 0; + + memcpy(&buffer_[index_], data, sizeof(int16_t) * length); + index_ += length; + return 0; +} + +int StandaloneVad::GetActivity(double* p, size_t length_p) { + if (index_ == 0) + return -1; + + const size_t num_frames = index_ / kLength10Ms; + if (num_frames > length_p) + return -1; + RTC_DCHECK_EQ(0, WebRtcVad_ValidRateAndFrameLength(kSampleRateHz, index_)); + + int activity = WebRtcVad_Process(vad_, kSampleRateHz, buffer_, index_); + if (activity < 0) + return -1; + else if (activity == 0) + p[0] = 0.01; // Arbitrary but small and non-zero. + else + p[0] = 0.5; // 0.5 is neutral values when combinned by other probabilities. + for (size_t n = 1; n < num_frames; n++) + p[n] = p[0]; + // Reset the buffer to start from the beginning. + index_ = 0; + return activity; +} + +int StandaloneVad::set_mode(int mode) { + if (mode < 0 || mode > 3) + return -1; + if (WebRtcVad_set_mode(vad_, mode) != 0) + return -1; + + mode_ = mode; + return 0; +} + +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/vad/standalone_vad.h b/VocieProcess/modules/audio_processing/vad/standalone_vad.h new file mode 100644 index 0000000..b084633 --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/standalone_vad.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC_STANDALONE_VAD_H_ +#define MODULES_AUDIO_PROCESSING_AGC_STANDALONE_VAD_H_ + +#include +#include + +#include "common_audio/vad/include/webrtc_vad.h" +#include "modules/audio_processing/vad/common.h" + +namespace webrtc { + +class StandaloneVad { + public: + static StandaloneVad* Create(); + ~StandaloneVad(); + + // Outputs + // p: a buffer where probabilities are written to. + // length_p: number of elements of `p`. + // + // return value: + // -1: if no audio is stored or VAD returns error. + // 0: in success. + // In case of error the content of `activity` is unchanged. + // + // Note that due to a high false-positive (VAD decision is active while the + // processed audio is just background noise) rate, stand-alone VAD is used as + // a one-sided indicator. The activity probability is 0.5 if the frame is + // classified as active, and the probability is 0.01 if the audio is + // classified as passive. In this way, when probabilities are combined, the + // effect of the stand-alone VAD is neutral if the input is classified as + // active. + int GetActivity(double* p, size_t length_p); + + // Expecting 10 ms of 16 kHz audio to be pushed in. + int AddAudio(const int16_t* data, size_t length); + + // Set aggressiveness of VAD, 0 is the least aggressive and 3 is the most + // aggressive mode. Returns -1 if the input is less than 0 or larger than 3, + // otherwise 0 is returned. + int set_mode(int mode); + // Get the agressiveness of the current VAD. + int mode() const { return mode_; } + + private: + explicit StandaloneVad(VadInst* vad); + + static const size_t kMaxNum10msFrames = 3; + + // TODO(turajs): Is there a way to use scoped-pointer here? + VadInst* vad_; + int16_t buffer_[kMaxNum10msFrames * kLength10Ms]; + size_t index_; + int mode_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC_STANDALONE_VAD_H_ diff --git a/VocieProcess/modules/audio_processing/vad/vad_audio_proc.cc b/VocieProcess/modules/audio_processing/vad/vad_audio_proc.cc new file mode 100644 index 0000000..aaf8214 --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/vad_audio_proc.cc @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/vad_audio_proc.h" + +#include +#include +#include + +#include "common_audio/third_party/ooura/fft_size_256/fft4g.h" +#include "modules/audio_processing/vad/pitch_internal.h" +#include "modules/audio_processing/vad/pole_zero_filter.h" +#include "modules/audio_processing/vad/vad_audio_proc_internal.h" +#include "rtc_base/checks.h" +extern "C" { +#include "modules/audio_coding/codecs/isac/main/source/filter_functions.h" +#include "modules/audio_coding/codecs/isac/main/source/isac_vad.h" +#include "modules/audio_coding/codecs/isac/main/source/pitch_estimator.h" +#include "modules/audio_coding/codecs/isac/main/source/structs.h" +} + +namespace webrtc { + +// The following structures are declared anonymous in iSAC's structs.h. To +// forward declare them, we use this derived class trick. +struct VadAudioProc::PitchAnalysisStruct : public ::PitchAnalysisStruct {}; +struct VadAudioProc::PreFiltBankstr : public ::PreFiltBankstr {}; + +static constexpr float kFrequencyResolution = + kSampleRateHz / static_cast(VadAudioProc::kDftSize); +static constexpr int kSilenceRms = 5; + +// TODO(turajs): Make a Create or Init for VadAudioProc. +VadAudioProc::VadAudioProc() + : audio_buffer_(), + num_buffer_samples_(kNumPastSignalSamples), + log_old_gain_(-2), + old_lag_(50), // Arbitrary but valid as pitch-lag (in samples). + pitch_analysis_handle_(new PitchAnalysisStruct), + pre_filter_handle_(new PreFiltBankstr), + high_pass_filter_(PoleZeroFilter::Create(kCoeffNumerator, + kFilterOrder, + kCoeffDenominator, + kFilterOrder)) { + static_assert(kNumPastSignalSamples + kNumSubframeSamples == + sizeof(kLpcAnalWin) / sizeof(kLpcAnalWin[0]), + "lpc analysis window incorrect size"); + static_assert(kLpcOrder + 1 == sizeof(kCorrWeight) / sizeof(kCorrWeight[0]), + "correlation weight incorrect size"); + + // TODO(turajs): Are we doing too much in the constructor? + float data[kDftSize]; + // Make FFT to initialize. + ip_[0] = 0; + WebRtc_rdft(kDftSize, 1, data, ip_, w_fft_); + // TODO(turajs): Need to initialize high-pass filter. + + // Initialize iSAC components. + WebRtcIsac_InitPreFilterbank(pre_filter_handle_.get()); + WebRtcIsac_InitPitchAnalysis(pitch_analysis_handle_.get()); +} + +VadAudioProc::~VadAudioProc() {} + +void VadAudioProc::ResetBuffer() { + memcpy(audio_buffer_, &audio_buffer_[kNumSamplesToProcess], + sizeof(audio_buffer_[0]) * kNumPastSignalSamples); + num_buffer_samples_ = kNumPastSignalSamples; +} + +int VadAudioProc::ExtractFeatures(const int16_t* frame, + size_t length, + AudioFeatures* features) { + features->num_frames = 0; + if (length != kNumSubframeSamples) { + return -1; + } + + // High-pass filter to remove the DC component and very low frequency content. + // We have experienced that this high-pass filtering improves voice/non-voiced + // classification. + if (high_pass_filter_->Filter(frame, kNumSubframeSamples, + &audio_buffer_[num_buffer_samples_]) != 0) { + return -1; + } + + num_buffer_samples_ += kNumSubframeSamples; + if (num_buffer_samples_ < kBufferLength) { + return 0; + } + RTC_DCHECK_EQ(num_buffer_samples_, kBufferLength); + features->num_frames = kNum10msSubframes; + features->silence = false; + + Rms(features->rms, kMaxNumFrames); + for (size_t i = 0; i < kNum10msSubframes; ++i) { + if (features->rms[i] < kSilenceRms) { + // PitchAnalysis can cause NaNs in the pitch gain if it's fed silence. + // Bail out here instead. + features->silence = true; + ResetBuffer(); + return 0; + } + } + + PitchAnalysis(features->log_pitch_gain, features->pitch_lag_hz, + kMaxNumFrames); + FindFirstSpectralPeaks(features->spectral_peak, kMaxNumFrames); + ResetBuffer(); + return 0; +} + +// Computes |kLpcOrder + 1| correlation coefficients. +void VadAudioProc::SubframeCorrelation(double* corr, + size_t length_corr, + size_t subframe_index) { + RTC_DCHECK_GE(length_corr, kLpcOrder + 1); + double windowed_audio[kNumSubframeSamples + kNumPastSignalSamples]; + size_t buffer_index = subframe_index * kNumSubframeSamples; + + for (size_t n = 0; n < kNumSubframeSamples + kNumPastSignalSamples; n++) + windowed_audio[n] = audio_buffer_[buffer_index++] * kLpcAnalWin[n]; + + WebRtcIsac_AutoCorr(corr, windowed_audio, + kNumSubframeSamples + kNumPastSignalSamples, kLpcOrder); +} + +// Compute `kNum10msSubframes` sets of LPC coefficients, one per 10 ms input. +// The analysis window is 15 ms long and it is centered on the first half of +// each 10ms sub-frame. This is equivalent to computing LPC coefficients for the +// first half of each 10 ms subframe. +void VadAudioProc::GetLpcPolynomials(double* lpc, size_t length_lpc) { + RTC_DCHECK_GE(length_lpc, kNum10msSubframes * (kLpcOrder + 1)); + double corr[kLpcOrder + 1]; + double reflec_coeff[kLpcOrder]; + for (size_t i = 0, offset_lpc = 0; i < kNum10msSubframes; + i++, offset_lpc += kLpcOrder + 1) { + SubframeCorrelation(corr, kLpcOrder + 1, i); + corr[0] *= 1.0001; + // This makes Lev-Durb a bit more stable. + for (size_t k = 0; k < kLpcOrder + 1; k++) { + corr[k] *= kCorrWeight[k]; + } + WebRtcIsac_LevDurb(&lpc[offset_lpc], reflec_coeff, corr, kLpcOrder); + } +} + +// Fit a second order curve to these 3 points and find the location of the +// extremum. The points are inverted before curve fitting. +static float QuadraticInterpolation(float prev_val, + float curr_val, + float next_val) { + // Doing the interpolation in |1 / A(z)|^2. + float fractional_index = 0; + next_val = 1.0f / next_val; + prev_val = 1.0f / prev_val; + curr_val = 1.0f / curr_val; + + fractional_index = + -(next_val - prev_val) * 0.5f / (next_val + prev_val - 2.f * curr_val); + RTC_DCHECK_LT(fabs(fractional_index), 1); + return fractional_index; +} + +// 1 / A(z), where A(z) is defined by `lpc` is a model of the spectral envelope +// of the input signal. The local maximum of the spectral envelope corresponds +// with the local minimum of A(z). It saves complexity, as we save one +// inversion. Furthermore, we find the first local maximum of magnitude squared, +// to save on one square root. +void VadAudioProc::FindFirstSpectralPeaks(double* f_peak, + size_t length_f_peak) { + RTC_DCHECK_GE(length_f_peak, kNum10msSubframes); + double lpc[kNum10msSubframes * (kLpcOrder + 1)]; + // For all sub-frames. + GetLpcPolynomials(lpc, kNum10msSubframes * (kLpcOrder + 1)); + + const size_t kNumDftCoefficients = kDftSize / 2 + 1; + float data[kDftSize]; + + for (size_t i = 0; i < kNum10msSubframes; i++) { + // Convert to float with zero pad. + memset(data, 0, sizeof(data)); + for (size_t n = 0; n < kLpcOrder + 1; n++) { + data[n] = static_cast(lpc[i * (kLpcOrder + 1) + n]); + } + // Transform to frequency domain. + WebRtc_rdft(kDftSize, 1, data, ip_, w_fft_); + + size_t index_peak = 0; + float prev_magn_sqr = data[0] * data[0]; + float curr_magn_sqr = data[2] * data[2] + data[3] * data[3]; + float next_magn_sqr; + bool found_peak = false; + for (size_t n = 2; n < kNumDftCoefficients - 1; n++) { + next_magn_sqr = + data[2 * n] * data[2 * n] + data[2 * n + 1] * data[2 * n + 1]; + if (curr_magn_sqr < prev_magn_sqr && curr_magn_sqr < next_magn_sqr) { + found_peak = true; + index_peak = n - 1; + break; + } + prev_magn_sqr = curr_magn_sqr; + curr_magn_sqr = next_magn_sqr; + } + float fractional_index = 0; + if (!found_peak) { + // Checking if |kNumDftCoefficients - 1| is the local minimum. + next_magn_sqr = data[1] * data[1]; + if (curr_magn_sqr < prev_magn_sqr && curr_magn_sqr < next_magn_sqr) { + index_peak = kNumDftCoefficients - 1; + } + } else { + // A peak is found, do a simple quadratic interpolation to get a more + // accurate estimate of the peak location. + fractional_index = + QuadraticInterpolation(prev_magn_sqr, curr_magn_sqr, next_magn_sqr); + } + f_peak[i] = (index_peak + fractional_index) * kFrequencyResolution; + } +} + +// Using iSAC functions to estimate pitch gains & lags. +void VadAudioProc::PitchAnalysis(double* log_pitch_gains, + double* pitch_lags_hz, + size_t length) { + // TODO(turajs): This can be "imported" from iSAC & and the next two + // constants. + RTC_DCHECK_GE(length, kNum10msSubframes); + const int kNumPitchSubframes = 4; + double gains[kNumPitchSubframes]; + double lags[kNumPitchSubframes]; + + const int kNumSubbandFrameSamples = 240; + const int kNumLookaheadSamples = 24; + + float lower[kNumSubbandFrameSamples]; + float upper[kNumSubbandFrameSamples]; + double lower_lookahead[kNumSubbandFrameSamples]; + double upper_lookahead[kNumSubbandFrameSamples]; + double lower_lookahead_pre_filter[kNumSubbandFrameSamples + + kNumLookaheadSamples]; + + // Split signal to lower and upper bands + WebRtcIsac_SplitAndFilterFloat(&audio_buffer_[kNumPastSignalSamples], lower, + upper, lower_lookahead, upper_lookahead, + pre_filter_handle_.get()); + WebRtcIsac_PitchAnalysis(lower_lookahead, lower_lookahead_pre_filter, + pitch_analysis_handle_.get(), lags, gains); + + // Lags are computed on lower-band signal with sampling rate half of the + // input signal. + GetSubframesPitchParameters( + kSampleRateHz / 2, gains, lags, kNumPitchSubframes, kNum10msSubframes, + &log_old_gain_, &old_lag_, log_pitch_gains, pitch_lags_hz); +} + +void VadAudioProc::Rms(double* rms, size_t length_rms) { + RTC_DCHECK_GE(length_rms, kNum10msSubframes); + size_t offset = kNumPastSignalSamples; + for (size_t i = 0; i < kNum10msSubframes; i++) { + rms[i] = 0; + for (size_t n = 0; n < kNumSubframeSamples; n++, offset++) + rms[i] += audio_buffer_[offset] * audio_buffer_[offset]; + rms[i] = sqrt(rms[i] / kNumSubframeSamples); + } +} + +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/vad/vad_audio_proc.h b/VocieProcess/modules/audio_processing/vad/vad_audio_proc.h new file mode 100644 index 0000000..905c687 --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/vad_audio_proc.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROC_H_ +#define MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROC_H_ + +#include +#include + +#include + +#include "modules/audio_processing/vad/common.h" // AudioFeatures, kSampleR... + +namespace webrtc { + +class PoleZeroFilter; + +class VadAudioProc { + public: + // Forward declare iSAC structs. + struct PitchAnalysisStruct; + struct PreFiltBankstr; + + VadAudioProc(); + ~VadAudioProc(); + + int ExtractFeatures(const int16_t* audio_frame, + size_t length, + AudioFeatures* audio_features); + + static constexpr size_t kDftSize = 512; + + private: + void PitchAnalysis(double* pitch_gains, double* pitch_lags_hz, size_t length); + void SubframeCorrelation(double* corr, + size_t length_corr, + size_t subframe_index); + void GetLpcPolynomials(double* lpc, size_t length_lpc); + void FindFirstSpectralPeaks(double* f_peak, size_t length_f_peak); + void Rms(double* rms, size_t length_rms); + void ResetBuffer(); + + // To compute spectral peak we perform LPC analysis to get spectral envelope. + // For every 30 ms we compute 3 spectral peak there for 3 LPC analysis. + // LPC is computed over 15 ms of windowed audio. For every 10 ms sub-frame + // we need 5 ms of past signal to create the input of LPC analysis. + static constexpr size_t kNumPastSignalSamples = size_t{kSampleRateHz / 200}; + + // TODO(turajs): maybe defining this at a higher level (maybe enum) so that + // all the code recognize it as "no-error." + static constexpr int kNoError = 0; + + static constexpr size_t kNum10msSubframes = 3; + static constexpr size_t kNumSubframeSamples = size_t{kSampleRateHz / 100}; + // Samples in 30 ms @ given sampling rate. + static constexpr size_t kNumSamplesToProcess = + kNum10msSubframes * kNumSubframeSamples; + static constexpr size_t kBufferLength = + kNumPastSignalSamples + kNumSamplesToProcess; + static constexpr size_t kIpLength = kDftSize >> 1; + static constexpr size_t kWLength = kDftSize >> 1; + static constexpr size_t kLpcOrder = 16; + + size_t ip_[kIpLength]; + float w_fft_[kWLength]; + + // A buffer of 5 ms (past audio) + 30 ms (one iSAC frame ). + float audio_buffer_[kBufferLength]; + size_t num_buffer_samples_; + + double log_old_gain_; + double old_lag_; + + std::unique_ptr pitch_analysis_handle_; + std::unique_ptr pre_filter_handle_; + std::unique_ptr high_pass_filter_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROC_H_ diff --git a/VocieProcess/modules/audio_processing/vad/vad_audio_proc_internal.h b/VocieProcess/modules/audio_processing/vad/vad_audio_proc_internal.h new file mode 100644 index 0000000..93589af --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/vad_audio_proc_internal.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROC_INTERNAL_H_ +#define MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROC_INTERNAL_H_ + +#include + +namespace webrtc { + +// These values should match MATLAB counterparts for unit-tests to pass. +static const double kCorrWeight[] = { + 1.000000, 0.985000, 0.970225, 0.955672, 0.941337, 0.927217, + 0.913308, 0.899609, 0.886115, 0.872823, 0.859730, 0.846834, + 0.834132, 0.821620, 0.809296, 0.797156, 0.785199}; + +static const double kLpcAnalWin[] = { + 0.00000000, 0.01314436, 0.02628645, 0.03942400, 0.05255473, 0.06567639, + 0.07878670, 0.09188339, 0.10496421, 0.11802689, 0.13106918, 0.14408883, + 0.15708358, 0.17005118, 0.18298941, 0.19589602, 0.20876878, 0.22160547, + 0.23440387, 0.24716177, 0.25987696, 0.27254725, 0.28517045, 0.29774438, + 0.31026687, 0.32273574, 0.33514885, 0.34750406, 0.35979922, 0.37203222, + 0.38420093, 0.39630327, 0.40833713, 0.42030043, 0.43219112, 0.44400713, + 0.45574642, 0.46740697, 0.47898676, 0.49048379, 0.50189608, 0.51322164, + 0.52445853, 0.53560481, 0.54665854, 0.55761782, 0.56848075, 0.57924546, + 0.58991008, 0.60047278, 0.61093173, 0.62128512, 0.63153117, 0.64166810, + 0.65169416, 0.66160761, 0.67140676, 0.68108990, 0.69065536, 0.70010148, + 0.70942664, 0.71862923, 0.72770765, 0.73666033, 0.74548573, 0.75418233, + 0.76274862, 0.77118312, 0.77948437, 0.78765094, 0.79568142, 0.80357442, + 0.81132858, 0.81894256, 0.82641504, 0.83374472, 0.84093036, 0.84797069, + 0.85486451, 0.86161063, 0.86820787, 0.87465511, 0.88095122, 0.88709512, + 0.89308574, 0.89892206, 0.90460306, 0.91012776, 0.91549520, 0.92070447, + 0.92575465, 0.93064488, 0.93537432, 0.93994213, 0.94434755, 0.94858979, + 0.95266814, 0.95658189, 0.96033035, 0.96391289, 0.96732888, 0.97057773, + 0.97365889, 0.97657181, 0.97931600, 0.98189099, 0.98429632, 0.98653158, + 0.98859639, 0.99049038, 0.99221324, 0.99376466, 0.99514438, 0.99635215, + 0.99738778, 0.99825107, 0.99894188, 0.99946010, 0.99980562, 0.99997840, + 0.99997840, 0.99980562, 0.99946010, 0.99894188, 0.99825107, 0.99738778, + 0.99635215, 0.99514438, 0.99376466, 0.99221324, 0.99049038, 0.98859639, + 0.98653158, 0.98429632, 0.98189099, 0.97931600, 0.97657181, 0.97365889, + 0.97057773, 0.96732888, 0.96391289, 0.96033035, 0.95658189, 0.95266814, + 0.94858979, 0.94434755, 0.93994213, 0.93537432, 0.93064488, 0.92575465, + 0.92070447, 0.91549520, 0.91012776, 0.90460306, 0.89892206, 0.89308574, + 0.88709512, 0.88095122, 0.87465511, 0.86820787, 0.86161063, 0.85486451, + 0.84797069, 0.84093036, 0.83374472, 0.82641504, 0.81894256, 0.81132858, + 0.80357442, 0.79568142, 0.78765094, 0.77948437, 0.77118312, 0.76274862, + 0.75418233, 0.74548573, 0.73666033, 0.72770765, 0.71862923, 0.70942664, + 0.70010148, 0.69065536, 0.68108990, 0.67140676, 0.66160761, 0.65169416, + 0.64166810, 0.63153117, 0.62128512, 0.61093173, 0.60047278, 0.58991008, + 0.57924546, 0.56848075, 0.55761782, 0.54665854, 0.53560481, 0.52445853, + 0.51322164, 0.50189608, 0.49048379, 0.47898676, 0.46740697, 0.45574642, + 0.44400713, 0.43219112, 0.42030043, 0.40833713, 0.39630327, 0.38420093, + 0.37203222, 0.35979922, 0.34750406, 0.33514885, 0.32273574, 0.31026687, + 0.29774438, 0.28517045, 0.27254725, 0.25987696, 0.24716177, 0.23440387, + 0.22160547, 0.20876878, 0.19589602, 0.18298941, 0.17005118, 0.15708358, + 0.14408883, 0.13106918, 0.11802689, 0.10496421, 0.09188339, 0.07878670, + 0.06567639, 0.05255473, 0.03942400, 0.02628645, 0.01314436, 0.00000000}; + +static const size_t kFilterOrder = 2; +static const float kCoeffNumerator[kFilterOrder + 1] = {0.974827f, -1.949650f, + 0.974827f}; +static const float kCoeffDenominator[kFilterOrder + 1] = {1.0f, -1.971999f, + 0.972457f}; + +static_assert(kFilterOrder + 1 == + sizeof(kCoeffNumerator) / sizeof(kCoeffNumerator[0]), + "numerator coefficients incorrect size"); +static_assert(kFilterOrder + 1 == + sizeof(kCoeffDenominator) / sizeof(kCoeffDenominator[0]), + "denominator coefficients incorrect size"); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROCESSING_H_ diff --git a/VocieProcess/modules/audio_processing/vad/vad_circular_buffer.cc b/VocieProcess/modules/audio_processing/vad/vad_circular_buffer.cc new file mode 100644 index 0000000..31f14d7 --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/vad_circular_buffer.cc @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/vad_circular_buffer.h" + +#include + +namespace webrtc { + +VadCircularBuffer::VadCircularBuffer(int buffer_size) + : buffer_(new double[buffer_size]), + is_full_(false), + index_(0), + buffer_size_(buffer_size), + sum_(0) {} + +VadCircularBuffer::~VadCircularBuffer() {} + +void VadCircularBuffer::Reset() { + is_full_ = false; + index_ = 0; + sum_ = 0; +} + +VadCircularBuffer* VadCircularBuffer::Create(int buffer_size) { + if (buffer_size <= 0) + return NULL; + return new VadCircularBuffer(buffer_size); +} + +double VadCircularBuffer::Oldest() const { + if (!is_full_) + return buffer_[0]; + else + return buffer_[index_]; +} + +double VadCircularBuffer::Mean() { + double m; + if (is_full_) { + m = sum_ / buffer_size_; + } else { + if (index_ > 0) + m = sum_ / index_; + else + m = 0; + } + return m; +} + +void VadCircularBuffer::Insert(double value) { + if (is_full_) { + sum_ -= buffer_[index_]; + } + sum_ += value; + buffer_[index_] = value; + index_++; + if (index_ >= buffer_size_) { + is_full_ = true; + index_ = 0; + } +} +int VadCircularBuffer::BufferLevel() { + if (is_full_) + return buffer_size_; + return index_; +} + +int VadCircularBuffer::Get(int index, double* value) const { + int err = ConvertToLinearIndex(&index); + if (err < 0) + return -1; + *value = buffer_[index]; + return 0; +} + +int VadCircularBuffer::Set(int index, double value) { + int err = ConvertToLinearIndex(&index); + if (err < 0) + return -1; + + sum_ -= buffer_[index]; + buffer_[index] = value; + sum_ += value; + return 0; +} + +int VadCircularBuffer::ConvertToLinearIndex(int* index) const { + if (*index < 0 || *index >= buffer_size_) + return -1; + + if (!is_full_ && *index >= index_) + return -1; + + *index = index_ - 1 - *index; + if (*index < 0) + *index += buffer_size_; + return 0; +} + +int VadCircularBuffer::RemoveTransient(int width_threshold, + double val_threshold) { + if (!is_full_ && index_ < width_threshold + 2) + return 0; + + int index_1 = 0; + int index_2 = width_threshold + 1; + double v = 0; + if (Get(index_1, &v) < 0) + return -1; + if (v < val_threshold) { + Set(index_1, 0); + int index; + for (index = index_2; index > index_1; index--) { + if (Get(index, &v) < 0) + return -1; + if (v < val_threshold) + break; + } + for (; index > index_1; index--) { + if (Set(index, 0.0) < 0) + return -1; + } + } + return 0; +} + +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/vad/vad_circular_buffer.h b/VocieProcess/modules/audio_processing/vad/vad_circular_buffer.h new file mode 100644 index 0000000..c1806f9 --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/vad_circular_buffer.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_VAD_CIRCULAR_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_VAD_VAD_CIRCULAR_BUFFER_H_ + +#include + +namespace webrtc { + +// A circular buffer tailored to the need of this project. It stores last +// K samples of the input, and keeps track of the mean of the last samples. +// +// It is used in class "PitchBasedActivity" to keep track of posterior +// probabilities in the past few seconds. The posterior probabilities are used +// to recursively update prior probabilities. +class VadCircularBuffer { + public: + static VadCircularBuffer* Create(int buffer_size); + ~VadCircularBuffer(); + + // If buffer is wrapped around. + bool is_full() const { return is_full_; } + // Get the oldest entry in the buffer. + double Oldest() const; + // Insert new value into the buffer. + void Insert(double value); + // Reset buffer, forget the past, start fresh. + void Reset(); + + // The mean value of the elements in the buffer. The return value is zero if + // buffer is empty, i.e. no value is inserted. + double Mean(); + // Remove transients. If the values exceed `val_threshold` for a period + // shorter then or equal to `width_threshold`, then that period is considered + // transient and set to zero. + int RemoveTransient(int width_threshold, double val_threshold); + + private: + explicit VadCircularBuffer(int buffer_size); + // Get previous values. |index = 0| corresponds to the most recent + // insertion. |index = 1| is the one before the most recent insertion, and + // so on. + int Get(int index, double* value) const; + // Set a given position to `value`. `index` is interpreted as above. + int Set(int index, double value); + // Return the number of valid elements in the buffer. + int BufferLevel(); + + // Convert an index with the interpretation as get() method to the + // corresponding linear index. + int ConvertToLinearIndex(int* index) const; + + std::unique_ptr buffer_; + bool is_full_; + int index_; + int buffer_size_; + double sum_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_PROCESSING_VAD_VAD_CIRCULAR_BUFFER_H_ diff --git a/VocieProcess/modules/audio_processing/vad/voice_activity_detector.cc b/VocieProcess/modules/audio_processing/vad/voice_activity_detector.cc new file mode 100644 index 0000000..02023d6 --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/voice_activity_detector.cc @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/vad/voice_activity_detector.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +const size_t kNumChannels = 1; + +const double kDefaultVoiceValue = 1.0; +const double kNeutralProbability = 0.5; +const double kLowProbability = 0.01; + +} // namespace + +VoiceActivityDetector::VoiceActivityDetector() + : last_voice_probability_(kDefaultVoiceValue), + standalone_vad_(StandaloneVad::Create()) {} + +VoiceActivityDetector::~VoiceActivityDetector() = default; + +// Because ISAC has a different chunk length, it updates +// `chunkwise_voice_probabilities_` and `chunkwise_rms_` when there is new data. +// Otherwise it clears them. +void VoiceActivityDetector::ProcessChunk(const int16_t* audio, + size_t length, + int sample_rate_hz) { + RTC_DCHECK_EQ(length, sample_rate_hz / 100); + // TODO(bugs.webrtc.org/7494): Remove resampling and force 16 kHz audio. + // Resample to the required rate. + const int16_t* resampled_ptr = audio; + if (sample_rate_hz != kSampleRateHz) { + RTC_CHECK_EQ( + resampler_.ResetIfNeeded(sample_rate_hz, kSampleRateHz, kNumChannels), + 0); + resampler_.Push(audio, length, resampled_, kLength10Ms, length); + resampled_ptr = resampled_; + } + RTC_DCHECK_EQ(length, kLength10Ms); + + // Each chunk needs to be passed into `standalone_vad_`, because internally it + // buffers the audio and processes it all at once when GetActivity() is + // called. + RTC_CHECK_EQ(standalone_vad_->AddAudio(resampled_ptr, length), 0); + + audio_processing_.ExtractFeatures(resampled_ptr, length, &features_); + + chunkwise_voice_probabilities_.resize(features_.num_frames); + chunkwise_rms_.resize(features_.num_frames); + std::copy(features_.rms, features_.rms + chunkwise_rms_.size(), + chunkwise_rms_.begin()); + if (features_.num_frames > 0) { + if (features_.silence) { + // The other features are invalid, so set the voice probabilities to an + // arbitrary low value. + std::fill(chunkwise_voice_probabilities_.begin(), + chunkwise_voice_probabilities_.end(), kLowProbability); + } else { + std::fill(chunkwise_voice_probabilities_.begin(), + chunkwise_voice_probabilities_.end(), kNeutralProbability); + RTC_CHECK_GE( + standalone_vad_->GetActivity(&chunkwise_voice_probabilities_[0], + chunkwise_voice_probabilities_.size()), + 0); + RTC_CHECK_GE(pitch_based_vad_.VoicingProbability( + features_, &chunkwise_voice_probabilities_[0]), + 0); + } + last_voice_probability_ = chunkwise_voice_probabilities_.back(); + } +} + +} // namespace webrtc diff --git a/VocieProcess/modules/audio_processing/vad/voice_activity_detector.h b/VocieProcess/modules/audio_processing/vad/voice_activity_detector.h new file mode 100644 index 0000000..92b9a8c --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/voice_activity_detector.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_VAD_VOICE_ACTIVITY_DETECTOR_H_ +#define MODULES_AUDIO_PROCESSING_VAD_VOICE_ACTIVITY_DETECTOR_H_ + +#include +#include + +#include +#include + +#include "common_audio/resampler/include/resampler.h" +#include "modules/audio_processing/vad/common.h" +#include "modules/audio_processing/vad/pitch_based_vad.h" +#include "modules/audio_processing/vad/standalone_vad.h" +#include "modules/audio_processing/vad/vad_audio_proc.h" + +namespace webrtc { + +// A Voice Activity Detector (VAD) that combines the voice probability from the +// StandaloneVad and PitchBasedVad to get a more robust estimation. +class VoiceActivityDetector { + public: + VoiceActivityDetector(); + ~VoiceActivityDetector(); + + // Processes each audio chunk and estimates the voice probability. + // TODO(bugs.webrtc.org/7494): Switch to rtc::ArrayView and remove + // `sample_rate_hz`. + void ProcessChunk(const int16_t* audio, size_t length, int sample_rate_hz); + + // Returns a vector of voice probabilities for each chunk. It can be empty for + // some chunks, but it catches up afterwards returning multiple values at + // once. + const std::vector& chunkwise_voice_probabilities() const { + return chunkwise_voice_probabilities_; + } + + // Returns a vector of RMS values for each chunk. It has the same length as + // chunkwise_voice_probabilities(). + const std::vector& chunkwise_rms() const { return chunkwise_rms_; } + + // Returns the last voice probability, regardless of the internal + // implementation, although it has a few chunks of delay. + float last_voice_probability() const { return last_voice_probability_; } + + private: + // TODO(aluebs): Change these to float. + std::vector chunkwise_voice_probabilities_; + std::vector chunkwise_rms_; + + float last_voice_probability_; + + Resampler resampler_; + VadAudioProc audio_processing_; + + std::unique_ptr standalone_vad_; + PitchBasedVad pitch_based_vad_; + + int16_t resampled_[kLength10Ms]; + AudioFeatures features_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_VAD_VOICE_ACTIVITY_DETECTOR_H_ diff --git a/VocieProcess/modules/audio_processing/vad/voice_gmm_tables.h b/VocieProcess/modules/audio_processing/vad/voice_gmm_tables.h new file mode 100644 index 0000000..ef4ad7e --- /dev/null +++ b/VocieProcess/modules/audio_processing/vad/voice_gmm_tables.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// GMM tables for active segments. Generated by MakeGmmTables.m. + +#ifndef MODULES_AUDIO_PROCESSING_VAD_VOICE_GMM_TABLES_H_ +#define MODULES_AUDIO_PROCESSING_VAD_VOICE_GMM_TABLES_H_ + +static const int kVoiceGmmNumMixtures = 12; +static const int kVoiceGmmDim = 3; + +static const double + kVoiceGmmCovarInverse[kVoiceGmmNumMixtures][kVoiceGmmDim][kVoiceGmmDim] = { + {{1.83673825579513e+00, -8.09791637570095e-04, 4.60106414365986e-03}, + {-8.09791637570095e-04, 8.89351738394608e-04, -9.80188953277734e-04}, + {4.60106414365986e-03, -9.80188953277734e-04, 1.38706060206582e-03}}, + {{6.76228912850703e+01, -1.98893120119660e-02, -3.53548357253551e-03}, + {-1.98893120119660e-02, 3.96216858500530e-05, -4.08492938394097e-05}, + {-3.53548357253551e-03, -4.08492938394097e-05, 9.31864352856416e-04}}, + {{9.98612435944558e+00, -5.27880954316893e-03, -6.30342541619017e-03}, + {-5.27880954316893e-03, 4.54359480225226e-05, 6.30804591626044e-05}, + {-6.30342541619017e-03, 6.30804591626044e-05, 5.36466441382942e-04}}, + {{3.39917474216349e+01, -1.56213579433191e-03, -4.01459014990225e-02}, + {-1.56213579433191e-03, 6.40415424897724e-05, 6.20076342427833e-05}, + {-4.01459014990225e-02, 6.20076342427833e-05, 3.51199070103063e-03}}, + {{1.34545062271428e+01, -7.94513610147144e-03, -5.34401019341728e-02}, + {-7.94513610147144e-03, 1.16511820098649e-04, 4.66063702069293e-05}, + {-5.34401019341728e-02, 4.66063702069293e-05, 2.72354323774163e-03}}, + {{1.08557844314806e+02, -1.54885805673668e-02, -1.88029692674851e-02}, + {-1.54885805673668e-02, 1.16404042786406e-04, 6.45579292702802e-06}, + {-1.88029692674851e-02, 6.45579292702802e-06, 4.32330478391416e-04}}, + {{8.22940066541450e+01, -1.15903110231303e-02, -4.92166764865343e-02}, + {-1.15903110231303e-02, 7.42510742165261e-05, 3.73007314191290e-06}, + {-4.92166764865343e-02, 3.73007314191290e-06, 3.64005221593244e-03}}, + {{2.31133605685660e+00, -7.83261568950254e-04, 7.45744012346313e-04}, + {-7.83261568950254e-04, 1.29460648214142e-05, -2.22774455093730e-06}, + {7.45744012346313e-04, -2.22774455093730e-06, 1.05117294093010e-04}}, + {{3.78767849189611e+02, 1.57759761011568e-03, -2.08551217988774e-02}, + {1.57759761011568e-03, 4.76066236886865e-05, -2.33977412299324e-05}, + {-2.08551217988774e-02, -2.33977412299324e-05, 5.24261005371196e-04}}, + {{6.98580096506135e-01, -5.13850255217378e-04, -4.01124551717056e-04}, + {-5.13850255217378e-04, 1.40501021984840e-06, -2.09496928716569e-06}, + {-4.01124551717056e-04, -2.09496928716569e-06, 2.82879357740037e-04}}, + {{2.62770945162399e+00, -2.31825753241430e-03, -5.30447217466318e-03}, + {-2.31825753241430e-03, 4.59108572227649e-05, 7.67631886355405e-05}, + {-5.30447217466318e-03, 7.67631886355405e-05, 2.28521601674098e-03}}, + {{1.89940391362152e+02, -4.23280856852379e-03, -2.70608873541399e-02}, + {-4.23280856852379e-03, 6.77547582742563e-05, 2.69154203800467e-05}, + {-2.70608873541399e-02, 2.69154203800467e-05, 3.88574543373470e-03}}}; + +static const double kVoiceGmmMean[kVoiceGmmNumMixtures][kVoiceGmmDim] = { + {-2.15020241646536e+00, 4.97079062999877e+02, 4.77078119504505e+02}, + {-8.92097680029190e-01, 5.92064964199921e+02, 1.81045145941059e+02}, + {-1.29435784144398e+00, 4.98450293410611e+02, 1.71991263804064e+02}, + {-1.03925228397884e+00, 4.99511274321571e+02, 1.05838336539105e+02}, + {-1.29229047206129e+00, 4.15026762566707e+02, 1.12861119017125e+02}, + {-7.88748114599810e-01, 4.48739336688113e+02, 1.89784216956337e+02}, + {-8.77777402332642e-01, 4.86620285054533e+02, 1.13477708016491e+02}, + {-2.06465957063057e+00, 6.33385049870607e+02, 2.32758546796149e+02}, + {-6.98893789231685e-01, 5.93622051503385e+02, 1.92536982473203e+02}, + {-2.55901217508894e+00, 1.55914919756205e+03, 1.39769980835570e+02}, + {-1.92070024165837e+00, 4.87983940444185e+02, 1.02745468128289e+02}, + {-7.29187507662854e-01, 5.22717685022855e+02, 1.16377942283991e+02}}; + +static const double kVoiceGmmWeights[kVoiceGmmNumMixtures] = { + -1.39789694361035e+01, -1.19527720202104e+01, -1.32396317929055e+01, + -1.09436815209238e+01, -1.13440027478149e+01, -1.12200721834504e+01, + -1.02537324043693e+01, -1.60789861938302e+01, -1.03394494048344e+01, + -1.83207938586818e+01, -1.31186044948288e+01, -9.52479998673554e+00}; +#endif // MODULES_AUDIO_PROCESSING_VAD_VOICE_GMM_TABLES_H_ diff --git a/VocieProcess/modules/third_party/fft/fft.c b/VocieProcess/modules/third_party/fft/fft.c new file mode 100644 index 0000000..7260462 --- /dev/null +++ b/VocieProcess/modules/third_party/fft/fft.c @@ -0,0 +1,942 @@ +/* + * Copyright(c)1995,97 Mark Olesen + * Queen's Univ at Kingston (Canada) + * + * Permission to use, copy, modify, and distribute this software for + * any purpose without fee is hereby granted, provided that this + * entire notice is included in all copies of any software which is + * or includes a copy or modification of this software and in all + * copies of the supporting documentation for such software. + * + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTY. IN PARTICULAR, NEITHER THE AUTHOR NOR QUEEN'S + * UNIVERSITY AT KINGSTON MAKES ANY REPRESENTATION OR WARRANTY OF ANY + * KIND CONCERNING THE MERCHANTABILITY OF THIS SOFTWARE OR ITS + * FITNESS FOR ANY PARTICULAR PURPOSE. + * + * All of which is to say that you can do what you like with this + * source code provided you don't try to sell it as your own and you + * include an unaltered copy of this message (including the + * copyright). + * + * It is also implicitly understood that bug fixes and improvements + * should make their way back to the general Internet community so + * that everyone benefits. + * + * Changes: + * Trivial type modifications by the WebRTC authors. + */ + + +/* + * File: + * WebRtcIsac_Fftn.c + * + * Public: + * WebRtcIsac_Fftn / fftnf (); + * + * Private: + * WebRtcIsac_Fftradix / fftradixf (); + * + * Descript: + * multivariate complex Fourier transform, computed in place + * using mixed-radix Fast Fourier Transform algorithm. + * + * Fortran code by: + * RC Singleton, Stanford Research Institute, Sept. 1968 + * + * translated by f2c (version 19950721). + * + * int WebRtcIsac_Fftn (int ndim, const int dims[], REAL Re[], REAL Im[], + * int iSign, double scaling); + * + * NDIM = the total number dimensions + * DIMS = a vector of array sizes + * if NDIM is zero then DIMS must be zero-terminated + * + * RE and IM hold the real and imaginary components of the data, and return + * the resulting real and imaginary Fourier coefficients. Multidimensional + * data *must* be allocated contiguously. There is no limit on the number + * of dimensions. + * + * ISIGN = the sign of the complex exponential (ie, forward or inverse FFT) + * the magnitude of ISIGN (normally 1) is used to determine the + * correct indexing increment (see below). + * + * SCALING = normalizing constant by which the final result is *divided* + * if SCALING == -1, normalize by total dimension of the transform + * if SCALING < -1, normalize by the square-root of the total dimension + * + * example: + * tri-variate transform with Re[n1][n2][n3], Im[n1][n2][n3] + * + * int dims[3] = {n1,n2,n3} + * WebRtcIsac_Fftn (3, dims, Re, Im, 1, scaling); + * + *-----------------------------------------------------------------------* + * int WebRtcIsac_Fftradix (REAL Re[], REAL Im[], size_t nTotal, size_t nPass, + * size_t nSpan, int iSign, size_t max_factors, + * size_t max_perm); + * + * RE, IM - see above documentation + * + * Although there is no limit on the number of dimensions, WebRtcIsac_Fftradix() must + * be called once for each dimension, but the calls may be in any order. + * + * NTOTAL = the total number of complex data values + * NPASS = the dimension of the current variable + * NSPAN/NPASS = the spacing of consecutive data values while indexing the + * current variable + * ISIGN - see above documentation + * + * example: + * tri-variate transform with Re[n1][n2][n3], Im[n1][n2][n3] + * + * WebRtcIsac_Fftradix (Re, Im, n1*n2*n3, n1, n1, 1, maxf, maxp); + * WebRtcIsac_Fftradix (Re, Im, n1*n2*n3, n2, n1*n2, 1, maxf, maxp); + * WebRtcIsac_Fftradix (Re, Im, n1*n2*n3, n3, n1*n2*n3, 1, maxf, maxp); + * + * single-variate transform, + * NTOTAL = N = NSPAN = (number of complex data values), + * + * WebRtcIsac_Fftradix (Re, Im, n, n, n, 1, maxf, maxp); + * + * The data can also be stored in a single array with alternating real and + * imaginary parts, the magnitude of ISIGN is changed to 2 to give correct + * indexing increment, and data [0] and data [1] used to pass the initial + * addresses for the sequences of real and imaginary values, + * + * example: + * REAL data [2*NTOTAL]; + * WebRtcIsac_Fftradix ( &data[0], &data[1], NTOTAL, nPass, nSpan, 2, maxf, maxp); + * + * for temporary allocation: + * + * MAX_FACTORS >= the maximum prime factor of NPASS + * MAX_PERM >= the number of prime factors of NPASS. In addition, + * if the square-free portion K of NPASS has two or more prime + * factors, then MAX_PERM >= (K-1) + * + * storage in FACTOR for a maximum of 15 prime factors of NPASS. if NPASS + * has more than one square-free factor, the product of the square-free + * factors must be <= 210 array storage for maximum prime factor of 23 the + * following two constants should agree with the array dimensions. + * + *----------------------------------------------------------------------*/ + +#include +#include + +#include "modules/third_party/fft/fft.h" + +/* double precision routine */ +static int +WebRtcIsac_Fftradix (double Re[], double Im[], + size_t nTotal, size_t nPass, size_t nSpan, int isign, + int max_factors, unsigned int max_perm, + FFTstr *fftstate); + + + +#ifndef M_PI +# define M_PI 3.14159265358979323846264338327950288 +#endif + +#ifndef SIN60 +# define SIN60 0.86602540378443865 /* sin(60 deg) */ +# define COS72 0.30901699437494742 /* cos(72 deg) */ +# define SIN72 0.95105651629515357 /* sin(72 deg) */ +#endif + +# define REAL double +# define FFTN WebRtcIsac_Fftn +# define FFTNS "fftn" +# define FFTRADIX WebRtcIsac_Fftradix +# define FFTRADIXS "fftradix" + + +int WebRtcIsac_Fftns(unsigned int ndim, const int dims[], + double Re[], + double Im[], + int iSign, + double scaling, + FFTstr *fftstate) +{ + + size_t nSpan, nPass, nTotal; + unsigned int i; + int ret, max_factors, max_perm; + + /* + * tally the number of elements in the data array + * and determine the number of dimensions + */ + nTotal = 1; + if (ndim && dims [0]) + { + for (i = 0; i < ndim; i++) + { + if (dims [i] <= 0) + { + return -1; + } + nTotal *= dims [i]; + } + } + else + { + ndim = 0; + for (i = 0; dims [i]; i++) + { + if (dims [i] <= 0) + { + return -1; + } + nTotal *= dims [i]; + ndim++; + } + } + + /* determine maximum number of factors and permuations */ +#if 1 + /* + * follow John Beale's example, just use the largest dimension and don't + * worry about excess allocation. May be someone else will do it? + */ + max_factors = max_perm = 1; + for (i = 0; i < ndim; i++) + { + nSpan = dims [i]; + if ((int)nSpan > max_factors) + { + max_factors = (int)nSpan; + } + if ((int)nSpan > max_perm) + { + max_perm = (int)nSpan; + } + } +#else + /* use the constants used in the original Fortran code */ + max_factors = 23; + max_perm = 209; +#endif + /* loop over the dimensions: */ + nPass = 1; + for (i = 0; i < ndim; i++) + { + nSpan = dims [i]; + nPass *= nSpan; + ret = FFTRADIX (Re, Im, nTotal, nSpan, nPass, iSign, + max_factors, max_perm, fftstate); + /* exit, clean-up already done */ + if (ret) + return ret; + } + + /* Divide through by the normalizing constant: */ + if (scaling && scaling != 1.0) + { + if (iSign < 0) iSign = -iSign; + if (scaling < 0.0) + { + scaling = (double)nTotal; + if (scaling < -1.0) + scaling = sqrt (scaling); + } + scaling = 1.0 / scaling; /* multiply is often faster */ + for (i = 0; i < nTotal; i += iSign) + { + Re [i] *= scaling; + Im [i] *= scaling; + } + } + return 0; +} + +/* + * singleton's mixed radix routine + * + * could move allocation out to WebRtcIsac_Fftn(), but leave it here so that it's + * possible to make this a standalone function + */ + +static int FFTRADIX (REAL Re[], + REAL Im[], + size_t nTotal, + size_t nPass, + size_t nSpan, + int iSign, + int max_factors, + unsigned int max_perm, + FFTstr *fftstate) +{ + int ii, mfactor, kspan, ispan, inc; + int j, jc, jf, jj, k, k1, k2, k3, k4, kk, kt, nn, ns, nt; + + + REAL radf; + REAL c1, c2, c3, cd, aa, aj, ak, ajm, ajp, akm, akp; + REAL s1, s2, s3, sd, bb, bj, bk, bjm, bjp, bkm, bkp; + + REAL *Rtmp = NULL; /* temp space for real part*/ + REAL *Itmp = NULL; /* temp space for imaginary part */ + REAL *Cos = NULL; /* Cosine values */ + REAL *Sin = NULL; /* Sine values */ + + REAL s60 = SIN60; /* sin(60 deg) */ + REAL c72 = COS72; /* cos(72 deg) */ + REAL s72 = SIN72; /* sin(72 deg) */ + REAL pi2 = M_PI; /* use PI first, 2 PI later */ + + + fftstate->SpaceAlloced = 0; + fftstate->MaxPermAlloced = 0; + + + // initialize to avoid warnings + k3 = c2 = c3 = s2 = s3 = 0.0; + + if (nPass < 2) + return 0; + + /* allocate storage */ + if (fftstate->SpaceAlloced < max_factors * sizeof (REAL)) + { +#ifdef SUN_BROKEN_REALLOC + if (!fftstate->SpaceAlloced) /* first time */ + { + fftstate->SpaceAlloced = max_factors * sizeof (REAL); + } + else + { +#endif + fftstate->SpaceAlloced = max_factors * sizeof (REAL); +#ifdef SUN_BROKEN_REALLOC + } +#endif + } + else + { + /* allow full use of alloc'd space */ + max_factors = fftstate->SpaceAlloced / sizeof (REAL); + } + if (fftstate->MaxPermAlloced < max_perm) + { +#ifdef SUN_BROKEN_REALLOC + if (!fftstate->MaxPermAlloced) /* first time */ + else +#endif + fftstate->MaxPermAlloced = max_perm; + } + else + { + /* allow full use of alloc'd space */ + max_perm = fftstate->MaxPermAlloced; + } + + /* assign pointers */ + Rtmp = (REAL *) fftstate->Tmp0; + Itmp = (REAL *) fftstate->Tmp1; + Cos = (REAL *) fftstate->Tmp2; + Sin = (REAL *) fftstate->Tmp3; + + /* + * Function Body + */ + inc = iSign; + if (iSign < 0) { + s72 = -s72; + s60 = -s60; + pi2 = -pi2; + inc = -inc; /* absolute value */ + } + + /* adjust for strange increments */ + nt = inc * (int)nTotal; + ns = inc * (int)nSpan; + kspan = ns; + + nn = nt - inc; + jc = ns / (int)nPass; + radf = pi2 * (double) jc; + pi2 *= 2.0; /* use 2 PI from here on */ + + ii = 0; + jf = 0; + /* determine the factors of n */ + mfactor = 0; + k = (int)nPass; + while (k % 16 == 0) { + mfactor++; + fftstate->factor [mfactor - 1] = 4; + k /= 16; + } + j = 3; + jj = 9; + do { + while (k % jj == 0) { + mfactor++; + fftstate->factor [mfactor - 1] = j; + k /= jj; + } + j += 2; + jj = j * j; + } while (jj <= k); + if (k <= 4) { + kt = mfactor; + fftstate->factor [mfactor] = k; + if (k != 1) + mfactor++; + } else { + if (k - (k / 4 << 2) == 0) { + mfactor++; + fftstate->factor [mfactor - 1] = 2; + k /= 4; + } + kt = mfactor; + j = 2; + do { + if (k % j == 0) { + mfactor++; + fftstate->factor [mfactor - 1] = j; + k /= j; + } + j = ((j + 1) / 2 << 1) + 1; + } while (j <= k); + } + if (kt) { + j = kt; + do { + mfactor++; + fftstate->factor [mfactor - 1] = fftstate->factor [j - 1]; + j--; + } while (j); + } + + /* test that mfactors is in range */ + if (mfactor > FFT_NFACTOR) + { + return -1; + } + + /* compute fourier transform */ + for (;;) { + sd = radf / (double) kspan; + cd = sin(sd); + cd = 2.0 * cd * cd; + sd = sin(sd + sd); + kk = 0; + ii++; + + switch (fftstate->factor [ii - 1]) { + case 2: + /* transform for factor of 2 (including rotation factor) */ + kspan /= 2; + k1 = kspan + 2; + do { + do { + k2 = kk + kspan; + ak = Re [k2]; + bk = Im [k2]; + Re [k2] = Re [kk] - ak; + Im [k2] = Im [kk] - bk; + Re [kk] += ak; + Im [kk] += bk; + kk = k2 + kspan; + } while (kk < nn); + kk -= nn; + } while (kk < jc); + if (kk >= kspan) + goto Permute_Results_Label; /* exit infinite loop */ + do { + c1 = 1.0 - cd; + s1 = sd; + do { + do { + do { + k2 = kk + kspan; + ak = Re [kk] - Re [k2]; + bk = Im [kk] - Im [k2]; + Re [kk] += Re [k2]; + Im [kk] += Im [k2]; + Re [k2] = c1 * ak - s1 * bk; + Im [k2] = s1 * ak + c1 * bk; + kk = k2 + kspan; + } while (kk < (nt-1)); + k2 = kk - nt; + c1 = -c1; + kk = k1 - k2; + } while (kk > k2); + ak = c1 - (cd * c1 + sd * s1); + s1 = sd * c1 - cd * s1 + s1; + c1 = 2.0 - (ak * ak + s1 * s1); + s1 *= c1; + c1 *= ak; + kk += jc; + } while (kk < k2); + k1 += inc + inc; + kk = (k1 - kspan + 1) / 2 + jc - 1; + } while (kk < (jc + jc)); + break; + + case 4: /* transform for factor of 4 */ + ispan = kspan; + kspan /= 4; + + do { + c1 = 1.0; + s1 = 0.0; + do { + do { + k1 = kk + kspan; + k2 = k1 + kspan; + k3 = k2 + kspan; + akp = Re [kk] + Re [k2]; + akm = Re [kk] - Re [k2]; + ajp = Re [k1] + Re [k3]; + ajm = Re [k1] - Re [k3]; + bkp = Im [kk] + Im [k2]; + bkm = Im [kk] - Im [k2]; + bjp = Im [k1] + Im [k3]; + bjm = Im [k1] - Im [k3]; + Re [kk] = akp + ajp; + Im [kk] = bkp + bjp; + ajp = akp - ajp; + bjp = bkp - bjp; + if (iSign < 0) { + akp = akm + bjm; + bkp = bkm - ajm; + akm -= bjm; + bkm += ajm; + } else { + akp = akm - bjm; + bkp = bkm + ajm; + akm += bjm; + bkm -= ajm; + } + /* avoid useless multiplies */ + if (s1 == 0.0) { + Re [k1] = akp; + Re [k2] = ajp; + Re [k3] = akm; + Im [k1] = bkp; + Im [k2] = bjp; + Im [k3] = bkm; + } else { + Re [k1] = akp * c1 - bkp * s1; + Re [k2] = ajp * c2 - bjp * s2; + Re [k3] = akm * c3 - bkm * s3; + Im [k1] = akp * s1 + bkp * c1; + Im [k2] = ajp * s2 + bjp * c2; + Im [k3] = akm * s3 + bkm * c3; + } + kk = k3 + kspan; + } while (kk < nt); + + c2 = c1 - (cd * c1 + sd * s1); + s1 = sd * c1 - cd * s1 + s1; + c1 = 2.0 - (c2 * c2 + s1 * s1); + s1 *= c1; + c1 *= c2; + /* values of c2, c3, s2, s3 that will get used next time */ + c2 = c1 * c1 - s1 * s1; + s2 = 2.0 * c1 * s1; + c3 = c2 * c1 - s2 * s1; + s3 = c2 * s1 + s2 * c1; + kk = kk - nt + jc; + } while (kk < kspan); + kk = kk - kspan + inc; + } while (kk < jc); + if (kspan == jc) + goto Permute_Results_Label; /* exit infinite loop */ + break; + + default: + /* transform for odd factors */ +#ifdef FFT_RADIX4 + return -1; + break; +#else /* FFT_RADIX4 */ + k = fftstate->factor [ii - 1]; + ispan = kspan; + kspan /= k; + + switch (k) { + case 3: /* transform for factor of 3 (optional code) */ + do { + do { + k1 = kk + kspan; + k2 = k1 + kspan; + ak = Re [kk]; + bk = Im [kk]; + aj = Re [k1] + Re [k2]; + bj = Im [k1] + Im [k2]; + Re [kk] = ak + aj; + Im [kk] = bk + bj; + ak -= 0.5 * aj; + bk -= 0.5 * bj; + aj = (Re [k1] - Re [k2]) * s60; + bj = (Im [k1] - Im [k2]) * s60; + Re [k1] = ak - bj; + Re [k2] = ak + bj; + Im [k1] = bk + aj; + Im [k2] = bk - aj; + kk = k2 + kspan; + } while (kk < (nn - 1)); + kk -= nn; + } while (kk < kspan); + break; + + case 5: /* transform for factor of 5 (optional code) */ + c2 = c72 * c72 - s72 * s72; + s2 = 2.0 * c72 * s72; + do { + do { + k1 = kk + kspan; + k2 = k1 + kspan; + k3 = k2 + kspan; + k4 = k3 + kspan; + akp = Re [k1] + Re [k4]; + akm = Re [k1] - Re [k4]; + bkp = Im [k1] + Im [k4]; + bkm = Im [k1] - Im [k4]; + ajp = Re [k2] + Re [k3]; + ajm = Re [k2] - Re [k3]; + bjp = Im [k2] + Im [k3]; + bjm = Im [k2] - Im [k3]; + aa = Re [kk]; + bb = Im [kk]; + Re [kk] = aa + akp + ajp; + Im [kk] = bb + bkp + bjp; + ak = akp * c72 + ajp * c2 + aa; + bk = bkp * c72 + bjp * c2 + bb; + aj = akm * s72 + ajm * s2; + bj = bkm * s72 + bjm * s2; + Re [k1] = ak - bj; + Re [k4] = ak + bj; + Im [k1] = bk + aj; + Im [k4] = bk - aj; + ak = akp * c2 + ajp * c72 + aa; + bk = bkp * c2 + bjp * c72 + bb; + aj = akm * s2 - ajm * s72; + bj = bkm * s2 - bjm * s72; + Re [k2] = ak - bj; + Re [k3] = ak + bj; + Im [k2] = bk + aj; + Im [k3] = bk - aj; + kk = k4 + kspan; + } while (kk < (nn-1)); + kk -= nn; + } while (kk < kspan); + break; + + default: + if (k != jf) { + jf = k; + s1 = pi2 / (double) k; + c1 = cos(s1); + s1 = sin(s1); + if (jf > max_factors){ + return -1; + } + Cos [jf - 1] = 1.0; + Sin [jf - 1] = 0.0; + j = 1; + do { + Cos [j - 1] = Cos [k - 1] * c1 + Sin [k - 1] * s1; + Sin [j - 1] = Cos [k - 1] * s1 - Sin [k - 1] * c1; + k--; + Cos [k - 1] = Cos [j - 1]; + Sin [k - 1] = -Sin [j - 1]; + j++; + } while (j < k); + } + do { + do { + k1 = kk; + k2 = kk + ispan; + ak = aa = Re [kk]; + bk = bb = Im [kk]; + j = 1; + k1 += kspan; + do { + k2 -= kspan; + j++; + Rtmp [j - 1] = Re [k1] + Re [k2]; + ak += Rtmp [j - 1]; + Itmp [j - 1] = Im [k1] + Im [k2]; + bk += Itmp [j - 1]; + j++; + Rtmp [j - 1] = Re [k1] - Re [k2]; + Itmp [j - 1] = Im [k1] - Im [k2]; + k1 += kspan; + } while (k1 < k2); + Re [kk] = ak; + Im [kk] = bk; + k1 = kk; + k2 = kk + ispan; + j = 1; + do { + k1 += kspan; + k2 -= kspan; + jj = j; + ak = aa; + bk = bb; + aj = 0.0; + bj = 0.0; + k = 1; + do { + k++; + ak += Rtmp [k - 1] * Cos [jj - 1]; + bk += Itmp [k - 1] * Cos [jj - 1]; + k++; + aj += Rtmp [k - 1] * Sin [jj - 1]; + bj += Itmp [k - 1] * Sin [jj - 1]; + jj += j; + if (jj > jf) { + jj -= jf; + } + } while (k < jf); + k = jf - j; + Re [k1] = ak - bj; + Im [k1] = bk + aj; + Re [k2] = ak + bj; + Im [k2] = bk - aj; + j++; + } while (j < k); + kk += ispan; + } while (kk < nn); + kk -= nn; + } while (kk < kspan); + break; + } + + /* multiply by rotation factor (except for factors of 2 and 4) */ + if (ii == mfactor) + goto Permute_Results_Label; /* exit infinite loop */ + kk = jc; + do { + c2 = 1.0 - cd; + s1 = sd; + do { + c1 = c2; + s2 = s1; + kk += kspan; + do { + do { + ak = Re [kk]; + Re [kk] = c2 * ak - s2 * Im [kk]; + Im [kk] = s2 * ak + c2 * Im [kk]; + kk += ispan; + } while (kk < nt); + ak = s1 * s2; + s2 = s1 * c2 + c1 * s2; + c2 = c1 * c2 - ak; + kk = kk - nt + kspan; + } while (kk < ispan); + c2 = c1 - (cd * c1 + sd * s1); + s1 += sd * c1 - cd * s1; + c1 = 2.0 - (c2 * c2 + s1 * s1); + s1 *= c1; + c2 *= c1; + kk = kk - ispan + jc; + } while (kk < kspan); + kk = kk - kspan + jc + inc; + } while (kk < (jc + jc)); + break; +#endif /* FFT_RADIX4 */ + } + } + + /* permute the results to normal order---done in two stages */ + /* permutation for square factors of n */ +Permute_Results_Label: + fftstate->Perm [0] = ns; + if (kt) { + k = kt + kt + 1; + if (mfactor < k) + k--; + j = 1; + fftstate->Perm [k] = jc; + do { + fftstate->Perm [j] = fftstate->Perm [j - 1] / fftstate->factor [j - 1]; + fftstate->Perm [k - 1] = fftstate->Perm [k] * fftstate->factor [j - 1]; + j++; + k--; + } while (j < k); + k3 = fftstate->Perm [k]; + kspan = fftstate->Perm [1]; + kk = jc; + k2 = kspan; + j = 1; + if (nPass != nTotal) { + /* permutation for multivariate transform */ + Permute_Multi_Label: + do { + do { + k = kk + jc; + do { + /* swap Re [kk] <> Re [k2], Im [kk] <> Im [k2] */ + ak = Re [kk]; Re [kk] = Re [k2]; Re [k2] = ak; + bk = Im [kk]; Im [kk] = Im [k2]; Im [k2] = bk; + kk += inc; + k2 += inc; + } while (kk < (k-1)); + kk += ns - jc; + k2 += ns - jc; + } while (kk < (nt-1)); + k2 = k2 - nt + kspan; + kk = kk - nt + jc; + } while (k2 < (ns-1)); + do { + do { + k2 -= fftstate->Perm [j - 1]; + j++; + k2 = fftstate->Perm [j] + k2; + } while (k2 > fftstate->Perm [j - 1]); + j = 1; + do { + if (kk < (k2-1)) + goto Permute_Multi_Label; + kk += jc; + k2 += kspan; + } while (k2 < (ns-1)); + } while (kk < (ns-1)); + } else { + /* permutation for single-variate transform (optional code) */ + Permute_Single_Label: + do { + /* swap Re [kk] <> Re [k2], Im [kk] <> Im [k2] */ + ak = Re [kk]; Re [kk] = Re [k2]; Re [k2] = ak; + bk = Im [kk]; Im [kk] = Im [k2]; Im [k2] = bk; + kk += inc; + k2 += kspan; + } while (k2 < (ns-1)); + do { + do { + k2 -= fftstate->Perm [j - 1]; + j++; + k2 = fftstate->Perm [j] + k2; + } while (k2 >= fftstate->Perm [j - 1]); + j = 1; + do { + if (kk < k2) + goto Permute_Single_Label; + kk += inc; + k2 += kspan; + } while (k2 < (ns-1)); + } while (kk < (ns-1)); + } + jc = k3; + } + + if ((kt << 1) + 1 >= mfactor) + return 0; + ispan = fftstate->Perm [kt]; + /* permutation for square-free factors of n */ + j = mfactor - kt; + fftstate->factor [j] = 1; + do { + fftstate->factor [j - 1] *= fftstate->factor [j]; + j--; + } while (j != kt); + kt++; + nn = fftstate->factor [kt - 1] - 1; + if (nn > (int) max_perm) { + return -1; + } + j = jj = 0; + for (;;) { + k = kt + 1; + k2 = fftstate->factor [kt - 1]; + kk = fftstate->factor [k - 1]; + j++; + if (j > nn) + break; /* exit infinite loop */ + jj += kk; + while (jj >= k2) { + jj -= k2; + k2 = kk; + k++; + kk = fftstate->factor [k - 1]; + jj += kk; + } + fftstate->Perm [j - 1] = jj; + } + /* determine the permutation cycles of length greater than 1 */ + j = 0; + for (;;) { + do { + j++; + kk = fftstate->Perm [j - 1]; + } while (kk < 0); + if (kk != j) { + do { + k = kk; + kk = fftstate->Perm [k - 1]; + fftstate->Perm [k - 1] = -kk; + } while (kk != j); + k3 = kk; + } else { + fftstate->Perm [j - 1] = -j; + if (j == nn) + break; /* exit infinite loop */ + } + } + max_factors *= inc; + /* reorder a and b, following the permutation cycles */ + for (;;) { + j = k3 + 1; + nt -= ispan; + ii = nt - inc + 1; + if (nt < 0) + break; /* exit infinite loop */ + do { + do { + j--; + } while (fftstate->Perm [j - 1] < 0); + jj = jc; + do { + kspan = jj; + if (jj > max_factors) { + kspan = max_factors; + } + jj -= kspan; + k = fftstate->Perm [j - 1]; + kk = jc * k + ii + jj; + k1 = kk + kspan - 1; + k2 = 0; + do { + k2++; + Rtmp [k2 - 1] = Re [k1]; + Itmp [k2 - 1] = Im [k1]; + k1 -= inc; + } while (k1 != (kk-1)); + do { + k1 = kk + kspan - 1; + k2 = k1 - jc * (k + fftstate->Perm [k - 1]); + k = -fftstate->Perm [k - 1]; + do { + Re [k1] = Re [k2]; + Im [k1] = Im [k2]; + k1 -= inc; + k2 -= inc; + } while (k1 != (kk-1)); + kk = k2 + 1; + } while (k != j); + k1 = kk + kspan - 1; + k2 = 0; + do { + k2++; + Re [k1] = Rtmp [k2 - 1]; + Im [k1] = Itmp [k2 - 1]; + k1 -= inc; + } while (k1 != (kk-1)); + } while (jj); + } while (j != 1); + } + return 0; /* exit point here */ +} +/* ---------------------- end-of-file (c source) ---------------------- */ + diff --git a/VocieProcess/modules/third_party/fft/fft.h b/VocieProcess/modules/third_party/fft/fft.h new file mode 100644 index 0000000..f8f8b6f --- /dev/null +++ b/VocieProcess/modules/third_party/fft/fft.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the ../../../LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/*--------------------------------*-C-*---------------------------------* + * File: + * fftn.h + * ---------------------------------------------------------------------* + * Re[]: real value array + * Im[]: imaginary value array + * nTotal: total number of complex values + * nPass: number of elements involved in this pass of transform + * nSpan: nspan/nPass = number of bytes to increment pointer + * in Re[] and Im[] + * isign: exponent: +1 = forward -1 = reverse + * scaling: normalizing constant by which the final result is *divided* + * scaling == -1, normalize by total dimension of the transform + * scaling < -1, normalize by the square-root of the total dimension + * + * ---------------------------------------------------------------------- + * See the comments in the code for correct usage! + */ + +#ifndef MODULES_THIRD_PARTY_FFT_FFT_H_ +#define MODULES_THIRD_PARTY_FFT_FFT_H_ + +#define FFT_MAXFFTSIZE 2048 +#define FFT_NFACTOR 11 + +typedef struct { + unsigned int SpaceAlloced; + unsigned int MaxPermAlloced; + double Tmp0[FFT_MAXFFTSIZE]; + double Tmp1[FFT_MAXFFTSIZE]; + double Tmp2[FFT_MAXFFTSIZE]; + double Tmp3[FFT_MAXFFTSIZE]; + int Perm[FFT_MAXFFTSIZE]; + int factor[FFT_NFACTOR]; + +} FFTstr; + +/* double precision routine */ + +int WebRtcIsac_Fftns(unsigned int ndim, + const int dims[], + double Re[], + double Im[], + int isign, + double scaling, + FFTstr* fftstate); + +#endif /* MODULES_THIRD_PARTY_FFT_FFT_H_ */ diff --git a/VocieProcess/rtc_base/event.cc b/VocieProcess/rtc_base/event.cc new file mode 100644 index 0000000..c2f6f8a --- /dev/null +++ b/VocieProcess/rtc_base/event.cc @@ -0,0 +1,210 @@ +/* + * Copyright 2004 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/event.h" + +#if defined(WEBRTC_WIN) +#include +#elif defined(WEBRTC_POSIX) +#include +#include +#include +#include +#else +#error "Must define either WEBRTC_WIN or WEBRTC_POSIX." +#endif + +#include "absl/types/optional.h" +#include "rtc_base/checks.h" +#include "rtc_base/synchronization/yield_policy.h" +#include "rtc_base/system/warn_current_thread_is_deadlocked.h" +#include "rtc_base/time_utils.h" + +namespace rtc { + +using ::webrtc::TimeDelta; + +Event::Event() : Event(false, false) {} + +#if defined(WEBRTC_WIN) + +Event::Event(bool manual_reset, bool initially_signaled) { + event_handle_ = ::CreateEvent(nullptr, // Security attributes. + manual_reset, initially_signaled, + nullptr); // Name. + RTC_CHECK(event_handle_); +} + +Event::~Event() { + CloseHandle(event_handle_); +} + +void Event::Set() { + SetEvent(event_handle_); +} + +void Event::Reset() { + ResetEvent(event_handle_); +} + +bool Event::Wait(TimeDelta give_up_after, TimeDelta /*warn_after*/) { + ScopedYieldPolicy::YieldExecution(); + const DWORD ms = + give_up_after.IsPlusInfinity() + ? INFINITE + : give_up_after.RoundUpTo(webrtc::TimeDelta::Millis(1)).ms(); + return (WaitForSingleObject(event_handle_, ms) == WAIT_OBJECT_0); +} + +#elif defined(WEBRTC_POSIX) + +// On MacOS, clock_gettime is available from version 10.12, and on +// iOS, from version 10.0. So we can't use it yet. +#if defined(WEBRTC_MAC) || defined(WEBRTC_IOS) +#define USE_CLOCK_GETTIME 0 +#define USE_PTHREAD_COND_TIMEDWAIT_MONOTONIC_NP 0 +// On Android, pthread_condattr_setclock is available from version 21. By +// default, we target a new enough version for 64-bit platforms but not for +// 32-bit platforms. For older versions, use +// pthread_cond_timedwait_monotonic_np. +#elif defined(WEBRTC_ANDROID) && (__ANDROID_API__ < 21) +#define USE_CLOCK_GETTIME 1 +#define USE_PTHREAD_COND_TIMEDWAIT_MONOTONIC_NP 1 +#else +#define USE_CLOCK_GETTIME 1 +#define USE_PTHREAD_COND_TIMEDWAIT_MONOTONIC_NP 0 +#endif + +Event::Event(bool manual_reset, bool initially_signaled) + : is_manual_reset_(manual_reset), event_status_(initially_signaled) { + RTC_CHECK(pthread_mutex_init(&event_mutex_, nullptr) == 0); + pthread_condattr_t cond_attr; + RTC_CHECK(pthread_condattr_init(&cond_attr) == 0); +#if USE_CLOCK_GETTIME && !USE_PTHREAD_COND_TIMEDWAIT_MONOTONIC_NP + RTC_CHECK(pthread_condattr_setclock(&cond_attr, CLOCK_MONOTONIC) == 0); +#endif + RTC_CHECK(pthread_cond_init(&event_cond_, &cond_attr) == 0); + pthread_condattr_destroy(&cond_attr); +} + +Event::~Event() { + pthread_mutex_destroy(&event_mutex_); + pthread_cond_destroy(&event_cond_); +} + +void Event::Set() { + pthread_mutex_lock(&event_mutex_); + event_status_ = true; + pthread_cond_broadcast(&event_cond_); + pthread_mutex_unlock(&event_mutex_); +} + +void Event::Reset() { + pthread_mutex_lock(&event_mutex_); + event_status_ = false; + pthread_mutex_unlock(&event_mutex_); +} + +namespace { + +timespec GetTimespec(TimeDelta duration_from_now) { + timespec ts; + + // Get the current time. +#if USE_CLOCK_GETTIME + clock_gettime(CLOCK_MONOTONIC, &ts); +#else + timeval tv; + gettimeofday(&tv, nullptr); + ts.tv_sec = tv.tv_sec; + ts.tv_nsec = tv.tv_usec * kNumNanosecsPerMicrosec; +#endif + + // Add the specified number of milliseconds to it. + int64_t microsecs_from_now = duration_from_now.us(); + ts.tv_sec += microsecs_from_now / kNumMicrosecsPerSec; + ts.tv_nsec += + (microsecs_from_now % kNumMicrosecsPerSec) * kNumNanosecsPerMicrosec; + + // Normalize. + if (ts.tv_nsec >= kNumNanosecsPerSec) { + ts.tv_sec++; + ts.tv_nsec -= kNumNanosecsPerSec; + } + + return ts; +} + +} // namespace + +bool Event::Wait(TimeDelta give_up_after, TimeDelta warn_after) { + // Instant when we'll log a warning message (because we've been waiting so + // long it might be a bug), but not yet give up waiting. nullopt if we + // shouldn't log a warning. + const absl::optional warn_ts = + warn_after >= give_up_after + ? absl::nullopt + : absl::make_optional(GetTimespec(warn_after)); + + // Instant when we'll stop waiting and return an error. nullopt if we should + // never give up. + const absl::optional give_up_ts = + give_up_after.IsPlusInfinity() + ? absl::nullopt + : absl::make_optional(GetTimespec(give_up_after)); + + ScopedYieldPolicy::YieldExecution(); + pthread_mutex_lock(&event_mutex_); + + // Wait for `event_cond_` to trigger and `event_status_` to be set, with the + // given timeout (or without a timeout if none is given). + const auto wait = [&](const absl::optional timeout_ts) { + int error = 0; + while (!event_status_ && error == 0) { + if (timeout_ts == absl::nullopt) { + error = pthread_cond_wait(&event_cond_, &event_mutex_); + } else { +#if USE_PTHREAD_COND_TIMEDWAIT_MONOTONIC_NP + error = pthread_cond_timedwait_monotonic_np(&event_cond_, &event_mutex_, + &*timeout_ts); +#else + error = + pthread_cond_timedwait(&event_cond_, &event_mutex_, &*timeout_ts); +#endif + } + } + return error; + }; + + int error; + if (warn_ts == absl::nullopt) { + error = wait(give_up_ts); + } else { + error = wait(warn_ts); + if (error == ETIMEDOUT) { + webrtc::WarnThatTheCurrentThreadIsProbablyDeadlocked(); + error = wait(give_up_ts); + } + } + + // NOTE(liulk): Exactly one thread will auto-reset this event. All + // the other threads will think it's unsignaled. This seems to be + // consistent with auto-reset events in WEBRTC_WIN + if (error == 0 && !is_manual_reset_) + event_status_ = false; + + pthread_mutex_unlock(&event_mutex_); + + return (error == 0); +} + +#endif + +} // namespace rtc diff --git a/VocieProcess/rtc_base/event.h b/VocieProcess/rtc_base/event.h new file mode 100644 index 0000000..12f6a7d --- /dev/null +++ b/VocieProcess/rtc_base/event.h @@ -0,0 +1,137 @@ +/* + * Copyright 2004 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_EVENT_H_ +#define RTC_BASE_EVENT_H_ + +#include "api/units/time_delta.h" + +#if defined(WEBRTC_WIN) +#include +#elif defined(WEBRTC_POSIX) +#include +#else +#error "Must define either WEBRTC_WIN or WEBRTC_POSIX." +#endif + +#include "rtc_base/synchronization/yield_policy.h" + +namespace rtc { + +// RTC_DISALLOW_WAIT() utility +// +// Sets a stack-scoped flag that disallows use of `rtc::Event::Wait` by means +// of raising a DCHECK when a call to `rtc::Event::Wait()` is made.. +// This is useful to guard synchronization-free scopes against regressions. +// +// Example of what this would catch (`ScopeToProtect` calls `Foo`): +// +// void Foo(TaskQueue* tq) { +// Event event; +// tq->PostTask([&event]() { +// event.Set(); +// }); +// event.Wait(Event::kForever); // <- Will trigger a DCHECK. +// } +// +// void ScopeToProtect() { +// TaskQueue* tq = GetSomeTaskQueue(); +// RTC_DISALLOW_WAIT(); // Policy takes effect. +// Foo(tq); +// } +// +#if RTC_DCHECK_IS_ON +#define RTC_DISALLOW_WAIT() ScopedDisallowWait disallow_wait_##__LINE__ +#else +#define RTC_DISALLOW_WAIT() +#endif + +class Event { + public: + // TODO(bugs.webrtc.org/14366): Consider removing this redundant alias. + static constexpr webrtc::TimeDelta kForever = + webrtc::TimeDelta::PlusInfinity(); + + Event(); + Event(bool manual_reset, bool initially_signaled); + Event(const Event&) = delete; + Event& operator=(const Event&) = delete; + ~Event(); + + void Set(); + void Reset(); + + // Waits for the event to become signaled, but logs a warning if it takes more + // than `warn_after`, and gives up completely if it takes more than + // `give_up_after`. (If `warn_after >= give_up_after`, no warning will be + // logged.) Either or both may be `kForever`, which means wait indefinitely. + // + // Care is taken so that the underlying OS wait call isn't requested to sleep + // shorter than `give_up_after`. + // + // Returns true if the event was signaled, false if there was a timeout or + // some other error. + bool Wait(webrtc::TimeDelta give_up_after, webrtc::TimeDelta warn_after); + + // Waits with the given timeout and a reasonable default warning timeout. + bool Wait(webrtc::TimeDelta give_up_after) { + return Wait(give_up_after, give_up_after.IsPlusInfinity() + ? webrtc::TimeDelta::Seconds(3) + : kForever); + } + + private: +#if defined(WEBRTC_WIN) + HANDLE event_handle_; +#elif defined(WEBRTC_POSIX) + pthread_mutex_t event_mutex_; + pthread_cond_t event_cond_; + const bool is_manual_reset_; + bool event_status_; +#endif +}; + +// These classes are provided for compatibility with Chromium. +// The rtc::Event implementation is overriden inside of Chromium for the +// purposes of detecting when threads are blocked that shouldn't be as well as +// to use the more accurate event implementation that's there than is provided +// by default on some platforms (e.g. Windows). +// When building with standalone WebRTC, this class is a noop. +// For further information, please see the +// ScopedAllowBaseSyncPrimitives(ForTesting) classes in Chromium. +class ScopedAllowBaseSyncPrimitives { + public: + ScopedAllowBaseSyncPrimitives() {} + ~ScopedAllowBaseSyncPrimitives() {} +}; + +class ScopedAllowBaseSyncPrimitivesForTesting { + public: + ScopedAllowBaseSyncPrimitivesForTesting() {} + ~ScopedAllowBaseSyncPrimitivesForTesting() {} +}; + +#if RTC_DCHECK_IS_ON +class ScopedDisallowWait { + public: + ScopedDisallowWait() = default; + + private: + class DisallowYieldHandler : public YieldInterface { + public: + void YieldExecution() override { RTC_DCHECK_NOTREACHED(); } + } handler_; + rtc::ScopedYieldPolicy policy{&handler_}; +}; +#endif + +} // namespace rtc + +#endif // RTC_BASE_EVENT_H_ diff --git a/VocieProcess/rtc_base/event_tracer.cc b/VocieProcess/rtc_base/event_tracer.cc new file mode 100644 index 0000000..7e9ecba --- /dev/null +++ b/VocieProcess/rtc_base/event_tracer.cc @@ -0,0 +1,449 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/event_tracer.h" + +#include + +#include "rtc_base/trace_event.h" + +#if defined(RTC_USE_PERFETTO) +#include "rtc_base/trace_categories.h" +#include "third_party/perfetto/include/perfetto/tracing/tracing.h" +#else +#include +#include +#include + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/sequence_checker.h" +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "rtc_base/logging.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/platform_thread_types.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/time_utils.h" +#endif + +namespace webrtc { + +namespace { + +#if !defined(RTC_USE_PERFETTO) +GetCategoryEnabledPtr g_get_category_enabled_ptr = nullptr; +AddTraceEventPtr g_add_trace_event_ptr = nullptr; +#endif + +} // namespace + +#if defined(RTC_USE_PERFETTO) +void RegisterPerfettoTrackEvents() { + if (perfetto::Tracing::IsInitialized()) { + webrtc::TrackEvent::Register(); + } +} +#else + +void SetupEventTracer(GetCategoryEnabledPtr get_category_enabled_ptr, + AddTraceEventPtr add_trace_event_ptr) { + g_get_category_enabled_ptr = get_category_enabled_ptr; + g_add_trace_event_ptr = add_trace_event_ptr; +} + +const unsigned char* EventTracer::GetCategoryEnabled(const char* name) { + if (g_get_category_enabled_ptr) + return g_get_category_enabled_ptr(name); + + // A string with null terminator means category is disabled. + return reinterpret_cast("\0"); +} + +// Arguments to this function (phase, etc.) are as defined in +// webrtc/rtc_base/trace_event.h. +void EventTracer::AddTraceEvent(char phase, + const unsigned char* category_enabled, + const char* name, + unsigned long long id, + int num_args, + const char** arg_names, + const unsigned char* arg_types, + const unsigned long long* arg_values, + unsigned char flags) { + if (g_add_trace_event_ptr) { + g_add_trace_event_ptr(phase, category_enabled, name, id, num_args, + arg_names, arg_types, arg_values, flags); + } +} +#endif + +} // namespace webrtc + +#if defined(RTC_USE_PERFETTO) +// TODO(bugs.webrtc.org/15917): Implement for perfetto. +namespace rtc::tracing { +void SetupInternalTracer(bool enable_all_categories) {} +bool StartInternalCapture(absl::string_view filename) { + return false; +} +void StartInternalCaptureToFile(FILE* file) {} +void StopInternalCapture() {} +void ShutdownInternalTracer() {} + +} // namespace rtc::tracing +#else + +// This is a guesstimate that should be enough in most cases. +static const size_t kEventLoggerArgsStrBufferInitialSize = 256; +static const size_t kTraceArgBufferLength = 32; + +namespace rtc { +namespace tracing { +namespace { + +// Atomic-int fast path for avoiding logging when disabled. +static std::atomic g_event_logging_active(0); + +// TODO(pbos): Log metadata for all threads, etc. +class EventLogger final { + public: + ~EventLogger() { RTC_DCHECK(thread_checker_.IsCurrent()); } + + void AddTraceEvent(const char* name, + const unsigned char* category_enabled, + char phase, + int num_args, + const char** arg_names, + const unsigned char* arg_types, + const unsigned long long* arg_values, + uint64_t timestamp, + int pid, + rtc::PlatformThreadId thread_id) { + std::vector args(num_args); + for (int i = 0; i < num_args; ++i) { + TraceArg& arg = args[i]; + arg.name = arg_names[i]; + arg.type = arg_types[i]; + arg.value.as_uint = arg_values[i]; + + // Value is a pointer to a temporary string, so we have to make a copy. + if (arg.type == TRACE_VALUE_TYPE_COPY_STRING) { + // Space for the string and for the terminating null character. + size_t str_length = strlen(arg.value.as_string) + 1; + char* str_copy = new char[str_length]; + memcpy(str_copy, arg.value.as_string, str_length); + arg.value.as_string = str_copy; + } + } + webrtc::MutexLock lock(&mutex_); + trace_events_.push_back( + {name, category_enabled, phase, args, timestamp, 1, thread_id}); + } + + // The TraceEvent format is documented here: + // https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview + void Log() { + RTC_DCHECK(output_file_); + static constexpr webrtc::TimeDelta kLoggingInterval = + webrtc::TimeDelta::Millis(100); + fprintf(output_file_, "{ \"traceEvents\": [\n"); + bool has_logged_event = false; + while (true) { + bool shutting_down = shutdown_event_.Wait(kLoggingInterval); + std::vector events; + { + webrtc::MutexLock lock(&mutex_); + trace_events_.swap(events); + } + std::string args_str; + args_str.reserve(kEventLoggerArgsStrBufferInitialSize); + for (TraceEvent& e : events) { + args_str.clear(); + if (!e.args.empty()) { + args_str += ", \"args\": {"; + bool is_first_argument = true; + for (TraceArg& arg : e.args) { + if (!is_first_argument) + args_str += ","; + is_first_argument = false; + args_str += " \""; + args_str += arg.name; + args_str += "\": "; + args_str += TraceArgValueAsString(arg); + + // Delete our copy of the string. + if (arg.type == TRACE_VALUE_TYPE_COPY_STRING) { + delete[] arg.value.as_string; + arg.value.as_string = nullptr; + } + } + args_str += " }"; + } + fprintf(output_file_, + "%s{ \"name\": \"%s\"" + ", \"cat\": \"%s\"" + ", \"ph\": \"%c\"" + ", \"ts\": %" PRIu64 + ", \"pid\": %d" +#if defined(WEBRTC_WIN) + ", \"tid\": %lu" +#else + ", \"tid\": %d" +#endif // defined(WEBRTC_WIN) + "%s" + "}\n", + has_logged_event ? "," : " ", e.name, e.category_enabled, + e.phase, e.timestamp, e.pid, e.tid, args_str.c_str()); + has_logged_event = true; + } + if (shutting_down) + break; + } + fprintf(output_file_, "]}\n"); + if (output_file_owned_) + fclose(output_file_); + output_file_ = nullptr; + } + + void Start(FILE* file, bool owned) { + RTC_DCHECK(thread_checker_.IsCurrent()); + RTC_DCHECK(file); + RTC_DCHECK(!output_file_); + output_file_ = file; + output_file_owned_ = owned; + { + webrtc::MutexLock lock(&mutex_); + // Since the atomic fast-path for adding events to the queue can be + // bypassed while the logging thread is shutting down there may be some + // stale events in the queue, hence the vector needs to be cleared to not + // log events from a previous logging session (which may be days old). + trace_events_.clear(); + } + // Enable event logging (fast-path). This should be disabled since starting + // shouldn't be done twice. + int zero = 0; + RTC_CHECK(g_event_logging_active.compare_exchange_strong(zero, 1)); + + // Finally start, everything should be set up now. + logging_thread_ = + PlatformThread::SpawnJoinable([this] { Log(); }, "EventTracingThread"); + TRACE_EVENT_INSTANT0("webrtc", "EventLogger::Start", + TRACE_EVENT_SCOPE_GLOBAL); + } + + void Stop() { + RTC_DCHECK(thread_checker_.IsCurrent()); + TRACE_EVENT_INSTANT0("webrtc", "EventLogger::Stop", + TRACE_EVENT_SCOPE_GLOBAL); + // Try to stop. Abort if we're not currently logging. + int one = 1; + if (g_event_logging_active.compare_exchange_strong(one, 0)) + return; + + // Wake up logging thread to finish writing. + shutdown_event_.Set(); + // Join the logging thread. + logging_thread_.Finalize(); + } + + private: + struct TraceArg { + const char* name; + unsigned char type; + // Copied from webrtc/rtc_base/trace_event.h TraceValueUnion. + union TraceArgValue { + bool as_bool; + unsigned long long as_uint; + long long as_int; + double as_double; + const void* as_pointer; + const char* as_string; + } value; + + // Assert that the size of the union is equal to the size of the as_uint + // field since we are assigning to arbitrary types using it. + static_assert(sizeof(TraceArgValue) == sizeof(unsigned long long), + "Size of TraceArg value union is not equal to the size of " + "the uint field of that union."); + }; + + struct TraceEvent { + const char* name; + const unsigned char* category_enabled; + char phase; + std::vector args; + uint64_t timestamp; + int pid; + rtc::PlatformThreadId tid; + }; + + static std::string TraceArgValueAsString(TraceArg arg) { + std::string output; + + if (arg.type == TRACE_VALUE_TYPE_STRING || + arg.type == TRACE_VALUE_TYPE_COPY_STRING) { + // Space for every character to be an espaced character + two for + // quatation marks. + output.reserve(strlen(arg.value.as_string) * 2 + 2); + output += '\"'; + const char* c = arg.value.as_string; + do { + if (*c == '"' || *c == '\\') { + output += '\\'; + output += *c; + } else { + output += *c; + } + } while (*++c); + output += '\"'; + } else { + output.resize(kTraceArgBufferLength); + size_t print_length = 0; + switch (arg.type) { + case TRACE_VALUE_TYPE_BOOL: + if (arg.value.as_bool) { + strcpy(&output[0], "true"); + print_length = 4; + } else { + strcpy(&output[0], "false"); + print_length = 5; + } + break; + case TRACE_VALUE_TYPE_UINT: + print_length = snprintf(&output[0], kTraceArgBufferLength, "%llu", + arg.value.as_uint); + break; + case TRACE_VALUE_TYPE_INT: + print_length = snprintf(&output[0], kTraceArgBufferLength, "%lld", + arg.value.as_int); + break; + case TRACE_VALUE_TYPE_DOUBLE: + print_length = snprintf(&output[0], kTraceArgBufferLength, "%f", + arg.value.as_double); + break; + case TRACE_VALUE_TYPE_POINTER: + print_length = snprintf(&output[0], kTraceArgBufferLength, "\"%p\"", + arg.value.as_pointer); + break; + } + size_t output_length = print_length < kTraceArgBufferLength + ? print_length + : kTraceArgBufferLength - 1; + // This will hopefully be very close to nop. On most implementations, it + // just writes null byte and sets the length field of the string. + output.resize(output_length); + } + + return output; + } + + webrtc::Mutex mutex_; + std::vector trace_events_ RTC_GUARDED_BY(mutex_); + rtc::PlatformThread logging_thread_; + rtc::Event shutdown_event_; + webrtc::SequenceChecker thread_checker_; + FILE* output_file_ = nullptr; + bool output_file_owned_ = false; +}; + +static std::atomic g_event_logger(nullptr); +static const char* const kDisabledTracePrefix = TRACE_DISABLED_BY_DEFAULT(""); +const unsigned char* InternalGetCategoryEnabled(const char* name) { + const char* prefix_ptr = &kDisabledTracePrefix[0]; + const char* name_ptr = name; + // Check whether name contains the default-disabled prefix. + while (*prefix_ptr == *name_ptr && *prefix_ptr != '\0') { + ++prefix_ptr; + ++name_ptr; + } + return reinterpret_cast(*prefix_ptr == '\0' ? "" + : name); +} + +const unsigned char* InternalEnableAllCategories(const char* name) { + return reinterpret_cast(name); +} + +void InternalAddTraceEvent(char phase, + const unsigned char* category_enabled, + const char* name, + unsigned long long id, + int num_args, + const char** arg_names, + const unsigned char* arg_types, + const unsigned long long* arg_values, + unsigned char flags) { + // Fast path for when event tracing is inactive. + if (g_event_logging_active.load() == 0) + return; + + g_event_logger.load()->AddTraceEvent( + name, category_enabled, phase, num_args, arg_names, arg_types, arg_values, + rtc::TimeMicros(), 1, rtc::CurrentThreadId()); +} + +} // namespace + +void SetupInternalTracer(bool enable_all_categories) { + EventLogger* null_logger = nullptr; + RTC_CHECK( + g_event_logger.compare_exchange_strong(null_logger, new EventLogger())); + webrtc::SetupEventTracer(enable_all_categories ? InternalEnableAllCategories + : InternalGetCategoryEnabled, + InternalAddTraceEvent); +} + +void StartInternalCaptureToFile(FILE* file) { + EventLogger* event_logger = g_event_logger.load(); + if (event_logger) { + event_logger->Start(file, false); + } +} + +bool StartInternalCapture(absl::string_view filename) { + EventLogger* event_logger = g_event_logger.load(); + if (!event_logger) + return false; + + FILE* file = fopen(std::string(filename).c_str(), "w"); + if (!file) { + RTC_LOG(LS_ERROR) << "Failed to open trace file '" << filename + << "' for writing."; + return false; + } + event_logger->Start(file, true); + return true; +} + +void StopInternalCapture() { + EventLogger* event_logger = g_event_logger.load(); + if (event_logger) { + event_logger->Stop(); + } +} + +void ShutdownInternalTracer() { + StopInternalCapture(); + EventLogger* old_logger = g_event_logger.load(std::memory_order_acquire); + RTC_DCHECK(old_logger); + RTC_CHECK(g_event_logger.compare_exchange_strong(old_logger, nullptr)); + delete old_logger; + webrtc::SetupEventTracer(nullptr, nullptr); +} + +} // namespace tracing +} // namespace rtc + +#endif // defined(RTC_USE_PERFETTO) diff --git a/VocieProcess/rtc_base/event_tracer.h b/VocieProcess/rtc_base/event_tracer.h new file mode 100644 index 0000000..7f8d3e6 --- /dev/null +++ b/VocieProcess/rtc_base/event_tracer.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_EVENT_TRACER_H_ +#define RTC_BASE_EVENT_TRACER_H_ + +// This file defines the interface for event tracing in WebRTC. +// +// Event log handlers are set through SetupEventTracer(). User of this API will +// provide two function pointers to handle event tracing calls. +// +// * GetCategoryEnabledPtr +// Event tracing system calls this function to determine if a particular +// event category is enabled. +// +// * AddTraceEventPtr +// Adds a tracing event. It is the user's responsibility to log the data +// provided. +// +// Parameters for the above two functions are described in trace_event.h. + +#include + +#include "absl/strings/string_view.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +#if defined(RTC_USE_PERFETTO) +void RegisterPerfettoTrackEvents(); +#else +typedef const unsigned char* (*GetCategoryEnabledPtr)(const char* name); +typedef void (*AddTraceEventPtr)(char phase, + const unsigned char* category_enabled, + const char* name, + unsigned long long id, + int num_args, + const char** arg_names, + const unsigned char* arg_types, + const unsigned long long* arg_values, + unsigned char flags); + +// User of WebRTC can call this method to setup event tracing. +// +// This method must be called before any WebRTC methods. Functions +// provided should be thread-safe. +void SetupEventTracer(GetCategoryEnabledPtr get_category_enabled_ptr, + AddTraceEventPtr add_trace_event_ptr); + +// This class defines interface for the event tracing system to call +// internally. Do not call these methods directly. +class EventTracer { + public: + static const unsigned char* GetCategoryEnabled(const char* name); + + static void AddTraceEvent(char phase, + const unsigned char* category_enabled, + const char* name, + unsigned long long id, + int num_args, + const char** arg_names, + const unsigned char* arg_types, + const unsigned long long* arg_values, + unsigned char flags); +}; +#endif + +} // namespace webrtc + +namespace rtc::tracing { +// Set up internal event tracer. +// TODO(webrtc:15917): Implement for perfetto. +RTC_EXPORT void SetupInternalTracer(bool enable_all_categories = true); +RTC_EXPORT bool StartInternalCapture(absl::string_view filename); +RTC_EXPORT void StartInternalCaptureToFile(FILE* file); +RTC_EXPORT void StopInternalCapture(); +// Make sure we run this, this will tear down the internal tracing. +RTC_EXPORT void ShutdownInternalTracer(); +} // namespace rtc::tracing + +#endif // RTC_BASE_EVENT_TRACER_H_ diff --git a/VocieProcess/rtc_base/platform_thread.cc b/VocieProcess/rtc_base/platform_thread.cc new file mode 100644 index 0000000..6433323 --- /dev/null +++ b/VocieProcess/rtc_base/platform_thread.cc @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/platform_thread.h" + +#include +#include + +#if !defined(WEBRTC_WIN) +#include +#endif + +#include "rtc_base/checks.h" + +namespace rtc { +namespace { + +#if defined(WEBRTC_WIN) +int Win32PriorityFromThreadPriority(ThreadPriority priority) { + switch (priority) { + case ThreadPriority::kLow: + return THREAD_PRIORITY_BELOW_NORMAL; + case ThreadPriority::kNormal: + return THREAD_PRIORITY_NORMAL; + case ThreadPriority::kHigh: + return THREAD_PRIORITY_ABOVE_NORMAL; + case ThreadPriority::kRealtime: + return THREAD_PRIORITY_TIME_CRITICAL; + } +} +#endif + +bool SetPriority(ThreadPriority priority) { +#if defined(WEBRTC_WIN) + return SetThreadPriority(GetCurrentThread(), + Win32PriorityFromThreadPriority(priority)) != FALSE; +#elif defined(__native_client__) || defined(WEBRTC_FUCHSIA) || \ + (defined(__EMSCRIPTEN__) && !defined(__EMSCRIPTEN_PTHREADS__)) + // Setting thread priorities is not supported in NaCl, Fuchsia or Emscripten + // without pthreads. + return true; +#elif defined(WEBRTC_CHROMIUM_BUILD) && defined(WEBRTC_LINUX) + // TODO(tommi): Switch to the same mechanism as Chromium uses for changing + // thread priorities. + return true; +#else + const int policy = SCHED_FIFO; + const int min_prio = sched_get_priority_min(policy); + const int max_prio = sched_get_priority_max(policy); + if (min_prio == -1 || max_prio == -1) { + return false; + } + + if (max_prio - min_prio <= 2) + return false; + + // Convert webrtc priority to system priorities: + sched_param param; + const int top_prio = max_prio - 1; + const int low_prio = min_prio + 1; + switch (priority) { + case ThreadPriority::kLow: + param.sched_priority = low_prio; + break; + case ThreadPriority::kNormal: + // The -1 ensures that the kHighPriority is always greater or equal to + // kNormalPriority. + param.sched_priority = (low_prio + top_prio - 1) / 2; + break; + case ThreadPriority::kHigh: + param.sched_priority = std::max(top_prio - 2, low_prio); + break; + case ThreadPriority::kRealtime: + param.sched_priority = top_prio; + break; + } + return pthread_setschedparam(pthread_self(), policy, ¶m) == 0; +#endif // defined(WEBRTC_WIN) +} + +#if defined(WEBRTC_WIN) +DWORD WINAPI RunPlatformThread(void* param) { + // The GetLastError() function only returns valid results when it is called + // after a Win32 API function that returns a "failed" result. A crash dump + // contains the result from GetLastError() and to make sure it does not + // falsely report a Windows error we call SetLastError here. + ::SetLastError(ERROR_SUCCESS); + auto function = static_cast*>(param); + (*function)(); + delete function; + return 0; +} +#else +void* RunPlatformThread(void* param) { + auto function = static_cast*>(param); + (*function)(); + delete function; + return 0; +} +#endif // defined(WEBRTC_WIN) + +} // namespace + +PlatformThread::PlatformThread(Handle handle, bool joinable) + : handle_(handle), joinable_(joinable) {} + +PlatformThread::PlatformThread(PlatformThread&& rhs) + : handle_(rhs.handle_), joinable_(rhs.joinable_) { + rhs.handle_ = absl::nullopt; +} + +PlatformThread& PlatformThread::operator=(PlatformThread&& rhs) { + Finalize(); + handle_ = rhs.handle_; + joinable_ = rhs.joinable_; + rhs.handle_ = absl::nullopt; + return *this; +} + +PlatformThread::~PlatformThread() { + Finalize(); +} + +PlatformThread PlatformThread::SpawnJoinable( + std::function thread_function, + absl::string_view name, + ThreadAttributes attributes) { + return SpawnThread(std::move(thread_function), name, attributes, + /*joinable=*/true); +} + +PlatformThread PlatformThread::SpawnDetached( + std::function thread_function, + absl::string_view name, + ThreadAttributes attributes) { + return SpawnThread(std::move(thread_function), name, attributes, + /*joinable=*/false); +} + +absl::optional PlatformThread::GetHandle() const { + return handle_; +} + +#if defined(WEBRTC_WIN) +bool PlatformThread::QueueAPC(PAPCFUNC function, ULONG_PTR data) { + RTC_DCHECK(handle_.has_value()); + return handle_.has_value() ? QueueUserAPC(function, *handle_, data) != FALSE + : false; +} +#endif + +void PlatformThread::Finalize() { + if (!handle_.has_value()) + return; +#if defined(WEBRTC_WIN) + if (joinable_) + WaitForSingleObject(*handle_, INFINITE); + CloseHandle(*handle_); +#else + if (joinable_) + RTC_CHECK_EQ(0, pthread_join(*handle_, nullptr)); +#endif + handle_ = absl::nullopt; +} + +PlatformThread PlatformThread::SpawnThread( + std::function thread_function, + absl::string_view name, + ThreadAttributes attributes, + bool joinable) { + RTC_DCHECK(thread_function); + RTC_DCHECK(!name.empty()); + // TODO(tommi): Consider lowering the limit to 15 (limit on Linux). + RTC_DCHECK(name.length() < 64); + auto start_thread_function_ptr = + new std::function([thread_function = std::move(thread_function), + name = std::string(name), attributes] { + rtc::SetCurrentThreadName(name.c_str()); + SetPriority(attributes.priority); + thread_function(); + }); +#if defined(WEBRTC_WIN) + // See bug 2902 for background on STACK_SIZE_PARAM_IS_A_RESERVATION. + // Set the reserved stack stack size to 1M, which is the default on Windows + // and Linux. + DWORD thread_id = 0; + PlatformThread::Handle handle = ::CreateThread( + nullptr, 1024 * 1024, &RunPlatformThread, start_thread_function_ptr, + STACK_SIZE_PARAM_IS_A_RESERVATION, &thread_id); + RTC_CHECK(handle) << "CreateThread failed"; +#else + pthread_attr_t attr; + pthread_attr_init(&attr); + // Set the stack stack size to 1M. + pthread_attr_setstacksize(&attr, 1024 * 1024); + pthread_attr_setdetachstate( + &attr, joinable ? PTHREAD_CREATE_JOINABLE : PTHREAD_CREATE_DETACHED); + PlatformThread::Handle handle; + RTC_CHECK_EQ(0, pthread_create(&handle, &attr, &RunPlatformThread, + start_thread_function_ptr)); + pthread_attr_destroy(&attr); +#endif // defined(WEBRTC_WIN) + return PlatformThread(handle, joinable); +} + +} // namespace rtc diff --git a/VocieProcess/rtc_base/platform_thread.h b/VocieProcess/rtc_base/platform_thread.h new file mode 100644 index 0000000..befd618 --- /dev/null +++ b/VocieProcess/rtc_base/platform_thread.h @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_PLATFORM_THREAD_H_ +#define RTC_BASE_PLATFORM_THREAD_H_ + +#include +#include +#if !defined(WEBRTC_WIN) +#include +#endif + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "rtc_base/platform_thread_types.h" + +namespace rtc { + +enum class ThreadPriority { + kLow = 1, + kNormal, + kHigh, + kRealtime, +}; + +struct ThreadAttributes { + ThreadPriority priority = ThreadPriority::kNormal; + ThreadAttributes& SetPriority(ThreadPriority priority_param) { + priority = priority_param; + return *this; + } +}; + +// Represents a simple worker thread. +class PlatformThread final { + public: + // Handle is the base platform thread handle. +#if defined(WEBRTC_WIN) + using Handle = HANDLE; +#else + using Handle = pthread_t; +#endif // defined(WEBRTC_WIN) + // This ctor creates the PlatformThread with an unset handle (returning true + // in empty()) and is provided for convenience. + // TODO(bugs.webrtc.org/12727) Look into if default and move support can be + // removed. + PlatformThread() = default; + + // Moves `rhs` into this, storing an empty state in `rhs`. + // TODO(bugs.webrtc.org/12727) Look into if default and move support can be + // removed. + PlatformThread(PlatformThread&& rhs); + + // Copies won't work since we'd have problems with joinable threads. + PlatformThread(const PlatformThread&) = delete; + PlatformThread& operator=(const PlatformThread&) = delete; + + // Moves `rhs` into this, storing an empty state in `rhs`. + // TODO(bugs.webrtc.org/12727) Look into if default and move support can be + // removed. + PlatformThread& operator=(PlatformThread&& rhs); + + // For a PlatformThread that's been spawned joinable, the destructor suspends + // the calling thread until the created thread exits unless the thread has + // already exited. + virtual ~PlatformThread(); + + // Finalizes any allocated resources. + // For a PlatformThread that's been spawned joinable, Finalize() suspends + // the calling thread until the created thread exits unless the thread has + // already exited. + // empty() returns true after completion. + void Finalize(); + + // Returns true if default constructed, moved from, or Finalize()ed. + bool empty() const { return !handle_.has_value(); } + + // Creates a started joinable thread which will be joined when the returned + // PlatformThread destructs or Finalize() is called. + static PlatformThread SpawnJoinable( + std::function thread_function, + absl::string_view name, + ThreadAttributes attributes = ThreadAttributes()); + + // Creates a started detached thread. The caller has to use external + // synchronization as nothing is provided by the PlatformThread construct. + static PlatformThread SpawnDetached( + std::function thread_function, + absl::string_view name, + ThreadAttributes attributes = ThreadAttributes()); + + // Returns the base platform thread handle of this thread. + absl::optional GetHandle() const; + +#if defined(WEBRTC_WIN) + // Queue a Windows APC function that runs when the thread is alertable. + bool QueueAPC(PAPCFUNC apc_function, ULONG_PTR data); +#endif + + private: + PlatformThread(Handle handle, bool joinable); + static PlatformThread SpawnThread(std::function thread_function, + absl::string_view name, + ThreadAttributes attributes, + bool joinable); + + absl::optional handle_; + bool joinable_ = false; +}; + +} // namespace rtc + +#endif // RTC_BASE_PLATFORM_THREAD_H_ diff --git a/VocieProcess/rtc_base/ref_count.h b/VocieProcess/rtc_base/ref_count.h new file mode 100644 index 0000000..45940e2 --- /dev/null +++ b/VocieProcess/rtc_base/ref_count.h @@ -0,0 +1,29 @@ +/* + * Copyright 2011 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef RTC_BASE_REF_COUNT_H_ +#define RTC_BASE_REF_COUNT_H_ + +// Transition file for backwards compatibility with source code +// that includes the non-API file. + +#include "api/ref_count.h" + +namespace rtc { + +// TODO(bugs.webrtc.org/15622): Deprecate and remove these aliases. +using RefCountInterface [[deprecated("Use webrtc::RefCountInterface")]] = + webrtc::RefCountInterface; +using RefCountReleaseStatus + [[deprecated("Use webrtc::RefCountReleaseStatus")]] = + webrtc::RefCountReleaseStatus; + +} // namespace rtc + +#endif // RTC_BASE_REF_COUNT_H_ diff --git a/VocieProcess/rtc_base/ref_counted_object.h b/VocieProcess/rtc_base/ref_counted_object.h new file mode 100644 index 0000000..007e9f9 --- /dev/null +++ b/VocieProcess/rtc_base/ref_counted_object.h @@ -0,0 +1,142 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef RTC_BASE_REF_COUNTED_OBJECT_H_ +#define RTC_BASE_REF_COUNTED_OBJECT_H_ + +#include "api/scoped_refptr.h" +#include "rtc_base/ref_count.h" +#include "rtc_base/ref_counter.h" + +namespace webrtc { + +template +class RefCountedObject : public T { + public: + RefCountedObject() {} + + RefCountedObject(const RefCountedObject&) = delete; + RefCountedObject& operator=(const RefCountedObject&) = delete; + + template + explicit RefCountedObject(P0&& p0) : T(std::forward(p0)) {} + + template + RefCountedObject(P0&& p0, P1&& p1, Args&&... args) + : T(std::forward(p0), + std::forward(p1), + std::forward(args)...) {} + + void AddRef() const override { ref_count_.IncRef(); } + + RefCountReleaseStatus Release() const override { + const auto status = ref_count_.DecRef(); + if (status == RefCountReleaseStatus::kDroppedLastRef) { + delete this; + } + return status; + } + + // Return whether the reference count is one. If the reference count is used + // in the conventional way, a reference count of 1 implies that the current + // thread owns the reference and no other thread shares it. This call + // performs the test for a reference count of one, and performs the memory + // barrier needed for the owning thread to act on the object, knowing that it + // has exclusive access to the object. + virtual bool HasOneRef() const { return ref_count_.HasOneRef(); } + + protected: + ~RefCountedObject() override {} + + mutable webrtc::webrtc_impl::RefCounter ref_count_{0}; +}; + +template +class FinalRefCountedObject final : public T { + public: + using T::T; + // Above using declaration propagates a default move constructor + // FinalRefCountedObject(FinalRefCountedObject&& other), but we also need + // move construction from T. + explicit FinalRefCountedObject(T&& other) : T(std::move(other)) {} + FinalRefCountedObject(const FinalRefCountedObject&) = delete; + FinalRefCountedObject& operator=(const FinalRefCountedObject&) = delete; + + void AddRef() const { ref_count_.IncRef(); } + RefCountReleaseStatus Release() const { + const auto status = ref_count_.DecRef(); + if (status == RefCountReleaseStatus::kDroppedLastRef) { + delete this; + } + return status; + } + bool HasOneRef() const { return ref_count_.HasOneRef(); } + + private: + ~FinalRefCountedObject() = default; + + mutable webrtc::webrtc_impl::RefCounter ref_count_{0}; +}; + +} // namespace webrtc + +// Backwards compatibe aliases. +// TODO: https://issues.webrtc.org/42225969 - deprecate and remove. +namespace rtc { +// Because there are users of this template that use "friend +// rtc::RefCountedObject<>" to give access to a private destructor, some +// indirection is needed; "friend" declarations cannot span an "using" +// declaration. Since a templated class on top of a templated class can't access +// the subclass' protected members, we duplicate the entire class instead. +template +class RefCountedObject : public T { + public: + RefCountedObject() {} + + RefCountedObject(const RefCountedObject&) = delete; + RefCountedObject& operator=(const RefCountedObject&) = delete; + + template + explicit RefCountedObject(P0&& p0) : T(std::forward(p0)) {} + + template + RefCountedObject(P0&& p0, P1&& p1, Args&&... args) + : T(std::forward(p0), + std::forward(p1), + std::forward(args)...) {} + + void AddRef() const override { ref_count_.IncRef(); } + + webrtc::RefCountReleaseStatus Release() const override { + const auto status = ref_count_.DecRef(); + if (status == webrtc::RefCountReleaseStatus::kDroppedLastRef) { + delete this; + } + return status; + } + + // Return whether the reference count is one. If the reference count is used + // in the conventional way, a reference count of 1 implies that the current + // thread owns the reference and no other thread shares it. This call + // performs the test for a reference count of one, and performs the memory + // barrier needed for the owning thread to act on the object, knowing that it + // has exclusive access to the object. + virtual bool HasOneRef() const { return ref_count_.HasOneRef(); } + + protected: + ~RefCountedObject() override {} + + mutable webrtc::webrtc_impl::RefCounter ref_count_{0}; +}; + +template +using FinalRefCountedObject = webrtc::FinalRefCountedObject; +} // namespace rtc + +#endif // RTC_BASE_REF_COUNTED_OBJECT_H_ diff --git a/VocieProcess/rtc_base/ref_counter.h b/VocieProcess/rtc_base/ref_counter.h new file mode 100644 index 0000000..93ae3d2 --- /dev/null +++ b/VocieProcess/rtc_base/ref_counter.h @@ -0,0 +1,75 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef RTC_BASE_REF_COUNTER_H_ +#define RTC_BASE_REF_COUNTER_H_ + +#include + +#include "rtc_base/ref_count.h" + +namespace webrtc { +namespace webrtc_impl { + +class RefCounter { + public: + explicit RefCounter(int ref_count) : ref_count_(ref_count) {} + RefCounter() = delete; + + void IncRef() { + // Relaxed memory order: The current thread is allowed to act on the + // resource protected by the reference counter both before and after the + // atomic op, so this function doesn't prevent memory access reordering. + ref_count_.fetch_add(1, std::memory_order_relaxed); + } + + // Returns kDroppedLastRef if this call dropped the last reference; the caller + // should therefore free the resource protected by the reference counter. + // Otherwise, returns kOtherRefsRemained (note that in case of multithreading, + // some other caller may have dropped the last reference by the time this call + // returns; all we know is that we didn't do it). + RefCountReleaseStatus DecRef() { + // Use release-acquire barrier to ensure all actions on the protected + // resource are finished before the resource can be freed. + // When ref_count_after_subtract > 0, this function require + // std::memory_order_release part of the barrier. + // When ref_count_after_subtract == 0, this function require + // std::memory_order_acquire part of the barrier. + // In addition std::memory_order_release is used for synchronization with + // the HasOneRef function to make sure all actions on the protected resource + // are finished before the resource is assumed to have exclusive access. + int ref_count_after_subtract = + ref_count_.fetch_sub(1, std::memory_order_acq_rel) - 1; + return ref_count_after_subtract == 0 + ? RefCountReleaseStatus::kDroppedLastRef + : RefCountReleaseStatus::kOtherRefsRemained; + } + + // Return whether the reference count is one. If the reference count is used + // in the conventional way, a reference count of 1 implies that the current + // thread owns the reference and no other thread shares it. This call performs + // the test for a reference count of one, and performs the memory barrier + // needed for the owning thread to act on the resource protected by the + // reference counter, knowing that it has exclusive access. + bool HasOneRef() const { + // To ensure resource protected by the reference counter has exclusive + // access, all changes to the resource before it was released by other + // threads must be visible by current thread. That is provided by release + // (in DecRef) and acquire (in this function) ordering. + return ref_count_.load(std::memory_order_acquire) == 1; + } + + private: + std::atomic ref_count_; +}; + +} // namespace webrtc_impl +} // namespace webrtc + +#endif // RTC_BASE_REF_COUNTER_H_ diff --git a/VocieProcess/rtc_base/synchronization/sequence_checker_internal.cc b/VocieProcess/rtc_base/synchronization/sequence_checker_internal.cc new file mode 100644 index 0000000..4b9583d --- /dev/null +++ b/VocieProcess/rtc_base/synchronization/sequence_checker_internal.cc @@ -0,0 +1,86 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "rtc_base/synchronization/sequence_checker_internal.h" + +#include + +#include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +namespace webrtc_sequence_checker_internal { + +SequenceCheckerImpl::SequenceCheckerImpl(bool attach_to_current_thread) + : attached_(attach_to_current_thread), + valid_thread_(rtc::CurrentThreadRef()), + valid_queue_(TaskQueueBase::Current()) {} + +SequenceCheckerImpl::SequenceCheckerImpl(TaskQueueBase* attached_queue) + : attached_(attached_queue != nullptr), + valid_thread_(rtc::PlatformThreadRef()), + valid_queue_(attached_queue) {} + +bool SequenceCheckerImpl::IsCurrent() const { + const TaskQueueBase* const current_queue = TaskQueueBase::Current(); + const rtc::PlatformThreadRef current_thread = rtc::CurrentThreadRef(); + MutexLock scoped_lock(&lock_); + if (!attached_) { // Previously detached. + attached_ = true; + valid_thread_ = current_thread; + valid_queue_ = current_queue; + return true; + } + if (valid_queue_) { + return valid_queue_ == current_queue; + } + return rtc::IsThreadRefEqual(valid_thread_, current_thread); +} + +void SequenceCheckerImpl::Detach() { + MutexLock scoped_lock(&lock_); + attached_ = false; + // We don't need to touch the other members here, they will be + // reset on the next call to IsCurrent(). +} + +#if RTC_DCHECK_IS_ON +std::string SequenceCheckerImpl::ExpectationToString() const { + const TaskQueueBase* const current_queue = TaskQueueBase::Current(); + const rtc::PlatformThreadRef current_thread = rtc::CurrentThreadRef(); + MutexLock scoped_lock(&lock_); + if (!attached_) + return "Checker currently not attached."; + + // The format of the string is meant to compliment the one we have inside of + // FatalLog() (checks.cc). Example: + // + // # Expected: TQ: 0x0 SysQ: 0x7fff69541330 Thread: 0x11dcf6dc0 + // # Actual: TQ: 0x7fa8f0604190 SysQ: 0x7fa8f0604a30 Thread: 0x700006f1a000 + // TaskQueue doesn't match + + rtc::StringBuilder message; + message.AppendFormat( + "# Expected: TQ: %p Thread: %p\n" + "# Actual: TQ: %p Thread: %p\n", + valid_queue_, reinterpret_cast(valid_thread_), current_queue, + reinterpret_cast(current_thread)); + + if ((valid_queue_ || current_queue) && valid_queue_ != current_queue) { + message << "TaskQueue doesn't match\n"; + } else if (!rtc::IsThreadRefEqual(valid_thread_, current_thread)) { + message << "Threads don't match\n"; + } + + return message.Release(); +} +#endif // RTC_DCHECK_IS_ON + +} // namespace webrtc_sequence_checker_internal +} // namespace webrtc diff --git a/VocieProcess/rtc_base/synchronization/sequence_checker_internal.h b/VocieProcess/rtc_base/synchronization/sequence_checker_internal.h new file mode 100644 index 0000000..a23ac08 --- /dev/null +++ b/VocieProcess/rtc_base/synchronization/sequence_checker_internal.h @@ -0,0 +1,90 @@ +/* + * Copyright 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef RTC_BASE_SYNCHRONIZATION_SEQUENCE_CHECKER_INTERNAL_H_ +#define RTC_BASE_SYNCHRONIZATION_SEQUENCE_CHECKER_INTERNAL_H_ + +#include +#include + +#include "api/task_queue/task_queue_base.h" +#include "rtc_base/platform_thread_types.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system/rtc_export.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { +namespace webrtc_sequence_checker_internal { + +// Real implementation of SequenceChecker, for use in debug mode, or +// for temporary use in release mode (e.g. to RTC_CHECK on a threading issue +// seen only in the wild). +// +// Note: You should almost always use the SequenceChecker class to get the +// right version for your build configuration. +class RTC_EXPORT SequenceCheckerImpl { + public: + explicit SequenceCheckerImpl(bool attach_to_current_thread); + explicit SequenceCheckerImpl(TaskQueueBase* attached_queue); + ~SequenceCheckerImpl() = default; + + bool IsCurrent() const; + // Changes the task queue or thread that is checked for in IsCurrent. This can + // be useful when an object may be created on one task queue / thread and then + // used exclusively on another thread. + void Detach(); + + // Returns a string that is formatted to match with the error string printed + // by RTC_CHECK() when a condition is not met. + // This is used in conjunction with the RTC_DCHECK_RUN_ON() macro. + std::string ExpectationToString() const; + + private: + mutable Mutex lock_; + // These are mutable so that IsCurrent can set them. + mutable bool attached_ RTC_GUARDED_BY(lock_); + mutable rtc::PlatformThreadRef valid_thread_ RTC_GUARDED_BY(lock_); + mutable const TaskQueueBase* valid_queue_ RTC_GUARDED_BY(lock_); +}; + +// Do nothing implementation, for use in release mode. +// +// Note: You should almost always use the SequenceChecker class to get the +// right version for your build configuration. +class SequenceCheckerDoNothing { + public: + explicit SequenceCheckerDoNothing(bool attach_to_current_thread) {} + explicit SequenceCheckerDoNothing(TaskQueueBase* attached_queue) {} + bool IsCurrent() const { return true; } + void Detach() {} +}; + +template +std::enable_if_t, + std::string> +ExpectationToString(const ThreadLikeObject* checker) { +#if RTC_DCHECK_IS_ON + return checker->ExpectationToString(); +#else + return std::string(); +#endif +} + +// Catch-all implementation for types other than explicitly supported above. +template +std::enable_if_t, + std::string> +ExpectationToString(const ThreadLikeObject*) { + return std::string(); +} + +} // namespace webrtc_sequence_checker_internal +} // namespace webrtc + +#endif // RTC_BASE_SYNCHRONIZATION_SEQUENCE_CHECKER_INTERNAL_H_ diff --git a/VocieProcess/rtc_base/synchronization/yield_policy.cc b/VocieProcess/rtc_base/synchronization/yield_policy.cc new file mode 100644 index 0000000..d883d42 --- /dev/null +++ b/VocieProcess/rtc_base/synchronization/yield_policy.cc @@ -0,0 +1,82 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "rtc_base/synchronization/yield_policy.h" + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "rtc_base/checks.h" +#if !defined(ABSL_HAVE_THREAD_LOCAL) && defined(WEBRTC_POSIX) +#include +#endif + +namespace rtc { +namespace { + +#if defined(ABSL_HAVE_THREAD_LOCAL) + +ABSL_CONST_INIT thread_local YieldInterface* current_yield_policy = nullptr; + +YieldInterface* GetCurrentYieldPolicy() { + return current_yield_policy; +} + +void SetCurrentYieldPolicy(YieldInterface* ptr) { + current_yield_policy = ptr; +} + +#elif defined(WEBRTC_POSIX) + +// Emscripten does not support the C++11 thread_local keyword but does support +// the pthread thread-local storage API. +// https://github.com/emscripten-core/emscripten/issues/3502 + +ABSL_CONST_INIT pthread_key_t g_current_yield_policy_tls = 0; + +void InitializeTls() { + RTC_CHECK_EQ(pthread_key_create(&g_current_yield_policy_tls, nullptr), 0); +} + +pthread_key_t GetCurrentYieldPolicyTls() { + static pthread_once_t init_once = PTHREAD_ONCE_INIT; + RTC_CHECK_EQ(pthread_once(&init_once, &InitializeTls), 0); + return g_current_yield_policy_tls; +} + +YieldInterface* GetCurrentYieldPolicy() { + return static_cast( + pthread_getspecific(GetCurrentYieldPolicyTls())); +} + +void SetCurrentYieldPolicy(YieldInterface* ptr) { + pthread_setspecific(GetCurrentYieldPolicyTls(), ptr); +} + +#else +#error Unsupported platform +#endif + +} // namespace + +ScopedYieldPolicy::ScopedYieldPolicy(YieldInterface* policy) + : previous_(GetCurrentYieldPolicy()) { + SetCurrentYieldPolicy(policy); +} + +ScopedYieldPolicy::~ScopedYieldPolicy() { + SetCurrentYieldPolicy(previous_); +} + +void ScopedYieldPolicy::YieldExecution() { + YieldInterface* current = GetCurrentYieldPolicy(); + if (current) + current->YieldExecution(); +} + +} // namespace rtc diff --git a/VocieProcess/rtc_base/synchronization/yield_policy.h b/VocieProcess/rtc_base/synchronization/yield_policy.h new file mode 100644 index 0000000..5def6b7 --- /dev/null +++ b/VocieProcess/rtc_base/synchronization/yield_policy.h @@ -0,0 +1,38 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef RTC_BASE_SYNCHRONIZATION_YIELD_POLICY_H_ +#define RTC_BASE_SYNCHRONIZATION_YIELD_POLICY_H_ + +namespace rtc { +class YieldInterface { + public: + virtual ~YieldInterface() = default; + virtual void YieldExecution() = 0; +}; + +// Sets the current thread-local yield policy while it's in scope and reverts +// to the previous policy when it leaves the scope. +class ScopedYieldPolicy final { + public: + explicit ScopedYieldPolicy(YieldInterface* policy); + ScopedYieldPolicy(const ScopedYieldPolicy&) = delete; + ScopedYieldPolicy& operator=(const ScopedYieldPolicy&) = delete; + ~ScopedYieldPolicy(); + // Will yield as specified by the currently active thread-local yield policy + // (which by default is a no-op). + static void YieldExecution(); + + private: + YieldInterface* const previous_; +}; + +} // namespace rtc + +#endif // RTC_BASE_SYNCHRONIZATION_YIELD_POLICY_H_ diff --git a/VocieProcess/rtc_base/system/ignore_warnings.h b/VocieProcess/rtc_base/system/ignore_warnings.h new file mode 100644 index 0000000..e891c50 --- /dev/null +++ b/VocieProcess/rtc_base/system/ignore_warnings.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_SYSTEM_IGNORE_WARNINGS_H_ +#define RTC_BASE_SYSTEM_IGNORE_WARNINGS_H_ + +#ifdef __clang__ +#define RTC_PUSH_IGNORING_WFRAME_LARGER_THAN() \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wframe-larger-than=\"") +#define RTC_POP_IGNORING_WFRAME_LARGER_THAN() _Pragma("clang diagnostic pop") +#elif __GNUC__ +#define RTC_PUSH_IGNORING_WFRAME_LARGER_THAN() \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wframe-larger-than=\"") +#define RTC_POP_IGNORING_WFRAME_LARGER_THAN() _Pragma("GCC diagnostic pop") +#else +#define RTC_PUSH_IGNORING_WFRAME_LARGER_THAN() +#define RTC_POP_IGNORING_WFRAME_LARGER_THAN() +#endif + +#endif // RTC_BASE_SYSTEM_IGNORE_WARNINGS_H_ diff --git a/VocieProcess/rtc_base/system/warn_current_thread_is_deadlocked.cc b/VocieProcess/rtc_base/system/warn_current_thread_is_deadlocked.cc new file mode 100644 index 0000000..f17b19a --- /dev/null +++ b/VocieProcess/rtc_base/system/warn_current_thread_is_deadlocked.cc @@ -0,0 +1,19 @@ +/* + * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/system/warn_current_thread_is_deadlocked.h" + +#include "rtc_base/logging.h" + +namespace webrtc { + + + +} // namespace webrtc diff --git a/VocieProcess/rtc_base/system/warn_current_thread_is_deadlocked.h b/VocieProcess/rtc_base/system/warn_current_thread_is_deadlocked.h new file mode 100644 index 0000000..4a0ba9d --- /dev/null +++ b/VocieProcess/rtc_base/system/warn_current_thread_is_deadlocked.h @@ -0,0 +1,24 @@ +/* + * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_SYSTEM_WARN_CURRENT_THREAD_IS_DEADLOCKED_H_ +#define RTC_BASE_SYSTEM_WARN_CURRENT_THREAD_IS_DEADLOCKED_H_ + +namespace webrtc { + +#if defined(WEBRTC_ANDROID) && !defined(WEBRTC_CHROMIUM_BUILD) +void WarnThatTheCurrentThreadIsProbablyDeadlocked(); +#else +inline void WarnThatTheCurrentThreadIsProbablyDeadlocked() {} +#endif + +} // namespace webrtc + +#endif // RTC_BASE_SYSTEM_WARN_CURRENT_THREAD_IS_DEADLOCKED_H_ diff --git a/VocieProcess/rtc_base/trace_event.h b/VocieProcess/rtc_base/trace_event.h new file mode 100644 index 0000000..fd0ceae --- /dev/null +++ b/VocieProcess/rtc_base/trace_event.h @@ -0,0 +1,807 @@ +/* + * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_TRACE_EVENT_H_ +#define RTC_BASE_TRACE_EVENT_H_ + +#if defined(RTC_DISABLE_TRACE_EVENTS) +#define RTC_TRACE_EVENTS_ENABLED 0 +#else +#define RTC_TRACE_EVENTS_ENABLED 1 +#endif + +// IWYU pragma: begin_exports +#if defined(RTC_USE_PERFETTO) +#include "rtc_base/trace_categories.h" +#endif +// IWYU pragma: end_exports + +#if !defined(RTC_USE_PERFETTO) +#include + +#include "rtc_base/event_tracer.h" + +#define RTC_NOOP() \ + do { \ + } while (0) + +// TODO(b/42226290): Add implementation for these events with Perfetto. +#define TRACE_EVENT_BEGIN(category, name, ...) RTC_NOOP(); +#define TRACE_EVENT_END(category, ...) RTC_NOOP(); +#define TRACE_EVENT(category, name, ...) RTC_NOOP(); +#define TRACE_EVENT_INSTANT(category, name, ...) RTC_NOOP(); +#define TRACE_EVENT_CATEGORY_ENABLED(category) RTC_NOOP(); +#define TRACE_COUNTER(category, track, ...) RTC_NOOP(); + +// Type values for identifying types in the TraceValue union. +#define TRACE_VALUE_TYPE_BOOL (static_cast(1)) +#define TRACE_VALUE_TYPE_UINT (static_cast(2)) +#define TRACE_VALUE_TYPE_INT (static_cast(3)) +#define TRACE_VALUE_TYPE_DOUBLE (static_cast(4)) +#define TRACE_VALUE_TYPE_POINTER (static_cast(5)) +#define TRACE_VALUE_TYPE_STRING (static_cast(6)) +#define TRACE_VALUE_TYPE_COPY_STRING (static_cast(7)) + +#if defined(TRACE_EVENT0) +#error "Another copy of trace_event.h has already been included." +#endif + +#if RTC_TRACE_EVENTS_ENABLED + +// Extracted from Chromium's src/base/debug/trace_event.h. + +// This header is designed to give you trace_event macros without specifying +// how the events actually get collected and stored. If you need to expose trace +// event to some other universe, you can copy-and-paste this file, +// implement the TRACE_EVENT_API macros, and do any other necessary fixup for +// the target platform. The end result is that multiple libraries can funnel +// events through to a shared trace event collector. + +// Trace events are for tracking application performance and resource usage. +// Macros are provided to track: +// Begin and end of function calls +// Counters +// +// Events are issued against categories. Whereas RTC_LOG's +// categories are statically defined, TRACE categories are created +// implicitly with a string. For example: +// TRACE_EVENT_INSTANT0("MY_SUBSYSTEM", "SomeImportantEvent") +// +// Events can be INSTANT, or can be pairs of BEGIN and END in the same scope: +// TRACE_EVENT_BEGIN0("MY_SUBSYSTEM", "SomethingCostly") +// doSomethingCostly() +// TRACE_EVENT_END0("MY_SUBSYSTEM", "SomethingCostly") +// Note: our tools can't always determine the correct BEGIN/END pairs unless +// these are used in the same scope. Use ASYNC_BEGIN/ASYNC_END macros if you +// need them to be in separate scopes. +// +// A common use case is to trace entire function scopes. This +// issues a trace BEGIN and END automatically: +// void doSomethingCostly() { +// TRACE_EVENT0("MY_SUBSYSTEM", "doSomethingCostly"); +// ... +// } +// +// Additional parameters can be associated with an event: +// void doSomethingCostly2(int howMuch) { +// TRACE_EVENT1("MY_SUBSYSTEM", "doSomethingCostly", +// "howMuch", howMuch); +// ... +// } +// +// The trace system will automatically add to this information the +// current process id, thread id, and a timestamp in microseconds. +// +// To trace an asynchronous procedure such as an IPC send/receive, use +// ASYNC_BEGIN and ASYNC_END: +// [single threaded sender code] +// static int send_count = 0; +// ++send_count; +// TRACE_EVENT_ASYNC_BEGIN0("ipc", "message", send_count); +// Send(new MyMessage(send_count)); +// [receive code] +// void OnMyMessage(send_count) { +// TRACE_EVENT_ASYNC_END0("ipc", "message", send_count); +// } +// The third parameter is a unique ID to match ASYNC_BEGIN/ASYNC_END pairs. +// ASYNC_BEGIN and ASYNC_END can occur on any thread of any traced process. +// Pointers can be used for the ID parameter, and they will be mangled +// internally so that the same pointer on two different processes will not +// match. For example: +// class MyTracedClass { +// public: +// MyTracedClass() { +// TRACE_EVENT_ASYNC_BEGIN0("category", "MyTracedClass", this); +// } +// ~MyTracedClass() { +// TRACE_EVENT_ASYNC_END0("category", "MyTracedClass", this); +// } +// } +// +// Trace event also supports counters, which is a way to track a quantity +// as it varies over time. Counters are created with the following macro: +// TRACE_COUNTER1("MY_SUBSYSTEM", "myCounter", g_myCounterValue); +// +// Counters are process-specific. The macro itself can be issued from any +// thread, however. +// +// Sometimes, you want to track two counters at once. You can do this with two +// counter macros: +// TRACE_COUNTER1("MY_SUBSYSTEM", "myCounter0", g_myCounterValue[0]); +// TRACE_COUNTER1("MY_SUBSYSTEM", "myCounter1", g_myCounterValue[1]); +// Or you can do it with a combined macro: +// TRACE_COUNTER2("MY_SUBSYSTEM", "myCounter", +// "bytesPinned", g_myCounterValue[0], +// "bytesAllocated", g_myCounterValue[1]); +// This indicates to the tracing UI that these counters should be displayed +// in a single graph, as a summed area chart. +// +// Since counters are in a global namespace, you may want to disembiguate with a +// unique ID, by using the TRACE_COUNTER_ID* variations. +// +// By default, trace collection is compiled in, but turned off at runtime. +// Collecting trace data is the responsibility of the embedding +// application. In Chrome's case, navigating to about:tracing will turn on +// tracing and display data collected across all active processes. +// +// When are string argument values copied: +// const char* arg_values are only referenced by default: +// TRACE_EVENT1("category", "name", +// "arg1", "literal string is only referenced"); +// Use TRACE_STR_COPY to force copying of a const char*: +// TRACE_EVENT1("category", "name", +// "arg1", TRACE_STR_COPY("string will be copied")); +// std::string arg_values are always copied: +// TRACE_EVENT1("category", "name", +// "arg1", std::string("string will be copied")); +// +// +// Thread Safety: +// Thread safety is provided by methods defined in event_tracer.h. See the file +// for details. + +// By default, const char* argument values are assumed to have long-lived scope +// and will not be copied. Use this macro to force a const char* to be copied. +#define TRACE_STR_COPY(str) \ + webrtc::trace_event_internal::TraceStringWithCopy(str) + +// This will mark the trace event as disabled by default. The user will need +// to explicitly enable the event. +#define TRACE_DISABLED_BY_DEFAULT(name) "disabled-by-default-" name + +// By default, uint64 ID argument values are not mangled with the Process ID in +// TRACE_EVENT_ASYNC macros. Use this macro to force Process ID mangling. +#define TRACE_ID_MANGLE(id) \ + webrtc::trace_event_internal::TraceID::ForceMangle(id) + +// Records a pair of begin and end events called "name" for the current +// scope, with 0, 1 or 2 associated arguments. If the category is not +// enabled, then this does nothing. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +#define TRACE_EVENT0(category, name) \ + INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name) +#define TRACE_EVENT1(category, name, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name, arg1_name, arg1_val) +#define TRACE_EVENT2(category, name, arg1_name, arg1_val, arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name, arg1_name, arg1_val, \ + arg2_name, arg2_val) + +// Enum reflecting the scope of an INSTANT event. Must fit within +// TRACE_EVENT_FLAG_SCOPE_MASK. +static constexpr uint8_t TRACE_EVENT_SCOPE_GLOBAL = 0u << 2; +static constexpr uint8_t TRACE_EVENT_SCOPE_PROCESS = 1u << 2; +static constexpr uint8_t TRACE_EVENT_SCOPE_THREAD = 2u << 2; + +// Records a single event called "name" immediately, with 0, 1 or 2 +// associated arguments. If the category is not enabled, then this +// does nothing. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +#define TRACE_EVENT_INSTANT0(category, name, scope) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_NONE) +#define TRACE_EVENT_INSTANT1(category, name, scope, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) +#define TRACE_EVENT_INSTANT2(category, name, scope, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, \ + arg2_name, arg2_val) + +// Records a single BEGIN event called "name" immediately, with 0, 1 or 2 +// associated arguments. If the category is not enabled, then this +// does nothing. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +#define TRACE_EVENT_BEGIN0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_NONE) +#define TRACE_EVENT_BEGIN1(category, name, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) +#define TRACE_EVENT_BEGIN2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, \ + arg2_name, arg2_val) + +// Records a single END event for "name" immediately. If the category +// is not enabled, then this does nothing. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +#define TRACE_EVENT_END0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_NONE) +#define TRACE_EVENT_END1(category, name, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) +#define TRACE_EVENT_END2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, \ + arg2_name, arg2_val) + +// Records the value of a counter called "name" immediately. Value +// must be representable as a 32 bit integer. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +#define TRACE_COUNTER1(category, name, value) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, category, name, \ + TRACE_EVENT_FLAG_NONE, "value", \ + static_cast(value)) + +// Records the values of a multi-parted counter called "name" immediately. +// The UI will treat value1 and value2 as parts of a whole, displaying their +// values as a stacked-bar chart. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +#define TRACE_COUNTER2(category, name, value1_name, value1_val, value2_name, \ + value2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, category, name, \ + TRACE_EVENT_FLAG_NONE, value1_name, \ + static_cast(value1_val), value2_name, \ + static_cast(value2_val)) + +// Records the value of a counter called "name" immediately. Value +// must be representable as a 32 bit integer. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +// - `id` is used to disambiguate counters with the same name. It must either +// be a pointer or an integer value up to 64 bits. If it's a pointer, the bits +// will be xored with a hash of the process ID so that the same pointer on +// two different processes will not collide. +#define TRACE_COUNTER_ID1(category, name, id, value) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_COUNTER, category, name, \ + id, TRACE_EVENT_FLAG_NONE, "value", \ + static_cast(value)) + +// Records the values of a multi-parted counter called "name" immediately. +// The UI will treat value1 and value2 as parts of a whole, displaying their +// values as a stacked-bar chart. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +// - `id` is used to disambiguate counters with the same name. It must either +// be a pointer or an integer value up to 64 bits. If it's a pointer, the bits +// will be xored with a hash of the process ID so that the same pointer on +// two different processes will not collide. +#define TRACE_COUNTER_ID2(category, name, id, value1_name, value1_val, \ + value2_name, value2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_COUNTER, category, name, \ + id, TRACE_EVENT_FLAG_NONE, value1_name, \ + static_cast(value1_val), value2_name, \ + static_cast(value2_val)) + +// Records a single ASYNC_BEGIN event called "name" immediately, with 0, 1 or 2 +// associated arguments. If the category is not enabled, then this +// does nothing. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +// - `id` is used to match the ASYNC_BEGIN event with the ASYNC_END event. ASYNC +// events are considered to match if their category, name and id values all +// match. `id` must either be a pointer or an integer value up to 64 bits. If +// it's a pointer, the bits will be xored with a hash of the process ID so +// that the same pointer on two different processes will not collide. +// An asynchronous operation can consist of multiple phases. The first phase is +// defined by the ASYNC_BEGIN calls. Additional phases can be defined using the +// ASYNC_STEP macros. When the operation completes, call ASYNC_END. +// An ASYNC trace typically occur on a single thread (if not, they will only be +// drawn on the thread defined in the ASYNC_BEGIN event), but all events in that +// operation must use the same `name` and `id`. Each event can have its own +// args. +#define TRACE_EVENT_ASYNC_BEGIN0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE) +#define TRACE_EVENT_ASYNC_BEGIN1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE, arg1_name, \ + arg1_val) +#define TRACE_EVENT_ASYNC_BEGIN2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE, arg1_name, \ + arg1_val, arg2_name, arg2_val) + +// Records a single ASYNC_STEP event for `step` immediately. If the category +// is not enabled, then this does nothing. The `name` and `id` must match the +// ASYNC_BEGIN event above. The `step` param identifies this step within the +// async event. This should be called at the beginning of the next phase of an +// asynchronous operation. +#define TRACE_EVENT_ASYNC_STEP_INTO0(category, name, id, step) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, category, \ + name, id, TRACE_EVENT_FLAG_NONE, "step", \ + step) +#define TRACE_EVENT_ASYNC_STEP_INTO1(category, name, id, step, arg1_name, \ + arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, category, \ + name, id, TRACE_EVENT_FLAG_NONE, "step", \ + step, arg1_name, arg1_val) + +// Records a single ASYNC_END event for "name" immediately. If the category +// is not enabled, then this does nothing. +#define TRACE_EVENT_ASYNC_END0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_NONE) +#define TRACE_EVENT_ASYNC_END1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_NONE, arg1_name, \ + arg1_val) +#define TRACE_EVENT_ASYNC_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_NONE, arg1_name, \ + arg1_val, arg2_name, arg2_val) + +//////////////////////////////////////////////////////////////////////////////// +// Implementation specific tracing API definitions. + +// Get a pointer to the enabled state of the given trace category. Only +// long-lived literal strings should be given as the category name. The returned +// pointer can be held permanently in a local static for example. If the +// unsigned char is non-zero, tracing is enabled. If tracing is enabled, +// TRACE_EVENT_API_ADD_TRACE_EVENT can be called. It's OK if tracing is disabled +// between the load of the tracing state and the call to +// TRACE_EVENT_API_ADD_TRACE_EVENT, because this flag only provides an early out +// for best performance when tracing is disabled. +// const unsigned char* +// TRACE_EVENT_API_GET_CATEGORY_ENABLED(const char* category_name) +#define TRACE_EVENT_API_GET_CATEGORY_ENABLED \ + webrtc::EventTracer::GetCategoryEnabled + +// Add a trace event to the platform tracing system. +// void TRACE_EVENT_API_ADD_TRACE_EVENT( +// char phase, +// const unsigned char* category_enabled, +// const char* name, +// unsigned long long id, +// int num_args, +// const char** arg_names, +// const unsigned char* arg_types, +// const unsigned long long* arg_values, +// unsigned char flags) +#define TRACE_EVENT_API_ADD_TRACE_EVENT webrtc::EventTracer::AddTraceEvent + +//////////////////////////////////////////////////////////////////////////////// + +// Implementation detail: trace event macros create temporary variables +// to keep instrumentation overhead low. These macros give each temporary +// variable a unique name based on the line number to prevent name collissions. +#define INTERNAL_TRACE_EVENT_UID3(a, b) trace_event_unique_##a##b +#define INTERNAL_TRACE_EVENT_UID2(a, b) INTERNAL_TRACE_EVENT_UID3(a, b) +#define INTERNAL_TRACE_EVENT_UID(name_prefix) \ + INTERNAL_TRACE_EVENT_UID2(name_prefix, __LINE__) + +#if WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS +#define INTERNAL_TRACE_EVENT_INFO_TYPE const unsigned char* +#else +#define INTERNAL_TRACE_EVENT_INFO_TYPE static const unsigned char* +#endif // WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS + +// Implementation detail: internal macro to create static category. +#define INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category) \ + INTERNAL_TRACE_EVENT_INFO_TYPE INTERNAL_TRACE_EVENT_UID(catstatic) = \ + TRACE_EVENT_API_GET_CATEGORY_ENABLED(category); + +// Implementation detail: internal macro to create static category and add +// event if the category is enabled. +#define INTERNAL_TRACE_EVENT_ADD(phase, category, name, flags, ...) \ + do { \ + INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category); \ + if (*INTERNAL_TRACE_EVENT_UID(catstatic)) { \ + webrtc::trace_event_internal::AddTraceEvent( \ + phase, INTERNAL_TRACE_EVENT_UID(catstatic), name, \ + webrtc::trace_event_internal::kNoEventId, flags, ##__VA_ARGS__); \ + } \ + } while (0) + +// Implementation detail: internal macro to create static category and add begin +// event if the category is enabled. Also adds the end event when the scope +// ends. +#define INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name, ...) \ + INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category); \ + webrtc::trace_event_internal::TraceEndOnScopeClose INTERNAL_TRACE_EVENT_UID( \ + profileScope); \ + if (*INTERNAL_TRACE_EVENT_UID(catstatic)) { \ + webrtc::trace_event_internal::AddTraceEvent( \ + TRACE_EVENT_PHASE_BEGIN, INTERNAL_TRACE_EVENT_UID(catstatic), name, \ + webrtc::trace_event_internal::kNoEventId, TRACE_EVENT_FLAG_NONE, \ + ##__VA_ARGS__); \ + INTERNAL_TRACE_EVENT_UID(profileScope) \ + .Initialize(INTERNAL_TRACE_EVENT_UID(catstatic), name); \ + } + +// Implementation detail: internal macro to create static category and add +// event if the category is enabled. +#define INTERNAL_TRACE_EVENT_ADD_WITH_ID(phase, category, name, id, flags, \ + ...) \ + do { \ + INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category); \ + if (*INTERNAL_TRACE_EVENT_UID(catstatic)) { \ + unsigned char trace_event_flags = flags | TRACE_EVENT_FLAG_HAS_ID; \ + webrtc::trace_event_internal::TraceID trace_event_trace_id( \ + id, &trace_event_flags); \ + webrtc::trace_event_internal::AddTraceEvent( \ + phase, INTERNAL_TRACE_EVENT_UID(catstatic), name, \ + trace_event_trace_id.data(), trace_event_flags, ##__VA_ARGS__); \ + } \ + } while (0) + +// Notes regarding the following definitions: +// New values can be added and propagated to third party libraries, but existing +// definitions must never be changed, because third party libraries may use old +// definitions. + +// Phase indicates the nature of an event entry. E.g. part of a begin/end pair. +#define TRACE_EVENT_PHASE_BEGIN ('B') +#define TRACE_EVENT_PHASE_END ('E') +#define TRACE_EVENT_PHASE_INSTANT ('I') +#define TRACE_EVENT_PHASE_ASYNC_BEGIN ('S') +#define TRACE_EVENT_PHASE_ASYNC_STEP ('T') +#define TRACE_EVENT_PHASE_ASYNC_END ('F') +#define TRACE_EVENT_PHASE_METADATA ('M') +#define TRACE_EVENT_PHASE_COUNTER ('C') + +// Flags for changing the behavior of TRACE_EVENT_API_ADD_TRACE_EVENT. +#define TRACE_EVENT_FLAG_NONE (static_cast(0)) +#define TRACE_EVENT_FLAG_HAS_ID (static_cast(1 << 1)) +#define TRACE_EVENT_FLAG_MANGLE_ID (static_cast(1 << 2)) + +namespace webrtc { +namespace trace_event_internal { + +// Specify these values when the corresponding argument of AddTraceEvent is not +// used. +const int kZeroNumArgs = 0; +const unsigned long long kNoEventId = 0; + +// TraceID encapsulates an ID that can either be an integer or pointer. Pointers +// are mangled with the Process ID so that they are unlikely to collide when the +// same pointer is used on different processes. +class TraceID { + public: + class ForceMangle { + public: + explicit ForceMangle(unsigned long long id) : data_(id) {} + explicit ForceMangle(unsigned long id) : data_(id) {} + explicit ForceMangle(unsigned int id) : data_(id) {} + explicit ForceMangle(unsigned short id) : data_(id) {} + explicit ForceMangle(unsigned char id) : data_(id) {} + explicit ForceMangle(long long id) + : data_(static_cast(id)) {} + explicit ForceMangle(long id) + : data_(static_cast(id)) {} + explicit ForceMangle(int id) : data_(static_cast(id)) {} + explicit ForceMangle(short id) + : data_(static_cast(id)) {} + explicit ForceMangle(signed char id) + : data_(static_cast(id)) {} + + unsigned long long data() const { return data_; } + + private: + unsigned long long data_; + }; + + explicit TraceID(const void* id, unsigned char* flags) + : data_( + static_cast(reinterpret_cast(id))) { + *flags |= TRACE_EVENT_FLAG_MANGLE_ID; + } + explicit TraceID(ForceMangle id, unsigned char* flags) : data_(id.data()) { + *flags |= TRACE_EVENT_FLAG_MANGLE_ID; + } + explicit TraceID(unsigned long long id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(unsigned long id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(unsigned int id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(unsigned short id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(unsigned char id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(long long id, unsigned char* flags) + : data_(static_cast(id)) { + (void)flags; + } + explicit TraceID(long id, unsigned char* flags) + : data_(static_cast(id)) { + (void)flags; + } + explicit TraceID(int id, unsigned char* flags) + : data_(static_cast(id)) { + (void)flags; + } + explicit TraceID(short id, unsigned char* flags) + : data_(static_cast(id)) { + (void)flags; + } + explicit TraceID(signed char id, unsigned char* flags) + : data_(static_cast(id)) { + (void)flags; + } + + unsigned long long data() const { return data_; } + + private: + unsigned long long data_; +}; + +// Simple union to store various types as unsigned long long. +union TraceValueUnion { + bool as_bool; + unsigned long long as_uint; + long long as_int; + double as_double; + const void* as_pointer; + const char* as_string; +}; + +// Simple container for const char* that should be copied instead of retained. +class TraceStringWithCopy { + public: + explicit TraceStringWithCopy(const char* str) : str_(str) {} + operator const char*() const { return str_; } + + private: + const char* str_; +}; + +// Define SetTraceValue for each allowed type. It stores the type and +// value in the return arguments. This allows this API to avoid declaring any +// structures so that it is portable to third_party libraries. +#define INTERNAL_DECLARE_SET_TRACE_VALUE(actual_type, union_member, \ + value_type_id) \ + static inline void SetTraceValue(actual_type arg, unsigned char* type, \ + unsigned long long* value) { \ + TraceValueUnion type_value; \ + type_value.union_member = arg; \ + *type = value_type_id; \ + *value = type_value.as_uint; \ + } +// Simpler form for int types that can be safely casted. +#define INTERNAL_DECLARE_SET_TRACE_VALUE_INT(actual_type, value_type_id) \ + static inline void SetTraceValue(actual_type arg, unsigned char* type, \ + unsigned long long* value) { \ + *type = value_type_id; \ + *value = static_cast(arg); \ + } + +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(unsigned long long, TRACE_VALUE_TYPE_UINT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(unsigned long, TRACE_VALUE_TYPE_UINT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(unsigned int, TRACE_VALUE_TYPE_UINT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(unsigned short, TRACE_VALUE_TYPE_UINT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(unsigned char, TRACE_VALUE_TYPE_UINT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(long long, TRACE_VALUE_TYPE_INT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(long, TRACE_VALUE_TYPE_INT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(int, TRACE_VALUE_TYPE_INT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(short, TRACE_VALUE_TYPE_INT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(signed char, TRACE_VALUE_TYPE_INT) +INTERNAL_DECLARE_SET_TRACE_VALUE(bool, as_bool, TRACE_VALUE_TYPE_BOOL) +INTERNAL_DECLARE_SET_TRACE_VALUE(double, as_double, TRACE_VALUE_TYPE_DOUBLE) +INTERNAL_DECLARE_SET_TRACE_VALUE(const void*, + as_pointer, + TRACE_VALUE_TYPE_POINTER) +INTERNAL_DECLARE_SET_TRACE_VALUE(const char*, + as_string, + TRACE_VALUE_TYPE_STRING) +INTERNAL_DECLARE_SET_TRACE_VALUE(const TraceStringWithCopy&, + as_string, + TRACE_VALUE_TYPE_COPY_STRING) + +#undef INTERNAL_DECLARE_SET_TRACE_VALUE +#undef INTERNAL_DECLARE_SET_TRACE_VALUE_INT + +// std::string version of SetTraceValue so that trace arguments can be strings. +static inline void SetTraceValue(const std::string& arg, + unsigned char* type, + unsigned long long* value) { + TraceValueUnion type_value; + type_value.as_string = arg.c_str(); + *type = TRACE_VALUE_TYPE_COPY_STRING; + *value = type_value.as_uint; +} + +// These AddTraceEvent template functions are defined here instead of in the +// macro, because the arg_values could be temporary objects, such as +// std::string. In order to store pointers to the internal c_str and pass +// through to the tracing API, the arg_values must live throughout +// these procedures. + +static inline void AddTraceEvent(char phase, + const unsigned char* category_enabled, + const char* name, + unsigned long long id, + unsigned char flags) { + TRACE_EVENT_API_ADD_TRACE_EVENT(phase, category_enabled, name, id, + kZeroNumArgs, nullptr, nullptr, nullptr, + flags); +} + +template +static inline void AddTraceEvent(char phase, + const unsigned char* category_enabled, + const char* name, + unsigned long long id, + unsigned char flags, + const char* arg1_name, + const ARG1_TYPE& arg1_val) { + const int num_args = 1; + unsigned char arg_types[1]; + unsigned long long arg_values[1]; + SetTraceValue(arg1_val, &arg_types[0], &arg_values[0]); + TRACE_EVENT_API_ADD_TRACE_EVENT(phase, category_enabled, name, id, num_args, + &arg1_name, arg_types, arg_values, flags); +} + +template +static inline void AddTraceEvent(char phase, + const unsigned char* category_enabled, + const char* name, + unsigned long long id, + unsigned char flags, + const char* arg1_name, + const ARG1_TYPE& arg1_val, + const char* arg2_name, + const ARG2_TYPE& arg2_val) { + const int num_args = 2; + const char* arg_names[2] = {arg1_name, arg2_name}; + unsigned char arg_types[2]; + unsigned long long arg_values[2]; + SetTraceValue(arg1_val, &arg_types[0], &arg_values[0]); + SetTraceValue(arg2_val, &arg_types[1], &arg_values[1]); + TRACE_EVENT_API_ADD_TRACE_EVENT(phase, category_enabled, name, id, num_args, + arg_names, arg_types, arg_values, flags); +} + +// Used by TRACE_EVENTx macro. Do not use directly. +class TraceEndOnScopeClose { + public: + // Note: members of data_ intentionally left uninitialized. See Initialize. + TraceEndOnScopeClose() : p_data_(nullptr) {} + ~TraceEndOnScopeClose() { + if (p_data_) + AddEventIfEnabled(); + } + + void Initialize(const unsigned char* category_enabled, const char* name) { + data_.category_enabled = category_enabled; + data_.name = name; + p_data_ = &data_; + } + + private: + // Add the end event if the category is still enabled. + void AddEventIfEnabled() { + // Only called when p_data_ is non-null. + if (*p_data_->category_enabled) { + TRACE_EVENT_API_ADD_TRACE_EVENT(TRACE_EVENT_PHASE_END, + p_data_->category_enabled, p_data_->name, + kNoEventId, kZeroNumArgs, nullptr, + nullptr, nullptr, TRACE_EVENT_FLAG_NONE); + } + } + + // This Data struct workaround is to avoid initializing all the members + // in Data during construction of this object, since this object is always + // constructed, even when tracing is disabled. If the members of Data were + // members of this class instead, compiler warnings occur about potential + // uninitialized accesses. + struct Data { + const unsigned char* category_enabled; + const char* name; + }; + Data* p_data_; + Data data_; +}; + +} // namespace trace_event_internal +} // namespace webrtc + +#else + +//////////////////////////////////////////////////////////////////////////////// +// This section defines no-op alternatives to the tracing macros when +// RTC_DISABLE_TRACE_EVENTS is defined. + +#define TRACE_DISABLED_BY_DEFAULT(name) "disabled-by-default-" name + +#define TRACE_ID_MANGLE(id) 0 + +#define TRACE_EVENT0(category, name) RTC_NOOP() +#define TRACE_EVENT1(category, name, arg1_name, arg1_val) RTC_NOOP() +#define TRACE_EVENT2(category, name, arg1_name, arg1_val, arg2_name, arg2_val) \ + RTC_NOOP() + +#define TRACE_EVENT_INSTANT0(category, name, scope) RTC_NOOP() +#define TRACE_EVENT_INSTANT1(category, name, scope, arg1_name, arg1_val) \ + RTC_NOOP() + +#define TRACE_EVENT_INSTANT2(category, name, scope, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + RTC_NOOP() + +#define TRACE_EVENT_BEGIN0(category, name) RTC_NOOP() +#define TRACE_EVENT_BEGIN1(category, name, arg1_name, arg1_val) RTC_NOOP() +#define TRACE_EVENT_BEGIN2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + RTC_NOOP() + +#define TRACE_EVENT_END0(category, name) RTC_NOOP() +#define TRACE_EVENT_END1(category, name, arg1_name, arg1_val) RTC_NOOP() +#define TRACE_EVENT_END2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + RTC_NOOP() + +#define TRACE_COUNTER1(category, name, value) RTC_NOOP() + +#define TRACE_COUNTER2(category, name, value1_name, value1_val, value2_name, \ + value2_val) \ + RTC_NOOP() + +#define TRACE_COUNTER_ID1(category, name, id, value) RTC_NOOP() + +#define TRACE_COUNTER_ID2(category, name, id, value1_name, value1_val, \ + value2_name, value2_val) \ + RTC_NOOP() + +#define TRACE_EVENT_ASYNC_BEGIN0(category, name, id) RTC_NOOP() +#define TRACE_EVENT_ASYNC_BEGIN1(category, name, id, arg1_name, arg1_val) \ + RTC_NOOP() +#define TRACE_EVENT_ASYNC_BEGIN2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + RTC_NOOP() + +#define TRACE_EVENT_ASYNC_STEP_INTO0(category, name, id, step) RTC_NOOP() +#define TRACE_EVENT_ASYNC_STEP_INTO1(category, name, id, step, arg1_name, \ + arg1_val) \ + RTC_NOOP() + +#define TRACE_EVENT_ASYNC_END0(category, name, id) RTC_NOOP() +#define TRACE_EVENT_ASYNC_END1(category, name, id, arg1_name, arg1_val) \ + RTC_NOOP() +#define TRACE_EVENT_ASYNC_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + RTC_NOOP() + +#define TRACE_EVENT_API_GET_CATEGORY_ENABLED "" + +#define TRACE_EVENT_API_ADD_TRACE_EVENT RTC_NOOP() + +#endif // RTC_TRACE_EVENTS_ENABLED +#endif // RTC_USE_PERFETTO + +#endif // RTC_BASE_TRACE_EVENT_H_ diff --git a/VocieProcess/system_wrappers/include/denormal_disabler.h b/VocieProcess/system_wrappers/include/denormal_disabler.h new file mode 100644 index 0000000..bd3d401 --- /dev/null +++ b/VocieProcess/system_wrappers/include/denormal_disabler.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SYSTEM_WRAPPERS_INCLUDE_DENORMAL_DISABLER_H_ +#define SYSTEM_WRAPPERS_INCLUDE_DENORMAL_DISABLER_H_ + +#include "rtc_base/system/arch.h" + +namespace webrtc { + +// Activates the hardware (HW) way to flush denormals (see [1]) to zero as they +// can very seriously impact performance. At destruction time restores the +// denormals handling state read by the ctor; hence, supports nested calls. +// Equals a no-op if the architecture is not x86 or ARM or if the compiler is +// not CLANG. +// [1] https://en.wikipedia.org/wiki/Denormal_number +// +// Example usage: +// +// void Foo() { +// DenormalDisabler d; +// ... +// } +class DenormalDisabler { + public: + // Ctor. If architecture and compiler are supported, stores the HW settings + // for denormals, disables denormals and sets `disabling_activated_` to true. + // Otherwise, only sets `disabling_activated_` to false. + DenormalDisabler(); + // Ctor. Same as above, but also requires `enabled` to be true to disable + // denormals. + explicit DenormalDisabler(bool enabled); + DenormalDisabler(const DenormalDisabler&) = delete; + DenormalDisabler& operator=(const DenormalDisabler&) = delete; + // Dtor. If `disabling_activated_` is true, restores the denormals HW settings + // read by the ctor before denormals were disabled. Otherwise it's a no-op. + ~DenormalDisabler(); + + // Returns true if architecture and compiler are supported. + static bool IsSupported(); + + private: + const int status_word_; + const bool disabling_activated_; +}; + +} // namespace webrtc + +#endif // SYSTEM_WRAPPERS_INCLUDE_DENORMAL_DISABLER_H_