/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/video_stream_encoder.h"
#include <algorithm>
#include <array>
#include <limits>
#include <memory>
#include <numeric>
#include <optional>
#include <utility>
#include "absl/algorithm/container.h"
#include "absl/cleanup/cleanup.h"
#include "absl/types/variant.h"
#include "api/field_trials_view.h"
#include "api/sequence_checker.h"
#include "api/task_queue/task_queue_base.h"
#include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h"
#include "api/video/render_resolution.h"
#include "api/video/video_adaptation_reason.h"
#include "api/video/video_bitrate_allocator_factory.h"
#include "api/video/video_codec_constants.h"
#include "api/video/video_layers_allocation.h"
#include "api/video/video_stream_encoder_settings.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_encoder.h"
#include "call/adaptation/resource_adaptation_processor.h"
#include "call/adaptation/video_source_restrictions.h"
#include "call/adaptation/video_stream_adapter.h"
#include "common_video/frame_instrumentation_data.h"
#include "media/base/media_channel.h"
#include "modules/video_coding/include/video_codec_initializer.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/svc/scalability_mode_util.h"
#include "modules/video_coding/svc/svc_rate_allocator.h"
#include "modules/video_coding/utility/vp8_constants.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h"
#include "rtc_base/experiments/encoder_info_settings.h"
#include "rtc_base/experiments/rate_control_settings.h"
#include "rtc_base/logging.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/system/no_unique_address.h"
#include "rtc_base/thread_annotations.h"
#include "rtc_base/trace_event.h"
#include "system_wrappers/include/metrics.h"
#include "video/adaptation/video_stream_encoder_resource_manager.h"
#include "video/alignment_adjuster.h"
#include "video/config/encoder_stream_factory.h"
#include "video/config/video_encoder_config.h"
#include "video/corruption_detection/frame_instrumentation_generator.h"
#include "video/frame_cadence_adapter.h"
#include "video/frame_dumping_encoder.h"
namespace webrtc {
namespace {
// Time interval for logging frame counts.
const int64_t kFrameLogIntervalMs = 60000;
// Time to keep a single cached pending frame in paused state.
const int64_t kPendingFrameTimeoutMs = 1000;
constexpr
char kFrameDropperFieldTrial[] =
"WebRTC-FrameDropper";
// TODO(bugs.webrtc.org/13572): Remove this kill switch after deploying the
// feature.
constexpr
char kSwitchEncoderOnInitializationFailuresFieldTrial[] =
"WebRTC-SwitchEncoderOnInitializationFailures";
const size_t kDefaultPayloadSize = 1440;
const int64_t kParameterUpdateIntervalMs = 1000;
constexpr
int kDefaultMinScreenSharebps = 1200000;
int GetNumSpatialLayers(
const VideoCodec& codec) {
if (codec.codecType == kVideoCodecVP9) {
return codec.VP9().numberOfSpatialLayers;
}
else if (codec.codecType == kVideoCodecAV1 &&
codec.GetScalabilityMode().has_value()) {
return ScalabilityModeToNumSpatialLayers(*(codec.GetScalabilityMode()));
}
else {
return 0;
}
}
std::optional<EncodedImageCallback::DropReason> MaybeConvertDropReason(
VideoStreamEncoderObserver::DropReason reason) {
switch (reason) {
case VideoStreamEncoderObserver::DropReason::kMediaOptimization:
return EncodedImageCallback::DropReason::kDroppedByMediaOptimizations;
case VideoStreamEncoderObserver::DropReason::kEncoder:
return EncodedImageCallback::DropReason::kDroppedByEncoder;
default:
return std::nullopt;
}
}
bool RequiresEncoderReset(
const VideoCodec& prev_send_codec,
const VideoCodec& new_send_codec,
bool was_encode_called_since_last_initialization) {
// Does not check max/minBitrate or maxFramerate.
if (new_send_codec.codecType != prev_send_codec.codecType ||
new_send_codec.width != prev_send_codec.width ||
new_send_codec.height != prev_send_codec.height ||
new_send_codec.qpMax != prev_send_codec.qpMax ||
new_send_codec.numberOfSimulcastStreams !=
prev_send_codec.numberOfSimulcastStreams ||
new_send_codec.mode != prev_send_codec.mode ||
new_send_codec.GetFrameDropEnabled() !=
prev_send_codec.GetFrameDropEnabled()) {
return true;
}
if (!was_encode_called_since_last_initialization &&
(new_send_codec.startBitrate != prev_send_codec.startBitrate)) {
// If start bitrate has changed reconfigure encoder only if encoding had not
// yet started.
return true;
}
switch (new_send_codec.codecType) {
case kVideoCodecVP8:
if (new_send_codec.VP8() != prev_send_codec.VP8()) {
return true;
}
break;
case kVideoCodecVP9:
if (new_send_codec.VP9() != prev_send_codec.VP9()) {
return true;
}
break;
case kVideoCodecH264:
if (new_send_codec.H264() != prev_send_codec.H264()) {
return true;
}
break;
case kVideoCodecH265:
// No H.265 specific handling needed.
[[fallthrough]];
default:
break;
}
for (
unsigned char i = 0; i < new_send_codec.numberOfSimulcastStreams; ++i) {
if (!new_send_codec.simulcastStream[i].active) {
// No need to reset when stream is inactive.
continue;
}
if (!prev_send_codec.simulcastStream[i].active ||
new_send_codec.simulcastStream[i].width !=
prev_send_codec.simulcastStream[i].width ||
new_send_codec.simulcastStream[i].height !=
prev_send_codec.simulcastStream[i].height ||
new_send_codec.simulcastStream[i].numberOfTemporalLayers !=
prev_send_codec.simulcastStream[i].numberOfTemporalLayers ||
new_send_codec.simulcastStream[i].qpMax !=
prev_send_codec.simulcastStream[i].qpMax) {
return true;
}
if (new_send_codec.simulcastStream[i].maxFramerate !=
prev_send_codec.simulcastStream[i].maxFramerate &&
new_send_codec.simulcastStream[i].maxFramerate !=
new_send_codec.maxFramerate) {
// SetRates can only represent maxFramerate for one layer. Reset the
// encoder if there are multiple layers that differ in maxFramerate.
return true;
}
}
if (new_send_codec.codecType == kVideoCodecVP9) {
size_t num_spatial_layers = new_send_codec.VP9().numberOfSpatialLayers;
for (
unsigned char i = 0; i < num_spatial_layers; ++i) {
if (!new_send_codec.spatialLayers[i].active) {
// No need to reset when layer is inactive.
continue;
}
if (new_send_codec.spatialLayers[i].width !=
prev_send_codec.spatialLayers[i].width ||
new_send_codec.spatialLayers[i].height !=
prev_send_codec.spatialLayers[i].height ||
new_send_codec.spatialLayers[i].numberOfTemporalLayers !=
prev_send_codec.spatialLayers[i].numberOfTemporalLayers ||
new_send_codec.spatialLayers[i].qpMax !=
prev_send_codec.spatialLayers[i].qpMax ||
!prev_send_codec.spatialLayers[i].active) {
return true;
}
}
}
if (new_send_codec.GetScalabilityMode() !=
prev_send_codec.GetScalabilityMode()) {
return true;
}
return false;
}
// Limit allocation across TLs in bitrate allocation according to number of TLs
// in EncoderInfo.
VideoBitrateAllocation UpdateAllocationFromEncoderInfo(
const VideoBitrateAllocation& allocation,
const VideoEncoder::EncoderInfo& encoder_info) {
if (allocation.get_sum_bps() == 0) {
return allocation;
}
VideoBitrateAllocation new_allocation;
for (
int si = 0; si < kMaxSpatialLayers; ++si) {
if (encoder_info.fps_allocation[si].size() == 1 &&
allocation.IsSpatialLayerUsed(si)) {
// One TL is signalled to be used by the encoder. Do not distribute
// bitrate allocation across TLs (use sum at ti:0).
new_allocation.SetBitrate(si, 0, allocation.GetSpatialLayerSum(si));
}
else {
for (
int ti = 0; ti < kMaxTemporalStreams; ++ti) {
if (allocation.HasBitrate(si, ti))
new_allocation.SetBitrate(si, ti, allocation.GetBitrate(si, ti));
}
}
}
new_allocation.set_bw_limited(allocation.is_bw_limited());
return new_allocation;
}
// Converts a VideoBitrateAllocation that contains allocated bitrate per layer,
// and an EncoderInfo that contains information about the actual encoder
// structure used by a codec. Stream structures can be Ksvc, Full SVC, Simulcast
// etc.
VideoLayersAllocation CreateVideoLayersAllocation(
const VideoCodec& encoder_config,
const VideoEncoder::RateControlParameters& current_rate,
const VideoEncoder::EncoderInfo& encoder_info) {
const VideoBitrateAllocation& target_bitrate = current_rate.target_bitrate;
VideoLayersAllocation layers_allocation;
if (target_bitrate.get_sum_bps() == 0) {
return layers_allocation;
}
if (encoder_config.numberOfSimulcastStreams > 1) {
layers_allocation.resolution_and_frame_rate_is_valid =
true;
for (
int si = 0; si < encoder_config.numberOfSimulcastStreams; ++si) {
if (!target_bitrate.IsSpatialLayerUsed(si) ||
target_bitrate.GetSpatialLayerSum(si) == 0) {
continue;
}
layers_allocation.active_spatial_layers.emplace_back();
VideoLayersAllocation::SpatialLayer& spatial_layer =
layers_allocation.active_spatial_layers.back();
spatial_layer.width = encoder_config.simulcastStream[si].width;
spatial_layer.height = encoder_config.simulcastStream[si].height;
spatial_layer.rtp_stream_index = si;
spatial_layer.spatial_id = 0;
auto frame_rate_fraction =
VideoEncoder::EncoderInfo::kMaxFramerateFraction;
if (encoder_info.fps_allocation[si].size() == 1) {
// One TL is signalled to be used by the encoder. Do not distribute
// bitrate allocation across TLs (use sum at tl:0).
spatial_layer.target_bitrate_per_temporal_layer.push_back(
DataRate::BitsPerSec(target_bitrate.GetSpatialLayerSum(si)));
frame_rate_fraction = encoder_info.fps_allocation[si][0];
}
else {
// Temporal layers are supported.
uint32_t temporal_layer_bitrate_bps = 0;
for (size_t ti = 0;
ti < encoder_config.simulcastStream[si].numberOfTemporalLayers;
++ti) {
if (!target_bitrate.HasBitrate(si, ti)) {
break;
}
if (ti < encoder_info.fps_allocation[si].size()) {
// Use frame rate of the top used temporal layer.
frame_rate_fraction = encoder_info.fps_allocation[si][ti];
}
temporal_layer_bitrate_bps += target_bitrate.GetBitrate(si, ti);
spatial_layer.target_bitrate_per_temporal_layer.push_back(
DataRate::BitsPerSec(temporal_layer_bitrate_bps));
}
}
// Encoder may drop frames internally if `maxFramerate` is set.
spatial_layer.frame_rate_fps = std::min<uint8_t>(
encoder_config.simulcastStream[si].maxFramerate,
rtc::saturated_cast<uint8_t>(
(current_rate.framerate_fps * frame_rate_fraction) /
VideoEncoder::EncoderInfo::kMaxFramerateFraction));
}
}
else if (encoder_config.numberOfSimulcastStreams == 1) {
// TODO(bugs.webrtc.org/12000): Implement support for AV1 with
// scalability.
const bool higher_spatial_depend_on_lower =
encoder_config.codecType == kVideoCodecVP9 &&
encoder_config.VP9().interLayerPred == InterLayerPredMode::kOn;
layers_allocation.resolution_and_frame_rate_is_valid =
true;
std::vector<DataRate> aggregated_spatial_bitrate(
webrtc::kMaxTemporalStreams, DataRate::Zero());
for (
int si = 0; si < webrtc::kMaxSpatialLayers; ++si) {
layers_allocation.resolution_and_frame_rate_is_valid =
true;
if (!target_bitrate.IsSpatialLayerUsed(si) ||
target_bitrate.GetSpatialLayerSum(si) == 0) {
break;
}
layers_allocation.active_spatial_layers.emplace_back();
VideoLayersAllocation::SpatialLayer& spatial_layer =
layers_allocation.active_spatial_layers.back();
spatial_layer.width = encoder_config.spatialLayers[si].width;
spatial_layer.height = encoder_config.spatialLayers[si].height;
spatial_layer.rtp_stream_index = 0;
spatial_layer.spatial_id = si;
auto frame_rate_fraction =
VideoEncoder::EncoderInfo::kMaxFramerateFraction;
if (encoder_info.fps_allocation[si].size() == 1) {
// One TL is signalled to be used by the encoder. Do not distribute
// bitrate allocation across TLs (use sum at tl:0).
DataRate aggregated_temporal_bitrate =
DataRate::BitsPerSec(target_bitrate.GetSpatialLayerSum(si));
aggregated_spatial_bitrate[0] += aggregated_temporal_bitrate;
if (higher_spatial_depend_on_lower) {
spatial_layer.target_bitrate_per_temporal_layer.push_back(
aggregated_spatial_bitrate[0]);
}
else {
spatial_layer.target_bitrate_per_temporal_layer.push_back(
aggregated_temporal_bitrate);
}
frame_rate_fraction = encoder_info.fps_allocation[si][0];
}
else {
// Temporal layers are supported.
DataRate aggregated_temporal_bitrate = DataRate::Zero();
for (size_t ti = 0;
ti < encoder_config.spatialLayers[si].numberOfTemporalLayers;
++ti) {
if (!target_bitrate.HasBitrate(si, ti)) {
break;
}
if (ti < encoder_info.fps_allocation[si].size()) {
// Use frame rate of the top used temporal layer.
frame_rate_fraction = encoder_info.fps_allocation[si][ti];
}
aggregated_temporal_bitrate +=
DataRate::BitsPerSec(target_bitrate.GetBitrate(si, ti));
if (higher_spatial_depend_on_lower) {
spatial_layer.target_bitrate_per_temporal_layer.push_back(
aggregated_temporal_bitrate + aggregated_spatial_bitrate[ti]);
aggregated_spatial_bitrate[ti] += aggregated_temporal_bitrate;
}
else {
spatial_layer.target_bitrate_per_temporal_layer.push_back(
aggregated_temporal_bitrate);
}
}
}
// Encoder may drop frames internally if `maxFramerate` is set.
spatial_layer.frame_rate_fps = std::min<uint8_t>(
encoder_config.spatialLayers[si].maxFramerate,
rtc::saturated_cast<uint8_t>(
(current_rate.framerate_fps * frame_rate_fraction) /
VideoEncoder::EncoderInfo::kMaxFramerateFraction));
}
}
return layers_allocation;
}
VideoEncoder::EncoderInfo GetEncoderInfoWithBitrateLimitUpdate(
const VideoEncoder::EncoderInfo& info,
const VideoEncoderConfig& encoder_config,
bool default_limits_allowed) {
if (!default_limits_allowed || !info.resolution_bitrate_limits.empty() ||
encoder_config.simulcast_layers.size() <= 1) {
return info;
}
// Bitrate limits are not configured and more than one layer is used, use
// the default limits (bitrate limits are not used for simulcast).
VideoEncoder::EncoderInfo new_info = info;
new_info.resolution_bitrate_limits =
EncoderInfoSettings::GetDefaultSinglecastBitrateLimits(
encoder_config.codec_type);
return new_info;
}
int NumActiveStreams(
const std::vector<VideoStream>& streams) {
int num_active = 0;
for (
const auto& stream : streams) {
if (stream.active)
++num_active;
}
return num_active;
}
void ApplySpatialLayerBitrateLimits(
const VideoEncoder::EncoderInfo& encoder_info,
const VideoEncoderConfig& encoder_config,
VideoCodec* codec) {
if (!(GetNumSpatialLayers(*codec) > 0)) {
// ApplySpatialLayerBitrateLimits() supports VP9 and AV1 (the latter with
// scalability mode set) only.
return;
}
if (VideoStreamEncoderResourceManager::IsSimulcastOrMultipleSpatialLayers(
encoder_config, *codec) ||
encoder_config.simulcast_layers.size() <= 1) {
// Resolution bitrate limits usage is restricted to singlecast.
return;
}
// Get bitrate limits for active stream.
std::optional<uint32_t> pixels =
VideoStreamAdapter::GetSingleActiveLayerPixels(*codec);
if (!pixels.has_value()) {
return;
}
std::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
encoder_info.GetEncoderBitrateLimitsForResolution(*pixels);
if (!bitrate_limits.has_value()) {
return;
}
// Index for the active stream.
std::optional<size_t> index;
for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) {
if (encoder_config.simulcast_layers[i].active)
index = i;
}
if (!index.has_value()) {
return;
}
int min_bitrate_bps;
if (encoder_config.simulcast_layers[*index].min_bitrate_bps <= 0) {
min_bitrate_bps = bitrate_limits->min_bitrate_bps;
}
else {
min_bitrate_bps = encoder_config.simulcast_layers[*index].min_bitrate_bps;
}
int max_bitrate_bps;
if (encoder_config.simulcast_layers[*index].max_bitrate_bps <= 0) {
max_bitrate_bps = bitrate_limits->max_bitrate_bps;
}
else {
max_bitrate_bps =
std::min(bitrate_limits->max_bitrate_bps,
encoder_config.simulcast_layers[*index].max_bitrate_bps);
}
if (min_bitrate_bps >= max_bitrate_bps) {
RTC_LOG(LS_WARNING) <<
"Bitrate limits not used, min_bitrate_bps "
<< min_bitrate_bps <<
" >= max_bitrate_bps "
<< max_bitrate_bps;
return;
}
for (
int i = 0; i < GetNumSpatialLayers(*codec); ++i) {
if (codec->spatialLayers[i].active) {
codec->spatialLayers[i].minBitrate = min_bitrate_bps / 1000;
codec->spatialLayers[i].maxBitrate = max_bitrate_bps / 1000;
codec->spatialLayers[i].targetBitrate =
std::min(codec->spatialLayers[i].targetBitrate,
codec->spatialLayers[i].maxBitrate);
break;
}
}
}
void ApplyEncoderBitrateLimitsIfSingleActiveStream(
const VideoEncoder::EncoderInfo& encoder_info,
const std::vector<VideoStream>& encoder_config_layers,
std::vector<VideoStream>* streams) {
// Apply limits if simulcast with one active stream (expect lowest).
bool single_active_stream =
streams->size() > 1 && NumActiveStreams(*streams) == 1 &&
!streams->front().active && NumActiveStreams(encoder_config_layers) == 1;
if (!single_active_stream) {
return;
}
// Index for the active stream.
size_t index = 0;
for (size_t i = 0; i < encoder_config_layers.size(); ++i) {
if (encoder_config_layers[i].active)
index = i;
}
if (streams->size() < (index + 1) || !(*streams)[index].active) {
return;
}
// Get bitrate limits for active stream.
std::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits =
encoder_info.GetEncoderBitrateLimitsForResolution(
(*streams)[index].width * (*streams)[index].height);
if (!encoder_bitrate_limits) {
return;
}
int min_bitrate_bps;
if (encoder_config_layers[index].min_bitrate_bps <= 0) {
min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps;
}
else {
min_bitrate_bps = (*streams)[index].min_bitrate_bps;
}
int max_bitrate_bps;
if (encoder_config_layers[index].max_bitrate_bps <= 0) {
max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps;
}
else {
max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps,
(*streams)[index].max_bitrate_bps);
}
if (min_bitrate_bps >= max_bitrate_bps) {
RTC_LOG(LS_WARNING) <<
"Encoder bitrate limits"
<<
" (min=" << encoder_bitrate_limits->min_bitrate_bps
<<
", max=" << encoder_bitrate_limits->max_bitrate_bps
<<
") do not intersect with stream limits"
<<
" (min=" << (*streams)[index].min_bitrate_bps
<<
", max=" << (*streams)[index].max_bitrate_bps
<<
"). Encoder bitrate limits not used.";
return;
}
(*streams)[index].min_bitrate_bps = min_bitrate_bps;
(*streams)[index].max_bitrate_bps = max_bitrate_bps;
(*streams)[index].target_bitrate_bps =
std::min((*streams)[index].target_bitrate_bps,
encoder_bitrate_limits->max_bitrate_bps);
}
std::optional<
int> ParseVp9LowTierCoreCountThreshold(
const FieldTrialsView& trials) {
FieldTrialFlag disable_low_tier(
"Disabled");
FieldTrialParameter<
int> max_core_count(
"max_core_count", 2);
ParseFieldTrial({&disable_low_tier, &max_core_count},
trials.Lookup(
"WebRTC-VP9-LowTierOptimizations"));
if (disable_low_tier.Get()) {
return std::nullopt;
}
return max_core_count.Get();
}
std::optional<
int> ParseEncoderThreadLimit(
const FieldTrialsView& trials) {
FieldTrialOptional<
int> encoder_thread_limit(
"encoder_thread_limit");
ParseFieldTrial({&encoder_thread_limit},
trials.Lookup(
"WebRTC-VideoEncoderSettings"));
return encoder_thread_limit.GetOptional();
}
}
// namespace
VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings()
: rate_control(),
encoder_target(DataRate::Zero()),
stable_encoder_target(DataRate::Zero()) {}
VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings(
const VideoBitrateAllocation& bitrate,
double framerate_fps,
DataRate bandwidth_allocation,
DataRate encoder_target,
DataRate stable_encoder_target)
: rate_control(bitrate, framerate_fps, bandwidth_allocation),
encoder_target(encoder_target),
stable_encoder_target(stable_encoder_target) {}
bool VideoStreamEncoder::EncoderRateSettings::
operator==(
const EncoderRateSettings& rhs)
const {
return rate_control == rhs.rate_control &&
encoder_target == rhs.encoder_target &&
stable_encoder_target == rhs.stable_encoder_target;
}
bool VideoStreamEncoder::EncoderRateSettings::
operator!=(
const EncoderRateSettings& rhs)
const {
return !(*
this == rhs);
}
class VideoStreamEncoder::DegradationPreferenceManager
:
public DegradationPreferenceProvider {
public:
explicit DegradationPreferenceManager(
VideoStreamAdapter* video_stream_adapter)
: degradation_preference_(DegradationPreference::DISABLED),
is_screenshare_(
false),
effective_degradation_preference_(DegradationPreference::DISABLED),
video_stream_adapter_(video_stream_adapter) {
RTC_DCHECK(video_stream_adapter_);
sequence_checker_.Detach();
}
~DegradationPreferenceManager() override =
default;
DegradationPreference degradation_preference()
const override {
RTC_DCHECK_RUN_ON(&sequence_checker_);
return effective_degradation_preference_;
}
void SetDegradationPreference(DegradationPreference degradation_preference) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
degradation_preference_ = degradation_preference;
MaybeUpdateEffectiveDegradationPreference();
}
void SetIsScreenshare(
bool is_screenshare) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
is_screenshare_ = is_screenshare;
MaybeUpdateEffectiveDegradationPreference();
}
private:
void MaybeUpdateEffectiveDegradationPreference()
RTC_RUN_ON(&sequence_checker_) {
DegradationPreference effective_degradation_preference =
(is_screenshare_ &&
degradation_preference_ == DegradationPreference::BALANCED)
? DegradationPreference::MAINTAIN_RESOLUTION
: degradation_preference_;
if (effective_degradation_preference != effective_degradation_preference_) {
effective_degradation_preference_ = effective_degradation_preference;
video_stream_adapter_->SetDegradationPreference(
effective_degradation_preference);
}
}
RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_;
DegradationPreference degradation_preference_
RTC_GUARDED_BY(&sequence_checker_);
bool is_screenshare_ RTC_GUARDED_BY(&sequence_checker_);
DegradationPreference effective_degradation_preference_
RTC_GUARDED_BY(&sequence_checker_);
VideoStreamAdapter* video_stream_adapter_ RTC_GUARDED_BY(&sequence_checker_);
};
VideoStreamEncoder::VideoStreamEncoder(
const Environment& env,
uint32_t number_of_cores,
VideoStreamEncoderObserver* encoder_stats_observer,
const VideoStreamEncoderSettings& settings,
std::unique_ptr<OveruseFrameDetector> overuse_detector,
std::unique_ptr<FrameCadenceAdapterInterface> frame_cadence_adapter,
std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
encoder_queue,
BitrateAllocationCallbackType allocation_cb_type,
webrtc::VideoEncoderFactory::EncoderSelectorInterface* encoder_selector)
: env_(env),
worker_queue_(TaskQueueBase::Current()),
number_of_cores_(number_of_cores),
settings_(settings),
allocation_cb_type_(allocation_cb_type),
rate_control_settings_(env_.field_trials()),
encoder_selector_from_constructor_(encoder_selector),
encoder_selector_from_factory_(
encoder_selector_from_constructor_
? nullptr
: settings.encoder_factory->GetEncoderSelector()),
encoder_selector_(encoder_selector_from_constructor_
? encoder_selector_from_constructor_
: encoder_selector_from_factory_.get()),
encoder_stats_observer_(encoder_stats_observer),
frame_cadence_adapter_(std::move(frame_cadence_adapter)),
delta_ntp_internal_ms_(env_.clock().CurrentNtpInMilliseconds() -
env_.clock().TimeInMilliseconds()),
last_frame_log_ms_(env_.clock().TimeInMilliseconds()),
next_frame_types_(1, VideoFrameType::kVideoFrameDelta),
input_state_provider_(encoder_stats_observer),
video_stream_adapter_(
std::make_unique<VideoStreamAdapter>(&input_state_provider_,
encoder_stats_observer,
env_.field_trials())),
degradation_preference_manager_(
std::make_unique<DegradationPreferenceManager>(
video_stream_adapter_.get())),
stream_resource_manager_(&input_state_provider_,
encoder_stats_observer,
&env_.clock(),
settings_.experiment_cpu_load_estimator,
std::move(overuse_detector),
degradation_preference_manager_.get(),
env_.field_trials()),
video_source_sink_controller_(
/*sink=*/frame_cadence_adapter_.get(),
/*source=*/nullptr),
default_limits_allowed_(!env_.field_trials().IsEnabled(
"WebRTC-DefaultBitrateLimitsKillSwitch")),
qp_parsing_allowed_(
!env_.field_trials().IsEnabled(
"WebRTC-QpParsingKillSwitch")),
switch_encoder_on_init_failures_(!env_.field_trials().IsDisabled(
kSwitchEncoderOnInitializationFailuresFieldTrial)),
vp9_low_tier_core_threshold_(
ParseVp9LowTierCoreCountThreshold(env_.field_trials())),
experimental_encoder_thread_limit_(
ParseEncoderThreadLimit(env_.field_trials())),
encoder_queue_(std::move(encoder_queue)) {
TRACE_EVENT0(
"webrtc",
"VideoStreamEncoder::VideoStreamEncoder");
RTC_DCHECK_RUN_ON(worker_queue_);
RTC_DCHECK(encoder_stats_observer);
RTC_DCHECK_GE(number_of_cores, 1);
frame_cadence_adapter_->Initialize(&cadence_callback_);
stream_resource_manager_.Initialize(encoder_queue_.get());
encoder_queue_->PostTask([
this] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
resource_adaptation_processor_ =
std::make_unique<ResourceAdaptationProcessor>(
video_stream_adapter_.get());
stream_resource_manager_.SetAdaptationProcessor(
resource_adaptation_processor_.get(), video_stream_adapter_.get());
resource_adaptation_processor_->AddResourceLimitationsListener(
&stream_resource_manager_);
video_stream_adapter_->AddRestrictionsListener(&stream_resource_manager_);
video_stream_adapter_->AddRestrictionsListener(
this);
stream_resource_manager_.MaybeInitializePixelLimitResource();
// Add the stream resource manager's resources to the processor.
adaptation_constraints_ = stream_resource_manager_.AdaptationConstraints();
for (
auto* constraint : adaptation_constraints_) {
video_stream_adapter_->AddAdaptationConstraint(constraint);
}
});
}
VideoStreamEncoder::~VideoStreamEncoder() {
RTC_DCHECK_RUN_ON(worker_queue_);
RTC_DCHECK(!video_source_sink_controller_.HasSource())
<<
"Must call ::Stop() before destruction.";
// The queue must be destroyed before its pointer is invalidated to avoid race
// between destructor and running task that check if function is called on the
// encoder_queue_.
// std::unique_ptr destructor does the same two operations in reverse order as
// it doesn't expect member would be used after its destruction has started.
encoder_queue_.get_deleter()(encoder_queue_.get());
encoder_queue_.release();
}
void VideoStreamEncoder::Stop() {
RTC_DCHECK_RUN_ON(worker_queue_);
video_source_sink_controller_.SetSource(nullptr);
rtc::Event shutdown_event;
absl::Cleanup shutdown = [&shutdown_event] { shutdown_event.Set(); };
encoder_queue_->PostTask([
this, shutdown = std::move(shutdown)] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
if (resource_adaptation_processor_) {
// We're no longer interested in restriction updates, which may get
// triggered as part of removing resources.
video_stream_adapter_->RemoveRestrictionsListener(
this);
video_stream_adapter_->RemoveRestrictionsListener(
&stream_resource_manager_);
resource_adaptation_processor_->RemoveResourceLimitationsListener(
&stream_resource_manager_);
// Stop and remove resources and delete adaptation processor.
stream_resource_manager_.StopManagedResources();
for (
auto* constraint : adaptation_constraints_) {
video_stream_adapter_->RemoveAdaptationConstraint(constraint);
}
for (
auto& resource : additional_resources_) {
stream_resource_manager_.RemoveResource(resource);
}
additional_resources_.clear();
stream_resource_manager_.SetAdaptationProcessor(nullptr, nullptr);
resource_adaptation_processor_.reset();
}
rate_allocator_ = nullptr;
ReleaseEncoder();
encoder_ = nullptr;
frame_cadence_adapter_ = nullptr;
frame_instrumentation_generator_ = nullptr;
});
shutdown_event.Wait(rtc::Event::kForever);
}
void VideoStreamEncoder::SetFecControllerOverride(
FecControllerOverride* fec_controller_override) {
encoder_queue_->PostTask([
this, fec_controller_override] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
RTC_DCHECK(!fec_controller_override_);
fec_controller_override_ = fec_controller_override;
if (encoder_) {
encoder_->SetFecControllerOverride(fec_controller_override_);
}
});
}
void VideoStreamEncoder::AddAdaptationResource(
rtc::scoped_refptr<Resource> resource) {
RTC_DCHECK_RUN_ON(worker_queue_);
TRACE_EVENT0(
"webrtc",
"VideoStreamEncoder::AddAdaptationResource");
// Map any externally added resources as kCpu for the sake of stats reporting.
// TODO(hbos): Make the manager map any unknown resources to kCpu and get rid
// of this MapResourceToReason() call.
TRACE_EVENT_ASYNC_BEGIN0(
"webrtc",
"VideoStreamEncoder::AddAdaptationResource(latency)",
this);
encoder_queue_->PostTask([
this, resource = std::move(resource)] {
TRACE_EVENT_ASYNC_END0(
"webrtc",
"VideoStreamEncoder::AddAdaptationResource(latency)",
this);
RTC_DCHECK_RUN_ON(encoder_queue_.get());
additional_resources_.push_back(resource);
stream_resource_manager_.AddResource(resource, VideoAdaptationReason::kCpu);
});
}
std::vector<rtc::scoped_refptr<Resource>>
VideoStreamEncoder::GetAdaptationResources() {
RTC_DCHECK_RUN_ON(worker_queue_);
// In practice, this method is only called by tests to verify operations that
// run on the encoder queue. So rather than force PostTask() operations to
// be accompanied by an event and a `Wait()`, we'll use PostTask + Wait()
// here.
rtc::Event event;
std::vector<rtc::scoped_refptr<Resource>> resources;
encoder_queue_->PostTask([&] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
resources = resource_adaptation_processor_->GetResources();
event.Set();
});
event.Wait(rtc::Event::kForever);
return resources;
}
void VideoStreamEncoder::SetSource(
rtc::VideoSourceInterface<VideoFrame>* source,
const DegradationPreference& degradation_preference) {
RTC_DCHECK_RUN_ON(worker_queue_);
video_source_sink_controller_.SetSource(source);
input_state_provider_.OnHasInputChanged(source);
// This may trigger reconfiguring the QualityScaler on the encoder queue.
encoder_queue_->PostTask([
this, degradation_preference] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
degradation_preference_manager_->SetDegradationPreference(
degradation_preference);
stream_resource_manager_.SetDegradationPreferences(degradation_preference);
if (encoder_) {
stream_resource_manager_.ConfigureQualityScaler(
encoder_->GetEncoderInfo());
stream_resource_manager_.ConfigureBandwidthQualityScaler(
encoder_->GetEncoderInfo());
}
});
}
void VideoStreamEncoder::SetSink(EncoderSink* sink,
bool rotation_applied) {
RTC_DCHECK_RUN_ON(worker_queue_);
video_source_sink_controller_.SetRotationApplied(rotation_applied);
video_source_sink_controller_.PushSourceSinkSettings();
encoder_queue_->PostTask([
this, sink] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
sink_ = sink;
});
}
void VideoStreamEncoder::SetStartBitrate(
int start_bitrate_bps) {
encoder_queue_->PostTask([
this, start_bitrate_bps] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
RTC_LOG(LS_INFO) <<
"SetStartBitrate " << start_bitrate_bps;
encoder_target_bitrate_bps_ =
start_bitrate_bps != 0 ? std::optional<uint32_t>(start_bitrate_bps)
: std::nullopt;
stream_resource_manager_.SetStartBitrate(
DataRate::BitsPerSec(start_bitrate_bps));
});
}
void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config,
size_t max_data_payload_length) {
ConfigureEncoder(std::move(config), max_data_payload_length, nullptr);
}
void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config,
size_t max_data_payload_length,
SetParametersCallback callback) {
RTC_DCHECK_RUN_ON(worker_queue_);
// Inform source about max configured framerate,
// scale_resolution_down_to and which layers are active.
int max_framerate = -1;
// Is any layer active.
bool active =
false;
// The max scale_resolution_down_to.
std::optional<rtc::VideoSinkWants::FrameSize> scale_resolution_down_to;
for (
const auto& stream : config.simulcast_layers) {
active |= stream.active;
if (stream.active) {
max_framerate = std::max(stream.max_framerate, max_framerate);
}
// Note: we propagate the highest scale_resolution_down_to regardless
// if layer is active or not.
if (stream.scale_resolution_down_to) {
if (!scale_resolution_down_to) {
scale_resolution_down_to.emplace(
stream.scale_resolution_down_to->width,
stream.scale_resolution_down_to->height);
}
else {
scale_resolution_down_to.emplace(
std::max(stream.scale_resolution_down_to->width,
scale_resolution_down_to->width),
std::max(stream.scale_resolution_down_to->height,
scale_resolution_down_to->height));
}
}
}
if (scale_resolution_down_to !=
video_source_sink_controller_.scale_resolution_down_to() ||
active != video_source_sink_controller_.active() ||
max_framerate !=
video_source_sink_controller_.frame_rate_upper_limit().value_or(-1)) {
video_source_sink_controller_.SetScaleResolutionDownTo(
scale_resolution_down_to);
if (max_framerate >= 0) {
video_source_sink_controller_.SetFrameRateUpperLimit(max_framerate);
}
else {
video_source_sink_controller_.SetFrameRateUpperLimit(std::nullopt);
}
video_source_sink_controller_.SetActive(active);
video_source_sink_controller_.PushSourceSinkSettings();
}
encoder_queue_->PostTask([
this, config = std::move(config),
max_data_payload_length,
callback = std::move(callback)]()
mutable {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
RTC_DCHECK(sink_);
RTC_LOG(LS_INFO) <<
"ConfigureEncoder requested.";
// Set up the frame cadence adapter according to if we're going to do
// screencast. The final number of spatial layers is based on info
// in `send_codec_`, which is computed based on incoming frame
// dimensions which can only be determined later.
//
// Note: zero-hertz mode isn't enabled by this alone. Constraints also
// have to be set up with min_fps = 0 and max_fps > 0.
if (config.content_type == VideoEncoderConfig::ContentType::kScreen) {
frame_cadence_adapter_->SetZeroHertzModeEnabled(
FrameCadenceAdapterInterface::ZeroHertzModeParams{});
}
else {
frame_cadence_adapter_->SetZeroHertzModeEnabled(std::nullopt);
}
pending_encoder_creation_ =
(!encoder_ || encoder_config_.video_format != config.video_format ||
max_data_payload_length_ != max_data_payload_length);
encoder_config_ = std::move(config);
max_data_payload_length_ = max_data_payload_length;
pending_encoder_reconfiguration_ =
true;
if (settings_.enable_frame_instrumentation_generator) {
frame_instrumentation_generator_ =
std::make_unique<FrameInstrumentationGenerator>(
encoder_config_.codec_type);
}
// Reconfigure the encoder now if the frame resolution is known.
// Otherwise, the reconfiguration is deferred until the next frame to
// minimize the number of reconfigurations. The codec configuration
// depends on incoming video frame size.
if (last_frame_info_) {
if (callback) {
encoder_configuration_callbacks_.push_back(std::move(callback));
}
ReconfigureEncoder();
}
else {
webrtc::InvokeSetParametersCallback(callback, webrtc::RTCError::OK());
}
});
}
// We should reduce the number of 'full' ReconfigureEncoder(). If only need
// subset of it at runtime, consider handle it in
// VideoStreamEncoder::EncodeVideoFrame() when encoder_info_ != info.
void VideoStreamEncoder::ReconfigureEncoder() {
// Running on the encoder queue.
RTC_DCHECK(pending_encoder_reconfiguration_);
RTC_LOG(LS_INFO) <<
"[VSE] " << __func__
<<
" [encoder_config=" << encoder_config_.ToString() <<
"]";
bool encoder_reset_required =
false;
if (pending_encoder_creation_) {
// Destroy existing encoder instance before creating a new one. Otherwise
// attempt to create another instance will fail if encoder factory
// supports only single instance of encoder of given type.
encoder_.reset();
encoder_ = MaybeCreateFrameDumpingEncoderWrapper(
settings_.encoder_factory->Create(env_, encoder_config_.video_format),
env_.field_trials());
if (!encoder_) {
RTC_LOG(LS_ERROR) <<
"CreateVideoEncoder failed, failing encoder format: "
<< encoder_config_.video_format.ToString();
RequestEncoderSwitch();
return;
}
if (encoder_selector_) {
encoder_selector_->OnCurrentEncoder(encoder_config_.video_format);
}
encoder_->SetFecControllerOverride(fec_controller_override_);
encoder_reset_required =
true;
}
// TODO(webrtc:14451) : Move AlignmentAdjuster into EncoderStreamFactory
// Possibly adjusts scale_resolution_down_by in `encoder_config_` to limit the
// alignment value.
AlignmentAdjuster::GetAlignmentAndMaybeAdjustScaleFactors(
encoder_->GetEncoderInfo(), &encoder_config_, std::nullopt);
std::vector<VideoStream> streams;
if (encoder_config_.video_stream_factory) {
// Note: only tests set their own EncoderStreamFactory...
encoder_config_.video_stream_factory->SetEncoderInfo(encoder_->GetEncoderInfo());
streams = encoder_config_.video_stream_factory->CreateEncoderStreams(
env_.field_trials(), last_frame_info_->width, last_frame_info_->height,
encoder_config_);
}
else {
auto factory = rtc::make_ref_counted<cricket::EncoderStreamFactory>(
encoder_->GetEncoderInfo(), latest_restrictions_);
streams = factory->CreateEncoderStreams(
env_.field_trials(), last_frame_info_->width, last_frame_info_->height,
encoder_config_);
}
// TODO(webrtc:14451) : Move AlignmentAdjuster into EncoderStreamFactory
// Get alignment when actual number of layers are known.
int alignment = AlignmentAdjuster::GetAlignmentAndMaybeAdjustScaleFactors(
encoder_->GetEncoderInfo(), &encoder_config_, streams.size());
// Check that the higher layers do not try to set number of temporal layers
// to less than 1.
// TODO(brandtr): Get rid of the wrapping optional as it serves no purpose
// at this layer.
#if RTC_DCHECK_IS_ON
for (
const auto& stream : streams) {
RTC_DCHECK_GE(stream.num_temporal_layers.value_or(1), 1);
}
#endif
// TODO(ilnik): If configured resolution is significantly less than provided,
// e.g. because there are not enough SSRCs for all simulcast streams,
// signal new resolutions via SinkWants to video source.
// Stream dimensions may be not equal to given because of a simulcast
// restrictions.
auto highest_stream = absl::c_max_element(
streams, [](
const webrtc::VideoStream& a,
const webrtc::VideoStream& b) {
return std::tie(a.width, a.height) < std::tie(b.width, b.height);
});
int highest_stream_width =
static_cast<
int>(highest_stream->width);
int highest_stream_height =
static_cast<
int>(highest_stream->height);
// Dimension may be reduced to be, e.g. divisible by 4.
RTC_CHECK_GE(last_frame_info_->width, highest_stream_width);
RTC_CHECK_GE(last_frame_info_->height, highest_stream_height);
crop_width_ = last_frame_info_->width - highest_stream_width;
crop_height_ = last_frame_info_->height - highest_stream_height;
if (!encoder_->GetEncoderInfo().is_qp_trusted.value_or(
true)) {
// when qp is not trusted, we priorities to using the
// |resolution_bitrate_limits| provided by the decoder.
const std::vector<VideoEncoder::ResolutionBitrateLimits>& bitrate_limits =
encoder_->GetEncoderInfo().resolution_bitrate_limits.empty()
? EncoderInfoSettings::
GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()
: encoder_->GetEncoderInfo().resolution_bitrate_limits;
// For BandwidthQualityScaler, its implement based on a certain pixel_count
// correspond a certain bps interval. In fact, WebRTC default max_bps is
// 2500Kbps when width * height > 960 * 540. For example, we assume:
// 1.the camera support 1080p.
// 2.ResolutionBitrateLimits set 720p bps interval is [1500Kbps,2000Kbps].
// 3.ResolutionBitrateLimits set 1080p bps interval is [2000Kbps,2500Kbps].
// We will never be stable at 720p due to actual encoding bps of 720p and
// 1080p are both 2500Kbps. So it is necessary to do a linear interpolation
// to get a certain bitrate for certain pixel_count. It also doesn't work
// for 960*540 and 640*520, we will nerver be stable at 640*520 due to their
// |target_bitrate_bps| are both 2000Kbps.
std::optional<VideoEncoder::ResolutionBitrateLimits>
qp_untrusted_bitrate_limit = EncoderInfoSettings::
GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
last_frame_info_->width * last_frame_info_->height,
bitrate_limits);
if (qp_untrusted_bitrate_limit) {
// bandwidth_quality_scaler is only used for singlecast.
if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) {
VideoStream& stream = streams.back();
stream.max_bitrate_bps =
std::min(stream.max_bitrate_bps,
qp_untrusted_bitrate_limit->max_bitrate_bps);
stream.min_bitrate_bps =
std::min(stream.max_bitrate_bps,
qp_untrusted_bitrate_limit->min_bitrate_bps);
// If it is screen share mode, the minimum value of max_bitrate should
// be greater than/equal to 1200kbps.
if (encoder_config_.content_type ==
VideoEncoderConfig::ContentType::kScreen) {
stream.max_bitrate_bps =
std::max(stream.max_bitrate_bps, kDefaultMinScreenSharebps);
}
stream.target_bitrate_bps = stream.max_bitrate_bps;
}
}
}
else {
std::optional<VideoEncoder::ResolutionBitrateLimits>
encoder_bitrate_limits =
encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
last_frame_info_->width * last_frame_info_->height);
if (encoder_bitrate_limits) {
if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) {
// Bitrate limits can be set by app (in SDP or RtpEncodingParameters)
// or/and can be provided by encoder. In presence of both set of
// limits, the final set is derived as their intersection.
int min_bitrate_bps;
if (encoder_config_.simulcast_layers[0].min_bitrate_bps <= 0) {
min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps;
}
else {
min_bitrate_bps = std::max(encoder_bitrate_limits->min_bitrate_bps,
streams.back().min_bitrate_bps);
}
int max_bitrate_bps;
// The API max bitrate comes from both `encoder_config_.max_bitrate_bps`
// and `encoder_config_.simulcast_layers[0].max_bitrate_bps`.
std::optional<
int> api_max_bitrate_bps;
if (encoder_config_.simulcast_layers[0].max_bitrate_bps > 0) {
api_max_bitrate_bps =
encoder_config_.simulcast_layers[0].max_bitrate_bps;
}
if (encoder_config_.max_bitrate_bps > 0) {
api_max_bitrate_bps = api_max_bitrate_bps.has_value()
? std::min(encoder_config_.max_bitrate_bps,
*api_max_bitrate_bps)
: encoder_config_.max_bitrate_bps;
}
if (!api_max_bitrate_bps.has_value()) {
max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps;
}
else {
max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps,
streams.back().max_bitrate_bps);
}
if (min_bitrate_bps < max_bitrate_bps) {
streams.back().min_bitrate_bps = min_bitrate_bps;
streams.back().max_bitrate_bps = max_bitrate_bps;
streams.back().target_bitrate_bps =
std::min(streams.back().target_bitrate_bps,
encoder_bitrate_limits->max_bitrate_bps);
}
else {
RTC_LOG(LS_WARNING)
<<
"Bitrate limits provided by encoder"
<<
" (min=" << encoder_bitrate_limits->min_bitrate_bps
<<
", max=" << encoder_bitrate_limits->max_bitrate_bps
<<
") do not intersect with limits set by app"
<<
" (min=" << streams.back().min_bitrate_bps
<<
", max=" << api_max_bitrate_bps.value_or(-1)
<<
"). The app bitrate limits will be used.";
}
}
}
}
ApplyEncoderBitrateLimitsIfSingleActiveStream(
GetEncoderInfoWithBitrateLimitUpdate(
encoder_->GetEncoderInfo(), encoder_config_, default_limits_allowed_),
encoder_config_.simulcast_layers, &streams);
VideoCodec codec = VideoCodecInitializer::SetupCodec(
env_.field_trials(), encoder_config_, streams);
if (encoder_config_.codec_type == kVideoCodecVP9 ||
encoder_config_.codec_type == kVideoCodecAV1
#ifdef RTC_ENABLE_H265
|| encoder_config_.codec_type == kVideoCodecH265
#endif
) {
// Spatial layers configuration might impose some parity restrictions,
// thus some cropping might be needed.
RTC_CHECK_GE(last_frame_info_->width, codec.width);
RTC_CHECK_GE(last_frame_info_->height, codec.height);
crop_width_ = last_frame_info_->width - codec.width;
crop_height_ = last_frame_info_->height - codec.height;
ApplySpatialLayerBitrateLimits(
GetEncoderInfoWithBitrateLimitUpdate(encoder_->GetEncoderInfo(),
encoder_config_,
default_limits_allowed_),
encoder_config_, &codec);
}
char log_stream_buf[4 * 1024];
rtc::SimpleStringBuilder log_stream(log_stream_buf);
log_stream <<
"ReconfigureEncoder: simulcast streams: ";
for (size_t i = 0; i < codec.numberOfSimulcastStreams; ++i) {
std::optional<ScalabilityMode> scalability_mode =
codec.simulcastStream[i].GetScalabilityMode();
if (scalability_mode) {
log_stream <<
"{" << i <<
": " << codec.simulcastStream[i].width <<
"x"
<< codec.simulcastStream[i].height <<
" "
<< ScalabilityModeToString(*scalability_mode)
<<
", min_kbps: " << codec.simulcastStream[i].minBitrate
<<
", target_kbps: " << codec.simulcastStream[i].targetBitrate
<<
", max_kbps: " << codec.simulcastStream[i].maxBitrate
<<
", max_fps: " << codec.simulcastStream[i].maxFramerate
<<
", max_qp: " << codec.simulcastStream[i].qpMax
<<
", num_tl: "
<< codec.simulcastStream[i].numberOfTemporalLayers
<<
", active: "
<< (codec.simulcastStream[i].active ?
"true" :
"false") <<
"}";
}
}
if (encoder_config_.codec_type == kVideoCodecVP9 ||
encoder_config_.codec_type == kVideoCodecAV1
#ifdef RTC_ENABLE_H265
|| encoder_config_.codec_type == kVideoCodecH265
#endif
) {
log_stream <<
", spatial layers: ";
for (
int i = 0; i < GetNumSpatialLayers(codec); ++i) {
log_stream <<
"{" << i <<
": " << codec.spatialLayers[i].width <<
"x"
<< codec.spatialLayers[i].height
<<
", min_kbps: " << codec.spatialLayers[i].minBitrate
<<
", target_kbps: " << codec.spatialLayers[i].targetBitrate
<<
", max_kbps: " << codec.spatialLayers[i].maxBitrate
<<
", max_fps: " << codec.spatialLayers[i].maxFramerate
<<
", max_qp: " << codec.spatialLayers[i].qpMax <<
", num_tl: "
<< codec.spatialLayers[i].numberOfTemporalLayers
<<
", active: "
<< (codec.spatialLayers[i].active ?
"true" :
"false") <<
"}";
}
}
RTC_LOG(LS_INFO) <<
"[VSE] " << log_stream.str();
codec.startBitrate = std::max(encoder_target_bitrate_bps_.value_or(0) / 1000,
codec.minBitrate);
codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate);
codec.expect_encode_from_texture = last_frame_info_->is_texture;
// Make sure the start bit rate is sane...
RTC_DCHECK_LE(codec.startBitrate, 1000000);
max_framerate_ = codec.maxFramerate;
// The resolutions that we're actually encoding with.
std::vector<rtc::VideoSinkWants::FrameSize> encoder_resolutions;
// TODO(hbos): For the case of SVC, also make use of `codec.spatialLayers`.
// For now, SVC layers are handled by the VP9 encoder.
for (
const auto& simulcastStream : codec.simulcastStream) {
if (!simulcastStream.active)
continue;
encoder_resolutions.emplace_back(simulcastStream.width,
simulcastStream.height);
}
worker_queue_->PostTask(SafeTask(
task_safety_.flag(),
[
this, alignment,
encoder_resolutions = std::move(encoder_resolutions)]() {
RTC_DCHECK_RUN_ON(worker_queue_);
if (alignment != video_source_sink_controller_.resolution_alignment() ||
encoder_resolutions !=
video_source_sink_controller_.resolutions()) {
video_source_sink_controller_.SetResolutionAlignment(alignment);
video_source_sink_controller_.SetResolutions(
std::move(encoder_resolutions));
video_source_sink_controller_.PushSourceSinkSettings();
}
}));
rate_allocator_ = settings_.bitrate_allocator_factory->Create(env_, codec);
rate_allocator_->SetLegacyConferenceMode(
encoder_config_.legacy_conference_mode);
// Reset (release existing encoder) if one exists and anything except
// start bitrate or max framerate has changed.
if (!encoder_reset_required) {
encoder_reset_required = RequiresEncoderReset(
send_codec_, codec, was_encode_called_since_last_initialization_);
}
if (codec.codecType == VideoCodecType::kVideoCodecVP9 &&
number_of_cores_ <= vp9_low_tier_core_threshold_.value_or(0)) {
codec.SetVideoEncoderComplexity(VideoCodecComplexity::kComplexityLow);
}
quality_convergence_controller_.Initialize(
codec.numberOfSimulcastStreams, encoder_->GetEncoderInfo().min_qp,
codec.codecType, env_.field_trials());
send_codec_ = codec;
// Keep the same encoder, as long as the video_format is unchanged.
// Encoder creation block is split in two since EncoderInfo needed to start
// CPU adaptation with the correct settings should be polled after
// encoder_->InitEncode().
if (encoder_reset_required) {
ReleaseEncoder();
const size_t max_data_payload_length = max_data_payload_length_ > 0
? max_data_payload_length_
: kDefaultPayloadSize;
VideoEncoder::Settings settings = VideoEncoder::Settings(
settings_.capabilities, number_of_cores_, max_data_payload_length);
settings.encoder_thread_limit = experimental_encoder_thread_limit_;
int error = encoder_->InitEncode(&send_codec_, settings);
if (error != 0) {
RTC_LOG(LS_ERROR) <<
"Failed to initialize the encoder associated with "
"codec type: "
<< CodecTypeToPayloadString(send_codec_.codecType)
<<
" (" << send_codec_.codecType
<<
"). Error: " << error;
ReleaseEncoder();
}
else {
encoder_initialized_ =
true;
encoder_->RegisterEncodeCompleteCallback(
this);
frame_encode_metadata_writer_.OnEncoderInit(send_codec_);
next_frame_types_.clear();
next_frame_types_.resize(
std::max(
static_cast<
int>(codec.numberOfSimulcastStreams), 1),
VideoFrameType::kVideoFrameKey);
}
frame_encode_metadata_writer_.Reset();
last_encode_info_ms_ = std::nullopt;
was_encode_called_since_last_initialization_ =
false;
}
// Inform dependents of updated encoder settings.
OnEncoderSettingsChanged();
if (encoder_initialized_) {
RTC_LOG(LS_VERBOSE) <<
" max bitrate " << codec.maxBitrate
<<
" start bitrate " << codec.startBitrate
<<
" max frame rate " << codec.maxFramerate
<<
" max payload size " << max_data_payload_length_;
}
else {
RTC_LOG(LS_ERROR) <<
"[VSE] Failed to configure encoder.";
rate_allocator_ = nullptr;
}
if (pending_encoder_creation_) {
stream_resource_manager_.ConfigureEncodeUsageResource();
pending_encoder_creation_ =
false;
}
int num_layers;
if (codec.codecType == kVideoCodecVP8) {
num_layers = codec.VP8()->numberOfTemporalLayers;
}
else if (codec.codecType == kVideoCodecVP9) {
num_layers = codec.VP9()->numberOfTemporalLayers;
}
else if ((codec.codecType == kVideoCodecAV1 ||
codec.codecType == kVideoCodecH265) &&
codec.GetScalabilityMode().has_value()) {
num_layers =
ScalabilityModeToNumTemporalLayers(*(codec.GetScalabilityMode()));
}
else if (codec.codecType == kVideoCodecH264) {
num_layers = codec.H264()->numberOfTemporalLayers;
}
else if (codec.codecType == kVideoCodecGeneric &&
codec.numberOfSimulcastStreams > 0) {
// This is mainly for unit testing, disabling frame dropping.
// TODO(sprang): Add a better way to disable frame dropping.
num_layers = codec.simulcastStream[0].numberOfTemporalLayers;
}
else {
num_layers = 1;
}
frame_dropper_.Reset();
frame_dropper_.SetRates(codec.startBitrate, max_framerate_);
// Force-disable frame dropper if either:
// * We have screensharing with layers.
// * "WebRTC-FrameDropper" field trial is "Disabled".
force_disable_frame_dropper_ =
env_.field_trials().IsDisabled(kFrameDropperFieldTrial) ||
(num_layers > 1 && codec.mode == VideoCodecMode::kScreensharing);
const VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo();
if (rate_control_settings_.UseEncoderBitrateAdjuster()) {
bitrate_adjuster_ = std::make_unique<EncoderBitrateAdjuster>(
codec, env_.field_trials(), env_.clock());
bitrate_adjuster_->OnEncoderInfo(info);
}
if (rate_allocator_ && last_encoder_rate_settings_) {
// We have a new rate allocator instance and already configured target
// bitrate. Update the rate allocation and notify observers.
// We must invalidate the last_encoder_rate_settings_ to ensure
// the changes get propagated to all listeners.
EncoderRateSettings rate_settings = *last_encoder_rate_settings_;
last_encoder_rate_settings_.reset();
rate_settings.rate_control.framerate_fps = GetInputFramerateFps();
SetEncoderRates(UpdateBitrateAllocation(rate_settings));
}
encoder_stats_observer_->OnEncoderReconfigured(encoder_config_, streams);
pending_encoder_reconfiguration_ =
false;
bool is_svc =
false;
bool single_stream_or_non_first_inactive =
true;
for (size_t i = 1; i < encoder_config_.simulcast_layers.size(); ++i) {
if (encoder_config_.simulcast_layers[i].active) {
single_stream_or_non_first_inactive =
false;
break;
}
}
// Set min_bitrate_bps, max_bitrate_bps, and max padding bit rate for VP9,
// AV1 and H.265, and leave only one stream containing all necessary
// information.
if ((
#ifdef RTC_ENABLE_H265
encoder_config_.codec_type == kVideoCodecH265 ||
#endif
encoder_config_.codec_type == kVideoCodecVP9 ||
encoder_config_.codec_type == kVideoCodecAV1) &&
single_stream_or_non_first_inactive) {
// Lower max bitrate to the level codec actually can produce.
streams[0].max_bitrate_bps =
std::min(streams[0].max_bitrate_bps,
SvcRateAllocator::GetMaxBitrate(codec).bps<
int>());
streams[0].min_bitrate_bps = codec.spatialLayers[0].minBitrate * 1000;
// target_bitrate_bps specifies the maximum padding bitrate.
streams[0].target_bitrate_bps =
SvcRateAllocator::GetPaddingBitrate(codec).bps<
int>();
streams[0].width = streams.back().width;
streams[0].height = streams.back().height;
is_svc = GetNumSpatialLayers(codec) > 1;
streams.resize(1);
}
sink_->OnEncoderConfigurationChanged(
std::move(streams), is_svc, encoder_config_.content_type,
encoder_config_.min_transmit_bitrate_bps);
stream_resource_manager_.ConfigureQualityScaler(info);
stream_resource_manager_.ConfigureBandwidthQualityScaler(info);
webrtc::RTCError encoder_configuration_result = webrtc::RTCError::OK();
if (!encoder_initialized_) {
RTC_LOG(LS_WARNING) <<
"Failed to initialize "
<< CodecTypeToPayloadString(codec.codecType)
<<
" encoder."
<<
"switch_encoder_on_init_failures: "
<< switch_encoder_on_init_failures_;
if (switch_encoder_on_init_failures_) {
RequestEncoderSwitch();
}
else {
encoder_configuration_result =
webrtc::RTCError(RTCErrorType::UNSUPPORTED_OPERATION);
}
}
if (!encoder_configuration_callbacks_.empty()) {
for (
auto& callback : encoder_configuration_callbacks_) {
webrtc::InvokeSetParametersCallback(callback,
encoder_configuration_result);
}
encoder_configuration_callbacks_.clear();
}
}
void VideoStreamEncoder::RequestEncoderSwitch() {
bool is_encoder_switching_supported =
settings_.encoder_switch_request_callback != nullptr;
bool is_encoder_selector_available = encoder_selector_ != nullptr;
RTC_LOG(LS_INFO) <<
"RequestEncoderSwitch."
<<
" is_encoder_selector_available: "
<< is_encoder_selector_available
<<
" is_encoder_switching_supported: "
<< is_encoder_switching_supported;
if (!is_encoder_switching_supported) {
return;
}
// If encoder selector is available, switch to the encoder it prefers.
// Otherwise try switching to VP8 (default WebRTC codec).
std::optional<SdpVideoFormat> preferred_fallback_encoder;
if (is_encoder_selector_available) {
preferred_fallback_encoder = encoder_selector_->OnEncoderBroken();
}
if (!preferred_fallback_encoder) {
preferred_fallback_encoder =
SdpVideoFormat(CodecTypeToPayloadString(kVideoCodecVP8));
}
settings_.encoder_switch_request_callback->RequestEncoderSwitch(
*preferred_fallback_encoder,
/*allow_default_fallback=*/true);
}
void VideoStreamEncoder::OnEncoderSettingsChanged() {
EncoderSettings encoder_settings(
GetEncoderInfoWithBitrateLimitUpdate(
encoder_->GetEncoderInfo(), encoder_config_, default_limits_allowed_),
encoder_config_.Copy(), send_codec_);
stream_resource_manager_.SetEncoderSettings(encoder_settings);
input_state_provider_.OnEncoderSettingsChanged(encoder_settings);
bool is_screenshare = encoder_settings.encoder_config().content_type ==
VideoEncoderConfig::ContentType::kScreen;
degradation_preference_manager_->SetIsScreenshare(is_screenshare);
if (is_screenshare) {
frame_cadence_adapter_->SetZeroHertzModeEnabled(
FrameCadenceAdapterInterface::ZeroHertzModeParams{
send_codec_.numberOfSimulcastStreams});
}
}
void VideoStreamEncoder::OnFrame(Timestamp post_time,
bool queue_overload,
const VideoFrame& video_frame) {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
VideoFrame incoming_frame = video_frame;
// In some cases, e.g., when the frame from decoder is fed to encoder,
// the timestamp may be set to the future. As the encoding pipeline assumes
// capture time to be less than present time, we should reset the capture
// timestamps here. Otherwise there may be issues with RTP send stream.
if (incoming_frame.timestamp_us() > post_time.us())
incoming_frame.set_timestamp_us(post_time.us());
// Capture time may come from clock with an offset and drift from clock_.
int64_t capture_ntp_time_ms;
if (video_frame.ntp_time_ms() > 0) {
capture_ntp_time_ms = video_frame.ntp_time_ms();
}
else if (video_frame.render_time_ms() != 0) {
capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_;
}
else {
capture_ntp_time_ms = post_time.ms() + delta_ntp_internal_ms_;
}
incoming_frame.set_ntp_time_ms(capture_ntp_time_ms);
// Convert NTP time, in ms, to RTP timestamp.
const int kMsToRtpTimestamp = 90;
incoming_frame.set_rtp_timestamp(
kMsToRtpTimestamp *
static_cast<uint32_t>(incoming_frame.ntp_time_ms()));
// Identifier should remain the same for newly produced incoming frame and the
// received |video_frame|.
incoming_frame.set_presentation_timestamp(
video_frame.presentation_timestamp());
if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) {
// We don't allow the same capture time for two frames, drop this one.
RTC_LOG(LS_WARNING) <<
"Same/old NTP timestamp ("
<< incoming_frame.ntp_time_ms()
<<
" <= " << last_captured_timestamp_
<<
") for incoming frame. Dropping.";
ProcessDroppedFrame(incoming_frame,
VideoStreamEncoderObserver::DropReason::kBadTimestamp);
return;
}
bool log_stats =
false;
if (post_time.ms() - last_frame_log_ms_ > kFrameLogIntervalMs) {
--> --------------------
--> maximum size reached
--> --------------------
