From 1936d42965d857f1ae2129faefe8dea678df08fe Mon Sep 17 00:00:00 2001 From: iphydf Date: Thu, 1 Jan 2026 21:20:24 +0000 Subject: [PATCH] test: add benchmark for toxav audio and video This includes RTP, but no networking. --- CMakeLists.txt | 12 ++ other/analysis/gen-file.sh | 4 + other/analysis/run-cppcheck | 3 +- toxav/BUILD.bazel | 65 +++++- toxav/audio_bench.cc | 199 ++++++++++++++++++ toxav/audio_test.cc | 388 ++++++++++++++++++++++++------------ toxav/av_test_support.cc | 171 ++++++++++++++++ toxav/av_test_support.hh | 89 +++++++++ toxav/rtp_bench.cc | 73 +++++++ toxav/video_bench.cc | 196 ++++++++++++++++++ toxav/video_test.cc | 337 +++++++++++++++++++------------ 11 files changed, 1280 insertions(+), 257 deletions(-) create mode 100644 toxav/audio_bench.cc create mode 100644 toxav/av_test_support.cc create mode 100644 toxav/av_test_support.hh create mode 100644 toxav/rtp_bench.cc create mode 100644 toxav/video_bench.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index bda1746a6..3c64c7def 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -578,12 +578,24 @@ endfunction() # if(UNITTEST AND TARGET GTest::gtest AND TARGET GTest::gmock) if(BUILD_TOXAV) + add_library(av_test_support STATIC + toxav/av_test_support.cc + toxav/av_test_support.hh) + if(TARGET toxcore_static) + target_link_libraries(av_test_support PRIVATE toxcore_static) + else() + target_link_libraries(av_test_support PRIVATE toxcore_shared) + endif() + target_link_libraries(av_test_support PRIVATE GTest::gtest) + unit_test(toxav audio) + target_link_libraries(unit_audio_test PRIVATE av_test_support) unit_test(toxav bwcontroller) unit_test(toxav msi) unit_test(toxav ring_buffer) unit_test(toxav rtp) unit_test(toxav video) + target_link_libraries(unit_video_test PRIVATE av_test_support) endif() unit_test(toxcore DHT) diff --git a/other/analysis/gen-file.sh b/other/analysis/gen-file.sh index 6af978c88..5dbcc1018 100644 --- a/other/analysis/gen-file.sh +++ b/other/analysis/gen-file.sh @@ -62,6 +62,10 @@ FIND_QUERY="$FIND_QUERY -and -not -wholename './testing/fuzzing/*'" FIND_QUERY="$FIND_QUERY -and -not -wholename './third_party/cmp/examples/*'" FIND_QUERY="$FIND_QUERY -and -not -wholename './third_party/cmp/test/*'" +if [ "$SKIP_BENCHMARK" == 1 ]; then + FIND_QUERY="$FIND_QUERY -and -not -name '*_bench.cc'" +fi + if [ "$SKIP_GTEST" == 1 ]; then FIND_QUERY="$FIND_QUERY -and -not -name '*_test.cc'" fi diff --git a/other/analysis/run-cppcheck b/other/analysis/run-cppcheck index e861851dc..70b9df75c 100755 --- a/other/analysis/run-cppcheck +++ b/other/analysis/run-cppcheck @@ -1,6 +1,7 @@ #!/bin/bash SKIP_GTEST=1 +SKIP_BENCHMARK=1 . other/analysis/gen-file.sh @@ -24,8 +25,6 @@ CPPCHECK+=("--suppress=knownConditionTrueFalse") CPPCHECK+=("--suppress=missingIncludeSystem") # TODO(iphydf): Maybe fix? CPPCHECK+=("--suppress=signConversion") -# We have suppressions in the code for the misra extension. -CPPCHECK+=("--suppress=unmatchedSuppression") # We use this for VLAs. CPPCHECK_CXX+=("--suppress=allocaCalled") diff --git a/toxav/BUILD.bazel b/toxav/BUILD.bazel index c6ce2018f..de644413f 100644 --- a/toxav/BUILD.bazel +++ b/toxav/BUILD.bazel @@ -1,6 +1,5 @@ -load("@rules_cc//cc:defs.bzl", "cc_test") +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") load("@rules_fuzzing//fuzzing:cc_defs.bzl", "cc_fuzz_test") -load("//tools:no_undefined.bzl", "cc_library") exports_files( srcs = ["toxav.h"], @@ -131,12 +130,29 @@ cc_library( ], ) +cc_library( + name = "av_test_support", + testonly = True, + srcs = ["av_test_support.cc"], + hdrs = ["av_test_support.hh"], + deps = [ + ":audio", + ":rtp", + ":video", + "//c-toxcore/toxcore:logger", + "//c-toxcore/toxcore:mono_time", + "//c-toxcore/toxcore:os_memory", + "@com_google_googletest//:gtest", + ], +) + cc_test( name = "audio_test", timeout = "moderate", srcs = ["audio_test.cc"], deps = [ ":audio", + ":av_test_support", ":rtp", "//c-toxcore/toxcore:logger", "//c-toxcore/toxcore:mono_time", @@ -166,6 +182,7 @@ cc_test( timeout = "moderate", srcs = ["video_test.cc"], deps = [ + ":av_test_support", ":rtp", ":video", "//c-toxcore/toxcore:logger", @@ -176,6 +193,50 @@ cc_test( ], ) +cc_binary( + name = "video_bench", + testonly = True, + srcs = ["video_bench.cc"], + deps = [ + ":av_test_support", + ":rtp", + ":video", + "//c-toxcore/toxcore:logger", + "//c-toxcore/toxcore:mono_time", + "//c-toxcore/toxcore:os_memory", + "@benchmark", + ], +) + +cc_binary( + name = "audio_bench", + testonly = True, + srcs = ["audio_bench.cc"], + deps = [ + ":audio", + ":av_test_support", + ":rtp", + "//c-toxcore/toxcore:logger", + "//c-toxcore/toxcore:mono_time", + "//c-toxcore/toxcore:os_memory", + "@benchmark", + ], +) + +cc_binary( + name = "rtp_bench", + testonly = True, + srcs = ["rtp_bench.cc"], + deps = [ + ":av_test_support", + ":rtp", + "//c-toxcore/toxcore:logger", + "//c-toxcore/toxcore:mono_time", + "//c-toxcore/toxcore:os_memory", + "@benchmark", + ], +) + cc_library( name = "msi", srcs = ["msi.c"], diff --git a/toxav/audio_bench.cc b/toxav/audio_bench.cc new file mode 100644 index 000000000..c218ca588 --- /dev/null +++ b/toxav/audio_bench.cc @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later + * Copyright © 2025 The TokTok team. + */ + +#include + +#include +#include +#include + +#include "../toxcore/logger.h" +#include "../toxcore/mono_time.h" +#include "../toxcore/network.h" +#include "../toxcore/os_memory.h" +#include "audio.h" +#include "av_test_support.hh" +#include "rtp.h" + +namespace { + +class AudioBench : public benchmark::Fixture { +public: + void SetUp(const ::benchmark::State &state) override + { + const Memory *mem = os_memory(); + log = logger_new(mem); + tm.t = 1000; + mono_time = mono_time_new(mem, mock_time_cb, &tm); + ac = ac_new(mono_time, log, 123, nullptr, nullptr); + + sampling_rate = static_cast(state.range(0)); + channels = static_cast(state.range(1)); + uint32_t bitrate = (channels == 1) ? 32000 : 64000; + + ac_reconfigure_encoder(ac, bitrate, sampling_rate, channels); + + sample_count = sampling_rate / 50; // 20ms frames + pcm.resize(sample_count * channels); + + rtp_mock.capture_packets = false; // Disable capturing for benchmarks + rtp_mock.auto_forward = true; + + rtp_mock.recv_session = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, + &rtp_mock, nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + } + + void TearDown(const ::benchmark::State &state) override + { + const Memory *mem = os_memory(); + if (rtp_mock.recv_session) { + rtp_kill(log, rtp_mock.recv_session); + } + if (ac) { + ac_kill(ac); + } + if (mono_time) { + mono_time_free(mem, mono_time); + } + if (log) { + logger_kill(log); + } + } + + Logger *log = nullptr; + Mono_Time *mono_time = nullptr; + MockTime tm; + ACSession *ac = nullptr; + RtpMock rtp_mock; + uint32_t sampling_rate = 0; + uint8_t channels = 0; + size_t sample_count = 0; + std::vector pcm; +}; + +// Benchmark encoding a sequence of silent audio frames. +BENCHMARK_DEFINE_F(AudioBench, EncodeSilentSequence)(benchmark::State &state) +{ + std::vector silent_pcm(sample_count * channels); + fill_silent_frame(channels, sample_count, silent_pcm); + + std::vector encoded(2000); + + for (auto _ : state) { + int encoded_size + = ac_encode(ac, silent_pcm.data(), sample_count, encoded.data(), encoded.size()); + benchmark::DoNotOptimize(encoded_size); + } +} + +BENCHMARK_REGISTER_F(AudioBench, EncodeSilentSequence) + ->Args({8000, 1}) + ->Args({48000, 1}) + ->Args({48000, 2}); + +// Benchmark encoding a sequence of audio frames. +BENCHMARK_DEFINE_F(AudioBench, EncodeSequence)(benchmark::State &state) +{ + int frame_index = 0; + const int num_prefilled = 50; + std::vector> pcms( + num_prefilled, std::vector(sample_count * channels)); + for (int i = 0; i < num_prefilled; ++i) { + fill_audio_frame(sampling_rate, channels, i, sample_count, pcms[i]); + } + + std::vector encoded(2000); + + for (auto _ : state) { + int idx = frame_index % num_prefilled; + int encoded_size + = ac_encode(ac, pcms[idx].data(), sample_count, encoded.data(), encoded.size()); + benchmark::DoNotOptimize(encoded_size); + frame_index++; + } +} + +BENCHMARK_REGISTER_F(AudioBench, EncodeSequence) + ->Args({8000, 1}) + ->Args({16000, 1}) + ->Args({24000, 1}) + ->Args({48000, 1}) + ->Args({48000, 2}); + +// Benchmark decoding a sequence of audio frames. +BENCHMARK_DEFINE_F(AudioBench, DecodeSequence)(benchmark::State &state) +{ + const int num_frames = 50; + std::vector> encoded_frames(num_frames); + + // Pre-encode + std::vector encoded_tmp(2000); + for (int i = 0; i < num_frames; ++i) { + fill_audio_frame(sampling_rate, channels, i, sample_count, pcm); + int size = ac_encode(ac, pcm.data(), sample_count, encoded_tmp.data(), encoded_tmp.size()); + + encoded_frames[i].resize(4 + size); + uint32_t net_sr = net_htonl(sampling_rate); + std::memcpy(encoded_frames[i].data(), &net_sr, 4); + std::memcpy(encoded_frames[i].data() + 4, encoded_tmp.data(), size); + } + + int frame_index = 0; + for (auto _ : state) { + int idx = frame_index % num_frames; + rtp_send_data(log, rtp_mock.recv_session, encoded_frames[idx].data(), + static_cast(encoded_frames[idx].size()), false); + ac_iterate(ac); + frame_index++; + } +} + +BENCHMARK_REGISTER_F(AudioBench, DecodeSequence) + ->Args({8000, 1}) + ->Args({16000, 1}) + ->Args({24000, 1}) + ->Args({48000, 1}) + ->Args({48000, 2}); + +// Full end-to-end sequence benchmark (Encode -> RTP -> Decode) +BENCHMARK_DEFINE_F(AudioBench, FullSequence)(benchmark::State &state) +{ + int frame_index = 0; + const int num_prefilled = 50; + std::vector> pcms( + num_prefilled, std::vector(sample_count * channels)); + for (int i = 0; i < num_prefilled; ++i) { + fill_audio_frame(sampling_rate, channels, i, sample_count, pcms[i]); + } + + std::vector encoded_tmp(2000); + + for (auto _ : state) { + int idx = frame_index % num_prefilled; + int size + = ac_encode(ac, pcms[idx].data(), sample_count, encoded_tmp.data(), encoded_tmp.size()); + + std::vector payload(4 + size); + uint32_t net_sr = net_htonl(sampling_rate); + std::memcpy(payload.data(), &net_sr, 4); + std::memcpy(payload.data() + 4, encoded_tmp.data(), size); + + rtp_send_data(log, rtp_mock.recv_session, payload.data(), + static_cast(payload.size()), false); + ac_iterate(ac); + + frame_index++; + } +} + +BENCHMARK_REGISTER_F(AudioBench, FullSequence) + ->Args({8000, 1}) + ->Args({16000, 1}) + ->Args({24000, 1}) + ->Args({48000, 1}) + ->Args({48000, 2}); + +} + +BENCHMARK_MAIN(); diff --git a/toxav/audio_test.cc b/toxav/audio_test.cc index fb6367e24..7bb65d6b1 100644 --- a/toxav/audio_test.cc +++ b/toxav/audio_test.cc @@ -3,84 +3,19 @@ #include #include +#include #include #include "../toxcore/logger.h" #include "../toxcore/mono_time.h" #include "../toxcore/network.h" #include "../toxcore/os_memory.h" +#include "av_test_support.hh" #include "rtp.h" namespace { -struct AudioTimeMock { - uint64_t t; -}; - -uint64_t audio_mock_time_cb(void *ud) { return static_cast(ud)->t; } - -struct AudioTestData { - uint32_t friend_number = 0; - std::vector last_pcm; - size_t sample_count = 0; - uint8_t channels = 0; - uint32_t sampling_rate = 0; - - static void receive_frame(uint32_t friend_number, const int16_t *pcm, size_t sample_count, - uint8_t channels, uint32_t sampling_rate, void *user_data) - { - auto *self = static_cast(user_data); - self->friend_number = friend_number; - self->last_pcm.assign(pcm, pcm + sample_count * channels); - self->sample_count = sample_count; - self->channels = channels; - self->sampling_rate = sampling_rate; - } -}; - -struct AudioRtpMock { - RTPSession *recv_session = nullptr; - std::vector> captured_packets; - bool auto_forward = true; - - static int send_packet(void *user_data, const uint8_t *data, uint16_t length) - { - auto *self = static_cast(user_data); - self->captured_packets.push_back(std::vector(data, data + length)); - if (self->auto_forward && self->recv_session) { - rtp_receive_packet(self->recv_session, data, length); - } - return 0; - } - - static int audio_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg) - { - return ac_queue_message(mono_time, cs, msg); - } -}; - -class AudioTest : public ::testing::Test { -protected: - void SetUp() override - { - const Memory *mem = os_memory(); - log = logger_new(mem); - tm.t = 1000; - mono_time = mono_time_new(mem, audio_mock_time_cb, &tm); - mono_time_update(mono_time); - } - - void TearDown() override - { - const Memory *mem = os_memory(); - mono_time_free(mem, mono_time); - logger_kill(log); - } - - Logger *log; - Mono_Time *mono_time; - AudioTimeMock tm; -}; +using AudioTest = AvTest; TEST_F(AudioTest, BasicNewKill) { @@ -96,11 +31,11 @@ TEST_F(AudioTest, EncodeDecodeLoop) ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); ASSERT_NE(ac, nullptr); - AudioRtpMock rtp_mock; - RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); - RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); + RtpMock rtp_mock; + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); rtp_mock.recv_session = recv_rtp; uint32_t sampling_rate = 48000; @@ -144,6 +79,203 @@ TEST_F(AudioTest, EncodeDecodeLoop) ac_kill(ac); } +TEST_F(AudioTest, EncodeDecodeRealistic) +{ + AudioTestData data; + ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); + ASSERT_NE(ac, nullptr); + + RtpMock rtp_mock; + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + rtp_mock.recv_session = recv_rtp; + + uint32_t sampling_rate = 48000; + uint8_t channels = 1; + size_t sample_count = 960; // 20ms at 48kHz + uint32_t bitrate = 48000; + + ASSERT_EQ(ac_reconfigure_encoder(ac, bitrate, sampling_rate, channels), 0); + + double frequency = 440.0; + double amplitude = 10000.0; + const double pi = std::acos(-1.0); + + std::vector all_sent; + std::vector all_recv; + + for (int frame = 0; frame < 50; ++frame) { + std::vector pcm(sample_count * channels); + for (size_t i = 0; i < sample_count; ++i) { + double t = static_cast(frame * sample_count + i) / sampling_rate; + pcm[i] = static_cast(std::sin(2.0 * pi * frequency * t) * amplitude); + } + all_sent.insert(all_sent.end(), pcm.begin(), pcm.end()); + + std::vector encoded(2000); + int encoded_size = ac_encode(ac, pcm.data(), sample_count, encoded.data(), encoded.size()); + ASSERT_GT(encoded_size, 0); + + std::vector payload(4 + static_cast(encoded_size)); + uint32_t net_sr = net_htonl(sampling_rate); + memcpy(payload.data(), &net_sr, 4); + memcpy(payload.data() + 4, encoded.data(), static_cast(encoded_size)); + + rtp_send_data(log, send_rtp, payload.data(), static_cast(payload.size()), false); + + ac_iterate(ac); + + if (data.sample_count > 0) { + all_recv.insert(all_recv.end(), data.last_pcm.begin(), data.last_pcm.end()); + } + } + + ASSERT_FALSE(all_recv.empty()); + + // Find the best match by trying different delays. + // Jitter buffer delay (3 frames = 2880 samples) + Opus lookahead (~312 samples) = ~3192. + double min_mse = 1e18; + int best_delay = 0; + + // Search around the expected delay + for (int delay = 3000; delay < 3500; ++delay) { + double mse = 0; + int count = 0; + for (size_t i = 0; i < 2000; ++i) { // Compare a decent chunk + if (i + delay < all_sent.size() && i < all_recv.size()) { + int diff = all_sent[i + delay] - all_recv[i]; + mse += static_cast(diff) * diff; + count++; + } + } + if (count > 1000) { + mse /= count; + if (mse < min_mse) { + min_mse = mse; + best_delay = delay; + } + } + } + + printf("Best audio delay found: %d samples, Min MSE: %f\n", best_delay, min_mse); + + // For 48kbps Opus, the MSE for a sine wave should be quite low once aligned. + // 10M is about 20% of the signal power (50M), which is a safe threshold for verification. + EXPECT_LT(min_mse, 10000000.0); + + rtp_kill(log, send_rtp); + rtp_kill(log, recv_rtp); + ac_kill(ac); +} + +TEST_F(AudioTest, EncodeDecodeSiren) +{ + AudioTestData data; + ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); + ASSERT_NE(ac, nullptr); + + RtpMock rtp_mock; + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + rtp_mock.recv_session = recv_rtp; + + uint32_t sampling_rate = 48000; + uint8_t channels = 1; + size_t sample_count = 960; // 20ms at 48kHz + uint32_t bitrate = 64000; + + ASSERT_EQ(ac_reconfigure_encoder(ac, bitrate, sampling_rate, channels), 0); + + double amplitude = 10000.0; + const double pi = std::acos(-1.0); + + std::vector all_sent; + std::vector all_recv; + + // 1 second of audio (50 frames) is enough for a siren test + for (int frame = 0; frame < 50; ++frame) { + std::vector pcm(sample_count * channels); + for (size_t i = 0; i < sample_count; ++i) { + double t = static_cast(frame * sample_count + i) / sampling_rate; + // Linear frequency sweep from 50Hz to 440Hz over 1 second + // f(t) = 50 + (440-50)/1 * t = 50 + 390t + // phi(t) = 2*pi * integral(f(t)) = 2*pi * (50t + 195t^2) + double phi = 2.0 * pi * (50.0 * t + 195.0 * t * t); + pcm[i] = static_cast(std::sin(phi) * amplitude); + } + all_sent.insert(all_sent.end(), pcm.begin(), pcm.end()); + + std::vector encoded(2000); + int encoded_size = ac_encode(ac, pcm.data(), sample_count, encoded.data(), encoded.size()); + ASSERT_GT(encoded_size, 0); + + std::vector payload(4 + static_cast(encoded_size)); + uint32_t net_sr = net_htonl(sampling_rate); + memcpy(payload.data(), &net_sr, 4); + memcpy(payload.data() + 4, encoded.data(), static_cast(encoded_size)); + + rtp_send_data(log, send_rtp, payload.data(), static_cast(payload.size()), false); + + ac_iterate(ac); + + if (data.sample_count > 0) { + all_recv.insert(all_recv.end(), data.last_pcm.begin(), data.last_pcm.end()); + } + } + + ASSERT_FALSE(all_recv.empty()); + + auto calculate_mse_at = [&](int delay, size_t window) { + double mse = 0; + int count = 0; + for (size_t i = 0; i < window; ++i) { + int sent_idx = static_cast(i) + delay; + if (sent_idx >= 0 && static_cast(sent_idx) < all_sent.size() + && i < all_recv.size()) { + int diff = all_sent[static_cast(sent_idx)] - all_recv[i]; + mse += static_cast(diff) * diff; + count++; + } + } + return count > 0 ? mse / count : 1e18; + }; + + // Two-stage search for speed + double min_mse = 1e18; + int coarse_best = 0; + + // 1. Coarse search + for (int delay = -5000; delay < 5000; delay += 100) { + double mse = calculate_mse_at(delay, 5000); + if (mse < min_mse) { + min_mse = mse; + coarse_best = delay; + } + } + + // 2. Fine search around coarse best + int best_delay = coarse_best; + for (int delay = coarse_best - 100; delay <= coarse_best + 100; ++delay) { + double mse = calculate_mse_at(delay, 10000); + if (mse < min_mse) { + min_mse = mse; + best_delay = delay; + } + } + + printf("Best siren audio delay found: %d samples, Min MSE: %f\n", best_delay, min_mse); + + // For 64kbps Opus, the MSE for a siren wave should be reasonably low once aligned. + EXPECT_LT(min_mse, 20000000.0); + + rtp_kill(log, send_rtp); + rtp_kill(log, recv_rtp); + ac_kill(ac); +} TEST_F(AudioTest, ReconfigureEncoder) { AudioTestData data; @@ -184,12 +316,12 @@ TEST_F(AudioTest, QueueInvalidMessage) ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); ASSERT_NE(ac, nullptr); - AudioRtpMock rtp_mock; + RtpMock rtp_mock; // Create a video RTP session but try to queue to audio session - RTPSession *video_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); - RTPSession *audio_recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); + RTPSession *video_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *audio_recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, + &rtp_mock, nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); rtp_mock.recv_session = audio_recv_rtp; std::vector dummy_video(100, 0); @@ -212,12 +344,12 @@ TEST_F(AudioTest, JitterBufferDuplicate) ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); ASSERT_NE(ac, nullptr); - AudioRtpMock rtp_mock; + RtpMock rtp_mock; rtp_mock.auto_forward = false; - RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); - RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); rtp_mock.recv_session = recv_rtp; uint8_t dummy_data[100] = {0}; @@ -253,12 +385,12 @@ TEST_F(AudioTest, JitterBufferOutOfOrder) ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); ASSERT_NE(ac, nullptr); - AudioRtpMock rtp_mock; + RtpMock rtp_mock; rtp_mock.auto_forward = false; - RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); - RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); rtp_mock.recv_session = recv_rtp; uint8_t dummy_data[100] = {0}; @@ -300,12 +432,12 @@ TEST_F(AudioTest, PacketLossConcealment) ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); ASSERT_NE(ac, nullptr); - AudioRtpMock rtp_mock; + RtpMock rtp_mock; rtp_mock.auto_forward = false; - RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); - RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); rtp_mock.recv_session = recv_rtp; uint8_t dummy_data[100] = {0}; @@ -346,12 +478,12 @@ TEST_F(AudioTest, JitterBufferReset) ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); ASSERT_NE(ac, nullptr); - AudioRtpMock rtp_mock; + RtpMock rtp_mock; rtp_mock.auto_forward = false; - RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); - RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); rtp_mock.recv_session = recv_rtp; uint8_t dummy_data[100] = {0}; @@ -391,12 +523,12 @@ TEST_F(AudioTest, DecoderReconfigureCooldown) ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); ASSERT_NE(ac, nullptr); - AudioRtpMock rtp_mock; + RtpMock rtp_mock; rtp_mock.auto_forward = false; - RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); - RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); rtp_mock.recv_session = recv_rtp; uint8_t dummy_data[100] = {0}; @@ -452,12 +584,12 @@ TEST_F(AudioTest, QueueDummyMessage) ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); ASSERT_NE(ac, nullptr); - AudioRtpMock rtp_mock; + RtpMock rtp_mock; // RTP_TYPE_AUDIO + 2 is the dummy type - RTPSession *dummy_rtp = rtp_new(log, RTP_TYPE_AUDIO + 2, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); - RTPSession *audio_recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); + RTPSession *dummy_rtp = rtp_new(log, RTP_TYPE_AUDIO + 2, mono_time, RtpMock::send_packet, + &rtp_mock, nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *audio_recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, + &rtp_mock, nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); rtp_mock.recv_session = audio_recv_rtp; std::vector dummy_payload(100, 0); @@ -480,12 +612,12 @@ TEST_F(AudioTest, LatePacketReset) ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); ASSERT_NE(ac, nullptr); - AudioRtpMock rtp_mock; + RtpMock rtp_mock; rtp_mock.auto_forward = false; - RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); - RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); rtp_mock.recv_session = recv_rtp; uint8_t dummy_data[100] = {0}; @@ -531,12 +663,12 @@ TEST_F(AudioTest, InvalidSamplingRate) ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); ASSERT_NE(ac, nullptr); - AudioRtpMock rtp_mock; + RtpMock rtp_mock; rtp_mock.auto_forward = false; - RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); - RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); rtp_mock.recv_session = recv_rtp; // 1. Send a packet with an absurdly large sampling rate. @@ -578,12 +710,12 @@ TEST_F(AudioTest, ShortPacket) ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); ASSERT_NE(ac, nullptr); - AudioRtpMock rtp_mock; + RtpMock rtp_mock; rtp_mock.auto_forward = false; - RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); - RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); rtp_mock.recv_session = recv_rtp; // 1. Send a packet that is too short (only sampling rate, no Opus data). @@ -610,12 +742,12 @@ TEST_F(AudioTest, JitterBufferWrapAround) ACSession *ac = ac_new(mono_time, log, 123, AudioTestData::receive_frame, &data); ASSERT_NE(ac, nullptr); - AudioRtpMock rtp_mock; + RtpMock rtp_mock; rtp_mock.auto_forward = false; - RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); - RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, AudioRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, ac, AudioRtpMock::audio_cb); + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, ac, RtpMock::audio_cb); rtp_mock.recv_session = recv_rtp; uint8_t dummy_data[100] = {0}; diff --git a/toxav/av_test_support.cc b/toxav/av_test_support.cc new file mode 100644 index 000000000..d77ae2e6e --- /dev/null +++ b/toxav/av_test_support.cc @@ -0,0 +1,171 @@ +#include "av_test_support.hh" + +#include +#include +#include + +#include "../toxcore/os_memory.h" + +// Mock Time +uint64_t mock_time_cb(void *ud) { return static_cast(ud)->t; } + +// RTP Mock +int RtpMock::send_packet(void *user_data, const uint8_t *data, uint16_t length) +{ + auto *self = static_cast(user_data); + if (self->capture_packets) { + if (self->store_last_packet_only) { + if (self->captured_packets.empty()) { + self->captured_packets.emplace_back(data, data + length); + } else { + self->captured_packets[0].assign(data, data + length); + } + } else { + self->captured_packets.push_back(std::vector(data, data + length)); + } + } + if (self->auto_forward && self->recv_session) { + rtp_receive_packet(self->recv_session, data, length); + } + return 0; +} + +int RtpMock::audio_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg) +{ + return ac_queue_message(mono_time, cs, msg); +} + +int RtpMock::video_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg) +{ + return vc_queue_message(mono_time, cs, msg); +} + +int RtpMock::noop_cb(const Mono_Time * /*mono_time*/, void * /*cs*/, RTPMessage *msg) +{ + std::free(msg); + return 0; +} + +// Audio Helpers +void fill_audio_frame(uint32_t sampling_rate, uint8_t channels, int frame_index, + size_t sample_count, std::vector &pcm) +{ + const double pi = std::acos(-1.0); + double amplitude = 10000.0; + + for (size_t i = 0; i < sample_count; ++i) { + double t = static_cast(frame_index * sample_count + i) / sampling_rate; + // Linear frequency sweep from 50Hz to 440Hz over 1 second (50 frames) + // f(t) = 50 + 390t + // phi(t) = 2*pi * (50t + 195t^2) + double phi = 2.0 * pi * (50.0 * t + 195.0 * t * t); + int16_t val = static_cast(std::sin(phi) * amplitude); + for (uint8_t c = 0; c < channels; ++c) { + pcm[i * channels + c] = val; + } + } +} + +void fill_silent_frame(uint8_t channels, size_t sample_count, std::vector &pcm) +{ + for (size_t i = 0; i < sample_count * channels; ++i) { + // Very low amplitude white noise (simulating silence with background hiss) + pcm[i] = (std::rand() % 21) - 10; + } +} + +AudioTestData::AudioTestData() = default; +AudioTestData::~AudioTestData() = default; + +void AudioTestData::receive_frame(uint32_t friend_number, const int16_t *pcm, size_t sample_count, + uint8_t channels, uint32_t sampling_rate, void *user_data) +{ + auto *self = static_cast(user_data); + self->friend_number = friend_number; + self->last_pcm.assign(pcm, pcm + sample_count * channels); + self->sample_count = sample_count; + self->channels = channels; + self->sampling_rate = sampling_rate; +} + +// Video Helpers +void fill_video_frame(uint16_t width, uint16_t height, int frame_index, std::vector &y, + std::vector &u, std::vector &v) +{ + // Background (dark gray) + std::fill(y.begin(), y.end(), 32); + std::fill(u.begin(), u.end(), 128); + std::fill(v.begin(), v.end(), 128); + + // Moving square (light gray) + int sq_size = height / 4; + if (sq_size < 16) + sq_size = 16; + int x0 = (frame_index * 8) % (width - sq_size); + int y0 = (frame_index * 4) % (height - sq_size); + for (int r = 0; r < sq_size; ++r) { + for (int c = 0; c < sq_size; ++c) { + y[(y0 + r) * width + (x0 + c)] = 200; + } + } +} + +double calculate_video_mse(uint16_t width, uint16_t height, int32_t ystride, + const std::vector &y_recv, const std::vector &y_orig) +{ + if (y_recv.empty() || y_orig.size() != static_cast(width) * height) { + return 1e10; + } + + double mse = 0; + for (int r = 0; r < height; ++r) { + for (int c = 0; c < width; ++c) { + int diff = static_cast(y_orig[r * width + c]) - y_recv[r * std::abs(ystride) + c]; + mse += diff * diff; + } + } + return mse / (static_cast(width) * height); +} + +// Video Test Data Helper +VideoTestData::VideoTestData() = default; +VideoTestData::~VideoTestData() = default; + +void VideoTestData::receive_frame(uint32_t friend_number, uint16_t width, uint16_t height, + const uint8_t *y, const uint8_t *u, const uint8_t *v, int32_t ystride, int32_t ustride, + int32_t vstride, void *user_data) +{ + auto *self = static_cast(user_data); + self->friend_number = friend_number; + self->width = width; + self->height = height; + self->ystride = ystride; + self->ustride = ustride; + self->vstride = vstride; + + self->y.assign(y, y + static_cast(std::abs(ystride)) * height); + self->u.assign(u, u + static_cast(std::abs(ustride)) * (height / 2)); + self->v.assign(v, v + static_cast(std::abs(vstride)) * (height / 2)); +} + +double VideoTestData::calculate_mse(const std::vector &y_orig) const +{ + return calculate_video_mse(width, height, ystride, y, y_orig); +} + +// Common Test Fixture +void AvTest::SetUp() +{ + const Memory *mem = os_memory(); + log = logger_new(mem); + tm.t = 1000; + mono_time = mono_time_new(mem, mock_time_cb, &tm); + mono_time_update(mono_time); +} + +void AvTest::TearDown() +{ + const Memory *mem = os_memory(); + mono_time_free(mem, mono_time); + logger_kill(log); +} diff --git a/toxav/av_test_support.hh b/toxav/av_test_support.hh new file mode 100644 index 000000000..4c0eb4adc --- /dev/null +++ b/toxav/av_test_support.hh @@ -0,0 +1,89 @@ +#ifndef TOXAV_AV_TEST_SUPPORT_H +#define TOXAV_AV_TEST_SUPPORT_H + +#include + +#include +#include + +#include "../toxcore/logger.h" +#include "../toxcore/mono_time.h" +#include "audio.h" +#include "rtp.h" +#include "video.h" + +// Mock Time +struct MockTime { + uint64_t t = 1000; +}; +uint64_t mock_time_cb(void *ud); + +// RTP Mock +struct RtpMock { + RTPSession *recv_session = nullptr; + std::vector> captured_packets; + bool auto_forward = true; + bool capture_packets = true; + bool store_last_packet_only = false; + + static int send_packet(void *user_data, const uint8_t *data, uint16_t length); + static int audio_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg); + static int video_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg); + static int noop_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg); +}; + +// Audio Helpers +void fill_audio_frame(uint32_t sampling_rate, uint8_t channels, int frame_index, + size_t sample_count, std::vector &pcm); +void fill_silent_frame(uint8_t channels, size_t sample_count, std::vector &pcm); + +struct AudioTestData { + uint32_t friend_number = 0; + std::vector last_pcm; + size_t sample_count = 0; + uint8_t channels = 0; + uint32_t sampling_rate = 0; + + AudioTestData(); + ~AudioTestData(); + + static void receive_frame(uint32_t friend_number, const int16_t *pcm, size_t sample_count, + uint8_t channels, uint32_t sampling_rate, void *user_data); +}; + +// Video Helpers +void fill_video_frame(uint16_t width, uint16_t height, int frame_index, std::vector &y, + std::vector &u, std::vector &v); +double calculate_video_mse(uint16_t width, uint16_t height, int32_t ystride, + const std::vector &y_recv, const std::vector &y_orig); + +// Video Test Data Helper +struct VideoTestData { + uint32_t friend_number = 0; + uint16_t width = 0; + uint16_t height = 0; + std::vector y, u, v; + int32_t ystride = 0, ustride = 0, vstride = 0; + + VideoTestData(); + ~VideoTestData(); + + static void receive_frame(uint32_t friend_number, uint16_t width, uint16_t height, + const uint8_t *y, const uint8_t *u, const uint8_t *v, int32_t ystride, int32_t ustride, + int32_t vstride, void *user_data); + + double calculate_mse(const std::vector &y_orig) const; +}; + +// Common Test Fixture +class AvTest : public ::testing::Test { +protected: + void SetUp() override; + void TearDown() override; + + Logger *log = nullptr; + Mono_Time *mono_time = nullptr; + MockTime tm; +}; + +#endif // TOXAV_AV_TEST_SUPPORT_H diff --git a/toxav/rtp_bench.cc b/toxav/rtp_bench.cc new file mode 100644 index 000000000..2e834ec2e --- /dev/null +++ b/toxav/rtp_bench.cc @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later + * Copyright © 2025 The TokTok team. + */ + +#include + +#include +#include + +#include "../toxcore/logger.h" +#include "../toxcore/mono_time.h" +#include "../toxcore/os_memory.h" +#include "av_test_support.hh" +#include "rtp.h" + +namespace { + +class RtpBench : public benchmark::Fixture { +public: + void SetUp(const ::benchmark::State &) override + { + const Memory *mem = os_memory(); + log = logger_new(mem); + mono_time = mono_time_new(mem, nullptr, nullptr); + + mock.store_last_packet_only = true; + + session = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &mock, nullptr, + nullptr, nullptr, &mock, RtpMock::noop_cb); + } + + void TearDown(const ::benchmark::State &) override + { + const Memory *mem = os_memory(); + rtp_kill(log, session); + mono_time_free(mem, mono_time); + logger_kill(log); + } + + Logger *log = nullptr; + Mono_Time *mono_time = nullptr; + RTPSession *session = nullptr; + RtpMock mock; +}; + +BENCHMARK_DEFINE_F(RtpBench, SendData)(benchmark::State &state) +{ + size_t data_size = static_cast(state.range(0)); + std::vector data(data_size, 0xAA); + + for (auto _ : state) { + rtp_send_data(log, session, data.data(), static_cast(data.size()), false); + benchmark::DoNotOptimize(mock.captured_packets.back()); + } +} +BENCHMARK_REGISTER_F(RtpBench, SendData)->Arg(100)->Arg(1000)->Arg(5000); + +BENCHMARK_DEFINE_F(RtpBench, ReceivePacket)(benchmark::State &state) +{ + size_t data_size = static_cast(state.range(0)); + std::vector data(data_size, 0xAA); + rtp_send_data(log, session, data.data(), static_cast(data.size()), false); + std::vector packet = mock.captured_packets.back(); + + for (auto _ : state) { + rtp_receive_packet(session, packet.data(), packet.size()); + } +} +BENCHMARK_REGISTER_F(RtpBench, ReceivePacket)->Arg(100)->Arg(1000); + +} // namespace + +BENCHMARK_MAIN(); diff --git a/toxav/video_bench.cc b/toxav/video_bench.cc new file mode 100644 index 000000000..4b376fb0b --- /dev/null +++ b/toxav/video_bench.cc @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later + * Copyright © 2025 The TokTok team. + */ + +#include + +#include + +#include "../toxcore/logger.h" +#include "../toxcore/mono_time.h" +#include "../toxcore/os_memory.h" +#include "av_test_support.hh" +#include "rtp.h" +#include "video.h" + +namespace { + +class VideoBench : public benchmark::Fixture { +public: + void SetUp(const ::benchmark::State &state) override + { + const Memory *mem = os_memory(); + log = logger_new(mem); + tm.t = 1000; + mono_time = mono_time_new(mem, mock_time_cb, &tm); + vc = vc_new(log, mono_time, 123, nullptr, nullptr); + + width = static_cast(state.range(0)); + height = static_cast(state.range(1)); + // Use a standard bitrate for benchmarks + vc_reconfigure_encoder(vc, 2000, width, height, -1); + + y.resize(static_cast(width) * height); + u.resize((static_cast(width) / 2) * (static_cast(height) / 2)); + v.resize((static_cast(width) / 2) * (static_cast(height) / 2)); + + rtp_mock.capture_packets = false; // Disable capturing for benchmarks + rtp_mock.auto_forward = true; + + rtp_mock.recv_session = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, + &rtp_mock, nullptr, nullptr, nullptr, vc, RtpMock::video_cb); + } + + void TearDown(const ::benchmark::State &state) override + { + const Memory *mem = os_memory(); + if (rtp_mock.recv_session) { + rtp_kill(log, rtp_mock.recv_session); + } + if (vc) { + vc_kill(vc); + } + if (mono_time) { + mono_time_free(mem, mono_time); + } + if (log) { + logger_kill(log); + } + } + + Logger *log = nullptr; + Mono_Time *mono_time = nullptr; + MockTime tm; + VCSession *vc = nullptr; + RtpMock rtp_mock; + uint16_t width = 0, height = 0; + std::vector y, u, v; +}; + +// Benchmark encoding a sequence of frames. +// Measures how the encoder performs as it builds up temporal state. +BENCHMARK_DEFINE_F(VideoBench, EncodeSequence)(benchmark::State &state) +{ + int frame_index = 0; + // Pre-fill frames to avoid measuring fill_frame time + const int num_prefilled = 100; + std::vector> ys(num_prefilled, std::vector(width * height)); + std::vector> us( + num_prefilled, std::vector((width / 2) * (height / 2))); + std::vector> vs( + num_prefilled, std::vector((width / 2) * (height / 2))); + for (int i = 0; i < num_prefilled; ++i) { + fill_video_frame(width, height, i, ys[i], us[i], vs[i]); + } + + for (auto _ : state) { + int idx = frame_index % num_prefilled; + // Force a keyframe every 100 frames to simulate real-world periodic keyframes + int flags = (frame_index % 100 == 0) ? VC_EFLAG_FORCE_KF : VC_EFLAG_NONE; + vc_encode(vc, width, height, ys[idx].data(), us[idx].data(), vs[idx].data(), flags); + vc_increment_frame_counter(vc); + + uint8_t *pkt_data; + uint32_t pkt_size; + bool is_keyframe; + while (vc_get_cx_data(vc, &pkt_data, &pkt_size, &is_keyframe)) { + benchmark::DoNotOptimize(pkt_data); + benchmark::DoNotOptimize(pkt_size); + } + frame_index++; + } +} + +BENCHMARK_REGISTER_F(VideoBench, EncodeSequence) + ->Args({320, 240}) + ->Args({640, 480}) + ->Args({1280, 720}) + ->Args({1920, 1080}); + +// Benchmark decoding a sequence of frames. +// First pre-encodes a sequence, then measures decoding performance. +BENCHMARK_DEFINE_F(VideoBench, DecodeSequence)(benchmark::State &state) +{ + const int num_frames = 100; + std::vector> encoded_frames(num_frames); + std::vector is_keyframe_list(num_frames); + + // Pre-encode + for (int i = 0; i < num_frames; ++i) { + fill_video_frame(width, height, i, y, u, v); + int flags = (i == 0) ? VC_EFLAG_FORCE_KF : VC_EFLAG_NONE; + vc_encode(vc, width, height, y.data(), u.data(), v.data(), flags); + vc_increment_frame_counter(vc); + + uint8_t *pkt_data; + uint32_t pkt_size; + bool is_kf; + while (vc_get_cx_data(vc, &pkt_data, &pkt_size, &is_kf)) { + encoded_frames[i].insert(encoded_frames[i].end(), pkt_data, pkt_data + pkt_size); + is_keyframe_list[i] = is_kf; + } + } + + int frame_index = 0; + for (auto _ : state) { + int idx = frame_index % num_frames; + const auto &encoded_data = encoded_frames[idx]; + rtp_send_data(log, rtp_mock.recv_session, encoded_data.data(), + static_cast(encoded_data.size()), is_keyframe_list[idx]); + vc_iterate(vc); + frame_index++; + } +} + +BENCHMARK_REGISTER_F(VideoBench, DecodeSequence) + ->Args({320, 240}) + ->Args({640, 480}) + ->Args({1280, 720}) + ->Args({1920, 1080}); + +// Full end-to-end sequence benchmark (Encode -> RTP -> Decode) +BENCHMARK_DEFINE_F(VideoBench, FullSequence)(benchmark::State &state) +{ + int frame_index = 0; + const int num_prefilled = 100; + std::vector> ys(num_prefilled, std::vector(width * height)); + std::vector> us( + num_prefilled, std::vector((width / 2) * (height / 2))); + std::vector> vs( + num_prefilled, std::vector((width / 2) * (height / 2))); + for (int i = 0; i < num_prefilled; ++i) { + fill_video_frame(width, height, i, ys[i], us[i], vs[i]); + } + + for (auto _ : state) { + int idx = frame_index % num_prefilled; + int flags = (frame_index % 100 == 0) ? VC_EFLAG_FORCE_KF : VC_EFLAG_NONE; + vc_encode(vc, width, height, ys[idx].data(), us[idx].data(), vs[idx].data(), flags); + vc_increment_frame_counter(vc); + + uint8_t *pkt_data; + uint32_t pkt_size; + bool is_keyframe = false; + // We need to collect all packets for the frame before sending to decoder + std::vector frame_data; + while (vc_get_cx_data(vc, &pkt_data, &pkt_size, &is_keyframe)) { + frame_data.insert(frame_data.end(), pkt_data, pkt_data + pkt_size); + } + + rtp_send_data(log, rtp_mock.recv_session, frame_data.data(), + static_cast(frame_data.size()), is_keyframe); + vc_iterate(vc); + + frame_index++; + } +} + +BENCHMARK_REGISTER_F(VideoBench, FullSequence) + ->Args({320, 240}) + ->Args({640, 480}) + ->Args({1280, 720}) + ->Args({1920, 1080}); + +} + +BENCHMARK_MAIN(); diff --git a/toxav/video_test.cc b/toxav/video_test.cc index cb52643a6..1aa8c1979 100644 --- a/toxav/video_test.cc +++ b/toxav/video_test.cc @@ -3,123 +3,19 @@ #include #include +#include #include #include "../toxcore/logger.h" #include "../toxcore/mono_time.h" #include "../toxcore/network.h" #include "../toxcore/os_memory.h" +#include "av_test_support.hh" #include "rtp.h" namespace { -struct VideoTimeMock { - uint64_t t; -}; - -uint64_t video_mock_time_cb(void *ud) { return static_cast(ud)->t; } - -void test_logger_cb(void *context, Logger_Level level, const char *file, uint32_t line, - const char *func, const char *message, void *userdata) -{ - (void)context; - (void)userdata; - const char *level_str = "UNKNOWN"; - switch (level) { - case LOGGER_LEVEL_TRACE: - level_str = "TRACE"; - break; - case LOGGER_LEVEL_DEBUG: - level_str = "DEBUG"; - break; - case LOGGER_LEVEL_INFO: - level_str = "INFO"; - break; - case LOGGER_LEVEL_WARNING: - level_str = "WARN"; - break; - case LOGGER_LEVEL_ERROR: - level_str = "ERROR"; - break; - } - printf("[%s] %s:%u %s: %s\n", level_str, file, line, func, message); -} - -struct VideoTestData { - uint32_t friend_number = 0; - uint16_t width = 0; - uint16_t height = 0; - std::vector y, u, v; - int32_t ystride = 0, ustride = 0, vstride = 0; - - VideoTestData(); - ~VideoTestData(); - - static void receive_frame(uint32_t friend_number, uint16_t width, uint16_t height, - const uint8_t *y, const uint8_t *u, const uint8_t *v, int32_t ystride, int32_t ustride, - int32_t vstride, void *user_data) - { - auto *self = static_cast(user_data); - self->friend_number = friend_number; - self->width = width; - self->height = height; - self->ystride = ystride; - self->ustride = ustride; - self->vstride = vstride; - - self->y.assign(y, y + static_cast(std::abs(ystride)) * height); - self->u.assign(u, u + static_cast(std::abs(ustride)) * (height / 2)); - self->v.assign(v, v + static_cast(std::abs(vstride)) * (height / 2)); - } -}; - -VideoTestData::VideoTestData() = default; -VideoTestData::~VideoTestData() = default; - -struct VideoRtpMock { - RTPSession *recv_session = nullptr; - std::vector> captured_packets; - bool auto_forward = true; - - static int send_packet(void *user_data, const uint8_t *data, uint16_t length) - { - auto *self = static_cast(user_data); - self->captured_packets.push_back(std::vector(data, data + length)); - if (self->auto_forward && self->recv_session) { - rtp_receive_packet(self->recv_session, data, length); - } - return 0; - } - - static int video_cb(const Mono_Time *mono_time, void *cs, RTPMessage *msg) - { - return vc_queue_message(mono_time, cs, msg); - } -}; - -class VideoTest : public ::testing::Test { -protected: - void SetUp() override - { - const Memory *mem = os_memory(); - log = logger_new(mem); - logger_callback_log(log, test_logger_cb, nullptr, nullptr); - tm.t = 1000; - mono_time = mono_time_new(mem, video_mock_time_cb, &tm); - mono_time_update(mono_time); - } - - void TearDown() override - { - const Memory *mem = os_memory(); - mono_time_free(mem, mono_time); - logger_kill(log); - } - - Logger *log; - Mono_Time *mono_time; - VideoTimeMock tm; -}; +using VideoTest = AvTest; TEST_F(VideoTest, BasicNewKill) { @@ -135,11 +31,11 @@ TEST_F(VideoTest, EncodeDecodeLoop) VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data); ASSERT_NE(vc, nullptr); - VideoRtpMock rtp_mock; - RTPSession *send_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, VideoRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb); - RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, VideoRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb); + RtpMock rtp_mock; + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, vc, RtpMock::video_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, vc, RtpMock::video_cb); rtp_mock.recv_session = recv_rtp; uint16_t width = 320; @@ -176,6 +72,197 @@ TEST_F(VideoTest, EncodeDecodeLoop) vc_kill(vc); } +TEST_F(VideoTest, EncodeDecodeSequence) +{ + VideoTestData data; + VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data); + ASSERT_NE(vc, nullptr); + + RtpMock rtp_mock; + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, vc, RtpMock::video_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, vc, RtpMock::video_cb); + rtp_mock.recv_session = recv_rtp; + + uint16_t width = 320; + uint16_t height = 240; + uint32_t bitrate = 2000; + + ASSERT_EQ(vc_reconfigure_encoder(vc, bitrate, width, height, -1), 0); + + for (int i = 0; i < 20; ++i) { + std::vector y(width * height); + std::vector u((width / 2) * (height / 2)); + std::vector v((width / 2) * (height / 2)); + + // Background + std::fill(y.begin(), y.end(), 16); + std::fill(u.begin(), u.end(), 128); + std::fill(v.begin(), v.end(), 128); + + // Moving square + int sq_size = 64; + int x0 = (i * 8) % (width - sq_size); + int y0 = (i * 4) % (height - sq_size); + for (int r = 0; r < sq_size; ++r) { + for (int c = 0; c < sq_size; ++c) { + y[(y0 + r) * width + (x0 + c)] = 200; + } + } + + ASSERT_EQ(vc_encode(vc, width, height, y.data(), u.data(), v.data(), + i == 0 ? VC_EFLAG_FORCE_KF : VC_EFLAG_NONE), + 0); + vc_increment_frame_counter(vc); + + uint8_t *pkt_data; + uint32_t pkt_size; + bool is_keyframe; + + while (vc_get_cx_data(vc, &pkt_data, &pkt_size, &is_keyframe)) { + rtp_send_data(log, send_rtp, pkt_data, pkt_size, is_keyframe); + } + + vc_iterate(vc); + + ASSERT_EQ(data.width, width); + ASSERT_EQ(data.height, height); + + double mse = data.calculate_mse(y); + // Expect MSE to be reasonably low for high bitrate + EXPECT_LT(mse, 100.0) << "Frame " << i << " MSE too high: " << mse; + } + + rtp_kill(log, send_rtp); + rtp_kill(log, recv_rtp); + vc_kill(vc); +} + +TEST_F(VideoTest, EncodeDecodeResolutionChange) +{ + VideoTestData data; + VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data); + ASSERT_NE(vc, nullptr); + + RtpMock rtp_mock; + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, vc, RtpMock::video_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, vc, RtpMock::video_cb); + rtp_mock.recv_session = recv_rtp; + + uint16_t widths[] = {320, 160, 480}; + uint16_t heights[] = {240, 120, 360}; + + for (int res = 0; res < 3; ++res) { + uint16_t width = widths[res]; + uint16_t height = heights[res]; + ASSERT_EQ(vc_reconfigure_encoder(vc, 2000, width, height, -1), 0); + + for (int i = 0; i < 5; ++i) { + std::vector y(width * height); + std::vector u((width / 2) * (height / 2)); + std::vector v((width / 2) * (height / 2)); + + std::fill(y.begin(), y.end(), static_cast((res * 50 + i * 10) % 256)); + std::fill(u.begin(), u.end(), 128); + std::fill(v.begin(), v.end(), 128); + + ASSERT_EQ(vc_encode(vc, width, height, y.data(), u.data(), v.data(), + i == 0 ? VC_EFLAG_FORCE_KF : VC_EFLAG_NONE), + 0); + vc_increment_frame_counter(vc); + + uint8_t *pkt_data; + uint32_t pkt_size; + bool is_keyframe; + + while (vc_get_cx_data(vc, &pkt_data, &pkt_size, &is_keyframe)) { + rtp_send_data(log, send_rtp, pkt_data, pkt_size, is_keyframe); + } + + vc_iterate(vc); + + ASSERT_EQ(data.width, width); + ASSERT_EQ(data.height, height); + + double mse = data.calculate_mse(y); + EXPECT_LT(mse, 100.0) << "Res " << res << " Frame " << i << " MSE too high: " << mse; + } + } + + rtp_kill(log, send_rtp); + rtp_kill(log, recv_rtp); + vc_kill(vc); +} + +TEST_F(VideoTest, EncodeDecodeBitrateImpact) +{ + uint32_t bitrates[] = {100, 500, 2000}; + double mses[3]; + + uint16_t width = 320; + uint16_t height = 240; + + for (int b = 0; b < 3; ++b) { + VideoTestData data; + VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data); + ASSERT_NE(vc, nullptr); + + RtpMock rtp_mock; + RTPSession *send_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, + &rtp_mock, nullptr, nullptr, nullptr, vc, RtpMock::video_cb); + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, + &rtp_mock, nullptr, nullptr, nullptr, vc, RtpMock::video_cb); + rtp_mock.recv_session = recv_rtp; + + ASSERT_EQ(vc_reconfigure_encoder(vc, bitrates[b], width, height, -1), 0); + + double total_mse = 0; + int frames = 10; + for (int i = 0; i < frames; ++i) { + std::vector y(width * height); + std::vector u((width / 2) * (height / 2)); + std::vector v((width / 2) * (height / 2)); + + for (size_t j = 0; j < y.size(); ++j) + y[j] = static_cast((j + i * 10) % 256); + std::fill(u.begin(), u.end(), 128); + std::fill(v.begin(), v.end(), 128); + + ASSERT_EQ(vc_encode(vc, width, height, y.data(), u.data(), v.data(), + i == 0 ? VC_EFLAG_FORCE_KF : VC_EFLAG_NONE), + 0); + vc_increment_frame_counter(vc); + + uint8_t *pkt_data; + uint32_t pkt_size; + bool is_keyframe; + + while (vc_get_cx_data(vc, &pkt_data, &pkt_size, &is_keyframe)) { + rtp_send_data(log, send_rtp, pkt_data, pkt_size, is_keyframe); + } + + vc_iterate(vc); + total_mse += data.calculate_mse(y); + } + + mses[b] = total_mse / frames; + + rtp_kill(log, send_rtp); + rtp_kill(log, recv_rtp); + vc_kill(vc); + } + + // Quality should generally improve (MSE decrease) as bitrate increases. + // 100kbps should have significantly higher MSE than 2000kbps for this complex pattern. + EXPECT_GT(mses[0], mses[1]); + EXPECT_GT(mses[1], mses[2]); + + printf("MSE results: 100kbps: %f, 500kbps: %f, 2000kbps: %f\n", mses[0], mses[1], mses[2]); +} + TEST_F(VideoTest, ReconfigureEncoder) { VideoTestData data; @@ -215,12 +302,12 @@ TEST_F(VideoTest, QueueInvalidMessage) VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data); ASSERT_NE(vc, nullptr); - VideoRtpMock rtp_mock; + RtpMock rtp_mock; // Create an audio RTP session but try to queue to video session - RTPSession *audio_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, VideoRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb); - RTPSession *video_recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, VideoRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb); + RTPSession *audio_rtp = rtp_new(log, RTP_TYPE_AUDIO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, vc, RtpMock::video_cb); + RTPSession *video_recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, + &rtp_mock, nullptr, nullptr, nullptr, vc, RtpMock::video_cb); rtp_mock.recv_session = video_recv_rtp; std::vector dummy_audio(100, 0); @@ -262,9 +349,9 @@ TEST_F(VideoTest, LcfdAndSpecialPackets) VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data); ASSERT_NE(vc, nullptr); - VideoRtpMock rtp_mock; - RTPSession *video_recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, VideoRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb); + RtpMock rtp_mock; + RTPSession *video_recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, + &rtp_mock, nullptr, nullptr, nullptr, vc, RtpMock::video_cb); rtp_mock.recv_session = video_recv_rtp; // 1. Test lcfd update @@ -286,8 +373,8 @@ TEST_F(VideoTest, LcfdAndSpecialPackets) EXPECT_EQ(vc_get_lcfd(vc), 50u); // Should still be 50 // 3. Test dummy packet PT = (RTP_TYPE_VIDEO + 2) % 128 - RTPSession *dummy_rtp = rtp_new(log, (RTP_TYPE_VIDEO + 2), mono_time, VideoRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb); + RTPSession *dummy_rtp = rtp_new(log, (RTP_TYPE_VIDEO + 2), mono_time, RtpMock::send_packet, + &rtp_mock, nullptr, nullptr, nullptr, vc, RtpMock::video_cb); rtp_mock.recv_session = dummy_rtp; rtp_send_data( log, dummy_rtp, dummy_frame.data(), static_cast(dummy_frame.size()), false); @@ -354,9 +441,9 @@ TEST_F(VideoTest, LyingLengthOOB) VCSession *vc = vc_new(log, mono_time, 123, VideoTestData::receive_frame, &data); ASSERT_NE(vc, nullptr); - VideoRtpMock rtp_mock; - RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, VideoRtpMock::send_packet, - &rtp_mock, nullptr, nullptr, nullptr, vc, VideoRtpMock::video_cb); + RtpMock rtp_mock; + RTPSession *recv_rtp = rtp_new(log, RTP_TYPE_VIDEO, mono_time, RtpMock::send_packet, &rtp_mock, + nullptr, nullptr, nullptr, vc, RtpMock::video_cb); rtp_mock.recv_session = recv_rtp; // Craft a malicious RTP packet