diff --git a/toxav/av_test.c b/toxav/av_test.c index 11de0138d..8fa493bf9 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -77,14 +77,13 @@ #define TEST_REJECT 0 #define TEST_CANCEL 0 #define TEST_MUTE_UNMUTE 0 -#define TEST_TRANSFER_A 0 -#define TEST_TRANSFER_V 1 +#define TEST_TRANSFER_A 1 +#define TEST_TRANSFER_V 0 typedef struct { bool incoming; uint32_t state; - uint32_t abitrate; pthread_mutex_t arb_mutex[1]; RingBuffer* arb; /* Audio ring buffer */ @@ -137,43 +136,8 @@ void t_toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool } void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, void *user_data) { + printf("Handling CALL STATE callback: %d\n", state); ((CallControl*)user_data)->state = state; - uint32_t abitrate = ((CallControl*)user_data)->abitrate; - - if (state & TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE) { - -// /* NOTE: I'm using values 8, 16, 24, 48, 64. You can use whatever OPUS supports. */ -// switch (abitrate) { -// case 8: abitrate = 16; break; -// case 16: abitrate = 24; break; -// case 24: abitrate = 48; break; -// case 48: abitrate = 64; break; -// default: return; -// } -// -// printf("Increasing bitrate to: %d\n", abitrate); -// toxav_set_audio_bit_rate(av, friend_number, abitrate, 0); - - } else if (state & TOXAV_CALL_STATE_DECREASE_AUDIO_BITRATE) { -// /* NOTE: I'm using values 8, 16, 24, 48, 64. You can use whatever OPUS supports. */ -// switch (abitrate) { -// case 16: abitrate = 8; break; -// case 24: abitrate = 16; break; -// case 48: abitrate = 24; break; -// case 64: abitrate = 48; break; -// default: return; -// } -// -// printf("Decreasing bitrate to: %d\n", abitrate); -// toxav_set_audio_bit_rate(av, friend_number, abitrate, 0); -// - } else if (state & TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE) { - - } else if (state & TOXAV_CALL_STATE_DECREASE_VIDEO_BITRATE) { - - } else { - printf("Handling CALL STATE callback: %d\n", state); - } } void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, @@ -223,6 +187,22 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, free(rb_write(cc->arb, f)); pthread_mutex_unlock(cc->arb_mutex); } +void t_toxav_audio_bitrate_control_cb(ToxAV *av, uint32_t friend_number, + bool good, uint32_t bit_rate, void *user_data) +{ + if (good) + printf ("Set new audio bitrate to: %d\n", bit_rate); + else + printf ("The network is overly saturated with audio bitrate at: %d\n", bit_rate); +} +void t_toxav_video_bitrate_control_cb(ToxAV *av, uint32_t friend_number, + bool good, uint32_t bit_rate, void *user_data) +{ + if (good) + printf ("Set new video bitrate to: %d", bit_rate); + else + printf ("The network is overly saturated with video bitrate at: %d", bit_rate); +} void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, size_t length, void *userdata) { if (length == 7 && memcmp("gentoo", data, 7) == 0) { @@ -299,12 +279,17 @@ void initialize_tox(Tox** bootstrap, ToxAV** AliceAV, CallControl* AliceCC, ToxA toxav_callback_call_state(*AliceAV, t_toxav_call_state_cb, AliceCC); toxav_callback_receive_video_frame(*AliceAV, t_toxav_receive_video_frame_cb, AliceCC); toxav_callback_receive_audio_frame(*AliceAV, t_toxav_receive_audio_frame_cb, AliceCC); + toxav_callback_audio_bitrate_control(*AliceAV, t_toxav_audio_bitrate_control_cb, AliceCC); + toxav_callback_video_bitrate_control(*AliceAV, t_toxav_video_bitrate_control_cb, AliceCC); /* Bob */ toxav_callback_call(*BobAV, t_toxav_call_cb, BobCC); toxav_callback_call_state(*BobAV, t_toxav_call_state_cb, BobCC); toxav_callback_receive_video_frame(*BobAV, t_toxav_receive_video_frame_cb, BobCC); toxav_callback_receive_audio_frame(*BobAV, t_toxav_receive_audio_frame_cb, BobCC); + toxav_callback_audio_bitrate_control(*BobAV, t_toxav_audio_bitrate_control_cb, BobCC); + toxav_callback_video_bitrate_control(*BobAV, t_toxav_video_bitrate_control_cb, BobCC); + printf("Created 2 instances of ToxAV\n"); printf("All set after %llu seconds!\n", time(NULL) - cur_time); @@ -760,11 +745,9 @@ int main (int argc, char** argv) AliceCC.arb = rb_new(16); BobCC.arb = rb_new(16); - AliceCC.abitrate = BobCC.abitrate = 48; - { /* Call */ TOXAV_ERR_CALL rc; - toxav_call(AliceAV, 0, AliceCC.abitrate, 0, &rc); + toxav_call(AliceAV, 0, 48, 0, &rc); if (rc != TOXAV_ERR_CALL_OK) { printf("toxav_call failed: %d\n", rc); @@ -777,7 +760,7 @@ int main (int argc, char** argv) { /* Answer */ TOXAV_ERR_ANSWER rc; - toxav_answer(BobAV, 0, BobCC.abitrate, 0, &rc); + toxav_answer(BobAV, 0, 48, 0, &rc); if (rc != TOXAV_ERR_ANSWER_OK) { printf("toxav_answer failed: %d\n", rc); @@ -794,7 +777,6 @@ int main (int argc, char** argv) exit(1); } - int16_t PCM[5760]; time_t start_time = time(NULL); @@ -802,7 +784,7 @@ int main (int argc, char** argv) /* Start decode thread */ - struct toxav_thread_data data = { + struct toxav_thread_data data = { .AliceAV = AliceAV, .BobAV = BobAV, .sig = 0 @@ -827,6 +809,8 @@ int main (int argc, char** argv) err = Pa_StartStream(adout); assert(err == paNoError); + assert(toxav_set_audio_bit_rate(AliceAV, 0, 64, false, NULL)); + /* Start write thread */ pthread_t t; pthread_create(&t, NULL, pa_write_thread, &BobCC); diff --git a/toxav/rtp.c b/toxav/rtp.c index a48eba20f..b16e3d733 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -95,7 +95,6 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num, vo retu->ssrc = random_int(); retu->payload_type = payload_type % 128; - retu->tstate = rtp_StateNormal; retu->m = messenger; retu->friend_id = friend_num; @@ -156,10 +155,10 @@ void rtp_kill ( RTPSession *session ) /* And finally free session */ free ( session ); } -void rtp_do(RTPSession *session) +int rtp_do(RTPSession *session) { if (!session || !session->rtcp_session) - return; + return rtp_StateNormal; if (current_time_monotonic() - session->rtcp_session->last_sent_report_ts >= RTCP_REPORT_INTERVAL_MS) { send_rtcp_report(session->rtcp_session, session->m, session->friend_id); @@ -182,7 +181,7 @@ void rtp_do(RTPSession *session) if (!rb_empty(session->rtcp_session->pl_stats)) { for (i = 0; reports[i] != NULL; i ++) free(reports[i]); - return; /* As some reports are timed out, we need more */ + return rtp_StateNormal; /* As some reports are timed out, we need more */ } /* We have 4 on-time reports so we can proceed */ @@ -194,16 +193,16 @@ void rtp_do(RTPSession *session) } if (quality <= 90) { - session->tstate = rtp_StateBad; LOGGER_WARNING("Stream quality: BAD"); + return rtp_StateBad; } else if (quality >= 99) { - session->tstate = rtp_StateGood; LOGGER_DEBUG("Stream quality: GOOD"); + return rtp_StateGood; } else { - session->tstate = rtp_StateNormal; LOGGER_DEBUG("Stream quality: NORMAL"); } } + return rtp_StateNormal; } int rtp_start_receiving(RTPSession* session) { @@ -267,7 +266,7 @@ int rtp_send_data ( RTPSession *session, const uint8_t *data, uint16_t length, b header->length = 12 /* Minimum header len */ + ( session->cc * size_32 ); uint32_t parsed_len = length + header->length + 1; - assert(parsed_len + (session->ext_header ? session->ext_header->length * size_32 : 0) > MAX_RTP_SIZE ); + assert(parsed_len + (session->ext_header ? session->ext_header->length * size_32 : 0) < MAX_RTP_SIZE ); parsed[0] = session->prefix; it = parse_header_out ( header, parsed + 1 ); @@ -485,7 +484,7 @@ int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data return -1; } - RTPHeader* header = parse_header_in ( data, length ); + RTPHeader* header = parse_header_in ( data + 1, length ); if ( !header ) { LOGGER_WARNING("Could not parse message: Header failed to extract!"); @@ -494,7 +493,7 @@ int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data RTPExtHeader* ext_header = NULL; - uint16_t from_pos = header->length; + uint16_t from_pos = header->length + 1; uint16_t msg_length = length - from_pos; if ( GET_FLAG_EXTENSION ( header ) ) { diff --git a/toxav/rtp.h b/toxav/rtp.h index fe07c4f6a..c973d2621 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -49,11 +49,11 @@ enum { rtp_TypeVideo, }; -typedef enum { +enum { rtp_StateBad = -1, rtp_StateNormal, rtp_StateGood, -} RTPTransmissionState; +}; /** * Standard rtp header. @@ -114,7 +114,6 @@ typedef struct { Messenger *m; int friend_id; - RTPTransmissionState tstate; struct RTCPSession_s *rtcp_session; @@ -136,7 +135,7 @@ void rtp_kill ( RTPSession* session ); /** * Do periodical rtp work. */ -void rtp_do(RTPSession *session); +int rtp_do(RTPSession *session); /** * By default rtp is in receiving state diff --git a/toxav/toxav.c b/toxav/toxav.c index 500ca9a8e..45b9e5e67 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -35,13 +35,13 @@ #include #define MAX_ENCODE_TIME_US ((1000 / 24) * 1000) +#define BITRATE_CHANGE_TESTING_TIME_MS 4000 typedef struct ToxAvBitrateAdapter_s { bool active; - uint32_t cycle_sent; - uint32_t cycle_ratio; - uint32_t cycle_repeat; - uint64_t start_time; + uint64_t end_time; + uint64_t next_send; + uint64_t next_send_interval; uint32_t bit_rate; } ToxAvBitrateAdapter; @@ -93,6 +93,8 @@ struct toxAV { PAIR(toxav_call_state_cb *, void *) scb; /* Call state callback */ PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ + PAIR(toxav_audio_bitrate_control_cb *, void *) abcb; /* Audio bitrate control callback */ + PAIR(toxav_video_bitrate_control_cb *, void *) vbcb; /* Video bitrate control callback */ /** Decode time measures */ int32_t dmssc; /** Measure count */ @@ -112,7 +114,6 @@ int callback_capabilites(void* toxav_inst, MSICall* call); bool audio_bitrate_invalid(uint32_t bitrate); bool video_bitrate_invalid(uint32_t bitrate); void invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state); -void qc_do(ToxAVCall* call); ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); ToxAVCall* call_get(ToxAV* av, uint32_t friend_number); void call_remove(ToxAVCall* call); @@ -120,7 +121,6 @@ bool call_prepare_transmission(ToxAVCall* call); void call_kill_transmission(ToxAVCall* call); void ba_set(ToxAvBitrateAdapter* ba, uint32_t bit_rate); bool ba_shoud_send_dummy(ToxAvBitrateAdapter* ba); -void ba_update_sent_regular(ToxAvBitrateAdapter* ba); ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) { @@ -235,13 +235,59 @@ void toxav_iterate(ToxAV* av) pthread_mutex_lock(i->mutex); pthread_mutex_unlock(av->mutex); - rtp_do(i->audio.first); ac_do(i->audio.second); + if (rtp_do(i->audio.first) < 0) { + /* Bad transmission */ + + uint32_t bb = i->audio_bit_rate; + + if (i->aba.active) { + bb = i->aba.bit_rate; + /* Stop sending dummy packets */ + memset(&i->aba, 0, sizeof(i->aba)); + } + + /* Notify app */ + if (av->abcb.first) + av->abcb.first (av, i->friend_id, false, bb, av->abcb.second); + } else if (i->aba.active && i->aba.end_time < current_time_monotonic()) { + + i->audio_bit_rate = i->aba.bit_rate; + + /* Notify user about the new bitrate */ + if (av->abcb.first) + av->abcb.first (av, i->friend_id, true, i->aba.bit_rate, av->abcb.second); + + /* Stop sending dummy packets */ + memset(&i->aba, 0, sizeof(i->aba)); + } - rtp_do(i->video.first); vc_do(i->video.second); - - qc_do(i); + if (rtp_do(i->video.first) < 0) { + /* Bad transmission */ + uint32_t bb = i->video_bit_rate; + + if (i->vba.active) { + bb = i->vba.bit_rate; + /* Stop sending dummy packets */ + memset(&i->vba, 0, sizeof(i->vba)); + } + + /* Notify app */ + if (av->vbcb.first) + av->vbcb.first (av, i->friend_id, false, bb, av->vbcb.second); + + } else if (i->vba.active && i->vba.end_time < current_time_monotonic()) { + + i->video_bit_rate = i->vba.bit_rate; + + /* Notify user about the new bitrate */ + if (av->vbcb.first) + av->vbcb.first (av, i->friend_id, true, i->vba.bit_rate, av->vbcb.second); + + /* Stop sending dummy packets */ + memset(&i->vba, 0, sizeof(i->vba)); + } if (i->msi_call->self_capabilities & msi_CapRAudio && i->msi_call->peer_capabilities & msi_CapSAudio) @@ -515,48 +561,12 @@ END: return rc == TOXAV_ERR_CALL_CONTROL_OK; } -bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, bool force, TOXAV_ERR_BIT_RATE* error) +void toxav_callback_video_bitrate_control(ToxAV* av, toxav_video_bitrate_control_cb* function, void* user_data) { - TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK; - ToxAVCall* call; - - if (m_friend_exists(av->m, friend_number) == 0) { - rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND; - goto END; - } - - if (audio_bitrate_invalid(audio_bit_rate)) { - rc = TOXAV_ERR_BIT_RATE_INVALID; - goto END; - } - pthread_mutex_lock(av->mutex); - call = call_get(av, friend_number); - if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - pthread_mutex_unlock(av->mutex); - rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; - goto END; - } - - if (call->audio_bit_rate == audio_bit_rate && call->aba.active && call->aba.bit_rate == audio_bit_rate) { - pthread_mutex_unlock(av->mutex); - goto END; - } - - pthread_mutex_lock(call->mutex); - if (force) { - call->audio_bit_rate = audio_bit_rate; - } else - ba_set(&call->aba, audio_bit_rate); - - pthread_mutex_unlock(call->mutex); + av->vbcb.first = function; + av->vbcb.second = user_data; pthread_mutex_unlock(av->mutex); - -END: - if (error) - *error = rc; - - return rc == TOXAV_ERR_BIT_RATE_OK; } bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, bool force, TOXAV_ERR_BIT_RATE* error) @@ -603,6 +613,58 @@ END: return rc == TOXAV_ERR_BIT_RATE_OK; } +void toxav_callback_audio_bitrate_control(ToxAV* av, toxav_audio_bitrate_control_cb* function, void* user_data) +{ + pthread_mutex_lock(av->mutex); + av->abcb.first = function; + av->abcb.second = user_data; + pthread_mutex_unlock(av->mutex); +} + +bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, bool force, TOXAV_ERR_BIT_RATE* error) +{ + TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK; + ToxAVCall* call; + + if (m_friend_exists(av->m, friend_number) == 0) { + rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND; + goto END; + } + + if (audio_bitrate_invalid(audio_bit_rate)) { + rc = TOXAV_ERR_BIT_RATE_INVALID; + goto END; + } + + pthread_mutex_lock(av->mutex); + call = call_get(av, friend_number); + if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { + pthread_mutex_unlock(av->mutex); + rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; + goto END; + } + + if (call->audio_bit_rate == audio_bit_rate && call->aba.active && call->aba.bit_rate == audio_bit_rate) { + pthread_mutex_unlock(av->mutex); + goto END; + } + + pthread_mutex_lock(call->mutex); + if (force) { + call->audio_bit_rate = audio_bit_rate; + } else + ba_set(&call->aba, audio_bit_rate); + + pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(av->mutex); + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_BIT_RATE_OK; +} + bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) { TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; @@ -790,7 +852,6 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc } } - ba_update_sent_regular(&call->aba); } pthread_mutex_unlock(call->mutex_audio); @@ -932,57 +993,6 @@ void invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state) av->scb.first(av, friend_number, state, av->scb.second); } -void qc_do(ToxAVCall* call) -{ - /* - switch(call->audio.first->tstate) { - case rtp_StateBad: - LOGGER_DEBUG("Suggesting lower bitrate for audio..."); - call->time_audio_good = 0; - call->last_bad_audio_bit_rate = call->audio_bit_rate; - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_DECREASE_AUDIO_BITRATE); - break; - case rtp_StateGood: - if (call->time_audio_good == 0) - call->time_audio_good = current_time_monotonic(); - else if (current_time_monotonic() - call->time_audio_good >= 30000 && - 64 > call->audio_bit_rate) - if (call->last_bad_audio_bit_rate > call->audio_bit_rate) { - if (current_time_monotonic() - call->time_audio_good >= 45000) - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE); - call->last_bad_audio_bit_rate = 0; - } else - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE); - break; - case rtp_StateNormal: - call->time_audio_good = 0; - break; - }*/ - /* - switch(call->video.first->tstate) { - case rtp_StateBad: - LOGGER_DEBUG("Suggesting lower bitrate for video..."); - call->time_video_good = 0; - call->last_bad_video_bit_rate = call->video_bit_rate; - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_DECREASE_VIDEO_BITRATE); - break; - case rtp_StateGood: - if (call->time_video_good == 0) - call->time_video_good = current_time_monotonic(); - else if (current_time_monotonic() - call->time_video_good >= 30000) - if (call->last_bad_video_bit_rate > call->video_bit_rate) { - if (current_time_monotonic() - call->time_video_good >= 45000) - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE); - call->last_bad_video_bit_rate = 0; - } else - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE); - break; - case rtp_StateNormal: - call->time_video_good = 0; - break; - }*/ -} - ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) { /* Assumes mutex locked */ @@ -1208,34 +1218,19 @@ void call_kill_transmission(ToxAVCall* call) void ba_set(ToxAvBitrateAdapter* ba, uint32_t bit_rate) { ba->bit_rate = bit_rate; - ba->start_time = current_time_monotonic(); - ba->cycle_sent = 0; - ba->cycle_repeat = 3; - ba->cycle_ratio = 4; + ba->next_send = current_time_monotonic(); + ba->end_time = ba->next_send + BITRATE_CHANGE_TESTING_TIME_MS; + ba->next_send_interval = 1000; ba->active = true; } bool ba_shoud_send_dummy(ToxAvBitrateAdapter* ba) { - if (!ba->active || ba->cycle_ratio == 0) + if (!ba->active || ba->next_send > current_time_monotonic()) return false; - if (ba->cycle_sent % ba->cycle_ratio == 0) - return true; -} - -void ba_update_sent_regular(ToxAvBitrateAdapter* ba) -{ - if (!ba->active || ba->cycle_ratio == 0) { - ba->active = false; - return; - } + ba->next_send_interval *= 0.8; + ba->next_send = current_time_monotonic() + ba->next_send_interval; - if (ba->cycle_sent % ba->cycle_ratio == 0) { - ba->cycle_sent = 0; - - if (--ba->cycle_repeat == 0) - --ba->cycle_ratio; - } else - ++ba->cycle_sent; -} + return true; +} \ No newline at end of file