From e57fb8c12ea7de1a5070ea0fc6f14c8e242c409f Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 17 Jan 2015 18:22:20 +0100 Subject: [PATCH 01/97] Progress --- toxav/msi.h | 28 +++++++++++----------- toxav/toxav.c | 64 +++++++++++++++++++++++++-------------------------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/toxav/msi.h b/toxav/msi.h index 660df05e4..29d44cccc 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -57,7 +57,7 @@ typedef enum { /** * Encoding settings. */ -typedef struct _MSICodecSettings { +typedef struct { MSICallType call_type; uint32_t video_bitrate; /* In kbits/s */ @@ -100,32 +100,32 @@ typedef enum { /** * The call struct. */ -typedef struct _MSICall { /* Call info structure */ - struct _MSISession *session; /* Session pointer */ +typedef struct { /* Call info structure */ + struct MSISession_s *session; /* Session pointer */ - MSICallState state; + MSICallState state; - MSICSettings csettings_local; /* Local call settings */ - MSICSettings *csettings_peer; /* Peers call settings */ + MSICSettings csettings_local; /* Local call settings */ + MSICSettings *csettings_peer; /* Peers call settings */ - MSICallIDType id; /* Random value identifying the call */ + MSICallIDType id; /* Random value identifying the call */ - int ringing_tout_ms; /* Ringing timeout in ms */ + int ringing_tout_ms; /* Ringing timeout in ms */ - int request_timer_id; /* Timer id for outgoing request/action */ - int ringing_timer_id; /* Timer id for ringing timeout */ + int request_timer_id; /* Timer id for outgoing request/action */ + int ringing_timer_id; /* Timer id for ringing timeout */ - uint32_t *peers; - uint16_t peer_count; + uint32_t *peers; + uint16_t peer_count; - int32_t call_idx; /* Index of this call in MSISession */ + int32_t call_idx; /* Index of this call in MSISession */ } MSICall; /** * Control session struct */ -typedef struct _MSISession { +typedef struct MSISession_s { /* Call handlers */ MSICall **calls; diff --git a/toxav/toxav.c b/toxav/toxav.c index f8605fd51..b0534ec5f 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -62,7 +62,7 @@ static const uint32_t jbuf_capacity = 6; static const uint8_t audio_index = 0, video_index = 1; typedef struct _ToxAvCall { - pthread_mutex_t mutex[1]; + pthread_mutex_t mutex_control[1]; pthread_mutex_t mutex_encoding_audio[1]; pthread_mutex_t mutex_encoding_video[1]; pthread_mutex_t mutex_do[1]; @@ -117,7 +117,7 @@ ToxAv *toxav_new( Tox *messenger, int32_t max_calls) unsigned int i; for (i = 0; i < max_calls; ++i) { - if (create_recursive_mutex(av->calls[i].mutex) != 0 ) { + if (create_recursive_mutex(av->calls[i].mutex_control) != 0 ) { LOGGER_WARNING("Failed to init call(%u) mutex!", i); msi_kill(av->msi_session); @@ -145,7 +145,7 @@ void toxav_kill ( ToxAv *av ) if ( av->calls[i].cs ) cs_kill(av->calls[i].cs); - pthread_mutex_destroy(av->calls[i].mutex); + pthread_mutex_destroy(av->calls[i].mutex_control); } msi_kill(av->msi_session); @@ -160,14 +160,14 @@ uint32_t toxav_do_interval(ToxAv *av) uint32_t rc = 200 + av->avgdectms; /* Return 200 if no call is active */ for (; i < av->max_calls; i ++) { - pthread_mutex_lock(av->calls[i].mutex); + pthread_mutex_lock(av->calls[i].mutex_control); if (av->calls[i].active) { /* This should work. Video payload will always come in greater intervals */ rc = MIN(av->calls[i].cs->audio_decoder_frame_duration, rc); } - pthread_mutex_unlock(av->calls[i].mutex); + pthread_mutex_unlock(av->calls[i].mutex_control); } return rc < av->avgdectms ? 0 : rc - av->avgdectms; @@ -182,15 +182,15 @@ void toxav_do(ToxAv *av) uint32_t i = 0; for (; i < av->max_calls; i ++) { - pthread_mutex_lock(av->calls[i].mutex); + pthread_mutex_lock(av->calls[i].mutex_control); if (av->calls[i].active) { pthread_mutex_lock(av->calls[i].mutex_do); - pthread_mutex_unlock(av->calls[i].mutex); + pthread_mutex_unlock(av->calls[i].mutex_control); cs_do(av->calls[i].cs); pthread_mutex_unlock(av->calls[i].mutex_do); } else { - pthread_mutex_unlock(av->calls[i].mutex); + pthread_mutex_unlock(av->calls[i].mutex_control); } } @@ -272,17 +272,17 @@ int toxav_prepare_transmission ( ToxAv *av, int32_t call_index, int support_vide ToxAvCall *call = &av->calls[call_index]; - pthread_mutex_lock(call->mutex); + pthread_mutex_lock(call->mutex_control); if (call->active) { - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); LOGGER_ERROR("Error while starting RTP session: call already active!\n"); return av_ErrorAlreadyInCallWithPeer; } if (pthread_mutex_init(call->mutex_encoding_audio, NULL) != 0 || pthread_mutex_init(call->mutex_encoding_video, NULL) != 0 || pthread_mutex_init(call->mutex_do, NULL) != 0) { - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); LOGGER_ERROR("Error while starting RTP session: mutex initializing failed!\n"); return av_ErrorUnknown; } @@ -312,7 +312,7 @@ int toxav_prepare_transmission ( ToxAv *av, int32_t call_index, int support_vide if ( !(call->cs = cs_new(c_self, c_peer, jbuf_capacity, support_video)) ) { LOGGER_ERROR("Error while starting Codec State!\n"); - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); return av_ErrorInitializingCodecs; } @@ -349,7 +349,7 @@ int toxav_prepare_transmission ( ToxAv *av, int32_t call_index, int support_vide } call->active = 1; - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); return av_ErrorNone; error: rtp_kill(call->crtps[audio_index], av->messenger); @@ -363,7 +363,7 @@ error: pthread_mutex_destroy(call->mutex_encoding_video); pthread_mutex_destroy(call->mutex_do); - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); return av_ErrorCreatingRtpSessions; } @@ -376,10 +376,10 @@ int toxav_kill_transmission ( ToxAv *av, int32_t call_index ) ToxAvCall *call = &av->calls[call_index]; - pthread_mutex_lock(call->mutex); + pthread_mutex_lock(call->mutex_control); if (!call->active) { - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); LOGGER_WARNING("Action on inactive call: %d", call_index); return av_ErrorInvalidState; } @@ -404,7 +404,7 @@ int toxav_kill_transmission ( ToxAv *av, int32_t call_index ) pthread_mutex_destroy(call->mutex_encoding_video); pthread_mutex_destroy(call->mutex_do); - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); return av_ErrorNone; } @@ -452,21 +452,21 @@ int toxav_prepare_video_frame ( ToxAv *av, int32_t call_index, uint8_t *dest, in ToxAvCall *call = &av->calls[call_index]; - pthread_mutex_lock(call->mutex); + pthread_mutex_lock(call->mutex_control); if (!call->active) { - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); LOGGER_WARNING("Action on inactive call: %d", call_index); return av_ErrorInvalidState; } if (cs_set_video_encoder_resolution(call->cs, input->d_w, input->d_h) < 0) { - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); return av_ErrorSettingVideoResolution; } pthread_mutex_lock(call->mutex_encoding_video); - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); int rc = vpx_codec_encode(&call->cs->v_encoder, input, call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); @@ -507,17 +507,17 @@ int toxav_send_video ( ToxAv *av, int32_t call_index, const uint8_t *frame, unsi } ToxAvCall *call = &av->calls[call_index]; - pthread_mutex_lock(call->mutex); + pthread_mutex_lock(call->mutex_control); if (!call->active) { - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); LOGGER_WARNING("Action on inactive call: %d", call_index); return av_ErrorInvalidState; } int rc = toxav_send_rtp_payload(av, call, av_TypeVideo, frame, frame_size); - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); return rc; } @@ -535,16 +535,16 @@ int toxav_prepare_audio_frame ( ToxAv *av, } ToxAvCall *call = &av->calls[call_index]; - pthread_mutex_lock(call->mutex); + pthread_mutex_lock(call->mutex_control); if (!call->active) { - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); LOGGER_WARNING("Action on inactive call: %d", call_index); return av_ErrorInvalidState; } pthread_mutex_lock(call->mutex_encoding_audio); - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); int32_t rc = opus_encode(call->cs->audio_encoder, frame, frame_size, dest, dest_max); pthread_mutex_unlock(call->mutex_encoding_audio); @@ -564,17 +564,17 @@ int toxav_send_audio ( ToxAv *av, int32_t call_index, const uint8_t *data, unsig } ToxAvCall *call = &av->calls[call_index]; - pthread_mutex_lock(call->mutex); + pthread_mutex_lock(call->mutex_control); if (!call->active) { - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); LOGGER_WARNING("Action on inactive call: %d", call_index); return av_ErrorInvalidState; } int rc = toxav_send_rtp_payload(av, call, av_TypeAudio, data, size); - pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(call->mutex_control); return rc; } @@ -624,11 +624,11 @@ int toxav_get_active_count(ToxAv *av) int rc = 0, i = 0; for (; i < av->max_calls; i++) { - pthread_mutex_lock(av->calls[i].mutex); + pthread_mutex_lock(av->calls[i].mutex_control); if (av->calls[i].active) rc++; - pthread_mutex_unlock(av->calls[i].mutex); + pthread_mutex_unlock(av->calls[i].mutex_control); } return rc; From 1450c22d01cbb5185ee8eac14657ddf3301d7e48 Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 24 Jan 2015 23:29:54 +0100 Subject: [PATCH 02/97] Current progress --- toxav/codec.c | 541 ++++++++++++++++++++++------------- toxav/codec.h | 67 +++-- toxav/msi.c | 14 +- toxav/msi.h | 4 +- toxav/rtp.c | 102 +++---- toxav/rtp.h | 8 +- toxav/toxav.c | 8 +- toxav/toxav_new.c | 710 ++++++++++++++++++++++++++++++++++++++++++++++ toxav/toxav_new.h | 481 +++++++++++++++++++++++++++++++ toxcore/logger.c | 4 +- toxcore/util.h | 1 + 11 files changed, 1649 insertions(+), 291 deletions(-) create mode 100644 toxav/toxav_new.c create mode 100644 toxav/toxav_new.h diff --git a/toxav/codec.c b/toxav/codec.c index dae35d54d..fd2f9f932 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -39,6 +40,9 @@ #include "rtp.h" #include "codec.h" + +#define DEFAULT_JBUF 6 + /* Good quality encode. */ #define MAX_ENCODE_TIME_US VPX_DL_GOOD_QUALITY #define MAX_DECODE_TIME_US 0 @@ -62,12 +66,12 @@ typedef struct { Payload **packets; } PayloadBuffer; -static _Bool buffer_full(const PayloadBuffer *b) +static bool buffer_full(const PayloadBuffer *b) { return (b->end + 1) % b->size == b->start; } -static _Bool buffer_empty(const PayloadBuffer *b) +static bool buffer_empty(const PayloadBuffer *b) { return b->end == b->start; } @@ -121,7 +125,7 @@ static void buffer_free(PayloadBuffer *b) } /* JITTER BUFFER WORK */ -typedef struct _JitterBuffer { +typedef struct { RTPMessage **queue; uint32_t size; uint32_t capacity; @@ -225,99 +229,9 @@ static RTPMessage *jbuf_read(JitterBuffer *q, int32_t *success) return NULL; } -static int init_video_decoder(CSSession *cs) -{ - int rc = vpx_codec_dec_init_ver(&cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, NULL, 0, VPX_DECODER_ABI_VERSION); - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); - return -1; - } - return 0; -} -static int init_audio_decoder(CSSession *cs) -{ - int rc; - cs->audio_decoder = opus_decoder_create(cs->audio_decoder_sample_rate, cs->audio_decoder_channels, &rc ); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc)); - return -1; - } - - return 0; -} - -static int init_video_encoder(CSSession *cs, uint16_t max_width, uint16_t max_height, uint32_t video_bitrate) -{ - vpx_codec_enc_cfg_t cfg; - int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); - - if (rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - cfg.rc_target_bitrate = video_bitrate; - cfg.g_w = max_width; - cfg.g_h = max_height; - cfg.g_pass = VPX_RC_ONE_PASS; - cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; - cfg.g_lag_in_frames = 0; - cfg.kf_min_dist = 0; - cfg.kf_max_dist = 48; - cfg.kf_mode = VPX_KF_AUTO; - - rc = vpx_codec_enc_init_ver(&cs->v_encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, VPX_ENCODER_ABI_VERSION); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - rc = vpx_codec_control(&cs->v_encoder, VP8E_SET_CPUUSED, 8); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - cs->max_width = max_width; - cs->max_height = max_height; - cs->video_bitrate = video_bitrate; - - return 0; -} - -static int init_audio_encoder(CSSession *cs) -{ - int rc = OPUS_OK; - cs->audio_encoder = opus_encoder_create(cs->audio_encoder_sample_rate, - cs->audio_encoder_channels, OPUS_APPLICATION_AUDIO, &rc); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); - return -1; - } - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(cs->audio_encoder_bitrate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); - return -1; - } - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); - return -1; - } - - return 0; -} /* PUBLIC */ int cs_split_video_payload(CSSession *cs, const uint8_t *payload, uint16_t length) @@ -406,17 +320,17 @@ void cs_do(CSSession *cs) /* Leave space for (possibly) other thread to queue more data after we read it here */ pthread_mutex_unlock(cs->queue_mutex); - rc = vpx_codec_decode(&cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); + rc = vpx_codec_decode(cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); free(p); if (rc != VPX_CODEC_OK) { LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc)); } else { vpx_codec_iter_t iter = NULL; - vpx_image_t *dest = vpx_codec_get_frame(&cs->v_decoder, &iter); + vpx_image_t *dest = vpx_codec_get_frame(cs->v_decoder, &iter); /* Play decoded images */ - for (; dest; dest = vpx_codec_get_frame(&cs->v_decoder, &iter)) { + for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) { if (cs->vcb.first) cs->vcb.first(cs->agent, cs->call_idx, dest, cs->vcb.second); @@ -430,13 +344,17 @@ void cs_do(CSSession *cs) pthread_mutex_unlock(cs->queue_mutex); } -int cs_set_video_encoder_resolution(CSSession *cs, uint16_t width, uint16_t height) +int cs_set_sending_video_resolution(CSSession *cs, uint16_t width, uint16_t height) { - vpx_codec_enc_cfg_t cfg = *cs->v_encoder.config.enc; - + if (!cs->v_encoding) + return -1; + + /* TODO FIXME reference is safe? */ + vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc; + if (cfg.g_w == width && cfg.g_h == height) return 0; - +/* if (width * height > cs->max_width * cs->max_height) { vpx_codec_ctx_t v_encoder = cs->v_encoder; @@ -447,12 +365,12 @@ int cs_set_video_encoder_resolution(CSSession *cs, uint16_t width, uint16_t heig vpx_codec_destroy(&v_encoder); return 0; - } + }*/ LOGGER_DEBUG("New video resolution: %u %u", width, height); cfg.g_w = width; cfg.g_h = height; - int rc = vpx_codec_enc_config_set(&cs->v_encoder, &cfg); + int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg); if ( rc != VPX_CODEC_OK) { LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); @@ -462,27 +380,107 @@ int cs_set_video_encoder_resolution(CSSession *cs, uint16_t width, uint16_t heig return 0; } -int cs_set_video_encoder_bitrate(CSSession *cs, uint32_t video_bitrate) +int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate) { - vpx_codec_enc_cfg_t cfg = *cs->v_encoder.config.enc; - - if (cfg.rc_target_bitrate == video_bitrate) + if (!cs->v_encoding) + return -1; + + /* TODO FIXME reference is safe? */ + vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc; + if (cfg.rc_target_bitrate == bitrate) return 0; LOGGER_DEBUG("New video bitrate: %u", video_bitrate); - cfg.rc_target_bitrate = video_bitrate; - int rc = vpx_codec_enc_config_set(&cs->v_encoder, &cfg); - + cfg.rc_target_bitrate = bitrate; + + int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg); if ( rc != VPX_CODEC_OK) { LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); return cs_ErrorSettingVideoBitrate; } - cs->video_bitrate = video_bitrate; return 0; } -CSSession *cs_new(const ToxAvCSettings *cs_self, const ToxAvCSettings *cs_peer, uint32_t jbuf_size, int has_video) +int cs_set_sending_audio_bitrate(CSSession *cs, int32_t rate) +{ + if (cs->audio_encoder == NULL) + return -1; + + int rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(rate)); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); + return -1; + } + + return 0; +} + +int cs_set_sending_audio_sampling_rate(CSSession* cs, int32_t rate) +{ + /* TODO Find a better way? */ + if (cs->audio_encoder == NULL) + return -1; + + int rc = OPUS_OK; + int last_rate = 0; + + rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(&last_rate)); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while getting encoder ctl: %s", opus_strerror(rc)); + return -1; + } + + if (rate == last_rate) + return 0; + + OpusEncoder* new_enc = opus_encoder_create(rate, cs->channels, OPUS_APPLICATION_AUDIO, &rc); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); + return -1; + } + + opus_encoder_destroy(cs->audio_encoder); + cs->audio_encoder = new_enc; + return 0; +} + +int cs_set_sending_audio_channels(CSSession* cs, int32_t count) +{ + /* TODO Find a better way? */ + if (cs->audio_encoder == NULL) + return -1; + + if (cs->channels == count) + return 0; + + int rc = OPUS_OK; + int bitrate = 0; + + rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(&bitrate)); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while getting encoder ctl: %s", opus_strerror(rc)); + return -1; + } + + cs->channels = count; + OpusEncoder* new_enc = opus_encoder_create(bitrate, cs->channels, OPUS_APPLICATION_AUDIO, &rc); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); + return -1; + } + + opus_encoder_destroy(cs->audio_encoder); + cs->audio_encoder = new_enc; + return 0; +} + +CSSession *cs_new(uint32_t s_audio_b, uint32_t p_audio_b, uint32_t s_video_b, uint32_t p_video_b) { CSSession *cs = calloc(sizeof(CSSession), 1); @@ -491,75 +489,40 @@ CSSession *cs_new(const ToxAvCSettings *cs_self, const ToxAvCSettings *cs_peer, return NULL; } - if (create_recursive_mutex(cs->queue_mutex) != 0) { - LOGGER_WARNING("Failed to create recursive mutex!"); - free(cs); - return NULL; + /* TODO this has to be exchanged in msi */ + cs->max_video_frame_size = MAX_VIDEOFRAME_SIZE; + cs->video_frame_piece_size = VIDEOFRAME_PIECE_SIZE; + + if (s_audio_b > 0 && 0 != cs_enable_audio_sending(cs, s_audio_b)) { /* Sending audio enabled */ + LOGGER_WARNING("Failed to enable audio sending!"); + goto FAILURE; } - - if ( !(cs->j_buf = jbuf_new(jbuf_size)) ) { - LOGGER_WARNING("Jitter buffer creaton failed!"); - goto error; + + if (p_audio_b > 0 && 0 != cs_enable_audio_receiving(cs)) { /* Receiving audio enabled */ + LOGGER_WARNING("Failed to enable audio receiving!"); + goto FAILURE; } - - cs->audio_encoder_bitrate = cs_self->audio_bitrate; - cs->audio_encoder_sample_rate = cs_self->audio_sample_rate; - cs->audio_encoder_channels = cs_self->audio_channels; - cs->audio_encoder_frame_duration = cs_self->audio_frame_duration; - - cs->audio_decoder_bitrate = cs_peer->audio_bitrate; - cs->audio_decoder_sample_rate = cs_peer->audio_sample_rate; - cs->audio_decoder_channels = cs_peer->audio_channels; - cs->audio_decoder_frame_duration = cs_peer->audio_frame_duration; - - - cs->capabilities |= ( 0 == init_audio_encoder(cs) ) ? cs_AudioEncoding : 0; - cs->capabilities |= ( 0 == init_audio_decoder(cs) ) ? cs_AudioDecoding : 0; - - if ( !(cs->capabilities & cs_AudioEncoding) || !(cs->capabilities & cs_AudioDecoding) ) goto error; - - if ((cs->support_video = has_video)) { - cs->max_video_frame_size = MAX_VIDEOFRAME_SIZE; - cs->video_frame_piece_size = VIDEOFRAME_PIECE_SIZE; - - cs->capabilities |= ( 0 == init_video_encoder(cs, cs_self->max_video_width, - cs_self->max_video_height, cs_self->video_bitrate) ) ? cs_VideoEncoding : 0; - cs->capabilities |= ( 0 == init_video_decoder(cs) ) ? cs_VideoDecoding : 0; - - if ( !(cs->capabilities & cs_VideoEncoding) || !(cs->capabilities & cs_VideoDecoding) ) goto error; - - if ( !(cs->frame_buf = calloc(cs->max_video_frame_size, 1)) ) goto error; - - if ( !(cs->split_video_frame = calloc(cs->video_frame_piece_size + VIDEOFRAME_HEADER_SIZE, 1)) ) - goto error; - - if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) ) goto error; + + if (s_video_b > 0 && 0 != cs_enable_video_sending(cs, s_video_b)) { /* Sending video enabled */ + LOGGER_WARNING("Failed to enable video sending!"); + goto FAILURE; + } + + if (p_video_b > 0 && 0 != cs_enable_video_receiving(cs)) { /* Receiving video enabled */ + LOGGER_WARNING("Failed to enable video receiving!"); + goto FAILURE; } return cs; -error: +FAILURE: LOGGER_WARNING("Error initializing codec session! Application might misbehave!"); - pthread_mutex_destroy(cs->queue_mutex); - - if ( cs->audio_encoder ) opus_encoder_destroy(cs->audio_encoder); - - if ( cs->audio_decoder ) opus_decoder_destroy(cs->audio_decoder); - - - if (has_video) { - if ( cs->capabilities & cs_VideoDecoding ) vpx_codec_destroy(&cs->v_decoder); - - if ( cs->capabilities & cs_VideoEncoding ) vpx_codec_destroy(&cs->v_encoder); - - buffer_free(cs->vbuf_raw); - - free(cs->frame_buf); - free(cs->split_video_frame); - } - - jbuf_free(cs->j_buf); + cs_disable_audio_sending(cs); + cs_disable_audio_receiving(cs); + cs_disable_video_sending(cs); + cs_disable_video_receiving(cs); + free(cs); return NULL; @@ -567,32 +530,226 @@ error: void cs_kill(CSSession *cs) { - if (!cs) return; + if (!cs) + return; - /* queue_message will not be called since it's unregistered before cs_kill is called */ - pthread_mutex_destroy(cs->queue_mutex); - - - if ( cs->audio_encoder ) - opus_encoder_destroy(cs->audio_encoder); - - if ( cs->audio_decoder ) - opus_decoder_destroy(cs->audio_decoder); - - if ( cs->capabilities & cs_VideoDecoding ) - vpx_codec_destroy(&cs->v_decoder); - - if ( cs->capabilities & cs_VideoEncoding ) - vpx_codec_destroy(&cs->v_encoder); - - jbuf_free(cs->j_buf); - buffer_free(cs->vbuf_raw); - free(cs->frame_buf); + /* NOTE: queue_message() will not be called since + * the callback is unregistered before cs_kill is called. + */ + + cs_disable_audio_sending(cs); + cs_disable_audio_receiving(cs); + cs_disable_video_sending(cs); + cs_disable_video_receiving(cs); LOGGER_DEBUG("Terminated codec state: %p", cs); free(cs); } +int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate) +{ + if (cs->audio_encoder) + return 0; + + /** + * Encoder is initialized with default values. These values (Sampling rate, channel count) + * change on the fly from toxav. + */ + + int rc = OPUS_OK; + cs->audio_encoder = opus_encoder_create(48000, 2, OPUS_APPLICATION_AUDIO, &rc); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); + return -1; + } + + rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate)); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); + goto FAILURE; + } + + rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10)); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); + goto FAILURE; + } + + cs->channels = 2; + return 0; + +FAILURE: + cs_disable_audio_sending(cs); + return -1; +} + +int cs_enable_audio_receiving(CSSession* cs) +{ + if (cs->audio_decoder) + return 0; + + /** + * Decoder is initialized with default values. These values (Sampling rate, channel count) + * change on the fly from toxav. + */ + + int rc; + cs->audio_decoder = opus_decoder_create(48000, 2, &rc ); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc)); + return -1; + } + + + if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) { + LOGGER_WARNING("Jitter buffer creaton failed!"); + opus_decoder_destroy(cs->audio_decoder); + cs->audio_decoder = NULL; + return -1; + } + + + return 0; +} + +int cs_enable_video_sending(CSSession* cs, uint32_t bitrate) +{ + if (cs->v_encoding) + return 0; + + vpx_codec_enc_cfg_t cfg; + int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); + + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); + return -1; + } + + rc = vpx_codec_enc_init_ver(cs->v_encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, + VPX_ENCODER_ABI_VERSION); + + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); + return -1; + } + + /* So that we can use cs_disable_video_sending to clean up */ + cs->v_encoding = true; + + if ( !(cs->split_video_frame = calloc(cs->video_frame_piece_size + VIDEOFRAME_HEADER_SIZE, 1)) ) + goto FAILURE; + + cfg.rc_target_bitrate = bitrate; + cfg.g_w = 800; + cfg.g_h = 600; + cfg.g_pass = VPX_RC_ONE_PASS; + cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; + cfg.g_lag_in_frames = 0; + cfg.kf_min_dist = 0; + cfg.kf_max_dist = 48; + cfg.kf_mode = VPX_KF_AUTO; + + + rc = vpx_codec_control(cs->v_encoder, VP8E_SET_CPUUSED, 8); + + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); + goto FAILURE; + } + + return 0; + +FAILURE: + cs_disable_video_sending(cs); + return -1; +} + +int cs_enable_video_receiving(CSSession* cs) +{ + if (cs->v_decoding) + return 0; + + if (create_recursive_mutex(cs->queue_mutex) != 0) { + LOGGER_WARNING("Failed to create recursive mutex!"); + return -1; + } + + int rc = vpx_codec_dec_init_ver(cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, + NULL, 0, VPX_DECODER_ABI_VERSION); + + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); + + pthread_mutex_destroy(cs->queue_mutex); + return -1; + } + + /* So that we can use cs_disable_video_sending to clean up */ + cs->v_decoding = true; + + if ( !(cs->frame_buf = calloc(cs->max_video_frame_size, 1)) ) + goto FAILURE; + + if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) ) + goto FAILURE; + + return 0; + +FAILURE: + cs_disable_video_receiving(cs); + return -1; +} + +void cs_disable_audio_sending(CSSession* cs) +{ + if ( cs->audio_encoder ) { + opus_encoder_destroy(cs->audio_encoder); + cs->audio_encoder = NULL; + cs->channels = 0; + } +} + +void cs_disable_audio_receiving(CSSession* cs) +{ + if ( cs->audio_decoder ) { + opus_decoder_destroy(cs->audio_decoder); + cs->audio_decoder = NULL; + jbuf_free(cs->j_buf); + cs->j_buf = NULL; + } +} + +void cs_disable_video_sending(CSSession* cs) +{ + if (cs->v_encoding) { + cs->v_encoding = false; + + free(cs->split_video_frame); + cs->split_video_frame = NULL; + + vpx_codec_destroy(cs->v_encoder); + } +} + +void cs_disable_video_receiving(CSSession* cs) +{ + if (cs->v_decoding) { + cs->v_decoding = false; + + buffer_free(cs->vbuf_raw); + cs->vbuf_raw = NULL; + free(cs->frame_buf); + cs->frame_buf = NULL; + + vpx_codec_destroy(cs->v_decoder); + pthread_mutex_destroy(cs->queue_mutex); + } +} + diff --git a/toxav/codec.h b/toxav/codec.h index 6018e5df6..92262ef83 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -75,18 +75,16 @@ typedef struct _CSSession { * * */ - int support_video; /* video encoding */ - vpx_codec_ctx_t v_encoder; + vpx_codec_ctx_t v_encoder[1]; + bool v_encoding; uint32_t frame_counter; /* video decoding */ - vpx_codec_ctx_t v_decoder; - int max_width; - int max_height; - unsigned int video_bitrate; - + vpx_codec_ctx_t v_decoder[1]; + bool v_decoding; + void *vbuf_raw; /* Un-decoded data */ /* Data handling */ uint8_t *frame_buf; /* buffer for split video payloads */ @@ -112,18 +110,10 @@ typedef struct _CSSession { /* audio encoding */ OpusEncoder *audio_encoder; - int audio_encoder_bitrate; - int audio_encoder_sample_rate; - int audio_encoder_frame_duration; - int audio_encoder_channels; - + int32_t channels; + /* audio decoding */ OpusDecoder *audio_decoder; - int audio_decoder_bitrate; - int audio_decoder_sample_rate; - int audio_decoder_frame_duration; - int audio_decoder_channels; - struct _JitterBuffer *j_buf; @@ -138,25 +128,16 @@ typedef struct _CSSession { * */ - uint64_t capabilities; /* supports*/ - /* Callbacks */ PAIR(CSAudioCallback, void *) acb; PAIR(CSVideoCallback, void *) vcb; - /* Buffering */ - void *vbuf_raw; /* Un-decoded data */ - pthread_mutex_t queue_mutex[1]; - void *agent; /* Pointer to ToxAv */ int32_t call_idx; + + pthread_mutex_t queue_mutex[1]; } CSSession; -/* Make sure to be called BEFORE corresponding rtp_new */ -CSSession *cs_new(const ToxAvCSettings *cs_self, const ToxAvCSettings *cs_peer, uint32_t jbuf_size, int has_video); -/* Make sure to be called AFTER corresponding rtp_kill */ -void cs_kill(CSSession *cs); - int cs_split_video_payload(CSSession *cs, const uint8_t *payload, uint16_t length); const uint8_t *cs_get_split_video_frame(CSSession *cs, uint16_t *size); @@ -165,11 +146,35 @@ const uint8_t *cs_get_split_video_frame(CSSession *cs, uint16_t *size); */ void cs_do(CSSession *cs); +/** + * Reconfigure video settings; return 0 on success or -1 on failure. + */ +int cs_set_sending_video_resolution(CSSession *cs, uint16_t width, uint16_t height); +int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate); -/* Reconfigure video encoder; return 0 on success or -1 on failure. */ -int cs_set_video_encoder_resolution(CSSession *cs, uint16_t width, uint16_t height); -int cs_set_video_encoder_bitrate(CSSession *cs, uint32_t video_bitrate); +int cs_set_sending_audio_bitrate(CSSession* cs, int32_t rate); +/* NOTE: Try not to call these a lot */ +int cs_set_sending_audio_sampling_rate(CSSession* cs, int32_t rate); +int cs_set_sending_audio_channels(CSSession* cs, int32_t count); +/** + * Make sure to be called BEFORE corresponding rtp_new + */ +CSSession *cs_new(uint32_t s_audio_b, uint32_t p_audio_b, uint32_t s_video_b, uint32_t p_video_b); +/** + * Make sure to be called AFTER corresponding rtp_kill + */ +void cs_kill(CSSession *cs); + +int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate); +int cs_enable_audio_receiving(CSSession* cs); +int cs_enable_video_sending(CSSession* cs, uint32_t bitrate); +int cs_enable_video_receiving(CSSession* cs); + +void cs_disable_audio_sending(CSSession* cs); +void cs_disable_audio_receiving(CSSession* cs); +void cs_disable_video_sending(CSSession* cs); +void cs_disable_video_receiving(CSSession* cs); /* Internal. Called from rtp_handle_message */ void queue_message(RTPSession *session, RTPMessage *msg); diff --git a/toxav/msi.c b/toxav/msi.c index 497af13b4..3de686ccc 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -840,7 +840,7 @@ static int handle_recv_invite ( MSISession *session, MSICall *call, MSIMessage * if ( call ) { if ( call->peers[0] == (uint32_t)msg->friend_id ) { - if (call->state == msi_CallInviting) { + if (call->state == msi_CallRequesting) { /* The glare case. A calls B when at the same time * B calls A. Who has advantage is set bey calculating * 'bigger' Call id and then that call id is being used in @@ -898,7 +898,7 @@ static int handle_recv_invite ( MSISession *session, MSICall *call, MSIMessage * } memcpy ( call->id, msg->callid.value, sizeof(msg->callid.value) ); - call->state = msi_CallStarting; + call->state = msi_CallRequested; add_peer( call, msg->friend_id); flush_peer_csettings ( call, msg, 0 ); @@ -1009,7 +1009,7 @@ static int handle_recv_starting ( MSISession *session, MSICall *call, MSIMessage invoke_callback(session, call->call_idx, msi_OnSelfCSChange); - } else if ( call->state == msi_CallInviting ) { + } else if ( call->state == msi_CallRequesting ) { LOGGER_DEBUG("Session: %p Handling 'starting' on call: %d", session, call->call_idx ); call->state = msi_CallActive; @@ -1344,7 +1344,7 @@ int msi_invite ( MSISession *session, send_message ( session, call, msg_invite, friend_id ); free( msg_invite ); - call->state = msi_CallInviting; + call->state = msi_CallRequesting; call->request_timer_id = timer_alloc ( session, handle_timeout, call->call_idx, m_deftout ); @@ -1402,7 +1402,7 @@ int msi_answer ( MSISession *session, int32_t call_index, const MSICSettings *cs return msi_ErrorNoCall; } - if ( session->calls[call_index]->state != msi_CallStarting ) { + if ( session->calls[call_index]->state != msi_CallRequested ) { LOGGER_ERROR("Call is in invalid state!"); pthread_mutex_unlock(session->mutex); return msi_ErrorInvalidState; @@ -1434,7 +1434,7 @@ int msi_cancel ( MSISession *session, int32_t call_index, uint32_t peer, const c return msi_ErrorNoCall; } - if ( session->calls[call_index]->state != msi_CallInviting ) { + if ( session->calls[call_index]->state != msi_CallRequesting ) { LOGGER_ERROR("Call is in invalid state!"); pthread_mutex_unlock(session->mutex); return msi_ErrorInvalidState; @@ -1477,7 +1477,7 @@ int msi_reject ( MSISession *session, int32_t call_index, const char *reason ) return msi_ErrorNoCall; } - if ( session->calls[call_index]->state != msi_CallStarting ) { + if ( session->calls[call_index]->state != msi_CallRequested ) { LOGGER_ERROR("Call is in invalid state!"); pthread_mutex_unlock(session->mutex); return msi_ErrorInvalidState; diff --git a/toxav/msi.h b/toxav/msi.h index 29d44cccc..bdd72e490 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -45,8 +45,8 @@ typedef enum { * Call state identifiers. */ typedef enum { - msi_CallInviting, /* when sending call invite */ - msi_CallStarting, /* when getting call invite */ + msi_CallRequesting, /* when sending call invite */ + msi_CallRequested, /* when getting call invite */ msi_CallActive, msi_CallHold, msi_CallOver diff --git a/toxav/rtp.c b/toxav/rtp.c index 2af89ebfb..ba93e7817 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -422,44 +422,6 @@ RTPMessage *rtp_new_message ( RTPSession *session, const uint8_t *data, uint32_t -int rtp_send_msg ( RTPSession *session, Messenger *messenger, const uint8_t *data, uint16_t length ) -{ - RTPMessage *msg = rtp_new_message (session, data, length); - - if ( !msg ) return -1; - - if ( -1 == send_custom_lossy_packet(messenger, session->dest, msg->data, msg->length) ) { - LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno)); - rtp_free_msg ( session, msg ); - return rtp_ErrorSending; - } - - - /* Set sequ number */ - session->sequnum = session->sequnum >= MAX_SEQU_NUM ? 0 : session->sequnum + 1; - rtp_free_msg ( session, msg ); - - return 0; -} - -void rtp_free_msg ( RTPSession *session, RTPMessage *msg ) -{ - if ( !session ) { - if ( msg->ext_header ) { - free ( msg->ext_header->table ); - free ( msg->ext_header ); - } - } else { - if ( msg->ext_header && session->ext_header != msg->ext_header ) { - free ( msg->ext_header->table ); - free ( msg->ext_header ); - } - } - - free ( msg->header ); - free ( msg ); -} - RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ) { RTPSession *retu = calloc(1, sizeof(RTPSession)); @@ -469,16 +431,10 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ) return NULL; } - if ( -1 == custom_lossy_packet_registerhandler(messenger, friend_num, payload_type, rtp_handle_packet, retu)) { - LOGGER_ERROR("Error setting custom register handler for rtp session"); - free(retu); - return NULL; - } - LOGGER_DEBUG("Registered packet handler: pt: %d; fid: %d", payload_type, friend_num); - retu->version = RTP_VERSION; /* It's always 2 */ - retu->padding = 0; /* If some additional data is needed about the packet */ + retu->version = RTP_VERSION; /* It's always 2 */ + retu->padding = 0; /* If some additional data is needed about the packet */ retu->extension = 0; /* If extension to header is needed */ retu->cc = 1; /* Amount of contributors */ retu->csrc = NULL; /* Container */ @@ -498,23 +454,24 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ) free(retu); return NULL; } - + retu->csrc[0] = retu->ssrc; /* Set my ssrc to the list receive */ /* Also set payload type as prefix */ retu->prefix = payload_type; - + + retu->m = messenger; /* * */ return retu; } -void rtp_kill ( RTPSession *session, Messenger *messenger ) +void rtp_kill ( RTPSession *session ) { if ( !session ) return; - custom_lossy_packet_registerhandler(messenger, session->dest, session->prefix, NULL, NULL); + custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, NULL, NULL); free ( session->ext_header ); free ( session->csrc ); @@ -523,5 +480,48 @@ void rtp_kill ( RTPSession *session, Messenger *messenger ) /* And finally free session */ free ( session ); - +} + +int rtp_register_for_receiving(RTPSession* session) +{ + return custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, + rtp_handle_packet, session); +} + +int rtp_send_msg ( RTPSession *session, Messenger *messenger, const uint8_t *data, uint16_t length ) +{ + RTPMessage *msg = rtp_new_message (session, data, length); + + if ( !msg ) return -1; + + if ( -1 == send_custom_lossy_packet(messenger, session->dest, msg->data, msg->length) ) { + LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno)); + rtp_free_msg ( session, msg ); + return rtp_ErrorSending; + } + + + /* Set sequ number */ + session->sequnum = session->sequnum >= MAX_SEQU_NUM ? 0 : session->sequnum + 1; + rtp_free_msg ( session, msg ); + + return 0; +} + +void rtp_free_msg ( RTPSession *session, RTPMessage *msg ) +{ + if ( !session ) { + if ( msg->ext_header ) { + free ( msg->ext_header->table ); + free ( msg->ext_header ); + } + } else { + if ( msg->ext_header && session->ext_header != msg->ext_header ) { + free ( msg->ext_header->table ); + free ( msg->ext_header ); + } + } + + free ( msg->header ); + free ( msg ); } diff --git a/toxav/rtp.h b/toxav/rtp.h index c98840acc..b25b13ba1 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -99,6 +99,7 @@ typedef struct _RTPSession { int dest; struct _CSSession *cs; + Messenger* m; } RTPSession; @@ -110,7 +111,12 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ); /** * Terminate the session. */ -void rtp_kill ( RTPSession *session, Messenger *messenger ); +void rtp_kill ( RTPSession* session ); + +/** + * By default rtp is not in receiving state + */ +int rtp_register_for_receiving (RTPSession *session); /** * Sends msg to _RTPSession::dest diff --git a/toxav/toxav.c b/toxav/toxav.c index b0534ec5f..684020205 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -460,7 +460,7 @@ int toxav_prepare_video_frame ( ToxAv *av, int32_t call_index, uint8_t *dest, in return av_ErrorInvalidState; } - if (cs_set_video_encoder_resolution(call->cs, input->d_w, input->d_h) < 0) { + if (cs_set_sending_video_resolution(call->cs, input->d_w, input->d_h) < 0) { pthread_mutex_unlock(call->mutex_control); return av_ErrorSettingVideoResolution; } @@ -468,7 +468,7 @@ int toxav_prepare_video_frame ( ToxAv *av, int32_t call_index, uint8_t *dest, in pthread_mutex_lock(call->mutex_encoding_video); pthread_mutex_unlock(call->mutex_control); - int rc = vpx_codec_encode(&call->cs->v_encoder, input, call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); + int rc = vpx_codec_encode(call->cs->v_encoder, input, call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); if ( rc != VPX_CODEC_OK) { LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(rc)); @@ -482,7 +482,7 @@ int toxav_prepare_video_frame ( ToxAv *av, int32_t call_index, uint8_t *dest, in const vpx_codec_cx_pkt_t *pkt; int copied = 0; - while ( (pkt = vpx_codec_get_cx_data(&call->cs->v_encoder, &iter)) ) { + while ( (pkt = vpx_codec_get_cx_data(call->cs->v_encoder, &iter)) ) { if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { if ( copied + pkt->data.frame.sz > dest_max ) { pthread_mutex_unlock(call->mutex_encoding_video); @@ -608,8 +608,6 @@ ToxAvCallState toxav_get_call_state(ToxAv *av, int32_t call_index) int toxav_capability_supported ( ToxAv *av, int32_t call_index, ToxAvCapabilities capability ) { - return av->calls[call_index].cs ? av->calls[call_index].cs->capabilities & (CSCapabilities) capability : 0; - /* 0 is error here */ } Tox *toxav_get_tox(ToxAv *av) diff --git a/toxav/toxav_new.c b/toxav/toxav_new.c new file mode 100644 index 000000000..d6c1872c8 --- /dev/null +++ b/toxav/toxav_new.c @@ -0,0 +1,710 @@ +/** toxav.c + * + * Copyright (C) 2013 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif /* HAVE_CONFIG_H */ + +#include "toxav_new.h" +#include "msi.h" /* Includes codec.h and rtp.h */ + +#include "../toxcore/Messenger.h" +#include "../toxcore/logger.h" +#include "../toxcore/util.h" + +#include +#include +#include + + +enum { + audio_index, + video_index, +}; + +typedef struct iToxAVCall +{ + pthread_mutex_t mutex_control[1]; + pthread_mutex_t mutex_encoding_audio[1]; + pthread_mutex_t mutex_encoding_video[1]; + pthread_mutex_t mutex_do[1]; + RTPSession *rtps[2]; /** Audio is first and video is second */ + CSSession *cs; + bool active; + int32_t friend_number; + int32_t call_idx; /* FIXME msi compat, remove */ + + struct iToxAVCall *prev; + struct iToxAVCall *next; +} IToxAVCall; + +struct toxAV +{ + Messenger* m; + MSISession* msi; + + /* Two-way storage: first is array of calls and second is list of calls with head and tail */ + IToxAVCall** calls; + uint32_t calls_tail; + uint32_t calls_head; + + PAIR(toxav_call_cb *, void*) ccb; /* Call callback */ + PAIR(toxav_call_state_cb *, void *) scb; /* Call state callback */ + PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ + PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ + + /** Decode time measures */ + int32_t dmssc; /** Measure count */ + int32_t dmsst; /** Last cycle total */ + int32_t dmssa; /** Average decoding time in ms */ +}; + + +void i_toxav_msi_callback_invite(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_ringing(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_start(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_cancel(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_reject(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_end(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_request_to(void* toxav_inst, int32_t call_idx, void *data); /* TODO remove */ +void i_toxav_msi_callback_peer_to(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_state_change(void* toxav_inst, int32_t call_idx, void *data); + +IToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number); +IToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number); +void i_toxav_remove_call(ToxAV* av, uint32_t friend_number); +bool i_toxav_audio_bitrate_invalid(uint32_t bitrate); +bool i_toxav_video_bitrate_invalid(uint32_t bitrate); +IToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error); +bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call); +void i_toxav_kill_transmission(ToxAV* av, IToxAVCall* call); + + + +ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) +{ + TOXAV_ERR_NEW rc = TOXAV_ERR_NEW_OK; + ToxAV *av = NULL; + + if (tox == NULL) { + rc = TOXAV_ERR_NEW_NULL; + goto FAILURE; + } + + if (((Messenger*)tox)->msi_packet) { + rc = TOXAV_ERR_NEW_MULTIPLE; + goto FAILURE; + } + + av = calloc ( sizeof(ToxAV), 1); + + if (av == NULL) { + LOGGER_WARNING("Allocation failed!"); + rc = TOXAV_ERR_NEW_MALLOC; + goto FAILURE; + } + + av->m = (Messenger *)tox; + av->msi = msi_new(av->m, 100); /* TODO remove max calls */ + + if (av->msi == NULL) { + rc = TOXAV_ERR_NEW_MALLOC; + goto FAILURE; + } + + av->msi->agent_handler = av; + + msi_register_callback(av->msi, i_toxav_msi_callback_invite, msi_OnInvite, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_ringing, msi_OnRinging, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_start, msi_OnStart, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_cancel, msi_OnCancel, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_reject, msi_OnReject, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_end, msi_OnEnd, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_request_to, msi_OnRequestTimeout, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_peer_to, msi_OnPeerTimeout, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_state_change, msi_OnPeerCSChange, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_state_change, msi_OnSelfCSChange, NULL); + + + if (error) + *error = rc; + + return av; + +FAILURE: + if (error) + *error = rc; + + free(av); + + return NULL; +} + +void toxav_kill(ToxAV* av) +{ + if (av == NULL) + return; + + msi_kill(av->msi); + /* TODO iterate over calls */ + free(av); +} + +Tox* toxav_get_tox(ToxAV* av) +{ + return (Tox*) av->m; +} + +uint32_t toxav_iteration_interval(const ToxAV* av) +{ + +} + +void toxav_iteration(ToxAV* av) +{ + +} + +bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) +{ + IToxAVCall* call = i_toxav_init_call(av, friend_number, audio_bit_rate, video_bit_rate, error); + if (call == NULL) { + return false; + } + + /* TODO remove csettings */ + MSICSettings csets; + csets.audio_bitrate = audio_bit_rate; + csets.video_bitrate = video_bit_rate; + + csets.call_type = video_bit_rate ? msi_TypeVideo : msi_TypeAudio; + + if (msi_invite(av->msi, &call->call_idx, &csets, 1000, friend_number) != 0) { + i_toxav_remove_call(av, friend_number); + if (error) + *error = TOXAV_ERR_CALL_MALLOC; /* FIXME: this should be the only reason to fail */ + return false; + } + + return true; +} + +void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data) +{ + av->ccb.first = function; + av->ccb.second = user_data; +} + +bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER* error) +{ + TOXAV_ERR_ANSWER rc = TOXAV_ERR_ANSWER_OK; + if (m_friend_exists(av->m, friend_number)) { + rc = TOXAV_ERR_ANSWER_FRIEND_NOT_FOUND; + goto END; + } + + if ((audio_bit_rate && i_toxav_audio_bitrate_invalid(audio_bit_rate)) + ||(video_bit_rate && i_toxav_video_bitrate_invalid(video_bit_rate)) + ) { + rc = TOXAV_ERR_CALL_INVALID_BIT_RATE; + goto END; + } + + IToxAVCall* call = i_toxav_get_call(av, friend_number); + if (call == NULL || av->msi->calls[call->call_idx]->state != msi_CallRequested) { + rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; + goto END; + } + + /* TODO remove csettings */ + MSICSettings csets; + csets.audio_bitrate = audio_bit_rate; + csets.video_bitrate = video_bit_rate; + + csets.call_type = video_bit_rate ? msi_TypeVideo : msi_TypeAudio; + + if (msi_answer(av->msi, call->call_idx, &csets) != 0) { + rc = TOXAV_ERR_ANSWER_MALLOC; /* TODO Some error here */ + /* TODO Reject call? */ + } + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_ANSWER_OK; +} + +void toxav_callback_call_state(ToxAV* av, toxav_call_state_cb* function, void* user_data) +{ + av->scb.first = function; + av->scb.second = user_data; +} + +bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL* error) +{ + +} + +bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE* error) +{ + +} + +bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE* error) +{ + +} + +void toxav_callback_request_video_frame(ToxAV* av, toxav_request_video_frame_cb* function, void* user_data) +{ + +} + +bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, const uint8_t* a, TOXAV_ERR_SEND_FRAME* error) +{ + +} + +void toxav_callback_request_audio_frame(ToxAV* av, toxav_request_audio_frame_cb* function, void* user_data) +{ + +} + +bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error) +{ + +} + +void toxav_callback_receive_video_frame(ToxAV* av, toxav_receive_video_frame_cb* function, void* user_data) +{ + +} + +void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* function, void* user_data) +{ + +} + + +/******************************************************************************* + * + * :: Internal + * + ******************************************************************************/ +/** TODO: + * - In msi call_idx can be the same as friend id + * - If crutial callback not present send error + * - Remove *data from msi + * - Remove CSettings from msi + */ +void i_toxav_msi_callback_invite(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + + uint32_t ab = toxav->msi->calls[call_idx]->csettings_peer[0].audio_bitrate; + uint32_t vb = toxav->msi->calls[call_idx]->csettings_peer[0].video_bitrate; + + IToxAVCall* call = i_toxav_init_call(toxav, toxav->msi->calls[call_idx]->peers[0], ab, vb, NULL); + if (call == NULL) { + msi_reject(toxav->msi, call_idx, NULL); + return false; + } + + call->call_idx = call_idx; + + if (toxav->ccb.first) + toxav->ccb.first(toxav, toxav->msi->calls[call_idx]->peers[0], true, true, toxav->ccb.second); +} + +void i_toxav_msi_callback_ringing(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + if (toxav->scb.first) + toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], + TOXAV_CALL_STATE_RINGING, toxav->scb.second); +} + +void i_toxav_msi_callback_start(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + + IToxAVCall* call = i_toxav_get_call(toxav, toxav->msi->calls[call_idx]->peers[0]); + + if (call == NULL || !i_toxav_prepare_transmission(toxav, call)) { + /* TODO send error */ + i_toxav_remove_call(toxav, toxav->msi->calls[call_idx]->peers[0]); + return; + } + + TOXAV_CALL_STATE state; + const MSICSettings* csets = toxav->msi->calls[call_idx]->csettings_peer[0]; + + if (csets->audio_bitrate && csets->video_bitrate) + state = TOXAV_CALL_STATE_SENDING_AV; + else if (csets->video_bitrate == 0) + state = TOXAV_CALL_STATE_SENDING_A; + else + state = TOXAV_CALL_STATE_SENDING_V; + + if (toxav->scb.first) /* TODO this */ + toxav->scb.first(toxav, call->friend_number, state, toxav->scb.second); +} + +void i_toxav_msi_callback_cancel(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + if (toxav->scb.first) + toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], + TOXAV_CALL_STATE_END, toxav->scb.second); +} + +void i_toxav_msi_callback_reject(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + if (toxav->scb.first) + toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], + TOXAV_CALL_STATE_END, toxav->scb.second); +} + +void i_toxav_msi_callback_end(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + if (toxav->scb.first) + toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], + TOXAV_CALL_STATE_END, toxav->scb.second); +} + +void i_toxav_msi_callback_request_to(void* toxav_inst, int32_t call_idx, void* data) +{ + /* TODO remove */ + ToxAV* toxav = toxav_inst; + if (toxav->scb.first) + toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], + TOXAV_CALL_STATE_ERROR, toxav->scb.second); +} + +void i_toxav_msi_callback_peer_to(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + if (toxav->scb.first) + toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], + TOXAV_CALL_STATE_ERROR, toxav->scb.second); +} + +void i_toxav_msi_callback_state_change(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + /* TODO something something msi */ +} + +IToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number) +{ + if (av->calls_tail < friend_number) + return NULL; + + return av->calls[friend_number]; +} + +IToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number) +{ + IToxAVCall* rc = calloc(sizeof(IToxAVCall), 1); + + if (rc == NULL) + return NULL; + + rc->friend_number = friend_number; + + if (create_recursive_mutex(rc->mutex_control) != 0) { + free(rc); + return NULL; + } + + if (create_recursive_mutex(rc->mutex_do) != 0) { + pthread_mutex_destroy(rc->mutex_control); + free(rc); + return NULL; + } + + + if (av->calls == NULL) { /* Creating */ + av->calls = calloc (sizeof(IToxAVCall*), friend_number + 1); + + if (av->calls == NULL) { + pthread_mutex_destroy(rc->mutex_control); + pthread_mutex_destroy(rc->mutex_do); + free(rc); + return NULL; + } + + av->calls_tail = av->calls_head = friend_number; + + } else if (av->calls_tail < friend_number) { /* Appending */ + void* tmp = realloc(av->calls, sizeof(IToxAVCall*) * friend_number + 1); + + if (tmp == NULL) { + pthread_mutex_destroy(rc->mutex_control); + pthread_mutex_destroy(rc->mutex_do); + free(rc); + return NULL; + } + + av->calls = tmp; + + /* Set fields in between to null */ + int32_t i = av->calls_tail; + for (; i < friend_number; i ++) + av->calls[i] = NULL; + + rc->prev = av->calls[av->calls_tail]; + av->calls[av->calls_tail]->next = rc; + + av->calls_tail = friend_number; + + } else if (av->calls_head > friend_number) { /* Inserting at front */ + rc->next = av->calls[av->calls_head]; + av->calls[av->calls_head]->prev = rc; + av->calls_head = friend_number; + } + + av->calls[friend_number] = rc; + return rc; +} + +void i_toxav_remove_call(ToxAV* av, uint32_t friend_number) +{ + IToxAVCall* tc = i_toxav_get_call(av, friend_number); + + if (tc == NULL) + return; + + IToxAVCall* prev = tc->prev; + IToxAVCall* next = tc->next; + + pthread_mutex_destroy(tc->mutex_control); + pthread_mutex_destroy(tc->mutex_do); + + free(tc); + + if (prev) + prev->next = next; + else if (next) + av->calls_head = next->friend_number; + else goto CLEAR; + + if (next) + next->prev = prev; + else if (prev) + av->calls_tail = prev->friend_number; + else goto CLEAR; + + av->calls[friend_number] = NULL; + return; + +CLEAR: + av->calls_head = av->calls_tail = 0; + free(av->calls); + av->calls = NULL; +} + +bool i_toxav_audio_bitrate_invalid(uint32_t bitrate) +{ + /* Opus RFC 6716 section-2.1.1 dictates the following: + * Opus supports all bitrates from 6 kbit/s to 510 kbit/s. + */ + return bitrate < 6 || bitrate > 510; +} + +bool i_toxav_video_bitrate_invalid(uint32_t bitrate) +{ + /* TODO: If anyone knows the answer to this one please fill it up */ + return false; +} + +IToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) +{ + TOXAV_ERR_CALL rc = TOXAV_ERR_CALL_OK; + IToxAVCall* call = NULL; + + if (m_friend_exists(av->m, friend_number)) { + rc = TOXAV_ERR_CALL_FRIEND_NOT_FOUND; + goto END; + } + + if (m_get_friend_connectionstatus(av->m, friend_number) != 1) { + rc = TOXAV_ERR_CALL_FRIEND_NOT_CONNECTED; + goto END; + } + + if (i_toxav_get_call(av, friend_number) != NULL) { + rc = TOXAV_ERR_CALL_FRIEND_ALREADY_IN_CALL; + goto END; + } + + if ((audio_bit_rate && i_toxav_audio_bitrate_invalid(audio_bit_rate)) + ||(video_bit_rate && i_toxav_video_bitrate_invalid(video_bit_rate)) + ) { + rc = TOXAV_ERR_CALL_INVALID_BIT_RATE; + goto END; + } + + IToxAVCall* call = i_toxav_add_call(av, friend_number); + if (call == NULL) { + rc = TOXAV_ERR_CALL_MALLOC; + } + +END: + if (error) + *error = rc; + + return call; +} + +bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call) +{ + pthread_mutex_lock(call->mutex_control); + + if (call->active) { + pthread_mutex_unlock(call->mutex_control); + LOGGER_WARNING("Call already active!\n"); + return true; + } + + if (pthread_mutex_init(call->mutex_encoding_audio, NULL) != 0) + goto MUTEX_INIT_ERROR; + + if (pthread_mutex_init(call->mutex_encoding_video, NULL) != 0) { + pthread_mutex_destroy(call->mutex_encoding_audio); + goto MUTEX_INIT_ERROR; + } + + if (pthread_mutex_init(call->mutex_do, NULL) != 0) { + pthread_mutex_destroy(call->mutex_encoding_audio); + pthread_mutex_destroy(call->mutex_encoding_video); + goto MUTEX_INIT_ERROR; + } + + const MSICSettings *c_peer = &av->msi->calls[call->call_idx]->csettings_peer[0]; + const MSICSettings *c_self = &av->msi->calls[call->call_idx]->csettings_local; + + call->cs = cs_new(c_self->audio_bitrate, c_peer->audio_bitrate, + c_self->video_bitrate, c_peer->video_bitrate); + + if ( !call->cs ) { + LOGGER_ERROR("Error while starting Codec State!\n"); + goto FAILURE; + } + + call->cs->agent = av; + call->cs->call_idx = call->call_idx; + + call->cs->acb.first = av->acb.first; + call->cs->acb.second = av->acb.second; + + call->cs->vcb.first = av->vcb.first; + call->cs->vcb.second = av->vcb.second; + + + if (c_self->audio_bitrate > 0 || c_peer->audio_bitrate > 0) { /* Prepare audio rtp */ + call->rtps[audio_index] = rtp_new(msi_TypeAudio, av->m, av->msi->calls[call->call_idx]->peers[0]); + + if ( !call->rtps[audio_index] ) { + LOGGER_ERROR("Error while starting audio RTP session!\n"); + goto FAILURE; + } + + call->rtps[audio_index]->cs = call->cs; + + if (c_peer->audio_bitrate > 0) + rtp_register_for_receiving(call->rtps[audio_index]); + } + + if (c_self->video_bitrate > 0 || c_peer->video_bitrate > 0) { /* Prepare video rtp */ + call->rtps[video_index] = rtp_new(msi_TypeVideo, av->m, av->msi->calls[call->call_idx]->peers[0]); + + if ( !call->rtps[video_index] ) { + LOGGER_ERROR("Error while starting video RTP session!\n"); + goto FAILURE; + } + + call->rtps[video_index]->cs = call->cs; + + if (c_peer->video_bitrate > 0) + rtp_register_for_receiving(call->rtps[audio_index]); + } + + call->active = 1; + pthread_mutex_unlock(call->mutex_control); + return true; + +FAILURE: + rtp_kill(call->rtps[audio_index]); + call->rtps[audio_index] = NULL; + rtp_kill(call->rtps[video_index]); + call->rtps[video_index] = NULL; + cs_kill(call->cs); + call->cs = NULL; + call->active = 0; + pthread_mutex_destroy(call->mutex_encoding_audio); + pthread_mutex_destroy(call->mutex_encoding_video); + pthread_mutex_destroy(call->mutex_do); + + pthread_mutex_unlock(call->mutex_control); + return false; + +MUTEX_INIT_ERROR: + pthread_mutex_unlock(call->mutex_control); + LOGGER_ERROR("Mutex initialization failed!\n"); + return false; +} + +void i_toxav_kill_transmission(ToxAV* av, IToxAVCall* call) +{ + pthread_mutex_lock(call->mutex_control); + + if (!call->active) { + pthread_mutex_unlock(call->mutex_control); + LOGGER_WARNING("Action on inactive call: %d", call->call_idx); + return; + } + + call->active = 0; + + pthread_mutex_lock(call->mutex_encoding_audio); + pthread_mutex_unlock(call->mutex_encoding_audio); + pthread_mutex_lock(call->mutex_encoding_video); + pthread_mutex_unlock(call->mutex_encoding_video); + pthread_mutex_lock(call->mutex_do); + pthread_mutex_unlock(call->mutex_do); + + rtp_kill(call->rtps[audio_index]); + call->rtps[audio_index] = NULL; + rtp_kill(call->rtps[video_index]); + call->rtps[video_index] = NULL; + cs_kill(call->cs); + call->cs = NULL; + + pthread_mutex_destroy(call->mutex_encoding_audio); + pthread_mutex_destroy(call->mutex_encoding_video); + pthread_mutex_destroy(call->mutex_do); + + pthread_mutex_unlock(call->mutex_control); +} diff --git a/toxav/toxav_new.h b/toxav/toxav_new.h new file mode 100644 index 000000000..78e793576 --- /dev/null +++ b/toxav/toxav_new.h @@ -0,0 +1,481 @@ +#pragma once +#include +#include +#include +/** \page av Public audio/video API for Tox clients. + * + * Unlike the Core API, this API is fully thread-safe. The library will ensure + * the proper synchronisation of parallel calls. + */ +/** + * The type of the Tox Audio/Video subsystem object. + */ +typedef struct toxAV ToxAV; +#ifndef TOX_DEFINED +#define TOX_DEFINED +/** + * The type of a Tox instance. Repeated here so this file does not have a direct + * dependency on the Core interface. + */ +typedef struct Tox Tox; +#endif +/******************************************************************************* + * + * :: Creation and destruction + * + ******************************************************************************/ +typedef enum TOXAV_ERR_NEW { + TOXAV_ERR_NEW_OK, + TOXAV_ERR_NEW_NULL, + /** + * Memory allocation failure while trying to allocate structures required for + * the A/V session. + */ + TOXAV_ERR_NEW_MALLOC, + /** + * Attempted to create a second session for the same Tox instance. + */ + TOXAV_ERR_NEW_MULTIPLE +} TOXAV_ERR_NEW; +/** + * Start new A/V session. There can only be only one session per Tox instance. + */ +ToxAV *toxav_new(Tox *tox, TOXAV_ERR_NEW *error); +/** + * Releases all resources associated with the A/V session. + * + * If any calls were ongoing, these will be forcibly terminated without + * notifying peers. After calling this function, no other functions may be + * called and the av pointer becomes invalid. + */ +void toxav_kill(ToxAV *av); +/** + * Returns the Tox instance the A/V object was created for. + */ +Tox *toxav_get_tox(ToxAV *av); +/******************************************************************************* + * + * :: A/V event loop + * + ******************************************************************************/ +/** + * Returns the interval in milliseconds when the next toxav_iteration should be + * called. If no call is active at the moment, this function returns 200. + */ +uint32_t toxav_iteration_interval(ToxAV const *av); +/** + * Main loop for the session. This function needs to be called in intervals of + * toxav_iteration_interval() milliseconds. It is best called in the same loop + * as tox_iteration. + */ +void toxav_iteration(ToxAV *av); +/******************************************************************************* + * + * :: Call setup + * + ******************************************************************************/ +typedef enum TOXAV_ERR_CALL { + TOXAV_ERR_CALL_OK, + /** + * A resource allocation error occurred while trying to create the structures + * required for the call. + */ + TOXAV_ERR_CALL_MALLOC, + /** + * The friend number did not designate a valid friend. + */ + TOXAV_ERR_CALL_FRIEND_NOT_FOUND, + /** + * The friend was valid, but not currently connected. + */ + TOXAV_ERR_CALL_FRIEND_NOT_CONNECTED, + /** + * Attempted to call a friend while already in an audio or video call with + * them. + */ + TOXAV_ERR_CALL_FRIEND_ALREADY_IN_CALL, + /** + * Audio or video bit rate is invalid. + */ + TOXAV_ERR_CALL_INVALID_BIT_RATE +} TOXAV_ERR_CALL; +/** + * Call a friend. This will start ringing the friend. + * + * It is the client's responsibility to stop ringing after a certain timeout, + * if such behaviour is desired. If the client does not stop ringing, the A/V + * library will not stop until the friend is disconnected. + * + * @param friend_number The friend number of the friend that should be called. + * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable + * audio sending. + * @param video_bit_rate Video bit rate in Kb/sec. Set this to 0 to disable + * video sending. + */ +bool toxav_call(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL *error); +/** + * The function type for the `call` callback. + */ +typedef void toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data); +/** + * Set the callback for the `call` event. Pass NULL to unset. + * + * This event is triggered when a call is received from a friend. + */ +void toxav_callback_call(ToxAV *av, toxav_call_cb *function, void *user_data); +typedef enum TOXAV_ERR_ANSWER { + TOXAV_ERR_ANSWER_OK, + /** + * A resource allocation error occurred while trying to create the structures + * required for the call. + */ + TOXAV_ERR_ANSWER_MALLOC, + /** + * The friend number did not designate a valid friend. + */ + TOXAV_ERR_ANSWER_FRIEND_NOT_FOUND, + /** + * The friend was valid, but they are not currently trying to initiate a call. + * This is also returned if this client is already in a call with the friend. + */ + TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING, + /** + * Audio or video bit rate is invalid. + */ + TOXAV_ERR_ANSWER_INVALID_BIT_RATE +} TOXAV_ERR_ANSWER; +/** + * Accept an incoming call. + * + * If an allocation error occurs while answering a call, both participants will + * receive TOXAV_CALL_STATE_ERROR and the call will end. + * + * @param friend_number The friend number of the friend that is calling. + * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable + * audio sending. + * @param video_bit_rate Video bit rate in Kb/sec. Set this to 0 to disable + * video sending. + */ +bool toxav_answer(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER *error); +/******************************************************************************* + * + * :: Call state graph + * + ******************************************************************************/ +typedef enum TOXAV_CALL_STATE { + /** + * The friend's client is aware of the call. This happens after calling + * toxav_call and the initial call request has been received. + */ + TOXAV_CALL_STATE_RINGING, + /** + * Not sending anything. Either the friend requested that this client stops + * sending anything, or the client turned off both audio and video by setting + * the respective bit rates to 0. + * + * If both sides are in this state, the call is effectively on hold, but not + * in the PAUSED state. + */ + TOXAV_CALL_STATE_NOT_SENDING, + /** + * Sending audio only. Either the friend requested that this client stops + * sending video, or the client turned off video by setting the video bit rate + * to 0. + */ + TOXAV_CALL_STATE_SENDING_A, + /** + * Sending video only. Either the friend requested that this client stops + * sending audio (muted), or the client turned off audio by setting the audio + * bit rate to 0. + */ + TOXAV_CALL_STATE_SENDING_V, + /** + * Sending both audio and video. + */ + TOXAV_CALL_STATE_SENDING_AV, + /** + * The call is on hold. Both sides stop sending and receiving. + */ + TOXAV_CALL_STATE_PAUSED, + /** + * The call has finished. This is the final state after which no more state + * transitions can occur for the call. + */ + TOXAV_CALL_STATE_END, + /** + * Sent by the AV core if an error occurred on the remote end. + */ + TOXAV_CALL_STATE_ERROR +} TOXAV_CALL_STATE; +/** + * The function type for the `call_state` callback. + * + * @param friend_number The friend number for which the call state changed. + * @param state The new call state. + */ +typedef void toxav_call_state_cb(ToxAV *av, uint32_t friend_number, TOXAV_CALL_STATE state, void *user_data); +/** + * Set the callback for the `call_state` event. Pass NULL to unset. + * + * This event is triggered when a call state transition occurs. + */ +void toxav_callback_call_state(ToxAV *av, toxav_call_state_cb *function, void *user_data); +/******************************************************************************* + * + * :: Call control + * + ******************************************************************************/ +typedef enum TOXAV_CALL_CONTROL { + /** + * Resume a previously paused call. Only valid if the pause was caused by this + * client. Not valid before the call is accepted. + */ + TOXAV_CALL_CONTROL_RESUME, + /** + * Put a call on hold. Not valid before the call is accepted. + */ + TOXAV_CALL_CONTROL_PAUSE, + /** + * Reject a call if it was not answered, yet. Cancel a call after it was + * answered. + */ + TOXAV_CALL_CONTROL_CANCEL, + /** + * Request that the friend stops sending audio. Regardless of the friend's + * compliance, this will cause the `receive_audio_frame` event to stop being + * triggered on receiving an audio frame from the friend. + */ + TOXAV_CALL_CONTROL_MUTE_AUDIO, + /** + * Request that the friend stops sending video. Regardless of the friend's + * compliance, this will cause the `receive_video_frame` event to stop being + * triggered on receiving an video frame from the friend. + */ + TOXAV_CALL_CONTROL_MUTE_VIDEO +} TOXAV_CALL_CONTROL; +typedef enum TOXAV_ERR_CALL_CONTROL { + TOXAV_ERR_CALL_CONTROL_OK, + /** + * The friend_number passed did not designate a valid friend. + */ + TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_FOUND, + /** + * This client is currently not in a call with the friend. Before the call is + * answered, only CANCEL is a valid control. + */ + TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL, + /** + * Attempted to resume a call that was not paused. + */ + TOXAV_ERR_CALL_CONTROL_NOT_PAUSED, + /** + * Attempted to resume a call that was paused by the other party. Also set if + * the client attempted to send a system-only control. + */ + TOXAV_ERR_CALL_CONTROL_DENIED, + /** + * The call was already paused on this client. It is valid to pause if the + * other party paused the call. The call will resume after both parties sent + * the RESUME control. + */ + TOXAV_ERR_CALL_CONTROL_ALREADY_PAUSED +} TOXAV_ERR_CALL_CONTROL; +/** + * Sends a call control command to a friend. + * + * @param friend_number The friend number of the friend this client is in a call + * with. + * @param control The control command to send. + * + * @return true on success. + */ +bool toxav_call_control(ToxAV *av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL *error); +/******************************************************************************* + * + * :: Controlling bit rates + * + ******************************************************************************/ +typedef enum TOXAV_ERR_BIT_RATE { + TOXAV_ERR_BIT_RATE_OK, + /** + * The bit rate passed was not one of the supported values. + */ + TOXAV_ERR_BIT_RATE_INVALID +} TOXAV_ERR_BIT_RATE; +/** + * Set the audio bit rate to be used in subsequent audio frames. + * + * @param friend_number The friend number of the friend for which to set the + * audio bit rate. + * @param audio_bit_rate The new audio bit rate in Kb/sec. Set to 0 to disable + * audio sending. + * + * @see toxav_call for the valid bit rates. + */ +bool toxav_set_audio_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE *error); +/** + * Set the video bit rate to be used in subsequent video frames. + * + * @param friend_number The friend number of the friend for which to set the + * video bit rate. + * @param video_bit_rate The new video bit rate in Kb/sec. Set to 0 to disable + * video sending. + * + * @see toxav_call for the valid bit rates. + */ +bool toxav_set_video_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE *error); +/******************************************************************************* + * + * :: A/V sending + * + ******************************************************************************/ +/** + * Common error codes for the send_*_frame functions. + */ +typedef enum TOXAV_ERR_SEND_FRAME { + TOXAV_ERR_SEND_FRAME_OK, + /** + * In case of video, one of Y, U, or V was NULL. In case of audio, the samples + * data pointer was NULL. + */ + TOXAV_ERR_SEND_FRAME_NULL, + /** + * The friend_number passed did not designate a valid friend. + */ + TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND, + /** + * This client is currently not in a call with the friend. + */ + TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL, + /** + * No video frame had been requested through the `request_video_frame` event, + * but the client tried to send one, anyway. + */ + TOXAV_ERR_SEND_FRAME_NOT_REQUESTED, + /** + * One of the frame parameters was invalid. E.g. the resolution may be too + * small or too large, or the audio sampling rate may be unsupported. + */ + TOXAV_ERR_SEND_FRAME_INVALID +} TOXAV_ERR_SEND_FRAME; +/** + * The function type for the `request_video_frame` callback. + * + * @param friend_number The friend number of the friend for which the next video + * frame should be sent. + */ +typedef void toxav_request_video_frame_cb(ToxAV *av, uint32_t friend_number, void *user_data); +/** + * Set the callback for the `request_video_frame` event. Pass NULL to unset. + */ +void toxav_callback_request_video_frame(ToxAV *av, toxav_request_video_frame_cb *function, void *user_data); +/** + * Send a video frame to a friend. + * + * This is called in response to receiving the `request_video_frame` event. + * + * Each plane should contain (width * height) pixels. The Alpha plane can be + * NULL, in which case every pixel is assumed fully opaque. + * + * @param friend_number The friend number of the friend to which to send a video + * frame. + * @param width Width of the frame in pixels. + * @param height Height of the frame in pixels. + * @param y Y (Luminance) plane data. + * @param u U (Chroma) plane data. + * @param v V (Chroma) plane data. + * @param a A (Alpha) plane data. + */ +bool toxav_send_video_frame(ToxAV *av, uint32_t friend_number, + uint16_t width, uint16_t height, + uint8_t const *y, uint8_t const *u, uint8_t const *v, uint8_t const *a, + TOXAV_ERR_SEND_FRAME *error); +/** + * The function type for the `request_audio_frame` callback. + * + * @param friend_number The friend number of the friend for which the next audio + * frame should be sent. + */ +typedef void toxav_request_audio_frame_cb(ToxAV *av, uint32_t friend_number, void *user_data); +/** + * Set the callback for the `request_audio_frame` event. Pass NULL to unset. + */ +void toxav_callback_request_audio_frame(ToxAV *av, toxav_request_audio_frame_cb *function, void *user_data); +/** + * Send an audio frame to a friend. + * + * This is called in response to receiving the `request_audio_frame` event. + * + * The expected format of the PCM data is: [s1c1][s1c2][...][s2c1][s2c2][...]... + * Meaning: sample 1 for channel 1, sample 1 for channel 2, ... + * For mono audio, this has no meaning, every sample is subsequent. For stereo, + * this means the expected format is LRLRLR... with samples for left and right + * alternating. + * + * @param friend_number The friend number of the friend to which to send an + * audio frame. + * @param pcm An array of audio samples. The size of this array must be + * sample_count * channels. + * @param sample_count Number of samples in this frame. Valid numbers here are + * ((sample rate) * (audio length) / 1000), where audio length can be + * 2.5, 5, 10, 20, 40 or 60 millseconds. + * @param channels Number of audio channels. Must be at least 1 for mono. + * For voice over IP, more than 2 channels (stereo) typically doesn't make + * sense, but up to 255 channels are supported. + * @param sampling_rate Audio sampling rate used in this frame. Valid sampling + * rates are 8000, 12000, 16000, 24000, or 48000. + */ +bool toxav_send_audio_frame(ToxAV *av, uint32_t friend_number, + int16_t const *pcm, + size_t sample_count, + uint8_t channels, + uint32_t sampling_rate, + TOXAV_ERR_SEND_FRAME *error); +/******************************************************************************* + * + * :: A/V receiving + * + ******************************************************************************/ +/** + * The function type for the `receive_video_frame` callback. + * + * Each plane contains (width * height) pixels. The Alpha plane can be NULL, in + * which case every pixel should be assumed fully opaque. + * + * @param friend_number The friend number of the friend who sent a video frame. + * @param width Width of the frame in pixels. + * @param height Height of the frame in pixels. + * @param y Y (Luminance) plane data. + * @param u U (Chroma) plane data. + * @param v V (Chroma) plane data. + * @param a A (Alpha) plane data. + */ +typedef void toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, + uint16_t width, uint16_t height, + uint8_t const *y, uint8_t const *u, uint8_t const *v, uint8_t const *a, + void *user_data); +/** + * Set the callback for the `receive_video_frame` event. Pass NULL to unset. + */ +void toxav_callback_receive_video_frame(ToxAV *av, toxav_receive_video_frame_cb *function, void *user_data); +/** + * The function type for the `receive_audio_frame` callback. + * + * @param friend_number The friend number of the friend who sent an audio frame. + * @param pcm An array of audio samples (sample_count * channels elements). + * @param sample_count The number of audio samples per channel in the PCM array. + * @param channels Number of audio channels. + * @param sampling_rate Sampling rate used in this frame. + * + * @see toxav_send_audio_frame for the audio format. + */ +typedef void toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, + int16_t const *pcm, + size_t sample_count, + uint8_t channels, + uint32_t sampling_rate, + void *user_data); +/** + * Set the callback for the `receive_audio_frame` event. Pass NULL to unset. + */ +void toxav_callback_receive_audio_frame(ToxAV *av, toxav_receive_audio_frame_cb *function, void *user_data); \ No newline at end of file diff --git a/toxcore/logger.c b/toxcore/logger.c index 2ef5f21ae..ac81a9007 100644 --- a/toxcore/logger.c +++ b/toxcore/logger.c @@ -42,7 +42,7 @@ #endif -typedef struct logger { +struct logger { FILE *log_file; LOG_LEVEL level; uint64_t start_time; /* Time when lib loaded */ @@ -55,7 +55,7 @@ typedef struct logger { /* For thread synchronisation */ pthread_mutex_t mutex[1]; -} logger; +}; logger *global = NULL; diff --git a/toxcore/util.h b/toxcore/util.h index 7cd6bb8b8..fab26e294 100644 --- a/toxcore/util.h +++ b/toxcore/util.h @@ -53,6 +53,7 @@ typedef int (*load_state_callback_func)(void *outer, const uint8_t *data, uint32 int load_state(load_state_callback_func load_state_callback, void *outer, const uint8_t *data, uint32_t length, uint16_t cookie_inner); +/* Returns -1 if failed or 0 if success */ int create_recursive_mutex(pthread_mutex_t *mutex); #endif /* __UTIL_H__ */ From 39680f31d0121cef2358507fcea84cacad69893a Mon Sep 17 00:00:00 2001 From: mannol Date: Sun, 1 Feb 2015 23:43:54 +0100 Subject: [PATCH 03/97] Progress --- toxav/codec.c | 644 ++++++++++++++++++++++++---------------------- toxav/codec.h | 66 ++--- toxav/rtp.c | 4 +- toxav/rtp.h | 2 +- toxav/toxav.c | 16 +- toxav/toxav_new.c | 250 ++++++++++++++++-- toxav/toxav_new.h | 8 +- 7 files changed, 615 insertions(+), 375 deletions(-) diff --git a/toxav/codec.c b/toxav/codec.c index fd2f9f932..43120e0f5 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -229,27 +229,196 @@ static RTPMessage *jbuf_read(JitterBuffer *q, int32_t *success) return NULL; } - +static int convert_bw_to_sampling_rate(int bw) +{ + switch(bw) + { + case OPUS_BANDWIDTH_NARROWBAND: return 8000; + case OPUS_BANDWIDTH_MEDIUMBAND: return 12000; + case OPUS_BANDWIDTH_WIDEBAND: return 16000; + case OPUS_BANDWIDTH_SUPERWIDEBAND: return 24000; + case OPUS_BANDWIDTH_FULLBAND: return 48000; + default: return -1; + } +} /* PUBLIC */ -int cs_split_video_payload(CSSession *cs, const uint8_t *payload, uint16_t length) -{ - if (!cs || !length || length > cs->max_video_frame_size) { - LOGGER_ERROR("Invalid CodecState or video frame size: %u", length); - return cs_ErrorSplittingVideoPayload; - } +void cs_do(CSSession *cs) +{ + /* Codec session should always be protected by call mutex so no need to check for cs validity + */ + + if (!cs) + return; + + Payload *p; + int rc; + + int success = 0; + + pthread_mutex_lock(cs->queue_mutex); + RTPMessage *msg; + + uint16_t fsize = 5760; /* Max frame size for 48 kHz */ + int16_t tmp[fsize * 2]; + + while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { + pthread_mutex_unlock(cs->queue_mutex); + + if (success == 2) { + rc = opus_decode(cs->audio_decoder, 0, 0, tmp, fsize, 1); + } else { + /* Get values from packet and decode. + * It also checks for validity of an opus packet + */ + rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); + if (rc != -1) { + cs->last_packet_sampling_rate = rc; + cs->last_pack_channels = opus_packet_get_nb_channels(msg->data); + + cs->last_packet_frame_duration = + ( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 ) + / cs->last_packet_sampling_rate; + + } else { + LOGGER_WARNING("Failed to load packet values!"); + rtp_free_msg(NULL, msg); + continue; + } + + rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, fsize, 0); + rtp_free_msg(NULL, msg); + } + + if (rc < 0) { + LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); + } else if (((ToxAV*)cs->agent)->acb.first) { + /* Play */ + ((ToxAV*)cs->agent)->acb.first(cs->agent, cs->call_idx, tmp, rc, + ((ToxAV*)cs->agent)->acb.second); + } + + pthread_mutex_lock(cs->queue_mutex); + } + + if (cs->vbuf_raw && !buffer_empty(cs->vbuf_raw)) { + /* Decode video */ + buffer_read(cs->vbuf_raw, &p); + + /* Leave space for (possibly) other thread to queue more data after we read it here */ + pthread_mutex_unlock(cs->queue_mutex); + + rc = vpx_codec_decode(cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); + free(p); + + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc)); + } else { + vpx_codec_iter_t iter = NULL; + vpx_image_t *dest = vpx_codec_get_frame(cs->v_decoder, &iter); + + /* Play decoded images */ + for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) { + if (((ToxAV*)cs->agent)->vcb.first) + ((ToxAV*)cs->agent)->vcb.first(cs->agent, cs->call_idx, dest, + ((ToxAV*)cs->agent)->vcb.second); + + vpx_img_free(dest); + } + } + + return; + } + + pthread_mutex_unlock(cs->queue_mutex); +} + +CSSession *cs_new(uint32_t s_audio_b, uint32_t p_audio_b, uint32_t s_video_b, uint32_t p_video_b) +{ + CSSession *cs = calloc(sizeof(CSSession), 1); + + if (!cs) { + LOGGER_WARNING("Allocation failed! Application might misbehave!"); + return NULL; + } + + /* TODO this has to be exchanged in msi */ + cs->max_video_frame_size = MAX_VIDEOFRAME_SIZE; + cs->video_frame_piece_size = VIDEOFRAME_PIECE_SIZE; + + if (s_audio_b > 0 && 0 != cs_enable_audio_sending(cs, s_audio_b, 2)) { /* Sending audio enabled */ + LOGGER_WARNING("Failed to enable audio sending!"); + goto FAILURE; + } + + if (p_audio_b > 0 && 0 != cs_enable_audio_receiving(cs)) { /* Receiving audio enabled */ + LOGGER_WARNING("Failed to enable audio receiving!"); + goto FAILURE; + } + + if (s_video_b > 0 && 0 != cs_enable_video_sending(cs, s_video_b)) { /* Sending video enabled */ + LOGGER_WARNING("Failed to enable video sending!"); + goto FAILURE; + } + + if (p_video_b > 0 && 0 != cs_enable_video_receiving(cs)) { /* Receiving video enabled */ + LOGGER_WARNING("Failed to enable video receiving!"); + goto FAILURE; + } + + return cs; + + FAILURE: + LOGGER_WARNING("Error initializing codec session! Application might misbehave!"); + + cs_disable_audio_sending(cs); + cs_disable_audio_receiving(cs); + cs_disable_video_sending(cs); + cs_disable_video_receiving(cs); + + free(cs); + + return NULL; +} + +void cs_kill(CSSession *cs) +{ + if (!cs) + return; + + /* NOTE: queue_message() will not be called since + * the callback is unregistered before cs_kill is called. + */ + + cs_disable_audio_sending(cs); + cs_disable_audio_receiving(cs); + cs_disable_video_sending(cs); + cs_disable_video_receiving(cs); + + LOGGER_DEBUG("Terminated codec state: %p", cs); + free(cs); +} + + + +void cs_init_video_splitter_cycle(CSSession* cs) +{ cs->split_video_frame[0] = cs->frameid_out++; cs->split_video_frame[1] = 0; +} + +int cs_update_video_splitter_cycle(CSSession *cs, const uint8_t *payload, uint16_t length) +{ cs->processing_video_frame = payload; cs->processing_video_frame_size = length; - + return ((length - 1) / cs->video_frame_piece_size) + 1; } -const uint8_t *cs_get_split_video_frame(CSSession *cs, uint16_t *size) +const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size) { if (!cs || !size) return NULL; @@ -275,74 +444,7 @@ const uint8_t *cs_get_split_video_frame(CSSession *cs, uint16_t *size) return cs->split_video_frame; } -void cs_do(CSSession *cs) -{ - /* Codec session should always be protected by call mutex so no need to check for cs validity - */ - if (!cs) return; - - Payload *p; - int rc; - - int success = 0; - - pthread_mutex_lock(cs->queue_mutex); - RTPMessage *msg; - - while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { - pthread_mutex_unlock(cs->queue_mutex); - - uint16_t fsize = ((cs->audio_decoder_sample_rate * cs->audio_decoder_frame_duration) / 1000); - int16_t tmp[fsize * cs->audio_decoder_channels]; - - if (success == 2) { - rc = opus_decode(cs->audio_decoder, 0, 0, tmp, fsize, 1); - } else { - rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, fsize, 0); - rtp_free_msg(NULL, msg); - } - - if (rc < 0) { - LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); - } else if (cs->acb.first) { - /* Play */ - cs->acb.first(cs->agent, cs->call_idx, tmp, rc, cs->acb.second); - } - - pthread_mutex_lock(cs->queue_mutex); - } - - if (cs->vbuf_raw && !buffer_empty(cs->vbuf_raw)) { - /* Decode video */ - buffer_read(cs->vbuf_raw, &p); - - /* Leave space for (possibly) other thread to queue more data after we read it here */ - pthread_mutex_unlock(cs->queue_mutex); - - rc = vpx_codec_decode(cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); - free(p); - - if (rc != VPX_CODEC_OK) { - LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc)); - } else { - vpx_codec_iter_t iter = NULL; - vpx_image_t *dest = vpx_codec_get_frame(cs->v_decoder, &iter); - - /* Play decoded images */ - for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) { - if (cs->vcb.first) - cs->vcb.first(cs->agent, cs->call_idx, dest, cs->vcb.second); - - vpx_img_free(dest); - } - } - - return; - } - - pthread_mutex_unlock(cs->queue_mutex); -} int cs_set_sending_video_resolution(CSSession *cs, uint16_t width, uint16_t height) { @@ -402,220 +504,6 @@ int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate) return 0; } -int cs_set_sending_audio_bitrate(CSSession *cs, int32_t rate) -{ - if (cs->audio_encoder == NULL) - return -1; - - int rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(rate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); - return -1; - } - - return 0; -} - -int cs_set_sending_audio_sampling_rate(CSSession* cs, int32_t rate) -{ - /* TODO Find a better way? */ - if (cs->audio_encoder == NULL) - return -1; - - int rc = OPUS_OK; - int last_rate = 0; - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(&last_rate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while getting encoder ctl: %s", opus_strerror(rc)); - return -1; - } - - if (rate == last_rate) - return 0; - - OpusEncoder* new_enc = opus_encoder_create(rate, cs->channels, OPUS_APPLICATION_AUDIO, &rc); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); - return -1; - } - - opus_encoder_destroy(cs->audio_encoder); - cs->audio_encoder = new_enc; - return 0; -} - -int cs_set_sending_audio_channels(CSSession* cs, int32_t count) -{ - /* TODO Find a better way? */ - if (cs->audio_encoder == NULL) - return -1; - - if (cs->channels == count) - return 0; - - int rc = OPUS_OK; - int bitrate = 0; - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(&bitrate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while getting encoder ctl: %s", opus_strerror(rc)); - return -1; - } - - cs->channels = count; - OpusEncoder* new_enc = opus_encoder_create(bitrate, cs->channels, OPUS_APPLICATION_AUDIO, &rc); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); - return -1; - } - - opus_encoder_destroy(cs->audio_encoder); - cs->audio_encoder = new_enc; - return 0; -} - -CSSession *cs_new(uint32_t s_audio_b, uint32_t p_audio_b, uint32_t s_video_b, uint32_t p_video_b) -{ - CSSession *cs = calloc(sizeof(CSSession), 1); - - if (!cs) { - LOGGER_WARNING("Allocation failed! Application might misbehave!"); - return NULL; - } - - /* TODO this has to be exchanged in msi */ - cs->max_video_frame_size = MAX_VIDEOFRAME_SIZE; - cs->video_frame_piece_size = VIDEOFRAME_PIECE_SIZE; - - if (s_audio_b > 0 && 0 != cs_enable_audio_sending(cs, s_audio_b)) { /* Sending audio enabled */ - LOGGER_WARNING("Failed to enable audio sending!"); - goto FAILURE; - } - - if (p_audio_b > 0 && 0 != cs_enable_audio_receiving(cs)) { /* Receiving audio enabled */ - LOGGER_WARNING("Failed to enable audio receiving!"); - goto FAILURE; - } - - if (s_video_b > 0 && 0 != cs_enable_video_sending(cs, s_video_b)) { /* Sending video enabled */ - LOGGER_WARNING("Failed to enable video sending!"); - goto FAILURE; - } - - if (p_video_b > 0 && 0 != cs_enable_video_receiving(cs)) { /* Receiving video enabled */ - LOGGER_WARNING("Failed to enable video receiving!"); - goto FAILURE; - } - - return cs; - -FAILURE: - LOGGER_WARNING("Error initializing codec session! Application might misbehave!"); - - cs_disable_audio_sending(cs); - cs_disable_audio_receiving(cs); - cs_disable_video_sending(cs); - cs_disable_video_receiving(cs); - - free(cs); - - return NULL; -} - -void cs_kill(CSSession *cs) -{ - if (!cs) - return; - - /* NOTE: queue_message() will not be called since - * the callback is unregistered before cs_kill is called. - */ - - cs_disable_audio_sending(cs); - cs_disable_audio_receiving(cs); - cs_disable_video_sending(cs); - cs_disable_video_receiving(cs); - - LOGGER_DEBUG("Terminated codec state: %p", cs); - free(cs); -} - -int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate) -{ - if (cs->audio_encoder) - return 0; - - /** - * Encoder is initialized with default values. These values (Sampling rate, channel count) - * change on the fly from toxav. - */ - - int rc = OPUS_OK; - cs->audio_encoder = opus_encoder_create(48000, 2, OPUS_APPLICATION_AUDIO, &rc); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); - return -1; - } - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); - goto FAILURE; - } - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); - goto FAILURE; - } - - cs->channels = 2; - return 0; - -FAILURE: - cs_disable_audio_sending(cs); - return -1; -} - -int cs_enable_audio_receiving(CSSession* cs) -{ - if (cs->audio_decoder) - return 0; - - /** - * Decoder is initialized with default values. These values (Sampling rate, channel count) - * change on the fly from toxav. - */ - - int rc; - cs->audio_decoder = opus_decoder_create(48000, 2, &rc ); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc)); - return -1; - } - - - if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) { - LOGGER_WARNING("Jitter buffer creaton failed!"); - opus_decoder_destroy(cs->audio_decoder); - cs->audio_decoder = NULL; - return -1; - } - - - return 0; -} - int cs_enable_video_sending(CSSession* cs, uint32_t bitrate) { if (cs->v_encoding) @@ -704,25 +592,6 @@ FAILURE: return -1; } -void cs_disable_audio_sending(CSSession* cs) -{ - if ( cs->audio_encoder ) { - opus_encoder_destroy(cs->audio_encoder); - cs->audio_encoder = NULL; - cs->channels = 0; - } -} - -void cs_disable_audio_receiving(CSSession* cs) -{ - if ( cs->audio_decoder ) { - opus_decoder_destroy(cs->audio_decoder); - cs->audio_decoder = NULL; - jbuf_free(cs->j_buf); - cs->j_buf = NULL; - } -} - void cs_disable_video_sending(CSSession* cs) { if (cs->v_encoding) { @@ -752,6 +621,163 @@ void cs_disable_video_receiving(CSSession* cs) +int cs_set_sending_audio_bitrate(CSSession *cs, int32_t rate) +{ + if (cs->audio_encoder == NULL) + return -1; + + int rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(rate)); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); + return -1; + } + + return 0; +} + +int cs_set_sending_audio_sampling_rate(CSSession* cs, int32_t rate) +{ + /* TODO Find a better way? */ + if (cs->audio_encoder == NULL) + return -1; + + int rc = OPUS_OK; + int bitrate = 0; + int channels = cs->encoder_channels; + + rc = opus_encoder_ctl(cs->audio_encoder, OPUS_GET_BITRATE(&bitrate)); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while getting encoder ctl: %s", opus_strerror(rc)); + return -1; + } + + cs_disable_audio_sending(cs); + return cs_enable_audio_sending(cs, bitrate, channels); +} + +int cs_set_sending_audio_channels(CSSession* cs, int32_t count) +{ + /* TODO Find a better way? */ + if (cs->audio_encoder == NULL) + return -1; + + if (cs->encoder_channels == count) + return 0; + + int rc = OPUS_OK; + int bitrate = 0; + + rc = opus_encoder_ctl(cs->audio_encoder, OPUS_GET_BITRATE(&bitrate)); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while getting encoder ctl: %s", opus_strerror(rc)); + return -1; + } + + cs_disable_audio_sending(cs); + return cs_enable_audio_sending(cs, bitrate, count); +} + +void cs_disable_audio_sending(CSSession* cs) +{ + if ( cs->audio_encoder ) { + opus_encoder_destroy(cs->audio_encoder); + cs->audio_encoder = NULL; + cs->encoder_channels = 0; + } +} + +void cs_disable_audio_receiving(CSSession* cs) +{ + if ( cs->audio_decoder ) { + opus_decoder_destroy(cs->audio_decoder); + cs->audio_decoder = NULL; + jbuf_free(cs->j_buf); + cs->j_buf = NULL; + + /* It's used for measuring iteration interval so this has to be some value. + * To avoid unecessary checking we set this to 500 + */ + cs->last_packet_frame_duration = 500; + } +} + +int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate, int channels) +{ + if (cs->audio_encoder) + return 0; + + /** + * Encoder is initialized with default values. These values (Sampling rate, channel count) + * change on the fly from toxav. + */ + + int rc = OPUS_OK; + cs->audio_encoder = opus_encoder_create(48000, channels, OPUS_APPLICATION_AUDIO, &rc); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); + return -1; + } + + rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate)); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); + goto FAILURE; + } + + rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10)); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); + goto FAILURE; + } + + cs->encoder_channels = channels; + return 0; + +FAILURE: + cs_disable_audio_sending(cs); + return -1; +} + +int cs_enable_audio_receiving(CSSession* cs) +{ + if (cs->audio_decoder) + return 0; + + /** + * Decoder is initialized with default values. These values (Sampling rate, channel count) + * change on the fly from toxav. + */ + + int rc; + cs->audio_decoder = opus_decoder_create(48000, 2, &rc ); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc)); + return -1; + } + + + if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) { + LOGGER_WARNING("Jitter buffer creaton failed!"); + opus_decoder_destroy(cs->audio_decoder); + cs->audio_decoder = NULL; + return -1; + } + + /* It's used for measuring iteration interval so this has to be some value. + * To avoid unecessary checking we set this to 500 + */ + cs->last_packet_frame_duration = 500; + return 0; +} + + /* Called from RTP */ void queue_message(RTPSession *session, RTPMessage *msg) diff --git a/toxav/codec.h b/toxav/codec.h index 92262ef83..951d6d2f8 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -96,7 +96,7 @@ typedef struct _CSSession { uint32_t video_frame_piece_size; uint32_t max_video_frame_size; - /* Reassembling */ + /* Splitting */ uint8_t *split_video_frame; const uint8_t *processing_video_frame; uint16_t processing_video_frame_size; @@ -110,10 +110,13 @@ typedef struct _CSSession { /* audio encoding */ OpusEncoder *audio_encoder; - int32_t channels; + int32_t encoder_channels; /* audio decoding */ OpusDecoder *audio_decoder; + int32_t last_pack_channels; + int32_t last_packet_sampling_rate; + int32_t last_packet_frame_duration; struct _JitterBuffer *j_buf; @@ -127,55 +130,56 @@ typedef struct _CSSession { * * */ - - /* Callbacks */ - PAIR(CSAudioCallback, void *) acb; - PAIR(CSVideoCallback, void *) vcb; - - void *agent; /* Pointer to ToxAv */ + void *agent; /* Pointer to ToxAV TODO make this pointer to ToxAV*/ int32_t call_idx; pthread_mutex_t queue_mutex[1]; } CSSession; -int cs_split_video_payload(CSSession *cs, const uint8_t *payload, uint16_t length); -const uint8_t *cs_get_split_video_frame(CSSession *cs, uint16_t *size); /** - * Call playback callbacks + * Generic */ void cs_do(CSSession *cs); -/** - * Reconfigure video settings; return 0 on success or -1 on failure. +/* Make sure to be called BEFORE corresponding rtp_new */ +CSSession *cs_new(uint32_t s_audio_b, uint32_t p_audio_b, uint32_t s_video_b, uint32_t p_video_b); +/* Make sure to be called AFTER corresponding rtp_kill */ +void cs_kill(CSSession *cs); + + +/** + * VIDEO HANDLING */ +void cs_init_video_splitter_cycle(CSSession *cs); +int cs_update_video_splitter_cycle(CSSession* cs, const uint8_t* payload, uint16_t length); +const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size); + int cs_set_sending_video_resolution(CSSession *cs, uint16_t width, uint16_t height); int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate); -int cs_set_sending_audio_bitrate(CSSession* cs, int32_t rate); -/* NOTE: Try not to call these a lot */ -int cs_set_sending_audio_sampling_rate(CSSession* cs, int32_t rate); -int cs_set_sending_audio_channels(CSSession* cs, int32_t count); - -/** - * Make sure to be called BEFORE corresponding rtp_new - */ -CSSession *cs_new(uint32_t s_audio_b, uint32_t p_audio_b, uint32_t s_video_b, uint32_t p_video_b); -/** - * Make sure to be called AFTER corresponding rtp_kill - */ -void cs_kill(CSSession *cs); - -int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate); -int cs_enable_audio_receiving(CSSession* cs); int cs_enable_video_sending(CSSession* cs, uint32_t bitrate); int cs_enable_video_receiving(CSSession* cs); -void cs_disable_audio_sending(CSSession* cs); -void cs_disable_audio_receiving(CSSession* cs); void cs_disable_video_sending(CSSession* cs); void cs_disable_video_receiving(CSSession* cs); +/** + * AUDIO HANDLING + */ +int cs_set_sending_audio_bitrate(CSSession* cs, int32_t rate); +int cs_set_sending_audio_sampling_rate(CSSession* cs, int32_t rate); +int cs_set_sending_audio_channels(CSSession* cs, int32_t count); + +int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate, int channels); +int cs_enable_audio_receiving(CSSession* cs); + +void cs_disable_audio_sending(CSSession* cs); +void cs_disable_audio_receiving(CSSession* cs); + + + + /* Internal. Called from rtp_handle_message */ void queue_message(RTPSession *session, RTPMessage *msg); #endif /* _CODEC_H_ */ diff --git a/toxav/rtp.c b/toxav/rtp.c index ba93e7817..a50dd7ce6 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -488,13 +488,13 @@ int rtp_register_for_receiving(RTPSession* session) rtp_handle_packet, session); } -int rtp_send_msg ( RTPSession *session, Messenger *messenger, const uint8_t *data, uint16_t length ) +int rtp_send_msg ( RTPSession *session, const uint8_t *data, uint16_t length ) { RTPMessage *msg = rtp_new_message (session, data, length); if ( !msg ) return -1; - if ( -1 == send_custom_lossy_packet(messenger, session->dest, msg->data, msg->length) ) { + if ( -1 == send_custom_lossy_packet(session->m, session->dest, msg->data, msg->length) ) { LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno)); rtp_free_msg ( session, msg ); return rtp_ErrorSending; diff --git a/toxav/rtp.h b/toxav/rtp.h index b25b13ba1..03b44719d 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -121,7 +121,7 @@ int rtp_register_for_receiving (RTPSession *session); /** * Sends msg to _RTPSession::dest */ -int rtp_send_msg ( RTPSession *session, Messenger *messenger, const uint8_t *data, uint16_t length ); +int rtp_send_msg ( RTPSession* session, const uint8_t* data, uint16_t length ); /** * Dealloc msg. diff --git a/toxav/toxav.c b/toxav/toxav.c index 684020205..ee7f49a64 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -176,14 +176,14 @@ uint32_t toxav_do_interval(ToxAv *av) void toxav_do(ToxAv *av) { msi_do(av->msi_session); - + uint64_t start = current_time_monotonic(); - + uint32_t i = 0; - + for (; i < av->max_calls; i ++) { pthread_mutex_lock(av->calls[i].mutex_control); - + if (av->calls[i].active) { pthread_mutex_lock(av->calls[i].mutex_do); pthread_mutex_unlock(av->calls[i].mutex_control); @@ -193,12 +193,12 @@ void toxav_do(ToxAv *av) pthread_mutex_unlock(av->calls[i].mutex_control); } } - + uint64_t end = current_time_monotonic(); - + /* TODO maybe use variable for sizes */ av->dectmsstotal += end - start; - + if (++av->dectmsscount == 3) { av->avgdectms = av->dectmsstotal / 3 + 2 /* NOTE Magic Offset */; av->dectmsscount = 0; @@ -432,7 +432,7 @@ static int toxav_send_rtp_payload(ToxAv *av, int i; for (i = 0; i < parts; i++) { - iter = cs_get_split_video_frame(call->cs, &part_size); + iter = cs_iterate_split_video_frame(call->cs, &part_size); if (rtp_send_msg(call->crtps[video_index], av->messenger, iter, part_size) < 0) return av_ErrorSendingPayload; diff --git a/toxav/toxav_new.c b/toxav/toxav_new.c index d6c1872c8..857d5a838 100644 --- a/toxav/toxav_new.c +++ b/toxav/toxav_new.c @@ -34,6 +34,7 @@ #include #include +#define MAX_ENCODE_TIME_US ((1000 / 24) * 1000) enum { audio_index, @@ -75,6 +76,8 @@ struct toxAV int32_t dmssc; /** Measure count */ int32_t dmsst; /** Last cycle total */ int32_t dmssa; /** Average decoding time in ms */ + + uint32_t interval; /** Calculated interval */ }; @@ -130,6 +133,7 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) goto FAILURE; } + av->interval = 200; av->msi->agent_handler = av; msi_register_callback(av->msi, i_toxav_msi_callback_invite, msi_OnInvite, NULL); @@ -144,7 +148,7 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) msi_register_callback(av->msi, i_toxav_msi_callback_state_change, msi_OnSelfCSChange, NULL); - if (error) + if (error) *error = rc; return av; @@ -175,12 +179,32 @@ Tox* toxav_get_tox(ToxAV* av) uint32_t toxav_iteration_interval(const ToxAV* av) { - + return av->interval; } void toxav_iteration(ToxAV* av) { - + msi_do(av->msi); + + uint64_t start = current_time_monotonic(); + uint32_t rc = 200 + av->dmssa; /* If no call is active interval is 200 */ + + IToxAVCall* i = av->calls[av->calls_head]; + for (; i; i = i->next) { + if (i->active) { + cs_do(i->cs); + rc = MIN(i->cs->last_packet_frame_duration, rc); + } + } + + av->interval = rc < av->dmssa ? 0 : rc - av->dmssa; + av->dmsst += current_time_monotonic() - start; + + if (++av->dmssc == 3) { + av->dmssa = av->dmsst / 3 + 2 /* NOTE Magic Offset for precission */; + av->dmssc = 0; + av->dmsst = 0; + } } bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) @@ -261,52 +285,244 @@ void toxav_callback_call_state(ToxAV* av, toxav_call_state_cb* function, void* u bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL* error) { - + TOXAV_ERR_CALL_CONTROL rc = TOXAV_ERR_CALL_CONTROL_OK; + + if (m_friend_exists(av->m, friend_number)) { + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_FOUND; + goto END; + } + + + IToxAVCall* call = i_toxav_get_call(av, friend_number); + if (call == NULL) { + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } + + /* TODO rest of these */ + switch (control) + { + case TOXAV_CALL_CONTROL_RESUME: { + + } break; + + case TOXAV_CALL_CONTROL_PAUSE: { + + } break; + + case TOXAV_CALL_CONTROL_CANCEL: { + if (av->msi->calls[call->call_idx]->state == msi_CallActive) { + /* Hang up */ + msi_hangup(av->msi, call->call_idx); + } else if (av->msi->calls[call->call_idx]->state == msi_CallRequested) { + /* Reject the call */ + msi_reject(av->msi, call->call_idx); + } else if (av->msi->calls[call->call_idx]->state == msi_CallRequesting) { + /* Cancel the call */ + msi_cancel(av->msi, call->call_idx); + } + } break; + + case TOXAV_CALL_CONTROL_MUTE_AUDIO: { + + } break; + + case TOXAV_CALL_CONTROL_MUTE_VIDEO: { + + } break; + } + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_CALL_CONTROL_OK; } bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE* error) { - + /* TODO */ } bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE* error) { - + /* TODO */ } void toxav_callback_request_video_frame(ToxAV* av, toxav_request_video_frame_cb* function, void* user_data) { - + /* TODO */ } -bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, const uint8_t* a, TOXAV_ERR_SEND_FRAME* error) +bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) { - + TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; + IToxAVCall* call; + + if (m_friend_exists(av->m, friend_number)) { + rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND; + goto END; + } + + call = i_toxav_get_call(av, friend_number); + if (call == NULL) { + rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; + goto END; + } + + if (av->msi->calls[call->call_idx]->state != msi_CallActive) { + /* TODO */ + rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; + goto END; + } + + if ( y == NULL || u == NULL || v == NULL ) { + rc = TOXAV_ERR_SEND_FRAME_NULL; + goto END; + } + + if ( cs_set_sending_video_resolution(call->cs, width, height) != 0 ) { + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + + { /* Encode */ + vpx_image_t img; + img.w = img.h = img.d_w = img.d_h = 0; + vpx_img_alloc(&img, VPX_IMG_FMT_VPXI420, width, height, 1); + + /* I420 "It comprises an NxM Y plane followed by (N/2)x(M/2) V and U planes." + * http://fourcc.org/yuv.php#IYUV + */ + memcpy(img.planes[VPX_PLANE_Y], y, width * height); + memcpy(img.planes[VPX_PLANE_U], u, (width/2) * (height/2)); + memcpy(img.planes[VPX_PLANE_V], v, (width/2) * (height/2)); + + int vrc = vpx_codec_encode(call->cs->v_encoder, &img, + call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); + + vpx_img_free(&img); /* FIXME don't free? */ + if ( vrc != VPX_CODEC_OK) { + LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + } + + ++call->cs->frame_counter; + + { /* Split and send */ + vpx_codec_iter_t iter = NULL; + const vpx_codec_cx_pkt_t *pkt; + + cs_init_video_splitter_cycle(call->cs); + + while ( (pkt = vpx_codec_get_cx_data(call->cs->v_encoder, &iter)) ) { + if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { + int parts = cs_update_video_splitter_cycle(call->cs, pkt->data.frame.buf, + pkt->data.frame.sz); + + if (parts < 0) /* Should never happen though */ + continue; + + uint16_t part_size; + const uint8_t *iter; + + int i; + for (i = 0; i < parts; i++) { + iter = cs_iterate_split_video_frame(call->cs, &part_size); + + if (rtp_send_msg(call->rtps[video_index], iter, part_size) < 0) + goto END; + } + } + } + } + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_SEND_FRAME_OK; } void toxav_callback_request_audio_frame(ToxAV* av, toxav_request_audio_frame_cb* function, void* user_data) { - + /* TODO */ } bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error) { - + TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; + IToxAVCall* call; + + if (m_friend_exists(av->m, friend_number)) { + rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND; + goto END; + } + + call = i_toxav_get_call(av, friend_number); + if (call == NULL) { + rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; + goto END; + } + + if (av->msi->calls[call->call_idx]->state != msi_CallActive) { + /* TODO */ + rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; + goto END; + } + + if ( pcm == NULL ) { + rc = TOXAV_ERR_SEND_FRAME_NULL; + goto END; + } + + if ( channels != 1 || channels != 2 ) { + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + + { /* Encode and send */ + /* TODO redundant? */ + cs_set_sending_audio_channels(call->cs, channels); + cs_set_sending_audio_sampling_rate(call->cs, sampling_rate); + + uint8_t dest[sample_count * channels * 2 /* sizeof(uint16_t) */]; + int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest, sizeof (dest)); + + if (vrc < 0) { + LOGGER_WARNING("Failed to encode frame"); + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + + vrc = rtp_send_msg(call->rtps[audio_index], dest, vrc); + /* TODO check for error? */ + } + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_SEND_FRAME_OK; } void toxav_callback_receive_video_frame(ToxAV* av, toxav_receive_video_frame_cb* function, void* user_data) { - + av->vcb.first = function; + av->vcb.second = user_data; } void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* function, void* user_data) { - + av->acb.first = function; + av->acb.second = user_data; } /******************************************************************************* - * + * * :: Internal * ******************************************************************************/ @@ -616,12 +832,6 @@ bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call) call->cs->agent = av; call->cs->call_idx = call->call_idx; - call->cs->acb.first = av->acb.first; - call->cs->acb.second = av->acb.second; - - call->cs->vcb.first = av->vcb.first; - call->cs->vcb.second = av->vcb.second; - if (c_self->audio_bitrate > 0 || c_peer->audio_bitrate > 0) { /* Prepare audio rtp */ call->rtps[audio_index] = rtp_new(msi_TypeAudio, av->m, av->msi->calls[call->call_idx]->peers[0]); diff --git a/toxav/toxav_new.h b/toxav/toxav_new.h index 78e793576..038ee99ac 100644 --- a/toxav/toxav_new.h +++ b/toxav/toxav_new.h @@ -374,8 +374,9 @@ void toxav_callback_request_video_frame(ToxAV *av, toxav_request_video_frame_cb * * This is called in response to receiving the `request_video_frame` event. * - * Each plane should contain (width * height) pixels. The Alpha plane can be - * NULL, in which case every pixel is assumed fully opaque. + * Y - plane should be of size: height * width + * U - plane should be of size: (height/2) * (width/2) + * V - plane should be of size: (height/2) * (width/2) * * @param friend_number The friend number of the friend to which to send a video * frame. @@ -384,11 +385,10 @@ void toxav_callback_request_video_frame(ToxAV *av, toxav_request_video_frame_cb * @param y Y (Luminance) plane data. * @param u U (Chroma) plane data. * @param v V (Chroma) plane data. - * @param a A (Alpha) plane data. */ bool toxav_send_video_frame(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, - uint8_t const *y, uint8_t const *u, uint8_t const *v, uint8_t const *a, + uint8_t const *y, uint8_t const *u, uint8_t const *v, TOXAV_ERR_SEND_FRAME *error); /** * The function type for the `request_audio_frame` callback. From aad857527cd63b5f79786df0c1aab50f4de87774 Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 14 Feb 2015 23:37:52 +0100 Subject: [PATCH 04/97] Control part of new api already kind of works --- toxav/Makefile.inc | 17 + toxav/av_test.c | 339 +++++++++++ toxav/codec.c | 84 +-- toxav/codec.h | 6 +- toxav/toxav.c | 1406 +++++++++++++++++++++++++------------------ toxav/toxav.h | 796 ++++++++++++++---------- toxav/toxav_new.c | 920 ---------------------------- toxav/toxav_new.h | 481 --------------- toxav/toxav_new_1.c | 689 +++++++++++++++++++++ toxav/toxav_new_1.h | 329 ++++++++++ 10 files changed, 2728 insertions(+), 2339 deletions(-) create mode 100644 toxav/av_test.c delete mode 100644 toxav/toxav_new.c delete mode 100644 toxav/toxav_new.h create mode 100644 toxav/toxav_new_1.c create mode 100644 toxav/toxav_new_1.h diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc index 0b4b869d4..6458260d0 100644 --- a/toxav/Makefile.inc +++ b/toxav/Makefile.inc @@ -35,4 +35,21 @@ libtoxav_la_LIBADD = libtoxcore.la \ $(PTHREAD_LIBS) \ $(AV_LIBS) + +noinst_PROGRAMS += av_test + +av_test_SOURCES = ../toxav/av_test.c + +av_test_CFLAGS = $(LIBSODIUM_CFLAGS) \ + $(NACL_CFLAGS) + +av_test_LDADD = $(LIBSODIUM_LDFLAGS) \ + $(NACL_LDFLAGS) \ + libtoxav.la \ + libtoxcore.la \ + $(LIBSODIUM_LIBS) \ + $(NACL_OBJECTS) \ + $(NACL_LIBS) + + endif \ No newline at end of file diff --git a/toxav/av_test.c b/toxav/av_test.c new file mode 100644 index 000000000..1e5e4ad73 --- /dev/null +++ b/toxav/av_test.c @@ -0,0 +1,339 @@ +#include "toxav.h" +#include "../toxcore/tox.h" + +#include +#include +#include +#include +#include + +#if defined(_WIN32) || defined(__WIN32__) || defined (WIN32) +#define c_sleep(x) Sleep(1*x) +#else +#include +#define c_sleep(x) usleep(1000*x) +#endif + +typedef struct { + bool incoming; + bool ringing; + bool ended; + bool errored; + bool sending; + bool paused; +} CallControl; + + +/** + * Callbacks + */ +void t_toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data) +{ + printf("Handling CALL callback\n"); + ((CallControl*)user_data)->incoming = true; +} +void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, TOXAV_CALL_STATE state, void *user_data) +{ + printf("Handling CALL STATE callback: "); + + if (((CallControl*)user_data)->ringing) + ((CallControl*)user_data)->ringing = false; + + if (((CallControl*)user_data)->paused) + ((CallControl*)user_data)->paused = false; + + switch (state) + { + case TOXAV_CALL_STATE_RINGING: { + printf("Ringing"); + ((CallControl*)user_data)->ringing = true; + } break; + + case TOXAV_CALL_STATE_NOT_SENDING: { + printf("Not sending"); + ((CallControl*)user_data)->sending = false; + } break; + + case TOXAV_CALL_STATE_SENDING_A: { + printf("Sending Audio"); + ((CallControl*)user_data)->sending = true; + } break; + + case TOXAV_CALL_STATE_SENDING_V: { + printf("Sending Video"); + ((CallControl*)user_data)->sending = true; + } break; + + case TOXAV_CALL_STATE_SENDING_AV: { + printf("Sending Both"); + ((CallControl*)user_data)->sending = true; + } break; + + case TOXAV_CALL_STATE_PAUSED: { + printf("Paused"); + ((CallControl*)user_data)->paused = true; + ((CallControl*)user_data)->sending = false; + } break; + + case TOXAV_CALL_STATE_END: { + printf("Ended"); + ((CallControl*)user_data)->ended = true; + ((CallControl*)user_data)->sending = false; + } break; + + case TOXAV_CALL_STATE_ERROR: { + printf("Error"); + ((CallControl*)user_data)->errored = true; + ((CallControl*)user_data)->sending = false; + } break; + } + + printf("\n"); +} +void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, + uint16_t width, uint16_t height, + uint8_t const *planes[], int32_t const stride[], + void *user_data) +{ + printf("Handling VIDEO FRAME callback\n"); +} +void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, + int16_t const *pcm, + size_t sample_count, + uint8_t channels, + uint32_t sampling_rate, + void *user_data) +{ + printf("Handling AUDIO FRAME callback\n"); +} +void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, uint16_t length, void *userdata) +{ + if (length == 7 && memcmp("gentoo", data, 7) == 0) { + tox_add_friend_norequest(m, public_key); + } +} + + +/** + */ +void prepare(Tox* Bsn, Tox* Alice, Tox* Bob) +{ + long long unsigned int cur_time = time(NULL); + + uint32_t to_compare = 974536; + uint8_t address[TOX_FRIEND_ADDRESS_SIZE]; + + tox_callback_friend_request(Alice, t_accept_friend_request_cb, &to_compare); + tox_get_address(Alice, address); + + assert(tox_add_friend(Bob, address, (uint8_t *)"gentoo", 7) >= 0); + + uint8_t off = 1; + + while (1) { + tox_do(Bsn); + tox_do(Alice); + tox_do(Bob); + + if (tox_isconnected(Bsn) && tox_isconnected(Alice) && tox_isconnected(Bob) && off) { + printf("Toxes are online, took %llu seconds\n", time(NULL) - cur_time); + off = 0; + } + + if (tox_get_friend_connection_status(Alice, 0) == 1 && tox_get_friend_connection_status(Bob, 0) == 1) + break; + + c_sleep(20); + } + + printf("All set after %llu seconds!\n", time(NULL) - cur_time); +} +void prepareAV(ToxAV* AliceAV, void* AliceUD, ToxAV* BobAV, void* BobUD) +{ + /* Alice */ + toxav_callback_call(AliceAV, t_toxav_call_cb, AliceUD); + toxav_callback_call_state(AliceAV, t_toxav_call_state_cb, AliceUD); + toxav_callback_receive_video_frame(AliceAV, t_toxav_receive_video_frame_cb, AliceUD); + toxav_callback_receive_audio_frame(AliceAV, t_toxav_receive_audio_frame_cb, AliceUD); + + /* Bob */ + toxav_callback_call(BobAV, t_toxav_call_cb, BobUD); + toxav_callback_call_state(BobAV, t_toxav_call_state_cb, BobUD); + toxav_callback_receive_video_frame(BobAV, t_toxav_receive_video_frame_cb, BobUD); + toxav_callback_receive_audio_frame(BobAV, t_toxav_receive_audio_frame_cb, BobUD); +} +void iterate(Tox* Bsn, ToxAV* AliceAV, ToxAV* BobAV) +{ + tox_do(Bsn); + tox_do(toxav_get_tox(AliceAV)); + tox_do(toxav_get_tox(BobAV)); + + toxav_iteration(AliceAV); + toxav_iteration(BobAV); + + c_sleep(20); +} + + +int main (int argc, char** argv) +{ + Tox *Bsn = tox_new(0); + Tox *Alice = tox_new(0); + Tox *Bob = tox_new(0); + + assert(Bsn && Alice && Bob); + + prepare(Bsn, Alice, Bob); + + + ToxAV *AliceAV, *BobAV; + CallControl AliceCC, BobCC; + + { + TOXAV_ERR_NEW rc; + AliceAV = toxav_new(Alice, &rc); + assert(rc == TOXAV_ERR_NEW_OK); + + BobAV = toxav_new(Bob, &rc); + assert(rc == TOXAV_ERR_NEW_OK); + + prepareAV(AliceAV, &AliceCC, BobAV, &BobCC); + printf("Created 2 instances of ToxAV\n"); + } + + +#define REGULAR_CALL_FLOW(A_BR, V_BR) \ + { \ + memset(&AliceCC, 0, sizeof(CallControl)); \ + memset(&BobCC, 0, sizeof(CallControl)); \ + \ + TOXAV_ERR_CALL rc; \ + toxav_call(AliceAV, 0, A_BR, V_BR, &rc); \ + \ + if (rc != TOXAV_ERR_CALL_OK) { \ + printf("toxav_call failed: %d\n", rc); \ + exit(1); \ + } \ + \ + \ + long long unsigned int start_time = time(NULL); \ + \ + \ + while (!AliceCC.ended || !BobCC.ended) { \ + \ + if (BobCC.incoming) { \ + TOXAV_ERR_ANSWER rc; \ + toxav_answer(BobAV, 0, 48, 4000, &rc); \ + \ + if (rc != TOXAV_ERR_ANSWER_OK) { \ + printf("toxav_answer failed: %d\n", rc); \ + exit(1); \ + } \ + BobCC.incoming = false; \ + } \ + else if (AliceCC.sending && BobCC.sending) { \ + /* TODO rtp */ \ + \ + if (time(NULL) - start_time == 5) { \ + \ + TOXAV_ERR_CALL_CONTROL rc; \ + toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); \ + \ + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { \ + printf("toxav_call_control failed: %d\n", rc); \ + exit(1); \ + } \ + } \ + } \ + \ + iterate(Bsn, AliceAV, BobAV); \ + } \ + printf("Success!\n");\ + } + + printf("\nTrying regular call (Audio and Video)...\n"); +// REGULAR_CALL_FLOW(48, 4000); + + printf("\nTrying regular call (Audio only)...\n"); +// REGULAR_CALL_FLOW(48, 0); + + printf("\nTrying regular call (Video only)...\n"); +// REGULAR_CALL_FLOW(0, 4000); + +#undef REGULAR_CALL_FLOW + + { /* Alice calls; Bob rejects */ + printf("\nTrying reject flow...\n"); + + memset(&AliceCC, 0, sizeof(CallControl)); + memset(&BobCC, 0, sizeof(CallControl)); + + { + TOXAV_ERR_CALL rc; + toxav_call(AliceAV, 0, 48, 0, &rc); + + if (rc != TOXAV_ERR_CALL_OK) { + printf("toxav_call failed: %d\n", rc); + exit(1); + } + } + + while (!BobCC.incoming) + iterate(Bsn, AliceAV, BobAV); + + /* Reject */ + { + TOXAV_ERR_CALL_CONTROL rc; + toxav_call_control(BobAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); + + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { + printf("toxav_call_control failed: %d\n", rc); + exit(1); + } + } + + while (!AliceCC.ended || !BobCC.ended) + iterate(Bsn, AliceAV, BobAV); + + printf("Success!\n"); + } + + { /* Alice calls; Alice cancels while ringing */ + printf("\nTrying cancel (while ringing) flow...\n"); + + memset(&AliceCC, 0, sizeof(CallControl)); + memset(&BobCC, 0, sizeof(CallControl)); + + { + TOXAV_ERR_CALL rc; + toxav_call(AliceAV, 0, 48, 0, &rc); + + if (rc != TOXAV_ERR_CALL_OK) { + printf("toxav_call failed: %d\n", rc); + exit(1); + } + } + + while (!BobCC.incoming) + iterate(Bsn, AliceAV, BobAV); + + /* Cancel */ + { + TOXAV_ERR_CALL_CONTROL rc; + toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); + + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { + printf("toxav_call_control failed: %d\n", rc); + exit(1); + } + } + + while (!AliceCC.ended || !BobCC.ended) + iterate(Bsn, AliceAV, BobAV); + + printf("Success!\n"); + } + + printf("\nTest successful!\n"); + return 0; +} \ No newline at end of file diff --git a/toxav/codec.c b/toxav/codec.c index 43120e0f5..e6fe713ea 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -40,7 +40,6 @@ #include "rtp.h" #include "codec.h" - #define DEFAULT_JBUF 6 /* Good quality encode. */ @@ -125,7 +124,7 @@ static void buffer_free(PayloadBuffer *b) } /* JITTER BUFFER WORK */ -typedef struct { +typedef struct JitterBuffer { RTPMessage **queue; uint32_t size; uint32_t capacity; @@ -260,48 +259,51 @@ void cs_do(CSSession *cs) int success = 0; pthread_mutex_lock(cs->queue_mutex); - RTPMessage *msg; - uint16_t fsize = 5760; /* Max frame size for 48 kHz */ - int16_t tmp[fsize * 2]; - - while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { - pthread_mutex_unlock(cs->queue_mutex); + if (cs->audio_decoder) { /* If receiving enabled */ + RTPMessage *msg; - if (success == 2) { - rc = opus_decode(cs->audio_decoder, 0, 0, tmp, fsize, 1); - } else { - /* Get values from packet and decode. - * It also checks for validity of an opus packet - */ - rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); - if (rc != -1) { - cs->last_packet_sampling_rate = rc; - cs->last_pack_channels = opus_packet_get_nb_channels(msg->data); + uint16_t fsize = 5760; /* Max frame size for 48 kHz */ + int16_t tmp[fsize * 2]; + + while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { + pthread_mutex_unlock(cs->queue_mutex); - cs->last_packet_frame_duration = - ( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 ) - / cs->last_packet_sampling_rate; - + if (success == 2) { + rc = opus_decode(cs->audio_decoder, 0, 0, tmp, fsize, 1); } else { - LOGGER_WARNING("Failed to load packet values!"); + /* Get values from packet and decode. + * It also checks for validity of an opus packet + */ + rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); + if (rc != -1) { + cs->last_packet_sampling_rate = rc; + cs->last_pack_channels = opus_packet_get_nb_channels(msg->data); + + cs->last_packet_frame_duration = + ( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 ) + / cs->last_packet_sampling_rate; + + } else { + LOGGER_WARNING("Failed to load packet values!"); + rtp_free_msg(NULL, msg); + continue; + } + + rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, fsize, 0); rtp_free_msg(NULL, msg); - continue; } - rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, fsize, 0); - rtp_free_msg(NULL, msg); + if (rc < 0) { + LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); + } else if (cs->acb.first) { + /* Play */ + cs->acb.first(cs->agent, cs->friend_number, tmp, rc, + cs->last_pack_channels, cs->last_packet_sampling_rate, cs->acb.second); + } + + pthread_mutex_lock(cs->queue_mutex); } - - if (rc < 0) { - LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); - } else if (((ToxAV*)cs->agent)->acb.first) { - /* Play */ - ((ToxAV*)cs->agent)->acb.first(cs->agent, cs->call_idx, tmp, rc, - ((ToxAV*)cs->agent)->acb.second); - } - - pthread_mutex_lock(cs->queue_mutex); } if (cs->vbuf_raw && !buffer_empty(cs->vbuf_raw)) { @@ -322,11 +324,11 @@ void cs_do(CSSession *cs) /* Play decoded images */ for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) { - if (((ToxAV*)cs->agent)->vcb.first) - ((ToxAV*)cs->agent)->vcb.first(cs->agent, cs->call_idx, dest, - ((ToxAV*)cs->agent)->vcb.second); - - vpx_img_free(dest); + if (cs->vcb.first) + cs->vcb.first(cs->agent, cs->call_idx, dest->d_w, dest->d_h, + (const uint8_t**)dest->planes, dest->stride, cs->vcb.second); + + vpx_img_free(dest); } } diff --git a/toxav/codec.h b/toxav/codec.h index 951d6d2f8..de5a6cd1c 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -117,7 +117,7 @@ typedef struct _CSSession { int32_t last_pack_channels; int32_t last_packet_sampling_rate; int32_t last_packet_frame_duration; - struct _JitterBuffer *j_buf; + struct JitterBuffer *j_buf; /* Voice activity detection */ @@ -132,6 +132,10 @@ typedef struct _CSSession { */ void *agent; /* Pointer to ToxAV TODO make this pointer to ToxAV*/ int32_t call_idx; + int32_t friend_number; + + PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ + PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ pthread_mutex_t queue_mutex[1]; } CSSession; diff --git a/toxav/toxav.c b/toxav/toxav.c index ee7f49a64..d71bbd6dc 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -1,5 +1,5 @@ /** toxav.c - * + * * Copyright (C) 2013 Tox project All Rights Reserved. * * This file is part of Tox. @@ -23,15 +23,9 @@ #include "config.h" #endif /* HAVE_CONFIG_H */ -#define __TOX_DEFINED__ -typedef struct Messenger Tox; - -#define _GNU_SOURCE /* implicit declaration warning */ - -#include "codec.h" -#include "msi.h" -#include "group.h" +#include "msi.h" /* Includes codec.h, rtp.h and toxav.h */ +#include "../toxcore/Messenger.h" #include "../toxcore/logger.h" #include "../toxcore/util.h" @@ -39,651 +33,913 @@ typedef struct Messenger Tox; #include #include -/* Assume 24 fps*/ #define MAX_ENCODE_TIME_US ((1000 / 24) * 1000) -/* true if invalid call index */ -#define CALL_INVALID_INDEX(idx, max) (idx < 0 || idx >= max) - -const ToxAvCSettings av_DefaultSettings = { - av_TypeAudio, - - 500, - 1280, - 720, - - 32000, - 20, - 48000, - 1 +enum { + audio_index, + video_index, }; -static const uint32_t jbuf_capacity = 6; -static const uint8_t audio_index = 0, video_index = 1; - -typedef struct _ToxAvCall { +typedef struct iToxAVCall +{ pthread_mutex_t mutex_control[1]; pthread_mutex_t mutex_encoding_audio[1]; pthread_mutex_t mutex_encoding_video[1]; pthread_mutex_t mutex_do[1]; - RTPSession *crtps[2]; /** Audio is first and video is second */ + RTPSession *rtps[2]; /** Audio is first and video is second */ CSSession *cs; - _Bool active; -} ToxAvCall; + bool active; + int32_t friend_number; + int32_t call_idx; /* FIXME msi compat, remove */ + + struct iToxAVCall *prev; + struct iToxAVCall *next; +} IToxAVCall; -struct _ToxAv { - Messenger *messenger; - MSISession *msi_session; /** Main msi session */ - ToxAvCall *calls; /** Per-call params */ - uint32_t max_calls; - - PAIR(ToxAvAudioCallback, void *) acb; - PAIR(ToxAvVideoCallback, void *) vcb; - - /* Decode time measure */ - int32_t dectmsscount; /** Measure count */ - int32_t dectmsstotal; /** Last cycle total */ - int32_t avgdectms; /** Average decoding time in ms */ +struct toxAV +{ + Messenger* m; + MSISession* msi; + + /* Two-way storage: first is array of calls and second is list of calls with head and tail */ + IToxAVCall** calls; + uint32_t calls_tail; + uint32_t calls_head; + + PAIR(toxav_call_cb *, void*) ccb; /* Call callback */ + PAIR(toxav_call_state_cb *, void *) scb; /* Call state callback */ + PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ + PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ + + /** Decode time measures */ + int32_t dmssc; /** Measure count */ + int32_t dmsst; /** Last cycle total */ + int32_t dmssa; /** Average decoding time in ms */ + + uint32_t interval; /** Calculated interval */ }; -static const MSICSettings *msicsettings_cast (const ToxAvCSettings *from) + +void i_toxav_msi_callback_invite(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_ringing(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_start(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_cancel(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_reject(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_end(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_request_to(void* toxav_inst, int32_t call_idx, void *data); /* TODO remove */ +void i_toxav_msi_callback_peer_to(void* toxav_inst, int32_t call_idx, void *data); +void i_toxav_msi_callback_state_change(void* toxav_inst, int32_t call_idx, void *data); + +IToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number); +IToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number); +void i_toxav_remove_call(ToxAV* av, uint32_t friend_number); +bool i_toxav_audio_bitrate_invalid(uint32_t bitrate); +bool i_toxav_video_bitrate_invalid(uint32_t bitrate); +IToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error); +bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call); +void i_toxav_kill_transmission(ToxAV* av, uint32_t friend_number); + + + +ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) { - assert(sizeof(MSICSettings) == sizeof(ToxAvCSettings)); - return (const MSICSettings *) from; -} - -static const ToxAvCSettings *toxavcsettings_cast (const MSICSettings *from) -{ - assert(sizeof(MSICSettings) == sizeof(ToxAvCSettings)); - return (const ToxAvCSettings *) from; - -} - -ToxAv *toxav_new( Tox *messenger, int32_t max_calls) -{ - ToxAv *av = calloc ( sizeof(ToxAv), 1); - + TOXAV_ERR_NEW rc = TOXAV_ERR_NEW_OK; + ToxAV *av = NULL; + + if (tox == NULL) { + rc = TOXAV_ERR_NEW_NULL; + goto FAILURE; + } + + if (((Messenger*)tox)->msi_packet) { + rc = TOXAV_ERR_NEW_MULTIPLE; + goto FAILURE; + } + + av = calloc ( sizeof(ToxAV), 1); + if (av == NULL) { LOGGER_WARNING("Allocation failed!"); - return NULL; + rc = TOXAV_ERR_NEW_MALLOC; + goto FAILURE; } - - av->messenger = (Messenger *)messenger; - av->msi_session = msi_new(av->messenger, max_calls); - av->msi_session->agent_handler = av; - av->calls = calloc(sizeof(ToxAvCall), max_calls); - av->max_calls = max_calls; - - unsigned int i; - - for (i = 0; i < max_calls; ++i) { - if (create_recursive_mutex(av->calls[i].mutex_control) != 0 ) { - LOGGER_WARNING("Failed to init call(%u) mutex!", i); - msi_kill(av->msi_session); - - free(av->calls); - free(av); - return NULL; - } + + av->m = (Messenger *)tox; + av->msi = msi_new(av->m, 100); /* TODO remove max calls */ + + if (av->msi == NULL) { + rc = TOXAV_ERR_NEW_MALLOC; + goto FAILURE; } - + + av->interval = 200; + av->msi->agent_handler = av; + + msi_register_callback(av->msi, i_toxav_msi_callback_invite, msi_OnInvite, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_ringing, msi_OnRinging, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_start, msi_OnStart, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_cancel, msi_OnCancel, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_reject, msi_OnReject, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_end, msi_OnEnd, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_request_to, msi_OnRequestTimeout, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_peer_to, msi_OnPeerTimeout, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_state_change, msi_OnPeerCSChange, NULL); + msi_register_callback(av->msi, i_toxav_msi_callback_state_change, msi_OnSelfCSChange, NULL); + + + if (error) + *error = rc; + return av; + +FAILURE: + if (error) + *error = rc; + + free(av); + + return NULL; } -void toxav_kill ( ToxAv *av ) +void toxav_kill(ToxAV* av) { - uint32_t i; - - for (i = 0; i < av->max_calls; i ++) { - if ( av->calls[i].crtps[audio_index] ) - rtp_kill(av->calls[i].crtps[audio_index], av->msi_session->messenger_handle); - - - if ( av->calls[i].crtps[video_index] ) - rtp_kill(av->calls[i].crtps[video_index], av->msi_session->messenger_handle); - - if ( av->calls[i].cs ) - cs_kill(av->calls[i].cs); - - pthread_mutex_destroy(av->calls[i].mutex_control); - } - - msi_kill(av->msi_session); - - free(av->calls); + if (av == NULL) + return; + + msi_kill(av->msi); + /* TODO iterate over calls */ free(av); } -uint32_t toxav_do_interval(ToxAv *av) +Tox* toxav_get_tox(ToxAV* av) { - int i = 0; - uint32_t rc = 200 + av->avgdectms; /* Return 200 if no call is active */ - - for (; i < av->max_calls; i ++) { - pthread_mutex_lock(av->calls[i].mutex_control); - - if (av->calls[i].active) { - /* This should work. Video payload will always come in greater intervals */ - rc = MIN(av->calls[i].cs->audio_decoder_frame_duration, rc); - } - - pthread_mutex_unlock(av->calls[i].mutex_control); - } - - return rc < av->avgdectms ? 0 : rc - av->avgdectms; + return (Tox*) av->m; } -void toxav_do(ToxAv *av) +uint32_t toxav_iteration_interval(const ToxAV* av) { - msi_do(av->msi_session); + return av->calls ? av->interval : 200; +} + +void toxav_iteration(ToxAV* av) +{ + msi_do(av->msi); + + if (av->calls == NULL) + return; uint64_t start = current_time_monotonic(); + uint32_t rc = 200 + av->dmssa; /* If no call is active interval is 200 */ - uint32_t i = 0; - - for (; i < av->max_calls; i ++) { - pthread_mutex_lock(av->calls[i].mutex_control); - - if (av->calls[i].active) { - pthread_mutex_lock(av->calls[i].mutex_do); - pthread_mutex_unlock(av->calls[i].mutex_control); - cs_do(av->calls[i].cs); - pthread_mutex_unlock(av->calls[i].mutex_do); - } else { - pthread_mutex_unlock(av->calls[i].mutex_control); + IToxAVCall* i = av->calls[av->calls_head]; + for (; i; i = i->next) { + if (i->active) { + cs_do(i->cs); + rc = MIN(i->cs->last_packet_frame_duration, rc); } } - uint64_t end = current_time_monotonic(); + av->interval = rc < av->dmssa ? 0 : rc - av->dmssa; + av->dmsst += current_time_monotonic() - start; - /* TODO maybe use variable for sizes */ - av->dectmsstotal += end - start; - - if (++av->dectmsscount == 3) { - av->avgdectms = av->dectmsstotal / 3 + 2 /* NOTE Magic Offset */; - av->dectmsscount = 0; - av->dectmsstotal = 0; + if (++av->dmssc == 3) { + av->dmssa = av->dmsst / 3 + 2 /* NOTE Magic Offset for precission */; + av->dmssc = 0; + av->dmsst = 0; } } -void toxav_register_callstate_callback ( ToxAv *av, ToxAVCallback cb, ToxAvCallbackID id, void *userdata ) +bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) { - msi_register_callback(av->msi_session, (MSICallbackType)cb, (MSICallbackID) id, userdata); -} - -void toxav_register_audio_callback(ToxAv *av, ToxAvAudioCallback cb, void *userdata) -{ - av->acb.first = cb; - av->acb.second = userdata; -} - -void toxav_register_video_callback(ToxAv *av, ToxAvVideoCallback cb, void *userdata) -{ - av->vcb.first = cb; - av->vcb.second = userdata; -} - -int toxav_call (ToxAv *av, - int32_t *call_index, - int user, - const ToxAvCSettings *csettings, - int ringing_seconds ) -{ - return msi_invite(av->msi_session, call_index, msicsettings_cast(csettings), ringing_seconds * 1000, user); -} - -int toxav_hangup ( ToxAv *av, int32_t call_index ) -{ - return msi_hangup(av->msi_session, call_index); -} - -int toxav_answer ( ToxAv *av, int32_t call_index, const ToxAvCSettings *csettings ) -{ - return msi_answer(av->msi_session, call_index, msicsettings_cast(csettings)); -} - -int toxav_reject ( ToxAv *av, int32_t call_index, const char *reason ) -{ - return msi_reject(av->msi_session, call_index, reason); -} - -int toxav_cancel ( ToxAv *av, int32_t call_index, int peer_id, const char *reason ) -{ - return msi_cancel(av->msi_session, call_index, peer_id, reason); -} - -int toxav_change_settings(ToxAv *av, int32_t call_index, const ToxAvCSettings *csettings) -{ - return msi_change_csettings(av->msi_session, call_index, msicsettings_cast(csettings)); -} - -int toxav_stop_call ( ToxAv *av, int32_t call_index ) -{ - return msi_stopcall(av->msi_session, call_index); -} - -int toxav_prepare_transmission ( ToxAv *av, int32_t call_index, int support_video ) -{ - if ( !av->msi_session || CALL_INVALID_INDEX(call_index, av->msi_session->max_calls) || - !av->msi_session->calls[call_index] || !av->msi_session->calls[call_index]->csettings_peer) { - LOGGER_ERROR("Error while starting RTP session: invalid call!\n"); - return av_ErrorNoCall; + IToxAVCall* call = i_toxav_init_call(av, friend_number, audio_bit_rate, video_bit_rate, error); + if (call == NULL) { + return false; } + + /* TODO remove csettings */ + MSICSettings csets; + csets.audio_bitrate = audio_bit_rate; + csets.video_bitrate = video_bit_rate; + + csets.call_type = video_bit_rate ? msi_TypeVideo : msi_TypeAudio; + + if (msi_invite(av->msi, &call->call_idx, &csets, 1000, friend_number) != 0) { + i_toxav_remove_call(av, friend_number); + if (error) + *error = TOXAV_ERR_CALL_MALLOC; /* FIXME: this should be the only reason to fail */ + return false; + } + + return true; +} - ToxAvCall *call = &av->calls[call_index]; +void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data) +{ + av->ccb.first = function; + av->ccb.second = user_data; +} +bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER* error) +{ + TOXAV_ERR_ANSWER rc = TOXAV_ERR_ANSWER_OK; + if (m_friend_exists(av->m, friend_number) == 0) { + rc = TOXAV_ERR_ANSWER_FRIEND_NOT_FOUND; + goto END; + } + + if ((audio_bit_rate && i_toxav_audio_bitrate_invalid(audio_bit_rate)) + ||(video_bit_rate && i_toxav_video_bitrate_invalid(video_bit_rate)) + ) { + rc = TOXAV_ERR_CALL_INVALID_BIT_RATE; + goto END; + } + + IToxAVCall* call = i_toxav_get_call(av, friend_number); + if (call == NULL || av->msi->calls[call->call_idx]->state != msi_CallRequested) { + rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; + goto END; + } + + /* TODO remove csettings */ + MSICSettings csets; + csets.audio_bitrate = audio_bit_rate; + csets.video_bitrate = video_bit_rate; + + csets.call_type = video_bit_rate ? msi_TypeVideo : msi_TypeAudio; + + if (msi_answer(av->msi, call->call_idx, &csets) != 0) { + rc = TOXAV_ERR_ANSWER_MALLOC; /* TODO Some error here */ + /* TODO Reject call? */ + } + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_ANSWER_OK; +} + +void toxav_callback_call_state(ToxAV* av, toxav_call_state_cb* function, void* user_data) +{ + av->scb.first = function; + av->scb.second = user_data; +} + +bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL* error) +{ + TOXAV_ERR_CALL_CONTROL rc = TOXAV_ERR_CALL_CONTROL_OK; + + if (m_friend_exists(av->m, friend_number) == 0) { + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_FOUND; + goto END; + } + + + IToxAVCall* call = i_toxav_get_call(av, friend_number); + if (call == NULL) { + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } + + /* TODO rest of these */ + switch (control) + { + case TOXAV_CALL_CONTROL_RESUME: { + + } break; + + case TOXAV_CALL_CONTROL_PAUSE: { + + } break; + + case TOXAV_CALL_CONTROL_CANCEL: { + if (av->msi->calls[call->call_idx]->state == msi_CallActive) { + /* Hang up */ + msi_hangup(av->msi, call->call_idx); + } else if (av->msi->calls[call->call_idx]->state == msi_CallRequested) { + /* Reject the call */ + msi_reject(av->msi, call->call_idx, NULL); + } else if (av->msi->calls[call->call_idx]->state == msi_CallRequesting) { + /* Cancel the call */ + msi_cancel(av->msi, call->call_idx, 0, NULL); + } + } break; + + case TOXAV_CALL_CONTROL_MUTE_AUDIO: { + + } break; + + case TOXAV_CALL_CONTROL_MUTE_VIDEO: { + + } break; + } + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_CALL_CONTROL_OK; +} + +bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE* error) +{ + /* TODO */ +} + +bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE* error) +{ + /* TODO */ +} + +void toxav_callback_request_video_frame(ToxAV* av, toxav_request_video_frame_cb* function, void* user_data) +{ + /* TODO */ +} + +bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) +{ + TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; + IToxAVCall* call; + + if (m_friend_exists(av->m, friend_number) == 0) { + rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND; + goto END; + } + + call = i_toxav_get_call(av, friend_number); + if (call == NULL) { + rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; + goto END; + } + + if (av->msi->calls[call->call_idx]->state != msi_CallActive) { + /* TODO */ + rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; + goto END; + } + + if ( y == NULL || u == NULL || v == NULL ) { + rc = TOXAV_ERR_SEND_FRAME_NULL; + goto END; + } + + if ( cs_set_sending_video_resolution(call->cs, width, height) != 0 ) { + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + + { /* Encode */ + vpx_image_t img; + img.w = img.h = img.d_w = img.d_h = 0; + vpx_img_alloc(&img, VPX_IMG_FMT_VPXI420, width, height, 1); + + /* I420 "It comprises an NxM Y plane followed by (N/2)x(M/2) V and U planes." + * http://fourcc.org/yuv.php#IYUV + */ + memcpy(img.planes[VPX_PLANE_Y], y, width * height); + memcpy(img.planes[VPX_PLANE_U], u, (width/2) * (height/2)); + memcpy(img.planes[VPX_PLANE_V], v, (width/2) * (height/2)); + + int vrc = vpx_codec_encode(call->cs->v_encoder, &img, + call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); + + vpx_img_free(&img); /* FIXME don't free? */ + if ( vrc != VPX_CODEC_OK) { + LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + } + + ++call->cs->frame_counter; + + { /* Split and send */ + vpx_codec_iter_t iter = NULL; + const vpx_codec_cx_pkt_t *pkt; + + cs_init_video_splitter_cycle(call->cs); + + while ( (pkt = vpx_codec_get_cx_data(call->cs->v_encoder, &iter)) ) { + if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { + int parts = cs_update_video_splitter_cycle(call->cs, pkt->data.frame.buf, + pkt->data.frame.sz); + + if (parts < 0) /* Should never happen though */ + continue; + + uint16_t part_size; + const uint8_t *iter; + + int i; + for (i = 0; i < parts; i++) { + iter = cs_iterate_split_video_frame(call->cs, &part_size); + + if (rtp_send_msg(call->rtps[video_index], iter, part_size) < 0) + goto END; + } + } + } + } + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_SEND_FRAME_OK; +} + +void toxav_callback_request_audio_frame(ToxAV* av, toxav_request_audio_frame_cb* function, void* user_data) +{ + /* TODO */ +} + +bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error) +{ + TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; + IToxAVCall* call; + + if (m_friend_exists(av->m, friend_number) == 0) { + rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND; + goto END; + } + + call = i_toxav_get_call(av, friend_number); + if (call == NULL) { + rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; + goto END; + } + + if (av->msi->calls[call->call_idx]->state != msi_CallActive) { + /* TODO */ + rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; + goto END; + } + + if ( pcm == NULL ) { + rc = TOXAV_ERR_SEND_FRAME_NULL; + goto END; + } + + if ( channels != 1 || channels != 2 ) { + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + + { /* Encode and send */ + /* TODO redundant? */ + cs_set_sending_audio_channels(call->cs, channels); + cs_set_sending_audio_sampling_rate(call->cs, sampling_rate); + + uint8_t dest[sample_count * channels * 2 /* sizeof(uint16_t) */]; + int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest, sizeof (dest)); + + if (vrc < 0) { + LOGGER_WARNING("Failed to encode frame"); + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + + vrc = rtp_send_msg(call->rtps[audio_index], dest, vrc); + /* TODO check for error? */ + } + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_SEND_FRAME_OK; +} + +void toxav_callback_receive_video_frame(ToxAV* av, toxav_receive_video_frame_cb* function, void* user_data) +{ + av->vcb.first = function; + av->vcb.second = user_data; +} + +void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* function, void* user_data) +{ + av->acb.first = function; + av->acb.second = user_data; +} + + +/******************************************************************************* + * + * :: Internal + * + ******************************************************************************/ +/** TODO: + * - In msi call_idx can be the same as friend id + * - If crutial callback not present send error + * - Remove *data from msi + * - Remove CSettings from msi + */ +void i_toxav_msi_callback_invite(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + + uint32_t ab = toxav->msi->calls[call_idx]->csettings_peer[0].audio_bitrate; + uint32_t vb = toxav->msi->calls[call_idx]->csettings_peer[0].video_bitrate; + + IToxAVCall* call = i_toxav_init_call(toxav, toxav->msi->calls[call_idx]->peers[0], ab, vb, NULL); + if (call == NULL) { + LOGGER_WARNING("No call, rejecting..."); + msi_reject(toxav->msi, call_idx, NULL); + } + + call->call_idx = call_idx; + + if (toxav->ccb.first) + toxav->ccb.first(toxav, toxav->msi->calls[call_idx]->peers[0], true, true, toxav->ccb.second); +} + +void i_toxav_msi_callback_ringing(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + if (toxav->scb.first) + toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], + TOXAV_CALL_STATE_RINGING, toxav->scb.second); +} + +void i_toxav_msi_callback_start(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + + IToxAVCall* call = i_toxav_get_call(toxav, toxav->msi->calls[call_idx]->peers[0]); + + if (call == NULL || !i_toxav_prepare_transmission(toxav, call)) { + /* TODO send error */ + i_toxav_remove_call(toxav, toxav->msi->calls[call_idx]->peers[0]); + return; + } + + TOXAV_CALL_STATE state; + const MSICSettings* csets = &toxav->msi->calls[call_idx]->csettings_peer[0]; + + if (csets->audio_bitrate && csets->video_bitrate) + state = TOXAV_CALL_STATE_SENDING_AV; + else if (csets->video_bitrate == 0) + state = TOXAV_CALL_STATE_SENDING_A; + else + state = TOXAV_CALL_STATE_SENDING_V; + + if (toxav->scb.first) /* TODO this */ + toxav->scb.first(toxav, call->friend_number, state, toxav->scb.second); +} + +void i_toxav_msi_callback_cancel(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + + i_toxav_remove_call(toxav, toxav->msi->calls[call_idx]->peers[0]); + + if (toxav->scb.first) + toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], + TOXAV_CALL_STATE_END, toxav->scb.second); +} + +void i_toxav_msi_callback_reject(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + + i_toxav_kill_transmission(toxav, toxav->msi->calls[call_idx]->peers[0]); + i_toxav_remove_call(toxav, toxav->msi->calls[call_idx]->peers[0]); + + if (toxav->scb.first) + toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], + TOXAV_CALL_STATE_END, toxav->scb.second); +} + +void i_toxav_msi_callback_end(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + + i_toxav_kill_transmission(toxav, toxav->msi->calls[call_idx]->peers[0]); + i_toxav_remove_call(toxav, toxav->msi->calls[call_idx]->peers[0]); + + if (toxav->scb.first) + toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], + TOXAV_CALL_STATE_END, toxav->scb.second); +} + +void i_toxav_msi_callback_request_to(void* toxav_inst, int32_t call_idx, void* data) +{ + /* TODO remove */ + ToxAV* toxav = toxav_inst; + if (toxav->scb.first) + toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], + TOXAV_CALL_STATE_ERROR, toxav->scb.second); +} + +void i_toxav_msi_callback_peer_to(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + if (toxav->scb.first) + toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], + TOXAV_CALL_STATE_ERROR, toxav->scb.second); +} + +void i_toxav_msi_callback_state_change(void* toxav_inst, int32_t call_idx, void* data) +{ + ToxAV* toxav = toxav_inst; + /* TODO something something msi */ +} + +IToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number) +{ + if (av->calls == NULL || av->calls_tail < friend_number) + return NULL; + + return av->calls[friend_number]; +} + +IToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number) +{ + IToxAVCall* rc = calloc(sizeof(IToxAVCall), 1); + + if (rc == NULL) + return NULL; + + rc->friend_number = friend_number; + + if (create_recursive_mutex(rc->mutex_control) != 0) { + free(rc); + return NULL; + } + + if (create_recursive_mutex(rc->mutex_do) != 0) { + pthread_mutex_destroy(rc->mutex_control); + free(rc); + return NULL; + } + + + if (av->calls == NULL) { /* Creating */ + av->calls = calloc (sizeof(IToxAVCall*), friend_number + 1); + + if (av->calls == NULL) { + pthread_mutex_destroy(rc->mutex_control); + pthread_mutex_destroy(rc->mutex_do); + free(rc); + return NULL; + } + + av->calls_tail = av->calls_head = friend_number; + + } else if (av->calls_tail < friend_number) { /* Appending */ + void* tmp = realloc(av->calls, sizeof(IToxAVCall*) * friend_number + 1); + + if (tmp == NULL) { + pthread_mutex_destroy(rc->mutex_control); + pthread_mutex_destroy(rc->mutex_do); + free(rc); + return NULL; + } + + av->calls = tmp; + + /* Set fields in between to null */ + int32_t i = av->calls_tail; + for (; i < friend_number; i ++) + av->calls[i] = NULL; + + rc->prev = av->calls[av->calls_tail]; + av->calls[av->calls_tail]->next = rc; + + av->calls_tail = friend_number; + + } else if (av->calls_head > friend_number) { /* Inserting at front */ + rc->next = av->calls[av->calls_head]; + av->calls[av->calls_head]->prev = rc; + av->calls_head = friend_number; + } + + av->calls[friend_number] = rc; + return rc; +} + +void i_toxav_remove_call(ToxAV* av, uint32_t friend_number) +{ + IToxAVCall* tc = i_toxav_get_call(av, friend_number); + + if (tc == NULL) + return; + + IToxAVCall* prev = tc->prev; + IToxAVCall* next = tc->next; + + pthread_mutex_destroy(tc->mutex_control); + pthread_mutex_destroy(tc->mutex_do); + + free(tc); + + if (prev) + prev->next = next; + else if (next) + av->calls_head = next->friend_number; + else goto CLEAR; + + if (next) + next->prev = prev; + else if (prev) + av->calls_tail = prev->friend_number; + else goto CLEAR; + + av->calls[friend_number] = NULL; + return; + +CLEAR: + av->calls_head = av->calls_tail = 0; + free(av->calls); + av->calls = NULL; +} + +bool i_toxav_audio_bitrate_invalid(uint32_t bitrate) +{ + /* Opus RFC 6716 section-2.1.1 dictates the following: + * Opus supports all bitrates from 6 kbit/s to 510 kbit/s. + */ + return bitrate < 6 || bitrate > 510; +} + +bool i_toxav_video_bitrate_invalid(uint32_t bitrate) +{ + /* TODO: If anyone knows the answer to this one please fill it up */ + return false; +} + +IToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) +{ + TOXAV_ERR_CALL rc = TOXAV_ERR_CALL_OK; + IToxAVCall* call = NULL; + + if (m_friend_exists(av->m, friend_number) == 0) { + rc = TOXAV_ERR_CALL_FRIEND_NOT_FOUND; + goto END; + } + + if (m_get_friend_connectionstatus(av->m, friend_number) != 1) { + rc = TOXAV_ERR_CALL_FRIEND_NOT_CONNECTED; + goto END; + } + + if (i_toxav_get_call(av, friend_number) != NULL) { + rc = TOXAV_ERR_CALL_FRIEND_ALREADY_IN_CALL; + goto END; + } + + if ((audio_bit_rate && i_toxav_audio_bitrate_invalid(audio_bit_rate)) + ||(video_bit_rate && i_toxav_video_bitrate_invalid(video_bit_rate)) + ) { + rc = TOXAV_ERR_CALL_INVALID_BIT_RATE; + goto END; + } + + call = i_toxav_add_call(av, friend_number); + if (call == NULL) { + rc = TOXAV_ERR_CALL_MALLOC; + } + +END: + if (error) + *error = rc; + + return call; +} + +bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call) +{ pthread_mutex_lock(call->mutex_control); - + if (call->active) { pthread_mutex_unlock(call->mutex_control); - LOGGER_ERROR("Error while starting RTP session: call already active!\n"); - return av_ErrorAlreadyInCallWithPeer; + LOGGER_WARNING("Call already active!\n"); + return true; } - - if (pthread_mutex_init(call->mutex_encoding_audio, NULL) != 0 - || pthread_mutex_init(call->mutex_encoding_video, NULL) != 0 || pthread_mutex_init(call->mutex_do, NULL) != 0) { - pthread_mutex_unlock(call->mutex_control); - LOGGER_ERROR("Error while starting RTP session: mutex initializing failed!\n"); - return av_ErrorUnknown; + + if (pthread_mutex_init(call->mutex_encoding_audio, NULL) != 0) + goto MUTEX_INIT_ERROR; + + if (pthread_mutex_init(call->mutex_encoding_video, NULL) != 0) { + pthread_mutex_destroy(call->mutex_encoding_audio); + goto MUTEX_INIT_ERROR; } - - const ToxAvCSettings *c_peer = toxavcsettings_cast - (&av->msi_session->calls[call_index]->csettings_peer[0]); - const ToxAvCSettings *c_self = toxavcsettings_cast - (&av->msi_session->calls[call_index]->csettings_local); - - LOGGER_DEBUG( - "Type: %u(s) %u(p)\n" - "Video bitrate: %u(s) %u(p)\n" - "Video height: %u(s) %u(p)\n" - "Video width: %u(s) %u(p)\n" - "Audio bitrate: %u(s) %u(p)\n" - "Audio framedur: %u(s) %u(p)\n" - "Audio sample rate: %u(s) %u(p)\n" - "Audio channels: %u(s) %u(p)\n", - c_self->call_type, c_peer->call_type, - c_self->video_bitrate, c_peer->video_bitrate, - c_self->max_video_height, c_peer->max_video_height, - c_self->max_video_width, c_peer->max_video_width, - c_self->audio_bitrate, c_peer->audio_bitrate, - c_self->audio_frame_duration, c_peer->audio_frame_duration, - c_self->audio_sample_rate, c_peer->audio_sample_rate, - c_self->audio_channels, c_peer->audio_channels ); - - if ( !(call->cs = cs_new(c_self, c_peer, jbuf_capacity, support_video)) ) { + + if (pthread_mutex_init(call->mutex_do, NULL) != 0) { + pthread_mutex_destroy(call->mutex_encoding_audio); + pthread_mutex_destroy(call->mutex_encoding_video); + goto MUTEX_INIT_ERROR; + } + + const MSICSettings *c_peer = &av->msi->calls[call->call_idx]->csettings_peer[0]; + const MSICSettings *c_self = &av->msi->calls[call->call_idx]->csettings_local; + + call->cs = cs_new(c_self->audio_bitrate, c_peer->audio_bitrate, + c_self->video_bitrate, c_peer->video_bitrate); + + if ( !call->cs ) { LOGGER_ERROR("Error while starting Codec State!\n"); - pthread_mutex_unlock(call->mutex_control); - return av_ErrorInitializingCodecs; + goto FAILURE; } - + call->cs->agent = av; - call->cs->call_idx = call_index; - - call->cs->acb.first = av->acb.first; - call->cs->acb.second = av->acb.second; - - call->cs->vcb.first = av->vcb.first; - call->cs->vcb.second = av->vcb.second; - - - call->crtps[audio_index] = - rtp_new(msi_TypeAudio, av->messenger, av->msi_session->calls[call_index]->peers[0]); - - if ( !call->crtps[audio_index] ) { - LOGGER_ERROR("Error while starting audio RTP session!\n"); - goto error; - } - - call->crtps[audio_index]->cs = call->cs; - - if ( support_video ) { - call->crtps[video_index] = - rtp_new(msi_TypeVideo, av->messenger, av->msi_session->calls[call_index]->peers[0]); - - if ( !call->crtps[video_index] ) { - LOGGER_ERROR("Error while starting video RTP session!\n"); - goto error; + + /* It makes no sense to have CSession without callbacks */ + assert(av->acb.first || av->vcb.first); + + memcpy(&call->cs->acb, &av->acb, sizeof(av->acb)); + memcpy(&call->cs->vcb, &av->vcb, sizeof(av->vcb)); + + call->cs->friend_number = call->friend_number; + call->cs->call_idx = call->call_idx; + + + if (c_self->audio_bitrate > 0 || c_peer->audio_bitrate > 0) { /* Prepare audio rtp */ + call->rtps[audio_index] = rtp_new(msi_TypeAudio, av->m, av->msi->calls[call->call_idx]->peers[0]); + + if ( !call->rtps[audio_index] ) { + LOGGER_ERROR("Error while starting audio RTP session!\n"); + goto FAILURE; } - - call->crtps[video_index]->cs = call->cs; + + call->rtps[audio_index]->cs = call->cs; + + if (c_peer->audio_bitrate > 0) + rtp_register_for_receiving(call->rtps[audio_index]); } - + + if (c_self->video_bitrate > 0 || c_peer->video_bitrate > 0) { /* Prepare video rtp */ + call->rtps[video_index] = rtp_new(msi_TypeVideo, av->m, av->msi->calls[call->call_idx]->peers[0]); + + if ( !call->rtps[video_index] ) { + LOGGER_ERROR("Error while starting video RTP session!\n"); + goto FAILURE; + } + + call->rtps[video_index]->cs = call->cs; + + if (c_peer->video_bitrate > 0) + rtp_register_for_receiving(call->rtps[audio_index]); + } + call->active = 1; pthread_mutex_unlock(call->mutex_control); - return av_ErrorNone; -error: - rtp_kill(call->crtps[audio_index], av->messenger); - call->crtps[audio_index] = NULL; - rtp_kill(call->crtps[video_index], av->messenger); - call->crtps[video_index] = NULL; + return true; + +FAILURE: + rtp_kill(call->rtps[audio_index]); + call->rtps[audio_index] = NULL; + rtp_kill(call->rtps[video_index]); + call->rtps[video_index] = NULL; cs_kill(call->cs); call->cs = NULL; call->active = 0; pthread_mutex_destroy(call->mutex_encoding_audio); pthread_mutex_destroy(call->mutex_encoding_video); pthread_mutex_destroy(call->mutex_do); - + pthread_mutex_unlock(call->mutex_control); - return av_ErrorCreatingRtpSessions; + return false; + +MUTEX_INIT_ERROR: + pthread_mutex_unlock(call->mutex_control); + LOGGER_ERROR("Mutex initialization failed!\n"); + return false; } -int toxav_kill_transmission ( ToxAv *av, int32_t call_index ) +void i_toxav_kill_transmission(ToxAV* av, uint32_t friend_number) { - if (CALL_INVALID_INDEX(call_index, av->msi_session->max_calls)) { - LOGGER_WARNING("Invalid call index: %d", call_index); - return av_ErrorNoCall; - } - - ToxAvCall *call = &av->calls[call_index]; - + IToxAVCall* call = i_toxav_get_call(av, friend_number); + if (!call) + return; + pthread_mutex_lock(call->mutex_control); - + if (!call->active) { pthread_mutex_unlock(call->mutex_control); - LOGGER_WARNING("Action on inactive call: %d", call_index); - return av_ErrorInvalidState; + LOGGER_WARNING("Action on inactive call: %d", call->call_idx); + return; } - + call->active = 0; - + pthread_mutex_lock(call->mutex_encoding_audio); pthread_mutex_unlock(call->mutex_encoding_audio); pthread_mutex_lock(call->mutex_encoding_video); pthread_mutex_unlock(call->mutex_encoding_video); pthread_mutex_lock(call->mutex_do); pthread_mutex_unlock(call->mutex_do); - - rtp_kill(call->crtps[audio_index], av->messenger); - call->crtps[audio_index] = NULL; - rtp_kill(call->crtps[video_index], av->messenger); - call->crtps[video_index] = NULL; + + rtp_kill(call->rtps[audio_index]); + call->rtps[audio_index] = NULL; + rtp_kill(call->rtps[video_index]); + call->rtps[video_index] = NULL; cs_kill(call->cs); call->cs = NULL; - + pthread_mutex_destroy(call->mutex_encoding_audio); pthread_mutex_destroy(call->mutex_encoding_video); pthread_mutex_destroy(call->mutex_do); - + pthread_mutex_unlock(call->mutex_control); - - return av_ErrorNone; } - -static int toxav_send_rtp_payload(ToxAv *av, - ToxAvCall *call, - ToxAvCallType type, - const uint8_t *payload, - unsigned int length) -{ - if (call->crtps[type - av_TypeAudio]) { - - /* Audio */ - if (type == av_TypeAudio) - return rtp_send_msg(call->crtps[audio_index], av->messenger, payload, length); - - /* Video */ - int parts = cs_split_video_payload(call->cs, payload, length); - - if (parts < 0) return parts; - - uint16_t part_size; - const uint8_t *iter; - - int i; - - for (i = 0; i < parts; i++) { - iter = cs_iterate_split_video_frame(call->cs, &part_size); - - if (rtp_send_msg(call->crtps[video_index], av->messenger, iter, part_size) < 0) - return av_ErrorSendingPayload; - } - - return av_ErrorNone; - - } else return av_ErrorNoRtpSession; -} - -int toxav_prepare_video_frame ( ToxAv *av, int32_t call_index, uint8_t *dest, int dest_max, vpx_image_t *input) -{ - if (CALL_INVALID_INDEX(call_index, av->msi_session->max_calls)) { - LOGGER_WARNING("Invalid call index: %d", call_index); - return av_ErrorNoCall; - } - - - ToxAvCall *call = &av->calls[call_index]; - pthread_mutex_lock(call->mutex_control); - - if (!call->active) { - pthread_mutex_unlock(call->mutex_control); - LOGGER_WARNING("Action on inactive call: %d", call_index); - return av_ErrorInvalidState; - } - - if (cs_set_sending_video_resolution(call->cs, input->d_w, input->d_h) < 0) { - pthread_mutex_unlock(call->mutex_control); - return av_ErrorSettingVideoResolution; - } - - pthread_mutex_lock(call->mutex_encoding_video); - pthread_mutex_unlock(call->mutex_control); - - int rc = vpx_codec_encode(call->cs->v_encoder, input, call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(rc)); - pthread_mutex_unlock(call->mutex_encoding_video); - return av_ErrorEncodingVideo; - } - - ++call->cs->frame_counter; - - vpx_codec_iter_t iter = NULL; - const vpx_codec_cx_pkt_t *pkt; - int copied = 0; - - while ( (pkt = vpx_codec_get_cx_data(call->cs->v_encoder, &iter)) ) { - if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { - if ( copied + pkt->data.frame.sz > dest_max ) { - pthread_mutex_unlock(call->mutex_encoding_video); - return av_ErrorPacketTooLarge; - } - - memcpy(dest + copied, pkt->data.frame.buf, pkt->data.frame.sz); - copied += pkt->data.frame.sz; - } - } - - pthread_mutex_unlock(call->mutex_encoding_video); - return copied; -} - -int toxav_send_video ( ToxAv *av, int32_t call_index, const uint8_t *frame, unsigned int frame_size) -{ - - if (CALL_INVALID_INDEX(call_index, av->msi_session->max_calls)) { - LOGGER_WARNING("Invalid call index: %d", call_index); - return av_ErrorNoCall; - } - - ToxAvCall *call = &av->calls[call_index]; - pthread_mutex_lock(call->mutex_control); - - - if (!call->active) { - pthread_mutex_unlock(call->mutex_control); - LOGGER_WARNING("Action on inactive call: %d", call_index); - return av_ErrorInvalidState; - } - - int rc = toxav_send_rtp_payload(av, call, av_TypeVideo, frame, frame_size); - pthread_mutex_unlock(call->mutex_control); - - return rc; -} - -int toxav_prepare_audio_frame ( ToxAv *av, - int32_t call_index, - uint8_t *dest, - int dest_max, - const int16_t *frame, - int frame_size) -{ - if (CALL_INVALID_INDEX(call_index, av->msi_session->max_calls)) { - LOGGER_WARNING("Action on nonexisting call: %d", call_index); - return av_ErrorNoCall; - } - - ToxAvCall *call = &av->calls[call_index]; - pthread_mutex_lock(call->mutex_control); - - if (!call->active) { - pthread_mutex_unlock(call->mutex_control); - LOGGER_WARNING("Action on inactive call: %d", call_index); - return av_ErrorInvalidState; - } - - pthread_mutex_lock(call->mutex_encoding_audio); - pthread_mutex_unlock(call->mutex_control); - int32_t rc = opus_encode(call->cs->audio_encoder, frame, frame_size, dest, dest_max); - pthread_mutex_unlock(call->mutex_encoding_audio); - - if (rc < 0) { - LOGGER_ERROR("Failed to encode payload: %s\n", opus_strerror(rc)); - return av_ErrorEncodingAudio; - } - - return rc; -} - -int toxav_send_audio ( ToxAv *av, int32_t call_index, const uint8_t *data, unsigned int size) -{ - if (CALL_INVALID_INDEX(call_index, av->msi_session->max_calls)) { - LOGGER_WARNING("Action on nonexisting call: %d", call_index); - return av_ErrorNoCall; - } - - ToxAvCall *call = &av->calls[call_index]; - pthread_mutex_lock(call->mutex_control); - - - if (!call->active) { - pthread_mutex_unlock(call->mutex_control); - LOGGER_WARNING("Action on inactive call: %d", call_index); - return av_ErrorInvalidState; - } - - int rc = toxav_send_rtp_payload(av, call, av_TypeAudio, data, size); - pthread_mutex_unlock(call->mutex_control); - return rc; -} - -int toxav_get_peer_csettings ( ToxAv *av, int32_t call_index, int peer, ToxAvCSettings *dest ) -{ - if ( peer < 0 || CALL_INVALID_INDEX(call_index, av->msi_session->max_calls) || - !av->msi_session->calls[call_index] || av->msi_session->calls[call_index]->peer_count <= peer ) - return av_ErrorNoCall; - - *dest = *toxavcsettings_cast(&av->msi_session->calls[call_index]->csettings_peer[peer]); - return av_ErrorNone; -} - -int toxav_get_peer_id ( ToxAv *av, int32_t call_index, int peer ) -{ - if ( peer < 0 || CALL_INVALID_INDEX(call_index, av->msi_session->max_calls) || !av->msi_session->calls[call_index] - || av->msi_session->calls[call_index]->peer_count <= peer ) - return av_ErrorNoCall; - - return av->msi_session->calls[call_index]->peers[peer]; -} - -ToxAvCallState toxav_get_call_state(ToxAv *av, int32_t call_index) -{ - if ( CALL_INVALID_INDEX(call_index, av->msi_session->max_calls) || !av->msi_session->calls[call_index] ) - return av_CallNonExistent; - - return av->msi_session->calls[call_index]->state; - -} - -int toxav_capability_supported ( ToxAv *av, int32_t call_index, ToxAvCapabilities capability ) -{ -} - -Tox *toxav_get_tox(ToxAv *av) -{ - return (Tox *)av->messenger; -} - -int toxav_get_active_count(ToxAv *av) -{ - if (!av) return -1; - - int rc = 0, i = 0; - - for (; i < av->max_calls; i++) { - pthread_mutex_lock(av->calls[i].mutex_control); - - if (av->calls[i].active) rc++; - - pthread_mutex_unlock(av->calls[i].mutex_control); - } - - return rc; -} - -/* Create a new toxav group. - * - * return group number on success. - * return -1 on failure. - * - * Audio data callback format: - * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) - * - * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). - */ -int toxav_add_av_groupchat(Tox *tox, void (*audio_callback)(Messenger *, int, int, const int16_t *, unsigned int, - uint8_t, unsigned int, void *), void *userdata) -{ - Messenger *m = tox; - return add_av_groupchat(m->group_chat_object, audio_callback, userdata); -} - -/* Join a AV group (you need to have been invited first.) - * - * returns group number on success - * returns -1 on failure. - * - * Audio data callback format (same as the one for toxav_add_av_groupchat()): - * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) - * - * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). - */ -int toxav_join_av_groupchat(Tox *tox, int32_t friendnumber, const uint8_t *data, uint16_t length, - void (*audio_callback)(Messenger *, int, int, const int16_t *, unsigned int, uint8_t, unsigned int, void *), - void *userdata) -{ - Messenger *m = tox; - return join_av_groupchat(m->group_chat_object, friendnumber, data, length, audio_callback, userdata); -} - -/* Send audio to the group chat. - * - * return 0 on success. - * return -1 on failure. - * - * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). - * - * Valid number of samples are ((sample rate) * (audio length (Valid ones are: 2.5, 5, 10, 20, 40 or 60 ms)) / 1000) - * Valid number of channels are 1 or 2. - * Valid sample rates are 8000, 12000, 16000, 24000, or 48000. - * - * Recommended values are: samples = 960, channels = 1, sample_rate = 48000 - */ -int toxav_group_send_audio(Tox *tox, int groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels, - unsigned int sample_rate) -{ - Messenger *m = tox; - return group_send_audio(m->group_chat_object, groupnumber, pcm, samples, channels, sample_rate); -} - diff --git a/toxav/toxav.h b/toxav/toxav.h index 3696f9619..69654f973 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -1,329 +1,483 @@ -/** toxav.h - * - * Copyright (C) 2013 Tox project All Rights Reserved. - * - * This file is part of Tox. - * - * Tox is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Tox is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Tox. If not, see . - * +#pragma once +#include +#include +#include +/** \page av Public audio/video API for Tox clients. + * + * Unlike the Core API, this API is fully thread-safe. The library will ensure + * the proper synchronisation of parallel calls. + */ +/** + * The type of the Tox Audio/Video subsystem object. + */ +typedef struct toxAV ToxAV; +#ifndef TOX_DEFINED +#define TOX_DEFINED +/** + * The type of a Tox instance. Repeated here so this file does not have a direct + * dependency on the Core interface. */ - - -#ifndef __TOXAV -#define __TOXAV -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct _ToxAv ToxAv; - -/* vpx_image_t */ -#include - -typedef void ( *ToxAVCallback ) ( void *agent, int32_t call_idx, void *arg ); -typedef void ( *ToxAvAudioCallback ) (void *agent, int32_t call_idx, const int16_t *PCM, uint16_t size, void *data); -typedef void ( *ToxAvVideoCallback ) (void *agent, int32_t call_idx, const vpx_image_t *img, void *data); - -#ifndef __TOX_DEFINED__ -#define __TOX_DEFINED__ typedef struct Tox Tox; #endif - -#define RTP_PAYLOAD_SIZE 65535 - - -/** - * Callbacks ids that handle the call states. - */ -typedef enum { - av_OnInvite, /* Incoming call */ - av_OnRinging, /* When peer is ready to accept/reject the call */ - av_OnStart, /* Call (RTP transmission) started */ - av_OnCancel, /* The side that initiated call canceled invite */ - av_OnReject, /* The side that was invited rejected the call */ - av_OnEnd, /* Call that was active ended */ - av_OnRequestTimeout, /* When the requested action didn't get response in specified time */ - av_OnPeerTimeout, /* Peer timed out; stop the call */ - av_OnPeerCSChange, /* Peer changing Csettings. Prepare for changed AV */ - av_OnSelfCSChange /* Csettings change confirmation. Once triggered peer is ready to recv changed AV */ -} ToxAvCallbackID; - - -/** - * Call type identifier. - */ -typedef enum { - av_TypeAudio = 192, - av_TypeVideo -} ToxAvCallType; - - -typedef enum { - av_CallNonExistent = -1, - av_CallInviting, /* when sending call invite */ - av_CallStarting, /* when getting call invite */ - av_CallActive, - av_CallHold, - av_CallHungUp -} ToxAvCallState; - -/** - * Error indicators. Values under -20 are reserved for toxcore. - */ -typedef enum { - av_ErrorNone = 0, - av_ErrorUnknown = -1, /* Unknown error */ - av_ErrorNoCall = -20, /* Trying to perform call action while not in a call */ - av_ErrorInvalidState = -21, /* Trying to perform call action while in invalid state*/ - av_ErrorAlreadyInCallWithPeer = -22, /* Trying to call peer when already in a call with peer */ - av_ErrorReachedCallLimit = -23, /* Cannot handle more calls */ - av_ErrorInitializingCodecs = -30, /* Failed creating CSSession */ - av_ErrorSettingVideoResolution = -31, /* Error setting resolution */ - av_ErrorSettingVideoBitrate = -32, /* Error setting bitrate */ - av_ErrorSplittingVideoPayload = -33, /* Error splitting video payload */ - av_ErrorEncodingVideo = -34, /* vpx_codec_encode failed */ - av_ErrorEncodingAudio = -35, /* opus_encode failed */ - av_ErrorSendingPayload = -40, /* Sending lossy packet failed */ - av_ErrorCreatingRtpSessions = -41, /* One of the rtp sessions failed to initialize */ - av_ErrorNoRtpSession = -50, /* Trying to perform rtp action on invalid session */ - av_ErrorInvalidCodecState = -51, /* Codec state not initialized */ - av_ErrorPacketTooLarge = -52, /* Split packet exceeds it's limit */ -} ToxAvError; - - -/** - * Locally supported capabilities. - */ -typedef enum { - av_AudioEncoding = 1 << 0, - av_AudioDecoding = 1 << 1, - av_VideoEncoding = 1 << 2, - av_VideoDecoding = 1 << 3 -} ToxAvCapabilities; - - -/** - * Encoding settings. - */ -typedef struct _ToxAvCSettings { - ToxAvCallType call_type; - - uint32_t video_bitrate; /* In kbits/s */ - uint16_t max_video_width; /* In px */ - uint16_t max_video_height; /* In px */ - - uint32_t audio_bitrate; /* In bits/s */ - uint16_t audio_frame_duration; /* In ms */ - uint32_t audio_sample_rate; /* In Hz */ - uint32_t audio_channels; -} ToxAvCSettings; - -extern const ToxAvCSettings av_DefaultSettings; - -/** - * Start new A/V session. There can only be one session at the time. - */ -ToxAv *toxav_new(Tox *messenger, int32_t max_calls); - -/** - * Remove A/V session. - */ -void toxav_kill(ToxAv *av); - -/** - * Returns the interval in milliseconds when the next toxav_do() should be called. - * If no call is active at the moment returns 200. - */ -uint32_t toxav_do_interval(ToxAv *av); - -/** - * Main loop for the session. Best called right after tox_do(); - */ -void toxav_do(ToxAv *av); - -/** - * Register callback for call state. - */ -void toxav_register_callstate_callback (ToxAv *av, ToxAVCallback cb, ToxAvCallbackID id, void *userdata); - -/** - * Register callback for audio data. - */ -void toxav_register_audio_callback (ToxAv *av, ToxAvAudioCallback cb, void *userdata); - -/** - * Register callback for video data. - */ -void toxav_register_video_callback (ToxAv *av, ToxAvVideoCallback cb, void *userdata); - -/** - * Call user. Use its friend_id. - */ -int toxav_call(ToxAv *av, - int32_t *call_index, - int friend_id, - const ToxAvCSettings *csettings, - int ringing_seconds); - -/** - * Hangup active call. - */ -int toxav_hangup(ToxAv *av, int32_t call_index); - -/** - * Answer incoming call. Pass the csettings that you will use. - */ -int toxav_answer(ToxAv *av, int32_t call_index, const ToxAvCSettings *csettings ); - -/** - * Reject incoming call. - */ -int toxav_reject(ToxAv *av, int32_t call_index, const char *reason); - -/** - * Cancel outgoing request. - */ -int toxav_cancel(ToxAv *av, int32_t call_index, int peer_id, const char *reason); - -/** - * Notify peer that we are changing codec settings. - */ -int toxav_change_settings(ToxAv *av, int32_t call_index, const ToxAvCSettings *csettings); - -/** - * Terminate transmission. Note that transmission will be - * terminated without informing remote peer. Usually called when we can't inform peer. - */ -int toxav_stop_call(ToxAv *av, int32_t call_index); - -/** - * Allocates transmission data. Must be call before calling toxav_prepare_* and toxav_send_*. - * Also, it must be called when call is started - */ -int toxav_prepare_transmission(ToxAv *av, int32_t call_index, int support_video); - -/** - * Clears transmission data. Call this at the end of the transmission. - */ -int toxav_kill_transmission(ToxAv *av, int32_t call_index); - -/** - * Encode video frame. - */ -int toxav_prepare_video_frame ( ToxAv *av, - int32_t call_index, - uint8_t *dest, - int dest_max, - vpx_image_t *input); - -/** - * Send encoded video packet. - */ -int toxav_send_video ( ToxAv *av, int32_t call_index, const uint8_t *frame, uint32_t frame_size); - -/** - * Encode audio frame. - */ -int toxav_prepare_audio_frame ( ToxAv *av, - int32_t call_index, - uint8_t *dest, - int dest_max, - const int16_t *frame, - int frame_size); - -/** - * Send encoded audio frame. - */ -int toxav_send_audio ( ToxAv *av, int32_t call_index, const uint8_t *frame, unsigned int size); - -/** - * Get codec settings from the peer. These were exchanged during call initialization - * or when peer send us new csettings. - */ -int toxav_get_peer_csettings ( ToxAv *av, int32_t call_index, int peer, ToxAvCSettings *dest ); - -/** - * Get friend id of peer participating in conversation. - */ -int toxav_get_peer_id ( ToxAv *av, int32_t call_index, int peer ); - -/** - * Get current call state. - */ -ToxAvCallState toxav_get_call_state ( ToxAv *av, int32_t call_index ); - -/** - * Is certain capability supported. Used to determine if encoding/decoding is ready. - */ -int toxav_capability_supported ( ToxAv *av, int32_t call_index, ToxAvCapabilities capability ); - -/** - * Returns tox reference. - */ -Tox *toxav_get_tox (ToxAv *av); - -/** - * Returns number of active calls or -1 on error. - */ -int toxav_get_active_count (ToxAv *av); - -/* Create a new toxav group. +/******************************************************************************* + * + * :: Creation and destruction * - * return group number on success. - * return -1 on failure. - * - * Audio data callback format: - * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) - * - * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + ******************************************************************************/ +typedef enum TOXAV_ERR_NEW { + TOXAV_ERR_NEW_OK, + TOXAV_ERR_NEW_NULL, + /** + * Memory allocation failure while trying to allocate structures required for + * the A/V session. + */ + TOXAV_ERR_NEW_MALLOC, + /** + * Attempted to create a second session for the same Tox instance. + */ + TOXAV_ERR_NEW_MULTIPLE +} TOXAV_ERR_NEW; +/** + * Start new A/V session. There can only be only one session per Tox instance. */ -int toxav_add_av_groupchat(Tox *tox, void (*audio_callback)(Tox *, int, int, const int16_t *, unsigned int, uint8_t, - unsigned int, void *), void *userdata); - -/* Join a AV group (you need to have been invited first.) +ToxAV *toxav_new(Tox *tox, TOXAV_ERR_NEW *error); +/** + * Releases all resources associated with the A/V session. * - * returns group number on success - * returns -1 on failure. - * - * Audio data callback format (same as the one for toxav_add_av_groupchat()): - * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) - * - * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + * If any calls were ongoing, these will be forcibly terminated without + * notifying peers. After calling this function, no other functions may be + * called and the av pointer becomes invalid. */ -int toxav_join_av_groupchat(Tox *tox, int32_t friendnumber, const uint8_t *data, uint16_t length, - void (*audio_callback)(Tox *, int, int, const int16_t *, unsigned int, uint8_t, unsigned int, void *), void *userdata); - -/* Send audio to the group chat. - * - * return 0 on success. - * return -1 on failure. - * - * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). - * - * Valid number of samples are ((sample rate) * (audio length (Valid ones are: 2.5, 5, 10, 20, 40 or 60 ms)) / 1000) - * Valid number of channels are 1 or 2. - * Valid sample rates are 8000, 12000, 16000, 24000, or 48000. - * - * Recommended values are: samples = 960, channels = 1, sample_rate = 48000 +void toxav_kill(ToxAV *av); +/** + * Returns the Tox instance the A/V object was created for. */ -int toxav_group_send_audio(Tox *tox, int groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels, - unsigned int sample_rate); - -#ifdef __cplusplus -} -#endif - -#endif /* __TOXAV */ +Tox *toxav_get_tox(ToxAV *av); +/******************************************************************************* + * + * :: A/V event loop + * + ******************************************************************************/ +/** + * Returns the interval in milliseconds when the next toxav_iteration should be + * called. If no call is active at the moment, this function returns 200. + */ +uint32_t toxav_iteration_interval(ToxAV const *av); +/** + * Main loop for the session. This function needs to be called in intervals of + * toxav_iteration_interval() milliseconds. It is best called in the same loop + * as tox_iteration. + */ +void toxav_iteration(ToxAV *av); +/******************************************************************************* + * + * :: Call setup + * + ******************************************************************************/ +typedef enum TOXAV_ERR_CALL { + TOXAV_ERR_CALL_OK, + /** + * A resource allocation error occurred while trying to create the structures + * required for the call. + */ + TOXAV_ERR_CALL_MALLOC, + /** + * The friend number did not designate a valid friend. + */ + TOXAV_ERR_CALL_FRIEND_NOT_FOUND, + /** + * The friend was valid, but not currently connected. + */ + TOXAV_ERR_CALL_FRIEND_NOT_CONNECTED, + /** + * Attempted to call a friend while already in an audio or video call with + * them. + */ + TOXAV_ERR_CALL_FRIEND_ALREADY_IN_CALL, + /** + * Audio or video bit rate is invalid. + */ + TOXAV_ERR_CALL_INVALID_BIT_RATE +} TOXAV_ERR_CALL; +/** + * Call a friend. This will start ringing the friend. + * + * It is the client's responsibility to stop ringing after a certain timeout, + * if such behaviour is desired. If the client does not stop ringing, the A/V + * library will not stop until the friend is disconnected. + * + * @param friend_number The friend number of the friend that should be called. + * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable + * audio sending. + * @param video_bit_rate Video bit rate in Kb/sec. Set this to 0 to disable + * video sending. + */ +bool toxav_call(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL *error); +/** + * The function type for the `call` callback. + */ +typedef void toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data); +/** + * Set the callback for the `call` event. Pass NULL to unset. + * + * This event is triggered when a call is received from a friend. + */ +void toxav_callback_call(ToxAV *av, toxav_call_cb *function, void *user_data); +typedef enum TOXAV_ERR_ANSWER { + TOXAV_ERR_ANSWER_OK, + /** + * A resource allocation error occurred while trying to create the structures + * required for the call. + */ + TOXAV_ERR_ANSWER_MALLOC, + /** + * The friend number did not designate a valid friend. + */ + TOXAV_ERR_ANSWER_FRIEND_NOT_FOUND, + /** + * The friend was valid, but they are not currently trying to initiate a call. + * This is also returned if this client is already in a call with the friend. + */ + TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING, + /** + * Audio or video bit rate is invalid. + */ + TOXAV_ERR_ANSWER_INVALID_BIT_RATE +} TOXAV_ERR_ANSWER; +/** + * Accept an incoming call. + * + * If an allocation error occurs while answering a call, both participants will + * receive TOXAV_CALL_STATE_ERROR and the call will end. + * + * @param friend_number The friend number of the friend that is calling. + * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable + * audio sending. + * @param video_bit_rate Video bit rate in Kb/sec. Set this to 0 to disable + * video sending. + */ +bool toxav_answer(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER *error); +/******************************************************************************* + * + * :: Call state graph + * + ******************************************************************************/ +typedef enum TOXAV_CALL_STATE { + /** + * The friend's client is aware of the call. This happens after calling + * toxav_call and the initial call request has been received. + */ + TOXAV_CALL_STATE_RINGING, + /** + * Not sending anything. Either the friend requested that this client stops + * sending anything, or the client turned off both audio and video by setting + * the respective bit rates to 0. + * + * If both sides are in this state, the call is effectively on hold, but not + * in the PAUSED state. + */ + TOXAV_CALL_STATE_NOT_SENDING, + /** + * Sending audio only. Either the friend requested that this client stops + * sending video, or the client turned off video by setting the video bit rate + * to 0. + */ + TOXAV_CALL_STATE_SENDING_A, + /** + * Sending video only. Either the friend requested that this client stops + * sending audio (muted), or the client turned off audio by setting the audio + * bit rate to 0. + */ + TOXAV_CALL_STATE_SENDING_V, + /** + * Sending both audio and video. + */ + TOXAV_CALL_STATE_SENDING_AV, + /** + * The call is on hold. Both sides stop sending and receiving. + */ + TOXAV_CALL_STATE_PAUSED, + /** + * The call has finished. This is the final state after which no more state + * transitions can occur for the call. + */ + TOXAV_CALL_STATE_END, + /** + * Sent by the AV core if an error occurred on the remote end. + */ + TOXAV_CALL_STATE_ERROR +} TOXAV_CALL_STATE; +/** + * The function type for the `call_state` callback. + * + * @param friend_number The friend number for which the call state changed. + * @param state The new call state. + */ +typedef void toxav_call_state_cb(ToxAV *av, uint32_t friend_number, TOXAV_CALL_STATE state, void *user_data); +/** + * Set the callback for the `call_state` event. Pass NULL to unset. + * + * This event is triggered when a call state transition occurs. + */ +void toxav_callback_call_state(ToxAV *av, toxav_call_state_cb *function, void *user_data); +/******************************************************************************* + * + * :: Call control + * + ******************************************************************************/ +typedef enum TOXAV_CALL_CONTROL { + /** + * Resume a previously paused call. Only valid if the pause was caused by this + * client. Not valid before the call is accepted. + */ + TOXAV_CALL_CONTROL_RESUME, + /** + * Put a call on hold. Not valid before the call is accepted. + */ + TOXAV_CALL_CONTROL_PAUSE, + /** + * Reject a call if it was not answered, yet. Cancel a call after it was + * answered. + */ + TOXAV_CALL_CONTROL_CANCEL, + /** + * Request that the friend stops sending audio. Regardless of the friend's + * compliance, this will cause the `receive_audio_frame` event to stop being + * triggered on receiving an audio frame from the friend. + */ + TOXAV_CALL_CONTROL_MUTE_AUDIO, + /** + * Request that the friend stops sending video. Regardless of the friend's + * compliance, this will cause the `receive_video_frame` event to stop being + * triggered on receiving an video frame from the friend. + */ + TOXAV_CALL_CONTROL_MUTE_VIDEO +} TOXAV_CALL_CONTROL; +typedef enum TOXAV_ERR_CALL_CONTROL { + TOXAV_ERR_CALL_CONTROL_OK, + /** + * The friend_number passed did not designate a valid friend. + */ + TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_FOUND, + /** + * This client is currently not in a call with the friend. Before the call is + * answered, only CANCEL is a valid control. + */ + TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL, + /** + * Attempted to resume a call that was not paused. + */ + TOXAV_ERR_CALL_CONTROL_NOT_PAUSED, + /** + * Attempted to resume a call that was paused by the other party. Also set if + * the client attempted to send a system-only control. + */ + TOXAV_ERR_CALL_CONTROL_DENIED, + /** + * The call was already paused on this client. It is valid to pause if the + * other party paused the call. The call will resume after both parties sent + * the RESUME control. + */ + TOXAV_ERR_CALL_CONTROL_ALREADY_PAUSED +} TOXAV_ERR_CALL_CONTROL; +/** + * Sends a call control command to a friend. + * + * @param friend_number The friend number of the friend this client is in a call + * with. + * @param control The control command to send. + * + * @return true on success. + */ +bool toxav_call_control(ToxAV *av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL *error); +/******************************************************************************* + * + * :: Controlling bit rates + * + ******************************************************************************/ +typedef enum TOXAV_ERR_BIT_RATE { + TOXAV_ERR_BIT_RATE_OK, + /** + * The bit rate passed was not one of the supported values. + */ + TOXAV_ERR_BIT_RATE_INVALID +} TOXAV_ERR_BIT_RATE; +/** + * Set the audio bit rate to be used in subsequent audio frames. + * + * @param friend_number The friend number of the friend for which to set the + * audio bit rate. + * @param audio_bit_rate The new audio bit rate in Kb/sec. Set to 0 to disable + * audio sending. + * + * @see toxav_call for the valid bit rates. + */ +bool toxav_set_audio_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE *error); +/** + * Set the video bit rate to be used in subsequent video frames. + * + * @param friend_number The friend number of the friend for which to set the + * video bit rate. + * @param video_bit_rate The new video bit rate in Kb/sec. Set to 0 to disable + * video sending. + * + * @see toxav_call for the valid bit rates. + */ +bool toxav_set_video_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE *error); +/******************************************************************************* + * + * :: A/V sending + * + ******************************************************************************/ +/** + * Common error codes for the send_*_frame functions. + */ +typedef enum TOXAV_ERR_SEND_FRAME { + TOXAV_ERR_SEND_FRAME_OK, + /** + * In case of video, one of Y, U, or V was NULL. In case of audio, the samples + * data pointer was NULL. + */ + TOXAV_ERR_SEND_FRAME_NULL, + /** + * The friend_number passed did not designate a valid friend. + */ + TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND, + /** + * This client is currently not in a call with the friend. + */ + TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL, + /** + * No video frame had been requested through the `request_video_frame` event, + * but the client tried to send one, anyway. + */ + TOXAV_ERR_SEND_FRAME_NOT_REQUESTED, + /** + * One of the frame parameters was invalid. E.g. the resolution may be too + * small or too large, or the audio sampling rate may be unsupported. + */ + TOXAV_ERR_SEND_FRAME_INVALID +} TOXAV_ERR_SEND_FRAME; +/** + * The function type for the `request_video_frame` callback. + * + * @param friend_number The friend number of the friend for which the next video + * frame should be sent. + */ +typedef void toxav_request_video_frame_cb(ToxAV *av, uint32_t friend_number, void *user_data); +/** + * Set the callback for the `request_video_frame` event. Pass NULL to unset. + */ +void toxav_callback_request_video_frame(ToxAV *av, toxav_request_video_frame_cb *function, void *user_data); +/** + * Send a video frame to a friend. + * + * This is called in response to receiving the `request_video_frame` event. + * + * Y - plane should be of size: height * width + * U - plane should be of size: (height/2) * (width/2) + * V - plane should be of size: (height/2) * (width/2) + * + * @param friend_number The friend number of the friend to which to send a video + * frame. + * @param width Width of the frame in pixels. + * @param height Height of the frame in pixels. + * @param y Y (Luminance) plane data. + * @param u U (Chroma) plane data. + * @param v V (Chroma) plane data. + */ +bool toxav_send_video_frame(ToxAV *av, uint32_t friend_number, + uint16_t width, uint16_t height, + uint8_t const *y, uint8_t const *u, uint8_t const *v, + TOXAV_ERR_SEND_FRAME *error); +/** + * The function type for the `request_audio_frame` callback. + * + * @param friend_number The friend number of the friend for which the next audio + * frame should be sent. + */ +typedef void toxav_request_audio_frame_cb(ToxAV *av, uint32_t friend_number, void *user_data); +/** + * Set the callback for the `request_audio_frame` event. Pass NULL to unset. + */ +void toxav_callback_request_audio_frame(ToxAV *av, toxav_request_audio_frame_cb *function, void *user_data); +/** + * Send an audio frame to a friend. + * + * This is called in response to receiving the `request_audio_frame` event. + * + * The expected format of the PCM data is: [s1c1][s1c2][...][s2c1][s2c2][...]... + * Meaning: sample 1 for channel 1, sample 1 for channel 2, ... + * For mono audio, this has no meaning, every sample is subsequent. For stereo, + * this means the expected format is LRLRLR... with samples for left and right + * alternating. + * + * @param friend_number The friend number of the friend to which to send an + * audio frame. + * @param pcm An array of audio samples. The size of this array must be + * sample_count * channels. + * @param sample_count Number of samples in this frame. Valid numbers here are + * ((sample rate) * (audio length) / 1000), where audio length can be + * 2.5, 5, 10, 20, 40 or 60 millseconds. + * @param channels Number of audio channels. Must be at least 1 for mono. + * For voice over IP, more than 2 channels (stereo) typically doesn't make + * sense, but up to 255 channels are supported. + * @param sampling_rate Audio sampling rate used in this frame. Valid sampling + * rates are 8000, 12000, 16000, 24000, or 48000. + */ +bool toxav_send_audio_frame(ToxAV *av, uint32_t friend_number, + int16_t const *pcm, + size_t sample_count, + uint8_t channels, + uint32_t sampling_rate, + TOXAV_ERR_SEND_FRAME *error); +/******************************************************************************* + * + * :: A/V receiving + * + ******************************************************************************/ +/** + * The function type for the `receive_video_frame` callback. + * + * Each plane contains (width * height) pixels. The Alpha plane can be NULL, in + * which case every pixel should be assumed fully opaque. + * + * @param friend_number The friend number of the friend who sent a video frame. + * @param width Width of the frame in pixels. + * @param height Height of the frame in pixels. + * @param planes Plane data. To access Y (Luminance) plane use index 0, + * To access U (Chroma) plane use index 1, + * To access V (Chroma) plane use index 2. + * The size of plane data is derived from width and height where + * Y = width * height, U = (width/2) * (height/2) and V = (width/2) * (height/2). + * @param stride Strides data. Indexing is the same as in 'planes' param. + */ +typedef void toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, + uint16_t width, uint16_t height, + uint8_t const *planes[], int32_t const stride[], + void *user_data); +/** + * Set the callback for the `receive_video_frame` event. Pass NULL to unset. + */ +void toxav_callback_receive_video_frame(ToxAV *av, toxav_receive_video_frame_cb *function, void *user_data); +/** + * The function type for the `receive_audio_frame` callback. + * + * @param friend_number The friend number of the friend who sent an audio frame. + * @param pcm An array of audio samples (sample_count * channels elements). + * @param sample_count The number of audio samples per channel in the PCM array. + * @param channels Number of audio channels. + * @param sampling_rate Sampling rate used in this frame. + * + * @see toxav_send_audio_frame for the audio format. + */ +typedef void toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, + int16_t const *pcm, + size_t sample_count, + uint8_t channels, + uint32_t sampling_rate, + void *user_data); +/** + * Set the callback for the `receive_audio_frame` event. Pass NULL to unset. + */ +void toxav_callback_receive_audio_frame(ToxAV *av, toxav_receive_audio_frame_cb *function, void *user_data); \ No newline at end of file diff --git a/toxav/toxav_new.c b/toxav/toxav_new.c deleted file mode 100644 index 857d5a838..000000000 --- a/toxav/toxav_new.c +++ /dev/null @@ -1,920 +0,0 @@ -/** toxav.c - * - * Copyright (C) 2013 Tox project All Rights Reserved. - * - * This file is part of Tox. - * - * Tox is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Tox is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Tox. If not, see . - * - */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif /* HAVE_CONFIG_H */ - -#include "toxav_new.h" -#include "msi.h" /* Includes codec.h and rtp.h */ - -#include "../toxcore/Messenger.h" -#include "../toxcore/logger.h" -#include "../toxcore/util.h" - -#include -#include -#include - -#define MAX_ENCODE_TIME_US ((1000 / 24) * 1000) - -enum { - audio_index, - video_index, -}; - -typedef struct iToxAVCall -{ - pthread_mutex_t mutex_control[1]; - pthread_mutex_t mutex_encoding_audio[1]; - pthread_mutex_t mutex_encoding_video[1]; - pthread_mutex_t mutex_do[1]; - RTPSession *rtps[2]; /** Audio is first and video is second */ - CSSession *cs; - bool active; - int32_t friend_number; - int32_t call_idx; /* FIXME msi compat, remove */ - - struct iToxAVCall *prev; - struct iToxAVCall *next; -} IToxAVCall; - -struct toxAV -{ - Messenger* m; - MSISession* msi; - - /* Two-way storage: first is array of calls and second is list of calls with head and tail */ - IToxAVCall** calls; - uint32_t calls_tail; - uint32_t calls_head; - - PAIR(toxav_call_cb *, void*) ccb; /* Call callback */ - PAIR(toxav_call_state_cb *, void *) scb; /* Call state callback */ - PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ - PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ - - /** Decode time measures */ - int32_t dmssc; /** Measure count */ - int32_t dmsst; /** Last cycle total */ - int32_t dmssa; /** Average decoding time in ms */ - - uint32_t interval; /** Calculated interval */ -}; - - -void i_toxav_msi_callback_invite(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_ringing(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_start(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_cancel(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_reject(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_end(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_request_to(void* toxav_inst, int32_t call_idx, void *data); /* TODO remove */ -void i_toxav_msi_callback_peer_to(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_state_change(void* toxav_inst, int32_t call_idx, void *data); - -IToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number); -IToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number); -void i_toxav_remove_call(ToxAV* av, uint32_t friend_number); -bool i_toxav_audio_bitrate_invalid(uint32_t bitrate); -bool i_toxav_video_bitrate_invalid(uint32_t bitrate); -IToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error); -bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call); -void i_toxav_kill_transmission(ToxAV* av, IToxAVCall* call); - - - -ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) -{ - TOXAV_ERR_NEW rc = TOXAV_ERR_NEW_OK; - ToxAV *av = NULL; - - if (tox == NULL) { - rc = TOXAV_ERR_NEW_NULL; - goto FAILURE; - } - - if (((Messenger*)tox)->msi_packet) { - rc = TOXAV_ERR_NEW_MULTIPLE; - goto FAILURE; - } - - av = calloc ( sizeof(ToxAV), 1); - - if (av == NULL) { - LOGGER_WARNING("Allocation failed!"); - rc = TOXAV_ERR_NEW_MALLOC; - goto FAILURE; - } - - av->m = (Messenger *)tox; - av->msi = msi_new(av->m, 100); /* TODO remove max calls */ - - if (av->msi == NULL) { - rc = TOXAV_ERR_NEW_MALLOC; - goto FAILURE; - } - - av->interval = 200; - av->msi->agent_handler = av; - - msi_register_callback(av->msi, i_toxav_msi_callback_invite, msi_OnInvite, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_ringing, msi_OnRinging, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_start, msi_OnStart, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_cancel, msi_OnCancel, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_reject, msi_OnReject, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_end, msi_OnEnd, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_request_to, msi_OnRequestTimeout, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_peer_to, msi_OnPeerTimeout, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_state_change, msi_OnPeerCSChange, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_state_change, msi_OnSelfCSChange, NULL); - - - if (error) - *error = rc; - - return av; - -FAILURE: - if (error) - *error = rc; - - free(av); - - return NULL; -} - -void toxav_kill(ToxAV* av) -{ - if (av == NULL) - return; - - msi_kill(av->msi); - /* TODO iterate over calls */ - free(av); -} - -Tox* toxav_get_tox(ToxAV* av) -{ - return (Tox*) av->m; -} - -uint32_t toxav_iteration_interval(const ToxAV* av) -{ - return av->interval; -} - -void toxav_iteration(ToxAV* av) -{ - msi_do(av->msi); - - uint64_t start = current_time_monotonic(); - uint32_t rc = 200 + av->dmssa; /* If no call is active interval is 200 */ - - IToxAVCall* i = av->calls[av->calls_head]; - for (; i; i = i->next) { - if (i->active) { - cs_do(i->cs); - rc = MIN(i->cs->last_packet_frame_duration, rc); - } - } - - av->interval = rc < av->dmssa ? 0 : rc - av->dmssa; - av->dmsst += current_time_monotonic() - start; - - if (++av->dmssc == 3) { - av->dmssa = av->dmsst / 3 + 2 /* NOTE Magic Offset for precission */; - av->dmssc = 0; - av->dmsst = 0; - } -} - -bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) -{ - IToxAVCall* call = i_toxav_init_call(av, friend_number, audio_bit_rate, video_bit_rate, error); - if (call == NULL) { - return false; - } - - /* TODO remove csettings */ - MSICSettings csets; - csets.audio_bitrate = audio_bit_rate; - csets.video_bitrate = video_bit_rate; - - csets.call_type = video_bit_rate ? msi_TypeVideo : msi_TypeAudio; - - if (msi_invite(av->msi, &call->call_idx, &csets, 1000, friend_number) != 0) { - i_toxav_remove_call(av, friend_number); - if (error) - *error = TOXAV_ERR_CALL_MALLOC; /* FIXME: this should be the only reason to fail */ - return false; - } - - return true; -} - -void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data) -{ - av->ccb.first = function; - av->ccb.second = user_data; -} - -bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER* error) -{ - TOXAV_ERR_ANSWER rc = TOXAV_ERR_ANSWER_OK; - if (m_friend_exists(av->m, friend_number)) { - rc = TOXAV_ERR_ANSWER_FRIEND_NOT_FOUND; - goto END; - } - - if ((audio_bit_rate && i_toxav_audio_bitrate_invalid(audio_bit_rate)) - ||(video_bit_rate && i_toxav_video_bitrate_invalid(video_bit_rate)) - ) { - rc = TOXAV_ERR_CALL_INVALID_BIT_RATE; - goto END; - } - - IToxAVCall* call = i_toxav_get_call(av, friend_number); - if (call == NULL || av->msi->calls[call->call_idx]->state != msi_CallRequested) { - rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; - goto END; - } - - /* TODO remove csettings */ - MSICSettings csets; - csets.audio_bitrate = audio_bit_rate; - csets.video_bitrate = video_bit_rate; - - csets.call_type = video_bit_rate ? msi_TypeVideo : msi_TypeAudio; - - if (msi_answer(av->msi, call->call_idx, &csets) != 0) { - rc = TOXAV_ERR_ANSWER_MALLOC; /* TODO Some error here */ - /* TODO Reject call? */ - } - -END: - if (error) - *error = rc; - - return rc == TOXAV_ERR_ANSWER_OK; -} - -void toxav_callback_call_state(ToxAV* av, toxav_call_state_cb* function, void* user_data) -{ - av->scb.first = function; - av->scb.second = user_data; -} - -bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL* error) -{ - TOXAV_ERR_CALL_CONTROL rc = TOXAV_ERR_CALL_CONTROL_OK; - - if (m_friend_exists(av->m, friend_number)) { - rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_FOUND; - goto END; - } - - - IToxAVCall* call = i_toxav_get_call(av, friend_number); - if (call == NULL) { - rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; - goto END; - } - - /* TODO rest of these */ - switch (control) - { - case TOXAV_CALL_CONTROL_RESUME: { - - } break; - - case TOXAV_CALL_CONTROL_PAUSE: { - - } break; - - case TOXAV_CALL_CONTROL_CANCEL: { - if (av->msi->calls[call->call_idx]->state == msi_CallActive) { - /* Hang up */ - msi_hangup(av->msi, call->call_idx); - } else if (av->msi->calls[call->call_idx]->state == msi_CallRequested) { - /* Reject the call */ - msi_reject(av->msi, call->call_idx); - } else if (av->msi->calls[call->call_idx]->state == msi_CallRequesting) { - /* Cancel the call */ - msi_cancel(av->msi, call->call_idx); - } - } break; - - case TOXAV_CALL_CONTROL_MUTE_AUDIO: { - - } break; - - case TOXAV_CALL_CONTROL_MUTE_VIDEO: { - - } break; - } - -END: - if (error) - *error = rc; - - return rc == TOXAV_ERR_CALL_CONTROL_OK; -} - -bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE* error) -{ - /* TODO */ -} - -bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE* error) -{ - /* TODO */ -} - -void toxav_callback_request_video_frame(ToxAV* av, toxav_request_video_frame_cb* function, void* user_data) -{ - /* TODO */ -} - -bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) -{ - TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; - IToxAVCall* call; - - if (m_friend_exists(av->m, friend_number)) { - rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND; - goto END; - } - - call = i_toxav_get_call(av, friend_number); - if (call == NULL) { - rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; - goto END; - } - - if (av->msi->calls[call->call_idx]->state != msi_CallActive) { - /* TODO */ - rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; - goto END; - } - - if ( y == NULL || u == NULL || v == NULL ) { - rc = TOXAV_ERR_SEND_FRAME_NULL; - goto END; - } - - if ( cs_set_sending_video_resolution(call->cs, width, height) != 0 ) { - rc = TOXAV_ERR_SEND_FRAME_INVALID; - goto END; - } - - { /* Encode */ - vpx_image_t img; - img.w = img.h = img.d_w = img.d_h = 0; - vpx_img_alloc(&img, VPX_IMG_FMT_VPXI420, width, height, 1); - - /* I420 "It comprises an NxM Y plane followed by (N/2)x(M/2) V and U planes." - * http://fourcc.org/yuv.php#IYUV - */ - memcpy(img.planes[VPX_PLANE_Y], y, width * height); - memcpy(img.planes[VPX_PLANE_U], u, (width/2) * (height/2)); - memcpy(img.planes[VPX_PLANE_V], v, (width/2) * (height/2)); - - int vrc = vpx_codec_encode(call->cs->v_encoder, &img, - call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); - - vpx_img_free(&img); /* FIXME don't free? */ - if ( vrc != VPX_CODEC_OK) { - LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); - rc = TOXAV_ERR_SEND_FRAME_INVALID; - goto END; - } - } - - ++call->cs->frame_counter; - - { /* Split and send */ - vpx_codec_iter_t iter = NULL; - const vpx_codec_cx_pkt_t *pkt; - - cs_init_video_splitter_cycle(call->cs); - - while ( (pkt = vpx_codec_get_cx_data(call->cs->v_encoder, &iter)) ) { - if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { - int parts = cs_update_video_splitter_cycle(call->cs, pkt->data.frame.buf, - pkt->data.frame.sz); - - if (parts < 0) /* Should never happen though */ - continue; - - uint16_t part_size; - const uint8_t *iter; - - int i; - for (i = 0; i < parts; i++) { - iter = cs_iterate_split_video_frame(call->cs, &part_size); - - if (rtp_send_msg(call->rtps[video_index], iter, part_size) < 0) - goto END; - } - } - } - } - -END: - if (error) - *error = rc; - - return rc == TOXAV_ERR_SEND_FRAME_OK; -} - -void toxav_callback_request_audio_frame(ToxAV* av, toxav_request_audio_frame_cb* function, void* user_data) -{ - /* TODO */ -} - -bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error) -{ - TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; - IToxAVCall* call; - - if (m_friend_exists(av->m, friend_number)) { - rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND; - goto END; - } - - call = i_toxav_get_call(av, friend_number); - if (call == NULL) { - rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; - goto END; - } - - if (av->msi->calls[call->call_idx]->state != msi_CallActive) { - /* TODO */ - rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; - goto END; - } - - if ( pcm == NULL ) { - rc = TOXAV_ERR_SEND_FRAME_NULL; - goto END; - } - - if ( channels != 1 || channels != 2 ) { - rc = TOXAV_ERR_SEND_FRAME_INVALID; - goto END; - } - - { /* Encode and send */ - /* TODO redundant? */ - cs_set_sending_audio_channels(call->cs, channels); - cs_set_sending_audio_sampling_rate(call->cs, sampling_rate); - - uint8_t dest[sample_count * channels * 2 /* sizeof(uint16_t) */]; - int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest, sizeof (dest)); - - if (vrc < 0) { - LOGGER_WARNING("Failed to encode frame"); - rc = TOXAV_ERR_SEND_FRAME_INVALID; - goto END; - } - - vrc = rtp_send_msg(call->rtps[audio_index], dest, vrc); - /* TODO check for error? */ - } - -END: - if (error) - *error = rc; - - return rc == TOXAV_ERR_SEND_FRAME_OK; -} - -void toxav_callback_receive_video_frame(ToxAV* av, toxav_receive_video_frame_cb* function, void* user_data) -{ - av->vcb.first = function; - av->vcb.second = user_data; -} - -void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* function, void* user_data) -{ - av->acb.first = function; - av->acb.second = user_data; -} - - -/******************************************************************************* - * - * :: Internal - * - ******************************************************************************/ -/** TODO: - * - In msi call_idx can be the same as friend id - * - If crutial callback not present send error - * - Remove *data from msi - * - Remove CSettings from msi - */ -void i_toxav_msi_callback_invite(void* toxav_inst, int32_t call_idx, void* data) -{ - ToxAV* toxav = toxav_inst; - - uint32_t ab = toxav->msi->calls[call_idx]->csettings_peer[0].audio_bitrate; - uint32_t vb = toxav->msi->calls[call_idx]->csettings_peer[0].video_bitrate; - - IToxAVCall* call = i_toxav_init_call(toxav, toxav->msi->calls[call_idx]->peers[0], ab, vb, NULL); - if (call == NULL) { - msi_reject(toxav->msi, call_idx, NULL); - return false; - } - - call->call_idx = call_idx; - - if (toxav->ccb.first) - toxav->ccb.first(toxav, toxav->msi->calls[call_idx]->peers[0], true, true, toxav->ccb.second); -} - -void i_toxav_msi_callback_ringing(void* toxav_inst, int32_t call_idx, void* data) -{ - ToxAV* toxav = toxav_inst; - if (toxav->scb.first) - toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], - TOXAV_CALL_STATE_RINGING, toxav->scb.second); -} - -void i_toxav_msi_callback_start(void* toxav_inst, int32_t call_idx, void* data) -{ - ToxAV* toxav = toxav_inst; - - IToxAVCall* call = i_toxav_get_call(toxav, toxav->msi->calls[call_idx]->peers[0]); - - if (call == NULL || !i_toxav_prepare_transmission(toxav, call)) { - /* TODO send error */ - i_toxav_remove_call(toxav, toxav->msi->calls[call_idx]->peers[0]); - return; - } - - TOXAV_CALL_STATE state; - const MSICSettings* csets = toxav->msi->calls[call_idx]->csettings_peer[0]; - - if (csets->audio_bitrate && csets->video_bitrate) - state = TOXAV_CALL_STATE_SENDING_AV; - else if (csets->video_bitrate == 0) - state = TOXAV_CALL_STATE_SENDING_A; - else - state = TOXAV_CALL_STATE_SENDING_V; - - if (toxav->scb.first) /* TODO this */ - toxav->scb.first(toxav, call->friend_number, state, toxav->scb.second); -} - -void i_toxav_msi_callback_cancel(void* toxav_inst, int32_t call_idx, void* data) -{ - ToxAV* toxav = toxav_inst; - if (toxav->scb.first) - toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], - TOXAV_CALL_STATE_END, toxav->scb.second); -} - -void i_toxav_msi_callback_reject(void* toxav_inst, int32_t call_idx, void* data) -{ - ToxAV* toxav = toxav_inst; - if (toxav->scb.first) - toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], - TOXAV_CALL_STATE_END, toxav->scb.second); -} - -void i_toxav_msi_callback_end(void* toxav_inst, int32_t call_idx, void* data) -{ - ToxAV* toxav = toxav_inst; - if (toxav->scb.first) - toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], - TOXAV_CALL_STATE_END, toxav->scb.second); -} - -void i_toxav_msi_callback_request_to(void* toxav_inst, int32_t call_idx, void* data) -{ - /* TODO remove */ - ToxAV* toxav = toxav_inst; - if (toxav->scb.first) - toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], - TOXAV_CALL_STATE_ERROR, toxav->scb.second); -} - -void i_toxav_msi_callback_peer_to(void* toxav_inst, int32_t call_idx, void* data) -{ - ToxAV* toxav = toxav_inst; - if (toxav->scb.first) - toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], - TOXAV_CALL_STATE_ERROR, toxav->scb.second); -} - -void i_toxav_msi_callback_state_change(void* toxav_inst, int32_t call_idx, void* data) -{ - ToxAV* toxav = toxav_inst; - /* TODO something something msi */ -} - -IToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number) -{ - if (av->calls_tail < friend_number) - return NULL; - - return av->calls[friend_number]; -} - -IToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number) -{ - IToxAVCall* rc = calloc(sizeof(IToxAVCall), 1); - - if (rc == NULL) - return NULL; - - rc->friend_number = friend_number; - - if (create_recursive_mutex(rc->mutex_control) != 0) { - free(rc); - return NULL; - } - - if (create_recursive_mutex(rc->mutex_do) != 0) { - pthread_mutex_destroy(rc->mutex_control); - free(rc); - return NULL; - } - - - if (av->calls == NULL) { /* Creating */ - av->calls = calloc (sizeof(IToxAVCall*), friend_number + 1); - - if (av->calls == NULL) { - pthread_mutex_destroy(rc->mutex_control); - pthread_mutex_destroy(rc->mutex_do); - free(rc); - return NULL; - } - - av->calls_tail = av->calls_head = friend_number; - - } else if (av->calls_tail < friend_number) { /* Appending */ - void* tmp = realloc(av->calls, sizeof(IToxAVCall*) * friend_number + 1); - - if (tmp == NULL) { - pthread_mutex_destroy(rc->mutex_control); - pthread_mutex_destroy(rc->mutex_do); - free(rc); - return NULL; - } - - av->calls = tmp; - - /* Set fields in between to null */ - int32_t i = av->calls_tail; - for (; i < friend_number; i ++) - av->calls[i] = NULL; - - rc->prev = av->calls[av->calls_tail]; - av->calls[av->calls_tail]->next = rc; - - av->calls_tail = friend_number; - - } else if (av->calls_head > friend_number) { /* Inserting at front */ - rc->next = av->calls[av->calls_head]; - av->calls[av->calls_head]->prev = rc; - av->calls_head = friend_number; - } - - av->calls[friend_number] = rc; - return rc; -} - -void i_toxav_remove_call(ToxAV* av, uint32_t friend_number) -{ - IToxAVCall* tc = i_toxav_get_call(av, friend_number); - - if (tc == NULL) - return; - - IToxAVCall* prev = tc->prev; - IToxAVCall* next = tc->next; - - pthread_mutex_destroy(tc->mutex_control); - pthread_mutex_destroy(tc->mutex_do); - - free(tc); - - if (prev) - prev->next = next; - else if (next) - av->calls_head = next->friend_number; - else goto CLEAR; - - if (next) - next->prev = prev; - else if (prev) - av->calls_tail = prev->friend_number; - else goto CLEAR; - - av->calls[friend_number] = NULL; - return; - -CLEAR: - av->calls_head = av->calls_tail = 0; - free(av->calls); - av->calls = NULL; -} - -bool i_toxav_audio_bitrate_invalid(uint32_t bitrate) -{ - /* Opus RFC 6716 section-2.1.1 dictates the following: - * Opus supports all bitrates from 6 kbit/s to 510 kbit/s. - */ - return bitrate < 6 || bitrate > 510; -} - -bool i_toxav_video_bitrate_invalid(uint32_t bitrate) -{ - /* TODO: If anyone knows the answer to this one please fill it up */ - return false; -} - -IToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) -{ - TOXAV_ERR_CALL rc = TOXAV_ERR_CALL_OK; - IToxAVCall* call = NULL; - - if (m_friend_exists(av->m, friend_number)) { - rc = TOXAV_ERR_CALL_FRIEND_NOT_FOUND; - goto END; - } - - if (m_get_friend_connectionstatus(av->m, friend_number) != 1) { - rc = TOXAV_ERR_CALL_FRIEND_NOT_CONNECTED; - goto END; - } - - if (i_toxav_get_call(av, friend_number) != NULL) { - rc = TOXAV_ERR_CALL_FRIEND_ALREADY_IN_CALL; - goto END; - } - - if ((audio_bit_rate && i_toxav_audio_bitrate_invalid(audio_bit_rate)) - ||(video_bit_rate && i_toxav_video_bitrate_invalid(video_bit_rate)) - ) { - rc = TOXAV_ERR_CALL_INVALID_BIT_RATE; - goto END; - } - - IToxAVCall* call = i_toxav_add_call(av, friend_number); - if (call == NULL) { - rc = TOXAV_ERR_CALL_MALLOC; - } - -END: - if (error) - *error = rc; - - return call; -} - -bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call) -{ - pthread_mutex_lock(call->mutex_control); - - if (call->active) { - pthread_mutex_unlock(call->mutex_control); - LOGGER_WARNING("Call already active!\n"); - return true; - } - - if (pthread_mutex_init(call->mutex_encoding_audio, NULL) != 0) - goto MUTEX_INIT_ERROR; - - if (pthread_mutex_init(call->mutex_encoding_video, NULL) != 0) { - pthread_mutex_destroy(call->mutex_encoding_audio); - goto MUTEX_INIT_ERROR; - } - - if (pthread_mutex_init(call->mutex_do, NULL) != 0) { - pthread_mutex_destroy(call->mutex_encoding_audio); - pthread_mutex_destroy(call->mutex_encoding_video); - goto MUTEX_INIT_ERROR; - } - - const MSICSettings *c_peer = &av->msi->calls[call->call_idx]->csettings_peer[0]; - const MSICSettings *c_self = &av->msi->calls[call->call_idx]->csettings_local; - - call->cs = cs_new(c_self->audio_bitrate, c_peer->audio_bitrate, - c_self->video_bitrate, c_peer->video_bitrate); - - if ( !call->cs ) { - LOGGER_ERROR("Error while starting Codec State!\n"); - goto FAILURE; - } - - call->cs->agent = av; - call->cs->call_idx = call->call_idx; - - - if (c_self->audio_bitrate > 0 || c_peer->audio_bitrate > 0) { /* Prepare audio rtp */ - call->rtps[audio_index] = rtp_new(msi_TypeAudio, av->m, av->msi->calls[call->call_idx]->peers[0]); - - if ( !call->rtps[audio_index] ) { - LOGGER_ERROR("Error while starting audio RTP session!\n"); - goto FAILURE; - } - - call->rtps[audio_index]->cs = call->cs; - - if (c_peer->audio_bitrate > 0) - rtp_register_for_receiving(call->rtps[audio_index]); - } - - if (c_self->video_bitrate > 0 || c_peer->video_bitrate > 0) { /* Prepare video rtp */ - call->rtps[video_index] = rtp_new(msi_TypeVideo, av->m, av->msi->calls[call->call_idx]->peers[0]); - - if ( !call->rtps[video_index] ) { - LOGGER_ERROR("Error while starting video RTP session!\n"); - goto FAILURE; - } - - call->rtps[video_index]->cs = call->cs; - - if (c_peer->video_bitrate > 0) - rtp_register_for_receiving(call->rtps[audio_index]); - } - - call->active = 1; - pthread_mutex_unlock(call->mutex_control); - return true; - -FAILURE: - rtp_kill(call->rtps[audio_index]); - call->rtps[audio_index] = NULL; - rtp_kill(call->rtps[video_index]); - call->rtps[video_index] = NULL; - cs_kill(call->cs); - call->cs = NULL; - call->active = 0; - pthread_mutex_destroy(call->mutex_encoding_audio); - pthread_mutex_destroy(call->mutex_encoding_video); - pthread_mutex_destroy(call->mutex_do); - - pthread_mutex_unlock(call->mutex_control); - return false; - -MUTEX_INIT_ERROR: - pthread_mutex_unlock(call->mutex_control); - LOGGER_ERROR("Mutex initialization failed!\n"); - return false; -} - -void i_toxav_kill_transmission(ToxAV* av, IToxAVCall* call) -{ - pthread_mutex_lock(call->mutex_control); - - if (!call->active) { - pthread_mutex_unlock(call->mutex_control); - LOGGER_WARNING("Action on inactive call: %d", call->call_idx); - return; - } - - call->active = 0; - - pthread_mutex_lock(call->mutex_encoding_audio); - pthread_mutex_unlock(call->mutex_encoding_audio); - pthread_mutex_lock(call->mutex_encoding_video); - pthread_mutex_unlock(call->mutex_encoding_video); - pthread_mutex_lock(call->mutex_do); - pthread_mutex_unlock(call->mutex_do); - - rtp_kill(call->rtps[audio_index]); - call->rtps[audio_index] = NULL; - rtp_kill(call->rtps[video_index]); - call->rtps[video_index] = NULL; - cs_kill(call->cs); - call->cs = NULL; - - pthread_mutex_destroy(call->mutex_encoding_audio); - pthread_mutex_destroy(call->mutex_encoding_video); - pthread_mutex_destroy(call->mutex_do); - - pthread_mutex_unlock(call->mutex_control); -} diff --git a/toxav/toxav_new.h b/toxav/toxav_new.h deleted file mode 100644 index 038ee99ac..000000000 --- a/toxav/toxav_new.h +++ /dev/null @@ -1,481 +0,0 @@ -#pragma once -#include -#include -#include -/** \page av Public audio/video API for Tox clients. - * - * Unlike the Core API, this API is fully thread-safe. The library will ensure - * the proper synchronisation of parallel calls. - */ -/** - * The type of the Tox Audio/Video subsystem object. - */ -typedef struct toxAV ToxAV; -#ifndef TOX_DEFINED -#define TOX_DEFINED -/** - * The type of a Tox instance. Repeated here so this file does not have a direct - * dependency on the Core interface. - */ -typedef struct Tox Tox; -#endif -/******************************************************************************* - * - * :: Creation and destruction - * - ******************************************************************************/ -typedef enum TOXAV_ERR_NEW { - TOXAV_ERR_NEW_OK, - TOXAV_ERR_NEW_NULL, - /** - * Memory allocation failure while trying to allocate structures required for - * the A/V session. - */ - TOXAV_ERR_NEW_MALLOC, - /** - * Attempted to create a second session for the same Tox instance. - */ - TOXAV_ERR_NEW_MULTIPLE -} TOXAV_ERR_NEW; -/** - * Start new A/V session. There can only be only one session per Tox instance. - */ -ToxAV *toxav_new(Tox *tox, TOXAV_ERR_NEW *error); -/** - * Releases all resources associated with the A/V session. - * - * If any calls were ongoing, these will be forcibly terminated without - * notifying peers. After calling this function, no other functions may be - * called and the av pointer becomes invalid. - */ -void toxav_kill(ToxAV *av); -/** - * Returns the Tox instance the A/V object was created for. - */ -Tox *toxav_get_tox(ToxAV *av); -/******************************************************************************* - * - * :: A/V event loop - * - ******************************************************************************/ -/** - * Returns the interval in milliseconds when the next toxav_iteration should be - * called. If no call is active at the moment, this function returns 200. - */ -uint32_t toxav_iteration_interval(ToxAV const *av); -/** - * Main loop for the session. This function needs to be called in intervals of - * toxav_iteration_interval() milliseconds. It is best called in the same loop - * as tox_iteration. - */ -void toxav_iteration(ToxAV *av); -/******************************************************************************* - * - * :: Call setup - * - ******************************************************************************/ -typedef enum TOXAV_ERR_CALL { - TOXAV_ERR_CALL_OK, - /** - * A resource allocation error occurred while trying to create the structures - * required for the call. - */ - TOXAV_ERR_CALL_MALLOC, - /** - * The friend number did not designate a valid friend. - */ - TOXAV_ERR_CALL_FRIEND_NOT_FOUND, - /** - * The friend was valid, but not currently connected. - */ - TOXAV_ERR_CALL_FRIEND_NOT_CONNECTED, - /** - * Attempted to call a friend while already in an audio or video call with - * them. - */ - TOXAV_ERR_CALL_FRIEND_ALREADY_IN_CALL, - /** - * Audio or video bit rate is invalid. - */ - TOXAV_ERR_CALL_INVALID_BIT_RATE -} TOXAV_ERR_CALL; -/** - * Call a friend. This will start ringing the friend. - * - * It is the client's responsibility to stop ringing after a certain timeout, - * if such behaviour is desired. If the client does not stop ringing, the A/V - * library will not stop until the friend is disconnected. - * - * @param friend_number The friend number of the friend that should be called. - * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable - * audio sending. - * @param video_bit_rate Video bit rate in Kb/sec. Set this to 0 to disable - * video sending. - */ -bool toxav_call(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL *error); -/** - * The function type for the `call` callback. - */ -typedef void toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data); -/** - * Set the callback for the `call` event. Pass NULL to unset. - * - * This event is triggered when a call is received from a friend. - */ -void toxav_callback_call(ToxAV *av, toxav_call_cb *function, void *user_data); -typedef enum TOXAV_ERR_ANSWER { - TOXAV_ERR_ANSWER_OK, - /** - * A resource allocation error occurred while trying to create the structures - * required for the call. - */ - TOXAV_ERR_ANSWER_MALLOC, - /** - * The friend number did not designate a valid friend. - */ - TOXAV_ERR_ANSWER_FRIEND_NOT_FOUND, - /** - * The friend was valid, but they are not currently trying to initiate a call. - * This is also returned if this client is already in a call with the friend. - */ - TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING, - /** - * Audio or video bit rate is invalid. - */ - TOXAV_ERR_ANSWER_INVALID_BIT_RATE -} TOXAV_ERR_ANSWER; -/** - * Accept an incoming call. - * - * If an allocation error occurs while answering a call, both participants will - * receive TOXAV_CALL_STATE_ERROR and the call will end. - * - * @param friend_number The friend number of the friend that is calling. - * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable - * audio sending. - * @param video_bit_rate Video bit rate in Kb/sec. Set this to 0 to disable - * video sending. - */ -bool toxav_answer(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER *error); -/******************************************************************************* - * - * :: Call state graph - * - ******************************************************************************/ -typedef enum TOXAV_CALL_STATE { - /** - * The friend's client is aware of the call. This happens after calling - * toxav_call and the initial call request has been received. - */ - TOXAV_CALL_STATE_RINGING, - /** - * Not sending anything. Either the friend requested that this client stops - * sending anything, or the client turned off both audio and video by setting - * the respective bit rates to 0. - * - * If both sides are in this state, the call is effectively on hold, but not - * in the PAUSED state. - */ - TOXAV_CALL_STATE_NOT_SENDING, - /** - * Sending audio only. Either the friend requested that this client stops - * sending video, or the client turned off video by setting the video bit rate - * to 0. - */ - TOXAV_CALL_STATE_SENDING_A, - /** - * Sending video only. Either the friend requested that this client stops - * sending audio (muted), or the client turned off audio by setting the audio - * bit rate to 0. - */ - TOXAV_CALL_STATE_SENDING_V, - /** - * Sending both audio and video. - */ - TOXAV_CALL_STATE_SENDING_AV, - /** - * The call is on hold. Both sides stop sending and receiving. - */ - TOXAV_CALL_STATE_PAUSED, - /** - * The call has finished. This is the final state after which no more state - * transitions can occur for the call. - */ - TOXAV_CALL_STATE_END, - /** - * Sent by the AV core if an error occurred on the remote end. - */ - TOXAV_CALL_STATE_ERROR -} TOXAV_CALL_STATE; -/** - * The function type for the `call_state` callback. - * - * @param friend_number The friend number for which the call state changed. - * @param state The new call state. - */ -typedef void toxav_call_state_cb(ToxAV *av, uint32_t friend_number, TOXAV_CALL_STATE state, void *user_data); -/** - * Set the callback for the `call_state` event. Pass NULL to unset. - * - * This event is triggered when a call state transition occurs. - */ -void toxav_callback_call_state(ToxAV *av, toxav_call_state_cb *function, void *user_data); -/******************************************************************************* - * - * :: Call control - * - ******************************************************************************/ -typedef enum TOXAV_CALL_CONTROL { - /** - * Resume a previously paused call. Only valid if the pause was caused by this - * client. Not valid before the call is accepted. - */ - TOXAV_CALL_CONTROL_RESUME, - /** - * Put a call on hold. Not valid before the call is accepted. - */ - TOXAV_CALL_CONTROL_PAUSE, - /** - * Reject a call if it was not answered, yet. Cancel a call after it was - * answered. - */ - TOXAV_CALL_CONTROL_CANCEL, - /** - * Request that the friend stops sending audio. Regardless of the friend's - * compliance, this will cause the `receive_audio_frame` event to stop being - * triggered on receiving an audio frame from the friend. - */ - TOXAV_CALL_CONTROL_MUTE_AUDIO, - /** - * Request that the friend stops sending video. Regardless of the friend's - * compliance, this will cause the `receive_video_frame` event to stop being - * triggered on receiving an video frame from the friend. - */ - TOXAV_CALL_CONTROL_MUTE_VIDEO -} TOXAV_CALL_CONTROL; -typedef enum TOXAV_ERR_CALL_CONTROL { - TOXAV_ERR_CALL_CONTROL_OK, - /** - * The friend_number passed did not designate a valid friend. - */ - TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_FOUND, - /** - * This client is currently not in a call with the friend. Before the call is - * answered, only CANCEL is a valid control. - */ - TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL, - /** - * Attempted to resume a call that was not paused. - */ - TOXAV_ERR_CALL_CONTROL_NOT_PAUSED, - /** - * Attempted to resume a call that was paused by the other party. Also set if - * the client attempted to send a system-only control. - */ - TOXAV_ERR_CALL_CONTROL_DENIED, - /** - * The call was already paused on this client. It is valid to pause if the - * other party paused the call. The call will resume after both parties sent - * the RESUME control. - */ - TOXAV_ERR_CALL_CONTROL_ALREADY_PAUSED -} TOXAV_ERR_CALL_CONTROL; -/** - * Sends a call control command to a friend. - * - * @param friend_number The friend number of the friend this client is in a call - * with. - * @param control The control command to send. - * - * @return true on success. - */ -bool toxav_call_control(ToxAV *av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL *error); -/******************************************************************************* - * - * :: Controlling bit rates - * - ******************************************************************************/ -typedef enum TOXAV_ERR_BIT_RATE { - TOXAV_ERR_BIT_RATE_OK, - /** - * The bit rate passed was not one of the supported values. - */ - TOXAV_ERR_BIT_RATE_INVALID -} TOXAV_ERR_BIT_RATE; -/** - * Set the audio bit rate to be used in subsequent audio frames. - * - * @param friend_number The friend number of the friend for which to set the - * audio bit rate. - * @param audio_bit_rate The new audio bit rate in Kb/sec. Set to 0 to disable - * audio sending. - * - * @see toxav_call for the valid bit rates. - */ -bool toxav_set_audio_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE *error); -/** - * Set the video bit rate to be used in subsequent video frames. - * - * @param friend_number The friend number of the friend for which to set the - * video bit rate. - * @param video_bit_rate The new video bit rate in Kb/sec. Set to 0 to disable - * video sending. - * - * @see toxav_call for the valid bit rates. - */ -bool toxav_set_video_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE *error); -/******************************************************************************* - * - * :: A/V sending - * - ******************************************************************************/ -/** - * Common error codes for the send_*_frame functions. - */ -typedef enum TOXAV_ERR_SEND_FRAME { - TOXAV_ERR_SEND_FRAME_OK, - /** - * In case of video, one of Y, U, or V was NULL. In case of audio, the samples - * data pointer was NULL. - */ - TOXAV_ERR_SEND_FRAME_NULL, - /** - * The friend_number passed did not designate a valid friend. - */ - TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND, - /** - * This client is currently not in a call with the friend. - */ - TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL, - /** - * No video frame had been requested through the `request_video_frame` event, - * but the client tried to send one, anyway. - */ - TOXAV_ERR_SEND_FRAME_NOT_REQUESTED, - /** - * One of the frame parameters was invalid. E.g. the resolution may be too - * small or too large, or the audio sampling rate may be unsupported. - */ - TOXAV_ERR_SEND_FRAME_INVALID -} TOXAV_ERR_SEND_FRAME; -/** - * The function type for the `request_video_frame` callback. - * - * @param friend_number The friend number of the friend for which the next video - * frame should be sent. - */ -typedef void toxav_request_video_frame_cb(ToxAV *av, uint32_t friend_number, void *user_data); -/** - * Set the callback for the `request_video_frame` event. Pass NULL to unset. - */ -void toxav_callback_request_video_frame(ToxAV *av, toxav_request_video_frame_cb *function, void *user_data); -/** - * Send a video frame to a friend. - * - * This is called in response to receiving the `request_video_frame` event. - * - * Y - plane should be of size: height * width - * U - plane should be of size: (height/2) * (width/2) - * V - plane should be of size: (height/2) * (width/2) - * - * @param friend_number The friend number of the friend to which to send a video - * frame. - * @param width Width of the frame in pixels. - * @param height Height of the frame in pixels. - * @param y Y (Luminance) plane data. - * @param u U (Chroma) plane data. - * @param v V (Chroma) plane data. - */ -bool toxav_send_video_frame(ToxAV *av, uint32_t friend_number, - uint16_t width, uint16_t height, - uint8_t const *y, uint8_t const *u, uint8_t const *v, - TOXAV_ERR_SEND_FRAME *error); -/** - * The function type for the `request_audio_frame` callback. - * - * @param friend_number The friend number of the friend for which the next audio - * frame should be sent. - */ -typedef void toxav_request_audio_frame_cb(ToxAV *av, uint32_t friend_number, void *user_data); -/** - * Set the callback for the `request_audio_frame` event. Pass NULL to unset. - */ -void toxav_callback_request_audio_frame(ToxAV *av, toxav_request_audio_frame_cb *function, void *user_data); -/** - * Send an audio frame to a friend. - * - * This is called in response to receiving the `request_audio_frame` event. - * - * The expected format of the PCM data is: [s1c1][s1c2][...][s2c1][s2c2][...]... - * Meaning: sample 1 for channel 1, sample 1 for channel 2, ... - * For mono audio, this has no meaning, every sample is subsequent. For stereo, - * this means the expected format is LRLRLR... with samples for left and right - * alternating. - * - * @param friend_number The friend number of the friend to which to send an - * audio frame. - * @param pcm An array of audio samples. The size of this array must be - * sample_count * channels. - * @param sample_count Number of samples in this frame. Valid numbers here are - * ((sample rate) * (audio length) / 1000), where audio length can be - * 2.5, 5, 10, 20, 40 or 60 millseconds. - * @param channels Number of audio channels. Must be at least 1 for mono. - * For voice over IP, more than 2 channels (stereo) typically doesn't make - * sense, but up to 255 channels are supported. - * @param sampling_rate Audio sampling rate used in this frame. Valid sampling - * rates are 8000, 12000, 16000, 24000, or 48000. - */ -bool toxav_send_audio_frame(ToxAV *av, uint32_t friend_number, - int16_t const *pcm, - size_t sample_count, - uint8_t channels, - uint32_t sampling_rate, - TOXAV_ERR_SEND_FRAME *error); -/******************************************************************************* - * - * :: A/V receiving - * - ******************************************************************************/ -/** - * The function type for the `receive_video_frame` callback. - * - * Each plane contains (width * height) pixels. The Alpha plane can be NULL, in - * which case every pixel should be assumed fully opaque. - * - * @param friend_number The friend number of the friend who sent a video frame. - * @param width Width of the frame in pixels. - * @param height Height of the frame in pixels. - * @param y Y (Luminance) plane data. - * @param u U (Chroma) plane data. - * @param v V (Chroma) plane data. - * @param a A (Alpha) plane data. - */ -typedef void toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, - uint16_t width, uint16_t height, - uint8_t const *y, uint8_t const *u, uint8_t const *v, uint8_t const *a, - void *user_data); -/** - * Set the callback for the `receive_video_frame` event. Pass NULL to unset. - */ -void toxav_callback_receive_video_frame(ToxAV *av, toxav_receive_video_frame_cb *function, void *user_data); -/** - * The function type for the `receive_audio_frame` callback. - * - * @param friend_number The friend number of the friend who sent an audio frame. - * @param pcm An array of audio samples (sample_count * channels elements). - * @param sample_count The number of audio samples per channel in the PCM array. - * @param channels Number of audio channels. - * @param sampling_rate Sampling rate used in this frame. - * - * @see toxav_send_audio_frame for the audio format. - */ -typedef void toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, - int16_t const *pcm, - size_t sample_count, - uint8_t channels, - uint32_t sampling_rate, - void *user_data); -/** - * Set the callback for the `receive_audio_frame` event. Pass NULL to unset. - */ -void toxav_callback_receive_audio_frame(ToxAV *av, toxav_receive_audio_frame_cb *function, void *user_data); \ No newline at end of file diff --git a/toxav/toxav_new_1.c b/toxav/toxav_new_1.c new file mode 100644 index 000000000..ee7f49a64 --- /dev/null +++ b/toxav/toxav_new_1.c @@ -0,0 +1,689 @@ +/** toxav.c + * + * Copyright (C) 2013 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif /* HAVE_CONFIG_H */ + +#define __TOX_DEFINED__ +typedef struct Messenger Tox; + +#define _GNU_SOURCE /* implicit declaration warning */ + +#include "codec.h" +#include "msi.h" +#include "group.h" + +#include "../toxcore/logger.h" +#include "../toxcore/util.h" + +#include +#include +#include + +/* Assume 24 fps*/ +#define MAX_ENCODE_TIME_US ((1000 / 24) * 1000) + +/* true if invalid call index */ +#define CALL_INVALID_INDEX(idx, max) (idx < 0 || idx >= max) + +const ToxAvCSettings av_DefaultSettings = { + av_TypeAudio, + + 500, + 1280, + 720, + + 32000, + 20, + 48000, + 1 +}; + +static const uint32_t jbuf_capacity = 6; +static const uint8_t audio_index = 0, video_index = 1; + +typedef struct _ToxAvCall { + pthread_mutex_t mutex_control[1]; + pthread_mutex_t mutex_encoding_audio[1]; + pthread_mutex_t mutex_encoding_video[1]; + pthread_mutex_t mutex_do[1]; + RTPSession *crtps[2]; /** Audio is first and video is second */ + CSSession *cs; + _Bool active; +} ToxAvCall; + +struct _ToxAv { + Messenger *messenger; + MSISession *msi_session; /** Main msi session */ + ToxAvCall *calls; /** Per-call params */ + uint32_t max_calls; + + PAIR(ToxAvAudioCallback, void *) acb; + PAIR(ToxAvVideoCallback, void *) vcb; + + /* Decode time measure */ + int32_t dectmsscount; /** Measure count */ + int32_t dectmsstotal; /** Last cycle total */ + int32_t avgdectms; /** Average decoding time in ms */ +}; + +static const MSICSettings *msicsettings_cast (const ToxAvCSettings *from) +{ + assert(sizeof(MSICSettings) == sizeof(ToxAvCSettings)); + return (const MSICSettings *) from; +} + +static const ToxAvCSettings *toxavcsettings_cast (const MSICSettings *from) +{ + assert(sizeof(MSICSettings) == sizeof(ToxAvCSettings)); + return (const ToxAvCSettings *) from; + +} + +ToxAv *toxav_new( Tox *messenger, int32_t max_calls) +{ + ToxAv *av = calloc ( sizeof(ToxAv), 1); + + if (av == NULL) { + LOGGER_WARNING("Allocation failed!"); + return NULL; + } + + av->messenger = (Messenger *)messenger; + av->msi_session = msi_new(av->messenger, max_calls); + av->msi_session->agent_handler = av; + av->calls = calloc(sizeof(ToxAvCall), max_calls); + av->max_calls = max_calls; + + unsigned int i; + + for (i = 0; i < max_calls; ++i) { + if (create_recursive_mutex(av->calls[i].mutex_control) != 0 ) { + LOGGER_WARNING("Failed to init call(%u) mutex!", i); + msi_kill(av->msi_session); + + free(av->calls); + free(av); + return NULL; + } + } + + return av; +} + +void toxav_kill ( ToxAv *av ) +{ + uint32_t i; + + for (i = 0; i < av->max_calls; i ++) { + if ( av->calls[i].crtps[audio_index] ) + rtp_kill(av->calls[i].crtps[audio_index], av->msi_session->messenger_handle); + + + if ( av->calls[i].crtps[video_index] ) + rtp_kill(av->calls[i].crtps[video_index], av->msi_session->messenger_handle); + + if ( av->calls[i].cs ) + cs_kill(av->calls[i].cs); + + pthread_mutex_destroy(av->calls[i].mutex_control); + } + + msi_kill(av->msi_session); + + free(av->calls); + free(av); +} + +uint32_t toxav_do_interval(ToxAv *av) +{ + int i = 0; + uint32_t rc = 200 + av->avgdectms; /* Return 200 if no call is active */ + + for (; i < av->max_calls; i ++) { + pthread_mutex_lock(av->calls[i].mutex_control); + + if (av->calls[i].active) { + /* This should work. Video payload will always come in greater intervals */ + rc = MIN(av->calls[i].cs->audio_decoder_frame_duration, rc); + } + + pthread_mutex_unlock(av->calls[i].mutex_control); + } + + return rc < av->avgdectms ? 0 : rc - av->avgdectms; +} + +void toxav_do(ToxAv *av) +{ + msi_do(av->msi_session); + + uint64_t start = current_time_monotonic(); + + uint32_t i = 0; + + for (; i < av->max_calls; i ++) { + pthread_mutex_lock(av->calls[i].mutex_control); + + if (av->calls[i].active) { + pthread_mutex_lock(av->calls[i].mutex_do); + pthread_mutex_unlock(av->calls[i].mutex_control); + cs_do(av->calls[i].cs); + pthread_mutex_unlock(av->calls[i].mutex_do); + } else { + pthread_mutex_unlock(av->calls[i].mutex_control); + } + } + + uint64_t end = current_time_monotonic(); + + /* TODO maybe use variable for sizes */ + av->dectmsstotal += end - start; + + if (++av->dectmsscount == 3) { + av->avgdectms = av->dectmsstotal / 3 + 2 /* NOTE Magic Offset */; + av->dectmsscount = 0; + av->dectmsstotal = 0; + } +} + +void toxav_register_callstate_callback ( ToxAv *av, ToxAVCallback cb, ToxAvCallbackID id, void *userdata ) +{ + msi_register_callback(av->msi_session, (MSICallbackType)cb, (MSICallbackID) id, userdata); +} + +void toxav_register_audio_callback(ToxAv *av, ToxAvAudioCallback cb, void *userdata) +{ + av->acb.first = cb; + av->acb.second = userdata; +} + +void toxav_register_video_callback(ToxAv *av, ToxAvVideoCallback cb, void *userdata) +{ + av->vcb.first = cb; + av->vcb.second = userdata; +} + +int toxav_call (ToxAv *av, + int32_t *call_index, + int user, + const ToxAvCSettings *csettings, + int ringing_seconds ) +{ + return msi_invite(av->msi_session, call_index, msicsettings_cast(csettings), ringing_seconds * 1000, user); +} + +int toxav_hangup ( ToxAv *av, int32_t call_index ) +{ + return msi_hangup(av->msi_session, call_index); +} + +int toxav_answer ( ToxAv *av, int32_t call_index, const ToxAvCSettings *csettings ) +{ + return msi_answer(av->msi_session, call_index, msicsettings_cast(csettings)); +} + +int toxav_reject ( ToxAv *av, int32_t call_index, const char *reason ) +{ + return msi_reject(av->msi_session, call_index, reason); +} + +int toxav_cancel ( ToxAv *av, int32_t call_index, int peer_id, const char *reason ) +{ + return msi_cancel(av->msi_session, call_index, peer_id, reason); +} + +int toxav_change_settings(ToxAv *av, int32_t call_index, const ToxAvCSettings *csettings) +{ + return msi_change_csettings(av->msi_session, call_index, msicsettings_cast(csettings)); +} + +int toxav_stop_call ( ToxAv *av, int32_t call_index ) +{ + return msi_stopcall(av->msi_session, call_index); +} + +int toxav_prepare_transmission ( ToxAv *av, int32_t call_index, int support_video ) +{ + if ( !av->msi_session || CALL_INVALID_INDEX(call_index, av->msi_session->max_calls) || + !av->msi_session->calls[call_index] || !av->msi_session->calls[call_index]->csettings_peer) { + LOGGER_ERROR("Error while starting RTP session: invalid call!\n"); + return av_ErrorNoCall; + } + + ToxAvCall *call = &av->calls[call_index]; + + pthread_mutex_lock(call->mutex_control); + + if (call->active) { + pthread_mutex_unlock(call->mutex_control); + LOGGER_ERROR("Error while starting RTP session: call already active!\n"); + return av_ErrorAlreadyInCallWithPeer; + } + + if (pthread_mutex_init(call->mutex_encoding_audio, NULL) != 0 + || pthread_mutex_init(call->mutex_encoding_video, NULL) != 0 || pthread_mutex_init(call->mutex_do, NULL) != 0) { + pthread_mutex_unlock(call->mutex_control); + LOGGER_ERROR("Error while starting RTP session: mutex initializing failed!\n"); + return av_ErrorUnknown; + } + + const ToxAvCSettings *c_peer = toxavcsettings_cast + (&av->msi_session->calls[call_index]->csettings_peer[0]); + const ToxAvCSettings *c_self = toxavcsettings_cast + (&av->msi_session->calls[call_index]->csettings_local); + + LOGGER_DEBUG( + "Type: %u(s) %u(p)\n" + "Video bitrate: %u(s) %u(p)\n" + "Video height: %u(s) %u(p)\n" + "Video width: %u(s) %u(p)\n" + "Audio bitrate: %u(s) %u(p)\n" + "Audio framedur: %u(s) %u(p)\n" + "Audio sample rate: %u(s) %u(p)\n" + "Audio channels: %u(s) %u(p)\n", + c_self->call_type, c_peer->call_type, + c_self->video_bitrate, c_peer->video_bitrate, + c_self->max_video_height, c_peer->max_video_height, + c_self->max_video_width, c_peer->max_video_width, + c_self->audio_bitrate, c_peer->audio_bitrate, + c_self->audio_frame_duration, c_peer->audio_frame_duration, + c_self->audio_sample_rate, c_peer->audio_sample_rate, + c_self->audio_channels, c_peer->audio_channels ); + + if ( !(call->cs = cs_new(c_self, c_peer, jbuf_capacity, support_video)) ) { + LOGGER_ERROR("Error while starting Codec State!\n"); + pthread_mutex_unlock(call->mutex_control); + return av_ErrorInitializingCodecs; + } + + call->cs->agent = av; + call->cs->call_idx = call_index; + + call->cs->acb.first = av->acb.first; + call->cs->acb.second = av->acb.second; + + call->cs->vcb.first = av->vcb.first; + call->cs->vcb.second = av->vcb.second; + + + call->crtps[audio_index] = + rtp_new(msi_TypeAudio, av->messenger, av->msi_session->calls[call_index]->peers[0]); + + if ( !call->crtps[audio_index] ) { + LOGGER_ERROR("Error while starting audio RTP session!\n"); + goto error; + } + + call->crtps[audio_index]->cs = call->cs; + + if ( support_video ) { + call->crtps[video_index] = + rtp_new(msi_TypeVideo, av->messenger, av->msi_session->calls[call_index]->peers[0]); + + if ( !call->crtps[video_index] ) { + LOGGER_ERROR("Error while starting video RTP session!\n"); + goto error; + } + + call->crtps[video_index]->cs = call->cs; + } + + call->active = 1; + pthread_mutex_unlock(call->mutex_control); + return av_ErrorNone; +error: + rtp_kill(call->crtps[audio_index], av->messenger); + call->crtps[audio_index] = NULL; + rtp_kill(call->crtps[video_index], av->messenger); + call->crtps[video_index] = NULL; + cs_kill(call->cs); + call->cs = NULL; + call->active = 0; + pthread_mutex_destroy(call->mutex_encoding_audio); + pthread_mutex_destroy(call->mutex_encoding_video); + pthread_mutex_destroy(call->mutex_do); + + pthread_mutex_unlock(call->mutex_control); + return av_ErrorCreatingRtpSessions; +} + +int toxav_kill_transmission ( ToxAv *av, int32_t call_index ) +{ + if (CALL_INVALID_INDEX(call_index, av->msi_session->max_calls)) { + LOGGER_WARNING("Invalid call index: %d", call_index); + return av_ErrorNoCall; + } + + ToxAvCall *call = &av->calls[call_index]; + + pthread_mutex_lock(call->mutex_control); + + if (!call->active) { + pthread_mutex_unlock(call->mutex_control); + LOGGER_WARNING("Action on inactive call: %d", call_index); + return av_ErrorInvalidState; + } + + call->active = 0; + + pthread_mutex_lock(call->mutex_encoding_audio); + pthread_mutex_unlock(call->mutex_encoding_audio); + pthread_mutex_lock(call->mutex_encoding_video); + pthread_mutex_unlock(call->mutex_encoding_video); + pthread_mutex_lock(call->mutex_do); + pthread_mutex_unlock(call->mutex_do); + + rtp_kill(call->crtps[audio_index], av->messenger); + call->crtps[audio_index] = NULL; + rtp_kill(call->crtps[video_index], av->messenger); + call->crtps[video_index] = NULL; + cs_kill(call->cs); + call->cs = NULL; + + pthread_mutex_destroy(call->mutex_encoding_audio); + pthread_mutex_destroy(call->mutex_encoding_video); + pthread_mutex_destroy(call->mutex_do); + + pthread_mutex_unlock(call->mutex_control); + + return av_ErrorNone; +} + +static int toxav_send_rtp_payload(ToxAv *av, + ToxAvCall *call, + ToxAvCallType type, + const uint8_t *payload, + unsigned int length) +{ + if (call->crtps[type - av_TypeAudio]) { + + /* Audio */ + if (type == av_TypeAudio) + return rtp_send_msg(call->crtps[audio_index], av->messenger, payload, length); + + /* Video */ + int parts = cs_split_video_payload(call->cs, payload, length); + + if (parts < 0) return parts; + + uint16_t part_size; + const uint8_t *iter; + + int i; + + for (i = 0; i < parts; i++) { + iter = cs_iterate_split_video_frame(call->cs, &part_size); + + if (rtp_send_msg(call->crtps[video_index], av->messenger, iter, part_size) < 0) + return av_ErrorSendingPayload; + } + + return av_ErrorNone; + + } else return av_ErrorNoRtpSession; +} + +int toxav_prepare_video_frame ( ToxAv *av, int32_t call_index, uint8_t *dest, int dest_max, vpx_image_t *input) +{ + if (CALL_INVALID_INDEX(call_index, av->msi_session->max_calls)) { + LOGGER_WARNING("Invalid call index: %d", call_index); + return av_ErrorNoCall; + } + + + ToxAvCall *call = &av->calls[call_index]; + pthread_mutex_lock(call->mutex_control); + + if (!call->active) { + pthread_mutex_unlock(call->mutex_control); + LOGGER_WARNING("Action on inactive call: %d", call_index); + return av_ErrorInvalidState; + } + + if (cs_set_sending_video_resolution(call->cs, input->d_w, input->d_h) < 0) { + pthread_mutex_unlock(call->mutex_control); + return av_ErrorSettingVideoResolution; + } + + pthread_mutex_lock(call->mutex_encoding_video); + pthread_mutex_unlock(call->mutex_control); + + int rc = vpx_codec_encode(call->cs->v_encoder, input, call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); + + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(rc)); + pthread_mutex_unlock(call->mutex_encoding_video); + return av_ErrorEncodingVideo; + } + + ++call->cs->frame_counter; + + vpx_codec_iter_t iter = NULL; + const vpx_codec_cx_pkt_t *pkt; + int copied = 0; + + while ( (pkt = vpx_codec_get_cx_data(call->cs->v_encoder, &iter)) ) { + if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { + if ( copied + pkt->data.frame.sz > dest_max ) { + pthread_mutex_unlock(call->mutex_encoding_video); + return av_ErrorPacketTooLarge; + } + + memcpy(dest + copied, pkt->data.frame.buf, pkt->data.frame.sz); + copied += pkt->data.frame.sz; + } + } + + pthread_mutex_unlock(call->mutex_encoding_video); + return copied; +} + +int toxav_send_video ( ToxAv *av, int32_t call_index, const uint8_t *frame, unsigned int frame_size) +{ + + if (CALL_INVALID_INDEX(call_index, av->msi_session->max_calls)) { + LOGGER_WARNING("Invalid call index: %d", call_index); + return av_ErrorNoCall; + } + + ToxAvCall *call = &av->calls[call_index]; + pthread_mutex_lock(call->mutex_control); + + + if (!call->active) { + pthread_mutex_unlock(call->mutex_control); + LOGGER_WARNING("Action on inactive call: %d", call_index); + return av_ErrorInvalidState; + } + + int rc = toxav_send_rtp_payload(av, call, av_TypeVideo, frame, frame_size); + pthread_mutex_unlock(call->mutex_control); + + return rc; +} + +int toxav_prepare_audio_frame ( ToxAv *av, + int32_t call_index, + uint8_t *dest, + int dest_max, + const int16_t *frame, + int frame_size) +{ + if (CALL_INVALID_INDEX(call_index, av->msi_session->max_calls)) { + LOGGER_WARNING("Action on nonexisting call: %d", call_index); + return av_ErrorNoCall; + } + + ToxAvCall *call = &av->calls[call_index]; + pthread_mutex_lock(call->mutex_control); + + if (!call->active) { + pthread_mutex_unlock(call->mutex_control); + LOGGER_WARNING("Action on inactive call: %d", call_index); + return av_ErrorInvalidState; + } + + pthread_mutex_lock(call->mutex_encoding_audio); + pthread_mutex_unlock(call->mutex_control); + int32_t rc = opus_encode(call->cs->audio_encoder, frame, frame_size, dest, dest_max); + pthread_mutex_unlock(call->mutex_encoding_audio); + + if (rc < 0) { + LOGGER_ERROR("Failed to encode payload: %s\n", opus_strerror(rc)); + return av_ErrorEncodingAudio; + } + + return rc; +} + +int toxav_send_audio ( ToxAv *av, int32_t call_index, const uint8_t *data, unsigned int size) +{ + if (CALL_INVALID_INDEX(call_index, av->msi_session->max_calls)) { + LOGGER_WARNING("Action on nonexisting call: %d", call_index); + return av_ErrorNoCall; + } + + ToxAvCall *call = &av->calls[call_index]; + pthread_mutex_lock(call->mutex_control); + + + if (!call->active) { + pthread_mutex_unlock(call->mutex_control); + LOGGER_WARNING("Action on inactive call: %d", call_index); + return av_ErrorInvalidState; + } + + int rc = toxav_send_rtp_payload(av, call, av_TypeAudio, data, size); + pthread_mutex_unlock(call->mutex_control); + return rc; +} + +int toxav_get_peer_csettings ( ToxAv *av, int32_t call_index, int peer, ToxAvCSettings *dest ) +{ + if ( peer < 0 || CALL_INVALID_INDEX(call_index, av->msi_session->max_calls) || + !av->msi_session->calls[call_index] || av->msi_session->calls[call_index]->peer_count <= peer ) + return av_ErrorNoCall; + + *dest = *toxavcsettings_cast(&av->msi_session->calls[call_index]->csettings_peer[peer]); + return av_ErrorNone; +} + +int toxav_get_peer_id ( ToxAv *av, int32_t call_index, int peer ) +{ + if ( peer < 0 || CALL_INVALID_INDEX(call_index, av->msi_session->max_calls) || !av->msi_session->calls[call_index] + || av->msi_session->calls[call_index]->peer_count <= peer ) + return av_ErrorNoCall; + + return av->msi_session->calls[call_index]->peers[peer]; +} + +ToxAvCallState toxav_get_call_state(ToxAv *av, int32_t call_index) +{ + if ( CALL_INVALID_INDEX(call_index, av->msi_session->max_calls) || !av->msi_session->calls[call_index] ) + return av_CallNonExistent; + + return av->msi_session->calls[call_index]->state; + +} + +int toxav_capability_supported ( ToxAv *av, int32_t call_index, ToxAvCapabilities capability ) +{ +} + +Tox *toxav_get_tox(ToxAv *av) +{ + return (Tox *)av->messenger; +} + +int toxav_get_active_count(ToxAv *av) +{ + if (!av) return -1; + + int rc = 0, i = 0; + + for (; i < av->max_calls; i++) { + pthread_mutex_lock(av->calls[i].mutex_control); + + if (av->calls[i].active) rc++; + + pthread_mutex_unlock(av->calls[i].mutex_control); + } + + return rc; +} + +/* Create a new toxav group. + * + * return group number on success. + * return -1 on failure. + * + * Audio data callback format: + * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + */ +int toxav_add_av_groupchat(Tox *tox, void (*audio_callback)(Messenger *, int, int, const int16_t *, unsigned int, + uint8_t, unsigned int, void *), void *userdata) +{ + Messenger *m = tox; + return add_av_groupchat(m->group_chat_object, audio_callback, userdata); +} + +/* Join a AV group (you need to have been invited first.) + * + * returns group number on success + * returns -1 on failure. + * + * Audio data callback format (same as the one for toxav_add_av_groupchat()): + * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + */ +int toxav_join_av_groupchat(Tox *tox, int32_t friendnumber, const uint8_t *data, uint16_t length, + void (*audio_callback)(Messenger *, int, int, const int16_t *, unsigned int, uint8_t, unsigned int, void *), + void *userdata) +{ + Messenger *m = tox; + return join_av_groupchat(m->group_chat_object, friendnumber, data, length, audio_callback, userdata); +} + +/* Send audio to the group chat. + * + * return 0 on success. + * return -1 on failure. + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + * + * Valid number of samples are ((sample rate) * (audio length (Valid ones are: 2.5, 5, 10, 20, 40 or 60 ms)) / 1000) + * Valid number of channels are 1 or 2. + * Valid sample rates are 8000, 12000, 16000, 24000, or 48000. + * + * Recommended values are: samples = 960, channels = 1, sample_rate = 48000 + */ +int toxav_group_send_audio(Tox *tox, int groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels, + unsigned int sample_rate) +{ + Messenger *m = tox; + return group_send_audio(m->group_chat_object, groupnumber, pcm, samples, channels, sample_rate); +} + diff --git a/toxav/toxav_new_1.h b/toxav/toxav_new_1.h new file mode 100644 index 000000000..3696f9619 --- /dev/null +++ b/toxav/toxav_new_1.h @@ -0,0 +1,329 @@ +/** toxav.h + * + * Copyright (C) 2013 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + */ + + +#ifndef __TOXAV +#define __TOXAV +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct _ToxAv ToxAv; + +/* vpx_image_t */ +#include + +typedef void ( *ToxAVCallback ) ( void *agent, int32_t call_idx, void *arg ); +typedef void ( *ToxAvAudioCallback ) (void *agent, int32_t call_idx, const int16_t *PCM, uint16_t size, void *data); +typedef void ( *ToxAvVideoCallback ) (void *agent, int32_t call_idx, const vpx_image_t *img, void *data); + +#ifndef __TOX_DEFINED__ +#define __TOX_DEFINED__ +typedef struct Tox Tox; +#endif + +#define RTP_PAYLOAD_SIZE 65535 + + +/** + * Callbacks ids that handle the call states. + */ +typedef enum { + av_OnInvite, /* Incoming call */ + av_OnRinging, /* When peer is ready to accept/reject the call */ + av_OnStart, /* Call (RTP transmission) started */ + av_OnCancel, /* The side that initiated call canceled invite */ + av_OnReject, /* The side that was invited rejected the call */ + av_OnEnd, /* Call that was active ended */ + av_OnRequestTimeout, /* When the requested action didn't get response in specified time */ + av_OnPeerTimeout, /* Peer timed out; stop the call */ + av_OnPeerCSChange, /* Peer changing Csettings. Prepare for changed AV */ + av_OnSelfCSChange /* Csettings change confirmation. Once triggered peer is ready to recv changed AV */ +} ToxAvCallbackID; + + +/** + * Call type identifier. + */ +typedef enum { + av_TypeAudio = 192, + av_TypeVideo +} ToxAvCallType; + + +typedef enum { + av_CallNonExistent = -1, + av_CallInviting, /* when sending call invite */ + av_CallStarting, /* when getting call invite */ + av_CallActive, + av_CallHold, + av_CallHungUp +} ToxAvCallState; + +/** + * Error indicators. Values under -20 are reserved for toxcore. + */ +typedef enum { + av_ErrorNone = 0, + av_ErrorUnknown = -1, /* Unknown error */ + av_ErrorNoCall = -20, /* Trying to perform call action while not in a call */ + av_ErrorInvalidState = -21, /* Trying to perform call action while in invalid state*/ + av_ErrorAlreadyInCallWithPeer = -22, /* Trying to call peer when already in a call with peer */ + av_ErrorReachedCallLimit = -23, /* Cannot handle more calls */ + av_ErrorInitializingCodecs = -30, /* Failed creating CSSession */ + av_ErrorSettingVideoResolution = -31, /* Error setting resolution */ + av_ErrorSettingVideoBitrate = -32, /* Error setting bitrate */ + av_ErrorSplittingVideoPayload = -33, /* Error splitting video payload */ + av_ErrorEncodingVideo = -34, /* vpx_codec_encode failed */ + av_ErrorEncodingAudio = -35, /* opus_encode failed */ + av_ErrorSendingPayload = -40, /* Sending lossy packet failed */ + av_ErrorCreatingRtpSessions = -41, /* One of the rtp sessions failed to initialize */ + av_ErrorNoRtpSession = -50, /* Trying to perform rtp action on invalid session */ + av_ErrorInvalidCodecState = -51, /* Codec state not initialized */ + av_ErrorPacketTooLarge = -52, /* Split packet exceeds it's limit */ +} ToxAvError; + + +/** + * Locally supported capabilities. + */ +typedef enum { + av_AudioEncoding = 1 << 0, + av_AudioDecoding = 1 << 1, + av_VideoEncoding = 1 << 2, + av_VideoDecoding = 1 << 3 +} ToxAvCapabilities; + + +/** + * Encoding settings. + */ +typedef struct _ToxAvCSettings { + ToxAvCallType call_type; + + uint32_t video_bitrate; /* In kbits/s */ + uint16_t max_video_width; /* In px */ + uint16_t max_video_height; /* In px */ + + uint32_t audio_bitrate; /* In bits/s */ + uint16_t audio_frame_duration; /* In ms */ + uint32_t audio_sample_rate; /* In Hz */ + uint32_t audio_channels; +} ToxAvCSettings; + +extern const ToxAvCSettings av_DefaultSettings; + +/** + * Start new A/V session. There can only be one session at the time. + */ +ToxAv *toxav_new(Tox *messenger, int32_t max_calls); + +/** + * Remove A/V session. + */ +void toxav_kill(ToxAv *av); + +/** + * Returns the interval in milliseconds when the next toxav_do() should be called. + * If no call is active at the moment returns 200. + */ +uint32_t toxav_do_interval(ToxAv *av); + +/** + * Main loop for the session. Best called right after tox_do(); + */ +void toxav_do(ToxAv *av); + +/** + * Register callback for call state. + */ +void toxav_register_callstate_callback (ToxAv *av, ToxAVCallback cb, ToxAvCallbackID id, void *userdata); + +/** + * Register callback for audio data. + */ +void toxav_register_audio_callback (ToxAv *av, ToxAvAudioCallback cb, void *userdata); + +/** + * Register callback for video data. + */ +void toxav_register_video_callback (ToxAv *av, ToxAvVideoCallback cb, void *userdata); + +/** + * Call user. Use its friend_id. + */ +int toxav_call(ToxAv *av, + int32_t *call_index, + int friend_id, + const ToxAvCSettings *csettings, + int ringing_seconds); + +/** + * Hangup active call. + */ +int toxav_hangup(ToxAv *av, int32_t call_index); + +/** + * Answer incoming call. Pass the csettings that you will use. + */ +int toxav_answer(ToxAv *av, int32_t call_index, const ToxAvCSettings *csettings ); + +/** + * Reject incoming call. + */ +int toxav_reject(ToxAv *av, int32_t call_index, const char *reason); + +/** + * Cancel outgoing request. + */ +int toxav_cancel(ToxAv *av, int32_t call_index, int peer_id, const char *reason); + +/** + * Notify peer that we are changing codec settings. + */ +int toxav_change_settings(ToxAv *av, int32_t call_index, const ToxAvCSettings *csettings); + +/** + * Terminate transmission. Note that transmission will be + * terminated without informing remote peer. Usually called when we can't inform peer. + */ +int toxav_stop_call(ToxAv *av, int32_t call_index); + +/** + * Allocates transmission data. Must be call before calling toxav_prepare_* and toxav_send_*. + * Also, it must be called when call is started + */ +int toxav_prepare_transmission(ToxAv *av, int32_t call_index, int support_video); + +/** + * Clears transmission data. Call this at the end of the transmission. + */ +int toxav_kill_transmission(ToxAv *av, int32_t call_index); + +/** + * Encode video frame. + */ +int toxav_prepare_video_frame ( ToxAv *av, + int32_t call_index, + uint8_t *dest, + int dest_max, + vpx_image_t *input); + +/** + * Send encoded video packet. + */ +int toxav_send_video ( ToxAv *av, int32_t call_index, const uint8_t *frame, uint32_t frame_size); + +/** + * Encode audio frame. + */ +int toxav_prepare_audio_frame ( ToxAv *av, + int32_t call_index, + uint8_t *dest, + int dest_max, + const int16_t *frame, + int frame_size); + +/** + * Send encoded audio frame. + */ +int toxav_send_audio ( ToxAv *av, int32_t call_index, const uint8_t *frame, unsigned int size); + +/** + * Get codec settings from the peer. These were exchanged during call initialization + * or when peer send us new csettings. + */ +int toxav_get_peer_csettings ( ToxAv *av, int32_t call_index, int peer, ToxAvCSettings *dest ); + +/** + * Get friend id of peer participating in conversation. + */ +int toxav_get_peer_id ( ToxAv *av, int32_t call_index, int peer ); + +/** + * Get current call state. + */ +ToxAvCallState toxav_get_call_state ( ToxAv *av, int32_t call_index ); + +/** + * Is certain capability supported. Used to determine if encoding/decoding is ready. + */ +int toxav_capability_supported ( ToxAv *av, int32_t call_index, ToxAvCapabilities capability ); + +/** + * Returns tox reference. + */ +Tox *toxav_get_tox (ToxAv *av); + +/** + * Returns number of active calls or -1 on error. + */ +int toxav_get_active_count (ToxAv *av); + +/* Create a new toxav group. + * + * return group number on success. + * return -1 on failure. + * + * Audio data callback format: + * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + */ +int toxav_add_av_groupchat(Tox *tox, void (*audio_callback)(Tox *, int, int, const int16_t *, unsigned int, uint8_t, + unsigned int, void *), void *userdata); + +/* Join a AV group (you need to have been invited first.) + * + * returns group number on success + * returns -1 on failure. + * + * Audio data callback format (same as the one for toxav_add_av_groupchat()): + * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + */ +int toxav_join_av_groupchat(Tox *tox, int32_t friendnumber, const uint8_t *data, uint16_t length, + void (*audio_callback)(Tox *, int, int, const int16_t *, unsigned int, uint8_t, unsigned int, void *), void *userdata); + +/* Send audio to the group chat. + * + * return 0 on success. + * return -1 on failure. + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + * + * Valid number of samples are ((sample rate) * (audio length (Valid ones are: 2.5, 5, 10, 20, 40 or 60 ms)) / 1000) + * Valid number of channels are 1 or 2. + * Valid sample rates are 8000, 12000, 16000, 24000, or 48000. + * + * Recommended values are: samples = 960, channels = 1, sample_rate = 48000 + */ +int toxav_group_send_audio(Tox *tox, int groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels, + unsigned int sample_rate); + +#ifdef __cplusplus +} +#endif + +#endif /* __TOXAV */ From 8c245affb1f7dac2baab263ad0c4e19314073d71 Mon Sep 17 00:00:00 2001 From: mannol Date: Sun, 15 Feb 2015 22:41:10 +0100 Subject: [PATCH 05/97] Started adjusting msi backend to new api --- toxav/av_test.c | 9 +- toxav/msi.c | 747 ++++++++++-------------------------------------- toxav/msi.h | 44 ++- toxav/toxav.c | 1 + 4 files changed, 175 insertions(+), 626 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 1e5e4ad73..3270f39c7 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -252,13 +252,13 @@ int main (int argc, char** argv) } printf("\nTrying regular call (Audio and Video)...\n"); -// REGULAR_CALL_FLOW(48, 4000); + REGULAR_CALL_FLOW(48, 4000); printf("\nTrying regular call (Audio only)...\n"); -// REGULAR_CALL_FLOW(48, 0); + REGULAR_CALL_FLOW(48, 0); printf("\nTrying regular call (Video only)...\n"); -// REGULAR_CALL_FLOW(0, 4000); + REGULAR_CALL_FLOW(0, 4000); #undef REGULAR_CALL_FLOW @@ -328,7 +328,8 @@ int main (int argc, char** argv) } } - while (!AliceCC.ended || !BobCC.ended) + /* Alice will not receive end state */ + while (!BobCC.ended) iterate(Bsn, AliceAV, BobAV); printf("Success!\n"); diff --git a/toxav/msi.c b/toxav/msi.c index 3de686ccc..706e21d7a 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -48,43 +48,50 @@ * |id [1 byte]| |size [1 byte]| |data [$size bytes]| |...{repeat}| |0 {end byte}| */ -typedef uint8_t MSIRawCSettingsType[23]; - typedef enum { IDRequest = 1, IDResponse, IDReason, - IDCallId, - IDCSettings, + IDCapabilities, } MSIHeaderID; +/** + * Headers + */ typedef enum { - TypeRequest, - TypeResponse, - + type_request, + type_response, } MSIMessageType; typedef enum { - invite, - start, - cancel, - reject, - end, - + requ_invite, + requ_start, + requ_cancel, + requ_reject, + requ_end, } MSIRequest; typedef enum { - ringing, - starting, - ending, - error - + resp_ringing, + resp_starting, + resp_ending, + resp_error, } MSIResponse; +typedef enum { + res_undisclosed, +} MSIReason; + +typedef enum { + cap_saudio, /* sending audio */ + cap_svideo, /* sending video */ + cap_raudio, /* receiving audio */ + cap_rvideo, /* receiving video */ +} MSICapabilities; #define GENERIC_HEADER(header, val_type) \ -typedef struct _MSIHeader##header { \ +typedef struct { \ val_type value; \ _Bool exists; \ } MSIHeader##header; @@ -92,44 +99,77 @@ _Bool exists; \ GENERIC_HEADER ( Request, MSIRequest ) GENERIC_HEADER ( Response, MSIResponse ) -GENERIC_HEADER ( CallId, MSICallIDType ) -GENERIC_HEADER ( Reason, MSIReasonStrType ) -GENERIC_HEADER ( CSettings, MSIRawCSettingsType ) +GENERIC_HEADER ( Reason, MSIReason ) +GENERIC_HEADER ( Capabilities, MSICapabilities ) -typedef struct _MSIMessage { - - MSIHeaderRequest request; - MSIHeaderResponse response; - MSIHeaderReason reason; - MSIHeaderCallId callid; - MSIHeaderCSettings csettings; - - int friend_id; - +typedef struct { + MSIHeaderRequest request; + MSIHeaderResponse response; + MSIHeaderReason reason; + MSIHeaderCapabilities capabilities; } MSIMessage; static void invoke_callback(MSISession *s, int32_t c, MSICallbackID i) { - if ( s->callbacks[i].first ) { + if ( s->callbacks[i] ) { LOGGER_DEBUG("Invoking callback function: %d", i); - - s->callbacks[i].first( s->agent_handler, c, s->callbacks[i].second ); + s->callbacks[i] ( s->agent_handler, c ); } } /** - * Parse raw 'data' received from socket into MSIMessage struct. - * Every message has to have end value of 'end_byte' or _undefined_ behavior - * occures. The best practice is to check the end of the message at the handle_packet. + * Create the message. + */ +static MSIMessage *msi_new_message ( MSIMessageType type, const uint8_t type_value ) +{ + MSIMessage *retu = calloc ( sizeof ( MSIMessage ), 1 ); + + if ( retu == NULL ) { + LOGGER_WARNING("Allocation failed! Program might misbehave!"); + return NULL; + } + + if ( type == type_request ) { + retu->request.exists = 1; + retu->request.value = type_value; + + } else { + retu->response.exists = 1; + retu->response.value = type_value; + } + + return retu; +} + + +/** + * Parse raw data received from socket into MSIMessage struct. */ static int parse_raw_data ( MSIMessage *msg, const uint8_t *data, uint16_t length ) { +#define PARSE_HEADER(bytes, header, constraint, enum_high_limit) do {\ + if ((constraint -= 3) < 1) { \ + LOGGER_ERROR("Read over length!"); \ + return -1; \ + } \ + \ + if ( bytes[1] != 1 ) { \ + LOGGER_ERROR("Invalid data size!"); \ + return -1; \ + } \ + \ + if ( bytes[2] > enum_high_limit ) { \ + LOGGER_ERROR("Failed enum high limit!"); \ + return -1; \ + } \ + \ + header.value = bytes[2]; \ + header.exists = 1; \ + bytes += 3; \ + } while(0) -#define FAIL_CONSTRAINT(constraint, wanted) if ((constraint -= wanted) < 1) { LOGGER_ERROR("Read over length!"); return -1; } -#define FAIL_SIZE(byte, valid) if ( byte != valid ) { LOGGER_ERROR("Invalid data size!"); return -1; } -#define FAIL_LIMITS(byte, high) if ( byte > high ) { LOGGER_ERROR("Failed limit!"); return -1; } if ( msg == NULL ) { LOGGER_ERROR("Could not parse message: no storage!"); @@ -147,49 +187,22 @@ static int parse_raw_data ( MSIMessage *msg, const uint8_t *data, uint16_t lengt while ( *it ) {/* until end byte is hit */ switch (*it) { case IDRequest: - FAIL_CONSTRAINT(size_constraint, 3); - FAIL_SIZE(it[1], 1); -// FAIL_LIMITS(it[2], invite, end); - FAIL_LIMITS(it[2], end); - msg->request.value = it[2]; - it += 3; - msg->request.exists = 1; + PARSE_HEADER(it, msg->request, size_constraint, requ_end); break; - + case IDResponse: - FAIL_CONSTRAINT(size_constraint, 3); - FAIL_SIZE(it[1], 1); -// FAIL_LIMITS(it[2], ringing, error); - FAIL_LIMITS(it[2], error); - msg->response.value = it[2]; + PARSE_HEADER(it, msg->response, size_constraint, resp_error); it += 3; - msg->response.exists = 1; break; - - case IDCallId: - FAIL_CONSTRAINT(size_constraint, sizeof(MSICallIDType) + 2); - FAIL_SIZE(it[1], sizeof(MSICallIDType)); - memcpy(msg->callid.value, it + 2, sizeof(MSICallIDType)); - it += sizeof(MSICallIDType) + 2; - msg->callid.exists = 1; - break; - + case IDReason: - FAIL_CONSTRAINT(size_constraint, sizeof(MSIReasonStrType) + 2); - FAIL_SIZE(it[1], sizeof(MSIReasonStrType)); - memcpy(msg->reason.value, it + 2, sizeof(MSIReasonStrType)); - it += sizeof(MSIReasonStrType) + 2; - msg->reason.exists = 1; + PARSE_HEADER(it, msg->reason, size_constraint, res_undisclosed); break; - - case IDCSettings: - FAIL_CONSTRAINT(size_constraint, sizeof(MSIRawCSettingsType) + 2); - FAIL_SIZE(it[1], sizeof(MSIRawCSettingsType)); - memcpy(msg->csettings.value, it + 2, sizeof(MSIRawCSettingsType)); - it += sizeof(MSIRawCSettingsType) + 2; - msg->csettings.exists = 1; + + case IDCapabilities: + PARSE_HEADER(it, msg->capabilities, size_constraint, requ_end); break; - + default: LOGGER_ERROR("Invalid id byte"); return -1; @@ -198,37 +211,14 @@ static int parse_raw_data ( MSIMessage *msg, const uint8_t *data, uint16_t lengt } return 0; + +#undef PARSE_HEADER } -/** - * Create the message. - */ -MSIMessage *msi_new_message ( MSIMessageType type, const uint8_t type_value ) -{ - MSIMessage *retu = calloc ( sizeof ( MSIMessage ), 1 ); - - if ( retu == NULL ) { - LOGGER_WARNING("Allocation failed! Program might misbehave!"); - return NULL; - } - - if ( type == TypeRequest ) { - retu->request.exists = 1; - retu->request.value = type_value; - - } else { - retu->response.exists = 1; - retu->response.value = type_value; - } - - return retu; -} - - /** * Parse data from handle_packet. */ -MSIMessage *parse_recv ( const uint8_t *data, uint16_t length ) +static MSIMessage *parse_in ( const uint8_t *data, uint16_t length ) { if ( data == NULL ) { LOGGER_WARNING("Tried to parse empty message!"); @@ -255,11 +245,8 @@ MSIMessage *parse_recv ( const uint8_t *data, uint16_t length ) /** * Speaks for itself. */ -uint8_t *format_output ( uint8_t *dest, - MSIHeaderID id, - const void *value, - uint8_t value_len, - uint16_t *length ) +static uint8_t *prepare_header ( MSIHeaderID id, uint8_t *dest, const void *value, + uint8_t value_len, uint16_t *length ) { if ( dest == NULL ) { LOGGER_ERROR("No destination space!"); @@ -285,9 +272,9 @@ uint8_t *format_output ( uint8_t *dest, /** - * Parse MSIMessage to send. + * Parse MSIMessage to send. Returns size in bytes of the parsed message */ -uint16_t parse_send ( MSIMessage *msg, uint8_t *dest ) +static uint16_t parse_out ( MSIMessage *msg, uint8_t *dest ) { if (msg == NULL) { LOGGER_ERROR("No message!"); @@ -304,24 +291,21 @@ uint16_t parse_send ( MSIMessage *msg, uint8_t *dest ) if (msg->request.exists) { uint8_t cast = msg->request.value; - it = format_output(it, IDRequest, &cast, 1, &size); + it = prepare_header(IDRequest, it, &cast, 1, &size); } if (msg->response.exists) { uint8_t cast = msg->response.value; - it = format_output(it, IDResponse, &cast, 1, &size); - } - - if (msg->callid.exists) { - it = format_output(it, IDCallId, &msg->callid.value, sizeof(msg->callid.value), &size); + it = prepare_header(IDResponse, it, &cast, 1, &size); } if (msg->reason.exists) { - it = format_output(it, IDReason, &msg->reason.value, sizeof(msg->reason.value), &size); + it = prepare_header(IDReason, it, &msg->reason.value, sizeof(msg->reason.value), &size); } - if (msg->csettings.exists) { - it = format_output(it, IDCSettings, &msg->csettings.value, sizeof(msg->csettings.value), &size); + if (msg->capabilities.exists) { + it = prepare_header(IDCapabilities, it, &msg->capabilities.value, + sizeof(msg->capabilities.value), &size); } *it = 0; @@ -330,267 +314,17 @@ uint16_t parse_send ( MSIMessage *msg, uint8_t *dest ) return size; } -void msi_msg_set_reason ( MSIMessage *msg, const MSIReasonStrType value ) +static int send_message ( MSICall *call, MSIMessage *msg, uint32_t to ) { - if ( !msg ) return; + uint8_t parsed [MSI_MAXMSG_SIZE]; + uint16_t length = parse_out ( msg, parsed ); - msg->reason.exists = 1; - memcpy(msg->reason.value, value, sizeof(MSIReasonStrType)); -} - -void msi_msg_set_callid ( MSIMessage *msg, const MSICallIDType value ) -{ - if ( !msg ) return; - - msg->callid.exists = 1; - memcpy(msg->callid.value, value, sizeof(MSICallIDType)); -} - -void msi_msg_set_csettings ( MSIMessage *msg, const MSICSettings *value ) -{ - if ( !msg ) return; - - msg->csettings.exists = 1; - - msg->csettings.value[0] = value->call_type; - uint8_t *iter = msg->csettings.value + 1; - - /* Video bitrate */ - uint32_t lval = htonl(value->video_bitrate); - memcpy(iter, &lval, 4); - iter += 4; - - /* Video max width */ - uint16_t sval = htons(value->max_video_width); - memcpy(iter, &sval, 2); - iter += 2; - - /* Video max height */ - sval = htons(value->max_video_height); - memcpy(iter, &sval, 2); - iter += 2; - - /* Audio bitrate */ - lval = htonl(value->audio_bitrate); - memcpy(iter, &lval, 4); - iter += 4; - - /* Audio frame duration */ - sval = htons(value->audio_frame_duration); - memcpy(iter, &sval, 2); - iter += 2; - - /* Audio sample rate */ - lval = htonl(value->audio_sample_rate); - memcpy(iter, &lval, 4); - iter += 4; - - /* Audio channels */ - lval = htonl(value->audio_channels); - memcpy(iter, &lval, 4); -} - -void msi_msg_get_csettings ( MSIMessage *msg, MSICSettings *dest ) -{ - if ( !msg || !dest || !msg->csettings.exists ) return; - - dest->call_type = msg->csettings.value[0]; - uint8_t *iter = msg->csettings.value + 1; - - memcpy(&dest->video_bitrate, iter, 4); - iter += 4; - dest->video_bitrate = ntohl(dest->video_bitrate); - - memcpy(&dest->max_video_width, iter, 2); - iter += 2; - dest->max_video_width = ntohs(dest->max_video_width); - - memcpy(&dest->max_video_height, iter, 2); - iter += 2; - dest->max_video_height = ntohs(dest->max_video_height); - - memcpy(&dest->audio_bitrate, iter, 4); - iter += 4; - dest->audio_bitrate = ntohl(dest->audio_bitrate); - - memcpy(&dest->audio_frame_duration, iter, 2); - iter += 2; - dest->audio_frame_duration = ntohs(dest->audio_frame_duration); - - memcpy(&dest->audio_sample_rate, iter, 4); - iter += 4; - dest->audio_sample_rate = ntohl(dest->audio_sample_rate); - - memcpy(&dest->audio_channels, iter, 4); - dest->audio_channels = ntohl(dest->audio_channels); -} - -typedef struct _Timer { - void (*func)(struct _Timer *); - uint64_t timeout; - MSISession *session; - int call_idx; - int id; - -} Timer; - -typedef struct _TimerHandler { - Timer **timers; - - uint32_t max_capacity; - uint32_t size; -} TimerHandler; - - -static int timer_alloc (MSISession *session , void (*func)(Timer *), int call_idx, uint32_t timeout) -{ - static int timer_id; - TimerHandler *timer_handler = session->timer_handler; - - uint32_t i = 0; - - for (; i < timer_handler->max_capacity && timer_handler->timers[i]; i ++); - - if (i == timer_handler->max_capacity) { - LOGGER_WARNING("Maximum capacity reached!"); - return -1; - } - - Timer *timer = timer_handler->timers[i] = calloc(sizeof(Timer), 1); - - if (timer == NULL) { - LOGGER_ERROR("Failed to allocate timer!"); - return -1; - } - - timer_handler->size ++; - - timer->func = func; - timer->session = session; - timer->call_idx = call_idx; - timer->timeout = timeout + current_time_monotonic(); /* In ms */ - ++timer_id; - timer->id = timer_id; - - /* reorder */ - if (i) { - int64_t j = i - 1; - - for (; j >= 0 && timeout < timer_handler->timers[j]->timeout; j--) { - Timer *tmp = timer_handler->timers[j]; - timer_handler->timers[j] = timer; - timer_handler->timers[j + 1] = tmp; - } - } - - LOGGER_DEBUG("Allocated timer index: %ull timeout: %ull, current size: %ull", i, timeout, timer_handler->size); - return timer->id; -} - -static int timer_release ( TimerHandler *timers_container, int id) -{ - Timer **timed_events = timers_container->timers; - - uint32_t i; - int rc = -1; - - for (i = 0; i < timers_container->max_capacity; ++i) { - if (timed_events[i] && timed_events[i]->id == id) { - rc = i; - break; - } - } - - if (rc == -1) { - LOGGER_WARNING("No event with id: %d", id); - return -1; - } - - free(timed_events[rc]); - - timed_events[rc] = NULL; - - i = rc + 1; - - for (; i < timers_container->max_capacity && timed_events[i]; i ++) { - timed_events[i - 1] = timed_events[i]; - timed_events[i] = NULL; - } - - timers_container->size--; - - LOGGER_DEBUG("Popped id: %d, current size: %ull ", id, timers_container->size); - return 0; -} - -/** - * Generate _random_ alphanumerical string. - */ -static void t_randomstr ( uint8_t *str, uint32_t size ) -{ - if (str == NULL) { - LOGGER_DEBUG("Empty destination!"); - return; - } - - static const uint8_t _bytes[] = - "0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz"; - - uint32_t _it = 0; - - for ( ; _it < size; _it++ ) { - str[_it] = _bytes[ random_int() % 61 ]; - } -} - -/* TODO: it would be nice to actually have some sane error codes */ -typedef enum { - error_none, - error_deadcall, /* has call id but it's from old call */ - error_id_mismatch, /* non-existing call */ - - error_no_callid, /* not having call id */ - error_no_call, /* no call in session */ - error_no_crypto_key, /* no crypto key */ - - error_busy - -} MSICallError; /* Error codes */ - - -/** - * Stringify error code. - */ -static const uint8_t *stringify_error ( MSICallError error_code ) -{ - static const uint8_t *strings[] = { - ( uint8_t *) "", - ( uint8_t *) "Using dead call", - ( uint8_t *) "Call id not set to any call", - ( uint8_t *) "Call id not available", - ( uint8_t *) "No active call in session", - ( uint8_t *) "No Crypto-key set", - ( uint8_t *) "Callee busy" - }; - - return strings[error_code]; -} - -static int send_message ( MSISession *session, MSICall *call, MSIMessage *msg, uint32_t to ) -{ - msi_msg_set_callid ( msg, call->id ); - - uint8_t msg_string_final [MSI_MAXMSG_SIZE]; - uint16_t length = parse_send ( msg, msg_string_final ); - - if (!length) { + if ( !length ) { LOGGER_WARNING("Parsing message failed; nothing sent!"); return -1; } - - if ( m_msi_packet(session->messenger_handle, to, msg_string_final, length) ) { + + if ( m_msi_packet(call->session->messenger_handle, to, parsed, length) ) { LOGGER_DEBUG("Sent message"); return 0; } @@ -598,157 +332,42 @@ static int send_message ( MSISession *session, MSICall *call, MSIMessage *msg, u return -1; } -static int send_reponse ( MSISession *session, MSICall *call, MSIResponse response, uint32_t to ) +static int send_reponse ( MSICall *call, MSIResponse response, uint32_t to ) { - MSIMessage *msg = msi_new_message ( TypeResponse, response ); - int ret = send_message ( session, call, msg, to ); + MSIMessage *msg = msi_new_message ( type_response, response ); + int ret = send_message ( call, msg, to ); free ( msg ); return ret; } -static int send_error ( MSISession *session, MSICall *call, MSICallError errid, uint32_t to ) +static int send_error ( MSICall *call, MSIReason reason, uint32_t to ) { if (!call) { LOGGER_WARNING("Cannot handle error on 'null' call"); return -1; } - LOGGER_DEBUG("Sending error: %d on call: %s", errid, call->id); + LOGGER_DEBUG("Sending error: %d on call: %d", reason, call->call_idx); - MSIMessage *msg_error = msi_new_message ( TypeResponse, error ); - - msi_msg_set_reason ( msg_error, stringify_error(errid) ); - send_message ( session, call, msg_error, to ); + MSIMessage *msg_error = msi_new_message ( type_response, resp_error ); + + if (!msg_error) + return -1; + + msg_error->reason.exists = 1; + msg_error->reason.value = reason; + + send_message ( call, msg_error, to ); free ( msg_error ); return 0; } -/** - * Determine 'bigger' call id - */ -static int call_id_bigger( const uint8_t *first, const uint8_t *second) -{ - return (memcmp(first, second, sizeof(MSICallIDType)) < 0); -} -/** - * Set/change peer csettings - */ -static int flush_peer_csettings ( MSICall *call, MSIMessage *msg, int peer_id ) -{ - if ( msg->csettings.exists ) { - msi_msg_get_csettings(msg, &call->csettings_peer[peer_id]); - - LOGGER_DEBUG("Peer: %d \n" - "Type: %u \n" - "Video bitrate: %u \n" - "Video height: %u \n" - "Video width: %u \n" - "Audio bitrate: %u \n" - "Audio framedur: %u \n" - "Audio sample rate: %u \n" - "Audio channels: %u \n", peer_id, - call->csettings_peer[peer_id].call_type, - call->csettings_peer[peer_id].video_bitrate, - call->csettings_peer[peer_id].max_video_height, - call->csettings_peer[peer_id].max_video_width, - call->csettings_peer[peer_id].audio_bitrate, - call->csettings_peer[peer_id].audio_frame_duration, - call->csettings_peer[peer_id].audio_sample_rate, - call->csettings_peer[peer_id].audio_channels ); - - return 0; - } - - LOGGER_WARNING("No csettings header!"); - return -1; -} - - -/** - * Add peer to peer list. - */ -static void add_peer( MSICall *call, int peer_id ) -{ - uint32_t *peers = !call->peers ? peers = calloc(sizeof(uint32_t), 1) : - realloc( call->peers, sizeof(uint32_t) * call->peer_count); - - if (!peers) { - LOGGER_WARNING("Allocation failed! Program might misbehave!"); - return; - } - - call->peer_count ++; - call->peers = peers; - call->peers[call->peer_count - 1] = peer_id; - - LOGGER_DEBUG("Added peer: %d", peer_id); -} - - -static MSICall *find_call ( MSISession *session, uint8_t *call_id ) -{ - if ( call_id == NULL ) return NULL; - - int32_t i = 0; - - for (; i < session->max_calls; i ++ ) - if ( session->calls[i] && memcmp(session->calls[i]->id, call_id, sizeof(session->calls[i]->id)) == 0 ) { - return session->calls[i]; - } - - return NULL; -} - static MSICall *init_call ( MSISession *session, int peers, int ringing_timeout ) { - - if (peers == 0) { - LOGGER_ERROR("No peers!"); - return NULL; - } - - int32_t call_idx = 0; - - for (; call_idx < session->max_calls; call_idx ++) { - if ( !session->calls[call_idx] ) { - - if (!(session->calls[call_idx] = calloc ( sizeof ( MSICall ), 1 ))) { - LOGGER_WARNING("Allocation failed! Program might misbehave!"); - return NULL; - } - - break; - } - } - - if ( call_idx == session->max_calls ) { - LOGGER_WARNING("Reached maximum amount of calls!"); - return NULL; - } - - - MSICall *call = session->calls[call_idx]; - - call->call_idx = call_idx; - - if ( !(call->csettings_peer = calloc ( sizeof ( MSICSettings ), peers )) ) { - LOGGER_WARNING("Allocation failed! Program might misbehave!"); - free(call); - return NULL; - } - - call->session = session; - - call->request_timer_id = 0; - call->ringing_timer_id = 0; - - call->ringing_tout_ms = ringing_timeout; - - LOGGER_DEBUG("Started new call with index: %u", call_idx); - return call; + } static int terminate_call ( MSISession *session, MSICall *call ) @@ -758,11 +377,6 @@ static int terminate_call ( MSISession *session, MSICall *call ) return -1; } - /* Check event loop and cancel timed events if there are any - */ - timer_release ( session->timer_handler, call->request_timer_id); - timer_release ( session->timer_handler, call->ringing_timer_id); - session->calls[call->call_idx] = NULL; LOGGER_DEBUG("Terminated call id: %d", call->call_idx); @@ -805,27 +419,6 @@ static void handle_remote_connection_change(Messenger *messenger, int friend_num } } -/** - * Function called at request timeout - */ -static void handle_timeout ( Timer *timer ) -{ - /* TODO: Cancel might not arrive there; set up - * timers on these cancels and terminate call on - * their timeout - */ - MSICall *call = timer->session->calls[timer->call_idx]; - - - if (call) { - LOGGER_DEBUG("[Call: %d] Request timed out!", call->call_idx); - - invoke_callback(timer->session, timer->call_idx, msi_OnRequestTimeout); - msi_cancel(timer->session, timer->call_idx, call->peers [0], "Request timed out"); - } -} - - /********** Request handlers **********/ static int handle_recv_invite ( MSISession *session, MSICall *call, MSIMessage *msg ) { @@ -873,7 +466,7 @@ static int handle_recv_invite ( MSISession *session, MSICall *call, MSIMessage * } LOGGER_DEBUG("Set new call type: %s", call->csettings_peer[0].call_type == msi_TypeAudio ? "audio" : "video"); - send_reponse(session, call, starting, msg->friend_id); + send_reponse(session, call, resp_starting, msg->friend_id); invoke_callback(session, call->call_idx, msi_OnPeerCSChange); return 1; } @@ -902,7 +495,7 @@ static int handle_recv_invite ( MSISession *session, MSICall *call, MSIMessage * add_peer( call, msg->friend_id); flush_peer_csettings ( call, msg, 0 ); - send_reponse(session, call, ringing, msg->friend_id); + send_reponse(session, call, resp_ringing, msg->friend_id); invoke_callback(session, call->call_idx, msi_OnInvite); return 1; @@ -935,7 +528,7 @@ static int handle_recv_reject ( MSISession *session, MSICall *call, MSIMessage * invoke_callback(session, call->call_idx, msi_OnReject); - send_reponse(session, call, ending, msg->friend_id); + send_reponse(session, call, resp_ending, msg->friend_id); terminate_call(session, call); return 1; @@ -968,7 +561,7 @@ static int handle_recv_end ( MSISession *session, MSICall *call, MSIMessage *msg LOGGER_DEBUG("Session: %p Handling 'end' on call: %d", session, call->call_idx); invoke_callback(session, call->call_idx, msi_OnEnd); - send_reponse(session, call, ending, msg->friend_id); + send_reponse(session, call, resp_ending, msg->friend_id); terminate_call ( session, call ); return 1; @@ -1014,7 +607,7 @@ static int handle_recv_starting ( MSISession *session, MSICall *call, MSIMessage call->state = msi_CallActive; - MSIMessage *msg_start = msi_new_message ( TypeRequest, start ); + MSIMessage *msg_start = msi_new_message ( type_request, requ_start ); send_message ( session, call, msg_start, msg->friend_id ); free ( msg_start ); @@ -1115,7 +708,7 @@ static void msi_handle_packet ( Messenger *messenger, int source, const uint8_t return; } - msg = parse_recv ( data, length ); + msg = parse_in ( data, length ); if ( !msg ) { LOGGER_WARNING("Error parsing message"); @@ -1124,58 +717,53 @@ static void msi_handle_packet ( Messenger *messenger, int source, const uint8_t LOGGER_DEBUG("Successfully parsed message"); } - msg->friend_id = source; - pthread_mutex_lock(session->mutex); /* Find what call */ - MSICall *call = msg->callid.exists ? find_call(session, msg->callid.value ) : NULL; - + MSICall *call = NULL; + /* Now handle message */ if ( msg->request.exists ) { /* Handle request */ switch (msg->request.value) { - case invite: + case requ_invite: handle_recv_invite ( session, call, msg ); break; - case start: + case requ_start: handle_recv_start ( session, call, msg ); break; - case cancel: + case requ_cancel: handle_recv_cancel ( session, call, msg ); break; - case reject: + case requ_reject: handle_recv_reject ( session, call, msg ); break; - case end: + case requ_end: handle_recv_end ( session, call, msg ); break; } } else if ( msg->response.exists ) { /* Handle response */ - - /* Got response so cancel timer */ - if ( call ) timer_release(session->timer_handler, call->request_timer_id); - + switch (msg->response.value) { - case ringing: + case resp_ringing: handle_recv_ringing ( session, call, msg ); break; - case starting: + case resp_starting: handle_recv_starting ( session, call, msg ); break; - case ending: + case resp_ending: handle_recv_ending ( session, call, msg ); break; - case error: + case resp_error: handle_recv_error ( session, call, msg ); break; } @@ -1192,24 +780,18 @@ static void msi_handle_packet ( Messenger *messenger, int source, const uint8_t /********** User functions **********/ -void msi_register_callback ( MSISession *session, MSICallbackType callback, MSICallbackID id, void *userdata ) +void msi_register_callback ( MSISession *session, MSICallbackType callback, MSICallbackID id) { - session->callbacks[id].first = callback; - session->callbacks[id].second = userdata; + session->callbacks[id] = callback; } - -MSISession *msi_new ( Messenger *messenger, int32_t max_calls ) +MSISession *msi_new ( Messenger *messenger ) { if (messenger == NULL) { LOGGER_ERROR("Could not init session on empty messenger!"); return NULL; } - if ( !max_calls ) { - LOGGER_WARNING("Invalid max call treshold!"); - return NULL; - } MSISession *retu = calloc ( sizeof ( MSISession ), 1 ); @@ -1218,43 +800,19 @@ MSISession *msi_new ( Messenger *messenger, int32_t max_calls ) return NULL; } - if (!(retu->calls = calloc( sizeof (MSICall *), max_calls ))) { - LOGGER_ERROR("Allocation failed! Program might misbehave!"); - goto error; - } - - retu->timer_handler = calloc(1, sizeof(TimerHandler)); - - if (retu->timer_handler == NULL) { - LOGGER_ERROR("Allocation failed! Program might misbehave!"); - goto error; - } - - /* Allocate space for timers */ - ((TimerHandler *)retu->timer_handler)->max_capacity = max_calls * 10; - - if (!(((TimerHandler *)retu->timer_handler)->timers = calloc(max_calls * 10, sizeof(Timer *)))) { - LOGGER_ERROR("Allocation failed! Program might misbehave!"); - goto error; - } - if (create_recursive_mutex(retu->mutex) != 0) { LOGGER_ERROR("Failed to init mutex! Program might misbehave"); goto error; } retu->messenger_handle = messenger; - retu->agent_handler = NULL; - retu->max_calls = max_calls; - retu->frequ = 10000; /* default value? */ - retu->call_timeout = 30000; /* default value? */ m_callback_msi_packet(messenger, msi_handle_packet, retu ); /* This is called when remote terminates session */ m_callback_connectionstatus_internal_av(messenger, handle_remote_connection_change, retu); - LOGGER_DEBUG("New msi session: %p max calls: %u", retu, max_calls); + LOGGER_DEBUG("New msi session: %p ", retu); return retu; error: @@ -1269,7 +827,6 @@ error: return NULL; } - int msi_kill ( MSISession *session ) { if (session == NULL) { @@ -1338,7 +895,7 @@ int msi_invite ( MSISession *session, call->csettings_local = *csettings; - MSIMessage *msg_invite = msi_new_message ( TypeRequest, invite ); + MSIMessage *msg_invite = msi_new_message ( type_request, requ_invite ); msi_msg_set_csettings(msg_invite, csettings); send_message ( session, call, msg_invite, friend_id ); @@ -1372,7 +929,7 @@ int msi_hangup ( MSISession *session, int32_t call_index ) return msi_ErrorInvalidState; } - MSIMessage *msg_end = msi_new_message ( TypeRequest, end ); + MSIMessage *msg_end = msi_new_message ( type_request, requ_end ); /* hangup for each peer */ int it = 0; @@ -1408,7 +965,7 @@ int msi_answer ( MSISession *session, int32_t call_index, const MSICSettings *cs return msi_ErrorInvalidState; } - MSIMessage *msg_starting = msi_new_message ( TypeResponse, starting ); + MSIMessage *msg_starting = msi_new_message ( type_response, resp_starting ); session->calls[call_index]->csettings_local = *csettings; @@ -1440,7 +997,7 @@ int msi_cancel ( MSISession *session, int32_t call_index, uint32_t peer, const c return msi_ErrorInvalidState; } - MSIMessage *msg_cancel = msi_new_message ( TypeRequest, cancel ); + MSIMessage *msg_cancel = msi_new_message ( type_request, requ_cancel ); /* FIXME */ #if 0 @@ -1483,7 +1040,7 @@ int msi_reject ( MSISession *session, int32_t call_index, const char *reason ) return msi_ErrorInvalidState; } - MSIMessage *msg_reject = msi_new_message ( TypeRequest, reject ); + MSIMessage *msg_reject = msi_new_message ( type_request, requ_reject ); /* FIXME */ #if 0 @@ -1568,7 +1125,7 @@ int msi_change_csettings(MSISession *session, int32_t call_index, const MSICSett *local = *csettings; - MSIMessage *msg_invite = msi_new_message ( TypeRequest, invite ); + MSIMessage *msg_invite = msi_new_message ( type_request, requ_invite ); msi_msg_set_csettings ( msg_invite, local ); send_message ( session, call, msg_invite, call->peers[0] ); diff --git a/toxav/msi.h b/toxav/msi.h index bdd72e490..c71c69dde 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -30,7 +30,7 @@ typedef uint8_t MSICallIDType[12]; typedef uint8_t MSIReasonStrType[255]; -typedef void ( *MSICallbackType ) ( void *agent, int32_t call_idx, void *arg ); +typedef void ( *MSICallbackType ) ( void *agent, int32_t call_idx); /** * Call type identifier. Also used as rtp callback prefix. @@ -70,6 +70,15 @@ typedef struct { uint32_t audio_channels; } MSICSettings; +/** + * Active capabilities masks + */ +typedef enum { + msi_SendingAudio = 1, + msi_SendingVideo = 2, + msi_RecvingAudio = 4, + msi_RecvingVideo = 8, +} MSICapMask; /** * Callbacks ids that handle the states @@ -100,25 +109,13 @@ typedef enum { /** * The call struct. */ -typedef struct { /* Call info structure */ - struct MSISession_s *session; /* Session pointer */ +typedef struct { + struct MSISession_s *session; /* Session pointer */ MSICallState state; - - MSICSettings csettings_local; /* Local call settings */ - MSICSettings *csettings_peer; /* Peers call settings */ - - MSICallIDType id; /* Random value identifying the call */ - - int ringing_tout_ms; /* Ringing timeout in ms */ - - int request_timer_id; /* Timer id for outgoing request/action */ - int ringing_timer_id; /* Timer id for ringing timeout */ - - uint32_t *peers; - uint16_t peer_count; - - int32_t call_idx; /* Index of this call in MSISession */ + uint8_t caps; /* Active capabilities */ + + uint32_t friend_id; /* Index of this call in MSISession */ } MSICall; @@ -126,21 +123,14 @@ typedef struct { /* Call info structure */ * Control session struct */ typedef struct MSISession_s { - /* Call handlers */ MSICall **calls; - int32_t max_calls; void *agent_handler; Messenger *messenger_handle; - uint32_t frequ; - uint32_t call_timeout; /* Time of the timeout for some action to end; 0 if infinite */ - pthread_mutex_t mutex[1]; - - void *timer_handler; - PAIR(MSICallbackType, void *) callbacks[10]; + MSICallbackType callbacks[10]; } MSISession; /** @@ -156,7 +146,7 @@ int msi_kill ( MSISession *session ); /** * Callback setter. */ -void msi_register_callback(MSISession *session, MSICallbackType callback, MSICallbackID id, void *userdata); +void msi_register_callback(MSISession *session, MSICallbackType callback, MSICallbackID id); /** * Send invite request to friend_id. diff --git a/toxav/toxav.c b/toxav/toxav.c index d71bbd6dc..b594cc29e 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -322,6 +322,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co } else if (av->msi->calls[call->call_idx]->state == msi_CallRequesting) { /* Cancel the call */ msi_cancel(av->msi, call->call_idx, 0, NULL); + i_toxav_remove_call(av, friend_number); } } break; From 7329f3b3d461a8f7738fa4b13a2dffc8d6a5b5f5 Mon Sep 17 00:00:00 2001 From: mannol Date: Mon, 16 Feb 2015 23:30:20 +0100 Subject: [PATCH 06/97] Fixed header protectors and cleaning up the msi --- toxav/codec.h | 6 +- toxav/msi.c | 867 +++++++++++++++++++------------------------------- toxav/msi.h | 89 +++--- toxav/rtp.h | 6 +- toxav/toxav.c | 14 - toxav/toxav.h | 8 +- 6 files changed, 382 insertions(+), 608 deletions(-) diff --git a/toxav/codec.h b/toxav/codec.h index de5a6cd1c..02a3b1b4f 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -21,8 +21,8 @@ * */ -#ifndef _CODEC_H_ -#define _CODEC_H_ +#ifndef CODEC_H +#define CODEC_H #include "toxav.h" #include "rtp.h" @@ -186,4 +186,4 @@ void cs_disable_audio_receiving(CSSession* cs); /* Internal. Called from rtp_handle_message */ void queue_message(RTPSession *session, RTPMessage *msg); -#endif /* _CODEC_H_ */ +#endif /* CODEC_H */ diff --git a/toxav/msi.c b/toxav/msi.c index 706e21d7a..e78c57feb 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -51,7 +51,7 @@ typedef enum { IDRequest = 1, IDResponse, - IDReason, + IDError, IDCapabilities, } MSIHeaderID; @@ -67,7 +67,6 @@ typedef enum { typedef enum { requ_invite, requ_start, - requ_cancel, requ_reject, requ_end, } MSIRequest; @@ -75,21 +74,9 @@ typedef enum { typedef enum { resp_ringing, resp_starting, - resp_ending, resp_error, } MSIResponse; -typedef enum { - res_undisclosed, -} MSIReason; - -typedef enum { - cap_saudio, /* sending audio */ - cap_svideo, /* sending video */ - cap_raudio, /* receiving audio */ - cap_rvideo, /* receiving video */ -} MSICapabilities; - #define GENERIC_HEADER(header, val_type) \ typedef struct { \ val_type value; \ @@ -99,23 +86,23 @@ _Bool exists; \ GENERIC_HEADER ( Request, MSIRequest ) GENERIC_HEADER ( Response, MSIResponse ) -GENERIC_HEADER ( Reason, MSIReason ) -GENERIC_HEADER ( Capabilities, MSICapabilities ) +GENERIC_HEADER ( Error, MSIError ) +GENERIC_HEADER ( Capabilities, uint8_t ) typedef struct { MSIHeaderRequest request; MSIHeaderResponse response; - MSIHeaderReason reason; + MSIHeaderError error; MSIHeaderCapabilities capabilities; } MSIMessage; -static void invoke_callback(MSISession *s, int32_t c, MSICallbackID i) +static void invoke_callback(MSICall* c, MSICallbackID i) { - if ( s->callbacks[i] ) { + if ( c->session->callbacks[i] ) { LOGGER_DEBUG("Invoking callback function: %d", i); - s->callbacks[i] ( s->agent_handler, c ); + c->session->callbacks[i] ( c->session->agent_handler, c ); } } @@ -149,22 +136,15 @@ static MSIMessage *msi_new_message ( MSIMessageType type, const uint8_t type_val */ static int parse_raw_data ( MSIMessage *msg, const uint8_t *data, uint16_t length ) { -#define PARSE_HEADER(bytes, header, constraint, enum_high_limit) do {\ - if ((constraint -= 3) < 1) { \ - LOGGER_ERROR("Read over length!"); \ - return -1; \ - } \ - \ - if ( bytes[1] != 1 ) { \ - LOGGER_ERROR("Invalid data size!"); \ - return -1; \ - } \ - \ - if ( bytes[2] > enum_high_limit ) { \ - LOGGER_ERROR("Failed enum high limit!"); \ - return -1; \ - } \ - \ + +#define CHECK_SIZE(bytes, constraint, size) \ + if ((constraint -= 3) < 1) { LOGGER_ERROR("Read over length!"); return -1; } \ + if ( bytes[1] != size ) { LOGGER_ERROR("Invalid data size!"); return -1; } + +#define CHECK_ENUM_HIGH(bytes, enum_high) \ + if ( bytes[2] > enum_high ) { LOGGER_ERROR("Failed enum high limit!"); return -1; } + +#define SET_VALUES(bytes, header) do { \ header.value = bytes[2]; \ header.exists = 1; \ bytes += 3; \ @@ -176,7 +156,7 @@ static int parse_raw_data ( MSIMessage *msg, const uint8_t *data, uint16_t lengt return -1; } - if ( data[length - 1] ) { /* End byte must have value 0 */ + if ( length == 0 || data[length - 1] ) { /* End byte must have value 0 */ LOGGER_ERROR("Invalid end byte"); return -1; } @@ -187,20 +167,27 @@ static int parse_raw_data ( MSIMessage *msg, const uint8_t *data, uint16_t lengt while ( *it ) {/* until end byte is hit */ switch (*it) { case IDRequest: - PARSE_HEADER(it, msg->request, size_constraint, requ_end); + CHECK_SIZE(it, size_constraint, 1); + CHECK_ENUM_HIGH(it, requ_end); + SET_VALUES(it, msg->request); break; case IDResponse: - PARSE_HEADER(it, msg->response, size_constraint, resp_error); + CHECK_SIZE(it, size_constraint, 1); + CHECK_ENUM_HIGH(it, resp_error); + SET_VALUES(it, msg->response); it += 3; break; - case IDReason: - PARSE_HEADER(it, msg->reason, size_constraint, res_undisclosed); + case IDError: + CHECK_SIZE(it, size_constraint, 1); + CHECK_ENUM_HIGH(it, msi_ErrUndisclosed); + SET_VALUES(it, msg->error); break; case IDCapabilities: - PARSE_HEADER(it, msg->capabilities, size_constraint, requ_end); + CHECK_SIZE(it, size_constraint, 1); + SET_VALUES(it, msg->capabilities); break; default: @@ -212,7 +199,9 @@ static int parse_raw_data ( MSIMessage *msg, const uint8_t *data, uint16_t lengt return 0; -#undef PARSE_HEADER +#undef CHECK_SIZE +#undef CHECK_ENUM_HIGH +#undef SET_VALUES } /** @@ -299,8 +288,8 @@ static uint16_t parse_out ( MSIMessage *msg, uint8_t *dest ) it = prepare_header(IDResponse, it, &cast, 1, &size); } - if (msg->reason.exists) { - it = prepare_header(IDReason, it, &msg->reason.value, sizeof(msg->reason.value), &size); + if (msg->error.exists) { + it = prepare_header(IDError, it, &msg->error.value, sizeof(msg->error.value), &size); } if (msg->capabilities.exists) { @@ -340,22 +329,22 @@ static int send_reponse ( MSICall *call, MSIResponse response, uint32_t to ) return ret; } -static int send_error ( MSICall *call, MSIReason reason, uint32_t to ) +static int send_error ( MSICall *call, MSIError error, uint32_t to ) { if (!call) { LOGGER_WARNING("Cannot handle error on 'null' call"); return -1; } - LOGGER_DEBUG("Sending error: %d on call: %d", reason, call->call_idx); + LOGGER_DEBUG("Sending error: %d on call: %d", error, call->call_idx); MSIMessage *msg_error = msi_new_message ( type_response, resp_error ); if (!msg_error) return -1; - msg_error->reason.exists = 1; - msg_error->reason.value = reason; + msg_error->error.exists = 1; + msg_error->error.value = error; send_message ( call, msg_error, to ); free ( msg_error ); @@ -364,53 +353,113 @@ static int send_error ( MSICall *call, MSIReason reason, uint32_t to ) } - -static MSICall *init_call ( MSISession *session, int peers, int ringing_timeout ) +static MSICall *get_call ( MSISession *session, uint32_t friend_id ) { + if (session->calls == NULL || session->calls_tail < friend_id) + return NULL; + return session->calls[friend_id]; } -static int terminate_call ( MSISession *session, MSICall *call ) +static MSICall *new_call ( MSISession *session, uint32_t friend_id ) { - if ( !call ) { - LOGGER_WARNING("Tried to terminate non-existing call!"); - return -1; + MSICall *rc = calloc(sizeof(MSICall), 1); + + if (rc == NULL) + return NULL; + + rc->friend_id = friend_id; + + if (session->calls == NULL) { /* Creating */ + session->calls = calloc (sizeof(MSICall*), friend_id + 1); + + if (session->calls == NULL) { + free(rc); + return NULL; + } + + session->calls_tail = session->calls_head = friend_id; + + } else if (session->calls_tail < friend_id) { /* Appending */ + void* tmp = realloc(session->calls, sizeof(MSICall*) * friend_id + 1); + + if (tmp == NULL) { + free(rc); + return NULL; + } + + session->calls = tmp; + + /* Set fields in between to null */ + int32_t i = session->calls_tail; + for (; i < friend_id; i ++) + session->calls[i] = NULL; + + rc->prev = session->calls[session->calls_tail]; + session->calls[session->calls_tail]->next = rc; + + session->calls_tail = friend_id; + + } else if (session->calls_head > friend_id) { /* Inserting at front */ + rc->next = session->calls[session->calls_head]; + session->calls[session->calls_head]->prev = rc; + session->calls_head = friend_id; } - - session->calls[call->call_idx] = NULL; - - LOGGER_DEBUG("Terminated call id: %d", call->call_idx); - - free ( call->csettings_peer ); - free ( call->peers ); - free ( call ); - - return 0; + + session->calls[friend_id] = rc; + return rc; } -static void handle_remote_connection_change(Messenger *messenger, int friend_num, uint8_t status, void *session_p) +static void kill_call ( MSICall *call ) +{ + if ( call == NULL ) + return; + + + MSISession* session = call->session; + + MSICall* prev = call->prev; + MSICall* next = call->next; + + if (prev) + prev->next = next; + else if (next) + session->calls_head = next->friend_id; + else goto CLEAR; + + if (next) + next->prev = prev; + else if (prev) + session->calls_tail = prev->friend_id; + else goto CLEAR; + + session->calls[call->friend_id] = NULL; + free(call); + return; + +CLEAR: + session->calls_head = session->calls_tail = 0; + free(session->calls); + session->calls = NULL; + free(call); +} + + + +static void handle_remote_connection_change(Messenger *messenger, int friend_id, uint8_t status, void *session_p) { (void)messenger; MSISession *session = session_p; switch ( status ) { case 0: { /* Went offline */ - int32_t j = 0; - - for ( ; j < session->max_calls; j ++ ) { - - if ( !session->calls[j] ) continue; - - uint16_t i = 0; - - for ( ; i < session->calls[j]->peer_count; i ++ ) - if ( session->calls[j]->peers[i] == (uint32_t)friend_num ) { - invoke_callback(session, j, msi_OnPeerTimeout); - terminate_call(session, session->calls[j]); - LOGGER_DEBUG("Remote: %d timed out!", friend_num); - return; /* TODO: On group calls change behaviour */ - } - } + MSICall* call = get_call(session, friend_id); + + if (call == NULL) + return; + + invoke_callback(call, msi_OnPeerTimeout); + kill_call(call); } break; @@ -419,247 +468,177 @@ static void handle_remote_connection_change(Messenger *messenger, int friend_num } } + + /********** Request handlers **********/ -static int handle_recv_invite ( MSISession *session, MSICall *call, MSIMessage *msg ) +static int handle_recv_invite ( MSICall *call, MSIMessage *msg ) { - LOGGER_DEBUG("Session: %p Handling 'invite' on call: %d", session, call ? call->call_idx : -1); + if ( call == NULL ) { + LOGGER_WARNING("Session: %p Handling 'invite' on no call"); + return -1; + } + + MSISession* session = call->session; + + LOGGER_DEBUG("Session: %p Handling 'invite' friend: %d", call->session, call->friend_id); - - if (!msg->csettings.exists) {/**/ - LOGGER_WARNING("Peer sent invalid codec settings!"); - send_error ( session, call, error_no_callid, msg->friend_id ); + + if ( call->state == msi_CallRequesting ) { + /* The rare glare case. + * Send starting and wait for starting by the other side. + * The peer will do the same. + * When you receive starting from peer send started. + * Don't notice the app until the start is received. + */ + + LOGGER_DEBUG("Glare detected!"); + + MSIMessage *msg_starting = msi_new_message ( type_response, resp_starting ); + + call->capabilities &= msg->capabilities; + + msg_starting->capabilities.value = call->capabilities; + msg_starting->capabilities.exists = 1; + + send_message ( call, msg_starting, call->friend_id ); + free ( msg_starting ); + return 0; } - - if ( call ) { - if ( call->peers[0] == (uint32_t)msg->friend_id ) { - if (call->state == msi_CallRequesting) { - /* The glare case. A calls B when at the same time - * B calls A. Who has advantage is set bey calculating - * 'bigger' Call id and then that call id is being used in - * future. User with 'bigger' Call id has the advantage - * as in he will wait the response from the other. - */ - LOGGER_DEBUG("Glare case; Peer: %d", call->peers[0]); - - if ( call_id_bigger (call->id, msg->callid.value) == 1 ) { /* Peer has advantage */ - - /* Terminate call; peer will timeout(call) if call initialization fails */ - terminate_call(session, call); - - call = init_call ( session, 1, 0 ); - - if ( !call ) { - LOGGER_ERROR("Starting call"); - return 0; - } - - } else { - return 0; /* Wait for ringing from peer */ - } - } else if (call->state == msi_CallActive) { - /* Request for media change; call callback and send starting response */ - if (flush_peer_csettings(call, msg, 0) != 0) { /**/ - LOGGER_WARNING("Peer sent invalid csetting!"); - send_error ( session, call, error_no_callid, msg->friend_id ); - return 0; - } - - LOGGER_DEBUG("Set new call type: %s", call->csettings_peer[0].call_type == msi_TypeAudio ? "audio" : "video"); - send_reponse(session, call, resp_starting, msg->friend_id); - invoke_callback(session, call->call_idx, msi_OnPeerCSChange); - return 1; - } - } else { - send_error ( session, call, error_busy, msg->friend_id ); /* TODO: Ugh*/ - terminate_call(session, call); - return 0; - } - } else { - call = init_call ( session, 1, 0 ); - - if ( !call ) { - LOGGER_ERROR("Starting call"); - return 0; - } - } - - if ( !msg->callid.exists ) { - send_error ( session, call, error_no_callid, msg->friend_id ); - terminate_call(session, call); - return 0; - } - - memcpy ( call->id, msg->callid.value, sizeof(msg->callid.value) ); + + call->capabilities = msg->capabilities; call->state = msi_CallRequested; + + send_reponse(call, resp_ringing, call->friend_id); + invoke_callback(call, msi_OnInvite); - add_peer( call, msg->friend_id); - flush_peer_csettings ( call, msg, 0 ); - send_reponse(session, call, resp_ringing, msg->friend_id); - invoke_callback(session, call->call_idx, msi_OnInvite); - - return 1; + return 0; } -static int handle_recv_start ( MSISession *session, MSICall *call, MSIMessage *msg ) +static int handle_recv_start ( MSICall *call, MSIMessage *msg ) { - if ( !call ) { + if ( call == NULL ) { LOGGER_WARNING("Session: %p Handling 'start' on no call"); - return 0; + return -1; } - + + if ( call->state != msi_CallRequested || call->state != msi_CallRequesting ) { + LOGGER_WARNING("Session: %p Invalid call state on 'start'"); + /* TODO send error */ + return -1; + } + (void)msg; - LOGGER_DEBUG("Session: %p Handling 'start' on call: %d, friend id: %d", session, call->call_idx, msg->friend_id ); + LOGGER_DEBUG("Session: %p Handling 'start', friend id: %d", call->session, call->friend_id ); call->state = msi_CallActive; - invoke_callback(session, call->call_idx, msi_OnStart); - return 1; + invoke_callback(call, msi_OnStart); + + return 0; } -static int handle_recv_reject ( MSISession *session, MSICall *call, MSIMessage *msg ) +static int handle_recv_reject ( MSICall *call, MSIMessage *msg ) { - if ( !call ) { + if ( call == NULL ) { LOGGER_WARNING("Session: %p Handling 'start' on no call"); - return 0; + return -1; } - - LOGGER_DEBUG("Session: %p Handling 'reject' on call: %u", session, call->call_idx); - - invoke_callback(session, call->call_idx, msi_OnReject); - - send_reponse(session, call, resp_ending, msg->friend_id); - terminate_call(session, call); - - return 1; -} - -static int handle_recv_cancel ( MSISession *session, MSICall *call, MSIMessage *msg ) -{ - if ( !call ) { - LOGGER_WARNING("Session: %p Handling 'start' on no call"); - return 0; - } - + (void)msg; + + if ( call->state != msi_CallRequesting ) { + LOGGER_WARNING("Session: %p Invalid call state on 'reject'"); + /* TODO send error */ + return -1; + } + + LOGGER_DEBUG("Session: %p Handling 'reject', friend id: %u", call->session, call->friend_id); - LOGGER_DEBUG("Session: %p Handling 'cancel' on call: %u", session, call->call_idx); + invoke_callback(call, msi_OnReject); + kill_call(call); - invoke_callback(session, call->call_idx, msi_OnCancel); - terminate_call ( session, call ); - - return 1; + return 0; } -static int handle_recv_end ( MSISession *session, MSICall *call, MSIMessage *msg ) +static int handle_recv_end ( MSICall *call, MSIMessage *msg ) { - if ( !call ) { + (void)msg; + + if ( call == NULL ) { LOGGER_WARNING("Session: %p Handling 'start' on no call"); - return 0; + return -1; } - LOGGER_DEBUG("Session: %p Handling 'end' on call: %d", session, call->call_idx); + LOGGER_DEBUG("Session: %p Handling 'end', friend id: %d", call->session, call->friend_id); - invoke_callback(session, call->call_idx, msi_OnEnd); - send_reponse(session, call, resp_ending, msg->friend_id); - terminate_call ( session, call ); + invoke_callback(call, msi_OnEnd); + kill_call ( call ); - return 1; + return 0; } /********** Response handlers **********/ -static int handle_recv_ringing ( MSISession *session, MSICall *call, MSIMessage *msg ) +static int handle_recv_ringing ( MSICall *call, MSIMessage *msg ) { - if ( !call ) { + if ( call == NULL ) { LOGGER_WARNING("Session: %p Handling 'start' on no call"); - return 0; + return -1; } (void)msg; - - if ( call->ringing_timer_id ) { - LOGGER_WARNING("Call already ringing"); - return 0; + + if ( call->state != msi_CallRequesting ) { + LOGGER_WARNING("Session: %p Invalid call state on 'ringing'"); + /* TODO send error */ + return -1; } - LOGGER_DEBUG("Session: %p Handling 'ringing' on call: %d", session, call->call_idx ); + LOGGER_DEBUG("Session: %p Handling 'ringing' friend id: %d", call->session, call->friend_id ); - call->ringing_timer_id = timer_alloc - ( session, handle_timeout, call->call_idx, call->ringing_tout_ms ); - invoke_callback(session, call->call_idx, msi_OnRinging); - return 1; + invoke_callback(call, msi_OnRinging); + return 0; } -static int handle_recv_starting ( MSISession *session, MSICall *call, MSIMessage *msg ) +static int handle_recv_starting ( MSICall *call, MSIMessage *msg ) { - if ( !call ) { + if ( call == NULL ) { LOGGER_WARNING("Session: %p Handling 'starting' on non-existing call"); return 0; } - - if ( call->state == msi_CallActive ) { /* Change media */ - - LOGGER_DEBUG("Session: %p Changing media on call: %d", session, call->call_idx ); - - invoke_callback(session, call->call_idx, msi_OnSelfCSChange); - - } else if ( call->state == msi_CallRequesting ) { - LOGGER_DEBUG("Session: %p Handling 'starting' on call: %d", session, call->call_idx ); - + + if ( call->state != msi_CallRequested || call->state != msi_CallRequesting ) { + LOGGER_WARNING("Session: %p Invalid call state on 'starting'"); + /* TODO send error */ + return -1; + } + + MSIMessage *msg_start = msi_new_message ( type_request, requ_start ); + send_message ( call, msg_start, call->friend_id ); + free ( msg_start ); + + if (call->state == msi_CallRequesting) { call->state = msi_CallActive; - - MSIMessage *msg_start = msi_new_message ( type_request, requ_start ); - send_message ( session, call, msg_start, msg->friend_id ); - free ( msg_start ); - - - flush_peer_csettings ( call, msg, 0 ); - - /* This is here in case of glare */ - timer_release(session->timer_handler, call->ringing_timer_id); - invoke_callback(session, call->call_idx, msi_OnStart); - } else { - LOGGER_ERROR("Invalid call state"); - terminate_call(session, call ); - return 0; - } - - return 1; + invoke_callback(call, msi_OnStart); + } + + /* Otherwise it's a glare case so don't start until 'start' is recved */ + + return 0; } -static int handle_recv_ending ( MSISession *session, MSICall *call, MSIMessage *msg ) +static int handle_recv_error ( MSICall *call, MSIMessage *msg ) { - if ( !call ) { - LOGGER_WARNING("Session: %p Handling 'start' on no call"); - return 0; - } - - (void)msg; - - LOGGER_DEBUG("Session: %p Handling 'ending' on call: %d", session, call->call_idx ); - - invoke_callback(session, call->call_idx, msi_OnEnd); - terminate_call ( session, call ); - - return 1; -} -static int handle_recv_error ( MSISession *session, MSICall *call, MSIMessage *msg ) -{ - if ( !call ) { + if ( call == NULL ) { LOGGER_WARNING("Handling 'error' on non-existing call!"); return -1; } - LOGGER_DEBUG("Session: %p Handling 'error' on call: %d", session, call->call_idx ); + LOGGER_DEBUG("Session: %p Handling 'error' friend id: %d", call->session, call->friend_id ); - invoke_callback(session, call->call_idx, msi_OnEnd); + invoke_callback(call, msi_OnError); - /* Handle error accordingly */ - if ( msg->reason.exists ) { - /* TODO */ - } + /* TODO Handle error accordingly */ - terminate_call ( session, call ); - - return 1; + return -1; } /** @@ -694,20 +673,16 @@ static int handle_recv_error ( MSISession *session, MSICall *call, MSIMessage *m * * */ -static void msi_handle_packet ( Messenger *messenger, int source, const uint8_t *data, uint16_t length, void *object ) +static void msi_handle_packet ( Messenger *messenger, int friend_id, const uint8_t *data, uint16_t length, void *object ) { LOGGER_DEBUG("Got msi message"); + /* Unused */ (void)messenger; MSISession *session = object; MSIMessage *msg; - if ( !length ) { - LOGGER_WARNING("Length param negative"); - return; - } - msg = parse_in ( data, length ); if ( !msg ) { @@ -716,64 +691,69 @@ static void msi_handle_packet ( Messenger *messenger, int source, const uint8_t } else { LOGGER_DEBUG("Successfully parsed message"); } - + pthread_mutex_lock(session->mutex); - /* Find what call */ - MSICall *call = NULL; + MSICall *call = get_call(session, friend_id); + + if (call == NULL) { + if (msg->request != requ_invite) { + /* TODO send error */ + return; + } + + call = new_call(session, friend_id); + if (call == NULL) { + /* TODO send error */ + return; + } + } + /* Now handle message */ - + int rc = 0; if ( msg->request.exists ) { /* Handle request */ - switch (msg->request.value) { case requ_invite: - handle_recv_invite ( session, call, msg ); + rc = handle_recv_invite ( call, msg ); break; case requ_start: - handle_recv_start ( session, call, msg ); - break; - - case requ_cancel: - handle_recv_cancel ( session, call, msg ); + rc = handle_recv_start ( call, msg ); break; case requ_reject: - handle_recv_reject ( session, call, msg ); + rc = handle_recv_reject ( call, msg ); break; case requ_end: - handle_recv_end ( session, call, msg ); + rc = handle_recv_end ( call, msg ); break; } - } else if ( msg->response.exists ) { /* Handle response */ - switch (msg->response.value) { case resp_ringing: - handle_recv_ringing ( session, call, msg ); + rc = handle_recv_ringing ( call, msg ); break; case resp_starting: - handle_recv_starting ( session, call, msg ); - break; - - case resp_ending: - handle_recv_ending ( session, call, msg ); + rc = handle_recv_starting ( call, msg ); break; case resp_error: - handle_recv_error ( session, call, msg ); + rc = handle_recv_error ( call, msg ); break; } - } else { LOGGER_WARNING("Invalid message: no resp nor requ headers"); + /* TODO send error */ + rc = -1; } - + + if (rc == -1) + kill_call(call); + free ( msg ); - pthread_mutex_unlock(session->mutex); } @@ -792,7 +772,6 @@ MSISession *msi_new ( Messenger *messenger ) return NULL; } - MSISession *retu = calloc ( sizeof ( MSISession ), 1 ); if (retu == NULL) { @@ -802,7 +781,8 @@ MSISession *msi_new ( Messenger *messenger ) if (create_recursive_mutex(retu->mutex) != 0) { LOGGER_ERROR("Failed to init mutex! Program might misbehave"); - goto error; + free(retu); + return NULL; } retu->messenger_handle = messenger; @@ -814,17 +794,6 @@ MSISession *msi_new ( Messenger *messenger ) LOGGER_DEBUG("New msi session: %p ", retu); return retu; - -error: - - if (retu->timer_handler) { - free(((TimerHandler *)retu->timer_handler)->timers); - free(retu->timer_handler); - } - - free(retu->calls); - free(retu); - return NULL; } int msi_kill ( MSISession *session ) @@ -837,19 +806,18 @@ int msi_kill ( MSISession *session ) m_callback_msi_packet((struct Messenger *) session->messenger_handle, NULL, NULL); pthread_mutex_lock(session->mutex); - /* Cancel active calls */ - int32_t idx = 0; - - for (; idx < session->max_calls; idx ++) if ( session->calls[idx] ) { - /* Cancel all? */ - uint16_t _it = 0; - /*for ( ; _it < session->calls[idx]->peer_count; _it++ ) - * FIXME: will not work on multiple peers, must cancel call for all peers - */ - msi_cancel ( session, idx, session->calls[idx]->peers [_it], "MSI session terminated!" ); + if (session->calls) { + MSIMessage *msg_end = msi_new_message ( type_request, requ_end ); + + MSICall* it = get_call(session, session->calls_head); + for (; it; it = it->next) { + send_message(it, msg_end, it->friend_id); + kill_call(it); } - - free ( session->calls ); + + free(msg_end); + } + pthread_mutex_unlock(session->mutex); pthread_mutex_destroy(session->mutex); @@ -858,247 +826,92 @@ int msi_kill ( MSISession *session ) return 0; } -int msi_invite ( MSISession *session, - int32_t *call_index, - const MSICSettings *csettings, - uint32_t rngsec, - uint32_t friend_id ) +int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_t capabilities ) { - pthread_mutex_lock(session->mutex); - LOGGER_DEBUG("Session: %p Inviting friend: %u", session, friend_id); - - - int i = 0; - - for (; i < session->max_calls; i ++) - if (session->calls[i] && session->calls[i]->peers[0] == friend_id) { - LOGGER_ERROR("Already in a call with friend %d", friend_id); - pthread_mutex_unlock(session->mutex); - return msi_ErrorAlreadyInCallWithPeer; - } - - - MSICall *call = init_call ( session, 1, rngsec ); /* Just one peer for now */ - - if ( !call ) { - pthread_mutex_unlock(session->mutex); - LOGGER_ERROR("Cannot handle more calls"); - return msi_ErrorReachedCallLimit; + + if (get_call(session, friend_id) != NULL) { + LOGGER_ERROR("Already in a call"); + return -1; } + + *call = new_call ( session, friend_id ); - *call_index = call->call_idx; - - t_randomstr ( call->id, sizeof(call->id) ); - - add_peer ( call, friend_id ); - - call->csettings_local = *csettings; - + if ( *call == NULL ) + return -1; + + *call->capabilities = capabilities; + MSIMessage *msg_invite = msi_new_message ( type_request, requ_invite ); - - msi_msg_set_csettings(msg_invite, csettings); - send_message ( session, call, msg_invite, friend_id ); + + msg_invite->capabilities.value = capabilities; + msg_invite->capabilities.exists = 1; + + send_message ( *call, msg_invite, friend_id ); free( msg_invite ); - call->state = msi_CallRequesting; - - call->request_timer_id = timer_alloc ( session, handle_timeout, call->call_idx, m_deftout ); - + *call->state = msi_CallRequesting; + LOGGER_DEBUG("Invite sent"); - - pthread_mutex_unlock(session->mutex); - return 0; } -int msi_hangup ( MSISession *session, int32_t call_index ) +int msi_hangup ( MSICall* call ) { - pthread_mutex_lock(session->mutex); LOGGER_DEBUG("Session: %p Hanging up call: %u", session, call_index); - - if ( call_index < 0 || call_index >= session->max_calls || !session->calls[call_index] ) { - LOGGER_ERROR("Invalid call index!"); - pthread_mutex_unlock(session->mutex); - return msi_ErrorNoCall; - } - - if ( session->calls[call_index]->state != msi_CallActive ) { - LOGGER_ERROR("Call is not active!"); - pthread_mutex_unlock(session->mutex); - return msi_ErrorInvalidState; - } - + MSIMessage *msg_end = msi_new_message ( type_request, requ_end ); - - /* hangup for each peer */ - int it = 0; - - for ( ; it < session->calls[call_index]->peer_count; it ++ ) - send_message ( session, session->calls[call_index], msg_end, session->calls[call_index]->peers[it] ); - - session->calls[call_index]->state = msi_CallOver; - + send_message ( call, msg_end, call->friend_id ); free ( msg_end ); - - session->calls[call_index]->request_timer_id = - timer_alloc ( session, handle_timeout, call_index, m_deftout ); - - pthread_mutex_unlock(session->mutex); + + kill_call(call); return 0; } -int msi_answer ( MSISession *session, int32_t call_index, const MSICSettings *csettings ) +int msi_answer ( MSICall* call, uint8_t capabilities ) { - pthread_mutex_lock(session->mutex); LOGGER_DEBUG("Session: %p Answering call: %u", session, call_index); - if ( call_index < 0 || call_index >= session->max_calls || !session->calls[call_index] ) { - LOGGER_ERROR("Invalid call index!"); - pthread_mutex_unlock(session->mutex); - return msi_ErrorNoCall; - } - - if ( session->calls[call_index]->state != msi_CallRequested ) { + if ( call->state != msi_CallRequested ) { LOGGER_ERROR("Call is in invalid state!"); - pthread_mutex_unlock(session->mutex); - return msi_ErrorInvalidState; + return -1; } - + + call->capabilities = capabilities; + MSIMessage *msg_starting = msi_new_message ( type_response, resp_starting ); - - session->calls[call_index]->csettings_local = *csettings; - - msi_msg_set_csettings(msg_starting, csettings); - - send_message ( session, session->calls[call_index], msg_starting, session->calls[call_index]->peers[0] ); + + msg_starting->capabilities.value = capabilities; + msg_starting->capabilities.exists = 1; + + send_message ( call, msg_starting, call->friend_id ); free ( msg_starting ); - session->calls[call_index]->state = msi_CallActive; - - pthread_mutex_unlock(session->mutex); return 0; } -int msi_cancel ( MSISession *session, int32_t call_index, uint32_t peer, const char *reason ) +int msi_reject ( MSICall* call ) { - pthread_mutex_lock(session->mutex); - LOGGER_DEBUG("Session: %p Canceling call: %u; reason: %s", session, call_index, reason ? reason : "Unknown"); - - if ( call_index < 0 || call_index >= session->max_calls || !session->calls[call_index] ) { - LOGGER_ERROR("Invalid call index!"); - pthread_mutex_unlock(session->mutex); - return msi_ErrorNoCall; - } - - if ( session->calls[call_index]->state != msi_CallRequesting ) { - LOGGER_ERROR("Call is in invalid state!"); - pthread_mutex_unlock(session->mutex); - return msi_ErrorInvalidState; - } - - MSIMessage *msg_cancel = msi_new_message ( type_request, requ_cancel ); - - /* FIXME */ -#if 0 - - if ( reason && strlen(reason) < sizeof(MSIReasonStrType) ) { - MSIReasonStrType reason_cast; - memset(reason_cast, '\0', sizeof(MSIReasonStrType)); - memcpy(reason_cast, reason, strlen(reason)); - msi_msg_set_reason(msg_cancel, reason_cast); - } - -#else - (void)reason; - -#endif - - send_message ( session, session->calls[call_index], msg_cancel, peer ); - free ( msg_cancel ); - - terminate_call ( session, session->calls[call_index] ); - pthread_mutex_unlock(session->mutex); - - return 0; -} - -int msi_reject ( MSISession *session, int32_t call_index, const char *reason ) -{ - pthread_mutex_lock(session->mutex); LOGGER_DEBUG("Session: %p Rejecting call: %u; reason: %s", session, call_index, reason ? reason : "Unknown"); - if ( call_index < 0 || call_index >= session->max_calls || !session->calls[call_index] ) { - LOGGER_ERROR("Invalid call index!"); - pthread_mutex_unlock(session->mutex); - return msi_ErrorNoCall; - } - - if ( session->calls[call_index]->state != msi_CallRequested ) { + if ( call->state != msi_CallRequested ) { LOGGER_ERROR("Call is in invalid state!"); - pthread_mutex_unlock(session->mutex); return msi_ErrorInvalidState; } MSIMessage *msg_reject = msi_new_message ( type_request, requ_reject ); - - /* FIXME */ -#if 0 - - if ( reason && strlen(reason) < sizeof(MSIReasonStrType) ) { - MSIReasonStrType reason_cast; - memset(reason_cast, '\0', sizeof(MSIReasonStrType)); - memcpy(reason_cast, reason, strlen(reason)); - msi_msg_set_reason(msg_reject, reason_cast); - } - -#else - (void)reason; - -#endif - - send_message ( session, session->calls[call_index], msg_reject, - session->calls[call_index]->peers[session->calls[call_index]->peer_count - 1] ); + send_message ( call, msg_reject, call->friend_id ); free ( msg_reject ); - session->calls[call_index]->state = msi_CallOver; - session->calls[call_index]->request_timer_id = - timer_alloc ( session, handle_timeout, call_index, m_deftout ); - - pthread_mutex_unlock(session->mutex); return 0; } -int msi_stopcall ( MSISession *session, int32_t call_index ) -{ - pthread_mutex_lock(session->mutex); - LOGGER_DEBUG("Session: %p Stopping call index: %u", session, call_index); - - if ( call_index < 0 || call_index >= session->max_calls || !session->calls[call_index] ) { - pthread_mutex_unlock(session->mutex); - return msi_ErrorNoCall; - } - - /* just terminate it */ - - terminate_call ( session, session->calls[call_index] ); - - pthread_mutex_unlock(session->mutex); - return 0; -} - -int msi_change_csettings(MSISession *session, int32_t call_index, const MSICSettings *csettings) +int msi_change_csettings( MSICall* call, uint8_t capabilities ) { pthread_mutex_lock(session->mutex); LOGGER_DEBUG("Changing media on call: %d", call_index); - if ( call_index < 0 || call_index >= session->max_calls || !session->calls[call_index] ) { - LOGGER_ERROR("Invalid call index!"); - pthread_mutex_unlock(session->mutex); - return msi_ErrorNoCall; - } - MSICall *call = session->calls[call_index]; if ( call->state != msi_CallActive ) { @@ -1136,26 +949,4 @@ int msi_change_csettings(MSISession *session, int32_t call_index, const MSICSett pthread_mutex_unlock(session->mutex); return 0; -} - -void msi_do(MSISession *session) -{ - pthread_mutex_lock(session->mutex); - - TimerHandler *timer = session->timer_handler; - - uint64_t time = current_time_monotonic(); - - while ( timer->timers[0] && timer->timers[0]->timeout < time ) { - LOGGER_DEBUG("Executing timer assigned at: %d", timer->timers[0]->timeout); - - int id = timer->timers[0]->id; - timer->timers[0]->func(timer->timers[0]); - - /* In case function has released timer */ - if (timer->timers[0] && timer->timers[0]->id == id) - timer_release(timer, id); - } - - pthread_mutex_unlock(session->mutex); -} +} \ No newline at end of file diff --git a/toxav/msi.h b/toxav/msi.h index c71c69dde..8fa309c30 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -19,8 +19,8 @@ * */ -#ifndef __TOXMSI -#define __TOXMSI +#ifndef MSI_H +#define MSI_H #include #include @@ -40,6 +40,23 @@ typedef enum { msi_TypeVideo } MSICallType; +/** + * Error codes. + */ +typedef enum { + msi_ErrUndisclosed, +} MSIError; + +/** + * Supported capabilities + */ +typedef enum { + msi_CapSAudio = 1, /* sending audio */ + msi_CapSVideo = 2, /* sending video */ + msi_CapRAudio = 4, /* receiving audio */ + msi_CapRVideo = 8, /* receiving video */ +} MSICapabilities; + /** * Call state identifiers. @@ -70,16 +87,6 @@ typedef struct { uint32_t audio_channels; } MSICSettings; -/** - * Active capabilities masks - */ -typedef enum { - msi_SendingAudio = 1, - msi_SendingVideo = 2, - msi_RecvingAudio = 4, - msi_RecvingVideo = 8, -} MSICapMask; - /** * Callbacks ids that handle the states */ @@ -87,9 +94,9 @@ typedef enum { msi_OnInvite, /* Incoming call */ msi_OnRinging, /* When peer is ready to accept/reject the call */ msi_OnStart, /* Call (RTP transmission) started */ - msi_OnCancel, /* The side that initiated call canceled invite */ msi_OnReject, /* The side that was invited rejected the call */ msi_OnEnd, /* Call that was active ended */ + msi_OnError, /* Call that was active ended */ msi_OnRequestTimeout, /* When the requested action didn't get response in specified time */ msi_OnPeerTimeout, /* Peer timed out; stop the call */ msi_OnPeerCSChange, /* Peer requested Csettings change */ @@ -107,25 +114,30 @@ typedef enum { } MSIError; /** - * The call struct. + * The call struct. Please do not modify outside msi.c */ -typedef struct { +typedef struct MSICall_s { struct MSISession_s *session; /* Session pointer */ MSICallState state; - uint8_t caps; /* Active capabilities */ + uint8_t capabilities; /* Active capabilities */ uint32_t friend_id; /* Index of this call in MSISession */ + + struct MSICall_s* next; + struct MSICall_s* prev; } MSICall; /** - * Control session struct + * Control session struct. Please do not modify outside msi.c */ typedef struct MSISession_s { /* Call handlers */ MSICall **calls; - + uint32_t calls_tail; + uint32_t calls_head; + void *agent_handler; Messenger *messenger_handle; @@ -139,7 +151,7 @@ typedef struct MSISession_s { MSISession *msi_new ( Messenger *messenger, int32_t max_calls ); /** - * Terminate control session. + * Terminate control session. NOTE: all calls will be freed */ int msi_kill ( MSISession *session ); @@ -151,45 +163,26 @@ void msi_register_callback(MSISession *session, MSICallbackType callback, MSICal /** * Send invite request to friend_id. */ -int msi_invite ( MSISession *session, - int32_t *call_index, - const MSICSettings *csettings, - uint32_t rngsec, - uint32_t friend_id ); +int msi_invite ( MSISession* session, MSICall** call, uint32_t friend_id, uint8_t capabilities ); /** - * Hangup active call. + * Hangup call. NOTE: 'call' will be freed */ -int msi_hangup ( MSISession *session, int32_t call_index ); +int msi_hangup ( MSICall* call ); /** - * Answer active call request. + * Answer call request. */ -int msi_answer ( MSISession *session, int32_t call_index, const MSICSettings *csettings ); +int msi_answer ( MSICall* call, uint8_t capabilities ); /** - * Cancel request. + * Reject incoming call. NOTE: 'call' will be freed */ -int msi_cancel ( MSISession *session, int32_t call_index, uint32_t peer, const char *reason ); +int msi_reject ( MSICall* call ); /** - * Reject incoming call. + * Change capabilities of the call. */ -int msi_reject ( MSISession *session, int32_t call_index, const char *reason ); +int msi_change_capabilities ( MSICall* call, uint8_t capabilities ); -/** - * Terminate the call. - */ -int msi_stopcall ( MSISession *session, int32_t call_index ); - -/** - * Change codec settings of the current call. - */ -int msi_change_csettings ( MSISession *session, int32_t call_index, const MSICSettings *csettings ); - -/** - * Main msi loop - */ -void msi_do( MSISession *session ); - -#endif /* __TOXMSI */ +#endif /* MSI_H */ diff --git a/toxav/rtp.h b/toxav/rtp.h index 03b44719d..b1888f6c7 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -19,8 +19,8 @@ * */ -#ifndef __TOXRTP -#define __TOXRTP +#ifndef RTP_H +#define RTP_H #define RTP_VERSION 2 #include @@ -130,4 +130,4 @@ void rtp_free_msg ( RTPSession *session, RTPMessage *msg ); -#endif /* __TOXRTP */ +#endif /* RTP_H */ diff --git a/toxav/toxav.c b/toxav/toxav.c index b594cc29e..72f995762 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -83,7 +83,6 @@ struct toxAV void i_toxav_msi_callback_invite(void* toxav_inst, int32_t call_idx, void *data); void i_toxav_msi_callback_ringing(void* toxav_inst, int32_t call_idx, void *data); void i_toxav_msi_callback_start(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_cancel(void* toxav_inst, int32_t call_idx, void *data); void i_toxav_msi_callback_reject(void* toxav_inst, int32_t call_idx, void *data); void i_toxav_msi_callback_end(void* toxav_inst, int32_t call_idx, void *data); void i_toxav_msi_callback_request_to(void* toxav_inst, int32_t call_idx, void *data); /* TODO remove */ @@ -138,7 +137,6 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) msi_register_callback(av->msi, i_toxav_msi_callback_invite, msi_OnInvite, NULL); msi_register_callback(av->msi, i_toxav_msi_callback_ringing, msi_OnRinging, NULL); msi_register_callback(av->msi, i_toxav_msi_callback_start, msi_OnStart, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_cancel, msi_OnCancel, NULL); msi_register_callback(av->msi, i_toxav_msi_callback_reject, msi_OnReject, NULL); msi_register_callback(av->msi, i_toxav_msi_callback_end, msi_OnEnd, NULL); msi_register_callback(av->msi, i_toxav_msi_callback_request_to, msi_OnRequestTimeout, NULL); @@ -588,17 +586,6 @@ void i_toxav_msi_callback_start(void* toxav_inst, int32_t call_idx, void* data) toxav->scb.first(toxav, call->friend_number, state, toxav->scb.second); } -void i_toxav_msi_callback_cancel(void* toxav_inst, int32_t call_idx, void* data) -{ - ToxAV* toxav = toxav_inst; - - i_toxav_remove_call(toxav, toxav->msi->calls[call_idx]->peers[0]); - - if (toxav->scb.first) - toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], - TOXAV_CALL_STATE_END, toxav->scb.second); -} - void i_toxav_msi_callback_reject(void* toxav_inst, int32_t call_idx, void* data) { ToxAV* toxav = toxav_inst; @@ -918,7 +905,6 @@ void i_toxav_kill_transmission(ToxAV* av, uint32_t friend_number) if (!call->active) { pthread_mutex_unlock(call->mutex_control); - LOGGER_WARNING("Action on inactive call: %d", call->call_idx); return; } diff --git a/toxav/toxav.h b/toxav/toxav.h index 69654f973..04ad3cd00 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -1,4 +1,6 @@ -#pragma once +#ifndef TOXAV_H +#define TOXAV_H + #include #include #include @@ -480,4 +482,6 @@ typedef void toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, /** * Set the callback for the `receive_audio_frame` event. Pass NULL to unset. */ -void toxav_callback_receive_audio_frame(ToxAV *av, toxav_receive_audio_frame_cb *function, void *user_data); \ No newline at end of file +void toxav_callback_receive_audio_frame(ToxAV *av, toxav_receive_audio_frame_cb *function, void *user_data); + +#endif /* TOXAV_H */ \ No newline at end of file From 483a6ffa328e8699d156633ab3e0016171e11dd8 Mon Sep 17 00:00:00 2001 From: mannol Date: Tue, 17 Feb 2015 23:34:40 +0100 Subject: [PATCH 07/97] Added 2 new headers into msi for buffer control --- toxav/codec.c | 20 +- toxav/codec.h | 8 +- toxav/msi.c | 601 ++++++++++++++++++++++++-------------------------- toxav/msi.h | 71 ++---- toxav/rtp.c | 8 +- toxav/rtp.h | 55 ++--- toxav/toxav.c | 6 +- toxav/toxav.h | 21 ++ 8 files changed, 369 insertions(+), 421 deletions(-) diff --git a/toxav/codec.c b/toxav/codec.c index e6fe713ea..9fc140711 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -1,8 +1,6 @@ /** codec.c * - * Audio and video codec intitialization, encoding/decoding and playback - * - * Copyright (C) 2013 Tox project All Rights Reserved. + * Copyright (C) 2013-2015 Tox project All Rights Reserved. * * This file is part of Tox. * @@ -124,7 +122,7 @@ static void buffer_free(PayloadBuffer *b) } /* JITTER BUFFER WORK */ -typedef struct JitterBuffer { +typedef struct JitterBuffer_s { RTPMessage **queue; uint32_t size; uint32_t capacity; @@ -711,11 +709,6 @@ int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate, int channels) if (cs->audio_encoder) return 0; - /** - * Encoder is initialized with default values. These values (Sampling rate, channel count) - * change on the fly from toxav. - */ - int rc = OPUS_OK; cs->audio_encoder = opus_encoder_create(48000, channels, OPUS_APPLICATION_AUDIO, &rc); @@ -750,12 +743,7 @@ int cs_enable_audio_receiving(CSSession* cs) { if (cs->audio_decoder) return 0; - - /** - * Decoder is initialized with default values. These values (Sampling rate, channel count) - * change on the fly from toxav. - */ - + int rc; cs->audio_decoder = opus_decoder_create(48000, 2, &rc ); @@ -792,7 +780,7 @@ void queue_message(RTPSession *session, RTPMessage *msg) if (!cs) return; /* Audio */ - if (session->payload_type == msi_TypeAudio % 128) { + if (session->payload_type == rtp_TypeAudio % 128) { pthread_mutex_lock(cs->queue_mutex); int ret = jbuf_write(cs->j_buf, msg); pthread_mutex_unlock(cs->queue_mutex); diff --git a/toxav/codec.h b/toxav/codec.h index 02a3b1b4f..b5eb19e21 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -1,8 +1,6 @@ /** codec.h * - * Audio and video codec intitialization, encoding/decoding and playback - * - * Copyright (C) 2013 Tox project All Rights Reserved. + * Copyright (C) 2013-2015 Tox project All Rights Reserved. * * This file is part of Tox. * @@ -69,7 +67,7 @@ typedef enum { /** * Codec session - controling codec */ -typedef struct _CSSession { +typedef struct CSSession_s { /* VIDEO * @@ -117,7 +115,7 @@ typedef struct _CSSession { int32_t last_pack_channels; int32_t last_packet_sampling_rate; int32_t last_packet_frame_duration; - struct JitterBuffer *j_buf; + struct JitterBuffer_s *j_buf; /* Voice activity detection */ diff --git a/toxav/msi.c b/toxav/msi.c index e78c57feb..c13429503 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -1,6 +1,6 @@ /** msi.c * - * Copyright (C) 2013 Tox project All Rights Reserved. + * Copyright (C) 2013-2015 Tox project All Rights Reserved. * * This file is part of Tox. * @@ -36,11 +36,6 @@ #define MSI_MAXMSG_SIZE 256 -/* Define default timeout for a request. - * There is no behavior specified by the msi on what will - * client do on timeout, but to call timeout callback. - */ -#define m_deftout 10000 /* in milliseconds */ /** * Protocol: @@ -53,17 +48,11 @@ typedef enum { IDResponse, IDError, IDCapabilities, + IDMVFSZ, + IDMVFPSZ, } MSIHeaderID; -/** - * Headers - */ -typedef enum { - type_request, - type_response, -} MSIMessageType; - typedef enum { requ_invite, requ_start, @@ -77,17 +66,20 @@ typedef enum { resp_error, } MSIResponse; + #define GENERIC_HEADER(header, val_type) \ typedef struct { \ val_type value; \ -_Bool exists; \ -} MSIHeader##header; +bool exists; \ +} MSIHeader##header -GENERIC_HEADER ( Request, MSIRequest ) -GENERIC_HEADER ( Response, MSIResponse ) -GENERIC_HEADER ( Error, MSIError ) -GENERIC_HEADER ( Capabilities, uint8_t ) +GENERIC_HEADER ( Request, MSIRequest ); +GENERIC_HEADER ( Response, MSIResponse ); +GENERIC_HEADER ( Error, MSIError ); +GENERIC_HEADER ( Capabilities, uint8_t ); +GENERIC_HEADER ( MVFSZ, uint16_t ); +GENERIC_HEADER ( MVFPSZ, uint16_t ); typedef struct { @@ -95,63 +87,36 @@ typedef struct { MSIHeaderResponse response; MSIHeaderError error; MSIHeaderCapabilities capabilities; + MSIHeaderMVFSZ mvfsz; /* Max video frame size. NOTE: Value must be in network b-order */ + MSIHeaderMVFPSZ mvfpsz; /* Max video frame piece size. NOTE: Value must be in network b-order */ } MSIMessage; -static void invoke_callback(MSICall* c, MSICallbackID i) +static int parse_input ( MSIMessage *dest, const uint8_t *data, uint16_t length ) { - if ( c->session->callbacks[i] ) { - LOGGER_DEBUG("Invoking callback function: %d", i); - c->session->callbacks[i] ( c->session->agent_handler, c ); - } -} - -/** - * Create the message. - */ -static MSIMessage *msi_new_message ( MSIMessageType type, const uint8_t type_value ) -{ - MSIMessage *retu = calloc ( sizeof ( MSIMessage ), 1 ); - - if ( retu == NULL ) { - LOGGER_WARNING("Allocation failed! Program might misbehave!"); - return NULL; - } - - if ( type == type_request ) { - retu->request.exists = 1; - retu->request.value = type_value; - - } else { - retu->response.exists = 1; - retu->response.value = type_value; - } - - return retu; -} - - -/** - * Parse raw data received from socket into MSIMessage struct. - */ -static int parse_raw_data ( MSIMessage *msg, const uint8_t *data, uint16_t length ) -{ - + /* Parse raw data received from socket into MSIMessage struct */ + #define CHECK_SIZE(bytes, constraint, size) \ - if ((constraint -= 3) < 1) { LOGGER_ERROR("Read over length!"); return -1; } \ + if ((constraint -= (2 + size)) < 1) { LOGGER_ERROR("Read over length!"); return -1; } \ if ( bytes[1] != size ) { LOGGER_ERROR("Invalid data size!"); return -1; } -#define CHECK_ENUM_HIGH(bytes, enum_high) \ +#define CHECK_ENUM_HIGH(bytes, enum_high) /* Assumes size == 1 */ \ if ( bytes[2] > enum_high ) { LOGGER_ERROR("Failed enum high limit!"); return -1; } -#define SET_VALUES(bytes, header) do { \ +#define SET_UINT8(bytes, header) do { \ header.value = bytes[2]; \ - header.exists = 1; \ + header.exists = true; \ bytes += 3; \ } while(0) - - if ( msg == NULL ) { +#define SET_UINT16(bytes, header) do { \ + memcpy(&header.value, bytes + 2, 2);\ + header.exists = true; \ + bytes += 4; \ + } while(0) + + + if ( dest == NULL ) { LOGGER_ERROR("Could not parse message: no storage!"); return -1; } @@ -169,25 +134,35 @@ static int parse_raw_data ( MSIMessage *msg, const uint8_t *data, uint16_t lengt case IDRequest: CHECK_SIZE(it, size_constraint, 1); CHECK_ENUM_HIGH(it, requ_end); - SET_VALUES(it, msg->request); + SET_UINT8(it, dest->request); break; case IDResponse: CHECK_SIZE(it, size_constraint, 1); CHECK_ENUM_HIGH(it, resp_error); - SET_VALUES(it, msg->response); + SET_UINT8(it, dest->response); it += 3; break; case IDError: CHECK_SIZE(it, size_constraint, 1); CHECK_ENUM_HIGH(it, msi_ErrUndisclosed); - SET_VALUES(it, msg->error); + SET_UINT8(it, dest->error); break; case IDCapabilities: CHECK_SIZE(it, size_constraint, 1); - SET_VALUES(it, msg->capabilities); + SET_UINT8(it, dest->capabilities); + break; + + case IDMVFSZ: + CHECK_SIZE(it, size_constraint, 2); + SET_UINT16(it, dest->mvfsz); + break; + + case IDMVFPSZ: + CHECK_SIZE(it, size_constraint, 2); + SET_UINT16(it, dest->mvfpsz); break; default: @@ -201,42 +176,15 @@ static int parse_raw_data ( MSIMessage *msg, const uint8_t *data, uint16_t lengt #undef CHECK_SIZE #undef CHECK_ENUM_HIGH -#undef SET_VALUES +#undef SET_UINT8 +#undef SET_UINT16 } -/** - * Parse data from handle_packet. - */ -static MSIMessage *parse_in ( const uint8_t *data, uint16_t length ) -{ - if ( data == NULL ) { - LOGGER_WARNING("Tried to parse empty message!"); - return NULL; - } - - MSIMessage *retu = calloc ( sizeof ( MSIMessage ), 1 ); - - if ( retu == NULL ) { - LOGGER_WARNING("Allocation failed! Program might misbehave!"); - return NULL; - } - - if ( parse_raw_data ( retu, data, length ) == -1 ) { - - free ( retu ); - return NULL; - } - - return retu; -} - - -/** - * Speaks for itself. - */ -static uint8_t *prepare_header ( MSIHeaderID id, uint8_t *dest, const void *value, - uint8_t value_len, uint16_t *length ) +static uint8_t *parse_header ( MSIHeaderID id, uint8_t *dest, const void *value, + uint8_t value_len, uint16_t *length ) { + /* Parse a single header for sending */ + if ( dest == NULL ) { LOGGER_ERROR("No destination space!"); return NULL; @@ -260,60 +208,65 @@ static uint8_t *prepare_header ( MSIHeaderID id, uint8_t *dest, const void *valu } -/** - * Parse MSIMessage to send. Returns size in bytes of the parsed message - */ -static uint16_t parse_out ( MSIMessage *msg, uint8_t *dest ) + +static void call_invoke_callback(MSICall* call, MSICallbackID cb) { - if (msg == NULL) { - LOGGER_ERROR("No message!"); - return 0; + if ( call->session->callbacks[cb] ) { + LOGGER_DEBUG("Invoking callback function: %d", cb); + call->session->callbacks[cb] ( call->session->agent_handler, call ); } - - if (dest == NULL ) { - LOGGER_ERROR("No destination!"); - return 0; - } - - uint8_t *it = dest; - uint16_t size = 0; - - if (msg->request.exists) { - uint8_t cast = msg->request.value; - it = prepare_header(IDRequest, it, &cast, 1, &size); - } - - if (msg->response.exists) { - uint8_t cast = msg->response.value; - it = prepare_header(IDResponse, it, &cast, 1, &size); - } - - if (msg->error.exists) { - it = prepare_header(IDError, it, &msg->error.value, sizeof(msg->error.value), &size); - } - - if (msg->capabilities.exists) { - it = prepare_header(IDCapabilities, it, &msg->capabilities.value, - sizeof(msg->capabilities.value), &size); - } - - *it = 0; - size ++; - - return size; } -static int send_message ( MSICall *call, MSIMessage *msg, uint32_t to ) +static int call_send_message ( MSICall *call, const MSIMessage *msg ) { + /* Parse and send message */ + uint8_t parsed [MSI_MAXMSG_SIZE]; - uint16_t length = parse_out ( msg, parsed ); - if ( !length ) { - LOGGER_WARNING("Parsing message failed; nothing sent!"); + uint8_t *it = parsed; + uint16_t size = 0; + + if (msg->request.exists) { + uint8_t cast = msg->request.value; + it = parse_header(IDRequest, it, &cast, + sizeof(cast), &size); + } + + if (msg->response.exists) { + uint8_t cast = msg->response.value; + it = parse_header(IDResponse, it, &cast, + sizeof(cast), &size); + } + + if (msg->error.exists) { + it = parse_header(IDError, it, &msg->error.value, + sizeof(msg->error.value), &size); + } + + if (msg->capabilities.exists) { + it = parse_header(IDCapabilities, it, &msg->capabilities.value, + sizeof(msg->capabilities.value), &size); + } + + if (msg->mvfsz.exists) { + it = parse_header(IDMVFSZ, it, &msg->mvfsz.value, + sizeof(msg->mvfsz.value), &size); + } + + if (msg->mvfpsz.exists) { + it = parse_header(IDMVFPSZ, it, &msg->mvfpsz.value, + sizeof(msg->mvfpsz.value), &size); + } + + *it = 0; + size ++; + + if ( it == parsed ) { + LOGGER_WARNING("Parsing message failed; empty message"); return -1; } - if ( m_msi_packet(call->session->messenger_handle, to, parsed, length) ) { + if ( m_msi_packet(call->session->messenger_handle, call->friend_id, parsed, size) ) { LOGGER_DEBUG("Sent message"); return 0; } @@ -321,34 +274,25 @@ static int send_message ( MSICall *call, MSIMessage *msg, uint32_t to ) return -1; } -static int send_reponse ( MSICall *call, MSIResponse response, uint32_t to ) -{ - MSIMessage *msg = msi_new_message ( type_response, response ); - int ret = send_message ( call, msg, to ); - free ( msg ); - return ret; -} - -static int send_error ( MSICall *call, MSIError error, uint32_t to ) +static int call_send_error ( MSICall *call, MSIError error ) { + /* Send error message */ + if (!call) { LOGGER_WARNING("Cannot handle error on 'null' call"); return -1; } - LOGGER_DEBUG("Sending error: %d on call: %d", error, call->call_idx); + LOGGER_DEBUG("Sending error: %d to friend: %d", error, call->friend_id); - MSIMessage *msg_error = msi_new_message ( type_response, resp_error ); + MSIMessage msg_error; + msg_error.response.exists = true; + msg_error.response.value = resp_error; - if (!msg_error) - return -1; + msg_error.error.exists = true; + msg_error.error.value = error; - msg_error->error.exists = 1; - msg_error->error.value = error; - - send_message ( call, msg_error, to ); - free ( msg_error ); - + call_send_message ( call, &msg_error ); return 0; } @@ -415,7 +359,6 @@ static void kill_call ( MSICall *call ) if ( call == NULL ) return; - MSISession* session = call->session; MSICall* prev = call->prev; @@ -446,7 +389,7 @@ CLEAR: -static void handle_remote_connection_change(Messenger *messenger, int friend_id, uint8_t status, void *session_p) +static void on_remote_connection_change(Messenger *messenger, int friend_id, uint8_t status, void *session_p) { (void)messenger; MSISession *session = session_p; @@ -458,7 +401,7 @@ static void handle_remote_connection_change(Messenger *messenger, int friend_id, if (call == NULL) return; - invoke_callback(call, msi_OnPeerTimeout); + call_invoke_callback(call, msi_OnPeerTimeout); kill_call(call); } break; @@ -471,7 +414,7 @@ static void handle_remote_connection_change(Messenger *messenger, int friend_id, /********** Request handlers **********/ -static int handle_recv_invite ( MSICall *call, MSIMessage *msg ) +static int handle_recv_invite ( MSICall *call, const MSIMessage *msg ) { if ( call == NULL ) { LOGGER_WARNING("Session: %p Handling 'invite' on no call"); @@ -482,6 +425,26 @@ static int handle_recv_invite ( MSICall *call, MSIMessage *msg ) LOGGER_DEBUG("Session: %p Handling 'invite' friend: %d", call->session, call->friend_id); + if (!msg->capabilities.exists) { + LOGGER_WARNING("Session: %p Invalid capabilities on 'invite'"); + /* TODO send error */ + return -1; + } + + if (!msg->mvfsz.exists) { + LOGGER_WARNING("Session: %p Invalid mvfsz on 'invite'"); + /* TODO send error */ + return -1; + } + + if (!msg->mvfpsz.exists) { + LOGGER_WARNING("Session: %p Invalid mvfpsz on 'invite'"); + /* TODO send error */ + return -1; + } + + MSIMessage response; + response.response.exists = true; if ( call->state == msi_CallRequesting ) { /* The rare glare case. @@ -493,29 +456,54 @@ static int handle_recv_invite ( MSICall *call, MSIMessage *msg ) LOGGER_DEBUG("Glare detected!"); - MSIMessage *msg_starting = msi_new_message ( type_response, resp_starting ); + call->peer_capabilities = msg->capabilities; - call->capabilities &= msg->capabilities; + call->peer_mvfsz = ntohs(msg->mvfsz.value); + call->peer_mvfpsz = ntohs(msg->mvfpsz.value); - msg_starting->capabilities.value = call->capabilities; - msg_starting->capabilities.exists = 1; + /* Send response */ + response.response.value = resp_starting; + call_send_message ( call, &response ); - send_message ( call, msg_starting, call->friend_id ); - free ( msg_starting ); + return 0; + } else if ( call->state == msi_CallActive ) { + /* Changing capabilities. + * We send starting but no response is expected. + * WARNING: if start is sent call is terminated with an error + */ + LOGGER_DEBUG("Peer is changing capabilities"); + call->peer_capabilities = msg->capabilities; + + call->peer_mvfsz = ntohs(msg->mvfsz.value); + call->peer_mvfpsz = ntohs(msg->mvfpsz.value); + + /* Send response */ + response.response.value = resp_starting; + call_send_message ( call, &response ); + + + call_invoke_callback(call, msi_OnCapabilities); return 0; } - call->capabilities = msg->capabilities; + call->peer_capabilities = msg->capabilities; + + call->peer_mvfsz = ntohs(msg->mvfsz.value); + call->peer_mvfpsz = ntohs(msg->mvfpsz.value); + call->state = msi_CallRequested; - send_reponse(call, resp_ringing, call->friend_id); - invoke_callback(call, msi_OnInvite); - + /* Send response */ + response.response.value = resp_ringing; + call_send_message ( call, &response ); + + + call_invoke_callback(call, msi_OnInvite); return 0; } -static int handle_recv_start ( MSICall *call, MSIMessage *msg ) +static int handle_recv_start ( MSICall *call, const MSIMessage *msg ) { if ( call == NULL ) { LOGGER_WARNING("Session: %p Handling 'start' on no call"); @@ -533,12 +521,12 @@ static int handle_recv_start ( MSICall *call, MSIMessage *msg ) LOGGER_DEBUG("Session: %p Handling 'start', friend id: %d", call->session, call->friend_id ); call->state = msi_CallActive; - invoke_callback(call, msi_OnStart); + call_invoke_callback(call, msi_OnStart); return 0; } -static int handle_recv_reject ( MSICall *call, MSIMessage *msg ) +static int handle_recv_reject ( MSICall *call, const MSIMessage *msg ) { if ( call == NULL ) { LOGGER_WARNING("Session: %p Handling 'start' on no call"); @@ -555,13 +543,13 @@ static int handle_recv_reject ( MSICall *call, MSIMessage *msg ) LOGGER_DEBUG("Session: %p Handling 'reject', friend id: %u", call->session, call->friend_id); - invoke_callback(call, msi_OnReject); + call_invoke_callback(call, msi_OnReject); kill_call(call); return 0; } -static int handle_recv_end ( MSICall *call, MSIMessage *msg ) +static int handle_recv_end ( MSICall *call, const MSIMessage *msg ) { (void)msg; @@ -572,14 +560,14 @@ static int handle_recv_end ( MSICall *call, MSIMessage *msg ) LOGGER_DEBUG("Session: %p Handling 'end', friend id: %d", call->session, call->friend_id); - invoke_callback(call, msi_OnEnd); + call_invoke_callback(call, msi_OnEnd); kill_call ( call ); return 0; } /********** Response handlers **********/ -static int handle_recv_ringing ( MSICall *call, MSIMessage *msg ) +static int handle_recv_ringing ( MSICall *call, const MSIMessage *msg ) { if ( call == NULL ) { LOGGER_WARNING("Session: %p Handling 'start' on no call"); @@ -596,36 +584,65 @@ static int handle_recv_ringing ( MSICall *call, MSIMessage *msg ) LOGGER_DEBUG("Session: %p Handling 'ringing' friend id: %d", call->session, call->friend_id ); - invoke_callback(call, msi_OnRinging); + call_invoke_callback(call, msi_OnRinging); return 0; } -static int handle_recv_starting ( MSICall *call, MSIMessage *msg ) + +static int handle_recv_starting ( MSICall *call, const MSIMessage *msg ) { if ( call == NULL ) { LOGGER_WARNING("Session: %p Handling 'starting' on non-existing call"); return 0; } - if ( call->state != msi_CallRequested || call->state != msi_CallRequesting ) { + if ( call->state == msi_CallActive ) { + LOGGER_DEBUG("Capabilities change confirmed"); + return 0; + } else if ( call->state != msi_CallRequested || call->state != msi_CallRequesting ) { LOGGER_WARNING("Session: %p Invalid call state on 'starting'"); /* TODO send error */ return -1; } - MSIMessage *msg_start = msi_new_message ( type_request, requ_start ); - send_message ( call, msg_start, call->friend_id ); - free ( msg_start ); - if (call->state == msi_CallRequesting) { + if (!msg->capabilities.exists) { + LOGGER_WARNING("Session: %p Invalid capabilities on 'starting'"); + /* TODO send error */ + return -1; + } + + if (!msg->mvfsz.exists) { + LOGGER_WARNING("Session: %p Invalid mvfsz on 'invite'"); + /* TODO send error */ + return -1; + } + + if (!msg->mvfpsz.exists) { + LOGGER_WARNING("Session: %p Invalid mvfpsz on 'invite'"); + /* TODO send error */ + return -1; + } + + call->peer_capabilities = msg->capabilities.value; + + call->peer_mvfsz = ntohs(msg->mvfsz.value); + call->peer_mvfpsz = ntohs(msg->mvfpsz.value); + call->state = msi_CallActive; - invoke_callback(call, msi_OnStart); - } - + call_invoke_callback(call, msi_OnStart); + } /* Otherwise it's a glare case so don't start until 'start' is recved */ + /* Send start in either case (glare or normal) */ + MSIMessage msg_start; + msg_start.request.exists = true; + msg_start.request.value = requ_start; + call_send_message ( call, &msg_start ); + return 0; } -static int handle_recv_error ( MSICall *call, MSIMessage *msg ) + +static int handle_recv_error ( MSICall *call, const MSIMessage *msg ) { if ( call == NULL ) { LOGGER_WARNING("Handling 'error' on non-existing call!"); @@ -634,46 +651,15 @@ static int handle_recv_error ( MSICall *call, MSIMessage *msg ) LOGGER_DEBUG("Session: %p Handling 'error' friend id: %d", call->session, call->friend_id ); - invoke_callback(call, msi_OnError); + call_invoke_callback(call, msi_OnError); /* TODO Handle error accordingly */ return -1; } -/** - * BASIC call flow: - * - * ALICE BOB - * | invite --> | - * | | - * | <-- ringing | - * | | - * | <-- starting | - * | | - * | start --> | - * | | - * | <-- MEDIA TRANS --> | - * | | - * | end --> | - * | | - * | <-- ending | - * - * Alice calls Bob by sending invite packet. - * Bob recvs the packet and sends an ringing packet; - * which notifies Alice that her invite is acknowledged. - * Ringing screen shown on both sides. - * Bob accepts the invite for a call by sending starting packet. - * Alice recvs the starting packet and sends the started packet to - * inform Bob that she recved the starting packet. - * Now the media transmission is established ( i.e. RTP transmission ). - * Alice hangs up and sends end packet. - * Bob recves the end packet and sends ending packet - * as the acknowledgement that the call is ending. - * - * - */ -static void msi_handle_packet ( Messenger *messenger, int friend_id, const uint8_t *data, uint16_t length, void *object ) +static void handle_msi_packet ( Messenger *messenger, int friend_id, const uint8_t *data, + uint16_t length, void *object ) { LOGGER_DEBUG("Got msi message"); @@ -681,11 +667,9 @@ static void msi_handle_packet ( Messenger *messenger, int friend_id, const uint8 (void)messenger; MSISession *session = object; - MSIMessage *msg; + MSIMessage msg; - msg = parse_in ( data, length ); - - if ( !msg ) { + if ( parse_input ( &msg, data, length ) == -1 ) { LOGGER_WARNING("Error parsing message"); return; } else { @@ -697,7 +681,7 @@ static void msi_handle_packet ( Messenger *messenger, int friend_id, const uint8 MSICall *call = get_call(session, friend_id); if (call == NULL) { - if (msg->request != requ_invite) { + if (msg.request != requ_invite) { /* TODO send error */ return; } @@ -712,36 +696,36 @@ static void msi_handle_packet ( Messenger *messenger, int friend_id, const uint8 /* Now handle message */ int rc = 0; - if ( msg->request.exists ) { /* Handle request */ - switch (msg->request.value) { + if ( msg.request.exists ) { /* Handle request */ + switch (msg.request.value) { case requ_invite: - rc = handle_recv_invite ( call, msg ); + rc = handle_recv_invite ( call, &msg ); break; case requ_start: - rc = handle_recv_start ( call, msg ); + rc = handle_recv_start ( call, &msg ); break; case requ_reject: - rc = handle_recv_reject ( call, msg ); + rc = handle_recv_reject ( call, &msg ); break; case requ_end: - rc = handle_recv_end ( call, msg ); + rc = handle_recv_end ( call, &msg ); break; } - } else if ( msg->response.exists ) { /* Handle response */ - switch (msg->response.value) { + } else if ( msg.response.exists ) { /* Handle response */ + switch (msg.response.value) { case resp_ringing: - rc = handle_recv_ringing ( call, msg ); + rc = handle_recv_ringing ( call, &msg ); break; case resp_starting: - rc = handle_recv_starting ( call, msg ); + rc = handle_recv_starting ( call, &msg ); break; case resp_error: - rc = handle_recv_error ( call, msg ); + rc = handle_recv_error ( call, &msg ); break; } } else { @@ -753,7 +737,6 @@ static void msi_handle_packet ( Messenger *messenger, int friend_id, const uint8 if (rc == -1) kill_call(call); - free ( msg ); pthread_mutex_unlock(session->mutex); } @@ -787,10 +770,10 @@ MSISession *msi_new ( Messenger *messenger ) retu->messenger_handle = messenger; - m_callback_msi_packet(messenger, msi_handle_packet, retu ); + m_callback_msi_packet(messenger, handle_msi_packet, retu ); /* This is called when remote terminates session */ - m_callback_connectionstatus_internal_av(messenger, handle_remote_connection_change, retu); + m_callback_connectionstatus_internal_av(messenger, on_remote_connection_change, retu); LOGGER_DEBUG("New msi session: %p ", retu); return retu; @@ -807,15 +790,15 @@ int msi_kill ( MSISession *session ) pthread_mutex_lock(session->mutex); if (session->calls) { - MSIMessage *msg_end = msi_new_message ( type_request, requ_end ); + MSIMessage msg_end; + msg_end.request.exists = true; + msg_end.request.value = requ_end; MSICall* it = get_call(session, session->calls_head); for (; it; it = it->next) { - send_message(it, msg_end, it->friend_id); - kill_call(it); + call_send_message(it, &msg_end); + kill_call(it); /* This will eventually free session->calls */ } - - free(msg_end); } pthread_mutex_unlock(session->mutex); @@ -840,15 +823,22 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ if ( *call == NULL ) return -1; - *call->capabilities = capabilities; + *call->self_capabilities = capabilities; - MSIMessage *msg_invite = msi_new_message ( type_request, requ_invite ); + MSIMessage msg_invite; + msg_invite.request.exists = true; + msg_invite.request.value = requ_invite; - msg_invite->capabilities.value = capabilities; - msg_invite->capabilities.exists = 1; + msg_invite.capabilities.exists = true; + msg_invite.capabilities.value = capabilities; - send_message ( *call, msg_invite, friend_id ); - free( msg_invite ); + msg_invite.mvfsz.exists = true; + msg_invite.mvfsz.value = htons(D_MVFSZ); + + msg_invite.mvfpsz.exists = true; + msg_invite.mvfpsz.value = htons(D_MVFPSZ); + + call_send_message ( *call, &msg_invite ); *call->state = msi_CallRequesting; @@ -860,9 +850,10 @@ int msi_hangup ( MSICall* call ) { LOGGER_DEBUG("Session: %p Hanging up call: %u", session, call_index); - MSIMessage *msg_end = msi_new_message ( type_request, requ_end ); - send_message ( call, msg_end, call->friend_id ); - free ( msg_end ); + MSIMessage msg_end; + msg_end.request.exists = true; + msg_end.request.value = requ_end; + call_send_message ( call, &msg_end ); kill_call(call); return 0; @@ -870,22 +861,29 @@ int msi_hangup ( MSICall* call ) int msi_answer ( MSICall* call, uint8_t capabilities ) { - LOGGER_DEBUG("Session: %p Answering call: %u", session, call_index); + LOGGER_DEBUG("Session: %p Answering call from: %u", call->session, call->friend_id); if ( call->state != msi_CallRequested ) { LOGGER_ERROR("Call is in invalid state!"); return -1; } - call->capabilities = capabilities; + call->self_capabilities = capabilities; - MSIMessage *msg_starting = msi_new_message ( type_response, resp_starting ); + MSIMessage msg_starting; + msg_starting.response.exists = true; + msg_starting.response.value = resp_starting; - msg_starting->capabilities.value = capabilities; - msg_starting->capabilities.exists = 1; + msg_starting.capabilities.exists = true; + msg_starting.capabilities.value = capabilities; - send_message ( call, msg_starting, call->friend_id ); - free ( msg_starting ); + msg_starting.mvfsz.exists = true; + msg_starting.mvfsz.value = htons(D_MVFSZ); + + msg_starting.mvfpsz.exists = true; + msg_starting.mvfpsz.value = htons(D_MVFPSZ); + + call_send_message ( call, &msg_starting ); return 0; } @@ -896,57 +894,30 @@ int msi_reject ( MSICall* call ) if ( call->state != msi_CallRequested ) { LOGGER_ERROR("Call is in invalid state!"); - return msi_ErrorInvalidState; + return -1; } - - MSIMessage *msg_reject = msi_new_message ( type_request, requ_reject ); - send_message ( call, msg_reject, call->friend_id ); - free ( msg_reject ); + + MSIMessage msg_reject; + msg_reject.request.exists = true; + msg_reject.request.value = requ_reject; + + call_send_message ( call, &msg_reject ); return 0; } int msi_change_csettings( MSICall* call, uint8_t capabilities ) { - pthread_mutex_lock(session->mutex); - - LOGGER_DEBUG("Changing media on call: %d", call_index); - - MSICall *call = session->calls[call_index]; - - if ( call->state != msi_CallActive ) { - LOGGER_ERROR("Call is not active!"); - pthread_mutex_unlock(session->mutex); - return msi_ErrorInvalidState; - } - - MSICSettings *local = &call->csettings_local; - - if ( - local->call_type == csettings->call_type && - local->video_bitrate == csettings->video_bitrate && - local->max_video_width == csettings->max_video_width && - local->max_video_height == csettings->max_video_height && - local->audio_bitrate == csettings->audio_bitrate && - local->audio_frame_duration == csettings->audio_frame_duration && - local->audio_sample_rate == csettings->audio_sample_rate && - local->audio_channels == csettings->audio_channels ) { - LOGGER_ERROR("Call is already set accordingly!"); - pthread_mutex_unlock(session->mutex); - return -1; - } - - *local = *csettings; - - MSIMessage *msg_invite = msi_new_message ( type_request, requ_invite ); - - msi_msg_set_csettings ( msg_invite, local ); - send_message ( session, call, msg_invite, call->peers[0] ); - free ( msg_invite ); - - LOGGER_DEBUG("Request for media change sent"); - - pthread_mutex_unlock(session->mutex); - + call->self_capabilities = capabilities; + + MSIMessage msg_invite; + msg_invite.request.exists = true; + msg_invite.request.value = requ_invite; + + msg_invite.capabilities.exists = true; + msg_invite.capabilities.value = capabilities; + + call_send_message ( *call, &msg_invite ); + return 0; } \ No newline at end of file diff --git a/toxav/msi.h b/toxav/msi.h index 8fa309c30..a1eb499bd 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -1,6 +1,6 @@ /** msi.h * - * Copyright (C) 2013 Tox project All Rights Reserved. + * Copyright (C) 2013-2015 Tox project All Rights Reserved. * * This file is part of Tox. * @@ -28,17 +28,9 @@ #include "codec.h" #include "../toxcore/Messenger.h" -typedef uint8_t MSICallIDType[12]; -typedef uint8_t MSIReasonStrType[255]; -typedef void ( *MSICallbackType ) ( void *agent, int32_t call_idx); - -/** - * Call type identifier. Also used as rtp callback prefix. - */ -typedef enum { - msi_TypeAudio = 192, - msi_TypeVideo -} MSICallType; +/** Preconfigured values for video splitting */ +#define D_MVFSZ 40000 /* 256KiB */ +#define D_MVFPSZ 500 /* 1.25 KiB*/ /** * Error codes. @@ -62,31 +54,12 @@ typedef enum { * Call state identifiers. */ typedef enum { + msi_CallInactive, /* Default */ + msi_CallActive, msi_CallRequesting, /* when sending call invite */ msi_CallRequested, /* when getting call invite */ - msi_CallActive, - msi_CallHold, - msi_CallOver - } MSICallState; - -/** - * Encoding settings. - */ -typedef struct { - MSICallType call_type; - - uint32_t video_bitrate; /* In kbits/s */ - uint16_t max_video_width; /* In px */ - uint16_t max_video_height; /* In px */ - - uint32_t audio_bitrate; /* In bits/s */ - uint16_t audio_frame_duration; /* In ms */ - uint32_t audio_sample_rate; /* In Hz */ - uint32_t audio_channels; -} MSICSettings; - /** * Callbacks ids that handle the states */ @@ -97,38 +70,36 @@ typedef enum { msi_OnReject, /* The side that was invited rejected the call */ msi_OnEnd, /* Call that was active ended */ msi_OnError, /* Call that was active ended */ - msi_OnRequestTimeout, /* When the requested action didn't get response in specified time */ msi_OnPeerTimeout, /* Peer timed out; stop the call */ - msi_OnPeerCSChange, /* Peer requested Csettings change */ - msi_OnSelfCSChange /* Csettings change confirmation */ + msi_OnCapabilities, /* Peer requested capabilities change */ } MSICallbackID; -/** - * Errors - */ -typedef enum { - msi_ErrorNoCall = -20, /* Trying to perform call action while not in a call */ - msi_ErrorInvalidState = -21, /* Trying to perform call action while in invalid state*/ - msi_ErrorAlreadyInCallWithPeer = -22, /* Trying to call peer when already in a call with peer */ - msi_ErrorReachedCallLimit = -23, /* Cannot handle more calls */ -} MSIError; - /** * The call struct. Please do not modify outside msi.c */ typedef struct MSICall_s { - struct MSISession_s *session; /* Session pointer */ + struct MSISession_s *session; /* Session pointer */ MSICallState state; - uint8_t capabilities; /* Active capabilities */ - uint32_t friend_id; /* Index of this call in MSISession */ + uint8_t peer_capabilities; /* Peer capabilities */ + uint8_t self_capabilities; /* Self capabilities */ + + uint16_t peer_mvfsz; /* Max video frame size */ + uint16_t peer_mvfpsz; /* Max video frame part size */ + + uint32_t friend_id; /* Index of this call in MSISession */ struct MSICall_s* next; struct MSICall_s* prev; } MSICall; +/** + * Msi callback type. 'agent' is a pointer to ToxAv + */ +typedef void ( *MSICallbackType ) ( void *agent, MSICall* call); + /** * Control session struct. Please do not modify outside msi.c */ @@ -142,7 +113,7 @@ typedef struct MSISession_s { Messenger *messenger_handle; pthread_mutex_t mutex[1]; - MSICallbackType callbacks[10]; + MSICallbackType callbacks[8]; } MSISession; /** diff --git a/toxav/rtp.c b/toxav/rtp.c index a50dd7ce6..396e02025 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -1,6 +1,6 @@ /** rtp.c * - * Copyright (C) 2013 Tox project All Rights Reserved. + * Copyright (C) 2013-2015 Tox project All Rights Reserved. * * This file is part of Tox. * @@ -333,8 +333,6 @@ RTPMessage *msg_parse ( const uint8_t *data, int length ) return NULL; } - retu->next = NULL; - return retu; } @@ -415,8 +413,6 @@ RTPMessage *rtp_new_message ( RTPSession *session, const uint8_t *data, uint32_t retu->length = total_length; - retu->next = NULL; - return retu; } @@ -497,7 +493,7 @@ int rtp_send_msg ( RTPSession *session, const uint8_t *data, uint16_t length ) if ( -1 == send_custom_lossy_packet(session->m, session->dest, msg->data, msg->length) ) { LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno)); rtp_free_msg ( session, msg ); - return rtp_ErrorSending; + return -1; } diff --git a/toxav/rtp.h b/toxav/rtp.h index b1888f6c7..e3b38a8ea 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -1,6 +1,6 @@ /** rtp.h * - * Copyright (C) 2013 Tox project All Rights Reserved. + * Copyright (C) 2013-2015 Tox project All Rights Reserved. * * This file is part of Tox. * @@ -31,13 +31,18 @@ #define MAX_SEQU_NUM 65535 #define MAX_RTP_SIZE 65535 +/** + * Payload type identifier. Also used as rtp callback prefix. + */ typedef enum { - rtp_ErrorSending = -40 -} RTPError; + rtp_TypeAudio = 192, + rtp_TypeVideo +} RTPPayloadType; + /** * Standard rtp header */ -typedef struct _RTPHeader { +typedef struct { uint8_t flags; /* Version(2),Padding(1), Ext(1), Cc(4) */ uint8_t marker_payloadt; /* Marker(1), PlayLoad Type(7) */ uint16_t sequnum; /* Sequence Number */ @@ -51,7 +56,7 @@ typedef struct _RTPHeader { /** * Standard rtp extension header. */ -typedef struct _RTPExtHeader { +typedef struct { uint16_t type; /* Extension profile */ uint16_t length; /* Number of extensions */ uint32_t *table; /* Extension's table */ @@ -61,45 +66,43 @@ typedef struct _RTPExtHeader { /** * Standard rtp message. */ -typedef struct _RTPMessage { +typedef struct { RTPHeader *header; RTPExtHeader *ext_header; uint8_t data[MAX_RTP_SIZE]; uint32_t length; - - struct _RTPMessage *next; } RTPMessage; /** * RTP control session. */ -typedef struct _RTPSession { - uint8_t version; - uint8_t padding; - uint8_t extension; - uint8_t cc; - uint8_t marker; - uint8_t payload_type; - uint16_t sequnum; /* Set when sending */ - uint16_t rsequnum; /* Check when recving msg */ - uint32_t timestamp; - uint32_t ssrc; - uint32_t *csrc; +typedef struct { + uint8_t version; + uint8_t padding; + uint8_t extension; + uint8_t cc; + uint8_t marker; + uint8_t payload_type; + uint16_t sequnum; /* Set when sending */ + uint16_t rsequnum; /* Check when recving msg */ + uint32_t timestamp; + uint32_t ssrc; + uint32_t *csrc; /* If some additional data must be sent via message * apply it here. Only by allocating this member you will be * automatically placing it within a message. */ - RTPExtHeader *ext_header; + RTPExtHeader *ext_header; /* Msg prefix for core to know when recving */ - uint8_t prefix; + uint8_t prefix; - int dest; + int dest; - struct _CSSession *cs; - Messenger* m; + struct CSSession_s *cs; + Messenger *m; } RTPSession; @@ -119,7 +122,7 @@ void rtp_kill ( RTPSession* session ); int rtp_register_for_receiving (RTPSession *session); /** - * Sends msg to _RTPSession::dest + * Sends msg to RTPSession::dest */ int rtp_send_msg ( RTPSession* session, const uint8_t* data, uint16_t length ); diff --git a/toxav/toxav.c b/toxav/toxav.c index 72f995762..864d16cfd 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -1,6 +1,6 @@ /** toxav.c * - * Copyright (C) 2013 Tox project All Rights Reserved. + * Copyright (C) 2013-2015 Tox project All Rights Reserved. * * This file is part of Tox. * @@ -843,7 +843,7 @@ bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call) if (c_self->audio_bitrate > 0 || c_peer->audio_bitrate > 0) { /* Prepare audio rtp */ - call->rtps[audio_index] = rtp_new(msi_TypeAudio, av->m, av->msi->calls[call->call_idx]->peers[0]); + call->rtps[audio_index] = rtp_new(rtp_TypeAudio, av->m, av->msi->calls[call->call_idx]->peers[0]); if ( !call->rtps[audio_index] ) { LOGGER_ERROR("Error while starting audio RTP session!\n"); @@ -857,7 +857,7 @@ bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call) } if (c_self->video_bitrate > 0 || c_peer->video_bitrate > 0) { /* Prepare video rtp */ - call->rtps[video_index] = rtp_new(msi_TypeVideo, av->m, av->msi->calls[call->call_idx]->peers[0]); + call->rtps[video_index] = rtp_new(rtp_TypeVideo, av->m, av->msi->calls[call->call_idx]->peers[0]); if ( !call->rtps[video_index] ) { LOGGER_ERROR("Error while starting video RTP session!\n"); diff --git a/toxav/toxav.h b/toxav/toxav.h index 04ad3cd00..c1c6019c3 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -1,3 +1,24 @@ +/** toxav.h + * + * Copyright (C) 2013-2015 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + */ + #ifndef TOXAV_H #define TOXAV_H From 6e259d5fcb4666ee0959ddb0bb91deace32703d4 Mon Sep 17 00:00:00 2001 From: mannol Date: Wed, 18 Feb 2015 23:23:46 +0100 Subject: [PATCH 08/97] Msi should be done; toxav.c need a cleanup and adjustments --- toxav/msi.c | 703 +++++++++++++++++++++++++------------------------- toxav/msi.h | 18 +- toxav/toxav.c | 241 ++++++++--------- 3 files changed, 464 insertions(+), 498 deletions(-) diff --git a/toxav/msi.c b/toxav/msi.c index c13429503..cc855613e 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -33,9 +33,11 @@ #include #include #include +#include #define MSI_MAXMSG_SIZE 256 +/* TODO send error on any calloc or etc */ /** * Protocol: @@ -63,14 +65,13 @@ typedef enum { typedef enum { resp_ringing, resp_starting, - resp_error, } MSIResponse; #define GENERIC_HEADER(header, val_type) \ typedef struct { \ -val_type value; \ -bool exists; \ + val_type value; \ + bool exists; \ } MSIHeader##header @@ -92,7 +93,207 @@ typedef struct { } MSIMessage; -static int parse_input ( MSIMessage *dest, const uint8_t *data, uint16_t length ) +int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ); +uint8_t *msg_parse_header_out ( MSIHeaderID id, uint8_t *dest, const void *value, uint8_t value_len, uint16_t *length ); +int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg ); +int send_error ( Messenger* m, uint32_t friend_id, MSIError error ); +static void invoke_callback(MSICall* call, MSICallbackID cb); +static MSICall *get_call ( MSISession *session, uint32_t friend_id ); +MSICall *new_call ( MSISession *session, uint32_t friend_id ); +void kill_call ( MSICall *call ); +void on_peer_status(Messenger *m, int friend_id, uint8_t status, void *data); +int handle_recv_invite ( MSICall *call, const MSIMessage *msg ); +int handle_recv_start ( MSICall *call, const MSIMessage *msg ); +int handle_recv_reject ( MSICall *call, const MSIMessage *msg ); +int handle_recv_end ( MSICall *call, const MSIMessage *msg ); +int handle_recv_ringing ( MSICall *call, const MSIMessage *msg ); +int handle_recv_starting ( MSICall *call, const MSIMessage *msg ); +int handle_recv_error ( MSICall *call, const MSIMessage *msg ); +void handle_msi_packet ( Messenger *m, int friend_id, const uint8_t *data, uint16_t length, void *object ); + + +/** + * Public functions + */ +void msi_register_callback ( MSISession *session, MSICallbackType callback, MSICallbackID id) +{ + session->callbacks[id] = callback; +} +MSISession *msi_new ( Messenger *messenger ) +{ + if (messenger == NULL) { + LOGGER_ERROR("Could not init session on empty messenger!"); + return NULL; + } + + MSISession *retu = calloc ( sizeof ( MSISession ), 1 ); + + if (retu == NULL) { + LOGGER_ERROR("Allocation failed! Program might misbehave!"); + return NULL; + } + + if (create_recursive_mutex(retu->mutex) != 0) { + LOGGER_ERROR("Failed to init mutex! Program might misbehave"); + free(retu); + return NULL; + } + + retu->messenger = messenger; + + m_callback_msi_packet(messenger, handle_msi_packet, retu ); + + /* This is called when remote terminates session */ + m_callback_connectionstatus_internal_av(messenger, on_peer_status, retu); + + LOGGER_DEBUG("New msi session: %p ", retu); + return retu; +} +int msi_kill ( MSISession *session ) +{ + if (session == NULL) { + LOGGER_ERROR("Tried to terminate non-existing session"); + return -1; + } + + m_callback_msi_packet((struct Messenger *) session->messenger, NULL, NULL); + pthread_mutex_lock(session->mutex); + + if (session->calls) { + MSIMessage msg_end; + msg_end.request.exists = true; + msg_end.request.value = requ_end; + + MSICall* it = get_call(session, session->calls_head); + for (; it; it = it->next) { + send_message(session->messenger, it->friend_id, &msg_end); + kill_call(it); /* This will eventually free session->calls */ + } + } + + pthread_mutex_unlock(session->mutex); + pthread_mutex_destroy(session->mutex); + + LOGGER_DEBUG("Terminated session: %p", session); + free ( session ); + return 0; +} +int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_t capabilities ) +{ + LOGGER_DEBUG("Session: %p Inviting friend: %u", session, friend_id); + + if (get_call(session, friend_id) != NULL) { + LOGGER_ERROR("Already in a call"); + return -1; + } + + (*call) = new_call ( session, friend_id ); + + if ( *call == NULL ) + return -1; + + (*call)->self_capabilities = capabilities; + + MSIMessage msg_invite; + msg_invite.request.exists = true; + msg_invite.request.value = requ_invite; + + msg_invite.capabilities.exists = true; + msg_invite.capabilities.value = capabilities; + + msg_invite.mvfsz.exists = true; + msg_invite.mvfsz.value = htons(D_MVFSZ); + + msg_invite.mvfpsz.exists = true; + msg_invite.mvfpsz.value = htons(D_MVFPSZ); + + send_message ( (*call)->session->messenger, (*call)->friend_id, &msg_invite ); + + (*call)->state = msi_CallRequesting; + + LOGGER_DEBUG("Invite sent"); + return 0; +} +int msi_hangup ( MSICall* call ) +{ + LOGGER_DEBUG("Session: %p Hanging up call: %u", session, call_index); + + MSIMessage msg_end; + msg_end.request.exists = true; + msg_end.request.value = requ_end; + send_message ( call->session->messenger, call->friend_id, &msg_end ); + + kill_call(call); + return 0; +} +int msi_answer ( MSICall* call, uint8_t capabilities ) +{ + LOGGER_DEBUG("Session: %p Answering call from: %u", call->session, call->friend_id); + + if ( call->state != msi_CallRequested ) { + LOGGER_ERROR("Call is in invalid state!"); + return -1; + } + + call->self_capabilities = capabilities; + + MSIMessage msg_starting; + msg_starting.response.exists = true; + msg_starting.response.value = resp_starting; + + msg_starting.capabilities.exists = true; + msg_starting.capabilities.value = capabilities; + + msg_starting.mvfsz.exists = true; + msg_starting.mvfsz.value = htons(D_MVFSZ); + + msg_starting.mvfpsz.exists = true; + msg_starting.mvfpsz.value = htons(D_MVFPSZ); + + send_message ( call->session->messenger, call->friend_id, &msg_starting ); + + return 0; +} +int msi_reject ( MSICall* call ) +{ + LOGGER_DEBUG("Session: %p Rejecting call: %u; reason: %s", session, call_index, reason ? reason : "Unknown"); + + if ( call->state != msi_CallRequested ) { + LOGGER_ERROR("Call is in invalid state!"); + return -1; + } + + MSIMessage msg_reject; + msg_reject.request.exists = true; + msg_reject.request.value = requ_reject; + + send_message ( call->session->messenger, call->friend_id, &msg_reject ); + + return 0; +} +int msi_change_csettings( MSICall* call, uint8_t capabilities ) +{ + call->self_capabilities = capabilities; + + MSIMessage msg_invite; + msg_invite.request.exists = true; + msg_invite.request.value = requ_invite; + + msg_invite.capabilities.exists = true; + msg_invite.capabilities.value = capabilities; + + send_message ( call->session->messenger, call->friend_id, &msg_invite ); + + return 0; +} + + + +/** + * Private functions + */ + +int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) { /* Parse raw data received from socket into MSIMessage struct */ @@ -116,10 +317,7 @@ static int parse_input ( MSIMessage *dest, const uint8_t *data, uint16_t length } while(0) - if ( dest == NULL ) { - LOGGER_ERROR("Could not parse message: no storage!"); - return -1; - } + assert(dest); if ( length == 0 || data[length - 1] ) { /* End byte must have value 0 */ LOGGER_ERROR("Invalid end byte"); @@ -139,7 +337,7 @@ static int parse_input ( MSIMessage *dest, const uint8_t *data, uint16_t length case IDResponse: CHECK_SIZE(it, size_constraint, 1); - CHECK_ENUM_HIGH(it, resp_error); + CHECK_ENUM_HIGH(it, resp_starting); SET_UINT8(it, dest->response); it += 3; break; @@ -179,22 +377,13 @@ static int parse_input ( MSIMessage *dest, const uint8_t *data, uint16_t length #undef SET_UINT8 #undef SET_UINT16 } - -static uint8_t *parse_header ( MSIHeaderID id, uint8_t *dest, const void *value, - uint8_t value_len, uint16_t *length ) +uint8_t *msg_parse_header_out ( MSIHeaderID id, uint8_t *dest, const void *value, uint8_t value_len, uint16_t *length ) { /* Parse a single header for sending */ + assert(dest); + assert(value); + assert(value_len); - if ( dest == NULL ) { - LOGGER_ERROR("No destination space!"); - return NULL; - } - - if (value == NULL || value_len == 0) { - LOGGER_ERROR("Empty header value"); - return NULL; - } - *dest = id; dest ++; *dest = value_len; @@ -206,20 +395,10 @@ static uint8_t *parse_header ( MSIHeaderID id, uint8_t *dest, const void *value, return dest + value_len; /* Set to next position ready to be written */ } - - - -static void call_invoke_callback(MSICall* call, MSICallbackID cb) -{ - if ( call->session->callbacks[cb] ) { - LOGGER_DEBUG("Invoking callback function: %d", cb); - call->session->callbacks[cb] ( call->session->agent_handler, call ); - } -} - -static int call_send_message ( MSICall *call, const MSIMessage *msg ) +int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg ) { /* Parse and send message */ + assert(m); uint8_t parsed [MSI_MAXMSG_SIZE]; @@ -228,33 +407,33 @@ static int call_send_message ( MSICall *call, const MSIMessage *msg ) if (msg->request.exists) { uint8_t cast = msg->request.value; - it = parse_header(IDRequest, it, &cast, + it = msg_parse_header_out(IDRequest, it, &cast, sizeof(cast), &size); } if (msg->response.exists) { uint8_t cast = msg->response.value; - it = parse_header(IDResponse, it, &cast, + it = msg_parse_header_out(IDResponse, it, &cast, sizeof(cast), &size); } if (msg->error.exists) { - it = parse_header(IDError, it, &msg->error.value, + it = msg_parse_header_out(IDError, it, &msg->error.value, sizeof(msg->error.value), &size); } if (msg->capabilities.exists) { - it = parse_header(IDCapabilities, it, &msg->capabilities.value, + it = msg_parse_header_out(IDCapabilities, it, &msg->capabilities.value, sizeof(msg->capabilities.value), &size); } if (msg->mvfsz.exists) { - it = parse_header(IDMVFSZ, it, &msg->mvfsz.value, + it = msg_parse_header_out(IDMVFSZ, it, &msg->mvfsz.value, sizeof(msg->mvfsz.value), &size); } if (msg->mvfpsz.exists) { - it = parse_header(IDMVFPSZ, it, &msg->mvfpsz.value, + it = msg_parse_header_out(IDMVFPSZ, it, &msg->mvfpsz.value, sizeof(msg->mvfpsz.value), &size); } @@ -266,47 +445,50 @@ static int call_send_message ( MSICall *call, const MSIMessage *msg ) return -1; } - if ( m_msi_packet(call->session->messenger_handle, call->friend_id, parsed, size) ) { + if ( m_msi_packet(m, friend_id, parsed, size) ) { LOGGER_DEBUG("Sent message"); return 0; } return -1; } - -static int call_send_error ( MSICall *call, MSIError error ) +int send_error ( Messenger* m, uint32_t friend_id, MSIError error ) { /* Send error message */ + assert(m); - if (!call) { - LOGGER_WARNING("Cannot handle error on 'null' call"); - return -1; - } - - LOGGER_DEBUG("Sending error: %d to friend: %d", error, call->friend_id); + LOGGER_DEBUG("Sending error: %d to friend: %d", error, friend_id); MSIMessage msg_error; - msg_error.response.exists = true; - msg_error.response.value = resp_error; msg_error.error.exists = true; msg_error.error.value = error; - call_send_message ( call, &msg_error ); + send_message ( m, friend_id, &msg_error ); return 0; } - - +static void invoke_callback(MSICall* call, MSICallbackID cb) +{ + assert(call); + + if ( call->session->callbacks[cb] ) { + LOGGER_DEBUG("Invoking callback function: %d", cb); + call->session->callbacks[cb] ( call->session->av, call ); + } +} static MSICall *get_call ( MSISession *session, uint32_t friend_id ) { + assert(session); + if (session->calls == NULL || session->calls_tail < friend_id) return NULL; return session->calls[friend_id]; } - -static MSICall *new_call ( MSISession *session, uint32_t friend_id ) +MSICall *new_call ( MSISession *session, uint32_t friend_id ) { + assert(session); + MSICall *rc = calloc(sizeof(MSICall), 1); if (rc == NULL) @@ -353,8 +535,7 @@ static MSICall *new_call ( MSISession *session, uint32_t friend_id ) session->calls[friend_id] = rc; return rc; } - -static void kill_call ( MSICall *call ) +void kill_call ( MSICall *call ) { if ( call == NULL ) return; @@ -386,22 +567,21 @@ CLEAR: session->calls = NULL; free(call); } - - - -static void on_remote_connection_change(Messenger *messenger, int friend_id, uint8_t status, void *session_p) +void on_peer_status(Messenger *m, int friend_id, uint8_t status, void *data) { - (void)messenger; - MSISession *session = session_p; + (void)m; + MSISession *session = data; switch ( status ) { - case 0: { /* Went offline */ + case 0: { /* Friend is now offline */ + LOGGER_DEBUG("Friend %d is now offline", friend_id); + MSICall* call = get_call(session, friend_id); if (call == NULL) return; - call_invoke_callback(call, msi_OnPeerTimeout); + invoke_callback(call, msi_OnPeerTimeout); kill_call(call); } break; @@ -410,16 +590,9 @@ static void on_remote_connection_change(Messenger *messenger, int friend_id, uin break; } } - - - -/********** Request handlers **********/ -static int handle_recv_invite ( MSICall *call, const MSIMessage *msg ) +int handle_recv_invite ( MSICall *call, const MSIMessage *msg ) { - if ( call == NULL ) { - LOGGER_WARNING("Session: %p Handling 'invite' on no call"); - return -1; - } + assert(call); MSISession* session = call->session; @@ -427,19 +600,7 @@ static int handle_recv_invite ( MSICall *call, const MSIMessage *msg ) if (!msg->capabilities.exists) { LOGGER_WARNING("Session: %p Invalid capabilities on 'invite'"); - /* TODO send error */ - return -1; - } - - if (!msg->mvfsz.exists) { - LOGGER_WARNING("Session: %p Invalid mvfsz on 'invite'"); - /* TODO send error */ - return -1; - } - - if (!msg->mvfpsz.exists) { - LOGGER_WARNING("Session: %p Invalid mvfpsz on 'invite'"); - /* TODO send error */ + call->error = msi_InvalidMessage; return -1; } @@ -456,14 +617,32 @@ static int handle_recv_invite ( MSICall *call, const MSIMessage *msg ) LOGGER_DEBUG("Glare detected!"); - call->peer_capabilities = msg->capabilities; + if (!msg->mvfsz.exists) { + LOGGER_WARNING("Session: %p Invalid mvfsz on 'invite'"); + call->error = msi_InvalidMessage; + return -1; + } + + if (!msg->mvfpsz.exists) { + LOGGER_WARNING("Session: %p Invalid mvfpsz on 'invite'"); + call->error = msi_InvalidMessage; + return -1; + } + + call->peer_capabilities = msg->capabilities.value; call->peer_mvfsz = ntohs(msg->mvfsz.value); call->peer_mvfpsz = ntohs(msg->mvfpsz.value); + if (call->peer_mvfsz > call->peer_mvfpsz) { + LOGGER_WARNING("Session: %p mvfsz param greater than mvfpsz on 'invite'"); + call->error = msi_InvalidParam; + return -1; + } + /* Send response */ response.response.value = resp_starting; - call_send_message ( call, &response ); + send_message ( call->session->messenger, call->friend_id, &response ); return 0; } else if ( call->state == msi_CallActive ) { @@ -473,21 +652,33 @@ static int handle_recv_invite ( MSICall *call, const MSIMessage *msg ) */ LOGGER_DEBUG("Peer is changing capabilities"); - call->peer_capabilities = msg->capabilities; - - call->peer_mvfsz = ntohs(msg->mvfsz.value); - call->peer_mvfpsz = ntohs(msg->mvfpsz.value); - /* Send response */ response.response.value = resp_starting; - call_send_message ( call, &response ); + send_message ( call->session->messenger, call->friend_id, &response ); + if ( call->peer_capabilities != msg->capabilities.value) { + /* Only invoke callback if capabilities changed */ + call->peer_capabilities = msg->capabilities.value; + invoke_callback(call, msi_OnCapabilities); + } - call_invoke_callback(call, msi_OnCapabilities); return 0; } - call->peer_capabilities = msg->capabilities; + + if (!msg->mvfsz.exists) { + LOGGER_WARNING("Session: %p Invalid mvfsz on 'invite'"); + call->error = msi_InvalidMessage; + return -1; + } + + if (!msg->mvfpsz.exists) { + LOGGER_WARNING("Session: %p Invalid mvfpsz on 'invite'"); + call->error = msi_InvalidMessage; + return -1; + } + + call->peer_capabilities = msg->capabilities.value; call->peer_mvfsz = ntohs(msg->mvfsz.value); call->peer_mvfpsz = ntohs(msg->mvfpsz.value); @@ -496,23 +687,19 @@ static int handle_recv_invite ( MSICall *call, const MSIMessage *msg ) /* Send response */ response.response.value = resp_ringing; - call_send_message ( call, &response ); + send_message ( call->session->messenger, call->friend_id, &response ); - call_invoke_callback(call, msi_OnInvite); + invoke_callback(call, msi_OnInvite); return 0; } - -static int handle_recv_start ( MSICall *call, const MSIMessage *msg ) +int handle_recv_start ( MSICall *call, const MSIMessage *msg ) { - if ( call == NULL ) { - LOGGER_WARNING("Session: %p Handling 'start' on no call"); - return -1; - } + assert(call); if ( call->state != msi_CallRequested || call->state != msi_CallRequesting ) { LOGGER_WARNING("Session: %p Invalid call state on 'start'"); - /* TODO send error */ + call->error = msi_InvalidState; return -1; } @@ -521,105 +708,88 @@ static int handle_recv_start ( MSICall *call, const MSIMessage *msg ) LOGGER_DEBUG("Session: %p Handling 'start', friend id: %d", call->session, call->friend_id ); call->state = msi_CallActive; - call_invoke_callback(call, msi_OnStart); + invoke_callback(call, msi_OnStart); return 0; } - -static int handle_recv_reject ( MSICall *call, const MSIMessage *msg ) +int handle_recv_reject ( MSICall *call, const MSIMessage *msg ) { - if ( call == NULL ) { - LOGGER_WARNING("Session: %p Handling 'start' on no call"); - return -1; - } + assert(call); (void)msg; if ( call->state != msi_CallRequesting ) { LOGGER_WARNING("Session: %p Invalid call state on 'reject'"); - /* TODO send error */ + call->error = msi_InvalidState; return -1; } LOGGER_DEBUG("Session: %p Handling 'reject', friend id: %u", call->session, call->friend_id); - call_invoke_callback(call, msi_OnReject); + invoke_callback(call, msi_OnReject); kill_call(call); return 0; } - -static int handle_recv_end ( MSICall *call, const MSIMessage *msg ) +int handle_recv_end ( MSICall *call, const MSIMessage *msg ) { - (void)msg; + assert(call); - if ( call == NULL ) { - LOGGER_WARNING("Session: %p Handling 'start' on no call"); - return -1; - } + (void)msg; LOGGER_DEBUG("Session: %p Handling 'end', friend id: %d", call->session, call->friend_id); - call_invoke_callback(call, msi_OnEnd); + invoke_callback(call, msi_OnEnd); kill_call ( call ); return 0; } - -/********** Response handlers **********/ -static int handle_recv_ringing ( MSICall *call, const MSIMessage *msg ) +int handle_recv_ringing ( MSICall *call, const MSIMessage *msg ) { - if ( call == NULL ) { - LOGGER_WARNING("Session: %p Handling 'start' on no call"); - return -1; - } + assert(call); (void)msg; if ( call->state != msi_CallRequesting ) { LOGGER_WARNING("Session: %p Invalid call state on 'ringing'"); - /* TODO send error */ + call->error = msi_InvalidState; return -1; } LOGGER_DEBUG("Session: %p Handling 'ringing' friend id: %d", call->session, call->friend_id ); - call_invoke_callback(call, msi_OnRinging); + invoke_callback(call, msi_OnRinging); return 0; } - -static int handle_recv_starting ( MSICall *call, const MSIMessage *msg ) +int handle_recv_starting ( MSICall *call, const MSIMessage *msg ) { - if ( call == NULL ) { - LOGGER_WARNING("Session: %p Handling 'starting' on non-existing call"); - return 0; - } + assert(call); if ( call->state == msi_CallActive ) { LOGGER_DEBUG("Capabilities change confirmed"); return 0; } else if ( call->state != msi_CallRequested || call->state != msi_CallRequesting ) { LOGGER_WARNING("Session: %p Invalid call state on 'starting'"); - /* TODO send error */ + call->error = msi_InvalidState; return -1; } if (call->state == msi_CallRequesting) { if (!msg->capabilities.exists) { LOGGER_WARNING("Session: %p Invalid capabilities on 'starting'"); - /* TODO send error */ + call->error = msi_InvalidParam; return -1; } if (!msg->mvfsz.exists) { LOGGER_WARNING("Session: %p Invalid mvfsz on 'invite'"); - /* TODO send error */ + call->error = msi_InvalidParam; return -1; } if (!msg->mvfpsz.exists) { LOGGER_WARNING("Session: %p Invalid mvfpsz on 'invite'"); - /* TODO send error */ + call->error = msi_InvalidParam; return -1; } @@ -628,8 +798,15 @@ static int handle_recv_starting ( MSICall *call, const MSIMessage *msg ) call->peer_mvfsz = ntohs(msg->mvfsz.value); call->peer_mvfpsz = ntohs(msg->mvfpsz.value); + + if (call->peer_mvfsz > call->peer_mvfpsz) { + LOGGER_WARNING("Session: %p mvfsz param greater than mvfpsz on 'invite'"); + call->error = msi_InvalidParam; + return -1; + } + call->state = msi_CallActive; - call_invoke_callback(call, msi_OnStart); + invoke_callback(call, msi_OnStart); } /* Otherwise it's a glare case so don't start until 'start' is recved */ @@ -637,65 +814,55 @@ static int handle_recv_starting ( MSICall *call, const MSIMessage *msg ) MSIMessage msg_start; msg_start.request.exists = true; msg_start.request.value = requ_start; - call_send_message ( call, &msg_start ); + send_message ( call->session->messenger, call->friend_id, &msg_start ); return 0; } - -static int handle_recv_error ( MSICall *call, const MSIMessage *msg ) +int handle_recv_error ( MSICall *call, const MSIMessage *msg ) { - if ( call == NULL ) { - LOGGER_WARNING("Handling 'error' on non-existing call!"); - return -1; - } + assert(call); LOGGER_DEBUG("Session: %p Handling 'error' friend id: %d", call->session, call->friend_id ); + + call->error = msg->error.value; + invoke_callback(call, msi_OnError); - call_invoke_callback(call, msi_OnError); - - /* TODO Handle error accordingly */ - - return -1; + return 0; } - -static void handle_msi_packet ( Messenger *messenger, int friend_id, const uint8_t *data, - uint16_t length, void *object ) +void handle_msi_packet ( Messenger *m, int friend_id, const uint8_t *data, uint16_t length, void *object ) { LOGGER_DEBUG("Got msi message"); - /* Unused */ - (void)messenger; - MSISession *session = object; MSIMessage msg; + + int rc = 0; - if ( parse_input ( &msg, data, length ) == -1 ) { + if ( msg_parse_in ( &msg, data, length ) == -1 ) { LOGGER_WARNING("Error parsing message"); + send_error(m, friend_id, msi_InvalidMessage); return; } else { LOGGER_DEBUG("Successfully parsed message"); } - pthread_mutex_lock(session->mutex); - MSICall *call = get_call(session, friend_id); if (call == NULL) { - if (msg.request != requ_invite) { - /* TODO send error */ + if (msg.request.value != requ_invite) { + send_error(m, friend_id, msi_StrayMessage); return; } call = new_call(session, friend_id); if (call == NULL) { - /* TODO send error */ + send_error(m, friend_id, msi_SystemError); return; } } /* Now handle message */ - int rc = 0; if ( msg.request.exists ) { /* Handle request */ switch (msg.request.value) { case requ_invite: @@ -723,201 +890,21 @@ static void handle_msi_packet ( Messenger *messenger, int friend_id, const uint8 case resp_starting: rc = handle_recv_starting ( call, &msg ); break; - - case resp_error: - rc = handle_recv_error ( call, &msg ); - break; } + } else if (msg.error.exists) { + handle_recv_error ( call, &msg ); + rc = -1; } else { - LOGGER_WARNING("Invalid message: no resp nor requ headers"); - /* TODO send error */ + LOGGER_WARNING("Invalid message; none of the request, response or error!"); + call->error = msi_InvalidMessage; rc = -1; } - if (rc == -1) - kill_call(call); - pthread_mutex_unlock(session->mutex); -} - - - -/********** User functions **********/ -void msi_register_callback ( MSISession *session, MSICallbackType callback, MSICallbackID id) -{ - session->callbacks[id] = callback; -} - -MSISession *msi_new ( Messenger *messenger ) -{ - if (messenger == NULL) { - LOGGER_ERROR("Could not init session on empty messenger!"); - return NULL; - } - - MSISession *retu = calloc ( sizeof ( MSISession ), 1 ); - - if (retu == NULL) { - LOGGER_ERROR("Allocation failed! Program might misbehave!"); - return NULL; - } - - if (create_recursive_mutex(retu->mutex) != 0) { - LOGGER_ERROR("Failed to init mutex! Program might misbehave"); - free(retu); - return NULL; - } - - retu->messenger_handle = messenger; - - m_callback_msi_packet(messenger, handle_msi_packet, retu ); - - /* This is called when remote terminates session */ - m_callback_connectionstatus_internal_av(messenger, on_remote_connection_change, retu); - - LOGGER_DEBUG("New msi session: %p ", retu); - return retu; -} - -int msi_kill ( MSISession *session ) -{ - if (session == NULL) { - LOGGER_ERROR("Tried to terminate non-existing session"); - return -1; - } - - m_callback_msi_packet((struct Messenger *) session->messenger_handle, NULL, NULL); - pthread_mutex_lock(session->mutex); - - if (session->calls) { - MSIMessage msg_end; - msg_end.request.exists = true; - msg_end.request.value = requ_end; + if (rc == -1) { + if (call->error != msi_ErrorNone) + send_error(m, friend_id, call->error); - MSICall* it = get_call(session, session->calls_head); - for (; it; it = it->next) { - call_send_message(it, &msg_end); - kill_call(it); /* This will eventually free session->calls */ - } + kill_call(call); } - - pthread_mutex_unlock(session->mutex); - pthread_mutex_destroy(session->mutex); - - LOGGER_DEBUG("Terminated session: %p", session); - free ( session ); - return 0; } - -int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_t capabilities ) -{ - LOGGER_DEBUG("Session: %p Inviting friend: %u", session, friend_id); - - if (get_call(session, friend_id) != NULL) { - LOGGER_ERROR("Already in a call"); - return -1; - } - - *call = new_call ( session, friend_id ); - - if ( *call == NULL ) - return -1; - - *call->self_capabilities = capabilities; - - MSIMessage msg_invite; - msg_invite.request.exists = true; - msg_invite.request.value = requ_invite; - - msg_invite.capabilities.exists = true; - msg_invite.capabilities.value = capabilities; - - msg_invite.mvfsz.exists = true; - msg_invite.mvfsz.value = htons(D_MVFSZ); - - msg_invite.mvfpsz.exists = true; - msg_invite.mvfpsz.value = htons(D_MVFPSZ); - - call_send_message ( *call, &msg_invite ); - - *call->state = msi_CallRequesting; - - LOGGER_DEBUG("Invite sent"); - return 0; -} - -int msi_hangup ( MSICall* call ) -{ - LOGGER_DEBUG("Session: %p Hanging up call: %u", session, call_index); - - MSIMessage msg_end; - msg_end.request.exists = true; - msg_end.request.value = requ_end; - call_send_message ( call, &msg_end ); - - kill_call(call); - return 0; -} - -int msi_answer ( MSICall* call, uint8_t capabilities ) -{ - LOGGER_DEBUG("Session: %p Answering call from: %u", call->session, call->friend_id); - - if ( call->state != msi_CallRequested ) { - LOGGER_ERROR("Call is in invalid state!"); - return -1; - } - - call->self_capabilities = capabilities; - - MSIMessage msg_starting; - msg_starting.response.exists = true; - msg_starting.response.value = resp_starting; - - msg_starting.capabilities.exists = true; - msg_starting.capabilities.value = capabilities; - - msg_starting.mvfsz.exists = true; - msg_starting.mvfsz.value = htons(D_MVFSZ); - - msg_starting.mvfpsz.exists = true; - msg_starting.mvfpsz.value = htons(D_MVFPSZ); - - call_send_message ( call, &msg_starting ); - - return 0; -} - -int msi_reject ( MSICall* call ) -{ - LOGGER_DEBUG("Session: %p Rejecting call: %u; reason: %s", session, call_index, reason ? reason : "Unknown"); - - if ( call->state != msi_CallRequested ) { - LOGGER_ERROR("Call is in invalid state!"); - return -1; - } - - MSIMessage msg_reject; - msg_reject.request.exists = true; - msg_reject.request.value = requ_reject; - - call_send_message ( call, &msg_reject ); - - return 0; -} - -int msi_change_csettings( MSICall* call, uint8_t capabilities ) -{ - call->self_capabilities = capabilities; - - MSIMessage msg_invite; - msg_invite.request.exists = true; - msg_invite.request.value = requ_invite; - - msg_invite.capabilities.exists = true; - msg_invite.capabilities.value = capabilities; - - call_send_message ( *call, &msg_invite ); - - return 0; -} \ No newline at end of file diff --git a/toxav/msi.h b/toxav/msi.h index a1eb499bd..4f27b9f8b 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -36,6 +36,12 @@ * Error codes. */ typedef enum { + msi_ErrorNone, + msi_InvalidMessage, + msi_InvalidParam, + msi_InvalidState, + msi_StrayMessage, + msi_SystemError, msi_ErrUndisclosed, } MSIError; @@ -69,7 +75,7 @@ typedef enum { msi_OnStart, /* Call (RTP transmission) started */ msi_OnReject, /* The side that was invited rejected the call */ msi_OnEnd, /* Call that was active ended */ - msi_OnError, /* Call that was active ended */ + msi_OnError, /* On protocol error */ msi_OnPeerTimeout, /* Peer timed out; stop the call */ msi_OnCapabilities, /* Peer requested capabilities change */ } MSICallbackID; @@ -90,6 +96,10 @@ typedef struct MSICall_s { uint32_t friend_id; /* Index of this call in MSISession */ + MSIError error; /* Last error */ + + void* av_call; /* Pointer to av call handler */ + struct MSICall_s* next; struct MSICall_s* prev; } MSICall; @@ -109,8 +119,8 @@ typedef struct MSISession_s { uint32_t calls_tail; uint32_t calls_head; - void *agent_handler; - Messenger *messenger_handle; + void *av; + Messenger *messenger; pthread_mutex_t mutex[1]; MSICallbackType callbacks[8]; @@ -119,7 +129,7 @@ typedef struct MSISession_s { /** * Start the control session. */ -MSISession *msi_new ( Messenger *messenger, int32_t max_calls ); +MSISession *msi_new ( Messenger *messenger ); /** * Terminate control session. NOTE: all calls will be freed diff --git a/toxav/toxav.c b/toxav/toxav.c index 864d16cfd..12a65737f 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -49,8 +49,8 @@ typedef struct iToxAVCall RTPSession *rtps[2]; /** Audio is first and video is second */ CSSession *cs; bool active; - int32_t friend_number; - int32_t call_idx; /* FIXME msi compat, remove */ + int32_t friend_id; + MSICall* msi_call; struct iToxAVCall *prev; struct iToxAVCall *next; @@ -80,21 +80,20 @@ struct toxAV }; -void i_toxav_msi_callback_invite(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_ringing(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_start(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_reject(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_end(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_request_to(void* toxav_inst, int32_t call_idx, void *data); /* TODO remove */ -void i_toxav_msi_callback_peer_to(void* toxav_inst, int32_t call_idx, void *data); -void i_toxav_msi_callback_state_change(void* toxav_inst, int32_t call_idx, void *data); +void i_callback_invite(void* toxav_inst, MSICall* call); +void i_callback_ringing(void* toxav_inst, MSICall* call); +void i_callback_start(void* toxav_inst, MSICall* call); +void i_callback_end(void* toxav_inst, MSICall* call); +void i_callback_error(void* toxav_inst, MSICall* call); +void i_callback_capabilites(void* toxav_inst, MSICall* call); +TOXAV_CALL_STATE capabilities_to_state(uint8_t capabilities); IToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number); IToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number); void i_toxav_remove_call(ToxAV* av, uint32_t friend_number); +IToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); bool i_toxav_audio_bitrate_invalid(uint32_t bitrate); bool i_toxav_video_bitrate_invalid(uint32_t bitrate); -IToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error); bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call); void i_toxav_kill_transmission(ToxAV* av, uint32_t friend_number); @@ -124,7 +123,7 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) } av->m = (Messenger *)tox; - av->msi = msi_new(av->m, 100); /* TODO remove max calls */ + av->msi = msi_new(av->m); if (av->msi == NULL) { rc = TOXAV_ERR_NEW_MALLOC; @@ -132,17 +131,16 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) } av->interval = 200; - av->msi->agent_handler = av; + av->msi->av = av; - msi_register_callback(av->msi, i_toxav_msi_callback_invite, msi_OnInvite, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_ringing, msi_OnRinging, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_start, msi_OnStart, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_reject, msi_OnReject, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_end, msi_OnEnd, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_request_to, msi_OnRequestTimeout, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_peer_to, msi_OnPeerTimeout, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_state_change, msi_OnPeerCSChange, NULL); - msi_register_callback(av->msi, i_toxav_msi_callback_state_change, msi_OnSelfCSChange, NULL); + msi_register_callback(av->msi, i_callback_invite, msi_OnInvite); + msi_register_callback(av->msi, i_callback_ringing, msi_OnRinging); + msi_register_callback(av->msi, i_callback_start, msi_OnStart); + msi_register_callback(av->msi, i_callback_end, msi_OnReject); + msi_register_callback(av->msi, i_callback_end, msi_OnEnd); + msi_register_callback(av->msi, i_callback_error, msi_OnError); + msi_register_callback(av->msi, i_callback_error, msi_OnPeerTimeout); + msi_register_callback(av->msi, i_callback_capabilites, msi_OnCapabilities); if (error) @@ -209,7 +207,7 @@ void toxav_iteration(ToxAV* av) bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) { - IToxAVCall* call = i_toxav_init_call(av, friend_number, audio_bit_rate, video_bit_rate, error); + IToxAVCall* call = i_toxav_init_call(av, friend_number, error); if (call == NULL) { return false; } @@ -528,111 +526,89 @@ void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* * ******************************************************************************/ /** TODO: - * - In msi call_idx can be the same as friend id - * - If crutial callback not present send error - * - Remove *data from msi - * - Remove CSettings from msi + * - If crutial callback not present send error. + * - Error handling by return values from callbacks and setting 'error'. */ -void i_toxav_msi_callback_invite(void* toxav_inst, int32_t call_idx, void* data) +void i_callback_invite(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - uint32_t ab = toxav->msi->calls[call_idx]->csettings_peer[0].audio_bitrate; - uint32_t vb = toxav->msi->calls[call_idx]->csettings_peer[0].video_bitrate; - - IToxAVCall* call = i_toxav_init_call(toxav, toxav->msi->calls[call_idx]->peers[0], ab, vb, NULL); - if (call == NULL) { - LOGGER_WARNING("No call, rejecting..."); - msi_reject(toxav->msi, call_idx, NULL); - } - - call->call_idx = call_idx; - - if (toxav->ccb.first) - toxav->ccb.first(toxav, toxav->msi->calls[call_idx]->peers[0], true, true, toxav->ccb.second); -} - -void i_toxav_msi_callback_ringing(void* toxav_inst, int32_t call_idx, void* data) -{ - ToxAV* toxav = toxav_inst; - if (toxav->scb.first) - toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], - TOXAV_CALL_STATE_RINGING, toxav->scb.second); -} - -void i_toxav_msi_callback_start(void* toxav_inst, int32_t call_idx, void* data) -{ - ToxAV* toxav = toxav_inst; - - IToxAVCall* call = i_toxav_get_call(toxav, toxav->msi->calls[call_idx]->peers[0]); - - if (call == NULL || !i_toxav_prepare_transmission(toxav, call)) { - /* TODO send error */ - i_toxav_remove_call(toxav, toxav->msi->calls[call_idx]->peers[0]); + IToxAVCall* av_call = i_toxav_init_call(toxav, call->friend_id, NULL); + if (av_call == NULL) { + LOGGER_WARNING("Failed to start call, rejecting..."); + msi_reject(toxav->msi, call); return; } - TOXAV_CALL_STATE state; - const MSICSettings* csets = &toxav->msi->calls[call_idx]->csettings_peer[0]; + call->av_call = av_call; + av_call->msi_call = call; - if (csets->audio_bitrate && csets->video_bitrate) - state = TOXAV_CALL_STATE_SENDING_AV; - else if (csets->video_bitrate == 0) - state = TOXAV_CALL_STATE_SENDING_A; - else - state = TOXAV_CALL_STATE_SENDING_V; - - if (toxav->scb.first) /* TODO this */ - toxav->scb.first(toxav, call->friend_number, state, toxav->scb.second); + if (toxav->ccb.first) + toxav->ccb.first(toxav, call->friend_id, call->peer_capabilities & msi_CapSAudio, + call->peer_capabilities & msi_CapSVideo, toxav->ccb.second); } -void i_toxav_msi_callback_reject(void* toxav_inst, int32_t call_idx, void* data) -{ - ToxAV* toxav = toxav_inst; - - i_toxav_kill_transmission(toxav, toxav->msi->calls[call_idx]->peers[0]); - i_toxav_remove_call(toxav, toxav->msi->calls[call_idx]->peers[0]); - - if (toxav->scb.first) - toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], - TOXAV_CALL_STATE_END, toxav->scb.second); -} - -void i_toxav_msi_callback_end(void* toxav_inst, int32_t call_idx, void* data) -{ - ToxAV* toxav = toxav_inst; - - i_toxav_kill_transmission(toxav, toxav->msi->calls[call_idx]->peers[0]); - i_toxav_remove_call(toxav, toxav->msi->calls[call_idx]->peers[0]); - - if (toxav->scb.first) - toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], - TOXAV_CALL_STATE_END, toxav->scb.second); -} - -void i_toxav_msi_callback_request_to(void* toxav_inst, int32_t call_idx, void* data) -{ - /* TODO remove */ - ToxAV* toxav = toxav_inst; - if (toxav->scb.first) - toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], - TOXAV_CALL_STATE_ERROR, toxav->scb.second); -} - -void i_toxav_msi_callback_peer_to(void* toxav_inst, int32_t call_idx, void* data) +void i_callback_ringing(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; if (toxav->scb.first) - toxav->scb.first(toxav, toxav->msi->calls[call_idx]->peers[0], - TOXAV_CALL_STATE_ERROR, toxav->scb.second); + toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_RINGING, toxav->scb.second); } -void i_toxav_msi_callback_state_change(void* toxav_inst, int32_t call_idx, void* data) +void i_callback_start(void* toxav_inst, MSICall* call) +{ + ToxAV* toxav = toxav_inst; + + IToxAVCall* call = i_toxav_get_call(toxav, call->friend_id); + + if (call == NULL || !i_toxav_prepare_transmission(toxav, call)) { + /* TODO send error */ + i_toxav_remove_call(toxav, call->friend_id); + return; + } + + TOXAV_CALL_STATE state = capabilities_to_state(call->msi_call->peer_capabilities); + + if (toxav->scb.first) + toxav->scb.first(toxav, call->friend_id, state, toxav->scb.second); +} + +void i_callback_end(void* toxav_inst, MSICall* call) +{ + ToxAV* toxav = toxav_inst; + + i_toxav_kill_transmission(toxav, call->friend_id); + i_toxav_remove_call(toxav, call->friend_id); + + if (toxav->scb.first) + toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_END, toxav->scb.second); +} + +void i_callback_error(void* toxav_inst, MSICall* call) +{ + ToxAV* toxav = toxav_inst; + if (toxav->scb.first) + toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_ERROR, toxav->scb.second); +} + +void i_callback_capabilites(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; /* TODO something something msi */ } +TOXAV_CALL_STATE capabilities_to_state(uint8_t capabilities) +{ + if ((capabilities & msi_CapSAudio) && (capabilities & msi_CapSVideo)) + return TOXAV_CALL_STATE_SENDING_AV; + else if (capabilities & msi_CapSAudio) + return TOXAV_CALL_STATE_SENDING_A; + else if (capabilities & msi_CapSVideo) + return TOXAV_CALL_STATE_SENDING_V; + else + return TOXAV_CALL_STATE_PAUSED; +} + IToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number) { if (av->calls == NULL || av->calls_tail < friend_number) @@ -648,7 +624,7 @@ IToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number) if (rc == NULL) return NULL; - rc->friend_number = friend_number; + rc->friend_id = friend_number; if (create_recursive_mutex(rc->mutex_control) != 0) { free(rc); @@ -724,13 +700,13 @@ void i_toxav_remove_call(ToxAV* av, uint32_t friend_number) if (prev) prev->next = next; else if (next) - av->calls_head = next->friend_number; + av->calls_head = next->friend_id; else goto CLEAR; if (next) next->prev = prev; else if (prev) - av->calls_tail = prev->friend_number; + av->calls_tail = prev->friend_id; else goto CLEAR; av->calls[friend_number] = NULL; @@ -742,21 +718,7 @@ CLEAR: av->calls = NULL; } -bool i_toxav_audio_bitrate_invalid(uint32_t bitrate) -{ - /* Opus RFC 6716 section-2.1.1 dictates the following: - * Opus supports all bitrates from 6 kbit/s to 510 kbit/s. - */ - return bitrate < 6 || bitrate > 510; -} - -bool i_toxav_video_bitrate_invalid(uint32_t bitrate) -{ - /* TODO: If anyone knows the answer to this one please fill it up */ - return false; -} - -IToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) +IToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) { TOXAV_ERR_CALL rc = TOXAV_ERR_CALL_OK; IToxAVCall* call = NULL; @@ -776,25 +738,32 @@ IToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, uint32_t audio_ goto END; } - if ((audio_bit_rate && i_toxav_audio_bitrate_invalid(audio_bit_rate)) - ||(video_bit_rate && i_toxav_video_bitrate_invalid(video_bit_rate)) - ) { - rc = TOXAV_ERR_CALL_INVALID_BIT_RATE; - goto END; - } - call = i_toxav_add_call(av, friend_number); if (call == NULL) { rc = TOXAV_ERR_CALL_MALLOC; } -END: + END: if (error) *error = rc; return call; } +bool i_toxav_audio_bitrate_invalid(uint32_t bitrate) +{ + /* Opus RFC 6716 section-2.1.1 dictates the following: + * Opus supports all bitrates from 6 kbit/s to 510 kbit/s. + */ + return bitrate < 6 || bitrate > 510; +} + +bool i_toxav_video_bitrate_invalid(uint32_t bitrate) +{ + /* TODO: If anyone knows the answer to this one please fill it up */ + return false; +} + bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call) { pthread_mutex_lock(call->mutex_control); @@ -838,7 +807,7 @@ bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call) memcpy(&call->cs->acb, &av->acb, sizeof(av->acb)); memcpy(&call->cs->vcb, &av->vcb, sizeof(av->vcb)); - call->cs->friend_number = call->friend_number; + call->cs->friend_number = call->friend_id; call->cs->call_idx = call->call_idx; @@ -929,4 +898,4 @@ void i_toxav_kill_transmission(ToxAV* av, uint32_t friend_number) pthread_mutex_destroy(call->mutex_do); pthread_mutex_unlock(call->mutex_control); -} +} \ No newline at end of file From efe31ec92f476faffd6502714d05cce0a7dfadc7 Mon Sep 17 00:00:00 2001 From: mannol Date: Fri, 20 Feb 2015 00:23:38 +0100 Subject: [PATCH 09/97] Removed extra msi header and started testing --- toxav/codec.c | 57 ++++---------- toxav/codec.h | 10 +-- toxav/msi.c | 111 +++++++------------------- toxav/msi.h | 12 +-- toxav/toxav.c | 188 ++++++++++++++++++++++++-------------------- toxav/toxav_new_1.c | 1 - 6 files changed, 154 insertions(+), 225 deletions(-) diff --git a/toxav/codec.c b/toxav/codec.c index 9fc140711..e44387dfe 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -44,9 +44,7 @@ #define MAX_ENCODE_TIME_US VPX_DL_GOOD_QUALITY #define MAX_DECODE_TIME_US 0 -// TODO this has to be exchanged in msi #define MAX_VIDEOFRAME_SIZE 0x40000 /* 256KiB */ -#define VIDEOFRAME_PIECE_SIZE 0x500 /* 1.25 KiB*/ #define VIDEOFRAME_HEADER_SIZE 0x2 /* FIXME: Might not be enough */ @@ -296,7 +294,7 @@ void cs_do(CSSession *cs) LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); } else if (cs->acb.first) { /* Play */ - cs->acb.first(cs->agent, cs->friend_number, tmp, rc, + cs->acb.first(cs->agent, cs->friend_id, tmp, rc, cs->last_pack_channels, cs->last_packet_sampling_rate, cs->acb.second); } @@ -323,7 +321,7 @@ void cs_do(CSSession *cs) /* Play decoded images */ for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) { if (cs->vcb.first) - cs->vcb.first(cs->agent, cs->call_idx, dest->d_w, dest->d_h, + cs->vcb.first(cs->agent, cs->friend_id, dest->d_w, dest->d_h, (const uint8_t**)dest->planes, dest->stride, cs->vcb.second); vpx_img_free(dest); @@ -336,7 +334,7 @@ void cs_do(CSSession *cs) pthread_mutex_unlock(cs->queue_mutex); } -CSSession *cs_new(uint32_t s_audio_b, uint32_t p_audio_b, uint32_t s_video_b, uint32_t p_video_b) +CSSession *cs_new(uint32_t peer_video_frame_piece_size) { CSSession *cs = calloc(sizeof(CSSession), 1); @@ -345,29 +343,7 @@ CSSession *cs_new(uint32_t s_audio_b, uint32_t p_audio_b, uint32_t s_video_b, ui return NULL; } - /* TODO this has to be exchanged in msi */ - cs->max_video_frame_size = MAX_VIDEOFRAME_SIZE; - cs->video_frame_piece_size = VIDEOFRAME_PIECE_SIZE; - - if (s_audio_b > 0 && 0 != cs_enable_audio_sending(cs, s_audio_b, 2)) { /* Sending audio enabled */ - LOGGER_WARNING("Failed to enable audio sending!"); - goto FAILURE; - } - - if (p_audio_b > 0 && 0 != cs_enable_audio_receiving(cs)) { /* Receiving audio enabled */ - LOGGER_WARNING("Failed to enable audio receiving!"); - goto FAILURE; - } - - if (s_video_b > 0 && 0 != cs_enable_video_sending(cs, s_video_b)) { /* Sending video enabled */ - LOGGER_WARNING("Failed to enable video sending!"); - goto FAILURE; - } - - if (p_video_b > 0 && 0 != cs_enable_video_receiving(cs)) { /* Receiving video enabled */ - LOGGER_WARNING("Failed to enable video receiving!"); - goto FAILURE; - } + cs->peer_video_frame_piece_size = peer_video_frame_piece_size; return cs; @@ -415,22 +391,22 @@ int cs_update_video_splitter_cycle(CSSession *cs, const uint8_t *payload, uint16 cs->processing_video_frame = payload; cs->processing_video_frame_size = length; - return ((length - 1) / cs->video_frame_piece_size) + 1; + return ((length - 1) / VIDEOFRAME_PIECE_SIZE) + 1; } const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size) { if (!cs || !size) return NULL; - if (cs->processing_video_frame_size > cs->video_frame_piece_size) { + if (cs->processing_video_frame_size > VIDEOFRAME_PIECE_SIZE) { memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE, cs->processing_video_frame, - cs->video_frame_piece_size); + VIDEOFRAME_PIECE_SIZE); - cs->processing_video_frame += cs->video_frame_piece_size; - cs->processing_video_frame_size -= cs->video_frame_piece_size; + cs->processing_video_frame += VIDEOFRAME_PIECE_SIZE; + cs->processing_video_frame_size -= VIDEOFRAME_PIECE_SIZE; - *size = cs->video_frame_piece_size + VIDEOFRAME_HEADER_SIZE; + *size = VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE; } else { memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE, cs->processing_video_frame, @@ -492,7 +468,7 @@ int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate) if (cfg.rc_target_bitrate == bitrate) return 0; - LOGGER_DEBUG("New video bitrate: %u", video_bitrate); + LOGGER_DEBUG("New video bitrate: %u", bitrate); cfg.rc_target_bitrate = bitrate; int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg); @@ -528,7 +504,7 @@ int cs_enable_video_sending(CSSession* cs, uint32_t bitrate) /* So that we can use cs_disable_video_sending to clean up */ cs->v_encoding = true; - if ( !(cs->split_video_frame = calloc(cs->video_frame_piece_size + VIDEOFRAME_HEADER_SIZE, 1)) ) + if ( !(cs->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) ) goto FAILURE; cfg.rc_target_bitrate = bitrate; @@ -579,7 +555,7 @@ int cs_enable_video_receiving(CSSession* cs) /* So that we can use cs_disable_video_sending to clean up */ cs->v_decoding = true; - if ( !(cs->frame_buf = calloc(cs->max_video_frame_size, 1)) ) + if ( !(cs->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) ) goto FAILURE; if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) ) @@ -837,10 +813,10 @@ void queue_message(RTPSession *session, RTPMessage *msg) uint8_t piece_number = packet[1]; - uint32_t length_before_piece = ((piece_number - 1) * cs->video_frame_piece_size); + uint32_t length_before_piece = ((piece_number - 1) * cs->peer_video_frame_piece_size); uint32_t framebuf_new_length = length_before_piece + (packet_size - VIDEOFRAME_HEADER_SIZE); - if (framebuf_new_length > cs->max_video_frame_size) { + if (framebuf_new_length > MAX_VIDEOFRAME_SIZE) { goto end; } @@ -851,9 +827,8 @@ void queue_message(RTPSession *session, RTPMessage *msg) packet + VIDEOFRAME_HEADER_SIZE, packet_size - VIDEOFRAME_HEADER_SIZE); - if (framebuf_new_length > cs->frame_size) { + if (framebuf_new_length > cs->frame_size) cs->frame_size = framebuf_new_length; - } end: rtp_free_msg(NULL, msg); diff --git a/toxav/codec.h b/toxav/codec.h index b5eb19e21..6a6739906 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -86,13 +86,12 @@ typedef struct CSSession_s { /* Data handling */ uint8_t *frame_buf; /* buffer for split video payloads */ - uint32_t frame_size; /* largest address written to in frame_buf for current input frame*/ + uint32_t frame_size; /* largest address written to in frame_buf for current input frame */ uint8_t frameid_in, frameid_out; /* id of input and output video frame */ uint32_t last_timestamp; /* calculating cycles */ /* Limits */ - uint32_t video_frame_piece_size; - uint32_t max_video_frame_size; + uint32_t peer_video_frame_piece_size; /* Splitting */ uint8_t *split_video_frame; @@ -129,8 +128,7 @@ typedef struct CSSession_s { * */ void *agent; /* Pointer to ToxAV TODO make this pointer to ToxAV*/ - int32_t call_idx; - int32_t friend_number; + int32_t friend_id; PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ @@ -145,7 +143,7 @@ typedef struct CSSession_s { void cs_do(CSSession *cs); /* Make sure to be called BEFORE corresponding rtp_new */ -CSSession *cs_new(uint32_t s_audio_b, uint32_t p_audio_b, uint32_t s_video_b, uint32_t p_video_b); +CSSession *cs_new(uint32_t peer_mvfpsz); /* Make sure to be called AFTER corresponding rtp_kill */ void cs_kill(CSSession *cs); diff --git a/toxav/msi.c b/toxav/msi.c index cc855613e..16476364c 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -50,8 +50,7 @@ typedef enum { IDResponse, IDError, IDCapabilities, - IDMVFSZ, - IDMVFPSZ, + IDVFPSZ, } MSIHeaderID; @@ -79,8 +78,7 @@ GENERIC_HEADER ( Request, MSIRequest ); GENERIC_HEADER ( Response, MSIResponse ); GENERIC_HEADER ( Error, MSIError ); GENERIC_HEADER ( Capabilities, uint8_t ); -GENERIC_HEADER ( MVFSZ, uint16_t ); -GENERIC_HEADER ( MVFPSZ, uint16_t ); +GENERIC_HEADER ( VFPSZ, uint16_t ); typedef struct { @@ -88,8 +86,7 @@ typedef struct { MSIHeaderResponse response; MSIHeaderError error; MSIHeaderCapabilities capabilities; - MSIHeaderMVFSZ mvfsz; /* Max video frame size. NOTE: Value must be in network b-order */ - MSIHeaderMVFPSZ mvfpsz; /* Max video frame piece size. NOTE: Value must be in network b-order */ + MSIHeaderVFPSZ vfpsz; /* Video frame piece size. NOTE: Value must be in network b-order */ } MSIMessage; @@ -200,12 +197,9 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ msg_invite.capabilities.exists = true; msg_invite.capabilities.value = capabilities; - - msg_invite.mvfsz.exists = true; - msg_invite.mvfsz.value = htons(D_MVFSZ); - - msg_invite.mvfpsz.exists = true; - msg_invite.mvfpsz.value = htons(D_MVFPSZ); + + msg_invite.vfpsz.exists = true; + msg_invite.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE); send_message ( (*call)->session->messenger, (*call)->friend_id, &msg_invite ); @@ -216,7 +210,7 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ } int msi_hangup ( MSICall* call ) { - LOGGER_DEBUG("Session: %p Hanging up call: %u", session, call_index); + LOGGER_DEBUG("Session: %p Hanging up call with friend: %u", call->session, call->friend_id); MSIMessage msg_end; msg_end.request.exists = true; @@ -244,11 +238,8 @@ int msi_answer ( MSICall* call, uint8_t capabilities ) msg_starting.capabilities.exists = true; msg_starting.capabilities.value = capabilities; - msg_starting.mvfsz.exists = true; - msg_starting.mvfsz.value = htons(D_MVFSZ); - - msg_starting.mvfpsz.exists = true; - msg_starting.mvfpsz.value = htons(D_MVFPSZ); + msg_starting.vfpsz.exists = true; + msg_starting.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE); send_message ( call->session->messenger, call->friend_id, &msg_starting ); @@ -256,7 +247,7 @@ int msi_answer ( MSICall* call, uint8_t capabilities ) } int msi_reject ( MSICall* call ) { - LOGGER_DEBUG("Session: %p Rejecting call: %u; reason: %s", session, call_index, reason ? reason : "Unknown"); + LOGGER_DEBUG("Session: %p Rejecting call with friend: %u", call->session, call->friend_id); if ( call->state != msi_CallRequested ) { LOGGER_ERROR("Call is in invalid state!"); @@ -353,14 +344,9 @@ int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) SET_UINT8(it, dest->capabilities); break; - case IDMVFSZ: + case IDVFPSZ: CHECK_SIZE(it, size_constraint, 2); - SET_UINT16(it, dest->mvfsz); - break; - - case IDMVFPSZ: - CHECK_SIZE(it, size_constraint, 2); - SET_UINT16(it, dest->mvfpsz); + SET_UINT16(it, dest->vfpsz); break; default: @@ -408,33 +394,28 @@ int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg ) if (msg->request.exists) { uint8_t cast = msg->request.value; it = msg_parse_header_out(IDRequest, it, &cast, - sizeof(cast), &size); + sizeof(cast), &size); } if (msg->response.exists) { uint8_t cast = msg->response.value; it = msg_parse_header_out(IDResponse, it, &cast, - sizeof(cast), &size); + sizeof(cast), &size); } if (msg->error.exists) { it = msg_parse_header_out(IDError, it, &msg->error.value, - sizeof(msg->error.value), &size); + sizeof(msg->error.value), &size); } if (msg->capabilities.exists) { it = msg_parse_header_out(IDCapabilities, it, &msg->capabilities.value, - sizeof(msg->capabilities.value), &size); + sizeof(msg->capabilities.value), &size); } - if (msg->mvfsz.exists) { - it = msg_parse_header_out(IDMVFSZ, it, &msg->mvfsz.value, - sizeof(msg->mvfsz.value), &size); - } - - if (msg->mvfpsz.exists) { - it = msg_parse_header_out(IDMVFPSZ, it, &msg->mvfpsz.value, - sizeof(msg->mvfpsz.value), &size); + if (msg->vfpsz.exists) { + it = msg_parse_header_out(IDVFPSZ, it, &msg->vfpsz.value, + sizeof(msg->vfpsz.value), &size); } *it = 0; @@ -494,6 +475,7 @@ MSICall *new_call ( MSISession *session, uint32_t friend_id ) if (rc == NULL) return NULL; + rc->session = session; rc->friend_id = friend_id; if (session->calls == NULL) { /* Creating */ @@ -617,28 +599,14 @@ int handle_recv_invite ( MSICall *call, const MSIMessage *msg ) LOGGER_DEBUG("Glare detected!"); - if (!msg->mvfsz.exists) { - LOGGER_WARNING("Session: %p Invalid mvfsz on 'invite'"); - call->error = msi_InvalidMessage; - return -1; - } - - if (!msg->mvfpsz.exists) { + if (!msg->vfpsz.exists) { LOGGER_WARNING("Session: %p Invalid mvfpsz on 'invite'"); call->error = msi_InvalidMessage; return -1; } call->peer_capabilities = msg->capabilities.value; - - call->peer_mvfsz = ntohs(msg->mvfsz.value); - call->peer_mvfpsz = ntohs(msg->mvfpsz.value); - - if (call->peer_mvfsz > call->peer_mvfpsz) { - LOGGER_WARNING("Session: %p mvfsz param greater than mvfpsz on 'invite'"); - call->error = msi_InvalidParam; - return -1; - } + call->peer_vfpsz = ntohs(msg->vfpsz.value); /* Send response */ response.response.value = resp_starting; @@ -665,24 +633,14 @@ int handle_recv_invite ( MSICall *call, const MSIMessage *msg ) return 0; } - - if (!msg->mvfsz.exists) { - LOGGER_WARNING("Session: %p Invalid mvfsz on 'invite'"); - call->error = msi_InvalidMessage; - return -1; - } - - if (!msg->mvfpsz.exists) { + if (!msg->vfpsz.exists) { LOGGER_WARNING("Session: %p Invalid mvfpsz on 'invite'"); call->error = msi_InvalidMessage; return -1; } call->peer_capabilities = msg->capabilities.value; - - call->peer_mvfsz = ntohs(msg->mvfsz.value); - call->peer_mvfpsz = ntohs(msg->mvfpsz.value); - + call->peer_vfpsz = ntohs(msg->vfpsz.value); call->state = msi_CallRequested; /* Send response */ @@ -781,31 +739,16 @@ int handle_recv_starting ( MSICall *call, const MSIMessage *msg ) return -1; } - if (!msg->mvfsz.exists) { - LOGGER_WARNING("Session: %p Invalid mvfsz on 'invite'"); - call->error = msi_InvalidParam; - return -1; - } - - if (!msg->mvfpsz.exists) { + if (!msg->vfpsz.exists) { LOGGER_WARNING("Session: %p Invalid mvfpsz on 'invite'"); call->error = msi_InvalidParam; return -1; } call->peer_capabilities = msg->capabilities.value; - - call->peer_mvfsz = ntohs(msg->mvfsz.value); - call->peer_mvfpsz = ntohs(msg->mvfpsz.value); - - - if (call->peer_mvfsz > call->peer_mvfpsz) { - LOGGER_WARNING("Session: %p mvfsz param greater than mvfpsz on 'invite'"); - call->error = msi_InvalidParam; - return -1; - } - + call->peer_vfpsz = ntohs(msg->vfpsz.value); call->state = msi_CallActive; + invoke_callback(call, msi_OnStart); } /* Otherwise it's a glare case so don't start until 'start' is recved */ diff --git a/toxav/msi.h b/toxav/msi.h index 4f27b9f8b..a55d8567a 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -28,9 +28,8 @@ #include "codec.h" #include "../toxcore/Messenger.h" -/** Preconfigured values for video splitting */ -#define D_MVFSZ 40000 /* 256KiB */ -#define D_MVFPSZ 500 /* 1.25 KiB*/ +/** Preconfigured value for video splitting */ +#define VIDEOFRAME_PIECE_SIZE 500 /* 1.25 KiB*/ /** * Error codes. @@ -87,15 +86,10 @@ typedef struct MSICall_s { struct MSISession_s *session; /* Session pointer */ MSICallState state; - uint8_t peer_capabilities; /* Peer capabilities */ uint8_t self_capabilities; /* Self capabilities */ - - uint16_t peer_mvfsz; /* Max video frame size */ - uint16_t peer_mvfpsz; /* Max video frame part size */ - + uint16_t peer_vfpsz; /* Video frame piece size */ uint32_t friend_id; /* Index of this call in MSISession */ - MSIError error; /* Last error */ void* av_call; /* Pointer to av call handler */ diff --git a/toxav/toxav.c b/toxav/toxav.c index 12a65737f..584b38982 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -40,21 +40,24 @@ enum { video_index, }; -typedef struct iToxAVCall +typedef struct ToxAVCall_s { pthread_mutex_t mutex_control[1]; pthread_mutex_t mutex_encoding_audio[1]; pthread_mutex_t mutex_encoding_video[1]; pthread_mutex_t mutex_do[1]; - RTPSession *rtps[2]; /** Audio is first and video is second */ + RTPSession *rtps[2]; /* Audio is first and video is second */ CSSession *cs; bool active; - int32_t friend_id; MSICall* msi_call; + uint32_t friend_id; - struct iToxAVCall *prev; - struct iToxAVCall *next; -} IToxAVCall; + uint32_t s_audio_b; /* Sending audio bitrate */ + uint32_t s_video_b; /* Sending video bitrate */ + + struct ToxAVCall_s *prev; + struct ToxAVCall_s *next; +} ToxAVCall; struct toxAV { @@ -62,7 +65,7 @@ struct toxAV MSISession* msi; /* Two-way storage: first is array of calls and second is list of calls with head and tail */ - IToxAVCall** calls; + ToxAVCall** calls; uint32_t calls_tail; uint32_t calls_head; @@ -88,13 +91,13 @@ void i_callback_error(void* toxav_inst, MSICall* call); void i_callback_capabilites(void* toxav_inst, MSICall* call); TOXAV_CALL_STATE capabilities_to_state(uint8_t capabilities); -IToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number); -IToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number); +ToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number); +ToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number); void i_toxav_remove_call(ToxAV* av, uint32_t friend_number); -IToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); +ToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); bool i_toxav_audio_bitrate_invalid(uint32_t bitrate); bool i_toxav_video_bitrate_invalid(uint32_t bitrate); -bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call); +bool i_toxav_prepare_transmission(ToxAV* av, ToxAVCall* call); void i_toxav_kill_transmission(ToxAV* av, uint32_t friend_number); @@ -179,15 +182,13 @@ uint32_t toxav_iteration_interval(const ToxAV* av) void toxav_iteration(ToxAV* av) { - msi_do(av->msi); - if (av->calls == NULL) return; uint64_t start = current_time_monotonic(); uint32_t rc = 200 + av->dmssa; /* If no call is active interval is 200 */ - IToxAVCall* i = av->calls[av->calls_head]; + ToxAVCall* i = av->calls[av->calls_head]; for (; i; i = i->next) { if (i->active) { cs_do(i->cs); @@ -207,22 +208,22 @@ void toxav_iteration(ToxAV* av) bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) { - IToxAVCall* call = i_toxav_init_call(av, friend_number, error); - if (call == NULL) { + ToxAVCall* call = i_toxav_init_call(av, friend_number, error); + if (call == NULL) return false; - } - /* TODO remove csettings */ - MSICSettings csets; - csets.audio_bitrate = audio_bit_rate; - csets.video_bitrate = video_bit_rate; + call->s_audio_b = audio_bit_rate; + call->s_video_b = video_bit_rate; - csets.call_type = video_bit_rate ? msi_TypeVideo : msi_TypeAudio; + uint8_t capabilities = 0; - if (msi_invite(av->msi, &call->call_idx, &csets, 1000, friend_number) != 0) { + capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; + capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; + + if (msi_invite(av->msi, &call->msi_call, friend_number, capabilities) != 0) { i_toxav_remove_call(av, friend_number); if (error) - *error = TOXAV_ERR_CALL_MALLOC; /* FIXME: this should be the only reason to fail */ + *error = TOXAV_ERR_CALL_MALLOC; return false; } @@ -250,23 +251,23 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui goto END; } - IToxAVCall* call = i_toxav_get_call(av, friend_number); - if (call == NULL || av->msi->calls[call->call_idx]->state != msi_CallRequested) { + ToxAVCall* call = i_toxav_get_call(av, friend_number); + if (call == NULL) { rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; goto END; } - /* TODO remove csettings */ - MSICSettings csets; - csets.audio_bitrate = audio_bit_rate; - csets.video_bitrate = video_bit_rate; + call->s_audio_b = audio_bit_rate; + call->s_video_b = video_bit_rate; - csets.call_type = video_bit_rate ? msi_TypeVideo : msi_TypeAudio; + uint8_t capabilities = 0; + + capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; + capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; + + if (msi_answer(call->msi_call, capabilities) != 0) + rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; /* the only reason for msi_answer to fail */ - if (msi_answer(av->msi, call->call_idx, &csets) != 0) { - rc = TOXAV_ERR_ANSWER_MALLOC; /* TODO Some error here */ - /* TODO Reject call? */ - } END: if (error) @@ -291,7 +292,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co } - IToxAVCall* call = i_toxav_get_call(av, friend_number); + ToxAVCall* call = i_toxav_get_call(av, friend_number); if (call == NULL) { rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; goto END; @@ -309,17 +310,17 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co } break; case TOXAV_CALL_CONTROL_CANCEL: { - if (av->msi->calls[call->call_idx]->state == msi_CallActive) { + if (call->msi_call->state == msi_CallActive + || call->msi_call->state == msi_CallRequesting) { /* Hang up */ - msi_hangup(av->msi, call->call_idx); - } else if (av->msi->calls[call->call_idx]->state == msi_CallRequested) { - /* Reject the call */ - msi_reject(av->msi, call->call_idx, NULL); - } else if (av->msi->calls[call->call_idx]->state == msi_CallRequesting) { - /* Cancel the call */ - msi_cancel(av->msi, call->call_idx, 0, NULL); - i_toxav_remove_call(av, friend_number); + msi_hangup(call->msi_call); + } else if (call->msi_call->state == msi_CallRequested) { + /* Reject the call */ + msi_reject(call->msi_call); } + + // No mather the case, terminate the call + i_toxav_remove_call(av, friend_number); } break; case TOXAV_CALL_CONTROL_MUTE_AUDIO: { @@ -356,7 +357,7 @@ void toxav_callback_request_video_frame(ToxAV* av, toxav_request_video_frame_cb* bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) { TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; - IToxAVCall* call; + ToxAVCall* call; if (m_friend_exists(av->m, friend_number) == 0) { rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND; @@ -369,7 +370,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u goto END; } - if (av->msi->calls[call->call_idx]->state != msi_CallActive) { + if (call->msi_call->state != msi_CallActive) { /* TODO */ rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; goto END; @@ -453,7 +454,7 @@ void toxav_callback_request_audio_frame(ToxAV* av, toxav_request_audio_frame_cb* bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error) { TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; - IToxAVCall* call; + ToxAVCall* call; if (m_friend_exists(av->m, friend_number) == 0) { rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND; @@ -466,7 +467,7 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } - if (av->msi->calls[call->call_idx]->state != msi_CallActive) { + if (call->msi_call->state != msi_CallActive) { /* TODO */ rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; goto END; @@ -533,10 +534,10 @@ void i_callback_invite(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - IToxAVCall* av_call = i_toxav_init_call(toxav, call->friend_id, NULL); + ToxAVCall* av_call = i_toxav_init_call(toxav, call->friend_id, NULL); if (av_call == NULL) { LOGGER_WARNING("Failed to start call, rejecting..."); - msi_reject(toxav->msi, call); + msi_reject(call); return; } @@ -559,15 +560,15 @@ void i_callback_start(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - IToxAVCall* call = i_toxav_get_call(toxav, call->friend_id); + ToxAVCall* av_call = i_toxav_get_call(toxav, call->friend_id); - if (call == NULL || !i_toxav_prepare_transmission(toxav, call)) { + if (av_call == NULL || !i_toxav_prepare_transmission(toxav, av_call)) { /* TODO send error */ i_toxav_remove_call(toxav, call->friend_id); return; } - TOXAV_CALL_STATE state = capabilities_to_state(call->msi_call->peer_capabilities); + TOXAV_CALL_STATE state = capabilities_to_state(av_call->msi_call->peer_capabilities); if (toxav->scb.first) toxav->scb.first(toxav, call->friend_id, state, toxav->scb.second); @@ -594,7 +595,7 @@ void i_callback_error(void* toxav_inst, MSICall* call) void i_callback_capabilites(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - /* TODO something something msi */ + /* TODO handle this */ } TOXAV_CALL_STATE capabilities_to_state(uint8_t capabilities) @@ -609,7 +610,7 @@ TOXAV_CALL_STATE capabilities_to_state(uint8_t capabilities) return TOXAV_CALL_STATE_PAUSED; } -IToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number) +ToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number) { if (av->calls == NULL || av->calls_tail < friend_number) return NULL; @@ -617,9 +618,9 @@ IToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number) return av->calls[friend_number]; } -IToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number) +ToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number) { - IToxAVCall* rc = calloc(sizeof(IToxAVCall), 1); + ToxAVCall* rc = calloc(sizeof(ToxAVCall), 1); if (rc == NULL) return NULL; @@ -639,7 +640,7 @@ IToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number) if (av->calls == NULL) { /* Creating */ - av->calls = calloc (sizeof(IToxAVCall*), friend_number + 1); + av->calls = calloc (sizeof(ToxAVCall*), friend_number + 1); if (av->calls == NULL) { pthread_mutex_destroy(rc->mutex_control); @@ -651,7 +652,7 @@ IToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number) av->calls_tail = av->calls_head = friend_number; } else if (av->calls_tail < friend_number) { /* Appending */ - void* tmp = realloc(av->calls, sizeof(IToxAVCall*) * friend_number + 1); + void* tmp = realloc(av->calls, sizeof(ToxAVCall*) * friend_number + 1); if (tmp == NULL) { pthread_mutex_destroy(rc->mutex_control); @@ -684,13 +685,13 @@ IToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number) void i_toxav_remove_call(ToxAV* av, uint32_t friend_number) { - IToxAVCall* tc = i_toxav_get_call(av, friend_number); + ToxAVCall* tc = i_toxav_get_call(av, friend_number); if (tc == NULL) return; - IToxAVCall* prev = tc->prev; - IToxAVCall* next = tc->next; + ToxAVCall* prev = tc->prev; + ToxAVCall* next = tc->next; pthread_mutex_destroy(tc->mutex_control); pthread_mutex_destroy(tc->mutex_do); @@ -718,10 +719,10 @@ CLEAR: av->calls = NULL; } -IToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) +ToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) { TOXAV_ERR_CALL rc = TOXAV_ERR_CALL_OK; - IToxAVCall* call = NULL; + ToxAVCall* call = NULL; if (m_friend_exists(av->m, friend_number) == 0) { rc = TOXAV_ERR_CALL_FRIEND_NOT_FOUND; @@ -764,8 +765,12 @@ bool i_toxav_video_bitrate_invalid(uint32_t bitrate) return false; } -bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call) +bool i_toxav_prepare_transmission(ToxAV* av, ToxAVCall* call) { + if (!av->acb.first && !av->vcb.first) + /* It makes no sense to have CSession without callbacks */ + return false; + pthread_mutex_lock(call->mutex_control); if (call->active) { @@ -788,11 +793,9 @@ bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call) goto MUTEX_INIT_ERROR; } - const MSICSettings *c_peer = &av->msi->calls[call->call_idx]->csettings_peer[0]; - const MSICSettings *c_self = &av->msi->calls[call->call_idx]->csettings_local; + uint8_t capabilities = call->msi_call->self_capabilities; - call->cs = cs_new(c_self->audio_bitrate, c_peer->audio_bitrate, - c_self->video_bitrate, c_peer->video_bitrate); + call->cs = cs_new(call->msi_call->peer_vfpsz); if ( !call->cs ) { LOGGER_ERROR("Error while starting Codec State!\n"); @@ -800,19 +803,14 @@ bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call) } call->cs->agent = av; - - /* It makes no sense to have CSession without callbacks */ - assert(av->acb.first || av->vcb.first); + call->cs->friend_id = call->friend_id; memcpy(&call->cs->acb, &av->acb, sizeof(av->acb)); memcpy(&call->cs->vcb, &av->vcb, sizeof(av->vcb)); - call->cs->friend_number = call->friend_id; - call->cs->call_idx = call->call_idx; - - - if (c_self->audio_bitrate > 0 || c_peer->audio_bitrate > 0) { /* Prepare audio rtp */ - call->rtps[audio_index] = rtp_new(rtp_TypeAudio, av->m, av->msi->calls[call->call_idx]->peers[0]); + if (capabilities & msi_CapSAudio || capabilities & msi_CapRAudio) { /* Prepare audio sending */ + + call->rtps[audio_index] = rtp_new(rtp_TypeAudio, av->m, call->friend_id); if ( !call->rtps[audio_index] ) { LOGGER_ERROR("Error while starting audio RTP session!\n"); @@ -821,12 +819,23 @@ bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call) call->rtps[audio_index]->cs = call->cs; - if (c_peer->audio_bitrate > 0) + if (cs_enable_audio_sending(call->cs, call->s_audio_b, 2) != 0) { + LOGGER_WARNING("Failed to enable audio sending!"); + goto FAILURE; + } + + if (capabilities & msi_CapRAudio) { + if (cs_enable_audio_receiving(call->cs) != 0) { + LOGGER_WARNING("Failed to enable audio receiving!"); + goto FAILURE; + } + rtp_register_for_receiving(call->rtps[audio_index]); + } } - if (c_self->video_bitrate > 0 || c_peer->video_bitrate > 0) { /* Prepare video rtp */ - call->rtps[video_index] = rtp_new(rtp_TypeVideo, av->m, av->msi->calls[call->call_idx]->peers[0]); + if (capabilities & msi_CapSVideo || capabilities & msi_CapRVideo) { /* Prepare video rtp */ + call->rtps[video_index] = rtp_new(rtp_TypeVideo, av->m, call->friend_id); if ( !call->rtps[video_index] ) { LOGGER_ERROR("Error while starting video RTP session!\n"); @@ -835,8 +844,19 @@ bool i_toxav_prepare_transmission(ToxAV* av, IToxAVCall* call) call->rtps[video_index]->cs = call->cs; - if (c_peer->video_bitrate > 0) + if (cs_enable_video_sending(call->cs, call->s_video_b) != 0) { + LOGGER_WARNING("Failed to enable video sending!"); + goto FAILURE; + } + + if (capabilities & msi_CapRVideo) { + if (cs_enable_video_receiving(call->cs) != 0) { + LOGGER_WARNING("Failed to enable video receiving!"); + goto FAILURE; + } + rtp_register_for_receiving(call->rtps[audio_index]); + } } call->active = 1; @@ -866,7 +886,7 @@ MUTEX_INIT_ERROR: void i_toxav_kill_transmission(ToxAV* av, uint32_t friend_number) { - IToxAVCall* call = i_toxav_get_call(av, friend_number); + ToxAVCall* call = i_toxav_get_call(av, friend_number); if (!call) return; diff --git a/toxav/toxav_new_1.c b/toxav/toxav_new_1.c index ee7f49a64..145dcf48a 100644 --- a/toxav/toxav_new_1.c +++ b/toxav/toxav_new_1.c @@ -317,7 +317,6 @@ int toxav_prepare_transmission ( ToxAv *av, int32_t call_index, int support_vide } call->cs->agent = av; - call->cs->call_idx = call_index; call->cs->acb.first = av->acb.first; call->cs->acb.second = av->acb.second; From 29601feb7656d1a75b4ab7cdb161f232f0ec92dc Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 21 Feb 2015 01:07:22 +0100 Subject: [PATCH 10/97] New msi protocol --- toxav/av_test.c | 20 +- toxav/msi.c | 500 ++++++++++++++++++------------------------------ toxav/msi.h | 17 +- toxav/rtp.c | 14 +- toxav/rtp.h | 7 +- toxav/toxav.c | 458 ++++++++++++++++++++++++-------------------- toxav/toxav.h | 5 - 7 files changed, 465 insertions(+), 556 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 3270f39c7..594e72323 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -36,19 +36,11 @@ void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, TOXAV_CALL_STATE s { printf("Handling CALL STATE callback: "); - if (((CallControl*)user_data)->ringing) - ((CallControl*)user_data)->ringing = false; - if (((CallControl*)user_data)->paused) ((CallControl*)user_data)->paused = false; switch (state) - { - case TOXAV_CALL_STATE_RINGING: { - printf("Ringing"); - ((CallControl*)user_data)->ringing = true; - } break; - + { case TOXAV_CALL_STATE_NOT_SENDING: { printf("Not sending"); ((CallControl*)user_data)->sending = false; @@ -230,6 +222,7 @@ int main (int argc, char** argv) exit(1); \ } \ BobCC.incoming = false; \ + BobCC.sending = true; /* There is no more start callback when answering */\ } \ else if (AliceCC.sending && BobCC.sending) { \ /* TODO rtp */ \ @@ -238,6 +231,7 @@ int main (int argc, char** argv) \ TOXAV_ERR_CALL_CONTROL rc; \ toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); \ + AliceCC.ended = true; /* There is no more end callback when hanging up */\ \ if (rc != TOXAV_ERR_CALL_CONTROL_OK) { \ printf("toxav_call_control failed: %d\n", rc); \ @@ -252,13 +246,13 @@ int main (int argc, char** argv) } printf("\nTrying regular call (Audio and Video)...\n"); - REGULAR_CALL_FLOW(48, 4000); +// REGULAR_CALL_FLOW(48, 4000); printf("\nTrying regular call (Audio only)...\n"); - REGULAR_CALL_FLOW(48, 0); +// REGULAR_CALL_FLOW(48, 0); printf("\nTrying regular call (Video only)...\n"); - REGULAR_CALL_FLOW(0, 4000); +// REGULAR_CALL_FLOW(0, 4000); #undef REGULAR_CALL_FLOW @@ -292,7 +286,7 @@ int main (int argc, char** argv) } } - while (!AliceCC.ended || !BobCC.ended) + while (!AliceCC.ended) iterate(Bsn, AliceAV, BobAV); printf("Success!\n"); diff --git a/toxav/msi.c b/toxav/msi.c index 16476364c..f179a7aba 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -37,8 +37,6 @@ #define MSI_MAXMSG_SIZE 256 -/* TODO send error on any calloc or etc */ - /** * Protocol: * @@ -47,24 +45,17 @@ typedef enum { IDRequest = 1, - IDResponse, IDError, IDCapabilities, IDVFPSZ, } MSIHeaderID; -typedef enum { - requ_invite, - requ_start, - requ_reject, - requ_end, -} MSIRequest; typedef enum { - resp_ringing, - resp_starting, -} MSIResponse; + requ_push, + requ_pop, +} MSIRequest; #define GENERIC_HEADER(header, val_type) \ @@ -75,7 +66,6 @@ typedef struct { \ GENERIC_HEADER ( Request, MSIRequest ); -GENERIC_HEADER ( Response, MSIResponse ); GENERIC_HEADER ( Error, MSIError ); GENERIC_HEADER ( Capabilities, uint8_t ); GENERIC_HEADER ( VFPSZ, uint16_t ); @@ -83,29 +73,24 @@ GENERIC_HEADER ( VFPSZ, uint16_t ); typedef struct { MSIHeaderRequest request; - MSIHeaderResponse response; MSIHeaderError error; MSIHeaderCapabilities capabilities; MSIHeaderVFPSZ vfpsz; /* Video frame piece size. NOTE: Value must be in network b-order */ } MSIMessage; +void msg_init (MSIMessage *dest, MSIRequest request); int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ); uint8_t *msg_parse_header_out ( MSIHeaderID id, uint8_t *dest, const void *value, uint8_t value_len, uint16_t *length ); int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg ); int send_error ( Messenger* m, uint32_t friend_id, MSIError error ); -static void invoke_callback(MSICall* call, MSICallbackID cb); +static int invoke_callback(MSICall* call, MSICallbackID cb); static MSICall *get_call ( MSISession *session, uint32_t friend_id ); MSICall *new_call ( MSISession *session, uint32_t friend_id ); void kill_call ( MSICall *call ); void on_peer_status(Messenger *m, int friend_id, uint8_t status, void *data); -int handle_recv_invite ( MSICall *call, const MSIMessage *msg ); -int handle_recv_start ( MSICall *call, const MSIMessage *msg ); -int handle_recv_reject ( MSICall *call, const MSIMessage *msg ); -int handle_recv_end ( MSICall *call, const MSIMessage *msg ); -int handle_recv_ringing ( MSICall *call, const MSIMessage *msg ); -int handle_recv_starting ( MSICall *call, const MSIMessage *msg ); -int handle_recv_error ( MSICall *call, const MSIMessage *msg ); +void handle_push ( MSICall *call, const MSIMessage *msg ); +void handle_pop ( MSICall *call, const MSIMessage *msg ); void handle_msi_packet ( Messenger *m, int friend_id, const uint8_t *data, uint16_t length, void *object ); @@ -157,13 +142,12 @@ int msi_kill ( MSISession *session ) pthread_mutex_lock(session->mutex); if (session->calls) { - MSIMessage msg_end; - msg_end.request.exists = true; - msg_end.request.value = requ_end; + MSIMessage msg; + msg_init(&msg, requ_pop); MSICall* it = get_call(session, session->calls_head); for (; it; it = it->next) { - send_message(session->messenger, it->friend_id, &msg_end); + send_message(session->messenger, it->friend_id, &msg); kill_call(it); /* This will eventually free session->calls */ } } @@ -191,17 +175,16 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ (*call)->self_capabilities = capabilities; - MSIMessage msg_invite; - msg_invite.request.exists = true; - msg_invite.request.value = requ_invite; + MSIMessage msg; + msg_init(&msg, requ_push); - msg_invite.capabilities.exists = true; - msg_invite.capabilities.value = capabilities; + msg.capabilities.exists = true; + msg.capabilities.value = capabilities; - msg_invite.vfpsz.exists = true; - msg_invite.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE); + msg.vfpsz.exists = true; + msg.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE); - send_message ( (*call)->session->messenger, (*call)->friend_id, &msg_invite ); + send_message ( (*call)->session->messenger, (*call)->friend_id, &msg ); (*call)->state = msi_CallRequesting; @@ -212,10 +195,10 @@ int msi_hangup ( MSICall* call ) { LOGGER_DEBUG("Session: %p Hanging up call with friend: %u", call->session, call->friend_id); - MSIMessage msg_end; - msg_end.request.exists = true; - msg_end.request.value = requ_end; - send_message ( call->session->messenger, call->friend_id, &msg_end ); + MSIMessage msg; + msg_init(&msg, requ_pop); + + send_message ( call->session->messenger, call->friend_id, &msg ); kill_call(call); return 0; @@ -225,65 +208,68 @@ int msi_answer ( MSICall* call, uint8_t capabilities ) LOGGER_DEBUG("Session: %p Answering call from: %u", call->session, call->friend_id); if ( call->state != msi_CallRequested ) { + /* Though sending in invalid state will not cause anything wierd + * Its better to not do it like a maniac */ LOGGER_ERROR("Call is in invalid state!"); return -1; } call->self_capabilities = capabilities; - MSIMessage msg_starting; - msg_starting.response.exists = true; - msg_starting.response.value = resp_starting; + MSIMessage msg; + msg_init(&msg, requ_push); - msg_starting.capabilities.exists = true; - msg_starting.capabilities.value = capabilities; + msg.capabilities.exists = true; + msg.capabilities.value = capabilities; - msg_starting.vfpsz.exists = true; - msg_starting.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE); + msg.vfpsz.exists = true; + msg.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE); - send_message ( call->session->messenger, call->friend_id, &msg_starting ); + send_message ( call->session->messenger, call->friend_id, &msg ); + + call->state = msi_CallActive; return 0; } -int msi_reject ( MSICall* call ) +int msi_change_capabilities( MSICall* call, uint8_t capabilities ) { - LOGGER_DEBUG("Session: %p Rejecting call with friend: %u", call->session, call->friend_id); + LOGGER_DEBUG("Session: %p Trying to change capabilities to friend %u", call->session, call->friend_id); - if ( call->state != msi_CallRequested ) { + if ( call->state != msi_CallActive ) { + /* Sending capabilities change can cause error on other side if + * the call is not active since we don't send header 'vfpsz'. + * If we were to send 'vfpsz' while call is active it would be + * ignored. However, if call is not active peer will expect + * the said header on 'push' so that it could handle the call + * like new. TODO: explain this better + */ LOGGER_ERROR("Call is in invalid state!"); return -1; } - MSIMessage msg_reject; - msg_reject.request.exists = true; - msg_reject.request.value = requ_reject; - - send_message ( call->session->messenger, call->friend_id, &msg_reject ); - - return 0; -} -int msi_change_csettings( MSICall* call, uint8_t capabilities ) -{ call->self_capabilities = capabilities; - MSIMessage msg_invite; - msg_invite.request.exists = true; - msg_invite.request.value = requ_invite; + MSIMessage msg; + msg_init(&msg, requ_push); - msg_invite.capabilities.exists = true; - msg_invite.capabilities.value = capabilities; + msg.capabilities.exists = true; + msg.capabilities.value = capabilities; - send_message ( call->session->messenger, call->friend_id, &msg_invite ); + send_message ( call->session->messenger, call->friend_id, &msg ); return 0; } - /** * Private functions */ - +void msg_init(MSIMessage* dest, MSIRequest request) +{ + memset(dest, 0, sizeof(*dest)); + dest->request.exists = true; + dest->request.value = request; +} int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) { /* Parse raw data received from socket into MSIMessage struct */ @@ -314,7 +300,9 @@ int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) LOGGER_ERROR("Invalid end byte"); return -1; } - + + memset(dest, 0, sizeof(*dest)); + const uint8_t *it = data; int size_constraint = length; @@ -322,17 +310,10 @@ int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) switch (*it) { case IDRequest: CHECK_SIZE(it, size_constraint, 1); - CHECK_ENUM_HIGH(it, requ_end); + CHECK_ENUM_HIGH(it, requ_pop); SET_UINT8(it, dest->request); break; - case IDResponse: - CHECK_SIZE(it, size_constraint, 1); - CHECK_ENUM_HIGH(it, resp_starting); - SET_UINT8(it, dest->response); - it += 3; - break; - case IDError: CHECK_SIZE(it, size_constraint, 1); CHECK_ENUM_HIGH(it, msi_ErrUndisclosed); @@ -356,6 +337,11 @@ int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) } } + if (dest->request.exists == false) { + LOGGER_ERROR("Invalid request field!"); + return -1; + } + return 0; #undef CHECK_SIZE @@ -395,17 +381,15 @@ int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg ) uint8_t cast = msg->request.value; it = msg_parse_header_out(IDRequest, it, &cast, sizeof(cast), &size); - } - - if (msg->response.exists) { - uint8_t cast = msg->response.value; - it = msg_parse_header_out(IDResponse, it, &cast, - sizeof(cast), &size); + } else { + LOGGER_DEBUG("Must have request field"); + return -1; } if (msg->error.exists) { - it = msg_parse_header_out(IDError, it, &msg->error.value, - sizeof(msg->error.value), &size); + uint8_t cast = msg->error.value; + it = msg_parse_header_out(IDError, it, &cast, + sizeof(cast), &size); } if (msg->capabilities.exists) { @@ -418,14 +402,14 @@ int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg ) sizeof(msg->vfpsz.value), &size); } - *it = 0; - size ++; - if ( it == parsed ) { LOGGER_WARNING("Parsing message failed; empty message"); return -1; } + *it = 0; + size ++; + if ( m_msi_packet(m, friend_id, parsed, size) ) { LOGGER_DEBUG("Sent message"); return 0; @@ -440,22 +424,36 @@ int send_error ( Messenger* m, uint32_t friend_id, MSIError error ) LOGGER_DEBUG("Sending error: %d to friend: %d", error, friend_id); - MSIMessage msg_error; + MSIMessage msg; + msg_init(&msg, requ_pop); - msg_error.error.exists = true; - msg_error.error.value = error; + msg.error.exists = true; + msg.error.value = error; - send_message ( m, friend_id, &msg_error ); + send_message ( m, friend_id, &msg ); return 0; } -static void invoke_callback(MSICall* call, MSICallbackID cb) +int invoke_callback(MSICall* call, MSICallbackID cb) { assert(call); if ( call->session->callbacks[cb] ) { LOGGER_DEBUG("Invoking callback function: %d", cb); - call->session->callbacks[cb] ( call->session->av, call ); + if ( call->session->callbacks[cb] ( call->session->av, call ) != 0 ) { + LOGGER_WARNING("Callback handling failed, sending error"); + goto FAILURE; + } + + return 0; } + +FAILURE: + /* If no callback present or error happened while handling, + * an error message will be send to friend + */ + + call->error = msi_HandleError; + return -1; } static MSICall *get_call ( MSISession *session, uint32_t friend_id ) { @@ -563,7 +561,7 @@ void on_peer_status(Messenger *m, int friend_id, uint8_t status, void *data) if (call == NULL) return; - invoke_callback(call, msi_OnPeerTimeout); + invoke_callback(call, msi_OnPeerTimeout); /* Failure is ignored */ kill_call(call); } break; @@ -572,205 +570,120 @@ void on_peer_status(Messenger *m, int friend_id, uint8_t status, void *data) break; } } -int handle_recv_invite ( MSICall *call, const MSIMessage *msg ) +void handle_push ( MSICall *call, const MSIMessage *msg ) { assert(call); MSISession* session = call->session; - LOGGER_DEBUG("Session: %p Handling 'invite' friend: %d", call->session, call->friend_id); + LOGGER_DEBUG("Session: %p Handling 'push' friend: %d", call->session, call->friend_id); if (!msg->capabilities.exists) { - LOGGER_WARNING("Session: %p Invalid capabilities on 'invite'"); + LOGGER_WARNING("Session: %p Invalid capabilities on 'push'"); call->error = msi_InvalidMessage; - return -1; + goto FAILURE; } - MSIMessage response; - response.response.exists = true; - - if ( call->state == msi_CallRequesting ) { - /* The rare glare case. - * Send starting and wait for starting by the other side. - * The peer will do the same. - * When you receive starting from peer send started. - * Don't notice the app until the start is received. - */ - - LOGGER_DEBUG("Glare detected!"); - + if (call->state != msi_CallActive) { if (!msg->vfpsz.exists) { - LOGGER_WARNING("Session: %p Invalid mvfpsz on 'invite'"); + LOGGER_WARNING("Session: %p Invalid vfpsz on 'push'"); call->error = msi_InvalidMessage; - return -1; + goto FAILURE; } - call->peer_capabilities = msg->capabilities.value; + /* Sending video frame piece size is ignored when call is active */ call->peer_vfpsz = ntohs(msg->vfpsz.value); - - /* Send response */ - response.response.value = resp_starting; - send_message ( call->session->messenger, call->friend_id, &response ); - - return 0; - } else if ( call->state == msi_CallActive ) { - /* Changing capabilities. - * We send starting but no response is expected. - * WARNING: if start is sent call is terminated with an error - */ - LOGGER_DEBUG("Peer is changing capabilities"); - - /* Send response */ - response.response.value = resp_starting; - send_message ( call->session->messenger, call->friend_id, &response ); - - if ( call->peer_capabilities != msg->capabilities.value) { - /* Only invoke callback if capabilities changed */ + } + + + switch (call->state) { + case msi_CallInactive: { + LOGGER_INFO("Friend is calling us"); + + /* Call requested */ call->peer_capabilities = msg->capabilities.value; - invoke_callback(call, msi_OnCapabilities); - } + call->state = msi_CallRequested; + + if ( invoke_callback(call, msi_OnInvite) == -1 ) + goto FAILURE; + + } break; - return 0; + case msi_CallActive: { + /* Only act if capabilities changed */ + if ( call->peer_capabilities != msg->capabilities.value) { + LOGGER_INFO("Friend is changing capabilities"); + + call->peer_capabilities = msg->capabilities.value; + if ( invoke_callback(call, msi_OnCapabilities) == -1 ) + goto FAILURE; + } + } break; + + case msi_CallRequesting: { + LOGGER_INFO("Friend answered our call"); + + /* Call started */ + call->peer_capabilities = msg->capabilities.value; + call->state = msi_CallActive; + + if ( invoke_callback(call, msi_OnStart) == -1 ) + goto FAILURE; + } break; + + case msi_CallRequested: { + /* Consecutive pushes during initialization state are ignored */ + LOGGER_WARNING("Consecutive push"); + } break; } - if (!msg->vfpsz.exists) { - LOGGER_WARNING("Session: %p Invalid mvfpsz on 'invite'"); - call->error = msi_InvalidMessage; - return -1; - } + return; - call->peer_capabilities = msg->capabilities.value; - call->peer_vfpsz = ntohs(msg->vfpsz.value); - call->state = msi_CallRequested; - - /* Send response */ - response.response.value = resp_ringing; - send_message ( call->session->messenger, call->friend_id, &response ); - - - invoke_callback(call, msi_OnInvite); - return 0; -} -int handle_recv_start ( MSICall *call, const MSIMessage *msg ) -{ - assert(call); - - if ( call->state != msi_CallRequested || call->state != msi_CallRequesting ) { - LOGGER_WARNING("Session: %p Invalid call state on 'start'"); - call->error = msi_InvalidState; - return -1; - } - - (void)msg; - - LOGGER_DEBUG("Session: %p Handling 'start', friend id: %d", call->session, call->friend_id ); - - call->state = msi_CallActive; - invoke_callback(call, msi_OnStart); - - return 0; -} -int handle_recv_reject ( MSICall *call, const MSIMessage *msg ) -{ - assert(call); - - (void)msg; - - if ( call->state != msi_CallRequesting ) { - LOGGER_WARNING("Session: %p Invalid call state on 'reject'"); - call->error = msi_InvalidState; - return -1; - } - - LOGGER_DEBUG("Session: %p Handling 'reject', friend id: %u", call->session, call->friend_id); - - invoke_callback(call, msi_OnReject); +FAILURE: + send_error(call->session->messenger, call->friend_id, call->error); kill_call(call); - - return 0; } -int handle_recv_end ( MSICall *call, const MSIMessage *msg ) +void handle_pop ( MSICall *call, const MSIMessage *msg ) { assert(call); + + LOGGER_DEBUG("Session: %p Handling 'pop', friend id: %d", call->session, call->friend_id); + + /* callback errors are ignored */ + + if (msg->error.exists) { + LOGGER_WARNING("Friend detected an error: %d", msg->error.value); + call->error = msg->error.value; + invoke_callback(call, msi_OnError); + + } else switch (call->state) { + case msi_CallInactive: { + LOGGER_ERROR("Handling what should be impossible case"); + abort(); + } break; + + case msi_CallActive: { + /* Hangup */ + LOGGER_INFO("Friend hung up on us"); + invoke_callback(call, msi_OnEnd); + } break; + + case msi_CallRequesting: { + /* Reject */ + LOGGER_INFO("Friend rejected our call"); + invoke_callback(call, msi_OnEnd); + } break; + + case msi_CallRequested: { + /* Cancel */ + LOGGER_INFO("Friend canceled call invite"); + invoke_callback(call, msi_OnEnd); + } break; + } - (void)msg; - - LOGGER_DEBUG("Session: %p Handling 'end', friend id: %d", call->session, call->friend_id); - - invoke_callback(call, msi_OnEnd); kill_call ( call ); - return 0; -} -int handle_recv_ringing ( MSICall *call, const MSIMessage *msg ) -{ - assert(call); - - (void)msg; - - if ( call->state != msi_CallRequesting ) { - LOGGER_WARNING("Session: %p Invalid call state on 'ringing'"); - call->error = msi_InvalidState; - return -1; - } - - LOGGER_DEBUG("Session: %p Handling 'ringing' friend id: %d", call->session, call->friend_id ); - - invoke_callback(call, msi_OnRinging); - return 0; -} -int handle_recv_starting ( MSICall *call, const MSIMessage *msg ) -{ - assert(call); - - if ( call->state == msi_CallActive ) { - LOGGER_DEBUG("Capabilities change confirmed"); - return 0; - } else if ( call->state != msi_CallRequested || call->state != msi_CallRequesting ) { - LOGGER_WARNING("Session: %p Invalid call state on 'starting'"); - call->error = msi_InvalidState; - return -1; - } - - if (call->state == msi_CallRequesting) { - if (!msg->capabilities.exists) { - LOGGER_WARNING("Session: %p Invalid capabilities on 'starting'"); - call->error = msi_InvalidParam; - return -1; - } - - if (!msg->vfpsz.exists) { - LOGGER_WARNING("Session: %p Invalid mvfpsz on 'invite'"); - call->error = msi_InvalidParam; - return -1; - } - - call->peer_capabilities = msg->capabilities.value; - call->peer_vfpsz = ntohs(msg->vfpsz.value); - call->state = msi_CallActive; - - invoke_callback(call, msi_OnStart); - } - /* Otherwise it's a glare case so don't start until 'start' is recved */ - - /* Send start in either case (glare or normal) */ - MSIMessage msg_start; - msg_start.request.exists = true; - msg_start.request.value = requ_start; - send_message ( call->session->messenger, call->friend_id, &msg_start ); - - return 0; -} -int handle_recv_error ( MSICall *call, const MSIMessage *msg ) -{ - assert(call); - - LOGGER_DEBUG("Session: %p Handling 'error' friend id: %d", call->session, call->friend_id ); - - call->error = msg->error.value; - invoke_callback(call, msi_OnError); - - return 0; + return; } void handle_msi_packet ( Messenger *m, int friend_id, const uint8_t *data, uint16_t length, void *object ) { @@ -779,8 +692,6 @@ void handle_msi_packet ( Messenger *m, int friend_id, const uint8_t *data, uint1 MSISession *session = object; MSIMessage msg; - int rc = 0; - if ( msg_parse_in ( &msg, data, length ) == -1 ) { LOGGER_WARNING("Error parsing message"); send_error(m, friend_id, msi_InvalidMessage); @@ -792,7 +703,7 @@ void handle_msi_packet ( Messenger *m, int friend_id, const uint8_t *data, uint1 MSICall *call = get_call(session, friend_id); if (call == NULL) { - if (msg.request.value != requ_invite) { + if (msg.request.value != requ_push) { send_error(m, friend_id, msi_StrayMessage); return; } @@ -805,49 +716,8 @@ void handle_msi_packet ( Messenger *m, int friend_id, const uint8_t *data, uint1 } - /* Now handle message */ - if ( msg.request.exists ) { /* Handle request */ - switch (msg.request.value) { - case requ_invite: - rc = handle_recv_invite ( call, &msg ); - break; - - case requ_start: - rc = handle_recv_start ( call, &msg ); - break; - - case requ_reject: - rc = handle_recv_reject ( call, &msg ); - break; - - case requ_end: - rc = handle_recv_end ( call, &msg ); - break; - } - } else if ( msg.response.exists ) { /* Handle response */ - switch (msg.response.value) { - case resp_ringing: - rc = handle_recv_ringing ( call, &msg ); - break; - - case resp_starting: - rc = handle_recv_starting ( call, &msg ); - break; - } - } else if (msg.error.exists) { - handle_recv_error ( call, &msg ); - rc = -1; - } else { - LOGGER_WARNING("Invalid message; none of the request, response or error!"); - call->error = msi_InvalidMessage; - rc = -1; - } - - - if (rc == -1) { - if (call->error != msi_ErrorNone) - send_error(m, friend_id, call->error); - - kill_call(call); - } -} + if (msg.request.value == requ_push) + handle_push(call, &msg); + else + handle_pop(call, &msg); /* always kills the call */ +} \ No newline at end of file diff --git a/toxav/msi.h b/toxav/msi.h index a55d8567a..783d3928c 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -42,6 +42,7 @@ typedef enum { msi_StrayMessage, msi_SystemError, msi_ErrUndisclosed, + msi_HandleError, } MSIError; /** @@ -70,9 +71,7 @@ typedef enum { */ typedef enum { msi_OnInvite, /* Incoming call */ - msi_OnRinging, /* When peer is ready to accept/reject the call */ msi_OnStart, /* Call (RTP transmission) started */ - msi_OnReject, /* The side that was invited rejected the call */ msi_OnEnd, /* Call that was active ended */ msi_OnError, /* On protocol error */ msi_OnPeerTimeout, /* Peer timed out; stop the call */ @@ -100,9 +99,12 @@ typedef struct MSICall_s { /** - * Msi callback type. 'agent' is a pointer to ToxAv + * Msi callback type. 'agent' is a pointer to ToxAv. + * Expected return on success is 0, if any other number is + * returned the call is considered errored and will be handled + * as such which means it will be terminated without any notice. */ -typedef void ( *MSICallbackType ) ( void *agent, MSICall* call); +typedef int ( *MSICallbackType ) ( void *agent, MSICall* call); /** * Control session struct. Please do not modify outside msi.c @@ -117,7 +119,7 @@ typedef struct MSISession_s { Messenger *messenger; pthread_mutex_t mutex[1]; - MSICallbackType callbacks[8]; + MSICallbackType callbacks[7]; } MSISession; /** @@ -150,11 +152,6 @@ int msi_hangup ( MSICall* call ); */ int msi_answer ( MSICall* call, uint8_t capabilities ); -/** - * Reject incoming call. NOTE: 'call' will be freed - */ -int msi_reject ( MSICall* call ); - /** * Change capabilities of the call. */ diff --git a/toxav/rtp.c b/toxav/rtp.c index 396e02025..8319c7dc9 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -478,12 +478,24 @@ void rtp_kill ( RTPSession *session ) free ( session ); } -int rtp_register_for_receiving(RTPSession* session) +int rtp_start_receiving(RTPSession* session) { + if (session == NULL) + return 0; + return custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, rtp_handle_packet, session); } +int rtp_stop_receiving(RTPSession* session) +{ + if (session == NULL) + return 0; + + return custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, + NULL, NULL); +} + int rtp_send_msg ( RTPSession *session, const uint8_t *data, uint16_t length ) { RTPMessage *msg = rtp_new_message (session, data, length); diff --git a/toxav/rtp.h b/toxav/rtp.h index e3b38a8ea..2950941bd 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -119,7 +119,12 @@ void rtp_kill ( RTPSession* session ); /** * By default rtp is not in receiving state */ -int rtp_register_for_receiving (RTPSession *session); +int rtp_start_receiving (RTPSession *session); + +/** + * Pause rtp receiving mode. + */ +int rtp_stop_receiving (RTPSession *session); /** * Sends msg to RTPSession::dest diff --git a/toxav/toxav.c b/toxav/toxav.c index 584b38982..5054d3996 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -42,6 +42,7 @@ enum { typedef struct ToxAVCall_s { + ToxAV* av; pthread_mutex_t mutex_control[1]; pthread_mutex_t mutex_encoding_audio[1]; pthread_mutex_t mutex_encoding_video[1]; @@ -55,6 +56,8 @@ typedef struct ToxAVCall_s uint32_t s_audio_b; /* Sending audio bitrate */ uint32_t s_video_b; /* Sending video bitrate */ + uint8_t last_capabilities; + struct ToxAVCall_s *prev; struct ToxAVCall_s *next; } ToxAVCall; @@ -83,22 +86,20 @@ struct toxAV }; -void i_callback_invite(void* toxav_inst, MSICall* call); -void i_callback_ringing(void* toxav_inst, MSICall* call); -void i_callback_start(void* toxav_inst, MSICall* call); -void i_callback_end(void* toxav_inst, MSICall* call); -void i_callback_error(void* toxav_inst, MSICall* call); -void i_callback_capabilites(void* toxav_inst, MSICall* call); +int callback_invite(void* toxav_inst, MSICall* call); +int callback_start(void* toxav_inst, MSICall* call); +int callback_end(void* toxav_inst, MSICall* call); +int callback_error(void* toxav_inst, MSICall* call); +int callback_capabilites(void* toxav_inst, MSICall* call); TOXAV_CALL_STATE capabilities_to_state(uint8_t capabilities); -ToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number); -ToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number); -void i_toxav_remove_call(ToxAV* av, uint32_t friend_number); -ToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); -bool i_toxav_audio_bitrate_invalid(uint32_t bitrate); -bool i_toxav_video_bitrate_invalid(uint32_t bitrate); -bool i_toxav_prepare_transmission(ToxAV* av, ToxAVCall* call); -void i_toxav_kill_transmission(ToxAV* av, uint32_t friend_number); +ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); +ToxAVCall* call_get(ToxAV* av, uint32_t friend_number); +void call_remove(ToxAVCall* call); +bool audio_bitrate_invalid(uint32_t bitrate); +bool video_bitrate_invalid(uint32_t bitrate); +bool call_prepare_transmission(ToxAVCall* call); +void call_kill_transmission(ToxAVCall* call); @@ -136,14 +137,12 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) av->interval = 200; av->msi->av = av; - msi_register_callback(av->msi, i_callback_invite, msi_OnInvite); - msi_register_callback(av->msi, i_callback_ringing, msi_OnRinging); - msi_register_callback(av->msi, i_callback_start, msi_OnStart); - msi_register_callback(av->msi, i_callback_end, msi_OnReject); - msi_register_callback(av->msi, i_callback_end, msi_OnEnd); - msi_register_callback(av->msi, i_callback_error, msi_OnError); - msi_register_callback(av->msi, i_callback_error, msi_OnPeerTimeout); - msi_register_callback(av->msi, i_callback_capabilites, msi_OnCapabilities); + msi_register_callback(av->msi, callback_invite, msi_OnInvite); + msi_register_callback(av->msi, callback_start, msi_OnStart); + msi_register_callback(av->msi, callback_end, msi_OnEnd); + msi_register_callback(av->msi, callback_error, msi_OnError); + msi_register_callback(av->msi, callback_error, msi_OnPeerTimeout); + msi_register_callback(av->msi, callback_capabilites, msi_OnCapabilities); if (error) @@ -166,7 +165,16 @@ void toxav_kill(ToxAV* av) return; msi_kill(av->msi); - /* TODO iterate over calls */ + + /* Msi kill will hang up all calls so just clean these calls */ + if (av->calls) { + ToxAVCall* it = call_get(av, av->calls_head); + for (; it; it = it->next) { + call_kill_transmission(it); + call_remove(it); /* This will eventually free av->calls */ + } + } + free(av); } @@ -208,20 +216,20 @@ void toxav_iteration(ToxAV* av) bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) { - ToxAVCall* call = i_toxav_init_call(av, friend_number, error); + ToxAVCall* call = call_new(av, friend_number, error); if (call == NULL) return false; call->s_audio_b = audio_bit_rate; call->s_video_b = video_bit_rate; - uint8_t capabilities = 0; + call->last_capabilities = msi_CapRAudio | msi_CapRVideo; - capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; - capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; + call->last_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; + call->last_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; - if (msi_invite(av->msi, &call->msi_call, friend_number, capabilities) != 0) { - i_toxav_remove_call(av, friend_number); + if (msi_invite(av->msi, &call->msi_call, friend_number, call->last_capabilities) != 0) { + call_remove(call); if (error) *error = TOXAV_ERR_CALL_MALLOC; return false; @@ -244,14 +252,14 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui goto END; } - if ((audio_bit_rate && i_toxav_audio_bitrate_invalid(audio_bit_rate)) - ||(video_bit_rate && i_toxav_video_bitrate_invalid(video_bit_rate)) + if ((audio_bit_rate && audio_bitrate_invalid(audio_bit_rate)) + ||(video_bit_rate && video_bitrate_invalid(video_bit_rate)) ) { rc = TOXAV_ERR_CALL_INVALID_BIT_RATE; goto END; } - ToxAVCall* call = i_toxav_get_call(av, friend_number); + ToxAVCall* call = call_get(av, friend_number); if (call == NULL) { rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; goto END; @@ -260,12 +268,12 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui call->s_audio_b = audio_bit_rate; call->s_video_b = video_bit_rate; - uint8_t capabilities = 0; + call->last_capabilities = msi_CapRAudio | msi_CapRVideo; - capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; - capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; + call->last_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; + call->last_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; - if (msi_answer(call->msi_call, capabilities) != 0) + if (msi_answer(call->msi_call, call->last_capabilities) != 0) rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; /* the only reason for msi_answer to fail */ @@ -292,7 +300,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co } - ToxAVCall* call = i_toxav_get_call(av, friend_number); + ToxAVCall* call = call_get(av, friend_number); if (call == NULL) { rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; goto END; @@ -302,29 +310,53 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co switch (control) { case TOXAV_CALL_CONTROL_RESUME: { - + if (call->msi_call->self_capabilities == 0 && + call->last_capabilities ) { + /* Only act if paused and had media transfer active before */ + + if (msi_change_capabilities(call->msi_call, call->last_capabilities) == -1) + return false; + + rtp_start_receiving(call->rtps[audio_index]); + rtp_start_receiving(call->rtps[video_index]); + } } break; case TOXAV_CALL_CONTROL_PAUSE: { - + if (call->msi_call->self_capabilities) { + /* Only act if not already paused */ + + call->last_capabilities = call->msi_call->self_capabilities; + + if (msi_change_capabilities(call->msi_call, 0) == -1 ) + return false; + + rtp_stop_receiving(call->rtps[audio_index]); + rtp_stop_receiving(call->rtps[video_index]); + } } break; case TOXAV_CALL_CONTROL_CANCEL: { - if (call->msi_call->state == msi_CallActive - || call->msi_call->state == msi_CallRequesting) { - /* Hang up */ - msi_hangup(call->msi_call); - } else if (call->msi_call->state == msi_CallRequested) { - /* Reject the call */ - msi_reject(call->msi_call); - } + /* Hang up */ + msi_hangup(call->msi_call); - // No mather the case, terminate the call - i_toxav_remove_call(av, friend_number); + /* No mather the case, terminate the call */ + call_remove(call); } break; case TOXAV_CALL_CONTROL_MUTE_AUDIO: { - + if (call->msi_call->self_capabilities & msi_CapRAudio || + call->msi_call->self_capabilities & msi_CapSAudio) { + + uint8_t capabilities = call->msi_call->self_capabilities; + capabilities ^= msi_CapRAudio; + capabilities ^= msi_CapRAudio; + + if (msi_change_capabilities(call->msi_call, call->msi_call->self_capabilities) == -1) + return false; + + rtp_stop_receiving(call->rtps[audio_index]); + } } break; case TOXAV_CALL_CONTROL_MUTE_VIDEO: { @@ -364,7 +396,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u goto END; } - call = i_toxav_get_call(av, friend_number); + call = call_get(av, friend_number); if (call == NULL) { rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; @@ -461,7 +493,7 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } - call = i_toxav_get_call(av, friend_number); + call = call_get(av, friend_number); if (call == NULL) { rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; @@ -526,19 +558,16 @@ void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* * :: Internal * ******************************************************************************/ -/** TODO: - * - If crutial callback not present send error. - * - Error handling by return values from callbacks and setting 'error'. +/** TODO: */ -void i_callback_invite(void* toxav_inst, MSICall* call) +int callback_invite(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - ToxAVCall* av_call = i_toxav_init_call(toxav, call->friend_id, NULL); + ToxAVCall* av_call = call_new(toxav, call->friend_id, NULL); if (av_call == NULL) { - LOGGER_WARNING("Failed to start call, rejecting..."); - msi_reject(call); - return; + LOGGER_WARNING("Failed to initialize call..."); + return -1; } call->av_call = av_call; @@ -547,55 +576,59 @@ void i_callback_invite(void* toxav_inst, MSICall* call) if (toxav->ccb.first) toxav->ccb.first(toxav, call->friend_id, call->peer_capabilities & msi_CapSAudio, call->peer_capabilities & msi_CapSVideo, toxav->ccb.second); + + return 0; } -void i_callback_ringing(void* toxav_inst, MSICall* call) -{ - ToxAV* toxav = toxav_inst; - if (toxav->scb.first) - toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_RINGING, toxav->scb.second); -} - -void i_callback_start(void* toxav_inst, MSICall* call) +int callback_start(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - ToxAVCall* av_call = i_toxav_get_call(toxav, call->friend_id); + ToxAVCall* av_call = call_get(toxav, call->friend_id); - if (av_call == NULL || !i_toxav_prepare_transmission(toxav, av_call)) { - /* TODO send error */ - i_toxav_remove_call(toxav, call->friend_id); - return; + if (av_call == NULL || !call_prepare_transmission(av_call)) { + call_remove(av_call); + return -1; } TOXAV_CALL_STATE state = capabilities_to_state(av_call->msi_call->peer_capabilities); if (toxav->scb.first) toxav->scb.first(toxav, call->friend_id, state, toxav->scb.second); + + return 0; } -void i_callback_end(void* toxav_inst, MSICall* call) +int callback_end(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - i_toxav_kill_transmission(toxav, call->friend_id); - i_toxav_remove_call(toxav, call->friend_id); + call_kill_transmission(call->av_call); + call_remove(call->av_call); if (toxav->scb.first) toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_END, toxav->scb.second); + + return 0; } -void i_callback_error(void* toxav_inst, MSICall* call) +int callback_error(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; if (toxav->scb.first) toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_ERROR, toxav->scb.second); + + return 0; } -void i_callback_capabilites(void* toxav_inst, MSICall* call) +int callback_capabilites(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - /* TODO handle this */ + if (toxav->scb.first) + toxav->scb.first(toxav, call->friend_id, + capabilities_to_state(call->peer_capabilities), toxav->scb.second); + + return 0; } TOXAV_CALL_STATE capabilities_to_state(uint8_t capabilities) @@ -610,116 +643,21 @@ TOXAV_CALL_STATE capabilities_to_state(uint8_t capabilities) return TOXAV_CALL_STATE_PAUSED; } -ToxAVCall* i_toxav_get_call(ToxAV* av, uint32_t friend_number) +bool audio_bitrate_invalid(uint32_t bitrate) { - if (av->calls == NULL || av->calls_tail < friend_number) - return NULL; - - return av->calls[friend_number]; + /* Opus RFC 6716 section-2.1.1 dictates the following: + * Opus supports all bitrates from 6 kbit/s to 510 kbit/s. + */ + return bitrate < 6 || bitrate > 510; } -ToxAVCall* i_toxav_add_call(ToxAV* av, uint32_t friend_number) +bool video_bitrate_invalid(uint32_t bitrate) { - ToxAVCall* rc = calloc(sizeof(ToxAVCall), 1); - - if (rc == NULL) - return NULL; - - rc->friend_id = friend_number; - - if (create_recursive_mutex(rc->mutex_control) != 0) { - free(rc); - return NULL; - } - - if (create_recursive_mutex(rc->mutex_do) != 0) { - pthread_mutex_destroy(rc->mutex_control); - free(rc); - return NULL; - } - - - if (av->calls == NULL) { /* Creating */ - av->calls = calloc (sizeof(ToxAVCall*), friend_number + 1); - - if (av->calls == NULL) { - pthread_mutex_destroy(rc->mutex_control); - pthread_mutex_destroy(rc->mutex_do); - free(rc); - return NULL; - } - - av->calls_tail = av->calls_head = friend_number; - - } else if (av->calls_tail < friend_number) { /* Appending */ - void* tmp = realloc(av->calls, sizeof(ToxAVCall*) * friend_number + 1); - - if (tmp == NULL) { - pthread_mutex_destroy(rc->mutex_control); - pthread_mutex_destroy(rc->mutex_do); - free(rc); - return NULL; - } - - av->calls = tmp; - - /* Set fields in between to null */ - int32_t i = av->calls_tail; - for (; i < friend_number; i ++) - av->calls[i] = NULL; - - rc->prev = av->calls[av->calls_tail]; - av->calls[av->calls_tail]->next = rc; - - av->calls_tail = friend_number; - - } else if (av->calls_head > friend_number) { /* Inserting at front */ - rc->next = av->calls[av->calls_head]; - av->calls[av->calls_head]->prev = rc; - av->calls_head = friend_number; - } - - av->calls[friend_number] = rc; - return rc; + /* TODO: If anyone knows the answer to this one please fill it up */ + return false; } -void i_toxav_remove_call(ToxAV* av, uint32_t friend_number) -{ - ToxAVCall* tc = i_toxav_get_call(av, friend_number); - - if (tc == NULL) - return; - - ToxAVCall* prev = tc->prev; - ToxAVCall* next = tc->next; - - pthread_mutex_destroy(tc->mutex_control); - pthread_mutex_destroy(tc->mutex_do); - - free(tc); - - if (prev) - prev->next = next; - else if (next) - av->calls_head = next->friend_id; - else goto CLEAR; - - if (next) - next->prev = prev; - else if (prev) - av->calls_tail = prev->friend_id; - else goto CLEAR; - - av->calls[friend_number] = NULL; - return; - -CLEAR: - av->calls_head = av->calls_tail = 0; - free(av->calls); - av->calls = NULL; -} - -ToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) +ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) { TOXAV_ERR_CALL rc = TOXAV_ERR_CALL_OK; ToxAVCall* call = NULL; @@ -734,39 +672,106 @@ ToxAVCall* i_toxav_init_call(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* goto END; } - if (i_toxav_get_call(av, friend_number) != NULL) { + if (call_get(av, friend_number) != NULL) { rc = TOXAV_ERR_CALL_FRIEND_ALREADY_IN_CALL; goto END; } - call = i_toxav_add_call(av, friend_number); + + call = calloc(sizeof(ToxAVCall), 1); + if (call == NULL) { rc = TOXAV_ERR_CALL_MALLOC; + goto END; } - END: + call->av = av; + call->friend_id = friend_number; + + if (create_recursive_mutex(call->mutex_control) != 0) { + free(call); + call = NULL; + rc = TOXAV_ERR_CALL_MALLOC; + goto END; + } + + if (create_recursive_mutex(call->mutex_do) != 0) { + pthread_mutex_destroy(call->mutex_control); + free(call); + call = NULL; + rc = TOXAV_ERR_CALL_MALLOC; + goto END; + } + + + if (av->calls == NULL) { /* Creating */ + av->calls = calloc (sizeof(ToxAVCall*), friend_number + 1); + + if (av->calls == NULL) { + pthread_mutex_destroy(call->mutex_control); + pthread_mutex_destroy(call->mutex_do); + free(call); + call = NULL; + rc = TOXAV_ERR_CALL_MALLOC; + goto END; + } + + av->calls_tail = av->calls_head = friend_number; + + } else if (av->calls_tail < friend_number) { /* Appending */ + void* tmp = realloc(av->calls, sizeof(ToxAVCall*) * friend_number + 1); + + if (tmp == NULL) { + pthread_mutex_destroy(call->mutex_control); + pthread_mutex_destroy(call->mutex_do); + free(call); + call = NULL; + rc = TOXAV_ERR_CALL_MALLOC; + goto END; + } + + av->calls = tmp; + + /* Set fields in between to null */ + int32_t i = av->calls_tail; + for (; i < friend_number; i ++) + av->calls[i] = NULL; + + call->prev = av->calls[av->calls_tail]; + av->calls[av->calls_tail]->next = call; + + av->calls_tail = friend_number; + + } else if (av->calls_head > friend_number) { /* Inserting at front */ + call->next = av->calls[av->calls_head]; + av->calls[av->calls_head]->prev = call; + av->calls_head = friend_number; + } + + av->calls[friend_number] = call; + +END: if (error) *error = rc; return call; } -bool i_toxav_audio_bitrate_invalid(uint32_t bitrate) +ToxAVCall* call_get(ToxAV* av, uint32_t friend_number) { - /* Opus RFC 6716 section-2.1.1 dictates the following: - * Opus supports all bitrates from 6 kbit/s to 510 kbit/s. - */ - return bitrate < 6 || bitrate > 510; + if (av->calls == NULL || av->calls_tail < friend_number) + return NULL; + + return av->calls[friend_number]; } -bool i_toxav_video_bitrate_invalid(uint32_t bitrate) -{ - /* TODO: If anyone knows the answer to this one please fill it up */ - return false; -} - -bool i_toxav_prepare_transmission(ToxAV* av, ToxAVCall* call) +bool call_prepare_transmission(ToxAVCall* call) { + if (call == NULL) + return false; + + ToxAV* av = call->av; + if (!av->acb.first && !av->vcb.first) /* It makes no sense to have CSession without callbacks */ return false; @@ -830,7 +835,7 @@ bool i_toxav_prepare_transmission(ToxAV* av, ToxAVCall* call) goto FAILURE; } - rtp_register_for_receiving(call->rtps[audio_index]); + rtp_start_receiving(call->rtps[audio_index]); } } @@ -854,8 +859,8 @@ bool i_toxav_prepare_transmission(ToxAV* av, ToxAVCall* call) LOGGER_WARNING("Failed to enable video receiving!"); goto FAILURE; } - - rtp_register_for_receiving(call->rtps[audio_index]); + + rtp_start_receiving(call->rtps[audio_index]); } } @@ -863,7 +868,7 @@ bool i_toxav_prepare_transmission(ToxAV* av, ToxAVCall* call) pthread_mutex_unlock(call->mutex_control); return true; -FAILURE: + FAILURE: rtp_kill(call->rtps[audio_index]); call->rtps[audio_index] = NULL; rtp_kill(call->rtps[video_index]); @@ -878,25 +883,19 @@ FAILURE: pthread_mutex_unlock(call->mutex_control); return false; -MUTEX_INIT_ERROR: + MUTEX_INIT_ERROR: pthread_mutex_unlock(call->mutex_control); LOGGER_ERROR("Mutex initialization failed!\n"); return false; } -void i_toxav_kill_transmission(ToxAV* av, uint32_t friend_number) +void call_kill_transmission(ToxAVCall* call) { - ToxAVCall* call = i_toxav_get_call(av, friend_number); - if (!call) + if (call == NULL || call->active == 0) return; pthread_mutex_lock(call->mutex_control); - if (!call->active) { - pthread_mutex_unlock(call->mutex_control); - return; - } - call->active = 0; pthread_mutex_lock(call->mutex_encoding_audio); @@ -918,4 +917,41 @@ void i_toxav_kill_transmission(ToxAV* av, uint32_t friend_number) pthread_mutex_destroy(call->mutex_do); pthread_mutex_unlock(call->mutex_control); +} + +void call_remove(ToxAVCall* call) +{ + if (call == NULL) + return; + + uint32_t friend_id = call->friend_id; + ToxAV* av = call->av; + + ToxAVCall* prev = call->prev; + ToxAVCall* next = call->next; + + pthread_mutex_destroy(call->mutex_control); + pthread_mutex_destroy(call->mutex_do); + + free(call); + + if (prev) + prev->next = next; + else if (next) + av->calls_head = next->friend_id; + else goto CLEAR; + + if (next) + next->prev = prev; + else if (prev) + av->calls_tail = prev->friend_id; + else goto CLEAR; + + av->calls[friend_id] = NULL; + return; + +CLEAR: + av->calls_head = av->calls_tail = 0; + free(av->calls); + av->calls = NULL; } \ No newline at end of file diff --git a/toxav/toxav.h b/toxav/toxav.h index c1c6019c3..5e82faa6a 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -186,11 +186,6 @@ bool toxav_answer(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, ui * ******************************************************************************/ typedef enum TOXAV_CALL_STATE { - /** - * The friend's client is aware of the call. This happens after calling - * toxav_call and the initial call request has been received. - */ - TOXAV_CALL_STATE_RINGING, /** * Not sending anything. Either the friend requested that this client stops * sending anything, or the client turned off both audio and video by setting From 9e65cd533735f1acc638241ed58400bf7029b1ef Mon Sep 17 00:00:00 2001 From: mannol Date: Sun, 22 Feb 2015 23:41:40 +0100 Subject: [PATCH 11/97] Implement pausing --- toxav/av_test.c | 173 +++++++++++++++++++++++++++++---------------- toxav/msi.h | 2 +- toxav/toxav.c | 184 ++++++++++++++++++++++++++++++++++-------------- toxav/toxav.h | 22 +++++- 4 files changed, 266 insertions(+), 115 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 594e72323..41f5a7585 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -16,13 +16,25 @@ typedef struct { bool incoming; - bool ringing; - bool ended; - bool errored; - bool sending; - bool paused; + TOXAV_CALL_STATE state; } CallControl; +const char* stringify_state(TOXAV_CALL_STATE s) +{ + static const char* strings[] = + { + "NOT SENDING", + "SENDING AUDIO", + "SENDING VIDEO", + "SENDING AUDIO AND VIDEO", + "PAUSED", + "END", + "ERROR" + }; + + return strings [s]; +}; + /** * Callbacks @@ -34,53 +46,9 @@ void t_toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool } void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, TOXAV_CALL_STATE state, void *user_data) { - printf("Handling CALL STATE callback: "); + printf("Handling CALL STATE callback: %s\n", stringify_state(state)); - if (((CallControl*)user_data)->paused) - ((CallControl*)user_data)->paused = false; - - switch (state) - { - case TOXAV_CALL_STATE_NOT_SENDING: { - printf("Not sending"); - ((CallControl*)user_data)->sending = false; - } break; - - case TOXAV_CALL_STATE_SENDING_A: { - printf("Sending Audio"); - ((CallControl*)user_data)->sending = true; - } break; - - case TOXAV_CALL_STATE_SENDING_V: { - printf("Sending Video"); - ((CallControl*)user_data)->sending = true; - } break; - - case TOXAV_CALL_STATE_SENDING_AV: { - printf("Sending Both"); - ((CallControl*)user_data)->sending = true; - } break; - - case TOXAV_CALL_STATE_PAUSED: { - printf("Paused"); - ((CallControl*)user_data)->paused = true; - ((CallControl*)user_data)->sending = false; - } break; - - case TOXAV_CALL_STATE_END: { - printf("Ended"); - ((CallControl*)user_data)->ended = true; - ((CallControl*)user_data)->sending = false; - } break; - - case TOXAV_CALL_STATE_ERROR: { - printf("Error"); - ((CallControl*)user_data)->errored = true; - ((CallControl*)user_data)->sending = false; - } break; - } - - printf("\n"); + ((CallControl*)user_data)->state = state; } void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, @@ -211,27 +179,24 @@ int main (int argc, char** argv) long long unsigned int start_time = time(NULL); \ \ \ - while (!AliceCC.ended || !BobCC.ended) { \ + while (BobCC.state != TOXAV_CALL_STATE_END) { \ \ if (BobCC.incoming) { \ TOXAV_ERR_ANSWER rc; \ - toxav_answer(BobAV, 0, 48, 4000, &rc); \ + toxav_answer(BobAV, 0, A_BR, V_BR, &rc); \ \ if (rc != TOXAV_ERR_ANSWER_OK) { \ printf("toxav_answer failed: %d\n", rc); \ exit(1); \ } \ BobCC.incoming = false; \ - BobCC.sending = true; /* There is no more start callback when answering */\ - } \ - else if (AliceCC.sending && BobCC.sending) { \ + } else { \ /* TODO rtp */ \ \ if (time(NULL) - start_time == 5) { \ \ TOXAV_ERR_CALL_CONTROL rc; \ toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); \ - AliceCC.ended = true; /* There is no more end callback when hanging up */\ \ if (rc != TOXAV_ERR_CALL_CONTROL_OK) { \ printf("toxav_call_control failed: %d\n", rc); \ @@ -286,7 +251,7 @@ int main (int argc, char** argv) } } - while (!AliceCC.ended) + while (AliceCC.state != TOXAV_CALL_STATE_END) iterate(Bsn, AliceAV, BobAV); printf("Success!\n"); @@ -323,12 +288,102 @@ int main (int argc, char** argv) } /* Alice will not receive end state */ - while (!BobCC.ended) + while (BobCC.state != TOXAV_CALL_STATE_END) iterate(Bsn, AliceAV, BobAV); printf("Success!\n"); } + { /* Check Mute-Unmute etc */ + printf("\nTrying mute functionality...\n"); + + + memset(&AliceCC, 0, sizeof(CallControl)); + memset(&BobCC, 0, sizeof(CallControl)); + + /* Assume sending audio and video */ + { + TOXAV_ERR_CALL rc; + toxav_call(AliceAV, 0, 48, 1000, &rc); + + if (rc != TOXAV_ERR_CALL_OK) { + printf("toxav_call failed: %d\n", rc); + exit(1); + } + } + + while (!BobCC.incoming) + iterate(Bsn, AliceAV, BobAV); + + /* At first try all stuff while in invalid state */ + assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); + assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); + assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_AUDIO, NULL)); + assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_VIDEO, NULL)); + assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_AUDIO, NULL)); + assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_VIDEO, NULL)); + + { + TOXAV_ERR_ANSWER rc; + toxav_answer(BobAV, 0, 48, 4000, &rc); + + if (rc != TOXAV_ERR_ANSWER_OK) { + printf("toxav_answer failed: %d\n", rc); + exit(1); + } + } + + iterate(Bsn, AliceAV, BobAV); + + /* Pause and Resume */ + printf("Pause and Resume\n"); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); + iterate(Bsn, AliceAV, BobAV); + assert(BobCC.state == TOXAV_CALL_STATE_PAUSED); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); + iterate(Bsn, AliceAV, BobAV); + assert(BobCC.state == TOXAV_CALL_STATE_SENDING_AV); + + /* Mute/Unmute single */ + printf("Mute/Unmute single\n"); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_AUDIO, NULL)); + iterate(Bsn, AliceAV, BobAV); + assert(BobCC.state == TOXAV_CALL_CONTROL_MUTE_AUDIO); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_AUDIO, NULL)); + iterate(Bsn, AliceAV, BobAV); + assert(BobCC.state == TOXAV_CALL_CONTROL_UNMUTE_AUDIO); + + /* Mute/Unmute both */ + printf("Mute/Unmute both\n"); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_AUDIO, NULL)); + iterate(Bsn, AliceAV, BobAV); + assert(BobCC.state == TOXAV_CALL_STATE_SENDING_V); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_VIDEO, NULL)); + iterate(Bsn, AliceAV, BobAV); + assert(BobCC.state == TOXAV_CALL_STATE_NOT_SENDING); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_AUDIO, NULL)); + iterate(Bsn, AliceAV, BobAV); + assert(BobCC.state == TOXAV_CALL_STATE_SENDING_A); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_VIDEO, NULL)); + iterate(Bsn, AliceAV, BobAV); + assert(BobCC.state == TOXAV_CALL_STATE_SENDING_AV); + + { + TOXAV_ERR_CALL_CONTROL rc; + toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); + + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { + printf("toxav_call_control failed: %d\n", rc); + exit(1); + } + } + + iterate(Bsn, AliceAV, BobAV); + assert(BobCC.state == TOXAV_CALL_STATE_END); + + printf("Success!\n"); + } + printf("\nTest successful!\n"); return 0; } \ No newline at end of file diff --git a/toxav/msi.h b/toxav/msi.h index 783d3928c..b846542d1 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -41,8 +41,8 @@ typedef enum { msi_InvalidState, msi_StrayMessage, msi_SystemError, - msi_ErrUndisclosed, msi_HandleError, + msi_ErrUndisclosed, /* NOTE: must be last enum otherwise parsing wont work */ } MSIError; /** diff --git a/toxav/toxav.c b/toxav/toxav.c index 5054d3996..e6b51ee4b 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -235,6 +235,8 @@ bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint return false; } + call->msi_call->av_call = call; + return true; } @@ -310,12 +312,22 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co switch (control) { case TOXAV_CALL_CONTROL_RESUME: { - if (call->msi_call->self_capabilities == 0 && + if (!call->active) { + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } + + /* Only act if paused and had media transfer active before */ + if (call->msi_call->self_capabilities == 0 && call->last_capabilities ) { - /* Only act if paused and had media transfer active before */ - if (msi_change_capabilities(call->msi_call, call->last_capabilities) == -1) - return false; + if (msi_change_capabilities(call->msi_call, + call->last_capabilities) == -1) { + /* The only reason for this function to fail is invalid state + * ( not active ) */ + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } rtp_start_receiving(call->rtps[audio_index]); rtp_start_receiving(call->rtps[video_index]); @@ -323,13 +335,21 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co } break; case TOXAV_CALL_CONTROL_PAUSE: { + if (!call->active) { + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } + + /* Only act if not already paused */ if (call->msi_call->self_capabilities) { - /* Only act if not already paused */ - call->last_capabilities = call->msi_call->self_capabilities; - if (msi_change_capabilities(call->msi_call, 0) == -1 ) - return false; + if (msi_change_capabilities(call->msi_call, 0) == -1 ) { + /* The only reason for this function to fail is invalid state + * ( not active ) */ + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } rtp_stop_receiving(call->rtps[audio_index]); rtp_stop_receiving(call->rtps[video_index]); @@ -341,26 +361,84 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co msi_hangup(call->msi_call); /* No mather the case, terminate the call */ + call_kill_transmission(call); call_remove(call); } break; case TOXAV_CALL_CONTROL_MUTE_AUDIO: { - if (call->msi_call->self_capabilities & msi_CapRAudio || - call->msi_call->self_capabilities & msi_CapSAudio) { - - uint8_t capabilities = call->msi_call->self_capabilities; - capabilities ^= msi_CapRAudio; - capabilities ^= msi_CapRAudio; - - if (msi_change_capabilities(call->msi_call, call->msi_call->self_capabilities) == -1) - return false; + if (!call->active) { + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } + + if (call->msi_call->self_capabilities & msi_CapRAudio) { + if (msi_change_capabilities(call->msi_call, call-> + msi_call->self_capabilities ^ msi_CapRAudio) == -1) { + /* The only reason for this function to fail is invalid state + * ( not active ) */ + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } rtp_stop_receiving(call->rtps[audio_index]); } } break; case TOXAV_CALL_CONTROL_MUTE_VIDEO: { + if (!call->active) { + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } + if (call->msi_call->self_capabilities & msi_CapRVideo) { + if (msi_change_capabilities(call->msi_call, call-> + msi_call->self_capabilities ^ msi_CapRVideo) == -1) { + /* The only reason for this function to fail is invalid state + * ( not active ) */ + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } + + rtp_stop_receiving(call->rtps[video_index]); + } + } break; + + case TOXAV_CALL_CONTROL_UNMUTE_AUDIO: { + if (!call->active) { + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } + + if (call->msi_call->self_capabilities & ~msi_CapRAudio) { + if (msi_change_capabilities(call->msi_call, call-> + msi_call->self_capabilities | msi_CapRAudio) == -1) { + /* The only reason for this function to fail is invalid state + * ( not active ) */ + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } + + rtp_start_receiving(call->rtps[audio_index]); + } + } break; + + case TOXAV_CALL_CONTROL_UNMUTE_VIDEO: { + if (!call->active) { + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } + + if (call->msi_call->self_capabilities & ~msi_CapRVideo) { + if (msi_change_capabilities(call->msi_call, call-> + msi_call->self_capabilities | msi_CapRVideo) == -1) { + /* The only reason for this function to fail is invalid state + * ( not active ) */ + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } + + rtp_start_receiving(call->rtps[video_index]); + } } break; } @@ -558,8 +636,6 @@ void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* * :: Internal * ******************************************************************************/ -/** TODO: - */ int callback_invite(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; @@ -586,15 +662,20 @@ int callback_start(void* toxav_inst, MSICall* call) ToxAVCall* av_call = call_get(toxav, call->friend_id); - if (av_call == NULL || !call_prepare_transmission(av_call)) { + if (av_call == NULL) { + /* Should this ever happen? */ + return -1; + } + + if (!call_prepare_transmission(av_call)) { + callback_error(toxav_inst, call); call_remove(av_call); return -1; } - TOXAV_CALL_STATE state = capabilities_to_state(av_call->msi_call->peer_capabilities); - if (toxav->scb.first) - toxav->scb.first(toxav, call->friend_id, state, toxav->scb.second); + toxav->scb.first(toxav, call->friend_id, + capabilities_to_state(call->peer_capabilities), toxav->scb.second); return 0; } @@ -603,18 +684,19 @@ int callback_end(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - call_kill_transmission(call->av_call); - call_remove(call->av_call); - if (toxav->scb.first) toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_END, toxav->scb.second); + call_kill_transmission(call->av_call); + call_remove(call->av_call); + return 0; } int callback_error(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; + if (toxav->scb.first) toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_ERROR, toxav->scb.second); @@ -633,14 +715,16 @@ int callback_capabilites(void* toxav_inst, MSICall* call) TOXAV_CALL_STATE capabilities_to_state(uint8_t capabilities) { - if ((capabilities & msi_CapSAudio) && (capabilities & msi_CapSVideo)) + if (capabilities == 0) + return TOXAV_CALL_STATE_PAUSED; + else if ((capabilities & msi_CapSAudio) && (capabilities & msi_CapSVideo)) return TOXAV_CALL_STATE_SENDING_AV; else if (capabilities & msi_CapSAudio) return TOXAV_CALL_STATE_SENDING_A; else if (capabilities & msi_CapSVideo) return TOXAV_CALL_STATE_SENDING_V; - else - return TOXAV_CALL_STATE_PAUSED; + + return TOXAV_CALL_STATE_NOT_SENDING; } bool audio_bitrate_invalid(uint32_t bitrate) @@ -798,8 +882,6 @@ bool call_prepare_transmission(ToxAVCall* call) goto MUTEX_INIT_ERROR; } - uint8_t capabilities = call->msi_call->self_capabilities; - call->cs = cs_new(call->msi_call->peer_vfpsz); if ( !call->cs ) { @@ -813,8 +895,7 @@ bool call_prepare_transmission(ToxAVCall* call) memcpy(&call->cs->acb, &av->acb, sizeof(av->acb)); memcpy(&call->cs->vcb, &av->vcb, sizeof(av->vcb)); - if (capabilities & msi_CapSAudio || capabilities & msi_CapRAudio) { /* Prepare audio sending */ - + { /* Prepare audio */ call->rtps[audio_index] = rtp_new(rtp_TypeAudio, av->m, call->friend_id); if ( !call->rtps[audio_index] ) { @@ -824,22 +905,21 @@ bool call_prepare_transmission(ToxAVCall* call) call->rtps[audio_index]->cs = call->cs; - if (cs_enable_audio_sending(call->cs, call->s_audio_b, 2) != 0) { + /* Only enable sending if bitrate is defined */ + if (call->s_audio_b > 0 && cs_enable_audio_sending(call->cs, call->s_audio_b, 2) != 0) { LOGGER_WARNING("Failed to enable audio sending!"); goto FAILURE; } - if (capabilities & msi_CapRAudio) { - if (cs_enable_audio_receiving(call->cs) != 0) { - LOGGER_WARNING("Failed to enable audio receiving!"); - goto FAILURE; - } - - rtp_start_receiving(call->rtps[audio_index]); + if (cs_enable_audio_receiving(call->cs) != 0) { + LOGGER_WARNING("Failed to enable audio receiving!"); + goto FAILURE; } + + rtp_start_receiving(call->rtps[audio_index]); } - if (capabilities & msi_CapSVideo || capabilities & msi_CapRVideo) { /* Prepare video rtp */ + { /* Prepare video */ call->rtps[video_index] = rtp_new(rtp_TypeVideo, av->m, call->friend_id); if ( !call->rtps[video_index] ) { @@ -849,26 +929,26 @@ bool call_prepare_transmission(ToxAVCall* call) call->rtps[video_index]->cs = call->cs; - if (cs_enable_video_sending(call->cs, call->s_video_b) != 0) { + /* Only enable sending if bitrate is defined */ + if (call->s_video_b > 0 && cs_enable_video_sending(call->cs, call->s_video_b) != 0) { LOGGER_WARNING("Failed to enable video sending!"); goto FAILURE; } - if (capabilities & msi_CapRVideo) { - if (cs_enable_video_receiving(call->cs) != 0) { - LOGGER_WARNING("Failed to enable video receiving!"); - goto FAILURE; - } - - rtp_start_receiving(call->rtps[audio_index]); + + if (cs_enable_video_receiving(call->cs) != 0) { + LOGGER_WARNING("Failed to enable video receiving!"); + goto FAILURE; } + + rtp_start_receiving(call->rtps[audio_index]); } call->active = 1; pthread_mutex_unlock(call->mutex_control); return true; - FAILURE: +FAILURE: rtp_kill(call->rtps[audio_index]); call->rtps[audio_index] = NULL; rtp_kill(call->rtps[video_index]); @@ -883,7 +963,7 @@ bool call_prepare_transmission(ToxAVCall* call) pthread_mutex_unlock(call->mutex_control); return false; - MUTEX_INIT_ERROR: +MUTEX_INIT_ERROR: pthread_mutex_unlock(call->mutex_control); LOGGER_ERROR("Mutex initialization failed!\n"); return false; diff --git a/toxav/toxav.h b/toxav/toxav.h index 5e82faa6a..ec232ddd1 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -246,7 +246,7 @@ void toxav_callback_call_state(ToxAV *av, toxav_call_state_cb *function, void *u typedef enum TOXAV_CALL_CONTROL { /** * Resume a previously paused call. Only valid if the pause was caused by this - * client. Not valid before the call is accepted. + * client, if not, this control is ignored. Not valid before the call is accepted. */ TOXAV_CALL_CONTROL_RESUME, /** @@ -269,7 +269,19 @@ typedef enum TOXAV_CALL_CONTROL { * compliance, this will cause the `receive_video_frame` event to stop being * triggered on receiving an video frame from the friend. */ - TOXAV_CALL_CONTROL_MUTE_VIDEO + TOXAV_CALL_CONTROL_MUTE_VIDEO, + /** + * Notify the friend that we are AGAIN ready to handle incoming audio. + * This control will not work if the call is not started with audio + * initiated. + */ + TOXAV_CALL_CONTROL_UNMUTE_AUDIO, + /** + * Notify the friend that we are AGAIN ready to handle incoming video. + * This control will not work if the call is not started with video + * initiated. + */ + TOXAV_CALL_CONTROL_UNMUTE_VIDEO } TOXAV_CALL_CONTROL; typedef enum TOXAV_ERR_CALL_CONTROL { TOXAV_ERR_CALL_CONTROL_OK, @@ -296,7 +308,11 @@ typedef enum TOXAV_ERR_CALL_CONTROL { * other party paused the call. The call will resume after both parties sent * the RESUME control. */ - TOXAV_ERR_CALL_CONTROL_ALREADY_PAUSED + TOXAV_ERR_CALL_CONTROL_ALREADY_PAUSED, + /** + * Tried to unmute a call that was not already muted. + */ + TOXAV_ERR_CALL_CONTROL_NOT_MUTED } TOXAV_ERR_CALL_CONTROL; /** * Sends a call control command to a friend. From 45e8807c1e693c105b97784d15b7eb19bcc87918 Mon Sep 17 00:00:00 2001 From: mannol Date: Sun, 1 Mar 2015 18:45:04 +0100 Subject: [PATCH 12/97] Make toxav thread safe --- toxav/av_test.c | 53 +++++---- toxav/codec.c | 32 ++--- toxav/msi.c | 111 +++++++++++++---- toxav/msi.h | 19 +-- toxav/toxav.c | 308 +++++++++++++++++++++++++++--------------------- toxav/toxav.h | 66 +++++------ 6 files changed, 339 insertions(+), 250 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 41f5a7585..bb79eedc9 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -16,7 +16,7 @@ typedef struct { bool incoming; - TOXAV_CALL_STATE state; + uint32_t state; } CallControl; const char* stringify_state(TOXAV_CALL_STATE s) @@ -44,9 +44,9 @@ void t_toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool printf("Handling CALL callback\n"); ((CallControl*)user_data)->incoming = true; } -void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, TOXAV_CALL_STATE state, void *user_data) +void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, void *user_data) { - printf("Handling CALL STATE callback: %s\n", stringify_state(state)); + printf("Handling CALL STATE callback: %d\n", state); ((CallControl*)user_data)->state = state; } @@ -211,13 +211,13 @@ int main (int argc, char** argv) } printf("\nTrying regular call (Audio and Video)...\n"); -// REGULAR_CALL_FLOW(48, 4000); + REGULAR_CALL_FLOW(48, 4000); printf("\nTrying regular call (Audio only)...\n"); -// REGULAR_CALL_FLOW(48, 0); + REGULAR_CALL_FLOW(48, 0); printf("\nTrying regular call (Video only)...\n"); -// REGULAR_CALL_FLOW(0, 4000); + REGULAR_CALL_FLOW(0, 4000); #undef REGULAR_CALL_FLOW @@ -318,10 +318,8 @@ int main (int argc, char** argv) /* At first try all stuff while in invalid state */ assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); - assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_AUDIO, NULL)); - assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_VIDEO, NULL)); - assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_AUDIO, NULL)); - assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_VIDEO, NULL)); + assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); + assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); { TOXAV_ERR_ANSWER rc; @@ -342,36 +340,36 @@ int main (int argc, char** argv) assert(BobCC.state == TOXAV_CALL_STATE_PAUSED); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); iterate(Bsn, AliceAV, BobAV); - assert(BobCC.state == TOXAV_CALL_STATE_SENDING_AV); + assert(BobCC.state & (TOXAV_CALL_STATE_SENDING_A | TOXAV_CALL_STATE_SENDING_V)); /* Mute/Unmute single */ printf("Mute/Unmute single\n"); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_AUDIO, NULL)); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); iterate(Bsn, AliceAV, BobAV); - assert(BobCC.state == TOXAV_CALL_CONTROL_MUTE_AUDIO); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_AUDIO, NULL)); + assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); iterate(Bsn, AliceAV, BobAV); - assert(BobCC.state == TOXAV_CALL_CONTROL_UNMUTE_AUDIO); + assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); /* Mute/Unmute both */ printf("Mute/Unmute both\n"); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_AUDIO, NULL)); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); iterate(Bsn, AliceAV, BobAV); - assert(BobCC.state == TOXAV_CALL_STATE_SENDING_V); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_VIDEO, NULL)); + assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); iterate(Bsn, AliceAV, BobAV); - assert(BobCC.state == TOXAV_CALL_STATE_NOT_SENDING); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_AUDIO, NULL)); + assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_V); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); iterate(Bsn, AliceAV, BobAV); - assert(BobCC.state == TOXAV_CALL_STATE_SENDING_A); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_VIDEO, NULL)); + assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); iterate(Bsn, AliceAV, BobAV); - assert(BobCC.state == TOXAV_CALL_STATE_SENDING_AV); + assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_V); { TOXAV_ERR_CALL_CONTROL rc; toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); - + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { printf("toxav_call_control failed: %d\n", rc); exit(1); @@ -384,6 +382,13 @@ int main (int argc, char** argv) printf("Success!\n"); } + + toxav_kill(BobAV); + toxav_kill(AliceAV); + tox_kill(Bob); + tox_kill(Alice); + tox_kill(Bsn); + printf("\nTest successful!\n"); return 0; } \ No newline at end of file diff --git a/toxav/codec.c b/toxav/codec.c index e44387dfe..645f7188d 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -343,21 +343,17 @@ CSSession *cs_new(uint32_t peer_video_frame_piece_size) return NULL; } + + if (create_recursive_mutex(cs->queue_mutex) != 0) { + LOGGER_WARNING("Failed to create recursive mutex!"); + free(cs); + return NULL; + } + + cs->peer_video_frame_piece_size = peer_video_frame_piece_size; return cs; - - FAILURE: - LOGGER_WARNING("Error initializing codec session! Application might misbehave!"); - - cs_disable_audio_sending(cs); - cs_disable_audio_receiving(cs); - cs_disable_video_sending(cs); - cs_disable_video_receiving(cs); - - free(cs); - - return NULL; } void cs_kill(CSSession *cs) @@ -374,6 +370,8 @@ void cs_kill(CSSession *cs) cs_disable_video_sending(cs); cs_disable_video_receiving(cs); + pthread_mutex_destroy(cs->queue_mutex); + LOGGER_DEBUG("Terminated codec state: %p", cs); free(cs); } @@ -536,19 +534,12 @@ int cs_enable_video_receiving(CSSession* cs) { if (cs->v_decoding) return 0; - - if (create_recursive_mutex(cs->queue_mutex) != 0) { - LOGGER_WARNING("Failed to create recursive mutex!"); - return -1; - } - + int rc = vpx_codec_dec_init_ver(cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, NULL, 0, VPX_DECODER_ABI_VERSION); if ( rc != VPX_CODEC_OK) { LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); - - pthread_mutex_destroy(cs->queue_mutex); return -1; } @@ -591,7 +582,6 @@ void cs_disable_video_receiving(CSSession* cs) cs->frame_buf = NULL; vpx_codec_destroy(cs->v_decoder); - pthread_mutex_destroy(cs->queue_mutex); } } diff --git a/toxav/msi.c b/toxav/msi.c index f179a7aba..ac900dac5 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -99,7 +99,9 @@ void handle_msi_packet ( Messenger *m, int friend_id, const uint8_t *data, uint1 */ void msi_register_callback ( MSISession *session, MSICallbackType callback, MSICallbackID id) { + pthread_mutex_lock(session->mutex); session->callbacks[id] = callback; + pthread_mutex_unlock(session->mutex); } MSISession *msi_new ( Messenger *messenger ) { @@ -163,15 +165,19 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ { LOGGER_DEBUG("Session: %p Inviting friend: %u", session, friend_id); + pthread_mutex_lock(session->mutex); if (get_call(session, friend_id) != NULL) { LOGGER_ERROR("Already in a call"); + pthread_mutex_unlock(session->mutex); return -1; } (*call) = new_call ( session, friend_id ); - if ( *call == NULL ) + if ( *call == NULL ) { + pthread_mutex_unlock(session->mutex); return -1; + } (*call)->self_capabilities = capabilities; @@ -180,7 +186,7 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ msg.capabilities.exists = true; msg.capabilities.value = capabilities; - + msg.vfpsz.exists = true; msg.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE); @@ -189,28 +195,37 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ (*call)->state = msi_CallRequesting; LOGGER_DEBUG("Invite sent"); + pthread_mutex_unlock(session->mutex); return 0; } int msi_hangup ( MSICall* call ) { LOGGER_DEBUG("Session: %p Hanging up call with friend: %u", call->session, call->friend_id); + MSISession* session = call->session; + pthread_mutex_lock(session->mutex); + MSIMessage msg; msg_init(&msg, requ_pop); - send_message ( call->session->messenger, call->friend_id, &msg ); + send_message ( session->messenger, call->friend_id, &msg ); kill_call(call); + pthread_mutex_unlock(session->mutex); return 0; } int msi_answer ( MSICall* call, uint8_t capabilities ) { LOGGER_DEBUG("Session: %p Answering call from: %u", call->session, call->friend_id); + MSISession* session = call->session; + pthread_mutex_lock(session->mutex); + if ( call->state != msi_CallRequested ) { /* Though sending in invalid state will not cause anything wierd * Its better to not do it like a maniac */ LOGGER_ERROR("Call is in invalid state!"); + pthread_mutex_unlock(session->mutex); return -1; } @@ -225,9 +240,10 @@ int msi_answer ( MSICall* call, uint8_t capabilities ) msg.vfpsz.exists = true; msg.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE); - send_message ( call->session->messenger, call->friend_id, &msg ); + send_message ( session->messenger, call->friend_id, &msg ); call->state = msi_CallActive; + pthread_mutex_unlock(session->mutex); return 0; } @@ -235,6 +251,9 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) { LOGGER_DEBUG("Session: %p Trying to change capabilities to friend %u", call->session, call->friend_id); + MSISession* session = call->session; + pthread_mutex_lock(session->mutex); + if ( call->state != msi_CallActive ) { /* Sending capabilities change can cause error on other side if * the call is not active since we don't send header 'vfpsz'. @@ -244,6 +263,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) * like new. TODO: explain this better */ LOGGER_ERROR("Call is in invalid state!"); + pthread_mutex_unlock(session->mutex); return -1; } @@ -257,6 +277,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) send_message ( call->session->messenger, call->friend_id, &msg ); + pthread_mutex_unlock(session->mutex); return 0; } @@ -316,7 +337,7 @@ int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) case IDError: CHECK_SIZE(it, size_constraint, 1); - CHECK_ENUM_HIGH(it, msi_ErrUndisclosed); + CHECK_ENUM_HIGH(it, msi_EUndisclosed); SET_UINT8(it, dest->error); break; @@ -440,7 +461,7 @@ int invoke_callback(MSICall* call, MSICallbackID cb) if ( call->session->callbacks[cb] ) { LOGGER_DEBUG("Invoking callback function: %d", cb); if ( call->session->callbacks[cb] ( call->session->av, call ) != 0 ) { - LOGGER_WARNING("Callback handling failed, sending error"); + LOGGER_WARNING("Callback state handling failed, sending error"); goto FAILURE; } @@ -449,10 +470,11 @@ int invoke_callback(MSICall* call, MSICallbackID cb) FAILURE: /* If no callback present or error happened while handling, - * an error message will be send to friend + * an error message will be sent to friend */ - call->error = msi_HandleError; + if (call->error == msi_ENone) + call->error = msi_EHandle; return -1; } static MSICall *get_call ( MSISession *session, uint32_t friend_id ) @@ -517,9 +539,12 @@ MSICall *new_call ( MSISession *session, uint32_t friend_id ) } void kill_call ( MSICall *call ) { + /* Assume that session mutex is locked */ if ( call == NULL ) return; + LOGGER_DEBUG("Killing call: %p", call); + MSISession* session = call->session; MSICall* prev = call->prev; @@ -529,23 +554,23 @@ void kill_call ( MSICall *call ) prev->next = next; else if (next) session->calls_head = next->friend_id; - else goto CLEAR; + else goto CLEAR_CONTAINER; if (next) next->prev = prev; else if (prev) session->calls_tail = prev->friend_id; - else goto CLEAR; + else goto CLEAR_CONTAINER; session->calls[call->friend_id] = NULL; free(call); return; -CLEAR: +CLEAR_CONTAINER: session->calls_head = session->calls_tail = 0; free(session->calls); - session->calls = NULL; free(call); + session->calls = NULL; } void on_peer_status(Messenger *m, int friend_id, uint8_t status, void *data) { @@ -556,13 +581,17 @@ void on_peer_status(Messenger *m, int friend_id, uint8_t status, void *data) case 0: { /* Friend is now offline */ LOGGER_DEBUG("Friend %d is now offline", friend_id); + pthread_mutex_lock(session->mutex); MSICall* call = get_call(session, friend_id); - if (call == NULL) + if (call == NULL) { + pthread_mutex_unlock(session->mutex); return; + } invoke_callback(call, msi_OnPeerTimeout); /* Failure is ignored */ kill_call(call); + pthread_mutex_unlock(session->mutex); } break; @@ -580,18 +609,17 @@ void handle_push ( MSICall *call, const MSIMessage *msg ) if (!msg->capabilities.exists) { LOGGER_WARNING("Session: %p Invalid capabilities on 'push'"); - call->error = msi_InvalidMessage; + call->error = msi_EInvalidMessage; goto FAILURE; } if (call->state != msi_CallActive) { if (!msg->vfpsz.exists) { LOGGER_WARNING("Session: %p Invalid vfpsz on 'push'"); - call->error = msi_InvalidMessage; + call->error = msi_EInvalidMessage; goto FAILURE; } - /* Sending video frame piece size is ignored when call is active */ call->peer_vfpsz = ntohs(msg->vfpsz.value); } @@ -610,6 +638,38 @@ void handle_push ( MSICall *call, const MSIMessage *msg ) } break; case msi_CallActive: { + if (msg->vfpsz.exists) { + /* If peer sended video frame piece size + * while the call is already active it's probable + * that he is trying to re-call us while the call + * is not terminated on our side. We can assume that + * in this case we can automatically answer the re-call. + */ + if (call->peer_vfpsz != ntohs(msg->vfpsz.value)) { + LOGGER_WARNING("Friend sent invalid parameters for re-call"); + call->error = msi_EInvalidParam; + invoke_callback(call, msi_OnError); + goto FAILURE; + } + + LOGGER_INFO("Friend is recalling us"); + + MSIMessage msg; + msg_init(&msg, requ_push); + + msg.capabilities.exists = true; + msg.capabilities.value = call->self_capabilities; + + msg.vfpsz.exists = true; + msg.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE); + + send_message ( call->session->messenger, call->friend_id, &msg ); + + /* If peer changed capabilities during re-call they will + * be handled accordingly during the next step + */ + } + /* Only act if capabilities changed */ if ( call->peer_capabilities != msg->capabilities.value) { LOGGER_INFO("Friend is changing capabilities"); @@ -629,6 +689,7 @@ void handle_push ( MSICall *call, const MSIMessage *msg ) if ( invoke_callback(call, msi_OnStart) == -1 ) goto FAILURE; + } break; case msi_CallRequested: { @@ -646,7 +707,7 @@ FAILURE: void handle_pop ( MSICall *call, const MSIMessage *msg ) { assert(call); - + LOGGER_DEBUG("Session: %p Handling 'pop', friend id: %d", call->session, call->friend_id); /* callback errors are ignored */ @@ -682,8 +743,6 @@ void handle_pop ( MSICall *call, const MSIMessage *msg ) } kill_call ( call ); - - return; } void handle_msi_packet ( Messenger *m, int friend_id, const uint8_t *data, uint16_t length, void *object ) { @@ -694,30 +753,34 @@ void handle_msi_packet ( Messenger *m, int friend_id, const uint8_t *data, uint1 if ( msg_parse_in ( &msg, data, length ) == -1 ) { LOGGER_WARNING("Error parsing message"); - send_error(m, friend_id, msi_InvalidMessage); + send_error(m, friend_id, msi_EInvalidMessage); return; } else { LOGGER_DEBUG("Successfully parsed message"); } + pthread_mutex_lock(session->mutex); MSICall *call = get_call(session, friend_id); if (call == NULL) { if (msg.request.value != requ_push) { - send_error(m, friend_id, msi_StrayMessage); + send_error(m, friend_id, msi_EStrayMessage); + pthread_mutex_unlock(session->mutex); return; } call = new_call(session, friend_id); if (call == NULL) { - send_error(m, friend_id, msi_SystemError); + send_error(m, friend_id, msi_ESystem); + pthread_mutex_unlock(session->mutex); return; } } - - if (msg.request.value == requ_push) + if (msg.request.value == requ_push) handle_push(call, &msg); else handle_pop(call, &msg); /* always kills the call */ + + pthread_mutex_unlock(session->mutex); } \ No newline at end of file diff --git a/toxav/msi.h b/toxav/msi.h index b846542d1..8404df191 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -35,14 +35,14 @@ * Error codes. */ typedef enum { - msi_ErrorNone, - msi_InvalidMessage, - msi_InvalidParam, - msi_InvalidState, - msi_StrayMessage, - msi_SystemError, - msi_HandleError, - msi_ErrUndisclosed, /* NOTE: must be last enum otherwise parsing wont work */ + msi_ENone, + msi_EInvalidMessage, + msi_EInvalidParam, + msi_EInvalidState, + msi_EStrayMessage, + msi_ESystem, + msi_EHandle, + msi_EUndisclosed, /* NOTE: must be last enum otherwise parsing wont work */ } MSIError; /** @@ -118,6 +118,9 @@ typedef struct MSISession_s { void *av; Messenger *messenger; + /* The mutex controls async access from control + * thread(s) and core thread. + */ pthread_mutex_t mutex[1]; MSICallbackType callbacks[7]; } MSISession; diff --git a/toxav/toxav.c b/toxav/toxav.c index e6b51ee4b..78243ae67 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -40,15 +40,16 @@ enum { video_index, }; -typedef struct ToxAVCall_s -{ +typedef struct ToxAVCall_s { ToxAV* av; - pthread_mutex_t mutex_control[1]; - pthread_mutex_t mutex_encoding_audio[1]; - pthread_mutex_t mutex_encoding_video[1]; - pthread_mutex_t mutex_do[1]; RTPSession *rtps[2]; /* Audio is first and video is second */ CSSession *cs; + + pthread_mutex_t mutex_audio_sending[1]; + pthread_mutex_t mutex_video_sending[1]; + /* Only audio or video can be decoded at one time */ + pthread_mutex_t mutex_decoding[1]; + bool active; MSICall* msi_call; uint32_t friend_id; @@ -56,14 +57,14 @@ typedef struct ToxAVCall_s uint32_t s_audio_b; /* Sending audio bitrate */ uint32_t s_video_b; /* Sending video bitrate */ - uint8_t last_capabilities; + uint8_t last_self_capabilities; + uint8_t last_peer_capabilities; struct ToxAVCall_s *prev; struct ToxAVCall_s *next; } ToxAVCall; -struct toxAV -{ +struct toxAV { Messenger* m; MSISession* msi; @@ -71,11 +72,14 @@ struct toxAV ToxAVCall** calls; uint32_t calls_tail; uint32_t calls_head; + pthread_mutex_t mutex[1]; PAIR(toxav_call_cb *, void*) ccb; /* Call callback */ PAIR(toxav_call_state_cb *, void *) scb; /* Call state callback */ PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ + PAIR(toxav_request_video_frame_cb *, void *) rvcb; /* Request video callback */ + PAIR(toxav_request_audio_frame_cb *, void *) racb; /* Request video callback */ /** Decode time measures */ int32_t dmssc; /** Measure count */ @@ -92,7 +96,6 @@ int callback_end(void* toxav_inst, MSICall* call); int callback_error(void* toxav_inst, MSICall* call); int callback_capabilites(void* toxav_inst, MSICall* call); -TOXAV_CALL_STATE capabilities_to_state(uint8_t capabilities); ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); ToxAVCall* call_get(ToxAV* av, uint32_t friend_number); void call_remove(ToxAVCall* call); @@ -118,7 +121,7 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) goto FAILURE; } - av = calloc ( sizeof(ToxAV), 1); + av = calloc (sizeof(ToxAV), 1); if (av == NULL) { LOGGER_WARNING("Allocation failed!"); @@ -126,10 +129,17 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) goto FAILURE; } + if (create_recursive_mutex(av->mutex) == -1) { + LOGGER_WARNING("Mutex creation failed!"); + rc = TOXAV_ERR_NEW_MALLOC; + goto FAILURE; + } + av->m = (Messenger *)tox; av->msi = msi_new(av->m); if (av->msi == NULL) { + pthread_mutex_destroy(av->mutex); rc = TOXAV_ERR_NEW_MALLOC; goto FAILURE; } @@ -163,6 +173,7 @@ void toxav_kill(ToxAV* av) { if (av == NULL) return; + pthread_mutex_lock(av->mutex); msi_kill(av->msi); @@ -175,6 +186,8 @@ void toxav_kill(ToxAV* av) } } + pthread_mutex_unlock(av->mutex); + pthread_mutex_destroy(av->mutex); free(av); } @@ -185,6 +198,7 @@ Tox* toxav_get_tox(ToxAV* av) uint32_t toxav_iteration_interval(const ToxAV* av) { + /* If no call is active interval is 200 */ return av->calls ? av->interval : 200; } @@ -194,14 +208,28 @@ void toxav_iteration(ToxAV* av) return; uint64_t start = current_time_monotonic(); - uint32_t rc = 200 + av->dmssa; /* If no call is active interval is 200 */ + uint32_t rc = 0; + pthread_mutex_lock(av->mutex); ToxAVCall* i = av->calls[av->calls_head]; for (; i; i = i->next) { if (i->active) { + pthread_mutex_lock(i->mutex_decoding); + + /* TODO make AV synchronisation */ + if (av->racb.first) + av->racb.first(av, i->friend_id, av->racb.second); + if (av->rvcb.first) + av->rvcb.first(av, i->friend_id, av->rvcb.second); + + pthread_mutex_unlock(av->mutex); cs_do(i->cs); rc = MIN(i->cs->last_packet_frame_duration, rc); - } + pthread_mutex_unlock(i->mutex_decoding); + } else + continue; + + pthread_mutex_lock(av->mutex); } av->interval = rc < av->dmssa ? 0 : rc - av->dmssa; @@ -216,38 +244,47 @@ void toxav_iteration(ToxAV* av) bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) { + pthread_mutex_lock(av->mutex); ToxAVCall* call = call_new(av, friend_number, error); - if (call == NULL) + if (call == NULL) { + pthread_mutex_unlock(av->mutex); return false; + } call->s_audio_b = audio_bit_rate; call->s_video_b = video_bit_rate; - call->last_capabilities = msi_CapRAudio | msi_CapRVideo; + call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; - call->last_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; - call->last_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; + call->last_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; + call->last_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; - if (msi_invite(av->msi, &call->msi_call, friend_number, call->last_capabilities) != 0) { + if (msi_invite(av->msi, &call->msi_call, friend_number, call->last_self_capabilities) != 0) { call_remove(call); if (error) *error = TOXAV_ERR_CALL_MALLOC; + pthread_mutex_unlock(av->mutex); return false; } call->msi_call->av_call = call; + pthread_mutex_unlock(av->mutex); return true; } void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data) { + pthread_mutex_lock(av->mutex); av->ccb.first = function; av->ccb.second = user_data; + pthread_mutex_unlock(av->mutex); } bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER* error) { + pthread_mutex_lock(av->mutex); + TOXAV_ERR_ANSWER rc = TOXAV_ERR_ANSWER_OK; if (m_friend_exists(av->m, friend_number) == 0) { rc = TOXAV_ERR_ANSWER_FRIEND_NOT_FOUND; @@ -270,16 +307,18 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui call->s_audio_b = audio_bit_rate; call->s_video_b = video_bit_rate; - call->last_capabilities = msi_CapRAudio | msi_CapRVideo; + call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; - call->last_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; - call->last_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; + call->last_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; + call->last_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; - if (msi_answer(call->msi_call, call->last_capabilities) != 0) + if (msi_answer(call->msi_call, call->last_self_capabilities) != 0) rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; /* the only reason for msi_answer to fail */ END: + pthread_mutex_unlock(av->mutex); + if (error) *error = rc; @@ -288,12 +327,15 @@ END: void toxav_callback_call_state(ToxAV* av, toxav_call_state_cb* function, void* user_data) { + pthread_mutex_lock(av->mutex); av->scb.first = function; av->scb.second = user_data; + pthread_mutex_unlock(av->mutex); } bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL* error) { + pthread_mutex_lock(av->mutex); TOXAV_ERR_CALL_CONTROL rc = TOXAV_ERR_CALL_CONTROL_OK; if (m_friend_exists(av->m, friend_number) == 0) { @@ -308,9 +350,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co goto END; } - /* TODO rest of these */ - switch (control) - { + switch (control) { case TOXAV_CALL_CONTROL_RESUME: { if (!call->active) { rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; @@ -319,10 +359,10 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co /* Only act if paused and had media transfer active before */ if (call->msi_call->self_capabilities == 0 && - call->last_capabilities ) { + call->last_self_capabilities ) { if (msi_change_capabilities(call->msi_call, - call->last_capabilities) == -1) { + call->last_self_capabilities) == -1) { /* The only reason for this function to fail is invalid state * ( not active ) */ rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; @@ -342,7 +382,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co /* Only act if not already paused */ if (call->msi_call->self_capabilities) { - call->last_capabilities = call->msi_call->self_capabilities; + call->last_self_capabilities = call->msi_call->self_capabilities; if (msi_change_capabilities(call->msi_call, 0) == -1 ) { /* The only reason for this function to fail is invalid state @@ -365,7 +405,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co call_remove(call); } break; - case TOXAV_CALL_CONTROL_MUTE_AUDIO: { + case TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO: { if (!call->active) { rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; goto END; @@ -381,10 +421,23 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co } rtp_stop_receiving(call->rtps[audio_index]); + } else { + /* This call was already muted so notify the friend that he can + * start sending audio again + */ + if (msi_change_capabilities(call->msi_call, call-> + msi_call->self_capabilities | msi_CapRAudio) == -1) { + /* The only reason for this function to fail is invalid state + * ( not active ) */ + rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + goto END; + } + + rtp_start_receiving(call->rtps[audio_index]); } } break; - case TOXAV_CALL_CONTROL_MUTE_VIDEO: { + case TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO: { if (!call->active) { rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; goto END; @@ -400,35 +453,10 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co } rtp_stop_receiving(call->rtps[video_index]); - } - } break; - - case TOXAV_CALL_CONTROL_UNMUTE_AUDIO: { - if (!call->active) { - rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; - goto END; - } - - if (call->msi_call->self_capabilities & ~msi_CapRAudio) { - if (msi_change_capabilities(call->msi_call, call-> - msi_call->self_capabilities | msi_CapRAudio) == -1) { - /* The only reason for this function to fail is invalid state - * ( not active ) */ - rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; - goto END; - } - - rtp_start_receiving(call->rtps[audio_index]); - } - } break; - - case TOXAV_CALL_CONTROL_UNMUTE_VIDEO: { - if (!call->active) { - rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; - goto END; - } - - if (call->msi_call->self_capabilities & ~msi_CapRVideo) { + } else { + /* This call was already muted so notify the friend that he can + * start sending video again + */ if (msi_change_capabilities(call->msi_call, call-> msi_call->self_capabilities | msi_CapRVideo) == -1) { /* The only reason for this function to fail is invalid state @@ -443,6 +471,8 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co } END: + pthread_mutex_unlock(av->mutex); + if (error) *error = rc; @@ -461,7 +491,10 @@ bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_ void toxav_callback_request_video_frame(ToxAV* av, toxav_request_video_frame_cb* function, void* user_data) { - /* TODO */ + pthread_mutex_lock(av->mutex); + av->rvcb.first = function; + av->rvcb.second = user_data; + pthread_mutex_unlock(av->mutex); } bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) @@ -474,24 +507,32 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u goto END; } + pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); - if (call == NULL) { + if (call == NULL || !call->active) { + pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; } + pthread_mutex_lock(call->mutex_video_sending); + pthread_mutex_unlock(av->mutex); + if (call->msi_call->state != msi_CallActive) { /* TODO */ + pthread_mutex_unlock(call->mutex_video_sending); rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; goto END; } if ( y == NULL || u == NULL || v == NULL ) { + pthread_mutex_unlock(call->mutex_video_sending); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } if ( cs_set_sending_video_resolution(call->cs, width, height) != 0 ) { + pthread_mutex_unlock(call->mutex_video_sending); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } @@ -513,6 +554,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u vpx_img_free(&img); /* FIXME don't free? */ if ( vrc != VPX_CODEC_OK) { + pthread_mutex_unlock(call->mutex_video_sending); LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; @@ -542,13 +584,18 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u for (i = 0; i < parts; i++) { iter = cs_iterate_split_video_frame(call->cs, &part_size); - if (rtp_send_msg(call->rtps[video_index], iter, part_size) < 0) + if (rtp_send_msg(call->rtps[video_index], iter, part_size) < 0) { + pthread_mutex_unlock(call->mutex_video_sending); + LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); goto END; + } } } } } + pthread_mutex_unlock(call->mutex_video_sending); + END: if (error) *error = rc; @@ -558,7 +605,10 @@ END: void toxav_callback_request_audio_frame(ToxAV* av, toxav_request_audio_frame_cb* function, void* user_data) { - /* TODO */ + pthread_mutex_lock(av->mutex); + av->racb.first = function; + av->racb.second = user_data; + pthread_mutex_unlock(av->mutex); } bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error) @@ -571,24 +621,32 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } + pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); - if (call == NULL) { + if (call == NULL || !call->active) { + pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; } + pthread_mutex_lock(call->mutex_audio_sending); + pthread_mutex_unlock(av->mutex); + if (call->msi_call->state != msi_CallActive) { /* TODO */ + pthread_mutex_unlock(call->mutex_audio_sending); rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; goto END; } if ( pcm == NULL ) { + pthread_mutex_unlock(call->mutex_audio_sending); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } if ( channels != 1 || channels != 2 ) { + pthread_mutex_unlock(call->mutex_audio_sending); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } @@ -604,6 +662,7 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc if (vrc < 0) { LOGGER_WARNING("Failed to encode frame"); rc = TOXAV_ERR_SEND_FRAME_INVALID; + pthread_mutex_unlock(call->mutex_audio_sending); goto END; } @@ -611,6 +670,8 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc /* TODO check for error? */ } + pthread_mutex_unlock(call->mutex_audio_sending); + END: if (error) *error = rc; @@ -620,14 +681,18 @@ END: void toxav_callback_receive_video_frame(ToxAV* av, toxav_receive_video_frame_cb* function, void* user_data) { + pthread_mutex_lock(av->mutex); av->vcb.first = function; av->vcb.second = user_data; + pthread_mutex_unlock(av->mutex); } void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* function, void* user_data) { + pthread_mutex_lock(av->mutex); av->acb.first = function; av->acb.second = user_data; + pthread_mutex_unlock(av->mutex); } @@ -639,10 +704,12 @@ void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* int callback_invite(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; + pthread_mutex_lock(toxav->mutex); ToxAVCall* av_call = call_new(toxav, call->friend_id, NULL); if (av_call == NULL) { LOGGER_WARNING("Failed to initialize call..."); + pthread_mutex_unlock(toxav->mutex); return -1; } @@ -653,36 +720,41 @@ int callback_invite(void* toxav_inst, MSICall* call) toxav->ccb.first(toxav, call->friend_id, call->peer_capabilities & msi_CapSAudio, call->peer_capabilities & msi_CapSVideo, toxav->ccb.second); + pthread_mutex_unlock(toxav->mutex); return 0; } int callback_start(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; + pthread_mutex_lock(toxav->mutex); ToxAVCall* av_call = call_get(toxav, call->friend_id); if (av_call == NULL) { /* Should this ever happen? */ + pthread_mutex_unlock(toxav->mutex); return -1; } if (!call_prepare_transmission(av_call)) { callback_error(toxav_inst, call); call_remove(av_call); + pthread_mutex_unlock(toxav->mutex); return -1; } if (toxav->scb.first) - toxav->scb.first(toxav, call->friend_id, - capabilities_to_state(call->peer_capabilities), toxav->scb.second); + toxav->scb.first(toxav, call->friend_id, call->peer_capabilities, toxav->scb.second); + pthread_mutex_unlock(toxav->mutex); return 0; } int callback_end(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; + pthread_mutex_lock(toxav->mutex); if (toxav->scb.first) toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_END, toxav->scb.second); @@ -690,43 +762,39 @@ int callback_end(void* toxav_inst, MSICall* call) call_kill_transmission(call->av_call); call_remove(call->av_call); + pthread_mutex_unlock(toxav->mutex); return 0; } int callback_error(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; + pthread_mutex_lock(toxav->mutex); if (toxav->scb.first) toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_ERROR, toxav->scb.second); + call_kill_transmission(call->av_call); + call_remove(call->av_call); + + pthread_mutex_unlock(toxav->mutex); return 0; } int callback_capabilites(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; + pthread_mutex_lock(toxav->mutex); + + /* TODO modify cs? */ + if (toxav->scb.first) - toxav->scb.first(toxav, call->friend_id, - capabilities_to_state(call->peer_capabilities), toxav->scb.second); + toxav->scb.first(toxav, call->friend_id, call->peer_capabilities, toxav->scb.second); + pthread_mutex_unlock(toxav->mutex); return 0; } -TOXAV_CALL_STATE capabilities_to_state(uint8_t capabilities) -{ - if (capabilities == 0) - return TOXAV_CALL_STATE_PAUSED; - else if ((capabilities & msi_CapSAudio) && (capabilities & msi_CapSVideo)) - return TOXAV_CALL_STATE_SENDING_AV; - else if (capabilities & msi_CapSAudio) - return TOXAV_CALL_STATE_SENDING_A; - else if (capabilities & msi_CapSVideo) - return TOXAV_CALL_STATE_SENDING_V; - - return TOXAV_CALL_STATE_NOT_SENDING; -} - bool audio_bitrate_invalid(uint32_t bitrate) { /* Opus RFC 6716 section-2.1.1 dictates the following: @@ -743,6 +811,7 @@ bool video_bitrate_invalid(uint32_t bitrate) ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) { + /* Assumes mutex locked */ TOXAV_ERR_CALL rc = TOXAV_ERR_CALL_OK; ToxAVCall* call = NULL; @@ -772,28 +841,10 @@ ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) call->av = av; call->friend_id = friend_number; - if (create_recursive_mutex(call->mutex_control) != 0) { - free(call); - call = NULL; - rc = TOXAV_ERR_CALL_MALLOC; - goto END; - } - - if (create_recursive_mutex(call->mutex_do) != 0) { - pthread_mutex_destroy(call->mutex_control); - free(call); - call = NULL; - rc = TOXAV_ERR_CALL_MALLOC; - goto END; - } - - if (av->calls == NULL) { /* Creating */ av->calls = calloc (sizeof(ToxAVCall*), friend_number + 1); if (av->calls == NULL) { - pthread_mutex_destroy(call->mutex_control); - pthread_mutex_destroy(call->mutex_do); free(call); call = NULL; rc = TOXAV_ERR_CALL_MALLOC; @@ -806,8 +857,6 @@ ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) void* tmp = realloc(av->calls, sizeof(ToxAVCall*) * friend_number + 1); if (tmp == NULL) { - pthread_mutex_destroy(call->mutex_control); - pthread_mutex_destroy(call->mutex_do); free(call); call = NULL; rc = TOXAV_ERR_CALL_MALLOC; @@ -843,6 +892,7 @@ END: ToxAVCall* call_get(ToxAV* av, uint32_t friend_number) { + /* Assumes mutex locked */ if (av->calls == NULL || av->calls_tail < friend_number) return NULL; @@ -851,6 +901,8 @@ ToxAVCall* call_get(ToxAV* av, uint32_t friend_number) bool call_prepare_transmission(ToxAVCall* call) { + /* Assumes mutex locked */ + if (call == NULL) return false; @@ -860,25 +912,22 @@ bool call_prepare_transmission(ToxAVCall* call) /* It makes no sense to have CSession without callbacks */ return false; - pthread_mutex_lock(call->mutex_control); - if (call->active) { - pthread_mutex_unlock(call->mutex_control); LOGGER_WARNING("Call already active!\n"); return true; } - if (pthread_mutex_init(call->mutex_encoding_audio, NULL) != 0) + if (pthread_mutex_init(call->mutex_audio_sending, NULL) != 0) goto MUTEX_INIT_ERROR; - if (pthread_mutex_init(call->mutex_encoding_video, NULL) != 0) { - pthread_mutex_destroy(call->mutex_encoding_audio); + if (pthread_mutex_init(call->mutex_video_sending, NULL) != 0) { + pthread_mutex_destroy(call->mutex_audio_sending); goto MUTEX_INIT_ERROR; } - if (pthread_mutex_init(call->mutex_do, NULL) != 0) { - pthread_mutex_destroy(call->mutex_encoding_audio); - pthread_mutex_destroy(call->mutex_encoding_video); + if (pthread_mutex_init(call->mutex_decoding, NULL) != 0) { + pthread_mutex_destroy(call->mutex_audio_sending); + pthread_mutex_destroy(call->mutex_video_sending); goto MUTEX_INIT_ERROR; } @@ -945,7 +994,6 @@ bool call_prepare_transmission(ToxAVCall* call) } call->active = 1; - pthread_mutex_unlock(call->mutex_control); return true; FAILURE: @@ -955,16 +1003,12 @@ FAILURE: call->rtps[video_index] = NULL; cs_kill(call->cs); call->cs = NULL; - call->active = 0; - pthread_mutex_destroy(call->mutex_encoding_audio); - pthread_mutex_destroy(call->mutex_encoding_video); - pthread_mutex_destroy(call->mutex_do); - - pthread_mutex_unlock(call->mutex_control); + pthread_mutex_destroy(call->mutex_audio_sending); + pthread_mutex_destroy(call->mutex_video_sending); + pthread_mutex_destroy(call->mutex_decoding); return false; MUTEX_INIT_ERROR: - pthread_mutex_unlock(call->mutex_control); LOGGER_ERROR("Mutex initialization failed!\n"); return false; } @@ -974,29 +1018,26 @@ void call_kill_transmission(ToxAVCall* call) if (call == NULL || call->active == 0) return; - pthread_mutex_lock(call->mutex_control); - call->active = 0; - pthread_mutex_lock(call->mutex_encoding_audio); - pthread_mutex_unlock(call->mutex_encoding_audio); - pthread_mutex_lock(call->mutex_encoding_video); - pthread_mutex_unlock(call->mutex_encoding_video); - pthread_mutex_lock(call->mutex_do); - pthread_mutex_unlock(call->mutex_do); + pthread_mutex_lock(call->mutex_audio_sending); + pthread_mutex_unlock(call->mutex_audio_sending); + pthread_mutex_lock(call->mutex_video_sending); + pthread_mutex_unlock(call->mutex_video_sending); + pthread_mutex_lock(call->mutex_decoding); + pthread_mutex_unlock(call->mutex_decoding); rtp_kill(call->rtps[audio_index]); call->rtps[audio_index] = NULL; rtp_kill(call->rtps[video_index]); call->rtps[video_index] = NULL; + cs_kill(call->cs); call->cs = NULL; - pthread_mutex_destroy(call->mutex_encoding_audio); - pthread_mutex_destroy(call->mutex_encoding_video); - pthread_mutex_destroy(call->mutex_do); - - pthread_mutex_unlock(call->mutex_control); + pthread_mutex_destroy(call->mutex_audio_sending); + pthread_mutex_destroy(call->mutex_video_sending); + pthread_mutex_destroy(call->mutex_decoding); } void call_remove(ToxAVCall* call) @@ -1010,9 +1051,6 @@ void call_remove(ToxAVCall* call) ToxAVCall* prev = call->prev; ToxAVCall* next = call->next; - pthread_mutex_destroy(call->mutex_control); - pthread_mutex_destroy(call->mutex_do); - free(call); if (prev) diff --git a/toxav/toxav.h b/toxav/toxav.h index ec232ddd1..101047edf 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -187,43 +187,41 @@ bool toxav_answer(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, ui ******************************************************************************/ typedef enum TOXAV_CALL_STATE { /** - * Not sending anything. Either the friend requested that this client stops - * sending anything, or the client turned off both audio and video by setting - * the respective bit rates to 0. - * - * If both sides are in this state, the call is effectively on hold, but not - * in the PAUSED state. + * Not sending nor receiving anything, meaning, one of the sides requested pause. + * The call will be resumed once the side that initiated pause resumes it. */ - TOXAV_CALL_STATE_NOT_SENDING, + TOXAV_CALL_STATE_PAUSED = 0, /** - * Sending audio only. Either the friend requested that this client stops - * sending video, or the client turned off video by setting the video bit rate - * to 0. + * The flag that marks that friend is sending audio. */ - TOXAV_CALL_STATE_SENDING_A, + TOXAV_CALL_STATE_SENDING_A = 1, /** - * Sending video only. Either the friend requested that this client stops - * sending audio (muted), or the client turned off audio by setting the audio - * bit rate to 0. + * The flag that marks that friend is sending video. */ - TOXAV_CALL_STATE_SENDING_V, + TOXAV_CALL_STATE_SENDING_V = 2, /** - * Sending both audio and video. + * The flag that marks that friend is receiving audio. */ - TOXAV_CALL_STATE_SENDING_AV, + TOXAV_CALL_STATE_RECEIVING_A = 4, /** - * The call is on hold. Both sides stop sending and receiving. + * The flag that marks that friend is receiving video. */ - TOXAV_CALL_STATE_PAUSED, + TOXAV_CALL_STATE_RECEIVING_V = 8, + + /** + * The core will never set TOXAV_CALL_STATE_END or TOXAV_CALL_STATE_ERROR + * together with other states. + */ + /** * The call has finished. This is the final state after which no more state * transitions can occur for the call. */ - TOXAV_CALL_STATE_END, + TOXAV_CALL_STATE_END = 16, /** * Sent by the AV core if an error occurred on the remote end. */ - TOXAV_CALL_STATE_ERROR + TOXAV_CALL_STATE_ERROR = 32 } TOXAV_CALL_STATE; /** * The function type for the `call_state` callback. @@ -231,7 +229,7 @@ typedef enum TOXAV_CALL_STATE { * @param friend_number The friend number for which the call state changed. * @param state The new call state. */ -typedef void toxav_call_state_cb(ToxAV *av, uint32_t friend_number, TOXAV_CALL_STATE state, void *user_data); +typedef void toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, void *user_data); /** * Set the callback for the `call_state` event. Pass NULL to unset. * @@ -261,27 +259,19 @@ typedef enum TOXAV_CALL_CONTROL { /** * Request that the friend stops sending audio. Regardless of the friend's * compliance, this will cause the `receive_audio_frame` event to stop being - * triggered on receiving an audio frame from the friend. + * triggered on receiving an audio frame from the friend. If the audio was + * already muted, calling this control will notify client to start sending + * audio again. */ - TOXAV_CALL_CONTROL_MUTE_AUDIO, + TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, /** * Request that the friend stops sending video. Regardless of the friend's * compliance, this will cause the `receive_video_frame` event to stop being - * triggered on receiving an video frame from the friend. + * triggered on receiving an video frame from the friend. If the video was + * already muted, calling this control will notify client to start sending + * video again. */ - TOXAV_CALL_CONTROL_MUTE_VIDEO, - /** - * Notify the friend that we are AGAIN ready to handle incoming audio. - * This control will not work if the call is not started with audio - * initiated. - */ - TOXAV_CALL_CONTROL_UNMUTE_AUDIO, - /** - * Notify the friend that we are AGAIN ready to handle incoming video. - * This control will not work if the call is not started with video - * initiated. - */ - TOXAV_CALL_CONTROL_UNMUTE_VIDEO + TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, } TOXAV_CALL_CONTROL; typedef enum TOXAV_ERR_CALL_CONTROL { TOXAV_ERR_CALL_CONTROL_OK, From 4f76a8c89eea77ebc190a324ad128e3c7dda298c Mon Sep 17 00:00:00 2001 From: mannol Date: Fri, 20 Mar 2015 19:07:23 +0100 Subject: [PATCH 13/97] Updating with master --- autogen.sh | 0 toxav/codec.c | 1 - 2 files changed, 1 deletion(-) mode change 100755 => 100644 autogen.sh diff --git a/autogen.sh b/autogen.sh old mode 100755 new mode 100644 diff --git a/toxav/codec.c b/toxav/codec.c index 645f7188d..95e0beb41 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -279,7 +279,6 @@ void cs_do(CSSession *cs) cs->last_packet_frame_duration = ( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 ) / cs->last_packet_sampling_rate; - } else { LOGGER_WARNING("Failed to load packet values!"); rtp_free_msg(NULL, msg); From 62af82705a805648a4dba54a7f516681e5163923 Mon Sep 17 00:00:00 2001 From: mannol Date: Sun, 22 Mar 2015 23:50:43 +0100 Subject: [PATCH 14/97] Some progress --- toxav/av_test.c | 61 +++++++++++++++++++++++++++++++++++++++++++++---- toxav/codec.c | 10 ++++---- toxav/rtp.c | 7 +++--- toxav/toxav.c | 33 +++++++++++++++----------- toxav/toxav.h | 6 ++++- 5 files changed, 91 insertions(+), 26 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 01484249b..46fd97e1a 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -39,6 +39,7 @@ typedef struct { bool incoming; uint32_t state; + uint32_t output_source; } CallControl; const char* stringify_state(TOXAV_CALL_STATE s) @@ -58,6 +59,8 @@ const char* stringify_state(TOXAV_CALL_STATE s) }; +int device_play_frame(uint32_t source, const int16_t* PCM, size_t size); + /** * Callbacks */ @@ -86,7 +89,7 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, uint32_t sampling_rate, void *user_data) { - printf("Handling AUDIO FRAME callback\n"); + device_play_frame(((CallControl*)user_data)->output_source, pcm, sample_count); } void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, uint16_t length, void *userdata) { @@ -173,7 +176,7 @@ int device_read_frame(ALCdevice* device, int32_t frame_dur, int16_t* PCM, size_t return f_size; } -int device_play_frame(uint32_t source, int16_t* PCM, size_t size) +int device_play_frame(uint32_t source, const int16_t* PCM, size_t size) { uint32_t bufid; int32_t processed, queued; @@ -604,17 +607,67 @@ int main (int argc, char** argv) if (TEST_TRANSFER_A) { /* Audio encoding/decoding and transfer */ printf("\nTrying audio enc/dec...\n"); + memset(&AliceCC, 0, sizeof(CallControl)); + memset(&BobCC, 0, sizeof(CallControl)); + + AliceCC.output_source = BobCC.output_source = source; + + { + TOXAV_ERR_CALL rc; + toxav_call(AliceAV, 0, 48, 0, &rc); + + if (rc != TOXAV_ERR_CALL_OK) { + printf("toxav_call failed: %d\n", rc); + exit(1); + } + } + + while (!BobCC.incoming) + iterate(Bsn, AliceAV, BobAV); + + { + TOXAV_ERR_ANSWER rc; + toxav_answer(BobAV, 0, 48, 0, &rc); + + if (rc != TOXAV_ERR_ANSWER_OK) { + printf("toxav_answer failed: %d\n", rc); + exit(1); + } + } + + iterate(Bsn, AliceAV, BobAV); + int16_t PCM[10000]; time_t start_time = time(NULL); /* Run for 5 seconds */ while ( start_time + 10 > time(NULL) ) { int frame_size = device_read_frame(in_device, 20, PCM, sizeof(PCM)); - if (frame_size > 0) - device_play_frame(source, PCM, frame_size); + if (frame_size > 0) { + TOXAV_ERR_SEND_FRAME rc; + if (toxav_send_audio_frame(AliceAV, 0, PCM, frame_size, 2, 48000, &rc) == false) { + printf("Error sending frame of size %d: %d\n", frame_size, rc); + exit (1); + } + } + + iterate(Bsn, AliceAV, BobAV); // c_sleep(20); } + { + TOXAV_ERR_CALL_CONTROL rc; + toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); + + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { + printf("toxav_call_control failed: %d\n", rc); + exit(1); + } + } + + iterate(Bsn, AliceAV, BobAV); + assert(BobCC.state == TOXAV_CALL_STATE_END); + printf("Success!"); } diff --git a/toxav/codec.c b/toxav/codec.c index bf94115e8..80975b708 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -245,7 +245,7 @@ void cs_do(CSSession *cs) /* Codec session should always be protected by call mutex so no need to check for cs validity */ - if (!cs) + if (!cs) return; Payload *p; @@ -258,7 +258,7 @@ void cs_do(CSSession *cs) if (cs->audio_decoder) { /* If receiving enabled */ RTPMessage *msg; - uint16_t fsize = 5760; /* Max frame size for 48 kHz */ + uint16_t fsize = 16000; /* Max frame size for 48 kHz */ int16_t tmp[fsize * 2]; while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { @@ -284,6 +284,7 @@ void cs_do(CSSession *cs) continue; } + LOGGER_DEBUG("Decoding packet of length: %d", msg->length); rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, fsize, 0); rtp_free_msg(NULL, msg); } @@ -741,8 +742,9 @@ void queue_message(RTPSession *session, RTPMessage *msg) */ CSSession *cs = session->cs; - if (!cs) return; - + if (!cs) + return; + /* Audio */ if (session->payload_type == rtp_TypeAudio % 128) { pthread_mutex_lock(cs->queue_mutex); diff --git a/toxav/rtp.c b/toxav/rtp.c index 8319c7dc9..e5f453103 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -363,7 +363,6 @@ int rtp_handle_packet ( Messenger *m, int32_t friendnumber, const uint8_t *data, } queue_message(session, msg); - return 0; } @@ -427,8 +426,6 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ) return NULL; } - LOGGER_DEBUG("Registered packet handler: pt: %d; fid: %d", payload_type, friend_num); - retu->version = RTP_VERSION; /* It's always 2 */ retu->padding = 0; /* If some additional data is needed about the packet */ retu->extension = 0; /* If extension to header is needed */ @@ -467,7 +464,7 @@ void rtp_kill ( RTPSession *session ) { if ( !session ) return; - custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, NULL, NULL); + rtp_stop_receiving (session); free ( session->ext_header ); free ( session->csrc ); @@ -483,6 +480,7 @@ int rtp_start_receiving(RTPSession* session) if (session == NULL) return 0; + LOGGER_DEBUG("Registering packet handler: pt: %d; friend: %d", session->prefix, session->dest); return custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, rtp_handle_packet, session); } @@ -492,6 +490,7 @@ int rtp_stop_receiving(RTPSession* session) if (session == NULL) return 0; + LOGGER_DEBUG("Unregistering packet handler: pt: %d; friend: %d", session->prefix, session->dest); return custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, NULL, NULL); } diff --git a/toxav/toxav.c b/toxav/toxav.c index 78243ae67..7cf8cec49 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -215,13 +215,6 @@ void toxav_iteration(ToxAV* av) for (; i; i = i->next) { if (i->active) { pthread_mutex_lock(i->mutex_decoding); - - /* TODO make AV synchronisation */ - if (av->racb.first) - av->racb.first(av, i->friend_id, av->racb.second); - if (av->rvcb.first) - av->rvcb.first(av, i->friend_id, av->rvcb.second); - pthread_mutex_unlock(av->mutex); cs_do(i->cs); rc = MIN(i->cs->last_packet_frame_duration, rc); @@ -304,6 +297,11 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui goto END; } + if (!call_prepare_transmission(call)) { + rc = TOXAV_ERR_ANSWER_MALLOC; + goto END; + } + call->s_audio_b = audio_bit_rate; call->s_video_b = video_bit_rate; @@ -645,7 +643,7 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } - if ( channels != 1 || channels != 2 ) { + if ( channels != 1 && channels != 2 ) { pthread_mutex_unlock(call->mutex_audio_sending); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; @@ -666,8 +664,12 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } - vrc = rtp_send_msg(call->rtps[audio_index], dest, vrc); - /* TODO check for error? */ + if (rtp_send_msg(call->rtps[audio_index], dest, vrc) != 0) { + LOGGER_WARNING("Failed to send audio packet"); + rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; + } + + LOGGER_DEBUG("Sent packet of size: %d (o %d)", vrc, sample_count); } pthread_mutex_unlock(call->mutex_audio_sending); @@ -965,7 +967,10 @@ bool call_prepare_transmission(ToxAVCall* call) goto FAILURE; } - rtp_start_receiving(call->rtps[audio_index]); + if (rtp_start_receiving(call->rtps[audio_index]) != 0) { + LOGGER_WARNING("Failed to enable audio receiving!"); + goto FAILURE; + } } { /* Prepare video */ @@ -984,13 +989,15 @@ bool call_prepare_transmission(ToxAVCall* call) goto FAILURE; } - if (cs_enable_video_receiving(call->cs) != 0) { LOGGER_WARNING("Failed to enable video receiving!"); goto FAILURE; } - rtp_start_receiving(call->rtps[audio_index]); + if (rtp_start_receiving(call->rtps[video_index]) != 0) { + LOGGER_WARNING("Failed to enable audio receiving!"); + goto FAILURE; + } } call->active = 1; diff --git a/toxav/toxav.h b/toxav/toxav.h index 101047edf..571282eda 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -380,7 +380,11 @@ typedef enum TOXAV_ERR_SEND_FRAME { * One of the frame parameters was invalid. E.g. the resolution may be too * small or too large, or the audio sampling rate may be unsupported. */ - TOXAV_ERR_SEND_FRAME_INVALID + TOXAV_ERR_SEND_FRAME_INVALID, + /** + * Failed to push frame through rtp interface. + */ + TOXAV_ERR_SEND_FRAME_RTP_FAILED } TOXAV_ERR_SEND_FRAME; /** * The function type for the `request_video_frame` callback. From 995bddbc26be5106cb33400311fbb669e314312a Mon Sep 17 00:00:00 2001 From: mannol Date: Mon, 23 Mar 2015 22:22:17 +0100 Subject: [PATCH 15/97] Audio works in test --- toxav/av_test.c | 3 +-- toxav/codec.c | 19 ++++++++++++++++--- toxav/codec.h | 1 + toxav/toxav.c | 2 +- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 46fd97e1a..9be648f6f 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -156,7 +156,7 @@ void iterate(Tox* Bsn, ToxAV* AliceAV, ToxAV* BobAV) toxav_iteration(AliceAV); toxav_iteration(BobAV); - c_sleep(20); + c_sleep(toxav_iteration_interval(AliceAV)); } int device_read_frame(ALCdevice* device, int32_t frame_dur, int16_t* PCM, size_t max_size) @@ -652,7 +652,6 @@ int main (int argc, char** argv) } iterate(Bsn, AliceAV, BobAV); -// c_sleep(20); } { diff --git a/toxav/codec.c b/toxav/codec.c index 80975b708..c3cb2622a 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -258,7 +258,7 @@ void cs_do(CSSession *cs) if (cs->audio_decoder) { /* If receiving enabled */ RTPMessage *msg; - uint16_t fsize = 16000; /* Max frame size for 48 kHz */ + uint16_t fsize = 10000; /* Should be enough for all normal frequences */ int16_t tmp[fsize * 2]; while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { @@ -293,6 +293,7 @@ void cs_do(CSSession *cs) LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); } else if (cs->acb.first) { /* Play */ + LOGGER_DEBUG("Playing audio frame size: %d chans: %d srate: %d", rc, cs->last_pack_channels, cs->last_packet_sampling_rate); cs->acb.first(cs->agent, cs->friend_id, tmp, rc, cs->last_pack_channels, cs->last_packet_sampling_rate, cs->acb.second); } @@ -598,6 +599,7 @@ int cs_set_sending_audio_bitrate(CSSession *cs, int32_t rate) return -1; } + LOGGER_DEBUG("Set new encoder bitrate to: %d", rate); return 0; } @@ -607,6 +609,9 @@ int cs_set_sending_audio_sampling_rate(CSSession* cs, int32_t rate) if (cs->audio_encoder == NULL) return -1; + if (cs->encoder_sample_rate == rate) + return 0; + int rc = OPUS_OK; int bitrate = 0; int channels = cs->encoder_channels; @@ -619,6 +624,9 @@ int cs_set_sending_audio_sampling_rate(CSSession* cs, int32_t rate) } cs_disable_audio_sending(cs); + cs->encoder_sample_rate = rate; + + LOGGER_DEBUG("Set new encoder sampling rate: %d", rate); return cs_enable_audio_sending(cs, bitrate, channels); } @@ -642,6 +650,8 @@ int cs_set_sending_audio_channels(CSSession* cs, int32_t count) } cs_disable_audio_sending(cs); + + LOGGER_DEBUG("Set new encoder channel count: %d", count); return cs_enable_audio_sending(cs, bitrate, count); } @@ -674,8 +684,12 @@ int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate, int channels) if (cs->audio_encoder) return 0; + if (!cs->encoder_sample_rate) + cs->encoder_sample_rate = 48000; + cs->encoder_channels = channels; + int rc = OPUS_OK; - cs->audio_encoder = opus_encoder_create(48000, channels, OPUS_APPLICATION_AUDIO, &rc); + cs->audio_encoder = opus_encoder_create(cs->encoder_sample_rate, channels, OPUS_APPLICATION_AUDIO, &rc); if ( rc != OPUS_OK ) { LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); @@ -696,7 +710,6 @@ int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate, int channels) goto FAILURE; } - cs->encoder_channels = channels; return 0; FAILURE: diff --git a/toxav/codec.h b/toxav/codec.h index 6a6739906..971a9e05e 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -108,6 +108,7 @@ typedef struct CSSession_s { /* audio encoding */ OpusEncoder *audio_encoder; int32_t encoder_channels; + int32_t encoder_sample_rate; /* audio decoding */ OpusDecoder *audio_decoder; diff --git a/toxav/toxav.c b/toxav/toxav.c index 7cf8cec49..850c2fc7d 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -957,7 +957,7 @@ bool call_prepare_transmission(ToxAVCall* call) call->rtps[audio_index]->cs = call->cs; /* Only enable sending if bitrate is defined */ - if (call->s_audio_b > 0 && cs_enable_audio_sending(call->cs, call->s_audio_b, 2) != 0) { + if (call->s_audio_b > 0 && cs_enable_audio_sending(call->cs, call->s_audio_b * 1000, 2) != 0) { LOGGER_WARNING("Failed to enable audio sending!"); goto FAILURE; } From 96ca88a0d63b9fd3ee1f986ab99627590fdbaa56 Mon Sep 17 00:00:00 2001 From: mannol Date: Mon, 23 Mar 2015 23:38:04 +0100 Subject: [PATCH 16/97] Make it possible to change channels/sample rate of the decoder --- toxav/av_test.c | 11 +++++--- toxav/codec.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++-- toxav/codec.h | 2 ++ toxav/toxav.c | 1 - 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 9be648f6f..7948445da 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -26,6 +26,8 @@ #define c_sleep(x) usleep(1000*x) #endif +#define MIN(a,b) (((a)<(b))?(a):(b)) + /* Enable/disable tests */ #define TEST_REGULAR_AV 0 #define TEST_REGULAR_A 0 @@ -156,12 +158,15 @@ void iterate(Tox* Bsn, ToxAV* AliceAV, ToxAV* BobAV) toxav_iteration(AliceAV); toxav_iteration(BobAV); - c_sleep(toxav_iteration_interval(AliceAV)); + int mina = MIN(tox_do_interval(toxav_get_tox(AliceAV)), toxav_iteration_interval(AliceAV)); + int minb = MIN(tox_do_interval(toxav_get_tox(BobAV)), toxav_iteration_interval(BobAV)); + + c_sleep(MIN(mina, minb)); } int device_read_frame(ALCdevice* device, int32_t frame_dur, int16_t* PCM, size_t max_size) { - int f_size = (48000 * frame_dur / 1000); + int f_size = (8000 * frame_dur / 1000); if (max_size < f_size) return -1; @@ -645,7 +650,7 @@ int main (int argc, char** argv) int frame_size = device_read_frame(in_device, 20, PCM, sizeof(PCM)); if (frame_size > 0) { TOXAV_ERR_SEND_FRAME rc; - if (toxav_send_audio_frame(AliceAV, 0, PCM, frame_size, 2, 48000, &rc) == false) { + if (toxav_send_audio_frame(AliceAV, 0, PCM, frame_size, 2, 8000, &rc) == false) { printf("Error sending frame of size %d: %d\n", frame_size, rc); exit (1); } diff --git a/toxav/codec.c b/toxav/codec.c index c3cb2622a..a72b2764c 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -237,6 +237,61 @@ static int convert_bw_to_sampling_rate(int bw) } +int cs_set_receiving_audio_bitrate(CSSession *cs, int32_t rate) +{ + if (cs->audio_decoder == NULL) + return -1; + + int rc = opus_decoder_ctl(cs->audio_decoder, OPUS_SET_BITRATE(rate)); + + if ( rc != OPUS_OK ) { + LOGGER_ERROR("Error while setting decoder ctl: %s", opus_strerror(rc)); + return -1; + } + + LOGGER_DEBUG("Set new decoder bitrate to: %d", rate); + return 0; +} + +int cs_set_receiving_audio_sampling_rate(CSSession* cs, int32_t rate) +{ + /* TODO Find a better way? */ + if (cs->audio_decoder == NULL) + return -1; + + if (cs->decoder_sample_rate == rate) + return 0; + + int channels = cs->decoder_channels; + + cs_disable_audio_receiving(cs); + + cs->decoder_sample_rate = rate; + cs->decoder_channels = channels; + + LOGGER_DEBUG("Set new encoder sampling rate: %d", rate); + return cs_enable_audio_receiving(cs); +} + +int cs_set_receiving_audio_channels(CSSession* cs, int32_t count) +{ + /* TODO Find a better way? */ + if (cs->audio_decoder == NULL) + return -1; + + if (cs->decoder_channels == count) + return 0; + + int srate = cs->decoder_sample_rate; + cs_disable_audio_receiving(cs); + + cs->decoder_channels = count; + cs->decoder_sample_rate = srate; + + LOGGER_DEBUG("Set new encoder channel count: %d", count); + return cs_enable_audio_receiving(cs); +} + /* PUBLIC */ @@ -284,6 +339,9 @@ void cs_do(CSSession *cs) continue; } + cs_set_receiving_audio_sampling_rate(cs, cs->last_packet_sampling_rate); + cs_set_receiving_audio_channels(cs, cs->last_pack_channels); + LOGGER_DEBUG("Decoding packet of length: %d", msg->length); rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, fsize, 0); rtp_free_msg(NULL, msg); @@ -353,6 +411,8 @@ CSSession *cs_new(uint32_t peer_video_frame_piece_size) cs->peer_video_frame_piece_size = peer_video_frame_piece_size; + cs->decoder_sample_rate = 48000; + cs->decoder_channels = 2; return cs; } @@ -676,6 +736,8 @@ void cs_disable_audio_receiving(CSSession* cs) * To avoid unecessary checking we set this to 500 */ cs->last_packet_frame_duration = 500; + cs->decoder_sample_rate = 48000; + cs->decoder_channels = 2; } } @@ -721,9 +783,9 @@ int cs_enable_audio_receiving(CSSession* cs) { if (cs->audio_decoder) return 0; - + int rc; - cs->audio_decoder = opus_decoder_create(48000, 2, &rc ); + cs->audio_decoder = opus_decoder_create(cs->decoder_sample_rate, cs->decoder_channels, &rc ); if ( rc != OPUS_OK ) { LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc)); @@ -742,6 +804,7 @@ int cs_enable_audio_receiving(CSSession* cs) * To avoid unecessary checking we set this to 500 */ cs->last_packet_frame_duration = 500; + return 0; } diff --git a/toxav/codec.h b/toxav/codec.h index 971a9e05e..5e015f397 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -112,6 +112,8 @@ typedef struct CSSession_s { /* audio decoding */ OpusDecoder *audio_decoder; + int32_t decoder_channels; + int32_t decoder_sample_rate; int32_t last_pack_channels; int32_t last_packet_sampling_rate; int32_t last_packet_frame_duration; diff --git a/toxav/toxav.c b/toxav/toxav.c index 850c2fc7d..df6fa8333 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -650,7 +650,6 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc } { /* Encode and send */ - /* TODO redundant? */ cs_set_sending_audio_channels(call->cs, channels); cs_set_sending_audio_sampling_rate(call->cs, sampling_rate); From c1aa5bac4f6c68eece7e78bc82241df5d30e05bc Mon Sep 17 00:00:00 2001 From: mannol Date: Tue, 24 Mar 2015 23:59:01 +0100 Subject: [PATCH 17/97] Current progress of video testing --- toxav/Makefile.inc | 2 + toxav/av_test.c | 149 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 148 insertions(+), 3 deletions(-) diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc index b61c1ea06..1dbabbcdf 100644 --- a/toxav/Makefile.inc +++ b/toxav/Makefile.inc @@ -36,6 +36,7 @@ libtoxav_la_LIBADD = libtoxcore.la \ $(AV_LIBS) + noinst_PROGRAMS += av_test av_test_SOURCES = ../toxav/av_test.c @@ -50,6 +51,7 @@ av_test_LDADD = $(LIBSODIUM_LDFLAGS) \ $(LIBSODIUM_LIBS) \ $(NACL_OBJECTS) \ -lopenal \ + -lopencv_calib3d -lopencv_contrib -lopencv_core -lopencv_features2d -lopencv_flann -lopencv_gpu -lopencv_highgui -lopencv_imgproc -lopencv_legacy -lopencv_ml -lopencv_objdetect -lopencv_ocl -lopencv_photo -lopencv_stitching -lopencv_superres -lopencv_ts -lopencv_video -lopencv_videostab \ $(NACL_LIBS) diff --git a/toxav/av_test.c b/toxav/av_test.c index 7948445da..2323588a1 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -13,6 +13,11 @@ # endif /* ALC_ALL_DEVICES_SPECIFIER */ #endif /* __APPLE__ */ +#include +#include + +#include + #include #include #include @@ -26,7 +31,6 @@ #define c_sleep(x) usleep(1000*x) #endif -#define MIN(a,b) (((a)<(b))?(a):(b)) /* Enable/disable tests */ #define TEST_REGULAR_AV 0 @@ -35,7 +39,8 @@ #define TEST_REJECT 0 #define TEST_CANCEL 0 #define TEST_MUTE_UNMUTE 0 -#define TEST_TRANSFER_A 1 +#define TEST_TRANSFER_A 0 +#define TEST_TRANSFER_V 1 typedef struct { @@ -44,6 +49,8 @@ typedef struct { uint32_t output_source; } CallControl; +const char* video_test_window = "AV Test"; + const char* stringify_state(TOXAV_CALL_STATE s) { static const char* strings[] = @@ -82,7 +89,36 @@ void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint8_t const *planes[], int32_t const stride[], void *user_data) { - printf("Handling VIDEO FRAME callback\n"); + IplImage output_img; + const int bpl = stride[VPX_PLANE_Y]; + const int cxbpl = stride[VPX_PLANE_V]; + + output_img.imageData = malloc(width * height * 3); + output_img.imageSize = width * height * 3; + output_img.width = width; + output_img.height = height; + + /* FIXME: possible bug? */ + const uint8_t* yData = planes[VPX_PLANE_Y]; + const uint8_t* uData = planes[VPX_PLANE_V]; + const uint8_t* vData = planes[VPX_PLANE_U]; + + // convert from planar to packed + for (int y = 0; y < height; ++y) + { + for (int x = 0; x < width; ++x) + { + uint8_t Y = planes[VPX_PLANE_Y][x + y * bpl]; + uint8_t U = planes[VPX_PLANE_V][x/(1 << 1) + y/(1 << 1)*cxbpl]; + uint8_t V = planes[VPX_PLANE_U][x/(1 << 1) + y/(1 << 1)*cxbpl]; + output_img.imageData[width * 3 * y + x * 3 + 0] = Y; + output_img.imageData[width * 3 * y + x * 3 + 1] = U; + output_img.imageData[width * 3 * y + x * 3 + 2] = V; + } + } + + cvShowImage(video_test_window, &output_img); + free(output_img.imageData); } void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, int16_t const *pcm, @@ -265,8 +301,84 @@ long get_device_idx(const char* arg) return res; } +int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) +{ + /* I use vpx image coz i'm noob TODO use opencv conversion */ + vpx_image vpx_img; + vpx_img.w = vpx_img.h = vpx_img.d_w = vpx_img.d_h = 0; + + const int w = img->width; + const int h = img->height; + + vpx_img_alloc(&vpx_img, VPX_IMG_FMT_VPXI420, w, h, 1); + + for (int y = 0; y < h; ++y) + { + for (int x = 0; x < w; ++x) + { + uint8_t b = img->imageData[(x + y * w) * 3 + 0]; + uint8_t g = img->imageData[(x + y * w) * 3 + 1]; + uint8_t r = img->imageData[(x + y * w) * 3 + 2]; + + vpx_img.planes[VPX_PLANE_Y][x + y * vpx_img.stride[VPX_PLANE_Y]] = ((66 * r + 129 * g + 25 * b) >> 8) + 16; + + if (!(x % (1 << vpx_img.x_chroma_shift)) && !(y % (1 << vpx_img.y_chroma_shift))) + { + const int i = x / (1 << vpx_img.x_chroma_shift); + const int j = y / (1 << vpx_img.y_chroma_shift); + vpx_img.planes[VPX_PLANE_U][i + j * vpx_img.stride[VPX_PLANE_U]] = ((112 * r + -94 * g + -18 * b) >> 8) + 128; + vpx_img.planes[VPX_PLANE_V][i + j * vpx_img.stride[VPX_PLANE_V]] = ((-38 * r + -74 * g + 112 * b) >> 8) + 128; + } + } + } + + int rc = toxav_send_video_frame(av, friend_number, w, h, + vpx_img.planes[VPX_PLANE_Y], + vpx_img.planes[VPX_PLANE_U], + vpx_img.planes[VPX_PLANE_V], NULL); + + vpx_img_free(&vpx_img); + return rc; +} + int main (int argc, char** argv) { + /* AV files for testing */ + const char* audio_in = ""; + const char* video_in = ""; + long audio_out_dev = 0; + +AGAIN: + switch (getopt(argc, argv, "a:v:o:")) + { + case 'a': + audio_in = optarg; + goto AGAIN; + break; + case 'v': + video_in = optarg; + goto AGAIN; + break; + case 'o': + char *d; + audio_out_dev = strtol(optarg, &d, 10); + if (*d) { + fprintf(stderr, "Invalid value for argument: 'o'"); + return 1; + } + goto AGAIN; + break; + case '?': + return 1; + break; + case -1: + break; + } + + + + return 0; + if (argc == 2) { if (strcmp(argv[1], "-d") == 0 || strcmp(argv[1], "--devices") == 0) { return print_devices(); @@ -675,6 +787,37 @@ int main (int argc, char** argv) printf("Success!"); } + if (TEST_TRANSFER_V) { + if (strlen(video_in) == 0) { + printf("Skipping video test...\n"); + goto CONTINUE; + } + + cvNamedWindow(video_test_window, CV_WINDOW_AUTOSIZE); + + CvCapture* capture = cvCreateFileCapture(video_in); + if (!capture) { + printf("No file named: %s\n", video_in); + return 1; + } + + IplImage* frame; + time_t start_time = time(NULL); + + while(start_time + 10 > time(NULL)) { + frame = cvQueryFrame( capture ); + if (!frame) + break; + + } + + cvReleaseCapture(&capture); + cvDestroyWindow(video_test_window); + + CONTINUE:; + } + + toxav_kill(BobAV); toxav_kill(AliceAV); tox_kill(Bob); From 56e5373a11eed9b4d89873f4668be2b1411a8d18 Mon Sep 17 00:00:00 2001 From: Dubslow Date: Wed, 25 Mar 2015 14:36:35 -0500 Subject: [PATCH 18/97] Consistency with core api --- toxav/av_test.c | 6 +++--- toxav/toxav.c | 12 ++++++------ toxav/toxav.h | 30 +++++++++++++++--------------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 7948445da..42b2fcc4e 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -155,8 +155,8 @@ void iterate(Tox* Bsn, ToxAV* AliceAV, ToxAV* BobAV) tox_do(toxav_get_tox(AliceAV)); tox_do(toxav_get_tox(BobAV)); - toxav_iteration(AliceAV); - toxav_iteration(BobAV); + toxav_iterate(AliceAV); + toxav_iterate(BobAV); int mina = MIN(tox_do_interval(toxav_get_tox(AliceAV)), toxav_iteration_interval(AliceAV)); int minb = MIN(tox_do_interval(toxav_get_tox(BobAV)), toxav_iteration_interval(BobAV)); @@ -686,4 +686,4 @@ int main (int argc, char** argv) alcCloseDevice(out_device); alcCaptureCloseDevice(in_device); return 0; -} \ No newline at end of file +} diff --git a/toxav/toxav.c b/toxav/toxav.c index df6fa8333..fcc6e86a2 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -78,8 +78,8 @@ struct toxAV { PAIR(toxav_call_state_cb *, void *) scb; /* Call state callback */ PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ - PAIR(toxav_request_video_frame_cb *, void *) rvcb; /* Request video callback */ - PAIR(toxav_request_audio_frame_cb *, void *) racb; /* Request video callback */ + PAIR(toxav_video_frame_request_cb *, void *) rvcb; /* Video request callback */ + PAIR(toxav_audio_frame_request_cb *, void *) racb; /* Audio request callback */ /** Decode time measures */ int32_t dmssc; /** Measure count */ @@ -202,7 +202,7 @@ uint32_t toxav_iteration_interval(const ToxAV* av) return av->calls ? av->interval : 200; } -void toxav_iteration(ToxAV* av) +void toxav_iterate(ToxAV* av) { if (av->calls == NULL) return; @@ -487,7 +487,7 @@ bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_ /* TODO */ } -void toxav_callback_request_video_frame(ToxAV* av, toxav_request_video_frame_cb* function, void* user_data) +void toxav_callback_video_frame_request(ToxAV* av, toxav_video_frame_request_cb* function, void* user_data) { pthread_mutex_lock(av->mutex); av->rvcb.first = function; @@ -601,7 +601,7 @@ END: return rc == TOXAV_ERR_SEND_FRAME_OK; } -void toxav_callback_request_audio_frame(ToxAV* av, toxav_request_audio_frame_cb* function, void* user_data) +void toxav_callback_audio_frame_request(ToxAV* av, toxav_audio_frame_request_cb* function, void* user_data) { pthread_mutex_lock(av->mutex); av->racb.first = function; @@ -1078,4 +1078,4 @@ CLEAR: av->calls_head = av->calls_tail = 0; free(av->calls); av->calls = NULL; -} \ No newline at end of file +} diff --git a/toxav/toxav.h b/toxav/toxav.h index 571282eda..48bb6b8c6 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -82,8 +82,8 @@ Tox *toxav_get_tox(ToxAV *av); * ******************************************************************************/ /** - * Returns the interval in milliseconds when the next toxav_iteration should be - * called. If no call is active at the moment, this function returns 200. + * Returns the interval in milliseconds when the next toxav_iterate call should + * be. If no call is active at the moment, this function returns 200. */ uint32_t toxav_iteration_interval(ToxAV const *av); /** @@ -91,7 +91,7 @@ uint32_t toxav_iteration_interval(ToxAV const *av); * toxav_iteration_interval() milliseconds. It is best called in the same loop * as tox_iteration. */ -void toxav_iteration(ToxAV *av); +void toxav_iterate(ToxAV *av); /******************************************************************************* * * :: Call setup @@ -372,7 +372,7 @@ typedef enum TOXAV_ERR_SEND_FRAME { */ TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL, /** - * No video frame had been requested through the `request_video_frame` event, + * No video frame had been requested through the `video_frame_request` event, * but the client tried to send one, anyway. */ TOXAV_ERR_SEND_FRAME_NOT_REQUESTED, @@ -387,20 +387,20 @@ typedef enum TOXAV_ERR_SEND_FRAME { TOXAV_ERR_SEND_FRAME_RTP_FAILED } TOXAV_ERR_SEND_FRAME; /** - * The function type for the `request_video_frame` callback. + * The function type for the `video_frame_request` callback. * * @param friend_number The friend number of the friend for which the next video * frame should be sent. */ -typedef void toxav_request_video_frame_cb(ToxAV *av, uint32_t friend_number, void *user_data); +typedef void toxav_video_frame_request_cb(ToxAV *av, uint32_t friend_number, void *user_data); /** - * Set the callback for the `request_video_frame` event. Pass NULL to unset. + * Set the callback for the `video_frame_request` event. Pass NULL to unset. */ -void toxav_callback_request_video_frame(ToxAV *av, toxav_request_video_frame_cb *function, void *user_data); +void toxav_callback_video_frame_request(ToxAV *av, toxav_video_frame_request_cb *function, void *user_data); /** * Send a video frame to a friend. * - * This is called in response to receiving the `request_video_frame` event. + * This is called in response to receiving the `video_frame_request` event. * * Y - plane should be of size: height * width * U - plane should be of size: (height/2) * (width/2) @@ -419,20 +419,20 @@ bool toxav_send_video_frame(ToxAV *av, uint32_t friend_number, uint8_t const *y, uint8_t const *u, uint8_t const *v, TOXAV_ERR_SEND_FRAME *error); /** - * The function type for the `request_audio_frame` callback. + * The function type for the `audio_frame_request` callback. * * @param friend_number The friend number of the friend for which the next audio * frame should be sent. */ -typedef void toxav_request_audio_frame_cb(ToxAV *av, uint32_t friend_number, void *user_data); +typedef void toxav_audio_frame_request_cb(ToxAV *av, uint32_t friend_number, void *user_data); /** - * Set the callback for the `request_audio_frame` event. Pass NULL to unset. + * Set the callback for the `audio_frame_request` event. Pass NULL to unset. */ -void toxav_callback_request_audio_frame(ToxAV *av, toxav_request_audio_frame_cb *function, void *user_data); +void toxav_callback_audio_frame_request(ToxAV *av, toxav_audio_frame_request_cb *function, void *user_data); /** * Send an audio frame to a friend. * - * This is called in response to receiving the `request_audio_frame` event. + * This is called in response to receiving the `audio_frame_request` event. * * The expected format of the PCM data is: [s1c1][s1c2][...][s2c1][s2c2][...]... * Meaning: sample 1 for channel 1, sample 1 for channel 2, ... @@ -510,4 +510,4 @@ typedef void toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, */ void toxav_callback_receive_audio_frame(ToxAV *av, toxav_receive_audio_frame_cb *function, void *user_data); -#endif /* TOXAV_H */ \ No newline at end of file +#endif /* TOXAV_H */ From e65efc8936ee029f735a22105843335c18026ad2 Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 28 Mar 2015 03:36:31 +0100 Subject: [PATCH 19/97] Improvement in test and removed some bloat from CS --- toxav/Makefile.inc | 1 + toxav/av_test.c | 606 +++++++++++++++++++++------------------------ toxav/codec.c | 187 ++++---------- toxav/codec.h | 16 +- toxav/toxav.c | 11 +- 5 files changed, 329 insertions(+), 492 deletions(-) diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc index 1dbabbcdf..8e2be25e5 100644 --- a/toxav/Makefile.inc +++ b/toxav/Makefile.inc @@ -52,6 +52,7 @@ av_test_LDADD = $(LIBSODIUM_LDFLAGS) \ $(NACL_OBJECTS) \ -lopenal \ -lopencv_calib3d -lopencv_contrib -lopencv_core -lopencv_features2d -lopencv_flann -lopencv_gpu -lopencv_highgui -lopencv_imgproc -lopencv_legacy -lopencv_ml -lopencv_objdetect -lopencv_ocl -lopencv_photo -lopencv_stitching -lopencv_superres -lopencv_ts -lopencv_video -lopencv_videostab \ + -lsndfile \ $(NACL_LIBS) diff --git a/toxav/av_test.c b/toxav/av_test.c index 249e0ab77..5ca53eb90 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -1,35 +1,32 @@ #include "toxav.h" #include "../toxcore/tox.h" -#ifdef __APPLE__ -# include -# include -#else -# include -# include -/* compatibility with older versions of OpenAL */ -# ifndef ALC_ALL_DEVICES_SPECIFIER -# include -# endif /* ALC_ALL_DEVICES_SPECIFIER */ -#endif /* __APPLE__ */ +/* For playing audio data */ +#include +#include +/* Processing wav's */ +#include + +/* For reading and displaying video data */ #include #include +/* For converting images TODO remove */ #include + +#include #include #include #include #include #include - -#if defined(_WIN32) || defined(__WIN32__) || defined (WIN32) -#define c_sleep(x) Sleep(1*x) -#else +#include #include + + #define c_sleep(x) usleep(1000*x) -#endif /* Enable/disable tests */ @@ -39,17 +36,17 @@ #define TEST_REJECT 0 #define TEST_CANCEL 0 #define TEST_MUTE_UNMUTE 0 -#define TEST_TRANSFER_A 0 -#define TEST_TRANSFER_V 1 +#define TEST_TRANSFER_A 1 +#define TEST_TRANSFER_V 0 typedef struct { bool incoming; uint32_t state; - uint32_t output_source; } CallControl; -const char* video_test_window = "AV Test"; +const char* vdout = "AV Test"; +uint32_t adout; const char* stringify_state(TOXAV_CALL_STATE s) { @@ -65,10 +62,7 @@ const char* stringify_state(TOXAV_CALL_STATE s) }; return strings [s]; -}; - - -int device_play_frame(uint32_t source, const int16_t* PCM, size_t size); +} /** * Callbacks @@ -104,9 +98,11 @@ void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, const uint8_t* vData = planes[VPX_PLANE_U]; // convert from planar to packed - for (int y = 0; y < height; ++y) + int y = 0; + for (; y < height; ++y) { - for (int x = 0; x < width; ++x) + int x = 0; + for (; x < width; ++x) { uint8_t Y = planes[VPX_PLANE_Y][x + y * bpl]; uint8_t U = planes[VPX_PLANE_V][x/(1 << 1) + y/(1 << 1)*cxbpl]; @@ -117,7 +113,7 @@ void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, } } - cvShowImage(video_test_window, &output_img); + cvShowImage(vdout, &output_img); free(output_img.imageData); } void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, @@ -127,7 +123,32 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, uint32_t sampling_rate, void *user_data) { - device_play_frame(((CallControl*)user_data)->output_source, pcm, sample_count); + uint32_t bufid; + int32_t processed, queued; + alGetSourcei(adout, AL_BUFFERS_PROCESSED, &processed); + alGetSourcei(adout, AL_BUFFERS_QUEUED, &queued); + + if(processed) { + uint32_t bufids[processed]; + alSourceUnqueueBuffers(adout, processed, bufids); + alDeleteBuffers(processed - 1, bufids + 1); + bufid = bufids[0]; + } +// else if(queued < 16) + alGenBuffers(1, &bufid); +// else +// return; + + + alBufferData(bufid, channels == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16, + pcm, sample_count * channels * 2, sampling_rate); + alSourceQueueBuffers(adout, 1, &bufid); + + int32_t state; + alGetSourcei(adout, AL_SOURCE_STATE, &state); + + if(state != AL_PLAYING) + alSourcePlay(adout); } void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, uint16_t length, void *userdata) { @@ -139,8 +160,20 @@ void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t /** */ -void prepare(Tox* Bsn, Tox* Alice, Tox* Bob) +void initialize_tox(Tox** bootstrap, ToxAV** AliceAV, CallControl* AliceCC, ToxAV** BobAV, CallControl* BobCC) { + Tox* Alice; + Tox* Bob; + + *bootstrap = tox_new(0); + Alice = tox_new(0); + Bob = tox_new(0); + + assert(bootstrap && Alice && Bob); + + printf("Created 3 instances of Tox\n"); + + printf("Preparing network...\n"); long long unsigned int cur_time = time(NULL); uint32_t to_compare = 974536; @@ -154,11 +187,11 @@ void prepare(Tox* Bsn, Tox* Alice, Tox* Bob) uint8_t off = 1; while (1) { - tox_do(Bsn); + tox_do(*bootstrap); tox_do(Alice); tox_do(Bob); - if (tox_isconnected(Bsn) && tox_isconnected(Alice) && tox_isconnected(Bob) && off) { + if (tox_isconnected(*bootstrap) && tox_isconnected(Alice) && tox_isconnected(Bob) && off) { printf("Toxes are online, took %llu seconds\n", time(NULL) - cur_time); off = 0; } @@ -169,25 +202,32 @@ void prepare(Tox* Bsn, Tox* Alice, Tox* Bob) c_sleep(20); } - printf("All set after %llu seconds!\n", time(NULL) - cur_time); -} -void prepareAV(ToxAV* AliceAV, void* AliceUD, ToxAV* BobAV, void* BobUD) -{ + + TOXAV_ERR_NEW rc; + *AliceAV = toxav_new(Alice, &rc); + assert(rc == TOXAV_ERR_NEW_OK); + + *BobAV = toxav_new(Bob, &rc); + assert(rc == TOXAV_ERR_NEW_OK); + /* Alice */ - toxav_callback_call(AliceAV, t_toxav_call_cb, AliceUD); - toxav_callback_call_state(AliceAV, t_toxav_call_state_cb, AliceUD); - toxav_callback_receive_video_frame(AliceAV, t_toxav_receive_video_frame_cb, AliceUD); - toxav_callback_receive_audio_frame(AliceAV, t_toxav_receive_audio_frame_cb, AliceUD); + toxav_callback_call(*AliceAV, t_toxav_call_cb, AliceCC); + toxav_callback_call_state(*AliceAV, t_toxav_call_state_cb, AliceCC); + toxav_callback_receive_video_frame(*AliceAV, t_toxav_receive_video_frame_cb, AliceCC); + toxav_callback_receive_audio_frame(*AliceAV, t_toxav_receive_audio_frame_cb, AliceCC); /* Bob */ - toxav_callback_call(BobAV, t_toxav_call_cb, BobUD); - toxav_callback_call_state(BobAV, t_toxav_call_state_cb, BobUD); - toxav_callback_receive_video_frame(BobAV, t_toxav_receive_video_frame_cb, BobUD); - toxav_callback_receive_audio_frame(BobAV, t_toxav_receive_audio_frame_cb, BobUD); + toxav_callback_call(*BobAV, t_toxav_call_cb, BobCC); + toxav_callback_call_state(*BobAV, t_toxav_call_state_cb, BobCC); + toxav_callback_receive_video_frame(*BobAV, t_toxav_receive_video_frame_cb, BobCC); + toxav_callback_receive_audio_frame(*BobAV, t_toxav_receive_audio_frame_cb, BobCC); + + printf("Created 2 instances of ToxAV\n"); + printf("All set after %llu seconds!\n", time(NULL) - cur_time); } -void iterate(Tox* Bsn, ToxAV* AliceAV, ToxAV* BobAV) +int iterate_tox(Tox* bootstrap, ToxAV* AliceAV, ToxAV* BobAV) { - tox_do(Bsn); + tox_do(bootstrap); tox_do(toxav_get_tox(AliceAV)); tox_do(toxav_get_tox(BobAV)); @@ -197,114 +237,16 @@ void iterate(Tox* Bsn, ToxAV* AliceAV, ToxAV* BobAV) int mina = MIN(tox_do_interval(toxav_get_tox(AliceAV)), toxav_iteration_interval(AliceAV)); int minb = MIN(tox_do_interval(toxav_get_tox(BobAV)), toxav_iteration_interval(BobAV)); - c_sleep(MIN(mina, minb)); -} - -int device_read_frame(ALCdevice* device, int32_t frame_dur, int16_t* PCM, size_t max_size) -{ - int f_size = (8000 * frame_dur / 1000); - - if (max_size < f_size) - return -1; - - /* Don't block if not enough data */ - int32_t samples; - alcGetIntegerv(device, ALC_CAPTURE_SAMPLES, sizeof(int32_t), &samples); - if (samples < f_size) - return 0; - - alcCaptureSamples(device, PCM, f_size); - return f_size; -} - -int device_play_frame(uint32_t source, const int16_t* PCM, size_t size) -{ - uint32_t bufid; - int32_t processed, queued; - alGetSourcei(source, AL_BUFFERS_PROCESSED, &processed); - alGetSourcei(source, AL_BUFFERS_QUEUED, &queued); - - if(processed) { - uint32_t bufids[processed]; - alSourceUnqueueBuffers(source, processed, bufids); - alDeleteBuffers(processed - 1, bufids + 1); - bufid = bufids[0]; - } - else if(queued < 16) - alGenBuffers(1, &bufid); - else - return 0; + int rc = MIN(mina, minb); + c_sleep(rc); - - alBufferData(bufid, AL_FORMAT_STEREO16, PCM, size * 2 * 2, 48000); - alSourceQueueBuffers(source, 1, &bufid); - - int32_t state; - alGetSourcei(source, AL_SOURCE_STATE, &state); - - if(state != AL_PLAYING) - alSourcePlay(source); - return 1; -} - -int print_devices() -{ - const char* default_input; - const char* default_output; - - const char *device; - - printf("Default input device: %s\n", alcGetString(NULL, ALC_CAPTURE_DEFAULT_DEVICE_SPECIFIER)); - printf("Default output device: %s\n", alcGetString(NULL, ALC_DEFAULT_DEVICE_SPECIFIER)); - - printf("\n"); - - printf("Input devices:\n"); - - int i = 0; - for(device = alcGetString(NULL, ALC_CAPTURE_DEVICE_SPECIFIER); *device; - device += strlen( device ) + 1, ++i) { - printf("%d) %s\n", i, device); - } - - printf("\n"); - printf("Output devices:\n"); - - i = 0; - for(device = alcGetString(NULL, ALC_DEVICE_SPECIFIER); *device; - device += strlen( device ) + 1, ++i) { - printf("%d) %s\n", i, device); - } - - return 0; -} - -int print_help(const char* name, int rc) -{ - fprintf(stderr, "Usage: %s [-h] \n", name); - return rc; -} - -long get_device_idx(const char* arg) -{ - if (strcmp(arg, "-") == 0) - return -1; /* Default */ - - char *p; - long res = strtol(arg, &p, 10); - - if (*p) { - fprintf(stderr, "Invalid device!"); - exit(1); - } - - return res; + return rc; } int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) { /* I use vpx image coz i'm noob TODO use opencv conversion */ - vpx_image vpx_img; + vpx_image_t vpx_img; vpx_img.w = vpx_img.h = vpx_img.d_w = vpx_img.d_h = 0; const int w = img->width; @@ -312,9 +254,11 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) vpx_img_alloc(&vpx_img, VPX_IMG_FMT_VPXI420, w, h, 1); - for (int y = 0; y < h; ++y) + int y = 0; + for (; y < h; ++y) { - for (int x = 0; x < w; ++x) + int x = 0; + for (; x < w; ++x) { uint8_t b = img->imageData[(x + y * w) * 3 + 0]; uint8_t g = img->imageData[(x + y * w) * 3 + 1]; @@ -341,113 +285,118 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) return rc; } -int main (int argc, char** argv) +int print_audio_devices() { - /* AV files for testing */ - const char* audio_in = ""; - const char* video_in = ""; - long audio_out_dev = 0; + const char *device; + + printf("Default output device: %s\n", alcGetString(NULL, ALC_DEFAULT_DEVICE_SPECIFIER)); + printf("Output devices:\n"); -AGAIN: - switch (getopt(argc, argv, "a:v:o:")) - { - case 'a': - audio_in = optarg; - goto AGAIN; - break; - case 'v': - video_in = optarg; - goto AGAIN; - break; - case 'o': - char *d; - audio_out_dev = strtol(optarg, &d, 10); - if (*d) { - fprintf(stderr, "Invalid value for argument: 'o'"); - return 1; - } - goto AGAIN; - break; - case '?': - return 1; - break; - case -1: - break; + int i = 0; + for(device = alcGetString(NULL, ALC_DEVICE_SPECIFIER); *device; + device += strlen( device ) + 1, ++i) { + printf("%d) %s\n", i, device); } - + return 0; +} + +int print_help (const char* name) +{ + printf("Usage: %s -[a:v:o:dh]\n" + "-a video input file\n" + "-v video input file\n" + "-o output audio device index\n" + "-d print output audio devices\n" + "-h print this help\n", name); return 0; +} + +int main (int argc, char** argv) +{ + struct stat st; - if (argc == 2) { - if (strcmp(argv[1], "-d") == 0 || strcmp(argv[1], "--devices") == 0) { - return print_devices(); - } - if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0) { - return print_help(argv[0], 0); - } - } - - if (argc != 3) { - fprintf(stderr, "Invalid input!\n"); - return print_help(argv[0], 1); - } - - int i; - - const char* in_device_name = ""; - const char* out_device_name = ""; - - { - long dev = get_device_idx(argv[1]); - if (dev == -1) - in_device_name = alcGetString(NULL, ALC_CAPTURE_DEFAULT_DEVICE_SPECIFIER); - else - { - const char* tmp; - i = -1; - for(tmp = alcGetString(NULL, ALC_CAPTURE_DEVICE_SPECIFIER); *tmp && i != dev; - tmp += strlen( tmp ) + 1, ++i) - in_device_name = tmp; - } - - printf("Input device: %s\n", in_device_name); - } - - { - long dev = get_device_idx(argv[1]); - if (dev == -1) - out_device_name = alcGetString(NULL, ALC_DEFAULT_DEVICE_SPECIFIER); - else - { - const char* tmp; - i = -1; - for(tmp = alcGetString(NULL, ALC_DEVICE_SPECIFIER); *tmp && i != dev; - tmp += strlen( tmp ) + 1, ++i) - out_device_name = tmp; - } - - printf("Output device: %s\n", out_device_name); - } - - ALCdevice* out_device; - ALCcontext* out_ctx; - uint32_t source; - uint32_t buffers[5]; + /* AV files for testing */ + const char* af_name = NULL; + const char* vf_name = NULL; + long audio_out_dev_idx = 0; + + /* Pasre settings */ + CHECK_ARG: switch (getopt(argc, argv, "a:v:o:dh")) { + case 'a': + af_name = optarg; + goto CHECK_ARG; + case 'v': + vf_name = optarg; + goto CHECK_ARG; + case 'o': { + char *d; + audio_out_dev_idx = strtol(optarg, &d, 10); + if (*d) { + printf("Invalid value for argument: 'o'"); + exit(1); + } + goto CHECK_ARG; + } + case 'd': + return print_audio_devices(); + case 'h': + return print_help(argv[0]); + case '?': + exit(1); + case -1:; + } + + { /* Check files */ + if (!af_name) { + printf("Required audio input file!\n"); + exit(1); + } + + if (!vf_name) { + printf("Required video input file!\n"); + exit(1); + } + + /* Check for files */ + if(stat(af_name, &st) != 0 || !S_ISREG(st.st_mode)) + { + printf("%s doesn't seem to be a regular file!\n", af_name); + exit(1); + } + + if(stat(vf_name, &st) != 0 || !S_ISREG(st.st_mode)) + { + printf("%s doesn't seem to be a regular file!\n", vf_name); + exit(1); + } + } + + ALCdevice* audio_out_device; { /* Open output device */ - out_device = alcOpenDevice(out_device_name); - if ( !out_device ) { - fprintf(stderr, "Failed to open playback device: %s: %d\n", out_device_name, alGetError()); - return 1; + const char* audio_out_dev_name = NULL; + + int i = 0; + for(audio_out_dev_name = alcGetString(NULL, ALC_DEVICE_SPECIFIER); i < audio_out_dev_idx; + audio_out_dev_name += strlen( audio_out_dev_name ) + 1, ++i) + if (!(audio_out_dev_name + strlen( audio_out_dev_name ) + 1)) + break; + + audio_out_device = alcOpenDevice(audio_out_dev_name); + if ( !audio_out_device ) { + printf("Failed to open playback device: %s: %d\n", audio_out_dev_name, alGetError()); + exit(1); } - out_ctx = alcCreateContext(out_device, NULL); + ALCcontext* out_ctx = alcCreateContext(audio_out_device, NULL); alcMakeContextCurrent(out_ctx); + uint32_t buffers[5]; alGenBuffers(5, buffers); - alGenSources((uint32_t)1, &source); - alSourcei(source, AL_LOOPING, AL_FALSE); + alGenSources((uint32_t)1, &adout); + alSourcei(adout, AL_LOOPING, AL_FALSE); uint16_t zeros[10000]; memset(zeros, 0, 10000); @@ -455,45 +404,31 @@ AGAIN: for ( i = 0; i < 5; ++i ) alBufferData(buffers[i], AL_FORMAT_STEREO16, zeros, 10000, 48000); - alSourceQueueBuffers(source, 5, buffers); - alSourcePlay(source); + alSourceQueueBuffers(adout, 5, buffers); + alSourcePlay(adout); + + printf("Using audio device: %s\n", audio_out_dev_name); } - ALCdevice* in_device; + printf("Using audio file: %s\n", af_name); + printf("Using video file: %s\n", vf_name); + - { /* Open input device */ - in_device = alcCaptureOpenDevice(in_device_name, 48000, AL_FORMAT_STEREO16, 10000); - if ( !in_device ) { - fprintf(stderr, "Failed to open capture device: %s: %d\n", in_device_name, alGetError()); - return 1; - } - - alcCaptureStart(in_device); - } - - Tox *Bsn = tox_new(0); - Tox *Alice = tox_new(0); - Tox *Bob = tox_new(0); - - assert(Bsn && Alice && Bob); - - prepare(Bsn, Alice, Bob); - ToxAV *AliceAV, *BobAV; - CallControl AliceCC, BobCC; - { - TOXAV_ERR_NEW rc; - AliceAV = toxav_new(Alice, &rc); - assert(rc == TOXAV_ERR_NEW_OK); - - BobAV = toxav_new(Bob, &rc); - assert(rc == TOXAV_ERR_NEW_OK); - - prepareAV(AliceAV, &AliceCC, BobAV, &BobCC); - printf("Created 2 instances of ToxAV\n"); - } + /* START TOX NETWORK */ + + Tox *bootstrap; + ToxAV *AliceAV; + ToxAV *BobAV; + + CallControl AliceCC; + CallControl BobCC; + + initialize_tox(&bootstrap, &AliceAV, &AliceCC, &BobAV, &BobCC); + + #define REGULAR_CALL_FLOW(A_BR, V_BR) \ @@ -539,7 +474,7 @@ AGAIN: } \ } \ \ - iterate(Bsn, AliceAV, BobAV); \ + iterate(bootstrap, AliceAV, BobAV); \ } \ printf("Success!\n");\ } while(0) @@ -578,7 +513,7 @@ AGAIN: } while (!BobCC.incoming) - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); /* Reject */ { @@ -592,7 +527,7 @@ AGAIN: } while (AliceCC.state != TOXAV_CALL_STATE_END) - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); printf("Success!\n"); } @@ -614,7 +549,7 @@ AGAIN: } while (!BobCC.incoming) - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); /* Cancel */ { @@ -629,7 +564,7 @@ AGAIN: /* Alice will not receive end state */ while (BobCC.state != TOXAV_CALL_STATE_END) - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); printf("Success!\n"); } @@ -652,7 +587,7 @@ AGAIN: } while (!BobCC.incoming) - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); /* At first try all stuff while in invalid state */ assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); @@ -670,39 +605,39 @@ AGAIN: } } - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); /* Pause and Resume */ printf("Pause and Resume\n"); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state == TOXAV_CALL_STATE_PAUSED); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state & (TOXAV_CALL_STATE_SENDING_A | TOXAV_CALL_STATE_SENDING_V)); /* Mute/Unmute single */ printf("Mute/Unmute single\n"); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); /* Mute/Unmute both */ printf("Mute/Unmute both\n"); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_V); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_V); { @@ -715,21 +650,22 @@ AGAIN: } } - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state == TOXAV_CALL_STATE_END); printf("Success!\n"); } if (TEST_TRANSFER_A) { /* Audio encoding/decoding and transfer */ + SNDFILE* af_handle; + SF_INFO af_info; + printf("\nTrying audio enc/dec...\n"); memset(&AliceCC, 0, sizeof(CallControl)); memset(&BobCC, 0, sizeof(CallControl)); - - AliceCC.output_source = BobCC.output_source = source; - { + { /* Call */ TOXAV_ERR_CALL rc; toxav_call(AliceAV, 0, 48, 0, &rc); @@ -740,11 +676,11 @@ AGAIN: } while (!BobCC.incoming) - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); - { + { /* Answer */ TOXAV_ERR_ANSWER rc; - toxav_answer(BobAV, 0, 48, 0, &rc); + toxav_answer(BobAV, 0, 64, 0, &rc); if (rc != TOXAV_ERR_ANSWER_OK) { printf("toxav_answer failed: %d\n", rc); @@ -752,26 +688,44 @@ AGAIN: } } - iterate(Bsn, AliceAV, BobAV); - - int16_t PCM[10000]; - time_t start_time = time(NULL); + iterate_tox(bootstrap, AliceAV, BobAV); + /* Open audio file */ + af_handle = sf_open(af_name, SFM_READ, &af_info); + if (af_handle == NULL) + { + printf("Failed to open the file.\n"); + exit(1); + } + /* Run for 5 seconds */ - while ( start_time + 10 > time(NULL) ) { - int frame_size = device_read_frame(in_device, 20, PCM, sizeof(PCM)); - if (frame_size > 0) { - TOXAV_ERR_SEND_FRAME rc; - if (toxav_send_audio_frame(AliceAV, 0, PCM, frame_size, 2, 8000, &rc) == false) { - printf("Error sending frame of size %d: %d\n", frame_size, rc); - exit (1); - } - } - - iterate(Bsn, AliceAV, BobAV); + + uint32_t frame_duration = 10; + int16_t PCM[10000]; + + time_t start_time = time(NULL); + time_t expected_time = af_info.frames / af_info.samplerate + 2; + + while ( start_time + expected_time > time(NULL) ) { + int frame_size = (af_info.samplerate * frame_duration / 1000); + + int64_t count = sf_read_short(af_handle, PCM, frame_size); + if (count > 0) { + TOXAV_ERR_SEND_FRAME rc; + if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) { + printf("Error sending frame of size %ld: %d\n", count, rc); + exit(1); + } + } + + iterate_tox(bootstrap, AliceAV, BobAV); } + + printf("Played file in: %lu\n", time(NULL) - start_time); + + sf_close(af_handle); - { + { /* Hangup */ TOXAV_ERR_CALL_CONTROL rc; toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); @@ -781,24 +735,19 @@ AGAIN: } } - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state == TOXAV_CALL_STATE_END); printf("Success!"); } if (TEST_TRANSFER_V) { - if (strlen(video_in) == 0) { - printf("Skipping video test...\n"); - goto CONTINUE; - } + cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); - cvNamedWindow(video_test_window, CV_WINDOW_AUTOSIZE); - - CvCapture* capture = cvCreateFileCapture(video_in); + CvCapture* capture = cvCreateFileCapture(vf_name); if (!capture) { - printf("No file named: %s\n", video_in); - return 1; + printf("Failed to open video file: %s\n", vf_name); + exit(1); } IplImage* frame; @@ -812,21 +761,20 @@ AGAIN: } cvReleaseCapture(&capture); - cvDestroyWindow(video_test_window); - - CONTINUE:; + cvDestroyWindow(vdout); } + Tox* Alice = toxav_get_tox(AliceAV); + Tox* Bob = toxav_get_tox(BobAV); toxav_kill(BobAV); toxav_kill(AliceAV); tox_kill(Bob); tox_kill(Alice); - tox_kill(Bsn); + tox_kill(bootstrap); printf("\nTest successful!\n"); - alcCloseDevice(out_device); - alcCaptureCloseDevice(in_device); + alcCloseDevice(audio_out_device); return 0; } diff --git a/toxav/codec.c b/toxav/codec.c index a72b2764c..b9cbbc06f 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -38,7 +38,7 @@ #include "rtp.h" #include "codec.h" -#define DEFAULT_JBUF 6 +#define DEFAULT_JBUF 3 /* Good quality encode. */ #define MAX_DECODE_TIME_US 0 @@ -175,6 +175,8 @@ static int jbuf_write(JitterBuffer *q, RTPMessage *m) unsigned int num = sequnum % q->size; if ((uint32_t)(sequnum - q->bottom) > q->size) { + LOGGER_DEBUG("Clearing jitter: %p", q); + jbuf_clear(q); q->bottom = sequnum - q->capacity; q->queue[num] = m; @@ -193,7 +195,7 @@ static int jbuf_write(JitterBuffer *q, RTPMessage *m) return 0; } -/* Success is 0 when there is nothing to dequeue, +/* success is set to 0 when there is nothing to dequeue, * 1 when there's a good packet, * 2 when there's a lost packet */ static RTPMessage *jbuf_read(JitterBuffer *q, int32_t *success) @@ -237,62 +239,6 @@ static int convert_bw_to_sampling_rate(int bw) } -int cs_set_receiving_audio_bitrate(CSSession *cs, int32_t rate) -{ - if (cs->audio_decoder == NULL) - return -1; - - int rc = opus_decoder_ctl(cs->audio_decoder, OPUS_SET_BITRATE(rate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while setting decoder ctl: %s", opus_strerror(rc)); - return -1; - } - - LOGGER_DEBUG("Set new decoder bitrate to: %d", rate); - return 0; -} - -int cs_set_receiving_audio_sampling_rate(CSSession* cs, int32_t rate) -{ - /* TODO Find a better way? */ - if (cs->audio_decoder == NULL) - return -1; - - if (cs->decoder_sample_rate == rate) - return 0; - - int channels = cs->decoder_channels; - - cs_disable_audio_receiving(cs); - - cs->decoder_sample_rate = rate; - cs->decoder_channels = channels; - - LOGGER_DEBUG("Set new encoder sampling rate: %d", rate); - return cs_enable_audio_receiving(cs); -} - -int cs_set_receiving_audio_channels(CSSession* cs, int32_t count) -{ - /* TODO Find a better way? */ - if (cs->audio_decoder == NULL) - return -1; - - if (cs->decoder_channels == count) - return 0; - - int srate = cs->decoder_sample_rate; - cs_disable_audio_receiving(cs); - - cs->decoder_channels = count; - cs->decoder_sample_rate = srate; - - LOGGER_DEBUG("Set new encoder channel count: %d", count); - return cs_enable_audio_receiving(cs); -} - - /* PUBLIC */ void cs_do(CSSession *cs) @@ -310,17 +256,19 @@ void cs_do(CSSession *cs) pthread_mutex_lock(cs->queue_mutex); + /********************* AUDIO *********************/ if (cs->audio_decoder) { /* If receiving enabled */ RTPMessage *msg; - uint16_t fsize = 10000; /* Should be enough for all normal frequences */ - int16_t tmp[fsize * 2]; + /* The maximum for 120 ms 48 KHz audio */ + int16_t tmp[20000]; while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { pthread_mutex_unlock(cs->queue_mutex); if (success == 2) { - rc = opus_decode(cs->audio_decoder, 0, 0, tmp, fsize, 1); + rc = opus_decode(cs->audio_decoder, NULL, 0, tmp, + cs->last_packet_sampling_rate * cs->last_packet_frame_duration / 1000, 1); } else { /* Get values from packet and decode. * It also checks for validity of an opus packet @@ -328,7 +276,7 @@ void cs_do(CSSession *cs) rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); if (rc != -1) { cs->last_packet_sampling_rate = rc; - cs->last_pack_channels = opus_packet_get_nb_channels(msg->data); + cs->last_packet_channels = opus_packet_get_nb_channels(msg->data); cs->last_packet_frame_duration = ( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 ) @@ -339,11 +287,7 @@ void cs_do(CSSession *cs) continue; } - cs_set_receiving_audio_sampling_rate(cs, cs->last_packet_sampling_rate); - cs_set_receiving_audio_channels(cs, cs->last_pack_channels); - - LOGGER_DEBUG("Decoding packet of length: %d", msg->length); - rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, fsize, 0); + rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, sizeof(tmp), 0); rtp_free_msg(NULL, msg); } @@ -351,15 +295,28 @@ void cs_do(CSSession *cs) LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); } else if (cs->acb.first) { /* Play */ - LOGGER_DEBUG("Playing audio frame size: %d chans: %d srate: %d", rc, cs->last_pack_channels, cs->last_packet_sampling_rate); + + LOGGER_DEBUG("Playing audio frame size: %d; channels: %d; srate: %d; duration %d", rc, + cs->last_packet_channels, cs->last_packet_sampling_rate, cs->last_packet_frame_duration); + + /* According to https://tools.ietf.org/html/rfc6716#section-2.1.2 + * Every encoder can encode both mono and stereo data so we must + * determine which format is selected. + */ + + if (cs->last_packet_channels == 2) { + /* The packet is encoded with stereo encoder */ + } + cs->acb.first(cs->agent, cs->friend_id, tmp, rc, - cs->last_pack_channels, cs->last_packet_sampling_rate, cs->acb.second); + cs->last_packet_channels, cs->last_packet_sampling_rate, cs->acb.second); } pthread_mutex_lock(cs->queue_mutex); } } + /********************* VIDEO *********************/ if (cs->vbuf_raw && !buffer_empty(cs->vbuf_raw)) { /* Decode video */ buffer_read(cs->vbuf_raw, &p); @@ -411,8 +368,6 @@ CSSession *cs_new(uint32_t peer_video_frame_piece_size) cs->peer_video_frame_piece_size = peer_video_frame_piece_size; - cs->decoder_sample_rate = 48000; - cs->decoder_channels = 2; return cs; } @@ -538,6 +493,8 @@ int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate) return 0; } + + int cs_enable_video_sending(CSSession* cs, uint32_t bitrate) { if (cs->v_encoding) @@ -619,6 +576,8 @@ FAILURE: return -1; } + + void cs_disable_video_sending(CSSession* cs) { if (cs->v_encoding) { @@ -663,64 +622,13 @@ int cs_set_sending_audio_bitrate(CSSession *cs, int32_t rate) return 0; } -int cs_set_sending_audio_sampling_rate(CSSession* cs, int32_t rate) -{ - /* TODO Find a better way? */ - if (cs->audio_encoder == NULL) - return -1; - - if (cs->encoder_sample_rate == rate) - return 0; - - int rc = OPUS_OK; - int bitrate = 0; - int channels = cs->encoder_channels; - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_GET_BITRATE(&bitrate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while getting encoder ctl: %s", opus_strerror(rc)); - return -1; - } - - cs_disable_audio_sending(cs); - cs->encoder_sample_rate = rate; - - LOGGER_DEBUG("Set new encoder sampling rate: %d", rate); - return cs_enable_audio_sending(cs, bitrate, channels); -} -int cs_set_sending_audio_channels(CSSession* cs, int32_t count) -{ - /* TODO Find a better way? */ - if (cs->audio_encoder == NULL) - return -1; - - if (cs->encoder_channels == count) - return 0; - - int rc = OPUS_OK; - int bitrate = 0; - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_GET_BITRATE(&bitrate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while getting encoder ctl: %s", opus_strerror(rc)); - return -1; - } - - cs_disable_audio_sending(cs); - - LOGGER_DEBUG("Set new encoder channel count: %d", count); - return cs_enable_audio_sending(cs, bitrate, count); -} void cs_disable_audio_sending(CSSession* cs) { if ( cs->audio_encoder ) { opus_encoder_destroy(cs->audio_encoder); cs->audio_encoder = NULL; - cs->encoder_channels = 0; } } @@ -732,26 +640,22 @@ void cs_disable_audio_receiving(CSSession* cs) jbuf_free(cs->j_buf); cs->j_buf = NULL; - /* It's used for measuring iteration interval so this has to be some value. - * To avoid unecessary checking we set this to 500 - */ - cs->last_packet_frame_duration = 500; - cs->decoder_sample_rate = 48000; - cs->decoder_channels = 2; + /* These need to be set in order to properly + * do error correction with opus */ + cs->last_packet_frame_duration = 120; + cs->last_packet_sampling_rate = 48000; } } -int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate, int channels) + + +int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate) { if (cs->audio_encoder) return 0; - if (!cs->encoder_sample_rate) - cs->encoder_sample_rate = 48000; - cs->encoder_channels = channels; - int rc = OPUS_OK; - cs->audio_encoder = opus_encoder_create(cs->encoder_sample_rate, channels, OPUS_APPLICATION_AUDIO, &rc); + cs->audio_encoder = opus_encoder_create(48000, 2, OPUS_APPLICATION_AUDIO, &rc); if ( rc != OPUS_OK ) { LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); @@ -785,7 +689,7 @@ int cs_enable_audio_receiving(CSSession* cs) return 0; int rc; - cs->audio_decoder = opus_decoder_create(cs->decoder_sample_rate, cs->decoder_channels, &rc ); + cs->audio_decoder = opus_decoder_create(48000, 2, &rc ); if ( rc != OPUS_OK ) { LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc)); @@ -800,10 +704,11 @@ int cs_enable_audio_receiving(CSSession* cs) return -1; } - /* It's used for measuring iteration interval so this has to be some value. - * To avoid unecessary checking we set this to 500 - */ - cs->last_packet_frame_duration = 500; + + /* These need to be set in order to properly + * do error correction with opus */ + cs->last_packet_frame_duration = 120; + cs->last_packet_sampling_rate = 48000; return 0; } @@ -813,8 +718,8 @@ int cs_enable_audio_receiving(CSSession* cs) /* Called from RTP */ void queue_message(RTPSession *session, RTPMessage *msg) { - /* This function is unregistered during call termination befor destroing - * Codec session so no need to check for validity of cs + /* This function is unregistered during call termination befor destroying + * Codec session so no need to check for validity of cs TODO properly check video cycle */ CSSession *cs = session->cs; diff --git a/toxav/codec.h b/toxav/codec.h index 5e015f397..526a80d5c 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -107,25 +107,15 @@ typedef struct CSSession_s { /* audio encoding */ OpusEncoder *audio_encoder; - int32_t encoder_channels; - int32_t encoder_sample_rate; /* audio decoding */ OpusDecoder *audio_decoder; - int32_t decoder_channels; - int32_t decoder_sample_rate; - int32_t last_pack_channels; + int32_t last_packet_channels; int32_t last_packet_sampling_rate; int32_t last_packet_frame_duration; struct JitterBuffer_s *j_buf; - /* Voice activity detection */ - uint32_t EVAD_tolerance; /* In frames */ - uint32_t EVAD_tolerance_cr; - - - /* OTHER * * @@ -171,10 +161,8 @@ void cs_disable_video_receiving(CSSession* cs); * AUDIO HANDLING */ int cs_set_sending_audio_bitrate(CSSession* cs, int32_t rate); -int cs_set_sending_audio_sampling_rate(CSSession* cs, int32_t rate); -int cs_set_sending_audio_channels(CSSession* cs, int32_t count); -int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate, int channels); +int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate); int cs_enable_audio_receiving(CSSession* cs); void cs_disable_audio_sending(CSSession* cs); diff --git a/toxav/toxav.c b/toxav/toxav.c index fcc6e86a2..84a0c43ae 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -208,7 +208,7 @@ void toxav_iterate(ToxAV* av) return; uint64_t start = current_time_monotonic(); - uint32_t rc = 0; + uint32_t rc = 500; pthread_mutex_lock(av->mutex); ToxAVCall* i = av->calls[av->calls_head]; @@ -225,7 +225,7 @@ void toxav_iterate(ToxAV* av) pthread_mutex_lock(av->mutex); } - av->interval = rc < av->dmssa ? 0 : rc - av->dmssa; + av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa); av->dmsst += current_time_monotonic() - start; if (++av->dmssc == 3) { @@ -650,9 +650,6 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc } { /* Encode and send */ - cs_set_sending_audio_channels(call->cs, channels); - cs_set_sending_audio_sampling_rate(call->cs, sampling_rate); - uint8_t dest[sample_count * channels * 2 /* sizeof(uint16_t) */]; int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest, sizeof (dest)); @@ -667,8 +664,6 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc LOGGER_WARNING("Failed to send audio packet"); rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; } - - LOGGER_DEBUG("Sent packet of size: %d (o %d)", vrc, sample_count); } pthread_mutex_unlock(call->mutex_audio_sending); @@ -956,7 +951,7 @@ bool call_prepare_transmission(ToxAVCall* call) call->rtps[audio_index]->cs = call->cs; /* Only enable sending if bitrate is defined */ - if (call->s_audio_b > 0 && cs_enable_audio_sending(call->cs, call->s_audio_b * 1000, 2) != 0) { + if (call->s_audio_b > 0 && cs_enable_audio_sending(call->cs, call->s_audio_b * 1000) != 0) { LOGGER_WARNING("Failed to enable audio sending!"); goto FAILURE; } From fdaad0b7c008b65646ca5cbd62ac0d305617ecfe Mon Sep 17 00:00:00 2001 From: mannol Date: Sun, 29 Mar 2015 01:10:34 +0100 Subject: [PATCH 20/97] A little CS cleanup --- toxav/av_test.c | 108 +++++------ toxav/codec.c | 479 +++++++++++++++++++----------------------------- toxav/codec.h | 81 ++------ toxav/rtp.h | 2 +- toxav/toxav.c | 160 +++++++++------- toxav/toxav.h | 10 +- 6 files changed, 363 insertions(+), 477 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 5ca53eb90..f0ad6a017 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -134,14 +134,14 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, alDeleteBuffers(processed - 1, bufids + 1); bufid = bufids[0]; } -// else if(queued < 16) + else if(queued < 16) alGenBuffers(1, &bufid); -// else -// return; + else + return; alBufferData(bufid, channels == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16, - pcm, sample_count * channels * 2, sampling_rate); + pcm, sample_count * 2, sampling_rate); alSourceQueueBuffers(adout, 1, &bufid); int32_t state; @@ -285,6 +285,36 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) return rc; } +ALCdevice* open_audio_device(const char* audio_out_dev_name) +{ + ALCdevice* rc; + rc = alcOpenDevice(audio_out_dev_name); + if ( !rc ) { + printf("Failed to open playback device: %s: %d\n", audio_out_dev_name, alGetError()); + exit(1); + } + + ALCcontext* out_ctx = alcCreateContext(rc, NULL); + alcMakeContextCurrent(out_ctx); + + uint32_t buffers[10]; + alGenBuffers(10, buffers); + alGenSources((uint32_t)1, &adout); + alSourcei(adout, AL_LOOPING, AL_FALSE); + + int16_t zeros[10000]; + memset(zeros, 0, sizeof(zeros)); + + int i; + for ( i = 0; i < 10; ++i ) + alBufferData(buffers[i], AL_FORMAT_STEREO16, zeros, sizeof(zeros), 48000); + + alSourceQueueBuffers(adout, 10, buffers); + alSourcePlay(adout); + + return rc; +} + int print_audio_devices() { const char *device; @@ -373,49 +403,18 @@ int main (int argc, char** argv) } } - ALCdevice* audio_out_device; - - { /* Open output device */ - const char* audio_out_dev_name = NULL; - - int i = 0; - for(audio_out_dev_name = alcGetString(NULL, ALC_DEVICE_SPECIFIER); i < audio_out_dev_idx; - audio_out_dev_name += strlen( audio_out_dev_name ) + 1, ++i) - if (!(audio_out_dev_name + strlen( audio_out_dev_name ) + 1)) - break; - - audio_out_device = alcOpenDevice(audio_out_dev_name); - if ( !audio_out_device ) { - printf("Failed to open playback device: %s: %d\n", audio_out_dev_name, alGetError()); - exit(1); - } - - ALCcontext* out_ctx = alcCreateContext(audio_out_device, NULL); - alcMakeContextCurrent(out_ctx); - - uint32_t buffers[5]; - alGenBuffers(5, buffers); - alGenSources((uint32_t)1, &adout); - alSourcei(adout, AL_LOOPING, AL_FALSE); - - uint16_t zeros[10000]; - memset(zeros, 0, 10000); - - for ( i = 0; i < 5; ++i ) - alBufferData(buffers[i], AL_FORMAT_STEREO16, zeros, 10000, 48000); - - alSourceQueueBuffers(adout, 5, buffers); - alSourcePlay(adout); - - printf("Using audio device: %s\n", audio_out_dev_name); - } + const char* audio_out_dev_name = NULL; + + int i = 0; + for(audio_out_dev_name = alcGetString(NULL, ALC_DEVICE_SPECIFIER); i < audio_out_dev_idx; + audio_out_dev_name += strlen( audio_out_dev_name ) + 1, ++i) + if (!(audio_out_dev_name + strlen( audio_out_dev_name ) + 1)) + break; + printf("Using audio device: %s\n", audio_out_dev_name); printf("Using audio file: %s\n", af_name); printf("Using video file: %s\n", vf_name); - - - /* START TOX NETWORK */ @@ -697,11 +696,11 @@ int main (int argc, char** argv) printf("Failed to open the file.\n"); exit(1); } + ALCdevice* audio_out_device = open_audio_device(audio_out_dev_name); - /* Run for 5 seconds */ uint32_t frame_duration = 10; - int16_t PCM[10000]; + int16_t PCM[5760]; time_t start_time = time(NULL); time_t expected_time = af_info.frames / af_info.samplerate + 2; @@ -711,18 +710,21 @@ int main (int argc, char** argv) int64_t count = sf_read_short(af_handle, PCM, frame_size); if (count > 0) { - TOXAV_ERR_SEND_FRAME rc; - if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) { - printf("Error sending frame of size %ld: %d\n", count, rc); - exit(1); - } + t_toxav_receive_audio_frame_cb(BobAV, 0, PCM, count, af_info.channels, af_info.samplerate, NULL); +// TOXAV_ERR_SEND_FRAME rc; +// if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) { +// printf("Error sending frame of size %ld: %d\n", count, rc); +// exit(1); +// } } - - iterate_tox(bootstrap, AliceAV, BobAV); + c_sleep(frame_duration); +// iterate_tox(bootstrap, AliceAV, BobAV); } + printf("Played file in: %lu\n", time(NULL) - start_time); + alcCloseDevice(audio_out_device); sf_close(af_handle); { /* Hangup */ @@ -774,7 +776,5 @@ int main (int argc, char** argv) tox_kill(bootstrap); printf("\nTest successful!\n"); - - alcCloseDevice(audio_out_device); return 0; } diff --git a/toxav/codec.c b/toxav/codec.c index b9cbbc06f..b0f2ed319 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -238,10 +238,80 @@ static int convert_bw_to_sampling_rate(int bw) } } +OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count) +{ + int status = OPUS_OK; + OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_AUDIO, &status); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(status)); + return NULL; + } + + status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bitrate)); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); + goto FAILURE; + } + + status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(10)); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); + goto FAILURE; + } + + return rc; + +FAILURE: + opus_encoder_destroy(rc); + return NULL; +} + +bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate) +{ + assert(dest); + + vpx_codec_enc_cfg_t cfg; + int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); + + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); + return false; + } + + rc = vpx_codec_enc_init_ver(dest, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, + VPX_ENCODER_ABI_VERSION); + + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); + return false; + } + + cfg.rc_target_bitrate = bitrate; + cfg.g_w = 800; + cfg.g_h = 600; + cfg.g_pass = VPX_RC_ONE_PASS; + cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; + cfg.g_lag_in_frames = 0; + cfg.kf_min_dist = 0; + cfg.kf_max_dist = 48; + cfg.kf_mode = VPX_KF_AUTO; + + rc = vpx_codec_control(dest, VP8E_SET_CPUUSED, 8); + + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); + vpx_codec_destroy(dest); + } + + return true; +} /* PUBLIC */ -void cs_do(CSSession *cs) +void cs_do(CSession *cs) { /* Codec session should always be protected by call mutex so no need to check for cs validity */ @@ -261,7 +331,7 @@ void cs_do(CSSession *cs) RTPMessage *msg; /* The maximum for 120 ms 48 KHz audio */ - int16_t tmp[20000]; + int16_t tmp[5760]; while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { pthread_mutex_unlock(cs->queue_mutex); @@ -276,7 +346,7 @@ void cs_do(CSSession *cs) rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); if (rc != -1) { cs->last_packet_sampling_rate = rc; - cs->last_packet_channels = opus_packet_get_nb_channels(msg->data); + cs->last_packet_channel_count = opus_packet_get_nb_channels(msg->data); cs->last_packet_frame_duration = ( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 ) @@ -287,7 +357,7 @@ void cs_do(CSSession *cs) continue; } - rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, sizeof(tmp), 0); + rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, 5760, 0); rtp_free_msg(NULL, msg); } @@ -295,21 +365,11 @@ void cs_do(CSSession *cs) LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); } else if (cs->acb.first) { /* Play */ - LOGGER_DEBUG("Playing audio frame size: %d; channels: %d; srate: %d; duration %d", rc, - cs->last_packet_channels, cs->last_packet_sampling_rate, cs->last_packet_frame_duration); + cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->last_packet_frame_duration); - /* According to https://tools.ietf.org/html/rfc6716#section-2.1.2 - * Every encoder can encode both mono and stereo data so we must - * determine which format is selected. - */ - - if (cs->last_packet_channels == 2) { - /* The packet is encoded with stereo encoder */ - } - - cs->acb.first(cs->agent, cs->friend_id, tmp, rc, - cs->last_packet_channels, cs->last_packet_sampling_rate, cs->acb.second); + cs->acb.first(cs->av, cs->friend_id, tmp, rc, + cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second); } pthread_mutex_lock(cs->queue_mutex); @@ -336,7 +396,7 @@ void cs_do(CSSession *cs) /* Play decoded images */ for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) { if (cs->vcb.first) - cs->vcb.first(cs->agent, cs->friend_id, dest->d_w, dest->d_h, + cs->vcb.first(cs->av, cs->friend_id, dest->d_w, dest->d_h, (const uint8_t**)dest->planes, dest->stride, cs->vcb.second); vpx_img_free(dest); @@ -349,29 +409,103 @@ void cs_do(CSSession *cs) pthread_mutex_unlock(cs->queue_mutex); } -CSSession *cs_new(uint32_t peer_video_frame_piece_size) +CSession *cs_new(uint32_t peer_video_frame_piece_size) { - CSSession *cs = calloc(sizeof(CSSession), 1); + CSession *cs = calloc(sizeof(CSession), 1); if (!cs) { LOGGER_WARNING("Allocation failed! Application might misbehave!"); return NULL; } - if (create_recursive_mutex(cs->queue_mutex) != 0) { LOGGER_WARNING("Failed to create recursive mutex!"); free(cs); return NULL; } + /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ + /* Create decoders and set up their values + */ + + /* + * AUDIO + */ + + int status; + cs->audio_decoder = opus_decoder_create(48000, 2, &status ); /* NOTE: Must be mono */ + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status)); + goto FAILURE; + } + + /* These need to be set in order to properly + * do error correction with opus */ + cs->last_packet_frame_duration = 120; + cs->last_packet_sampling_rate = 48000; + + if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) { + LOGGER_WARNING("Jitter buffer creaton failed!"); + opus_decoder_destroy(cs->audio_decoder); + goto FAILURE; + } + + /* + * VIDEO + */ + int rc = vpx_codec_dec_init_ver(cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, + NULL, 0, VPX_DECODER_ABI_VERSION); + + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); + goto AUDIO_DECODER_CLEANUP; + } + + if ( !(cs->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) ) { + vpx_codec_destroy(cs->v_decoder); + goto AUDIO_DECODER_CLEANUP; + } + + if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) ) { + free(cs->frame_buf); + vpx_codec_destroy(cs->v_decoder); + goto AUDIO_DECODER_CLEANUP; + } + /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ + + /* Initialize encoders with default values */ + cs->audio_encoder = create_audio_encoder(48000, 48000, 2); + if (cs->audio_encoder == NULL) + goto VIDEO_DECODER_CLEANUP; + + cs->last_encoding_bitrate = 48000; + cs->last_encoding_sampling_rate = 48000; + cs->last_encoding_channel_count = 2; + + if (!create_video_encoder(cs->v_encoder, 500000)) { + opus_encoder_destroy(cs->audio_encoder); + goto VIDEO_DECODER_CLEANUP; + } cs->peer_video_frame_piece_size = peer_video_frame_piece_size; return cs; + +VIDEO_DECODER_CLEANUP: + buffer_free(cs->vbuf_raw); + free(cs->frame_buf); + vpx_codec_destroy(cs->v_decoder); +AUDIO_DECODER_CLEANUP: + opus_decoder_destroy(cs->audio_decoder); + jbuf_free(cs->j_buf); +FAILURE: + pthread_mutex_destroy(cs->queue_mutex); + free(cs); + return NULL; } -void cs_kill(CSSession *cs) +void cs_kill(CSession *cs) { if (!cs) return; @@ -380,10 +514,13 @@ void cs_kill(CSSession *cs) * the callback is unregistered before cs_kill is called. */ - cs_disable_audio_sending(cs); - cs_disable_audio_receiving(cs); - cs_disable_video_sending(cs); - cs_disable_video_receiving(cs); + vpx_codec_destroy(cs->v_encoder); + vpx_codec_destroy(cs->v_decoder); + opus_encoder_destroy(cs->audio_encoder); + opus_decoder_destroy(cs->audio_decoder); + buffer_free(cs->vbuf_raw); + jbuf_free(cs->j_buf); + free(cs->frame_buf); pthread_mutex_destroy(cs->queue_mutex); @@ -391,15 +528,13 @@ void cs_kill(CSSession *cs) free(cs); } - - -void cs_init_video_splitter_cycle(CSSession* cs) +void cs_init_video_splitter_cycle(CSession* cs) { cs->split_video_frame[0] = cs->frameid_out++; cs->split_video_frame[1] = 0; } -int cs_update_video_splitter_cycle(CSSession *cs, const uint8_t *payload, uint16_t length) +int cs_update_video_splitter_cycle(CSession *cs, const uint8_t *payload, uint16_t length) { cs->processing_video_frame = payload; cs->processing_video_frame_size = length; @@ -407,7 +542,7 @@ int cs_update_video_splitter_cycle(CSSession *cs, const uint8_t *payload, uint16 return ((length - 1) / VIDEOFRAME_PIECE_SIZE) + 1; } -const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size) +const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size) { if (!cs || !size) return NULL; @@ -433,295 +568,61 @@ const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size) return cs->split_video_frame; } - - -int cs_set_sending_video_resolution(CSSession *cs, uint16_t width, uint16_t height) +int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height) { - if (!cs->v_encoding) - return -1; - - /* TODO FIXME reference is safe? */ vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc; + if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) + return 0; /* Nothing changed */ - if (cfg.g_w == width && cfg.g_h == height) - return 0; -/* - if (width * height > cs->max_width * cs->max_height) { - vpx_codec_ctx_t v_encoder = cs->v_encoder; - - if (init_video_encoder(cs, width, height, cs->video_bitrate) == -1) { - cs->v_encoder = v_encoder; - return cs_ErrorSettingVideoResolution; - } - - vpx_codec_destroy(&v_encoder); - return 0; - }*/ - - LOGGER_DEBUG("New video resolution: %u %u", width, height); + cfg.rc_target_bitrate = bitrate; cfg.g_w = width; cfg.g_h = height; - int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - return cs_ErrorSettingVideoResolution; - } - - return 0; -} - -int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate) -{ - if (!cs->v_encoding) - return -1; - - /* TODO FIXME reference is safe? */ - vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc; - if (cfg.rc_target_bitrate == bitrate) - return 0; - - LOGGER_DEBUG("New video bitrate: %u", bitrate); - cfg.rc_target_bitrate = bitrate; int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg); if ( rc != VPX_CODEC_OK) { LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - return cs_ErrorSettingVideoBitrate; + return -1; } return 0; } - - -int cs_enable_video_sending(CSSession* cs, uint32_t bitrate) +int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels) { - if (cs->v_encoding) - return 0; + /* Values are checked in toxav.c */ - vpx_codec_enc_cfg_t cfg; - int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); - - if (rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - rc = vpx_codec_enc_init_ver(cs->v_encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, - VPX_ENCODER_ABI_VERSION); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - /* So that we can use cs_disable_video_sending to clean up */ - cs->v_encoding = true; - - if ( !(cs->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) ) - goto FAILURE; - - cfg.rc_target_bitrate = bitrate; - cfg.g_w = 800; - cfg.g_h = 600; - cfg.g_pass = VPX_RC_ONE_PASS; - cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; - cfg.g_lag_in_frames = 0; - cfg.kf_min_dist = 0; - cfg.kf_max_dist = 48; - cfg.kf_mode = VPX_KF_AUTO; - - - rc = vpx_codec_control(cs->v_encoder, VP8E_SET_CPUUSED, 8); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - goto FAILURE; - } - - return 0; - -FAILURE: - cs_disable_video_sending(cs); - return -1; -} - -int cs_enable_video_receiving(CSSession* cs) -{ - if (cs->v_decoding) - return 0; - - int rc = vpx_codec_dec_init_ver(cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, - NULL, 0, VPX_DECODER_ABI_VERSION); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - /* So that we can use cs_disable_video_sending to clean up */ - cs->v_decoding = true; - - if ( !(cs->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) ) - goto FAILURE; - - if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) ) - goto FAILURE; - - return 0; - -FAILURE: - cs_disable_video_receiving(cs); - return -1; -} - - - -void cs_disable_video_sending(CSSession* cs) -{ - if (cs->v_encoding) { - cs->v_encoding = false; + if (cs->last_encoding_sampling_rate != sampling_rate || cs->last_encoding_channel_count != channels) { + OpusEncoder* new_encoder = create_audio_encoder(bitrate, sampling_rate, channels); + if (new_encoder == NULL) + return -1; - free(cs->split_video_frame); - cs->split_video_frame = NULL; - - vpx_codec_destroy(cs->v_encoder); - } -} - -void cs_disable_video_receiving(CSSession* cs) -{ - if (cs->v_decoding) { - cs->v_decoding = false; - - buffer_free(cs->vbuf_raw); - cs->vbuf_raw = NULL; - free(cs->frame_buf); - cs->frame_buf = NULL; - - vpx_codec_destroy(cs->v_decoder); - } -} - - - -int cs_set_sending_audio_bitrate(CSSession *cs, int32_t rate) -{ - if (cs->audio_encoder == NULL) - return -1; - - int rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(rate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); - return -1; - } - - LOGGER_DEBUG("Set new encoder bitrate to: %d", rate); - return 0; -} - - - -void cs_disable_audio_sending(CSSession* cs) -{ - if ( cs->audio_encoder ) { opus_encoder_destroy(cs->audio_encoder); - cs->audio_encoder = NULL; - } -} - -void cs_disable_audio_receiving(CSSession* cs) -{ - if ( cs->audio_decoder ) { - opus_decoder_destroy(cs->audio_decoder); - cs->audio_decoder = NULL; - jbuf_free(cs->j_buf); - cs->j_buf = NULL; + cs->audio_encoder = new_encoder; + } else if (cs->last_encoding_bitrate == bitrate) + return 0; /* Nothing changed */ + else { + int status = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate)); - /* These need to be set in order to properly - * do error correction with opus */ - cs->last_packet_frame_duration = 120; - cs->last_packet_sampling_rate = 48000; + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); + return -1; + } } -} - - -int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate) -{ - if (cs->audio_encoder) - return 0; - - int rc = OPUS_OK; - cs->audio_encoder = opus_encoder_create(48000, 2, OPUS_APPLICATION_AUDIO, &rc); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); - return -1; - } - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); - goto FAILURE; - } - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); - goto FAILURE; - } - - return 0; - -FAILURE: - cs_disable_audio_sending(cs); - return -1; -} - -int cs_enable_audio_receiving(CSSession* cs) -{ - if (cs->audio_decoder) - return 0; - - int rc; - cs->audio_decoder = opus_decoder_create(48000, 2, &rc ); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc)); - return -1; - } - - - if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) { - LOGGER_WARNING("Jitter buffer creaton failed!"); - opus_decoder_destroy(cs->audio_decoder); - cs->audio_decoder = NULL; - return -1; - } - - - /* These need to be set in order to properly - * do error correction with opus */ - cs->last_packet_frame_duration = 120; - cs->last_packet_sampling_rate = 48000; - + cs->last_encoding_bitrate = bitrate; + cs->last_encoding_sampling_rate = sampling_rate; + cs->last_encoding_channel_count = channels; return 0; } - /* Called from RTP */ void queue_message(RTPSession *session, RTPMessage *msg) { /* This function is unregistered during call termination befor destroying * Codec session so no need to check for validity of cs TODO properly check video cycle */ - CSSession *cs = session->cs; + CSession *cs = session->cs; if (!cs) return; diff --git a/toxav/codec.h b/toxav/codec.h index 526a80d5c..b13203f1d 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -42,32 +42,7 @@ #define PAIR(TYPE1__, TYPE2__) struct { TYPE1__ first; TYPE2__ second; } -typedef void (*CSAudioCallback) (void *agent, int32_t call_idx, const int16_t *PCM, uint16_t size, void *data); -typedef void (*CSVideoCallback) (void *agent, int32_t call_idx, const vpx_image_t *img, void *data); - -/** - * Codec capabilities - */ -typedef enum { - cs_AudioEncoding = 1 << 0, - cs_AudioDecoding = 1 << 1, - cs_VideoEncoding = 1 << 2, - cs_VideoDecoding = 1 << 3 -} CSCapabilities; - -/** - * Codec errors. - */ -typedef enum { - cs_ErrorSettingVideoResolution = -30, - cs_ErrorSettingVideoBitrate = -31, - cs_ErrorSplittingVideoPayload = -32, -} CSError; - -/** - * Codec session - controling codec - */ -typedef struct CSSession_s { +typedef struct CSession_s { /* VIDEO * @@ -76,12 +51,10 @@ typedef struct CSSession_s { /* video encoding */ vpx_codec_ctx_t v_encoder[1]; - bool v_encoding; uint32_t frame_counter; /* video decoding */ vpx_codec_ctx_t v_decoder[1]; - bool v_decoding; void *vbuf_raw; /* Un-decoded data */ /* Data handling */ @@ -107,10 +80,13 @@ typedef struct CSSession_s { /* audio encoding */ OpusEncoder *audio_encoder; + int32_t last_encoding_sampling_rate; + int32_t last_encoding_channel_count; + int32_t last_encoding_bitrate; /* audio decoding */ OpusDecoder *audio_decoder; - int32_t last_packet_channels; + int32_t last_packet_channel_count; int32_t last_packet_sampling_rate; int32_t last_packet_frame_duration; struct JitterBuffer_s *j_buf; @@ -120,55 +96,28 @@ typedef struct CSSession_s { * * */ - void *agent; /* Pointer to ToxAV TODO make this pointer to ToxAV*/ + ToxAV *av; int32_t friend_id; PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ pthread_mutex_t queue_mutex[1]; -} CSSession; +} CSession; -/** - * Generic - */ -void cs_do(CSSession *cs); - +void cs_do(CSession *cs); /* Make sure to be called BEFORE corresponding rtp_new */ -CSSession *cs_new(uint32_t peer_mvfpsz); +CSession *cs_new(uint32_t peer_mvfpsz); /* Make sure to be called AFTER corresponding rtp_kill */ -void cs_kill(CSSession *cs); - - -/** - * VIDEO HANDLING - */ -void cs_init_video_splitter_cycle(CSSession *cs); -int cs_update_video_splitter_cycle(CSSession* cs, const uint8_t* payload, uint16_t length); -const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size); - -int cs_set_sending_video_resolution(CSSession *cs, uint16_t width, uint16_t height); -int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate); - -int cs_enable_video_sending(CSSession* cs, uint32_t bitrate); -int cs_enable_video_receiving(CSSession* cs); - -void cs_disable_video_sending(CSSession* cs); -void cs_disable_video_receiving(CSSession* cs); - -/** - * AUDIO HANDLING - */ -int cs_set_sending_audio_bitrate(CSSession* cs, int32_t rate); - -int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate); -int cs_enable_audio_receiving(CSSession* cs); - -void cs_disable_audio_sending(CSSession* cs); -void cs_disable_audio_receiving(CSSession* cs); +void cs_kill(CSession *cs); +void cs_init_video_splitter_cycle(CSession *cs); +int cs_update_video_splitter_cycle(CSession* cs, const uint8_t* payload, uint16_t length); +const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size); +int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height); +int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels); /* Internal. Called from rtp_handle_message */ diff --git a/toxav/rtp.h b/toxav/rtp.h index 2950941bd..3bfbb7afc 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -101,7 +101,7 @@ typedef struct { int dest; - struct CSSession_s *cs; + struct CSession_s *cs; Messenger *m; } RTPSession; diff --git a/toxav/toxav.c b/toxav/toxav.c index 84a0c43ae..62fed33f4 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -43,7 +43,7 @@ enum { typedef struct ToxAVCall_s { ToxAV* av; RTPSession *rtps[2]; /* Audio is first and video is second */ - CSSession *cs; + CSession *cs; pthread_mutex_t mutex_audio_sending[1]; pthread_mutex_t mutex_video_sending[1]; @@ -54,8 +54,8 @@ typedef struct ToxAVCall_s { MSICall* msi_call; uint32_t friend_id; - uint32_t s_audio_b; /* Sending audio bitrate */ - uint32_t s_video_b; /* Sending video bitrate */ + uint32_t audio_bit_rate; /* Sending audio bitrate */ + uint32_t video_bit_rate; /* Sending video bitrate */ uint8_t last_self_capabilities; uint8_t last_peer_capabilities; @@ -244,8 +244,8 @@ bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint return false; } - call->s_audio_b = audio_bit_rate; - call->s_video_b = video_bit_rate; + call->audio_bit_rate = audio_bit_rate; + call->video_bit_rate = video_bit_rate; call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; @@ -302,8 +302,8 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui goto END; } - call->s_audio_b = audio_bit_rate; - call->s_video_b = video_bit_rate; + call->audio_bit_rate = audio_bit_rate; + call->video_bit_rate = video_bit_rate; call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; @@ -479,12 +479,70 @@ END: bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE* error) { - /* TODO */ + TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK; + ToxAVCall* call; + + if (m_friend_exists(av->m, friend_number) == 0) { + rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND; + goto END; + } + + if (audio_bitrate_invalid(audio_bit_rate)) { + rc = TOXAV_ERR_BIT_RATE_INVALID; + goto END; + } + + pthread_mutex_lock(av->mutex); + call = call_get(av, friend_number); + if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { + pthread_mutex_unlock(av->mutex); + rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; + goto END; + } + + /* NOTE: no need to lock*/ + call->audio_bit_rate = audio_bit_rate; + pthread_mutex_unlock(av->mutex); + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_BIT_RATE_OK; } bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE* error) { - /* TODO */ + TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK; + ToxAVCall* call; + + if (m_friend_exists(av->m, friend_number) == 0) { + rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND; + goto END; + } + + if (video_bitrate_invalid(video_bit_rate)) { + rc = TOXAV_ERR_BIT_RATE_INVALID; + goto END; + } + + pthread_mutex_lock(av->mutex); + call = call_get(av, friend_number); + if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { + pthread_mutex_unlock(av->mutex); + rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; + goto END; + } + + /* NOTE: no need to lock*/ + call->video_bit_rate = video_bit_rate; + pthread_mutex_unlock(av->mutex); + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_BIT_RATE_OK; } void toxav_callback_video_frame_request(ToxAV* av, toxav_video_frame_request_cb* function, void* user_data) @@ -507,7 +565,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); - if (call == NULL || !call->active) { + if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; @@ -516,20 +574,13 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u pthread_mutex_lock(call->mutex_video_sending); pthread_mutex_unlock(av->mutex); - if (call->msi_call->state != msi_CallActive) { - /* TODO */ - pthread_mutex_unlock(call->mutex_video_sending); - rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; - goto END; - } - if ( y == NULL || u == NULL || v == NULL ) { pthread_mutex_unlock(call->mutex_video_sending); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } - if ( cs_set_sending_video_resolution(call->cs, width, height) != 0 ) { + if ( cs_reconfigure_video_encoder(call->cs, call->video_bit_rate, width, height) != 0 ) { pthread_mutex_unlock(call->mutex_video_sending); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; @@ -550,7 +601,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u int vrc = vpx_codec_encode(call->cs->v_encoder, &img, call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); - vpx_img_free(&img); /* FIXME don't free? */ + vpx_img_free(&img); if ( vrc != VPX_CODEC_OK) { pthread_mutex_unlock(call->mutex_video_sending); LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); @@ -621,7 +672,7 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); - if (call == NULL || !call->active) { + if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; @@ -630,33 +681,34 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc pthread_mutex_lock(call->mutex_audio_sending); pthread_mutex_unlock(av->mutex); - if (call->msi_call->state != msi_CallActive) { - /* TODO */ - pthread_mutex_unlock(call->mutex_audio_sending); - rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; - goto END; - } - if ( pcm == NULL ) { pthread_mutex_unlock(call->mutex_audio_sending); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } - if ( channels != 1 && channels != 2 ) { + if ( channels > 2 ) { pthread_mutex_unlock(call->mutex_audio_sending); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } { /* Encode and send */ + if (cs_reconfigure_audio_encoder(call->cs, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) { + pthread_mutex_unlock(call->mutex_audio_sending); + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + + + LOGGER_DEBUG("Sending audio frame size: %d; channels: %d; srate: %d", sample_count, channels, sampling_rate); uint8_t dest[sample_count * channels * 2 /* sizeof(uint16_t) */]; int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest, sizeof (dest)); if (vrc < 0) { LOGGER_WARNING("Failed to encode frame"); - rc = TOXAV_ERR_SEND_FRAME_INVALID; pthread_mutex_unlock(call->mutex_audio_sending); + rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } @@ -914,19 +966,19 @@ bool call_prepare_transmission(ToxAVCall* call) } if (pthread_mutex_init(call->mutex_audio_sending, NULL) != 0) - goto MUTEX_INIT_ERROR; + return false; if (pthread_mutex_init(call->mutex_video_sending, NULL) != 0) { - pthread_mutex_destroy(call->mutex_audio_sending); - goto MUTEX_INIT_ERROR; + goto AUDIO_SENDING_MUTEX_CLEANUP; } if (pthread_mutex_init(call->mutex_decoding, NULL) != 0) { - pthread_mutex_destroy(call->mutex_audio_sending); - pthread_mutex_destroy(call->mutex_video_sending); - goto MUTEX_INIT_ERROR; + goto VIDEO_SENDING_MUTEX_CLEANUP; } + /* Creates both audio and video encoders and decoders with some default values. + * Make sure to reconfigure encoders dynamically when sending data + */ call->cs = cs_new(call->msi_call->peer_vfpsz); if ( !call->cs ) { @@ -934,13 +986,13 @@ bool call_prepare_transmission(ToxAVCall* call) goto FAILURE; } - call->cs->agent = av; + call->cs->av = av; call->cs->friend_id = call->friend_id; memcpy(&call->cs->acb, &av->acb, sizeof(av->acb)); memcpy(&call->cs->vcb, &av->vcb, sizeof(av->vcb)); - { /* Prepare audio */ + { /* Prepare audio RTP */ call->rtps[audio_index] = rtp_new(rtp_TypeAudio, av->m, call->friend_id); if ( !call->rtps[audio_index] ) { @@ -950,24 +1002,13 @@ bool call_prepare_transmission(ToxAVCall* call) call->rtps[audio_index]->cs = call->cs; - /* Only enable sending if bitrate is defined */ - if (call->s_audio_b > 0 && cs_enable_audio_sending(call->cs, call->s_audio_b * 1000) != 0) { - LOGGER_WARNING("Failed to enable audio sending!"); - goto FAILURE; - } - - if (cs_enable_audio_receiving(call->cs) != 0) { - LOGGER_WARNING("Failed to enable audio receiving!"); - goto FAILURE; - } - if (rtp_start_receiving(call->rtps[audio_index]) != 0) { LOGGER_WARNING("Failed to enable audio receiving!"); goto FAILURE; } } - { /* Prepare video */ + { /* Prepare video RTP */ call->rtps[video_index] = rtp_new(rtp_TypeVideo, av->m, call->friend_id); if ( !call->rtps[video_index] ) { @@ -977,17 +1018,6 @@ bool call_prepare_transmission(ToxAVCall* call) call->rtps[video_index]->cs = call->cs; - /* Only enable sending if bitrate is defined */ - if (call->s_video_b > 0 && cs_enable_video_sending(call->cs, call->s_video_b) != 0) { - LOGGER_WARNING("Failed to enable video sending!"); - goto FAILURE; - } - - if (cs_enable_video_receiving(call->cs) != 0) { - LOGGER_WARNING("Failed to enable video receiving!"); - goto FAILURE; - } - if (rtp_start_receiving(call->rtps[video_index]) != 0) { LOGGER_WARNING("Failed to enable audio receiving!"); goto FAILURE; @@ -1004,13 +1034,11 @@ FAILURE: call->rtps[video_index] = NULL; cs_kill(call->cs); call->cs = NULL; - pthread_mutex_destroy(call->mutex_audio_sending); - pthread_mutex_destroy(call->mutex_video_sending); pthread_mutex_destroy(call->mutex_decoding); - return false; - -MUTEX_INIT_ERROR: - LOGGER_ERROR("Mutex initialization failed!\n"); +VIDEO_SENDING_MUTEX_CLEANUP: + pthread_mutex_destroy(call->mutex_video_sending); +AUDIO_SENDING_MUTEX_CLEANUP: + pthread_mutex_destroy(call->mutex_audio_sending); return false; } diff --git a/toxav/toxav.h b/toxav/toxav.h index 48bb6b8c6..ae95c61b2 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -324,7 +324,15 @@ typedef enum TOXAV_ERR_BIT_RATE { /** * The bit rate passed was not one of the supported values. */ - TOXAV_ERR_BIT_RATE_INVALID + TOXAV_ERR_BIT_RATE_INVALID, + /** + * The friend_number passed did not designate a valid friend. + */ + TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND, + /** + * This client is currently not in a call with the friend. + */ + TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL } TOXAV_ERR_BIT_RATE; /** * Set the audio bit rate to be used in subsequent audio frames. From 62c4fd7409c47a30a8290c0bd1c486ad4b4b1973 Mon Sep 17 00:00:00 2001 From: mannol Date: Thu, 2 Apr 2015 02:04:45 +0200 Subject: [PATCH 21/97] Video preview works (lmao) --- toxav/Makefile.inc | 8 +- toxav/av_test.c | 232 ++++++++++++++++++++++++++++++--------------- toxav/codec.c | 17 +++- toxav/toxav.c | 3 +- 4 files changed, 175 insertions(+), 85 deletions(-) diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc index 8e2be25e5..a73ddc91f 100644 --- a/toxav/Makefile.inc +++ b/toxav/Makefile.inc @@ -37,14 +37,14 @@ libtoxav_la_LIBADD = libtoxcore.la \ -noinst_PROGRAMS += av_test +#noinst_PROGRAMS += av_test -av_test_SOURCES = ../toxav/av_test.c +#av_test_SOURCES = ../toxav/av_test.c -av_test_CFLAGS = $(LIBSODIUM_CFLAGS) \ +#av_test_CFLAGS = $(LIBSODIUM_CFLAGS) \ $(NACL_CFLAGS) -av_test_LDADD = $(LIBSODIUM_LDFLAGS) \ +#av_test_LDADD = $(LIBSODIUM_LDFLAGS) \ $(NACL_LDFLAGS) \ libtoxav.la \ libtoxcore.la \ diff --git a/toxav/av_test.c b/toxav/av_test.c index f0ad6a017..b7260d7d0 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -1,4 +1,4 @@ -#include "toxav.h" +#include "../toxav/toxav.h" #include "../toxcore/tox.h" /* For playing audio data */ @@ -11,9 +11,7 @@ /* For reading and displaying video data */ #include #include - -/* For converting images TODO remove */ -#include +#include #include @@ -29,6 +27,23 @@ #define c_sleep(x) usleep(1000*x) +#define CLIP(X) ( (X) > 255 ? 255 : (X) < 0 ? 0 : X) + +// RGB -> YUV +#define RGB2Y(R, G, B) CLIP(( ( 66 * (R) + 129 * (G) + 25 * (B) + 128) >> 8) + 16) +#define RGB2U(R, G, B) CLIP(( ( -38 * (R) - 74 * (G) + 112 * (B) + 128) >> 8) + 128) +#define RGB2V(R, G, B) CLIP(( ( 112 * (R) - 94 * (G) - 18 * (B) + 128) >> 8) + 128) + +// YUV -> RGB +#define C(Y) ( (Y) - 16 ) +#define D(U) ( (U) - 128 ) +#define E(V) ( (V) - 128 ) + +#define YUV2R(Y, U, V) CLIP(( 298 * C(Y) + 409 * E(V) + 128) >> 8) +#define YUV2G(Y, U, V) CLIP(( 298 * C(Y) - 100 * D(U) - 208 * E(V) + 128) >> 8) +#define YUV2B(Y, U, V) CLIP(( 298 * C(Y) + 516 * D(U) + 128) >> 8) + + /* Enable/disable tests */ #define TEST_REGULAR_AV 0 #define TEST_REGULAR_A 0 @@ -36,8 +51,8 @@ #define TEST_REJECT 0 #define TEST_CANCEL 0 #define TEST_MUTE_UNMUTE 0 -#define TEST_TRANSFER_A 1 -#define TEST_TRANSFER_V 0 +#define TEST_TRANSFER_A 0 +#define TEST_TRANSFER_V 1 typedef struct { @@ -83,38 +98,32 @@ void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint8_t const *planes[], int32_t const stride[], void *user_data) { - IplImage output_img; - const int bpl = stride[VPX_PLANE_Y]; - const int cxbpl = stride[VPX_PLANE_V]; + uint16_t *img_data = malloc(height * width * 6); - output_img.imageData = malloc(width * height * 3); - output_img.imageSize = width * height * 3; - output_img.width = width; - output_img.height = height; - - /* FIXME: possible bug? */ - const uint8_t* yData = planes[VPX_PLANE_Y]; - const uint8_t* uData = planes[VPX_PLANE_V]; - const uint8_t* vData = planes[VPX_PLANE_U]; - - // convert from planar to packed - int y = 0; - for (; y < height; ++y) - { - int x = 0; - for (; x < width; ++x) - { - uint8_t Y = planes[VPX_PLANE_Y][x + y * bpl]; - uint8_t U = planes[VPX_PLANE_V][x/(1 << 1) + y/(1 << 1)*cxbpl]; - uint8_t V = planes[VPX_PLANE_U][x/(1 << 1) + y/(1 << 1)*cxbpl]; - output_img.imageData[width * 3 * y + x * 3 + 0] = Y; - output_img.imageData[width * 3 * y + x * 3 + 1] = U; - output_img.imageData[width * 3 * y + x * 3 + 2] = V; + unsigned long int i, j; + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + uint8_t *point = (void*)img_data + 3 * ((i * width) + j); + int y = planes[0][(i * stride[0]) + j]; + int u = planes[1][((i / 2) * stride[1]) + (j / 2)]; + int v = planes[2][((i / 2) * stride[2]) + (j / 2)]; + + point[0] = YUV2R(y, u, v); + point[1] = YUV2G(y, u, v); + point[2] = YUV2B(y, u, v); } } - cvShowImage(vdout, &output_img); - free(output_img.imageData); + + CvMat mat = cvMat(height, width, CV_8UC3, img_data); + + CvSize sz = {.height = height, .width = width}; + + IplImage* header = cvCreateImageHeader(sz, 1, 3); + IplImage* img = cvGetImage(&mat, header); + cvShowImage(vdout, img); + cvWaitKey(1); + free(img_data); } void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, int16_t const *pcm, @@ -140,7 +149,7 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, return; - alBufferData(bufid, channels == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16, + alBufferData(bufid, channels == 1 ? AL_FORMAT_MONO16 : AL_FORMAT_STEREO16, pcm, sample_count * 2, sampling_rate); alSourceQueueBuffers(adout, 1, &bufid); @@ -244,45 +253,42 @@ int iterate_tox(Tox* bootstrap, ToxAV* AliceAV, ToxAV* BobAV) } int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) -{ - /* I use vpx image coz i'm noob TODO use opencv conversion */ - vpx_image_t vpx_img; - vpx_img.w = vpx_img.h = vpx_img.d_w = vpx_img.d_h = 0; +{ + int32_t strides[3] = { 1280, 640, 640 }; + uint8_t* planes[3] = { + malloc(img->height * img->width), + malloc(img->height * img->width / 2), + malloc(img->height * img->width / 2), + }; - const int w = img->width; - const int h = img->height; + int x_chroma_shift = 1; + int y_chroma_shift = 1; - vpx_img_alloc(&vpx_img, VPX_IMG_FMT_VPXI420, w, h, 1); - - int y = 0; - for (; y < h; ++y) - { - int x = 0; - for (; x < w; ++x) - { - uint8_t b = img->imageData[(x + y * w) * 3 + 0]; - uint8_t g = img->imageData[(x + y * w) * 3 + 1]; - uint8_t r = img->imageData[(x + y * w) * 3 + 2]; + int x, y; + for (y = 0; y < img->height; ++y) { + for (x = 0; x < img->width; ++x) { + uint8_t r = img->imageData[(x + y * img->width) * 3 + 0]; + uint8_t g = img->imageData[(x + y * img->width) * 3 + 1]; + uint8_t b = img->imageData[(x + y * img->width) * 3 + 2]; - vpx_img.planes[VPX_PLANE_Y][x + y * vpx_img.stride[VPX_PLANE_Y]] = ((66 * r + 129 * g + 25 * b) >> 8) + 16; - - if (!(x % (1 << vpx_img.x_chroma_shift)) && !(y % (1 << vpx_img.y_chroma_shift))) - { - const int i = x / (1 << vpx_img.x_chroma_shift); - const int j = y / (1 << vpx_img.y_chroma_shift); - vpx_img.planes[VPX_PLANE_U][i + j * vpx_img.stride[VPX_PLANE_U]] = ((112 * r + -94 * g + -18 * b) >> 8) + 128; - vpx_img.planes[VPX_PLANE_V][i + j * vpx_img.stride[VPX_PLANE_V]] = ((-38 * r + -74 * g + 112 * b) >> 8) + 128; + planes[0][x + y * strides[0]] = RGB2Y(r, g, b); + if (!(x % (1 << x_chroma_shift)) && !(y % (1 << y_chroma_shift))) { + const int i = x / (1 << x_chroma_shift); + const int j = y / (1 << y_chroma_shift); + planes[1][i + j * strides[1]] = RGB2U(r, g, b); + planes[2][i + j * strides[2]] = RGB2V(r, g, b); } } } - int rc = toxav_send_video_frame(av, friend_number, w, h, - vpx_img.planes[VPX_PLANE_Y], - vpx_img.planes[VPX_PLANE_U], - vpx_img.planes[VPX_PLANE_V], NULL); - vpx_img_free(&vpx_img); - return rc; +// int rc = toxav_send_video_frame(av, friend_number, img->width, img->height, planes[0], planes[1], planes[2], NULL); + t_toxav_receive_video_frame_cb(av, 0, img->width, img->height, (const uint8_t**) planes, strides, NULL); + free(planes[0]); + free(planes[1]); + free(planes[2]); +// return rc; + return 0; } ALCdevice* open_audio_device(const char* audio_out_dev_name) @@ -403,6 +409,32 @@ int main (int argc, char** argv) } } + + cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); + + CvCapture* capture = cvCreateFileCapture(vf_name); + if (!capture) { + printf("Failed to open video file: %s\n", vf_name); + exit(1); + } + + IplImage* frame; + time_t start_time = time(NULL); + + int frame_rate = cvGetCaptureProperty(capture, CV_CAP_PROP_FPS); + while(start_time + 4 > time(NULL)) { + frame = cvQueryFrame( capture ); + if (!frame) + break; + + send_opencv_img(NULL, 0, frame); + c_sleep(1); + } + + cvReleaseCapture(&capture); + cvDestroyWindow(vdout); + + return 0; const char* audio_out_dev_name = NULL; int i = 0; @@ -706,19 +738,19 @@ int main (int argc, char** argv) time_t expected_time = af_info.frames / af_info.samplerate + 2; while ( start_time + expected_time > time(NULL) ) { - int frame_size = (af_info.samplerate * frame_duration / 1000); + int frame_size = (af_info.samplerate * frame_duration / 1000) * af_info.channels; int64_t count = sf_read_short(af_handle, PCM, frame_size); if (count > 0) { - t_toxav_receive_audio_frame_cb(BobAV, 0, PCM, count, af_info.channels, af_info.samplerate, NULL); -// TOXAV_ERR_SEND_FRAME rc; -// if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) { -// printf("Error sending frame of size %ld: %d\n", count, rc); -// exit(1); -// } +// t_toxav_receive_audio_frame_cb(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, NULL); + TOXAV_ERR_SEND_FRAME rc; + if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) { + printf("Error sending frame of size %ld: %d\n", count, rc); + exit(1); + } } - c_sleep(frame_duration); -// iterate_tox(bootstrap, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); +// c_sleep(frame_duration); } @@ -744,6 +776,36 @@ int main (int argc, char** argv) } if (TEST_TRANSFER_V) { + printf("\nTrying video enc/dec...\n"); + + memset(&AliceCC, 0, sizeof(CallControl)); + memset(&BobCC, 0, sizeof(CallControl)); + + { /* Call */ + TOXAV_ERR_CALL rc; + toxav_call(AliceAV, 0, 0, 500000, &rc); + + if (rc != TOXAV_ERR_CALL_OK) { + printf("toxav_call failed: %d\n", rc); + exit(1); + } + } + + while (!BobCC.incoming) + iterate_tox(bootstrap, AliceAV, BobAV); + + { /* Answer */ + TOXAV_ERR_ANSWER rc; + toxav_answer(BobAV, 0, 0, 500000, &rc); + + if (rc != TOXAV_ERR_ANSWER_OK) { + printf("toxav_answer failed: %d\n", rc); + exit(1); + } + } + + iterate_tox(bootstrap, AliceAV, BobAV); + cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); CvCapture* capture = cvCreateFileCapture(vf_name); @@ -755,15 +817,35 @@ int main (int argc, char** argv) IplImage* frame; time_t start_time = time(NULL); + int frame_rate = cvGetCaptureProperty(capture, CV_CAP_PROP_FPS); while(start_time + 10 > time(NULL)) { frame = cvQueryFrame( capture ); if (!frame) break; +// cvShowImage(vdout, frame); +// cvWaitKey(frame_rate); + send_opencv_img(AliceAV, 0, frame); + iterate_tox(bootstrap, AliceAV, BobAV); } cvReleaseCapture(&capture); cvDestroyWindow(vdout); + + { /* Hangup */ + TOXAV_ERR_CALL_CONTROL rc; + toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); + + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { + printf("toxav_call_control failed: %d\n", rc); + exit(1); + } + } + + iterate_tox(bootstrap, AliceAV, BobAV); + assert(BobCC.state == TOXAV_CALL_STATE_END); + + printf("Success!"); } diff --git a/toxav/codec.c b/toxav/codec.c index b0f2ed319..4ae5a10b6 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -337,12 +337,14 @@ void cs_do(CSession *cs) pthread_mutex_unlock(cs->queue_mutex); if (success == 2) { + LOGGER_DEBUG("OPUS correction"); rc = opus_decode(cs->audio_decoder, NULL, 0, tmp, - cs->last_packet_sampling_rate * cs->last_packet_frame_duration / 1000, 1); + (cs->last_packet_sampling_rate * cs->last_packet_frame_duration / 1000) * + cs->last_packet_channel_count, 1); } else { /* Get values from packet and decode. - * It also checks for validity of an opus packet - */ + * It also checks for validity of an opus packet + */ rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); if (rc != -1) { cs->last_packet_sampling_rate = rc; @@ -351,6 +353,9 @@ void cs_do(CSession *cs) cs->last_packet_frame_duration = ( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 ) / cs->last_packet_sampling_rate; + + /* TODO FIXME WARNING calculate properly according to propper channel count */ + cs->last_packet_frame_duration /= cs->last_packet_channel_count; } else { LOGGER_WARNING("Failed to load packet values!"); rtp_free_msg(NULL, msg); @@ -433,7 +438,7 @@ CSession *cs_new(uint32_t peer_video_frame_piece_size) */ int status; - cs->audio_decoder = opus_decoder_create(48000, 2, &status ); /* NOTE: Must be mono */ + cs->audio_decoder = opus_decoder_create(48000, 1, &status ); /* NOTE: Must be mono */ if ( status != OPUS_OK ) { LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status)); @@ -472,6 +477,10 @@ CSession *cs_new(uint32_t peer_video_frame_piece_size) vpx_codec_destroy(cs->v_decoder); goto AUDIO_DECODER_CLEANUP; } + + if ( !(cs->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) ) + goto FAILURE; + /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ /* Initialize encoders with default values */ diff --git a/toxav/toxav.c b/toxav/toxav.c index 62fed33f4..9febd3a92 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -700,9 +700,8 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } - LOGGER_DEBUG("Sending audio frame size: %d; channels: %d; srate: %d", sample_count, channels, sampling_rate); - uint8_t dest[sample_count * channels * 2 /* sizeof(uint16_t) */]; + uint8_t dest[sample_count * channels * sizeof(int16_t)]; int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest, sizeof (dest)); if (vrc < 0) { From bf9f7e2ae8dc5f1b83c45e383ec70cb08c4caf5f Mon Sep 17 00:00:00 2001 From: mannol Date: Tue, 7 Apr 2015 01:24:36 +0200 Subject: [PATCH 22/97] Video works but there is one deadlock left and video replay is slow --- toxav/av_test.c | 112 ++++++++++++++++++++++++++---------------------- toxav/codec.c | 2 +- toxav/rtp.h | 12 ++++++ toxav/toxav.c | 33 ++++++++------ 4 files changed, 92 insertions(+), 67 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index b7260d7d0..28d19ec8d 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -60,6 +60,12 @@ typedef struct { uint32_t state; } CallControl; +struct toxav_thread_data { + ToxAV* AliceAV; + ToxAV* BobAV; + int32_t sig; +}; + const char* vdout = "AV Test"; uint32_t adout; @@ -240,17 +246,24 @@ int iterate_tox(Tox* bootstrap, ToxAV* AliceAV, ToxAV* BobAV) tox_do(toxav_get_tox(AliceAV)); tox_do(toxav_get_tox(BobAV)); - toxav_iterate(AliceAV); - toxav_iterate(BobAV); - - int mina = MIN(tox_do_interval(toxav_get_tox(AliceAV)), toxav_iteration_interval(AliceAV)); - int minb = MIN(tox_do_interval(toxav_get_tox(BobAV)), toxav_iteration_interval(BobAV)); - - int rc = MIN(mina, minb); + uint32_t rc = MIN(tox_do_interval(toxav_get_tox(AliceAV)), tox_do_interval(toxav_get_tox(BobAV))); c_sleep(rc); - return rc; } +void* iterate_toxav (void * data) +{ + struct toxav_thread_data* data_cast = data; + while (data_cast->sig == 0) { + toxav_iterate(data_cast->AliceAV); + toxav_iterate(data_cast->BobAV); +// c_sleep(1); + } + + data_cast->sig = 1; + + cvDestroyWindow(vdout); + pthread_exit(NULL); +} int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) { @@ -282,13 +295,11 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) } -// int rc = toxav_send_video_frame(av, friend_number, img->width, img->height, planes[0], planes[1], planes[2], NULL); - t_toxav_receive_video_frame_cb(av, 0, img->width, img->height, (const uint8_t**) planes, strides, NULL); + int rc = toxav_send_video_frame(av, friend_number, img->width, img->height, planes[0], planes[1], planes[2], NULL); free(planes[0]); free(planes[1]); free(planes[2]); -// return rc; - return 0; + return rc; } ALCdevice* open_audio_device(const char* audio_out_dev_name) @@ -349,8 +360,10 @@ int print_help (const char* name) return 0; } + int main (int argc, char** argv) { + cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); struct stat st; /* AV files for testing */ @@ -409,32 +422,6 @@ int main (int argc, char** argv) } } - - cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); - - CvCapture* capture = cvCreateFileCapture(vf_name); - if (!capture) { - printf("Failed to open video file: %s\n", vf_name); - exit(1); - } - - IplImage* frame; - time_t start_time = time(NULL); - - int frame_rate = cvGetCaptureProperty(capture, CV_CAP_PROP_FPS); - while(start_time + 4 > time(NULL)) { - frame = cvQueryFrame( capture ); - if (!frame) - break; - - send_opencv_img(NULL, 0, frame); - c_sleep(1); - } - - cvReleaseCapture(&capture); - cvDestroyWindow(vdout); - - return 0; const char* audio_out_dev_name = NULL; int i = 0; @@ -459,9 +446,6 @@ int main (int argc, char** argv) initialize_tox(&bootstrap, &AliceAV, &AliceCC, &BobAV, &BobCC); - - - #define REGULAR_CALL_FLOW(A_BR, V_BR) \ do { \ memset(&AliceCC, 0, sizeof(CallControl)); \ @@ -737,6 +721,18 @@ int main (int argc, char** argv) time_t start_time = time(NULL); time_t expected_time = af_info.frames / af_info.samplerate + 2; + + /* Start decode thread */ + struct toxav_thread_data data = { + .AliceAV = AliceAV, + .BobAV = BobAV, + .sig = 0 + }; + + pthread_t dect; + pthread_create(&dect, NULL, iterate_toxav, &data); + pthread_detach(dect); + while ( start_time + expected_time > time(NULL) ) { int frame_size = (af_info.samplerate * frame_duration / 1000) * af_info.channels; @@ -772,6 +768,10 @@ int main (int argc, char** argv) iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state == TOXAV_CALL_STATE_END); + /* Stop decode thread */ + data.sig = -1; + while(data.sig != 1); + printf("Success!"); } @@ -783,7 +783,7 @@ int main (int argc, char** argv) { /* Call */ TOXAV_ERR_CALL rc; - toxav_call(AliceAV, 0, 0, 500000, &rc); + toxav_call(AliceAV, 0, 0, 5000000, &rc); if (rc != TOXAV_ERR_CALL_OK) { printf("toxav_call failed: %d\n", rc); @@ -796,7 +796,7 @@ int main (int argc, char** argv) { /* Answer */ TOXAV_ERR_ANSWER rc; - toxav_answer(BobAV, 0, 0, 500000, &rc); + toxav_answer(BobAV, 0, 0, 5000000, &rc); if (rc != TOXAV_ERR_ANSWER_OK) { printf("toxav_answer failed: %d\n", rc); @@ -806,7 +806,16 @@ int main (int argc, char** argv) iterate_tox(bootstrap, AliceAV, BobAV); - cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); + /* Start decode thread */ + struct toxav_thread_data data = { + .AliceAV = AliceAV, + .BobAV = BobAV, + .sig = 0 + }; + + pthread_t dect; + pthread_create(&dect, NULL, iterate_toxav, &data); + pthread_detach(dect); CvCapture* capture = cvCreateFileCapture(vf_name); if (!capture) { @@ -814,23 +823,17 @@ int main (int argc, char** argv) exit(1); } - IplImage* frame; time_t start_time = time(NULL); - - int frame_rate = cvGetCaptureProperty(capture, CV_CAP_PROP_FPS); - while(start_time + 10 > time(NULL)) { - frame = cvQueryFrame( capture ); + while(start_time + 6 > time(NULL)) { + IplImage* frame = cvQueryFrame( capture ); if (!frame) break; -// cvShowImage(vdout, frame); -// cvWaitKey(frame_rate); send_opencv_img(AliceAV, 0, frame); iterate_tox(bootstrap, AliceAV, BobAV); } cvReleaseCapture(&capture); - cvDestroyWindow(vdout); { /* Hangup */ TOXAV_ERR_CALL_CONTROL rc; @@ -845,6 +848,11 @@ int main (int argc, char** argv) iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state == TOXAV_CALL_STATE_END); + /* Stop decode thread */ + printf("Stopping decode thread"); + data.sig = -1; + while(data.sig != 1); + printf("Success!"); } diff --git a/toxav/codec.c b/toxav/codec.c index 4ae5a10b6..c3f5d7402 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -423,7 +423,7 @@ CSession *cs_new(uint32_t peer_video_frame_piece_size) return NULL; } - if (create_recursive_mutex(cs->queue_mutex) != 0) { + if (pthread_mutex_init(cs->queue_mutex, NULL) != 0) { LOGGER_WARNING("Failed to create recursive mutex!"); free(cs); return NULL; diff --git a/toxav/rtp.h b/toxav/rtp.h index 3bfbb7afc..b47be18ab 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -28,6 +28,18 @@ #include "../toxcore/Messenger.h" +#define LOGGED_LOCK(mutex) do { \ + LOGGER_DEBUG("Locking mutex: %p", mutex);\ + pthread_mutex_lock(mutex);\ + LOGGER_DEBUG("Locked mutex: %p", mutex);\ +} while(0) + +#define LOGGED_UNLOCK(mutex) do { \ + LOGGER_DEBUG("Unlocking mutex: %p", mutex);\ + pthread_mutex_unlock(mutex);\ + LOGGER_DEBUG("Unlocked mutex: %p", mutex);\ +} while(0) + #define MAX_SEQU_NUM 65535 #define MAX_RTP_SIZE 65535 diff --git a/toxav/toxav.c b/toxav/toxav.c index 9febd3a92..54554918f 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -47,7 +47,7 @@ typedef struct ToxAVCall_s { pthread_mutex_t mutex_audio_sending[1]; pthread_mutex_t mutex_video_sending[1]; - /* Only audio or video can be decoded at one time */ + /* Only audio or video can be decoded at the time */ pthread_mutex_t mutex_decoding[1]; bool active; @@ -129,7 +129,8 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) goto FAILURE; } - if (create_recursive_mutex(av->mutex) == -1) { +// if (create_recursive_mutex(av->mutex) == -1) { + if (pthread_mutex_init(av->mutex, NULL) == -1) { LOGGER_WARNING("Mutex creation failed!"); rc = TOXAV_ERR_NEW_MALLOC; goto FAILURE; @@ -204,26 +205,30 @@ uint32_t toxav_iteration_interval(const ToxAV* av) void toxav_iterate(ToxAV* av) { - if (av->calls == NULL) + pthread_mutex_lock(av->mutex); + if (av->calls == NULL) { + pthread_mutex_unlock(av->mutex); return; + } uint64_t start = current_time_monotonic(); uint32_t rc = 500; - pthread_mutex_lock(av->mutex); ToxAVCall* i = av->calls[av->calls_head]; - for (; i; i = i->next) { + while (i) { if (i->active) { pthread_mutex_lock(i->mutex_decoding); pthread_mutex_unlock(av->mutex); + cs_do(i->cs); rc = MIN(i->cs->last_packet_frame_duration, rc); + + pthread_mutex_lock(av->mutex); pthread_mutex_unlock(i->mutex_decoding); - } else - continue; - - pthread_mutex_lock(av->mutex); + i = i->next; + } } + pthread_mutex_unlock(av->mutex); av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa); av->dmsst += current_time_monotonic() - start; @@ -1048,6 +1053,11 @@ void call_kill_transmission(ToxAVCall* call) call->active = 0; + rtp_kill(call->rtps[audio_index]); + call->rtps[audio_index] = NULL; + rtp_kill(call->rtps[video_index]); + call->rtps[video_index] = NULL; + pthread_mutex_lock(call->mutex_audio_sending); pthread_mutex_unlock(call->mutex_audio_sending); pthread_mutex_lock(call->mutex_video_sending); @@ -1055,11 +1065,6 @@ void call_kill_transmission(ToxAVCall* call) pthread_mutex_lock(call->mutex_decoding); pthread_mutex_unlock(call->mutex_decoding); - rtp_kill(call->rtps[audio_index]); - call->rtps[audio_index] = NULL; - rtp_kill(call->rtps[video_index]); - call->rtps[video_index] = NULL; - cs_kill(call->cs); call->cs = NULL; From 9c003c9dd215d5f6bb2c1a0fbdc2c0f7fd9def7c Mon Sep 17 00:00:00 2001 From: mannol Date: Wed, 8 Apr 2015 01:00:19 +0200 Subject: [PATCH 23/97] Video works now --- toxav/av_test.c | 126 +++++++++++++++++++++++++----------- toxav/codec.c | 39 ++++++----- toxav/codec.h | 5 +- toxav/msi.c | 46 ++++++------- toxav/rtp.h | 8 +-- toxav/toxav.c | 167 +++++++++++++++++++++++++----------------------- 6 files changed, 227 insertions(+), 164 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 28d19ec8d..a04aff7c1 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -51,8 +51,8 @@ #define TEST_REJECT 0 #define TEST_CANCEL 0 #define TEST_MUTE_UNMUTE 0 -#define TEST_TRANSFER_A 0 -#define TEST_TRANSFER_V 1 +#define TEST_TRANSFER_A 1 +#define TEST_TRANSFER_V 0 typedef struct { @@ -128,7 +128,6 @@ void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, IplImage* header = cvCreateImageHeader(sz, 1, 3); IplImage* img = cvGetImage(&mat, header); cvShowImage(vdout, img); - cvWaitKey(1); free(img_data); } void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, @@ -139,7 +138,7 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, void *user_data) { uint32_t bufid; - int32_t processed, queued; + int32_t processed = 0, queued = 16; alGetSourcei(adout, AL_BUFFERS_PROCESSED, &processed); alGetSourcei(adout, AL_BUFFERS_QUEUED, &queued); @@ -147,12 +146,12 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, uint32_t bufids[processed]; alSourceUnqueueBuffers(adout, processed, bufids); alDeleteBuffers(processed - 1, bufids + 1); - bufid = bufids[0]; +// bufid = bufids[0]; } - else if(queued < 16) +// else if(queued < 16) alGenBuffers(1, &bufid); - else - return; +// else +// return; alBufferData(bufid, channels == 1 ? AL_FORMAT_MONO16 : AL_FORMAT_STEREO16, @@ -246,22 +245,27 @@ int iterate_tox(Tox* bootstrap, ToxAV* AliceAV, ToxAV* BobAV) tox_do(toxav_get_tox(AliceAV)); tox_do(toxav_get_tox(BobAV)); - uint32_t rc = MIN(tox_do_interval(toxav_get_tox(AliceAV)), tox_do_interval(toxav_get_tox(BobAV))); - c_sleep(rc); - return rc; + return MIN(tox_do_interval(toxav_get_tox(AliceAV)), tox_do_interval(toxav_get_tox(BobAV))); } void* iterate_toxav (void * data) { struct toxav_thread_data* data_cast = data; + +// cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); + while (data_cast->sig == 0) { toxav_iterate(data_cast->AliceAV); toxav_iterate(data_cast->BobAV); -// c_sleep(1); + int rc = MIN(toxav_iteration_interval(data_cast->AliceAV), toxav_iteration_interval(data_cast->BobAV)); + + printf("\rToxAV interval: %d ", rc); + fflush(stdout); + cvWaitKey(rc); } data_cast->sig = 1; - cvDestroyWindow(vdout); +// cvDestroyWindow(vdout); pthread_exit(NULL); } @@ -314,19 +318,8 @@ ALCdevice* open_audio_device(const char* audio_out_dev_name) ALCcontext* out_ctx = alcCreateContext(rc, NULL); alcMakeContextCurrent(out_ctx); - uint32_t buffers[10]; - alGenBuffers(10, buffers); alGenSources((uint32_t)1, &adout); alSourcei(adout, AL_LOOPING, AL_FALSE); - - int16_t zeros[10000]; - memset(zeros, 0, sizeof(zeros)); - - int i; - for ( i = 0; i < 10; ++i ) - alBufferData(buffers[i], AL_FORMAT_STEREO16, zeros, sizeof(zeros), 48000); - - alSourceQueueBuffers(adout, 10, buffers); alSourcePlay(adout); return rc; @@ -351,8 +344,10 @@ int print_audio_devices() int print_help (const char* name) { printf("Usage: %s -[a:v:o:dh]\n" - "-a video input file\n" + "-a audio input file\n" + "-b audio frame duration\n" "-v video input file\n" + "-x video frame duration\n" "-o output audio device index\n" "-d print output audio devices\n" "-h print this help\n", name); @@ -363,7 +358,6 @@ int print_help (const char* name) int main (int argc, char** argv) { - cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); struct stat st; /* AV files for testing */ @@ -371,14 +365,35 @@ int main (int argc, char** argv) const char* vf_name = NULL; long audio_out_dev_idx = 0; - /* Pasre settings */ - CHECK_ARG: switch (getopt(argc, argv, "a:v:o:dh")) { + int32_t audio_frame_duration = 20; + int32_t video_frame_duration = 10; + + /* Parse settings */ + CHECK_ARG: switch (getopt(argc, argv, "a:b:v:x:o:dh")) { case 'a': af_name = optarg; goto CHECK_ARG; + case 'b':{ + char *d; + audio_frame_duration = strtol(optarg, &d, 10); + if (*d) { + printf("Invalid value for argument: 'b'"); + exit(1); + } + goto CHECK_ARG; + } case 'v': vf_name = optarg; goto CHECK_ARG; + case 'x':{ + char *d; + video_frame_duration = strtol(optarg, &d, 10); + if (*d) { + printf("Invalid value for argument: 'x'"); + exit(1); + } + goto CHECK_ARG; + } case 'o': { char *d; audio_out_dev_idx = strtol(optarg, &d, 10); @@ -434,7 +449,40 @@ int main (int argc, char** argv) printf("Using audio file: %s\n", af_name); printf("Using video file: %s\n", vf_name); + if (0) { + /* Open audio file */ + SF_INFO af_info; + SNDFILE* af_handle = sf_open(af_name, SFM_READ, &af_info); + if (af_handle == NULL) + { + printf("Failed to open the file.\n"); + exit(1); + } + ALCdevice* audio_out_device = open_audio_device(audio_out_dev_name); + + + int16_t PCM[5760]; + + time_t start_time = time(NULL); + time_t expected_time = af_info.frames / af_info.samplerate + 2; + + printf("Sample rate %d\n", af_info.samplerate); + while ( start_time + expected_time > time(NULL) ) { + int frame_size = (af_info.samplerate * audio_frame_duration / 1000) * af_info.channels; + + int64_t count = sf_read_short(af_handle, PCM, frame_size); + if (count > 0) + t_toxav_receive_audio_frame_cb(NULL, 0, PCM, count, af_info.channels, af_info.samplerate, NULL); + c_sleep(audio_frame_duration); + } + + printf("Played file in: %lu\n", time(NULL) - start_time); + + alcCloseDevice(audio_out_device); + sf_close(af_handle); + return 0; + } /* START TOX NETWORK */ Tox *bootstrap; @@ -714,8 +762,6 @@ int main (int argc, char** argv) } ALCdevice* audio_out_device = open_audio_device(audio_out_dev_name); - - uint32_t frame_duration = 10; int16_t PCM[5760]; time_t start_time = time(NULL); @@ -724,8 +770,8 @@ int main (int argc, char** argv) /* Start decode thread */ struct toxav_thread_data data = { - .AliceAV = AliceAV, - .BobAV = BobAV, + .AliceAV = AliceAV, + .BobAV = BobAV, .sig = 0 }; @@ -733,8 +779,9 @@ int main (int argc, char** argv) pthread_create(&dect, NULL, iterate_toxav, &data); pthread_detach(dect); + printf("Sample rate %d\n", af_info.samplerate); while ( start_time + expected_time > time(NULL) ) { - int frame_size = (af_info.samplerate * frame_duration / 1000) * af_info.channels; + int frame_size = (af_info.samplerate * audio_frame_duration / 1000) * af_info.channels; int64_t count = sf_read_short(af_handle, PCM, frame_size); if (count > 0) { @@ -746,7 +793,7 @@ int main (int argc, char** argv) } } iterate_tox(bootstrap, AliceAV, BobAV); -// c_sleep(frame_duration); + c_sleep(audio_frame_duration); } @@ -770,7 +817,8 @@ int main (int argc, char** argv) /* Stop decode thread */ data.sig = -1; - while(data.sig != 1); + while(data.sig != 1) + pthread_yield(); printf("Success!"); } @@ -824,13 +872,14 @@ int main (int argc, char** argv) } time_t start_time = time(NULL); - while(start_time + 6 > time(NULL)) { + while(start_time + 90 > time(NULL)) { IplImage* frame = cvQueryFrame( capture ); if (!frame) break; send_opencv_img(AliceAV, 0, frame); iterate_tox(bootstrap, AliceAV, BobAV); + c_sleep(video_frame_duration); } cvReleaseCapture(&capture); @@ -849,9 +898,10 @@ int main (int argc, char** argv) assert(BobCC.state == TOXAV_CALL_STATE_END); /* Stop decode thread */ - printf("Stopping decode thread"); + printf("Stopping decode thread\n"); data.sig = -1; - while(data.sig != 1); + while(data.sig != 1) + pthread_yield(); printf("Success!"); } diff --git a/toxav/codec.c b/toxav/codec.c index c3f5d7402..cd26d1e36 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -38,7 +38,7 @@ #include "rtp.h" #include "codec.h" -#define DEFAULT_JBUF 3 +#define DEFAULT_JBUF 6 /* Good quality encode. */ #define MAX_DECODE_TIME_US 0 @@ -324,17 +324,17 @@ void cs_do(CSession *cs) int success = 0; - pthread_mutex_lock(cs->queue_mutex); + LOGGED_LOCK(cs->queue_mutex); /********************* AUDIO *********************/ - if (cs->audio_decoder) { /* If receiving enabled */ + if (cs->audio_decoder) { RTPMessage *msg; /* The maximum for 120 ms 48 KHz audio */ int16_t tmp[5760]; while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { - pthread_mutex_unlock(cs->queue_mutex); + LOGGED_UNLOCK(cs->queue_mutex); if (success == 2) { LOGGER_DEBUG("OPUS correction"); @@ -377,7 +377,7 @@ void cs_do(CSession *cs) cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second); } - pthread_mutex_lock(cs->queue_mutex); + LOGGED_LOCK(cs->queue_mutex); } } @@ -387,7 +387,7 @@ void cs_do(CSession *cs) buffer_read(cs->vbuf_raw, &p); /* Leave space for (possibly) other thread to queue more data after we read it here */ - pthread_mutex_unlock(cs->queue_mutex); + LOGGED_UNLOCK(cs->queue_mutex); rc = vpx_codec_decode(cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); free(p); @@ -411,7 +411,7 @@ void cs_do(CSession *cs) return; } - pthread_mutex_unlock(cs->queue_mutex); + LOGGED_UNLOCK(cs->queue_mutex); } CSession *cs_new(uint32_t peer_video_frame_piece_size) @@ -423,7 +423,7 @@ CSession *cs_new(uint32_t peer_video_frame_piece_size) return NULL; } - if (pthread_mutex_init(cs->queue_mutex, NULL) != 0) { + if (create_recursive_mutex(cs->queue_mutex) != 0) { LOGGER_WARNING("Failed to create recursive mutex!"); free(cs); return NULL; @@ -481,6 +481,8 @@ CSession *cs_new(uint32_t peer_video_frame_piece_size) if ( !(cs->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) ) goto FAILURE; + cs->linfts = current_time_monotonic(); + cs->lcfd = 10; /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ /* Initialize encoders with default values */ @@ -621,6 +623,8 @@ int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling cs->last_encoding_bitrate = bitrate; cs->last_encoding_sampling_rate = sampling_rate; cs->last_encoding_channel_count = channels; + + LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels); return 0; } @@ -628,9 +632,6 @@ int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling /* Called from RTP */ void queue_message(RTPSession *session, RTPMessage *msg) { - /* This function is unregistered during call termination befor destroying - * Codec session so no need to check for validity of cs TODO properly check video cycle - */ CSession *cs = session->cs; if (!cs) @@ -638,9 +639,9 @@ void queue_message(RTPSession *session, RTPMessage *msg) /* Audio */ if (session->payload_type == rtp_TypeAudio % 128) { - pthread_mutex_lock(cs->queue_mutex); + LOGGED_LOCK(cs->queue_mutex); int ret = jbuf_write(cs->j_buf, msg); - pthread_mutex_unlock(cs->queue_mutex); + LOGGED_UNLOCK(cs->queue_mutex); if (ret == -1) { rtp_free_msg(NULL, msg); @@ -662,7 +663,7 @@ void queue_message(RTPSession *session, RTPMessage *msg) Payload *p = malloc(sizeof(Payload) + cs->frame_size); if (p) { - pthread_mutex_lock(cs->queue_mutex); + LOGGED_LOCK(cs->queue_mutex); if (buffer_full(cs->vbuf_raw)) { LOGGER_DEBUG("Dropped video frame"); @@ -673,15 +674,19 @@ void queue_message(RTPSession *session, RTPMessage *msg) p->size = cs->frame_size; memcpy(p->data, cs->frame_buf, cs->frame_size); } - + + /* Calculate time took for peer to send us this frame */ + uint32_t t_lcfd = current_time_monotonic() - cs->linfts; + cs->lcfd = t_lcfd > 100 ? cs->lcfd : t_lcfd; + cs->linfts = current_time_monotonic(); + buffer_write(cs->vbuf_raw, p); - pthread_mutex_unlock(cs->queue_mutex); + LOGGED_UNLOCK(cs->queue_mutex); } else { LOGGER_WARNING("Allocation failed! Program might misbehave!"); goto end; } - cs->last_timestamp = msg->header->timestamp; cs->frameid_in = packet[0]; memset(cs->frame_buf, 0, cs->frame_size); cs->frame_size = 0; diff --git a/toxav/codec.h b/toxav/codec.h index b13203f1d..93b08cd26 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -61,8 +61,9 @@ typedef struct CSession_s { uint8_t *frame_buf; /* buffer for split video payloads */ uint32_t frame_size; /* largest address written to in frame_buf for current input frame */ uint8_t frameid_in, frameid_out; /* id of input and output video frame */ - uint32_t last_timestamp; /* calculating cycles */ - + uint64_t linfts; /* Last received frame time stamp */ + uint32_t lcfd; /* Last calculated frame duration for incoming video payload */ + /* Limits */ uint32_t peer_video_frame_piece_size; diff --git a/toxav/msi.c b/toxav/msi.c index e21ec948a..29e053ec6 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -99,9 +99,9 @@ void handle_msi_packet ( Messenger *m, int friend_id, const uint8_t *data, uint1 */ void msi_register_callback ( MSISession *session, MSICallbackType callback, MSICallbackID id) { - pthread_mutex_lock(session->mutex); + LOGGED_LOCK(session->mutex); session->callbacks[id] = callback; - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); } MSISession *msi_new ( Messenger *messenger ) { @@ -141,7 +141,7 @@ int msi_kill ( MSISession *session ) } m_callback_msi_packet((struct Messenger *) session->messenger, NULL, NULL); - pthread_mutex_lock(session->mutex); + LOGGED_LOCK(session->mutex); if (session->calls) { MSIMessage msg; @@ -154,7 +154,7 @@ int msi_kill ( MSISession *session ) } } - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); pthread_mutex_destroy(session->mutex); LOGGER_DEBUG("Terminated session: %p", session); @@ -165,17 +165,17 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ { LOGGER_DEBUG("Session: %p Inviting friend: %u", session, friend_id); - pthread_mutex_lock(session->mutex); + LOGGED_LOCK(session->mutex); if (get_call(session, friend_id) != NULL) { LOGGER_ERROR("Already in a call"); - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); return -1; } (*call) = new_call ( session, friend_id ); if ( *call == NULL ) { - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); return -1; } @@ -195,7 +195,7 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ (*call)->state = msi_CallRequesting; LOGGER_DEBUG("Invite sent"); - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); return 0; } int msi_hangup ( MSICall* call ) @@ -203,7 +203,7 @@ int msi_hangup ( MSICall* call ) LOGGER_DEBUG("Session: %p Hanging up call with friend: %u", call->session, call->friend_id); MSISession* session = call->session; - pthread_mutex_lock(session->mutex); + LOGGED_LOCK(session->mutex); MSIMessage msg; msg_init(&msg, requ_pop); @@ -211,7 +211,7 @@ int msi_hangup ( MSICall* call ) send_message ( session->messenger, call->friend_id, &msg ); kill_call(call); - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); return 0; } int msi_answer ( MSICall* call, uint8_t capabilities ) @@ -219,13 +219,13 @@ int msi_answer ( MSICall* call, uint8_t capabilities ) LOGGER_DEBUG("Session: %p Answering call from: %u", call->session, call->friend_id); MSISession* session = call->session; - pthread_mutex_lock(session->mutex); + LOGGED_LOCK(session->mutex); if ( call->state != msi_CallRequested ) { /* Though sending in invalid state will not cause anything wierd * Its better to not do it like a maniac */ LOGGER_ERROR("Call is in invalid state!"); - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); return -1; } @@ -243,7 +243,7 @@ int msi_answer ( MSICall* call, uint8_t capabilities ) send_message ( session->messenger, call->friend_id, &msg ); call->state = msi_CallActive; - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); return 0; } @@ -252,7 +252,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) LOGGER_DEBUG("Session: %p Trying to change capabilities to friend %u", call->session, call->friend_id); MSISession* session = call->session; - pthread_mutex_lock(session->mutex); + LOGGED_LOCK(session->mutex); if ( call->state != msi_CallActive ) { /* Sending capabilities change can cause error on other side if @@ -263,7 +263,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) * like new. TODO: explain this better */ LOGGER_ERROR("Call is in invalid state!"); - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); return -1; } @@ -277,7 +277,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) send_message ( call->session->messenger, call->friend_id, &msg ); - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); return 0; } @@ -581,17 +581,17 @@ void on_peer_status(Messenger *m, int friend_id, uint8_t status, void *data) case 0: { /* Friend is now offline */ LOGGER_DEBUG("Friend %d is now offline", friend_id); - pthread_mutex_lock(session->mutex); + LOGGED_LOCK(session->mutex); MSICall* call = get_call(session, friend_id); if (call == NULL) { - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); return; } invoke_callback(call, msi_OnPeerTimeout); /* Failure is ignored */ kill_call(call); - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); } break; @@ -759,20 +759,20 @@ void handle_msi_packet ( Messenger *m, int friend_id, const uint8_t *data, uint1 LOGGER_DEBUG("Successfully parsed message"); } - pthread_mutex_lock(session->mutex); + LOGGED_LOCK(session->mutex); MSICall *call = get_call(session, friend_id); if (call == NULL) { if (msg.request.value != requ_push) { send_error(m, friend_id, msi_EStrayMessage); - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); return; } call = new_call(session, friend_id); if (call == NULL) { send_error(m, friend_id, msi_ESystem); - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); return; } } @@ -782,5 +782,5 @@ void handle_msi_packet ( Messenger *m, int friend_id, const uint8_t *data, uint1 else handle_pop(call, &msg); /* always kills the call */ - pthread_mutex_unlock(session->mutex); + LOGGED_UNLOCK(session->mutex); } diff --git a/toxav/rtp.h b/toxav/rtp.h index b47be18ab..6b796d5a9 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -29,15 +29,15 @@ #include "../toxcore/Messenger.h" #define LOGGED_LOCK(mutex) do { \ - LOGGER_DEBUG("Locking mutex: %p", mutex);\ + /*LOGGER_DEBUG("Locking mutex: %p", mutex);*/\ pthread_mutex_lock(mutex);\ - LOGGER_DEBUG("Locked mutex: %p", mutex);\ + /*LOGGER_DEBUG("Locked mutex: %p", mutex);*/\ } while(0) #define LOGGED_UNLOCK(mutex) do { \ - LOGGER_DEBUG("Unlocking mutex: %p", mutex);\ + /*LOGGER_DEBUG("Unlocking mutex: %p", mutex);*/\ pthread_mutex_unlock(mutex);\ - LOGGER_DEBUG("Unlocked mutex: %p", mutex);\ + /*LOGGER_DEBUG("Unlocked mutex: %p", mutex);*/\ } while(0) #define MAX_SEQU_NUM 65535 diff --git a/toxav/toxav.c b/toxav/toxav.c index 54554918f..0f16fde25 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -129,8 +129,7 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) goto FAILURE; } -// if (create_recursive_mutex(av->mutex) == -1) { - if (pthread_mutex_init(av->mutex, NULL) == -1) { + if (create_recursive_mutex(av->mutex) != 0) { LOGGER_WARNING("Mutex creation failed!"); rc = TOXAV_ERR_NEW_MALLOC; goto FAILURE; @@ -174,7 +173,7 @@ void toxav_kill(ToxAV* av) { if (av == NULL) return; - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); msi_kill(av->msi); @@ -187,7 +186,7 @@ void toxav_kill(ToxAV* av) } } - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); pthread_mutex_destroy(av->mutex); free(av); } @@ -205,9 +204,9 @@ uint32_t toxav_iteration_interval(const ToxAV* av) void toxav_iterate(ToxAV* av) { - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); if (av->calls == NULL) { - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); return; } @@ -215,20 +214,28 @@ void toxav_iterate(ToxAV* av) uint32_t rc = 500; ToxAVCall* i = av->calls[av->calls_head]; - while (i) { + for (; i; i = i->next) { if (i->active) { - pthread_mutex_lock(i->mutex_decoding); - pthread_mutex_unlock(av->mutex); + LOGGED_LOCK(i->mutex_decoding); + LOGGED_UNLOCK(av->mutex); cs_do(i->cs); - rc = MIN(i->cs->last_packet_frame_duration, rc); + if (i->last_self_capabilities & msi_CapRAudio) /* Receiving audio */ + rc = MIN(i->cs->last_packet_frame_duration, rc); + if (i->last_self_capabilities & msi_CapRVideo) /* Receiving video */ + rc = MIN(i->cs->lcfd, rc); - pthread_mutex_lock(av->mutex); - pthread_mutex_unlock(i->mutex_decoding); - i = i->next; + uint32_t fid = i->friend_id; + + LOGGED_UNLOCK(i->mutex_decoding); + LOGGED_LOCK(av->mutex); + + /* In case this call is popped from container stop iteration */ + if (call_get(av, fid) != i) + break; } } - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa); av->dmsst += current_time_monotonic() - start; @@ -242,10 +249,10 @@ void toxav_iterate(ToxAV* av) bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) { - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); ToxAVCall* call = call_new(av, friend_number, error); if (call == NULL) { - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); return false; } @@ -261,27 +268,27 @@ bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint call_remove(call); if (error) *error = TOXAV_ERR_CALL_MALLOC; - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); return false; } call->msi_call->av_call = call; - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); return true; } void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data) { - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); av->ccb.first = function; av->ccb.second = user_data; - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); } bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER* error) { - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); TOXAV_ERR_ANSWER rc = TOXAV_ERR_ANSWER_OK; if (m_friend_exists(av->m, friend_number) == 0) { @@ -320,7 +327,7 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui END: - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); if (error) *error = rc; @@ -330,15 +337,15 @@ END: void toxav_callback_call_state(ToxAV* av, toxav_call_state_cb* function, void* user_data) { - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); av->scb.first = function; av->scb.second = user_data; - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); } bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL* error) { - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); TOXAV_ERR_CALL_CONTROL rc = TOXAV_ERR_CALL_CONTROL_OK; if (m_friend_exists(av->m, friend_number) == 0) { @@ -474,7 +481,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co } END: - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); if (error) *error = rc; @@ -497,17 +504,17 @@ bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_ goto END; } - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; goto END; } /* NOTE: no need to lock*/ call->audio_bit_rate = audio_bit_rate; - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); END: if (error) @@ -531,17 +538,17 @@ bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_ goto END; } - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; goto END; } /* NOTE: no need to lock*/ call->video_bit_rate = video_bit_rate; - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); END: if (error) @@ -552,10 +559,10 @@ END: void toxav_callback_video_frame_request(ToxAV* av, toxav_video_frame_request_cb* function, void* user_data) { - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); av->rvcb.first = function; av->rvcb.second = user_data; - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); } bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) @@ -568,25 +575,25 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u goto END; } - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; } - pthread_mutex_lock(call->mutex_video_sending); - pthread_mutex_unlock(av->mutex); + LOGGED_LOCK(call->mutex_video_sending); + LOGGED_UNLOCK(av->mutex); if ( y == NULL || u == NULL || v == NULL ) { - pthread_mutex_unlock(call->mutex_video_sending); + LOGGED_UNLOCK(call->mutex_video_sending); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } if ( cs_reconfigure_video_encoder(call->cs, call->video_bit_rate, width, height) != 0 ) { - pthread_mutex_unlock(call->mutex_video_sending); + LOGGED_UNLOCK(call->mutex_video_sending); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } @@ -608,7 +615,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u vpx_img_free(&img); if ( vrc != VPX_CODEC_OK) { - pthread_mutex_unlock(call->mutex_video_sending); + LOGGED_UNLOCK(call->mutex_video_sending); LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; @@ -639,7 +646,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u iter = cs_iterate_split_video_frame(call->cs, &part_size); if (rtp_send_msg(call->rtps[video_index], iter, part_size) < 0) { - pthread_mutex_unlock(call->mutex_video_sending); + LOGGED_UNLOCK(call->mutex_video_sending); LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); goto END; } @@ -648,7 +655,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u } } - pthread_mutex_unlock(call->mutex_video_sending); + LOGGED_UNLOCK(call->mutex_video_sending); END: if (error) @@ -659,10 +666,10 @@ END: void toxav_callback_audio_frame_request(ToxAV* av, toxav_audio_frame_request_cb* function, void* user_data) { - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); av->racb.first = function; av->racb.second = user_data; - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); } bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error) @@ -675,32 +682,32 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; } - pthread_mutex_lock(call->mutex_audio_sending); - pthread_mutex_unlock(av->mutex); + LOGGED_LOCK(call->mutex_audio_sending); + LOGGED_UNLOCK(av->mutex); if ( pcm == NULL ) { - pthread_mutex_unlock(call->mutex_audio_sending); + LOGGED_UNLOCK(call->mutex_audio_sending); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } if ( channels > 2 ) { - pthread_mutex_unlock(call->mutex_audio_sending); + LOGGED_UNLOCK(call->mutex_audio_sending); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } { /* Encode and send */ if (cs_reconfigure_audio_encoder(call->cs, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) { - pthread_mutex_unlock(call->mutex_audio_sending); + LOGGED_UNLOCK(call->mutex_audio_sending); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } @@ -711,7 +718,7 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc if (vrc < 0) { LOGGER_WARNING("Failed to encode frame"); - pthread_mutex_unlock(call->mutex_audio_sending); + LOGGED_UNLOCK(call->mutex_audio_sending); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } @@ -722,7 +729,7 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc } } - pthread_mutex_unlock(call->mutex_audio_sending); + LOGGED_UNLOCK(call->mutex_audio_sending); END: if (error) @@ -733,18 +740,18 @@ END: void toxav_callback_receive_video_frame(ToxAV* av, toxav_receive_video_frame_cb* function, void* user_data) { - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); av->vcb.first = function; av->vcb.second = user_data; - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); } void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* function, void* user_data) { - pthread_mutex_lock(av->mutex); + LOGGED_LOCK(av->mutex); av->acb.first = function; av->acb.second = user_data; - pthread_mutex_unlock(av->mutex); + LOGGED_UNLOCK(av->mutex); } @@ -756,12 +763,12 @@ void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* int callback_invite(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - pthread_mutex_lock(toxav->mutex); + LOGGED_LOCK(toxav->mutex); ToxAVCall* av_call = call_new(toxav, call->friend_id, NULL); if (av_call == NULL) { LOGGER_WARNING("Failed to initialize call..."); - pthread_mutex_unlock(toxav->mutex); + LOGGED_UNLOCK(toxav->mutex); return -1; } @@ -772,41 +779,41 @@ int callback_invite(void* toxav_inst, MSICall* call) toxav->ccb.first(toxav, call->friend_id, call->peer_capabilities & msi_CapSAudio, call->peer_capabilities & msi_CapSVideo, toxav->ccb.second); - pthread_mutex_unlock(toxav->mutex); + LOGGED_UNLOCK(toxav->mutex); return 0; } int callback_start(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - pthread_mutex_lock(toxav->mutex); + LOGGED_LOCK(toxav->mutex); ToxAVCall* av_call = call_get(toxav, call->friend_id); if (av_call == NULL) { /* Should this ever happen? */ - pthread_mutex_unlock(toxav->mutex); + LOGGED_UNLOCK(toxav->mutex); return -1; } if (!call_prepare_transmission(av_call)) { callback_error(toxav_inst, call); call_remove(av_call); - pthread_mutex_unlock(toxav->mutex); + LOGGED_UNLOCK(toxav->mutex); return -1; } if (toxav->scb.first) toxav->scb.first(toxav, call->friend_id, call->peer_capabilities, toxav->scb.second); - pthread_mutex_unlock(toxav->mutex); + LOGGED_UNLOCK(toxav->mutex); return 0; } int callback_end(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - pthread_mutex_lock(toxav->mutex); + LOGGED_LOCK(toxav->mutex); if (toxav->scb.first) toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_END, toxav->scb.second); @@ -814,14 +821,14 @@ int callback_end(void* toxav_inst, MSICall* call) call_kill_transmission(call->av_call); call_remove(call->av_call); - pthread_mutex_unlock(toxav->mutex); + LOGGED_UNLOCK(toxav->mutex); return 0; } int callback_error(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - pthread_mutex_lock(toxav->mutex); + LOGGED_LOCK(toxav->mutex); if (toxav->scb.first) toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_ERROR, toxav->scb.second); @@ -829,21 +836,21 @@ int callback_error(void* toxav_inst, MSICall* call) call_kill_transmission(call->av_call); call_remove(call->av_call); - pthread_mutex_unlock(toxav->mutex); + LOGGED_UNLOCK(toxav->mutex); return 0; } int callback_capabilites(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - pthread_mutex_lock(toxav->mutex); + LOGGED_LOCK(toxav->mutex); /* TODO modify cs? */ if (toxav->scb.first) toxav->scb.first(toxav, call->friend_id, call->peer_capabilities, toxav->scb.second); - pthread_mutex_unlock(toxav->mutex); + LOGGED_UNLOCK(toxav->mutex); return 0; } @@ -969,14 +976,14 @@ bool call_prepare_transmission(ToxAVCall* call) return true; } - if (pthread_mutex_init(call->mutex_audio_sending, NULL) != 0) + if (create_recursive_mutex(call->mutex_audio_sending) != 0) return false; - if (pthread_mutex_init(call->mutex_video_sending, NULL) != 0) { + if (create_recursive_mutex(call->mutex_video_sending) != 0) { goto AUDIO_SENDING_MUTEX_CLEANUP; } - if (pthread_mutex_init(call->mutex_decoding, NULL) != 0) { + if (create_recursive_mutex(call->mutex_decoding) != 0) { goto VIDEO_SENDING_MUTEX_CLEANUP; } @@ -1058,12 +1065,12 @@ void call_kill_transmission(ToxAVCall* call) rtp_kill(call->rtps[video_index]); call->rtps[video_index] = NULL; - pthread_mutex_lock(call->mutex_audio_sending); - pthread_mutex_unlock(call->mutex_audio_sending); - pthread_mutex_lock(call->mutex_video_sending); - pthread_mutex_unlock(call->mutex_video_sending); - pthread_mutex_lock(call->mutex_decoding); - pthread_mutex_unlock(call->mutex_decoding); + LOGGED_LOCK(call->mutex_audio_sending); + LOGGED_UNLOCK(call->mutex_audio_sending); + LOGGED_LOCK(call->mutex_video_sending); + LOGGED_UNLOCK(call->mutex_video_sending); + LOGGED_LOCK(call->mutex_decoding); + LOGGED_UNLOCK(call->mutex_decoding); cs_kill(call->cs); call->cs = NULL; From 4fa31d14cf53dd54b182508df31b5524b1f24cb6 Mon Sep 17 00:00:00 2001 From: mannol Date: Thu, 9 Apr 2015 02:43:13 +0200 Subject: [PATCH 24/97] Make it possible to decode mono audio with stereo decoder --- toxav/av_test.c | 26 +++++++++++--------- toxav/codec.c | 64 ++++++++++++++++++++++++++++++++---------------- toxav/codec.h | 3 +++ toxav/toxav.c | 15 ++++++++---- toxcore/logger.c | 12 ++++----- 5 files changed, 76 insertions(+), 44 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index a04aff7c1..007a1a109 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -141,28 +141,30 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, int32_t processed = 0, queued = 16; alGetSourcei(adout, AL_BUFFERS_PROCESSED, &processed); alGetSourcei(adout, AL_BUFFERS_QUEUED, &queued); - + if(processed) { uint32_t bufids[processed]; alSourceUnqueueBuffers(adout, processed, bufids); alDeleteBuffers(processed - 1, bufids + 1); -// bufid = bufids[0]; + bufid = bufids[0]; } -// else if(queued < 16) +// else if(queued < 16) { alGenBuffers(1, &bufid); +// } // else // return; - + alBufferData(bufid, channels == 1 ? AL_FORMAT_MONO16 : AL_FORMAT_STEREO16, - pcm, sample_count * 2, sampling_rate); + pcm, sample_count * 2 * channels, sampling_rate); alSourceQueueBuffers(adout, 1, &bufid); - + int32_t state; alGetSourcei(adout, AL_SOURCE_STATE, &state); - - if(state != AL_PLAYING) + if(state != AL_PLAYING) { + printf("Here\n"); alSourcePlay(adout); + } } void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, uint16_t length, void *userdata) { @@ -260,7 +262,8 @@ void* iterate_toxav (void * data) printf("\rToxAV interval: %d ", rc); fflush(stdout); - cvWaitKey(rc); +// cvWaitKey(rc); + c_sleep(rc/2); } data_cast->sig = 1; @@ -319,7 +322,7 @@ ALCdevice* open_audio_device(const char* audio_out_dev_name) alcMakeContextCurrent(out_ctx); alGenSources((uint32_t)1, &adout); - alSourcei(adout, AL_LOOPING, AL_FALSE); + alSourcei(adout, AL_LOOPING, AL_FALSE); alSourcePlay(adout); return rc; @@ -730,7 +733,7 @@ int main (int argc, char** argv) { /* Call */ TOXAV_ERR_CALL rc; - toxav_call(AliceAV, 0, 48, 0, &rc); + toxav_call(AliceAV, 0, 8, 0, &rc); if (rc != TOXAV_ERR_CALL_OK) { printf("toxav_call failed: %d\n", rc); @@ -785,7 +788,6 @@ int main (int argc, char** argv) int64_t count = sf_read_short(af_handle, PCM, frame_size); if (count > 0) { -// t_toxav_receive_audio_frame_cb(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, NULL); TOXAV_ERR_SEND_FRAME rc; if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) { printf("Error sending frame of size %ld: %d\n", count, rc); diff --git a/toxav/codec.c b/toxav/codec.c index cd26d1e36..be69ee70e 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -38,7 +38,7 @@ #include "rtp.h" #include "codec.h" -#define DEFAULT_JBUF 6 +#define DEFAULT_JBUF 3 /* Good quality encode. */ #define MAX_DECODE_TIME_US 0 @@ -342,39 +342,59 @@ void cs_do(CSession *cs) (cs->last_packet_sampling_rate * cs->last_packet_frame_duration / 1000) * cs->last_packet_channel_count, 1); } else { - /* Get values from packet and decode. - * It also checks for validity of an opus packet - */ + /* Get values from packet and decode. */ + /* NOTE: This didn't work very well rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); if (rc != -1) { cs->last_packet_sampling_rate = rc; - cs->last_packet_channel_count = opus_packet_get_nb_channels(msg->data); - - cs->last_packet_frame_duration = - ( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 ) - / cs->last_packet_sampling_rate; - - /* TODO FIXME WARNING calculate properly according to propper channel count */ - cs->last_packet_frame_duration /= cs->last_packet_channel_count; } else { LOGGER_WARNING("Failed to load packet values!"); rtp_free_msg(NULL, msg); continue; - } + }*/ - rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, 5760, 0); + + /* Pick up sampling rate from packet */ + memcpy(&cs->last_packet_sampling_rate, msg->data, 4); + cs->last_packet_sampling_rate = ntohl(cs->last_packet_sampling_rate); + + cs->last_packet_channel_count = opus_packet_get_nb_channels(msg->data + 4); + rc = opus_decode(cs->audio_decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0); rtp_free_msg(NULL, msg); } if (rc < 0) { LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); } else if (cs->acb.first) { - /* Play */ - LOGGER_DEBUG("Playing audio frame size: %d; channels: %d; srate: %d; duration %d", rc, - cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->last_packet_frame_duration); - cs->acb.first(cs->av, cs->friend_id, tmp, rc, - cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second); + /* Extract channels */ + int16_t left[rc/2]; + int16_t right[rc/2]; + int i = 0; + for (; i < rc/2; i ++) { + left[i] = tmp[i * 2]; + right[i] = tmp[(i * 2) + 1]; + } + + if (memcmp(left, right, sizeof(int16_t)) == 0) { + cs->last_packet_channel_count = 1; + cs->last_packet_frame_duration = (rc * 1000) / cs->last_packet_sampling_rate * cs->last_packet_channel_count; + + LOGGER_DEBUG("Playing mono audio frame size: %d; srate: %d; duration %d", rc, + cs->last_packet_sampling_rate, cs->last_packet_frame_duration); + + cs->acb.first(cs->av, cs->friend_id, right, rc / 2, + cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second); + } else { + cs->last_packet_channel_count = 2; + cs->last_packet_frame_duration = (rc * 1000) / cs->last_packet_sampling_rate * cs->last_packet_channel_count; + + LOGGER_DEBUG("Playing stereo audio frame size: %d; channels: %d; srate: %d; duration %d", rc, + cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->last_packet_frame_duration); + + cs->acb.first(cs->av, cs->friend_id, tmp, rc, + cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second); + } } LOGGED_LOCK(cs->queue_mutex); @@ -438,7 +458,7 @@ CSession *cs_new(uint32_t peer_video_frame_piece_size) */ int status; - cs->audio_decoder = opus_decoder_create(48000, 1, &status ); /* NOTE: Must be mono */ + cs->audio_decoder = opus_decoder_create(48000, 2, &status ); /* NOTE: Must be stereo */ if ( status != OPUS_OK ) { LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status)); @@ -482,7 +502,7 @@ CSession *cs_new(uint32_t peer_video_frame_piece_size) goto FAILURE; cs->linfts = current_time_monotonic(); - cs->lcfd = 10; + cs->lcfd = 60; /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ /* Initialize encoders with default values */ @@ -579,6 +599,8 @@ const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size) return cs->split_video_frame; } + + int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height) { vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc; diff --git a/toxav/codec.h b/toxav/codec.h index 93b08cd26..4e2b995bc 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -42,6 +42,9 @@ #define PAIR(TYPE1__, TYPE2__) struct { TYPE1__ first; TYPE2__ second; } +#define PACKED_AUDIO_SIZE(x) (x + 5) +#define UNPACKED_AUDIO_SIZE(x) (x - 5) + typedef struct CSession_s { /* VIDEO diff --git a/toxav/toxav.c b/toxav/toxav.c index 0f16fde25..12f8b561d 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -223,7 +223,7 @@ void toxav_iterate(ToxAV* av) if (i->last_self_capabilities & msi_CapRAudio) /* Receiving audio */ rc = MIN(i->cs->last_packet_frame_duration, rc); if (i->last_self_capabilities & msi_CapRVideo) /* Receiving video */ - rc = MIN(i->cs->lcfd, rc); + rc = MIN(i->cs->lcfd, rc); /* TODO handle on/off */ uint32_t fid = i->friend_id; @@ -712,9 +712,11 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } - LOGGER_DEBUG("Sending audio frame size: %d; channels: %d; srate: %d", sample_count, channels, sampling_rate); - uint8_t dest[sample_count * channels * sizeof(int16_t)]; - int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest, sizeof (dest)); + uint8_t dest[sample_count * channels + sizeof(sampling_rate)]; /* This is more than enough always */ + + sampling_rate = htonl(sampling_rate); + memcpy(dest, &sampling_rate, sizeof(sampling_rate)); + int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest + sizeof(sampling_rate), sizeof(dest) - sizeof(sampling_rate)); if (vrc < 0) { LOGGER_WARNING("Failed to encode frame"); @@ -723,7 +725,10 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } - if (rtp_send_msg(call->rtps[audio_index], dest, vrc) != 0) { + LOGGER_DEBUG("Sending encoded audio frame size: %d; channels: %d; srate: %d", vrc, channels, + ntohl(sampling_rate)); + + if (rtp_send_msg(call->rtps[audio_index], dest, vrc + sizeof(sampling_rate)) != 0) { LOGGER_WARNING("Failed to send audio packet"); rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; } diff --git a/toxcore/logger.c b/toxcore/logger.c index e8aef7e0b..fc6a989aa 100644 --- a/toxcore/logger.c +++ b/toxcore/logger.c @@ -200,12 +200,12 @@ void logger_write (Logger *log, LOG_LEVEL level, const char *file, int line, con #endif static const char *logger_format = - "%s " /* Logger id string */ - "%-16s" /* Time string of format: %m:%d %H:%M:%S */ - "%u " /* Thread id */ - "%-5s " /* Logger lever string */ - "%-20s " /* File:line string */ - "- %s" /* Output message */ + "%s " /* Logger id string */ + "%-16s" /* Time string of format: %m:%d %H:%M:%S */ + "%-12u " /* Thread id */ + "%-5s " /* Logger lever string */ + "%-20s " /* File:line string */ + "- %s" /* Output message */ WIN_CR "\n"; /* Every new print new line */ From b2d88a4544a81a217db18b60d91a44d85821db3d Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 11 Apr 2015 02:07:54 +0200 Subject: [PATCH 25/97] Random fixes --- toxav/av_test.c | 174 ++++++++++++++++++++---------------------------- toxav/codec.c | 78 +++++++++++++--------- toxav/codec.h | 3 + toxav/toxav.c | 14 ++-- 4 files changed, 132 insertions(+), 137 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 007a1a109..dab1f6efd 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -1,14 +1,12 @@ #include "../toxav/toxav.h" #include "../toxcore/tox.h" -/* For playing audio data */ -#include -#include - -/* Processing wav's */ +/* Playing audio data */ +#include +/* Reading audio */ #include -/* For reading and displaying video data */ +/* Reading and Displaying video data */ #include #include #include @@ -23,7 +21,6 @@ #include #include - #define c_sleep(x) usleep(1000*x) @@ -67,7 +64,7 @@ struct toxav_thread_data { }; const char* vdout = "AV Test"; -uint32_t adout; +PaStream* adout = NULL; const char* stringify_state(TOXAV_CALL_STATE s) { @@ -137,34 +134,7 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, uint32_t sampling_rate, void *user_data) { - uint32_t bufid; - int32_t processed = 0, queued = 16; - alGetSourcei(adout, AL_BUFFERS_PROCESSED, &processed); - alGetSourcei(adout, AL_BUFFERS_QUEUED, &queued); - - if(processed) { - uint32_t bufids[processed]; - alSourceUnqueueBuffers(adout, processed, bufids); - alDeleteBuffers(processed - 1, bufids + 1); - bufid = bufids[0]; - } -// else if(queued < 16) { - alGenBuffers(1, &bufid); -// } -// else -// return; - - - alBufferData(bufid, channels == 1 ? AL_FORMAT_MONO16 : AL_FORMAT_STEREO16, - pcm, sample_count * 2 * channels, sampling_rate); - alSourceQueueBuffers(adout, 1, &bufid); - - int32_t state; - alGetSourcei(adout, AL_SOURCE_STATE, &state); - if(state != AL_PLAYING) { - printf("Here\n"); - alSourcePlay(adout); - } + Pa_WriteStream(adout, pcm, sample_count/channels); } void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, uint16_t length, void *userdata) { @@ -260,10 +230,8 @@ void* iterate_toxav (void * data) toxav_iterate(data_cast->BobAV); int rc = MIN(toxav_iteration_interval(data_cast->AliceAV), toxav_iteration_interval(data_cast->BobAV)); - printf("\rToxAV interval: %d ", rc); - fflush(stdout); // cvWaitKey(rc); - c_sleep(rc/2); + c_sleep(10); } data_cast->sig = 1; @@ -309,36 +277,13 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) return rc; } -ALCdevice* open_audio_device(const char* audio_out_dev_name) -{ - ALCdevice* rc; - rc = alcOpenDevice(audio_out_dev_name); - if ( !rc ) { - printf("Failed to open playback device: %s: %d\n", audio_out_dev_name, alGetError()); - exit(1); - } - - ALCcontext* out_ctx = alcCreateContext(rc, NULL); - alcMakeContextCurrent(out_ctx); - - alGenSources((uint32_t)1, &adout); - alSourcei(adout, AL_LOOPING, AL_FALSE); - alSourcePlay(adout); - - return rc; -} - int print_audio_devices() { - const char *device; - - printf("Default output device: %s\n", alcGetString(NULL, ALC_DEFAULT_DEVICE_SPECIFIER)); - printf("Output devices:\n"); - int i = 0; - for(device = alcGetString(NULL, ALC_DEVICE_SPECIFIER); *device; - device += strlen( device ) + 1, ++i) { - printf("%d) %s\n", i, device); + for (i = 0; i < Pa_GetDeviceCount(); ++i) { + const PaDeviceInfo* info = Pa_GetDeviceInfo(i); + if (info) + printf("%d) %s\n", i, info->name); } return 0; @@ -361,12 +306,13 @@ int print_help (const char* name) int main (int argc, char** argv) { + Pa_Initialize(); struct stat st; /* AV files for testing */ const char* af_name = NULL; const char* vf_name = NULL; - long audio_out_dev_idx = 0; + long audio_out_dev_idx = -1; int32_t audio_frame_duration = 20; int32_t video_frame_duration = 10; @@ -440,52 +386,66 @@ int main (int argc, char** argv) } } - const char* audio_out_dev_name = NULL; + if (audio_out_dev_idx < 0) + audio_out_dev_idx = Pa_GetDefaultOutputDevice(); - int i = 0; - for(audio_out_dev_name = alcGetString(NULL, ALC_DEVICE_SPECIFIER); i < audio_out_dev_idx; - audio_out_dev_name += strlen( audio_out_dev_name ) + 1, ++i) - if (!(audio_out_dev_name + strlen( audio_out_dev_name ) + 1)) - break; - - printf("Using audio device: %s\n", audio_out_dev_name); - printf("Using audio file: %s\n", af_name); - printf("Using video file: %s\n", vf_name); + const PaDeviceInfo* audio_dev = Pa_GetDeviceInfo(audio_out_dev_idx); + if (!audio_dev) { + fprintf(stderr, "Device under index: %ld invalid", audio_out_dev_idx); + return 1; + } if (0) { - /* Open audio file */ + SNDFILE* af_handle; SF_INFO af_info; - SNDFILE* af_handle = sf_open(af_name, SFM_READ, &af_info); - if (af_handle == NULL) - { + + /* Open audio file */ + af_handle = sf_open(af_name, SFM_READ, &af_info); + if (af_handle == NULL) { printf("Failed to open the file.\n"); exit(1); } - ALCdevice* audio_out_device = open_audio_device(audio_out_dev_name); + + int frame_size = (af_info.samplerate * audio_frame_duration / 1000) * af_info.channels; + + struct PaStreamParameters output; + output.device = audio_out_dev_idx; /* default output device */ + output.channelCount = af_info.channels; + output.sampleFormat = paInt16; + output.suggestedLatency = audio_dev->defaultHighOutputLatency; + output.hostApiSpecificStreamInfo = NULL; - int16_t PCM[5760]; + PaError err = Pa_OpenStream(&adout, NULL, &output, af_info.samplerate, frame_size, paNoFlag, NULL, NULL); + assert(err == paNoError); + + err = Pa_StartStream(adout); + assert(err == paNoError); + + int16_t PCM[frame_size]; time_t start_time = time(NULL); time_t expected_time = af_info.frames / af_info.samplerate + 2; printf("Sample rate %d\n", af_info.samplerate); while ( start_time + expected_time > time(NULL) ) { - int frame_size = (af_info.samplerate * audio_frame_duration / 1000) * af_info.channels; int64_t count = sf_read_short(af_handle, PCM, frame_size); - if (count > 0) + if (count > 0) { t_toxav_receive_audio_frame_cb(NULL, 0, PCM, count, af_info.channels, af_info.samplerate, NULL); - c_sleep(audio_frame_duration); + } + + c_sleep(audio_frame_duration / 2); } - - printf("Played file in: %lu\n", time(NULL) - start_time); - - alcCloseDevice(audio_out_device); - sf_close(af_handle); + Pa_Terminate(); return 0; } + + printf("Using audio device: %s\n", audio_dev->name); + printf("Using audio file: %s\n", af_name); + printf("Using video file: %s\n", vf_name); + /* START TOX NETWORK */ Tox *bootstrap; @@ -733,7 +693,7 @@ int main (int argc, char** argv) { /* Call */ TOXAV_ERR_CALL rc; - toxav_call(AliceAV, 0, 8, 0, &rc); + toxav_call(AliceAV, 0, 48, 0, &rc); if (rc != TOXAV_ERR_CALL_OK) { printf("toxav_call failed: %d\n", rc); @@ -746,7 +706,7 @@ int main (int argc, char** argv) { /* Answer */ TOXAV_ERR_ANSWER rc; - toxav_answer(BobAV, 0, 64, 0, &rc); + toxav_answer(BobAV, 0, 48, 0, &rc); if (rc != TOXAV_ERR_ANSWER_OK) { printf("toxav_answer failed: %d\n", rc); @@ -758,12 +718,26 @@ int main (int argc, char** argv) /* Open audio file */ af_handle = sf_open(af_name, SFM_READ, &af_info); - if (af_handle == NULL) - { + if (af_handle == NULL) { printf("Failed to open the file.\n"); exit(1); } - ALCdevice* audio_out_device = open_audio_device(audio_out_dev_name); + + + int frame_size = (af_info.samplerate * audio_frame_duration / 1000) * af_info.channels; + + struct PaStreamParameters output; + output.device = audio_out_dev_idx; + output.channelCount = af_info.channels; + output.sampleFormat = paInt16; + output.suggestedLatency = audio_dev->defaultHighOutputLatency; + output.hostApiSpecificStreamInfo = NULL; + + PaError err = Pa_OpenStream(&adout, NULL, &output, af_info.samplerate, frame_size, paNoFlag, NULL, NULL); + assert(err == paNoError); + + err = Pa_StartStream(adout); + assert(err == paNoError); int16_t PCM[5760]; @@ -784,24 +758,22 @@ int main (int argc, char** argv) printf("Sample rate %d\n", af_info.samplerate); while ( start_time + expected_time > time(NULL) ) { - int frame_size = (af_info.samplerate * audio_frame_duration / 1000) * af_info.channels; int64_t count = sf_read_short(af_handle, PCM, frame_size); if (count > 0) { TOXAV_ERR_SEND_FRAME rc; - if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) { + if (toxav_send_audio_frame(AliceAV, 0, PCM, count/af_info.channels, af_info.channels, af_info.samplerate, &rc) == false) { printf("Error sending frame of size %ld: %d\n", count, rc); - exit(1); +// exit(1); } } iterate_tox(bootstrap, AliceAV, BobAV); - c_sleep(audio_frame_duration); + c_sleep(30); } printf("Played file in: %lu\n", time(NULL) - start_time); - alcCloseDevice(audio_out_device); sf_close(af_handle); { /* Hangup */ diff --git a/toxav/codec.c b/toxav/codec.c index be69ee70e..d55cc3456 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -175,7 +175,7 @@ static int jbuf_write(JitterBuffer *q, RTPMessage *m) unsigned int num = sequnum % q->size; if ((uint32_t)(sequnum - q->bottom) > q->size) { - LOGGER_DEBUG("Clearing jitter: %p", q); + LOGGER_DEBUG("Clearing filled jitter buffer: %p", q); jbuf_clear(q); q->bottom = sequnum - q->capacity; @@ -309,6 +309,32 @@ bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate) return true; } +bool reconfigure_audio_decoder(CSession* cs, int32_t sampling_rate, int8_t channels) +{ + if (sampling_rate != cs->last_decoding_sampling_rate || channels != cs->last_decoding_channel_count) { + if (current_time_monotonic() - cs->last_decoder_reconfiguration < 500) + return false; + + int status; + OpusDecoder* new_dec = opus_decoder_create(sampling_rate, channels, &status ); + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status)); + return false; + } + + cs->last_decoding_sampling_rate = sampling_rate; + cs->last_decoding_channel_count = channels; + cs->last_decoder_reconfiguration = current_time_monotonic(); + + opus_decoder_destroy(cs->audio_decoder); + cs->audio_decoder = new_dec; + + LOGGER_DEBUG("Reconfigured audio decoder sr: %d cc: %d", sampling_rate, channels); + } + + return true; +} + /* PUBLIC */ void cs_do(CSession *cs) @@ -339,7 +365,7 @@ void cs_do(CSession *cs) if (success == 2) { LOGGER_DEBUG("OPUS correction"); rc = opus_decode(cs->audio_decoder, NULL, 0, tmp, - (cs->last_packet_sampling_rate * cs->last_packet_frame_duration / 1000) * + (cs->last_packet_sampling_rate * cs->last_packet_frame_duration / 1000) / cs->last_packet_channel_count, 1); } else { /* Get values from packet and decode. */ @@ -359,6 +385,17 @@ void cs_do(CSession *cs) cs->last_packet_sampling_rate = ntohl(cs->last_packet_sampling_rate); cs->last_packet_channel_count = opus_packet_get_nb_channels(msg->data + 4); + + /* + * NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa, + * it didn't work quite well. + */ + if (!reconfigure_audio_decoder(cs, cs->last_packet_sampling_rate, cs->last_packet_channel_count)) { + LOGGER_WARNING("Failed to reconfigure decoder!"); + rtp_free_msg(NULL, msg); + continue; + } + rc = opus_decode(cs->audio_decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0); rtp_free_msg(NULL, msg); } @@ -366,35 +403,12 @@ void cs_do(CSession *cs) if (rc < 0) { LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); } else if (cs->acb.first) { + cs->last_packet_channel_count = 2; + cs->last_packet_frame_duration = (rc * 1000) / cs->last_packet_sampling_rate * cs->last_packet_channel_count; - /* Extract channels */ - int16_t left[rc/2]; - int16_t right[rc/2]; - int i = 0; - for (; i < rc/2; i ++) { - left[i] = tmp[i * 2]; - right[i] = tmp[(i * 2) + 1]; - } + cs->acb.first(cs->av, cs->friend_id, tmp, rc * cs->last_packet_channel_count, + cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second); - if (memcmp(left, right, sizeof(int16_t)) == 0) { - cs->last_packet_channel_count = 1; - cs->last_packet_frame_duration = (rc * 1000) / cs->last_packet_sampling_rate * cs->last_packet_channel_count; - - LOGGER_DEBUG("Playing mono audio frame size: %d; srate: %d; duration %d", rc, - cs->last_packet_sampling_rate, cs->last_packet_frame_duration); - - cs->acb.first(cs->av, cs->friend_id, right, rc / 2, - cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second); - } else { - cs->last_packet_channel_count = 2; - cs->last_packet_frame_duration = (rc * 1000) / cs->last_packet_sampling_rate * cs->last_packet_channel_count; - - LOGGER_DEBUG("Playing stereo audio frame size: %d; channels: %d; srate: %d; duration %d", rc, - cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->last_packet_frame_duration); - - cs->acb.first(cs->av, cs->friend_id, tmp, rc, - cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second); - } } LOGGED_LOCK(cs->queue_mutex); @@ -458,13 +472,17 @@ CSession *cs_new(uint32_t peer_video_frame_piece_size) */ int status; - cs->audio_decoder = opus_decoder_create(48000, 2, &status ); /* NOTE: Must be stereo */ + cs->audio_decoder = opus_decoder_create(48000, 2, &status ); if ( status != OPUS_OK ) { LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status)); goto FAILURE; } + cs->last_decoding_channel_count = 2; + cs->last_decoding_sampling_rate = 48000; + cs->last_decoder_reconfiguration = 0; /* Make it possible to reconfigure straight away */ + /* These need to be set in order to properly * do error correction with opus */ cs->last_packet_frame_duration = 120; diff --git a/toxav/codec.h b/toxav/codec.h index 4e2b995bc..830dbbf66 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -93,6 +93,9 @@ typedef struct CSession_s { int32_t last_packet_channel_count; int32_t last_packet_sampling_rate; int32_t last_packet_frame_duration; + int32_t last_decoding_sampling_rate; + int32_t last_decoding_channel_count; + uint64_t last_decoder_reconfiguration; struct JitterBuffer_s *j_buf; diff --git a/toxav/toxav.c b/toxav/toxav.c index 12f8b561d..bd788d7d7 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -237,7 +237,8 @@ void toxav_iterate(ToxAV* av) } LOGGED_UNLOCK(av->mutex); - av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa); +// av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa); + av->interval = rc < 5 ? 1: rc - 5; av->dmsst += current_time_monotonic() - start; if (++av->dmssc == 3) { @@ -712,21 +713,22 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } - uint8_t dest[sample_count * channels + sizeof(sampling_rate)]; /* This is more than enough always */ + uint8_t dest[sample_count + sizeof(sampling_rate)]; /* This is more than enough always */ sampling_rate = htonl(sampling_rate); memcpy(dest, &sampling_rate, sizeof(sampling_rate)); - int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest + sizeof(sampling_rate), sizeof(dest) - sizeof(sampling_rate)); + int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, + dest + sizeof(sampling_rate), sizeof(dest) - sizeof(sampling_rate)); if (vrc < 0) { - LOGGER_WARNING("Failed to encode frame"); + LOGGER_WARNING("Failed to encode frame %s", opus_strerror(vrc)); LOGGED_UNLOCK(call->mutex_audio_sending); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } - LOGGER_DEBUG("Sending encoded audio frame size: %d; channels: %d; srate: %d", vrc, channels, - ntohl(sampling_rate)); +// LOGGER_DEBUG("Sending encoded audio frame size: %d; channels: %d; srate: %d", vrc, channels, +// ntohl(sampling_rate)); if (rtp_send_msg(call->rtps[audio_index], dest, vrc + sizeof(sampling_rate)) != 0) { LOGGER_WARNING("Failed to send audio packet"); From 2465f486acd90ed8395c8a83a13af09ecd024c98 Mon Sep 17 00:00:00 2001 From: mannol Date: Mon, 13 Apr 2015 01:45:53 +0200 Subject: [PATCH 26/97] Started custom RTCP --- toxav/av_test.c | 107 +++++---- toxav/codec.c | 98 +------- toxav/codec.h | 8 +- toxav/msi.h | 4 +- toxav/rtp.c | 595 ++++++++++++++++++++++++++---------------------- toxav/rtp.h | 49 ++-- toxav/toxav.c | 3 + toxcore/util.c | 73 ++++++ toxcore/util.h | 10 + 9 files changed, 500 insertions(+), 447 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index dab1f6efd..c20d459b7 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -1,5 +1,31 @@ +/** av_test.c + * + * Copyright (C) 2013-2015 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + * Compile with (Linux only; in newly created directory toxcore/dir_name): + * gcc -o av_test ../toxav/av_test.c ../build/.libs/libtox*.a -lopencv_core \ + * -lopencv_highgui -lopencv_imgproc -lsndfile -pthread -lvpx -lopus -lsodium -lportaudio + */ + + #include "../toxav/toxav.h" #include "../toxcore/tox.h" +#include "../toxcore/util.h" /* Playing audio data */ #include @@ -11,7 +37,6 @@ #include #include - #include #include #include @@ -63,8 +88,8 @@ struct toxav_thread_data { int32_t sig; }; -const char* vdout = "AV Test"; -PaStream* adout = NULL; +const char* vdout = "AV Test"; /* Video output */ +PaStream* adout = NULL; /* Audio output */ const char* stringify_state(TOXAV_CALL_STATE s) { @@ -230,7 +255,7 @@ void* iterate_toxav (void * data) toxav_iterate(data_cast->BobAV); int rc = MIN(toxav_iteration_interval(data_cast->AliceAV), toxav_iteration_interval(data_cast->BobAV)); -// cvWaitKey(rc); +// cvWaitKey(10); c_sleep(10); } @@ -306,7 +331,26 @@ int print_help (const char* name) int main (int argc, char** argv) { + RingBuffer* rb = rb_new(4); + int a[5] = {0, 1, 2, 3, 4}; + int* x; + rb_write(rb, a + 0); + rb_write(rb, a + 1); + rb_write(rb, a + 2); + rb_write(rb, a + 3); +// rb_write(rb, a + 4); + + x = rb_write(rb, a + 4); + while (rb_read(rb, (void**) &x)) +// rb_read(rb, (void**)&x); + printf("%d ", *x); + + printf("\n"); +// int r = 43; +// printf("%d\n", r >= 40 ? 3 : r / 10); + return 0; Pa_Initialize(); + struct stat st; /* AV files for testing */ @@ -395,53 +439,6 @@ int main (int argc, char** argv) return 1; } - if (0) { - SNDFILE* af_handle; - SF_INFO af_info; - - /* Open audio file */ - af_handle = sf_open(af_name, SFM_READ, &af_info); - if (af_handle == NULL) { - printf("Failed to open the file.\n"); - exit(1); - } - - int frame_size = (af_info.samplerate * audio_frame_duration / 1000) * af_info.channels; - - struct PaStreamParameters output; - output.device = audio_out_dev_idx; /* default output device */ - output.channelCount = af_info.channels; - output.sampleFormat = paInt16; - output.suggestedLatency = audio_dev->defaultHighOutputLatency; - output.hostApiSpecificStreamInfo = NULL; - - - PaError err = Pa_OpenStream(&adout, NULL, &output, af_info.samplerate, frame_size, paNoFlag, NULL, NULL); - assert(err == paNoError); - - err = Pa_StartStream(adout); - assert(err == paNoError); - - int16_t PCM[frame_size]; - - time_t start_time = time(NULL); - time_t expected_time = af_info.frames / af_info.samplerate + 2; - - printf("Sample rate %d\n", af_info.samplerate); - while ( start_time + expected_time > time(NULL) ) { - - int64_t count = sf_read_short(af_handle, PCM, frame_size); - if (count > 0) { - t_toxav_receive_audio_frame_cb(NULL, 0, PCM, count, af_info.channels, af_info.samplerate, NULL); - } - - c_sleep(audio_frame_duration / 2); - } - - Pa_Terminate(); - return 0; - } - printf("Using audio device: %s\n", audio_dev->name); printf("Using audio file: %s\n", af_name); printf("Using video file: %s\n", vf_name); @@ -758,17 +755,15 @@ int main (int argc, char** argv) printf("Sample rate %d\n", af_info.samplerate); while ( start_time + expected_time > time(NULL) ) { - int64_t count = sf_read_short(af_handle, PCM, frame_size); if (count > 0) { TOXAV_ERR_SEND_FRAME rc; if (toxav_send_audio_frame(AliceAV, 0, PCM, count/af_info.channels, af_info.channels, af_info.samplerate, &rc) == false) { printf("Error sending frame of size %ld: %d\n", count, rc); -// exit(1); } } iterate_tox(bootstrap, AliceAV, BobAV); - c_sleep(30); + c_sleep(53); } @@ -794,6 +789,8 @@ int main (int argc, char** argv) while(data.sig != 1) pthread_yield(); + Pa_StopStream(adout); + printf("Success!"); } @@ -890,5 +887,7 @@ int main (int argc, char** argv) tox_kill(bootstrap); printf("\nTest successful!\n"); + + Pa_Terminate(); return 0; } diff --git a/toxav/codec.c b/toxav/codec.c index d55cc3456..671be1ac1 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -46,78 +46,13 @@ #define MAX_VIDEOFRAME_SIZE 0x40000 /* 256KiB */ #define VIDEOFRAME_HEADER_SIZE 0x2 -/* FIXME: Might not be enough */ +/* FIXME: Might not be enough? NOTE: I think it is enough */ #define VIDEO_DECODE_BUFFER_SIZE 20 #define ARRAY(TYPE__) struct { uint16_t size; TYPE__ data[]; } typedef ARRAY(uint8_t) Payload; -typedef struct { - uint16_t size; /* Max size */ - uint16_t start; - uint16_t end; - Payload **packets; -} PayloadBuffer; - -static bool buffer_full(const PayloadBuffer *b) -{ - return (b->end + 1) % b->size == b->start; -} - -static bool buffer_empty(const PayloadBuffer *b) -{ - return b->end == b->start; -} - -static void buffer_write(PayloadBuffer *b, Payload *p) -{ - b->packets[b->end] = p; - b->end = (b->end + 1) % b->size; - - if (b->end == b->start) b->start = (b->start + 1) % b->size; /* full, overwrite */ -} - -static void buffer_read(PayloadBuffer *b, Payload **p) -{ - *p = b->packets[b->start]; - b->start = (b->start + 1) % b->size; -} - -static void buffer_clear(PayloadBuffer *b) -{ - while (!buffer_empty(b)) { - Payload *p; - buffer_read(b, &p); - free(p); - } -} - -static PayloadBuffer *buffer_new(int size) -{ - PayloadBuffer *buf = calloc(sizeof(PayloadBuffer), 1); - - if (!buf) return NULL; - - buf->size = size + 1; /* include empty elem */ - - if (!(buf->packets = calloc(buf->size, sizeof(Payload *)))) { - free(buf); - return NULL; - } - - return buf; -} - -static void buffer_free(PayloadBuffer *b) -{ - if (b) { - buffer_clear(b); - free(b->packets); - free(b); - } -} - /* JITTER BUFFER WORK */ typedef struct JitterBuffer_s { RTPMessage **queue; @@ -318,7 +253,7 @@ bool reconfigure_audio_decoder(CSession* cs, int32_t sampling_rate, int8_t chann int status; OpusDecoder* new_dec = opus_decoder_create(sampling_rate, channels, &status ); if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status)); + LOGGER_ERROR("Error while starting audio decoder(%d %d): %s", sampling_rate, channels, opus_strerror(status)); return false; } @@ -336,7 +271,6 @@ bool reconfigure_audio_decoder(CSession* cs, int32_t sampling_rate, int8_t chann } /* PUBLIC */ - void cs_do(CSession *cs) { /* Codec session should always be protected by call mutex so no need to check for cs validity @@ -416,9 +350,9 @@ void cs_do(CSession *cs) } /********************* VIDEO *********************/ - if (cs->vbuf_raw && !buffer_empty(cs->vbuf_raw)) { + if (cs->vbuf_raw && !rb_empty(cs->vbuf_raw)) { /* Decode video */ - buffer_read(cs->vbuf_raw, &p); + rb_read(cs->vbuf_raw, (void**)&p); /* Leave space for (possibly) other thread to queue more data after we read it here */ LOGGED_UNLOCK(cs->queue_mutex); @@ -447,7 +381,6 @@ void cs_do(CSession *cs) LOGGED_UNLOCK(cs->queue_mutex); } - CSession *cs_new(uint32_t peer_video_frame_piece_size) { CSession *cs = calloc(sizeof(CSession), 1); @@ -510,7 +443,7 @@ CSession *cs_new(uint32_t peer_video_frame_piece_size) goto AUDIO_DECODER_CLEANUP; } - if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) ) { + if ( !(cs->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE)) ) { free(cs->frame_buf); vpx_codec_destroy(cs->v_decoder); goto AUDIO_DECODER_CLEANUP; @@ -542,7 +475,7 @@ CSession *cs_new(uint32_t peer_video_frame_piece_size) return cs; VIDEO_DECODER_CLEANUP: - buffer_free(cs->vbuf_raw); + rb_free(cs->vbuf_raw); free(cs->frame_buf); vpx_codec_destroy(cs->v_decoder); AUDIO_DECODER_CLEANUP: @@ -553,7 +486,6 @@ FAILURE: free(cs); return NULL; } - void cs_kill(CSession *cs) { if (!cs) @@ -567,22 +499,21 @@ void cs_kill(CSession *cs) vpx_codec_destroy(cs->v_decoder); opus_encoder_destroy(cs->audio_encoder); opus_decoder_destroy(cs->audio_decoder); - buffer_free(cs->vbuf_raw); + rb_free(cs->vbuf_raw); jbuf_free(cs->j_buf); free(cs->frame_buf); + free(cs->split_video_frame); pthread_mutex_destroy(cs->queue_mutex); LOGGER_DEBUG("Terminated codec state: %p", cs); free(cs); } - void cs_init_video_splitter_cycle(CSession* cs) { cs->split_video_frame[0] = cs->frameid_out++; cs->split_video_frame[1] = 0; } - int cs_update_video_splitter_cycle(CSession *cs, const uint8_t *payload, uint16_t length) { cs->processing_video_frame = payload; @@ -590,7 +521,6 @@ int cs_update_video_splitter_cycle(CSession *cs, const uint8_t *payload, uint16_ return ((length - 1) / VIDEOFRAME_PIECE_SIZE) + 1; } - const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size) { if (!cs || !size) return NULL; @@ -616,9 +546,6 @@ const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size) return cs->split_video_frame; } - - - int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height) { vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc; @@ -637,7 +564,6 @@ int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, return 0; } - int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels) { /* Values are checked in toxav.c */ @@ -667,8 +593,6 @@ int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels); return 0; } - - /* Called from RTP */ void queue_message(RTPSession *session, RTPMessage *msg) { @@ -705,10 +629,10 @@ void queue_message(RTPSession *session, RTPMessage *msg) if (p) { LOGGED_LOCK(cs->queue_mutex); - if (buffer_full(cs->vbuf_raw)) { + if (rb_full(cs->vbuf_raw)) { LOGGER_DEBUG("Dropped video frame"); Payload *tp; - buffer_read(cs->vbuf_raw, &tp); + rb_read(cs->vbuf_raw, (void**)&tp); free(tp); } else { p->size = cs->frame_size; @@ -720,7 +644,7 @@ void queue_message(RTPSession *session, RTPMessage *msg) cs->lcfd = t_lcfd > 100 ? cs->lcfd : t_lcfd; cs->linfts = current_time_monotonic(); - buffer_write(cs->vbuf_raw, p); + rb_write(cs->vbuf_raw, p); LOGGED_UNLOCK(cs->queue_mutex); } else { LOGGER_WARNING("Allocation failed! Program might misbehave!"); diff --git a/toxav/codec.h b/toxav/codec.h index 830dbbf66..7cc9b15d7 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -25,6 +25,8 @@ #include "toxav.h" #include "rtp.h" +#include "../toxcore/util.h" + #include #include #include @@ -40,8 +42,6 @@ /* Audio encoding/decoding */ #include -#define PAIR(TYPE1__, TYPE2__) struct { TYPE1__ first; TYPE2__ second; } - #define PACKED_AUDIO_SIZE(x) (x + 5) #define UNPACKED_AUDIO_SIZE(x) (x - 5) @@ -125,8 +125,4 @@ const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size); int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height); int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels); - - -/* Internal. Called from rtp_handle_message */ -void queue_message(RTPSession *session, RTPMessage *msg); #endif /* CODEC_H */ diff --git a/toxav/msi.h b/toxav/msi.h index 8404df191..7d82afc81 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -29,7 +29,7 @@ #include "../toxcore/Messenger.h" /** Preconfigured value for video splitting */ -#define VIDEOFRAME_PIECE_SIZE 500 /* 1.25 KiB*/ +#define VIDEOFRAME_PIECE_SIZE 500 /** * Error codes. @@ -42,7 +42,7 @@ typedef enum { msi_EStrayMessage, msi_ESystem, msi_EHandle, - msi_EUndisclosed, /* NOTE: must be last enum otherwise parsing wont work */ + msi_EUndisclosed, /* NOTE: must be last enum otherwise parsing will not work */ } MSIError; /** diff --git a/toxav/rtp.c b/toxav/rtp.c index e5f453103..77fce0569 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -28,9 +28,9 @@ #include "rtp.h" #include -void queue_message(RTPSession *_session, RTPMessage *_msg); #define size_32 4 +#define RTCP_REPORT_INTERVAL_MS 500 #define ADD_FLAG_VERSION(_h, _v) do { ( _h->flags ) &= 0x3F; ( _h->flags ) |= ( ( ( _v ) << 6 ) & 0xC0 ); } while(0) #define ADD_FLAG_PADDING(_h, _v) do { if ( _v > 0 ) _v = 1; ( _h->flags ) &= 0xDF; ( _h->flags ) |= ( ( ( _v ) << 5 ) & 0x20 ); } while(0) @@ -46,24 +46,236 @@ void queue_message(RTPSession *_session, RTPMessage *_msg); #define GET_SETTING_MARKER(_h) (( _h->marker_payloadt ) >> 7) #define GET_SETTING_PAYLOAD(_h) ((_h->marker_payloadt) & 0x7f) -/** - * Checks if message came in late. - */ -static int check_late_message (RTPSession *session, RTPMessage *msg) + +typedef struct { + uint64_t timestamp; /* in ms */ + + uint32_t packets_missing; + uint32_t expected_packets; + /* ... other stuff in the future */ +} RTCPReport; + +typedef struct RTCPSession_s { + uint8_t prefix; + uint64_t last_sent_report_ts; + uint32_t last_missing_packets; + uint32_t last_expected_packets; + + RingBuffer* pl_stats; /* Packet loss stats over time */ +} RTCPSession; + + + +/* queue_message() is defined in codec.c */ +void queue_message(RTPSession *session, RTPMessage *msg); +RTPHeader *parse_header_in ( const uint8_t *payload, int length ); +RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length ); +RTPMessage *msg_parse ( const uint8_t *data, int length ); +uint8_t *parse_header_out ( const RTPHeader* header, uint8_t* payload ); +uint8_t *parse_ext_header_out ( const RTPExtHeader* header, uint8_t* payload ); +void build_header ( RTPSession* session, RTPHeader* header ); +void send_rtcp_report ( RTCPSession* session, Messenger* m, int32_t friendnumber ); +int handle_rtp_packet ( Messenger *m, int32_t friendnumber, const uint8_t *data, uint32_t length, void *object ); +int handle_rtcp_packet ( Messenger *m, int32_t friendnumber, const uint8_t *data, uint32_t length, void *object ); + + + + +RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ) { - /* - * Check Sequence number. If this new msg has lesser number then the session->rsequnum - * it shows that the message came in late. Also check timestamp to be 100% certain. - * - */ - return ( msg->header->sequnum < session->rsequnum && msg->header->timestamp < session->timestamp ) ? 0 : -1; + RTPSession *retu = calloc(1, sizeof(RTPSession)); + + if ( !retu ) { + LOGGER_WARNING("Alloc failed! Program might misbehave!"); + return NULL; + } + + retu->version = RTP_VERSION; /* It's always 2 */ + retu->padding = 0; /* If some additional data is needed about the packet */ + retu->extension = 0; /* If extension to header is needed */ + retu->cc = 1; /* Amount of contributors */ + retu->csrc = NULL; /* Container */ + retu->ssrc = random_int(); + retu->marker = 0; + retu->payload_type = payload_type % 128; + + retu->m = messenger; + retu->dest = friend_num; + retu->rsequnum = retu->sequnum = 0; + retu->ext_header = NULL; /* When needed allocate */ + + if ( !(retu->csrc = calloc(1, sizeof(uint32_t))) ) { + LOGGER_WARNING("Alloc failed! Program might misbehave!"); + free(retu); + return NULL; + } + + retu->csrc[0] = retu->ssrc; /* Set my ssrc to the list receive */ + + /* Also set payload type as prefix */ + retu->prefix = payload_type; + + + /* Initialize rtcp session */ + if (!(retu->rtcp = calloc(1, sizeof(RTCPSession)))) { + LOGGER_WARNING("Alloc failed! Program might misbehave!"); + free(retu->csrc); + free(retu); + return NULL; + } + + retu->rtcp->prefix = 222 + payload_type % 192; + retu->rtcp->pl_stats = rb_new(4); + + return retu; +} +void rtp_kill ( RTPSession *session ) +{ + if ( !session ) return; + + rtp_stop_receiving (session); + + free ( session->ext_header ); + free ( session->csrc ); + + void* t; + while (!rb_empty(session->rtcp->pl_stats)) { + rb_read(session->rtcp->pl_stats, (void**) &t); + free(t); + } + rb_free(session->rtcp->pl_stats); + + LOGGER_DEBUG("Terminated RTP session: %p", session); + + /* And finally free session */ + free ( session ); +} +void rtp_do(RTPSession *session) +{ + if (!session || !session->rtcp) + return; + + if (current_time_monotonic() - session->rtcp->last_sent_report_ts >= RTCP_REPORT_INTERVAL_MS) { + send_rtcp_report(session->rtcp, session->m, session->dest); + } + + if (rb_full(session->rtcp->pl_stats)) { + RTCPReport* reports[4]; + + int i = 0; + for (; rb_read(session->rtcp->pl_stats, (void**) reports + i); i++); + + /* Check for timed out reports (> 6 sec) */ + uint64_t now = current_time_monotonic(); + for (i = 0; i < 4 && now - reports[i]->timestamp < 6000; i ++); + for (; i < 4; i ++) { + rb_write(session->rtcp->pl_stats, reports[i]); + reports[i] = NULL; + } + if (!rb_empty(session->rtcp->pl_stats)) { + for (i = 0; reports[i] != NULL; i ++) + free(reports[i]); + return; /* As some reports are timed out, we need more... */ + } + + /* We have 4 on-time reports so we can proceed */ + uint32_t quality_loss = 0; + for (i = 0; i < 4; i++) { + uint32_t idx = reports[i]->packets_missing * 100 / reports[i]->expected_packets; + quality_loss += idx; + } + + if (quality_loss > 40) { + LOGGER_DEBUG("Packet loss detected"); + } + } +} +int rtp_start_receiving(RTPSession* session) +{ + if (session == NULL) + return -1; + + if (custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, + handle_rtp_packet, session) == -1) { + LOGGER_WARNING("Failed to register rtp receive handler"); + return -1; + } + if (custom_lossy_packet_registerhandler(session->m, session->dest, session->rtcp->prefix, + handle_rtcp_packet, session->rtcp) == -1) { + LOGGER_WARNING("Failed to register rtcp receive handler"); + custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, NULL, NULL); + return -1; + } + + return 0; +} +int rtp_stop_receiving(RTPSession* session) +{ + if (session == NULL) + return -1; + + custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, NULL, NULL); + custom_lossy_packet_registerhandler(session->m, session->dest, session->rtcp->prefix, NULL, NULL); /* RTCP */ + + return 0; +} +int rtp_send_msg ( RTPSession *session, const uint8_t *data, uint16_t length ) +{ + if ( !session ) { + LOGGER_WARNING("No session!"); + return -1; + } + + uint8_t parsed[MAX_RTP_SIZE]; + uint8_t *it; + + RTPHeader header[1]; + build_header(session, header); + + uint32_t parsed_len = length + header->length + 1; + + parsed[0] = session->prefix; + + it = parse_header_out ( header, parsed + 1 ); + + if ( session->ext_header ) { + parsed_len += ( 4 /* Minimum ext header len */ + session->ext_header->length * size_32 ); + it = parse_ext_header_out ( session->ext_header, it ); + } + + memcpy ( it, data, length ); + + if ( -1 == send_custom_lossy_packet(session->m, session->dest, parsed, parsed_len) ) { + LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno)); + return -1; + } + + /* Set sequ number */ + session->sequnum = session->sequnum >= MAX_SEQU_NUM ? 0 : session->sequnum + 1; + return 0; +} +void rtp_free_msg ( RTPSession *session, RTPMessage *msg ) +{ + if ( !session ) { + if ( msg->ext_header ) { + free ( msg->ext_header->table ); + free ( msg->ext_header ); + } + } else { + if ( msg->ext_header && session->ext_header != msg->ext_header ) { + free ( msg->ext_header->table ); + free ( msg->ext_header ); + } + } + + free ( msg->header ); + free ( msg ); } -/** - * Extracts header from payload. - */ -RTPHeader *extract_header ( const uint8_t *payload, int length ) + + +RTPHeader *parse_header_in ( const uint8_t *payload, int length ) { if ( !payload || !length ) { LOGGER_WARNING("No payload to extract!"); @@ -111,8 +323,6 @@ RTPHeader *extract_header ( const uint8_t *payload, int length ) return NULL; } - memset(retu->csrc, 0, 16 * sizeof (uint32_t)); - retu->marker_payloadt = *it; ++it; retu->length = total; @@ -125,7 +335,6 @@ RTPHeader *extract_header ( const uint8_t *payload, int length ) retu->ssrc = ntohl(retu->ssrc); uint8_t x; - for ( x = 0; x < cc; x++ ) { it += 4; memcpy(&retu->csrc[x], it, sizeof(retu->csrc[x])); @@ -134,11 +343,7 @@ RTPHeader *extract_header ( const uint8_t *payload, int length ) return retu; } - -/** - * Extracts external header from payload. Must be called AFTER extract_header()! - */ -RTPExtHeader *extract_ext_header ( const uint8_t *payload, uint16_t length ) +RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length ) { const uint8_t *it = payload; @@ -182,11 +387,47 @@ RTPExtHeader *extract_ext_header ( const uint8_t *payload, uint16_t length ) return retu; } +RTPMessage *msg_parse ( const uint8_t *data, int length ) +{ + RTPMessage *retu = calloc(1, sizeof (RTPMessage)); -/** - * Adds header to payload. Make sure _payload_ has enough space. - */ -uint8_t *add_header ( RTPHeader *header, uint8_t *payload ) + retu->header = parse_header_in ( data, length ); /* It allocates memory and all */ + + if ( !retu->header ) { + LOGGER_WARNING("Header failed to extract!"); + free(retu); + return NULL; + } + + uint16_t from_pos = retu->header->length; + retu->length = length - from_pos; + + if ( GET_FLAG_EXTENSION ( retu->header ) ) { + retu->ext_header = parse_ext_header_in ( data + from_pos, length ); + + if ( retu->ext_header ) { + retu->length -= ( 4 /* Minimum ext header len */ + retu->ext_header->length * size_32 ); + from_pos += ( 4 /* Minimum ext header len */ + retu->ext_header->length * size_32 ); + } else { /* Error */ + LOGGER_WARNING("Ext Header failed to extract!"); + rtp_free_msg(NULL, retu); + return NULL; + } + } else { + retu->ext_header = NULL; + } + + if ( length - from_pos <= MAX_RTP_SIZE ) + memcpy ( retu->data, data + from_pos, length - from_pos ); + else { + LOGGER_WARNING("Invalid length!"); + rtp_free_msg(NULL, retu); + return NULL; + } + + return retu; +} +uint8_t *parse_header_out ( const RTPHeader *header, uint8_t *payload ) { uint8_t cc = GET_FLAG_CSRCC ( header ); uint8_t *it = payload; @@ -223,11 +464,7 @@ uint8_t *add_header ( RTPHeader *header, uint8_t *payload ) return it + 4; } - -/** - * Adds extension header to payload. Make sure _payload_ has enough space. - */ -uint8_t *add_ext_header ( RTPExtHeader *header, uint8_t *payload ) +uint8_t *parse_ext_header_out ( const RTPExtHeader *header, uint8_t *payload ) { uint8_t *it = payload; uint16_t length; @@ -242,9 +479,7 @@ uint8_t *add_ext_header ( RTPExtHeader *header, uint8_t *payload ) it -= 2; /* Return to 0 position */ if ( header->table ) { - uint16_t x; - for ( x = 0; x < header->length; x++ ) { it += 4; entry = htonl(header->table[x]); @@ -254,92 +489,45 @@ uint8_t *add_ext_header ( RTPExtHeader *header, uint8_t *payload ) return it + 4; } - -/** - * Builds header from control session values. - */ -RTPHeader *build_header ( RTPSession *session ) +void build_header ( RTPSession *session, RTPHeader *header ) { - RTPHeader *retu = calloc ( 1, sizeof (RTPHeader) ); + ADD_FLAG_VERSION ( header, session->version ); + ADD_FLAG_PADDING ( header, session->padding ); + ADD_FLAG_EXTENSION ( header, session->extension ); + ADD_FLAG_CSRCC ( header, session->cc ); + ADD_SETTING_MARKER ( header, session->marker ); + ADD_SETTING_PAYLOAD ( header, session->payload_type ); - if ( !retu ) { - LOGGER_WARNING("Alloc failed! Program might misbehave!"); - return NULL; - } - - ADD_FLAG_VERSION ( retu, session->version ); - ADD_FLAG_PADDING ( retu, session->padding ); - ADD_FLAG_EXTENSION ( retu, session->extension ); - ADD_FLAG_CSRCC ( retu, session->cc ); - ADD_SETTING_MARKER ( retu, session->marker ); - ADD_SETTING_PAYLOAD ( retu, session->payload_type ); - - retu->sequnum = session->sequnum; - retu->timestamp = current_time_monotonic(); /* milliseconds */ - retu->ssrc = session->ssrc; + header->sequnum = session->sequnum; + header->timestamp = current_time_monotonic(); /* milliseconds */ + header->ssrc = session->ssrc; int i; - for ( i = 0; i < session->cc; i++ ) - retu->csrc[i] = session->csrc[i]; + header->csrc[i] = session->csrc[i]; - retu->length = 12 /* Minimum header len */ + ( session->cc * size_32 ); - - return retu; + header->length = 12 /* Minimum header len */ + ( session->cc * size_32 ); } - - -/** - * Parses data into RTPMessage struct. Stores headers separately from the payload data - * and so the length variable is set accordingly. - */ -RTPMessage *msg_parse ( const uint8_t *data, int length ) +void send_rtcp_report(RTCPSession* session, Messenger* m, int32_t friendnumber) { - RTPMessage *retu = calloc(1, sizeof (RTPMessage)); - - retu->header = extract_header ( data, length ); /* It allocates memory and all */ - - if ( !retu->header ) { - LOGGER_WARNING("Header failed to extract!"); - free(retu); - return NULL; - } - - uint16_t from_pos = retu->header->length; - retu->length = length - from_pos; - - - - if ( GET_FLAG_EXTENSION ( retu->header ) ) { - retu->ext_header = extract_ext_header ( data + from_pos, length ); - - if ( retu->ext_header ) { - retu->length -= ( 4 /* Minimum ext header len */ + retu->ext_header->length * size_32 ); - from_pos += ( 4 /* Minimum ext header len */ + retu->ext_header->length * size_32 ); - } else { /* Error */ - LOGGER_WARNING("Ext Header failed to extract!"); - rtp_free_msg(NULL, retu); - return NULL; - } - } else { - retu->ext_header = NULL; - } - - if ( length - from_pos <= MAX_RTP_SIZE ) - memcpy ( retu->data, data + from_pos, length - from_pos ); - else { - LOGGER_WARNING("Invalid length!"); - rtp_free_msg(NULL, retu); - return NULL; - } - - return retu; + if (session->last_expected_packets == 0) + return; + + uint8_t parsed[9]; + parsed[0] = session->prefix; + + uint32_t packets_missing = htonl(session->last_missing_packets); + uint32_t expected_packets = htonl(session->last_expected_packets); + + memcpy(parsed + 1, &packets_missing, 4); + memcpy(parsed + 5, &expected_packets, 4); + + if (-1 == send_custom_lossy_packet(m, friendnumber, parsed, sizeof(parsed))) + LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", sizeof(parsed), strerror(errno)); + else + session->last_sent_report_ts = current_time_monotonic(); } - -/** - * Callback for networking core. - */ -int rtp_handle_packet ( Messenger *m, int32_t friendnumber, const uint8_t *data, uint32_t length, void *object ) +int handle_rtp_packet ( Messenger *m, int32_t friendnumber, const uint8_t *data, uint32_t length, void *object ) { RTPSession *session = object; RTPMessage *msg; @@ -357,178 +545,37 @@ int rtp_handle_packet ( Messenger *m, int32_t friendnumber, const uint8_t *data, } /* Check if message came in late */ - if ( check_late_message(session, msg) < 0 ) { /* Not late */ + if ( msg->header->sequnum > session->rsequnum && msg->header->timestamp > session->rtimestamp ) { + /* Not late */ session->rsequnum = msg->header->sequnum; - session->timestamp = msg->header->timestamp; + session->rtimestamp = msg->header->timestamp; } queue_message(session, msg); return 0; } - -/** - * Allocate message and store data there - */ -RTPMessage *rtp_new_message ( RTPSession *session, const uint8_t *data, uint32_t length ) +int handle_rtcp_packet ( Messenger *m, int32_t friendnumber, const uint8_t *data, uint32_t length, void *object ) { - if ( !session ) { - LOGGER_WARNING("No session!"); - return NULL; - } - - uint8_t *from_pos; - RTPMessage *retu = calloc(1, sizeof (RTPMessage)); - - if ( !retu ) { - LOGGER_WARNING("Alloc failed! Program might misbehave!"); - return NULL; - } - - /* Sets header values and copies the extension header in retu */ - retu->header = build_header ( session ); /* It allocates memory and all */ - retu->ext_header = session->ext_header; - - - uint32_t total_length = length + retu->header->length + 1; - - retu->data[0] = session->prefix; - - if ( retu->ext_header ) { - total_length += ( 4 /* Minimum ext header len */ + retu->ext_header->length * size_32 ); - - from_pos = add_header ( retu->header, retu->data + 1 ); - from_pos = add_ext_header ( retu->ext_header, from_pos + 1 ); - } else { - from_pos = add_header ( retu->header, retu->data + 1 ); - } - - /* - * Parses the extension header into the message - * Of course if any - */ - - /* Appends data on to retu->data */ - memcpy ( from_pos, data, length ); - - retu->length = total_length; - - return retu; -} - - - -RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ) -{ - RTPSession *retu = calloc(1, sizeof(RTPSession)); - - if ( !retu ) { - LOGGER_WARNING("Alloc failed! Program might misbehave!"); - return NULL; - } - - retu->version = RTP_VERSION; /* It's always 2 */ - retu->padding = 0; /* If some additional data is needed about the packet */ - retu->extension = 0; /* If extension to header is needed */ - retu->cc = 1; /* Amount of contributors */ - retu->csrc = NULL; /* Container */ - retu->ssrc = random_int(); - retu->marker = 0; - retu->payload_type = payload_type % 128; - - retu->dest = friend_num; - - retu->rsequnum = retu->sequnum = 0; - - retu->ext_header = NULL; /* When needed allocate */ - - - if ( !(retu->csrc = calloc(1, sizeof (uint32_t))) ) { - LOGGER_WARNING("Alloc failed! Program might misbehave!"); - free(retu); - return NULL; - } - - retu->csrc[0] = retu->ssrc; /* Set my ssrc to the list receive */ - - /* Also set payload type as prefix */ - retu->prefix = payload_type; - - retu->m = messenger; - /* - * - */ - return retu; -} - -void rtp_kill ( RTPSession *session ) -{ - if ( !session ) return; - - rtp_stop_receiving (session); - - free ( session->ext_header ); - free ( session->csrc ); - - LOGGER_DEBUG("Terminated RTP session: %p", session); - - /* And finally free session */ - free ( session ); -} - -int rtp_start_receiving(RTPSession* session) -{ - if (session == NULL) - return 0; - - LOGGER_DEBUG("Registering packet handler: pt: %d; friend: %d", session->prefix, session->dest); - return custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, - rtp_handle_packet, session); -} - -int rtp_stop_receiving(RTPSession* session) -{ - if (session == NULL) - return 0; - - LOGGER_DEBUG("Unregistering packet handler: pt: %d; friend: %d", session->prefix, session->dest); - return custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, - NULL, NULL); -} - -int rtp_send_msg ( RTPSession *session, const uint8_t *data, uint16_t length ) -{ - RTPMessage *msg = rtp_new_message (session, data, length); - - if ( !msg ) return -1; - - if ( -1 == send_custom_lossy_packet(session->m, session->dest, msg->data, msg->length) ) { - LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno)); - rtp_free_msg ( session, msg ); + if (length < 9) return -1; + + RTCPSession* session = object; + RTCPReport* report = malloc(sizeof(RTCPReport)); + + memcpy(&report->packets_missing, data + 1, 4); + memcpy(&report->expected_packets, data + 5, 4); + + report->packets_missing = ntohl(report->packets_missing); + report->expected_packets = ntohl(report->expected_packets); + + /* This would cause undefined behaviour */ + if (report->expected_packets == 0) { + free(report); + return 0; } + report->timestamp = current_time_monotonic(); - /* Set sequ number */ - session->sequnum = session->sequnum >= MAX_SEQU_NUM ? 0 : session->sequnum + 1; - rtp_free_msg ( session, msg ); - + free(rb_write(session->pl_stats, report)); return 0; -} - -void rtp_free_msg ( RTPSession *session, RTPMessage *msg ) -{ - if ( !session ) { - if ( msg->ext_header ) { - free ( msg->ext_header->table ); - free ( msg->ext_header ); - } - } else { - if ( msg->ext_header && session->ext_header != msg->ext_header ) { - free ( msg->ext_header->table ); - free ( msg->ext_header ); - } - } - - free ( msg->header ); - free ( msg ); -} +} \ No newline at end of file diff --git a/toxav/rtp.h b/toxav/rtp.h index 6b796d5a9..fa5af9fe1 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -23,8 +23,6 @@ #define RTP_H #define RTP_VERSION 2 -#include -// #include #include "../toxcore/Messenger.h" @@ -51,8 +49,8 @@ typedef enum { rtp_TypeVideo } RTPPayloadType; -/** - * Standard rtp header +/** + * Standard rtp header. */ typedef struct { uint8_t flags; /* Version(2),Padding(1), Ext(1), Cc(4) */ @@ -62,17 +60,14 @@ typedef struct { uint32_t ssrc; /* SSRC */ uint32_t csrc[16]; /* CSRC's table */ uint32_t length; /* Length of the header in payload string. */ - } RTPHeader; - -/** +/** * Standard rtp extension header. */ typedef struct { uint16_t type; /* Extension profile */ uint16_t length; /* Number of extensions */ uint32_t *table; /* Extension's table */ - } RTPExtHeader; /** @@ -90,31 +85,32 @@ typedef struct { * RTP control session. */ typedef struct { - uint8_t version; - uint8_t padding; - uint8_t extension; - uint8_t cc; - uint8_t marker; - uint8_t payload_type; - uint16_t sequnum; /* Set when sending */ - uint16_t rsequnum; /* Check when recving msg */ - uint32_t timestamp; - uint32_t ssrc; - uint32_t *csrc; + uint8_t version; + uint8_t padding; + uint8_t extension; + uint8_t cc; + uint8_t marker; + uint8_t payload_type; + uint16_t sequnum; /* Sending sequence number */ + uint16_t rsequnum; /* Receiving sequence number */ + uint32_t rtimestamp; + uint32_t ssrc; + uint32_t *csrc; /* If some additional data must be sent via message * apply it here. Only by allocating this member you will be * automatically placing it within a message. */ - RTPExtHeader *ext_header; + RTPExtHeader *ext_header; /* Msg prefix for core to know when recving */ - uint8_t prefix; + uint8_t prefix; - int dest; + int dest; - struct CSession_s *cs; - Messenger *m; + struct RTCPSession_s *rtcp; + struct CSession_s *cs; + Messenger *m; } RTPSession; @@ -128,6 +124,11 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ); */ void rtp_kill ( RTPSession* session ); +/** + * Do periodical rtp work. + */ +void rtp_do(RTPSession *session); + /** * By default rtp is not in receiving state */ diff --git a/toxav/toxav.c b/toxav/toxav.c index bd788d7d7..721b9d91f 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -220,6 +220,9 @@ void toxav_iterate(ToxAV* av) LOGGED_UNLOCK(av->mutex); cs_do(i->cs); + rtp_do(i->rtps[0]); + rtp_do(i->rtps[1]); + if (i->last_self_capabilities & msi_CapRAudio) /* Receiving audio */ rc = MIN(i->cs->last_packet_frame_duration, rc); if (i->last_self_capabilities & msi_CapRVideo) /* Receiving video */ diff --git a/toxcore/util.c b/toxcore/util.c index 5a72c4a48..d6db946da 100644 --- a/toxcore/util.c +++ b/toxcore/util.c @@ -185,3 +185,76 @@ int create_recursive_mutex(pthread_mutex_t *mutex) return 0; } + + +struct RingBuffer { + uint16_t size; /* Max size */ + uint16_t start; + uint16_t end; + void **data; +}; + +bool rb_full(const RingBuffer *b) +{ + return (b->end + 1) % b->size == b->start; +} +bool rb_empty(const RingBuffer *b) +{ + return b->end == b->start; +} +void* rb_write(RingBuffer *b, void *p) +{ + void* rc = NULL; + if ((b->end + 1) % b->size == b->start) /* full */ + rc = b->data[b->start]; + + b->data[b->end] = p; + b->end = (b->end + 1) % b->size; + + if (b->end == b->start) + b->start = (b->start + 1) % b->size; + + return rc; +} +bool rb_read(RingBuffer *b, void **p) +{ + if (b->end == b->start) { /* Empty */ + *p = NULL; + return false; + } + + *p = b->data[b->start]; + b->start = (b->start + 1) % b->size; + return true; +} +void rb_clear(RingBuffer *b) +{ + while (!rb_empty(b)) { + void *p; + rb_read(b, &p); + free(p); + } +} +RingBuffer *rb_new(int size) +{ + RingBuffer *buf = calloc(sizeof(RingBuffer), 1); + + if (!buf) return NULL; + + buf->size = size + 1; /* include empty elem */ + + if (!(buf->data = calloc(buf->size, sizeof(void *)))) { + free(buf); + return NULL; + } + + return buf; +} +void rb_free(RingBuffer *b) +{ + if (b) { + rb_clear(b); + free(b->data); + free(b); + } +} \ No newline at end of file diff --git a/toxcore/util.h b/toxcore/util.h index fab26e294..6c3d3b389 100644 --- a/toxcore/util.h +++ b/toxcore/util.h @@ -30,6 +30,7 @@ #include #define MIN(a,b) (((a)<(b))?(a):(b)) +#define PAIR(TYPE1__, TYPE2__) struct { TYPE1__ first; TYPE2__ second; } void unix_time_update(); uint64_t unix_time(); @@ -56,4 +57,13 @@ int load_state(load_state_callback_func load_state_callback, void *outer, /* Returns -1 if failed or 0 if success */ int create_recursive_mutex(pthread_mutex_t *mutex); +/* Ring buffer */ +typedef struct RingBuffer RingBuffer; +bool rb_full(const RingBuffer *b); +bool rb_empty(const RingBuffer *b); +void* rb_write(RingBuffer* b, void* p); +bool rb_read(RingBuffer* b, void** p); +void rb_clear(RingBuffer *b); +RingBuffer *rb_new(int size); +void rb_free(RingBuffer *b); #endif /* __UTIL_H__ */ From da6c17222f54c933c826e9c557e66629ee57790f Mon Sep 17 00:00:00 2001 From: mannol Date: Thu, 16 Apr 2015 02:00:34 +0200 Subject: [PATCH 27/97] The pretty basic adaptive bitrate is *working* --- toxav/av_test.c | 59 ++++++----------- toxav/codec.c | 5 +- toxav/rtp.c | 101 ++++++++++++++++++----------- toxav/rtp.h | 16 +++-- toxav/toxav.c | 167 ++++++++++++++++++++++++++++++++++-------------- toxav/toxav.h | 26 +++++--- 6 files changed, 233 insertions(+), 141 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index c20d459b7..6ebe0421e 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -27,6 +27,9 @@ #include "../toxcore/tox.h" #include "../toxcore/util.h" +#define LOGGING +#include "../toxcore/logger.h" + /* Playing audio data */ #include /* Reading audio */ @@ -80,6 +83,7 @@ typedef struct { bool incoming; uint32_t state; + uint32_t abitrate; } CallControl; struct toxav_thread_data { @@ -91,22 +95,6 @@ struct toxav_thread_data { const char* vdout = "AV Test"; /* Video output */ PaStream* adout = NULL; /* Audio output */ -const char* stringify_state(TOXAV_CALL_STATE s) -{ - static const char* strings[] = - { - "NOT SENDING", - "SENDING AUDIO", - "SENDING VIDEO", - "SENDING AUDIO AND VIDEO", - "PAUSED", - "END", - "ERROR" - }; - - return strings [s]; -} - /** * Callbacks */ @@ -117,9 +105,20 @@ void t_toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool } void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, void *user_data) { - printf("Handling CALL STATE callback: %d\n", state); - ((CallControl*)user_data)->state = state; + + if (state & TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE) { + uint32_t bitrate = ((CallControl*)user_data)->abitrate; + + if (bitrate < 64) { + printf("Changing bitrate to: %d\n", 64); + toxav_set_audio_bit_rate(av, friend_number, 64, 0); + } + } else if (state & TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE) { + + } else { + printf("Handling CALL STATE callback: %d\n", state); + } } void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, @@ -331,24 +330,6 @@ int print_help (const char* name) int main (int argc, char** argv) { - RingBuffer* rb = rb_new(4); - int a[5] = {0, 1, 2, 3, 4}; - int* x; - rb_write(rb, a + 0); - rb_write(rb, a + 1); - rb_write(rb, a + 2); - rb_write(rb, a + 3); -// rb_write(rb, a + 4); - - x = rb_write(rb, a + 4); - while (rb_read(rb, (void**) &x)) -// rb_read(rb, (void**)&x); - printf("%d ", *x); - - printf("\n"); -// int r = 43; -// printf("%d\n", r >= 40 ? 3 : r / 10); - return 0; Pa_Initialize(); struct stat st; @@ -688,9 +669,11 @@ int main (int argc, char** argv) memset(&AliceCC, 0, sizeof(CallControl)); memset(&BobCC, 0, sizeof(CallControl)); + AliceCC.abitrate = BobCC.abitrate = 8; + { /* Call */ TOXAV_ERR_CALL rc; - toxav_call(AliceAV, 0, 48, 0, &rc); + toxav_call(AliceAV, 0, AliceCC.abitrate, 0, &rc); if (rc != TOXAV_ERR_CALL_OK) { printf("toxav_call failed: %d\n", rc); @@ -703,7 +686,7 @@ int main (int argc, char** argv) { /* Answer */ TOXAV_ERR_ANSWER rc; - toxav_answer(BobAV, 0, 48, 0, &rc); + toxav_answer(BobAV, 0, BobCC.abitrate, 0, &rc); if (rc != TOXAV_ERR_ANSWER_OK) { printf("toxav_answer failed: %d\n", rc); diff --git a/toxav/codec.c b/toxav/codec.c index 671be1ac1..1e8754e91 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -293,7 +293,7 @@ void cs_do(CSession *cs) /* The maximum for 120 ms 48 KHz audio */ int16_t tmp[5760]; - while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { + if ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { LOGGED_UNLOCK(cs->queue_mutex); if (success == 2) { @@ -327,7 +327,7 @@ void cs_do(CSession *cs) if (!reconfigure_audio_decoder(cs, cs->last_packet_sampling_rate, cs->last_packet_channel_count)) { LOGGER_WARNING("Failed to reconfigure decoder!"); rtp_free_msg(NULL, msg); - continue; + goto DONE; } rc = opus_decode(cs->audio_decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0); @@ -347,6 +347,7 @@ void cs_do(CSession *cs) LOGGED_LOCK(cs->queue_mutex); } + DONE:; } /********************* VIDEO *********************/ diff --git a/toxav/rtp.c b/toxav/rtp.c index 77fce0569..665ebf860 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -50,15 +50,17 @@ typedef struct { uint64_t timestamp; /* in ms */ - uint32_t packets_missing; + uint32_t received_packets; uint32_t expected_packets; /* ... other stuff in the future */ } RTCPReport; typedef struct RTCPSession_s { + RTPSession *rtp_session; + uint8_t prefix; uint64_t last_sent_report_ts; - uint32_t last_missing_packets; + uint32_t last_received_packets; uint32_t last_expected_packets; RingBuffer* pl_stats; /* Packet loss stats over time */ @@ -66,6 +68,7 @@ typedef struct RTCPSession_s { + /* queue_message() is defined in codec.c */ void queue_message(RTPSession *session, RTPMessage *msg); RTPHeader *parse_header_in ( const uint8_t *payload, int length ); @@ -91,18 +94,12 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ) } retu->version = RTP_VERSION; /* It's always 2 */ - retu->padding = 0; /* If some additional data is needed about the packet */ - retu->extension = 0; /* If extension to header is needed */ - retu->cc = 1; /* Amount of contributors */ - retu->csrc = NULL; /* Container */ retu->ssrc = random_int(); - retu->marker = 0; retu->payload_type = payload_type % 128; - + + retu->tstate = rtp_StateNormal; retu->m = messenger; retu->dest = friend_num; - retu->rsequnum = retu->sequnum = 0; - retu->ext_header = NULL; /* When needed allocate */ if ( !(retu->csrc = calloc(1, sizeof(uint32_t))) ) { LOGGER_WARNING("Alloc failed! Program might misbehave!"); @@ -117,15 +114,16 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ) /* Initialize rtcp session */ - if (!(retu->rtcp = calloc(1, sizeof(RTCPSession)))) { + if (!(retu->rtcp_session = calloc(1, sizeof(RTCPSession)))) { LOGGER_WARNING("Alloc failed! Program might misbehave!"); free(retu->csrc); free(retu); return NULL; } - retu->rtcp->prefix = 222 + payload_type % 192; - retu->rtcp->pl_stats = rb_new(4); + retu->rtcp_session->prefix = 222 + payload_type % 192; + retu->rtcp_session->pl_stats = rb_new(4); + retu->rtcp_session->rtp_session = retu; return retu; } @@ -139,11 +137,11 @@ void rtp_kill ( RTPSession *session ) free ( session->csrc ); void* t; - while (!rb_empty(session->rtcp->pl_stats)) { - rb_read(session->rtcp->pl_stats, (void**) &t); + while (!rb_empty(session->rtcp_session->pl_stats)) { + rb_read(session->rtcp_session->pl_stats, (void**) &t); free(t); } - rb_free(session->rtcp->pl_stats); + rb_free(session->rtcp_session->pl_stats); LOGGER_DEBUG("Terminated RTP session: %p", session); @@ -152,41 +150,49 @@ void rtp_kill ( RTPSession *session ) } void rtp_do(RTPSession *session) { - if (!session || !session->rtcp) + if (!session || !session->rtcp_session) return; - if (current_time_monotonic() - session->rtcp->last_sent_report_ts >= RTCP_REPORT_INTERVAL_MS) { - send_rtcp_report(session->rtcp, session->m, session->dest); + if (current_time_monotonic() - session->rtcp_session->last_sent_report_ts >= RTCP_REPORT_INTERVAL_MS) { + send_rtcp_report(session->rtcp_session, session->m, session->dest); } - if (rb_full(session->rtcp->pl_stats)) { + if (rb_full(session->rtcp_session->pl_stats)) { RTCPReport* reports[4]; int i = 0; - for (; rb_read(session->rtcp->pl_stats, (void**) reports + i); i++); + for (; rb_read(session->rtcp_session->pl_stats, (void**) reports + i); i++); /* Check for timed out reports (> 6 sec) */ uint64_t now = current_time_monotonic(); for (i = 0; i < 4 && now - reports[i]->timestamp < 6000; i ++); for (; i < 4; i ++) { - rb_write(session->rtcp->pl_stats, reports[i]); + rb_write(session->rtcp_session->pl_stats, reports[i]); reports[i] = NULL; } - if (!rb_empty(session->rtcp->pl_stats)) { + if (!rb_empty(session->rtcp_session->pl_stats)) { for (i = 0; reports[i] != NULL; i ++) free(reports[i]); return; /* As some reports are timed out, we need more... */ } /* We have 4 on-time reports so we can proceed */ - uint32_t quality_loss = 0; + uint32_t quality = 100; for (i = 0; i < 4; i++) { - uint32_t idx = reports[i]->packets_missing * 100 / reports[i]->expected_packets; - quality_loss += idx; + uint32_t idx = reports[i]->received_packets * 100 / reports[i]->expected_packets; + quality = MIN(quality, idx); + free(reports[i]); } - if (quality_loss > 40) { - LOGGER_DEBUG("Packet loss detected"); + if (quality <= 70) { + session->tstate = rtp_StateBad; + LOGGER_WARNING("Stream quality: BAD"); + } else if (quality >= 99) { + session->tstate = rtp_StateGood; + LOGGER_DEBUG("Stream quality: GOOD"); + } else { + session->tstate = rtp_StateNormal; + LOGGER_DEBUG("Stream quality: NORMAL"); } } } @@ -200,8 +206,8 @@ int rtp_start_receiving(RTPSession* session) LOGGER_WARNING("Failed to register rtp receive handler"); return -1; } - if (custom_lossy_packet_registerhandler(session->m, session->dest, session->rtcp->prefix, - handle_rtcp_packet, session->rtcp) == -1) { + if (custom_lossy_packet_registerhandler(session->m, session->dest, session->rtcp_session->prefix, + handle_rtcp_packet, session->rtcp_session) == -1) { LOGGER_WARNING("Failed to register rtcp receive handler"); custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, NULL, NULL); return -1; @@ -215,7 +221,7 @@ int rtp_stop_receiving(RTPSession* session) return -1; custom_lossy_packet_registerhandler(session->m, session->dest, session->prefix, NULL, NULL); - custom_lossy_packet_registerhandler(session->m, session->dest, session->rtcp->prefix, NULL, NULL); /* RTCP */ + custom_lossy_packet_registerhandler(session->m, session->dest, session->rtcp_session->prefix, NULL, NULL); /* RTCP */ return 0; } @@ -516,16 +522,21 @@ void send_rtcp_report(RTCPSession* session, Messenger* m, int32_t friendnumber) uint8_t parsed[9]; parsed[0] = session->prefix; - uint32_t packets_missing = htonl(session->last_missing_packets); + uint32_t received_packets = htonl(session->last_received_packets); uint32_t expected_packets = htonl(session->last_expected_packets); - memcpy(parsed + 1, &packets_missing, 4); + memcpy(parsed + 1, &received_packets, 4); memcpy(parsed + 5, &expected_packets, 4); if (-1 == send_custom_lossy_packet(m, friendnumber, parsed, sizeof(parsed))) LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", sizeof(parsed), strerror(errno)); - else + else { + LOGGER_DEBUG("Sent rtcp report: ex: %d rc: %d", session->last_expected_packets, session->last_received_packets); + + session->last_received_packets = 0; + session->last_expected_packets = 0; session->last_sent_report_ts = current_time_monotonic(); + } } int handle_rtp_packet ( Messenger *m, int32_t friendnumber, const uint8_t *data, uint32_t length, void *object ) { @@ -545,12 +556,21 @@ int handle_rtp_packet ( Messenger *m, int32_t friendnumber, const uint8_t *data, } /* Check if message came in late */ - if ( msg->header->sequnum > session->rsequnum && msg->header->timestamp > session->rtimestamp ) { + if ( msg->header->sequnum > session->rsequnum || msg->header->timestamp > session->rtimestamp ) { /* Not late */ + if (msg->header->sequnum > session->rsequnum) + session->rtcp_session->last_expected_packets += msg->header->sequnum - session->rsequnum; + else if (msg->header->sequnum < session->rsequnum) + session->rtcp_session->last_expected_packets += (msg->header->sequnum + 65535) - session->rsequnum; + else /* Usual case when transmission starts */ + session->rtcp_session->last_expected_packets ++; + session->rsequnum = msg->header->sequnum; session->rtimestamp = msg->header->timestamp; } + session->rtcp_session->last_received_packets ++; + queue_message(session, msg); return 0; } @@ -562,14 +582,15 @@ int handle_rtcp_packet ( Messenger *m, int32_t friendnumber, const uint8_t *data RTCPSession* session = object; RTCPReport* report = malloc(sizeof(RTCPReport)); - memcpy(&report->packets_missing, data + 1, 4); + memcpy(&report->received_packets, data + 1, 4); memcpy(&report->expected_packets, data + 5, 4); - report->packets_missing = ntohl(report->packets_missing); + report->received_packets = ntohl(report->received_packets); report->expected_packets = ntohl(report->expected_packets); - /* This would cause undefined behaviour */ - if (report->expected_packets == 0) { + /* Invalid values */ + if (report->expected_packets == 0 || report->received_packets > report->expected_packets) { + LOGGER_WARNING("Malformed rtcp report! %d %d", report->expected_packets, report->received_packets); free(report); return 0; } @@ -577,5 +598,7 @@ int handle_rtcp_packet ( Messenger *m, int32_t friendnumber, const uint8_t *data report->timestamp = current_time_monotonic(); free(rb_write(session->pl_stats, report)); + + LOGGER_DEBUG("Got rtcp report: ex: %d rc: %d", report->expected_packets, report->received_packets); return 0; } \ No newline at end of file diff --git a/toxav/rtp.h b/toxav/rtp.h index fa5af9fe1..a28ae7bc4 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -44,10 +44,16 @@ /** * Payload type identifier. Also used as rtp callback prefix. */ -typedef enum { +enum { rtp_TypeAudio = 192, rtp_TypeVideo -} RTPPayloadType; +}; + +typedef enum { + rtp_StateBad = -1, + rtp_StateNormal, + rtp_StateGood, +} RTPTransmissionState; /** * Standard rtp header. @@ -108,10 +114,11 @@ typedef struct { int dest; - struct RTCPSession_s *rtcp; + struct RTCPSession_s *rtcp_session; struct CSession_s *cs; Messenger *m; - + + RTPTransmissionState tstate; } RTPSession; /** @@ -150,5 +157,4 @@ int rtp_send_msg ( RTPSession* session, const uint8_t* data, uint16_t length ); void rtp_free_msg ( RTPSession *session, RTPMessage *msg ); - #endif /* RTP_H */ diff --git a/toxav/toxav.c b/toxav/toxav.c index 721b9d91f..0e94e5ddf 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -60,6 +60,12 @@ typedef struct ToxAVCall_s { uint8_t last_self_capabilities; uint8_t last_peer_capabilities; + /** Quality control */ + uint64_t time_audio_good; + uint32_t last_bad_audio_bit_rate; + uint64_t time_video_good; + uint32_t last_bad_video_bit_rate; + struct ToxAVCall_s *prev; struct ToxAVCall_s *next; } ToxAVCall; @@ -96,11 +102,13 @@ int callback_end(void* toxav_inst, MSICall* call); int callback_error(void* toxav_inst, MSICall* call); int callback_capabilites(void* toxav_inst, MSICall* call); -ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); -ToxAVCall* call_get(ToxAV* av, uint32_t friend_number); -void call_remove(ToxAVCall* call); bool audio_bitrate_invalid(uint32_t bitrate); bool video_bitrate_invalid(uint32_t bitrate); +void invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state); +ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); +ToxAVCall* call_get(ToxAV* av, uint32_t friend_number); +void qc_do(ToxAVCall* call); +void call_remove(ToxAVCall* call); bool call_prepare_transmission(ToxAVCall* call); void call_kill_transmission(ToxAVCall* call); @@ -220,8 +228,9 @@ void toxav_iterate(ToxAV* av) LOGGED_UNLOCK(av->mutex); cs_do(i->cs); - rtp_do(i->rtps[0]); - rtp_do(i->rtps[1]); + rtp_do(i->rtps[audio_index]); + rtp_do(i->rtps[video_index]); + qc_do(i); if (i->last_self_capabilities & msi_CapRAudio) /* Receiving audio */ rc = MIN(i->cs->last_packet_frame_duration, rc); @@ -516,8 +525,10 @@ bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_ goto END; } - /* NOTE: no need to lock*/ + /* Decoding mutex is locked because of quality control */ + LOGGED_LOCK(call->mutex_decoding); call->audio_bit_rate = audio_bit_rate; + LOGGED_UNLOCK(call->mutex_decoding); LOGGED_UNLOCK(av->mutex); END: @@ -550,8 +561,10 @@ bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_ goto END; } - /* NOTE: no need to lock*/ + /* Decoding mutex is locked because of quality control */ + LOGGED_LOCK(call->mutex_decoding); call->video_bit_rate = video_bit_rate; + LOGGED_UNLOCK(call->mutex_decoding); LOGGED_UNLOCK(av->mutex); END: @@ -813,8 +826,7 @@ int callback_start(void* toxav_inst, MSICall* call) return -1; } - if (toxav->scb.first) - toxav->scb.first(toxav, call->friend_id, call->peer_capabilities, toxav->scb.second); + invoke_call_state(toxav, call->friend_id, call->peer_capabilities); LOGGED_UNLOCK(toxav->mutex); return 0; @@ -825,8 +837,7 @@ int callback_end(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; LOGGED_LOCK(toxav->mutex); - if (toxav->scb.first) - toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_END, toxav->scb.second); + invoke_call_state(toxav, call->friend_id, TOXAV_CALL_STATE_END); call_kill_transmission(call->av_call); call_remove(call->av_call); @@ -840,8 +851,7 @@ int callback_error(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; LOGGED_LOCK(toxav->mutex); - if (toxav->scb.first) - toxav->scb.first(toxav, call->friend_id, TOXAV_CALL_STATE_ERROR, toxav->scb.second); + invoke_call_state(toxav, call->friend_id, TOXAV_CALL_STATE_ERROR); call_kill_transmission(call->av_call); call_remove(call->av_call); @@ -857,8 +867,7 @@ int callback_capabilites(void* toxav_inst, MSICall* call) /* TODO modify cs? */ - if (toxav->scb.first) - toxav->scb.first(toxav, call->friend_id, call->peer_capabilities, toxav->scb.second); + invoke_call_state(toxav, call->friend_id, call->peer_capabilities); LOGGED_UNLOCK(toxav->mutex); return 0; @@ -878,6 +887,12 @@ bool video_bitrate_invalid(uint32_t bitrate) return false; } +void invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state) +{ + if (av->scb.first) + av->scb.first(av, friend_number, state, av->scb.second); +} + ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) { /* Assumes mutex locked */ @@ -968,6 +983,94 @@ ToxAVCall* call_get(ToxAV* av, uint32_t friend_number) return av->calls[friend_number]; } +void qc_do(ToxAVCall* call) +{ + /* Please NOTE: The quality control is rather basic, + * advanced algorithms will be applied in the future + */ + switch(call->rtps[audio_index]->tstate) { + case rtp_StateBad: + LOGGER_DEBUG("Suggesting lower bitrate for audio..."); + call->time_audio_good = 0; + call->last_bad_audio_bit_rate = call->audio_bit_rate; + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_LOWER_AUDIO_BITRATE); + break; + case rtp_StateGood: + if (call->time_audio_good == 0) + call->time_audio_good = current_time_monotonic(); + else if (current_time_monotonic() - call->time_audio_good >= 30000 && + 64 > call->audio_bit_rate) + if (call->last_bad_audio_bit_rate > call->audio_bit_rate) { + if (current_time_monotonic() - call->time_audio_good >= 45000) + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE); + call->last_bad_audio_bit_rate = 0; + } else + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE); + break; + case rtp_StateNormal: + call->time_audio_good = 0; + break; + } + + switch(call->rtps[video_index]->tstate) { + case rtp_StateBad: + LOGGER_DEBUG("Suggesting lower bitrate for video..."); + call->time_video_good = 0; + call->last_bad_video_bit_rate = call->video_bit_rate; + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_LOWER_VIDEO_BITRATE); + break; + case rtp_StateGood: + if (call->time_video_good == 0) + call->time_video_good = current_time_monotonic(); + else if (current_time_monotonic() - call->time_video_good >= 30000) + if (call->last_bad_video_bit_rate > call->video_bit_rate) { + if (current_time_monotonic() - call->time_video_good >= 45000) + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE); + call->last_bad_video_bit_rate = 0; + } else + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE); + break; + case rtp_StateNormal: + call->time_video_good = 0; + break; + } + +} + +void call_remove(ToxAVCall* call) +{ + if (call == NULL) + return; + + uint32_t friend_id = call->friend_id; + ToxAV* av = call->av; + + ToxAVCall* prev = call->prev; + ToxAVCall* next = call->next; + + free(call); + + if (prev) + prev->next = next; + else if (next) + av->calls_head = next->friend_id; + else goto CLEAR; + + if (next) + next->prev = prev; + else if (prev) + av->calls_tail = prev->friend_id; + else goto CLEAR; + + av->calls[friend_id] = NULL; + return; + +CLEAR: + av->calls_head = av->calls_tail = 0; + free(av->calls); + av->calls = NULL; +} + bool call_prepare_transmission(ToxAVCall* call) { /* Assumes mutex locked */ @@ -1089,37 +1192,3 @@ void call_kill_transmission(ToxAVCall* call) pthread_mutex_destroy(call->mutex_video_sending); pthread_mutex_destroy(call->mutex_decoding); } - -void call_remove(ToxAVCall* call) -{ - if (call == NULL) - return; - - uint32_t friend_id = call->friend_id; - ToxAV* av = call->av; - - ToxAVCall* prev = call->prev; - ToxAVCall* next = call->next; - - free(call); - - if (prev) - prev->next = next; - else if (next) - av->calls_head = next->friend_id; - else goto CLEAR; - - if (next) - next->prev = prev; - else if (prev) - av->calls_tail = prev->friend_id; - else goto CLEAR; - - av->calls[friend_id] = NULL; - return; - -CLEAR: - av->calls_head = av->calls_tail = 0; - free(av->calls); - av->calls = NULL; -} diff --git a/toxav/toxav.h b/toxav/toxav.h index ae95c61b2..098075b38 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -207,21 +207,31 @@ typedef enum TOXAV_CALL_STATE { * The flag that marks that friend is receiving video. */ TOXAV_CALL_STATE_RECEIVING_V = 8, - - /** - * The core will never set TOXAV_CALL_STATE_END or TOXAV_CALL_STATE_ERROR - * together with other states. - */ - /** * The call has finished. This is the final state after which no more state * transitions can occur for the call. */ TOXAV_CALL_STATE_END = 16, /** - * Sent by the AV core if an error occurred on the remote end. + * AV core suggests you to lower bitrate for audio. */ - TOXAV_CALL_STATE_ERROR = 32 + TOXAV_CALL_STATE_LOWER_AUDIO_BITRATE = 32, + /** + * AV core suggests you to lower bitrate for video. + */ + TOXAV_CALL_STATE_LOWER_VIDEO_BITRATE = 64, + /** + * AV core suggests you to increase bitrate for audio. + */ + TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE = 128, + /** + * AV core suggests you to increase bitrate for video. + */ + TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE = 256, + /** + * Set by the AV core if an error occurred on the remote end. + */ + TOXAV_CALL_STATE_ERROR = 32768 } TOXAV_CALL_STATE; /** * The function type for the `call_state` callback. From 969367b72aebaa2ecfc168cf2e86b20501298c20 Mon Sep 17 00:00:00 2001 From: mannol Date: Fri, 17 Apr 2015 15:31:14 +0200 Subject: [PATCH 28/97] Update latest --- toxav/av_test.c | 137 ++++++++++++++++++++++++++++++++++++++---------- toxav/codec.c | 8 +-- toxav/toxav.c | 8 +-- toxav/toxav.h | 4 +- 4 files changed, 120 insertions(+), 37 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 6ebe0421e..507a708e0 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -26,6 +26,7 @@ #include "../toxav/toxav.h" #include "../toxcore/tox.h" #include "../toxcore/util.h" +#include "../toxcore/network.h" /* current_time_monotonic() */ #define LOGGING #include "../toxcore/logger.h" @@ -84,6 +85,9 @@ typedef struct { bool incoming; uint32_t state; uint32_t abitrate; + pthread_mutex_t arb_mutex[1]; + RingBuffer* arb; /* Audio ring buffer */ + } CallControl; struct toxav_thread_data { @@ -95,6 +99,32 @@ struct toxav_thread_data { const char* vdout = "AV Test"; /* Video output */ PaStream* adout = NULL; /* Audio output */ + +typedef struct { + uint16_t size; + int16_t data[]; +} frame; + +void* pa_write_thread (void* d) +{ + /* The purpose of this thread is to make sure Pa_WriteStream will not block + * toxav_iterate thread + */ + CallControl* cc = d; + + while (Pa_IsStreamActive(adout)) { + frame* f; + pthread_mutex_lock(cc->arb_mutex); + if (rb_read(cc->arb, (void**)&f)) { + pthread_mutex_unlock(cc->arb_mutex); + Pa_WriteStream(adout, f->data, f->size); + free(f); + } else + pthread_mutex_unlock(cc->arb_mutex); + } +} + + /** * Callbacks */ @@ -106,15 +136,38 @@ void t_toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, void *user_data) { ((CallControl*)user_data)->state = state; + uint32_t abitrate = ((CallControl*)user_data)->abitrate; if (state & TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE) { - uint32_t bitrate = ((CallControl*)user_data)->abitrate; - if (bitrate < 64) { - printf("Changing bitrate to: %d\n", 64); - toxav_set_audio_bit_rate(av, friend_number, 64, 0); - } - } else if (state & TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE) { +// /* NOTE: I'm using values 8, 16, 24, 48, 64. You can use whatever OPUS supports. */ +// switch (abitrate) { +// case 8: abitrate = 16; break; +// case 16: abitrate = 24; break; +// case 24: abitrate = 48; break; +// case 48: abitrate = 64; break; +// default: return; +// } +// +// printf("Increasing bitrate to: %d\n", abitrate); +// toxav_set_audio_bit_rate(av, friend_number, abitrate, 0); + + } else if (state & TOXAV_CALL_STATE_DECREASE_AUDIO_BITRATE) { +// /* NOTE: I'm using values 8, 16, 24, 48, 64. You can use whatever OPUS supports. */ +// switch (abitrate) { +// case 16: abitrate = 8; break; +// case 24: abitrate = 16; break; +// case 48: abitrate = 24; break; +// case 64: abitrate = 48; break; +// default: return; +// } +// +// printf("Decreasing bitrate to: %d\n", abitrate); +// toxav_set_audio_bit_rate(av, friend_number, abitrate, 0); +// + } + + if (state & TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE) { } else { printf("Handling CALL STATE callback: %d\n", state); @@ -158,7 +211,16 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, uint32_t sampling_rate, void *user_data) { - Pa_WriteStream(adout, pcm, sample_count/channels); + CallControl* cc = user_data; + frame* f = malloc(sizeof(frame) + sample_count * sizeof(int16_t)); + memcpy(f->data, pcm, sample_count); + f->size = sample_count/channels; + + pthread_mutex_lock(cc->arb_mutex); + free(rb_write(cc->arb, f)); + pthread_mutex_unlock(cc->arb_mutex); + +// Pa_WriteStream(adout, pcm, sample_count/channels); } void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, uint16_t length, void *userdata) { @@ -220,6 +282,7 @@ void initialize_tox(Tox** bootstrap, ToxAV** AliceAV, CallControl* AliceCC, ToxA *BobAV = toxav_new(Bob, &rc); assert(rc == TOXAV_ERR_NEW_OK); + /* Alice */ toxav_callback_call(*AliceAV, t_toxav_call_cb, AliceCC); toxav_callback_call_state(*AliceAV, t_toxav_call_state_cb, AliceCC); @@ -255,7 +318,9 @@ void* iterate_toxav (void * data) int rc = MIN(toxav_iteration_interval(data_cast->AliceAV), toxav_iteration_interval(data_cast->BobAV)); // cvWaitKey(10); - c_sleep(10); + printf("\rSleeping for: %d ", rc); + fflush(stdout); + c_sleep(rc); } data_cast->sig = 1; @@ -330,6 +395,7 @@ int print_help (const char* name) int main (int argc, char** argv) { + freopen("/dev/zero", "w", stderr); Pa_Initialize(); struct stat st; @@ -669,7 +735,13 @@ int main (int argc, char** argv) memset(&AliceCC, 0, sizeof(CallControl)); memset(&BobCC, 0, sizeof(CallControl)); - AliceCC.abitrate = BobCC.abitrate = 8; + pthread_mutex_init(AliceCC.arb_mutex, NULL); + pthread_mutex_init(BobCC.arb_mutex, NULL); + + AliceCC.arb = rb_new(16); + BobCC.arb = rb_new(16); + + AliceCC.abitrate = BobCC.abitrate = 16; { /* Call */ TOXAV_ERR_CALL rc; @@ -704,21 +776,6 @@ int main (int argc, char** argv) } - int frame_size = (af_info.samplerate * audio_frame_duration / 1000) * af_info.channels; - - struct PaStreamParameters output; - output.device = audio_out_dev_idx; - output.channelCount = af_info.channels; - output.sampleFormat = paInt16; - output.suggestedLatency = audio_dev->defaultHighOutputLatency; - output.hostApiSpecificStreamInfo = NULL; - - PaError err = Pa_OpenStream(&adout, NULL, &output, af_info.samplerate, frame_size, paNoFlag, NULL, NULL); - assert(err == paNoError); - - err = Pa_StartStream(adout); - assert(err == paNoError); - int16_t PCM[5760]; time_t start_time = time(NULL); @@ -736,8 +793,25 @@ int main (int argc, char** argv) pthread_create(&dect, NULL, iterate_toxav, &data); pthread_detach(dect); + + int frame_size = (af_info.samplerate * audio_frame_duration / 1000) * af_info.channels; + + struct PaStreamParameters output; + output.device = audio_out_dev_idx; + output.channelCount = af_info.channels; + output.sampleFormat = paInt16; + output.suggestedLatency = audio_dev->defaultHighOutputLatency; + output.hostApiSpecificStreamInfo = NULL; + + PaError err = Pa_OpenStream(&adout, NULL, &output, af_info.samplerate, frame_size, paNoFlag, NULL, NULL); + assert(err == paNoError); + + err = Pa_StartStream(adout); + assert(err == paNoError); + printf("Sample rate %d\n", af_info.samplerate); while ( start_time + expected_time > time(NULL) ) { + uint64_t enc_start_time = current_time_monotonic(); int64_t count = sf_read_short(af_handle, PCM, frame_size); if (count > 0) { TOXAV_ERR_SEND_FRAME rc; @@ -746,9 +820,10 @@ int main (int argc, char** argv) } } iterate_tox(bootstrap, AliceAV, BobAV); - c_sleep(53); + c_sleep(abs(audio_frame_duration - (current_time_monotonic() - enc_start_time) - 1)); } - + + Pa_StopStream(adout); printf("Played file in: %lu\n", time(NULL) - start_time); @@ -772,7 +847,15 @@ int main (int argc, char** argv) while(data.sig != 1) pthread_yield(); - Pa_StopStream(adout); + pthread_mutex_destroy(AliceCC.arb_mutex); + pthread_mutex_destroy(BobCC.arb_mutex); + + void* f = NULL; + while(rb_read(AliceCC.arb, &f)) + free(f); + + while(rb_read(BobCC.arb, &f)) + free(f); printf("Success!"); } diff --git a/toxav/codec.c b/toxav/codec.c index 1e8754e91..95b4d4067 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -293,7 +293,7 @@ void cs_do(CSession *cs) /* The maximum for 120 ms 48 KHz audio */ int16_t tmp[5760]; - if ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { + while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { LOGGED_UNLOCK(cs->queue_mutex); if (success == 2) { @@ -327,7 +327,8 @@ void cs_do(CSession *cs) if (!reconfigure_audio_decoder(cs, cs->last_packet_sampling_rate, cs->last_packet_channel_count)) { LOGGER_WARNING("Failed to reconfigure decoder!"); rtp_free_msg(NULL, msg); - goto DONE; + continue; +// goto DONE; } rc = opus_decode(cs->audio_decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0); @@ -337,7 +338,6 @@ void cs_do(CSession *cs) if (rc < 0) { LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); } else if (cs->acb.first) { - cs->last_packet_channel_count = 2; cs->last_packet_frame_duration = (rc * 1000) / cs->last_packet_sampling_rate * cs->last_packet_channel_count; cs->acb.first(cs->av, cs->friend_id, tmp, rc * cs->last_packet_channel_count, @@ -347,7 +347,7 @@ void cs_do(CSession *cs) LOGGED_LOCK(cs->queue_mutex); } - DONE:; +// DONE:; } /********************* VIDEO *********************/ diff --git a/toxav/toxav.c b/toxav/toxav.c index 0e94e5ddf..dbc065871 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -249,8 +249,8 @@ void toxav_iterate(ToxAV* av) } LOGGED_UNLOCK(av->mutex); -// av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa); - av->interval = rc < 5 ? 1: rc - 5; + av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa); +// av->interval = rc < 5 ? 1: rc - 5; av->dmsst += current_time_monotonic() - start; if (++av->dmssc == 3) { @@ -993,7 +993,7 @@ void qc_do(ToxAVCall* call) LOGGER_DEBUG("Suggesting lower bitrate for audio..."); call->time_audio_good = 0; call->last_bad_audio_bit_rate = call->audio_bit_rate; - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_LOWER_AUDIO_BITRATE); + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_DECREASE_AUDIO_BITRATE); break; case rtp_StateGood: if (call->time_audio_good == 0) @@ -1017,7 +1017,7 @@ void qc_do(ToxAVCall* call) LOGGER_DEBUG("Suggesting lower bitrate for video..."); call->time_video_good = 0; call->last_bad_video_bit_rate = call->video_bit_rate; - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_LOWER_VIDEO_BITRATE); + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_DECREASE_VIDEO_BITRATE); break; case rtp_StateGood: if (call->time_video_good == 0) diff --git a/toxav/toxav.h b/toxav/toxav.h index 098075b38..73e5a86c8 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -215,11 +215,11 @@ typedef enum TOXAV_CALL_STATE { /** * AV core suggests you to lower bitrate for audio. */ - TOXAV_CALL_STATE_LOWER_AUDIO_BITRATE = 32, + TOXAV_CALL_STATE_DECREASE_AUDIO_BITRATE = 32, /** * AV core suggests you to lower bitrate for video. */ - TOXAV_CALL_STATE_LOWER_VIDEO_BITRATE = 64, + TOXAV_CALL_STATE_DECREASE_VIDEO_BITRATE = 64, /** * AV core suggests you to increase bitrate for audio. */ From eb9f4ad592e1f1988c90ac9af9799e2f1def4002 Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 18 Apr 2015 19:08:18 +0200 Subject: [PATCH 29/97] Fix toxav callback --- toxav/av_test.c | 64 +++++++++++++++++++++++++------------------------ toxav/codec.c | 3 ++- toxav/toxav.c | 3 --- toxav/toxav.h | 3 ++- 4 files changed, 37 insertions(+), 36 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index e24f0a069..d669e3805 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -77,8 +77,8 @@ #define TEST_REJECT 0 #define TEST_CANCEL 0 #define TEST_MUTE_UNMUTE 0 -#define TEST_TRANSFER_A 1 -#define TEST_TRANSFER_V 0 +#define TEST_TRANSFER_A 0 +#define TEST_TRANSFER_V 1 typedef struct { @@ -119,8 +119,10 @@ void* pa_write_thread (void* d) pthread_mutex_unlock(cc->arb_mutex); Pa_WriteStream(adout, f->data, f->size); free(f); - } else + } else { pthread_mutex_unlock(cc->arb_mutex); + c_sleep(10); + } } } @@ -165,9 +167,9 @@ void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, vo // printf("Decreasing bitrate to: %d\n", abitrate); // toxav_set_audio_bit_rate(av, friend_number, abitrate, 0); // - } - - if (state & TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE) { + } else if (state & TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE) { + + } else if (state & TOXAV_CALL_STATE_DECREASE_VIDEO_BITRATE) { } else { printf("Handling CALL STATE callback: %d\n", state); @@ -175,7 +177,8 @@ void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, vo } void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, - uint8_t const *planes[], int32_t const stride[], + uint8_t const *y, uint8_t const *u, uint8_t const *v, + int32_t ystride, int32_t ustride, int32_t vstride, void *user_data) { uint16_t *img_data = malloc(height * width * 6); @@ -184,13 +187,13 @@ void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, for (i = 0; i < height; ++i) { for (j = 0; j < width; ++j) { uint8_t *point = (void*)img_data + 3 * ((i * width) + j); - int y = planes[0][(i * stride[0]) + j]; - int u = planes[1][((i / 2) * stride[1]) + (j / 2)]; - int v = planes[2][((i / 2) * stride[2]) + (j / 2)]; + int yx = y[(i * ystride) + j]; + int ux = u[((i / 2) * ustride) + (j / 2)]; + int vx = v[((i / 2) * vstride) + (j / 2)]; - point[0] = YUV2R(y, u, v); - point[1] = YUV2G(y, u, v); - point[2] = YUV2B(y, u, v); + point[0] = YUV2R(yx, ux, vx); + point[1] = YUV2G(yx, ux, vx); + point[2] = YUV2B(yx, ux, vx); } } @@ -211,16 +214,14 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, uint32_t sampling_rate, void *user_data) { -// CallControl* cc = user_data; -// frame* f = malloc(sizeof(frame) + sample_count * sizeof(int16_t)); -// memcpy(f->data, pcm, sample_count); -// f->size = sample_count/channels; -// -// pthread_mutex_lock(cc->arb_mutex); -// free(rb_write(cc->arb, f)); -// pthread_mutex_unlock(cc->arb_mutex); + CallControl* cc = user_data; + frame* f = malloc(sizeof(uint16_t) + sample_count * sizeof(int16_t)); + memcpy(f->data, pcm, sample_count * sizeof(int16_t)); + f->size = sample_count/channels; - Pa_WriteStream(adout, pcm, sample_count/channels); + pthread_mutex_lock(cc->arb_mutex); + free(rb_write(cc->arb, f)); + pthread_mutex_unlock(cc->arb_mutex); } void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, size_t length, void *userdata) { @@ -317,7 +318,7 @@ int iterate_tox(Tox* bootstrap, ToxAV* AliceAV, ToxAV* BobAV) return MIN(tox_iteration_interval(toxav_get_tox(AliceAV)), tox_iteration_interval(toxav_get_tox(BobAV))); } void* iterate_toxav (void * data) -{ +{ struct toxav_thread_data* data_cast = data; // cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); @@ -327,9 +328,7 @@ void* iterate_toxav (void * data) toxav_iterate(data_cast->BobAV); int rc = MIN(toxav_iteration_interval(data_cast->AliceAV), toxav_iteration_interval(data_cast->BobAV)); -// cvWaitKey(10); - printf("\rSleeping for: %d ", rc); - fflush(stdout); +// cvWaitKey(rc); c_sleep(rc); } @@ -751,7 +750,7 @@ int main (int argc, char** argv) AliceCC.arb = rb_new(16); BobCC.arb = rb_new(16); - AliceCC.abitrate = BobCC.abitrate = 16; + AliceCC.abitrate = BobCC.abitrate = 48; { /* Call */ TOXAV_ERR_CALL rc; @@ -803,7 +802,6 @@ int main (int argc, char** argv) pthread_create(&dect, NULL, iterate_toxav, &data); pthread_detach(dect); - int frame_size = (af_info.samplerate * audio_frame_duration / 1000) * af_info.channels; struct PaStreamParameters output; @@ -819,6 +817,11 @@ int main (int argc, char** argv) err = Pa_StartStream(adout); assert(err == paNoError); + /* Start write thread */ + pthread_t t; + pthread_create(&t, NULL, pa_write_thread, &BobCC); + pthread_detach(t); + printf("Sample rate %d\n", af_info.samplerate); while ( start_time + expected_time > time(NULL) ) { uint64_t enc_start_time = current_time_monotonic(); @@ -833,10 +836,9 @@ int main (int argc, char** argv) c_sleep(abs(audio_frame_duration - (current_time_monotonic() - enc_start_time) - 1)); } + printf("Played file in: %lu; stopping stream...\n", time(NULL) - start_time); + Pa_StopStream(adout); - - printf("Played file in: %lu\n", time(NULL) - start_time); - sf_close(af_handle); { /* Hangup */ diff --git a/toxav/codec.c b/toxav/codec.c index 95b4d4067..e64c4d52f 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -371,7 +371,8 @@ void cs_do(CSession *cs) for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) { if (cs->vcb.first) cs->vcb.first(cs->av, cs->friend_id, dest->d_w, dest->d_h, - (const uint8_t**)dest->planes, dest->stride, cs->vcb.second); + (const uint8_t*)dest->planes[0], (const uint8_t*)dest->planes[1], (const uint8_t*)dest->planes[2], + dest->stride[0], dest->stride[1], dest->stride[2], cs->vcb.second); vpx_img_free(dest); } diff --git a/toxav/toxav.c b/toxav/toxav.c index 2e6326fd8..01b2670d9 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -250,7 +250,6 @@ void toxav_iterate(ToxAV* av) LOGGED_UNLOCK(av->mutex); av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa); -// av->interval = rc < 5 ? 1: rc - 5; av->dmsst += current_time_monotonic() - start; if (++av->dmssc == 3) { @@ -865,8 +864,6 @@ int callback_capabilites(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; LOGGED_LOCK(toxav->mutex); - /* TODO modify cs? */ - invoke_call_state(toxav, call->friend_id, call->peer_capabilities); LOGGED_UNLOCK(toxav->mutex); diff --git a/toxav/toxav.h b/toxav/toxav.h index 73e5a86c8..0edd90242 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -500,7 +500,8 @@ bool toxav_send_audio_frame(ToxAV *av, uint32_t friend_number, */ typedef void toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, - uint8_t const *planes[], int32_t const stride[], + uint8_t const *y, uint8_t const *u, uint8_t const *v, + int32_t ystride, int32_t ustride, int32_t vstride, void *user_data); /** * Set the callback for the `receive_video_frame` event. Pass NULL to unset. From 3a12a3b3291b34ef18edcd0ca66449944bc8c933 Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 18 Apr 2015 19:15:46 +0200 Subject: [PATCH 30/97] Update comment --- toxav/toxav.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/toxav/toxav.h b/toxav/toxav.h index 0edd90242..f2501d7ae 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -485,18 +485,17 @@ bool toxav_send_audio_frame(ToxAV *av, uint32_t friend_number, /** * The function type for the `receive_video_frame` callback. * - * Each plane contains (width * height) pixels. The Alpha plane can be NULL, in - * which case every pixel should be assumed fully opaque. - * * @param friend_number The friend number of the friend who sent a video frame. * @param width Width of the frame in pixels. * @param height Height of the frame in pixels. - * @param planes Plane data. To access Y (Luminance) plane use index 0, - * To access U (Chroma) plane use index 1, - * To access V (Chroma) plane use index 2. - * The size of plane data is derived from width and height where - * Y = width * height, U = (width/2) * (height/2) and V = (width/2) * (height/2). - * @param stride Strides data. Indexing is the same as in 'planes' param. + * @param y + * @param u + * @param v Plane data. + * The size of plane data is derived from width and height where + * Y = width * height, U = (width/2) * (height/2) and V = (width/2) * (height/2). + * @param ystride + * @param ustride + * @param vstride Strides data. Indexing is the same as in 'planes' param. */ typedef void toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, From cbb8fdd4eab3bae4db1179f1fa04cdaa35957aeb Mon Sep 17 00:00:00 2001 From: mannol Date: Mon, 20 Apr 2015 11:21:55 +0200 Subject: [PATCH 31/97] Add extern "C" --- toxav/toxav.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/toxav/toxav.h b/toxav/toxav.h index f2501d7ae..2571bfff8 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -25,6 +25,10 @@ #include #include #include + +#ifdef __cplusplus +extern "C" { +#endif /** \page av Public audio/video API for Tox clients. * * Unlike the Core API, this API is fully thread-safe. The library will ensure @@ -528,4 +532,8 @@ typedef void toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, */ void toxav_callback_receive_audio_frame(ToxAV *av, toxav_receive_audio_frame_cb *function, void *user_data); +#ifdef __cplusplus +} +#endif + #endif /* TOXAV_H */ From 3fd0ee5f0873924b4881b0e33eb1c17ea877ab4a Mon Sep 17 00:00:00 2001 From: mannol Date: Tue, 21 Apr 2015 02:31:12 +0200 Subject: [PATCH 32/97] Final touchups --- toxav/Makefile.inc | 4 + toxav/audio.c | 383 +++++++++++++++++++++++++++++++++++++++++++++ toxav/audio.h | 60 +++++++ toxav/av_test.c | 19 ++- toxav/codec.c | 10 +- toxav/codec.h | 3 - toxav/msi.c | 21 ++- toxav/msi.h | 3 +- toxav/rtp.c | 27 +++- toxav/toxav.c | 11 +- toxav/toxav.h | 2 +- toxav/video.c | 332 +++++++++++++++++++++++++++++++++++++++ toxav/video.h | 81 ++++++++++ 13 files changed, 923 insertions(+), 33 deletions(-) create mode 100644 toxav/audio.c create mode 100644 toxav/audio.h create mode 100644 toxav/video.c create mode 100644 toxav/video.h diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc index a73ddc91f..d9adb4fea 100644 --- a/toxav/Makefile.inc +++ b/toxav/Makefile.inc @@ -12,6 +12,10 @@ libtoxav_la_SOURCES = ../toxav/rtp.h \ ../toxav/group.c \ ../toxav/codec.h \ ../toxav/codec.c \ + ../toxav/audio.h \ + ../toxav/audio.c \ + ../toxav/video.h \ + ../toxav/video.c \ ../toxav/toxav.h \ ../toxav/toxav.c diff --git a/toxav/audio.c b/toxav/audio.c new file mode 100644 index 000000000..f3e969e91 --- /dev/null +++ b/toxav/audio.c @@ -0,0 +1,383 @@ +/** audio.c + * + * Copyright (C) 2013-2015 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + */ + +#include + +#include "audio.h" +#include "rtp.h" + +#include "../toxcore/logger.h" + +static struct JitterBuffer *jbuf_new(uint32_t capacity); +static void jbuf_clear(struct JitterBuffer *q); +static void jbuf_free(struct JitterBuffer *q); +static int jbuf_write(struct JitterBuffer *q, RTPMessage *m); +static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success); + +OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count); +bool reconfigure_audio_decoder(ACSession* ac, int32_t sampling_rate, int8_t channels); + + + +ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *cb, void *cb_data) +{ + ACSession *ac = calloc(sizeof(ACSession), 1); + + if (!ac) { + LOGGER_WARNING("Allocation failed! Application might misbehave!"); + return NULL; + } + + if (create_recursive_mutex(ac->queue_mutex) != 0) { + LOGGER_WARNING("Failed to create recursive mutex!"); + free(ac); + return NULL; + } + + int status; + ac->decoder = opus_decoder_create(48000, 2, &status ); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status)); + goto BASE_CLEANUP; + } + + if ( !(ac->j_buf = jbuf_new(3)) ) { + LOGGER_WARNING("Jitter buffer creaton failed!"); + opus_decoder_destroy(ac->decoder); + goto BASE_CLEANUP; + } + + /* Initialize encoders with default values */ + ac->encoder = create_audio_encoder(48000, 48000, 2); + if (ac->encoder == NULL) + goto DECODER_CLEANUP; + + ac->last_encoding_bitrate = 48000; + ac->last_encoding_sampling_rate = 48000; + ac->last_encoding_channel_count = 2; + + ac->last_decoding_channel_count = 2; + ac->last_decoding_sampling_rate = 48000; + ac->last_decoder_reconfiguration = 0; /* Make it possible to reconfigure straight away */ + + /* These need to be set in order to properly + * do error correction with opus */ + ac->last_packet_frame_duration = 120; + ac->last_packet_sampling_rate = 48000; + + ac->av = av; + ac->friend_id = friend_id; + ac->acb.first = cb; + ac->acb.second = cb_data; + + return ac; + +DECODER_CLEANUP: + opus_decoder_destroy(ac->decoder); + jbuf_free(ac->j_buf); +BASE_CLEANUP: + pthread_mutex_destroy(ac->queue_mutex); + free(ac); + return NULL; +} +void ac_kill(ACSession* ac) +{ + if (!ac) + return; + + opus_encoder_destroy(ac->encoder); + opus_decoder_destroy(ac->decoder); + jbuf_free(ac->j_buf); + + pthread_mutex_destroy(ac->queue_mutex); + + LOGGER_DEBUG("Terminated audio handler: %p", ac); + free(ac); +} +void ac_do(ACSession* ac) +{ + if (!ac) + return; + + /* Enough space for the maximum frame size (120 ms 48 KHz audio) */ + int16_t tmp[5760]; + + RTPMessage *msg; + int rc = 0; + + pthread_mutex_lock(ac->queue_mutex); + while ((msg = jbuf_read(ac->j_buf, &rc)) || rc == 2) { + pthread_mutex_unlock(ac->queue_mutex); + + if (rc == 2) { + LOGGER_DEBUG("OPUS correction"); + rc = opus_decode(ac->decoder, NULL, 0, tmp, + (ac->last_packet_sampling_rate * ac->last_packet_frame_duration / 1000) / + ac->last_packet_channel_count, 1); + } else { + /* Get values from packet and decode. */ + /* NOTE: This didn't work very well + rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); + if (rc != -1) { + cs->last_packet_sampling_rate = rc; + } else { + LOGGER_WARNING("Failed to load packet values!"); + rtp_free_msg(NULL, msg); + continue; + }*/ + + + /* Pick up sampling rate from packet */ + memcpy(&ac->last_packet_sampling_rate, msg->data, 4); + ac->last_packet_sampling_rate = ntohl(ac->last_packet_sampling_rate); + + ac->last_packet_channel_count = opus_packet_get_nb_channels(msg->data + 4); + + /* + * NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa, + * it didn't work quite well. + */ + if (!reconfigure_audio_decoder(ac, ac->last_packet_sampling_rate, ac->last_packet_channel_count)) { + LOGGER_WARNING("Failed to reconfigure decoder!"); + rtp_free_msg(NULL, msg); + continue; + } + + rc = opus_decode(ac->decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0); + rtp_free_msg(NULL, msg); + } + + if (rc < 0) { + LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); + } else if (ac->acb.first) { + ac->last_packet_frame_duration = (rc * 1000) / ac->last_packet_sampling_rate * ac->last_packet_channel_count; + + ac->acb.first(ac->av, ac->friend_id, tmp, rc * ac->last_packet_channel_count, + ac->last_packet_channel_count, ac->last_packet_sampling_rate, ac->acb.second); + } + + return; + } + pthread_mutex_unlock(ac->queue_mutex); +} +int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels) +{ + if (!ac) + return; + + /* Values are checked in toxav.c */ + if (ac->last_encoding_sampling_rate != sampling_rate || ac->last_encoding_channel_count != channels) { + OpusEncoder* new_encoder = create_audio_encoder(bitrate, sampling_rate, channels); + if (new_encoder == NULL) + return -1; + + opus_encoder_destroy(ac->encoder); + ac->encoder = new_encoder; + } else if (ac->last_encoding_bitrate == bitrate) + return 0; /* Nothing changed */ + else { + int status = opus_encoder_ctl(ac->encoder, OPUS_SET_BITRATE(bitrate)); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); + return -1; + } + } + + ac->last_encoding_bitrate = bitrate; + ac->last_encoding_sampling_rate = sampling_rate; + ac->last_encoding_channel_count = channels; + + LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels); + return 0; +} +/* called from rtp */ +void ac_queue_message(void* acp, RTPMessage *msg) +{ + if (!acp || !msg) + return; + + ACSession* ac = acp; + + pthread_mutex_lock(ac->queue_mutex); + int ret = jbuf_write(ac->j_buf, msg); + pthread_mutex_unlock(ac->queue_mutex); + + if (ret == -1) + rtp_free_msg(NULL, msg); +} + + +/* JITTER BUFFER WORK */ +struct JitterBuffer { + RTPMessage **queue; + uint32_t size; + uint32_t capacity; + uint16_t bottom; + uint16_t top; +}; + +static struct JitterBuffer *jbuf_new(uint32_t capacity) +{ + unsigned int size = 1; + + while (size <= (capacity * 4)) { + size *= 2; + } + + struct JitterBuffer *q; + + if ( !(q = calloc(sizeof(struct JitterBuffer), 1)) ) return NULL; + + if (!(q->queue = calloc(sizeof(RTPMessage *), size))) { + free(q); + return NULL; + } + + q->size = size; + q->capacity = capacity; + return q; +} +static void jbuf_clear(struct JitterBuffer *q) +{ + for (; q->bottom != q->top; ++q->bottom) { + if (q->queue[q->bottom % q->size]) { + rtp_free_msg(NULL, q->queue[q->bottom % q->size]); + q->queue[q->bottom % q->size] = NULL; + } + } +} +static void jbuf_free(struct JitterBuffer *q) +{ + if (!q) return; + + jbuf_clear(q); + free(q->queue); + free(q); +} +static int jbuf_write(struct JitterBuffer *q, RTPMessage *m) +{ + uint16_t sequnum = m->header->sequnum; + + unsigned int num = sequnum % q->size; + + if ((uint32_t)(sequnum - q->bottom) > q->size) { + LOGGER_DEBUG("Clearing filled jitter buffer: %p", q); + + jbuf_clear(q); + q->bottom = sequnum - q->capacity; + q->queue[num] = m; + q->top = sequnum + 1; + return 0; + } + + if (q->queue[num]) + return -1; + + q->queue[num] = m; + + if ((sequnum - q->bottom) >= (q->top - q->bottom)) + q->top = sequnum + 1; + + return 0; +} +static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success) +{ + if (q->top == q->bottom) { + *success = 0; + return NULL; + } + + unsigned int num = q->bottom % q->size; + + if (q->queue[num]) { + RTPMessage *ret = q->queue[num]; + q->queue[num] = NULL; + ++q->bottom; + *success = 1; + return ret; + } + + if ((uint32_t)(q->top - q->bottom) > q->capacity) { + ++q->bottom; + *success = 2; + return NULL; + } + + *success = 0; + return NULL; +} +OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count) +{ + int status = OPUS_OK; + OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_AUDIO, &status); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(status)); + return NULL; + } + + status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bitrate)); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); + goto FAILURE; + } + + status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(10)); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); + goto FAILURE; + } + + return rc; + +FAILURE: + opus_encoder_destroy(rc); + return NULL; +} +bool reconfigure_audio_decoder(ACSession* ac, int32_t sampling_rate, int8_t channels) +{ + if (sampling_rate != ac->last_decoding_sampling_rate || channels != ac->last_decoding_channel_count) { + if (current_time_monotonic() - ac->last_decoder_reconfiguration < 500) + return false; + + int status; + OpusDecoder* new_dec = opus_decoder_create(sampling_rate, channels, &status ); + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio decoder(%d %d): %s", sampling_rate, channels, opus_strerror(status)); + return false; + } + + ac->last_decoding_sampling_rate = sampling_rate; + ac->last_decoding_channel_count = channels; + ac->last_decoder_reconfiguration = current_time_monotonic(); + + opus_decoder_destroy(ac->decoder); + ac->decoder = new_dec; + + LOGGER_DEBUG("Reconfigured audio decoder sr: %d cc: %d", sampling_rate, channels); + } + + return true; +} \ No newline at end of file diff --git a/toxav/audio.h b/toxav/audio.h new file mode 100644 index 000000000..62a28cdf5 --- /dev/null +++ b/toxav/audio.h @@ -0,0 +1,60 @@ +/** audio.h + * + * Copyright (C) 2013-2015 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + */ + +#ifndef AUDIO_H +#define AUDIO_H + +#include +#include + +#include "toxav.h" + +#include "../toxcore/util.h" + +typedef struct ACSession_s { + /* encoding */ + OpusEncoder *encoder; + int32_t last_encoding_sampling_rate; + int32_t last_encoding_channel_count; + int32_t last_encoding_bitrate; + + /* decoding */ + OpusDecoder *decoder; + int32_t last_packet_channel_count; + int32_t last_packet_sampling_rate; + int32_t last_packet_frame_duration; + int32_t last_decoding_sampling_rate; + int32_t last_decoding_channel_count; + uint64_t last_decoder_reconfiguration; + void *j_buf; + + pthread_mutex_t queue_mutex[1]; + + ToxAV* av; + uint32_t friend_id; + PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ +} ACSession; + +ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *cb, void *cb_data); +void ac_kill(ACSession* ac); +void ac_do(ACSession* ac); +int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels); +#endif /* AUDIO_H */ \ No newline at end of file diff --git a/toxav/av_test.c b/toxav/av_test.c index d669e3805..994f9cf6e 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -77,8 +77,8 @@ #define TEST_REJECT 0 #define TEST_CANCEL 0 #define TEST_MUTE_UNMUTE 0 -#define TEST_TRANSFER_A 0 -#define TEST_TRANSFER_V 1 +#define TEST_TRANSFER_A 1 +#define TEST_TRANSFER_V 0 typedef struct { @@ -320,21 +320,28 @@ int iterate_tox(Tox* bootstrap, ToxAV* AliceAV, ToxAV* BobAV) void* iterate_toxav (void * data) { struct toxav_thread_data* data_cast = data; - -// cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); +#if defined TEST_TRANSFER_V && TEST_TRANSFER_V == 1 + cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); +#endif while (data_cast->sig == 0) { toxav_iterate(data_cast->AliceAV); toxav_iterate(data_cast->BobAV); int rc = MIN(toxav_iteration_interval(data_cast->AliceAV), toxav_iteration_interval(data_cast->BobAV)); -// cvWaitKey(rc); +#if defined TEST_TRANSFER_V && TEST_TRANSFER_V == 1 + cvWaitKey(rc); +#else c_sleep(rc); +#endif } data_cast->sig = 1; -// cvDestroyWindow(vdout); +#if defined TEST_TRANSFER_V && TEST_TRANSFER_V == 1 + cvDestroyWindow(vdout); +#endif + pthread_exit(NULL); } diff --git a/toxav/codec.c b/toxav/codec.c index e64c4d52f..57e43c67f 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -328,7 +328,6 @@ void cs_do(CSession *cs) LOGGER_WARNING("Failed to reconfigure decoder!"); rtp_free_msg(NULL, msg); continue; -// goto DONE; } rc = opus_decode(cs->audio_decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0); @@ -347,7 +346,6 @@ void cs_do(CSession *cs) LOGGED_LOCK(cs->queue_mutex); } -// DONE:; } /********************* VIDEO *********************/ @@ -497,12 +495,12 @@ void cs_kill(CSession *cs) * the callback is unregistered before cs_kill is called. */ - vpx_codec_destroy(cs->v_encoder); - vpx_codec_destroy(cs->v_decoder); opus_encoder_destroy(cs->audio_encoder); opus_decoder_destroy(cs->audio_decoder); - rb_free(cs->vbuf_raw); jbuf_free(cs->j_buf); + vpx_codec_destroy(cs->v_encoder); + vpx_codec_destroy(cs->v_decoder); + rb_free(cs->vbuf_raw); free(cs->frame_buf); free(cs->split_video_frame); @@ -600,7 +598,7 @@ void queue_message(RTPSession *session, RTPMessage *msg) { CSession *cs = session->cs; - if (!cs) + if (!cs) return; /* Audio */ diff --git a/toxav/codec.h b/toxav/codec.h index 7cc9b15d7..497016eb3 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -42,9 +42,6 @@ /* Audio encoding/decoding */ #include -#define PACKED_AUDIO_SIZE(x) (x + 5) -#define UNPACKED_AUDIO_SIZE(x) (x - 5) - typedef struct CSession_s { /* VIDEO diff --git a/toxav/msi.c b/toxav/msi.c index c313e288c..b7926e074 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -188,7 +188,7 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ msg.capabilities.value = capabilities; msg.vfpsz.exists = true; - msg.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE); + msg.vfpsz.value = VIDEOFRAME_PIECE_SIZE; send_message ( (*call)->session->messenger, (*call)->friend_id, &msg ); @@ -238,7 +238,7 @@ int msi_answer ( MSICall* call, uint8_t capabilities ) msg.capabilities.value = capabilities; msg.vfpsz.exists = true; - msg.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE); + msg.vfpsz.value = VIDEOFRAME_PIECE_SIZE; send_message ( session->messenger, call->friend_id, &msg ); @@ -349,6 +349,12 @@ int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) case IDVFPSZ: CHECK_SIZE(it, size_constraint, 2); SET_UINT16(it, dest->vfpsz); + dest->vfpsz = ntohs(dest->vfpsz); + + if (dest->vfpsz.value > 1200) { + LOGGER_ERROR("Invalid vfpsz param"); + return -1; + } break; default: @@ -419,8 +425,9 @@ int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg ) } if (msg->vfpsz.exists) { - it = msg_parse_header_out(IDVFPSZ, it, &msg->vfpsz.value, - sizeof(msg->vfpsz.value), &size); + uint16_t nb_vfpsz = htons(msg->vfpsz); + it = msg_parse_header_out(IDVFPSZ, it, &nb_vfpsz, + sizeof(nb_vfpsz), &size); } if ( it == parsed ) { @@ -620,7 +627,7 @@ void handle_push ( MSICall *call, const MSIMessage *msg ) goto FAILURE; } - call->peer_vfpsz = ntohs(msg->vfpsz.value); + call->peer_vfpsz = msg->vfpsz.value; } @@ -645,7 +652,7 @@ void handle_push ( MSICall *call, const MSIMessage *msg ) * is not terminated on our side. We can assume that * in this case we can automatically answer the re-call. */ - if (call->peer_vfpsz != ntohs(msg->vfpsz.value)) { + if (call->peer_vfpsz != msg->vfpsz.value) { LOGGER_WARNING("Friend sent invalid parameters for re-call"); call->error = msi_EInvalidParam; invoke_callback(call, msi_OnError); @@ -661,7 +668,7 @@ void handle_push ( MSICall *call, const MSIMessage *msg ) msg.capabilities.value = call->self_capabilities; msg.vfpsz.exists = true; - msg.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE); + msg.vfpsz.value = VIDEOFRAME_PIECE_SIZE; send_message ( call->session->messenger, call->friend_id, &msg ); diff --git a/toxav/msi.h b/toxav/msi.h index 7d82afc81..457d31488 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -25,7 +25,8 @@ #include #include -#include "codec.h" +#include "audio.h" +#include "video.h" #include "../toxcore/Messenger.h" /** Preconfigured value for video splitting */ diff --git a/toxav/rtp.c b/toxav/rtp.c index e1e6dbac8..9ef41b357 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -69,9 +69,11 @@ typedef struct RTCPSession_s { +/* These are defined externally */ +void ac_queue_message(void *acp, RTPMessage *msg); +void vc_queue_message(void *vcp, RTPMessage *msg); + -/* queue_message() is defined in codec.c */ -void queue_message(RTPSession *session, RTPMessage *msg); RTPHeader *parse_header_in ( const uint8_t *payload, int length ); RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length ); RTPMessage *msg_parse ( const uint8_t *data, int length ); @@ -395,6 +397,10 @@ RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length ) } RTPMessage *msg_parse ( const uint8_t *data, int length ) { + /* TODO: data dynamic, [0] + * TODO: dummy payload type + * TODO: parse header before allocating message + */ RTPMessage *retu = calloc(1, sizeof (RTPMessage)); retu->header = parse_header_in ( data, length ); /* It allocates memory and all */ @@ -540,6 +546,7 @@ void send_rtcp_report(RTCPSession* session, Messenger* m, uint32_t friendnumber) } int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) { + /* TODO on message callback */ RTPSession *session = object; RTPMessage *msg; @@ -570,8 +577,20 @@ int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data } session->rtcp_session->last_received_packets ++; - - queue_message(session, msg); + + /* Check if this session can handle the packet */ + if (session->payload_type != session->prefix % 128) { + LOGGER_WARNING("Friend %d sent invalid payload type!", session->dest); + rtp_free_msg(msg); + return -1; + } + + /* Handle */ + if (session->payload_type == rtp_TypeAudio % 128) + ac_queue_message(session->cs, msg); + else /* It can only be video */ + vc_queue_message(session->cs, msg); + return 0; } int handle_rtcp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) diff --git a/toxav/toxav.c b/toxav/toxav.c index 01b2670d9..6f712af92 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -1170,11 +1170,6 @@ void call_kill_transmission(ToxAVCall* call) call->active = 0; - rtp_kill(call->rtps[audio_index]); - call->rtps[audio_index] = NULL; - rtp_kill(call->rtps[video_index]); - call->rtps[video_index] = NULL; - LOGGED_LOCK(call->mutex_audio_sending); LOGGED_UNLOCK(call->mutex_audio_sending); LOGGED_LOCK(call->mutex_video_sending); @@ -1182,6 +1177,12 @@ void call_kill_transmission(ToxAVCall* call) LOGGED_LOCK(call->mutex_decoding); LOGGED_UNLOCK(call->mutex_decoding); + + rtp_kill(call->rtps[audio_index]); + call->rtps[audio_index] = NULL; + rtp_kill(call->rtps[video_index]); + call->rtps[video_index] = NULL; + cs_kill(call->cs); call->cs = NULL; diff --git a/toxav/toxav.h b/toxav/toxav.h index 2571bfff8..bf8756ebb 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -499,7 +499,7 @@ bool toxav_send_audio_frame(ToxAV *av, uint32_t friend_number, * Y = width * height, U = (width/2) * (height/2) and V = (width/2) * (height/2). * @param ystride * @param ustride - * @param vstride Strides data. Indexing is the same as in 'planes' param. + * @param vstride Strides data. */ typedef void toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, diff --git a/toxav/video.c b/toxav/video.c new file mode 100644 index 000000000..d51cfd4a4 --- /dev/null +++ b/toxav/video.c @@ -0,0 +1,332 @@ +/** video.c + * + * Copyright (C) 2013-2015 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + */ + +#include +#include + +#include "video.h" +#include "msi.h" + +#include "../toxcore/logger.h" +#include "../toxcore/network.h" + +/* Good quality encode. */ +#define MAX_DECODE_TIME_US 0 + +#define MAX_VIDEOFRAME_SIZE 0x40000 /* 256KiB */ +#define VIDEOFRAME_HEADER_SIZE 0x2 + +/* FIXME: Might not be enough? NOTE: I think it is enough */ +#define VIDEO_DECODE_BUFFER_SIZE 20 + +typedef struct { uint16_t size; uint8_t data[]; } Payload; + +bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate); + + +VCSession* vc_new(ToxAV* av, uint32_t friend_id, toxav_receive_video_frame_cb* cb, void* cb_data, uint32_t mvfpsz) +{ + VCSession *vc = calloc(sizeof(VCSession), 1); + + if (!vc) { + LOGGER_WARNING("Allocation failed! Application might misbehave!"); + return NULL; + } + + if (create_recursive_mutex(vc->queue_mutex) != 0) { + LOGGER_WARNING("Failed to create recursive mutex!"); + free(vc); + return NULL; + } + + if ( !(vc->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) ) + goto BASE_CLEANUP; + if ( !(vc->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) ) + goto BASE_CLEANUP; + if ( !(vc->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE)) ) + goto BASE_CLEANUP; + + int rc = vpx_codec_dec_init_ver(vc->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, + NULL, 0, VPX_DECODER_ABI_VERSION); + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); + goto BASE_CLEANUP; + } + + if (!create_video_encoder(vc->v_encoder, 500000)) { + vpx_codec_destroy(vc->v_decoder); + goto BASE_CLEANUP; + } + + vc->linfts = current_time_monotonic(); + vc->lcfd = 60; + + vc->peer_video_frame_piece_size = mvfpsz; + + return vc; + +BASE_CLEANUP: + pthread_mutex_destroy(vc->queue_mutex); + rb_free(vc->vbuf_raw); + free(vc->split_video_frame); + free(vc->frame_buf); + free(vc); + return NULL; +} +void vc_kill(VCSession* vc) +{ + if (!vc) + return; + + vpx_codec_destroy(vc->v_encoder); + vpx_codec_destroy(vc->v_decoder); + rb_free(vc->vbuf_raw); + free(vc->split_video_frame); + free(vc->frame_buf); + + pthread_mutex_destroy(vc->queue_mutex); + + LOGGER_DEBUG("Terminated video handler: %p", vc); + free(vc); +} +void vc_do(VCSession* vc) +{ + if (!vc) + return; + + Payload *p; + int rc; + + pthread_mutex_lock(vc->queue_mutex); + if (rb_read(vc->vbuf_raw, (void**)&p)) { + pthread_mutex_unlock(vc->queue_mutex); + + rc = vpx_codec_decode(vc->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); + free(p); + + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc)); + } else { + vpx_codec_iter_t iter = NULL; + vpx_image_t *dest = vpx_codec_get_frame(vc->v_decoder, &iter); + + /* Play decoded images */ + for (; dest; dest = vpx_codec_get_frame(vc->v_decoder, &iter)) { + if (vc->vcb.first) + vc->vcb.first(vc->av, vc->friend_id, dest->d_w, dest->d_h, + (const uint8_t*)dest->planes[0], (const uint8_t*)dest->planes[1], (const uint8_t*)dest->planes[2], + dest->stride[0], dest->stride[1], dest->stride[2], vc->vcb.second); + + vpx_img_free(dest); + } + } + + return; + } + pthread_mutex_unlock(vc->queue_mutex); +} +void vc_init_video_splitter_cycle(VCSession* vc) +{ + if (!vc) + return; + + vc->split_video_frame[0] = vc->frameid_out++; + vc->split_video_frame[1] = 0; +} +int vc_update_video_splitter_cycle(VCSession* vc, const uint8_t* payload, uint16_t length) +{ + if (!vc) + return; + + vc->processing_video_frame = payload; + vc->processing_video_frame_size = length; + + return ((length - 1) / VIDEOFRAME_PIECE_SIZE) + 1; +} +const uint8_t* vc_iterate_split_video_frame(VCSession* vc, uint16_t* size) +{ + if (!vc || !size) + return NULL; + + if (vc->processing_video_frame_size > VIDEOFRAME_PIECE_SIZE) { + memcpy(vc->split_video_frame + VIDEOFRAME_HEADER_SIZE, + vc->processing_video_frame, + VIDEOFRAME_PIECE_SIZE); + + vc->processing_video_frame += VIDEOFRAME_PIECE_SIZE; + vc->processing_video_frame_size -= VIDEOFRAME_PIECE_SIZE; + + *size = VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE; + } else { + memcpy(vc->split_video_frame + VIDEOFRAME_HEADER_SIZE, + vc->processing_video_frame, + vc->processing_video_frame_size); + + *size = vc->processing_video_frame_size + VIDEOFRAME_HEADER_SIZE; + } + + vc->split_video_frame[1]++; + + return vc->split_video_frame; +} +int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height) +{ + if (!vc) + return; + + vpx_codec_enc_cfg_t cfg = *vc->v_encoder[0].config.enc; + if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) + return 0; /* Nothing changed */ + + cfg.rc_target_bitrate = bitrate; + cfg.g_w = width; + cfg.g_h = height; + + int rc = vpx_codec_enc_config_set(vc->v_encoder, &cfg); + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); + return -1; + } + + return 0; +} +/* Called from RTP */ +void vc_queue_message(void* vcp, RTPMessage *msg) +{ + /* This function does the reconstruction of video packets. + * See more info about video splitting in docs + */ + if (!vcp || !msg) + return; + + VCSession* vc = vcp; + + uint8_t *packet = msg->data; + uint32_t packet_size = msg->length; + + if (packet_size < VIDEOFRAME_HEADER_SIZE) + goto end; + + uint8_t diff = packet[0] - vc->frameid_in; + + if (diff != 0) { + if (diff < 225) { /* New frame */ + /* Flush last frames' data and get ready for this frame */ + Payload *p = malloc(sizeof(Payload) + vc->frame_size); + + if (p) { + LOGGED_LOCK(vc->queue_mutex); + + if (rb_full(vc->vbuf_raw)) { + LOGGER_DEBUG("Dropped video frame"); + Payload *tp; + rb_read(vc->vbuf_raw, (void**)&tp); + free(tp); + } else { + p->size = vc->frame_size; + memcpy(p->data, vc->frame_buf, vc->frame_size); + } + + /* Calculate time took for peer to send us this frame */ + uint32_t t_lcfd = current_time_monotonic() - vc->linfts; + vc->lcfd = t_lcfd > 100 ? vc->lcfd : t_lcfd; + vc->linfts = current_time_monotonic(); + + rb_write(vc->vbuf_raw, p); + LOGGED_UNLOCK(vc->queue_mutex); + } else { + LOGGER_WARNING("Allocation failed! Program might misbehave!"); + goto end; + } + + vc->frameid_in = packet[0]; + memset(vc->frame_buf, 0, vc->frame_size); + vc->frame_size = 0; + + } else { /* Old frame; drop */ + LOGGER_DEBUG("Old packet: %u", packet[0]); + goto end; + } + } + + uint8_t piece_number = packet[1]; + + uint32_t length_before_piece = ((piece_number - 1) * vc->peer_video_frame_piece_size); + uint32_t framebuf_new_length = length_before_piece + (packet_size - VIDEOFRAME_HEADER_SIZE); + + if (framebuf_new_length > MAX_VIDEOFRAME_SIZE) + goto end; + + + /* Otherwise it's part of the frame so just process */ + /* LOGGER_DEBUG("Video Packet: %u %u", packet[0], packet[1]); */ + + memcpy(vc->frame_buf + length_before_piece, + packet + VIDEOFRAME_HEADER_SIZE, + packet_size - VIDEOFRAME_HEADER_SIZE); + + if (framebuf_new_length > vc->frame_size) + vc->frame_size = framebuf_new_length; + +end: + rtp_free_msg(NULL, msg); +} + + +bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate) +{ + assert(dest); + + vpx_codec_enc_cfg_t cfg; + int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); + + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); + return false; + } + + rc = vpx_codec_enc_init_ver(dest, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, + VPX_ENCODER_ABI_VERSION); + + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); + return false; + } + + cfg.rc_target_bitrate = bitrate; + cfg.g_w = 800; + cfg.g_h = 600; + cfg.g_pass = VPX_RC_ONE_PASS; + cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; + cfg.g_lag_in_frames = 0; + cfg.kf_min_dist = 0; + cfg.kf_max_dist = 48; + cfg.kf_mode = VPX_KF_AUTO; + + rc = vpx_codec_control(dest, VP8E_SET_CPUUSED, 8); + + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); + vpx_codec_destroy(dest); + } + + return true; +} \ No newline at end of file diff --git a/toxav/video.h b/toxav/video.h new file mode 100644 index 000000000..c1678ad20 --- /dev/null +++ b/toxav/video.h @@ -0,0 +1,81 @@ +/** video.h + * + * Copyright (C) 2013-2015 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + */ + +#ifndef VIDEO_H +#define VIDEO_H + +#include +#include +#include +#include +#include +#define VIDEO_CODEC_DECODER_INTERFACE (vpx_codec_vp8_dx()) +#define VIDEO_CODEC_ENCODER_INTERFACE (vpx_codec_vp8_cx()) + +#include + +#include "toxav.h" + +#include "../toxcore/util.h" + +typedef struct VCSession_s { + + /* encoding */ + vpx_codec_ctx_t v_encoder[1]; + uint32_t frame_counter; + + /* decoding */ + vpx_codec_ctx_t v_decoder[1]; + void *vbuf_raw; /* Un-decoded data */ + + /* Data handling */ + uint8_t *frame_buf; /* buffer for split video payloads */ + uint32_t frame_size; /* largest address written to in frame_buf for current input frame */ + uint8_t frameid_in, frameid_out; /* id of input and output video frame */ + uint64_t linfts; /* Last received frame time stamp */ + uint32_t lcfd; /* Last calculated frame duration for incoming video payload */ + + /* Limits */ + uint32_t peer_video_frame_piece_size; + + /* Splitting */ + uint8_t *split_video_frame; + const uint8_t *processing_video_frame; + uint16_t processing_video_frame_size; + + + ToxAV *av; + int32_t friend_id; + + PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ + + pthread_mutex_t queue_mutex[1]; +} VCSession; + +VCSession* vc_new(ToxAV* av, uint32_t friend_id, toxav_receive_video_frame_cb *cb, void *cb_data, uint32_t mvfpsz); +void vc_kill(VCSession* vc); +void vc_do(VCSession* vc); +void vc_init_video_splitter_cycle(VCSession* vc); +int vc_update_video_splitter_cycle(VCSession* vc, const uint8_t* payload, uint16_t length); +const uint8_t *vc_iterate_split_video_frame(VCSession* vc, uint16_t *size); +int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height); + +#endif /* VIDEO_H */ \ No newline at end of file From 1bfd93e64a2a6d3bf9c90a9aa89abd29f3d826a7 Mon Sep 17 00:00:00 2001 From: mannol Date: Wed, 22 Apr 2015 02:09:37 +0200 Subject: [PATCH 33/97] Finished refactoring --- toxav/Makefile.inc | 2 - toxav/audio.c | 62 ++-- toxav/audio.h | 3 + toxav/av_test.c | 7 +- toxav/codec.c | 686 --------------------------------------------- toxav/codec.h | 125 --------- toxav/msi.c | 50 ++-- toxav/rtp.c | 41 +-- toxav/rtp.h | 46 +-- toxav/toxav.c | 365 ++++++++++++------------ toxav/video.c | 70 +++-- toxav/video.h | 3 + 12 files changed, 335 insertions(+), 1125 deletions(-) delete mode 100644 toxav/codec.c delete mode 100644 toxav/codec.h diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc index d9adb4fea..0434a3c6a 100644 --- a/toxav/Makefile.inc +++ b/toxav/Makefile.inc @@ -10,8 +10,6 @@ libtoxav_la_SOURCES = ../toxav/rtp.h \ ../toxav/msi.c \ ../toxav/group.h \ ../toxav/group.c \ - ../toxav/codec.h \ - ../toxav/codec.c \ ../toxav/audio.h \ ../toxav/audio.c \ ../toxav/video.h \ diff --git a/toxav/audio.c b/toxav/audio.c index f3e969e91..dc85452a8 100644 --- a/toxav/audio.c +++ b/toxav/audio.c @@ -83,6 +83,7 @@ ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *c * do error correction with opus */ ac->last_packet_frame_duration = 120; ac->last_packet_sampling_rate = 48000; + ac->last_packet_channel_count = 1; ac->av = av; ac->friend_id = friend_id; @@ -119,7 +120,7 @@ void ac_do(ACSession* ac) return; /* Enough space for the maximum frame size (120 ms 48 KHz audio) */ - int16_t tmp[5760]; + int16_t tmp[5760 * 2]; RTPMessage *msg; int rc = 0; @@ -130,9 +131,8 @@ void ac_do(ACSession* ac) if (rc == 2) { LOGGER_DEBUG("OPUS correction"); - rc = opus_decode(ac->decoder, NULL, 0, tmp, - (ac->last_packet_sampling_rate * ac->last_packet_frame_duration / 1000) / - ac->last_packet_channel_count, 1); + int fs = (ac->last_packet_sampling_rate * ac->last_packet_frame_duration) / 1000; + rc = opus_decode(ac->decoder, NULL, 0, tmp, fs, 1); } else { /* Get values from packet and decode. */ /* NOTE: This didn't work very well @@ -152,10 +152,9 @@ void ac_do(ACSession* ac) ac->last_packet_channel_count = opus_packet_get_nb_channels(msg->data + 4); - /* - * NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa, - * it didn't work quite well. - */ + /** NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa, + * it didn't work quite well. + */ if (!reconfigure_audio_decoder(ac, ac->last_packet_sampling_rate, ac->last_packet_channel_count)) { LOGGER_WARNING("Failed to reconfigure decoder!"); rtp_free_msg(NULL, msg); @@ -169,7 +168,7 @@ void ac_do(ACSession* ac) if (rc < 0) { LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); } else if (ac->acb.first) { - ac->last_packet_frame_duration = (rc * 1000) / ac->last_packet_sampling_rate * ac->last_packet_channel_count; + ac->last_packet_frame_duration = (rc * 1000) / ac->last_packet_sampling_rate; ac->acb.first(ac->av, ac->friend_id, tmp, rc * ac->last_packet_channel_count, ac->last_packet_channel_count, ac->last_packet_sampling_rate, ac->acb.second); @@ -179,6 +178,37 @@ void ac_do(ACSession* ac) } pthread_mutex_unlock(ac->queue_mutex); } +int ac_queue_message(void* acp, struct RTPMessage_s *msg) +{ + if (!acp || !msg) + return -1; + + if ((msg->header->marker_payloadt & 0x7f) == rtp_TypeDummyAudio % 128) { + LOGGER_WARNING("Got dummy!"); + rtp_free_msg(NULL, msg); + return 0; + } + + if ((msg->header->marker_payloadt & 0x7f) != rtp_TypeAudio % 128) { + LOGGER_WARNING("Invalid payload type!"); + rtp_free_msg(NULL, msg); + return -1; + } + + ACSession* ac = acp; + + pthread_mutex_lock(ac->queue_mutex); + int rc = jbuf_write(ac->j_buf, msg); + pthread_mutex_unlock(ac->queue_mutex); + + if (rc == -1) { + LOGGER_WARNING("Could not queue the message!"); + rtp_free_msg(NULL, msg); + return -1; + } + + return 0; +} int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels) { if (!ac) @@ -210,21 +240,7 @@ int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels); return 0; } -/* called from rtp */ -void ac_queue_message(void* acp, RTPMessage *msg) -{ - if (!acp || !msg) - return; - - ACSession* ac = acp; - pthread_mutex_lock(ac->queue_mutex); - int ret = jbuf_write(ac->j_buf, msg); - pthread_mutex_unlock(ac->queue_mutex); - - if (ret == -1) - rtp_free_msg(NULL, msg); -} /* JITTER BUFFER WORK */ diff --git a/toxav/audio.h b/toxav/audio.h index 62a28cdf5..2cb0d8f6e 100644 --- a/toxav/audio.h +++ b/toxav/audio.h @@ -29,6 +29,8 @@ #include "../toxcore/util.h" +struct RTPMessage_s; + typedef struct ACSession_s { /* encoding */ OpusEncoder *encoder; @@ -56,5 +58,6 @@ typedef struct ACSession_s { ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *cb, void *cb_data); void ac_kill(ACSession* ac); void ac_do(ACSession* ac); +int ac_queue_message(void *acp, struct RTPMessage_s *msg); int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels); #endif /* AUDIO_H */ \ No newline at end of file diff --git a/toxav/av_test.c b/toxav/av_test.c index 994f9cf6e..dce63184f 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -77,8 +77,8 @@ #define TEST_REJECT 0 #define TEST_CANCEL 0 #define TEST_MUTE_UNMUTE 0 -#define TEST_TRANSFER_A 1 -#define TEST_TRANSFER_V 0 +#define TEST_TRANSFER_A 0 +#define TEST_TRANSFER_V 1 typedef struct { @@ -329,6 +329,9 @@ void* iterate_toxav (void * data) toxav_iterate(data_cast->BobAV); int rc = MIN(toxav_iteration_interval(data_cast->AliceAV), toxav_iteration_interval(data_cast->BobAV)); + printf("\rIteration interval: %d ", rc); + fflush(stdout); + #if defined TEST_TRANSFER_V && TEST_TRANSFER_V == 1 cvWaitKey(rc); #else diff --git a/toxav/codec.c b/toxav/codec.c deleted file mode 100644 index 57e43c67f..000000000 --- a/toxav/codec.c +++ /dev/null @@ -1,686 +0,0 @@ -/** codec.c - * - * Copyright (C) 2013-2015 Tox project All Rights Reserved. - * - * This file is part of Tox. - * - * Tox is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Tox is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Tox. If not, see . - * - */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif /* HAVE_CONFIG_H */ - -#include "../toxcore/logger.h" -#include "../toxcore/util.h" - -#include -#include -#include -#include -#include -#include - -#include "msi.h" -#include "rtp.h" -#include "codec.h" - -#define DEFAULT_JBUF 3 - -/* Good quality encode. */ -#define MAX_DECODE_TIME_US 0 - -#define MAX_VIDEOFRAME_SIZE 0x40000 /* 256KiB */ -#define VIDEOFRAME_HEADER_SIZE 0x2 - -/* FIXME: Might not be enough? NOTE: I think it is enough */ -#define VIDEO_DECODE_BUFFER_SIZE 20 - -#define ARRAY(TYPE__) struct { uint16_t size; TYPE__ data[]; } - -typedef ARRAY(uint8_t) Payload; - -/* JITTER BUFFER WORK */ -typedef struct JitterBuffer_s { - RTPMessage **queue; - uint32_t size; - uint32_t capacity; - uint16_t bottom; - uint16_t top; -} JitterBuffer; - -static JitterBuffer *jbuf_new(uint32_t capacity) -{ - unsigned int size = 1; - - while (size <= (capacity * 4)) { - size *= 2; - } - - JitterBuffer *q; - - if ( !(q = calloc(sizeof(JitterBuffer), 1)) ) return NULL; - - if (!(q->queue = calloc(sizeof(RTPMessage *), size))) { - free(q); - return NULL; - } - - q->size = size; - q->capacity = capacity; - return q; -} - -static void jbuf_clear(JitterBuffer *q) -{ - for (; q->bottom != q->top; ++q->bottom) { - if (q->queue[q->bottom % q->size]) { - rtp_free_msg(NULL, q->queue[q->bottom % q->size]); - q->queue[q->bottom % q->size] = NULL; - } - } -} - -static void jbuf_free(JitterBuffer *q) -{ - if (!q) return; - - jbuf_clear(q); - free(q->queue); - free(q); -} - -static int jbuf_write(JitterBuffer *q, RTPMessage *m) -{ - uint16_t sequnum = m->header->sequnum; - - unsigned int num = sequnum % q->size; - - if ((uint32_t)(sequnum - q->bottom) > q->size) { - LOGGER_DEBUG("Clearing filled jitter buffer: %p", q); - - jbuf_clear(q); - q->bottom = sequnum - q->capacity; - q->queue[num] = m; - q->top = sequnum + 1; - return 0; - } - - if (q->queue[num]) - return -1; - - q->queue[num] = m; - - if ((sequnum - q->bottom) >= (q->top - q->bottom)) - q->top = sequnum + 1; - - return 0; -} - -/* success is set to 0 when there is nothing to dequeue, - * 1 when there's a good packet, - * 2 when there's a lost packet */ -static RTPMessage *jbuf_read(JitterBuffer *q, int32_t *success) -{ - if (q->top == q->bottom) { - *success = 0; - return NULL; - } - - unsigned int num = q->bottom % q->size; - - if (q->queue[num]) { - RTPMessage *ret = q->queue[num]; - q->queue[num] = NULL; - ++q->bottom; - *success = 1; - return ret; - } - - if ((uint32_t)(q->top - q->bottom) > q->capacity) { - ++q->bottom; - *success = 2; - return NULL; - } - - *success = 0; - return NULL; -} - -static int convert_bw_to_sampling_rate(int bw) -{ - switch(bw) - { - case OPUS_BANDWIDTH_NARROWBAND: return 8000; - case OPUS_BANDWIDTH_MEDIUMBAND: return 12000; - case OPUS_BANDWIDTH_WIDEBAND: return 16000; - case OPUS_BANDWIDTH_SUPERWIDEBAND: return 24000; - case OPUS_BANDWIDTH_FULLBAND: return 48000; - default: return -1; - } -} - -OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count) -{ - int status = OPUS_OK; - OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_AUDIO, &status); - - if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(status)); - return NULL; - } - - status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bitrate)); - - if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); - goto FAILURE; - } - - status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(10)); - - if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); - goto FAILURE; - } - - return rc; - -FAILURE: - opus_encoder_destroy(rc); - return NULL; -} - -bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate) -{ - assert(dest); - - vpx_codec_enc_cfg_t cfg; - int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); - - if (rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); - return false; - } - - rc = vpx_codec_enc_init_ver(dest, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, - VPX_ENCODER_ABI_VERSION); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); - return false; - } - - cfg.rc_target_bitrate = bitrate; - cfg.g_w = 800; - cfg.g_h = 600; - cfg.g_pass = VPX_RC_ONE_PASS; - cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; - cfg.g_lag_in_frames = 0; - cfg.kf_min_dist = 0; - cfg.kf_max_dist = 48; - cfg.kf_mode = VPX_KF_AUTO; - - rc = vpx_codec_control(dest, VP8E_SET_CPUUSED, 8); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - vpx_codec_destroy(dest); - } - - return true; -} - -bool reconfigure_audio_decoder(CSession* cs, int32_t sampling_rate, int8_t channels) -{ - if (sampling_rate != cs->last_decoding_sampling_rate || channels != cs->last_decoding_channel_count) { - if (current_time_monotonic() - cs->last_decoder_reconfiguration < 500) - return false; - - int status; - OpusDecoder* new_dec = opus_decoder_create(sampling_rate, channels, &status ); - if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio decoder(%d %d): %s", sampling_rate, channels, opus_strerror(status)); - return false; - } - - cs->last_decoding_sampling_rate = sampling_rate; - cs->last_decoding_channel_count = channels; - cs->last_decoder_reconfiguration = current_time_monotonic(); - - opus_decoder_destroy(cs->audio_decoder); - cs->audio_decoder = new_dec; - - LOGGER_DEBUG("Reconfigured audio decoder sr: %d cc: %d", sampling_rate, channels); - } - - return true; -} - -/* PUBLIC */ -void cs_do(CSession *cs) -{ - /* Codec session should always be protected by call mutex so no need to check for cs validity - */ - - if (!cs) - return; - - Payload *p; - int rc; - - int success = 0; - - LOGGED_LOCK(cs->queue_mutex); - - /********************* AUDIO *********************/ - if (cs->audio_decoder) { - RTPMessage *msg; - - /* The maximum for 120 ms 48 KHz audio */ - int16_t tmp[5760]; - - while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { - LOGGED_UNLOCK(cs->queue_mutex); - - if (success == 2) { - LOGGER_DEBUG("OPUS correction"); - rc = opus_decode(cs->audio_decoder, NULL, 0, tmp, - (cs->last_packet_sampling_rate * cs->last_packet_frame_duration / 1000) / - cs->last_packet_channel_count, 1); - } else { - /* Get values from packet and decode. */ - /* NOTE: This didn't work very well - rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); - if (rc != -1) { - cs->last_packet_sampling_rate = rc; - } else { - LOGGER_WARNING("Failed to load packet values!"); - rtp_free_msg(NULL, msg); - continue; - }*/ - - - /* Pick up sampling rate from packet */ - memcpy(&cs->last_packet_sampling_rate, msg->data, 4); - cs->last_packet_sampling_rate = ntohl(cs->last_packet_sampling_rate); - - cs->last_packet_channel_count = opus_packet_get_nb_channels(msg->data + 4); - - /* - * NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa, - * it didn't work quite well. - */ - if (!reconfigure_audio_decoder(cs, cs->last_packet_sampling_rate, cs->last_packet_channel_count)) { - LOGGER_WARNING("Failed to reconfigure decoder!"); - rtp_free_msg(NULL, msg); - continue; - } - - rc = opus_decode(cs->audio_decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0); - rtp_free_msg(NULL, msg); - } - - if (rc < 0) { - LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); - } else if (cs->acb.first) { - cs->last_packet_frame_duration = (rc * 1000) / cs->last_packet_sampling_rate * cs->last_packet_channel_count; - - cs->acb.first(cs->av, cs->friend_id, tmp, rc * cs->last_packet_channel_count, - cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second); - - } - - LOGGED_LOCK(cs->queue_mutex); - } - } - - /********************* VIDEO *********************/ - if (cs->vbuf_raw && !rb_empty(cs->vbuf_raw)) { - /* Decode video */ - rb_read(cs->vbuf_raw, (void**)&p); - - /* Leave space for (possibly) other thread to queue more data after we read it here */ - LOGGED_UNLOCK(cs->queue_mutex); - - rc = vpx_codec_decode(cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); - free(p); - - if (rc != VPX_CODEC_OK) { - LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc)); - } else { - vpx_codec_iter_t iter = NULL; - vpx_image_t *dest = vpx_codec_get_frame(cs->v_decoder, &iter); - - /* Play decoded images */ - for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) { - if (cs->vcb.first) - cs->vcb.first(cs->av, cs->friend_id, dest->d_w, dest->d_h, - (const uint8_t*)dest->planes[0], (const uint8_t*)dest->planes[1], (const uint8_t*)dest->planes[2], - dest->stride[0], dest->stride[1], dest->stride[2], cs->vcb.second); - - vpx_img_free(dest); - } - } - - return; - } - - LOGGED_UNLOCK(cs->queue_mutex); -} -CSession *cs_new(uint32_t peer_video_frame_piece_size) -{ - CSession *cs = calloc(sizeof(CSession), 1); - - if (!cs) { - LOGGER_WARNING("Allocation failed! Application might misbehave!"); - return NULL; - } - - if (create_recursive_mutex(cs->queue_mutex) != 0) { - LOGGER_WARNING("Failed to create recursive mutex!"); - free(cs); - return NULL; - } - - /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ - /* Create decoders and set up their values - */ - - /* - * AUDIO - */ - - int status; - cs->audio_decoder = opus_decoder_create(48000, 2, &status ); - - if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status)); - goto FAILURE; - } - - cs->last_decoding_channel_count = 2; - cs->last_decoding_sampling_rate = 48000; - cs->last_decoder_reconfiguration = 0; /* Make it possible to reconfigure straight away */ - - /* These need to be set in order to properly - * do error correction with opus */ - cs->last_packet_frame_duration = 120; - cs->last_packet_sampling_rate = 48000; - - if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) { - LOGGER_WARNING("Jitter buffer creaton failed!"); - opus_decoder_destroy(cs->audio_decoder); - goto FAILURE; - } - - /* - * VIDEO - */ - int rc = vpx_codec_dec_init_ver(cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, - NULL, 0, VPX_DECODER_ABI_VERSION); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); - goto AUDIO_DECODER_CLEANUP; - } - - if ( !(cs->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) ) { - vpx_codec_destroy(cs->v_decoder); - goto AUDIO_DECODER_CLEANUP; - } - - if ( !(cs->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE)) ) { - free(cs->frame_buf); - vpx_codec_destroy(cs->v_decoder); - goto AUDIO_DECODER_CLEANUP; - } - - if ( !(cs->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) ) - goto FAILURE; - - cs->linfts = current_time_monotonic(); - cs->lcfd = 60; - /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ - - /* Initialize encoders with default values */ - cs->audio_encoder = create_audio_encoder(48000, 48000, 2); - if (cs->audio_encoder == NULL) - goto VIDEO_DECODER_CLEANUP; - - cs->last_encoding_bitrate = 48000; - cs->last_encoding_sampling_rate = 48000; - cs->last_encoding_channel_count = 2; - - if (!create_video_encoder(cs->v_encoder, 500000)) { - opus_encoder_destroy(cs->audio_encoder); - goto VIDEO_DECODER_CLEANUP; - } - - cs->peer_video_frame_piece_size = peer_video_frame_piece_size; - - return cs; - -VIDEO_DECODER_CLEANUP: - rb_free(cs->vbuf_raw); - free(cs->frame_buf); - vpx_codec_destroy(cs->v_decoder); -AUDIO_DECODER_CLEANUP: - opus_decoder_destroy(cs->audio_decoder); - jbuf_free(cs->j_buf); -FAILURE: - pthread_mutex_destroy(cs->queue_mutex); - free(cs); - return NULL; -} -void cs_kill(CSession *cs) -{ - if (!cs) - return; - - /* NOTE: queue_message() will not be called since - * the callback is unregistered before cs_kill is called. - */ - - opus_encoder_destroy(cs->audio_encoder); - opus_decoder_destroy(cs->audio_decoder); - jbuf_free(cs->j_buf); - vpx_codec_destroy(cs->v_encoder); - vpx_codec_destroy(cs->v_decoder); - rb_free(cs->vbuf_raw); - free(cs->frame_buf); - free(cs->split_video_frame); - - pthread_mutex_destroy(cs->queue_mutex); - - LOGGER_DEBUG("Terminated codec state: %p", cs); - free(cs); -} -void cs_init_video_splitter_cycle(CSession* cs) -{ - cs->split_video_frame[0] = cs->frameid_out++; - cs->split_video_frame[1] = 0; -} -int cs_update_video_splitter_cycle(CSession *cs, const uint8_t *payload, uint16_t length) -{ - cs->processing_video_frame = payload; - cs->processing_video_frame_size = length; - - return ((length - 1) / VIDEOFRAME_PIECE_SIZE) + 1; -} -const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size) -{ - if (!cs || !size) return NULL; - - if (cs->processing_video_frame_size > VIDEOFRAME_PIECE_SIZE) { - memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE, - cs->processing_video_frame, - VIDEOFRAME_PIECE_SIZE); - - cs->processing_video_frame += VIDEOFRAME_PIECE_SIZE; - cs->processing_video_frame_size -= VIDEOFRAME_PIECE_SIZE; - - *size = VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE; - } else { - memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE, - cs->processing_video_frame, - cs->processing_video_frame_size); - - *size = cs->processing_video_frame_size + VIDEOFRAME_HEADER_SIZE; - } - - cs->split_video_frame[1]++; - - return cs->split_video_frame; -} -int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height) -{ - vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc; - if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) - return 0; /* Nothing changed */ - - cfg.rc_target_bitrate = bitrate; - cfg.g_w = width; - cfg.g_h = height; - - int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg); - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - return 0; -} -int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels) -{ - /* Values are checked in toxav.c */ - - if (cs->last_encoding_sampling_rate != sampling_rate || cs->last_encoding_channel_count != channels) { - OpusEncoder* new_encoder = create_audio_encoder(bitrate, sampling_rate, channels); - if (new_encoder == NULL) - return -1; - - opus_encoder_destroy(cs->audio_encoder); - cs->audio_encoder = new_encoder; - } else if (cs->last_encoding_bitrate == bitrate) - return 0; /* Nothing changed */ - else { - int status = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate)); - - if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); - return -1; - } - } - - cs->last_encoding_bitrate = bitrate; - cs->last_encoding_sampling_rate = sampling_rate; - cs->last_encoding_channel_count = channels; - - LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels); - return 0; -} -/* Called from RTP */ -void queue_message(RTPSession *session, RTPMessage *msg) -{ - CSession *cs = session->cs; - - if (!cs) - return; - - /* Audio */ - if (session->payload_type == rtp_TypeAudio % 128) { - LOGGED_LOCK(cs->queue_mutex); - int ret = jbuf_write(cs->j_buf, msg); - LOGGED_UNLOCK(cs->queue_mutex); - - if (ret == -1) { - rtp_free_msg(NULL, msg); - } - } - /* Video */ - else { - uint8_t *packet = msg->data; - uint32_t packet_size = msg->length; - - if (packet_size < VIDEOFRAME_HEADER_SIZE) - goto end; - - uint8_t diff = packet[0] - cs->frameid_in; - - if (diff != 0) { - if (diff < 225) { /* New frame */ - /* Flush last frames' data and get ready for this frame */ - Payload *p = malloc(sizeof(Payload) + cs->frame_size); - - if (p) { - LOGGED_LOCK(cs->queue_mutex); - - if (rb_full(cs->vbuf_raw)) { - LOGGER_DEBUG("Dropped video frame"); - Payload *tp; - rb_read(cs->vbuf_raw, (void**)&tp); - free(tp); - } else { - p->size = cs->frame_size; - memcpy(p->data, cs->frame_buf, cs->frame_size); - } - - /* Calculate time took for peer to send us this frame */ - uint32_t t_lcfd = current_time_monotonic() - cs->linfts; - cs->lcfd = t_lcfd > 100 ? cs->lcfd : t_lcfd; - cs->linfts = current_time_monotonic(); - - rb_write(cs->vbuf_raw, p); - LOGGED_UNLOCK(cs->queue_mutex); - } else { - LOGGER_WARNING("Allocation failed! Program might misbehave!"); - goto end; - } - - cs->frameid_in = packet[0]; - memset(cs->frame_buf, 0, cs->frame_size); - cs->frame_size = 0; - - } else { /* Old frame; drop */ - LOGGER_DEBUG("Old packet: %u", packet[0]); - goto end; - } - } - - uint8_t piece_number = packet[1]; - - uint32_t length_before_piece = ((piece_number - 1) * cs->peer_video_frame_piece_size); - uint32_t framebuf_new_length = length_before_piece + (packet_size - VIDEOFRAME_HEADER_SIZE); - - if (framebuf_new_length > MAX_VIDEOFRAME_SIZE) { - goto end; - } - - /* Otherwise it's part of the frame so just process */ - /* LOGGER_DEBUG("Video Packet: %u %u", packet[0], packet[1]); */ - - memcpy(cs->frame_buf + length_before_piece, - packet + VIDEOFRAME_HEADER_SIZE, - packet_size - VIDEOFRAME_HEADER_SIZE); - - if (framebuf_new_length > cs->frame_size) - cs->frame_size = framebuf_new_length; - -end: - rtp_free_msg(NULL, msg); - } -} diff --git a/toxav/codec.h b/toxav/codec.h deleted file mode 100644 index 497016eb3..000000000 --- a/toxav/codec.h +++ /dev/null @@ -1,125 +0,0 @@ -/** codec.h - * - * Copyright (C) 2013-2015 Tox project All Rights Reserved. - * - * This file is part of Tox. - * - * Tox is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Tox is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Tox. If not, see . - * - */ - -#ifndef CODEC_H -#define CODEC_H - -#include "toxav.h" -#include "rtp.h" - -#include "../toxcore/util.h" - -#include -#include -#include - -#include -#include -#include -#include -#include -#define VIDEO_CODEC_DECODER_INTERFACE (vpx_codec_vp8_dx()) -#define VIDEO_CODEC_ENCODER_INTERFACE (vpx_codec_vp8_cx()) - -/* Audio encoding/decoding */ -#include - -typedef struct CSession_s { - - /* VIDEO - * - * - */ - - /* video encoding */ - vpx_codec_ctx_t v_encoder[1]; - uint32_t frame_counter; - - /* video decoding */ - vpx_codec_ctx_t v_decoder[1]; - void *vbuf_raw; /* Un-decoded data */ - - /* Data handling */ - uint8_t *frame_buf; /* buffer for split video payloads */ - uint32_t frame_size; /* largest address written to in frame_buf for current input frame */ - uint8_t frameid_in, frameid_out; /* id of input and output video frame */ - uint64_t linfts; /* Last received frame time stamp */ - uint32_t lcfd; /* Last calculated frame duration for incoming video payload */ - - /* Limits */ - uint32_t peer_video_frame_piece_size; - - /* Splitting */ - uint8_t *split_video_frame; - const uint8_t *processing_video_frame; - uint16_t processing_video_frame_size; - - - - /* AUDIO - * - * - */ - - /* audio encoding */ - OpusEncoder *audio_encoder; - int32_t last_encoding_sampling_rate; - int32_t last_encoding_channel_count; - int32_t last_encoding_bitrate; - - /* audio decoding */ - OpusDecoder *audio_decoder; - int32_t last_packet_channel_count; - int32_t last_packet_sampling_rate; - int32_t last_packet_frame_duration; - int32_t last_decoding_sampling_rate; - int32_t last_decoding_channel_count; - uint64_t last_decoder_reconfiguration; - struct JitterBuffer_s *j_buf; - - - /* OTHER - * - * - */ - ToxAV *av; - int32_t friend_id; - - PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ - PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ - - pthread_mutex_t queue_mutex[1]; -} CSession; - - -void cs_do(CSession *cs); -/* Make sure to be called BEFORE corresponding rtp_new */ -CSession *cs_new(uint32_t peer_mvfpsz); -/* Make sure to be called AFTER corresponding rtp_kill */ -void cs_kill(CSession *cs); - -void cs_init_video_splitter_cycle(CSession *cs); -int cs_update_video_splitter_cycle(CSession* cs, const uint8_t* payload, uint16_t length); -const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size); - -int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height); -int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels); -#endif /* CODEC_H */ diff --git a/toxav/msi.c b/toxav/msi.c index b7926e074..0bd04c568 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -99,9 +99,9 @@ void handle_msi_packet ( Messenger *m, uint32_t friend_id, const uint8_t *data, */ void msi_register_callback ( MSISession *session, MSICallbackType callback, MSICallbackID id) { - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); session->callbacks[id] = callback; - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); } MSISession *msi_new ( Messenger *messenger ) { @@ -141,7 +141,7 @@ int msi_kill ( MSISession *session ) } m_callback_msi_packet((struct Messenger *) session->messenger, NULL, NULL); - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); if (session->calls) { MSIMessage msg; @@ -154,7 +154,7 @@ int msi_kill ( MSISession *session ) } } - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); pthread_mutex_destroy(session->mutex); LOGGER_DEBUG("Terminated session: %p", session); @@ -165,17 +165,17 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ { LOGGER_DEBUG("Session: %p Inviting friend: %u", session, friend_id); - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); if (get_call(session, friend_id) != NULL) { LOGGER_ERROR("Already in a call"); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return -1; } (*call) = new_call ( session, friend_id ); if ( *call == NULL ) { - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return -1; } @@ -195,7 +195,7 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ (*call)->state = msi_CallRequesting; LOGGER_DEBUG("Invite sent"); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return 0; } int msi_hangup ( MSICall* call ) @@ -203,7 +203,7 @@ int msi_hangup ( MSICall* call ) LOGGER_DEBUG("Session: %p Hanging up call with friend: %u", call->session, call->friend_id); MSISession* session = call->session; - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); MSIMessage msg; msg_init(&msg, requ_pop); @@ -211,7 +211,7 @@ int msi_hangup ( MSICall* call ) send_message ( session->messenger, call->friend_id, &msg ); kill_call(call); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return 0; } int msi_answer ( MSICall* call, uint8_t capabilities ) @@ -219,13 +219,13 @@ int msi_answer ( MSICall* call, uint8_t capabilities ) LOGGER_DEBUG("Session: %p Answering call from: %u", call->session, call->friend_id); MSISession* session = call->session; - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); if ( call->state != msi_CallRequested ) { /* Though sending in invalid state will not cause anything wierd * Its better to not do it like a maniac */ LOGGER_ERROR("Call is in invalid state!"); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return -1; } @@ -243,7 +243,7 @@ int msi_answer ( MSICall* call, uint8_t capabilities ) send_message ( session->messenger, call->friend_id, &msg ); call->state = msi_CallActive; - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return 0; } @@ -252,7 +252,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) LOGGER_DEBUG("Session: %p Trying to change capabilities to friend %u", call->session, call->friend_id); MSISession* session = call->session; - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); if ( call->state != msi_CallActive ) { /* Sending capabilities change can cause error on other side if @@ -263,7 +263,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) * like new. TODO: explain this better */ LOGGER_ERROR("Call is in invalid state!"); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return -1; } @@ -277,7 +277,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) send_message ( call->session->messenger, call->friend_id, &msg ); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return 0; } @@ -349,7 +349,7 @@ int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) case IDVFPSZ: CHECK_SIZE(it, size_constraint, 2); SET_UINT16(it, dest->vfpsz); - dest->vfpsz = ntohs(dest->vfpsz); + dest->vfpsz.value = ntohs(dest->vfpsz.value); if (dest->vfpsz.value > 1200) { LOGGER_ERROR("Invalid vfpsz param"); @@ -425,7 +425,7 @@ int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg ) } if (msg->vfpsz.exists) { - uint16_t nb_vfpsz = htons(msg->vfpsz); + uint16_t nb_vfpsz = htons(msg->vfpsz.value); it = msg_parse_header_out(IDVFPSZ, it, &nb_vfpsz, sizeof(nb_vfpsz), &size); } @@ -588,17 +588,17 @@ void on_peer_status(Messenger* m, uint32_t friend_id, uint8_t status, void* data case 0: { /* Friend is now offline */ LOGGER_DEBUG("Friend %d is now offline", friend_id); - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); MSICall* call = get_call(session, friend_id); if (call == NULL) { - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return; } invoke_callback(call, msi_OnPeerTimeout); /* Failure is ignored */ kill_call(call); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); } break; @@ -766,20 +766,20 @@ void handle_msi_packet ( Messenger* m, uint32_t friend_id, const uint8_t* data, LOGGER_DEBUG("Successfully parsed message"); } - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); MSICall *call = get_call(session, friend_id); if (call == NULL) { if (msg.request.value != requ_push) { send_error(m, friend_id, msi_EStrayMessage); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return; } call = new_call(session, friend_id); if (call == NULL) { send_error(m, friend_id, msi_ESystem); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return; } } @@ -789,5 +789,5 @@ void handle_msi_packet ( Messenger* m, uint32_t friend_id, const uint8_t* data, else handle_pop(call, &msg); /* always kills the call */ - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); } diff --git a/toxav/rtp.c b/toxav/rtp.c index 9ef41b357..9657da67c 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -68,12 +68,6 @@ typedef struct RTCPSession_s { } RTCPSession; - -/* These are defined externally */ -void ac_queue_message(void *acp, RTPMessage *msg); -void vc_queue_message(void *vcp, RTPMessage *msg); - - RTPHeader *parse_header_in ( const uint8_t *payload, int length ); RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length ); RTPMessage *msg_parse ( const uint8_t *data, int length ); @@ -100,7 +94,7 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ) retu->tstate = rtp_StateNormal; retu->m = messenger; - retu->dest = friend_num; + retu->friend_id = friend_num; if ( !(retu->csrc = calloc(1, sizeof(uint32_t))) ) { LOGGER_WARNING("Alloc failed! Program might misbehave!"); @@ -155,7 +149,7 @@ void rtp_do(RTPSession *session) return; if (current_time_monotonic() - session->rtcp_session->last_sent_report_ts >= RTCP_REPORT_INTERVAL_MS) { - send_rtcp_report(session->rtcp_session, session->m, session->dest); + send_rtcp_report(session->rtcp_session, session->m, session->friend_id); } if (rb_full(session->rtcp_session->pl_stats)) { @@ -202,15 +196,15 @@ int rtp_start_receiving(RTPSession* session) if (session == NULL) return -1; - if (m_callback_rtp_packet(session->m, session->dest, session->prefix, + if (m_callback_rtp_packet(session->m, session->friend_id, session->prefix, handle_rtp_packet, session) == -1) { LOGGER_WARNING("Failed to register rtp receive handler"); return -1; } - if (m_callback_rtp_packet(session->m, session->dest, session->rtcp_session->prefix, + if (m_callback_rtp_packet(session->m, session->friend_id, session->rtcp_session->prefix, handle_rtcp_packet, session->rtcp_session) == -1) { LOGGER_WARNING("Failed to register rtcp receive handler"); - m_callback_rtp_packet(session->m, session->dest, session->prefix, NULL, NULL); + m_callback_rtp_packet(session->m, session->friend_id, session->prefix, NULL, NULL); return -1; } @@ -221,8 +215,8 @@ int rtp_stop_receiving(RTPSession* session) if (session == NULL) return -1; - m_callback_rtp_packet(session->m, session->dest, session->prefix, NULL, NULL); - m_callback_rtp_packet(session->m, session->dest, session->rtcp_session->prefix, NULL, NULL); /* RTCP */ + m_callback_rtp_packet(session->m, session->friend_id, session->prefix, NULL, NULL); + m_callback_rtp_packet(session->m, session->friend_id, session->rtcp_session->prefix, NULL, NULL); /* RTCP */ return 0; } @@ -253,7 +247,7 @@ int rtp_send_msg ( RTPSession *session, const uint8_t *data, uint16_t length ) memcpy ( it, data, length ); - if ( -1 == send_custom_lossy_packet(session->m, session->dest, parsed, parsed_len) ) { + if ( -1 == send_custom_lossy_packet(session->m, session->friend_id, parsed, parsed_len) ) { LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno)); return -1; } @@ -546,7 +540,6 @@ void send_rtcp_report(RTCPSession* session, Messenger* m, uint32_t friendnumber) } int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) { - /* TODO on message callback */ RTPSession *session = object; RTPMessage *msg; @@ -578,20 +571,12 @@ int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data session->rtcp_session->last_received_packets ++; - /* Check if this session can handle the packet */ - if (session->payload_type != session->prefix % 128) { - LOGGER_WARNING("Friend %d sent invalid payload type!", session->dest); - rtp_free_msg(msg); - return -1; + if (session->mcb) + return session->mcb (session->cs, msg); + else { + rtp_free_msg(session, msg); + return 0; } - - /* Handle */ - if (session->payload_type == rtp_TypeAudio % 128) - ac_queue_message(session->cs, msg); - else /* It can only be video */ - vc_queue_message(session->cs, msg); - - return 0; } int handle_rtcp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) { diff --git a/toxav/rtp.h b/toxav/rtp.h index a28ae7bc4..3056b54ef 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -42,11 +42,13 @@ #define MAX_RTP_SIZE 65535 /** - * Payload type identifier. Also used as rtp callback prefix. + * Payload type identifier. Also used as rtp callback prefix. (Not dummies) */ enum { rtp_TypeAudio = 192, - rtp_TypeVideo + rtp_TypeVideo, + rtp_TypeDummyAudio, + rtp_TypeDummyVideo, }; typedef enum { @@ -79,7 +81,7 @@ typedef struct { /** * Standard rtp message. */ -typedef struct { +typedef struct RTPMessage_s { RTPHeader *header; RTPExtHeader *ext_header; @@ -91,34 +93,36 @@ typedef struct { * RTP control session. */ typedef struct { - uint8_t version; - uint8_t padding; - uint8_t extension; - uint8_t cc; - uint8_t marker; - uint8_t payload_type; - uint16_t sequnum; /* Sending sequence number */ - uint16_t rsequnum; /* Receiving sequence number */ - uint32_t rtimestamp; - uint32_t ssrc; - uint32_t *csrc; + uint8_t version; + uint8_t padding; + uint8_t extension; + uint8_t cc; + uint8_t marker; + uint8_t payload_type; + uint16_t sequnum; /* Sending sequence number */ + uint16_t rsequnum; /* Receiving sequence number */ + uint32_t rtimestamp; + uint32_t ssrc; + uint32_t *csrc; /* If some additional data must be sent via message * apply it here. Only by allocating this member you will be * automatically placing it within a message. */ - RTPExtHeader *ext_header; + RTPExtHeader *ext_header; /* Msg prefix for core to know when recving */ - uint8_t prefix; - - int dest; + uint8_t prefix; + Messenger *m; + int friend_id; + RTPTransmissionState tstate; struct RTCPSession_s *rtcp_session; - struct CSession_s *cs; - Messenger *m; + + + void *cs; + int (*mcb) (void*, RTPMessage* msg); - RTPTransmissionState tstate; } RTPSession; /** diff --git a/toxav/toxav.c b/toxav/toxav.c index 6f712af92..5cb614d4c 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -23,7 +23,8 @@ #include "config.h" #endif /* HAVE_CONFIG_H */ -#include "msi.h" /* Includes codec.h, rtp.h and toxav.h */ +#include "msi.h" +#include "rtp.h" #include "../toxcore/Messenger.h" #include "../toxcore/logger.h" @@ -35,20 +36,17 @@ #define MAX_ENCODE_TIME_US ((1000 / 24) * 1000) -enum { - audio_index, - video_index, -}; typedef struct ToxAVCall_s { ToxAV* av; - RTPSession *rtps[2]; /* Audio is first and video is second */ - CSession *cs; - pthread_mutex_t mutex_audio_sending[1]; - pthread_mutex_t mutex_video_sending[1]; - /* Only audio or video can be decoded at the time */ - pthread_mutex_t mutex_decoding[1]; + pthread_mutex_t mutex_audio[1]; + PAIR(RTPSession *, ACSession *) audio; + + pthread_mutex_t mutex_video[1]; + PAIR(RTPSession *, VCSession *) video; + + pthread_mutex_t mutex[1]; bool active; MSICall* msi_call; @@ -57,8 +55,8 @@ typedef struct ToxAVCall_s { uint32_t audio_bit_rate; /* Sending audio bitrate */ uint32_t video_bit_rate; /* Sending video bitrate */ - uint8_t last_self_capabilities; - uint8_t last_peer_capabilities; + /** Required for monitoring */ + uint8_t previous_self_capabilities; /** Quality control */ uint64_t time_audio_good; @@ -181,7 +179,7 @@ void toxav_kill(ToxAV* av) { if (av == NULL) return; - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); msi_kill(av->msi); @@ -194,7 +192,7 @@ void toxav_kill(ToxAV* av) } } - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); pthread_mutex_destroy(av->mutex); free(av); } @@ -212,9 +210,9 @@ uint32_t toxav_iteration_interval(const ToxAV* av) void toxav_iterate(ToxAV* av) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); if (av->calls == NULL) { - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); return; } @@ -224,30 +222,36 @@ void toxav_iterate(ToxAV* av) ToxAVCall* i = av->calls[av->calls_head]; for (; i; i = i->next) { if (i->active) { - LOGGED_LOCK(i->mutex_decoding); - LOGGED_UNLOCK(av->mutex); + pthread_mutex_lock(i->mutex); + pthread_mutex_unlock(av->mutex); + + rtp_do(i->audio.first); + ac_do(i->audio.second); + + rtp_do(i->video.first); + vc_do(i->video.second); - cs_do(i->cs); - rtp_do(i->rtps[audio_index]); - rtp_do(i->rtps[video_index]); qc_do(i); - if (i->last_self_capabilities & msi_CapRAudio) /* Receiving audio */ - rc = MIN(i->cs->last_packet_frame_duration, rc); - if (i->last_self_capabilities & msi_CapRVideo) /* Receiving video */ - rc = MIN(i->cs->lcfd, rc); /* TODO handle on/off */ + if (i->msi_call->self_capabilities & msi_CapRAudio && + i->msi_call->peer_capabilities & msi_CapSAudio) + rc = MIN(i->audio.second->last_packet_frame_duration, rc); + + if (i->msi_call->self_capabilities & msi_CapRVideo && + i->msi_call->peer_capabilities & msi_CapSVideo) + rc = MIN(i->video.second->lcfd, rc); uint32_t fid = i->friend_id; - LOGGED_UNLOCK(i->mutex_decoding); - LOGGED_LOCK(av->mutex); + pthread_mutex_unlock(i->mutex); + pthread_mutex_lock(av->mutex); /* In case this call is popped from container stop iteration */ if (call_get(av, fid) != i) break; } } - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa); av->dmsst += current_time_monotonic() - start; @@ -261,46 +265,46 @@ void toxav_iterate(ToxAV* av) bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); ToxAVCall* call = call_new(av, friend_number, error); if (call == NULL) { - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); return false; } call->audio_bit_rate = audio_bit_rate; call->video_bit_rate = video_bit_rate; - call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; + call->previous_self_capabilities = msi_CapRAudio | msi_CapRVideo; - call->last_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; - call->last_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; + call->previous_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; + call->previous_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; - if (msi_invite(av->msi, &call->msi_call, friend_number, call->last_self_capabilities) != 0) { + if (msi_invite(av->msi, &call->msi_call, friend_number, call->previous_self_capabilities) != 0) { call_remove(call); if (error) *error = TOXAV_ERR_CALL_MALLOC; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); return false; } call->msi_call->av_call = call; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); return true; } void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); av->ccb.first = function; av->ccb.second = user_data; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); } bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER* error) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); TOXAV_ERR_ANSWER rc = TOXAV_ERR_ANSWER_OK; if (m_friend_exists(av->m, friend_number) == 0) { @@ -329,17 +333,17 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui call->audio_bit_rate = audio_bit_rate; call->video_bit_rate = video_bit_rate; - call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; + call->previous_self_capabilities = msi_CapRAudio | msi_CapRVideo; - call->last_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; - call->last_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; + call->previous_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; + call->previous_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; - if (msi_answer(call->msi_call, call->last_self_capabilities) != 0) + if (msi_answer(call->msi_call, call->previous_self_capabilities) != 0) rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; /* the only reason for msi_answer to fail */ END: - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); if (error) *error = rc; @@ -349,15 +353,15 @@ END: void toxav_callback_call_state(ToxAV* av, toxav_call_state_cb* function, void* user_data) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); av->scb.first = function; av->scb.second = user_data; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); } bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL* error) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); TOXAV_ERR_CALL_CONTROL rc = TOXAV_ERR_CALL_CONTROL_OK; if (m_friend_exists(av->m, friend_number) == 0) { @@ -381,18 +385,18 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co /* Only act if paused and had media transfer active before */ if (call->msi_call->self_capabilities == 0 && - call->last_self_capabilities ) { + call->previous_self_capabilities ) { if (msi_change_capabilities(call->msi_call, - call->last_self_capabilities) == -1) { + call->previous_self_capabilities) == -1) { /* The only reason for this function to fail is invalid state * ( not active ) */ rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; goto END; } - rtp_start_receiving(call->rtps[audio_index]); - rtp_start_receiving(call->rtps[video_index]); + rtp_start_receiving(call->audio.first); + rtp_start_receiving(call->video.first); } } break; @@ -404,7 +408,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co /* Only act if not already paused */ if (call->msi_call->self_capabilities) { - call->last_self_capabilities = call->msi_call->self_capabilities; + call->previous_self_capabilities = call->msi_call->self_capabilities; if (msi_change_capabilities(call->msi_call, 0) == -1 ) { /* The only reason for this function to fail is invalid state @@ -413,8 +417,8 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co goto END; } - rtp_stop_receiving(call->rtps[audio_index]); - rtp_stop_receiving(call->rtps[video_index]); + rtp_stop_receiving(call->audio.first); + rtp_stop_receiving(call->video.first); } } break; @@ -442,7 +446,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co goto END; } - rtp_stop_receiving(call->rtps[audio_index]); + rtp_stop_receiving(call->audio.first); } else { /* This call was already muted so notify the friend that he can * start sending audio again @@ -455,7 +459,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co goto END; } - rtp_start_receiving(call->rtps[audio_index]); + rtp_start_receiving(call->audio.first); } } break; @@ -474,7 +478,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co goto END; } - rtp_stop_receiving(call->rtps[video_index]); + rtp_stop_receiving(call->video.first); } else { /* This call was already muted so notify the friend that he can * start sending video again @@ -487,13 +491,13 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co goto END; } - rtp_start_receiving(call->rtps[video_index]); + rtp_start_receiving(call->video.first); } } break; } END: - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); if (error) *error = rc; @@ -516,19 +520,19 @@ bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_ goto END; } - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; goto END; } /* Decoding mutex is locked because of quality control */ - LOGGED_LOCK(call->mutex_decoding); + pthread_mutex_lock(call->mutex); call->audio_bit_rate = audio_bit_rate; - LOGGED_UNLOCK(call->mutex_decoding); - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(av->mutex); END: if (error) @@ -552,19 +556,19 @@ bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_ goto END; } - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; goto END; } /* Decoding mutex is locked because of quality control */ - LOGGED_LOCK(call->mutex_decoding); + pthread_mutex_lock(call->mutex); call->video_bit_rate = video_bit_rate; - LOGGED_UNLOCK(call->mutex_decoding); - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(av->mutex); END: if (error) @@ -575,10 +579,10 @@ END: void toxav_callback_video_frame_request(ToxAV* av, toxav_video_frame_request_cb* function, void* user_data) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); av->rvcb.first = function; av->rvcb.second = user_data; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); } bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) @@ -591,25 +595,25 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u goto END; } - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; } - LOGGED_LOCK(call->mutex_video_sending); - LOGGED_UNLOCK(av->mutex); + pthread_mutex_lock(call->mutex_video); + pthread_mutex_unlock(av->mutex); if ( y == NULL || u == NULL || v == NULL ) { - LOGGED_UNLOCK(call->mutex_video_sending); + pthread_mutex_unlock(call->mutex_video); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } - if ( cs_reconfigure_video_encoder(call->cs, call->video_bit_rate, width, height) != 0 ) { - LOGGED_UNLOCK(call->mutex_video_sending); + if ( vc_reconfigure_encoder(call->video.second, call->video_bit_rate, width, height) != 0 ) { + pthread_mutex_unlock(call->mutex_video); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } @@ -626,29 +630,29 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u memcpy(img.planes[VPX_PLANE_U], u, (width/2) * (height/2)); memcpy(img.planes[VPX_PLANE_V], v, (width/2) * (height/2)); - int vrc = vpx_codec_encode(call->cs->v_encoder, &img, - call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); + int vrc = vpx_codec_encode(call->video.second->v_encoder, &img, + call->video.second->frame_counter, 1, 0, MAX_ENCODE_TIME_US); vpx_img_free(&img); if ( vrc != VPX_CODEC_OK) { - LOGGED_UNLOCK(call->mutex_video_sending); + pthread_mutex_unlock(call->mutex_video); LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } } - ++call->cs->frame_counter; + ++call->video.second->frame_counter; { /* Split and send */ vpx_codec_iter_t iter = NULL; const vpx_codec_cx_pkt_t *pkt; - cs_init_video_splitter_cycle(call->cs); + vc_init_video_splitter_cycle(call->video.second); - while ( (pkt = vpx_codec_get_cx_data(call->cs->v_encoder, &iter)) ) { + while ( (pkt = vpx_codec_get_cx_data(call->video.second->v_encoder, &iter)) ) { if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { - int parts = cs_update_video_splitter_cycle(call->cs, pkt->data.frame.buf, + int parts = vc_update_video_splitter_cycle(call->video.second, pkt->data.frame.buf, pkt->data.frame.sz); if (parts < 0) /* Should never happen though */ @@ -659,10 +663,10 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u int i; for (i = 0; i < parts; i++) { - iter = cs_iterate_split_video_frame(call->cs, &part_size); + iter = vc_iterate_split_video_frame(call->video.second, &part_size); - if (rtp_send_msg(call->rtps[video_index], iter, part_size) < 0) { - LOGGED_UNLOCK(call->mutex_video_sending); + if (rtp_send_msg(call->video.first, iter, part_size) < 0) { + pthread_mutex_unlock(call->mutex_video); LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); goto END; } @@ -671,7 +675,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u } } - LOGGED_UNLOCK(call->mutex_video_sending); + pthread_mutex_unlock(call->mutex_video); END: if (error) @@ -682,10 +686,10 @@ END: void toxav_callback_audio_frame_request(ToxAV* av, toxav_audio_frame_request_cb* function, void* user_data) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); av->racb.first = function; av->racb.second = user_data; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); } bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error) @@ -698,32 +702,32 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; } - LOGGED_LOCK(call->mutex_audio_sending); - LOGGED_UNLOCK(av->mutex); + pthread_mutex_lock(call->mutex_audio); + pthread_mutex_unlock(av->mutex); if ( pcm == NULL ) { - LOGGED_UNLOCK(call->mutex_audio_sending); + pthread_mutex_unlock(call->mutex_audio); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } if ( channels > 2 ) { - LOGGED_UNLOCK(call->mutex_audio_sending); + pthread_mutex_unlock(call->mutex_audio); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } { /* Encode and send */ - if (cs_reconfigure_audio_encoder(call->cs, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) { - LOGGED_UNLOCK(call->mutex_audio_sending); + if (ac_reconfigure_encoder(call->audio.second, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) { + pthread_mutex_unlock(call->mutex_audio); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } @@ -732,12 +736,12 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc sampling_rate = htonl(sampling_rate); memcpy(dest, &sampling_rate, sizeof(sampling_rate)); - int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, + int vrc = opus_encode(call->audio.second->encoder, pcm, sample_count, dest + sizeof(sampling_rate), sizeof(dest) - sizeof(sampling_rate)); if (vrc < 0) { LOGGER_WARNING("Failed to encode frame %s", opus_strerror(vrc)); - LOGGED_UNLOCK(call->mutex_audio_sending); + pthread_mutex_unlock(call->mutex_audio); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } @@ -745,13 +749,13 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc // LOGGER_DEBUG("Sending encoded audio frame size: %d; channels: %d; srate: %d", vrc, channels, // ntohl(sampling_rate)); - if (rtp_send_msg(call->rtps[audio_index], dest, vrc + sizeof(sampling_rate)) != 0) { + if (rtp_send_msg(call->audio.first, dest, vrc + sizeof(sampling_rate)) != 0) { LOGGER_WARNING("Failed to send audio packet"); rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; } } - LOGGED_UNLOCK(call->mutex_audio_sending); + pthread_mutex_unlock(call->mutex_audio); END: if (error) @@ -762,18 +766,18 @@ END: void toxav_callback_receive_video_frame(ToxAV* av, toxav_receive_video_frame_cb* function, void* user_data) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); av->vcb.first = function; av->vcb.second = user_data; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); } void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* function, void* user_data) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); av->acb.first = function; av->acb.second = user_data; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); } @@ -785,12 +789,12 @@ void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* int callback_invite(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - LOGGED_LOCK(toxav->mutex); + pthread_mutex_lock(toxav->mutex); ToxAVCall* av_call = call_new(toxav, call->friend_id, NULL); if (av_call == NULL) { LOGGER_WARNING("Failed to initialize call..."); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return -1; } @@ -801,72 +805,72 @@ int callback_invite(void* toxav_inst, MSICall* call) toxav->ccb.first(toxav, call->friend_id, call->peer_capabilities & msi_CapSAudio, call->peer_capabilities & msi_CapSVideo, toxav->ccb.second); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return 0; } int callback_start(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - LOGGED_LOCK(toxav->mutex); + pthread_mutex_lock(toxav->mutex); ToxAVCall* av_call = call_get(toxav, call->friend_id); if (av_call == NULL) { /* Should this ever happen? */ - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return -1; } if (!call_prepare_transmission(av_call)) { callback_error(toxav_inst, call); call_remove(av_call); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return -1; } invoke_call_state(toxav, call->friend_id, call->peer_capabilities); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return 0; } int callback_end(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - LOGGED_LOCK(toxav->mutex); + pthread_mutex_lock(toxav->mutex); invoke_call_state(toxav, call->friend_id, TOXAV_CALL_STATE_END); call_kill_transmission(call->av_call); call_remove(call->av_call); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return 0; } int callback_error(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - LOGGED_LOCK(toxav->mutex); + pthread_mutex_lock(toxav->mutex); invoke_call_state(toxav, call->friend_id, TOXAV_CALL_STATE_ERROR); call_kill_transmission(call->av_call); call_remove(call->av_call); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return 0; } int callback_capabilites(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - LOGGED_LOCK(toxav->mutex); + pthread_mutex_lock(toxav->mutex); invoke_call_state(toxav, call->friend_id, call->peer_capabilities); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return 0; } @@ -982,10 +986,8 @@ ToxAVCall* call_get(ToxAV* av, uint32_t friend_number) void qc_do(ToxAVCall* call) { - /* Please NOTE: The quality control is rather basic, - * advanced algorithms will be applied in the future - */ - switch(call->rtps[audio_index]->tstate) { + /* + switch(call->audio.first->tstate) { case rtp_StateBad: LOGGER_DEBUG("Suggesting lower bitrate for audio..."); call->time_audio_good = 0; @@ -1007,9 +1009,9 @@ void qc_do(ToxAVCall* call) case rtp_StateNormal: call->time_audio_good = 0; break; - } - - switch(call->rtps[video_index]->tstate) { + }*/ + /* + switch(call->video.first->tstate) { case rtp_StateBad: LOGGER_DEBUG("Suggesting lower bitrate for video..."); call->time_video_good = 0; @@ -1030,8 +1032,7 @@ void qc_do(ToxAVCall* call) case rtp_StateNormal: call->time_video_good = 0; break; - } - + }*/ } void call_remove(ToxAVCall* call) @@ -1086,61 +1087,50 @@ bool call_prepare_transmission(ToxAVCall* call) return true; } - if (create_recursive_mutex(call->mutex_audio_sending) != 0) + if (create_recursive_mutex(call->mutex_audio) != 0) return false; - if (create_recursive_mutex(call->mutex_video_sending) != 0) { + if (create_recursive_mutex(call->mutex_video) != 0) { goto AUDIO_SENDING_MUTEX_CLEANUP; } - if (create_recursive_mutex(call->mutex_decoding) != 0) { + if (create_recursive_mutex(call->mutex) != 0) { goto VIDEO_SENDING_MUTEX_CLEANUP; } - /* Creates both audio and video encoders and decoders with some default values. - * Make sure to reconfigure encoders dynamically when sending data - */ - call->cs = cs_new(call->msi_call->peer_vfpsz); - if ( !call->cs ) { - LOGGER_ERROR("Error while starting Codec State!\n"); - goto FAILURE; - } - - call->cs->av = av; - call->cs->friend_id = call->friend_id; - - memcpy(&call->cs->acb, &av->acb, sizeof(av->acb)); - memcpy(&call->cs->vcb, &av->vcb, sizeof(av->vcb)); - - { /* Prepare audio RTP */ - call->rtps[audio_index] = rtp_new(rtp_TypeAudio, av->m, call->friend_id); + { /* Prepare audio */ + call->audio.first = rtp_new(rtp_TypeAudio, av->m, call->friend_id); + call->audio.second = ac_new(av, call->friend_id, av->acb.first, av->acb.second); - if ( !call->rtps[audio_index] ) { - LOGGER_ERROR("Error while starting audio RTP session!\n"); + if ( !call->audio.first || !call->audio.second ) { + LOGGER_ERROR("Error while starting audio!\n"); goto FAILURE; } - call->rtps[audio_index]->cs = call->cs; + call->audio.first->cs = call->audio.second; + call->audio.first->mcb = ac_queue_message; - if (rtp_start_receiving(call->rtps[audio_index]) != 0) { + if (rtp_start_receiving(call->audio.first) != 0) { LOGGER_WARNING("Failed to enable audio receiving!"); goto FAILURE; } } - { /* Prepare video RTP */ - call->rtps[video_index] = rtp_new(rtp_TypeVideo, av->m, call->friend_id); + { /* Prepare video */ + call->video.first = rtp_new(rtp_TypeVideo, av->m, call->friend_id); + call->video.second = vc_new(av, call->friend_id, av->vcb.first, av->vcb.second, call->msi_call->peer_vfpsz); - if ( !call->rtps[video_index] ) { - LOGGER_ERROR("Error while starting video RTP session!\n"); + if ( !call->video.first || !call->video.second ) { + LOGGER_ERROR("Error while starting video!\n"); goto FAILURE; } - call->rtps[video_index]->cs = call->cs; + call->video.first->cs = call->video.second; + call->video.first->mcb = vc_queue_message; - if (rtp_start_receiving(call->rtps[video_index]) != 0) { - LOGGER_WARNING("Failed to enable audio receiving!"); + if (rtp_start_receiving(call->video.first) != 0) { + LOGGER_WARNING("Failed to enable video receiving!"); goto FAILURE; } } @@ -1149,17 +1139,19 @@ bool call_prepare_transmission(ToxAVCall* call) return true; FAILURE: - rtp_kill(call->rtps[audio_index]); - call->rtps[audio_index] = NULL; - rtp_kill(call->rtps[video_index]); - call->rtps[video_index] = NULL; - cs_kill(call->cs); - call->cs = NULL; - pthread_mutex_destroy(call->mutex_decoding); + rtp_kill(call->audio.first); + ac_kill(call->audio.second); + call->audio.first = NULL; + call->audio.second = NULL; + rtp_kill(call->video.first); + vc_kill(call->video.second); + call->video.first = NULL; + call->video.second = NULL; + pthread_mutex_destroy(call->mutex); VIDEO_SENDING_MUTEX_CLEANUP: - pthread_mutex_destroy(call->mutex_video_sending); + pthread_mutex_destroy(call->mutex_video); AUDIO_SENDING_MUTEX_CLEANUP: - pthread_mutex_destroy(call->mutex_audio_sending); + pthread_mutex_destroy(call->mutex_audio); return false; } @@ -1170,23 +1162,24 @@ void call_kill_transmission(ToxAVCall* call) call->active = 0; - LOGGED_LOCK(call->mutex_audio_sending); - LOGGED_UNLOCK(call->mutex_audio_sending); - LOGGED_LOCK(call->mutex_video_sending); - LOGGED_UNLOCK(call->mutex_video_sending); - LOGGED_LOCK(call->mutex_decoding); - LOGGED_UNLOCK(call->mutex_decoding); + pthread_mutex_lock(call->mutex_audio); + pthread_mutex_unlock(call->mutex_audio); + pthread_mutex_lock(call->mutex_video); + pthread_mutex_unlock(call->mutex_video); + pthread_mutex_lock(call->mutex); + pthread_mutex_unlock(call->mutex); + rtp_kill(call->audio.first); + ac_kill(call->audio.second); + call->audio.first = NULL; + call->audio.second = NULL; - rtp_kill(call->rtps[audio_index]); - call->rtps[audio_index] = NULL; - rtp_kill(call->rtps[video_index]); - call->rtps[video_index] = NULL; + rtp_kill(call->video.first); + vc_kill(call->video.second); + call->video.first = NULL; + call->video.second = NULL; - cs_kill(call->cs); - call->cs = NULL; - - pthread_mutex_destroy(call->mutex_audio_sending); - pthread_mutex_destroy(call->mutex_video_sending); - pthread_mutex_destroy(call->mutex_decoding); + pthread_mutex_destroy(call->mutex_audio); + pthread_mutex_destroy(call->mutex_video); + pthread_mutex_destroy(call->mutex); } diff --git a/toxav/video.c b/toxav/video.c index d51cfd4a4..039fc2a09 100644 --- a/toxav/video.c +++ b/toxav/video.c @@ -24,6 +24,7 @@ #include "video.h" #include "msi.h" +#include "rtp.h" #include "../toxcore/logger.h" #include "../toxcore/network.h" @@ -78,7 +79,9 @@ VCSession* vc_new(ToxAV* av, uint32_t friend_id, toxav_receive_video_frame_cb* c vc->linfts = current_time_monotonic(); vc->lcfd = 60; - + vc->vcb.first = cb; + vc->vcb.second = cb_data; + vc->friend_id = friend_id; vc->peer_video_frame_piece_size = mvfpsz; return vc; @@ -187,35 +190,25 @@ const uint8_t* vc_iterate_split_video_frame(VCSession* vc, uint16_t* size) return vc->split_video_frame; } -int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height) -{ - if (!vc) - return; - - vpx_codec_enc_cfg_t cfg = *vc->v_encoder[0].config.enc; - if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) - return 0; /* Nothing changed */ - - cfg.rc_target_bitrate = bitrate; - cfg.g_w = width; - cfg.g_h = height; - - int rc = vpx_codec_enc_config_set(vc->v_encoder, &cfg); - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - return 0; -} -/* Called from RTP */ -void vc_queue_message(void* vcp, RTPMessage *msg) +int vc_queue_message(void* vcp, struct RTPMessage_s *msg) { /* This function does the reconstruction of video packets. * See more info about video splitting in docs */ if (!vcp || !msg) - return; + return -1; + + if ((msg->header->marker_payloadt & 0x7f) == rtp_TypeDummyVideo % 128) { + LOGGER_WARNING("Got dummy!"); + rtp_free_msg(NULL, msg); + return 0; + } + + if ((msg->header->marker_payloadt & 0x7f) != rtp_TypeVideo % 128) { + LOGGER_WARNING("Invalid payload type!"); + rtp_free_msg(NULL, msg); + return -1; + } VCSession* vc = vcp; @@ -233,7 +226,7 @@ void vc_queue_message(void* vcp, RTPMessage *msg) Payload *p = malloc(sizeof(Payload) + vc->frame_size); if (p) { - LOGGED_LOCK(vc->queue_mutex); + pthread_mutex_lock(vc->queue_mutex); if (rb_full(vc->vbuf_raw)) { LOGGER_DEBUG("Dropped video frame"); @@ -251,7 +244,7 @@ void vc_queue_message(void* vcp, RTPMessage *msg) vc->linfts = current_time_monotonic(); rb_write(vc->vbuf_raw, p); - LOGGED_UNLOCK(vc->queue_mutex); + pthread_mutex_unlock(vc->queue_mutex); } else { LOGGER_WARNING("Allocation failed! Program might misbehave!"); goto end; @@ -288,7 +281,30 @@ void vc_queue_message(void* vcp, RTPMessage *msg) end: rtp_free_msg(NULL, msg); + return 0; } +int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height) +{ + if (!vc) + return; + + vpx_codec_enc_cfg_t cfg = *vc->v_encoder[0].config.enc; + if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) + return 0; /* Nothing changed */ + + cfg.rc_target_bitrate = bitrate; + cfg.g_w = width; + cfg.g_h = height; + + int rc = vpx_codec_enc_config_set(vc->v_encoder, &cfg); + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); + return -1; + } + + return 0; +} + bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate) diff --git a/toxav/video.h b/toxav/video.h index c1678ad20..ed264f367 100644 --- a/toxav/video.h +++ b/toxav/video.h @@ -36,6 +36,8 @@ #include "../toxcore/util.h" +struct RTPMessage_s; + typedef struct VCSession_s { /* encoding */ @@ -76,6 +78,7 @@ void vc_do(VCSession* vc); void vc_init_video_splitter_cycle(VCSession* vc); int vc_update_video_splitter_cycle(VCSession* vc, const uint8_t* payload, uint16_t length); const uint8_t *vc_iterate_split_video_frame(VCSession* vc, uint16_t *size); +int vc_queue_message(void *vcp, struct RTPMessage_s *msg); int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height); #endif /* VIDEO_H */ \ No newline at end of file From 144fc94d6987c8c6f74d8024af5a5c1738fe4678 Mon Sep 17 00:00:00 2001 From: mannol Date: Sun, 26 Apr 2015 00:31:03 +0200 Subject: [PATCH 34/97] Almost done --- toxav/audio.c | 94 +++++++++++------ toxav/audio.h | 7 ++ toxav/av_test.c | 8 +- toxav/rtp.c | 276 +++++++++++++++++++++++++----------------------- toxav/rtp.h | 14 ++- toxav/toxav.c | 268 +++++++++++++++++++++++++++------------------- toxav/toxav.h | 78 +++++++------- toxav/video.c | 32 +++--- toxav/video.h | 4 +- 9 files changed, 439 insertions(+), 342 deletions(-) diff --git a/toxav/audio.c b/toxav/audio.c index dc85452a8..2f068c857 100644 --- a/toxav/audio.c +++ b/toxav/audio.c @@ -33,10 +33,11 @@ static int jbuf_write(struct JitterBuffer *q, RTPMessage *m); static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success); OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count); +bool reconfigure_audio_encoder(OpusEncoder** e, int32_t new_br, int32_t new_sr, uint8_t new_ch, + int32_t *old_br, int32_t *old_sr, int32_t *old_ch); bool reconfigure_audio_decoder(ACSession* ac, int32_t sampling_rate, int8_t channels); - ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *cb, void *cb_data) { ACSession *ac = calloc(sizeof(ACSession), 1); @@ -71,10 +72,20 @@ ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *c if (ac->encoder == NULL) goto DECODER_CLEANUP; + ac->test_encoder = create_audio_encoder(48000, 48000, 2); + if (ac->test_encoder == NULL) { + opus_encoder_destroy(ac->encoder); + goto DECODER_CLEANUP; + } + ac->last_encoding_bitrate = 48000; ac->last_encoding_sampling_rate = 48000; ac->last_encoding_channel_count = 2; + ac->last_test_encoding_bitrate = 48000; + ac->last_test_encoding_sampling_rate = 48000; + ac->last_test_encoding_channel_count = 2; + ac->last_decoding_channel_count = 2; ac->last_decoding_sampling_rate = 48000; ac->last_decoder_reconfiguration = 0; /* Make it possible to reconfigure straight away */ @@ -141,7 +152,7 @@ void ac_do(ACSession* ac) cs->last_packet_sampling_rate = rc; } else { LOGGER_WARNING("Failed to load packet values!"); - rtp_free_msg(NULL, msg); + rtp_free_msg(msg); continue; }*/ @@ -157,12 +168,12 @@ void ac_do(ACSession* ac) */ if (!reconfigure_audio_decoder(ac, ac->last_packet_sampling_rate, ac->last_packet_channel_count)) { LOGGER_WARNING("Failed to reconfigure decoder!"); - rtp_free_msg(NULL, msg); + rtp_free_msg(msg); continue; } rc = opus_decode(ac->decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0); - rtp_free_msg(NULL, msg); + rtp_free_msg(msg); } if (rc < 0) { @@ -183,15 +194,15 @@ int ac_queue_message(void* acp, struct RTPMessage_s *msg) if (!acp || !msg) return -1; - if ((msg->header->marker_payloadt & 0x7f) == rtp_TypeDummyAudio % 128) { + if ((msg->header->marker_payloadt & 0x7f) == (rtp_TypeAudio + 2) % 128) { LOGGER_WARNING("Got dummy!"); - rtp_free_msg(NULL, msg); + rtp_free_msg(msg); return 0; } if ((msg->header->marker_payloadt & 0x7f) != rtp_TypeAudio % 128) { LOGGER_WARNING("Invalid payload type!"); - rtp_free_msg(NULL, msg); + rtp_free_msg(msg); return -1; } @@ -203,7 +214,7 @@ int ac_queue_message(void* acp, struct RTPMessage_s *msg) if (rc == -1) { LOGGER_WARNING("Could not queue the message!"); - rtp_free_msg(NULL, msg); + rtp_free_msg(msg); return -1; } @@ -211,35 +222,22 @@ int ac_queue_message(void* acp, struct RTPMessage_s *msg) } int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels) { - if (!ac) - return; - - /* Values are checked in toxav.c */ - if (ac->last_encoding_sampling_rate != sampling_rate || ac->last_encoding_channel_count != channels) { - OpusEncoder* new_encoder = create_audio_encoder(bitrate, sampling_rate, channels); - if (new_encoder == NULL) - return -1; - - opus_encoder_destroy(ac->encoder); - ac->encoder = new_encoder; - } else if (ac->last_encoding_bitrate == bitrate) - return 0; /* Nothing changed */ - else { - int status = opus_encoder_ctl(ac->encoder, OPUS_SET_BITRATE(bitrate)); - - if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); - return -1; - } - } - - ac->last_encoding_bitrate = bitrate; - ac->last_encoding_sampling_rate = sampling_rate; - ac->last_encoding_channel_count = channels; + if (!ac || !reconfigure_audio_encoder(&ac->encoder, bitrate, sampling_rate, channels, + &ac->last_encoding_bitrate, &ac->last_encoding_sampling_rate, &ac->last_encoding_channel_count)) + return -1; LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels); return 0; } +int ac_reconfigure_test_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels) +{ + if (!ac || !reconfigure_audio_encoder(&ac->test_encoder, bitrate, sampling_rate, channels, + &ac->last_encoding_bitrate, &ac->last_encoding_sampling_rate, &ac->last_encoding_channel_count)) + return -1; + + LOGGER_DEBUG ("Reconfigured test audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels); + return 0; +} @@ -277,7 +275,7 @@ static void jbuf_clear(struct JitterBuffer *q) { for (; q->bottom != q->top; ++q->bottom) { if (q->queue[q->bottom % q->size]) { - rtp_free_msg(NULL, q->queue[q->bottom % q->size]); + rtp_free_msg(q->queue[q->bottom % q->size]); q->queue[q->bottom % q->size] = NULL; } } @@ -372,6 +370,34 @@ FAILURE: opus_encoder_destroy(rc); return NULL; } +bool reconfigure_audio_encoder(OpusEncoder** e, int32_t new_br, int32_t new_sr, uint8_t new_ch, + int32_t* old_br, int32_t* old_sr, int32_t* old_ch) +{ + /* Values are checked in toxav.c */ + if (*old_sr != new_sr || *old_ch != new_ch) { + OpusEncoder* new_encoder = create_audio_encoder(new_br, new_sr, new_ch); + if (new_encoder == NULL) + return false; + + opus_encoder_destroy(*e); + *e = new_encoder; + } else if (*old_br == new_br) + return true; /* Nothing changed */ + else { + int status = opus_encoder_ctl(*e, OPUS_SET_BITRATE(new_br)); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); + return false; + } + } + + *old_br = new_br; + *old_sr = new_sr; + *old_ch = new_ch; + + return true; +} bool reconfigure_audio_decoder(ACSession* ac, int32_t sampling_rate, int8_t channels) { if (sampling_rate != ac->last_decoding_sampling_rate || channels != ac->last_decoding_channel_count) { diff --git a/toxav/audio.h b/toxav/audio.h index 2cb0d8f6e..a36396f15 100644 --- a/toxav/audio.h +++ b/toxav/audio.h @@ -38,6 +38,12 @@ typedef struct ACSession_s { int32_t last_encoding_channel_count; int32_t last_encoding_bitrate; + /* Testing encoder for dynamic bitrate streaming */ + OpusEncoder *test_encoder; + int32_t last_test_encoding_sampling_rate; + int32_t last_test_encoding_channel_count; + int32_t last_test_encoding_bitrate; + /* decoding */ OpusDecoder *decoder; int32_t last_packet_channel_count; @@ -60,4 +66,5 @@ void ac_kill(ACSession* ac); void ac_do(ACSession* ac); int ac_queue_message(void *acp, struct RTPMessage_s *msg); int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels); +int ac_reconfigure_test_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels); #endif /* AUDIO_H */ \ No newline at end of file diff --git a/toxav/av_test.c b/toxav/av_test.c index dce63184f..11de0138d 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -186,7 +186,7 @@ void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, unsigned long int i, j; for (i = 0; i < height; ++i) { for (j = 0; j < width; ++j) { - uint8_t *point = (void*)img_data + 3 * ((i * width) + j); + uint8_t *point = (uint8_t*) img_data + 3 * ((i * width) + j); int yx = y[(i * ystride) + j]; int ux = u[((i / 2) * ustride) + (j / 2)]; int vx = v[((i / 2) * vstride) + (j / 2)]; @@ -226,7 +226,7 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, size_t length, void *userdata) { if (length == 7 && memcmp("gentoo", data, 7) == 0) { - assert(tox_friend_add_norequest(m, public_key, NULL) != ~0); + assert(tox_friend_add_norequest(m, public_key, NULL) != (uint32_t) ~0); } } @@ -262,7 +262,7 @@ void initialize_tox(Tox** bootstrap, ToxAV** AliceAV, CallControl* AliceCC, ToxA tox_self_get_address(Alice, address); - assert(tox_friend_add(Bob, address, (uint8_t *)"gentoo", 7, NULL) != ~0); + assert(tox_friend_add(Bob, address, (uint8_t *)"gentoo", 7, NULL) != (uint32_t) ~0); uint8_t off = 1; @@ -563,7 +563,7 @@ int main (int argc, char** argv) } \ } \ \ - iterate(bootstrap, AliceAV, BobAV); \ + iterate_tox(bootstrap, AliceAV, BobAV); \ } \ printf("Success!\n");\ } while(0) diff --git a/toxav/rtp.c b/toxav/rtp.c index 9657da67c..a48eba20f 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -29,6 +29,7 @@ #include "rtp.h" #include +#include #define size_32 4 #define RTCP_REPORT_INTERVAL_MS 500 @@ -37,8 +38,8 @@ #define ADD_FLAG_PADDING(_h, _v) do { if ( _v > 0 ) _v = 1; ( _h->flags ) &= 0xDF; ( _h->flags ) |= ( ( ( _v ) << 5 ) & 0x20 ); } while(0) #define ADD_FLAG_EXTENSION(_h, _v) do { if ( _v > 0 ) _v = 1; ( _h->flags ) &= 0xEF;( _h->flags ) |= ( ( ( _v ) << 4 ) & 0x10 ); } while(0) #define ADD_FLAG_CSRCC(_h, _v) do { ( _h->flags ) &= 0xF0; ( _h->flags ) |= ( ( _v ) & 0x0F ); } while(0) -#define ADD_SETTING_MARKER(_h, _v) do { if ( _v > 1 ) _v = 1; ( _h->marker_payloadt ) &= 0x7F; ( _h->marker_payloadt ) |= ( ( ( _v ) << 7 ) /*& 0x80 */ ); } while(0) -#define ADD_SETTING_PAYLOAD(_h, _v) do { if ( _v > 127 ) _v = 127; ( _h->marker_payloadt ) &= 0x80; ( _h->marker_payloadt ) |= ( ( _v ) /* & 0x7F */ ); } while(0) +#define ADD_SETTING_MARKER(_h, _v) do { ( _h->marker_payloadt ) &= 0x7F; ( _h->marker_payloadt ) |= ( ( ( _v ) << 7 ) /*& 0x80 */ ); } while(0) +#define ADD_SETTING_PAYLOAD(_h, _v) do { ( _h->marker_payloadt ) &= 0x80; ( _h->marker_payloadt ) |= ( ( _v ) /* & 0x7F */ ); } while(0) #define GET_FLAG_VERSION(_h) (( _h->flags & 0xd0 ) >> 6) #define GET_FLAG_PADDING(_h) (( _h->flags & 0x20 ) >> 5) @@ -70,17 +71,19 @@ typedef struct RTCPSession_s { RTPHeader *parse_header_in ( const uint8_t *payload, int length ); RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length ); -RTPMessage *msg_parse ( const uint8_t *data, int length ); uint8_t *parse_header_out ( const RTPHeader* header, uint8_t* payload ); uint8_t *parse_ext_header_out ( const RTPExtHeader* header, uint8_t* payload ); -void build_header ( RTPSession* session, RTPHeader* header ); -void send_rtcp_report ( RTCPSession* session, Messenger* m, uint32_t friendnumber ); int handle_rtp_packet ( Messenger *m, uint32_t friendnumber, const uint8_t *data, uint16_t length, void *object ); int handle_rtcp_packet ( Messenger *m, uint32_t friendnumber, const uint8_t *data, uint16_t length, void *object ); +void send_rtcp_report ( RTCPSession* session, Messenger* m, uint32_t friendnumber ); -RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ) +RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num, void* cs, int (*mcb) (void*, RTPMessage*) ) { + assert(mcb); + assert(cs); + assert(messenger); + RTPSession *retu = calloc(1, sizeof(RTPSession)); if ( !retu ) { @@ -107,6 +110,8 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ) /* Also set payload type as prefix */ retu->prefix = payload_type; + retu->cs = cs; + retu->mcb = mcb; /* Initialize rtcp session */ if (!(retu->rtcp_session = calloc(1, sizeof(RTCPSession)))) { @@ -120,6 +125,14 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ) retu->rtcp_session->pl_stats = rb_new(4); retu->rtcp_session->rtp_session = retu; + if (-1 == rtp_start_receiving(retu)) { + LOGGER_WARNING("Failed to start rtp receiving mode"); + free(retu->rtcp_session); + free(retu->csrc); + free(retu); + return NULL; + } + return retu; } void rtp_kill ( RTPSession *session ) @@ -156,11 +169,12 @@ void rtp_do(RTPSession *session) RTCPReport* reports[4]; int i = 0; - for (; rb_read(session->rtcp_session->pl_stats, (void**) reports + i); i++); + for (; i < 4; i++) + rb_read(session->rtcp_session->pl_stats, (void**) reports + i); /* Check for timed out reports (> 6 sec) */ uint64_t now = current_time_monotonic(); - for (i = 0; i < 4 && now - reports[i]->timestamp < 6000; i ++); + for (i = 0; i < 4 && (now - reports[i]->timestamp) < 6000; i ++); for (; i < 4; i ++) { rb_write(session->rtcp_session->pl_stats, reports[i]); reports[i] = NULL; @@ -168,18 +182,18 @@ void rtp_do(RTPSession *session) if (!rb_empty(session->rtcp_session->pl_stats)) { for (i = 0; reports[i] != NULL; i ++) free(reports[i]); - return; /* As some reports are timed out, we need more... */ + return; /* As some reports are timed out, we need more */ } /* We have 4 on-time reports so we can proceed */ uint32_t quality = 100; for (i = 0; i < 4; i++) { - uint32_t idx = reports[i]->received_packets * 100 / reports[i]->expected_packets; - quality = MIN(quality, idx); + uint32_t current = reports[i]->received_packets * 100 / reports[i]->expected_packets; + quality = MIN(quality, current); free(reports[i]); } - if (quality <= 70) { + if (quality <= 90) { session->tstate = rtp_StateBad; LOGGER_WARNING("Stream quality: BAD"); } else if (quality >= 99) { @@ -220,7 +234,7 @@ int rtp_stop_receiving(RTPSession* session) return 0; } -int rtp_send_msg ( RTPSession *session, const uint8_t *data, uint16_t length ) +int rtp_send_data ( RTPSession *session, const uint8_t *data, uint16_t length, bool dummy ) { if ( !session ) { LOGGER_WARNING("No session!"); @@ -231,12 +245,31 @@ int rtp_send_msg ( RTPSession *session, const uint8_t *data, uint16_t length ) uint8_t *it; RTPHeader header[1]; - build_header(session, header); + ADD_FLAG_VERSION ( header, session->version ); + ADD_FLAG_PADDING ( header, session->padding ); + ADD_FLAG_EXTENSION ( header, session->extension ); + ADD_FLAG_CSRCC ( header, session->cc ); + ADD_SETTING_MARKER ( header, session->marker ); + + if (dummy) + ADD_SETTING_PAYLOAD ( header, (session->payload_type + 2) % 128 ); + else + ADD_SETTING_PAYLOAD ( header, session->payload_type ); + header->sequnum = session->sequnum; + header->timestamp = current_time_monotonic(); + header->ssrc = session->ssrc; + + int i; + for ( i = 0; i < session->cc; i++ ) + header->csrc[i] = session->csrc[i]; + + header->length = 12 /* Minimum header len */ + ( session->cc * size_32 ); + uint32_t parsed_len = length + header->length + 1; + assert(parsed_len + (session->ext_header ? session->ext_header->length * size_32 : 0) > MAX_RTP_SIZE ); parsed[0] = session->prefix; - it = parse_header_out ( header, parsed + 1 ); if ( session->ext_header ) { @@ -244,8 +277,7 @@ int rtp_send_msg ( RTPSession *session, const uint8_t *data, uint16_t length ) it = parse_ext_header_out ( session->ext_header, it ); } - memcpy ( it, data, length ); - + memcpy(it, data, length); if ( -1 == send_custom_lossy_packet(session->m, session->friend_id, parsed, parsed_len) ) { LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno)); @@ -256,18 +288,11 @@ int rtp_send_msg ( RTPSession *session, const uint8_t *data, uint16_t length ) session->sequnum = session->sequnum >= MAX_SEQU_NUM ? 0 : session->sequnum + 1; return 0; } -void rtp_free_msg ( RTPSession *session, RTPMessage *msg ) +void rtp_free_msg ( RTPMessage *msg ) { - if ( !session ) { - if ( msg->ext_header ) { - free ( msg->ext_header->table ); - free ( msg->ext_header ); - } - } else { - if ( msg->ext_header && session->ext_header != msg->ext_header ) { - free ( msg->ext_header->table ); - free ( msg->ext_header ); - } + if ( msg->ext_header ) { + free ( msg->ext_header->table ); + free ( msg->ext_header ); } free ( msg->header ); @@ -389,50 +414,6 @@ RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length ) return retu; } -RTPMessage *msg_parse ( const uint8_t *data, int length ) -{ - /* TODO: data dynamic, [0] - * TODO: dummy payload type - * TODO: parse header before allocating message - */ - RTPMessage *retu = calloc(1, sizeof (RTPMessage)); - - retu->header = parse_header_in ( data, length ); /* It allocates memory and all */ - - if ( !retu->header ) { - LOGGER_WARNING("Header failed to extract!"); - free(retu); - return NULL; - } - - uint16_t from_pos = retu->header->length; - retu->length = length - from_pos; - - if ( GET_FLAG_EXTENSION ( retu->header ) ) { - retu->ext_header = parse_ext_header_in ( data + from_pos, length ); - - if ( retu->ext_header ) { - retu->length -= ( 4 /* Minimum ext header len */ + retu->ext_header->length * size_32 ); - from_pos += ( 4 /* Minimum ext header len */ + retu->ext_header->length * size_32 ); - } else { /* Error */ - LOGGER_WARNING("Ext Header failed to extract!"); - rtp_free_msg(NULL, retu); - return NULL; - } - } else { - retu->ext_header = NULL; - } - - if ( length - from_pos <= MAX_RTP_SIZE ) - memcpy ( retu->data, data + from_pos, length - from_pos ); - else { - LOGGER_WARNING("Invalid length!"); - rtp_free_msg(NULL, retu); - return NULL; - } - - return retu; -} uint8_t *parse_header_out ( const RTPHeader *header, uint8_t *payload ) { uint8_t cc = GET_FLAG_CSRCC ( header ); @@ -495,88 +476,95 @@ uint8_t *parse_ext_header_out ( const RTPExtHeader *header, uint8_t *payload ) return it + 4; } -void build_header ( RTPSession *session, RTPHeader *header ) -{ - ADD_FLAG_VERSION ( header, session->version ); - ADD_FLAG_PADDING ( header, session->padding ); - ADD_FLAG_EXTENSION ( header, session->extension ); - ADD_FLAG_CSRCC ( header, session->cc ); - ADD_SETTING_MARKER ( header, session->marker ); - ADD_SETTING_PAYLOAD ( header, session->payload_type ); - - header->sequnum = session->sequnum; - header->timestamp = current_time_monotonic(); /* milliseconds */ - header->ssrc = session->ssrc; - - int i; - for ( i = 0; i < session->cc; i++ ) - header->csrc[i] = session->csrc[i]; - - header->length = 12 /* Minimum header len */ + ( session->cc * size_32 ); -} -void send_rtcp_report(RTCPSession* session, Messenger* m, uint32_t friendnumber) -{ - if (session->last_expected_packets == 0) - return; - - uint8_t parsed[9]; - parsed[0] = session->prefix; - - uint32_t received_packets = htonl(session->last_received_packets); - uint32_t expected_packets = htonl(session->last_expected_packets); - - memcpy(parsed + 1, &received_packets, 4); - memcpy(parsed + 5, &expected_packets, 4); - - if (-1 == send_custom_lossy_packet(m, friendnumber, parsed, sizeof(parsed))) - LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", sizeof(parsed), strerror(errno)); - else { - LOGGER_DEBUG("Sent rtcp report: ex: %d rc: %d", session->last_expected_packets, session->last_received_packets); - - session->last_received_packets = 0; - session->last_expected_packets = 0; - session->last_sent_report_ts = current_time_monotonic(); - } -} int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) { RTPSession *session = object; - RTPMessage *msg; - if ( !session || length < 13 ) { /* 12 is the minimum length for rtp + desc. byte */ + if ( !session || length < 13 || length > MAX_RTP_SIZE ) { LOGGER_WARNING("No session or invalid length of received buffer!"); return -1; } + + RTPHeader* header = parse_header_in ( data, length ); - msg = msg_parse ( data + 1, length - 1 ); - - if ( !msg ) { - LOGGER_WARNING("Could not parse message!"); + if ( !header ) { + LOGGER_WARNING("Could not parse message: Header failed to extract!"); return -1; } + + RTPExtHeader* ext_header = NULL; + + uint16_t from_pos = header->length; + uint16_t msg_length = length - from_pos; + if ( GET_FLAG_EXTENSION ( header ) ) { + ext_header = parse_ext_header_in ( data + from_pos, length ); + + if ( ext_header ) { + msg_length -= ( 4 /* Minimum ext header len */ + ext_header->length * size_32 ); + from_pos += ( 4 /* Minimum ext header len */ + ext_header->length * size_32 ); + } else { /* Error */ + LOGGER_WARNING("Could not parse message: Ext Header failed to extract!"); + free(header); + return -1; + } + } + + if (msg_length > MAX_RTP_SIZE) { + LOGGER_WARNING("Could not parse message: Invalid length!"); + free(header); + free(ext_header); + return -1; + } + /* Check if message came in late */ - if ( msg->header->sequnum > session->rsequnum || msg->header->timestamp > session->rtimestamp ) { + if ( header->sequnum > session->rsequnum || header->timestamp > session->rtimestamp ) { /* Not late */ - if (msg->header->sequnum > session->rsequnum) - session->rtcp_session->last_expected_packets += msg->header->sequnum - session->rsequnum; - else if (msg->header->sequnum < session->rsequnum) - session->rtcp_session->last_expected_packets += (msg->header->sequnum + 65535) - session->rsequnum; + if (header->sequnum > session->rsequnum) + session->rtcp_session->last_expected_packets += header->sequnum - session->rsequnum; + else if (header->sequnum < session->rsequnum) + session->rtcp_session->last_expected_packets += (header->sequnum + 65535) - session->rsequnum; else /* Usual case when transmission starts */ session->rtcp_session->last_expected_packets ++; - session->rsequnum = msg->header->sequnum; - session->rtimestamp = msg->header->timestamp; + session->rsequnum = header->sequnum; + session->rtimestamp = header->timestamp; } session->rtcp_session->last_received_packets ++; - if (session->mcb) - return session->mcb (session->cs, msg); - else { - rtp_free_msg(session, msg); + /* Check if the message is dummy. We don't keep dummy messages */ + if (GET_SETTING_PAYLOAD(header) == (session->payload_type + 2) % 128) { + LOGGER_DEBUG("Received dummy rtp message"); + free(header); + free(ext_header); return 0; } + + /* Otherwise we will store the message if we have an appropriate handler */ + if (!session->mcb) { + LOGGER_DEBUG("No handler for the message of %d payload", GET_SETTING_PAYLOAD(header)); + free(header); + free(ext_header); + return 0; + } + + RTPMessage *msg = calloc(1, sizeof (RTPMessage) + msg_length); + + if ( !msg ) { + LOGGER_WARNING("Could not parse message: Allocation failed!"); + free(header); + free(ext_header); + return -1; + } + + msg->header = header; + msg->ext_header = ext_header; + msg->length = msg_length; + + memcpy ( msg->data, data + from_pos, msg_length ); + + return session->mcb (session->cs, msg); } int handle_rtcp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) { @@ -605,4 +593,28 @@ int handle_rtcp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* dat LOGGER_DEBUG("Got rtcp report: ex: %d rc: %d", report->expected_packets, report->received_packets); return 0; +} +void send_rtcp_report(RTCPSession* session, Messenger* m, uint32_t friendnumber) +{ + if (session->last_expected_packets == 0) + return; + + uint8_t parsed[9]; + parsed[0] = session->prefix; + + uint32_t received_packets = htonl(session->last_received_packets); + uint32_t expected_packets = htonl(session->last_expected_packets); + + memcpy(parsed + 1, &received_packets, 4); + memcpy(parsed + 5, &expected_packets, 4); + + if (-1 == send_custom_lossy_packet(m, friendnumber, parsed, sizeof(parsed))) + LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", sizeof(parsed), strerror(errno)); + else { + LOGGER_DEBUG("Sent rtcp report: ex: %d rc: %d", session->last_expected_packets, session->last_received_packets); + + session->last_received_packets = 0; + session->last_expected_packets = 0; + session->last_sent_report_ts = current_time_monotonic(); + } } \ No newline at end of file diff --git a/toxav/rtp.h b/toxav/rtp.h index 3056b54ef..fe07c4f6a 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -39,7 +39,7 @@ } while(0) #define MAX_SEQU_NUM 65535 -#define MAX_RTP_SIZE 65535 +#define MAX_RTP_SIZE 1500 /** * Payload type identifier. Also used as rtp callback prefix. (Not dummies) @@ -47,8 +47,6 @@ enum { rtp_TypeAudio = 192, rtp_TypeVideo, - rtp_TypeDummyAudio, - rtp_TypeDummyVideo, }; typedef enum { @@ -85,8 +83,8 @@ typedef struct RTPMessage_s { RTPHeader *header; RTPExtHeader *ext_header; - uint8_t data[MAX_RTP_SIZE]; uint32_t length; + uint8_t data[]; } RTPMessage; /** @@ -128,7 +126,7 @@ typedef struct { /** * Must be called before calling any other rtp function. */ -RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ); +RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num, void* cs, int (*mcb) (void*, RTPMessage*) ); /** * Terminate the session. @@ -141,7 +139,7 @@ void rtp_kill ( RTPSession* session ); void rtp_do(RTPSession *session); /** - * By default rtp is not in receiving state + * By default rtp is in receiving state */ int rtp_start_receiving (RTPSession *session); @@ -153,12 +151,12 @@ int rtp_stop_receiving (RTPSession *session); /** * Sends msg to RTPSession::dest */ -int rtp_send_msg ( RTPSession* session, const uint8_t* data, uint16_t length ); +int rtp_send_data ( RTPSession* session, const uint8_t* data, uint16_t length, bool dummy ); /** * Dealloc msg. */ -void rtp_free_msg ( RTPSession *session, RTPMessage *msg ); +void rtp_free_msg ( RTPMessage *msg ); #endif /* RTP_H */ diff --git a/toxav/toxav.c b/toxav/toxav.c index 5cb614d4c..500ca9a8e 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -36,6 +36,14 @@ #define MAX_ENCODE_TIME_US ((1000 / 24) * 1000) +typedef struct ToxAvBitrateAdapter_s { + bool active; + uint32_t cycle_sent; + uint32_t cycle_ratio; + uint32_t cycle_repeat; + uint64_t start_time; + uint32_t bit_rate; +} ToxAvBitrateAdapter; typedef struct ToxAVCall_s { ToxAV* av; @@ -55,7 +63,10 @@ typedef struct ToxAVCall_s { uint32_t audio_bit_rate; /* Sending audio bitrate */ uint32_t video_bit_rate; /* Sending video bitrate */ - /** Required for monitoring */ + ToxAvBitrateAdapter aba; + ToxAvBitrateAdapter vba; + + /** Required for monitoring changes in states */ uint8_t previous_self_capabilities; /** Quality control */ @@ -82,8 +93,6 @@ struct toxAV { PAIR(toxav_call_state_cb *, void *) scb; /* Call state callback */ PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ - PAIR(toxav_video_frame_request_cb *, void *) rvcb; /* Video request callback */ - PAIR(toxav_audio_frame_request_cb *, void *) racb; /* Audio request callback */ /** Decode time measures */ int32_t dmssc; /** Measure count */ @@ -103,14 +112,15 @@ int callback_capabilites(void* toxav_inst, MSICall* call); bool audio_bitrate_invalid(uint32_t bitrate); bool video_bitrate_invalid(uint32_t bitrate); void invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state); +void qc_do(ToxAVCall* call); ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); ToxAVCall* call_get(ToxAV* av, uint32_t friend_number); -void qc_do(ToxAVCall* call); void call_remove(ToxAVCall* call); bool call_prepare_transmission(ToxAVCall* call); void call_kill_transmission(ToxAVCall* call); - - +void ba_set(ToxAvBitrateAdapter* ba, uint32_t bit_rate); +bool ba_shoud_send_dummy(ToxAvBitrateAdapter* ba); +void ba_update_sent_regular(ToxAvBitrateAdapter* ba); ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) { @@ -300,7 +310,7 @@ void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data) av->ccb.first = function; av->ccb.second = user_data; pthread_mutex_unlock(av->mutex); -} +}/** Required for monitoring */ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER* error) { @@ -505,7 +515,7 @@ END: return rc == TOXAV_ERR_CALL_CONTROL_OK; } -bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE* error) +bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, bool force, TOXAV_ERR_BIT_RATE* error) { TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK; ToxAVCall* call; @@ -528,9 +538,17 @@ bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_ goto END; } - /* Decoding mutex is locked because of quality control */ + if (call->audio_bit_rate == audio_bit_rate && call->aba.active && call->aba.bit_rate == audio_bit_rate) { + pthread_mutex_unlock(av->mutex); + goto END; + } + pthread_mutex_lock(call->mutex); - call->audio_bit_rate = audio_bit_rate; + if (force) { + call->audio_bit_rate = audio_bit_rate; + } else + ba_set(&call->aba, audio_bit_rate); + pthread_mutex_unlock(call->mutex); pthread_mutex_unlock(av->mutex); @@ -541,7 +559,7 @@ END: return rc == TOXAV_ERR_BIT_RATE_OK; } -bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE* error) +bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, bool force, TOXAV_ERR_BIT_RATE* error) { TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK; ToxAVCall* call; @@ -564,9 +582,17 @@ bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_ goto END; } - /* Decoding mutex is locked because of quality control */ + if (call->video_bit_rate == video_bit_rate && call->vba.active && call->vba.bit_rate == video_bit_rate) { + pthread_mutex_unlock(av->mutex); + goto END; + } + pthread_mutex_lock(call->mutex); - call->video_bit_rate = video_bit_rate; + if (force) { + call->video_bit_rate = video_bit_rate; + } else { + ba_set(&call->vba, video_bit_rate); + } pthread_mutex_unlock(call->mutex); pthread_mutex_unlock(av->mutex); @@ -577,14 +603,6 @@ END: return rc == TOXAV_ERR_BIT_RATE_OK; } -void toxav_callback_video_frame_request(ToxAV* av, toxav_video_frame_request_cb* function, void* user_data) -{ - pthread_mutex_lock(av->mutex); - av->rvcb.first = function; - av->rvcb.second = user_data; - pthread_mutex_unlock(av->mutex); -} - bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) { TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; @@ -630,7 +648,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u memcpy(img.planes[VPX_PLANE_U], u, (width/2) * (height/2)); memcpy(img.planes[VPX_PLANE_V], v, (width/2) * (height/2)); - int vrc = vpx_codec_encode(call->video.second->v_encoder, &img, + int vrc = vpx_codec_encode(call->video.second->encoder, &img, call->video.second->frame_counter, 1, 0, MAX_ENCODE_TIME_US); vpx_img_free(&img); @@ -650,7 +668,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u vc_init_video_splitter_cycle(call->video.second); - while ( (pkt = vpx_codec_get_cx_data(call->video.second->v_encoder, &iter)) ) { + while ( (pkt = vpx_codec_get_cx_data(call->video.second->encoder, &iter)) ) { if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { int parts = vc_update_video_splitter_cycle(call->video.second, pkt->data.frame.buf, pkt->data.frame.sz); @@ -665,7 +683,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u for (i = 0; i < parts; i++) { iter = vc_iterate_split_video_frame(call->video.second, &part_size); - if (rtp_send_msg(call->video.first, iter, part_size) < 0) { + if (rtp_send_data(call->video.first, iter, part_size, false) < 0) { pthread_mutex_unlock(call->mutex_video); LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); goto END; @@ -684,14 +702,6 @@ END: return rc == TOXAV_ERR_SEND_FRAME_OK; } -void toxav_callback_audio_frame_request(ToxAV* av, toxav_audio_frame_request_cb* function, void* user_data) -{ - pthread_mutex_lock(av->mutex); - av->racb.first = function; - av->racb.second = user_data; - pthread_mutex_unlock(av->mutex); -} - bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error) { TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; @@ -746,13 +756,41 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } -// LOGGER_DEBUG("Sending encoded audio frame size: %d; channels: %d; srate: %d", vrc, channels, -// ntohl(sampling_rate)); - - if (rtp_send_msg(call->audio.first, dest, vrc + sizeof(sampling_rate)) != 0) { + if (rtp_send_data(call->audio.first, dest, vrc + sizeof(sampling_rate), false) != 0) { LOGGER_WARNING("Failed to send audio packet"); rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; } + + + /* For bitrate measurement; send dummy packet */ + if (ba_shoud_send_dummy(&call->aba)) { + sampling_rate = ntohl(sampling_rate); + if (ac_reconfigure_test_encoder(call->audio.second, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) { + /* FIXME should the bitrate changing fail here? */ + pthread_mutex_unlock(call->mutex_audio); + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + + sampling_rate = htonl(sampling_rate); + memcpy(dest, &sampling_rate, sizeof(sampling_rate)); + vrc = opus_encode(call->audio.second->test_encoder, pcm, sample_count, + dest + sizeof(sampling_rate), sizeof(dest) - sizeof(sampling_rate)); + + if (vrc < 0) { + LOGGER_WARNING("Failed to encode frame %s", opus_strerror(vrc)); + pthread_mutex_unlock(call->mutex_audio); + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + + if (rtp_send_data(call->audio.first, dest, vrc + sizeof(sampling_rate), true) != 0) { + LOGGER_WARNING("Failed to send audio packet"); + rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; + } + } + + ba_update_sent_regular(&call->aba); } pthread_mutex_unlock(call->mutex_audio); @@ -894,6 +932,57 @@ void invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state) av->scb.first(av, friend_number, state, av->scb.second); } +void qc_do(ToxAVCall* call) +{ + /* + switch(call->audio.first->tstate) { + case rtp_StateBad: + LOGGER_DEBUG("Suggesting lower bitrate for audio..."); + call->time_audio_good = 0; + call->last_bad_audio_bit_rate = call->audio_bit_rate; + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_DECREASE_AUDIO_BITRATE); + break; + case rtp_StateGood: + if (call->time_audio_good == 0) + call->time_audio_good = current_time_monotonic(); + else if (current_time_monotonic() - call->time_audio_good >= 30000 && + 64 > call->audio_bit_rate) + if (call->last_bad_audio_bit_rate > call->audio_bit_rate) { + if (current_time_monotonic() - call->time_audio_good >= 45000) + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE); + call->last_bad_audio_bit_rate = 0; + } else + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE); + break; + case rtp_StateNormal: + call->time_audio_good = 0; + break; + }*/ + /* + switch(call->video.first->tstate) { + case rtp_StateBad: + LOGGER_DEBUG("Suggesting lower bitrate for video..."); + call->time_video_good = 0; + call->last_bad_video_bit_rate = call->video_bit_rate; + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_DECREASE_VIDEO_BITRATE); + break; + case rtp_StateGood: + if (call->time_video_good == 0) + call->time_video_good = current_time_monotonic(); + else if (current_time_monotonic() - call->time_video_good >= 30000) + if (call->last_bad_video_bit_rate > call->video_bit_rate) { + if (current_time_monotonic() - call->time_video_good >= 45000) + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE); + call->last_bad_video_bit_rate = 0; + } else + invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE); + break; + case rtp_StateNormal: + call->time_video_good = 0; + break; + }*/ +} + ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) { /* Assumes mutex locked */ @@ -984,57 +1073,6 @@ ToxAVCall* call_get(ToxAV* av, uint32_t friend_number) return av->calls[friend_number]; } -void qc_do(ToxAVCall* call) -{ - /* - switch(call->audio.first->tstate) { - case rtp_StateBad: - LOGGER_DEBUG("Suggesting lower bitrate for audio..."); - call->time_audio_good = 0; - call->last_bad_audio_bit_rate = call->audio_bit_rate; - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_DECREASE_AUDIO_BITRATE); - break; - case rtp_StateGood: - if (call->time_audio_good == 0) - call->time_audio_good = current_time_monotonic(); - else if (current_time_monotonic() - call->time_audio_good >= 30000 && - 64 > call->audio_bit_rate) - if (call->last_bad_audio_bit_rate > call->audio_bit_rate) { - if (current_time_monotonic() - call->time_audio_good >= 45000) - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE); - call->last_bad_audio_bit_rate = 0; - } else - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE); - break; - case rtp_StateNormal: - call->time_audio_good = 0; - break; - }*/ - /* - switch(call->video.first->tstate) { - case rtp_StateBad: - LOGGER_DEBUG("Suggesting lower bitrate for video..."); - call->time_video_good = 0; - call->last_bad_video_bit_rate = call->video_bit_rate; - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_DECREASE_VIDEO_BITRATE); - break; - case rtp_StateGood: - if (call->time_video_good == 0) - call->time_video_good = current_time_monotonic(); - else if (current_time_monotonic() - call->time_video_good >= 30000) - if (call->last_bad_video_bit_rate > call->video_bit_rate) { - if (current_time_monotonic() - call->time_video_good >= 45000) - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE); - call->last_bad_video_bit_rate = 0; - } else - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE); - break; - case rtp_StateNormal: - call->time_video_good = 0; - break; - }*/ -} - void call_remove(ToxAVCall* call) { if (call == NULL) @@ -1098,41 +1136,24 @@ bool call_prepare_transmission(ToxAVCall* call) goto VIDEO_SENDING_MUTEX_CLEANUP; } - { /* Prepare audio */ - call->audio.first = rtp_new(rtp_TypeAudio, av->m, call->friend_id); call->audio.second = ac_new(av, call->friend_id, av->acb.first, av->acb.second); + call->audio.first = rtp_new(rtp_TypeAudio, av->m, call->friend_id, call->audio.second, ac_queue_message); if ( !call->audio.first || !call->audio.second ) { LOGGER_ERROR("Error while starting audio!\n"); goto FAILURE; } - - call->audio.first->cs = call->audio.second; - call->audio.first->mcb = ac_queue_message; - - if (rtp_start_receiving(call->audio.first) != 0) { - LOGGER_WARNING("Failed to enable audio receiving!"); - goto FAILURE; - } } { /* Prepare video */ - call->video.first = rtp_new(rtp_TypeVideo, av->m, call->friend_id); call->video.second = vc_new(av, call->friend_id, av->vcb.first, av->vcb.second, call->msi_call->peer_vfpsz); + call->video.first = rtp_new(rtp_TypeVideo, av->m, call->friend_id, call->video.second, vc_queue_message); if ( !call->video.first || !call->video.second ) { LOGGER_ERROR("Error while starting video!\n"); goto FAILURE; } - - call->video.first->cs = call->video.second; - call->video.first->mcb = vc_queue_message; - - if (rtp_start_receiving(call->video.first) != 0) { - LOGGER_WARNING("Failed to enable video receiving!"); - goto FAILURE; - } } call->active = 1; @@ -1183,3 +1204,38 @@ void call_kill_transmission(ToxAVCall* call) pthread_mutex_destroy(call->mutex_video); pthread_mutex_destroy(call->mutex); } + +void ba_set(ToxAvBitrateAdapter* ba, uint32_t bit_rate) +{ + ba->bit_rate = bit_rate; + ba->start_time = current_time_monotonic(); + ba->cycle_sent = 0; + ba->cycle_repeat = 3; + ba->cycle_ratio = 4; + ba->active = true; +} + +bool ba_shoud_send_dummy(ToxAvBitrateAdapter* ba) +{ + if (!ba->active || ba->cycle_ratio == 0) + return false; + + if (ba->cycle_sent % ba->cycle_ratio == 0) + return true; +} + +void ba_update_sent_regular(ToxAvBitrateAdapter* ba) +{ + if (!ba->active || ba->cycle_ratio == 0) { + ba->active = false; + return; + } + + if (ba->cycle_sent % ba->cycle_ratio == 0) { + ba->cycle_sent = 0; + + if (--ba->cycle_repeat == 0) + --ba->cycle_ratio; + } else + ++ba->cycle_sent; +} diff --git a/toxav/toxav.h b/toxav/toxav.h index bf8756ebb..f2c3b2b3c 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -216,22 +216,6 @@ typedef enum TOXAV_CALL_STATE { * transitions can occur for the call. */ TOXAV_CALL_STATE_END = 16, - /** - * AV core suggests you to lower bitrate for audio. - */ - TOXAV_CALL_STATE_DECREASE_AUDIO_BITRATE = 32, - /** - * AV core suggests you to lower bitrate for video. - */ - TOXAV_CALL_STATE_DECREASE_VIDEO_BITRATE = 64, - /** - * AV core suggests you to increase bitrate for audio. - */ - TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE = 128, - /** - * AV core suggests you to increase bitrate for video. - */ - TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE = 256, /** * Set by the AV core if an error occurred on the remote end. */ @@ -348,6 +332,24 @@ typedef enum TOXAV_ERR_BIT_RATE { */ TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL } TOXAV_ERR_BIT_RATE; +/** + * The function type for the `audio_bitrate_control` callback. + * + * @param friend_number The friend number of the friend for which to set the + * audio bit rate. + * @param good Is the stream good enough to keep the said bitrate. Upon failed + * non forceful bit rate setup this will be set to false and 'bit_rate' + * will be set to the bit rate that failed, otherwise 'good' will be set to + * true with 'bit_rate' set to new bit rate. If the stream becomes bad, + * the 'good' wil be set to false with 'bit_rate' set to the current bit rate. + * This callback will never be called when the stream is good. + * @param bit_rate The bit rate in Kb/sec. + */ +typedef void toxav_audio_bitrate_control_cb(ToxAV *av, uint32_t friend_number, bool good, uint32_t bit_rate, void *user_data); +/** + * Set the callback for the `audio_bitrate_control` event. Pass NULL to unset. + */ +void toxav_callback_audio_bitrate_control(ToxAV *av, toxav_audio_bitrate_control_cb *function, void *user_data); /** * Set the audio bit rate to be used in subsequent audio frames. * @@ -358,7 +360,25 @@ typedef enum TOXAV_ERR_BIT_RATE { * * @see toxav_call for the valid bit rates. */ -bool toxav_set_audio_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE *error); +bool toxav_set_audio_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, bool force, TOXAV_ERR_BIT_RATE *error); +/** + * The function type for the `video_bitrate_control` callback. + * + * @param friend_number The friend number of the friend for which to set the + * video bit rate. + * @param good Is the stream good enough to keep the said bitrate. Upon failed + * non forceful bit rate setup this will be set to false and 'bit_rate' + * will be set to the bit rate that failed, otherwise 'good' will be set to + * true with 'bit_rate' set to new bit rate. If the stream becomes bad, + * the 'good' wil be set to false with 'bit_rate' set to the current bit rate. + * This callback will never be called when the stream is good. + * @param bit_rate The bit rate in Kb/sec. + */ +typedef void toxav_video_bitrate_control_cb(ToxAV *av, uint32_t friend_number, bool good, uint32_t bit_rate, void *user_data); +/** + * Set the callback for the `video_bitrate_control` event. Pass NULL to unset. + */ +void toxav_callback_video_bitrate_control(ToxAV *av, toxav_video_bitrate_control_cb *function, void *user_data); /** * Set the video bit rate to be used in subsequent video frames. * @@ -369,7 +389,7 @@ bool toxav_set_audio_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t audio_ * * @see toxav_call for the valid bit rates. */ -bool toxav_set_video_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE *error); +bool toxav_set_video_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t video_bit_rate, bool force, TOXAV_ERR_BIT_RATE *error); /******************************************************************************* * * :: A/V sending @@ -408,17 +428,6 @@ typedef enum TOXAV_ERR_SEND_FRAME { */ TOXAV_ERR_SEND_FRAME_RTP_FAILED } TOXAV_ERR_SEND_FRAME; -/** - * The function type for the `video_frame_request` callback. - * - * @param friend_number The friend number of the friend for which the next video - * frame should be sent. - */ -typedef void toxav_video_frame_request_cb(ToxAV *av, uint32_t friend_number, void *user_data); -/** - * Set the callback for the `video_frame_request` event. Pass NULL to unset. - */ -void toxav_callback_video_frame_request(ToxAV *av, toxav_video_frame_request_cb *function, void *user_data); /** * Send a video frame to a friend. * @@ -440,17 +449,6 @@ bool toxav_send_video_frame(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, uint8_t const *y, uint8_t const *u, uint8_t const *v, TOXAV_ERR_SEND_FRAME *error); -/** - * The function type for the `audio_frame_request` callback. - * - * @param friend_number The friend number of the friend for which the next audio - * frame should be sent. - */ -typedef void toxav_audio_frame_request_cb(ToxAV *av, uint32_t friend_number, void *user_data); -/** - * Set the callback for the `audio_frame_request` event. Pass NULL to unset. - */ -void toxav_callback_audio_frame_request(ToxAV *av, toxav_audio_frame_request_cb *function, void *user_data); /** * Send an audio frame to a friend. * diff --git a/toxav/video.c b/toxav/video.c index 039fc2a09..cdc3c0ae2 100644 --- a/toxav/video.c +++ b/toxav/video.c @@ -65,15 +65,15 @@ VCSession* vc_new(ToxAV* av, uint32_t friend_id, toxav_receive_video_frame_cb* c if ( !(vc->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE)) ) goto BASE_CLEANUP; - int rc = vpx_codec_dec_init_ver(vc->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, + int rc = vpx_codec_dec_init_ver(vc->decoder, VIDEO_CODEC_DECODER_INTERFACE, NULL, 0, VPX_DECODER_ABI_VERSION); if ( rc != VPX_CODEC_OK) { LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); goto BASE_CLEANUP; } - if (!create_video_encoder(vc->v_encoder, 500000)) { - vpx_codec_destroy(vc->v_decoder); + if (!create_video_encoder(vc->encoder, 500000)) { + vpx_codec_destroy(vc->decoder); goto BASE_CLEANUP; } @@ -99,8 +99,8 @@ void vc_kill(VCSession* vc) if (!vc) return; - vpx_codec_destroy(vc->v_encoder); - vpx_codec_destroy(vc->v_decoder); + vpx_codec_destroy(vc->encoder); + vpx_codec_destroy(vc->decoder); rb_free(vc->vbuf_raw); free(vc->split_video_frame); free(vc->frame_buf); @@ -122,17 +122,17 @@ void vc_do(VCSession* vc) if (rb_read(vc->vbuf_raw, (void**)&p)) { pthread_mutex_unlock(vc->queue_mutex); - rc = vpx_codec_decode(vc->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); + rc = vpx_codec_decode(vc->decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); free(p); if (rc != VPX_CODEC_OK) { LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc)); } else { vpx_codec_iter_t iter = NULL; - vpx_image_t *dest = vpx_codec_get_frame(vc->v_decoder, &iter); + vpx_image_t *dest = vpx_codec_get_frame(vc->decoder, &iter); /* Play decoded images */ - for (; dest; dest = vpx_codec_get_frame(vc->v_decoder, &iter)) { + for (; dest; dest = vpx_codec_get_frame(vc->decoder, &iter)) { if (vc->vcb.first) vc->vcb.first(vc->av, vc->friend_id, dest->d_w, dest->d_h, (const uint8_t*)dest->planes[0], (const uint8_t*)dest->planes[1], (const uint8_t*)dest->planes[2], @@ -157,7 +157,7 @@ void vc_init_video_splitter_cycle(VCSession* vc) int vc_update_video_splitter_cycle(VCSession* vc, const uint8_t* payload, uint16_t length) { if (!vc) - return; + return 0; vc->processing_video_frame = payload; vc->processing_video_frame_size = length; @@ -198,15 +198,15 @@ int vc_queue_message(void* vcp, struct RTPMessage_s *msg) if (!vcp || !msg) return -1; - if ((msg->header->marker_payloadt & 0x7f) == rtp_TypeDummyVideo % 128) { + if ((msg->header->marker_payloadt & 0x7f) == (rtp_TypeVideo + 2) % 128) { LOGGER_WARNING("Got dummy!"); - rtp_free_msg(NULL, msg); + rtp_free_msg(msg); return 0; } if ((msg->header->marker_payloadt & 0x7f) != rtp_TypeVideo % 128) { LOGGER_WARNING("Invalid payload type!"); - rtp_free_msg(NULL, msg); + rtp_free_msg(msg); return -1; } @@ -280,15 +280,15 @@ int vc_queue_message(void* vcp, struct RTPMessage_s *msg) vc->frame_size = framebuf_new_length; end: - rtp_free_msg(NULL, msg); + rtp_free_msg(msg); return 0; } int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height) { if (!vc) - return; + return -1; - vpx_codec_enc_cfg_t cfg = *vc->v_encoder[0].config.enc; + vpx_codec_enc_cfg_t cfg = *vc->encoder[0].config.enc; if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) return 0; /* Nothing changed */ @@ -296,7 +296,7 @@ int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint1 cfg.g_w = width; cfg.g_h = height; - int rc = vpx_codec_enc_config_set(vc->v_encoder, &cfg); + int rc = vpx_codec_enc_config_set(vc->encoder, &cfg); if ( rc != VPX_CODEC_OK) { LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); return -1; diff --git a/toxav/video.h b/toxav/video.h index ed264f367..9c5836a39 100644 --- a/toxav/video.h +++ b/toxav/video.h @@ -41,11 +41,11 @@ struct RTPMessage_s; typedef struct VCSession_s { /* encoding */ - vpx_codec_ctx_t v_encoder[1]; + vpx_codec_ctx_t encoder[1]; uint32_t frame_counter; /* decoding */ - vpx_codec_ctx_t v_decoder[1]; + vpx_codec_ctx_t decoder[1]; void *vbuf_raw; /* Un-decoded data */ /* Data handling */ From 27e0254a98a32fad89ecc1c19121394754cfda2d Mon Sep 17 00:00:00 2001 From: mannol Date: Mon, 27 Apr 2015 00:15:57 +0200 Subject: [PATCH 35/97] Almooooooost --- toxav/av_test.c | 74 ++++++--------- toxav/rtp.c | 19 ++-- toxav/rtp.h | 7 +- toxav/toxav.c | 243 ++++++++++++++++++++++++------------------------ 4 files changed, 160 insertions(+), 183 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 11de0138d..8fa493bf9 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -77,14 +77,13 @@ #define TEST_REJECT 0 #define TEST_CANCEL 0 #define TEST_MUTE_UNMUTE 0 -#define TEST_TRANSFER_A 0 -#define TEST_TRANSFER_V 1 +#define TEST_TRANSFER_A 1 +#define TEST_TRANSFER_V 0 typedef struct { bool incoming; uint32_t state; - uint32_t abitrate; pthread_mutex_t arb_mutex[1]; RingBuffer* arb; /* Audio ring buffer */ @@ -137,43 +136,8 @@ void t_toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool } void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, void *user_data) { + printf("Handling CALL STATE callback: %d\n", state); ((CallControl*)user_data)->state = state; - uint32_t abitrate = ((CallControl*)user_data)->abitrate; - - if (state & TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE) { - -// /* NOTE: I'm using values 8, 16, 24, 48, 64. You can use whatever OPUS supports. */ -// switch (abitrate) { -// case 8: abitrate = 16; break; -// case 16: abitrate = 24; break; -// case 24: abitrate = 48; break; -// case 48: abitrate = 64; break; -// default: return; -// } -// -// printf("Increasing bitrate to: %d\n", abitrate); -// toxav_set_audio_bit_rate(av, friend_number, abitrate, 0); - - } else if (state & TOXAV_CALL_STATE_DECREASE_AUDIO_BITRATE) { -// /* NOTE: I'm using values 8, 16, 24, 48, 64. You can use whatever OPUS supports. */ -// switch (abitrate) { -// case 16: abitrate = 8; break; -// case 24: abitrate = 16; break; -// case 48: abitrate = 24; break; -// case 64: abitrate = 48; break; -// default: return; -// } -// -// printf("Decreasing bitrate to: %d\n", abitrate); -// toxav_set_audio_bit_rate(av, friend_number, abitrate, 0); -// - } else if (state & TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE) { - - } else if (state & TOXAV_CALL_STATE_DECREASE_VIDEO_BITRATE) { - - } else { - printf("Handling CALL STATE callback: %d\n", state); - } } void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, @@ -223,6 +187,22 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, free(rb_write(cc->arb, f)); pthread_mutex_unlock(cc->arb_mutex); } +void t_toxav_audio_bitrate_control_cb(ToxAV *av, uint32_t friend_number, + bool good, uint32_t bit_rate, void *user_data) +{ + if (good) + printf ("Set new audio bitrate to: %d\n", bit_rate); + else + printf ("The network is overly saturated with audio bitrate at: %d\n", bit_rate); +} +void t_toxav_video_bitrate_control_cb(ToxAV *av, uint32_t friend_number, + bool good, uint32_t bit_rate, void *user_data) +{ + if (good) + printf ("Set new video bitrate to: %d", bit_rate); + else + printf ("The network is overly saturated with video bitrate at: %d", bit_rate); +} void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, size_t length, void *userdata) { if (length == 7 && memcmp("gentoo", data, 7) == 0) { @@ -299,12 +279,17 @@ void initialize_tox(Tox** bootstrap, ToxAV** AliceAV, CallControl* AliceCC, ToxA toxav_callback_call_state(*AliceAV, t_toxav_call_state_cb, AliceCC); toxav_callback_receive_video_frame(*AliceAV, t_toxav_receive_video_frame_cb, AliceCC); toxav_callback_receive_audio_frame(*AliceAV, t_toxav_receive_audio_frame_cb, AliceCC); + toxav_callback_audio_bitrate_control(*AliceAV, t_toxav_audio_bitrate_control_cb, AliceCC); + toxav_callback_video_bitrate_control(*AliceAV, t_toxav_video_bitrate_control_cb, AliceCC); /* Bob */ toxav_callback_call(*BobAV, t_toxav_call_cb, BobCC); toxav_callback_call_state(*BobAV, t_toxav_call_state_cb, BobCC); toxav_callback_receive_video_frame(*BobAV, t_toxav_receive_video_frame_cb, BobCC); toxav_callback_receive_audio_frame(*BobAV, t_toxav_receive_audio_frame_cb, BobCC); + toxav_callback_audio_bitrate_control(*BobAV, t_toxav_audio_bitrate_control_cb, BobCC); + toxav_callback_video_bitrate_control(*BobAV, t_toxav_video_bitrate_control_cb, BobCC); + printf("Created 2 instances of ToxAV\n"); printf("All set after %llu seconds!\n", time(NULL) - cur_time); @@ -760,11 +745,9 @@ int main (int argc, char** argv) AliceCC.arb = rb_new(16); BobCC.arb = rb_new(16); - AliceCC.abitrate = BobCC.abitrate = 48; - { /* Call */ TOXAV_ERR_CALL rc; - toxav_call(AliceAV, 0, AliceCC.abitrate, 0, &rc); + toxav_call(AliceAV, 0, 48, 0, &rc); if (rc != TOXAV_ERR_CALL_OK) { printf("toxav_call failed: %d\n", rc); @@ -777,7 +760,7 @@ int main (int argc, char** argv) { /* Answer */ TOXAV_ERR_ANSWER rc; - toxav_answer(BobAV, 0, BobCC.abitrate, 0, &rc); + toxav_answer(BobAV, 0, 48, 0, &rc); if (rc != TOXAV_ERR_ANSWER_OK) { printf("toxav_answer failed: %d\n", rc); @@ -794,7 +777,6 @@ int main (int argc, char** argv) exit(1); } - int16_t PCM[5760]; time_t start_time = time(NULL); @@ -802,7 +784,7 @@ int main (int argc, char** argv) /* Start decode thread */ - struct toxav_thread_data data = { + struct toxav_thread_data data = { .AliceAV = AliceAV, .BobAV = BobAV, .sig = 0 @@ -827,6 +809,8 @@ int main (int argc, char** argv) err = Pa_StartStream(adout); assert(err == paNoError); + assert(toxav_set_audio_bit_rate(AliceAV, 0, 64, false, NULL)); + /* Start write thread */ pthread_t t; pthread_create(&t, NULL, pa_write_thread, &BobCC); diff --git a/toxav/rtp.c b/toxav/rtp.c index a48eba20f..b16e3d733 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -95,7 +95,6 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num, vo retu->ssrc = random_int(); retu->payload_type = payload_type % 128; - retu->tstate = rtp_StateNormal; retu->m = messenger; retu->friend_id = friend_num; @@ -156,10 +155,10 @@ void rtp_kill ( RTPSession *session ) /* And finally free session */ free ( session ); } -void rtp_do(RTPSession *session) +int rtp_do(RTPSession *session) { if (!session || !session->rtcp_session) - return; + return rtp_StateNormal; if (current_time_monotonic() - session->rtcp_session->last_sent_report_ts >= RTCP_REPORT_INTERVAL_MS) { send_rtcp_report(session->rtcp_session, session->m, session->friend_id); @@ -182,7 +181,7 @@ void rtp_do(RTPSession *session) if (!rb_empty(session->rtcp_session->pl_stats)) { for (i = 0; reports[i] != NULL; i ++) free(reports[i]); - return; /* As some reports are timed out, we need more */ + return rtp_StateNormal; /* As some reports are timed out, we need more */ } /* We have 4 on-time reports so we can proceed */ @@ -194,16 +193,16 @@ void rtp_do(RTPSession *session) } if (quality <= 90) { - session->tstate = rtp_StateBad; LOGGER_WARNING("Stream quality: BAD"); + return rtp_StateBad; } else if (quality >= 99) { - session->tstate = rtp_StateGood; LOGGER_DEBUG("Stream quality: GOOD"); + return rtp_StateGood; } else { - session->tstate = rtp_StateNormal; LOGGER_DEBUG("Stream quality: NORMAL"); } } + return rtp_StateNormal; } int rtp_start_receiving(RTPSession* session) { @@ -267,7 +266,7 @@ int rtp_send_data ( RTPSession *session, const uint8_t *data, uint16_t length, b header->length = 12 /* Minimum header len */ + ( session->cc * size_32 ); uint32_t parsed_len = length + header->length + 1; - assert(parsed_len + (session->ext_header ? session->ext_header->length * size_32 : 0) > MAX_RTP_SIZE ); + assert(parsed_len + (session->ext_header ? session->ext_header->length * size_32 : 0) < MAX_RTP_SIZE ); parsed[0] = session->prefix; it = parse_header_out ( header, parsed + 1 ); @@ -485,7 +484,7 @@ int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data return -1; } - RTPHeader* header = parse_header_in ( data, length ); + RTPHeader* header = parse_header_in ( data + 1, length ); if ( !header ) { LOGGER_WARNING("Could not parse message: Header failed to extract!"); @@ -494,7 +493,7 @@ int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data RTPExtHeader* ext_header = NULL; - uint16_t from_pos = header->length; + uint16_t from_pos = header->length + 1; uint16_t msg_length = length - from_pos; if ( GET_FLAG_EXTENSION ( header ) ) { diff --git a/toxav/rtp.h b/toxav/rtp.h index fe07c4f6a..c973d2621 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -49,11 +49,11 @@ enum { rtp_TypeVideo, }; -typedef enum { +enum { rtp_StateBad = -1, rtp_StateNormal, rtp_StateGood, -} RTPTransmissionState; +}; /** * Standard rtp header. @@ -114,7 +114,6 @@ typedef struct { Messenger *m; int friend_id; - RTPTransmissionState tstate; struct RTCPSession_s *rtcp_session; @@ -136,7 +135,7 @@ void rtp_kill ( RTPSession* session ); /** * Do periodical rtp work. */ -void rtp_do(RTPSession *session); +int rtp_do(RTPSession *session); /** * By default rtp is in receiving state diff --git a/toxav/toxav.c b/toxav/toxav.c index 500ca9a8e..45b9e5e67 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -35,13 +35,13 @@ #include #define MAX_ENCODE_TIME_US ((1000 / 24) * 1000) +#define BITRATE_CHANGE_TESTING_TIME_MS 4000 typedef struct ToxAvBitrateAdapter_s { bool active; - uint32_t cycle_sent; - uint32_t cycle_ratio; - uint32_t cycle_repeat; - uint64_t start_time; + uint64_t end_time; + uint64_t next_send; + uint64_t next_send_interval; uint32_t bit_rate; } ToxAvBitrateAdapter; @@ -93,6 +93,8 @@ struct toxAV { PAIR(toxav_call_state_cb *, void *) scb; /* Call state callback */ PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ + PAIR(toxav_audio_bitrate_control_cb *, void *) abcb; /* Audio bitrate control callback */ + PAIR(toxav_video_bitrate_control_cb *, void *) vbcb; /* Video bitrate control callback */ /** Decode time measures */ int32_t dmssc; /** Measure count */ @@ -112,7 +114,6 @@ int callback_capabilites(void* toxav_inst, MSICall* call); bool audio_bitrate_invalid(uint32_t bitrate); bool video_bitrate_invalid(uint32_t bitrate); void invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state); -void qc_do(ToxAVCall* call); ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); ToxAVCall* call_get(ToxAV* av, uint32_t friend_number); void call_remove(ToxAVCall* call); @@ -120,7 +121,6 @@ bool call_prepare_transmission(ToxAVCall* call); void call_kill_transmission(ToxAVCall* call); void ba_set(ToxAvBitrateAdapter* ba, uint32_t bit_rate); bool ba_shoud_send_dummy(ToxAvBitrateAdapter* ba); -void ba_update_sent_regular(ToxAvBitrateAdapter* ba); ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) { @@ -235,13 +235,59 @@ void toxav_iterate(ToxAV* av) pthread_mutex_lock(i->mutex); pthread_mutex_unlock(av->mutex); - rtp_do(i->audio.first); ac_do(i->audio.second); + if (rtp_do(i->audio.first) < 0) { + /* Bad transmission */ + + uint32_t bb = i->audio_bit_rate; + + if (i->aba.active) { + bb = i->aba.bit_rate; + /* Stop sending dummy packets */ + memset(&i->aba, 0, sizeof(i->aba)); + } + + /* Notify app */ + if (av->abcb.first) + av->abcb.first (av, i->friend_id, false, bb, av->abcb.second); + } else if (i->aba.active && i->aba.end_time < current_time_monotonic()) { + + i->audio_bit_rate = i->aba.bit_rate; + + /* Notify user about the new bitrate */ + if (av->abcb.first) + av->abcb.first (av, i->friend_id, true, i->aba.bit_rate, av->abcb.second); + + /* Stop sending dummy packets */ + memset(&i->aba, 0, sizeof(i->aba)); + } - rtp_do(i->video.first); vc_do(i->video.second); - - qc_do(i); + if (rtp_do(i->video.first) < 0) { + /* Bad transmission */ + uint32_t bb = i->video_bit_rate; + + if (i->vba.active) { + bb = i->vba.bit_rate; + /* Stop sending dummy packets */ + memset(&i->vba, 0, sizeof(i->vba)); + } + + /* Notify app */ + if (av->vbcb.first) + av->vbcb.first (av, i->friend_id, false, bb, av->vbcb.second); + + } else if (i->vba.active && i->vba.end_time < current_time_monotonic()) { + + i->video_bit_rate = i->vba.bit_rate; + + /* Notify user about the new bitrate */ + if (av->vbcb.first) + av->vbcb.first (av, i->friend_id, true, i->vba.bit_rate, av->vbcb.second); + + /* Stop sending dummy packets */ + memset(&i->vba, 0, sizeof(i->vba)); + } if (i->msi_call->self_capabilities & msi_CapRAudio && i->msi_call->peer_capabilities & msi_CapSAudio) @@ -515,48 +561,12 @@ END: return rc == TOXAV_ERR_CALL_CONTROL_OK; } -bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, bool force, TOXAV_ERR_BIT_RATE* error) +void toxav_callback_video_bitrate_control(ToxAV* av, toxav_video_bitrate_control_cb* function, void* user_data) { - TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK; - ToxAVCall* call; - - if (m_friend_exists(av->m, friend_number) == 0) { - rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND; - goto END; - } - - if (audio_bitrate_invalid(audio_bit_rate)) { - rc = TOXAV_ERR_BIT_RATE_INVALID; - goto END; - } - pthread_mutex_lock(av->mutex); - call = call_get(av, friend_number); - if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - pthread_mutex_unlock(av->mutex); - rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; - goto END; - } - - if (call->audio_bit_rate == audio_bit_rate && call->aba.active && call->aba.bit_rate == audio_bit_rate) { - pthread_mutex_unlock(av->mutex); - goto END; - } - - pthread_mutex_lock(call->mutex); - if (force) { - call->audio_bit_rate = audio_bit_rate; - } else - ba_set(&call->aba, audio_bit_rate); - - pthread_mutex_unlock(call->mutex); + av->vbcb.first = function; + av->vbcb.second = user_data; pthread_mutex_unlock(av->mutex); - -END: - if (error) - *error = rc; - - return rc == TOXAV_ERR_BIT_RATE_OK; } bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, bool force, TOXAV_ERR_BIT_RATE* error) @@ -603,6 +613,58 @@ END: return rc == TOXAV_ERR_BIT_RATE_OK; } +void toxav_callback_audio_bitrate_control(ToxAV* av, toxav_audio_bitrate_control_cb* function, void* user_data) +{ + pthread_mutex_lock(av->mutex); + av->abcb.first = function; + av->abcb.second = user_data; + pthread_mutex_unlock(av->mutex); +} + +bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, bool force, TOXAV_ERR_BIT_RATE* error) +{ + TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK; + ToxAVCall* call; + + if (m_friend_exists(av->m, friend_number) == 0) { + rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND; + goto END; + } + + if (audio_bitrate_invalid(audio_bit_rate)) { + rc = TOXAV_ERR_BIT_RATE_INVALID; + goto END; + } + + pthread_mutex_lock(av->mutex); + call = call_get(av, friend_number); + if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { + pthread_mutex_unlock(av->mutex); + rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; + goto END; + } + + if (call->audio_bit_rate == audio_bit_rate && call->aba.active && call->aba.bit_rate == audio_bit_rate) { + pthread_mutex_unlock(av->mutex); + goto END; + } + + pthread_mutex_lock(call->mutex); + if (force) { + call->audio_bit_rate = audio_bit_rate; + } else + ba_set(&call->aba, audio_bit_rate); + + pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(av->mutex); + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_BIT_RATE_OK; +} + bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) { TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; @@ -790,7 +852,6 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc } } - ba_update_sent_regular(&call->aba); } pthread_mutex_unlock(call->mutex_audio); @@ -932,57 +993,6 @@ void invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state) av->scb.first(av, friend_number, state, av->scb.second); } -void qc_do(ToxAVCall* call) -{ - /* - switch(call->audio.first->tstate) { - case rtp_StateBad: - LOGGER_DEBUG("Suggesting lower bitrate for audio..."); - call->time_audio_good = 0; - call->last_bad_audio_bit_rate = call->audio_bit_rate; - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_DECREASE_AUDIO_BITRATE); - break; - case rtp_StateGood: - if (call->time_audio_good == 0) - call->time_audio_good = current_time_monotonic(); - else if (current_time_monotonic() - call->time_audio_good >= 30000 && - 64 > call->audio_bit_rate) - if (call->last_bad_audio_bit_rate > call->audio_bit_rate) { - if (current_time_monotonic() - call->time_audio_good >= 45000) - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE); - call->last_bad_audio_bit_rate = 0; - } else - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_AUDIO_BITRATE); - break; - case rtp_StateNormal: - call->time_audio_good = 0; - break; - }*/ - /* - switch(call->video.first->tstate) { - case rtp_StateBad: - LOGGER_DEBUG("Suggesting lower bitrate for video..."); - call->time_video_good = 0; - call->last_bad_video_bit_rate = call->video_bit_rate; - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_DECREASE_VIDEO_BITRATE); - break; - case rtp_StateGood: - if (call->time_video_good == 0) - call->time_video_good = current_time_monotonic(); - else if (current_time_monotonic() - call->time_video_good >= 30000) - if (call->last_bad_video_bit_rate > call->video_bit_rate) { - if (current_time_monotonic() - call->time_video_good >= 45000) - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE); - call->last_bad_video_bit_rate = 0; - } else - invoke_call_state(call->av, call->friend_id, TOXAV_CALL_STATE_INCREASE_VIDEO_BITRATE); - break; - case rtp_StateNormal: - call->time_video_good = 0; - break; - }*/ -} - ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) { /* Assumes mutex locked */ @@ -1208,34 +1218,19 @@ void call_kill_transmission(ToxAVCall* call) void ba_set(ToxAvBitrateAdapter* ba, uint32_t bit_rate) { ba->bit_rate = bit_rate; - ba->start_time = current_time_monotonic(); - ba->cycle_sent = 0; - ba->cycle_repeat = 3; - ba->cycle_ratio = 4; + ba->next_send = current_time_monotonic(); + ba->end_time = ba->next_send + BITRATE_CHANGE_TESTING_TIME_MS; + ba->next_send_interval = 1000; ba->active = true; } bool ba_shoud_send_dummy(ToxAvBitrateAdapter* ba) { - if (!ba->active || ba->cycle_ratio == 0) + if (!ba->active || ba->next_send > current_time_monotonic()) return false; - if (ba->cycle_sent % ba->cycle_ratio == 0) - return true; -} - -void ba_update_sent_regular(ToxAvBitrateAdapter* ba) -{ - if (!ba->active || ba->cycle_ratio == 0) { - ba->active = false; - return; - } + ba->next_send_interval *= 0.8; + ba->next_send = current_time_monotonic() + ba->next_send_interval; - if (ba->cycle_sent % ba->cycle_ratio == 0) { - ba->cycle_sent = 0; - - if (--ba->cycle_repeat == 0) - --ba->cycle_ratio; - } else - ++ba->cycle_sent; -} + return true; +} \ No newline at end of file From e4a020333d76bc30172f54f2545677f01bdd54b6 Mon Sep 17 00:00:00 2001 From: mannol Date: Tue, 28 Apr 2015 00:55:57 +0200 Subject: [PATCH 36/97] working av new api --- toxav/av_test.c | 35 ++++++++++------ toxav/rtp.c | 6 +-- toxav/toxav.c | 104 +++++++++++++++++++++++++++++++++++++++++------- toxav/video.c | 29 +++++++++++++- toxav/video.h | 3 ++ 5 files changed, 147 insertions(+), 30 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 8fa493bf9..86deebdf6 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -77,8 +77,8 @@ #define TEST_REJECT 0 #define TEST_CANCEL 0 #define TEST_MUTE_UNMUTE 0 -#define TEST_TRANSFER_A 1 -#define TEST_TRANSFER_V 0 +#define TEST_TRANSFER_A 0 +#define TEST_TRANSFER_V 1 typedef struct { @@ -218,20 +218,28 @@ void initialize_tox(Tox** bootstrap, ToxAV** AliceAV, CallControl* AliceCC, ToxA Tox* Alice; Tox* Bob; + struct Tox_Options opts; + tox_options_default(&opts); + + opts.end_port = 0; + { TOX_ERR_NEW error; - *bootstrap = tox_new(NULL, NULL, 0, &error); + + opts.start_port = 33445; + *bootstrap = tox_new(&opts, NULL, 0, &error); assert(error == TOX_ERR_NEW_OK); - Alice = tox_new(NULL, NULL, 0, &error); + opts.start_port = 33455; + Alice = tox_new(&opts, NULL, 0, &error); assert(error == TOX_ERR_NEW_OK); - Bob = tox_new(NULL, NULL, 0, &error); + opts.start_port = 33465; + Bob = tox_new(&opts, NULL, 0, &error); assert(error == TOX_ERR_NEW_OK); } printf("Created 3 instances of Tox\n"); - printf("Preparing network...\n"); long long unsigned int cur_time = time(NULL); @@ -768,7 +776,8 @@ int main (int argc, char** argv) } } - iterate_tox(bootstrap, AliceAV, BobAV); + while (AliceCC.state == 0) + iterate_tox(bootstrap, AliceAV, BobAV); /* Open audio file */ af_handle = sf_open(af_name, SFM_READ, &af_info); @@ -809,7 +818,7 @@ int main (int argc, char** argv) err = Pa_StartStream(adout); assert(err == paNoError); - assert(toxav_set_audio_bit_rate(AliceAV, 0, 64, false, NULL)); + toxav_set_audio_bit_rate(AliceAV, 0, 64, false, NULL); /* Start write thread */ pthread_t t; @@ -874,7 +883,7 @@ int main (int argc, char** argv) { /* Call */ TOXAV_ERR_CALL rc; - toxav_call(AliceAV, 0, 0, 5000000, &rc); + toxav_call(AliceAV, 0, 0, 3000, &rc); if (rc != TOXAV_ERR_CALL_OK) { printf("toxav_call failed: %d\n", rc); @@ -887,7 +896,7 @@ int main (int argc, char** argv) { /* Answer */ TOXAV_ERR_ANSWER rc; - toxav_answer(BobAV, 0, 0, 5000000, &rc); + toxav_answer(BobAV, 0, 0, 500, &rc); if (rc != TOXAV_ERR_ANSWER_OK) { printf("toxav_answer failed: %d\n", rc); @@ -899,8 +908,8 @@ int main (int argc, char** argv) /* Start decode thread */ struct toxav_thread_data data = { - .AliceAV = AliceAV, - .BobAV = BobAV, + .AliceAV = AliceAV, + .BobAV = BobAV, .sig = 0 }; @@ -914,6 +923,8 @@ int main (int argc, char** argv) exit(1); } + toxav_set_video_bit_rate(AliceAV, 0, 5000, false, NULL); + time_t start_time = time(NULL); while(start_time + 90 > time(NULL)) { IplImage* frame = cvQueryFrame( capture ); diff --git a/toxav/rtp.c b/toxav/rtp.c index b16e3d733..6c603f79c 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -193,13 +193,13 @@ int rtp_do(RTPSession *session) } if (quality <= 90) { - LOGGER_WARNING("Stream quality: BAD"); + LOGGER_WARNING("Stream quality: BAD (%d)", quality); return rtp_StateBad; } else if (quality >= 99) { - LOGGER_DEBUG("Stream quality: GOOD"); + LOGGER_DEBUG("Stream quality: GOOD (%d)", quality); return rtp_StateGood; } else { - LOGGER_DEBUG("Stream quality: NORMAL"); + LOGGER_DEBUG("Stream quality: NORMAL (%d)", quality); } } return rtp_StateNormal; diff --git a/toxav/toxav.c b/toxav/toxav.c index 45b9e5e67..8d47f5cd6 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -592,17 +592,22 @@ bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_ goto END; } - if (call->video_bit_rate == video_bit_rate && call->vba.active && call->vba.bit_rate == video_bit_rate) { + if (call->video_bit_rate == video_bit_rate || call->vba.active || call->vba.bit_rate == video_bit_rate) { pthread_mutex_unlock(av->mutex); goto END; } pthread_mutex_lock(call->mutex); - if (force) { - call->video_bit_rate = video_bit_rate; - } else { + + if (video_bit_rate > call->video_bit_rate && !force) ba_set(&call->vba, video_bit_rate); + else { + call->video_bit_rate = video_bit_rate; + + if (!force && av->vbcb.first) + av->vbcb.first (av, call->friend_id, true, video_bit_rate, av->vbcb.second); } + pthread_mutex_unlock(call->mutex); pthread_mutex_unlock(av->mutex); @@ -644,16 +649,21 @@ bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_ goto END; } - if (call->audio_bit_rate == audio_bit_rate && call->aba.active && call->aba.bit_rate == audio_bit_rate) { + if (call->audio_bit_rate == audio_bit_rate || call->aba.active || call->aba.bit_rate == audio_bit_rate) { pthread_mutex_unlock(av->mutex); goto END; } pthread_mutex_lock(call->mutex); - if (force) { - call->audio_bit_rate = audio_bit_rate; - } else + + if (audio_bit_rate > call->audio_bit_rate && !force) ba_set(&call->aba, audio_bit_rate); + else { + call->audio_bit_rate = audio_bit_rate; + + if (!force && av->abcb.first) + av->abcb.first (av, call->friend_id, true, audio_bit_rate, av->abcb.second); + } pthread_mutex_unlock(call->mutex); pthread_mutex_unlock(av->mutex); @@ -692,7 +702,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u goto END; } - if ( vc_reconfigure_encoder(call->video.second, call->video_bit_rate, width, height) != 0 ) { + if ( vc_reconfigure_encoder(call->video.second, call->video_bit_rate * 1000, width, height) != 0 ) { pthread_mutex_unlock(call->mutex_video); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; @@ -755,6 +765,69 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u } } + if (ba_shoud_send_dummy(&call->vba)) { + if ( vc_reconfigure_test_encoder(call->video.second, call->vba.bit_rate * 1000, width, height) != 0 ) { + pthread_mutex_unlock(call->mutex_video); + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + + /* FIXME use the same image as before */ + vpx_image_t img; + img.w = img.h = img.d_w = img.d_h = 0; + vpx_img_alloc(&img, VPX_IMG_FMT_VPXI420, width, height, 1); + + /* I420 "It comprises an NxM Y plane followed by (N/2)x(M/2) V and U planes." + * http://fourcc.org/yuv.php#IYUV + */ + memcpy(img.planes[VPX_PLANE_Y], y, width * height); + memcpy(img.planes[VPX_PLANE_U], u, (width/2) * (height/2)); + memcpy(img.planes[VPX_PLANE_V], v, (width/2) * (height/2)); + + int vrc = vpx_codec_encode(call->video.second->test_encoder, &img, + call->video.second->test_frame_counter, 1, 0, MAX_ENCODE_TIME_US); + + vpx_img_free(&img); + if ( vrc != VPX_CODEC_OK) { + pthread_mutex_unlock(call->mutex_video); + LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + + call->video.second->test_frame_counter++; + + vpx_codec_iter_t iter = NULL; + const vpx_codec_cx_pkt_t *pkt; + + /* Send the encoded data as dummy packets */ + while ( (pkt = vpx_codec_get_cx_data(call->video.second->test_encoder, &iter)) ) { + if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { + + int parts = pkt->data.frame.sz / 1300; + int i; + for (i = 0; i < parts; i++) { + if (rtp_send_data(call->video.first, pkt->data.frame.buf + i * 1300, 1300, true) < 0) { + pthread_mutex_unlock(call->mutex_video); + LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); + goto END; + } + } + + if (pkt->data.frame.sz % 1300) { + if (rtp_send_data(call->video.first, pkt->data.frame.buf + parts * 1300, pkt->data.frame.sz % 1300, true) < 0) { + pthread_mutex_unlock(call->mutex_video); + LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); + goto END; + } + } + } + } + + if (call->vba.end_time == ~0) + call->vba.end_time = current_time_monotonic() + BITRATE_CHANGE_TESTING_TIME_MS; + } + pthread_mutex_unlock(call->mutex_video); END: @@ -824,8 +897,8 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc } - /* For bitrate measurement; send dummy packet */ - if (ba_shoud_send_dummy(&call->aba)) { + /* For bitrate measurement; send dummy packet */ + if (ba_shoud_send_dummy(&call->aba)) { sampling_rate = ntohl(sampling_rate); if (ac_reconfigure_test_encoder(call->audio.second, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) { /* FIXME should the bitrate changing fail here? */ @@ -837,7 +910,7 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc sampling_rate = htonl(sampling_rate); memcpy(dest, &sampling_rate, sizeof(sampling_rate)); vrc = opus_encode(call->audio.second->test_encoder, pcm, sample_count, - dest + sizeof(sampling_rate), sizeof(dest) - sizeof(sampling_rate)); + dest + sizeof(sampling_rate), sizeof(dest) - sizeof(sampling_rate)); if (vrc < 0) { LOGGER_WARNING("Failed to encode frame %s", opus_strerror(vrc)); @@ -850,10 +923,13 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc LOGGER_WARNING("Failed to send audio packet"); rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; } + + if (call->aba.end_time == ~0) + call->aba.end_time = current_time_monotonic() + BITRATE_CHANGE_TESTING_TIME_MS; } - } + pthread_mutex_unlock(call->mutex_audio); END: @@ -1219,7 +1295,7 @@ void ba_set(ToxAvBitrateAdapter* ba, uint32_t bit_rate) { ba->bit_rate = bit_rate; ba->next_send = current_time_monotonic(); - ba->end_time = ba->next_send + BITRATE_CHANGE_TESTING_TIME_MS; + ba->end_time = ~0; ba->next_send_interval = 1000; ba->active = true; } diff --git a/toxav/video.c b/toxav/video.c index cdc3c0ae2..c540af3b8 100644 --- a/toxav/video.c +++ b/toxav/video.c @@ -76,6 +76,11 @@ VCSession* vc_new(ToxAV* av, uint32_t friend_id, toxav_receive_video_frame_cb* c vpx_codec_destroy(vc->decoder); goto BASE_CLEANUP; } + if (!create_video_encoder(vc->test_encoder, 500000)) { + vpx_codec_destroy(vc->encoder); + vpx_codec_destroy(vc->decoder); + goto BASE_CLEANUP; + } vc->linfts = current_time_monotonic(); vc->lcfd = 60; @@ -100,6 +105,7 @@ void vc_kill(VCSession* vc) return; vpx_codec_destroy(vc->encoder); + vpx_codec_destroy(vc->test_encoder); vpx_codec_destroy(vc->decoder); rb_free(vc->vbuf_raw); free(vc->split_video_frame); @@ -288,7 +294,7 @@ int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint1 if (!vc) return -1; - vpx_codec_enc_cfg_t cfg = *vc->encoder[0].config.enc; + vpx_codec_enc_cfg_t cfg = *vc->encoder->config.enc; if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) return 0; /* Nothing changed */ @@ -304,6 +310,27 @@ int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint1 return 0; } +int vc_reconfigure_test_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height) +{ + if (!vc) + return -1; + + vpx_codec_enc_cfg_t cfg = *vc->test_encoder->config.enc; + if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) + return 0; /* Nothing changed */ + + cfg.rc_target_bitrate = bitrate; + cfg.g_w = width; + cfg.g_h = height; + + int rc = vpx_codec_enc_config_set(vc->test_encoder, &cfg); + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to set test encoder control setting: %s", vpx_codec_err_to_string(rc)); + return -1; + } + + return 0; +} diff --git a/toxav/video.h b/toxav/video.h index 9c5836a39..78003ef3a 100644 --- a/toxav/video.h +++ b/toxav/video.h @@ -42,7 +42,9 @@ typedef struct VCSession_s { /* encoding */ vpx_codec_ctx_t encoder[1]; + vpx_codec_ctx_t test_encoder[1]; uint32_t frame_counter; + uint32_t test_frame_counter; /* decoding */ vpx_codec_ctx_t decoder[1]; @@ -80,5 +82,6 @@ int vc_update_video_splitter_cycle(VCSession* vc, const uint8_t* payload, uint16 const uint8_t *vc_iterate_split_video_frame(VCSession* vc, uint16_t *size); int vc_queue_message(void *vcp, struct RTPMessage_s *msg); int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height); +int vc_reconfigure_test_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height); #endif /* VIDEO_H */ \ No newline at end of file From 9bba7a0434d0967d5dd76b8afc7783ea2edad0cf Mon Sep 17 00:00:00 2001 From: mannol Date: Wed, 29 Apr 2015 01:01:25 +0200 Subject: [PATCH 37/97] Done --- auto_tests/Makefile.inc | 10 +- auto_tests/toxav_basic_test.c | 913 +++++++++++++--------------------- auto_tests/toxav_many_test.c | 475 ++++++------------ {toxav => testing}/av_test.c | 264 +--------- toxav/Makefile.inc | 21 - toxav/audio.c | 35 +- toxav/audio.h | 33 +- toxav/msi.c | 124 +++-- toxav/msi.h | 24 +- toxav/rtp.c | 67 +-- toxav/rtp.h | 11 +- toxav/toxav.c | 132 +++-- toxav/toxav.h | 40 +- toxav/video.c | 25 +- toxav/video.h | 36 +- 15 files changed, 791 insertions(+), 1419 deletions(-) rename {toxav => testing}/av_test.c (70%) diff --git a/auto_tests/Makefile.inc b/auto_tests/Makefile.inc index d78a6a5a3..741ca7faf 100644 --- a/auto_tests/Makefile.inc +++ b/auto_tests/Makefile.inc @@ -21,8 +21,8 @@ AUTOTEST_LDADD = \ if BUILD_AV -TESTS += toxav_basic_test toxav_many_test -check_PROGRAMS += toxav_basic_test toxav_many_test +TESTS += toxav_basic_test #toxav_many_test +check_PROGRAMS += toxav_basic_test #toxav_many_test AUTOTEST_LDADD += libtoxav.la endif @@ -90,11 +90,11 @@ toxav_basic_test_CFLAGS = $(AUTOTEST_CFLAGS) toxav_basic_test_LDADD = $(AUTOTEST_LDADD) $(AV_LIBS) -toxav_many_test_SOURCES = ../auto_tests/toxav_many_test.c +#toxav_many_test_SOURCES = ../auto_tests/toxav_many_test.c -toxav_many_test_CFLAGS = $(AUTOTEST_CFLAGS) +#toxav_many_test_CFLAGS = $(AUTOTEST_CFLAGS) -toxav_many_test_LDADD = $(AUTOTEST_LDADD) +#toxav_many_test_LDADD = $(AUTOTEST_LDADD) endif endif diff --git a/auto_tests/toxav_basic_test.c b/auto_tests/toxav_basic_test.c index a43b7c2fd..7598c0db9 100644 --- a/auto_tests/toxav_basic_test.c +++ b/auto_tests/toxav_basic_test.c @@ -14,6 +14,7 @@ #include #include "../toxcore/tox.h" +#include "../toxcore/util.h" #include "../toxcore/logger.h" #include "../toxcore/crypto_core.h" #include "../toxav/toxav.h" @@ -28,600 +29,406 @@ #endif +#define TEST_REGULAR_AV 1 +#define TEST_REGULAR_A 1 +#define TEST_REGULAR_V 1 +#define TEST_REJECT 1 +#define TEST_CANCEL 1 +#define TEST_MUTE_UNMUTE 1 -typedef enum _CallStatus { - none, - InCall, - Ringing, - Ended, - Rejected, - Canceled, - TimedOut -} CallStatus; +typedef struct { + bool incoming; + uint32_t state; + +} CallControl; -typedef struct _Party { - CallStatus status; - ToxAv *av; - time_t *CallStarted; - int call_index; -} Party; -typedef struct _Status { - Party Alice; - Party Bob; -} Status; - -/* My default settings */ -static ToxAvCSettings muhcaps; - -void accept_friend_request(Tox *m, const uint8_t *public_key, const uint8_t *data, size_t length, void *userdata) +/** + * Callbacks + */ +void t_toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data) +{ + printf("Handling CALL callback\n"); + ((CallControl*)user_data)->incoming = true; +} +void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, void *user_data) +{ + printf("Handling CALL STATE callback: %d\n", state); + ((CallControl*)user_data)->state = state; +} +void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, + uint16_t width, uint16_t height, + uint8_t const *y, uint8_t const *u, uint8_t const *v, + int32_t ystride, int32_t ustride, int32_t vstride, + void *user_data) +{ + (void) av; + (void) friend_number; + (void) width; + (void) height; + (void) y; + (void) u; + (void) v; + (void) ystride; + (void) ustride; + (void) vstride; + (void) user_data; +} +void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, + int16_t const *pcm, + size_t sample_count, + uint8_t channels, + uint32_t sampling_rate, + void *user_data) +{ + (void) av; + (void) friend_number; + (void) pcm; + (void) sample_count; + (void) channels; + (void) sampling_rate; + (void) user_data; +} +void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, size_t length, void *userdata) { if (length == 7 && memcmp("gentoo", data, 7) == 0) { - tox_friend_add_norequest(m, public_key, 0); + assert(tox_friend_add_norequest(m, public_key, NULL) != (uint32_t) ~0); } } -/******************************************************************************/ -void callback_recv_invite ( void *av, int32_t call_index, void *_arg ) -{ - Status *cast = _arg; - - if (cast->Alice.av == av) { - // ... - } else if (cast->Bob.av == av) { - /* Bob always receives invite */ - cast->Bob.status = Ringing; - cast->Bob.call_index = call_index; - } -} -void callback_recv_ringing ( void *av, int32_t call_index, void *_arg ) -{ - Status *cast = _arg; - - if (cast->Alice.av == av) { - /* Alice always sends invite */ - cast->Alice.status = Ringing; - } else if (cast->Bob.av == av) { - // ... - } -} - - -void callback_call_started ( void *av, int32_t call_index, void *_arg ) -{ - Status *cast = _arg; - - if (cast->Alice.av == av) { - printf("Call started on Alices side...\n"); - cast->Alice.status = InCall; - toxav_prepare_transmission(av, call_index, 1); - } else if (cast->Bob.av == av) { - printf("Call started on Bob side...\n"); - cast->Bob.status = InCall; - toxav_prepare_transmission(av, call_index, 1); - } -} -void callback_call_canceled ( void *av, int32_t call_index, void *_arg ) -{ - Status *cast = _arg; - - if (cast->Alice.av == av) { - // ... - } else if (cast->Bob.av == av) { - printf ( "Call Canceled for Bob!\n" ); - cast->Bob.status = Canceled; - } -} -void callback_call_rejected ( void *av, int32_t call_index, void *_arg ) -{ - Status *cast = _arg; - - printf ( "Call rejected by Bob!\n" - "Call ended for Alice!\n" ); - - /* If Bob rejects, call is ended for alice and she sends ending */ - if (cast->Alice.av == av) { - cast->Alice.status = Rejected; - } else if (cast->Bob.av == av) { - //... ignor - } -} -void callback_call_ended ( void *av, int32_t call_index, void *_arg ) -{ - Status *cast = _arg; - - if (cast->Alice.av == av) { - printf ( "Call ended for Alice!\n" ); - cast->Alice.status = Ended; - } else if (cast->Bob.av == av) { - printf ( "Call ended for Bob!\n" ); - cast->Bob.status = Ended; - } -} - -void callback_peer_cs_change ( void *av, int32_t call_index, void *_arg ) -{ - ToxAvCSettings csettings; - toxav_get_peer_csettings(av, call_index, 0, &csettings); - - printf("Peer changing settings to: \n" - "Type: %u \n" - "Video bitrate: %u \n" - "Video height: %u \n" - "Video width: %u \n" - "Audio bitrate: %u \n" - "Audio framedur: %u \n" - "Audio sample rate: %u \n" - "Audio channels: %u \n", - csettings.call_type, - csettings.video_bitrate, - csettings.max_video_height, - csettings.max_video_width, - csettings.audio_bitrate, - csettings.audio_frame_duration, - csettings.audio_sample_rate, - csettings.audio_channels - ); -} - -void callback_self_cs_change ( void *av, int32_t call_index, void *_arg ) -{ - ToxAvCSettings csettings; - toxav_get_peer_csettings(av, call_index, 0, &csettings); - - printf("Changed settings to: \n" - "Type: %u \n" - "Video bitrate: %u \n" - "Video height: %u \n" - "Video width: %u \n" - "Audio bitrate: %u \n" - "Audio framedur: %u \n" - "Audio sample rate: %u \n" - "Audio channels: %u \n", - csettings.call_type, - csettings.video_bitrate, - csettings.max_video_height, - csettings.max_video_width, - csettings.audio_bitrate, - csettings.audio_frame_duration, - csettings.audio_sample_rate, - csettings.audio_channels - ); -} - -void callback_requ_timeout ( void *av, int32_t call_index, void *_arg ) -{ - Status *cast = _arg; - printf("Call timed-out!\n"); - - if (cast->Alice.av == av) { - cast->Alice.status = TimedOut; - } else if (cast->Bob.av == av) { - cast->Bob.status = TimedOut; - } -} - -void callback_audio (void *agent, int32_t call_idx, const int16_t *PCM, uint16_t size, void *data) -{} - -void callback_video (void *agent, int32_t call_idx, const vpx_image_t *img, void *data) -{} - -void register_callbacks(ToxAv *av, void *data) -{ - toxav_register_callstate_callback(av, callback_call_started, av_OnStart, data); - toxav_register_callstate_callback(av, callback_call_canceled, av_OnCancel, data); - toxav_register_callstate_callback(av, callback_call_rejected, av_OnReject, data); - toxav_register_callstate_callback(av, callback_call_ended, av_OnEnd, data); - toxav_register_callstate_callback(av, callback_recv_invite, av_OnInvite, data); - toxav_register_callstate_callback(av, callback_recv_ringing, av_OnRinging, data); - toxav_register_callstate_callback(av, callback_requ_timeout, av_OnRequestTimeout, data); - toxav_register_callstate_callback(av, callback_peer_cs_change, av_OnPeerCSChange, data); - toxav_register_callstate_callback(av, callback_self_cs_change, av_OnSelfCSChange, data); - toxav_register_audio_callback(av, callback_audio, NULL); - toxav_register_video_callback(av, callback_video, NULL); -} - - -/*************************************************************************************************/ - -/* Alice calls bob and the call starts. - * What happens during the call is defined after. To quit the loop use: step++; +/** + * Iterate helper */ -#define CALL_AND_START_LOOP(AliceCallType, BobCallType) \ -{ int step = 0, running = 1; while (running) {\ - tox_iterate(bootstrap_node); tox_iterate(Alice); tox_iterate(Bob); \ - toxav_do(status_control.Bob.av); toxav_do(status_control.Alice.av); \ - switch ( step ) {\ - case 0: /* Alice */ printf("Alice is calling...\n");\ - toxav_call(status_control.Alice.av, &status_control.Alice.call_index, 0, &muhcaps, 10); step++; break;\ - case 1: /* Bob */ if (status_control.Bob.status == Ringing) { printf("Bob answers...\n");\ - cur_time = time(NULL); toxav_answer(status_control.Bob.av, status_control.Bob.call_index, &muhcaps); step++; } break; \ - case 2: /* Rtp transmission */ \ - if (status_control.Bob.status == InCall && status_control.Alice.status == InCall) +int iterate_tox(Tox* bootstrap, Tox* Alice, Tox* Bob) +{ + tox_iterate(bootstrap); + tox_iterate(Alice); + tox_iterate(Bob); + + return MIN(tox_iteration_interval(Alice), tox_iteration_interval(Bob)); +} -#define TERMINATE_SCOPE() break;\ -case 3: /* Wait for Both to have status ended */\ -if (status_control.Alice.status == Ended && status_control.Bob.status == Ended) running = 0; break; } c_sleep(20); } } printf("\n"); START_TEST(test_AV_flows) { + Tox* Alice, *Bob, *bootstrap; + ToxAV* AliceAV, *BobAV; + + CallControl AliceCC, BobCC; + + { + TOX_ERR_NEW error; + + bootstrap = tox_new(NULL, NULL, 0, &error); + assert(error == TOX_ERR_NEW_OK); + + Alice = tox_new(NULL, NULL, 0, &error); + assert(error == TOX_ERR_NEW_OK); + + Bob = tox_new(NULL, NULL, 0, &error); + assert(error == TOX_ERR_NEW_OK); + } + + printf("Created 3 instances of Tox\n"); + printf("Preparing network...\n"); long long unsigned int cur_time = time(NULL); - Tox *bootstrap_node = tox_new(0, 0, 0, 0); - Tox *Alice = tox_new(0, 0, 0, 0); - Tox *Bob = tox_new(0, 0, 0, 0); - - ck_assert_msg(bootstrap_node || Alice || Bob, "Failed to create 3 tox instances"); - + uint32_t to_compare = 974536; - tox_callback_friend_request(Alice, accept_friend_request, &to_compare); uint8_t address[TOX_ADDRESS_SIZE]; + + tox_callback_friend_request(Alice, t_accept_friend_request_cb, &to_compare); tox_self_get_address(Alice, address); - uint32_t test = tox_friend_add(Bob, address, (uint8_t *)"gentoo", 7, 0); - - ck_assert_msg(test == 0, "Failed to add friend error code: %i", test); - + + + assert(tox_friend_add(Bob, address, (uint8_t *)"gentoo", 7, NULL) != (uint32_t) ~0); + uint8_t off = 1; - + while (1) { - tox_iterate(bootstrap_node); - tox_iterate(Alice); - tox_iterate(Bob); - - if (tox_self_get_connection_status(bootstrap_node) && tox_self_get_connection_status(Alice) - && tox_self_get_connection_status(Bob) - && off) { + iterate_tox(bootstrap, Alice, Bob); + + if (tox_self_get_connection_status(bootstrap) && + tox_self_get_connection_status(Alice) && + tox_self_get_connection_status(Bob) && off) { printf("Toxes are online, took %llu seconds\n", time(NULL) - cur_time); off = 0; } - - if (tox_friend_get_connection_status(Alice, 0, 0) && tox_friend_get_connection_status(Bob, 0, 0)) + + if (tox_friend_get_connection_status(Alice, 0, NULL) == TOX_CONNECTION_UDP && + tox_friend_get_connection_status(Bob, 0, NULL) == TOX_CONNECTION_UDP) break; - + c_sleep(20); } - - printf("All set after %llu seconds! Starting call...\n", time(NULL) - cur_time); - - muhcaps = av_DefaultSettings; - muhcaps.max_video_height = muhcaps.max_video_width = 128; - - Status status_control = { - {none, toxav_new(Alice, 1), NULL, -1}, - {none, toxav_new(Bob, 1), NULL, -1}, - }; - - ck_assert_msg(status_control.Alice.av || status_control.Bob.av, "Failed to create 2 toxav instances"); - - - register_callbacks(status_control.Alice.av, &status_control); - register_callbacks(status_control.Bob.av, &status_control); - - const int frame_size = (av_DefaultSettings.audio_sample_rate * av_DefaultSettings.audio_frame_duration / 1000); - int16_t sample_payload[frame_size]; - randombytes((uint8_t *)sample_payload, sizeof(int16_t) * frame_size); - - uint8_t prepared_payload[RTP_PAYLOAD_SIZE]; - int payload_size; - - vpx_image_t *sample_image = vpx_img_alloc(NULL, VPX_IMG_FMT_I420, 128, 128, 1); - - memcpy(sample_image->planes[VPX_PLANE_Y], sample_payload, 10); - memcpy(sample_image->planes[VPX_PLANE_U], sample_payload, 10); - memcpy(sample_image->planes[VPX_PLANE_V], sample_payload, 10); - - - /************************************************************************************************* - * Successful flows (when call starts) - */ - - /* - * Call with audio only on both sides. Alice calls Bob. - */ - - - CALL_AND_START_LOOP(TypeAudio, TypeAudio) { - /* Both send */ - payload_size = toxav_prepare_audio_frame(status_control.Alice.av, status_control.Alice.call_index, prepared_payload, - 1000, sample_payload, frame_size); - - if ( payload_size < 0 ) { - ck_assert_msg ( 0, "Failed to encode payload" ); - } - - toxav_send_audio(status_control.Alice.av, status_control.Alice.call_index, prepared_payload, payload_size); - - payload_size = toxav_prepare_audio_frame(status_control.Bob.av, status_control.Bob.call_index, prepared_payload, 1000, - sample_payload, frame_size); - - if ( payload_size < 0 ) { - ck_assert_msg ( 0, "Failed to encode payload" ); - } - - toxav_send_audio(status_control.Bob.av, status_control.Bob.call_index, prepared_payload, payload_size); - - if (time(NULL) - cur_time > 10) { /* Transmit for 10 seconds */ - step++; /* This terminates the loop */ - toxav_kill_transmission(status_control.Alice.av, status_control.Alice.call_index); - toxav_kill_transmission(status_control.Bob.av, status_control.Bob.call_index); - - /* Call over Alice hangs up */ - toxav_hangup(status_control.Alice.av, status_control.Alice.call_index); - } - } - TERMINATE_SCOPE() - - - /* - * Call with audio on both sides and video on one side. Alice calls Bob. - */ - CALL_AND_START_LOOP(TypeAudio, TypeVideo) { - /* Both send */ - payload_size = toxav_prepare_audio_frame(status_control.Alice.av, status_control.Alice.call_index, prepared_payload, - 1000, sample_payload, frame_size); - - if ( payload_size < 0 ) { - ck_assert_msg ( 0, "Failed to encode payload" ); - } - - toxav_send_audio(status_control.Alice.av, status_control.Alice.call_index, prepared_payload, payload_size); - - payload_size = toxav_prepare_audio_frame(status_control.Bob.av, status_control.Bob.call_index, prepared_payload, 1000, - sample_payload, frame_size); - - if ( payload_size < 0 ) { - ck_assert_msg ( 0, "Failed to encode payload" ); - } - - toxav_send_audio(status_control.Bob.av, status_control.Bob.call_index, prepared_payload, payload_size); - -// toxav_send_video(status_control.Bob.av, status_control.Bob.call_index, sample_image); - - if (time(NULL) - cur_time > 10) { /* Transmit for 10 seconds */ - step++; /* This terminates the loop */ - toxav_kill_transmission(status_control.Alice.av, status_control.Alice.call_index); - toxav_kill_transmission(status_control.Bob.av, status_control.Bob.call_index); - - /* Call over Alice hangs up */ - toxav_hangup(status_control.Alice.av, status_control.Alice.call_index); - } - } - TERMINATE_SCOPE() - - - /* - * Call with audio and video on both sides. Alice calls Bob. - */ - CALL_AND_START_LOOP(TypeVideo, TypeVideo) { - /* Both send */ - - payload_size = toxav_prepare_audio_frame(status_control.Alice.av, status_control.Alice.call_index, prepared_payload, - 1000, sample_payload, frame_size); - - if ( payload_size < 0 ) { - ck_assert_msg ( 0, "Failed to encode payload" ); - } - - toxav_send_audio(status_control.Alice.av, status_control.Alice.call_index, prepared_payload, payload_size); - - payload_size = toxav_prepare_audio_frame(status_control.Bob.av, status_control.Bob.call_index, prepared_payload, 1000, - sample_payload, frame_size); - - if ( payload_size < 0 ) { - ck_assert_msg ( 0, "Failed to encode payload" ); - } - - toxav_send_audio(status_control.Bob.av, status_control.Bob.call_index, prepared_payload, payload_size); - -// toxav_send_video(status_control.Alice.av, status_control.Alice.call_index, sample_image); -// toxav_send_video(status_control.Bob.av, status_control.Bob.call_index, sample_image); - - - if (time(NULL) - cur_time > 10) { /* Transmit for 10 seconds */ - step++; /* This terminates the loop */ - toxav_kill_transmission(status_control.Alice.av, status_control.Alice.call_index); - toxav_kill_transmission(status_control.Bob.av, status_control.Bob.call_index); - - /* Call over Alice hangs up */ - toxav_hangup(status_control.Alice.av, status_control.Alice.call_index); - } - } - TERMINATE_SCOPE() - - - uint64_t times_they_are_a_changin = time(NULL); - /* Media change */ - CALL_AND_START_LOOP(TypeAudio, TypeAudio) { - /* Both send */ - payload_size = toxav_prepare_audio_frame(status_control.Alice.av, status_control.Alice.call_index, prepared_payload, - 1000, sample_payload, frame_size); - - if ( payload_size < 0 ) { - ck_assert_msg ( 0, "Failed to encode payload" ); - } - - toxav_send_audio(status_control.Alice.av, status_control.Alice.call_index, prepared_payload, payload_size); - - payload_size = toxav_prepare_audio_frame(status_control.Bob.av, status_control.Bob.call_index, prepared_payload, 1000, - sample_payload, frame_size); - - if ( payload_size < 0 ) { - ck_assert_msg ( 0, "Failed to encode payload" ); - } - - toxav_send_audio(status_control.Bob.av, status_control.Bob.call_index, prepared_payload, payload_size); - - /* Wait 2 seconds and change transmission type */ - if (time(NULL) - times_they_are_a_changin > 2) { - times_they_are_a_changin = time(NULL); - muhcaps.audio_bitrate ++; - toxav_change_settings(status_control.Alice.av, status_control.Alice.call_index, &muhcaps); - } - - if (time(NULL) - cur_time > 10) { /* Transmit for 10 seconds */ - step++; /* This terminates the loop */ - toxav_kill_transmission(status_control.Alice.av, status_control.Alice.call_index); - toxav_kill_transmission(status_control.Bob.av, status_control.Bob.call_index); - - /* Call over Alice hangs up */ - toxav_hangup(status_control.Alice.av, status_control.Alice.call_index); - } - } - TERMINATE_SCOPE() - - - /************************************************************************************************* - * Other flows - */ - - /* - * Call and reject - */ + + { - int step = 0; - int running = 1; - - while (running) { - tox_iterate(bootstrap_node); - tox_iterate(Alice); - tox_iterate(Bob); - - switch ( step ) { - case 0: /* Alice */ - printf("Alice is calling...\n"); - toxav_call(status_control.Alice.av, &status_control.Alice.call_index, 0, &muhcaps, 10); - step++; - break; - - case 1: /* Bob */ - if (status_control.Bob.status == Ringing) { - printf("Bob rejects...\n"); - toxav_reject(status_control.Bob.av, status_control.Bob.call_index, "Who likes D's anyway?"); - step++; - } - - break; - - case 2: /* Wait for Both to have status ended */ - if (status_control.Alice.status == Rejected && status_control.Bob.status == Ended) running = 0; - - break; - } - - c_sleep(20); - } - - printf("\n"); + TOXAV_ERR_NEW error; + AliceAV = toxav_new(Alice, &error); + assert(error == TOXAV_ERR_NEW_OK); + + BobAV = toxav_new(Bob, &error); + assert(error == TOXAV_ERR_NEW_OK); } - - - /* - * Call and cancel - */ - { - int step = 0; - int running = 1; - - while (running) { - tox_iterate(bootstrap_node); - tox_iterate(Alice); - tox_iterate(Bob); - - toxav_do(status_control.Alice.av); - toxav_do(status_control.Bob.av); - - - switch ( step ) { - case 0: /* Alice */ - printf("Alice is calling...\n"); - toxav_call(status_control.Alice.av, &status_control.Alice.call_index, 0, &muhcaps, 10); - step++; - break; - - - case 1: /* Alice again */ - if (status_control.Bob.status == Ringing) { - printf("Alice cancels...\n"); - toxav_cancel(status_control.Alice.av, status_control.Alice.call_index, 0, "Who likes D's anyway?"); - step++; - } - - break; - - case 2: /* Wait for Both to have status ended */ - if (status_control.Bob.status == Canceled) running = 0; - - break; - } - - c_sleep(20); - } - - printf("\n"); + + toxav_callback_call(AliceAV, t_toxav_call_cb, &AliceCC); + toxav_callback_call_state(AliceAV, t_toxav_call_state_cb, &AliceCC); + toxav_callback_receive_video_frame(AliceAV, t_toxav_receive_video_frame_cb, &AliceCC); + toxav_callback_receive_audio_frame(AliceAV, t_toxav_receive_audio_frame_cb, &AliceCC); + + toxav_callback_call(BobAV, t_toxav_call_cb, &BobCC); + toxav_callback_call_state(BobAV, t_toxav_call_state_cb, &BobCC); + toxav_callback_receive_video_frame(BobAV, t_toxav_receive_video_frame_cb, &BobCC); + toxav_callback_receive_audio_frame(BobAV, t_toxav_receive_audio_frame_cb, &BobCC); + + printf("Created 2 instances of ToxAV\n"); + printf("All set after %llu seconds!\n", time(NULL) - cur_time); + + +#define REGULAR_CALL_FLOW(A_BR, V_BR) \ + do { \ + memset(&AliceCC, 0, sizeof(CallControl)); \ + memset(&BobCC, 0, sizeof(CallControl)); \ + \ + TOXAV_ERR_CALL rc; \ + toxav_call(AliceAV, 0, A_BR, V_BR, &rc); \ + \ + if (rc != TOXAV_ERR_CALL_OK) { \ + printf("toxav_call failed: %d\n", rc); \ + exit(1); \ + } \ + \ + \ + long long unsigned int start_time = time(NULL); \ + \ + \ + while (BobCC.state != TOXAV_CALL_STATE_END) { \ + \ + if (BobCC.incoming) { \ + TOXAV_ERR_ANSWER rc; \ + toxav_answer(BobAV, 0, A_BR, V_BR, &rc); \ + \ + if (rc != TOXAV_ERR_ANSWER_OK) { \ + printf("toxav_answer failed: %d\n", rc); \ + exit(1); \ + } \ + BobCC.incoming = false; \ + } else { \ + /* TODO rtp */ \ + \ + if (time(NULL) - start_time == 5) { \ + \ + TOXAV_ERR_CALL_CONTROL rc; \ + toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); \ + \ + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { \ + printf("toxav_call_control failed: %d\n", rc); \ + exit(1); \ + } \ + } \ + } \ + \ + iterate_tox(bootstrap, Alice, Bob); \ + } \ + printf("Success!\n");\ + } while(0) + + if (TEST_REGULAR_AV) { + printf("\nTrying regular call (Audio and Video)...\n"); + REGULAR_CALL_FLOW(48, 4000); } - - /* - * Timeout - */ - { - int step = 0; - int running = 1; - - while (running) { - tox_iterate(bootstrap_node); - tox_iterate(Alice); - tox_iterate(Bob); - - toxav_do(status_control.Alice.av); - toxav_do(status_control.Bob.av); - - switch ( step ) { - case 0: - printf("Alice is calling...\n"); - toxav_call(status_control.Alice.av, &status_control.Alice.call_index, 0, &muhcaps, 10); - step++; - break; - - case 1: - if (status_control.Alice.status == TimedOut) running = 0; - - break; - } - - c_sleep(20); - } - - printf("\n"); + + if (TEST_REGULAR_A) { + printf("\nTrying regular call (Audio only)...\n"); + REGULAR_CALL_FLOW(48, 0); } - - vpx_img_free(sample_image); - toxav_kill(status_control.Alice.av); - toxav_kill(status_control.Bob.av); - tox_kill(bootstrap_node); - tox_kill(Alice); + + if (TEST_REGULAR_V) { + printf("\nTrying regular call (Video only)...\n"); + REGULAR_CALL_FLOW(0, 4000); + } + +#undef REGULAR_CALL_FLOW + + if (TEST_REJECT) { /* Alice calls; Bob rejects */ + printf("\nTrying reject flow...\n"); + + memset(&AliceCC, 0, sizeof(CallControl)); + memset(&BobCC, 0, sizeof(CallControl)); + + { + TOXAV_ERR_CALL rc; + toxav_call(AliceAV, 0, 48, 0, &rc); + + if (rc != TOXAV_ERR_CALL_OK) { + printf("toxav_call failed: %d\n", rc); + exit(1); + } + } + + while (!BobCC.incoming) + iterate_tox(bootstrap, Alice, Bob); + + /* Reject */ + { + TOXAV_ERR_CALL_CONTROL rc; + toxav_call_control(BobAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); + + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { + printf("toxav_call_control failed: %d\n", rc); + exit(1); + } + } + + while (AliceCC.state != TOXAV_CALL_STATE_END) + iterate_tox(bootstrap, Alice, Bob); + + printf("Success!\n"); + } + + if (TEST_CANCEL) { /* Alice calls; Alice cancels while ringing */ + printf("\nTrying cancel (while ringing) flow...\n"); + + memset(&AliceCC, 0, sizeof(CallControl)); + memset(&BobCC, 0, sizeof(CallControl)); + + { + TOXAV_ERR_CALL rc; + toxav_call(AliceAV, 0, 48, 0, &rc); + + if (rc != TOXAV_ERR_CALL_OK) { + printf("toxav_call failed: %d\n", rc); + exit(1); + } + } + + while (!BobCC.incoming) + iterate_tox(bootstrap, Alice, Bob); + + /* Cancel */ + { + TOXAV_ERR_CALL_CONTROL rc; + toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); + + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { + printf("toxav_call_control failed: %d\n", rc); + exit(1); + } + } + + /* Alice will not receive end state */ + while (BobCC.state != TOXAV_CALL_STATE_END) + iterate_tox(bootstrap, Alice, Bob); + + printf("Success!\n"); + } + + if (TEST_MUTE_UNMUTE) { /* Check Mute-Unmute etc */ + printf("\nTrying mute functionality...\n"); + + memset(&AliceCC, 0, sizeof(CallControl)); + memset(&BobCC, 0, sizeof(CallControl)); + + /* Assume sending audio and video */ + { + TOXAV_ERR_CALL rc; + toxav_call(AliceAV, 0, 48, 1000, &rc); + + if (rc != TOXAV_ERR_CALL_OK) { + printf("toxav_call failed: %d\n", rc); + exit(1); + } + } + + while (!BobCC.incoming) + iterate_tox(bootstrap, Alice, Bob); + + /* At first try all stuff while in invalid state */ + assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); + assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); + assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); + assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); + + { + TOXAV_ERR_ANSWER rc; + toxav_answer(BobAV, 0, 48, 4000, &rc); + + if (rc != TOXAV_ERR_ANSWER_OK) { + printf("toxav_answer failed: %d\n", rc); + exit(1); + } + } + + iterate_tox(bootstrap, Alice, Bob); + + /* Pause and Resume */ + printf("Pause and Resume\n"); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); + iterate_tox(bootstrap, Alice, Bob); + assert(BobCC.state == TOXAV_CALL_STATE_PAUSED); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); + iterate_tox(bootstrap, Alice, Bob); + assert(BobCC.state & (TOXAV_CALL_STATE_SENDING_A | TOXAV_CALL_STATE_SENDING_V)); + + /* Mute/Unmute single */ + printf("Mute/Unmute single\n"); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); + iterate_tox(bootstrap, Alice, Bob); + assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); + iterate_tox(bootstrap, Alice, Bob); + assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); + + /* Mute/Unmute both */ + printf("Mute/Unmute both\n"); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); + iterate_tox(bootstrap, Alice, Bob); + assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); + iterate_tox(bootstrap, Alice, Bob); + assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_V); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); + iterate_tox(bootstrap, Alice, Bob); + assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); + assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); + iterate_tox(bootstrap, Alice, Bob); + assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_V); + + { + TOXAV_ERR_CALL_CONTROL rc; + toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); + + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { + printf("toxav_call_control failed: %d\n", rc); + exit(1); + } + } + + iterate_tox(bootstrap, Alice, Bob); + assert(BobCC.state == TOXAV_CALL_STATE_END); + + printf("Success!\n"); + } + + toxav_kill(BobAV); + toxav_kill(AliceAV); tox_kill(Bob); - - printf("Calls ended!\n"); + tox_kill(Alice); + tox_kill(bootstrap); + + printf("\nTest successful!\n"); } END_TEST -/*************************************************************************************************/ - - -/*************************************************************************************************/ - -/*************************************************************************************************/ - Suite *tox_suite(void) { Suite *s = suite_create("ToxAV"); DEFTESTCASE_SLOW(AV_flows, 200); - return s; } int main(int argc, char *argv[]) @@ -637,6 +444,4 @@ int main(int argc, char *argv[]) srunner_free(test_runner); return number_failed; - -// return test_AV_flows(); } diff --git a/auto_tests/toxav_many_test.c b/auto_tests/toxav_many_test.c index 4287118f7..ef59b2b2e 100644 --- a/auto_tests/toxav_many_test.c +++ b/auto_tests/toxav_many_test.c @@ -14,6 +14,7 @@ #include #include "../toxcore/tox.h" +#include "../toxcore/util.h" #include "../toxcore/logger.h" #include "../toxcore/crypto_core.h" #include "../toxav/toxav.h" @@ -26,359 +27,177 @@ #define c_sleep(x) usleep(1000*x) #endif -pthread_mutex_t muhmutex; -typedef enum _CallStatus { - none, - InCall, - Ringing, - Ended, - Rejected, - Canceled +typedef struct { + bool incoming; + uint32_t state; + +} CallControl; -} CallStatus; -typedef struct _Party { - CallStatus status; - ToxAv *av; - int id; -} Party; - -typedef struct _ACall { - pthread_t tid; - int idx; - - Party Caller; - Party Callee; -} ACall; - -typedef struct _Status { - ACall calls[3]; /* Make 3 calls for this test */ -} Status; - -Status status_control; - -void accept_friend_request(Tox *m, const uint8_t *public_key, const uint8_t *data, size_t length, void *userdata) +/** + * Callbacks + */ +void t_toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data) +{ + printf("Handling CALL callback\n"); + ((CallControl*)user_data)->incoming = true; +} +void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, void *user_data) +{ + printf("Handling CALL STATE callback: %d\n", state); + ((CallControl*)user_data)->state = state; +} +void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, + uint16_t width, uint16_t height, + uint8_t const *y, uint8_t const *u, uint8_t const *v, + int32_t ystride, int32_t ustride, int32_t vstride, + void *user_data) +{ + (void) av; + (void) friend_number; + (void) width; + (void) height; + (void) y; + (void) u; + (void) v; + (void) ystride; + (void) ustride; + (void) vstride; + (void) user_data; +} +void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, + int16_t const *pcm, + size_t sample_count, + uint8_t channels, + uint32_t sampling_rate, + void *user_data) +{ + (void) av; + (void) friend_number; + (void) pcm; + (void) sample_count; + (void) channels; + (void) sampling_rate; + (void) user_data; +} +void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, size_t length, void *userdata) { if (length == 7 && memcmp("gentoo", data, 7) == 0) { - tox_friend_add_norequest(m, public_key, 0); + assert(tox_friend_add_norequest(m, public_key, NULL) != (uint32_t) ~0); } } -/******************************************************************************/ -void callback_recv_invite ( void *av, int32_t call_index, void *_arg ) +/** + * Iterate helper + */ +ToxAV* setup_av_instance(Tox* tox, CallControl *CC) { - Status *cast = _arg; - cast->calls[call_index].Callee.status = Ringing; + TOXAV_ERR_NEW error; + + ToxAV* av = toxav_new(tox, &error); + assert(error == TOXAV_ERR_NEW_OK); + + toxav_callback_call(av, t_toxav_call_cb, CC); + toxav_callback_call_state(av, t_toxav_call_state_cb, CC); + toxav_callback_receive_video_frame(av, t_toxav_receive_video_frame_cb, CC); + toxav_callback_receive_audio_frame(av, t_toxav_receive_audio_frame_cb, CC); + + return av; } -void callback_recv_ringing ( void *av, int32_t call_index, void *_arg ) +void* call_thread(ToxAV* Alice, ToxAV* Bob) { - Status *cast = _arg; - cast->calls[call_index].Caller.status = Ringing; -} -void callback_call_ended ( void *av, int32_t call_index, void *_arg ) -{ - Status *cast = _arg; - - if (av == cast->calls[call_index].Caller.av) - cast->calls[call_index].Caller.status = Ended; - else - cast->calls[call_index].Callee.status = Ended; -} -void callback_call_started ( void *av, int32_t call_index, void *_arg ) -{ - Status *cast = _arg; - - if (av == cast->calls[call_index].Caller.av) - cast->calls[call_index].Caller.status = InCall; - else - cast->calls[call_index].Callee.status = InCall; -} -void callback_call_canceled ( void *av, int32_t call_index, void *_arg ) -{ -} -void callback_call_rejected ( void *av, int32_t call_index, void *_arg ) -{ - Status *cast = _arg; - cast->calls[call_index].Caller.status = Rejected; -} - -void callback_requ_timeout ( void *av, int32_t call_index, void *_arg ) -{ - ck_assert_msg(0, "No answer!"); -} - -void callback_audio (void *agent, int32_t call_idx, const int16_t *PCM, uint16_t size, void *data) -{} - -void callback_video (void *agent, int32_t call_idx, const vpx_image_t *img, void *data) -{} - -void register_callbacks(ToxAv *av, void *data) -{ - toxav_register_callstate_callback(av, callback_call_started, av_OnStart, data); - toxav_register_callstate_callback(av, callback_call_canceled, av_OnCancel, data); - toxav_register_callstate_callback(av, callback_call_rejected, av_OnReject, data); - toxav_register_callstate_callback(av, callback_call_ended, av_OnEnd, data); - toxav_register_callstate_callback(av, callback_recv_invite, av_OnInvite, data); - toxav_register_callstate_callback(av, callback_recv_ringing, av_OnRinging, data); - toxav_register_callstate_callback(av, callback_requ_timeout, av_OnRequestTimeout, data); - - - toxav_register_audio_callback(av, callback_audio, NULL); - toxav_register_video_callback(av, callback_video, NULL); -} -/*************************************************************************************************/ - -int call_running[3]; - -void *in_thread_call (void *arg) -{ -#define call_print(call, what, args...) printf("[%d] " what "\n", call, ##args) - - ACall *this_call = arg; - uint64_t start = 0; - int step = 0; - int call_idx; - - const int frame_size = (av_DefaultSettings.audio_sample_rate * av_DefaultSettings.audio_frame_duration / 1000); - int16_t sample_payload[frame_size]; - randombytes((uint8_t *)sample_payload, sizeof(int16_t) * frame_size); - - uint8_t prepared_payload[RTP_PAYLOAD_SIZE]; - - register_callbacks(this_call->Caller.av, &status_control); - register_callbacks(this_call->Callee.av, arg); - - /* NOTE: CALLEE WILL ALWAHYS NEED CALL_IDX == 0 */ - pthread_mutex_lock(&muhmutex); - - while (call_running[this_call->idx]) { - - pthread_mutex_unlock(&muhmutex); - - switch ( step ) { - case 0: /* CALLER */ - toxav_call(this_call->Caller.av, &call_idx, this_call->Callee.id, &av_DefaultSettings, 10); - call_print(call_idx, "Calling ..."); - step++; - break; - - case 1: /* CALLEE */ - pthread_mutex_lock(&muhmutex); - - if (this_call->Caller.status == Ringing) { - call_print(call_idx, "Callee answers ..."); - pthread_mutex_unlock(&muhmutex); - toxav_answer(this_call->Callee.av, 0, &av_DefaultSettings); - step++; - start = time(NULL); - pthread_mutex_lock(&muhmutex); - } - - pthread_mutex_unlock(&muhmutex); - break; - - case 2: /* Rtp transmission */ - pthread_mutex_lock(&muhmutex); - - if (this_call->Caller.status == InCall) { /* I think this is okay */ - call_print(call_idx, "Sending rtp ..."); - pthread_mutex_unlock(&muhmutex); - - c_sleep(1000); /* We have race condition here */ - toxav_prepare_transmission(this_call->Callee.av, 0, 1); - toxav_prepare_transmission(this_call->Caller.av, call_idx, 1); - - int payload_size = toxav_prepare_audio_frame(this_call->Caller.av, call_idx, prepared_payload, RTP_PAYLOAD_SIZE, - sample_payload, frame_size); - - if ( payload_size < 0 ) { - ck_assert_msg ( 0, "Failed to encode payload" ); - } - - - while (time(NULL) - start < 10) { /* 10 seconds */ - /* Both send */ - toxav_send_audio(this_call->Caller.av, call_idx, prepared_payload, payload_size); - - toxav_send_audio(this_call->Callee.av, 0, prepared_payload, payload_size); - - /* Both receive */ - int16_t storage[RTP_PAYLOAD_SIZE]; - int recved; - - c_sleep(20); - } - - step++; /* This terminates the loop */ - - pthread_mutex_lock(&muhmutex); - toxav_kill_transmission(this_call->Callee.av, 0); - toxav_kill_transmission(this_call->Caller.av, call_idx); - pthread_mutex_unlock(&muhmutex); - - /* Call over CALLER hangs up */ - toxav_hangup(this_call->Caller.av, call_idx); - call_print(call_idx, "Hanging up ..."); - - pthread_mutex_lock(&muhmutex); - } - - pthread_mutex_unlock(&muhmutex); - break; - - case 3: /* Wait for Both to have status ended */ - pthread_mutex_lock(&muhmutex); - - if (this_call->Caller.status == Ended) { - pthread_mutex_unlock(&muhmutex); - c_sleep(1000); /* race condition */ - pthread_mutex_lock(&muhmutex); - this_call->Callee.status = Ended; - call_running[this_call->idx] = 0; - } - - pthread_mutex_unlock(&muhmutex); - - break; - - } - - c_sleep(20); - - pthread_mutex_lock(&muhmutex); - } - - pthread_mutex_unlock(&muhmutex); - call_print(call_idx, "Call ended successfully!"); pthread_exit(NULL); } - - - - START_TEST(test_AV_three_calls) -// void test_AV_three_calls() { - long long unsigned int cur_time = time(NULL); - Tox *bootstrap_node = tox_new(0, 0, 0, 0); - Tox *caller = tox_new(0, 0, 0, 0); - Tox *callees[3] = { - tox_new(0, 0, 0, 0), - tox_new(0, 0, 0, 0), - tox_new(0, 0, 0, 0), - }; - - - ck_assert_msg(bootstrap_node != NULL, "Failed to create bootstrap node"); - + Tox* Alice, *bootstrap, *Bobs[3]; + ToxAV* AliceAV, *BobsAV[3]; + + CallControl AliceCC[3], BobsCC[3]; + int i = 0; - - for (; i < 3; i ++) { - ck_assert_msg(callees[i] != NULL, "Failed to create 3 tox instances"); - } - - for ( i = 0; i < 3; i ++ ) { - uint32_t to_compare = 974536; - tox_callback_friend_request(callees[i], accept_friend_request, &to_compare); - uint8_t address[TOX_ADDRESS_SIZE]; - tox_self_get_address(callees[i], address); - - uint32_t test = tox_friend_add(caller, address, (uint8_t *)"gentoo", 7, 0); - ck_assert_msg( test == i, "Failed to add friend error code: %i", test); - } - - uint8_t off = 1; - - while (1) { - tox_iterate(bootstrap_node); - tox_iterate(caller); - - for (i = 0; i < 3; i ++) { - tox_iterate(callees[i]); + { + TOX_ERR_NEW error; + + bootstrap = tox_new(NULL, NULL, 0, &error); + assert(error == TOX_ERR_NEW_OK); + + Alice = tox_new(NULL, NULL, 0, &error); + assert(error == TOX_ERR_NEW_OK); + + for (; i < 3; i ++) { + BobsAV[i] = tox_new(NULL, NULL, 0, &error); + assert(error == TOX_ERR_NEW_OK); } - - - if (tox_self_get_connection_status(bootstrap_node) && - tox_self_get_connection_status(caller) && - tox_self_get_connection_status(callees[0]) && - tox_self_get_connection_status(callees[1]) && - tox_self_get_connection_status(callees[2]) && off) { + } + + printf("Created 5 instances of Tox\n"); + printf("Preparing network...\n"); + long long unsigned int cur_time = time(NULL); + + uint32_t to_compare = 974536; + uint8_t address[TOX_ADDRESS_SIZE]; + + tox_callback_friend_request(Alice, t_accept_friend_request_cb, &to_compare); + tox_self_get_address(Alice, address); + + + assert(tox_friend_add(Bobs[0], address, (uint8_t *)"gentoo", 7, NULL) != (uint32_t) ~0); + assert(tox_friend_add(Bobs[1], address, (uint8_t *)"gentoo", 7, NULL) != (uint32_t) ~0); + assert(tox_friend_add(Bobs[2], address, (uint8_t *)"gentoo", 7, NULL) != (uint32_t) ~0); + + uint8_t off = 1; + + while (1) { + tox_iterate(bootstrap); + tox_iterate(Alice); + tox_iterate(Bobs[0]); + tox_iterate(Bobs[1]); + tox_iterate(Bobs[2]); + + if (tox_self_get_connection_status(bootstrap) && + tox_self_get_connection_status(Alice) && + tox_self_get_connection_status(Bobs[0]) && + tox_self_get_connection_status(Bobs[1]) && + tox_self_get_connection_status(Bobs[2]) && off) { printf("Toxes are online, took %llu seconds\n", time(NULL) - cur_time); off = 0; } - - - if (tox_friend_get_connection_status(caller, 0, 0) && - tox_friend_get_connection_status(caller, 1, 0) && - tox_friend_get_connection_status(caller, 2, 0) ) + + if (tox_friend_get_connection_status(Alice, 0, NULL) == TOX_CONNECTION_UDP && + tox_friend_get_connection_status(Alice, 1, NULL) == TOX_CONNECTION_UDP && + tox_friend_get_connection_status(Alice, 2, NULL) == TOX_CONNECTION_UDP && + tox_friend_get_connection_status(Bobs[0], 0, NULL) == TOX_CONNECTION_UDP && + tox_friend_get_connection_status(Bobs[1], 0, NULL) == TOX_CONNECTION_UDP && + tox_friend_get_connection_status(Bobs[2], 0, NULL) == TOX_CONNECTION_UDP) break; - + c_sleep(20); } - - printf("All set after %llu seconds! Starting call...\n", time(NULL) - cur_time); - - ToxAv *uniqcallerav = toxav_new(caller, 3); - + + AliceAV = setup_av_instance(Alice, &AliceCC); + BobsAV[0] = setup_av_instance(Bobs[0], &BobsCC[0]); + BobsAV[1] = setup_av_instance(Bobs[1], &BobsCC[1]); + BobsAV[2] = setup_av_instance(Bobs[2], &BobsCC[2]); + + printf("Created 4 instances of ToxAV\n"); + printf("All set after %llu seconds!\n", time(NULL) - cur_time); + + + + tox_kill(bootstrap); + tox_kill(Alice); + toxav_kill(AliceAV); + for (i = 0; i < 3; i ++) { - status_control.calls[i].idx = i; - - status_control.calls[i].Caller.av = uniqcallerav; - status_control.calls[i].Caller.id = 0; - status_control.calls[i].Caller.status = none; - - status_control.calls[i].Callee.av = toxav_new(callees[i], 1); - status_control.calls[i].Callee.id = i; - status_control.calls[i].Callee.status = none; + tox_kill(Bobs[i]); + toxav_kill(BobsAV[i]); } - - pthread_mutex_init(&muhmutex, NULL); - - for ( i = 0; i < 3; i++ ) { - call_running[i] = 1; - pthread_create(&status_control.calls[i].tid, NULL, in_thread_call, &status_control.calls[i]); - } - - /* Now start 3 calls and they'll run for 10 s */ - - for ( i = 0; i < 3; i++ ) - pthread_detach(status_control.calls[i].tid); - - while (call_running[0] || call_running[1] || call_running[2]) { - pthread_mutex_lock(&muhmutex); - - tox_iterate(bootstrap_node); - tox_iterate(caller); - tox_iterate(callees[0]); - tox_iterate(callees[1]); - tox_iterate(callees[2]); - - for ( i = 0; i < 3; i++ ) - toxav_do(status_control.calls[0].Caller.av); - - toxav_do(status_control.calls[0].Callee.av); - toxav_do(status_control.calls[1].Callee.av); - toxav_do(status_control.calls[2].Callee.av); - - pthread_mutex_unlock(&muhmutex); - c_sleep(20); - } - - toxav_kill(status_control.calls[0].Caller.av); - toxav_kill(status_control.calls[0].Callee.av); - toxav_kill(status_control.calls[1].Callee.av); - toxav_kill(status_control.calls[2].Callee.av); - - tox_kill(bootstrap_node); - tox_kill(caller); - - for ( i = 0; i < 3; i ++) - tox_kill(callees[i]); - + + printf("\nTest successful!\n"); } END_TEST @@ -410,8 +229,4 @@ int main(int argc, char *argv[]) srunner_free(test_runner); return number_failed; - -// test_AV_three_calls(); - -// return 0; } diff --git a/toxav/av_test.c b/testing/av_test.c similarity index 70% rename from toxav/av_test.c rename to testing/av_test.c index 86deebdf6..7298ed235 100644 --- a/toxav/av_test.c +++ b/testing/av_test.c @@ -70,13 +70,6 @@ #define YUV2B(Y, U, V) CLIP(( 298 * C(Y) + 516 * D(U) + 128) >> 8) -/* Enable/disable tests */ -#define TEST_REGULAR_AV 0 -#define TEST_REGULAR_A 0 -#define TEST_REGULAR_V 0 -#define TEST_REJECT 0 -#define TEST_CANCEL 0 -#define TEST_MUTE_UNMUTE 0 #define TEST_TRANSFER_A 0 #define TEST_TRANSFER_V 1 @@ -126,7 +119,7 @@ void* pa_write_thread (void* d) } -/** +/** * Callbacks */ void t_toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data) @@ -187,21 +180,21 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, free(rb_write(cc->arb, f)); pthread_mutex_unlock(cc->arb_mutex); } -void t_toxav_audio_bitrate_control_cb(ToxAV *av, uint32_t friend_number, - bool good, uint32_t bit_rate, void *user_data) +void t_toxav_audio_bit_rate_status_cb(ToxAV *av, uint32_t friend_number, + bool stable, uint32_t bit_rate, void *user_data) { - if (good) - printf ("Set new audio bitrate to: %d\n", bit_rate); + if (stable) + printf ("Set new audio bit rate to: %d\n", bit_rate); else - printf ("The network is overly saturated with audio bitrate at: %d\n", bit_rate); + printf ("The network is overly saturated with audio bit rate at: %d\n", bit_rate); } -void t_toxav_video_bitrate_control_cb(ToxAV *av, uint32_t friend_number, - bool good, uint32_t bit_rate, void *user_data) +void t_toxav_video_bit_rate_status_cb(ToxAV *av, uint32_t friend_number, + bool stable, uint32_t bit_rate, void *user_data) { - if (good) - printf ("Set new video bitrate to: %d", bit_rate); + if (stable) + printf ("Set new video bit rate to: %d", bit_rate); else - printf ("The network is overly saturated with video bitrate at: %d", bit_rate); + printf ("The network is overly saturated with video bit rate at: %d", bit_rate); } void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, size_t length, void *userdata) { @@ -287,16 +280,16 @@ void initialize_tox(Tox** bootstrap, ToxAV** AliceAV, CallControl* AliceCC, ToxA toxav_callback_call_state(*AliceAV, t_toxav_call_state_cb, AliceCC); toxav_callback_receive_video_frame(*AliceAV, t_toxav_receive_video_frame_cb, AliceCC); toxav_callback_receive_audio_frame(*AliceAV, t_toxav_receive_audio_frame_cb, AliceCC); - toxav_callback_audio_bitrate_control(*AliceAV, t_toxav_audio_bitrate_control_cb, AliceCC); - toxav_callback_video_bitrate_control(*AliceAV, t_toxav_video_bitrate_control_cb, AliceCC); + toxav_callback_audio_bit_rate_status(*AliceAV, t_toxav_audio_bit_rate_status_cb, AliceCC); + toxav_callback_video_bit_rate_status(*AliceAV, t_toxav_video_bit_rate_status_cb, AliceCC); /* Bob */ toxav_callback_call(*BobAV, t_toxav_call_cb, BobCC); toxav_callback_call_state(*BobAV, t_toxav_call_state_cb, BobCC); toxav_callback_receive_video_frame(*BobAV, t_toxav_receive_video_frame_cb, BobCC); toxav_callback_receive_audio_frame(*BobAV, t_toxav_receive_audio_frame_cb, BobCC); - toxav_callback_audio_bitrate_control(*BobAV, t_toxav_audio_bitrate_control_cb, BobCC); - toxav_callback_video_bitrate_control(*BobAV, t_toxav_video_bitrate_control_cb, BobCC); + toxav_callback_audio_bit_rate_status(*BobAV, t_toxav_audio_bit_rate_status_cb, BobCC); + toxav_callback_video_bit_rate_status(*BobAV, t_toxav_video_bit_rate_status_cb, BobCC); printf("Created 2 instances of ToxAV\n"); @@ -513,232 +506,7 @@ int main (int argc, char** argv) initialize_tox(&bootstrap, &AliceAV, &AliceCC, &BobAV, &BobCC); -#define REGULAR_CALL_FLOW(A_BR, V_BR) \ - do { \ - memset(&AliceCC, 0, sizeof(CallControl)); \ - memset(&BobCC, 0, sizeof(CallControl)); \ - \ - TOXAV_ERR_CALL rc; \ - toxav_call(AliceAV, 0, A_BR, V_BR, &rc); \ - \ - if (rc != TOXAV_ERR_CALL_OK) { \ - printf("toxav_call failed: %d\n", rc); \ - exit(1); \ - } \ - \ - \ - long long unsigned int start_time = time(NULL); \ - \ - \ - while (BobCC.state != TOXAV_CALL_STATE_END) { \ - \ - if (BobCC.incoming) { \ - TOXAV_ERR_ANSWER rc; \ - toxav_answer(BobAV, 0, A_BR, V_BR, &rc); \ - \ - if (rc != TOXAV_ERR_ANSWER_OK) { \ - printf("toxav_answer failed: %d\n", rc); \ - exit(1); \ - } \ - BobCC.incoming = false; \ - } else { \ - /* TODO rtp */ \ - \ - if (time(NULL) - start_time == 5) { \ - \ - TOXAV_ERR_CALL_CONTROL rc; \ - toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); \ - \ - if (rc != TOXAV_ERR_CALL_CONTROL_OK) { \ - printf("toxav_call_control failed: %d\n", rc); \ - exit(1); \ - } \ - } \ - } \ - \ - iterate_tox(bootstrap, AliceAV, BobAV); \ - } \ - printf("Success!\n");\ - } while(0) - - if (TEST_REGULAR_AV) { - printf("\nTrying regular call (Audio and Video)...\n"); - REGULAR_CALL_FLOW(48, 4000); - } - - if (TEST_REGULAR_A) { - printf("\nTrying regular call (Audio only)...\n"); - REGULAR_CALL_FLOW(48, 0); - } - - if (TEST_REGULAR_V) { - printf("\nTrying regular call (Video only)...\n"); - REGULAR_CALL_FLOW(0, 4000); - } - -#undef REGULAR_CALL_FLOW - - if (TEST_REJECT) { /* Alice calls; Bob rejects */ - printf("\nTrying reject flow...\n"); - - memset(&AliceCC, 0, sizeof(CallControl)); - memset(&BobCC, 0, sizeof(CallControl)); - - { - TOXAV_ERR_CALL rc; - toxav_call(AliceAV, 0, 48, 0, &rc); - - if (rc != TOXAV_ERR_CALL_OK) { - printf("toxav_call failed: %d\n", rc); - exit(1); - } - } - - while (!BobCC.incoming) - iterate_tox(bootstrap, AliceAV, BobAV); - - /* Reject */ - { - TOXAV_ERR_CALL_CONTROL rc; - toxav_call_control(BobAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); - - if (rc != TOXAV_ERR_CALL_CONTROL_OK) { - printf("toxav_call_control failed: %d\n", rc); - exit(1); - } - } - - while (AliceCC.state != TOXAV_CALL_STATE_END) - iterate_tox(bootstrap, AliceAV, BobAV); - - printf("Success!\n"); - } - - if (TEST_CANCEL) { /* Alice calls; Alice cancels while ringing */ - printf("\nTrying cancel (while ringing) flow...\n"); - - memset(&AliceCC, 0, sizeof(CallControl)); - memset(&BobCC, 0, sizeof(CallControl)); - - { - TOXAV_ERR_CALL rc; - toxav_call(AliceAV, 0, 48, 0, &rc); - - if (rc != TOXAV_ERR_CALL_OK) { - printf("toxav_call failed: %d\n", rc); - exit(1); - } - } - - while (!BobCC.incoming) - iterate_tox(bootstrap, AliceAV, BobAV); - - /* Cancel */ - { - TOXAV_ERR_CALL_CONTROL rc; - toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); - - if (rc != TOXAV_ERR_CALL_CONTROL_OK) { - printf("toxav_call_control failed: %d\n", rc); - exit(1); - } - } - - /* Alice will not receive end state */ - while (BobCC.state != TOXAV_CALL_STATE_END) - iterate_tox(bootstrap, AliceAV, BobAV); - - printf("Success!\n"); - } - - if (TEST_MUTE_UNMUTE) { /* Check Mute-Unmute etc */ - printf("\nTrying mute functionality...\n"); - - memset(&AliceCC, 0, sizeof(CallControl)); - memset(&BobCC, 0, sizeof(CallControl)); - - /* Assume sending audio and video */ - { - TOXAV_ERR_CALL rc; - toxav_call(AliceAV, 0, 48, 1000, &rc); - - if (rc != TOXAV_ERR_CALL_OK) { - printf("toxav_call failed: %d\n", rc); - exit(1); - } - } - - while (!BobCC.incoming) - iterate_tox(bootstrap, AliceAV, BobAV); - - /* At first try all stuff while in invalid state */ - assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); - assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); - assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); - - { - TOXAV_ERR_ANSWER rc; - toxav_answer(BobAV, 0, 48, 4000, &rc); - - if (rc != TOXAV_ERR_ANSWER_OK) { - printf("toxav_answer failed: %d\n", rc); - exit(1); - } - } - - iterate_tox(bootstrap, AliceAV, BobAV); - - /* Pause and Resume */ - printf("Pause and Resume\n"); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); - iterate_tox(bootstrap, AliceAV, BobAV); - assert(BobCC.state == TOXAV_CALL_STATE_PAUSED); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); - iterate_tox(bootstrap, AliceAV, BobAV); - assert(BobCC.state & (TOXAV_CALL_STATE_SENDING_A | TOXAV_CALL_STATE_SENDING_V)); - - /* Mute/Unmute single */ - printf("Mute/Unmute single\n"); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - iterate_tox(bootstrap, AliceAV, BobAV); - assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - iterate_tox(bootstrap, AliceAV, BobAV); - assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); - - /* Mute/Unmute both */ - printf("Mute/Unmute both\n"); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - iterate_tox(bootstrap, AliceAV, BobAV); - assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); - iterate_tox(bootstrap, AliceAV, BobAV); - assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_V); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - iterate_tox(bootstrap, AliceAV, BobAV); - assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); - iterate_tox(bootstrap, AliceAV, BobAV); - assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_V); - - { - TOXAV_ERR_CALL_CONTROL rc; - toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); - - if (rc != TOXAV_ERR_CALL_CONTROL_OK) { - printf("toxav_call_control failed: %d\n", rc); - exit(1); - } - } - - iterate_tox(bootstrap, AliceAV, BobAV); - assert(BobCC.state == TOXAV_CALL_STATE_END); - - printf("Success!\n"); - } - - if (TEST_TRANSFER_A) { /* Audio encoding/decoding and transfer */ + if (TEST_TRANSFER_A) { SNDFILE* af_handle; SF_INFO af_info; diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc index 0434a3c6a..3907951e3 100644 --- a/toxav/Makefile.inc +++ b/toxav/Makefile.inc @@ -37,25 +37,4 @@ libtoxav_la_LIBADD = libtoxcore.la \ $(PTHREAD_LIBS) \ $(AV_LIBS) - - -#noinst_PROGRAMS += av_test - -#av_test_SOURCES = ../toxav/av_test.c - -#av_test_CFLAGS = $(LIBSODIUM_CFLAGS) \ - $(NACL_CFLAGS) - -#av_test_LDADD = $(LIBSODIUM_LDFLAGS) \ - $(NACL_LDFLAGS) \ - libtoxav.la \ - libtoxcore.la \ - $(LIBSODIUM_LIBS) \ - $(NACL_OBJECTS) \ - -lopenal \ - -lopencv_calib3d -lopencv_contrib -lopencv_core -lopencv_features2d -lopencv_flann -lopencv_gpu -lopencv_highgui -lopencv_imgproc -lopencv_legacy -lopencv_ml -lopencv_objdetect -lopencv_ocl -lopencv_photo -lopencv_stitching -lopencv_superres -lopencv_ts -lopencv_video -lopencv_videostab \ - -lsndfile \ - $(NACL_LIBS) - - endif \ No newline at end of file diff --git a/toxav/audio.c b/toxav/audio.c index 2f068c857..c592a7dac 100644 --- a/toxav/audio.c +++ b/toxav/audio.c @@ -31,14 +31,14 @@ static void jbuf_clear(struct JitterBuffer *q); static void jbuf_free(struct JitterBuffer *q); static int jbuf_write(struct JitterBuffer *q, RTPMessage *m); static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success); - -OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count); +OpusEncoder* create_audio_encoder (int32_t bit_rate, int32_t sampling_rate, int32_t channel_count); bool reconfigure_audio_encoder(OpusEncoder** e, int32_t new_br, int32_t new_sr, uint8_t new_ch, int32_t *old_br, int32_t *old_sr, int32_t *old_ch); bool reconfigure_audio_decoder(ACSession* ac, int32_t sampling_rate, int8_t channels); -ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *cb, void *cb_data) + +ACSession* ac_new(ToxAV* av, uint32_t friend_number, toxav_receive_audio_frame_cb *cb, void *cb_data) { ACSession *ac = calloc(sizeof(ACSession), 1); @@ -78,11 +78,11 @@ ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *c goto DECODER_CLEANUP; } - ac->last_encoding_bitrate = 48000; + ac->last_encoding_bit_rate = 48000; ac->last_encoding_sampling_rate = 48000; ac->last_encoding_channel_count = 2; - ac->last_test_encoding_bitrate = 48000; + ac->last_test_encoding_bit_rate = 48000; ac->last_test_encoding_sampling_rate = 48000; ac->last_test_encoding_channel_count = 2; @@ -97,7 +97,7 @@ ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *c ac->last_packet_channel_count = 1; ac->av = av; - ac->friend_id = friend_id; + ac->friend_number = friend_number; ac->acb.first = cb; ac->acb.second = cb_data; @@ -181,7 +181,7 @@ void ac_do(ACSession* ac) } else if (ac->acb.first) { ac->last_packet_frame_duration = (rc * 1000) / ac->last_packet_sampling_rate; - ac->acb.first(ac->av, ac->friend_id, tmp, rc * ac->last_packet_channel_count, + ac->acb.first(ac->av, ac->friend_number, tmp, rc * ac->last_packet_channel_count, ac->last_packet_channel_count, ac->last_packet_sampling_rate, ac->acb.second); } @@ -220,28 +220,27 @@ int ac_queue_message(void* acp, struct RTPMessage_s *msg) return 0; } -int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels) +int ac_reconfigure_encoder(ACSession* ac, int32_t bit_rate, int32_t sampling_rate, uint8_t channels) { - if (!ac || !reconfigure_audio_encoder(&ac->encoder, bitrate, sampling_rate, channels, - &ac->last_encoding_bitrate, &ac->last_encoding_sampling_rate, &ac->last_encoding_channel_count)) + if (!ac || !reconfigure_audio_encoder(&ac->encoder, bit_rate, sampling_rate, channels, + &ac->last_encoding_bit_rate, &ac->last_encoding_sampling_rate, &ac->last_encoding_channel_count)) return -1; - LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels); + LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bit_rate, sampling_rate, channels); return 0; } -int ac_reconfigure_test_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels) +int ac_reconfigure_test_encoder(ACSession* ac, int32_t bit_rate, int32_t sampling_rate, uint8_t channels) { - if (!ac || !reconfigure_audio_encoder(&ac->test_encoder, bitrate, sampling_rate, channels, - &ac->last_encoding_bitrate, &ac->last_encoding_sampling_rate, &ac->last_encoding_channel_count)) + if (!ac || !reconfigure_audio_encoder(&ac->test_encoder, bit_rate, sampling_rate, channels, + &ac->last_encoding_bit_rate, &ac->last_encoding_sampling_rate, &ac->last_encoding_channel_count)) return -1; - LOGGER_DEBUG ("Reconfigured test audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels); + LOGGER_DEBUG ("Reconfigured test audio encoder br: %d sr: %d cc:%d", bit_rate, sampling_rate, channels); return 0; } -/* JITTER BUFFER WORK */ struct JitterBuffer { RTPMessage **queue; uint32_t size; @@ -340,7 +339,7 @@ static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success) *success = 0; return NULL; } -OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count) +OpusEncoder* create_audio_encoder (int32_t bit_rate, int32_t sampling_rate, int32_t channel_count) { int status = OPUS_OK; OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_AUDIO, &status); @@ -350,7 +349,7 @@ OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32 return NULL; } - status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bitrate)); + status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bit_rate)); if ( status != OPUS_OK ) { LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); diff --git a/toxav/audio.h b/toxav/audio.h index a36396f15..c003bac01 100644 --- a/toxav/audio.h +++ b/toxav/audio.h @@ -31,18 +31,21 @@ struct RTPMessage_s; +/* + * Base Audio Codec session type. + */ typedef struct ACSession_s { /* encoding */ OpusEncoder *encoder; int32_t last_encoding_sampling_rate; int32_t last_encoding_channel_count; - int32_t last_encoding_bitrate; + int32_t last_encoding_bit_rate; - /* Testing encoder for dynamic bitrate streaming */ + /* Testing encoder for dynamic bit rate streaming */ OpusEncoder *test_encoder; int32_t last_test_encoding_sampling_rate; int32_t last_test_encoding_channel_count; - int32_t last_test_encoding_bitrate; + int32_t last_test_encoding_bit_rate; /* decoding */ OpusDecoder *decoder; @@ -57,14 +60,30 @@ typedef struct ACSession_s { pthread_mutex_t queue_mutex[1]; ToxAV* av; - uint32_t friend_id; + uint32_t friend_number; PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ } ACSession; -ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *cb, void *cb_data); +/* + * Create new Audio Codec session. + */ +ACSession* ac_new(ToxAV* av, uint32_t friend_number, toxav_receive_audio_frame_cb *cb, void *cb_data); +/* + * Kill the Audio Codec session. + */ void ac_kill(ACSession* ac); +/* + * Do periodic work. Work is consisted out of decoding only. + */ void ac_do(ACSession* ac); +/* + * Queue new rtp message. + */ int ac_queue_message(void *acp, struct RTPMessage_s *msg); -int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels); -int ac_reconfigure_test_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels); +/* + * Set new values to the encoders. + */ +int ac_reconfigure_encoder(ACSession* ac, int32_t bit_rate, int32_t sampling_rate, uint8_t channels); +int ac_reconfigure_test_encoder(ACSession* ac, int32_t bit_rate, int32_t sampling_rate, uint8_t channels); + #endif /* AUDIO_H */ \ No newline at end of file diff --git a/toxav/msi.c b/toxav/msi.c index 0bd04c568..f8bc8451b 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -75,37 +75,37 @@ typedef struct { MSIHeaderRequest request; MSIHeaderError error; MSIHeaderCapabilities capabilities; - MSIHeaderVFPSZ vfpsz; /* Video frame piece size. NOTE: Value must be in network b-order */ + MSIHeaderVFPSZ vfpsz; /* Video frame piece size. NOTE: Value must be in network b-order TODO: get rid of this eventually */ } MSIMessage; void msg_init (MSIMessage *dest, MSIRequest request); int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ); uint8_t *msg_parse_header_out ( MSIHeaderID id, uint8_t *dest, const void *value, uint8_t value_len, uint16_t *length ); -int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg ); -int send_error ( Messenger* m, uint32_t friend_id, MSIError error ); +int send_message ( Messenger* m, uint32_t friend_number, const MSIMessage *msg ); +int send_error ( Messenger* m, uint32_t friend_number, MSIError error ); static int invoke_callback(MSICall* call, MSICallbackID cb); -static MSICall *get_call ( MSISession *session, uint32_t friend_id ); -MSICall *new_call ( MSISession *session, uint32_t friend_id ); +static MSICall *get_call ( MSISession *session, uint32_t friend_number ); +MSICall *new_call ( MSISession *session, uint32_t friend_number ); void kill_call ( MSICall *call ); -void on_peer_status(Messenger *m, uint32_t friend_id, uint8_t status, void *data); +void on_peer_status(Messenger *m, uint32_t friend_number, uint8_t status, void *data); void handle_push ( MSICall *call, const MSIMessage *msg ); void handle_pop ( MSICall *call, const MSIMessage *msg ); -void handle_msi_packet ( Messenger *m, uint32_t friend_id, const uint8_t *data, uint16_t length, void *object ); +void handle_msi_packet ( Messenger *m, uint32_t friend_number, const uint8_t *data, uint16_t length, void *object ); /** * Public functions */ -void msi_register_callback ( MSISession *session, MSICallbackType callback, MSICallbackID id) +void msi_register_callback ( MSISession* session, msi_action_cb* callback, MSICallbackID id) { pthread_mutex_lock(session->mutex); session->callbacks[id] = callback; pthread_mutex_unlock(session->mutex); } -MSISession *msi_new ( Messenger *messenger ) +MSISession *msi_new ( Messenger *m ) { - if (messenger == NULL) { + if (m == NULL) { LOGGER_ERROR("Could not init session on empty messenger!"); return NULL; } @@ -123,12 +123,12 @@ MSISession *msi_new ( Messenger *messenger ) return NULL; } - retu->messenger = messenger; + retu->messenger = m; - m_callback_msi_packet(messenger, handle_msi_packet, retu ); + m_callback_msi_packet(m, handle_msi_packet, retu ); /* This is called when remote terminates session */ - m_callback_connectionstatus_internal_av(messenger, on_peer_status, retu); + m_callback_connectionstatus_internal_av(m, on_peer_status, retu); LOGGER_DEBUG("New msi session: %p ", retu); return retu; @@ -149,7 +149,7 @@ int msi_kill ( MSISession *session ) MSICall* it = get_call(session, session->calls_head); for (; it; it = it->next) { - send_message(session->messenger, it->friend_id, &msg); + send_message(session->messenger, it->friend_number, &msg); kill_call(it); /* This will eventually free session->calls */ } } @@ -161,18 +161,18 @@ int msi_kill ( MSISession *session ) free ( session ); return 0; } -int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_t capabilities ) +int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_number, uint8_t capabilities ) { - LOGGER_DEBUG("Session: %p Inviting friend: %u", session, friend_id); + LOGGER_DEBUG("Session: %p Inviting friend: %u", session, friend_number); pthread_mutex_lock(session->mutex); - if (get_call(session, friend_id) != NULL) { + if (get_call(session, friend_number) != NULL) { LOGGER_ERROR("Already in a call"); pthread_mutex_unlock(session->mutex); return -1; } - (*call) = new_call ( session, friend_id ); + (*call) = new_call ( session, friend_number ); if ( *call == NULL ) { pthread_mutex_unlock(session->mutex); @@ -190,7 +190,7 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ msg.vfpsz.exists = true; msg.vfpsz.value = VIDEOFRAME_PIECE_SIZE; - send_message ( (*call)->session->messenger, (*call)->friend_id, &msg ); + send_message ( (*call)->session->messenger, (*call)->friend_number, &msg ); (*call)->state = msi_CallRequesting; @@ -200,7 +200,7 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ } int msi_hangup ( MSICall* call ) { - LOGGER_DEBUG("Session: %p Hanging up call with friend: %u", call->session, call->friend_id); + LOGGER_DEBUG("Session: %p Hanging up call with friend: %u", call->session, call->friend_number); MSISession* session = call->session; pthread_mutex_lock(session->mutex); @@ -208,7 +208,7 @@ int msi_hangup ( MSICall* call ) MSIMessage msg; msg_init(&msg, requ_pop); - send_message ( session->messenger, call->friend_id, &msg ); + send_message ( session->messenger, call->friend_number, &msg ); kill_call(call); pthread_mutex_unlock(session->mutex); @@ -216,7 +216,7 @@ int msi_hangup ( MSICall* call ) } int msi_answer ( MSICall* call, uint8_t capabilities ) { - LOGGER_DEBUG("Session: %p Answering call from: %u", call->session, call->friend_id); + LOGGER_DEBUG("Session: %p Answering call from: %u", call->session, call->friend_number); MSISession* session = call->session; pthread_mutex_lock(session->mutex); @@ -240,7 +240,7 @@ int msi_answer ( MSICall* call, uint8_t capabilities ) msg.vfpsz.exists = true; msg.vfpsz.value = VIDEOFRAME_PIECE_SIZE; - send_message ( session->messenger, call->friend_id, &msg ); + send_message ( session->messenger, call->friend_number, &msg ); call->state = msi_CallActive; pthread_mutex_unlock(session->mutex); @@ -249,7 +249,7 @@ int msi_answer ( MSICall* call, uint8_t capabilities ) } int msi_change_capabilities( MSICall* call, uint8_t capabilities ) { - LOGGER_DEBUG("Session: %p Trying to change capabilities to friend %u", call->session, call->friend_id); + LOGGER_DEBUG("Session: %p Trying to change capabilities to friend %u", call->session, call->friend_number); MSISession* session = call->session; pthread_mutex_lock(session->mutex); @@ -275,7 +275,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) msg.capabilities.exists = true; msg.capabilities.value = capabilities; - send_message ( call->session->messenger, call->friend_id, &msg ); + send_message ( call->session->messenger, call->friend_number, &msg ); pthread_mutex_unlock(session->mutex); return 0; @@ -394,7 +394,7 @@ uint8_t *msg_parse_header_out ( MSIHeaderID id, uint8_t *dest, const void *value return dest + value_len; /* Set to next position ready to be written */ } -int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg ) +int send_message ( Messenger* m, uint32_t friend_number, const MSIMessage *msg ) { /* Parse and send message */ assert(m); @@ -438,19 +438,19 @@ int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg ) *it = 0; size ++; - if ( m_msi_packet(m, friend_id, parsed, size) ) { + if ( m_msi_packet(m, friend_number, parsed, size) ) { LOGGER_DEBUG("Sent message"); return 0; } return -1; } -int send_error ( Messenger* m, uint32_t friend_id, MSIError error ) +int send_error ( Messenger* m, uint32_t friend_number, MSIError error ) { /* Send error message */ assert(m); - LOGGER_DEBUG("Sending error: %d to friend: %d", error, friend_id); + LOGGER_DEBUG("Sending error: %d to friend: %d", error, friend_number); MSIMessage msg; msg_init(&msg, requ_pop); @@ -458,7 +458,7 @@ int send_error ( Messenger* m, uint32_t friend_id, MSIError error ) msg.error.exists = true; msg.error.value = error; - send_message ( m, friend_id, &msg ); + send_message ( m, friend_number, &msg ); return 0; } int invoke_callback(MSICall* call, MSICallbackID cb) @@ -484,16 +484,16 @@ FAILURE: call->error = msi_EHandle; return -1; } -static MSICall *get_call ( MSISession *session, uint32_t friend_id ) +static MSICall *get_call ( MSISession *session, uint32_t friend_number ) { assert(session); - if (session->calls == NULL || session->calls_tail < friend_id) + if (session->calls == NULL || session->calls_tail < friend_number) return NULL; - return session->calls[friend_id]; + return session->calls[friend_number]; } -MSICall *new_call ( MSISession *session, uint32_t friend_id ) +MSICall *new_call ( MSISession *session, uint32_t friend_number ) { assert(session); @@ -503,20 +503,20 @@ MSICall *new_call ( MSISession *session, uint32_t friend_id ) return NULL; rc->session = session; - rc->friend_id = friend_id; + rc->friend_number = friend_number; if (session->calls == NULL) { /* Creating */ - session->calls = calloc (sizeof(MSICall*), friend_id + 1); + session->calls = calloc (sizeof(MSICall*), friend_number + 1); if (session->calls == NULL) { free(rc); return NULL; } - session->calls_tail = session->calls_head = friend_id; + session->calls_tail = session->calls_head = friend_number; - } else if (session->calls_tail < friend_id) { /* Appending */ - void* tmp = realloc(session->calls, sizeof(MSICall*) * friend_id + 1); + } else if (session->calls_tail < friend_number) { /* Appending */ + void* tmp = realloc(session->calls, sizeof(MSICall*) * friend_number + 1); if (tmp == NULL) { free(rc); @@ -526,22 +526,22 @@ MSICall *new_call ( MSISession *session, uint32_t friend_id ) session->calls = tmp; /* Set fields in between to null */ - int32_t i = session->calls_tail; - for (; i < friend_id; i ++) + int32_t i = session->calls_tail + 1; + for (; i < friend_number; i ++) session->calls[i] = NULL; rc->prev = session->calls[session->calls_tail]; session->calls[session->calls_tail]->next = rc; - session->calls_tail = friend_id; + session->calls_tail = friend_number; - } else if (session->calls_head > friend_id) { /* Inserting at front */ + } else if (session->calls_head > friend_number) { /* Inserting at front */ rc->next = session->calls[session->calls_head]; session->calls[session->calls_head]->prev = rc; - session->calls_head = friend_id; + session->calls_head = friend_number; } - session->calls[friend_id] = rc; + session->calls[friend_number] = rc; return rc; } void kill_call ( MSICall *call ) @@ -560,16 +560,16 @@ void kill_call ( MSICall *call ) if (prev) prev->next = next; else if (next) - session->calls_head = next->friend_id; + session->calls_head = next->friend_number; else goto CLEAR_CONTAINER; if (next) next->prev = prev; else if (prev) - session->calls_tail = prev->friend_id; + session->calls_tail = prev->friend_number; else goto CLEAR_CONTAINER; - session->calls[call->friend_id] = NULL; + session->calls[call->friend_number] = NULL; free(call); return; @@ -579,17 +579,17 @@ CLEAR_CONTAINER: free(call); session->calls = NULL; } -void on_peer_status(Messenger* m, uint32_t friend_id, uint8_t status, void* data) +void on_peer_status(Messenger* m, uint32_t friend_number, uint8_t status, void* data) { (void)m; MSISession *session = data; switch ( status ) { case 0: { /* Friend is now offline */ - LOGGER_DEBUG("Friend %d is now offline", friend_id); + LOGGER_DEBUG("Friend %d is now offline", friend_number); pthread_mutex_lock(session->mutex); - MSICall* call = get_call(session, friend_id); + MSICall* call = get_call(session, friend_number); if (call == NULL) { pthread_mutex_unlock(session->mutex); @@ -610,9 +610,7 @@ void handle_push ( MSICall *call, const MSIMessage *msg ) { assert(call); - MSISession* session = call->session; - - LOGGER_DEBUG("Session: %p Handling 'push' friend: %d", call->session, call->friend_id); + LOGGER_DEBUG("Session: %p Handling 'push' friend: %d", call->session, call->friend_number); if (!msg->capabilities.exists) { LOGGER_WARNING("Session: %p Invalid capabilities on 'push'"); @@ -670,7 +668,7 @@ void handle_push ( MSICall *call, const MSIMessage *msg ) msg.vfpsz.exists = true; msg.vfpsz.value = VIDEOFRAME_PIECE_SIZE; - send_message ( call->session->messenger, call->friend_id, &msg ); + send_message ( call->session->messenger, call->friend_number, &msg ); /* If peer changed capabilities during re-call they will * be handled accordingly during the next step @@ -708,14 +706,14 @@ void handle_push ( MSICall *call, const MSIMessage *msg ) return; FAILURE: - send_error(call->session->messenger, call->friend_id, call->error); + send_error(call->session->messenger, call->friend_number, call->error); kill_call(call); } void handle_pop ( MSICall *call, const MSIMessage *msg ) { assert(call); - LOGGER_DEBUG("Session: %p Handling 'pop', friend id: %d", call->session, call->friend_id); + LOGGER_DEBUG("Session: %p Handling 'pop', friend id: %d", call->session, call->friend_number); /* callback errors are ignored */ @@ -751,7 +749,7 @@ void handle_pop ( MSICall *call, const MSIMessage *msg ) kill_call ( call ); } -void handle_msi_packet ( Messenger* m, uint32_t friend_id, const uint8_t* data, uint16_t length, void* object ) +void handle_msi_packet ( Messenger* m, uint32_t friend_number, const uint8_t* data, uint16_t length, void* object ) { LOGGER_DEBUG("Got msi message"); @@ -760,25 +758,25 @@ void handle_msi_packet ( Messenger* m, uint32_t friend_id, const uint8_t* data, if ( msg_parse_in ( &msg, data, length ) == -1 ) { LOGGER_WARNING("Error parsing message"); - send_error(m, friend_id, msi_EInvalidMessage); + send_error(m, friend_number, msi_EInvalidMessage); return; } else { LOGGER_DEBUG("Successfully parsed message"); } pthread_mutex_lock(session->mutex); - MSICall *call = get_call(session, friend_id); + MSICall *call = get_call(session, friend_number); if (call == NULL) { if (msg.request.value != requ_push) { - send_error(m, friend_id, msi_EStrayMessage); + send_error(m, friend_number, msi_EStrayMessage); pthread_mutex_unlock(session->mutex); return; } - call = new_call(session, friend_id); + call = new_call(session, friend_number); if (call == NULL) { - send_error(m, friend_id, msi_ESystem); + send_error(m, friend_number, msi_ESystem); pthread_mutex_unlock(session->mutex); return; } diff --git a/toxav/msi.h b/toxav/msi.h index 457d31488..4836ae89d 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -89,7 +89,7 @@ typedef struct MSICall_s { uint8_t peer_capabilities; /* Peer capabilities */ uint8_t self_capabilities; /* Self capabilities */ uint16_t peer_vfpsz; /* Video frame piece size */ - uint32_t friend_id; /* Index of this call in MSISession */ + uint32_t friend_number; /* Index of this call in MSISession */ MSIError error; /* Last error */ void* av_call; /* Pointer to av call handler */ @@ -100,12 +100,11 @@ typedef struct MSICall_s { /** - * Msi callback type. 'agent' is a pointer to ToxAv. * Expected return on success is 0, if any other number is * returned the call is considered errored and will be handled * as such which means it will be terminated without any notice. */ -typedef int ( *MSICallbackType ) ( void *agent, MSICall* call); +typedef int msi_action_cb ( void *av, MSICall* call); /** * Control session struct. Please do not modify outside msi.c @@ -119,43 +118,34 @@ typedef struct MSISession_s { void *av; Messenger *messenger; - /* The mutex controls async access from control - * thread(s) and core thread. - */ pthread_mutex_t mutex[1]; - MSICallbackType callbacks[7]; + msi_action_cb* callbacks[7]; } MSISession; /** * Start the control session. */ -MSISession *msi_new ( Messenger *messenger ); - +MSISession *msi_new ( Messenger *m ); /** * Terminate control session. NOTE: all calls will be freed */ int msi_kill ( MSISession *session ); - /** * Callback setter. */ -void msi_register_callback(MSISession *session, MSICallbackType callback, MSICallbackID id); - +void msi_register_callback(MSISession *session, msi_action_cb* callback, MSICallbackID id); /** - * Send invite request to friend_id. + * Send invite request to friend_number. */ -int msi_invite ( MSISession* session, MSICall** call, uint32_t friend_id, uint8_t capabilities ); - +int msi_invite ( MSISession* session, MSICall** call, uint32_t friend_number, uint8_t capabilities ); /** * Hangup call. NOTE: 'call' will be freed */ int msi_hangup ( MSICall* call ); - /** * Answer call request. */ int msi_answer ( MSICall* call, uint8_t capabilities ); - /** * Change capabilities of the call. */ diff --git a/toxav/rtp.c b/toxav/rtp.c index 6c603f79c..2219805be 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -78,11 +78,11 @@ int handle_rtcp_packet ( Messenger *m, uint32_t friendnumber, const uint8_t *dat void send_rtcp_report ( RTCPSession* session, Messenger* m, uint32_t friendnumber ); -RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num, void* cs, int (*mcb) (void*, RTPMessage*) ) +RTPSession *rtp_new ( int payload_type, Messenger *m, int friend_num, void* cs, int (*mcb) (void*, RTPMessage*) ) { assert(mcb); assert(cs); - assert(messenger); + assert(m); RTPSession *retu = calloc(1, sizeof(RTPSession)); @@ -95,8 +95,8 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num, vo retu->ssrc = random_int(); retu->payload_type = payload_type % 128; - retu->m = messenger; - retu->friend_id = friend_num; + retu->m = m; + retu->friend_number = friend_num; if ( !(retu->csrc = calloc(1, sizeof(uint32_t))) ) { LOGGER_WARNING("Alloc failed! Program might misbehave!"); @@ -161,7 +161,7 @@ int rtp_do(RTPSession *session) return rtp_StateNormal; if (current_time_monotonic() - session->rtcp_session->last_sent_report_ts >= RTCP_REPORT_INTERVAL_MS) { - send_rtcp_report(session->rtcp_session, session->m, session->friend_id); + send_rtcp_report(session->rtcp_session, session->m, session->friend_number); } if (rb_full(session->rtcp_session->pl_stats)) { @@ -209,15 +209,15 @@ int rtp_start_receiving(RTPSession* session) if (session == NULL) return -1; - if (m_callback_rtp_packet(session->m, session->friend_id, session->prefix, + if (m_callback_rtp_packet(session->m, session->friend_number, session->prefix, handle_rtp_packet, session) == -1) { LOGGER_WARNING("Failed to register rtp receive handler"); return -1; } - if (m_callback_rtp_packet(session->m, session->friend_id, session->rtcp_session->prefix, + if (m_callback_rtp_packet(session->m, session->friend_number, session->rtcp_session->prefix, handle_rtcp_packet, session->rtcp_session) == -1) { LOGGER_WARNING("Failed to register rtcp receive handler"); - m_callback_rtp_packet(session->m, session->friend_id, session->prefix, NULL, NULL); + m_callback_rtp_packet(session->m, session->friend_number, session->prefix, NULL, NULL); return -1; } @@ -228,8 +228,8 @@ int rtp_stop_receiving(RTPSession* session) if (session == NULL) return -1; - m_callback_rtp_packet(session->m, session->friend_id, session->prefix, NULL, NULL); - m_callback_rtp_packet(session->m, session->friend_id, session->rtcp_session->prefix, NULL, NULL); /* RTCP */ + m_callback_rtp_packet(session->m, session->friend_number, session->prefix, NULL, NULL); + m_callback_rtp_packet(session->m, session->friend_number, session->rtcp_session->prefix, NULL, NULL); /* RTCP */ return 0; } @@ -243,7 +243,8 @@ int rtp_send_data ( RTPSession *session, const uint8_t *data, uint16_t length, b uint8_t parsed[MAX_RTP_SIZE]; uint8_t *it; - RTPHeader header[1]; + RTPHeader header[1] = {0}; + ADD_FLAG_VERSION ( header, session->version ); ADD_FLAG_PADDING ( header, session->padding ); ADD_FLAG_EXTENSION ( header, session->extension ); @@ -278,12 +279,11 @@ int rtp_send_data ( RTPSession *session, const uint8_t *data, uint16_t length, b memcpy(it, data, length); - if ( -1 == send_custom_lossy_packet(session->m, session->friend_id, parsed, parsed_len) ) { + if ( -1 == send_custom_lossy_packet(session->m, session->friend_number, parsed, parsed_len) ) { LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno)); return -1; } - /* Set sequ number */ session->sequnum = session->sequnum >= MAX_SEQU_NUM ? 0 : session->sequnum + 1; return 0; } @@ -300,7 +300,6 @@ void rtp_free_msg ( RTPMessage *msg ) - RTPHeader *parse_header_in ( const uint8_t *payload, int length ) { if ( !payload || !length ) { @@ -322,12 +321,7 @@ RTPHeader *parse_header_in ( const uint8_t *payload, int length ) retu->flags = *it; ++it; - - /* This indicates if the first 2 bits are valid. - * Now it may happen that this is out of order but - * it cuts down chances of parsing some invalid value - */ - + if ( GET_FLAG_VERSION(retu) != RTP_VERSION ) { /* Deallocate */ LOGGER_WARNING("Invalid version!"); @@ -335,15 +329,10 @@ RTPHeader *parse_header_in ( const uint8_t *payload, int length ) return NULL; } - /* - * Added a check for the size of the header little sooner so - * I don't need to parse the other stuff if it's bad - */ uint8_t cc = GET_FLAG_CSRCC ( retu ); int total = 12 /* Minimum header len */ + ( cc * 4 ); if ( length < total ) { - /* Deallocate */ LOGGER_WARNING("Length invalid!"); free(retu); return NULL; @@ -355,9 +344,10 @@ RTPHeader *parse_header_in ( const uint8_t *payload, int length ) memcpy(&retu->timestamp, it, sizeof(retu->timestamp)); - retu->timestamp = ntohl(retu->timestamp); it += 4; memcpy(&retu->ssrc, it, sizeof(retu->ssrc)); + + retu->timestamp = ntohl(retu->timestamp); retu->ssrc = ntohl(retu->ssrc); uint8_t x; @@ -380,34 +370,31 @@ RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length ) return NULL; } - uint16_t ext_length; - memcpy(&ext_length, it, sizeof(ext_length)); - ext_length = ntohs(ext_length); + memcpy(&retu->length, it, sizeof(retu->length)); + retu->length = ntohs(retu->length); it += 2; - - - if ( length < ( ext_length * sizeof(uint32_t) ) ) { + + if ( length < ( retu->length * sizeof(uint32_t) ) ) { LOGGER_WARNING("Length invalid!"); free(retu); return NULL; } - - retu->length = ext_length; + memcpy(&retu->type, it, sizeof(retu->type)); retu->type = ntohs(retu->type); + it += 2; - - if ( !(retu->table = calloc(ext_length, sizeof (uint32_t))) ) { + + if ( !(retu->table = calloc(retu->length, sizeof (uint32_t))) ) { LOGGER_WARNING("Alloc failed! Program might misbehave!"); free(retu); return NULL; } uint16_t x; - - for ( x = 0; x < ext_length; x++ ) { + for ( x = 0; x < retu->length; x++ ) { it += 4; - memcpy(&(retu->table[x]), it, sizeof(retu->table[x])); + memcpy(retu->table + x, it, sizeof(*retu->table)); retu->table[x] = ntohl(retu->table[x]); } @@ -433,7 +420,6 @@ uint8_t *parse_header_out ( const RTPHeader *header, uint8_t *payload ) *it = header->marker_payloadt; ++it; - timestamp = htonl(header->timestamp); memcpy(it, ×tamp, sizeof(timestamp)); it += 4; @@ -579,7 +565,6 @@ int handle_rtcp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* dat report->received_packets = ntohl(report->received_packets); report->expected_packets = ntohl(report->expected_packets); - /* Invalid values */ if (report->expected_packets == 0 || report->received_packets > report->expected_packets) { LOGGER_WARNING("Malformed rtcp report! %d %d", report->expected_packets, report->received_packets); free(report); diff --git a/toxav/rtp.h b/toxav/rtp.h index c973d2621..a158d7245 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -113,10 +113,9 @@ typedef struct { uint8_t prefix; Messenger *m; - int friend_id; + int friend_number; struct RTCPSession_s *rtcp_session; - void *cs; int (*mcb) (void*, RTPMessage* msg); @@ -125,33 +124,27 @@ typedef struct { /** * Must be called before calling any other rtp function. */ -RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num, void* cs, int (*mcb) (void*, RTPMessage*) ); - +RTPSession *rtp_new ( int payload_type, Messenger *m, int friend_num, void* cs, int (*mcb) (void*, RTPMessage*) ); /** * Terminate the session. */ void rtp_kill ( RTPSession* session ); - /** * Do periodical rtp work. */ int rtp_do(RTPSession *session); - /** * By default rtp is in receiving state */ int rtp_start_receiving (RTPSession *session); - /** * Pause rtp receiving mode. */ int rtp_stop_receiving (RTPSession *session); - /** * Sends msg to RTPSession::dest */ int rtp_send_data ( RTPSession* session, const uint8_t* data, uint16_t length, bool dummy ); - /** * Dealloc msg. */ diff --git a/toxav/toxav.c b/toxav/toxav.c index 8d47f5cd6..25a2857c2 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -58,10 +58,10 @@ typedef struct ToxAVCall_s { bool active; MSICall* msi_call; - uint32_t friend_id; + uint32_t friend_number; - uint32_t audio_bit_rate; /* Sending audio bitrate */ - uint32_t video_bit_rate; /* Sending video bitrate */ + uint32_t audio_bit_rate; /* Sending audio bit rate */ + uint32_t video_bit_rate; /* Sending video bit rate */ ToxAvBitrateAdapter aba; ToxAvBitrateAdapter vba; @@ -93,8 +93,8 @@ struct toxAV { PAIR(toxav_call_state_cb *, void *) scb; /* Call state callback */ PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ - PAIR(toxav_audio_bitrate_control_cb *, void *) abcb; /* Audio bitrate control callback */ - PAIR(toxav_video_bitrate_control_cb *, void *) vbcb; /* Video bitrate control callback */ + PAIR(toxav_audio_bit_rate_status_cb *, void *) abcb; /* Audio bit rate control callback */ + PAIR(toxav_video_bit_rate_status_cb *, void *) vbcb; /* Video bit rate control callback */ /** Decode time measures */ int32_t dmssc; /** Measure count */ @@ -111,8 +111,8 @@ int callback_end(void* toxav_inst, MSICall* call); int callback_error(void* toxav_inst, MSICall* call); int callback_capabilites(void* toxav_inst, MSICall* call); -bool audio_bitrate_invalid(uint32_t bitrate); -bool video_bitrate_invalid(uint32_t bitrate); +bool audio_bit_rate_invalid(uint32_t bit_rate); +bool video_bit_rate_invalid(uint32_t bit_rate); void invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state); ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); ToxAVCall* call_get(ToxAV* av, uint32_t friend_number); @@ -129,12 +129,12 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) if (tox == NULL) { rc = TOXAV_ERR_NEW_NULL; - goto FAILURE; + goto END; } if (((Messenger*)tox)->msi_packet) { rc = TOXAV_ERR_NEW_MULTIPLE; - goto FAILURE; + goto END; } av = calloc (sizeof(ToxAV), 1); @@ -142,13 +142,13 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) if (av == NULL) { LOGGER_WARNING("Allocation failed!"); rc = TOXAV_ERR_NEW_MALLOC; - goto FAILURE; + goto END; } if (create_recursive_mutex(av->mutex) != 0) { LOGGER_WARNING("Mutex creation failed!"); rc = TOXAV_ERR_NEW_MALLOC; - goto FAILURE; + goto END; } av->m = (Messenger *)tox; @@ -157,7 +157,7 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) if (av->msi == NULL) { pthread_mutex_destroy(av->mutex); rc = TOXAV_ERR_NEW_MALLOC; - goto FAILURE; + goto END; } av->interval = 200; @@ -170,19 +170,16 @@ ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) msi_register_callback(av->msi, callback_error, msi_OnPeerTimeout); msi_register_callback(av->msi, callback_capabilites, msi_OnCapabilities); - +END: if (error) *error = rc; + if (rc != TOXAV_ERR_NEW_OK) { + free(av); + av = NULL; + } + return av; - -FAILURE: - if (error) - *error = rc; - - free(av); - - return NULL; } void toxav_kill(ToxAV* av) @@ -249,14 +246,14 @@ void toxav_iterate(ToxAV* av) /* Notify app */ if (av->abcb.first) - av->abcb.first (av, i->friend_id, false, bb, av->abcb.second); + av->abcb.first (av, i->friend_number, false, bb, av->abcb.second); } else if (i->aba.active && i->aba.end_time < current_time_monotonic()) { i->audio_bit_rate = i->aba.bit_rate; - /* Notify user about the new bitrate */ + /* Notify user about the new bit rate */ if (av->abcb.first) - av->abcb.first (av, i->friend_id, true, i->aba.bit_rate, av->abcb.second); + av->abcb.first (av, i->friend_number, true, i->aba.bit_rate, av->abcb.second); /* Stop sending dummy packets */ memset(&i->aba, 0, sizeof(i->aba)); @@ -275,15 +272,15 @@ void toxav_iterate(ToxAV* av) /* Notify app */ if (av->vbcb.first) - av->vbcb.first (av, i->friend_id, false, bb, av->vbcb.second); + av->vbcb.first (av, i->friend_number, false, bb, av->vbcb.second); } else if (i->vba.active && i->vba.end_time < current_time_monotonic()) { i->video_bit_rate = i->vba.bit_rate; - /* Notify user about the new bitrate */ + /* Notify user about the new bit rate */ if (av->vbcb.first) - av->vbcb.first (av, i->friend_id, true, i->vba.bit_rate, av->vbcb.second); + av->vbcb.first (av, i->friend_number, true, i->vba.bit_rate, av->vbcb.second); /* Stop sending dummy packets */ memset(&i->vba, 0, sizeof(i->vba)); @@ -297,7 +294,7 @@ void toxav_iterate(ToxAV* av) i->msi_call->peer_capabilities & msi_CapSVideo) rc = MIN(i->video.second->lcfd, rc); - uint32_t fid = i->friend_id; + uint32_t fid = i->friend_number; pthread_mutex_unlock(i->mutex); pthread_mutex_lock(av->mutex); @@ -356,7 +353,7 @@ void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data) av->ccb.first = function; av->ccb.second = user_data; pthread_mutex_unlock(av->mutex); -}/** Required for monitoring */ +} bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER* error) { @@ -368,8 +365,8 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui goto END; } - if ((audio_bit_rate && audio_bitrate_invalid(audio_bit_rate)) - ||(video_bit_rate && video_bitrate_invalid(video_bit_rate)) + if ((audio_bit_rate && audio_bit_rate_invalid(audio_bit_rate)) + ||(video_bit_rate && video_bit_rate_invalid(video_bit_rate)) ) { rc = TOXAV_ERR_CALL_INVALID_BIT_RATE; goto END; @@ -561,7 +558,7 @@ END: return rc == TOXAV_ERR_CALL_CONTROL_OK; } -void toxav_callback_video_bitrate_control(ToxAV* av, toxav_video_bitrate_control_cb* function, void* user_data) +void toxav_callback_video_bit_rate_status(ToxAV* av, toxav_video_bit_rate_status_cb* function, void* user_data) { pthread_mutex_lock(av->mutex); av->vbcb.first = function; @@ -579,7 +576,7 @@ bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_ goto END; } - if (video_bitrate_invalid(video_bit_rate)) { + if (video_bit_rate_invalid(video_bit_rate)) { rc = TOXAV_ERR_BIT_RATE_INVALID; goto END; } @@ -605,7 +602,7 @@ bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_ call->video_bit_rate = video_bit_rate; if (!force && av->vbcb.first) - av->vbcb.first (av, call->friend_id, true, video_bit_rate, av->vbcb.second); + av->vbcb.first (av, call->friend_number, true, video_bit_rate, av->vbcb.second); } pthread_mutex_unlock(call->mutex); @@ -618,7 +615,7 @@ END: return rc == TOXAV_ERR_BIT_RATE_OK; } -void toxav_callback_audio_bitrate_control(ToxAV* av, toxav_audio_bitrate_control_cb* function, void* user_data) +void toxav_callback_audio_bit_rate_status(ToxAV* av, toxav_audio_bit_rate_status_cb* function, void* user_data) { pthread_mutex_lock(av->mutex); av->abcb.first = function; @@ -636,7 +633,7 @@ bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_ goto END; } - if (audio_bitrate_invalid(audio_bit_rate)) { + if (audio_bit_rate_invalid(audio_bit_rate)) { rc = TOXAV_ERR_BIT_RATE_INVALID; goto END; } @@ -662,7 +659,7 @@ bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_ call->audio_bit_rate = audio_bit_rate; if (!force && av->abcb.first) - av->abcb.first (av, call->friend_id, true, audio_bit_rate, av->abcb.second); + av->abcb.first (av, call->friend_number, true, audio_bit_rate, av->abcb.second); } pthread_mutex_unlock(call->mutex); @@ -897,11 +894,11 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc } - /* For bitrate measurement; send dummy packet */ + /* For bit rate measurement; send dummy packet */ if (ba_shoud_send_dummy(&call->aba)) { sampling_rate = ntohl(sampling_rate); if (ac_reconfigure_test_encoder(call->audio.second, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) { - /* FIXME should the bitrate changing fail here? */ + /* FIXME should the bit rate changing fail here? */ pthread_mutex_unlock(call->mutex_audio); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; @@ -966,7 +963,7 @@ int callback_invite(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - ToxAVCall* av_call = call_new(toxav, call->friend_id, NULL); + ToxAVCall* av_call = call_new(toxav, call->friend_number, NULL); if (av_call == NULL) { LOGGER_WARNING("Failed to initialize call..."); pthread_mutex_unlock(toxav->mutex); @@ -977,7 +974,7 @@ int callback_invite(void* toxav_inst, MSICall* call) av_call->msi_call = call; if (toxav->ccb.first) - toxav->ccb.first(toxav, call->friend_id, call->peer_capabilities & msi_CapSAudio, + toxav->ccb.first(toxav, call->friend_number, call->peer_capabilities & msi_CapSAudio, call->peer_capabilities & msi_CapSVideo, toxav->ccb.second); pthread_mutex_unlock(toxav->mutex); @@ -989,7 +986,7 @@ int callback_start(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - ToxAVCall* av_call = call_get(toxav, call->friend_id); + ToxAVCall* av_call = call_get(toxav, call->friend_number); if (av_call == NULL) { /* Should this ever happen? */ @@ -1004,7 +1001,7 @@ int callback_start(void* toxav_inst, MSICall* call) return -1; } - invoke_call_state(toxav, call->friend_id, call->peer_capabilities); + invoke_call_state(toxav, call->friend_number, call->peer_capabilities); pthread_mutex_unlock(toxav->mutex); return 0; @@ -1015,7 +1012,7 @@ int callback_end(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - invoke_call_state(toxav, call->friend_id, TOXAV_CALL_STATE_END); + invoke_call_state(toxav, call->friend_number, TOXAV_CALL_STATE_END); call_kill_transmission(call->av_call); call_remove(call->av_call); @@ -1029,7 +1026,7 @@ int callback_error(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - invoke_call_state(toxav, call->friend_id, TOXAV_CALL_STATE_ERROR); + invoke_call_state(toxav, call->friend_number, TOXAV_CALL_STATE_ERROR); call_kill_transmission(call->av_call); call_remove(call->av_call); @@ -1043,21 +1040,21 @@ int callback_capabilites(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - invoke_call_state(toxav, call->friend_id, call->peer_capabilities); + invoke_call_state(toxav, call->friend_number, call->peer_capabilities); pthread_mutex_unlock(toxav->mutex); return 0; } -bool audio_bitrate_invalid(uint32_t bitrate) +bool audio_bit_rate_invalid(uint32_t bit_rate) { /* Opus RFC 6716 section-2.1.1 dictates the following: - * Opus supports all bitrates from 6 kbit/s to 510 kbit/s. + * Opus supports all bit rates from 6 kbit/s to 510 kbit/s. */ - return bitrate < 6 || bitrate > 510; + return bit_rate < 6 || bit_rate > 510; } -bool video_bitrate_invalid(uint32_t bitrate) +bool video_bit_rate_invalid(uint32_t bit_rate) { /* TODO: If anyone knows the answer to this one please fill it up */ return false; @@ -1099,7 +1096,7 @@ ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) } call->av = av; - call->friend_id = friend_number; + call->friend_number = friend_number; if (av->calls == NULL) { /* Creating */ av->calls = calloc (sizeof(ToxAVCall*), friend_number + 1); @@ -1126,7 +1123,7 @@ ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) av->calls = tmp; /* Set fields in between to null */ - int32_t i = av->calls_tail; + int32_t i = av->calls_tail + 1; for (; i < friend_number; i ++) av->calls[i] = NULL; @@ -1164,7 +1161,7 @@ void call_remove(ToxAVCall* call) if (call == NULL) return; - uint32_t friend_id = call->friend_id; + uint32_t friend_number = call->friend_number; ToxAV* av = call->av; ToxAVCall* prev = call->prev; @@ -1175,16 +1172,16 @@ void call_remove(ToxAVCall* call) if (prev) prev->next = next; else if (next) - av->calls_head = next->friend_id; + av->calls_head = next->friend_number; else goto CLEAR; if (next) next->prev = prev; else if (prev) - av->calls_tail = prev->friend_id; + av->calls_tail = prev->friend_number; else goto CLEAR; - av->calls[friend_id] = NULL; + av->calls[friend_number] = NULL; return; CLEAR: @@ -1214,17 +1211,16 @@ bool call_prepare_transmission(ToxAVCall* call) if (create_recursive_mutex(call->mutex_audio) != 0) return false; - if (create_recursive_mutex(call->mutex_video) != 0) { - goto AUDIO_SENDING_MUTEX_CLEANUP; - } + if (create_recursive_mutex(call->mutex_video) != 0) + goto FAILURE_3; + + if (create_recursive_mutex(call->mutex) != 0) + goto FAILURE_2; - if (create_recursive_mutex(call->mutex) != 0) { - goto VIDEO_SENDING_MUTEX_CLEANUP; - } { /* Prepare audio */ - call->audio.second = ac_new(av, call->friend_id, av->acb.first, av->acb.second); - call->audio.first = rtp_new(rtp_TypeAudio, av->m, call->friend_id, call->audio.second, ac_queue_message); + call->audio.second = ac_new(av, call->friend_number, av->acb.first, av->acb.second); + call->audio.first = rtp_new(rtp_TypeAudio, av->m, call->friend_number, call->audio.second, ac_queue_message); if ( !call->audio.first || !call->audio.second ) { LOGGER_ERROR("Error while starting audio!\n"); @@ -1233,8 +1229,8 @@ bool call_prepare_transmission(ToxAVCall* call) } { /* Prepare video */ - call->video.second = vc_new(av, call->friend_id, av->vcb.first, av->vcb.second, call->msi_call->peer_vfpsz); - call->video.first = rtp_new(rtp_TypeVideo, av->m, call->friend_id, call->video.second, vc_queue_message); + call->video.second = vc_new(av, call->friend_number, av->vcb.first, av->vcb.second, call->msi_call->peer_vfpsz); + call->video.first = rtp_new(rtp_TypeVideo, av->m, call->friend_number, call->video.second, vc_queue_message); if ( !call->video.first || !call->video.second ) { LOGGER_ERROR("Error while starting video!\n"); @@ -1255,9 +1251,9 @@ FAILURE: call->video.first = NULL; call->video.second = NULL; pthread_mutex_destroy(call->mutex); -VIDEO_SENDING_MUTEX_CLEANUP: +FAILURE_2: pthread_mutex_destroy(call->mutex_video); -AUDIO_SENDING_MUTEX_CLEANUP: +FAILURE_3: pthread_mutex_destroy(call->mutex_audio); return false; } diff --git a/toxav/toxav.h b/toxav/toxav.h index f2c3b2b3c..b8db223e7 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -333,23 +333,23 @@ typedef enum TOXAV_ERR_BIT_RATE { TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL } TOXAV_ERR_BIT_RATE; /** - * The function type for the `audio_bitrate_control` callback. + * The function type for the `audio_bit_rate_status` callback. * * @param friend_number The friend number of the friend for which to set the * audio bit rate. - * @param good Is the stream good enough to keep the said bitrate. Upon failed - * non forceful bit rate setup this will be set to false and 'bit_rate' - * will be set to the bit rate that failed, otherwise 'good' will be set to - * true with 'bit_rate' set to new bit rate. If the stream becomes bad, - * the 'good' wil be set to false with 'bit_rate' set to the current bit rate. - * This callback will never be called when the stream is good. + * @param stable Is the stream stable enough to keep the bit rate. + * Upon successful, non forceful, bit rate change, this is set to + * true and 'bit_rate' is set to new bit rate. + * The stable is set to false with bit_rate set to the unstable + * bit rate when either current stream is unstable with said bit rate + * or the non forceful change failed. * @param bit_rate The bit rate in Kb/sec. */ -typedef void toxav_audio_bitrate_control_cb(ToxAV *av, uint32_t friend_number, bool good, uint32_t bit_rate, void *user_data); +typedef void toxav_audio_bit_rate_status_cb(ToxAV *av, uint32_t friend_number, bool stable, uint32_t bit_rate, void *user_data); /** - * Set the callback for the `audio_bitrate_control` event. Pass NULL to unset. + * Set the callback for the `audio_bit_rate_status` event. Pass NULL to unset. */ -void toxav_callback_audio_bitrate_control(ToxAV *av, toxav_audio_bitrate_control_cb *function, void *user_data); +void toxav_callback_audio_bit_rate_status(ToxAV *av, toxav_audio_bit_rate_status_cb *function, void *user_data); /** * Set the audio bit rate to be used in subsequent audio frames. * @@ -362,23 +362,23 @@ void toxav_callback_audio_bitrate_control(ToxAV *av, toxav_audio_bitrate_control */ bool toxav_set_audio_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, bool force, TOXAV_ERR_BIT_RATE *error); /** - * The function type for the `video_bitrate_control` callback. + * The function type for the `video_bit_rate_status` callback. * * @param friend_number The friend number of the friend for which to set the * video bit rate. - * @param good Is the stream good enough to keep the said bitrate. Upon failed - * non forceful bit rate setup this will be set to false and 'bit_rate' - * will be set to the bit rate that failed, otherwise 'good' will be set to - * true with 'bit_rate' set to new bit rate. If the stream becomes bad, - * the 'good' wil be set to false with 'bit_rate' set to the current bit rate. - * This callback will never be called when the stream is good. + * @param stable Is the stream stable enough to keep the bit rate. + * Upon successful, non forceful, bit rate change, this is set to + * true and 'bit_rate' is set to new bit rate. + * The stable is set to false with bit_rate set to the unstable + * bit rate when either current stream is unstable with said bit rate + * or the non forceful change failed. * @param bit_rate The bit rate in Kb/sec. */ -typedef void toxav_video_bitrate_control_cb(ToxAV *av, uint32_t friend_number, bool good, uint32_t bit_rate, void *user_data); +typedef void toxav_video_bit_rate_status_cb(ToxAV *av, uint32_t friend_number, bool stable, uint32_t bit_rate, void *user_data); /** - * Set the callback for the `video_bitrate_control` event. Pass NULL to unset. + * Set the callback for the `video_bit_rate_status` event. Pass NULL to unset. */ -void toxav_callback_video_bitrate_control(ToxAV *av, toxav_video_bitrate_control_cb *function, void *user_data); +void toxav_callback_video_bit_rate_status(ToxAV *av, toxav_video_bit_rate_status_cb *function, void *user_data); /** * Set the video bit rate to be used in subsequent video frames. * diff --git a/toxav/video.c b/toxav/video.c index c540af3b8..22ca2beef 100644 --- a/toxav/video.c +++ b/toxav/video.c @@ -35,15 +35,14 @@ #define MAX_VIDEOFRAME_SIZE 0x40000 /* 256KiB */ #define VIDEOFRAME_HEADER_SIZE 0x2 -/* FIXME: Might not be enough? NOTE: I think it is enough */ #define VIDEO_DECODE_BUFFER_SIZE 20 typedef struct { uint16_t size; uint8_t data[]; } Payload; -bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate); +bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bit_rate); -VCSession* vc_new(ToxAV* av, uint32_t friend_id, toxav_receive_video_frame_cb* cb, void* cb_data, uint32_t mvfpsz) +VCSession* vc_new(ToxAV* av, uint32_t friend_number, toxav_receive_video_frame_cb* cb, void* cb_data, uint32_t mvfpsz) { VCSession *vc = calloc(sizeof(VCSession), 1); @@ -86,7 +85,7 @@ VCSession* vc_new(ToxAV* av, uint32_t friend_id, toxav_receive_video_frame_cb* c vc->lcfd = 60; vc->vcb.first = cb; vc->vcb.second = cb_data; - vc->friend_id = friend_id; + vc->friend_number = friend_number; vc->peer_video_frame_piece_size = mvfpsz; return vc; @@ -140,7 +139,7 @@ void vc_do(VCSession* vc) /* Play decoded images */ for (; dest; dest = vpx_codec_get_frame(vc->decoder, &iter)) { if (vc->vcb.first) - vc->vcb.first(vc->av, vc->friend_id, dest->d_w, dest->d_h, + vc->vcb.first(vc->av, vc->friend_number, dest->d_w, dest->d_h, (const uint8_t*)dest->planes[0], (const uint8_t*)dest->planes[1], (const uint8_t*)dest->planes[2], dest->stride[0], dest->stride[1], dest->stride[2], vc->vcb.second); @@ -289,16 +288,16 @@ end: rtp_free_msg(msg); return 0; } -int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height) +int vc_reconfigure_encoder(VCSession* vc, int32_t bit_rate, uint16_t width, uint16_t height) { if (!vc) return -1; vpx_codec_enc_cfg_t cfg = *vc->encoder->config.enc; - if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) + if (cfg.rc_target_bitrate == bit_rate && cfg.g_w == width && cfg.g_h == height) return 0; /* Nothing changed */ - cfg.rc_target_bitrate = bitrate; + cfg.rc_target_bitrate = bit_rate; cfg.g_w = width; cfg.g_h = height; @@ -310,16 +309,16 @@ int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint1 return 0; } -int vc_reconfigure_test_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height) +int vc_reconfigure_test_encoder(VCSession* vc, int32_t bit_rate, uint16_t width, uint16_t height) { if (!vc) return -1; vpx_codec_enc_cfg_t cfg = *vc->test_encoder->config.enc; - if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) + if (cfg.rc_target_bitrate == bit_rate && cfg.g_w == width && cfg.g_h == height) return 0; /* Nothing changed */ - cfg.rc_target_bitrate = bitrate; + cfg.rc_target_bitrate = bit_rate; cfg.g_w = width; cfg.g_h = height; @@ -334,7 +333,7 @@ int vc_reconfigure_test_encoder(VCSession* vc, int32_t bitrate, uint16_t width, -bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate) +bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bit_rate) { assert(dest); @@ -354,7 +353,7 @@ bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate) return false; } - cfg.rc_target_bitrate = bitrate; + cfg.rc_target_bitrate = bit_rate; cfg.g_w = 800; cfg.g_h = 600; cfg.g_pass = VPX_RC_ONE_PASS; diff --git a/toxav/video.h b/toxav/video.h index 78003ef3a..8da155787 100644 --- a/toxav/video.h +++ b/toxav/video.h @@ -38,6 +38,9 @@ struct RTPMessage_s; +/* + * Base Video Codec session type. + */ typedef struct VCSession_s { /* encoding */ @@ -65,23 +68,46 @@ typedef struct VCSession_s { const uint8_t *processing_video_frame; uint16_t processing_video_frame_size; - ToxAV *av; - int32_t friend_id; + uint32_t friend_number; PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ pthread_mutex_t queue_mutex[1]; } VCSession; -VCSession* vc_new(ToxAV* av, uint32_t friend_id, toxav_receive_video_frame_cb *cb, void *cb_data, uint32_t mvfpsz); +/* + * Create new Video Codec session. + */ +VCSession* vc_new(ToxAV* av, uint32_t friend_number, toxav_receive_video_frame_cb *cb, void *cb_data, uint32_t mvfpsz); +/* + * Kill the Video Codec session. + */ void vc_kill(VCSession* vc); +/* + * Do periodic work. Work is consisted out of decoding only. + */ void vc_do(VCSession* vc); +/* + * Set new video splitting cycle. This is requirement in order to send video packets. + */ void vc_init_video_splitter_cycle(VCSession* vc); +/* + * Update the video splitter cycle with new data. + */ int vc_update_video_splitter_cycle(VCSession* vc, const uint8_t* payload, uint16_t length); +/* + * Iterate over splitted cycle. + */ const uint8_t *vc_iterate_split_video_frame(VCSession* vc, uint16_t *size); +/* + * Queue new rtp message. + */ int vc_queue_message(void *vcp, struct RTPMessage_s *msg); -int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height); -int vc_reconfigure_test_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height); +/* + * Set new values to the encoders. + */ +int vc_reconfigure_encoder(VCSession* vc, int32_t bit_rate, uint16_t width, uint16_t height); +int vc_reconfigure_test_encoder(VCSession* vc, int32_t bit_rate, uint16_t width, uint16_t height); #endif /* VIDEO_H */ \ No newline at end of file From 45c45841cf099682f47b081e07c01ca52583b8e9 Mon Sep 17 00:00:00 2001 From: krobelus Date: Wed, 29 Apr 2015 09:21:28 +0200 Subject: [PATCH 38/97] Fixed typo --- toxav/toxav.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toxav/toxav.h b/toxav/toxav.h index b8db223e7..67e5496e8 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -93,7 +93,7 @@ uint32_t toxav_iteration_interval(ToxAV const *av); /** * Main loop for the session. This function needs to be called in intervals of * toxav_iteration_interval() milliseconds. It is best called in the same loop - * as tox_iteration. + * as tox_iterate. */ void toxav_iterate(ToxAV *av); /******************************************************************************* From b2b11f0fba52bcba5acb3ba9e4aa4da119d76093 Mon Sep 17 00:00:00 2001 From: mannol Date: Thu, 30 Apr 2015 00:40:30 +0200 Subject: [PATCH 39/97] Added many test and fixed various warnings --- auto_tests/Makefile.inc | 10 +- auto_tests/toxav_basic_test.c | 13 +++ auto_tests/toxav_many_test.c | 168 ++++++++++++++++++++++++++++++---- toxav/msi.c | 2 +- toxav/rtp.c | 9 +- toxav/toxav.c | 11 ++- 6 files changed, 182 insertions(+), 31 deletions(-) diff --git a/auto_tests/Makefile.inc b/auto_tests/Makefile.inc index 741ca7faf..d78a6a5a3 100644 --- a/auto_tests/Makefile.inc +++ b/auto_tests/Makefile.inc @@ -21,8 +21,8 @@ AUTOTEST_LDADD = \ if BUILD_AV -TESTS += toxav_basic_test #toxav_many_test -check_PROGRAMS += toxav_basic_test #toxav_many_test +TESTS += toxav_basic_test toxav_many_test +check_PROGRAMS += toxav_basic_test toxav_many_test AUTOTEST_LDADD += libtoxav.la endif @@ -90,11 +90,11 @@ toxav_basic_test_CFLAGS = $(AUTOTEST_CFLAGS) toxav_basic_test_LDADD = $(AUTOTEST_LDADD) $(AV_LIBS) -#toxav_many_test_SOURCES = ../auto_tests/toxav_many_test.c +toxav_many_test_SOURCES = ../auto_tests/toxav_many_test.c -#toxav_many_test_CFLAGS = $(AUTOTEST_CFLAGS) +toxav_many_test_CFLAGS = $(AUTOTEST_CFLAGS) -#toxav_many_test_LDADD = $(AUTOTEST_LDADD) +toxav_many_test_LDADD = $(AUTOTEST_LDADD) endif endif diff --git a/auto_tests/toxav_basic_test.c b/auto_tests/toxav_basic_test.c index 7598c0db9..423cd03de 100644 --- a/auto_tests/toxav_basic_test.c +++ b/auto_tests/toxav_basic_test.c @@ -49,11 +49,19 @@ typedef struct { */ void t_toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data) { + (void) av; + (void) friend_number; + (void) audio_enabled; + (void) video_enabled; + printf("Handling CALL callback\n"); ((CallControl*)user_data)->incoming = true; } void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, void *user_data) { + (void) av; + (void) friend_number; + printf("Handling CALL STATE callback: %d\n", state); ((CallControl*)user_data)->state = state; } @@ -92,6 +100,8 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, } void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, size_t length, void *userdata) { + (void) userdata; + if (length == 7 && memcmp("gentoo", data, 7) == 0) { assert(tox_friend_add_norequest(m, public_key, NULL) != (uint32_t) ~0); } @@ -433,6 +443,9 @@ Suite *tox_suite(void) } int main(int argc, char *argv[]) { + (void) argc; + (void) argv; + Suite *tox = tox_suite(); SRunner *test_runner = srunner_create(tox); diff --git a/auto_tests/toxav_many_test.c b/auto_tests/toxav_many_test.c index ef59b2b2e..84f94e961 100644 --- a/auto_tests/toxav_many_test.c +++ b/auto_tests/toxav_many_test.c @@ -31,22 +31,34 @@ typedef struct { bool incoming; uint32_t state; - } CallControl; +typedef struct { + ToxAV* AliceAV; + ToxAV* BobAV; + CallControl* AliceCC; + CallControl* BobCC; + uint32_t friend_number; +} thread_data; /** * Callbacks */ void t_toxav_call_cb(ToxAV *av, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data) { + (void) av; + (void) audio_enabled; + (void) video_enabled; + printf("Handling CALL callback\n"); - ((CallControl*)user_data)->incoming = true; + ((CallControl*)user_data)[friend_number].incoming = true; } void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, void *user_data) { + (void) av; + printf("Handling CALL STATE callback: %d\n", state); - ((CallControl*)user_data)->state = state; + ((CallControl*)user_data)[friend_number].state = state; } void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, @@ -83,6 +95,7 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, } void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, size_t length, void *userdata) { + (void) userdata; if (length == 7 && memcmp("gentoo", data, 7) == 0) { assert(tox_friend_add_norequest(m, public_key, NULL) != (uint32_t) ~0); } @@ -106,11 +119,82 @@ ToxAV* setup_av_instance(Tox* tox, CallControl *CC) return av; } -void* call_thread(ToxAV* Alice, ToxAV* Bob) +void* call_thread(void* pd) { + ToxAV* AliceAV = ((thread_data*) pd)->AliceAV; + ToxAV* BobAV = ((thread_data*) pd)->BobAV; + CallControl* AliceCC = ((thread_data*) pd)->AliceCC; + CallControl* BobCC = ((thread_data*) pd)->BobCC; + uint32_t friend_number = ((thread_data*) pd)->friend_number; + + + memset(AliceCC, 0, sizeof(CallControl)); + memset(BobCC, 0, sizeof(CallControl)); + + { /* Call */ + TOXAV_ERR_CALL rc; + toxav_call(AliceAV, friend_number, 48, 3000, &rc); + + if (rc != TOXAV_ERR_CALL_OK) { + printf("toxav_call failed: %d\n", rc); + exit(1); + } + } + + while (!BobCC->incoming) + c_sleep(10); + + { /* Answer */ + TOXAV_ERR_ANSWER rc; + toxav_answer(BobAV, 0, 8, 500, &rc); + + if (rc != TOXAV_ERR_ANSWER_OK) { + printf("toxav_answer failed: %d\n", rc); + exit(1); + } + } + + c_sleep(30); + + int16_t PCM[960]; + uint8_t video_y[800*600]; + uint8_t video_u[800*600 / 2]; + uint8_t video_v[800*600 / 2]; + + memset(PCM, 0, sizeof(PCM)); + memset(video_y, 0, sizeof(video_y)); + memset(video_u, 0, sizeof(video_u)); + memset(video_v, 0, sizeof(video_v)); + + time_t start_time = time(NULL); + while(time(NULL) - start_time < 9) { + toxav_iterate(AliceAV); + toxav_iterate(BobAV); + + toxav_send_audio_frame(AliceAV, friend_number, PCM, 960, 1, 48000, NULL); + toxav_send_audio_frame(BobAV, 0, PCM, 960, 1, 48000, NULL); + + toxav_send_video_frame(AliceAV, friend_number, 800, 600, video_y, video_u, video_v, NULL); + toxav_send_video_frame(BobAV, 0, 800, 600, video_y, video_u, video_v, NULL); + + c_sleep(10); + } + + { /* Hangup */ + TOXAV_ERR_CALL_CONTROL rc; + toxav_call_control(AliceAV, friend_number, TOXAV_CALL_CONTROL_CANCEL, &rc); + + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { + printf("toxav_call_control failed: %d\n", rc); + exit(1); + } + } + + c_sleep(30); pthread_exit(NULL); } + START_TEST(test_AV_three_calls) { Tox* Alice, *bootstrap, *Bobs[3]; @@ -118,7 +202,6 @@ START_TEST(test_AV_three_calls) CallControl AliceCC[3], BobsCC[3]; - int i = 0; { TOX_ERR_NEW error; @@ -128,10 +211,14 @@ START_TEST(test_AV_three_calls) Alice = tox_new(NULL, NULL, 0, &error); assert(error == TOX_ERR_NEW_OK); - for (; i < 3; i ++) { - BobsAV[i] = tox_new(NULL, NULL, 0, &error); - assert(error == TOX_ERR_NEW_OK); - } + Bobs[0] = tox_new(NULL, NULL, 0, &error); + assert(error == TOX_ERR_NEW_OK); + + Bobs[1] = tox_new(NULL, NULL, 0, &error); + assert(error == TOX_ERR_NEW_OK); + + Bobs[2] = tox_new(NULL, NULL, 0, &error); + assert(error == TOX_ERR_NEW_OK); } printf("Created 5 instances of Tox\n"); @@ -178,25 +265,65 @@ START_TEST(test_AV_three_calls) c_sleep(20); } - AliceAV = setup_av_instance(Alice, &AliceCC); - BobsAV[0] = setup_av_instance(Bobs[0], &BobsCC[0]); - BobsAV[1] = setup_av_instance(Bobs[1], &BobsCC[1]); - BobsAV[2] = setup_av_instance(Bobs[2], &BobsCC[2]); + AliceAV = setup_av_instance(Alice, AliceCC); + BobsAV[0] = setup_av_instance(Bobs[0], BobsCC + 0); + BobsAV[1] = setup_av_instance(Bobs[1], BobsCC + 1); + BobsAV[2] = setup_av_instance(Bobs[2], BobsCC + 2); printf("Created 4 instances of ToxAV\n"); printf("All set after %llu seconds!\n", time(NULL) - cur_time); + thread_data tds[3]; + tds[0].AliceAV = AliceAV; + tds[0].BobAV = BobsAV[0]; + tds[0].AliceCC = AliceCC + 0; + tds[0].BobCC = BobsCC + 0; + tds[0].friend_number = 0; + tds[1].AliceAV = AliceAV; + tds[1].BobAV = BobsAV[1]; + tds[1].AliceCC = AliceCC + 1; + tds[1].BobCC = BobsCC + 1; + tds[1].friend_number = 1; - tox_kill(bootstrap); - tox_kill(Alice); - toxav_kill(AliceAV); + tds[2].AliceAV = AliceAV; + tds[2].BobAV = BobsAV[2]; + tds[2].AliceCC = AliceCC + 2; + tds[2].BobCC = BobsCC + 2; + tds[2].friend_number = 2; - for (i = 0; i < 3; i ++) { - tox_kill(Bobs[i]); - toxav_kill(BobsAV[i]); + pthread_t tids[3]; + (void) pthread_create(tids + 0, NULL, call_thread, tds + 0); + (void) pthread_create(tids + 1, NULL, call_thread, tds + 1); + (void) pthread_create(tids + 2, NULL, call_thread, tds + 2); + + (void) pthread_detach(tids[0]); + (void) pthread_detach(tids[1]); + (void) pthread_detach(tids[2]); + + time_t start_time = time(NULL); + while (time(NULL) - start_time < 10) { + tox_iterate(Alice); + tox_iterate(Bobs[0]); + tox_iterate(Bobs[1]); + tox_iterate(Bobs[2]); + c_sleep(20); } + (void) pthread_join(tids[0], NULL); + (void) pthread_join(tids[1], NULL); + (void) pthread_join(tids[2], NULL); + + toxav_kill(BobsAV[0]); + toxav_kill(BobsAV[1]); + toxav_kill(BobsAV[2]); + toxav_kill(AliceAV); + tox_kill(Bobs[0]); + tox_kill(Bobs[1]); + tox_kill(Bobs[2]); + tox_kill(Alice); + tox_kill(bootstrap); + printf("\nTest successful!\n"); } END_TEST @@ -218,6 +345,9 @@ Suite *tox_suite(void) int main(int argc, char *argv[]) { + (void) argc; + (void) argv; + Suite *tox = tox_suite(); SRunner *test_runner = srunner_create(tox); diff --git a/toxav/msi.c b/toxav/msi.c index f8bc8451b..d68e4a9c5 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -526,7 +526,7 @@ MSICall *new_call ( MSISession *session, uint32_t friend_number ) session->calls = tmp; /* Set fields in between to null */ - int32_t i = session->calls_tail + 1; + uint32_t i = session->calls_tail + 1; for (; i < friend_number; i ++) session->calls[i] = NULL; diff --git a/toxav/rtp.c b/toxav/rtp.c index 2219805be..4ca23d2a6 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -243,7 +243,8 @@ int rtp_send_data ( RTPSession *session, const uint8_t *data, uint16_t length, b uint8_t parsed[MAX_RTP_SIZE]; uint8_t *it; - RTPHeader header[1] = {0}; + RTPHeader header[1]; + memset(header, 0, sizeof(header)); ADD_FLAG_VERSION ( header, session->version ); ADD_FLAG_PADDING ( header, session->padding ); @@ -463,6 +464,9 @@ uint8_t *parse_ext_header_out ( const RTPExtHeader *header, uint8_t *payload ) } int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) { + (void) m; + (void) friendnumber; + RTPSession *session = object; if ( !session || length < 13 || length > MAX_RTP_SIZE ) { @@ -553,6 +557,9 @@ int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data } int handle_rtcp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) { + (void) m; + (void) friendnumber; + if (length < 9) return -1; diff --git a/toxav/toxav.c b/toxav/toxav.c index 25a2857c2..e7807592d 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -224,7 +224,7 @@ void toxav_iterate(ToxAV* av) } uint64_t start = current_time_monotonic(); - uint32_t rc = 500; + int32_t rc = 500; ToxAVCall* i = av->calls[av->calls_head]; for (; i; i = i->next) { @@ -292,7 +292,7 @@ void toxav_iterate(ToxAV* av) if (i->msi_call->self_capabilities & msi_CapRVideo && i->msi_call->peer_capabilities & msi_CapSVideo) - rc = MIN(i->video.second->lcfd, rc); + rc = MIN(i->video.second->lcfd, (uint32_t) rc); uint32_t fid = i->friend_number; @@ -821,7 +821,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u } } - if (call->vba.end_time == ~0) + if (call->vba.end_time == (uint64_t) ~0) call->vba.end_time = current_time_monotonic() + BITRATE_CHANGE_TESTING_TIME_MS; } @@ -921,7 +921,7 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; } - if (call->aba.end_time == ~0) + if (call->aba.end_time == (uint64_t) ~0) call->aba.end_time = current_time_monotonic() + BITRATE_CHANGE_TESTING_TIME_MS; } } @@ -1056,6 +1056,7 @@ bool audio_bit_rate_invalid(uint32_t bit_rate) bool video_bit_rate_invalid(uint32_t bit_rate) { + (void) bit_rate; /* TODO: If anyone knows the answer to this one please fill it up */ return false; } @@ -1123,7 +1124,7 @@ ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) av->calls = tmp; /* Set fields in between to null */ - int32_t i = av->calls_tail + 1; + uint32_t i = av->calls_tail + 1; for (; i < friend_number; i ++) av->calls[i] = NULL; From eb6e8aa290a7423b666d037981fb453a6f897861 Mon Sep 17 00:00:00 2001 From: mannol Date: Fri, 1 May 2015 22:15:12 +0200 Subject: [PATCH 40/97] Fixed* api comments and some bugs --- auto_tests/toxav_basic_test.c | 77 +++++++++++++++++------------------ auto_tests/toxav_many_test.c | 36 ++++++++-------- toxav/toxav.c | 48 ++++++++++++++++------ toxav/toxav.h | 62 ++++++++++++---------------- toxav/video.c | 5 ++- 5 files changed, 124 insertions(+), 104 deletions(-) diff --git a/auto_tests/toxav_basic_test.c b/auto_tests/toxav_basic_test.c index 423cd03de..bdf6c9201 100644 --- a/auto_tests/toxav_basic_test.c +++ b/auto_tests/toxav_basic_test.c @@ -9,7 +9,6 @@ #include #include #include -#include #include @@ -103,7 +102,7 @@ void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t (void) userdata; if (length == 7 && memcmp("gentoo", data, 7) == 0) { - assert(tox_friend_add_norequest(m, public_key, NULL) != (uint32_t) ~0); + ck_assert(tox_friend_add_norequest(m, public_key, NULL) != (uint32_t) ~0); } } @@ -133,13 +132,13 @@ START_TEST(test_AV_flows) TOX_ERR_NEW error; bootstrap = tox_new(NULL, NULL, 0, &error); - assert(error == TOX_ERR_NEW_OK); + ck_assert(error == TOX_ERR_NEW_OK); Alice = tox_new(NULL, NULL, 0, &error); - assert(error == TOX_ERR_NEW_OK); + ck_assert(error == TOX_ERR_NEW_OK); Bob = tox_new(NULL, NULL, 0, &error); - assert(error == TOX_ERR_NEW_OK); + ck_assert(error == TOX_ERR_NEW_OK); } printf("Created 3 instances of Tox\n"); @@ -153,7 +152,7 @@ START_TEST(test_AV_flows) tox_self_get_address(Alice, address); - assert(tox_friend_add(Bob, address, (uint8_t *)"gentoo", 7, NULL) != (uint32_t) ~0); + ck_assert(tox_friend_add(Bob, address, (uint8_t *)"gentoo", 7, NULL) != (uint32_t) ~0); uint8_t off = 1; @@ -178,10 +177,10 @@ START_TEST(test_AV_flows) { TOXAV_ERR_NEW error; AliceAV = toxav_new(Alice, &error); - assert(error == TOXAV_ERR_NEW_OK); + ck_assert(error == TOXAV_ERR_NEW_OK); BobAV = toxav_new(Bob, &error); - assert(error == TOXAV_ERR_NEW_OK); + ck_assert(error == TOXAV_ERR_NEW_OK); } toxav_callback_call(AliceAV, t_toxav_call_cb, &AliceCC); @@ -208,7 +207,7 @@ START_TEST(test_AV_flows) \ if (rc != TOXAV_ERR_CALL_OK) { \ printf("toxav_call failed: %d\n", rc); \ - exit(1); \ + ck_assert(0); \ } \ \ \ @@ -223,7 +222,7 @@ START_TEST(test_AV_flows) \ if (rc != TOXAV_ERR_ANSWER_OK) { \ printf("toxav_answer failed: %d\n", rc); \ - exit(1); \ + ck_assert(0); \ } \ BobCC.incoming = false; \ } else { \ @@ -236,7 +235,7 @@ START_TEST(test_AV_flows) \ if (rc != TOXAV_ERR_CALL_CONTROL_OK) { \ printf("toxav_call_control failed: %d\n", rc); \ - exit(1); \ + ck_assert(0); \ } \ } \ } \ @@ -275,7 +274,7 @@ START_TEST(test_AV_flows) if (rc != TOXAV_ERR_CALL_OK) { printf("toxav_call failed: %d\n", rc); - exit(1); + ck_assert(0); } } @@ -289,7 +288,7 @@ START_TEST(test_AV_flows) if (rc != TOXAV_ERR_CALL_CONTROL_OK) { printf("toxav_call_control failed: %d\n", rc); - exit(1); + ck_assert(0); } } @@ -311,7 +310,7 @@ START_TEST(test_AV_flows) if (rc != TOXAV_ERR_CALL_OK) { printf("toxav_call failed: %d\n", rc); - exit(1); + ck_assert(0); } } @@ -325,7 +324,7 @@ START_TEST(test_AV_flows) if (rc != TOXAV_ERR_CALL_CONTROL_OK) { printf("toxav_call_control failed: %d\n", rc); - exit(1); + ck_assert(0); } } @@ -349,7 +348,7 @@ START_TEST(test_AV_flows) if (rc != TOXAV_ERR_CALL_OK) { printf("toxav_call failed: %d\n", rc); - exit(1); + ck_assert(0); } } @@ -357,10 +356,10 @@ START_TEST(test_AV_flows) iterate_tox(bootstrap, Alice, Bob); /* At first try all stuff while in invalid state */ - assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); - assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); - assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); + ck_assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); + ck_assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); + ck_assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); + ck_assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_HIDE_VIDEO, NULL)); { TOXAV_ERR_ANSWER rc; @@ -368,7 +367,7 @@ START_TEST(test_AV_flows) if (rc != TOXAV_ERR_ANSWER_OK) { printf("toxav_answer failed: %d\n", rc); - exit(1); + ck_assert(0); } } @@ -376,36 +375,36 @@ START_TEST(test_AV_flows) /* Pause and Resume */ printf("Pause and Resume\n"); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); + ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); iterate_tox(bootstrap, Alice, Bob); - assert(BobCC.state == TOXAV_CALL_STATE_PAUSED); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); + ck_assert(BobCC.state == TOXAV_CALL_STATE_PAUSED); + ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); iterate_tox(bootstrap, Alice, Bob); - assert(BobCC.state & (TOXAV_CALL_STATE_SENDING_A | TOXAV_CALL_STATE_SENDING_V)); + ck_assert(BobCC.state & (TOXAV_CALL_STATE_SENDING_A | TOXAV_CALL_STATE_SENDING_V)); /* Mute/Unmute single */ printf("Mute/Unmute single\n"); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); + ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); iterate_tox(bootstrap, Alice, Bob); - assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); + ck_assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); + ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); iterate_tox(bootstrap, Alice, Bob); - assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); + ck_assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); /* Mute/Unmute both */ printf("Mute/Unmute both\n"); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); + ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); iterate_tox(bootstrap, Alice, Bob); - assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); + ck_assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); + ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_HIDE_VIDEO, NULL)); iterate_tox(bootstrap, Alice, Bob); - assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_V); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); + ck_assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_V); + ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); iterate_tox(bootstrap, Alice, Bob); - assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); - assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); + ck_assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); + ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_HIDE_VIDEO, NULL)); iterate_tox(bootstrap, Alice, Bob); - assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_V); + ck_assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_V); { TOXAV_ERR_CALL_CONTROL rc; @@ -413,12 +412,12 @@ START_TEST(test_AV_flows) if (rc != TOXAV_ERR_CALL_CONTROL_OK) { printf("toxav_call_control failed: %d\n", rc); - exit(1); + ck_assert(0); } } iterate_tox(bootstrap, Alice, Bob); - assert(BobCC.state == TOXAV_CALL_STATE_END); + ck_assert(BobCC.state == TOXAV_CALL_STATE_END); printf("Success!\n"); } diff --git a/auto_tests/toxav_many_test.c b/auto_tests/toxav_many_test.c index 84f94e961..f913c9d30 100644 --- a/auto_tests/toxav_many_test.c +++ b/auto_tests/toxav_many_test.c @@ -9,10 +9,11 @@ #include #include #include -#include #include +#include "helpers.h" + #include "../toxcore/tox.h" #include "../toxcore/util.h" #include "../toxcore/logger.h" @@ -97,7 +98,7 @@ void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t { (void) userdata; if (length == 7 && memcmp("gentoo", data, 7) == 0) { - assert(tox_friend_add_norequest(m, public_key, NULL) != (uint32_t) ~0); + ck_assert(tox_friend_add_norequest(m, public_key, NULL) != (uint32_t) ~0); } } @@ -110,7 +111,7 @@ ToxAV* setup_av_instance(Tox* tox, CallControl *CC) TOXAV_ERR_NEW error; ToxAV* av = toxav_new(tox, &error); - assert(error == TOXAV_ERR_NEW_OK); + ck_assert(error == TOXAV_ERR_NEW_OK); toxav_callback_call(av, t_toxav_call_cb, CC); toxav_callback_call_state(av, t_toxav_call_state_cb, CC); @@ -137,7 +138,7 @@ void* call_thread(void* pd) if (rc != TOXAV_ERR_CALL_OK) { printf("toxav_call failed: %d\n", rc); - exit(1); + ck_assert(0); } } @@ -150,7 +151,7 @@ void* call_thread(void* pd) if (rc != TOXAV_ERR_ANSWER_OK) { printf("toxav_answer failed: %d\n", rc); - exit(1); + ck_assert(0); } } @@ -167,7 +168,7 @@ void* call_thread(void* pd) memset(video_v, 0, sizeof(video_v)); time_t start_time = time(NULL); - while(time(NULL) - start_time < 9) { + while(time(NULL) - start_time < 4) { toxav_iterate(AliceAV); toxav_iterate(BobAV); @@ -186,11 +187,13 @@ void* call_thread(void* pd) if (rc != TOXAV_ERR_CALL_CONTROL_OK) { printf("toxav_call_control failed: %d\n", rc); - exit(1); + ck_assert(0); } } c_sleep(30); + + printf ("Closing thread\n"); pthread_exit(NULL); } @@ -206,19 +209,19 @@ START_TEST(test_AV_three_calls) TOX_ERR_NEW error; bootstrap = tox_new(NULL, NULL, 0, &error); - assert(error == TOX_ERR_NEW_OK); + ck_assert(error == TOX_ERR_NEW_OK); Alice = tox_new(NULL, NULL, 0, &error); - assert(error == TOX_ERR_NEW_OK); + ck_assert(error == TOX_ERR_NEW_OK); Bobs[0] = tox_new(NULL, NULL, 0, &error); - assert(error == TOX_ERR_NEW_OK); + ck_assert(error == TOX_ERR_NEW_OK); Bobs[1] = tox_new(NULL, NULL, 0, &error); - assert(error == TOX_ERR_NEW_OK); + ck_assert(error == TOX_ERR_NEW_OK); Bobs[2] = tox_new(NULL, NULL, 0, &error); - assert(error == TOX_ERR_NEW_OK); + ck_assert(error == TOX_ERR_NEW_OK); } printf("Created 5 instances of Tox\n"); @@ -232,9 +235,9 @@ START_TEST(test_AV_three_calls) tox_self_get_address(Alice, address); - assert(tox_friend_add(Bobs[0], address, (uint8_t *)"gentoo", 7, NULL) != (uint32_t) ~0); - assert(tox_friend_add(Bobs[1], address, (uint8_t *)"gentoo", 7, NULL) != (uint32_t) ~0); - assert(tox_friend_add(Bobs[2], address, (uint8_t *)"gentoo", 7, NULL) != (uint32_t) ~0); + ck_assert(tox_friend_add(Bobs[0], address, (uint8_t *)"gentoo", 7, NULL) != (uint32_t) ~0); + ck_assert(tox_friend_add(Bobs[1], address, (uint8_t *)"gentoo", 7, NULL) != (uint32_t) ~0); + ck_assert(tox_friend_add(Bobs[2], address, (uint8_t *)"gentoo", 7, NULL) != (uint32_t) ~0); uint8_t off = 1; @@ -302,7 +305,7 @@ START_TEST(test_AV_three_calls) (void) pthread_detach(tids[2]); time_t start_time = time(NULL); - while (time(NULL) - start_time < 10) { + while (time(NULL) - start_time < 5) { tox_iterate(Alice); tox_iterate(Bobs[0]); tox_iterate(Bobs[1]); @@ -314,6 +317,7 @@ START_TEST(test_AV_three_calls) (void) pthread_join(tids[1], NULL); (void) pthread_join(tids[2], NULL); + printf ("Killing all instances\n"); toxav_kill(BobsAV[0]); toxav_kill(BobsAV[1]); toxav_kill(BobsAV[2]); diff --git a/toxav/toxav.c b/toxav/toxav.c index e7807592d..1817916b1 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -79,7 +79,7 @@ typedef struct ToxAVCall_s { struct ToxAVCall_s *next; } ToxAVCall; -struct toxAV { +struct ToxAV { Messenger* m; MSISession* msi; @@ -116,7 +116,7 @@ bool video_bit_rate_invalid(uint32_t bit_rate); void invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state); ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); ToxAVCall* call_get(ToxAV* av, uint32_t friend_number); -void call_remove(ToxAVCall* call); +ToxAVCall* call_remove(ToxAVCall* call); bool call_prepare_transmission(ToxAVCall* call); void call_kill_transmission(ToxAVCall* call); void ba_set(ToxAvBitrateAdapter* ba, uint32_t bit_rate); @@ -193,9 +193,9 @@ void toxav_kill(ToxAV* av) /* Msi kill will hang up all calls so just clean these calls */ if (av->calls) { ToxAVCall* it = call_get(av, av->calls_head); - for (; it; it = it->next) { + while (it) { call_kill_transmission(it); - call_remove(it); /* This will eventually free av->calls */ + it = call_remove(it); /* This will eventually free av->calls */ } } @@ -318,6 +318,14 @@ void toxav_iterate(ToxAV* av) bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) { + if ((audio_bit_rate && audio_bit_rate_invalid(audio_bit_rate)) + ||(video_bit_rate && video_bit_rate_invalid(video_bit_rate)) + ) { + if (error) + *error = TOXAV_ERR_CALL_INVALID_BIT_RATE; + return false; + } + pthread_mutex_lock(av->mutex); ToxAVCall* call = call_new(av, friend_number, error); if (call == NULL) { @@ -368,7 +376,7 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui if ((audio_bit_rate && audio_bit_rate_invalid(audio_bit_rate)) ||(video_bit_rate && video_bit_rate_invalid(video_bit_rate)) ) { - rc = TOXAV_ERR_CALL_INVALID_BIT_RATE; + rc = TOXAV_ERR_ANSWER_INVALID_BIT_RATE; goto END; } @@ -379,7 +387,7 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui } if (!call_prepare_transmission(call)) { - rc = TOXAV_ERR_ANSWER_MALLOC; + rc = TOXAV_ERR_ANSWER_CODEC_INITIALIZATION; goto END; } @@ -450,6 +458,9 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co rtp_start_receiving(call->audio.first); rtp_start_receiving(call->video.first); + } else { + rc = TOXAV_ERR_CALL_CONTROL_NOT_PAUSED; + goto END; } } break; @@ -472,6 +483,9 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co rtp_stop_receiving(call->audio.first); rtp_stop_receiving(call->video.first); + } else { + rc = TOXAV_ERR_CALL_CONTROL_ALREADY_PAUSED; + goto END; } } break; @@ -516,7 +530,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co } } break; - case TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO: { + case TOXAV_CALL_CONTROL_TOGGLE_HIDE_VIDEO: { if (!call->active) { rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; goto END; @@ -589,7 +603,7 @@ bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_ goto END; } - if (call->video_bit_rate == video_bit_rate || call->vba.active || call->vba.bit_rate == video_bit_rate) { + if (call->video_bit_rate == video_bit_rate || (call->vba.active && call->vba.bit_rate == video_bit_rate)) { pthread_mutex_unlock(av->mutex); goto END; } @@ -599,6 +613,8 @@ bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_ if (video_bit_rate > call->video_bit_rate && !force) ba_set(&call->vba, video_bit_rate); else { + /* Cancel any previous non forceful bitrate change request */ + memset(&call->vba, 0, sizeof(call->vba)); call->video_bit_rate = video_bit_rate; if (!force && av->vbcb.first) @@ -646,16 +662,19 @@ bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_ goto END; } - if (call->audio_bit_rate == audio_bit_rate || call->aba.active || call->aba.bit_rate == audio_bit_rate) { + if (call->audio_bit_rate == audio_bit_rate || (call->aba.active && call->aba.bit_rate == audio_bit_rate)) { pthread_mutex_unlock(av->mutex); goto END; } + pthread_mutex_lock(call->mutex); if (audio_bit_rate > call->audio_bit_rate && !force) ba_set(&call->aba, audio_bit_rate); else { + /* Cancel any previous non forceful bitrate change request */ + memset(&call->aba, 0, sizeof(call->aba)); call->audio_bit_rate = audio_bit_rate; if (!force && av->abcb.first) @@ -755,6 +774,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u if (rtp_send_data(call->video.first, iter, part_size, false) < 0) { pthread_mutex_unlock(call->mutex_video); LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); + rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; goto END; } } @@ -807,6 +827,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u if (rtp_send_data(call->video.first, pkt->data.frame.buf + i * 1300, 1300, true) < 0) { pthread_mutex_unlock(call->mutex_video); LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); + rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; goto END; } } @@ -815,6 +836,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u if (rtp_send_data(call->video.first, pkt->data.frame.buf + parts * 1300, pkt->data.frame.sz % 1300, true) < 0) { pthread_mutex_unlock(call->mutex_video); LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); + rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; goto END; } } @@ -1157,10 +1179,10 @@ ToxAVCall* call_get(ToxAV* av, uint32_t friend_number) return av->calls[friend_number]; } -void call_remove(ToxAVCall* call) +ToxAVCall* call_remove(ToxAVCall* call) { if (call == NULL) - return; + return NULL; uint32_t friend_number = call->friend_number; ToxAV* av = call->av; @@ -1183,12 +1205,14 @@ void call_remove(ToxAVCall* call) else goto CLEAR; av->calls[friend_number] = NULL; - return; + return next; CLEAR: av->calls_head = av->calls_tail = 0; free(av->calls); av->calls = NULL; + + return NULL; } bool call_prepare_transmission(ToxAVCall* call) diff --git a/toxav/toxav.h b/toxav/toxav.h index 67e5496e8..125422eb4 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -37,7 +37,7 @@ extern "C" { /** * The type of the Tox Audio/Video subsystem object. */ -typedef struct toxAV ToxAV; +typedef struct ToxAV ToxAV; #ifndef TOX_DEFINED #define TOX_DEFINED /** @@ -92,8 +92,8 @@ Tox *toxav_get_tox(ToxAV *av); uint32_t toxav_iteration_interval(ToxAV const *av); /** * Main loop for the session. This function needs to be called in intervals of - * toxav_iteration_interval() milliseconds. It is best called in the same loop - * as tox_iterate. + * toxav_iteration_interval() milliseconds. It is best called in the separate + * thread from tox_iterate. */ void toxav_iterate(ToxAV *av); /******************************************************************************* @@ -153,10 +153,11 @@ void toxav_callback_call(ToxAV *av, toxav_call_cb *function, void *user_data); typedef enum TOXAV_ERR_ANSWER { TOXAV_ERR_ANSWER_OK, /** - * A resource allocation error occurred while trying to create the structures - * required for the call. + * Failed to initialize codecs for call session. Note that codec initiation + * will fail if there is no receive callback registered for either audio or + * video. */ - TOXAV_ERR_ANSWER_MALLOC, + TOXAV_ERR_ANSWER_CODEC_INITIALIZATION, /** * The friend number did not designate a valid friend. */ @@ -174,8 +175,8 @@ typedef enum TOXAV_ERR_ANSWER { /** * Accept an incoming call. * - * If an allocation error occurs while answering a call, both participants will - * receive TOXAV_CALL_STATE_ERROR and the call will end. + * If answering fails for any reason, the call will still be pending and it is + * possible to try and answer it later. * * @param friend_number The friend number of the friend that is calling. * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable @@ -217,7 +218,8 @@ typedef enum TOXAV_CALL_STATE { */ TOXAV_CALL_STATE_END = 16, /** - * Set by the AV core if an error occurred on the remote end. + * Set by the AV core if an error occurred on the remote end. This call + * state will never be triggered in combination with other call states. */ TOXAV_CALL_STATE_ERROR = 32768 } TOXAV_CALL_STATE; @@ -266,10 +268,10 @@ typedef enum TOXAV_CALL_CONTROL { * Request that the friend stops sending video. Regardless of the friend's * compliance, this will cause the `receive_video_frame` event to stop being * triggered on receiving an video frame from the friend. If the video was - * already muted, calling this control will notify client to start sending + * already hidden, calling this control will notify client to start sending * video again. */ - TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, + TOXAV_CALL_CONTROL_TOGGLE_HIDE_VIDEO } TOXAV_CALL_CONTROL; typedef enum TOXAV_ERR_CALL_CONTROL { TOXAV_ERR_CALL_CONTROL_OK, @@ -286,21 +288,12 @@ typedef enum TOXAV_ERR_CALL_CONTROL { * Attempted to resume a call that was not paused. */ TOXAV_ERR_CALL_CONTROL_NOT_PAUSED, - /** - * Attempted to resume a call that was paused by the other party. Also set if - * the client attempted to send a system-only control. - */ - TOXAV_ERR_CALL_CONTROL_DENIED, /** * The call was already paused on this client. It is valid to pause if the * other party paused the call. The call will resume after both parties sent * the RESUME control. */ - TOXAV_ERR_CALL_CONTROL_ALREADY_PAUSED, - /** - * Tried to unmute a call that was not already muted. - */ - TOXAV_ERR_CALL_CONTROL_NOT_MUTED + TOXAV_ERR_CALL_CONTROL_ALREADY_PAUSED } TOXAV_ERR_CALL_CONTROL; /** * Sends a call control command to a friend. @@ -351,7 +344,12 @@ typedef void toxav_audio_bit_rate_status_cb(ToxAV *av, uint32_t friend_number, b */ void toxav_callback_audio_bit_rate_status(ToxAV *av, toxav_audio_bit_rate_status_cb *function, void *user_data); /** - * Set the audio bit rate to be used in subsequent audio frames. + * Set the audio bit rate to be used in subsequent audio frames. If the passed + * bit rate is the same as the current bit rate this function will return true + * without calling a callback. If there is an active non forceful setup with the + * passed audio bit rate and the new set request is forceful, the bit rate is + * forcefully set and the previous non forceful request is cancelled. The active + * non forceful setup will be canceled in favour of new non forceful setup. * * @param friend_number The friend number of the friend for which to set the * audio bit rate. @@ -380,7 +378,12 @@ typedef void toxav_video_bit_rate_status_cb(ToxAV *av, uint32_t friend_number, b */ void toxav_callback_video_bit_rate_status(ToxAV *av, toxav_video_bit_rate_status_cb *function, void *user_data); /** - * Set the video bit rate to be used in subsequent video frames. + * Set the video bit rate to be used in subsequent video frames. If the passed + * bit rate is the same as the current bit rate this function will return true + * without calling a callback. If there is an active non forceful setup with the + * passed bit rate and the new set request is forceful, the bit rate is + * forcefully set and the previous non forceful request is cancelled. The active + * non forceful setup will be canceled in favour of new non forceful setup. * * @param friend_number The friend number of the friend for which to set the * video bit rate. @@ -413,11 +416,6 @@ typedef enum TOXAV_ERR_SEND_FRAME { * This client is currently not in a call with the friend. */ TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL, - /** - * No video frame had been requested through the `video_frame_request` event, - * but the client tried to send one, anyway. - */ - TOXAV_ERR_SEND_FRAME_NOT_REQUESTED, /** * One of the frame parameters was invalid. E.g. the resolution may be too * small or too large, or the audio sampling rate may be unsupported. @@ -431,8 +429,6 @@ typedef enum TOXAV_ERR_SEND_FRAME { /** * Send a video frame to a friend. * - * This is called in response to receiving the `video_frame_request` event. - * * Y - plane should be of size: height * width * U - plane should be of size: (height/2) * (width/2) * V - plane should be of size: (height/2) * (width/2) @@ -452,8 +448,6 @@ bool toxav_send_video_frame(ToxAV *av, uint32_t friend_number, /** * Send an audio frame to a friend. * - * This is called in response to receiving the `audio_frame_request` event. - * * The expected format of the PCM data is: [s1c1][s1c2][...][s2c1][s2c2][...]... * Meaning: sample 1 for channel 1, sample 1 for channel 2, ... * For mono audio, this has no meaning, every sample is subsequent. For stereo, @@ -467,9 +461,7 @@ bool toxav_send_video_frame(ToxAV *av, uint32_t friend_number, * @param sample_count Number of samples in this frame. Valid numbers here are * ((sample rate) * (audio length) / 1000), where audio length can be * 2.5, 5, 10, 20, 40 or 60 millseconds. - * @param channels Number of audio channels. Must be at least 1 for mono. - * For voice over IP, more than 2 channels (stereo) typically doesn't make - * sense, but up to 255 channels are supported. + * @param channels Number of audio channels. Supported values are 1 and 2. * @param sampling_rate Audio sampling rate used in this frame. Valid sampling * rates are 8000, 12000, 16000, 24000, or 48000. */ diff --git a/toxav/video.c b/toxav/video.c index 22ca2beef..fe57387f7 100644 --- a/toxav/video.c +++ b/toxav/video.c @@ -87,6 +87,7 @@ VCSession* vc_new(ToxAV* av, uint32_t friend_number, toxav_receive_video_frame_c vc->vcb.second = cb_data; vc->friend_number = friend_number; vc->peer_video_frame_piece_size = mvfpsz; + vc->av = av; return vc; @@ -294,7 +295,7 @@ int vc_reconfigure_encoder(VCSession* vc, int32_t bit_rate, uint16_t width, uint return -1; vpx_codec_enc_cfg_t cfg = *vc->encoder->config.enc; - if (cfg.rc_target_bitrate == bit_rate && cfg.g_w == width && cfg.g_h == height) + if (cfg.rc_target_bitrate == (uint32_t) bit_rate && cfg.g_w == width && cfg.g_h == height) return 0; /* Nothing changed */ cfg.rc_target_bitrate = bit_rate; @@ -315,7 +316,7 @@ int vc_reconfigure_test_encoder(VCSession* vc, int32_t bit_rate, uint16_t width, return -1; vpx_codec_enc_cfg_t cfg = *vc->test_encoder->config.enc; - if (cfg.rc_target_bitrate == bit_rate && cfg.g_w == width && cfg.g_h == height) + if (cfg.rc_target_bitrate == (uint32_t) bit_rate && cfg.g_w == width && cfg.g_h == height) return 0; /* Nothing changed */ cfg.rc_target_bitrate = bit_rate; From 73fbc2296117f8a507e8ed9399e1cf13f19ddb19 Mon Sep 17 00:00:00 2001 From: mannol Date: Thu, 7 May 2015 23:14:03 +0200 Subject: [PATCH 41/97] Fixed inconsistencies --- auto_tests/toxav_basic_test.c | 4 ++-- testing/av_test.c | 4 ++-- toxav/rtp.c | 2 +- toxav/rtp.h | 1 - toxav/toxav.c | 2 +- toxav/toxav.h | 10 +++------- 6 files changed, 9 insertions(+), 14 deletions(-) diff --git a/auto_tests/toxav_basic_test.c b/auto_tests/toxav_basic_test.c index bdf6c9201..9f04dc113 100644 --- a/auto_tests/toxav_basic_test.c +++ b/auto_tests/toxav_basic_test.c @@ -292,7 +292,7 @@ START_TEST(test_AV_flows) } } - while (AliceCC.state != TOXAV_CALL_STATE_END) + while (AliceCC.state != TOXAV_CALL_STATE_FINISHED) iterate_tox(bootstrap, Alice, Bob); printf("Success!\n"); @@ -329,7 +329,7 @@ START_TEST(test_AV_flows) } /* Alice will not receive end state */ - while (BobCC.state != TOXAV_CALL_STATE_END) + while (BobCC.state != TOXAV_CALL_STATE_FINISHED) iterate_tox(bootstrap, Alice, Bob); printf("Success!\n"); diff --git a/testing/av_test.c b/testing/av_test.c index 7298ed235..0e7af66ef 100644 --- a/testing/av_test.c +++ b/testing/av_test.c @@ -623,7 +623,7 @@ int main (int argc, char** argv) } iterate_tox(bootstrap, AliceAV, BobAV); - assert(BobCC.state == TOXAV_CALL_STATE_END); + assert(BobCC.state == TOXAV_CALL_STATE_FINISHED); /* Stop decode thread */ data.sig = -1; @@ -717,7 +717,7 @@ int main (int argc, char** argv) } iterate_tox(bootstrap, AliceAV, BobAV); - assert(BobCC.state == TOXAV_CALL_STATE_END); + assert(BobCC.state == TOXAV_CALL_STATE_FINISHED); /* Stop decode thread */ printf("Stopping decode thread\n"); diff --git a/toxav/rtp.c b/toxav/rtp.c index 4ca23d2a6..7b3a5ed0f 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -285,7 +285,7 @@ int rtp_send_data ( RTPSession *session, const uint8_t *data, uint16_t length, b return -1; } - session->sequnum = session->sequnum >= MAX_SEQU_NUM ? 0 : session->sequnum + 1; + session->sequnum ++; return 0; } void rtp_free_msg ( RTPMessage *msg ) diff --git a/toxav/rtp.h b/toxav/rtp.h index a158d7245..b18d6f8dd 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -38,7 +38,6 @@ /*LOGGER_DEBUG("Unlocked mutex: %p", mutex);*/\ } while(0) -#define MAX_SEQU_NUM 65535 #define MAX_RTP_SIZE 1500 /** diff --git a/toxav/toxav.c b/toxav/toxav.c index 1817916b1..31629c291 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -1034,7 +1034,7 @@ int callback_end(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - invoke_call_state(toxav, call->friend_number, TOXAV_CALL_STATE_END); + invoke_call_state(toxav, call->friend_number, TOXAV_CALL_STATE_FINISHED); call_kill_transmission(call->av_call); call_remove(call->av_call); diff --git a/toxav/toxav.h b/toxav/toxav.h index 125422eb4..994b30094 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -191,11 +191,6 @@ bool toxav_answer(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, ui * ******************************************************************************/ typedef enum TOXAV_CALL_STATE { - /** - * Not sending nor receiving anything, meaning, one of the sides requested pause. - * The call will be resumed once the side that initiated pause resumes it. - */ - TOXAV_CALL_STATE_PAUSED = 0, /** * The flag that marks that friend is sending audio. */ @@ -214,9 +209,10 @@ typedef enum TOXAV_CALL_STATE { TOXAV_CALL_STATE_RECEIVING_V = 8, /** * The call has finished. This is the final state after which no more state - * transitions can occur for the call. + * transitions can occur for the call. This call state will never be + * triggered in combination with other call states. */ - TOXAV_CALL_STATE_END = 16, + TOXAV_CALL_STATE_FINISHED = 16, /** * Set by the AV core if an error occurred on the remote end. This call * state will never be triggered in combination with other call states. From 64037017cc3804d9921bd9780570685871df8f0b Mon Sep 17 00:00:00 2001 From: mannol Date: Tue, 12 May 2015 22:16:00 +0200 Subject: [PATCH 42/97] Fix bug --- toxav/msi.c | 6 ++++++ toxav/toxav.c | 19 +++++++++++++++---- toxav/toxav.h | 4 ++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/toxav/msi.c b/toxav/msi.c index d68e4a9c5..65b6c4e45 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -205,6 +205,12 @@ int msi_hangup ( MSICall* call ) MSISession* session = call->session; pthread_mutex_lock(session->mutex); + if ( call->state == msi_CallInactive ) { + LOGGER_ERROR("Call is in invalid state!"); + pthread_mutex_unlock(session->mutex); + return -1; + } + MSIMessage msg; msg_init(&msg, requ_pop); diff --git a/toxav/toxav.c b/toxav/toxav.c index 31629c291..5e32f1969 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -113,7 +113,7 @@ int callback_capabilites(void* toxav_inst, MSICall* call); bool audio_bit_rate_invalid(uint32_t bit_rate); bool video_bit_rate_invalid(uint32_t bit_rate); -void invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state); +bool invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state); ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); ToxAVCall* call_get(ToxAV* av, uint32_t friend_number); ToxAVCall* call_remove(ToxAVCall* call); @@ -998,6 +998,11 @@ int callback_invite(void* toxav_inst, MSICall* call) if (toxav->ccb.first) toxav->ccb.first(toxav, call->friend_number, call->peer_capabilities & msi_CapSAudio, call->peer_capabilities & msi_CapSVideo, toxav->ccb.second); + else { + /* No handler to capture the call request, send failure */ + pthread_mutex_unlock(toxav->mutex); + return -1; + } pthread_mutex_unlock(toxav->mutex); return 0; @@ -1018,12 +1023,15 @@ int callback_start(void* toxav_inst, MSICall* call) if (!call_prepare_transmission(av_call)) { callback_error(toxav_inst, call); - call_remove(av_call); pthread_mutex_unlock(toxav->mutex); return -1; } - invoke_call_state(toxav, call->friend_number, call->peer_capabilities); + if (!invoke_call_state(toxav, call->friend_number, call->peer_capabilities)) { + callback_error(toxav_inst, call); + pthread_mutex_unlock(toxav->mutex); + return -1; + } pthread_mutex_unlock(toxav->mutex); return 0; @@ -1083,10 +1091,13 @@ bool video_bit_rate_invalid(uint32_t bit_rate) return false; } -void invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state) +bool invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state) { if (av->scb.first) av->scb.first(av, friend_number, state, av->scb.second); + else + return false; + return true; } ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) diff --git a/toxav/toxav.h b/toxav/toxav.h index 994b30094..b0e7e37d1 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -190,7 +190,7 @@ bool toxav_answer(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, ui * :: Call state graph * ******************************************************************************/ -typedef enum TOXAV_CALL_STATE { +enum TOXAV_CALL_STATE { /** * The flag that marks that friend is sending audio. */ @@ -218,7 +218,7 @@ typedef enum TOXAV_CALL_STATE { * state will never be triggered in combination with other call states. */ TOXAV_CALL_STATE_ERROR = 32768 -} TOXAV_CALL_STATE; +}; /** * The function type for the `call_state` callback. * From 62c40af1a0c557ba8c77583c972ae3af9af15cf1 Mon Sep 17 00:00:00 2001 From: mannol Date: Tue, 12 May 2015 23:27:49 +0200 Subject: [PATCH 43/97] Added apidsl input file --- other/apidsl/toxav.in.h | 625 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 625 insertions(+) create mode 100644 other/apidsl/toxav.in.h diff --git a/other/apidsl/toxav.in.h b/other/apidsl/toxav.in.h new file mode 100644 index 000000000..9de1e1841 --- /dev/null +++ b/other/apidsl/toxav.in.h @@ -0,0 +1,625 @@ +%{ +/* toxav.h + * + * Copyright (C) 2013-2015 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + */ + +#ifndef TOXAV_H +#define TOXAV_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif +%} + +/** \page av Public audio/video API for Tox clients. + * + * This API can handle multiple calls. Each call has its state, in very rare + * occasions the library can change the state of the call without apps knowledge. + * + */ + +/** \subsection events Events and callbacks + * + * As in Core API, events are handled by callbacks. One callback can be + * registered per event. All events have a callback function type named + * `toxav_{event}_cb` and a function to register it named `tox_callback_{event}`. + * Passing a NULL callback will result in no callback being registered for that + * event. Only one callback per event can be registered, so if a client needs + * multiple event listeners, it needs to implement the dispatch functionality + * itself. Unlike Core API, lack of some event handlers will cause the the + * library to drop calls before they are started. Hanging up call from a + * callback causes undefined behaviour. + * + */ + +/** \subsection threading Threading implications + * + * Unlike the Core API, this API is fully thread-safe. The library will ensure + * the proper synchronisation of parallel calls. + * + * A common way to run ToxAV (multiple or single instance) is to have a thread, + * separate from tox instance thread, running a simple ${toxAV.iterate} loop, + * sleeping for ${toxAV.iteration_interval} * milliseconds on each iteration. + * + */ + +/** + * External Tox type. + */ +class tox { + struct this; +} + +/** + * ToxAV. + */ +class toxAV { + +/** + * The ToxAV instance type. Each ToxAV instance can be bound to only one Tox + * instance, and Tox instance can have only one ToxAV instance. One must make + * sure to close ToxAV instance prior closing Tox instance otherwise undefined + * behaviour occurs. Upon closing of ToxAV instance, all active calls will be + * forcibly terminated without notifying peers. + * + */ +struct this; +/******************************************************************************* + * + * :: API version + * + ******************************************************************************/ +/** + * The major version number. Incremented when the API or ABI changes in an + * incompatible way. + */ +#define TOXAV_VERSION_MAJOR 0u +/** + * The minor version number. Incremented when functionality is added without + * breaking the API or ABI. Set to 0 when the major version number is + * incremented. + */ +#define TOXAV_VERSION_MINOR 0u +/** + * The patch or revision number. Incremented when bugfixes are applied without + * changing any functionality or API or ABI. + */ +#define TOXAV_VERSION_PATCH 0u + +/** + * A macro to check at preprocessing time whether the client code is compatible + * with the installed version of ToxAV. + */ +#define TOXAV_VERSION_IS_API_COMPATIBLE(MAJOR, MINOR, PATCH) \ + (TOXAV_VERSION_MAJOR == MAJOR && \ + (TOXAV_VERSION_MINOR > MINOR || \ + (TOXAV_VERSION_MINOR == MINOR && \ + TOXAV_VERSION_PATCH >= PATCH))) + +/** + * A macro to make compilation fail if the client code is not compatible with + * the installed version of ToxAV. + */ +#define TOXAV_VERSION_REQUIRE(MAJOR, MINOR, PATCH) \ + typedef char toxav_required_version[TOXAV_IS_COMPATIBLE(MAJOR, MINOR, PATCH) ? 1 : -1] + +/** + * A convenience macro to call ${version.is_compatible} with the currently + * compiling API version. + */ +#define TOXAV_VERSION_IS_ABI_COMPATIBLE() \ + toxav_version_is_compatible(TOXAV_VERSION_MAJOR, TOXAV_VERSION_MINOR, TOXAV_VERSION_PATCH) + + +static namespace version { + + /** + * Return the major version number of the library. Can be used to display the + * ToxAV library version or to check whether the client is compatible with the + * dynamically linked version of ToxAV. + */ + uint32_t major(); + + /** + * Return the minor version number of the library. + */ + uint32_t minor(); + + /** + * Return the patch number of the library. + */ + uint32_t patch(); + + /** + * Return whether the compiled library version is compatible with the passed + * version numbers. + */ + bool is_compatible(uint32_t major, uint32_t minor, uint32_t patch); + +} +/******************************************************************************* + * + * :: Creation and destruction + * + ******************************************************************************/ +/** + * Start new A/V session. There can only be only one session per Tox instance. + */ +static this new (tox::this *tox) { + NULL, + /** + * Memory allocation failure while trying to allocate structures required for + * the A/V session. + */ + MALLOC, + /** + * Attempted to create a second session for the same Tox instance. + */ + MULTIPLE, +} +/** + * Releases all resources associated with the A/V session. + * + * If any calls were ongoing, these will be forcibly terminated without + * notifying peers. After calling this function, no other functions may be + * called and the av pointer becomes invalid. + */ +void kill(); +/** + * Returns the Tox instance the A/V object was created for. + */ +tox::this *tox { get(); } +/******************************************************************************* + * + * :: A/V event loop + * + ******************************************************************************/ +/** + * Returns the interval in milliseconds when the next toxav_iterate call should + * be. If no call is active at the moment, this function returns 200. + */ +const uint32_t iteration_interval(); +/** + * Main loop for the session. This function needs to be called in intervals of + * toxav_iteration_interval() milliseconds. It is best called in the separate + * thread from tox_iterate. + */ +void iterate(); +/******************************************************************************* + * + * :: Call setup + * + ******************************************************************************/ +/** + * Call a friend. This will start ringing the friend. + * + * It is the client's responsibility to stop ringing after a certain timeout, + * if such behaviour is desired. If the client does not stop ringing, the + * library will not stop until the friend is disconnected. + * + * @param friend_number The friend number of the friend that should be called. + * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable + * audio sending. + * @param video_bit_rate Video bit rate in Kb/sec. Set this to 0 to disable + * video sending. + */ +bool call(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) { + /** + * A resource allocation error occurred while trying to create the structures + * required for the call. + */ + MALLOC, + /** + * The friend number did not designate a valid friend. + */ + FRIEND_NOT_FOUND, + /** + * The friend was valid, but not currently connected. + */ + FRIEND_NOT_CONNECTED, + /** + * Attempted to call a friend while already in an audio or video call with + * them. + */ + FRIEND_ALREADY_IN_CALL, + /** + * Audio or video bit rate is invalid. + */ + INVALID_BIT_RATE, +} +event call { + /** + * The function type for the ${event call} callback. + * + * @param friend_number The friend number from which the call is incoming. + * @param audio_enabled True if friend is sending audio. + * @param video_enabled True if friend is sending video. + */ + typedef void(uint32_t friend_number, bool audio_enabled, bool video_enabled); +} +/** + * Accept an incoming call. + * + * If answering fails for any reason, the call will still be pending and it is + * possible to try and answer it later. + * + * @param friend_number The friend number of the friend that is calling. + * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable + * audio sending. + * @param video_bit_rate Video bit rate in Kb/sec. Set this to 0 to disable + * video sending. + */ +bool answer(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) { + /** + * Failed to initialize codecs for call session. Note that codec initiation + * will fail if there is no receive callback registered for either audio or + * video. + */ + CODEC_INITIALIZATION, + /** + * The friend number did not designate a valid friend. + */ + FRIEND_NOT_FOUND, + /** + * The friend was valid, but they are not currently trying to initiate a call. + * This is also returned if this client is already in a call with the friend. + */ + FRIEND_NOT_CALLING, + /** + * Audio or video bit rate is invalid. + */ + INVALID_BIT_RATE, +} +/******************************************************************************* + * + * :: Call state graph + * + ******************************************************************************/ +bitmask CALL_STATE { + /** + * Set by the AV core if an error occurred on the remote end or if friend + * timed out. This is the final state after which no more state + * transitions can occur for the call. This call state will never be triggered + * in combination with other call states. + */ + ERROR, + /** + * The call has finished. This is the final state after which no more state + * transitions can occur for the call. This call state will never be + * triggered in combination with other call states. + */ + FINISHED, + /** + * The flag that marks that friend is sending audio. + */ + SENDING_A, + /** + * The flag that marks that friend is sending video. + */ + SENDING_V, + /** + * The flag that marks that friend is receiving audio. + */ + RECEIVING_A, + /** + * The flag that marks that friend is receiving video. + */ + RECEIVING_V, +} +event call_state { + /** + * The function type for the ${event call_state} callback. + * + * @param friend_number The friend number for which the call state changed. + * @param state The new call state which is guaranteed to be different than + * the previous state. The state is set to 0 when the call is paused. + */ + typedef void(uint32_t friend_number, uint32_t state); +} +/******************************************************************************* + * + * :: Call control + * + ******************************************************************************/ +enum class CALL_CONTROL { + /** + * Resume a previously paused call. Only valid if the pause was caused by this + * client, if not, this control is ignored. Not valid before the call is accepted. + */ + RESUME, + /** + * Put a call on hold. Not valid before the call is accepted. + */ + PAUSE, + /** + * Reject a call if it was not answered, yet. Cancel a call after it was + * answered. + */ + CANCEL, + /** + * Request that the friend stops sending audio. Regardless of the friend's + * compliance, this will cause the ${event audio.receive_frame} event to stop being + * triggered on receiving an audio frame from the friend. + */ + MUTE_AUDIO, + /** + * Calling this control will notify client to start sending audio again. + */ + UNMUTE_AUDIO, + /** + * Request that the friend stops sending video. Regardless of the friend's + * compliance, this will cause the ${event video.receive_frame} event to stop being + * triggered on receiving an video frame from the friend. + */ + HIDE_VIDEO, + /** + * Calling this control will notify client to start sending video again. + */ + SHOW_VIDEO, +} +/** + * Sends a call control command to a friend. + * + * @param friend_number The friend number of the friend this client is in a call + * with. + * @param control The control command to send. + * + * @return true on success. + */ +bool call_control (uint32_t friend_number, CALL_CONTROL control) { + /** + * The friend_number passed did not designate a valid friend. + */ + FRIEND_NOT_FOUND, + /** + * This client is currently not in a call with the friend. Before the call is + * answered, only CANCEL is a valid control. + */ + FRIEND_NOT_IN_CALL, + /** + * Happens if user tried to pause an already paused call or if trying to + * resume a call that is not paused. + */ + INVALID_TRANSITION, +} +/******************************************************************************* + * + * :: Controlling bit rates + * + ******************************************************************************/ +error for set_bit_rate { + /** + * The bit rate passed was not one of the supported values. + */ + INVALID, + /** + * The friend_number passed did not designate a valid friend. + */ + FRIEND_NOT_FOUND, + /** + * This client is currently not in a call with the friend. + */ + FRIEND_NOT_IN_CALL, +} +namespace audio { + namespace bit_rate { + event status { + /** + * The function type for the ${event status} callback. + * + * @param friend_number The friend number of the friend for which to set the + * audio bit rate. + * @param stable Is the stream stable enough to keep the bit rate. + * Upon successful, non forceful, bit rate change, this is set to + * true and 'bit_rate' is set to new bit rate. + * The stable is set to false with bit_rate set to the unstable + * bit rate when either current stream is unstable with said bit rate + * or the non forceful change failed. + * @param bit_rate The bit rate in Kb/sec. + */ + typedef void(uint32_t friend_number, bool stable, uint32_t bit_rate); + } + /** + * Set the audio bit rate to be used in subsequent audio frames. If the passed + * bit rate is the same as the current bit rate this function will return true + * without calling a callback. If there is an active non forceful setup with the + * passed audio bit rate and the new set request is forceful, the bit rate is + * forcefully set and the previous non forceful request is cancelled. The active + * non forceful setup will be canceled in favour of new non forceful setup. + * + * @param friend_number The friend number of the friend for which to set the + * audio bit rate. + * @param audio_bit_rate The new audio bit rate in Kb/sec. Set to 0 to disable + * audio sending. + * @param force True if the bit rate change is forceful. + * + */ + bool set(uint32_t friend_number, uint32_t audio_bit_rate, bool force) with error for set_bit_rate; + } +} +namespace video { + namespace bit_rate { + event status { + /** + * The function type for the ${event status} callback. + * + * @param friend_number The friend number of the friend for which to set the + * video bit rate. + * @param stable Is the stream stable enough to keep the bit rate. + * Upon successful, non forceful, bit rate change, this is set to + * true and 'bit_rate' is set to new bit rate. + * The stable is set to false with bit_rate set to the unstable + * bit rate when either current stream is unstable with said bit rate + * or the non forceful change failed. + * @param bit_rate The bit rate in Kb/sec. + */ + typedef void(uint32_t friend_number, bool stable, uint32_t bit_rate); + } + /** + * Set the video bit rate to be used in subsequent video frames. If the passed + * bit rate is the same as the current bit rate this function will return true + * without calling a callback. If there is an active non forceful setup with the + * passed video bit rate and the new set request is forceful, the bit rate is + * forcefully set and the previous non forceful request is cancelled. The active + * non forceful setup will be canceled in favour of new non forceful setup. + * + * @param friend_number The friend number of the friend for which to set the + * video bit rate. + * @param audio_bit_rate The new video bit rate in Kb/sec. Set to 0 to disable + * video sending. + * @param force True if the bit rate change is forceful. + * + */ + bool set(uint32_t friend_number, uint32_t audio_bit_rate, bool force) with error for set_bit_rate; + } +} +/******************************************************************************* + * + * :: A/V sending + * + ******************************************************************************/ +error for send_frame { + /** + * In case of video, one of Y, U, or V was NULL. In case of audio, the samples + * data pointer was NULL. + */ + NULL, + /** + * The friend_number passed did not designate a valid friend. + */ + FRIEND_NOT_FOUND, + /** + * This client is currently not in a call with the friend. + */ + FRIEND_NOT_IN_CALL, + /** + * One of the frame parameters was invalid. E.g. the resolution may be too + * small or too large, or the audio sampling rate may be unsupported. + */ + INVALID, + /** + * Failed to push frame through rtp interface. + */ + RTP_FAILED, +} +namespace audio { + /** + * Send an audio frame to a friend. + * + * The expected format of the PCM data is: [s1c1][s1c2][...][s2c1][s2c2][...]... + * Meaning: sample 1 for channel 1, sample 1 for channel 2, ... + * For mono audio, this has no meaning, every sample is subsequent. For stereo, + * this means the expected format is LRLRLR... with samples for left and right + * alternating. + * + * @param friend_number The friend number of the friend to which to send an + * audio frame. + * @param pcm An array of audio samples. The size of this array must be + * sample_count * channels. + * @param sample_count Number of samples in this frame. Valid numbers here are + * ((sample rate) * (audio length) / 1000), where audio length can be + * 2.5, 5, 10, 20, 40 or 60 millseconds. + * @param channels Number of audio channels. Supported values are 1 and 2. + * @param sampling_rate Audio sampling rate used in this frame. Valid sampling + * rates are 8000, 12000, 16000, 24000, or 48000. + */ + bool send_frame(uint32_t friend_number, const int16_t *pcm, size_t sample_count, + uint8_t channels, uint32_t sampling_rate) with error for send_frame; +} +namespace video { + /** + * Send a video frame to a friend. + * + * Y - plane should be of size: height * width + * U - plane should be of size: (height/2) * (width/2) + * V - plane should be of size: (height/2) * (width/2) + * + * @param friend_number The friend number of the friend to which to send a video + * frame. + * @param width Width of the frame in pixels. + * @param height Height of the frame in pixels. + * @param y Y (Luminance) plane data. + * @param u U (Chroma) plane data. + * @param v V (Chroma) plane data. + * @param a A (Alpha) plane data. + */ + bool send_frame(uint32_t friend_number, uint16_t width, uint16_t height, + const uint8_t *y, const uint8_t *u, const uint8_t *v, const uint8_t *a) with error for send_frame; +} +/******************************************************************************* + * + * :: A/V receiving + * + ******************************************************************************/ +namespace audio { + event receive_frame { + /** + * The function type for the ${event receive_frame} callback. + * + * @param friend_number The friend number of the friend who sent an audio frame. + * @param pcm An array of audio samples (sample_count * channels elements). + * @param sample_count The number of audio samples per channel in the PCM array. + * @param channels Number of audio channels. + * @param sampling_rate Sampling rate used in this frame. + * + */ + typedef void(uint32_t friend_number, const int16_t *pcm, size_t sample_count, + uint8_t channels, uint32_t sampling_rate); + } +} +namespace video { + event receive_frame { + /** + * The function type for the ${event receive_frame} callback. + * + * @param friend_number The friend number of the friend who sent a video frame. + * @param width Width of the frame in pixels. + * @param height Height of the frame in pixels. + * @param y + * @param u + * @param v Plane data. + * The size of plane data is derived from width and height where + * Y = MAX(width, abs(ystride)) * height, + * U = MAX(width/2, abs(ustride)) * (height/2) and + * V = MAX(width/2, abs(vstride)) * (height/2). + * A = MAX(width, abs(astride)) * height. + * @param ystride + * @param ustride + * @param vstride + * @param astride Strides data. Strides represent padding for each plane + * that may or may not be present. You must handle strides in + * your image processing code. Strides are negative if the + * image is bottom-up hence why you MUST abs() it when + * calculating plane buffer size. + */ + typedef void(uint32_t friend_number, uint16_t width, uint16_t height, + const uint8_t *y, const uint8_t *u, const uint8_t *v, const uint8_t *a, + int32_t ystride, int32_t ustride, int32_t vstride, int32_t astride); + } +} + +} +%{ +#endif +%} \ No newline at end of file From 979d7730aa85502881788223a1fb4e30badaa159 Mon Sep 17 00:00:00 2001 From: mannol Date: Fri, 5 Jun 2015 22:18:21 +0200 Subject: [PATCH 44/97] Forgot about this --- other/apidsl/toxav.in.h | 3 +++ toxav/toxav.h | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/other/apidsl/toxav.in.h b/other/apidsl/toxav.in.h index 0631c827a..f3aa6f100 100644 --- a/other/apidsl/toxav.in.h +++ b/other/apidsl/toxav.in.h @@ -618,5 +618,8 @@ namespace video { } %{ +#ifdef __cplusplus +} #endif +#endif /* TOXAV_H */ %} diff --git a/toxav/toxav.h b/toxav/toxav.h index 0b9be2413..ffe9fbb97 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -630,4 +630,9 @@ typedef void toxav_video_receive_frame_cb(ToxAV *toxAV, uint32_t friend_number, * */ void toxav_callback_video_receive_frame(ToxAV *toxAV, toxav_video_receive_frame_cb *callback, void *user_data); + +#ifdef __cplusplus +} #endif + +#endif /* TOXAV_H */ From a3132feddb25656e33c7ce8c9bc6abc78657a01e Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 13 Jun 2015 15:00:34 +0200 Subject: [PATCH 45/97] Fixed sample size in callback and other stuff --- auto_tests/toxav_basic_test.c | 4 ++-- auto_tests/toxav_many_test.c | 10 ++++------ testing/av_test.c | 23 +++++++++++------------ toxav/audio.c | 4 ++-- 4 files changed, 19 insertions(+), 22 deletions(-) diff --git a/auto_tests/toxav_basic_test.c b/auto_tests/toxav_basic_test.c index a3847640e..abe5d0346 100644 --- a/auto_tests/toxav_basic_test.c +++ b/auto_tests/toxav_basic_test.c @@ -66,8 +66,8 @@ void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, vo } void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, - uint8_t const *y, uint8_t const *u, uint8_t const *v, uint8_t const *a, - int32_t ystride, int32_t ustride, int32_t vstride, int32_t astride, + uint8_t const *y, uint8_t const *u, uint8_t const *v, + int32_t ystride, int32_t ustride, int32_t vstride, void *user_data) { (void) av; diff --git a/auto_tests/toxav_many_test.c b/auto_tests/toxav_many_test.c index 761a4525b..438f27895 100644 --- a/auto_tests/toxav_many_test.c +++ b/auto_tests/toxav_many_test.c @@ -61,8 +61,8 @@ void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, vo } void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, - uint8_t const *y, uint8_t const *u, uint8_t const *v, uint8_t const *a, - int32_t ystride, int32_t ustride, int32_t vstride, int32_t stride, + uint8_t const *y, uint8_t const *u, uint8_t const *v, + int32_t ystride, int32_t ustride, int32_t vstride, void *user_data) { (void) av; @@ -159,13 +159,11 @@ void* call_thread(void* pd) uint8_t video_y[800*600]; uint8_t video_u[800*600 / 2]; uint8_t video_v[800*600 / 2]; - uint8_t video_a[800*600]; memset(PCM, 0, sizeof(PCM)); memset(video_y, 0, sizeof(video_y)); memset(video_u, 0, sizeof(video_u)); memset(video_v, 0, sizeof(video_v)); - memset(video_a, 0, sizeof(video_a)); time_t start_time = time(NULL); while(time(NULL) - start_time < 4) { @@ -175,8 +173,8 @@ void* call_thread(void* pd) toxav_audio_send_frame(AliceAV, friend_number, PCM, 960, 1, 48000, NULL); toxav_audio_send_frame(BobAV, 0, PCM, 960, 1, 48000, NULL); - toxav_video_send_frame(AliceAV, friend_number, 800, 600, video_y, video_u, video_v, video_a, NULL); - toxav_video_send_frame(BobAV, 0, 800, 600, video_y, video_u, video_v, video_a, NULL); + toxav_video_send_frame(AliceAV, friend_number, 800, 600, video_y, video_u, video_v, NULL); + toxav_video_send_frame(BobAV, 0, 800, 600, video_y, video_u, video_v, NULL); c_sleep(10); } diff --git a/testing/av_test.c b/testing/av_test.c index de973d916..cd9608b60 100644 --- a/testing/av_test.c +++ b/testing/av_test.c @@ -70,8 +70,8 @@ #define YUV2B(Y, U, V) CLIP(( 298 * C(Y) + 516 * D(U) + 128) >> 8) -#define TEST_TRANSFER_A 0 -#define TEST_TRANSFER_V 1 +#define TEST_TRANSFER_A 1 +#define TEST_TRANSFER_V 0 typedef struct { @@ -134,14 +134,13 @@ void t_toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, vo } void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, - uint8_t const *y, uint8_t const *u, uint8_t const *v, uint8_t const *a, - int32_t ystride, int32_t ustride, int32_t vstride, int32_t astride, + uint8_t const *y, uint8_t const *u, uint8_t const *v, + int32_t ystride, int32_t ustride, int32_t vstride, void *user_data) { ystride = abs(ystride); ustride = abs(ustride); vstride = abs(vstride); - astride = abs(astride); uint16_t *img_data = malloc(height * width * 6); @@ -177,9 +176,9 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, void *user_data) { CallControl* cc = user_data; - frame* f = malloc(sizeof(uint16_t) + sample_count * sizeof(int16_t)); - memcpy(f->data, pcm, sample_count * sizeof(int16_t)); - f->size = sample_count/channels; + frame* f = malloc(sizeof(uint16_t) + sample_count * sizeof(int16_t) * channels); + memcpy(f->data, pcm, sample_count * sizeof(int16_t) * channels); + f->size = sample_count; pthread_mutex_lock(cc->arb_mutex); free(rb_write(cc->arb, f)); @@ -225,15 +224,15 @@ void initialize_tox(Tox** bootstrap, ToxAV** AliceAV, CallControl* AliceCC, ToxA TOX_ERR_NEW error; opts.start_port = 33445; - *bootstrap = tox_new(&opts, NULL, 0, &error); + *bootstrap = tox_new(&opts, &error); assert(error == TOX_ERR_NEW_OK); opts.start_port = 33455; - Alice = tox_new(&opts, NULL, 0, &error); + Alice = tox_new(&opts, &error); assert(error == TOX_ERR_NEW_OK); opts.start_port = 33465; - Bob = tox_new(&opts, NULL, 0, &error); + Bob = tox_new(&opts, &error); assert(error == TOX_ERR_NEW_OK); } @@ -369,7 +368,7 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) } - int rc = toxav_video_send_frame(av, friend_number, img->width, img->height, planes[0], planes[1], planes[2], NULL, NULL); + int rc = toxav_video_send_frame(av, friend_number, img->width, img->height, planes[0], planes[1], planes[2], NULL); free(planes[0]); free(planes[1]); free(planes[2]); diff --git a/toxav/audio.c b/toxav/audio.c index f6993a1d0..0a6f04e39 100644 --- a/toxav/audio.c +++ b/toxav/audio.c @@ -181,8 +181,8 @@ void ac_do(ACSession* ac) } else if (ac->acb.first) { ac->last_packet_frame_duration = (rc * 1000) / ac->last_packet_sampling_rate; - ac->acb.first(ac->av, ac->friend_number, tmp, rc * ac->last_packet_channel_count, - ac->last_packet_channel_count, ac->last_packet_sampling_rate, ac->acb.second); + ac->acb.first(ac->av, ac->friend_number, tmp, rc, ac->last_packet_channel_count, + ac->last_packet_sampling_rate, ac->acb.second); } return; From 9aba4ec273b782fd34a869a902cc2a0b8275dbff Mon Sep 17 00:00:00 2001 From: mannol Date: Thu, 25 Jun 2015 01:04:31 +0200 Subject: [PATCH 46/97] Random fixes --- toxav/toxav.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/toxav/toxav.c b/toxav/toxav.c index 35523319d..c7012cb40 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -331,7 +331,7 @@ void toxav_iterate(ToxAV* av) av->dmsst += current_time_monotonic() - start; if (++av->dmssc == 3) { - av->dmssa = av->dmsst / 3 + 2 /* NOTE Magic Offset for precission */; + av->dmssa = av->dmsst / 3 + 5 /* NOTE Magic Offset for precission */; av->dmssc = 0; av->dmsst = 0; } @@ -1270,20 +1270,28 @@ bool call_prepare_transmission(ToxAVCall* call) { /* Prepare audio */ call->audio.second = ac_new(av, call->friend_number, av->acb.first, av->acb.second); - call->audio.first = rtp_new(rtp_TypeAudio, av->m, call->friend_number, call->audio.second, ac_queue_message); + if (!call->audio.second) { + LOGGER_ERROR("Failed to create audio codec session"); + goto FAILURE; + } - if ( !call->audio.first || !call->audio.second ) { - LOGGER_ERROR("Error while starting audio!\n"); + call->audio.first = rtp_new(rtp_TypeAudio, av->m, call->friend_number, call->audio.second, ac_queue_message); + if (!call->audio.first) { + LOGGER_ERROR("Failed to create audio rtp session");; goto FAILURE; } } { /* Prepare video */ call->video.second = vc_new(av, call->friend_number, av->vcb.first, av->vcb.second, call->msi_call->peer_vfpsz); - call->video.first = rtp_new(rtp_TypeVideo, av->m, call->friend_number, call->video.second, vc_queue_message); + if (!call->video.second) { + LOGGER_ERROR("Failed to create video codec session"); + goto FAILURE; + } - if ( !call->video.first || !call->video.second ) { - LOGGER_ERROR("Error while starting video!\n"); + call->video.first = rtp_new(rtp_TypeVideo, av->m, call->friend_number, call->video.second, vc_queue_message); + if (!call->video.first) { + LOGGER_ERROR("Failed to create video rtp session"); goto FAILURE; } } From 08bc4eb0e09cb4d4d9724f7bfeae5f4feb3aaf29 Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 27 Jun 2015 01:55:28 +0200 Subject: [PATCH 47/97] Added payload turning off by setting bit rate to 0 --- auto_tests/toxav_basic_test.c | 53 ++++++++++++++++++++++ other/apidsl/toxav.in.h | 4 ++ toxav/toxav.c | 83 ++++++++++++++++++++++++++++++----- toxav/toxav.h | 4 ++ 4 files changed, 132 insertions(+), 12 deletions(-) diff --git a/auto_tests/toxav_basic_test.c b/auto_tests/toxav_basic_test.c index abe5d0346..2735635f4 100644 --- a/auto_tests/toxav_basic_test.c +++ b/auto_tests/toxav_basic_test.c @@ -34,6 +34,7 @@ #define TEST_REJECT 1 #define TEST_CANCEL 1 #define TEST_MUTE_UNMUTE 1 +#define TEST_STOP_RESUME_PAYLOAD 1 typedef struct { @@ -424,6 +425,58 @@ START_TEST(test_AV_flows) printf("Success!\n"); } + if (TEST_STOP_RESUME_PAYLOAD) { /* Stop and resume audio/video payload */ + printf("\nTrying stop/resume functionality...\n"); + + memset(&AliceCC, 0, sizeof(CallControl)); + memset(&BobCC, 0, sizeof(CallControl)); + + /* Assume sending audio and video */ + { + TOXAV_ERR_CALL rc; + toxav_call(AliceAV, 0, 48, 0, &rc); + + if (rc != TOXAV_ERR_CALL_OK) { + printf("toxav_call failed: %d\n", rc); + ck_assert(0); + } + } + + while (!BobCC.incoming) + iterate_tox(bootstrap, Alice, Bob); + + { + TOXAV_ERR_ANSWER rc; + toxav_answer(BobAV, 0, 48, 0, &rc); + + if (rc != TOXAV_ERR_ANSWER_OK) { + printf("toxav_answer failed: %d\n", rc); + ck_assert(0); + } + } + + iterate_tox(bootstrap, Alice, Bob); + + printf("Call started as audio only\n"); + printf("Turning on video for Alice...\n"); + ck_assert(toxav_video_bit_rate_set(AliceAV, 0, 1000, false, NULL)); + + iterate_tox(bootstrap, Alice, Bob); + ck_assert(BobCC.state & TOXAV_CALL_STATE_SENDING_V); + + printf("Turning off video for Alice...\n"); + ck_assert(toxav_video_bit_rate_set(AliceAV, 0, 0, false, NULL)); + + iterate_tox(bootstrap, Alice, Bob); + ck_assert(!(BobCC.state & TOXAV_CALL_STATE_SENDING_V)); + + printf("Turning off audio for Alice...\n"); + ck_assert(toxav_audio_bit_rate_set(AliceAV, 0, 0, false, NULL)); + + iterate_tox(bootstrap, Alice, Bob); + ck_assert(!(BobCC.state & TOXAV_CALL_STATE_SENDING_A)); + } + toxav_kill(BobAV); toxav_kill(AliceAV); tox_kill(Bob); diff --git a/other/apidsl/toxav.in.h b/other/apidsl/toxav.in.h index f3aa6f100..59a46612b 100644 --- a/other/apidsl/toxav.in.h +++ b/other/apidsl/toxav.in.h @@ -518,6 +518,10 @@ error for send_frame { * small or too large, or the audio sampling rate may be unsupported. */ INVALID, + /** + * Bit rate for this payload type was not set up. + */ + BIT_RATE_NOT_SET, /** * Failed to push frame through rtp interface. */ diff --git a/toxav/toxav.c b/toxav/toxav.c index c7012cb40..59afe654d 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -596,6 +596,8 @@ void toxav_callback_audio_bit_rate_status(ToxAV* av, toxav_audio_bit_rate_status bool toxav_audio_bit_rate_set(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, bool force, TOXAV_ERR_SET_BIT_RATE* error) { + LOGGER_DEBUG("Setting new audio bitrate to: %d", audio_bit_rate); + TOXAV_ERR_SET_BIT_RATE rc = TOXAV_ERR_SET_BIT_RATE_OK; ToxAVCall* call; @@ -604,7 +606,7 @@ bool toxav_audio_bit_rate_set(ToxAV* av, uint32_t friend_number, uint32_t audio_ goto END; } - if (audio_bit_rate_invalid(audio_bit_rate)) { + if (audio_bit_rate && audio_bit_rate_invalid(audio_bit_rate)) { rc = TOXAV_ERR_SET_BIT_RATE_INVALID; goto END; } @@ -622,18 +624,39 @@ bool toxav_audio_bit_rate_set(ToxAV* av, uint32_t friend_number, uint32_t audio_ goto END; } + /* Video sending is turned off; notify peer */ + if (audio_bit_rate == 0) { + call->audio_bit_rate = 0; + + msi_change_capabilities(call->msi_call, call->msi_call-> + self_capabilities ^ msi_CapSAudio); + pthread_mutex_unlock(av->mutex); + goto END; + } pthread_mutex_lock(call->mutex); - if (audio_bit_rate > call->audio_bit_rate && !force) - ba_set(&call->aba, audio_bit_rate); - else { - /* Cancel any previous non forceful bitrate change request */ - memset(&call->aba, 0, sizeof(call->aba)); + if (call->audio_bit_rate == 0) { + /* The audio has been turned off before this */ call->audio_bit_rate = audio_bit_rate; + msi_change_capabilities(call->msi_call, call-> + msi_call->self_capabilities | msi_CapSAudio); + if (!force && av->abcb.first) av->abcb.first (av, call->friend_number, true, audio_bit_rate, av->abcb.second); + } else { + /* The audio was active before this */ + if (audio_bit_rate > call->audio_bit_rate && !force) + ba_set(&call->aba, audio_bit_rate); + else { + /* Cancel any previous non forceful bitrate change request */ + memset(&call->aba, 0, sizeof(call->aba)); + call->audio_bit_rate = audio_bit_rate; + + if (!force && av->abcb.first) + av->abcb.first (av, call->friend_number, true, audio_bit_rate, av->abcb.second); + } } pthread_mutex_unlock(call->mutex); @@ -656,6 +679,8 @@ void toxav_callback_video_bit_rate_status(ToxAV* av, toxav_video_bit_rate_status bool toxav_video_bit_rate_set(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, bool force, TOXAV_ERR_SET_BIT_RATE* error) { + LOGGER_DEBUG("Setting new video bitrate to: %d", video_bit_rate); + TOXAV_ERR_SET_BIT_RATE rc = TOXAV_ERR_SET_BIT_RATE_OK; ToxAVCall* call; @@ -664,7 +689,7 @@ bool toxav_video_bit_rate_set(ToxAV* av, uint32_t friend_number, uint32_t video_ goto END; } - if (video_bit_rate_invalid(video_bit_rate)) { + if (video_bit_rate && video_bit_rate_invalid(video_bit_rate)) { rc = TOXAV_ERR_SET_BIT_RATE_INVALID; goto END; } @@ -682,17 +707,39 @@ bool toxav_video_bit_rate_set(ToxAV* av, uint32_t friend_number, uint32_t video_ goto END; } + /* Video sending is turned off; notify peer */ + if (video_bit_rate == 0) { + call->video_bit_rate = 0; + + msi_change_capabilities(call->msi_call, call->msi_call-> + self_capabilities ^ msi_CapSVideo); + pthread_mutex_unlock(av->mutex); + goto END; + } + pthread_mutex_lock(call->mutex); - if (video_bit_rate > call->video_bit_rate && !force) - ba_set(&call->vba, video_bit_rate); - else { - /* Cancel any previous non forceful bitrate change request */ - memset(&call->vba, 0, sizeof(call->vba)); + if (call->video_bit_rate == 0) { + /* The video has been turned off before this */ call->video_bit_rate = video_bit_rate; + msi_change_capabilities(call->msi_call, call-> + msi_call->self_capabilities | msi_CapSVideo); + if (!force && av->vbcb.first) av->vbcb.first (av, call->friend_number, true, video_bit_rate, av->vbcb.second); + } else { + /* The video was active before this */ + if (video_bit_rate > call->video_bit_rate && !force) + ba_set(&call->vba, video_bit_rate); + else { + /* Cancel any previous non forceful bitrate change request */ + memset(&call->vba, 0, sizeof(call->vba)); + call->video_bit_rate = video_bit_rate; + + if (!force && av->vbcb.first) + av->vbcb.first (av, call->friend_number, true, video_bit_rate, av->vbcb.second); + } } pthread_mutex_unlock(call->mutex); @@ -723,6 +770,12 @@ bool toxav_audio_send_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } + if (call->audio_bit_rate == 0) { + pthread_mutex_unlock(av->mutex); + rc = TOXAV_ERR_SEND_FRAME_BIT_RATE_NOT_SET; + goto END; + } + pthread_mutex_lock(call->mutex_audio); pthread_mutex_unlock(av->mutex); @@ -825,6 +878,12 @@ bool toxav_video_send_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u goto END; } + if (call->video_bit_rate == 0) { + pthread_mutex_unlock(av->mutex); + rc = TOXAV_ERR_SEND_FRAME_BIT_RATE_NOT_SET; + goto END; + } + pthread_mutex_lock(call->mutex_video); pthread_mutex_unlock(av->mutex); diff --git a/toxav/toxav.h b/toxav/toxav.h index ffe9fbb97..2094f66cd 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -540,6 +540,10 @@ typedef enum TOXAV_ERR_SEND_FRAME { * small or too large, or the audio sampling rate may be unsupported. */ TOXAV_ERR_SEND_FRAME_INVALID, + /** + * Bit rate for this payload type was not set up. + */ + TOXAV_ERR_SEND_FRAME_BIT_RATE_NOT_SET, /** * Failed to push frame through rtp interface. */ From 1ba55d0ca737c21b78471180e90e6e81780aa52c Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 27 Jun 2015 16:12:33 +0200 Subject: [PATCH 48/97] Added explanation about how audio callback works --- other/apidsl/toxav.in.h | 6 ++++-- toxav/toxav.h | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/other/apidsl/toxav.in.h b/other/apidsl/toxav.in.h index 59a46612b..6e50d0d5f 100644 --- a/other/apidsl/toxav.in.h +++ b/other/apidsl/toxav.in.h @@ -578,8 +578,10 @@ namespace video { namespace audio { event receive_frame { /** - * The function type for the ${event receive_frame} callback. - * + * The function type for the ${event receive_frame} callback. The callback can be + * called multiple times per single iteration depending on the amount of queued + * frames in the buffer. + * * @param friend_number The friend number of the friend who sent an audio frame. * @param pcm An array of audio samples (sample_count * channels elements). * @param sample_count The number of audio samples per channel in the PCM array. diff --git a/toxav/toxav.h b/toxav/toxav.h index 2094f66cd..917a6bf57 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -592,7 +592,9 @@ bool toxav_video_send_frame(ToxAV *toxAV, uint32_t friend_number, uint16_t width * ******************************************************************************/ /** - * The function type for the audio_receive_frame callback. + * The function type for the audio_receive_frame callback. The callback can be + * called multiple times per single iteration depending on the amount of queued + * frames in the buffer. * * @param friend_number The friend number of the friend who sent an audio frame. * @param pcm An array of audio samples (sample_count * channels elements). From 2ecb71bb1e02a3542700ba9e8bcc372e203b8437 Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 27 Jun 2015 17:28:07 +0200 Subject: [PATCH 49/97] Renamed TOXAV_CALL_STATE to TOXAV_FRIEND_CALL_STATE --- auto_tests/toxav_basic_test.c | 28 ++++++++++++++-------------- other/apidsl/toxav.in.h | 2 +- testing/av_test.c | 4 ++-- toxav/toxav.c | 4 ++-- toxav/toxav.h | 14 +++++++------- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/auto_tests/toxav_basic_test.c b/auto_tests/toxav_basic_test.c index 2735635f4..305257fb6 100644 --- a/auto_tests/toxav_basic_test.c +++ b/auto_tests/toxav_basic_test.c @@ -215,7 +215,7 @@ START_TEST(test_AV_flows) long long unsigned int start_time = time(NULL); \ \ \ - while (BobCC.state != TOXAV_CALL_STATE_FINISHED) { \ + while (BobCC.state != TOXAV_FRIEND_CALL_STATE_FINISHED) { \ \ if (BobCC.incoming) { \ TOXAV_ERR_ANSWER rc; \ @@ -293,7 +293,7 @@ START_TEST(test_AV_flows) } } - while (AliceCC.state != TOXAV_CALL_STATE_FINISHED) + while (AliceCC.state != TOXAV_FRIEND_CALL_STATE_FINISHED) iterate_tox(bootstrap, Alice, Bob); printf("Success!\n"); @@ -330,7 +330,7 @@ START_TEST(test_AV_flows) } /* Alice will not receive end state */ - while (BobCC.state != TOXAV_CALL_STATE_FINISHED) + while (BobCC.state != TOXAV_FRIEND_CALL_STATE_FINISHED) iterate_tox(bootstrap, Alice, Bob); printf("Success!\n"); @@ -383,31 +383,31 @@ START_TEST(test_AV_flows) ck_assert(BobCC.state == 0); ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state & (TOXAV_CALL_STATE_SENDING_A | TOXAV_CALL_STATE_SENDING_V)); + ck_assert(BobCC.state & (TOXAV_FRIEND_CALL_STATE_SENDING_A | TOXAV_FRIEND_CALL_STATE_SENDING_V)); /* Mute/Unmute single */ printf("Mute/Unmute single\n"); ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_AUDIO, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); + ck_assert(BobCC.state ^ TOXAV_FRIEND_CALL_STATE_RECEIVING_A); ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_AUDIO, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); + ck_assert(BobCC.state & TOXAV_FRIEND_CALL_STATE_RECEIVING_A); /* Mute/Unmute both */ printf("Mute/Unmute both\n"); ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_AUDIO, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); + ck_assert(BobCC.state ^ TOXAV_FRIEND_CALL_STATE_RECEIVING_A); ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_HIDE_VIDEO, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_V); + ck_assert(BobCC.state ^ TOXAV_FRIEND_CALL_STATE_RECEIVING_V); ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_AUDIO, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); + ck_assert(BobCC.state & TOXAV_FRIEND_CALL_STATE_RECEIVING_A); ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_SHOW_VIDEO, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_V); + ck_assert(BobCC.state & TOXAV_FRIEND_CALL_STATE_RECEIVING_V); { TOXAV_ERR_CALL_CONTROL rc; @@ -420,7 +420,7 @@ START_TEST(test_AV_flows) } iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state == TOXAV_CALL_STATE_FINISHED); + ck_assert(BobCC.state == TOXAV_FRIEND_CALL_STATE_FINISHED); printf("Success!\n"); } @@ -462,19 +462,19 @@ START_TEST(test_AV_flows) ck_assert(toxav_video_bit_rate_set(AliceAV, 0, 1000, false, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state & TOXAV_CALL_STATE_SENDING_V); + ck_assert(BobCC.state & TOXAV_FRIEND_CALL_STATE_SENDING_V); printf("Turning off video for Alice...\n"); ck_assert(toxav_video_bit_rate_set(AliceAV, 0, 0, false, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(!(BobCC.state & TOXAV_CALL_STATE_SENDING_V)); + ck_assert(!(BobCC.state & TOXAV_FRIEND_CALL_STATE_SENDING_V)); printf("Turning off audio for Alice...\n"); ck_assert(toxav_audio_bit_rate_set(AliceAV, 0, 0, false, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(!(BobCC.state & TOXAV_CALL_STATE_SENDING_A)); + ck_assert(!(BobCC.state & TOXAV_FRIEND_CALL_STATE_SENDING_A)); } toxav_kill(BobAV); diff --git a/other/apidsl/toxav.in.h b/other/apidsl/toxav.in.h index 6e50d0d5f..4a96defc5 100644 --- a/other/apidsl/toxav.in.h +++ b/other/apidsl/toxav.in.h @@ -296,7 +296,7 @@ bool answer(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_ * :: Call state graph * ******************************************************************************/ -bitmask CALL_STATE { +bitmask FRIEND_CALL_STATE { /** * Set by the AV core if an error occurred on the remote end or if friend * timed out. This is the final state after which no more state diff --git a/testing/av_test.c b/testing/av_test.c index cd9608b60..20e00c12e 100644 --- a/testing/av_test.c +++ b/testing/av_test.c @@ -627,7 +627,7 @@ int main (int argc, char** argv) } iterate_tox(bootstrap, AliceAV, BobAV); - assert(BobCC.state == TOXAV_CALL_STATE_FINISHED); + assert(BobCC.state == TOXAV_FRIEND_CALL_STATE_FINISHED); /* Stop decode thread */ data.sig = -1; @@ -721,7 +721,7 @@ int main (int argc, char** argv) } iterate_tox(bootstrap, AliceAV, BobAV); - assert(BobCC.state == TOXAV_CALL_STATE_FINISHED); + assert(BobCC.state == TOXAV_FRIEND_CALL_STATE_FINISHED); /* Stop decode thread */ printf("Stopping decode thread\n"); diff --git a/toxav/toxav.c b/toxav/toxav.c index 59afe654d..8a39259a3 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -1115,7 +1115,7 @@ int callback_end(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - invoke_call_state(toxav, call->friend_number, TOXAV_CALL_STATE_FINISHED); + invoke_call_state(toxav, call->friend_number, TOXAV_FRIEND_CALL_STATE_FINISHED); call_kill_transmission(call->av_call); call_remove(call->av_call); @@ -1129,7 +1129,7 @@ int callback_error(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - invoke_call_state(toxav, call->friend_number, TOXAV_CALL_STATE_ERROR); + invoke_call_state(toxav, call->friend_number, TOXAV_FRIEND_CALL_STATE_ERROR); call_kill_transmission(call->av_call); call_remove(call->av_call); diff --git a/toxav/toxav.h b/toxav/toxav.h index 917a6bf57..8362edb37 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -302,36 +302,36 @@ bool toxav_answer(ToxAV *toxAV, uint32_t friend_number, uint32_t audio_bit_rate, * :: Call state graph * ******************************************************************************/ -enum TOXAV_CALL_STATE { +enum TOXAV_FRIEND_CALL_STATE { /** * Set by the AV core if an error occurred on the remote end or if friend * timed out. This is the final state after which no more state * transitions can occur for the call. This call state will never be triggered * in combination with other call states. */ - TOXAV_CALL_STATE_ERROR = 1, + TOXAV_FRIEND_CALL_STATE_ERROR = 1, /** * The call has finished. This is the final state after which no more state * transitions can occur for the call. This call state will never be * triggered in combination with other call states. */ - TOXAV_CALL_STATE_FINISHED = 2, + TOXAV_FRIEND_CALL_STATE_FINISHED = 2, /** * The flag that marks that friend is sending audio. */ - TOXAV_CALL_STATE_SENDING_A = 4, + TOXAV_FRIEND_CALL_STATE_SENDING_A = 4, /** * The flag that marks that friend is sending video. */ - TOXAV_CALL_STATE_SENDING_V = 8, + TOXAV_FRIEND_CALL_STATE_SENDING_V = 8, /** * The flag that marks that friend is receiving audio. */ - TOXAV_CALL_STATE_RECEIVING_A = 16, + TOXAV_FRIEND_CALL_STATE_RECEIVING_A = 16, /** * The flag that marks that friend is receiving video. */ - TOXAV_CALL_STATE_RECEIVING_V = 32, + TOXAV_FRIEND_CALL_STATE_RECEIVING_V = 32, }; From 13148d7d7c7fc6ea6ef7e592dfd8b0bf3cfd1244 Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 27 Jun 2015 19:20:52 +0200 Subject: [PATCH 50/97] Added support for old groupchats --- toxav/Makefile.inc | 4 ++- toxav/toxav.h | 8 +++-- toxav/toxav_old.c | 81 ++++++++++++++++++++++++++++++++++++++++++++++ toxav/toxav_old.h | 66 +++++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 toxav/toxav_old.c create mode 100644 toxav/toxav_old.h diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc index 3907951e3..79275c9bb 100644 --- a/toxav/Makefile.inc +++ b/toxav/Makefile.inc @@ -15,7 +15,9 @@ libtoxav_la_SOURCES = ../toxav/rtp.h \ ../toxav/video.h \ ../toxav/video.c \ ../toxav/toxav.h \ - ../toxav/toxav.c + ../toxav/toxav.c \ + ../toxav/toxav_old.h \ + ../toxav/toxav_old.c libtoxav_la_CFLAGS = -I../toxcore \ diff --git a/toxav/toxav.h b/toxav/toxav.h index 8362edb37..0a25fc96f 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -237,7 +237,8 @@ typedef enum TOXAV_ERR_CALL { * * It is the client's responsibility to stop ringing after a certain timeout, * if such behaviour is desired. If the client does not stop ringing, the - * library will not stop until the friend is disconnected. + * library will not stop until the friend is disconnected. Audio and video + * receiving are both enabled by default. * * @param friend_number The friend number of the friend that should be called. * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable @@ -288,7 +289,8 @@ typedef enum TOXAV_ERR_ANSWER { * Accept an incoming call. * * If answering fails for any reason, the call will still be pending and it is - * possible to try and answer it later. + * possible to try and answer it later. Audio and video receiving are both + * enabled by default. * * @param friend_number The friend number of the friend that is calling. * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable @@ -637,6 +639,8 @@ typedef void toxav_video_receive_frame_cb(ToxAV *toxAV, uint32_t friend_number, */ void toxav_callback_video_receive_frame(ToxAV *toxAV, toxav_video_receive_frame_cb *callback, void *user_data); +#include "toxav_old.h" + #ifdef __cplusplus } #endif diff --git a/toxav/toxav_old.c b/toxav/toxav_old.c new file mode 100644 index 000000000..1f553fc39 --- /dev/null +++ b/toxav/toxav_old.c @@ -0,0 +1,81 @@ +/* toxav_old.h + * + * Copyright (C) 2013-2015 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + */ +/** + * This file contains the group chats code for the backwards compatibility. + */ + +#include "toxav_old.h" +#include "group.h" + +/* Create a new toxav group. + * + * return group number on success. + * return -1 on failure. + * + * Audio data callback format: + * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + */ +int toxav_add_av_groupchat(struct Tox *tox, void (*audio_callback)(void *, int, int, const int16_t *, unsigned int, + uint8_t, unsigned int, void *), void *userdata) +{ + Messenger *m = (Messenger *)tox; + return add_av_groupchat(m->group_chat_object, audio_callback, userdata); +} + +/* Join a AV group (you need to have been invited first.) + * + * returns group number on success + * returns -1 on failure. + * + * Audio data callback format (same as the one for toxav_add_av_groupchat()): + * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + */ +int toxav_join_av_groupchat(struct Tox *tox, int32_t friendnumber, const uint8_t *data, uint16_t length, + void (*audio_callback)(void *, int, int, const int16_t *, unsigned int, uint8_t, unsigned int, void *), + void *userdata) +{ + Messenger *m = (Messenger *)tox; + return join_av_groupchat(m->group_chat_object, friendnumber, data, length, audio_callback, userdata); +} + +/* Send audio to the group chat. + * + * return 0 on success. + * return -1 on failure. + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + * + * Valid number of samples are ((sample rate) * (audio length (Valid ones are: 2.5, 5, 10, 20, 40 or 60 ms)) / 1000) + * Valid number of channels are 1 or 2. + * Valid sample rates are 8000, 12000, 16000, 24000, or 48000. + * + * Recommended values are: samples = 960, channels = 1, sample_rate = 48000 + */ +int toxav_group_send_audio(struct Tox *tox, int groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels, + unsigned int sample_rate) +{ + Messenger *m = (Messenger *)tox; + return group_send_audio(m->group_chat_object, groupnumber, pcm, samples, channels, sample_rate); +} \ No newline at end of file diff --git a/toxav/toxav_old.h b/toxav/toxav_old.h new file mode 100644 index 000000000..89fe3014f --- /dev/null +++ b/toxav/toxav_old.h @@ -0,0 +1,66 @@ +/* toxav_old.h + * + * Copyright (C) 2013-2015 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + */ +/** + * This file contains the group chats code for the backwards compatibility. + */ +#include "../toxcore/tox.h" + +/* Create a new toxav group. + * + * return group number on success. + * return -1 on failure. + * + * Audio data callback format: + * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + */ +int toxav_add_av_groupchat(struct Tox *tox, void (*audio_callback)(void*, int, int, const int16_t *, unsigned int, uint8_t, + unsigned int, void *), void *userdata); + +/* Join a AV group (you need to have been invited first.) + * + * returns group number on success + * returns -1 on failure. + * + * Audio data callback format (same as the one for toxav_add_av_groupchat()): + * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + */ +int toxav_join_av_groupchat(struct Tox *tox, int32_t friendnumber, const uint8_t *data, uint16_t length, + void (*audio_callback)(void*, int, int, const int16_t *, unsigned int, uint8_t, unsigned int, void *), void *userdata); + +/* Send audio to the group chat. + * + * return 0 on success. + * return -1 on failure. + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + * + * Valid number of samples are ((sample rate) * (audio length (Valid ones are: 2.5, 5, 10, 20, 40 or 60 ms)) / 1000) + * Valid number of channels are 1 or 2. + * Valid sample rates are 8000, 12000, 16000, 24000, or 48000. + * + * Recommended values are: samples = 960, channels = 1, sample_rate = 48000 + */ +int toxav_group_send_audio(struct Tox *tox, int groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels, + unsigned int sample_rate); \ No newline at end of file From 6c126e34e677da42849b9002ad376117bff5852e Mon Sep 17 00:00:00 2001 From: mannol Date: Tue, 30 Jun 2015 01:41:38 +0200 Subject: [PATCH 51/97] Implement handling capability change on rtp level --- auto_tests/toxav_basic_test.c | 77 +++++++++++++++++++++++++++++++++++ other/apidsl/toxav.in.h | 11 +++-- toxav/rtp.c | 2 + toxav/toxav.c | 22 ++++++++-- toxav/toxav.h | 5 ++- 5 files changed, 107 insertions(+), 10 deletions(-) diff --git a/auto_tests/toxav_basic_test.c b/auto_tests/toxav_basic_test.c index 305257fb6..f859856d6 100644 --- a/auto_tests/toxav_basic_test.c +++ b/auto_tests/toxav_basic_test.c @@ -35,6 +35,7 @@ #define TEST_CANCEL 1 #define TEST_MUTE_UNMUTE 1 #define TEST_STOP_RESUME_PAYLOAD 1 +#define TEST_PAUSE_RESUME_SEND 1 typedef struct { @@ -82,6 +83,7 @@ void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, (void) ustride; (void) vstride; (void) user_data; + printf("Received video payload\n"); } void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, int16_t const *pcm, @@ -97,6 +99,7 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, (void) channels; (void) sampling_rate; (void) user_data; + printf("Received audio payload\n"); } void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, size_t length, void *userdata) { @@ -475,6 +478,80 @@ START_TEST(test_AV_flows) iterate_tox(bootstrap, Alice, Bob); ck_assert(!(BobCC.state & TOXAV_FRIEND_CALL_STATE_SENDING_A)); + + { + TOXAV_ERR_CALL_CONTROL rc; + toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); + + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { + printf("toxav_call_control failed: %d\n", rc); + ck_assert(0); + } + } + + iterate_tox(bootstrap, Alice, Bob); + ck_assert(BobCC.state == TOXAV_FRIEND_CALL_STATE_FINISHED); + + printf("Success!\n"); + } + + if (TEST_PAUSE_RESUME_SEND) { /* Stop and resume audio/video payload and test send options */ + printf("\nTrying stop/resume functionality...\n"); + + memset(&AliceCC, 0, sizeof(CallControl)); + memset(&BobCC, 0, sizeof(CallControl)); + + /* Assume sending audio and video */ + { + TOXAV_ERR_CALL rc; + toxav_call(AliceAV, 0, 48, 0, &rc); + + if (rc != TOXAV_ERR_CALL_OK) { + printf("toxav_call failed: %d\n", rc); + ck_assert(0); + } + } + + while (!BobCC.incoming) + iterate_tox(bootstrap, Alice, Bob); + + { + TOXAV_ERR_ANSWER rc; + toxav_answer(BobAV, 0, 48, 0, &rc); + + if (rc != TOXAV_ERR_ANSWER_OK) { + printf("toxav_answer failed: %d\n", rc); + ck_assert(0); + } + } + + int16_t PCM[5670]; + + iterate_tox(bootstrap, Alice, Bob); + ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); + iterate_tox(bootstrap, Alice, Bob); + ck_assert(!toxav_audio_send_frame(AliceAV, 0, PCM, 960, 1, 48000, NULL)); + ck_assert(!toxav_audio_send_frame(BobAV, 0, PCM, 960, 1, 48000, NULL)); + ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); + iterate_tox(bootstrap, Alice, Bob); + ck_assert(toxav_audio_send_frame(AliceAV, 0, PCM, 960, 1, 48000, NULL)); + ck_assert(toxav_audio_send_frame(BobAV, 0, PCM, 960, 1, 48000, NULL)); + iterate_tox(bootstrap, Alice, Bob); + + { + TOXAV_ERR_CALL_CONTROL rc; + toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); + + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { + printf("toxav_call_control failed: %d\n", rc); + ck_assert(0); + } + } + + iterate_tox(bootstrap, Alice, Bob); + ck_assert(BobCC.state == TOXAV_FRIEND_CALL_STATE_FINISHED); + + printf("Success!\n"); } toxav_kill(BobAV); diff --git a/other/apidsl/toxav.in.h b/other/apidsl/toxav.in.h index 4a96defc5..8d1d8f6e6 100644 --- a/other/apidsl/toxav.in.h +++ b/other/apidsl/toxav.in.h @@ -216,7 +216,8 @@ void iterate(); * * It is the client's responsibility to stop ringing after a certain timeout, * if such behaviour is desired. If the client does not stop ringing, the - * library will not stop until the friend is disconnected. + * library will not stop until the friend is disconnected. Audio and video + * receiving are both enabled by default. * * @param friend_number The friend number of the friend that should be called. * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable @@ -262,7 +263,8 @@ event call { * Accept an incoming call. * * If answering fails for any reason, the call will still be pending and it is - * possible to try and answer it later. + * possible to try and answer it later. Audio and video receiving are both + * enabled by default. * * @param friend_number The friend number of the friend that is calling. * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable @@ -519,9 +521,10 @@ error for send_frame { */ INVALID, /** - * Bit rate for this payload type was not set up. + * Either friend turned off audio or video receiving or we turned off sending + * for the said payload. */ - BIT_RATE_NOT_SET, + PAYLOAD_TYPE_DISABLED, /** * Failed to push frame through rtp interface. */ diff --git a/toxav/rtp.c b/toxav/rtp.c index 7b3a5ed0f..148c4238d 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -221,6 +221,7 @@ int rtp_start_receiving(RTPSession* session) return -1; } + LOGGER_DEBUG("Started receiving on session: %p", session); return 0; } int rtp_stop_receiving(RTPSession* session) @@ -231,6 +232,7 @@ int rtp_stop_receiving(RTPSession* session) m_callback_rtp_packet(session->m, session->friend_number, session->prefix, NULL, NULL); m_callback_rtp_packet(session->m, session->friend_number, session->rtcp_session->prefix, NULL, NULL); /* RTCP */ + LOGGER_DEBUG("Stopped receiving on session: %p", session); return 0; } int rtp_send_data ( RTPSession *session, const uint8_t *data, uint16_t length, bool dummy ) diff --git a/toxav/toxav.c b/toxav/toxav.c index 8a39259a3..58e083765 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -770,9 +770,11 @@ bool toxav_audio_send_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } - if (call->audio_bit_rate == 0) { + if (call->audio_bit_rate == 0 || + !(call->msi_call->self_capabilities & msi_CapSAudio) || + !(call->msi_call->peer_capabilities & msi_CapRAudio)) { pthread_mutex_unlock(av->mutex); - rc = TOXAV_ERR_SEND_FRAME_BIT_RATE_NOT_SET; + rc = TOXAV_ERR_SEND_FRAME_PAYLOAD_TYPE_DISABLED; goto END; } @@ -878,9 +880,11 @@ bool toxav_video_send_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u goto END; } - if (call->video_bit_rate == 0) { + if (call->video_bit_rate == 0 || + !(call->msi_call->self_capabilities & msi_CapSVideo) || + !(call->msi_call->peer_capabilities & msi_CapRVideo)) { pthread_mutex_unlock(av->mutex); - rc = TOXAV_ERR_SEND_FRAME_BIT_RATE_NOT_SET; + rc = TOXAV_ERR_SEND_FRAME_PAYLOAD_TYPE_DISABLED; goto END; } @@ -1143,6 +1147,16 @@ int callback_capabilites(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); + if (call->peer_capabilities & msi_CapSAudio) + rtp_start_receiving(((ToxAVCall*)call->av_call)->audio.first); + else + rtp_stop_receiving(((ToxAVCall*)call->av_call)->audio.first); + + if (call->peer_capabilities & msi_CapSVideo) + rtp_start_receiving(((ToxAVCall*)call->av_call)->video.first); + else + rtp_stop_receiving(((ToxAVCall*)call->av_call)->video.first); + invoke_call_state(toxav, call->friend_number, call->peer_capabilities); pthread_mutex_unlock(toxav->mutex); diff --git a/toxav/toxav.h b/toxav/toxav.h index 0a25fc96f..d5641d74b 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -543,9 +543,10 @@ typedef enum TOXAV_ERR_SEND_FRAME { */ TOXAV_ERR_SEND_FRAME_INVALID, /** - * Bit rate for this payload type was not set up. + * Either friend turned off audio or video receiving or we turned off sending + * for the said payload. */ - TOXAV_ERR_SEND_FRAME_BIT_RATE_NOT_SET, + TOXAV_ERR_SEND_FRAME_PAYLOAD_TYPE_DISABLED, /** * Failed to push frame through rtp interface. */ From 4bc2413c4451560e912617fd7432c4b6f141ccf7 Mon Sep 17 00:00:00 2001 From: mannol Date: Wed, 1 Jul 2015 22:50:06 +0200 Subject: [PATCH 52/97] Fixed missing include toxav_old.h --- toxav/toxav.h | 45 +++++++++++++++++++++++++++++++- toxav/toxav_old.c | 2 +- toxav/toxav_old.h | 66 ----------------------------------------------- 3 files changed, 45 insertions(+), 68 deletions(-) delete mode 100644 toxav/toxav_old.h diff --git a/toxav/toxav.h b/toxav/toxav.h index d5641d74b..f9d29de4e 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -640,7 +640,50 @@ typedef void toxav_video_receive_frame_cb(ToxAV *toxAV, uint32_t friend_number, */ void toxav_callback_video_receive_frame(ToxAV *toxAV, toxav_video_receive_frame_cb *callback, void *user_data); -#include "toxav_old.h" +/** + * NOTE Compatibility with old toxav group calls TODO remove + */ +/* Create a new toxav group. + * + * return group number on success. + * return -1 on failure. + * + * Audio data callback format: + * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + */ +int toxav_add_av_groupchat(Tox *tox, void (*audio_callback)(void*, int, int, const int16_t *, unsigned int, uint8_t, + unsigned int, void *), void *userdata); + +/* Join a AV group (you need to have been invited first.) + * + * returns group number on success + * returns -1 on failure. + * + * Audio data callback format (same as the one for toxav_add_av_groupchat()): + * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + */ +int toxav_join_av_groupchat(Tox *tox, int32_t friendnumber, const uint8_t *data, uint16_t length, + void (*audio_callback)(void*, int, int, const int16_t *, unsigned int, uint8_t, unsigned int, void *), void *userdata); + +/* Send audio to the group chat. + * + * return 0 on success. + * return -1 on failure. + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + * + * Valid number of samples are ((sample rate) * (audio length (Valid ones are: 2.5, 5, 10, 20, 40 or 60 ms)) / 1000) + * Valid number of channels are 1 or 2. + * Valid sample rates are 8000, 12000, 16000, 24000, or 48000. + * + * Recommended values are: samples = 960, channels = 1, sample_rate = 48000 + */ +int toxav_group_send_audio(Tox *tox, int groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels, + unsigned int sample_rate); #ifdef __cplusplus } diff --git a/toxav/toxav_old.c b/toxav/toxav_old.c index 1f553fc39..61c2f0207 100644 --- a/toxav/toxav_old.c +++ b/toxav/toxav_old.c @@ -22,7 +22,7 @@ * This file contains the group chats code for the backwards compatibility. */ -#include "toxav_old.h" +#include "toxav.h" #include "group.h" /* Create a new toxav group. diff --git a/toxav/toxav_old.h b/toxav/toxav_old.h deleted file mode 100644 index 89fe3014f..000000000 --- a/toxav/toxav_old.h +++ /dev/null @@ -1,66 +0,0 @@ -/* toxav_old.h - * - * Copyright (C) 2013-2015 Tox project All Rights Reserved. - * - * This file is part of Tox. - * - * Tox is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Tox is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Tox. If not, see . - * - */ -/** - * This file contains the group chats code for the backwards compatibility. - */ -#include "../toxcore/tox.h" - -/* Create a new toxav group. - * - * return group number on success. - * return -1 on failure. - * - * Audio data callback format: - * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) - * - * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). - */ -int toxav_add_av_groupchat(struct Tox *tox, void (*audio_callback)(void*, int, int, const int16_t *, unsigned int, uint8_t, - unsigned int, void *), void *userdata); - -/* Join a AV group (you need to have been invited first.) - * - * returns group number on success - * returns -1 on failure. - * - * Audio data callback format (same as the one for toxav_add_av_groupchat()): - * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) - * - * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). - */ -int toxav_join_av_groupchat(struct Tox *tox, int32_t friendnumber, const uint8_t *data, uint16_t length, - void (*audio_callback)(void*, int, int, const int16_t *, unsigned int, uint8_t, unsigned int, void *), void *userdata); - -/* Send audio to the group chat. - * - * return 0 on success. - * return -1 on failure. - * - * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). - * - * Valid number of samples are ((sample rate) * (audio length (Valid ones are: 2.5, 5, 10, 20, 40 or 60 ms)) / 1000) - * Valid number of channels are 1 or 2. - * Valid sample rates are 8000, 12000, 16000, 24000, or 48000. - * - * Recommended values are: samples = 960, channels = 1, sample_rate = 48000 - */ -int toxav_group_send_audio(struct Tox *tox, int groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels, - unsigned int sample_rate); \ No newline at end of file From ce9003d432e4ed70f227f8c6b6f4bc2841d516a9 Mon Sep 17 00:00:00 2001 From: GrayHatter Date: Wed, 8 Jul 2015 15:09:19 -0700 Subject: [PATCH 53/97] send_message() is too global to be non-static --- toxav/msi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toxav/msi.c b/toxav/msi.c index d35591603..ec2f4bf20 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -82,7 +82,7 @@ typedef struct { void msg_init (MSIMessage *dest, MSIRequest request); int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ); uint8_t *msg_parse_header_out ( MSIHeaderID id, uint8_t *dest, const void *value, uint8_t value_len, uint16_t *length ); -int send_message ( Messenger* m, uint32_t friend_number, const MSIMessage *msg ); +static int send_message ( Messenger* m, uint32_t friend_number, const MSIMessage *msg ); int send_error ( Messenger* m, uint32_t friend_number, MSIError error ); static int invoke_callback(MSICall* call, MSICallbackID cb); static MSICall *get_call ( MSISession *session, uint32_t friend_number ); From dd0880af3850c44ad143330827adf5a99cbe07be Mon Sep 17 00:00:00 2001 From: mannol Date: Thu, 9 Jul 2015 03:40:47 +0200 Subject: [PATCH 54/97] Fixed documentation. --- other/apidsl/toxav.in.h | 8 +++++--- toxav/toxav.h | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/other/apidsl/toxav.in.h b/other/apidsl/toxav.in.h index 8d1d8f6e6..4b328a30d 100644 --- a/other/apidsl/toxav.in.h +++ b/other/apidsl/toxav.in.h @@ -334,8 +334,10 @@ event call_state { * The function type for the ${event call_state} callback. * * @param friend_number The friend number for which the call state changed. - * @param state The new call state which is guaranteed to be different than - * the previous state. The state is set to 0 when the call is paused. + * @param state The bitmask of the new call state which is guaranteed to be + * different than the previous state. The state is set to 0 when the call is + * paused. The bitmask represents all the activities currently performed by the + * friend. */ typedef void(uint32_t friend_number, uint32_t state); } @@ -583,7 +585,7 @@ namespace audio { /** * The function type for the ${event receive_frame} callback. The callback can be * called multiple times per single iteration depending on the amount of queued - * frames in the buffer. + * frames in the buffer. The received format is the same as in send function. * * @param friend_number The friend number of the friend who sent an audio frame. * @param pcm An array of audio samples (sample_count * channels elements). diff --git a/toxav/toxav.h b/toxav/toxav.h index f9d29de4e..30ac66d13 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -341,8 +341,10 @@ enum TOXAV_FRIEND_CALL_STATE { * The function type for the call_state callback. * * @param friend_number The friend number for which the call state changed. - * @param state The new call state which is guaranteed to be different than - * the previous state. The state is set to 0 when the call is paused. + * @param state The bitmask of the new call state which is guaranteed to be + * different than the previous state. The state is set to 0 when the call is + * paused. The bitmask represents all the activities currently performed by the + * friend. */ typedef void toxav_call_state_cb(ToxAV *toxAV, uint32_t friend_number, uint32_t state, void *user_data); /** @@ -597,7 +599,7 @@ bool toxav_video_send_frame(ToxAV *toxAV, uint32_t friend_number, uint16_t width /** * The function type for the audio_receive_frame callback. The callback can be * called multiple times per single iteration depending on the amount of queued - * frames in the buffer. + * frames in the buffer. The received format is the same as in send function. * * @param friend_number The friend number of the friend who sent an audio frame. * @param pcm An array of audio samples (sample_count * channels elements). From 7d5de68364f83662a19e45e6e3ab021414178df5 Mon Sep 17 00:00:00 2001 From: mannol Date: Fri, 10 Jul 2015 14:28:43 +0200 Subject: [PATCH 55/97] Fixed CALL_STATE naming --- auto_tests/toxav_basic_test.c | 12 ++++++------ other/apidsl/toxav.in.h | 4 ++-- toxav/toxav.h | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/auto_tests/toxav_basic_test.c b/auto_tests/toxav_basic_test.c index f859856d6..41fb6787a 100644 --- a/auto_tests/toxav_basic_test.c +++ b/auto_tests/toxav_basic_test.c @@ -392,25 +392,25 @@ START_TEST(test_AV_flows) printf("Mute/Unmute single\n"); ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_AUDIO, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state ^ TOXAV_FRIEND_CALL_STATE_RECEIVING_A); + ck_assert(BobCC.state ^ TOXAV_FRIEND_CALL_STATE_ACCEPTING_A); ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_AUDIO, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state & TOXAV_FRIEND_CALL_STATE_RECEIVING_A); + ck_assert(BobCC.state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_A); /* Mute/Unmute both */ printf("Mute/Unmute both\n"); ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_MUTE_AUDIO, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state ^ TOXAV_FRIEND_CALL_STATE_RECEIVING_A); + ck_assert(BobCC.state ^ TOXAV_FRIEND_CALL_STATE_ACCEPTING_A); ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_HIDE_VIDEO, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state ^ TOXAV_FRIEND_CALL_STATE_RECEIVING_V); + ck_assert(BobCC.state ^ TOXAV_FRIEND_CALL_STATE_ACCEPTING_V); ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_UNMUTE_AUDIO, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state & TOXAV_FRIEND_CALL_STATE_RECEIVING_A); + ck_assert(BobCC.state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_A); ck_assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_SHOW_VIDEO, NULL)); iterate_tox(bootstrap, Alice, Bob); - ck_assert(BobCC.state & TOXAV_FRIEND_CALL_STATE_RECEIVING_V); + ck_assert(BobCC.state & TOXAV_FRIEND_CALL_STATE_ACCEPTING_V); { TOXAV_ERR_CALL_CONTROL rc; diff --git a/other/apidsl/toxav.in.h b/other/apidsl/toxav.in.h index 4b328a30d..22a01602c 100644 --- a/other/apidsl/toxav.in.h +++ b/other/apidsl/toxav.in.h @@ -323,11 +323,11 @@ bitmask FRIEND_CALL_STATE { /** * The flag that marks that friend is receiving audio. */ - RECEIVING_A, + ACCEPTING_A, /** * The flag that marks that friend is receiving video. */ - RECEIVING_V, + ACCEPTING_V, } event call_state { /** diff --git a/toxav/toxav.h b/toxav/toxav.h index 30ac66d13..3c745b364 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -329,11 +329,11 @@ enum TOXAV_FRIEND_CALL_STATE { /** * The flag that marks that friend is receiving audio. */ - TOXAV_FRIEND_CALL_STATE_RECEIVING_A = 16, + TOXAV_FRIEND_CALL_STATE_ACCEPTING_A = 16, /** * The flag that marks that friend is receiving video. */ - TOXAV_FRIEND_CALL_STATE_RECEIVING_V = 32, + TOXAV_FRIEND_CALL_STATE_ACCEPTING_V = 32, }; From 657a57b406717a3ff08233eef14f20818c137f47 Mon Sep 17 00:00:00 2001 From: mannol Date: Fri, 7 Aug 2015 02:04:04 +0200 Subject: [PATCH 56/97] Fixed memory leak during session cleanup and fixed naming. --- other/apidsl/toxav.in.h | 4 ++-- testing/av_test.c | 5 ----- toxav/audio.c | 1 + toxav/rtp.c | 3 ++- toxav/toxav.h | 4 ++-- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/other/apidsl/toxav.in.h b/other/apidsl/toxav.in.h index 22a01602c..f437eeb35 100644 --- a/other/apidsl/toxav.in.h +++ b/other/apidsl/toxav.in.h @@ -490,12 +490,12 @@ namespace video { * * @param friend_number The friend number of the friend for which to set the * video bit rate. - * @param audio_bit_rate The new video bit rate in Kb/sec. Set to 0 to disable + * @param video_bit_rate The new video bit rate in Kb/sec. Set to 0 to disable * video sending. * @param force True if the bit rate change is forceful. * */ - bool set(uint32_t friend_number, uint32_t audio_bit_rate, bool force) with error for set_bit_rate; + bool set(uint32_t friend_number, uint32_t video_bit_rate, bool force) with error for set_bit_rate; } } /******************************************************************************* diff --git a/testing/av_test.c b/testing/av_test.c index 20e00c12e..8e048d02c 100644 --- a/testing/av_test.c +++ b/testing/av_test.c @@ -91,7 +91,6 @@ struct toxav_thread_data { const char* vdout = "AV Test"; /* Video output */ PaStream* adout = NULL; /* Audio output */ - typedef struct { uint16_t size; int16_t data[]; @@ -118,7 +117,6 @@ void* pa_write_thread (void* d) } } - /** * Callbacks */ @@ -207,7 +205,6 @@ void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t } } - /** */ void initialize_tox(Tox** bootstrap, ToxAV** AliceAV, CallControl* AliceCC, ToxAV** BobAV, CallControl* BobCC) @@ -374,7 +371,6 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) free(planes[2]); return rc; } - int print_audio_devices() { int i = 0; @@ -386,7 +382,6 @@ int print_audio_devices() return 0; } - int print_help (const char* name) { printf("Usage: %s -[a:v:o:dh]\n" diff --git a/toxav/audio.c b/toxav/audio.c index 0a6f04e39..afb9f26ce 100644 --- a/toxav/audio.c +++ b/toxav/audio.c @@ -117,6 +117,7 @@ void ac_kill(ACSession* ac) return; opus_encoder_destroy(ac->encoder); + opus_encoder_destroy(ac->test_encoder); opus_decoder_destroy(ac->decoder); jbuf_free(ac->j_buf); diff --git a/toxav/rtp.c b/toxav/rtp.c index 148c4238d..7a20877fa 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -138,7 +138,7 @@ void rtp_kill ( RTPSession *session ) { if ( !session ) return; - rtp_stop_receiving (session); + rtp_stop_receiving (session); free ( session->ext_header ); free ( session->csrc ); @@ -153,6 +153,7 @@ void rtp_kill ( RTPSession *session ) LOGGER_DEBUG("Terminated RTP session: %p", session); /* And finally free session */ + free ( session->rtcp_session ); free ( session ); } int rtp_do(RTPSession *session) diff --git a/toxav/toxav.h b/toxav/toxav.h index 3c745b364..58d5503f5 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -510,12 +510,12 @@ void toxav_callback_video_bit_rate_status(ToxAV *toxAV, toxav_video_bit_rate_sta * * @param friend_number The friend number of the friend for which to set the * video bit rate. - * @param audio_bit_rate The new video bit rate in Kb/sec. Set to 0 to disable + * @param video_bit_rate The new video bit rate in Kb/sec. Set to 0 to disable * video sending. * @param force True if the bit rate change is forceful. * */ -bool toxav_video_bit_rate_set(ToxAV *toxAV, uint32_t friend_number, uint32_t audio_bit_rate, bool force, TOXAV_ERR_SET_BIT_RATE *error); +bool toxav_video_bit_rate_set(ToxAV *toxAV, uint32_t friend_number, uint32_t video_bit_rate, bool force, TOXAV_ERR_SET_BIT_RATE *error); /******************************************************************************* * * :: A/V sending From 3c8cae72d08cabe870e2fd6b6ffdd78a32c2b410 Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 8 Aug 2015 17:45:24 +0200 Subject: [PATCH 57/97] Removed redundant function from video.[h|c] --- testing/av_test.c | 10 +++++----- toxav/toxav.c | 4 ++-- toxav/video.c | 32 +++++--------------------------- toxav/video.h | 3 +-- 4 files changed, 13 insertions(+), 36 deletions(-) diff --git a/testing/av_test.c b/testing/av_test.c index 8e048d02c..1c13ebad8 100644 --- a/testing/av_test.c +++ b/testing/av_test.c @@ -70,8 +70,8 @@ #define YUV2B(Y, U, V) CLIP(( 298 * C(Y) + 516 * D(U) + 128) >> 8) -#define TEST_TRANSFER_A 1 -#define TEST_TRANSFER_V 0 +#define TEST_TRANSFER_A 0 +#define TEST_TRANSFER_V 1 typedef struct { @@ -650,7 +650,7 @@ int main (int argc, char** argv) { /* Call */ TOXAV_ERR_CALL rc; - toxav_call(AliceAV, 0, 0, 3000, &rc); + toxav_call(AliceAV, 0, 0, 2000, &rc); if (rc != TOXAV_ERR_CALL_OK) { printf("toxav_call failed: %d\n", rc); @@ -663,7 +663,7 @@ int main (int argc, char** argv) { /* Answer */ TOXAV_ERR_ANSWER rc; - toxav_answer(BobAV, 0, 0, 500, &rc); + toxav_answer(BobAV, 0, 0, 5000, &rc); if (rc != TOXAV_ERR_ANSWER_OK) { printf("toxav_answer failed: %d\n", rc); @@ -690,7 +690,7 @@ int main (int argc, char** argv) exit(1); } - toxav_video_bit_rate_set(AliceAV, 0, 5000, false, NULL); +// toxav_video_bit_rate_set(AliceAV, 0, 5000, false, NULL); time_t start_time = time(NULL); while(start_time + 90 > time(NULL)) { diff --git a/toxav/toxav.c b/toxav/toxav.c index 58e083765..aaad2f144 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -897,7 +897,7 @@ bool toxav_video_send_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u goto END; } - if ( vc_reconfigure_encoder(call->video.second, call->video_bit_rate * 1000, width, height) != 0 ) { + if ( vc_reconfigure_encoder(call->video.second->encoder, call->video_bit_rate * 1000, width, height) != 0 ) { pthread_mutex_unlock(call->mutex_video); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; @@ -962,7 +962,7 @@ bool toxav_video_send_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u } if (ba_shoud_send_dummy(&call->vba)) { - if ( vc_reconfigure_test_encoder(call->video.second, call->vba.bit_rate * 1000, width, height) != 0 ) { + if ( vc_reconfigure_encoder(call->video.second->test_encoder, call->vba.bit_rate * 1000, width, height) != 0 ) { pthread_mutex_unlock(call->mutex_video); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; diff --git a/toxav/video.c b/toxav/video.c index ee49c0a14..f5f9f5138 100644 --- a/toxav/video.c +++ b/toxav/video.c @@ -289,20 +289,20 @@ end: rtp_free_msg(msg); return 0; } -int vc_reconfigure_encoder(VCSession* vc, int32_t bit_rate, uint16_t width, uint16_t height) +int vc_reconfigure_encoder(vpx_codec_ctx_t* vccdc, uint32_t bit_rate, uint16_t width, uint16_t height) { - if (!vc) + if (!vccdc) return -1; - vpx_codec_enc_cfg_t cfg = *vc->encoder->config.enc; - if (cfg.rc_target_bitrate == (uint32_t) bit_rate && cfg.g_w == width && cfg.g_h == height) + vpx_codec_enc_cfg_t cfg = *vccdc->config.enc; + if (cfg.rc_target_bitrate == bit_rate && cfg.g_w == width && cfg.g_h == height) return 0; /* Nothing changed */ cfg.rc_target_bitrate = bit_rate; cfg.g_w = width; cfg.g_h = height; - int rc = vpx_codec_enc_config_set(vc->encoder, &cfg); + int rc = vpx_codec_enc_config_set(vccdc, &cfg); if ( rc != VPX_CODEC_OK) { LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); return -1; @@ -310,28 +310,6 @@ int vc_reconfigure_encoder(VCSession* vc, int32_t bit_rate, uint16_t width, uint return 0; } -int vc_reconfigure_test_encoder(VCSession* vc, int32_t bit_rate, uint16_t width, uint16_t height) -{ - if (!vc) - return -1; - - vpx_codec_enc_cfg_t cfg = *vc->test_encoder->config.enc; - if (cfg.rc_target_bitrate == (uint32_t) bit_rate && cfg.g_w == width && cfg.g_h == height) - return 0; /* Nothing changed */ - - cfg.rc_target_bitrate = bit_rate; - cfg.g_w = width; - cfg.g_h = height; - - int rc = vpx_codec_enc_config_set(vc->test_encoder, &cfg); - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set test encoder control setting: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - return 0; -} - bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bit_rate) diff --git a/toxav/video.h b/toxav/video.h index 96d3205d3..ac165df67 100644 --- a/toxav/video.h +++ b/toxav/video.h @@ -107,7 +107,6 @@ int vc_queue_message(void *vcp, struct RTPMessage_s *msg); /* * Set new values to the encoders. */ -int vc_reconfigure_encoder(VCSession* vc, int32_t bit_rate, uint16_t width, uint16_t height); -int vc_reconfigure_test_encoder(VCSession* vc, int32_t bit_rate, uint16_t width, uint16_t height); +int vc_reconfigure_encoder(vpx_codec_ctx_t* vccdc, uint32_t bit_rate, uint16_t width, uint16_t height); #endif /* VIDEO_H */ \ No newline at end of file From 6ab4308581f6b06b2a4516614b4f1e2f9b3667a9 Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 8 Aug 2015 18:01:36 +0200 Subject: [PATCH 58/97] Apply OPUS codec fixes --- toxav/audio.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/toxav/audio.c b/toxav/audio.c index afb9f26ce..ff1e17825 100644 --- a/toxav/audio.c +++ b/toxav/audio.c @@ -343,7 +343,7 @@ static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success) OpusEncoder* create_audio_encoder (int32_t bit_rate, int32_t sampling_rate, int32_t channel_count) { int status = OPUS_OK; - OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_AUDIO, &status); + OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_VOIP, &status); if ( status != OPUS_OK ) { LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(status)); @@ -357,6 +357,26 @@ OpusEncoder* create_audio_encoder (int32_t bit_rate, int32_t sampling_rate, int3 goto FAILURE; } + /* Enable in-band forward error correction in codec */ + status = opus_encoder_ctl(rc, OPUS_SET_INBAND_FEC(1)); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); + goto FAILURE; + } + + /* Make codec resistant to up to 10% packet loss + * NOTE This could also be adjusted on the fly, rather than hard-coded, + * with feedback from the receiving client. + */ + status = opus_encoder_ctl(rc, OPUS_SET_PACKET_LOSS_PERC(10)); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); + goto FAILURE; + } + + /* Set algorithm to the highest complexity, maximizing compression */ status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(10)); if ( status != OPUS_OK ) { From 12d3f9396b53aa71f41cd16703cdeb7befd99a4a Mon Sep 17 00:00:00 2001 From: mannol Date: Sun, 9 Aug 2015 11:57:39 +0200 Subject: [PATCH 59/97] Fix possible double free --- toxav/toxav.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/toxav/toxav.c b/toxav/toxav.c index aaad2f144..881ade1f5 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -113,7 +113,7 @@ int callback_capabilites(void* toxav_inst, MSICall* call); bool audio_bit_rate_invalid(uint32_t bit_rate); bool video_bit_rate_invalid(uint32_t bit_rate); -bool invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state); +bool invoke_call_state_callback(ToxAV* av, uint32_t friend_number, uint32_t state); ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); ToxAVCall* call_get(ToxAV* av, uint32_t friend_number); ToxAVCall* call_remove(ToxAVCall* call); @@ -1104,7 +1104,7 @@ int callback_start(void* toxav_inst, MSICall* call) return -1; } - if (!invoke_call_state(toxav, call->friend_number, call->peer_capabilities)) { + if (!invoke_call_state_callback(toxav, call->friend_number, call->peer_capabilities)) { callback_error(toxav_inst, call); pthread_mutex_unlock(toxav->mutex); return -1; @@ -1119,10 +1119,12 @@ int callback_end(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - invoke_call_state(toxav, call->friend_number, TOXAV_FRIEND_CALL_STATE_FINISHED); + invoke_call_state_callback(toxav, call->friend_number, TOXAV_FRIEND_CALL_STATE_FINISHED); - call_kill_transmission(call->av_call); - call_remove(call->av_call); + if (call->av_call) { + call_kill_transmission(call->av_call); + call_remove(call->av_call); + } pthread_mutex_unlock(toxav->mutex); return 0; @@ -1133,10 +1135,12 @@ int callback_error(void* toxav_inst, MSICall* call) ToxAV* toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - invoke_call_state(toxav, call->friend_number, TOXAV_FRIEND_CALL_STATE_ERROR); + invoke_call_state_callback(toxav, call->friend_number, TOXAV_FRIEND_CALL_STATE_ERROR); - call_kill_transmission(call->av_call); - call_remove(call->av_call); + if (call->av_call) { + call_kill_transmission(call->av_call); + call_remove(call->av_call); + } pthread_mutex_unlock(toxav->mutex); return 0; @@ -1157,7 +1161,7 @@ int callback_capabilites(void* toxav_inst, MSICall* call) else rtp_stop_receiving(((ToxAVCall*)call->av_call)->video.first); - invoke_call_state(toxav, call->friend_number, call->peer_capabilities); + invoke_call_state_callback(toxav, call->friend_number, call->peer_capabilities); pthread_mutex_unlock(toxav->mutex); return 0; @@ -1178,7 +1182,7 @@ bool video_bit_rate_invalid(uint32_t bit_rate) return false; } -bool invoke_call_state(ToxAV* av, uint32_t friend_number, uint32_t state) +bool invoke_call_state_callback(ToxAV* av, uint32_t friend_number, uint32_t state) { if (av->scb.first) av->scb.first(av, friend_number, state, av->scb.second); @@ -1288,6 +1292,10 @@ ToxAVCall* call_remove(ToxAVCall* call) ToxAVCall* prev = call->prev; ToxAVCall* next = call->next; + /* Set av call in msi to NULL in order to know if call if ToxAVCall is + * removed from the msi call. + */ + call->msi_call->av_call = NULL; free(call); if (prev) From 0be0e88f3ec0cd81147a1418aa0afe61c24112b7 Mon Sep 17 00:00:00 2001 From: mannol Date: Sun, 9 Aug 2015 12:02:52 +0200 Subject: [PATCH 60/97] Remove empty lines --- toxav/toxav.c | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/toxav/toxav.c b/toxav/toxav.c index 881ade1f5..8624a6b1f 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -142,7 +142,6 @@ bool toxav_version_is_compatible(uint32_t major, uint32_t minor, uint32_t patch) return 1; } - ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) { TOXAV_ERR_NEW rc = TOXAV_ERR_NEW_OK; @@ -202,7 +201,6 @@ END: return av; } - void toxav_kill(ToxAV* av) { if (av == NULL) @@ -224,18 +222,15 @@ void toxav_kill(ToxAV* av) pthread_mutex_destroy(av->mutex); free(av); } - Tox* toxav_get_tox(const ToxAV* av) { return (Tox*) av->m; } - uint32_t toxav_iteration_interval(const ToxAV* av) { /* If no call is active interval is 200 */ return av->calls ? av->interval : 200; } - void toxav_iterate(ToxAV* av) { pthread_mutex_lock(av->mutex); @@ -336,7 +331,6 @@ void toxav_iterate(ToxAV* av) av->dmsst = 0; } } - bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) { if ((audio_bit_rate && audio_bit_rate_invalid(audio_bit_rate)) @@ -375,7 +369,6 @@ bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint return true; } - void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data) { pthread_mutex_lock(av->mutex); @@ -383,7 +376,6 @@ void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data) av->ccb.second = user_data; pthread_mutex_unlock(av->mutex); } - bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER* error) { pthread_mutex_lock(av->mutex); @@ -432,7 +424,6 @@ END: return rc == TOXAV_ERR_ANSWER_OK; } - void toxav_callback_call_state(ToxAV* av, toxav_call_state_cb* function, void* user_data) { pthread_mutex_lock(av->mutex); @@ -440,7 +431,6 @@ void toxav_callback_call_state(ToxAV* av, toxav_call_state_cb* function, void* u av->scb.second = user_data; pthread_mutex_unlock(av->mutex); } - bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL* error) { pthread_mutex_lock(av->mutex); @@ -585,7 +575,6 @@ END: return rc == TOXAV_ERR_CALL_CONTROL_OK; } - void toxav_callback_audio_bit_rate_status(ToxAV* av, toxav_audio_bit_rate_status_cb* function, void* user_data) { pthread_mutex_lock(av->mutex); @@ -593,7 +582,6 @@ void toxav_callback_audio_bit_rate_status(ToxAV* av, toxav_audio_bit_rate_status av->abcb.second = user_data; pthread_mutex_unlock(av->mutex); } - bool toxav_audio_bit_rate_set(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, bool force, TOXAV_ERR_SET_BIT_RATE* error) { LOGGER_DEBUG("Setting new audio bitrate to: %d", audio_bit_rate); @@ -668,7 +656,6 @@ END: return rc == TOXAV_ERR_SET_BIT_RATE_OK; } - void toxav_callback_video_bit_rate_status(ToxAV* av, toxav_video_bit_rate_status_cb* function, void* user_data) { pthread_mutex_lock(av->mutex); @@ -676,7 +663,6 @@ void toxav_callback_video_bit_rate_status(ToxAV* av, toxav_video_bit_rate_status av->vbcb.second = user_data; pthread_mutex_unlock(av->mutex); } - bool toxav_video_bit_rate_set(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, bool force, TOXAV_ERR_SET_BIT_RATE* error) { LOGGER_DEBUG("Setting new video bitrate to: %d", video_bit_rate); @@ -751,7 +737,6 @@ END: return rc == TOXAV_ERR_SET_BIT_RATE_OK; } - bool toxav_audio_send_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error) { TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; @@ -861,7 +846,6 @@ END: return rc == TOXAV_ERR_SEND_FRAME_OK; } - bool toxav_video_send_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) { TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; @@ -1034,7 +1018,6 @@ END: return rc == TOXAV_ERR_SEND_FRAME_OK; } - void toxav_callback_audio_receive_frame(ToxAV* av, toxav_audio_receive_frame_cb* function, void* user_data) { pthread_mutex_lock(av->mutex); @@ -1042,7 +1025,6 @@ void toxav_callback_audio_receive_frame(ToxAV* av, toxav_audio_receive_frame_cb* av->acb.second = user_data; pthread_mutex_unlock(av->mutex); } - void toxav_callback_video_receive_frame(ToxAV* av, toxav_video_receive_frame_cb* function, void* user_data) { pthread_mutex_lock(av->mutex); @@ -1084,7 +1066,6 @@ int callback_invite(void* toxav_inst, MSICall* call) pthread_mutex_unlock(toxav->mutex); return 0; } - int callback_start(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; @@ -1113,7 +1094,6 @@ int callback_start(void* toxav_inst, MSICall* call) pthread_mutex_unlock(toxav->mutex); return 0; } - int callback_end(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; @@ -1129,7 +1109,6 @@ int callback_end(void* toxav_inst, MSICall* call) pthread_mutex_unlock(toxav->mutex); return 0; } - int callback_error(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; @@ -1145,7 +1124,6 @@ int callback_error(void* toxav_inst, MSICall* call) pthread_mutex_unlock(toxav->mutex); return 0; } - int callback_capabilites(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; @@ -1166,7 +1144,6 @@ int callback_capabilites(void* toxav_inst, MSICall* call) pthread_mutex_unlock(toxav->mutex); return 0; } - bool audio_bit_rate_invalid(uint32_t bit_rate) { /* Opus RFC 6716 section-2.1.1 dictates the following: @@ -1174,14 +1151,12 @@ bool audio_bit_rate_invalid(uint32_t bit_rate) */ return bit_rate < 6 || bit_rate > 510; } - bool video_bit_rate_invalid(uint32_t bit_rate) { (void) bit_rate; /* TODO: If anyone knows the answer to this one please fill it up */ return false; } - bool invoke_call_state_callback(ToxAV* av, uint32_t friend_number, uint32_t state) { if (av->scb.first) @@ -1190,7 +1165,6 @@ bool invoke_call_state_callback(ToxAV* av, uint32_t friend_number, uint32_t stat return false; return true; } - ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) { /* Assumes mutex locked */ @@ -1271,7 +1245,6 @@ END: return call; } - ToxAVCall* call_get(ToxAV* av, uint32_t friend_number) { /* Assumes mutex locked */ @@ -1280,7 +1253,6 @@ ToxAVCall* call_get(ToxAV* av, uint32_t friend_number) return av->calls[friend_number]; } - ToxAVCall* call_remove(ToxAVCall* call) { if (call == NULL) @@ -1320,7 +1292,6 @@ CLEAR: return NULL; } - bool call_prepare_transmission(ToxAVCall* call) { /* Assumes mutex locked */ @@ -1396,7 +1367,6 @@ FAILURE_3: pthread_mutex_destroy(call->mutex_audio); return false; } - void call_kill_transmission(ToxAVCall* call) { if (call == NULL || call->active == 0) @@ -1425,7 +1395,6 @@ void call_kill_transmission(ToxAVCall* call) pthread_mutex_destroy(call->mutex_video); pthread_mutex_destroy(call->mutex); } - void ba_set(ToxAvBitrateAdapter* ba, uint32_t bit_rate) { ba->bit_rate = bit_rate; @@ -1434,7 +1403,6 @@ void ba_set(ToxAvBitrateAdapter* ba, uint32_t bit_rate) ba->next_send_interval = 1000; ba->active = true; } - bool ba_shoud_send_dummy(ToxAvBitrateAdapter* ba) { if (!ba->active || ba->next_send > current_time_monotonic()) From bf5e9b89d2a67c293aae503c03e193307ea7990b Mon Sep 17 00:00:00 2001 From: mannol Date: Thu, 13 Aug 2015 00:19:24 +0200 Subject: [PATCH 61/97] Fix bug in codec initialization --- toxav/video.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/toxav/video.c b/toxav/video.c index f5f9f5138..389d2e1c2 100644 --- a/toxav/video.c +++ b/toxav/video.c @@ -324,6 +324,16 @@ bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bit_rate) return false; } + cfg.rc_target_bitrate = bit_rate; + cfg.g_w = 4000; + cfg.g_h = 4000; + cfg.g_pass = VPX_RC_ONE_PASS; + cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; + cfg.g_lag_in_frames = 0; + cfg.kf_min_dist = 0; + cfg.kf_max_dist = 48; + cfg.kf_mode = VPX_KF_AUTO; + rc = vpx_codec_enc_init_ver(dest, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, VPX_ENCODER_ABI_VERSION); @@ -332,16 +342,6 @@ bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bit_rate) return false; } - cfg.rc_target_bitrate = bit_rate; - cfg.g_w = 800; - cfg.g_h = 600; - cfg.g_pass = VPX_RC_ONE_PASS; - cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; - cfg.g_lag_in_frames = 0; - cfg.kf_min_dist = 0; - cfg.kf_max_dist = 48; - cfg.kf_mode = VPX_KF_AUTO; - rc = vpx_codec_control(dest, VP8E_SET_CPUUSED, 8); if ( rc != VPX_CODEC_OK) { From 4ca82235a0e6dd0a8af31a6c106434af1e41f07f Mon Sep 17 00:00:00 2001 From: krobelus Date: Tue, 18 Aug 2015 21:28:25 +0200 Subject: [PATCH 62/97] typo --- toxav/toxav.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toxav/toxav.h b/toxav/toxav.h index 58d5503f5..50a1c36b4 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -40,7 +40,7 @@ extern "C" { * * As in Core API, events are handled by callbacks. One callback can be * registered per event. All events have a callback function type named - * `toxav_{event}_cb` and a function to register it named `tox_callback_{event}`. + * `toxav_{event}_cb` and a function to register it named `toxav_callback_{event}`. * Passing a NULL callback will result in no callback being registered for that * event. Only one callback per event can be registered, so if a client needs * multiple event listeners, it needs to implement the dispatch functionality From d6fdf16520b6f242935ca95eeb739ec9a8eaa14c Mon Sep 17 00:00:00 2001 From: Eniz Vukovic Date: Sat, 10 Oct 2015 23:54:23 +0200 Subject: [PATCH 63/97] New Adaptive BR algorithm, cleanups and fixes --- auto_tests/toxav_basic_test.c | 30 +- auto_tests/toxav_many_test.c | 26 +- configure.ac | 14 +- other/apidsl/toxav.in.h | 130 ++-- testing/av_test.c | 74 +-- toxav/Makefile.inc | 61 +- toxav/audio.c | 282 ++++---- toxav/audio.h | 65 +- toxav/bwcontroler.c | 207 ++++++ toxav/bwcontroler.h | 37 ++ toxav/group.c | 12 +- toxav/msi.c | 628 +++++++++--------- toxav/msi.h | 35 +- toxav/rtp.c | 768 ++++++++------------- toxav/rtp.h | 139 ++-- toxav/toxav.c | 1182 ++++++++++++++------------------- toxav/toxav.h | 238 +++---- toxav/toxav_old.c | 4 +- toxav/video.c | 292 +++----- toxav/video.h | 65 +- toxcore/Messenger.c | 6 +- toxcore/assoc.c | 8 +- toxcore/assoc.h | 4 +- toxcore/logger.c | 18 +- toxcore/logger.h | 19 +- toxcore/network.c | 12 +- toxcore/util.c | 29 +- toxcore/util.h | 6 +- 28 files changed, 2030 insertions(+), 2361 deletions(-) create mode 100644 toxav/bwcontroler.c create mode 100644 toxav/bwcontroler.h diff --git a/auto_tests/toxav_basic_test.c b/auto_tests/toxav_basic_test.c index 41fb6787a..5821a6d45 100644 --- a/auto_tests/toxav_basic_test.c +++ b/auto_tests/toxav_basic_test.c @@ -2,11 +2,20 @@ #include "config.h" #endif +#ifndef HAVE_LIBCHECK +# include + +# define ck_assert(X) assert(X); +# define START_TEST(NAME) void NAME () +# define END_TEST +#else +# include "helpers.h" +#endif + #include #include #include #include -#include #include #include @@ -18,7 +27,6 @@ #include "../toxcore/crypto_core.h" #include "../toxav/toxav.h" -#include "helpers.h" #if defined(_WIN32) || defined(__WIN32__) || defined (WIN32) #define c_sleep(x) Sleep(1*x) @@ -462,19 +470,19 @@ START_TEST(test_AV_flows) printf("Call started as audio only\n"); printf("Turning on video for Alice...\n"); - ck_assert(toxav_video_bit_rate_set(AliceAV, 0, 1000, false, NULL)); + ck_assert(toxav_bit_rate_set(AliceAV, 0, -1, 1000, NULL)); iterate_tox(bootstrap, Alice, Bob); ck_assert(BobCC.state & TOXAV_FRIEND_CALL_STATE_SENDING_V); printf("Turning off video for Alice...\n"); - ck_assert(toxav_video_bit_rate_set(AliceAV, 0, 0, false, NULL)); + ck_assert(toxav_bit_rate_set(AliceAV, 0, -1, 0, NULL)); iterate_tox(bootstrap, Alice, Bob); ck_assert(!(BobCC.state & TOXAV_FRIEND_CALL_STATE_SENDING_V)); printf("Turning off audio for Alice...\n"); - ck_assert(toxav_audio_bit_rate_set(AliceAV, 0, 0, false, NULL)); + ck_assert(toxav_bit_rate_set(AliceAV, 0, 0, -1, NULL)); iterate_tox(bootstrap, Alice, Bob); ck_assert(!(BobCC.state & TOXAV_FRIEND_CALL_STATE_SENDING_A)); @@ -564,7 +572,16 @@ START_TEST(test_AV_flows) } END_TEST - +#ifndef HAVE_LIBCHECK +int main(int argc, char *argv[]) +{ + (void) argc; + (void) argv; + + test_AV_flows(); + return 0; +} +#else Suite *tox_suite(void) { Suite *s = suite_create("ToxAV"); @@ -589,3 +606,4 @@ int main(int argc, char *argv[]) return number_failed; } +#endif diff --git a/auto_tests/toxav_many_test.c b/auto_tests/toxav_many_test.c index 438f27895..7dc82c6f2 100644 --- a/auto_tests/toxav_many_test.c +++ b/auto_tests/toxav_many_test.c @@ -2,18 +2,25 @@ #include "config.h" #endif +#ifndef HAVE_LIBCHECK +# include + +# define ck_assert(X) assert(X); +# define START_TEST(NAME) void NAME () +# define END_TEST +#else +# include "helpers.h" +#endif + #include #include #include #include -#include #include #include #include -#include "helpers.h" - #include "../toxcore/tox.h" #include "../toxcore/util.h" #include "../toxcore/logger.h" @@ -331,8 +338,16 @@ START_TEST(test_AV_three_calls) END_TEST - - +#ifndef HAVE_LIBCHECK +int main(int argc, char *argv[]) +{ + (void) argc; + (void) argv; + + test_AV_three_calls(); + return 0; +} +#else Suite *tox_suite(void) { Suite *s = suite_create("ToxAV"); @@ -362,3 +377,4 @@ int main(int argc, char *argv[]) return number_failed; } +#endif diff --git a/configure.ac b/configure.ac index 2b7f3a2e2..639fc20c5 100644 --- a/configure.ac +++ b/configure.ac @@ -33,7 +33,7 @@ BUILD_TESTS="yes" BUILD_AV="yes" BUILD_TESTING="yes" -LOGGING="no" +TOX_LOGGER="no" LOGGING_OUTNAM="libtoxcore.log" NCURSES_FOUND="no" @@ -82,13 +82,13 @@ AC_ARG_ENABLE([randombytes-stir], ] ) -AC_ARG_ENABLE([log], - [AC_HELP_STRING([--enable-log], [enable logging (default: auto)]) ], +AC_ARG_ENABLE([logger], + [AC_HELP_STRING([--enable-logger], [enable logging (default: auto)]) ], [ if test "x$enableval" = "xyes"; then - LOGGING="yes" + TOX_LOGGER="yes" - AC_DEFINE([LOGGING], [], [If logging enabled]) + AC_DEFINE([TOX_LOGGER], [], [If logging enabled]) AC_DEFINE([LOGGER_LEVEL], [LOG_DEBUG], [LOG_LEVEL value]) AC_DEFINE_UNQUOTED([LOGGER_OUTPUT_FILE], ["$LOGGING_OUTNAM"], [Output of logger]) fi @@ -99,7 +99,7 @@ AC_ARG_WITH(log-level, AC_HELP_STRING([--with-log-level=LEVEL], [Logger levels: TRACE; DEBUG; INFO; WARNING; ERROR ]), [ - if test "x$LOGGING" = "xno"; then + if test "x$TOX_LOGGER" = "xno"; then AC_MSG_WARN([Logging disabled!]) else if test "x$withval" = "xTRACE"; then @@ -127,7 +127,7 @@ AC_ARG_WITH(log-path, AC_HELP_STRING([--with-log-path=DIR], [Path of logger output]), [ - if test "x$LOGGING" = "xno"; then + if test "x$TOX_LOGGER" = "xno"; then AC_MSG_WARN([Logging disabled!]) else AC_DEFINE_UNQUOTED([LOGGER_OUTPUT_FILE], ["$withval""/""$LOGGING_OUTNAM"], [Output of logger]) diff --git a/other/apidsl/toxav.in.h b/other/apidsl/toxav.in.h index f437eeb35..c272b9347 100644 --- a/other/apidsl/toxav.in.h +++ b/other/apidsl/toxav.in.h @@ -56,12 +56,19 @@ extern "C" { /** \subsection threading Threading implications * * Unlike the Core API, this API is fully thread-safe. The library will ensure - * the proper synchronisation of parallel calls. + * the proper synchronization of parallel calls. * * A common way to run ToxAV (multiple or single instance) is to have a thread, * separate from tox instance thread, running a simple ${toxAV.iterate} loop, * sleeping for ${toxAV.iteration_interval} * milliseconds on each iteration. * + * An important thing to note is that events are triggered from both tox and + * toxav thread (see above). audio and video receive frame events are triggered + * from toxav thread while all the other events are triggered from tox thread. + * + * Tox thread has priority with mutex mechanisms. Any api function can + * fail if mutexes are held by tox thread in which case they will set SYNC + * error code. */ /** @@ -231,6 +238,10 @@ bool call(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_ra * required for the call. */ MALLOC, + /** + * Synchronization error occurred. + */ + SYNC, /** * The friend number did not designate a valid friend. */ @@ -273,6 +284,10 @@ event call { * video sending. */ bool answer(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate) { + /** + * Synchronization error occurred. + */ + SYNC, /** * Failed to initialize codecs for call session. Note that codec initiation * will fail if there is no receive callback registered for either audio or @@ -347,7 +362,7 @@ event call_state { * ******************************************************************************/ enum class CALL_CONTROL { - /** + /** * Resume a previously paused call. Only valid if the pause was caused by this * client, if not, this control is ignored. Not valid before the call is accepted. */ @@ -392,6 +407,10 @@ enum class CALL_CONTROL { * @return true on success. */ bool call_control (uint32_t friend_number, CALL_CONTROL control) { + /** + * Synchronization error occurred. + */ + SYNC, /** * The friend_number passed did not designate a valid friend. */ @@ -412,38 +431,7 @@ bool call_control (uint32_t friend_number, CALL_CONTROL control) { * :: Controlling bit rates * ******************************************************************************/ -error for set_bit_rate { - /** - * The bit rate passed was not one of the supported values. - */ - INVALID, - /** - * The friend_number passed did not designate a valid friend. - */ - FRIEND_NOT_FOUND, - /** - * This client is currently not in a call with the friend. - */ - FRIEND_NOT_IN_CALL, -} -namespace audio { - namespace bit_rate { - event status { - /** - * The function type for the ${event status} callback. - * - * @param friend_number The friend number of the friend for which to set the - * audio bit rate. - * @param stable Is the stream stable enough to keep the bit rate. - * Upon successful, non forceful, bit rate change, this is set to - * true and 'bit_rate' is set to new bit rate. - * The stable is set to false with bit_rate set to the unstable - * bit rate when either current stream is unstable with said bit rate - * or the non forceful change failed. - * @param bit_rate The bit rate in Kb/sec. - */ - typedef void(uint32_t friend_number, bool stable, uint32_t bit_rate); - } +namespace bit_rate { /** * Set the audio bit rate to be used in subsequent audio frames. If the passed * bit rate is the same as the current bit rate this function will return true @@ -452,51 +440,43 @@ namespace audio { * forcefully set and the previous non forceful request is cancelled. The active * non forceful setup will be canceled in favour of new non forceful setup. * - * @param friend_number The friend number of the friend for which to set the - * audio bit rate. + * @param friend_number The friend number. * @param audio_bit_rate The new audio bit rate in Kb/sec. Set to 0 to disable - * audio sending. - * @param force True if the bit rate change is forceful. - * - */ - bool set(uint32_t friend_number, uint32_t audio_bit_rate, bool force) with error for set_bit_rate; - } -} -namespace video { - namespace bit_rate { - event status { - /** - * The function type for the ${event status} callback. - * - * @param friend_number The friend number of the friend for which to set the - * video bit rate. - * @param stable Is the stream stable enough to keep the bit rate. - * Upon successful, non forceful, bit rate change, this is set to - * true and 'bit_rate' is set to new bit rate. - * The stable is set to false with bit_rate set to the unstable - * bit rate when either current stream is unstable with said bit rate - * or the non forceful change failed. - * @param bit_rate The bit rate in Kb/sec. - */ - typedef void(uint32_t friend_number, bool stable, uint32_t bit_rate); - } - /** - * Set the video bit rate to be used in subsequent video frames. If the passed - * bit rate is the same as the current bit rate this function will return true - * without calling a callback. If there is an active non forceful setup with the - * passed video bit rate and the new set request is forceful, the bit rate is - * forcefully set and the previous non forceful request is cancelled. The active - * non forceful setup will be canceled in favour of new non forceful setup. - * - * @param friend_number The friend number of the friend for which to set the - * video bit rate. + * audio sending. Set to -1 to leave unchanged. * @param video_bit_rate The new video bit rate in Kb/sec. Set to 0 to disable - * video sending. - * @param force True if the bit rate change is forceful. + * video sending. Set to -1 to leave unchanged. * */ - bool set(uint32_t friend_number, uint32_t video_bit_rate, bool force) with error for set_bit_rate; - } + bool set(uint32_t friend_number, int32_t audio_bit_rate, int32_t video_bit_rate) { + /** + * Synchronization error occurred. + */ + SYNC, + /** + * The bit rate passed was not one of the supported values. + */ + INVALID, + /** + * The friend_number passed did not designate a valid friend. + */ + FRIEND_NOT_FOUND, + /** + * This client is currently not in a call with the friend. + */ + FRIEND_NOT_IN_CALL, + } + event status { + /** + * The function type for the ${event status} callback. The event is triggered + * when the network becomes too saturated for current bit rates at which + * point core suggests new bit rates. + * + * @param friend_number The friend number. + * @param audio_bit_rate Suggested maximum audio bit rate in Kb/sec. + * @param video_bit_rate Suggested maximum video bit rate in Kb/sec. + */ + typedef void(uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate); + } } /******************************************************************************* * diff --git a/testing/av_test.c b/testing/av_test.c index 1c13ebad8..fa6a831f8 100644 --- a/testing/av_test.c +++ b/testing/av_test.c @@ -28,9 +28,6 @@ #include "../toxcore/util.h" #include "../toxcore/network.h" /* current_time_monotonic() */ -#define LOGGING -#include "../toxcore/logger.h" - /* Playing audio data */ #include /* Reading audio */ @@ -53,21 +50,21 @@ #define c_sleep(x) usleep(1000*x) -#define CLIP(X) ( (X) > 255 ? 255 : (X) < 0 ? 0 : X) +#define CLIP(X) ((X) > 255 ? 255 : (X) < 0 ? 0 : X) // RGB -> YUV -#define RGB2Y(R, G, B) CLIP(( ( 66 * (R) + 129 * (G) + 25 * (B) + 128) >> 8) + 16) -#define RGB2U(R, G, B) CLIP(( ( -38 * (R) - 74 * (G) + 112 * (B) + 128) >> 8) + 128) -#define RGB2V(R, G, B) CLIP(( ( 112 * (R) - 94 * (G) - 18 * (B) + 128) >> 8) + 128) +#define RGB2Y(R, G, B) CLIP((( 66 * (R) + 129 * (G) + 25 * (B) + 128) >> 8) + 16) +#define RGB2U(R, G, B) CLIP(((-38 * (R) - 74 * (G) + 112 * (B) + 128) >> 8) + 128) +#define RGB2V(R, G, B) CLIP(((112 * (R) - 94 * (G) - 18 * (B) + 128) >> 8) + 128) // YUV -> RGB -#define C(Y) ( (Y) - 16 ) -#define D(U) ( (U) - 128 ) -#define E(V) ( (V) - 128 ) +#define C(Y) ((Y) - 16 ) +#define D(U) ((U) - 128 ) +#define E(V) ((V) - 128 ) -#define YUV2R(Y, U, V) CLIP(( 298 * C(Y) + 409 * E(V) + 128) >> 8) -#define YUV2G(Y, U, V) CLIP(( 298 * C(Y) - 100 * D(U) - 208 * E(V) + 128) >> 8) -#define YUV2B(Y, U, V) CLIP(( 298 * C(Y) + 516 * D(U) + 128) >> 8) +#define YUV2R(Y, U, V) CLIP((298 * C(Y) + 409 * E(V) + 128) >> 8) +#define YUV2G(Y, U, V) CLIP((298 * C(Y) - 100 * D(U) - 208 * E(V) + 128) >> 8) +#define YUV2B(Y, U, V) CLIP((298 * C(Y) + 516 * D(U) + 128) >> 8) #define TEST_TRANSFER_A 0 @@ -182,21 +179,11 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, free(rb_write(cc->arb, f)); pthread_mutex_unlock(cc->arb_mutex); } -void t_toxav_audio_bit_rate_status_cb(ToxAV *av, uint32_t friend_number, - bool stable, uint32_t bit_rate, void *user_data) +void t_toxav_bit_rate_status_cb(ToxAV *av, uint32_t friend_number, + uint32_t audio_bit_rate, uint32_t video_bit_rate, + void *user_data) { - if (stable) - printf ("Set new audio bit rate to: %d\n", bit_rate); - else - printf ("The network is overly saturated with audio bit rate at: %d\n", bit_rate); -} -void t_toxav_video_bit_rate_status_cb(ToxAV *av, uint32_t friend_number, - bool stable, uint32_t bit_rate, void *user_data) -{ - if (stable) - printf ("Set new video bit rate to: %d", bit_rate); - else - printf ("The network is overly saturated with video bit rate at: %d", bit_rate); + printf ("Suggested bit rates: audio: %d video: %d\n", audio_bit_rate, video_bit_rate); } void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, size_t length, void *userdata) { @@ -216,6 +203,7 @@ void initialize_tox(Tox** bootstrap, ToxAV** AliceAV, CallControl* AliceCC, ToxA tox_options_default(&opts); opts.end_port = 0; + opts.ipv6_enabled = false; { TOX_ERR_NEW error; @@ -279,18 +267,16 @@ void initialize_tox(Tox** bootstrap, ToxAV** AliceAV, CallControl* AliceCC, ToxA /* Alice */ toxav_callback_call(*AliceAV, t_toxav_call_cb, AliceCC); toxav_callback_call_state(*AliceAV, t_toxav_call_state_cb, AliceCC); + toxav_callback_bit_rate_status(*AliceAV, t_toxav_bit_rate_status_cb, AliceCC); toxav_callback_video_receive_frame(*AliceAV, t_toxav_receive_video_frame_cb, AliceCC); toxav_callback_audio_receive_frame(*AliceAV, t_toxav_receive_audio_frame_cb, AliceCC); - toxav_callback_video_bit_rate_status(*AliceAV, t_toxav_video_bit_rate_status_cb, AliceCC); - toxav_callback_audio_bit_rate_status(*AliceAV, t_toxav_audio_bit_rate_status_cb, AliceCC); /* Bob */ toxav_callback_call(*BobAV, t_toxav_call_cb, BobCC); toxav_callback_call_state(*BobAV, t_toxav_call_state_cb, BobCC); + toxav_callback_bit_rate_status(*BobAV, t_toxav_bit_rate_status_cb, BobCC); toxav_callback_video_receive_frame(*BobAV, t_toxav_receive_video_frame_cb, BobCC); toxav_callback_audio_receive_frame(*BobAV, t_toxav_receive_audio_frame_cb, BobCC); - toxav_callback_video_bit_rate_status(*BobAV, t_toxav_video_bit_rate_status_cb, BobCC); - toxav_callback_audio_bit_rate_status(*BobAV, t_toxav_audio_bit_rate_status_cb, BobCC); printf("Created 2 instances of ToxAV\n"); @@ -320,6 +306,9 @@ void* iterate_toxav (void * data) fflush(stdout); #if defined TEST_TRANSFER_V && TEST_TRANSFER_V == 1 + if (!rc) + rc = 1; + cvWaitKey(rc); #else c_sleep(rc); @@ -340,8 +329,8 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) int32_t strides[3] = { 1280, 640, 640 }; uint8_t* planes[3] = { malloc(img->height * img->width), - malloc(img->height * img->width / 2), - malloc(img->height * img->width / 2), + malloc(img->height * img->width / 4), + malloc(img->height * img->width / 4), }; int x_chroma_shift = 1; @@ -363,9 +352,9 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) } } } - - - int rc = toxav_video_send_frame(av, friend_number, img->width, img->height, planes[0], planes[1], planes[2], NULL); + + int rc = toxav_video_send_frame(av, friend_number, img->width, img->height, + planes[0], planes[1], planes[2], NULL); free(planes[0]); free(planes[1]); free(planes[2]); @@ -396,9 +385,8 @@ int print_help (const char* name) return 0; } - int main (int argc, char** argv) -{ +{ freopen("/dev/zero", "w", stderr); Pa_Initialize(); @@ -585,7 +573,7 @@ int main (int argc, char** argv) err = Pa_StartStream(adout); assert(err == paNoError); - toxav_audio_bit_rate_set(AliceAV, 0, 64, false, NULL); +// toxav_audio_bit_rate_set(AliceAV, 0, 64, false, NULL); /* Start write thread */ pthread_t t; @@ -593,7 +581,7 @@ int main (int argc, char** argv) pthread_detach(t); printf("Sample rate %d\n", af_info.samplerate); - while ( start_time + expected_time > time(NULL) ) { + while (start_time + expected_time > time(NULL) ) { uint64_t enc_start_time = current_time_monotonic(); int64_t count = sf_read_short(af_handle, PCM, frame_size); if (count > 0) { @@ -674,7 +662,7 @@ int main (int argc, char** argv) iterate_tox(bootstrap, AliceAV, BobAV); /* Start decode thread */ - struct toxav_thread_data data = { + struct toxav_thread_data data = { .AliceAV = AliceAV, .BobAV = BobAV, .sig = 0 @@ -694,13 +682,13 @@ int main (int argc, char** argv) time_t start_time = time(NULL); while(start_time + 90 > time(NULL)) { - IplImage* frame = cvQueryFrame( capture ); + IplImage* frame = cvQueryFrame(capture ); if (!frame) break; send_opencv_img(AliceAV, 0, frame); iterate_tox(bootstrap, AliceAV, BobAV); - c_sleep(video_frame_duration); + c_sleep(10); } cvReleaseCapture(&capture); diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc index 79275c9bb..232c06de5 100644 --- a/toxav/Makefile.inc +++ b/toxav/Makefile.inc @@ -1,42 +1,43 @@ if BUILD_AV -lib_LTLIBRARIES += libtoxav.la -libtoxav_la_include_HEADERS = ../toxav/toxav.h -libtoxav_la_includedir = $(includedir)/tox +lib_LTLIBRARIES += libtoxav.la + libtoxav_la_include_HEADERS = ../toxav/toxav.h + libtoxav_la_includedir = $(includedir)/tox libtoxav_la_SOURCES = ../toxav/rtp.h \ - ../toxav/rtp.c \ - ../toxav/msi.h \ - ../toxav/msi.c \ - ../toxav/group.h \ - ../toxav/group.c \ - ../toxav/audio.h \ - ../toxav/audio.c \ - ../toxav/video.h \ - ../toxav/video.c \ - ../toxav/toxav.h \ - ../toxav/toxav.c \ - ../toxav/toxav_old.h \ - ../toxav/toxav_old.c - + ../toxav/rtp.c \ + ../toxav/msi.h \ + ../toxav/msi.c \ + ../toxav/group.h \ + ../toxav/group.c \ + ../toxav/audio.h \ + ../toxav/audio.c \ + ../toxav/video.h \ + ../toxav/video.c \ + ../toxav/bwcontroler.h \ + ../toxav/bwcontroler.c \ + ../toxav/toxav.h \ + ../toxav/toxav.c \ + ../toxav/toxav_old.h \ + ../toxav/toxav_old.c libtoxav_la_CFLAGS = -I../toxcore \ - -I../toxav \ - $(LIBSODIUM_CFLAGS) \ - $(NACL_CFLAGS) \ - $(AV_CFLAGS) \ - $(PTHREAD_CFLAGS) + -I../toxav \ + $(LIBSODIUM_CFLAGS) \ + $(NACL_CFLAGS) \ + $(AV_CFLAGS) \ + $(PTHREAD_CFLAGS) libtoxav_la_LDFLAGS = $(TOXAV_LT_LDFLAGS) \ - $(LIBSODIUM_LDFLAGS) \ - $(NACL_LDFLAGS) \ - $(EXTRA_LT_LDFLAGS) \ - $(WINSOCK2_LIBS) + $(LIBSODIUM_LDFLAGS) \ + $(NACL_LDFLAGS) \ + $(EXTRA_LT_LDFLAGS) \ + $(WINSOCK2_LIBS) libtoxav_la_LIBADD = libtoxcore.la \ - $(LIBSODIUM_LIBS) \ - $(NACL_LIBS) \ - $(PTHREAD_LIBS) \ - $(AV_LIBS) + $(LIBSODIUM_LIBS) \ + $(NACL_LIBS) \ + $(PTHREAD_LIBS) \ + $(AV_LIBS) endif \ No newline at end of file diff --git a/toxav/audio.c b/toxav/audio.c index ff1e17825..3ba95c03e 100644 --- a/toxav/audio.c +++ b/toxav/audio.c @@ -19,6 +19,10 @@ * */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif /* HAVE_CONFIG_H */ + #include #include "audio.h" @@ -29,80 +33,71 @@ static struct JitterBuffer *jbuf_new(uint32_t capacity); static void jbuf_clear(struct JitterBuffer *q); static void jbuf_free(struct JitterBuffer *q); -static int jbuf_write(struct JitterBuffer *q, RTPMessage *m); -static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success); -OpusEncoder* create_audio_encoder (int32_t bit_rate, int32_t sampling_rate, int32_t channel_count); -bool reconfigure_audio_encoder(OpusEncoder** e, int32_t new_br, int32_t new_sr, uint8_t new_ch, +static int jbuf_write(struct JitterBuffer *q, struct RTPMessage *m); +static struct RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success); +OpusEncoder *create_audio_encoder (int32_t bit_rate, int32_t sampling_rate, int32_t channel_count); +bool reconfigure_audio_encoder(OpusEncoder **e, int32_t new_br, int32_t new_sr, uint8_t new_ch, int32_t *old_br, int32_t *old_sr, int32_t *old_ch); -bool reconfigure_audio_decoder(ACSession* ac, int32_t sampling_rate, int8_t channels); +bool reconfigure_audio_decoder(ACSession *ac, int32_t sampling_rate, int8_t channels); -ACSession* ac_new(ToxAV* av, uint32_t friend_number, toxav_audio_receive_frame_cb *cb, void *cb_data) +ACSession *ac_new(ToxAV *av, uint32_t friend_number, toxav_audio_receive_frame_cb *cb, void *cb_data) { ACSession *ac = calloc(sizeof(ACSession), 1); - + if (!ac) { LOGGER_WARNING("Allocation failed! Application might misbehave!"); return NULL; } - + if (create_recursive_mutex(ac->queue_mutex) != 0) { LOGGER_WARNING("Failed to create recursive mutex!"); free(ac); return NULL; } - + int status; - ac->decoder = opus_decoder_create(48000, 2, &status ); - - if ( status != OPUS_OK ) { + ac->decoder = opus_decoder_create(48000, 2, &status); + + if (status != OPUS_OK) { LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status)); goto BASE_CLEANUP; } - - if ( !(ac->j_buf = jbuf_new(3)) ) { + + if (!(ac->j_buf = jbuf_new(3))) { LOGGER_WARNING("Jitter buffer creaton failed!"); opus_decoder_destroy(ac->decoder); goto BASE_CLEANUP; } - + /* Initialize encoders with default values */ ac->encoder = create_audio_encoder(48000, 48000, 2); + if (ac->encoder == NULL) goto DECODER_CLEANUP; - ac->test_encoder = create_audio_encoder(48000, 48000, 2); - if (ac->test_encoder == NULL) { - opus_encoder_destroy(ac->encoder); - goto DECODER_CLEANUP; - } - - ac->last_encoding_bit_rate = 48000; - ac->last_encoding_sampling_rate = 48000; - ac->last_encoding_channel_count = 2; - - ac->last_test_encoding_bit_rate = 48000; - ac->last_test_encoding_sampling_rate = 48000; - ac->last_test_encoding_channel_count = 2; - - ac->last_decoding_channel_count = 2; - ac->last_decoding_sampling_rate = 48000; - ac->last_decoder_reconfiguration = 0; /* Make it possible to reconfigure straight away */ + ac->le_bit_rate = 48000; + ac->le_sample_rate = 48000; + ac->le_channel_count = 2; + ac->ld_channel_count = 2; + ac->ld_sample_rate = 48000; + ac->ldrts = 0; /* Make it possible to reconfigure straight away */ + /* These need to be set in order to properly * do error correction with opus */ - ac->last_packet_frame_duration = 120; - ac->last_packet_sampling_rate = 48000; - ac->last_packet_channel_count = 1; - + ac->lp_frame_duration = 120; + ac->lp_sampling_rate = 48000; + ac->lp_channel_count = 1; + ac->av = av; ac->friend_number = friend_number; ac->acb.first = cb; ac->acb.second = cb_data; - + return ac; - + DECODER_CLEANUP: opus_decoder_destroy(ac->decoder); jbuf_free(ac->j_buf); @@ -111,39 +106,41 @@ BASE_CLEANUP: free(ac); return NULL; } -void ac_kill(ACSession* ac) +void ac_kill(ACSession *ac) { if (!ac) return; - + opus_encoder_destroy(ac->encoder); - opus_encoder_destroy(ac->test_encoder); opus_decoder_destroy(ac->decoder); jbuf_free(ac->j_buf); - + pthread_mutex_destroy(ac->queue_mutex); - + LOGGER_DEBUG("Terminated audio handler: %p", ac); free(ac); } -void ac_do(ACSession* ac) +void ac_iterate(ACSession *ac) { if (!ac) return; + + /* TODO fix this and jitter buffering */ - /* Enough space for the maximum frame size (120 ms 48 KHz audio) */ + /* Enough space for the maximum frame size (120 ms 48 KHz stereo audio) */ int16_t tmp[5760 * 2]; - - RTPMessage *msg; + + struct RTPMessage *msg; int rc = 0; - + pthread_mutex_lock(ac->queue_mutex); + while ((msg = jbuf_read(ac->j_buf, &rc)) || rc == 2) { pthread_mutex_unlock(ac->queue_mutex); - + if (rc == 2) { LOGGER_DEBUG("OPUS correction"); - int fs = (ac->last_packet_sampling_rate * ac->last_packet_frame_duration) / 1000; + int fs = (ac->lp_sampling_rate * ac->lp_frame_duration) / 1000; rc = opus_decode(ac->decoder, NULL, 0, tmp, fs, 1); } else { /* Get values from packet and decode. */ @@ -156,98 +153,93 @@ void ac_do(ACSession* ac) rtp_free_msg(msg); continue; }*/ - - + + /* Pick up sampling rate from packet */ - memcpy(&ac->last_packet_sampling_rate, msg->data, 4); - ac->last_packet_sampling_rate = ntohl(ac->last_packet_sampling_rate); - - ac->last_packet_channel_count = opus_packet_get_nb_channels(msg->data + 4); - + memcpy(&ac->lp_sampling_rate, msg->data, 4); + ac->lp_sampling_rate = ntohl(ac->lp_sampling_rate); + + ac->lp_channel_count = opus_packet_get_nb_channels(msg->data + 4); + /** NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa, * it didn't work quite well. */ - if (!reconfigure_audio_decoder(ac, ac->last_packet_sampling_rate, ac->last_packet_channel_count)) { + if (!reconfigure_audio_decoder(ac, ac->lp_sampling_rate, ac->lp_channel_count)) { LOGGER_WARNING("Failed to reconfigure decoder!"); - rtp_free_msg(msg); + free(msg); continue; } - - rc = opus_decode(ac->decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0); - rtp_free_msg(msg); + + rc = opus_decode(ac->decoder, msg->data + 4, msg->len - 4, tmp, 5760, 0); + free(msg); } - + if (rc < 0) { LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); } else if (ac->acb.first) { - ac->last_packet_frame_duration = (rc * 1000) / ac->last_packet_sampling_rate; - - ac->acb.first(ac->av, ac->friend_number, tmp, rc, ac->last_packet_channel_count, - ac->last_packet_sampling_rate, ac->acb.second); + ac->lp_frame_duration = (rc * 1000) / ac->lp_sampling_rate; + + ac->acb.first(ac->av, ac->friend_number, tmp, rc, ac->lp_channel_count, + ac->lp_sampling_rate, ac->acb.second); } - + return; } + pthread_mutex_unlock(ac->queue_mutex); } -int ac_queue_message(void* acp, struct RTPMessage_s *msg) +int ac_queue_message(void *acp, struct RTPMessage *msg) { if (!acp || !msg) return -1; - - if ((msg->header->marker_payloadt & 0x7f) == (rtp_TypeAudio + 2) % 128) { + + if ((msg->header.pt & 0x7f) == (rtp_TypeAudio + 2) % 128) { LOGGER_WARNING("Got dummy!"); - rtp_free_msg(msg); + free(msg); return 0; } - - if ((msg->header->marker_payloadt & 0x7f) != rtp_TypeAudio % 128) { + + if ((msg->header.pt & 0x7f) != rtp_TypeAudio % 128) { LOGGER_WARNING("Invalid payload type!"); - rtp_free_msg(msg); + free(msg); return -1; } - - ACSession* ac = acp; - + + ACSession *ac = acp; + pthread_mutex_lock(ac->queue_mutex); int rc = jbuf_write(ac->j_buf, msg); pthread_mutex_unlock(ac->queue_mutex); - + if (rc == -1) { LOGGER_WARNING("Could not queue the message!"); - rtp_free_msg(msg); + free(msg); return -1; } - + return 0; } -int ac_reconfigure_encoder(ACSession* ac, int32_t bit_rate, int32_t sampling_rate, uint8_t channels) +int ac_reconfigure_encoder(ACSession *ac, int32_t bit_rate, int32_t sampling_rate, uint8_t channels) { - if (!ac || !reconfigure_audio_encoder(&ac->encoder, bit_rate, sampling_rate, channels, - &ac->last_encoding_bit_rate, &ac->last_encoding_sampling_rate, &ac->last_encoding_channel_count)) + if (!ac || !reconfigure_audio_encoder(&ac->encoder, bit_rate, + sampling_rate, channels, + &ac->le_bit_rate, + &ac->le_sample_rate, + &ac->le_channel_count)) return -1; - + LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bit_rate, sampling_rate, channels); return 0; } -int ac_reconfigure_test_encoder(ACSession* ac, int32_t bit_rate, int32_t sampling_rate, uint8_t channels) -{ - if (!ac || !reconfigure_audio_encoder(&ac->test_encoder, bit_rate, sampling_rate, channels, - &ac->last_encoding_bit_rate, &ac->last_encoding_sampling_rate, &ac->last_encoding_channel_count)) - return -1; - - LOGGER_DEBUG ("Reconfigured test audio encoder br: %d sr: %d cc:%d", bit_rate, sampling_rate, channels); - return 0; -} struct JitterBuffer { - RTPMessage **queue; - uint32_t size; - uint32_t capacity; - uint16_t bottom; - uint16_t top; + struct RTPMessage **queue; + uint32_t size; + uint32_t capacity; + uint16_t bottom; + uint16_t top; }; static struct JitterBuffer *jbuf_new(uint32_t capacity) @@ -260,9 +252,9 @@ static struct JitterBuffer *jbuf_new(uint32_t capacity) struct JitterBuffer *q; - if ( !(q = calloc(sizeof(struct JitterBuffer), 1)) ) return NULL; + if (!(q = calloc(sizeof(struct JitterBuffer), 1))) return NULL; - if (!(q->queue = calloc(sizeof(RTPMessage *), size))) { + if (!(q->queue = calloc(sizeof(struct RTPMessage *), size))) { free(q); return NULL; } @@ -275,7 +267,7 @@ static void jbuf_clear(struct JitterBuffer *q) { for (; q->bottom != q->top; ++q->bottom) { if (q->queue[q->bottom % q->size]) { - rtp_free_msg(q->queue[q->bottom % q->size]); + free(q->queue[q->bottom % q->size]); q->queue[q->bottom % q->size] = NULL; } } @@ -288,15 +280,15 @@ static void jbuf_free(struct JitterBuffer *q) free(q->queue); free(q); } -static int jbuf_write(struct JitterBuffer *q, RTPMessage *m) +static int jbuf_write(struct JitterBuffer *q, struct RTPMessage *m) { - uint16_t sequnum = m->header->sequnum; + uint16_t sequnum = m->header.sequnum; unsigned int num = sequnum % q->size; if ((uint32_t)(sequnum - q->bottom) > q->size) { LOGGER_DEBUG("Clearing filled jitter buffer: %p", q); - + jbuf_clear(q); q->bottom = sequnum - q->capacity; q->queue[num] = m; @@ -314,7 +306,7 @@ static int jbuf_write(struct JitterBuffer *q, RTPMessage *m) return 0; } -static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success) +static struct RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success) { if (q->top == q->bottom) { *success = 0; @@ -324,7 +316,7 @@ static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success) unsigned int num = q->bottom % q->size; if (q->queue[num]) { - RTPMessage *ret = q->queue[num]; + struct RTPMessage *ret = q->queue[num]; q->queue[num] = NULL; ++q->bottom; *success = 1; @@ -340,73 +332,74 @@ static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success) *success = 0; return NULL; } -OpusEncoder* create_audio_encoder (int32_t bit_rate, int32_t sampling_rate, int32_t channel_count) +OpusEncoder *create_audio_encoder (int32_t bit_rate, int32_t sampling_rate, int32_t channel_count) { int status = OPUS_OK; - OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_VOIP, &status); - - if ( status != OPUS_OK ) { + OpusEncoder *rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_VOIP, &status); + + if (status != OPUS_OK) { LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(status)); return NULL; } - + status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bit_rate)); - - if ( status != OPUS_OK ) { + + if (status != OPUS_OK) { LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); goto FAILURE; } - + /* Enable in-band forward error correction in codec */ status = opus_encoder_ctl(rc, OPUS_SET_INBAND_FEC(1)); - if ( status != OPUS_OK ) { + if (status != OPUS_OK) { LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); goto FAILURE; } - + /* Make codec resistant to up to 10% packet loss * NOTE This could also be adjusted on the fly, rather than hard-coded, * with feedback from the receiving client. */ status = opus_encoder_ctl(rc, OPUS_SET_PACKET_LOSS_PERC(10)); - if ( status != OPUS_OK ) { + if (status != OPUS_OK) { LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); goto FAILURE; } - + /* Set algorithm to the highest complexity, maximizing compression */ status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(10)); - - if ( status != OPUS_OK ) { + + if (status != OPUS_OK) { LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); goto FAILURE; } - + return rc; - + FAILURE: opus_encoder_destroy(rc); return NULL; } -bool reconfigure_audio_encoder(OpusEncoder** e, int32_t new_br, int32_t new_sr, uint8_t new_ch, - int32_t* old_br, int32_t* old_sr, int32_t* old_ch) +bool reconfigure_audio_encoder(OpusEncoder **e, int32_t new_br, int32_t new_sr, uint8_t new_ch, + int32_t *old_br, int32_t *old_sr, int32_t *old_ch) { /* Values are checked in toxav.c */ if (*old_sr != new_sr || *old_ch != new_ch) { - OpusEncoder* new_encoder = create_audio_encoder(new_br, new_sr, new_ch); + OpusEncoder *new_encoder = create_audio_encoder(new_br, new_sr, new_ch); + if (new_encoder == NULL) return false; - + opus_encoder_destroy(*e); *e = new_encoder; } else if (*old_br == new_br) return true; /* Nothing changed */ else { int status = opus_encoder_ctl(*e, OPUS_SET_BITRATE(new_br)); - - if ( status != OPUS_OK ) { + + if (status != OPUS_OK) { LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); return false; } @@ -415,31 +408,32 @@ bool reconfigure_audio_encoder(OpusEncoder** e, int32_t new_br, int32_t new_sr, *old_br = new_br; *old_sr = new_sr; *old_ch = new_ch; - + return true; } -bool reconfigure_audio_decoder(ACSession* ac, int32_t sampling_rate, int8_t channels) +bool reconfigure_audio_decoder(ACSession *ac, int32_t sampling_rate, int8_t channels) { - if (sampling_rate != ac->last_decoding_sampling_rate || channels != ac->last_decoding_channel_count) { - if (current_time_monotonic() - ac->last_decoder_reconfiguration < 500) + if (sampling_rate != ac->ld_sample_rate || channels != ac->ld_channel_count) { + if (current_time_monotonic() - ac->ldrts < 500) return false; - + int status; - OpusDecoder* new_dec = opus_decoder_create(sampling_rate, channels, &status ); - if ( status != OPUS_OK ) { + OpusDecoder *new_dec = opus_decoder_create(sampling_rate, channels, &status); + + if (status != OPUS_OK) { LOGGER_ERROR("Error while starting audio decoder(%d %d): %s", sampling_rate, channels, opus_strerror(status)); return false; } - - ac->last_decoding_sampling_rate = sampling_rate; - ac->last_decoding_channel_count = channels; - ac->last_decoder_reconfiguration = current_time_monotonic(); - + + ac->ld_sample_rate = sampling_rate; + ac->ld_channel_count = channels; + ac->ldrts = current_time_monotonic(); + opus_decoder_destroy(ac->decoder); ac->decoder = new_dec; - + LOGGER_DEBUG("Reconfigured audio decoder sr: %d cc: %d", sampling_rate, channels); } - + return true; -} \ No newline at end of file +} diff --git a/toxav/audio.h b/toxav/audio.h index 9ef10ae40..b1db7448a 100644 --- a/toxav/audio.h +++ b/toxav/audio.h @@ -29,61 +29,36 @@ #include "../toxcore/util.h" -struct RTPMessage_s; +struct RTPMessage; -/* - * Base Audio Codec session type. - */ typedef struct ACSession_s { /* encoding */ OpusEncoder *encoder; - int32_t last_encoding_sampling_rate; - int32_t last_encoding_channel_count; - int32_t last_encoding_bit_rate; - - /* Testing encoder for dynamic bit rate streaming */ - OpusEncoder *test_encoder; - int32_t last_test_encoding_sampling_rate; - int32_t last_test_encoding_channel_count; - int32_t last_test_encoding_bit_rate; - + int32_t le_sample_rate; /* Last encoder sample rate */ + int32_t le_channel_count; /* Last encoder channel count */ + int32_t le_bit_rate; /* Last encoder bit rate */ + /* decoding */ OpusDecoder *decoder; - int32_t last_packet_channel_count; - int32_t last_packet_sampling_rate; - int32_t last_packet_frame_duration; - int32_t last_decoding_sampling_rate; - int32_t last_decoding_channel_count; - uint64_t last_decoder_reconfiguration; + int32_t lp_channel_count; /* Last packet channel count */ + int32_t lp_sampling_rate; /* Last packet sample rate */ + int32_t lp_frame_duration; /* Last packet frame duration */ + int32_t ld_sample_rate; /* Last decoder sample rate */ + int32_t ld_channel_count; /* Last decoder channel count */ + uint64_t ldrts; /* Last decoder reconfiguration time stamp */ void *j_buf; - + pthread_mutex_t queue_mutex[1]; - - ToxAV* av; + + ToxAV *av; uint32_t friend_number; PAIR(toxav_audio_receive_frame_cb *, void *) acb; /* Audio frame receive callback */ } ACSession; -/* - * Create new Audio Codec session. - */ -ACSession* ac_new(ToxAV* av, uint32_t friend_number, toxav_audio_receive_frame_cb *cb, void *cb_data); -/* - * Kill the Audio Codec session. - */ -void ac_kill(ACSession* ac); -/* - * Do periodic work. Work is consisted out of decoding only. - */ -void ac_do(ACSession* ac); -/* - * Queue new rtp message. - */ -int ac_queue_message(void *acp, struct RTPMessage_s *msg); -/* - * Set new values to the encoders. - */ -int ac_reconfigure_encoder(ACSession* ac, int32_t bit_rate, int32_t sampling_rate, uint8_t channels); -int ac_reconfigure_test_encoder(ACSession* ac, int32_t bit_rate, int32_t sampling_rate, uint8_t channels); +ACSession *ac_new(ToxAV *av, uint32_t friend_number, toxav_audio_receive_frame_cb *cb, void *cb_data); +void ac_kill(ACSession *ac); +void ac_iterate(ACSession *ac); +int ac_queue_message(void *acp, struct RTPMessage *msg); +int ac_reconfigure_encoder(ACSession *ac, int32_t bit_rate, int32_t sampling_rate, uint8_t channels); -#endif /* AUDIO_H */ \ No newline at end of file +#endif /* AUDIO_H */ diff --git a/toxav/bwcontroler.c b/toxav/bwcontroler.c new file mode 100644 index 000000000..2c468ce3f --- /dev/null +++ b/toxav/bwcontroler.c @@ -0,0 +1,207 @@ +/** bwcontroler.c + * + * Copyright (C) 2013-2015 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif /* HAVE_CONFIG_H */ + +#include +#include "bwcontroler.h" +#include "../toxcore/logger.h" +#include "../toxcore/util.h" + +#define BWC_PACKET_ID 196 +#define BWC_SEND_INTERVAL_MS 1000 +#define BWC_REFRESH_INTERVAL_MS 10000 +#define BWC_AVG_PKT_COUNT 20 + +/** + * + */ + +struct BWControler_s { + void (*mcb) (BWControler *, uint32_t, float, void *); + void *mcb_data; + + Messenger *m; + uint32_t friend_number; + + struct { + uint32_t lru; /* Last recv update time stamp */ + uint32_t lsu; /* Last sent update time stamp */ + uint32_t lfu; /* Last refresh time stamp */ + + uint32_t lost; + uint32_t recv; + } cycle; + + struct { + uint32_t rb_s[BWC_AVG_PKT_COUNT]; + RingBuffer *rb; + } rcvpkt; /* To calculate average received packet */ +}; + +int bwc_handle_data(Messenger *m, uint32_t friendnumber, const uint8_t *data, uint16_t length, void *object); +void send_update(BWControler *bwc); + +BWControler *bwc_new(Messenger *m, uint32_t friendnumber, + void (*mcb) (BWControler *, uint32_t, float, void *), + void *udata) +{ + BWControler *retu = calloc(sizeof(struct BWControler_s), 1); + + retu->mcb = mcb; + retu->mcb_data = udata; + retu->m = m; + retu->friend_number = friendnumber; + retu->cycle.lsu = retu->cycle.lfu = current_time_monotonic(); + retu->rcvpkt.rb = rb_new(BWC_AVG_PKT_COUNT); + + /* Fill with zeros */ + int i = 0; + for (; i < BWC_AVG_PKT_COUNT; i ++) + rb_write(retu->rcvpkt.rb, retu->rcvpkt.rb_s + i); + + m_callback_rtp_packet(m, friendnumber, BWC_PACKET_ID, bwc_handle_data, retu); + + return retu; +} +void bwc_kill(BWControler *bwc) +{ + if (!bwc) + return; + + m_callback_rtp_packet(bwc->m, bwc->friend_number, BWC_PACKET_ID, NULL, NULL); + + rb_kill(bwc->rcvpkt.rb); + free(bwc); +} +void bwc_feed_avg(BWControler* bwc, uint32_t bytes) +{ + uint32_t *p; + + rb_read(bwc->rcvpkt.rb, (void**) &p); + rb_write(bwc->rcvpkt.rb, p); + + *p = bytes; +} +void bwc_add_lost(BWControler *bwc, uint32_t bytes) +{ + if (!bwc) + return; + + if (!bytes) { + uint32_t* t_avg[BWC_AVG_PKT_COUNT], c = 1; + + rb_data(bwc->rcvpkt.rb, (void**) t_avg); + + int i = 0; + for (; i < BWC_AVG_PKT_COUNT; i ++) { + bytes += *(t_avg[i]); + + if (*(t_avg[i])) + c++; + } + + bytes /= c; + } + + bwc->cycle.lost += bytes; + send_update(bwc); +} +void bwc_add_recv(BWControler *bwc, uint32_t bytes) +{ + if (!bwc || !bytes) + return; + + bwc->cycle.recv += bytes; + send_update(bwc); +} + + +struct BWCMessage { + uint8_t core_type; /* Aligner for payload type which is always 196 */ + + uint32_t lost; + uint32_t recv; +} __attribute__((packed)); + +/* Check alignment */ +typedef char __fail_if_misaligned [ sizeof(struct BWCMessage) == 9 ? 1 : -1 ]; + +void send_update(BWControler *bwc) +{ + if (current_time_monotonic() - bwc->cycle.lfu > BWC_REFRESH_INTERVAL_MS) { + + bwc->cycle.lost /= 10; + bwc->cycle.recv /= 10; + bwc->cycle.lfu = current_time_monotonic(); + } + else if (current_time_monotonic() - bwc->cycle.lsu > BWC_SEND_INTERVAL_MS) { + + if (bwc->cycle.lost) + { + LOGGER_DEBUG ("%p Sent update", bwc); + + struct BWCMessage msg; + msg.core_type = BWC_PACKET_ID; + msg.lost = htonl(bwc->cycle.lost); + msg.recv = htonl(bwc->cycle.recv); + + if (-1 == send_custom_lossy_packet(bwc->m, bwc->friend_number, (uint8_t *)&msg, sizeof(msg))) + LOGGER_WARNING("BWC send failed (len: %d)! std error: %s", sizeof(msg), strerror(errno)); + } + + bwc->cycle.lsu = current_time_monotonic(); + } +} +int on_update (BWControler *bwc, struct BWCMessage *msg) +{ + LOGGER_DEBUG ("%p Got update from peer", bwc); + + /* Peer must respect time boundary */ + if (current_time_monotonic() < bwc->cycle.lru + BWC_SEND_INTERVAL_MS) { + LOGGER_DEBUG("%p Rejecting extra update", bwc); + return -1; + } + + bwc->cycle.lru = current_time_monotonic(); + + msg->recv = ntohl(msg->recv); + msg->lost = ntohl(msg->lost); + + LOGGER_DEBUG ("recved: %u lost: %u", msg->recv, msg->lost); + + if (msg->lost && bwc->mcb) + bwc->mcb(bwc, bwc->friend_number, + ((float) (msg->lost) / (msg->recv + msg->lost)), + bwc->mcb_data); + + return 0; +} +int bwc_handle_data(Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object) +{ + if (length != sizeof(struct BWCMessage)) + return; + + /* NOTE the data is mutable */ + return on_update(object, (struct BWCMessage *) data); +} diff --git a/toxav/bwcontroler.h b/toxav/bwcontroler.h new file mode 100644 index 000000000..53b07d380 --- /dev/null +++ b/toxav/bwcontroler.h @@ -0,0 +1,37 @@ +/** bwcontroler.h + * + * Copyright (C) 2013-2015 Tox project All Rights Reserved. + * + * This file is part of Tox. + * + * Tox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Tox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tox. If not, see . + * + */ + +#ifndef BWCONROLER_H +#define BWCONROLER_H +#include "../toxcore/Messenger.h" + +typedef struct BWControler_s BWControler; + +BWControler *bwc_new(Messenger *m, uint32_t friendnumber, + void (*mcb) (BWControler *, uint32_t, float, void *), + void *udata); +void bwc_kill(BWControler *bwc); + +void bwc_feed_avg(BWControler *bwc, uint32_t bytes); +void bwc_add_lost(BWControler *bwc, uint32_t bytes); +void bwc_add_recv(BWControler *bwc, uint32_t bytes); + +#endif /* BWCONROLER_H */ diff --git a/toxav/group.c b/toxav/group.c index 817ee6e69..190c2c3d7 100644 --- a/toxav/group.c +++ b/toxav/group.c @@ -20,7 +20,7 @@ #ifdef HAVE_CONFIG_H #include "config.h" -#endif +#endif /* HAVE_CONFIG_H */ #include "group.h" #include "../toxcore/util.h" @@ -54,7 +54,7 @@ static Group_JitterBuffer *create_queue(unsigned int capacity) Group_JitterBuffer *q; - if ( !(q = calloc(sizeof(Group_JitterBuffer), 1)) ) return NULL; + if (!(q = calloc(sizeof(Group_JitterBuffer), 1))) return NULL; if (!(q->queue = calloc(sizeof(Group_Audio_Packet *), size))) { free(q); @@ -190,7 +190,7 @@ static int recreate_encoder(Group_AV *group_av) group_av->audio_encoder = opus_encoder_create(group_av->audio_sample_rate, group_av->audio_channels, OPUS_APPLICATION_AUDIO, &rc); - if ( rc != OPUS_OK ) { + if (rc != OPUS_OK) { LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); group_av->audio_encoder = NULL; return -1; @@ -198,7 +198,7 @@ static int recreate_encoder(Group_AV *group_av) rc = opus_encoder_ctl(group_av->audio_encoder, OPUS_SET_BITRATE(group_av->audio_bitrate)); - if ( rc != OPUS_OK ) { + if (rc != OPUS_OK) { LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); opus_encoder_destroy(group_av->audio_encoder); group_av->audio_encoder = NULL; @@ -207,7 +207,7 @@ static int recreate_encoder(Group_AV *group_av) rc = opus_encoder_ctl(group_av->audio_encoder, OPUS_SET_COMPLEXITY(10)); - if ( rc != OPUS_OK ) { + if (rc != OPUS_OK) { LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); opus_encoder_destroy(group_av->audio_encoder); group_av->audio_encoder = NULL; @@ -306,7 +306,7 @@ static int decode_audio_packet(Group_AV *group_av, Group_Peer_AV *peer_av, int g int rc; peer_av->audio_decoder = opus_decoder_create(sample_rate, channels, &rc); - if ( rc != OPUS_OK ) { + if (rc != OPUS_OK) { LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc)); free(pk); return -1; diff --git a/toxav/msi.c b/toxav/msi.c index b262e9a06..ef307bcbd 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -46,12 +46,12 @@ typedef enum { IDRequest = 1, IDError, IDCapabilities, - IDVFPSZ, } MSIHeaderID; typedef enum { + requ_init, requ_push, requ_pop, } MSIRequest; @@ -64,224 +64,246 @@ typedef struct { \ } MSIHeader##header -GENERIC_HEADER ( Request, MSIRequest ); -GENERIC_HEADER ( Error, MSIError ); -GENERIC_HEADER ( Capabilities, uint8_t ); -GENERIC_HEADER ( VFPSZ, uint16_t ); +GENERIC_HEADER (Request, MSIRequest); +GENERIC_HEADER (Error, MSIError); +GENERIC_HEADER (Capabilities, uint8_t); typedef struct { MSIHeaderRequest request; MSIHeaderError error; MSIHeaderCapabilities capabilities; - MSIHeaderVFPSZ vfpsz; /* Video frame piece size. NOTE: Value must be in network b-order TODO: get rid of this eventually */ } MSIMessage; void msg_init (MSIMessage *dest, MSIRequest request); -int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ); -uint8_t *msg_parse_header_out ( MSIHeaderID id, uint8_t *dest, const void *value, uint8_t value_len, uint16_t *length ); -static int send_message ( Messenger* m, uint32_t friend_number, const MSIMessage *msg ); -int send_error ( Messenger* m, uint32_t friend_number, MSIError error ); -static int invoke_callback(MSICall* call, MSICallbackID cb); -static MSICall *get_call ( MSISession *session, uint32_t friend_number ); -MSICall *new_call ( MSISession *session, uint32_t friend_number ); -void kill_call ( MSICall *call ); +int msg_parse_in (MSIMessage *dest, const uint8_t *data, uint16_t length); +uint8_t *msg_parse_header_out (MSIHeaderID id, uint8_t *dest, const void *value, uint8_t value_len, uint16_t *length); +static int send_message (Messenger *m, uint32_t friend_number, const MSIMessage *msg); +int send_error (Messenger *m, uint32_t friend_number, MSIError error); +static int invoke_callback(MSICall *call, MSICallbackID cb); +static MSICall *get_call (MSISession *session, uint32_t friend_number); +MSICall *new_call (MSISession *session, uint32_t friend_number); +void kill_call (MSICall *call); void on_peer_status(Messenger *m, uint32_t friend_number, uint8_t status, void *data); -void handle_push ( MSICall *call, const MSIMessage *msg ); -void handle_pop ( MSICall *call, const MSIMessage *msg ); -void handle_msi_packet ( Messenger *m, uint32_t friend_number, const uint8_t *data, uint16_t length, void *object ); +void handle_init (MSICall *call, const MSIMessage *msg); +void handle_push (MSICall *call, const MSIMessage *msg); +void handle_pop (MSICall *call, const MSIMessage *msg); +void handle_msi_packet (Messenger *m, uint32_t friend_number, const uint8_t *data, uint16_t length, void *object); /** * Public functions */ -void msi_register_callback ( MSISession* session, msi_action_cb* callback, MSICallbackID id) +void msi_register_callback (MSISession *session, msi_action_cb *callback, MSICallbackID id) { + if (!session) + return; + pthread_mutex_lock(session->mutex); session->callbacks[id] = callback; pthread_mutex_unlock(session->mutex); } -MSISession *msi_new ( Messenger *m ) +MSISession *msi_new (Messenger *m) { if (m == NULL) { LOGGER_ERROR("Could not init session on empty messenger!"); return NULL; } - - MSISession *retu = calloc ( sizeof ( MSISession ), 1 ); - + + MSISession *retu = calloc (sizeof (MSISession), 1); + if (retu == NULL) { LOGGER_ERROR("Allocation failed! Program might misbehave!"); return NULL; } - + if (create_recursive_mutex(retu->mutex) != 0) { LOGGER_ERROR("Failed to init mutex! Program might misbehave"); free(retu); return NULL; } - + retu->messenger = m; - - m_callback_msi_packet(m, handle_msi_packet, retu ); - + + m_callback_msi_packet(m, handle_msi_packet, retu); + /* This is called when remote terminates session */ m_callback_connectionstatus_internal_av(m, on_peer_status, retu); - + LOGGER_DEBUG("New msi session: %p ", retu); return retu; } -int msi_kill ( MSISession *session ) +int msi_kill (MSISession *session) { if (session == NULL) { LOGGER_ERROR("Tried to terminate non-existing session"); return -1; } - + m_callback_msi_packet((struct Messenger *) session->messenger, NULL, NULL); - pthread_mutex_lock(session->mutex); + if (pthread_mutex_trylock(session->mutex) != 0) { + LOGGER_ERROR ("Failed to aquire lock on msi mutex"); + return -1; + } + if (session->calls) { MSIMessage msg; msg_init(&msg, requ_pop); - - MSICall* it = get_call(session, session->calls_head); + + MSICall *it = get_call(session, session->calls_head); + for (; it; it = it->next) { send_message(session->messenger, it->friend_number, &msg); kill_call(it); /* This will eventually free session->calls */ } } - + pthread_mutex_unlock(session->mutex); pthread_mutex_destroy(session->mutex); - + LOGGER_DEBUG("Terminated session: %p", session); - free ( session ); + free (session); return 0; } -int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_number, uint8_t capabilities ) +int msi_invite (MSISession *session, MSICall **call, uint32_t friend_number, uint8_t capabilities) { - LOGGER_DEBUG("Session: %p Inviting friend: %u", session, friend_number); + if (!session) + return -1; - pthread_mutex_lock(session->mutex); + LOGGER_DEBUG("Session: %p Inviting friend: %u", session, friend_number); + + if (pthread_mutex_trylock(session->mutex) != 0) { + LOGGER_ERROR ("Failed to aquire lock on msi mutex"); + return -1; + } + if (get_call(session, friend_number) != NULL) { LOGGER_ERROR("Already in a call"); pthread_mutex_unlock(session->mutex); return -1; } - - (*call) = new_call ( session, friend_number ); - - if ( *call == NULL ) { + + (*call) = new_call (session, friend_number); + + if (*call == NULL) { pthread_mutex_unlock(session->mutex); return -1; } - + (*call)->self_capabilities = capabilities; - + MSIMessage msg; - msg_init(&msg, requ_push); - + msg_init(&msg, requ_init); + msg.capabilities.exists = true; msg.capabilities.value = capabilities; - - msg.vfpsz.exists = true; - msg.vfpsz.value = VIDEOFRAME_PIECE_SIZE; - - send_message ( (*call)->session->messenger, (*call)->friend_number, &msg ); - + + send_message ((*call)->session->messenger, (*call)->friend_number, &msg); + (*call)->state = msi_CallRequesting; - + LOGGER_DEBUG("Invite sent"); pthread_mutex_unlock(session->mutex); return 0; } -int msi_hangup ( MSICall* call ) +int msi_hangup (MSICall *call) { + if (!call || !call->session) + return -1; + LOGGER_DEBUG("Session: %p Hanging up call with friend: %u", call->session, call->friend_number); + + MSISession *session = call->session; - MSISession* session = call->session; - pthread_mutex_lock(session->mutex); - - if ( call->state == msi_CallInactive ) { + if (pthread_mutex_trylock(session->mutex) != 0) { + LOGGER_ERROR ("Failed to aquire lock on msi mutex"); + return -1; + } + + if (call->state == msi_CallInactive) { LOGGER_ERROR("Call is in invalid state!"); pthread_mutex_unlock(session->mutex); return -1; } - + MSIMessage msg; msg_init(&msg, requ_pop); - - send_message ( session->messenger, call->friend_number, &msg ); - + + send_message (session->messenger, call->friend_number, &msg); + kill_call(call); pthread_mutex_unlock(session->mutex); return 0; } -int msi_answer ( MSICall* call, uint8_t capabilities ) +int msi_answer (MSICall *call, uint8_t capabilities) { + if (!call || !call->session) + return -1; + LOGGER_DEBUG("Session: %p Answering call from: %u", call->session, call->friend_number); - - MSISession* session = call->session; - pthread_mutex_lock(session->mutex); - - if ( call->state != msi_CallRequested ) { - /* Though sending in invalid state will not cause anything wierd + + MSISession *session = call->session; + + if (pthread_mutex_trylock(session->mutex) != 0) { + LOGGER_ERROR ("Failed to aquire lock on msi mutex"); + return -1; + } + + if (call->state != msi_CallRequested) { + /* Though sending in invalid state will not cause anything wierd * Its better to not do it like a maniac */ LOGGER_ERROR("Call is in invalid state!"); pthread_mutex_unlock(session->mutex); return -1; } - + call->self_capabilities = capabilities; - + MSIMessage msg; msg_init(&msg, requ_push); - + msg.capabilities.exists = true; msg.capabilities.value = capabilities; - - msg.vfpsz.exists = true; - msg.vfpsz.value = VIDEOFRAME_PIECE_SIZE; - - send_message ( session->messenger, call->friend_number, &msg ); - + + send_message (session->messenger, call->friend_number, &msg); + call->state = msi_CallActive; pthread_mutex_unlock(session->mutex); - + return 0; } -int msi_change_capabilities( MSICall* call, uint8_t capabilities ) +int msi_change_capabilities(MSICall *call, uint8_t capabilities) { + if (!call || !call->session) + return -1; + LOGGER_DEBUG("Session: %p Trying to change capabilities to friend %u", call->session, call->friend_number); - - MSISession* session = call->session; - pthread_mutex_lock(session->mutex); - - if ( call->state != msi_CallActive ) { - /* Sending capabilities change can cause error on other side if - * the call is not active since we don't send header 'vfpsz'. - * If we were to send 'vfpsz' while call is active it would be - * ignored. However, if call is not active peer will expect - * the said header on 'push' so that it could handle the call - * like new. TODO: explain this better - */ + + MSISession *session = call->session; + + if (pthread_mutex_trylock(session->mutex) != 0) { + LOGGER_ERROR ("Failed to aquire lock on msi mutex"); + return -1; + } + + if (call->state != msi_CallActive) { LOGGER_ERROR("Call is in invalid state!"); pthread_mutex_unlock(session->mutex); return -1; } - + call->self_capabilities = capabilities; - + MSIMessage msg; msg_init(&msg, requ_push); - + msg.capabilities.exists = true; msg.capabilities.value = capabilities; - - send_message ( call->session->messenger, call->friend_number, &msg ); - + + send_message (call->session->messenger, call->friend_number, &msg); + pthread_mutex_unlock(session->mutex); return 0; } @@ -290,23 +312,23 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) /** * Private functions */ -void msg_init(MSIMessage* dest, MSIRequest request) +void msg_init(MSIMessage *dest, MSIRequest request) { memset(dest, 0, sizeof(*dest)); dest->request.exists = true; dest->request.value = request; } -int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) +int msg_parse_in (MSIMessage *dest, const uint8_t *data, uint16_t length) { /* Parse raw data received from socket into MSIMessage struct */ - + #define CHECK_SIZE(bytes, constraint, size) \ if ((constraint -= (2 + size)) < 1) { LOGGER_ERROR("Read over length!"); return -1; } \ - if ( bytes[1] != size ) { LOGGER_ERROR("Invalid data size!"); return -1; } - + if (bytes[1] != size) { LOGGER_ERROR("Invalid data size!"); return -1; } + #define CHECK_ENUM_HIGH(bytes, enum_high) /* Assumes size == 1 */ \ - if ( bytes[2] > enum_high ) { LOGGER_ERROR("Failed enum high limit!"); return -1; } - + if (bytes[2] > enum_high) { LOGGER_ERROR("Failed enum high limit!"); return -1; } + #define SET_UINT8(bytes, header) do { \ header.value = bytes[2]; \ header.exists = true; \ @@ -318,50 +340,39 @@ int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) header.exists = true; \ bytes += 4; \ } while(0) - - + + assert(dest); - if ( length == 0 || data[length - 1] ) { /* End byte must have value 0 */ + if (length == 0 || data[length - 1]) { /* End byte must have value 0 */ LOGGER_ERROR("Invalid end byte"); return -1; } - + memset(dest, 0, sizeof(*dest)); - + const uint8_t *it = data; int size_constraint = length; - while ( *it ) {/* until end byte is hit */ + while (*it) {/* until end byte is hit */ switch (*it) { case IDRequest: CHECK_SIZE(it, size_constraint, 1); CHECK_ENUM_HIGH(it, requ_pop); SET_UINT8(it, dest->request); break; - + case IDError: CHECK_SIZE(it, size_constraint, 1); CHECK_ENUM_HIGH(it, msi_EUndisclosed); SET_UINT8(it, dest->error); break; - + case IDCapabilities: CHECK_SIZE(it, size_constraint, 1); SET_UINT8(it, dest->capabilities); break; - - case IDVFPSZ: - CHECK_SIZE(it, size_constraint, 2); - SET_UINT16(it, dest->vfpsz); - dest->vfpsz.value = ntohs(dest->vfpsz.value); - - if (dest->vfpsz.value > 1200) { - LOGGER_ERROR("Invalid vfpsz param"); - return -1; - } - break; - + default: LOGGER_ERROR("Invalid id byte"); return -1; @@ -373,7 +384,7 @@ int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) LOGGER_ERROR("Invalid request field!"); return -1; } - + return 0; #undef CHECK_SIZE @@ -381,13 +392,13 @@ int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) #undef SET_UINT8 #undef SET_UINT16 } -uint8_t *msg_parse_header_out ( MSIHeaderID id, uint8_t *dest, const void *value, uint8_t value_len, uint16_t *length ) +uint8_t *msg_parse_header_out (MSIHeaderID id, uint8_t *dest, const void *value, uint8_t value_len, uint16_t *length) { /* Parse a single header for sending */ assert(dest); assert(value); assert(value_len); - + *dest = id; dest ++; *dest = value_len; @@ -399,208 +410,205 @@ uint8_t *msg_parse_header_out ( MSIHeaderID id, uint8_t *dest, const void *value return dest + value_len; /* Set to next position ready to be written */ } -int send_message ( Messenger* m, uint32_t friend_number, const MSIMessage *msg ) +int send_message (Messenger *m, uint32_t friend_number, const MSIMessage *msg) { /* Parse and send message */ assert(m); - + uint8_t parsed [MSI_MAXMSG_SIZE]; uint8_t *it = parsed; uint16_t size = 0; - + if (msg->request.exists) { uint8_t cast = msg->request.value; - it = msg_parse_header_out(IDRequest, it, &cast, + it = msg_parse_header_out(IDRequest, it, &cast, sizeof(cast), &size); } else { LOGGER_DEBUG("Must have request field"); return -1; } - + if (msg->error.exists) { uint8_t cast = msg->error.value; - it = msg_parse_header_out(IDError, it, &cast, + it = msg_parse_header_out(IDError, it, &cast, sizeof(cast), &size); } - + if (msg->capabilities.exists) { - it = msg_parse_header_out(IDCapabilities, it, &msg->capabilities.value, + it = msg_parse_header_out(IDCapabilities, it, &msg->capabilities.value, sizeof(msg->capabilities.value), &size); } - - if (msg->vfpsz.exists) { - uint16_t nb_vfpsz = htons(msg->vfpsz.value); - it = msg_parse_header_out(IDVFPSZ, it, &nb_vfpsz, - sizeof(nb_vfpsz), &size); - } - - if ( it == parsed ) { + + if (it == parsed) { LOGGER_WARNING("Parsing message failed; empty message"); return -1; } - + *it = 0; size ++; - - if ( m_msi_packet(m, friend_number, parsed, size) ) { + + if (m_msi_packet(m, friend_number, parsed, size)) { LOGGER_DEBUG("Sent message"); return 0; } return -1; } -int send_error ( Messenger* m, uint32_t friend_number, MSIError error ) +int send_error (Messenger *m, uint32_t friend_number, MSIError error) { /* Send error message */ assert(m); - + LOGGER_DEBUG("Sending error: %d to friend: %d", error, friend_number); MSIMessage msg; msg_init(&msg, requ_pop); - + msg.error.exists = true; msg.error.value = error; - - send_message ( m, friend_number, &msg ); + + send_message (m, friend_number, &msg); return 0; } -int invoke_callback(MSICall* call, MSICallbackID cb) +int invoke_callback(MSICall *call, MSICallbackID cb) { assert(call); - - if ( call->session->callbacks[cb] ) { + + if (call->session->callbacks[cb]) { LOGGER_DEBUG("Invoking callback function: %d", cb); - if ( call->session->callbacks[cb] ( call->session->av, call ) != 0 ) { + + if (call->session->callbacks[cb] (call->session->av, call) != 0) { LOGGER_WARNING("Callback state handling failed, sending error"); goto FAILURE; } - + return 0; } - + FAILURE: /* If no callback present or error happened while handling, * an error message will be sent to friend */ - + if (call->error == msi_ENone) call->error = msi_EHandle; + return -1; } -static MSICall *get_call ( MSISession *session, uint32_t friend_number ) +static MSICall *get_call (MSISession *session, uint32_t friend_number) { assert(session); - + if (session->calls == NULL || session->calls_tail < friend_number) return NULL; - + return session->calls[friend_number]; } -MSICall *new_call ( MSISession *session, uint32_t friend_number ) +MSICall *new_call (MSISession *session, uint32_t friend_number) { assert(session); - + MSICall *rc = calloc(sizeof(MSICall), 1); - + if (rc == NULL) return NULL; - + rc->session = session; rc->friend_number = friend_number; - + if (session->calls == NULL) { /* Creating */ - session->calls = calloc (sizeof(MSICall*), friend_number + 1); - + session->calls = calloc (sizeof(MSICall *), friend_number + 1); + if (session->calls == NULL) { free(rc); return NULL; } - + session->calls_tail = session->calls_head = friend_number; - + } else if (session->calls_tail < friend_number) { /* Appending */ - void* tmp = realloc(session->calls, sizeof(MSICall*) * friend_number + 1); - + void *tmp = realloc(session->calls, sizeof(MSICall *) * friend_number + 1); + if (tmp == NULL) { free(rc); return NULL; } - + session->calls = tmp; - + /* Set fields in between to null */ uint32_t i = session->calls_tail + 1; + for (; i < friend_number; i ++) session->calls[i] = NULL; - + rc->prev = session->calls[session->calls_tail]; session->calls[session->calls_tail]->next = rc; - + session->calls_tail = friend_number; - + } else if (session->calls_head > friend_number) { /* Inserting at front */ rc->next = session->calls[session->calls_head]; session->calls[session->calls_head]->prev = rc; session->calls_head = friend_number; } - + session->calls[friend_number] = rc; return rc; } -void kill_call ( MSICall *call ) +void kill_call (MSICall *call) { /* Assume that session mutex is locked */ - if ( call == NULL ) + if (call == NULL) return; - + LOGGER_DEBUG("Killing call: %p", call); - - MSISession* session = call->session; - - MSICall* prev = call->prev; - MSICall* next = call->next; - + + MSISession *session = call->session; + + MSICall *prev = call->prev; + MSICall *next = call->next; + if (prev) prev->next = next; else if (next) session->calls_head = next->friend_number; else goto CLEAR_CONTAINER; - + if (next) next->prev = prev; else if (prev) session->calls_tail = prev->friend_number; else goto CLEAR_CONTAINER; - + session->calls[call->friend_number] = NULL; free(call); return; - + CLEAR_CONTAINER: session->calls_head = session->calls_tail = 0; free(session->calls); free(call); session->calls = NULL; } -void on_peer_status(Messenger* m, uint32_t friend_number, uint8_t status, void* data) +void on_peer_status(Messenger *m, uint32_t friend_number, uint8_t status, void *data) { (void)m; MSISession *session = data; - switch ( status ) { + switch (status) { case 0: { /* Friend is now offline */ LOGGER_DEBUG("Friend %d is now offline", friend_number); - + pthread_mutex_lock(session->mutex); - MSICall* call = get_call(session, friend_number); - + MSICall *call = get_call(session, friend_number); + if (call == NULL) { pthread_mutex_unlock(session->mutex); return; } - + invoke_callback(call, msi_OnPeerTimeout); /* Failure is ignored */ kill_call(call); pthread_mutex_unlock(session->mutex); @@ -611,10 +619,70 @@ void on_peer_status(Messenger* m, uint32_t friend_number, uint8_t status, void* break; } } -void handle_push ( MSICall *call, const MSIMessage *msg ) +void handle_init (MSICall* call, const MSIMessage* msg) { assert(call); + LOGGER_DEBUG("Session: %p Handling 'init' friend: %d", call->session, call->friend_number); + if (!msg->capabilities.exists) { + LOGGER_WARNING("Session: %p Invalid capabilities on 'init'"); + call->error = msi_EInvalidMessage; + goto FAILURE; + } + + switch (call->state) + { + case msi_CallInactive: { + /* Call requested */ + call->peer_capabilities = msg->capabilities.value; + call->state = msi_CallRequested; + + if (invoke_callback(call, msi_OnInvite) == -1) + goto FAILURE; + } + break; + + case msi_CallActive: { + /* If peer sent init while the call is already + * active it's probable that he is trying to + * re-call us while the call is not terminated + * on our side. We can assume that in this case + * we can automatically answer the re-call. + */ + + LOGGER_INFO("Friend is recalling us"); + + MSIMessage msg; + msg_init(&msg, requ_push); + + msg.capabilities.exists = true; + msg.capabilities.value = call->self_capabilities; + + send_message (call->session->messenger, call->friend_number, &msg); + + /* If peer changed capabilities during re-call they will + * be handled accordingly during the next step + */ + } + break; + + default: { + LOGGER_WARNING("Session: %p Invalid state on 'init'"); + call->error = msi_EInvalidState; + goto FAILURE; + } + break; + } + + return; +FAILURE: + send_error(call->session->messenger, call->friend_number, call->error); + kill_call(call); +} +void handle_push (MSICall *call, const MSIMessage *msg) +{ + assert(call); + LOGGER_DEBUG("Session: %p Handling 'push' friend: %d", call->session, call->friend_number); if (!msg->capabilities.exists) { @@ -622,175 +690,137 @@ void handle_push ( MSICall *call, const MSIMessage *msg ) call->error = msi_EInvalidMessage; goto FAILURE; } - - if (call->state != msi_CallActive) { - if (!msg->vfpsz.exists) { - LOGGER_WARNING("Session: %p Invalid vfpsz on 'push'"); - call->error = msi_EInvalidMessage; - goto FAILURE; - } - - call->peer_vfpsz = msg->vfpsz.value; - } - - + switch (call->state) { - case msi_CallInactive: { - LOGGER_INFO("Friend is calling us"); - - /* Call requested */ - call->peer_capabilities = msg->capabilities.value; - call->state = msi_CallRequested; - - if ( invoke_callback(call, msi_OnInvite) == -1 ) - goto FAILURE; - - } break; - case msi_CallActive: { - if (msg->vfpsz.exists) { - /* If peer sended video frame piece size - * while the call is already active it's probable - * that he is trying to re-call us while the call - * is not terminated on our side. We can assume that - * in this case we can automatically answer the re-call. - */ - if (call->peer_vfpsz != msg->vfpsz.value) { - LOGGER_WARNING("Friend sent invalid parameters for re-call"); - call->error = msi_EInvalidParam; - invoke_callback(call, msi_OnError); - goto FAILURE; - } - - LOGGER_INFO("Friend is recalling us"); - - MSIMessage msg; - msg_init(&msg, requ_push); - - msg.capabilities.exists = true; - msg.capabilities.value = call->self_capabilities; - - msg.vfpsz.exists = true; - msg.vfpsz.value = VIDEOFRAME_PIECE_SIZE; - - send_message ( call->session->messenger, call->friend_number, &msg ); - - /* If peer changed capabilities during re-call they will - * be handled accordingly during the next step - */ - } - /* Only act if capabilities changed */ - if ( call->peer_capabilities != msg->capabilities.value) { + if (call->peer_capabilities != msg->capabilities.value) { LOGGER_INFO("Friend is changing capabilities to: %u", msg->capabilities.value); - + call->peer_capabilities = msg->capabilities.value; - if ( invoke_callback(call, msi_OnCapabilities) == -1 ) + + if (invoke_callback(call, msi_OnCapabilities) == -1) goto FAILURE; } - } break; - + } + break; + case msi_CallRequesting: { LOGGER_INFO("Friend answered our call"); - + /* Call started */ call->peer_capabilities = msg->capabilities.value; call->state = msi_CallActive; - - if ( invoke_callback(call, msi_OnStart) == -1 ) + + if (invoke_callback(call, msi_OnStart) == -1) goto FAILURE; - - } break; - + + } + break; + + /* Pushes during initialization state are ignored */ + case msi_CallInactive: case msi_CallRequested: { - /* Consecutive pushes during initialization state are ignored */ - LOGGER_WARNING("Consecutive push"); - } break; + LOGGER_WARNING("Ignoring invalid push"); + } + break; } - + return; - + FAILURE: send_error(call->session->messenger, call->friend_number, call->error); kill_call(call); } -void handle_pop ( MSICall *call, const MSIMessage *msg ) +void handle_pop (MSICall *call, const MSIMessage *msg) { assert(call); - + LOGGER_DEBUG("Session: %p Handling 'pop', friend id: %d", call->session, call->friend_number); - + /* callback errors are ignored */ - + if (msg->error.exists) { LOGGER_WARNING("Friend detected an error: %d", msg->error.value); call->error = msg->error.value; invoke_callback(call, msi_OnError); - + } else switch (call->state) { case msi_CallInactive: { LOGGER_ERROR("Handling what should be impossible case"); abort(); - } break; - + } + break; + case msi_CallActive: { /* Hangup */ LOGGER_INFO("Friend hung up on us"); invoke_callback(call, msi_OnEnd); - } break; - + } + break; + case msi_CallRequesting: { /* Reject */ LOGGER_INFO("Friend rejected our call"); invoke_callback(call, msi_OnEnd); - } break; - + } + break; + case msi_CallRequested: { /* Cancel */ LOGGER_INFO("Friend canceled call invite"); invoke_callback(call, msi_OnEnd); - } break; + } + break; } - - kill_call ( call ); + + kill_call (call); } -void handle_msi_packet ( Messenger* m, uint32_t friend_number, const uint8_t* data, uint16_t length, void* object ) +void handle_msi_packet (Messenger *m, uint32_t friend_number, const uint8_t *data, uint16_t length, void *object) { LOGGER_DEBUG("Got msi message"); - + MSISession *session = object; MSIMessage msg; - - if ( msg_parse_in ( &msg, data, length ) == -1 ) { + + if (msg_parse_in (&msg, data, length) == -1) { LOGGER_WARNING("Error parsing message"); send_error(m, friend_number, msi_EInvalidMessage); return; } else { LOGGER_DEBUG("Successfully parsed message"); } - + pthread_mutex_lock(session->mutex); MSICall *call = get_call(session, friend_number); - + if (call == NULL) { - if (msg.request.value != requ_push) { + if (msg.request.value != requ_init) { send_error(m, friend_number, msi_EStrayMessage); pthread_mutex_unlock(session->mutex); return; } - + call = new_call(session, friend_number); + if (call == NULL) { send_error(m, friend_number, msi_ESystem); pthread_mutex_unlock(session->mutex); return; } } - - if (msg.request.value == requ_push) - handle_push(call, &msg); - else - handle_pop(call, &msg); /* always kills the call */ - + + switch (msg.request.value) { + case requ_init: + handle_init(call, &msg); + break; + case requ_push: + handle_push(call, &msg); + break; + case requ_pop: + handle_pop(call, &msg); /* always kills the call */ + break; + } + pthread_mutex_unlock(session->mutex); } diff --git a/toxav/msi.h b/toxav/msi.h index 59f32c1d8..e69581d15 100644 --- a/toxav/msi.h +++ b/toxav/msi.h @@ -29,9 +29,6 @@ #include "video.h" #include "../toxcore/Messenger.h" -/** Preconfigured value for video splitting */ -#define VIDEOFRAME_PIECE_SIZE 500 - /** * Error codes. */ @@ -89,13 +86,13 @@ typedef struct MSICall_s { uint8_t peer_capabilities; /* Peer capabilities */ uint8_t self_capabilities; /* Self capabilities */ uint16_t peer_vfpsz; /* Video frame piece size */ - uint32_t friend_number; /* Index of this call in MSISession */ + uint32_t friend_number; /* Index of this call in MSISession */ MSIError error; /* Last error */ - - void* av_call; /* Pointer to av call handler */ - - struct MSICall_s* next; - struct MSICall_s* prev; + + void *av_call; /* Pointer to av call handler */ + + struct MSICall_s *next; + struct MSICall_s *prev; } MSICall; @@ -104,7 +101,7 @@ typedef struct MSICall_s { * returned the call is considered errored and will be handled * as such which means it will be terminated without any notice. */ -typedef int msi_action_cb ( void *av, MSICall* call); +typedef int msi_action_cb (void *av, MSICall *call); /** * Control session struct. Please do not modify outside msi.c @@ -114,41 +111,41 @@ typedef struct MSISession_s { MSICall **calls; uint32_t calls_tail; uint32_t calls_head; - + void *av; Messenger *messenger; pthread_mutex_t mutex[1]; - msi_action_cb* callbacks[7]; + msi_action_cb *callbacks[7]; } MSISession; /** * Start the control session. */ -MSISession *msi_new ( Messenger *m ); +MSISession *msi_new(Messenger *m); /** * Terminate control session. NOTE: all calls will be freed */ -int msi_kill ( MSISession *session ); +int msi_kill(MSISession *session); /** * Callback setter. */ -void msi_register_callback(MSISession *session, msi_action_cb* callback, MSICallbackID id); +void msi_register_callback(MSISession *session, msi_action_cb *callback, MSICallbackID id); /** * Send invite request to friend_number. */ -int msi_invite ( MSISession* session, MSICall** call, uint32_t friend_number, uint8_t capabilities ); +int msi_invite(MSISession *session, MSICall **call, uint32_t friend_number, uint8_t capabilities); /** * Hangup call. NOTE: 'call' will be freed */ -int msi_hangup ( MSICall* call ); +int msi_hangup(MSICall *call); /** * Answer call request. */ -int msi_answer ( MSICall* call, uint8_t capabilities ); +int msi_answer(MSICall *call, uint8_t capabilities); /** * Change capabilities of the call. */ -int msi_change_capabilities ( MSICall* call, uint8_t capabilities ); +int msi_change_capabilities(MSICall *call, uint8_t capabilities); #endif /* MSI_H */ diff --git a/toxav/rtp.c b/toxav/rtp.c index 4df2e2d5f..763166cd1 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -24,6 +24,7 @@ #endif /* HAVE_CONFIG_H */ #include "rtp.h" +#include "bwcontroler.h" #include "../toxcore/logger.h" #include "../toxcore/util.h" #include "../toxcore/Messenger.h" @@ -31,584 +32,361 @@ #include #include -#define RTCP_REPORT_INTERVAL_MS 500 -#define RTP_MAX_SIZE 1500 -#define ADD_FLAG_VERSION(_h, _v) do { ( _h->flags ) &= 0x3F; ( _h->flags ) |= ( ( ( _v ) << 6 ) & 0xC0 ); } while(0) -#define ADD_FLAG_PADDING(_h, _v) do { if ( _v > 0 ) _v = 1; ( _h->flags ) &= 0xDF; ( _h->flags ) |= ( ( ( _v ) << 5 ) & 0x20 ); } while(0) -#define ADD_FLAG_EXTENSION(_h, _v) do { if ( _v > 0 ) _v = 1; ( _h->flags ) &= 0xEF;( _h->flags ) |= ( ( ( _v ) << 4 ) & 0x10 ); } while(0) -#define ADD_FLAG_CSRCC(_h, _v) do { ( _h->flags ) &= 0xF0; ( _h->flags ) |= ( ( _v ) & 0x0F ); } while(0) -#define ADD_SETTING_MARKER(_h, _v) do { ( _h->marker_payloadt ) &= 0x7F; ( _h->marker_payloadt ) |= ( ( ( _v ) << 7 ) /*& 0x80 */ ); } while(0) -#define ADD_SETTING_PAYLOAD(_h, _v) do { ( _h->marker_payloadt ) &= 0x80; ( _h->marker_payloadt ) |= ( ( _v ) /* & 0x7F */ ); } while(0) - -#define GET_FLAG_VERSION(_h) (( _h->flags & 0xd0 ) >> 6) -#define GET_FLAG_PADDING(_h) (( _h->flags & 0x20 ) >> 5) -#define GET_FLAG_EXTENSION(_h) (( _h->flags & 0x10 ) >> 4) -#define GET_FLAG_CSRCC(_h) ( _h->flags & 0x0f ) -#define GET_SETTING_MARKER(_h) (( _h->marker_payloadt ) >> 7) -#define GET_SETTING_PAYLOAD(_h) ((_h->marker_payloadt) & 0x7f) +int handle_rtp_packet (Messenger *m, uint32_t friendnumber, const uint8_t *data, uint16_t length, void *object); -typedef struct { - uint64_t timestamp; /* in ms */ - - uint32_t received_packets; - uint32_t expected_packets; - /* ... other stuff in the future */ -} RTCPReport; - -typedef struct RTCPSession_s { - RTPSession *rtp_session; - - uint8_t prefix; - uint64_t last_sent_report_ts; - uint32_t last_received_packets; - uint32_t last_expected_packets; - - RingBuffer* pl_stats; /* Packet loss stats over time */ -} RTCPSession; - - -RTPHeader *parse_header_in ( const uint8_t *payload, int length ); -RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length ); -uint8_t *parse_header_out ( const RTPHeader* header, uint8_t* payload ); -uint8_t *parse_ext_header_out ( const RTPExtHeader* header, uint8_t* payload ); -int handle_rtp_packet ( Messenger *m, uint32_t friendnumber, const uint8_t *data, uint16_t length, void *object ); -int handle_rtcp_packet ( Messenger *m, uint32_t friendnumber, const uint8_t *data, uint16_t length, void *object ); -void send_rtcp_report ( RTCPSession* session, Messenger* m, uint32_t friendnumber ); - - -RTPSession *rtp_new ( int payload_type, Messenger *m, int friend_num, void* cs, int (*mcb) (void*, RTPMessage*) ) +RTPSession *rtp_new (int payload_type, Messenger *m, uint32_t friendnumber, + BWControler *bwc, void *cs, + int (*mcb) (void *, struct RTPMessage *)) { assert(mcb); assert(cs); assert(m); - + RTPSession *retu = calloc(1, sizeof(RTPSession)); - if ( !retu ) { + if (!retu) { LOGGER_WARNING("Alloc failed! Program might misbehave!"); return NULL; } - retu->version = RTP_VERSION; /* It's always 2 */ - retu->ssrc = random_int(); - retu->payload_type = payload_type % 128; - + retu->ssrc = random_int(); + retu->payload_type = payload_type; + retu->m = m; - retu->friend_number = friend_num; - - if ( !(retu->csrc = calloc(1, sizeof(uint32_t))) ) { - LOGGER_WARNING("Alloc failed! Program might misbehave!"); - free(retu); - return NULL; - } - - retu->csrc[0] = retu->ssrc; /* Set my ssrc to the list receive */ + retu->friend_number = friendnumber; /* Also set payload type as prefix */ - retu->prefix = payload_type; - + + retu->bwc = bwc; retu->cs = cs; retu->mcb = mcb; - - /* Initialize rtcp session */ - if (!(retu->rtcp_session = calloc(1, sizeof(RTCPSession)))) { - LOGGER_WARNING("Alloc failed! Program might misbehave!"); - free(retu->csrc); - free(retu); - return NULL; - } - - retu->rtcp_session->prefix = payload_type + 2; - retu->rtcp_session->pl_stats = rb_new(4); - retu->rtcp_session->rtp_session = retu; - - if (-1 == rtp_start_receiving(retu)) { + + if (-1 == rtp_allow_receiving(retu)) { LOGGER_WARNING("Failed to start rtp receiving mode"); - free(retu->rtcp_session); - free(retu->csrc); free(retu); return NULL; } - + return retu; } -void rtp_kill ( RTPSession *session ) +void rtp_kill (RTPSession *session) { - if ( !session ) return; + if (!session) + return; - rtp_stop_receiving (session); - - free ( session->ext_header ); - free ( session->csrc ); - - void* t; - while (!rb_empty(session->rtcp_session->pl_stats)) { - rb_read(session->rtcp_session->pl_stats, (void**) &t); - free(t); - } - rb_free(session->rtcp_session->pl_stats); - LOGGER_DEBUG("Terminated RTP session: %p", session); - /* And finally free session */ - free ( session->rtcp_session ); - free ( session ); + rtp_stop_receiving (session); + free (session); } -int rtp_do(RTPSession *session) -{ - if (!session || !session->rtcp_session) - return rtp_StateNormal; - - if (current_time_monotonic() - session->rtcp_session->last_sent_report_ts >= RTCP_REPORT_INTERVAL_MS) { - send_rtcp_report(session->rtcp_session, session->m, session->friend_number); - } - - if (rb_full(session->rtcp_session->pl_stats)) { - RTCPReport* reports[4]; - - int i = 0; - for (; i < 4; i++) - rb_read(session->rtcp_session->pl_stats, (void**) reports + i); - - /* Check for timed out reports (> 6 sec) */ - uint64_t now = current_time_monotonic(); - for (i = 0; i < 4 && (now - reports[i]->timestamp) < 6000; i ++); - for (; i < 4; i ++) { - rb_write(session->rtcp_session->pl_stats, reports[i]); - reports[i] = NULL; - } - if (!rb_empty(session->rtcp_session->pl_stats)) { - for (i = 0; reports[i] != NULL; i ++) - free(reports[i]); - return rtp_StateNormal; /* As some reports are timed out, we need more */ - } - - /* We have 4 on-time reports so we can proceed */ - uint32_t quality = 100; - for (i = 0; i < 4; i++) { - uint32_t current = reports[i]->received_packets * 100 / reports[i]->expected_packets; - quality = MIN(quality, current); - free(reports[i]); - } - - if (quality <= 90) { - LOGGER_WARNING("Stream quality: BAD (%d)", quality); - return rtp_StateBad; - } else if (quality >= 99) { - LOGGER_DEBUG("Stream quality: GOOD (%d)", quality); - return rtp_StateGood; - } else { - LOGGER_DEBUG("Stream quality: NORMAL (%d)", quality); - } - } - return rtp_StateNormal; -} -int rtp_start_receiving(RTPSession* session) +int rtp_allow_receiving(RTPSession *session) { if (session == NULL) return -1; - - if (m_callback_rtp_packet(session->m, session->friend_number, session->prefix, - handle_rtp_packet, session) == -1) { + + if (m_callback_rtp_packet(session->m, session->friend_number, session->payload_type, + handle_rtp_packet, session) == -1) { LOGGER_WARNING("Failed to register rtp receive handler"); return -1; } - if (m_callback_rtp_packet(session->m, session->friend_number, session->rtcp_session->prefix, - handle_rtcp_packet, session->rtcp_session) == -1) { - LOGGER_WARNING("Failed to register rtcp receive handler"); - m_callback_rtp_packet(session->m, session->friend_number, session->prefix, NULL, NULL); - return -1; - } - + LOGGER_DEBUG("Started receiving on session: %p", session); return 0; } -int rtp_stop_receiving(RTPSession* session) +int rtp_stop_receiving(RTPSession *session) { if (session == NULL) return -1; - - m_callback_rtp_packet(session->m, session->friend_number, session->prefix, NULL, NULL); - m_callback_rtp_packet(session->m, session->friend_number, session->rtcp_session->prefix, NULL, NULL); /* RTCP */ - + + m_callback_rtp_packet(session->m, session->friend_number, session->payload_type, NULL, NULL); + LOGGER_DEBUG("Stopped receiving on session: %p", session); return 0; } -int rtp_send_data ( RTPSession *session, const uint8_t *data, uint16_t length, bool dummy ) +int rtp_send_data (RTPSession *session, const uint8_t *data, uint16_t length) { - if ( !session ) { + if (!session) { LOGGER_WARNING("No session!"); return -1; } - - uint8_t parsed[RTP_MAX_SIZE]; - uint8_t *it; - RTPHeader header[1]; - memset(header, 0, sizeof(header)); - - ADD_FLAG_VERSION ( header, session->version ); - ADD_FLAG_PADDING ( header, session->padding ); - ADD_FLAG_EXTENSION ( header, session->extension ); - ADD_FLAG_CSRCC ( header, session->cc ); - ADD_SETTING_MARKER ( header, session->marker ); - - if (dummy) - ADD_SETTING_PAYLOAD ( header, (session->payload_type + 2) % 128 ); - else - ADD_SETTING_PAYLOAD ( header, session->payload_type ); + uint8_t rdata[length + sizeof(struct RTPHeader) + 1]; + memset(rdata, 0, sizeof(rdata)); - header->sequnum = session->sequnum; - header->timestamp = current_time_monotonic(); - header->ssrc = session->ssrc; + rdata[0] = session->payload_type; - int i; - for ( i = 0; i < session->cc; i++ ) - header->csrc[i] = session->csrc[i]; + struct RTPHeader *header = (struct RTPHeader *)(rdata + 1); - header->length = 12 /* Minimum header len */ + ( session->cc * 4 ); - - uint32_t parsed_len = length + header->length + 1; - assert(parsed_len + (session->ext_header ? session->ext_header->length * 4 : 0) < RTP_MAX_SIZE ); + header->ve = 2; + header->pe = 0; + header->xe = 0; + header->cc = 0; - parsed[0] = session->prefix; - it = parse_header_out ( header, parsed + 1 ); - - if ( session->ext_header ) { - parsed_len += ( 4 /* Minimum ext header len */ + session->ext_header->length * 4 ); - it = parse_ext_header_out ( session->ext_header, it ); + header->ma = 0; + header->pt = session->payload_type % 128; + + header->sequnum = htons(session->sequnum); + header->timestamp = htonl(current_time_monotonic()); + header->ssrc = htonl(session->ssrc); + + header->cpart = 0; + header->tlen = htons(length); + + if (MAX_CRYPTO_DATA_SIZE > length + sizeof(struct RTPHeader) + 1) { + + /** + * The lenght is lesser than the maximum allowed lenght (including header) + * Send the packet in single piece. + */ + + memcpy(rdata + 1 + sizeof(struct RTPHeader), data, length); + + if (-1 == send_custom_lossy_packet(session->m, session->friend_number, rdata, sizeof(rdata))) + LOGGER_WARNING("RTP send failed (len: %d)! std error: %s", sizeof(rdata), strerror(errno)); + } else { + + /** + * The lenght is greater than the maximum allowed lenght (including header) + * Send the packet in multiple pieces. + */ + + uint16_t sent = 0; + uint16_t piece = MAX_CRYPTO_DATA_SIZE - (sizeof(struct RTPHeader) + 1); + + while ((length - sent) + sizeof(struct RTPHeader) + 1 > MAX_CRYPTO_DATA_SIZE) { + memcpy(rdata + 1 + sizeof(struct RTPHeader), data + sent, piece); + + if (-1 == send_custom_lossy_packet(session->m, session->friend_number, + rdata, piece + sizeof(struct RTPHeader) + 1)) + LOGGER_WARNING("RTP send failed (len: %d)! std error: %s", + piece + sizeof(struct RTPHeader) + 1, strerror(errno)); + + sent += piece; + header->cpart = htons(sent); + } + + /* Send remaining */ + piece = length - sent; + + if (piece) { + memcpy(rdata + 1 + sizeof(struct RTPHeader), data + sent, piece); + + if (-1 == send_custom_lossy_packet(session->m, session->friend_number, rdata, + piece + sizeof(struct RTPHeader) + 1)) + LOGGER_WARNING("RTP send failed (len: %d)! std error: %s", + piece + sizeof(struct RTPHeader) + 1, strerror(errno)); + } } - memcpy(it, data, length); - - if ( -1 == send_custom_lossy_packet(session->m, session->friend_number, parsed, parsed_len) ) { - LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno)); - return -1; - } - session->sequnum ++; return 0; } -void rtp_free_msg ( RTPMessage *msg ) + + +bool chloss (const RTPSession *session, const struct RTPHeader *header) { - if ( msg->ext_header ) { - free ( msg->ext_header->table ); - free ( msg->ext_header ); + if (ntohl(header->timestamp) < session->rtimestamp) { + uint16_t hosq, lost = 0; + + hosq = ntohs(header->sequnum); + + lost = (hosq > session->rsequnum) ? + (session->rsequnum + 65535) - hosq : + session->rsequnum - hosq; + + puts ("Lost packet"); + while (lost --) + bwc_add_lost(session->bwc ,0); + + return true; } - - free ( msg->header ); - free ( msg ); + + return false; } - - - -RTPHeader *parse_header_in ( const uint8_t *payload, int length ) +struct RTPMessage *new_message (size_t allocate_len, const uint8_t *data, uint16_t data_length) { - if ( !payload || !length ) { - LOGGER_WARNING("No payload to extract!"); - return NULL; - } + assert(allocate_len >= data_length); - RTPHeader *retu = calloc(1, sizeof (RTPHeader)); + struct RTPMessage *msg = calloc(sizeof(struct RTPMessage) + (allocate_len - sizeof(struct RTPHeader)), 1); - if ( !retu ) { - LOGGER_WARNING("Alloc failed! Program might misbehave!"); - return NULL; - } + msg->len = data_length - sizeof(struct RTPHeader); + memcpy(&msg->header, data, data_length); - memcpy(&retu->sequnum, payload, sizeof(retu->sequnum)); - retu->sequnum = ntohs(retu->sequnum); + msg->header.sequnum = ntohs(msg->header.sequnum); + msg->header.timestamp = ntohl(msg->header.timestamp); + msg->header.ssrc = ntohl(msg->header.ssrc); - const uint8_t *it = payload + 2; + msg->header.cpart = ntohs(msg->header.cpart); + msg->header.tlen = ntohs(msg->header.tlen); - retu->flags = *it; - ++it; - - if ( GET_FLAG_VERSION(retu) != RTP_VERSION ) { - /* Deallocate */ - LOGGER_WARNING("Invalid version!"); - free(retu); - return NULL; - } - - uint8_t cc = GET_FLAG_CSRCC ( retu ); - int total = 12 /* Minimum header len */ + ( cc * 4 ); - - if ( length < total ) { - LOGGER_WARNING("Length invalid!"); - free(retu); - return NULL; - } - - retu->marker_payloadt = *it; - ++it; - retu->length = total; - - - memcpy(&retu->timestamp, it, sizeof(retu->timestamp)); - it += 4; - memcpy(&retu->ssrc, it, sizeof(retu->ssrc)); - - retu->timestamp = ntohl(retu->timestamp); - retu->ssrc = ntohl(retu->ssrc); - - uint8_t x; - for ( x = 0; x < cc; x++ ) { - it += 4; - memcpy(&retu->csrc[x], it, sizeof(retu->csrc[x])); - retu->csrc[x] = ntohl(retu->csrc[x]); - } - - return retu; + return msg; } -RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length ) -{ - const uint8_t *it = payload; - - RTPExtHeader *retu = calloc(1, sizeof (RTPExtHeader)); - - if ( !retu ) { - LOGGER_WARNING("Alloc failed! Program might misbehave!"); - return NULL; - } - - memcpy(&retu->length, it, sizeof(retu->length)); - retu->length = ntohs(retu->length); - it += 2; - - if ( length < ( retu->length * sizeof(uint32_t) ) ) { - LOGGER_WARNING("Length invalid!"); - free(retu); - return NULL; - } - - memcpy(&retu->type, it, sizeof(retu->type)); - retu->type = ntohs(retu->type); - - it += 2; - - if ( !(retu->table = calloc(retu->length, sizeof (uint32_t))) ) { - LOGGER_WARNING("Alloc failed! Program might misbehave!"); - free(retu); - return NULL; - } - - uint16_t x; - for ( x = 0; x < retu->length; x++ ) { - it += 4; - memcpy(retu->table + x, it, sizeof(*retu->table)); - retu->table[x] = ntohl(retu->table[x]); - } - - return retu; -} -uint8_t *parse_header_out ( const RTPHeader *header, uint8_t *payload ) -{ - uint8_t cc = GET_FLAG_CSRCC ( header ); - uint8_t *it = payload; - uint16_t sequnum; - uint32_t timestamp; - uint32_t ssrc; - uint32_t csrc; - - - /* Add sequence number first */ - sequnum = htons(header->sequnum); - memcpy(it, &sequnum, sizeof(sequnum)); - it += 2; - - *it = header->flags; - ++it; - *it = header->marker_payloadt; - ++it; - - timestamp = htonl(header->timestamp); - memcpy(it, ×tamp, sizeof(timestamp)); - it += 4; - ssrc = htonl(header->ssrc); - memcpy(it, &ssrc, sizeof(ssrc)); - - uint8_t x; - - for ( x = 0; x < cc; x++ ) { - it += 4; - csrc = htonl(header->csrc[x]); - memcpy(it, &csrc, sizeof(csrc)); - } - - return it + 4; -} -uint8_t *parse_ext_header_out ( const RTPExtHeader *header, uint8_t *payload ) -{ - uint8_t *it = payload; - uint16_t length; - uint16_t type; - uint32_t entry; - - length = htons(header->length); - memcpy(it, &length, sizeof(length)); - it += 2; - type = htons(header->type); - memcpy(it, &type, sizeof(type)); - it -= 2; /* Return to 0 position */ - - if ( header->table ) { - uint16_t x; - for ( x = 0; x < header->length; x++ ) { - it += 4; - entry = htonl(header->table[x]); - memcpy(it, &entry, sizeof(entry)); - } - } - - return it + 4; -} -int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) +int handle_rtp_packet (Messenger *m, uint32_t friendnumber, const uint8_t *data, uint16_t length, void *object) { (void) m; (void) friendnumber; - + RTPSession *session = object; - if ( !session || length < 13 || length > RTP_MAX_SIZE ) { + data ++; + length--; + + if (!session || length < sizeof (struct RTPHeader)) { LOGGER_WARNING("No session or invalid length of received buffer!"); return -1; } - - RTPHeader* header = parse_header_in ( data + 1, length ); - if ( !header ) { - LOGGER_WARNING("Could not parse message: Header failed to extract!"); + const struct RTPHeader *header = (struct RTPHeader *) data; + + if (header->pt != session->payload_type % 128) { + LOGGER_WARNING("Invalid payload type with the session"); return -1; } - - RTPExtHeader* ext_header = NULL; - - uint16_t from_pos = header->length + 1; - uint16_t msg_length = length - from_pos; - if ( GET_FLAG_EXTENSION ( header ) ) { - ext_header = parse_ext_header_in ( data + from_pos, length ); + if (ntohs(header->cpart) >= ntohs(header->tlen)) { + /* Never allow this case to happen */ + return -1; + } - if ( ext_header ) { - msg_length -= ( 4 /* Minimum ext header len */ + ext_header->length * 4 ); - from_pos += ( 4 /* Minimum ext header len */ + ext_header->length * 4 ); - } else { /* Error */ - LOGGER_WARNING("Could not parse message: Ext Header failed to extract!"); - free(header); - return -1; + bwc_feed_avg(session->bwc, length); + + if (ntohs(header->tlen) == length - sizeof (struct RTPHeader)) { + /* The message is sent in single part */ + + /* Only allow messages which have arrived in order; + * drop late messages + */ + if (chloss(session, header)) { + return 0; + } else { + /* Message is not late; pick up the latest parameters */ + session->rsequnum = ntohs(header->sequnum); + session->rtimestamp = ntohl(header->timestamp); + } + + bwc_add_recv(session->bwc, length); + + /* Invoke processing of active multiparted message */ + if (session->mp) { + if (session->mcb) + session->mcb (session->cs, session->mp); + else + free(session->mp); + + session->mp = NULL; + } + + /* The message came in the allowed time; + * process it only if handler for the session is present. + */ + + if (!session->mcb) + return 0; + + return session->mcb (session->cs, new_message(length, data, length)); + } else { + /* The message is sent in multiple parts */ + + if (session->mp) { + /* There are 2 possible situations in this case: + * 1) being that we got the part of already processing message. + * 2) being that we got the part of a new/old message. + * + * We handle them differently as we only allow a single multiparted + * processing message + */ + + if (session->mp->header.sequnum == ntohs(header->sequnum) && + session->mp->header.timestamp == ntohl(header->timestamp)) { + /* First case */ + + /* Make sure we have enough allocated memory */ + if (session->mp->header.tlen - session->mp->len < length - sizeof(struct RTPHeader) || + session->mp->header.tlen <= ntohs(header->cpart)) { + /* There happened to be some corruption on the stream; + * continue wihtout this part + */ + return 0; + } + + memcpy(session->mp->data + ntohs(header->cpart), data + sizeof(struct RTPHeader), + length - sizeof(struct RTPHeader)); + + session->mp->len += length - sizeof(struct RTPHeader); + + bwc_add_recv(session->bwc, length); + + if (session->mp->len == session->mp->header.tlen) { + /* Received a full message; now push it for the further + * processing. + */ + if (session->mcb) + session->mcb (session->cs, session->mp); + else + free(session->mp); + + session->mp = NULL; + } + } else { + /* Second case */ + + if (session->mp->header.timestamp > ntohl(header->timestamp)) + /* The received message part is from the old message; + * discard it. + */ + return 0; + + /* Measure missing parts of the old message */ + bwc_add_lost(session->bwc, + (session->mp->header.tlen - session->mp->len) + + + /* Must account sizes of rtp headers too */ + ((session->mp->header.tlen - session->mp->len) / + MAX_CRYPTO_DATA_SIZE) * sizeof(struct RTPHeader) ); + + /* Push the previous message for processing */ + if (session->mcb) + session->mcb (session->cs, session->mp); + else + free(session->mp); + + session->mp = NULL; + goto NEW_MULTIPARTED; + } + } else { + /* In this case threat the message as if it was received in order + */ + + /* This is also a point for new multiparted messages */ +NEW_MULTIPARTED: + + /* Only allow messages which have arrived in order; + * drop late messages + */ + if (chloss(session, header)) { + return 0; + } else { + /* Message is not late; pick up the latest parameters */ + session->rsequnum = ntohs(header->sequnum); + session->rtimestamp = ntohl(header->timestamp); + } + + bwc_add_recv(session->bwc, length); + + /* Again, only store message if handler is present + */ + if (session->mcb) { + session->mp = new_message(ntohs(header->tlen) + sizeof(struct RTPHeader), data, length); + + /* Reposition data if necessary */ + if (ntohs(header->cpart)); + + memmove(session->mp->data + ntohs(header->cpart), session->mp->data, session->mp->len); + } } } - - if (msg_length > RTP_MAX_SIZE) { - LOGGER_WARNING("Could not parse message: Invalid length!"); - free(header); - free(ext_header); - return -1; - } - - /* Check if message came in late */ - if ( header->sequnum > session->rsequnum || header->timestamp > session->rtimestamp ) { - /* Not late */ - if (header->sequnum > session->rsequnum) - session->rtcp_session->last_expected_packets += header->sequnum - session->rsequnum; - else if (header->sequnum < session->rsequnum) - session->rtcp_session->last_expected_packets += (header->sequnum + 65535) - session->rsequnum; - else /* Usual case when transmission starts */ - session->rtcp_session->last_expected_packets ++; - - session->rsequnum = header->sequnum; - session->rtimestamp = header->timestamp; - } - session->rtcp_session->last_received_packets ++; - - /* Check if the message is dummy. We don't keep dummy messages */ - if (GET_SETTING_PAYLOAD(header) == (session->payload_type + 2) % 128) { - LOGGER_DEBUG("Received dummy rtp message"); - free(header); - free(ext_header); - return 0; - } - - /* Otherwise we will store the message if we have an appropriate handler */ - if (!session->mcb) { - LOGGER_DEBUG("No handler for the message of %d payload", GET_SETTING_PAYLOAD(header)); - free(header); - free(ext_header); - return 0; - } - - RTPMessage *msg = calloc(1, sizeof (RTPMessage) + msg_length); - - if ( !msg ) { - LOGGER_WARNING("Could not parse message: Allocation failed!"); - free(header); - free(ext_header); - return -1; - } - - msg->header = header; - msg->ext_header = ext_header; - msg->length = msg_length; - - memcpy ( msg->data, data + from_pos, msg_length ); - - return session->mcb (session->cs, msg); -} -int handle_rtcp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) -{ - (void) m; - (void) friendnumber; - - if (length < 9) - return -1; - - RTCPSession* session = object; - RTCPReport* report = malloc(sizeof(RTCPReport)); - - memcpy(&report->received_packets, data + 1, 4); - memcpy(&report->expected_packets, data + 5, 4); - - report->received_packets = ntohl(report->received_packets); - report->expected_packets = ntohl(report->expected_packets); - - if (report->expected_packets == 0 || report->received_packets > report->expected_packets) { - LOGGER_WARNING("Malformed rtcp report! %d %d", report->expected_packets, report->received_packets); - free(report); - return 0; - } - - report->timestamp = current_time_monotonic(); - - free(rb_write(session->pl_stats, report)); - - LOGGER_DEBUG("Got rtcp report: ex: %d rc: %d", report->expected_packets, report->received_packets); return 0; } -void send_rtcp_report(RTCPSession* session, Messenger* m, uint32_t friendnumber) -{ - if (session->last_expected_packets == 0) - return; - - uint8_t parsed[9]; - parsed[0] = session->prefix; - - uint32_t received_packets = htonl(session->last_received_packets); - uint32_t expected_packets = htonl(session->last_expected_packets); - - memcpy(parsed + 1, &received_packets, 4); - memcpy(parsed + 5, &expected_packets, 4); - - if (-1 == send_custom_lossy_packet(m, friendnumber, parsed, sizeof(parsed))) - LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", sizeof(parsed), strerror(errno)); - else { - LOGGER_DEBUG("Sent rtcp report: ex: %d rc: %d", session->last_expected_packets, session->last_received_packets); - - session->last_received_packets = 0; - session->last_expected_packets = 0; - session->last_sent_report_ts = current_time_monotonic(); - } -} diff --git a/toxav/rtp.h b/toxav/rtp.h index 9c5cf07d6..fddbce3de 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -22,119 +22,88 @@ #ifndef RTP_H #define RTP_H -#define RTP_VERSION 2 - +#include "bwcontroler.h" #include "../toxcore/Messenger.h" #include "stdbool.h" /** - * Payload type identifier. Also used as rtp callback prefix. (Not dummies) + * Payload type identifier. Also used as rtp callback prefix. */ enum { rtp_TypeAudio = 192, rtp_TypeVideo, }; -enum { - rtp_StateBad = -1, - rtp_StateNormal, - rtp_StateGood, -}; +struct RTPHeader { + /* Standard RTP header */ +#ifndef WORDS_BIGENDIAN + unsigned cc: 4; /* Contributing sources count */ + unsigned xe: 1; /* Extra header */ + unsigned pe: 1; /* Padding */ + unsigned ve: 2; /* Version */ -/** - * Standard rtp header. - */ -typedef struct { - uint8_t flags; /* Version(2),Padding(1), Ext(1), Cc(4) */ - uint8_t marker_payloadt; /* Marker(1), PlayLoad Type(7) */ - uint16_t sequnum; /* Sequence Number */ - uint32_t timestamp; /* Timestamp */ - uint32_t ssrc; /* SSRC */ - uint32_t csrc[16]; /* CSRC's table */ - uint32_t length; /* Length of the header in payload string. */ -} RTPHeader; -/** - * Standard rtp extension header. - */ -typedef struct { - uint16_t type; /* Extension profile */ - uint16_t length; /* Number of extensions */ - uint32_t *table; /* Extension's table */ -} RTPExtHeader; + unsigned pt: 7; /* Payload type */ + unsigned ma: 1; /* Marker */ +#else + unsigned ve: 2; /* Version */ + unsigned pe: 1; /* Padding */ + unsigned xe: 1; /* Extra header */ + unsigned cc: 4; /* Contributing sources count */ -/** - * Standard rtp message. - */ -typedef struct RTPMessage_s { - RTPHeader *header; - RTPExtHeader *ext_header; + unsigned ma: 1; /* Marker */ + unsigned pt: 7; /* Payload type */ +#endif - uint32_t length; - uint8_t data[]; -} RTPMessage; + uint16_t sequnum; + uint32_t timestamp; + uint32_t ssrc; + uint32_t csrc[16]; + + /* Non-standard TOX-specific fields */ + uint16_t cpart;/* Data offset of the current part */ + uint16_t tlen; /* Total message lenght */ +} __attribute__ ((packed)); + +/* Check alignment */ +typedef char __fail_if_misaligned [ sizeof(struct RTPHeader) == 80 ? 1 : -1 ]; + +struct RTPMessage { + uint16_t len; + + struct RTPHeader header; + uint8_t data[]; +} __attribute__ ((packed)); + +/* Check alignment */ +typedef char __fail_if_misaligned [ sizeof(struct RTPMessage) == 82 ? 1 : -1 ]; /** * RTP control session. */ typedef struct { - uint8_t version; - uint8_t padding; - uint8_t extension; - uint8_t cc; - uint8_t marker; uint8_t payload_type; - uint16_t sequnum; /* Sending sequence number */ - uint16_t rsequnum; /* Receiving sequence number */ + uint16_t sequnum; /* Sending sequence number */ + uint16_t rsequnum; /* Receiving sequence number */ uint32_t rtimestamp; uint32_t ssrc; - uint32_t *csrc; - /* If some additional data must be sent via message - * apply it here. Only by allocating this member you will be - * automatically placing it within a message. - */ - RTPExtHeader *ext_header; - - /* Msg prefix for core to know when recving */ - uint8_t prefix; + struct RTPMessage *mp; /* Expected parted message */ Messenger *m; - int friend_number; - struct RTCPSession_s *rtcp_session; + uint32_t friend_number; + BWControler *bwc; void *cs; - int (*mcb) (void*, RTPMessage* msg); - + int (*mcb) (void *, struct RTPMessage *msg); } RTPSession; -/** - * Must be called before calling any other rtp function. - */ -RTPSession *rtp_new ( int payload_type, Messenger *m, int friend_num, void* cs, int (*mcb) (void*, RTPMessage*) ); -/** - * Terminate the session. - */ -void rtp_kill ( RTPSession* session ); -/** - * Do periodical rtp work. - */ -int rtp_do(RTPSession *session); -/** - * By default rtp is in receiving state - */ -int rtp_start_receiving (RTPSession *session); -/** - * Pause rtp receiving mode. - */ -int rtp_stop_receiving (RTPSession *session); -/** - * Sends msg to RTPSession::dest - */ -int rtp_send_data ( RTPSession* session, const uint8_t* data, uint16_t length, bool dummy ); -/** - * Dealloc msg. - */ -void rtp_free_msg ( RTPMessage *msg ); +RTPSession *rtp_new (int payload_type, Messenger *m, uint32_t friend_num, + BWControler *bwc, void *cs, + int (*mcb) (void *, struct RTPMessage *)); +void rtp_kill (RTPSession *session); +int rtp_allow_receiving (RTPSession *session); +int rtp_stop_receiving (RTPSession *session); +int rtp_send_data (RTPSession *session, const uint8_t *data, uint16_t length); #endif /* RTP_H */ diff --git a/toxav/toxav.c b/toxav/toxav.c index 8624a6b1f..4a413b66c 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -1,5 +1,5 @@ /** toxav.c - * + * * Copyright (C) 2013-2015 Tox project All Rights Reserved. * * This file is part of Tox. @@ -35,92 +35,74 @@ #include #define MAX_ENCODE_TIME_US ((1000 / 24) * 1000) -#define BITRATE_CHANGE_TESTING_TIME_MS 4000 - -typedef struct ToxAvBitrateAdapter_s { - bool active; - uint64_t end_time; - uint64_t next_send; - uint64_t next_send_interval; - uint32_t bit_rate; -} ToxAvBitrateAdapter; typedef struct ToxAVCall_s { - ToxAV* av; - + ToxAV *av; + pthread_mutex_t mutex_audio[1]; PAIR(RTPSession *, ACSession *) audio; - + pthread_mutex_t mutex_video[1]; PAIR(RTPSession *, VCSession *) video; - - pthread_mutex_t mutex[1]; - + + BWControler *bwc; + bool active; - MSICall* msi_call; + MSICall *msi_call; uint32_t friend_number; - + uint32_t audio_bit_rate; /* Sending audio bit rate */ uint32_t video_bit_rate; /* Sending video bit rate */ - - ToxAvBitrateAdapter aba; - ToxAvBitrateAdapter vba; - + /** Required for monitoring changes in states */ uint8_t previous_self_capabilities; - - /** Quality control */ - uint64_t time_audio_good; - uint32_t last_bad_audio_bit_rate; - uint64_t time_video_good; - uint32_t last_bad_video_bit_rate; - + + pthread_mutex_t mutex[1]; + struct ToxAVCall_s *prev; struct ToxAVCall_s *next; } ToxAVCall; -struct ToxAV { - Messenger* m; - MSISession* msi; - +struct ToxAV_s { + Messenger *m; + MSISession *msi; + /* Two-way storage: first is array of calls and second is list of calls with head and tail */ - ToxAVCall** calls; + ToxAVCall **calls; uint32_t calls_tail; uint32_t calls_head; pthread_mutex_t mutex[1]; - - PAIR(toxav_call_cb *, void*) ccb; /* Call callback */ + + PAIR(toxav_call_cb *, void *) ccb; /* Call callback */ PAIR(toxav_call_state_cb *, void *) scb; /* Call state callback */ PAIR(toxav_audio_receive_frame_cb *, void *) acb; /* Audio frame receive callback */ PAIR(toxav_video_receive_frame_cb *, void *) vcb; /* Video frame receive callback */ - PAIR(toxav_audio_bit_rate_status_cb *, void *) abcb; /* Audio bit rate control callback */ - PAIR(toxav_video_bit_rate_status_cb *, void *) vbcb; /* Video bit rate control callback */ - + PAIR(toxav_bit_rate_status_cb *, void *) bcb; /* Bit rate control callback */ + /** Decode time measures */ int32_t dmssc; /** Measure count */ int32_t dmsst; /** Last cycle total */ int32_t dmssa; /** Average decoding time in ms */ - + uint32_t interval; /** Calculated interval */ }; +void callback_bwc (BWControler *bwc, uint32_t friend_number, float loss, void *user_data); -int callback_invite(void* toxav_inst, MSICall* call); -int callback_start(void* toxav_inst, MSICall* call); -int callback_end(void* toxav_inst, MSICall* call); -int callback_error(void* toxav_inst, MSICall* call); -int callback_capabilites(void* toxav_inst, MSICall* call); +int callback_invite(void *toxav_inst, MSICall *call); +int callback_start(void *toxav_inst, MSICall *call); +int callback_end(void *toxav_inst, MSICall *call); +int callback_error(void *toxav_inst, MSICall *call); +int callback_capabilites(void *toxav_inst, MSICall *call); bool audio_bit_rate_invalid(uint32_t bit_rate); bool video_bit_rate_invalid(uint32_t bit_rate); -bool invoke_call_state_callback(ToxAV* av, uint32_t friend_number, uint32_t state); -ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error); -ToxAVCall* call_get(ToxAV* av, uint32_t friend_number); -ToxAVCall* call_remove(ToxAVCall* call); -bool call_prepare_transmission(ToxAVCall* call); -void call_kill_transmission(ToxAVCall* call); -void ba_set(ToxAvBitrateAdapter* ba, uint32_t bit_rate); -bool ba_shoud_send_dummy(ToxAvBitrateAdapter* ba); +bool invoke_call_state_callback(ToxAV *av, uint32_t friend_number, uint32_t state); +ToxAVCall *call_new(ToxAV *av, uint32_t friend_number, TOXAV_ERR_CALL *error); +ToxAVCall *call_get(ToxAV *av, uint32_t friend_number); +ToxAVCall *call_remove(ToxAVCall *call); +bool call_prepare_transmission(ToxAVCall *call); +void call_kill_transmission(ToxAVCall *call); uint32_t toxav_version_major(void) { @@ -139,79 +121,86 @@ bool toxav_version_is_compatible(uint32_t major, uint32_t minor, uint32_t patch) (void)major; (void)minor; (void)patch; - + return 1; } -ToxAV* toxav_new(Tox* tox, TOXAV_ERR_NEW* error) +ToxAV *toxav_new(Tox *tox, TOXAV_ERR_NEW *error) { TOXAV_ERR_NEW rc = TOXAV_ERR_NEW_OK; ToxAV *av = NULL; - + if (tox == NULL) { rc = TOXAV_ERR_NEW_NULL; goto END; } - - if (((Messenger*)tox)->msi_packet) { + + if (((Messenger *)tox)->msi_packet) { rc = TOXAV_ERR_NEW_MULTIPLE; goto END; } - + av = calloc (sizeof(ToxAV), 1); - + if (av == NULL) { LOGGER_WARNING("Allocation failed!"); rc = TOXAV_ERR_NEW_MALLOC; goto END; } - + if (create_recursive_mutex(av->mutex) != 0) { LOGGER_WARNING("Mutex creation failed!"); rc = TOXAV_ERR_NEW_MALLOC; goto END; } - + av->m = (Messenger *)tox; av->msi = msi_new(av->m); - + if (av->msi == NULL) { pthread_mutex_destroy(av->mutex); rc = TOXAV_ERR_NEW_MALLOC; goto END; } - + av->interval = 200; av->msi->av = av; - + msi_register_callback(av->msi, callback_invite, msi_OnInvite); msi_register_callback(av->msi, callback_start, msi_OnStart); msi_register_callback(av->msi, callback_end, msi_OnEnd); msi_register_callback(av->msi, callback_error, msi_OnError); msi_register_callback(av->msi, callback_error, msi_OnPeerTimeout); msi_register_callback(av->msi, callback_capabilites, msi_OnCapabilities); - + END: + if (error) *error = rc; - + if (rc != TOXAV_ERR_NEW_OK) { free(av); av = NULL; } - + return av; } -void toxav_kill(ToxAV* av) +void toxav_kill(ToxAV *av) { if (av == NULL) return; + pthread_mutex_lock(av->mutex); - msi_kill(av->msi); - + /* To avoid possible deadlocks */ + while (av->msi && msi_kill(av->msi) != 0) { + pthread_mutex_unlock(av->mutex); + pthread_mutex_lock(av->mutex); + } + /* Msi kill will hang up all calls so just clean these calls */ if (av->calls) { - ToxAVCall* it = call_get(av, av->calls_head); + ToxAVCall *it = call_get(av, av->calls_head); + while (it) { call_kill_transmission(it); it = call_remove(it); /* This will eventually free av->calls */ @@ -220,812 +209,614 @@ void toxav_kill(ToxAV* av) pthread_mutex_unlock(av->mutex); pthread_mutex_destroy(av->mutex); + free(av); } -Tox* toxav_get_tox(const ToxAV* av) +Tox *toxav_get_tox(const ToxAV *av) { - return (Tox*) av->m; + return (Tox *) av->m; } -uint32_t toxav_iteration_interval(const ToxAV* av) +uint32_t toxav_iteration_interval(const ToxAV *av) { /* If no call is active interval is 200 */ return av->calls ? av->interval : 200; } -void toxav_iterate(ToxAV* av) +void toxav_iterate(ToxAV *av) { pthread_mutex_lock(av->mutex); + if (av->calls == NULL) { pthread_mutex_unlock(av->mutex); return; } - + uint64_t start = current_time_monotonic(); int32_t rc = 500; - - ToxAVCall* i = av->calls[av->calls_head]; + + ToxAVCall *i = av->calls[av->calls_head]; + for (; i; i = i->next) { if (i->active) { pthread_mutex_lock(i->mutex); pthread_mutex_unlock(av->mutex); - - ac_do(i->audio.second); - if (rtp_do(i->audio.first) < 0) { - /* Bad transmission */ - - uint32_t bb = i->audio_bit_rate; - - if (i->aba.active) { - bb = i->aba.bit_rate; - /* Stop sending dummy packets */ - memset(&i->aba, 0, sizeof(i->aba)); - } - - /* Notify app */ - if (av->abcb.first) - av->abcb.first (av, i->friend_number, false, bb, av->abcb.second); - } else if (i->aba.active && i->aba.end_time < current_time_monotonic()) { - - i->audio_bit_rate = i->aba.bit_rate; - - /* Notify user about the new bit rate */ - if (av->abcb.first) - av->abcb.first (av, i->friend_number, true, i->aba.bit_rate, av->abcb.second); - - /* Stop sending dummy packets */ - memset(&i->aba, 0, sizeof(i->aba)); - } - - vc_do(i->video.second); - if (rtp_do(i->video.first) < 0) { - /* Bad transmission */ - uint32_t bb = i->video_bit_rate; - - if (i->vba.active) { - bb = i->vba.bit_rate; - /* Stop sending dummy packets */ - memset(&i->vba, 0, sizeof(i->vba)); - } - - /* Notify app */ - if (av->vbcb.first) - av->vbcb.first (av, i->friend_number, false, bb, av->vbcb.second); - - } else if (i->vba.active && i->vba.end_time < current_time_monotonic()) { - - i->video_bit_rate = i->vba.bit_rate; - - /* Notify user about the new bit rate */ - if (av->vbcb.first) - av->vbcb.first (av, i->friend_number, true, i->vba.bit_rate, av->vbcb.second); - - /* Stop sending dummy packets */ - memset(&i->vba, 0, sizeof(i->vba)); - } - - if (i->msi_call->self_capabilities & msi_CapRAudio && - i->msi_call->peer_capabilities & msi_CapSAudio) - rc = MIN(i->audio.second->last_packet_frame_duration, rc); - - if (i->msi_call->self_capabilities & msi_CapRVideo && - i->msi_call->peer_capabilities & msi_CapSVideo) + + ac_iterate(i->audio.second); + vc_iterate(i->video.second); + + if (i->msi_call->self_capabilities & msi_CapRAudio && + i->msi_call->peer_capabilities & msi_CapSAudio) + rc = MIN(i->audio.second->lp_frame_duration, rc); + + if (i->msi_call->self_capabilities & msi_CapRVideo && + i->msi_call->peer_capabilities & msi_CapSVideo) rc = MIN(i->video.second->lcfd, (uint32_t) rc); - + uint32_t fid = i->friend_number; - + pthread_mutex_unlock(i->mutex); pthread_mutex_lock(av->mutex); - + /* In case this call is popped from container stop iteration */ if (call_get(av, fid) != i) break; } } + pthread_mutex_unlock(av->mutex); - + av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa); av->dmsst += current_time_monotonic() - start; - + if (++av->dmssc == 3) { av->dmssa = av->dmsst / 3 + 5 /* NOTE Magic Offset for precission */; av->dmssc = 0; av->dmsst = 0; } } -bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) +bool toxav_call(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, + TOXAV_ERR_CALL *error) { - if ((audio_bit_rate && audio_bit_rate_invalid(audio_bit_rate)) - ||(video_bit_rate && video_bit_rate_invalid(video_bit_rate)) - ) { - if (error) - *error = TOXAV_ERR_CALL_INVALID_BIT_RATE; - return false; - } + TOXAV_ERR_CALL rc = TOXAV_ERR_CALL_OK; pthread_mutex_lock(av->mutex); - ToxAVCall* call = call_new(av, friend_number, error); - if (call == NULL) { - pthread_mutex_unlock(av->mutex); - return false; - } + if ((audio_bit_rate && audio_bit_rate_invalid(audio_bit_rate)) + || (video_bit_rate && video_bit_rate_invalid(video_bit_rate))) { + rc = TOXAV_ERR_CALL_INVALID_BIT_RATE; + goto END; + } + + ToxAVCall *call = call_new(av, friend_number, error); + + if (call == NULL) { + rc = TOXAV_ERR_CALL_MALLOC; + goto END; + } + call->audio_bit_rate = audio_bit_rate; call->video_bit_rate = video_bit_rate; - + call->previous_self_capabilities = msi_CapRAudio | msi_CapRVideo; - + call->previous_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; call->previous_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; - + if (msi_invite(av->msi, &call->msi_call, friend_number, call->previous_self_capabilities) != 0) { call_remove(call); - if (error) - *error = TOXAV_ERR_CALL_MALLOC; - pthread_mutex_unlock(av->mutex); - return false; + rc = TOXAV_ERR_CALL_SYNC; + goto END; } - + call->msi_call->av_call = call; + +END: pthread_mutex_unlock(av->mutex); - return true; + if (error) + *error = rc; + + return rc == TOXAV_ERR_CALL_OK; } -void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data) +void toxav_callback_call(ToxAV *av, toxav_call_cb *function, void *user_data) { pthread_mutex_lock(av->mutex); av->ccb.first = function; av->ccb.second = user_data; pthread_mutex_unlock(av->mutex); } -bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER* error) +bool toxav_answer(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, + TOXAV_ERR_ANSWER *error) { pthread_mutex_lock(av->mutex); - + TOXAV_ERR_ANSWER rc = TOXAV_ERR_ANSWER_OK; + if (m_friend_exists(av->m, friend_number) == 0) { rc = TOXAV_ERR_ANSWER_FRIEND_NOT_FOUND; goto END; } - + if ((audio_bit_rate && audio_bit_rate_invalid(audio_bit_rate)) - ||(video_bit_rate && video_bit_rate_invalid(video_bit_rate)) - ) { + || (video_bit_rate && video_bit_rate_invalid(video_bit_rate)) + ) { rc = TOXAV_ERR_ANSWER_INVALID_BIT_RATE; goto END; } - - ToxAVCall* call = call_get(av, friend_number); + + ToxAVCall *call = call_get(av, friend_number); + if (call == NULL) { rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; goto END; } - + if (!call_prepare_transmission(call)) { - rc = TOXAV_ERR_ANSWER_CODEC_INITIALIZATION; - goto END; - } - + rc = TOXAV_ERR_ANSWER_CODEC_INITIALIZATION; + goto END; + } + call->audio_bit_rate = audio_bit_rate; call->video_bit_rate = video_bit_rate; - + call->previous_self_capabilities = msi_CapRAudio | msi_CapRVideo; - + call->previous_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; call->previous_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; - + if (msi_answer(call->msi_call, call->previous_self_capabilities) != 0) - rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; /* the only reason for msi_answer to fail */ - - + rc = TOXAV_ERR_ANSWER_SYNC; + END: pthread_mutex_unlock(av->mutex); - + if (error) *error = rc; - + return rc == TOXAV_ERR_ANSWER_OK; } -void toxav_callback_call_state(ToxAV* av, toxav_call_state_cb* function, void* user_data) +void toxav_callback_call_state(ToxAV *av, toxav_call_state_cb *function, void *user_data) { pthread_mutex_lock(av->mutex); av->scb.first = function; av->scb.second = user_data; pthread_mutex_unlock(av->mutex); } -bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL* error) +bool toxav_call_control(ToxAV *av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL *error) { pthread_mutex_lock(av->mutex); TOXAV_ERR_CALL_CONTROL rc = TOXAV_ERR_CALL_CONTROL_OK; - + if (m_friend_exists(av->m, friend_number) == 0) { rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_FOUND; goto END; } - - ToxAVCall* call = call_get(av, friend_number); + + ToxAVCall *call = call_get(av, friend_number); + if (call == NULL || (!call->active && control != TOXAV_CALL_CONTROL_CANCEL)) { rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; goto END; } - + switch (control) { case TOXAV_CALL_CONTROL_RESUME: { /* Only act if paused and had media transfer active before */ - if (call->msi_call->self_capabilities == 0 && - call->previous_self_capabilities ) { - - if (msi_change_capabilities(call->msi_call, - call->previous_self_capabilities) == -1) { - /* The only reason for this function to fail is invalid state - * ( not active ) */ - rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + if (call->msi_call->self_capabilities == 0 && + call->previous_self_capabilities) { + + if (msi_change_capabilities(call->msi_call, + call->previous_self_capabilities) == -1) { + rc = TOXAV_ERR_CALL_CONTROL_SYNC; goto END; } - - rtp_start_receiving(call->audio.first); - rtp_start_receiving(call->video.first); + + rtp_allow_receiving(call->audio.first); + rtp_allow_receiving(call->video.first); } else { rc = TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION; goto END; } - } break; - + } + break; + case TOXAV_CALL_CONTROL_PAUSE: { /* Only act if not already paused */ if (call->msi_call->self_capabilities) { call->previous_self_capabilities = call->msi_call->self_capabilities; - - if (msi_change_capabilities(call->msi_call, 0) == -1 ) { - /* The only reason for this function to fail is invalid state - * ( not active ) */ - rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + + if (msi_change_capabilities(call->msi_call, 0) == -1) { + rc = TOXAV_ERR_CALL_CONTROL_SYNC; goto END; } - + rtp_stop_receiving(call->audio.first); rtp_stop_receiving(call->video.first); } else { rc = TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION; goto END; } - } break; - + } + break; + case TOXAV_CALL_CONTROL_CANCEL: { /* Hang up */ - msi_hangup(call->msi_call); - + if (msi_hangup(call->msi_call) != 0) { + rc = TOXAV_ERR_CALL_CONTROL_SYNC; + goto END; + } + /* No mather the case, terminate the call */ call_kill_transmission(call); call_remove(call); - } break; - + } + break; + case TOXAV_CALL_CONTROL_MUTE_AUDIO: { if (call->msi_call->self_capabilities & msi_CapRAudio) { if (msi_change_capabilities(call->msi_call, call-> - msi_call->self_capabilities ^ msi_CapRAudio) == -1) { - /* The only reason for this function to fail is invalid state - * ( not active ) */ - rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + msi_call->self_capabilities ^ msi_CapRAudio) == -1) { + rc = TOXAV_ERR_CALL_CONTROL_SYNC; goto END; } - + rtp_stop_receiving(call->audio.first); } else { rc = TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION; goto END; } - } break; - + } + break; + case TOXAV_CALL_CONTROL_UNMUTE_AUDIO: { if (call->msi_call->self_capabilities ^ msi_CapRAudio) { if (msi_change_capabilities(call->msi_call, call-> - msi_call->self_capabilities | msi_CapRAudio) == -1) { - /* The only reason for this function to fail is invalid state - * ( not active ) */ - rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + msi_call->self_capabilities | msi_CapRAudio) == -1) { + rc = TOXAV_ERR_CALL_CONTROL_SYNC; goto END; } - - rtp_start_receiving(call->audio.first); + + rtp_allow_receiving(call->audio.first); } else { rc = TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION; goto END; } - } break; - + } + break; + case TOXAV_CALL_CONTROL_HIDE_VIDEO: { if (call->msi_call->self_capabilities & msi_CapRVideo) { if (msi_change_capabilities(call->msi_call, call-> - msi_call->self_capabilities ^ msi_CapRVideo) == -1) { - /* The only reason for this function to fail is invalid state - * ( not active ) */ - rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + msi_call->self_capabilities ^ msi_CapRVideo) == -1) { + rc = TOXAV_ERR_CALL_CONTROL_SYNC; goto END; } - + rtp_stop_receiving(call->video.first); } else { rc = TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION; goto END; } - } break; - + } + break; + case TOXAV_CALL_CONTROL_SHOW_VIDEO: { if (call->msi_call->self_capabilities ^ msi_CapRVideo) { if (msi_change_capabilities(call->msi_call, call-> - msi_call->self_capabilities | msi_CapRVideo) == -1) { - /* The only reason for this function to fail is invalid state - * ( not active ) */ - rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; + msi_call->self_capabilities | msi_CapRVideo) == -1) { + rc = TOXAV_ERR_CALL_CONTROL_SYNC; goto END; } - - rtp_start_receiving(call->audio.first); + + rtp_allow_receiving(call->audio.first); } else { rc = TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION; goto END; } - } break; + } + break; } - + END: pthread_mutex_unlock(av->mutex); - + if (error) *error = rc; - + return rc == TOXAV_ERR_CALL_CONTROL_OK; } -void toxav_callback_audio_bit_rate_status(ToxAV* av, toxav_audio_bit_rate_status_cb* function, void* user_data) +bool toxav_bit_rate_set(ToxAV *av, uint32_t friend_number, int32_t audio_bit_rate, + int32_t video_bit_rate, TOXAV_ERR_BIT_RATE_SET *error) { - pthread_mutex_lock(av->mutex); - av->abcb.first = function; - av->abcb.second = user_data; - pthread_mutex_unlock(av->mutex); -} -bool toxav_audio_bit_rate_set(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, bool force, TOXAV_ERR_SET_BIT_RATE* error) -{ - LOGGER_DEBUG("Setting new audio bitrate to: %d", audio_bit_rate); - - TOXAV_ERR_SET_BIT_RATE rc = TOXAV_ERR_SET_BIT_RATE_OK; - ToxAVCall* call; - + TOXAV_ERR_BIT_RATE_SET rc = TOXAV_ERR_BIT_RATE_SET_OK; + ToxAVCall *call; + if (m_friend_exists(av->m, friend_number) == 0) { - rc = TOXAV_ERR_SET_BIT_RATE_FRIEND_NOT_FOUND; + rc = TOXAV_ERR_BIT_RATE_SET_FRIEND_NOT_FOUND; goto END; } - - if (audio_bit_rate && audio_bit_rate_invalid(audio_bit_rate)) { - rc = TOXAV_ERR_SET_BIT_RATE_INVALID; + + if (audio_bit_rate > 0 && audio_bit_rate_invalid(audio_bit_rate)) { + rc = TOXAV_ERR_BIT_RATE_SET_INVALID_AUDIO_BIT_RATE; goto END; } - + + if (video_bit_rate > 0 && video_bit_rate_invalid(video_bit_rate)) { + rc = TOXAV_ERR_BIT_RATE_SET_INVALID_VIDEO_BIT_RATE; + goto END; + } + pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); + if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { pthread_mutex_unlock(av->mutex); - rc = TOXAV_ERR_SET_BIT_RATE_FRIEND_NOT_IN_CALL; + rc = TOXAV_ERR_BIT_RATE_SET_FRIEND_NOT_IN_CALL; goto END; } - - if (call->audio_bit_rate == audio_bit_rate || (call->aba.active && call->aba.bit_rate == audio_bit_rate)) { - pthread_mutex_unlock(av->mutex); - goto END; - } - - /* Video sending is turned off; notify peer */ - if (audio_bit_rate == 0) { - call->audio_bit_rate = 0; - - msi_change_capabilities(call->msi_call, call->msi_call-> - self_capabilities ^ msi_CapSAudio); - pthread_mutex_unlock(av->mutex); - goto END; - } - - pthread_mutex_lock(call->mutex); - - if (call->audio_bit_rate == 0) { - /* The audio has been turned off before this */ - call->audio_bit_rate = audio_bit_rate; - - msi_change_capabilities(call->msi_call, call-> - msi_call->self_capabilities | msi_CapSAudio); - - if (!force && av->abcb.first) - av->abcb.first (av, call->friend_number, true, audio_bit_rate, av->abcb.second); - } else { - /* The audio was active before this */ - if (audio_bit_rate > call->audio_bit_rate && !force) - ba_set(&call->aba, audio_bit_rate); - else { - /* Cancel any previous non forceful bitrate change request */ - memset(&call->aba, 0, sizeof(call->aba)); + + if (audio_bit_rate >= 0) { + LOGGER_DEBUG("Setting new audio bitrate to: %d", audio_bit_rate); + + if (call->audio_bit_rate == audio_bit_rate) { + LOGGER_DEBUG("Audio bitrate already set to: %d", audio_bit_rate); + } else if (audio_bit_rate == 0) { + LOGGER_DEBUG("Turned off audio sending"); + if (msi_change_capabilities(call->msi_call, call->msi_call-> + self_capabilities ^ msi_CapSAudio) != 0) { + pthread_mutex_unlock(av->mutex); + rc = TOXAV_ERR_BIT_RATE_SET_SYNC; + goto END; + } + /* Audio sending is turned off; notify peer */ + call->audio_bit_rate = 0; + } else { + pthread_mutex_lock(call->mutex); + if (call->audio_bit_rate == 0) { + LOGGER_DEBUG("Turned on audio sending"); + /* The audio has been turned off before this */ + if (msi_change_capabilities(call->msi_call, call-> + msi_call->self_capabilities | msi_CapSAudio) != 0) { + pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(av->mutex); + rc = TOXAV_ERR_BIT_RATE_SET_SYNC; + goto END; + } + } else + LOGGER_DEBUG("Set new audio bit rate %d", audio_bit_rate); call->audio_bit_rate = audio_bit_rate; - - if (!force && av->abcb.first) - av->abcb.first (av, call->friend_number, true, audio_bit_rate, av->abcb.second); + pthread_mutex_unlock(call->mutex); } } - - pthread_mutex_unlock(call->mutex); - pthread_mutex_unlock(av->mutex); - -END: - if (error) - *error = rc; - - return rc == TOXAV_ERR_SET_BIT_RATE_OK; -} -void toxav_callback_video_bit_rate_status(ToxAV* av, toxav_video_bit_rate_status_cb* function, void* user_data) -{ - pthread_mutex_lock(av->mutex); - av->vbcb.first = function; - av->vbcb.second = user_data; - pthread_mutex_unlock(av->mutex); -} -bool toxav_video_bit_rate_set(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, bool force, TOXAV_ERR_SET_BIT_RATE* error) -{ - LOGGER_DEBUG("Setting new video bitrate to: %d", video_bit_rate); - - TOXAV_ERR_SET_BIT_RATE rc = TOXAV_ERR_SET_BIT_RATE_OK; - ToxAVCall* call; - - if (m_friend_exists(av->m, friend_number) == 0) { - rc = TOXAV_ERR_SET_BIT_RATE_FRIEND_NOT_FOUND; - goto END; - } - - if (video_bit_rate && video_bit_rate_invalid(video_bit_rate)) { - rc = TOXAV_ERR_SET_BIT_RATE_INVALID; - goto END; - } - - pthread_mutex_lock(av->mutex); - call = call_get(av, friend_number); - if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - pthread_mutex_unlock(av->mutex); - rc = TOXAV_ERR_SET_BIT_RATE_FRIEND_NOT_IN_CALL; - goto END; - } - - if (call->video_bit_rate == video_bit_rate || (call->vba.active && call->vba.bit_rate == video_bit_rate)) { - pthread_mutex_unlock(av->mutex); - goto END; - } - - /* Video sending is turned off; notify peer */ - if (video_bit_rate == 0) { - call->video_bit_rate = 0; - - msi_change_capabilities(call->msi_call, call->msi_call-> - self_capabilities ^ msi_CapSVideo); - pthread_mutex_unlock(av->mutex); - goto END; - } - - pthread_mutex_lock(call->mutex); - - if (call->video_bit_rate == 0) { - /* The video has been turned off before this */ - call->video_bit_rate = video_bit_rate; - - msi_change_capabilities(call->msi_call, call-> - msi_call->self_capabilities | msi_CapSVideo); - - if (!force && av->vbcb.first) - av->vbcb.first (av, call->friend_number, true, video_bit_rate, av->vbcb.second); - } else { - /* The video was active before this */ - if (video_bit_rate > call->video_bit_rate && !force) - ba_set(&call->vba, video_bit_rate); - else { - /* Cancel any previous non forceful bitrate change request */ - memset(&call->vba, 0, sizeof(call->vba)); + + if (video_bit_rate >= 0) { + LOGGER_DEBUG("Setting new video bitrate to: %d", video_bit_rate); + + if (call->video_bit_rate == video_bit_rate) { + LOGGER_DEBUG("Video bitrate already set to: %d", video_bit_rate); + } else if (video_bit_rate == 0) { + LOGGER_DEBUG("Turned off video sending"); + /* Video sending is turned off; notify peer */ + if (msi_change_capabilities(call->msi_call, call->msi_call-> + self_capabilities ^ msi_CapSVideo) != 0) { + pthread_mutex_unlock(av->mutex); + rc = TOXAV_ERR_BIT_RATE_SET_SYNC; + goto END; + } + call->video_bit_rate = 0; + } else { + pthread_mutex_lock(call->mutex); + if (call->video_bit_rate == 0) { + LOGGER_DEBUG("Turned on video sending"); + /* The video has been turned off before this */ + if (msi_change_capabilities(call->msi_call, call-> + msi_call->self_capabilities | msi_CapSVideo) != 0) { + pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(av->mutex); + rc = TOXAV_ERR_BIT_RATE_SET_SYNC; + goto END; + } + } else + LOGGER_DEBUG("Set new video bit rate %d", video_bit_rate); call->video_bit_rate = video_bit_rate; - - if (!force && av->vbcb.first) - av->vbcb.first (av, call->friend_number, true, video_bit_rate, av->vbcb.second); + pthread_mutex_unlock(call->mutex); } } - pthread_mutex_unlock(call->mutex); pthread_mutex_unlock(av->mutex); - END: if (error) *error = rc; - - return rc == TOXAV_ERR_SET_BIT_RATE_OK; + + return rc == TOXAV_ERR_BIT_RATE_SET_OK; } -bool toxav_audio_send_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error) +void toxav_callback_bit_rate_status(ToxAV *av, toxav_bit_rate_status_cb *function, void *user_data) +{ + pthread_mutex_lock(av->mutex); + av->bcb.first = function; + av->bcb.second = user_data; + pthread_mutex_unlock(av->mutex); +} +bool toxav_audio_send_frame(ToxAV *av, uint32_t friend_number, const int16_t *pcm, size_t sample_count, + uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME *error) { TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; - ToxAVCall* call; - + ToxAVCall *call; + if (m_friend_exists(av->m, friend_number) == 0) { rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND; goto END; } - + pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); + if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; } - + if (call->audio_bit_rate == 0 || - !(call->msi_call->self_capabilities & msi_CapSAudio) || - !(call->msi_call->peer_capabilities & msi_CapRAudio)) { + !(call->msi_call->self_capabilities & msi_CapSAudio) || + !(call->msi_call->peer_capabilities & msi_CapRAudio)) { pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_PAYLOAD_TYPE_DISABLED; goto END; } - + pthread_mutex_lock(call->mutex_audio); pthread_mutex_unlock(av->mutex); - - if ( pcm == NULL ) { + + if (pcm == NULL) { pthread_mutex_unlock(call->mutex_audio); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } - - if ( channels > 2 ) { + + if (channels > 2) { pthread_mutex_unlock(call->mutex_audio); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } - + { /* Encode and send */ if (ac_reconfigure_encoder(call->audio.second, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) { pthread_mutex_unlock(call->mutex_audio); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } - + uint8_t dest[sample_count + sizeof(sampling_rate)]; /* This is more than enough always */ - + sampling_rate = htonl(sampling_rate); memcpy(dest, &sampling_rate, sizeof(sampling_rate)); int vrc = opus_encode(call->audio.second->encoder, pcm, sample_count, dest + sizeof(sampling_rate), sizeof(dest) - sizeof(sampling_rate)); - + if (vrc < 0) { LOGGER_WARNING("Failed to encode frame %s", opus_strerror(vrc)); pthread_mutex_unlock(call->mutex_audio); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } - - if (rtp_send_data(call->audio.first, dest, vrc + sizeof(sampling_rate), false) != 0) { + + if (rtp_send_data(call->audio.first, dest, vrc + sizeof(sampling_rate)) != 0) { LOGGER_WARNING("Failed to send audio packet"); rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; } - - - /* For bit rate measurement; send dummy packet */ - if (ba_shoud_send_dummy(&call->aba)) { - sampling_rate = ntohl(sampling_rate); - if (ac_reconfigure_test_encoder(call->audio.second, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) { - /* FIXME should the bit rate changing fail here? */ - pthread_mutex_unlock(call->mutex_audio); - rc = TOXAV_ERR_SEND_FRAME_INVALID; - goto END; - } - - sampling_rate = htonl(sampling_rate); - memcpy(dest, &sampling_rate, sizeof(sampling_rate)); - vrc = opus_encode(call->audio.second->test_encoder, pcm, sample_count, - dest + sizeof(sampling_rate), sizeof(dest) - sizeof(sampling_rate)); - - if (vrc < 0) { - LOGGER_WARNING("Failed to encode frame %s", opus_strerror(vrc)); - pthread_mutex_unlock(call->mutex_audio); - rc = TOXAV_ERR_SEND_FRAME_INVALID; - goto END; - } - - if (rtp_send_data(call->audio.first, dest, vrc + sizeof(sampling_rate), true) != 0) { - LOGGER_WARNING("Failed to send audio packet"); - rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; - } - - if (call->aba.end_time == (uint64_t) ~0) - call->aba.end_time = current_time_monotonic() + BITRATE_CHANGE_TESTING_TIME_MS; - } } - - + + pthread_mutex_unlock(call->mutex_audio); - + END: if (error) *error = rc; - + return rc == TOXAV_ERR_SEND_FRAME_OK; } -bool toxav_video_send_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) +bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t *y, + const uint8_t *u, const uint8_t *v, TOXAV_ERR_SEND_FRAME *error) { TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; - ToxAVCall* call; - + ToxAVCall *call; + if (m_friend_exists(av->m, friend_number) == 0) { rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND; goto END; } - + pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); + if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; } - + if (call->video_bit_rate == 0 || - !(call->msi_call->self_capabilities & msi_CapSVideo) || - !(call->msi_call->peer_capabilities & msi_CapRVideo)) { + !(call->msi_call->self_capabilities & msi_CapSVideo) || + !(call->msi_call->peer_capabilities & msi_CapRVideo)) { pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_PAYLOAD_TYPE_DISABLED; goto END; } - + pthread_mutex_lock(call->mutex_video); pthread_mutex_unlock(av->mutex); - - if ( y == NULL || u == NULL || v == NULL ) { + + if (y == NULL || u == NULL || v == NULL) { pthread_mutex_unlock(call->mutex_video); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } - - if ( vc_reconfigure_encoder(call->video.second->encoder, call->video_bit_rate * 1000, width, height) != 0 ) { + + if (vc_reconfigure_encoder(call->video.second->encoder, call->video_bit_rate * 1000, width, height) != 0) { pthread_mutex_unlock(call->mutex_video); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } - + { /* Encode */ vpx_image_t img; img.w = img.h = img.d_w = img.d_h = 0; - vpx_img_alloc(&img, VPX_IMG_FMT_VPXI420, width, height, 1); - - /* I420 "It comprises an NxM Y plane followed by (N/2)x(M/2) V and U planes." + vpx_img_alloc(&img, VPX_IMG_FMT_I420, width, height, 0); + + /* I420 "It comprises an NxM Y plane followed by (N/2)x(M/2) V and U planes." * http://fourcc.org/yuv.php#IYUV */ memcpy(img.planes[VPX_PLANE_Y], y, width * height); - memcpy(img.planes[VPX_PLANE_U], u, (width/2) * (height/2)); - memcpy(img.planes[VPX_PLANE_V], v, (width/2) * (height/2)); - - int vrc = vpx_codec_encode(call->video.second->encoder, &img, + memcpy(img.planes[VPX_PLANE_U], u, (width / 2) * (height / 2)); + memcpy(img.planes[VPX_PLANE_V], v, (width / 2) * (height / 2)); + + int vrc = vpx_codec_encode(call->video.second->encoder, &img, call->video.second->frame_counter, 1, 0, MAX_ENCODE_TIME_US); - + vpx_img_free(&img); - if ( vrc != VPX_CODEC_OK) { + + if (vrc != VPX_CODEC_OK) { pthread_mutex_unlock(call->mutex_video); LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } } - + ++call->video.second->frame_counter; - - { /* Split and send */ + + { /* Send frames */ vpx_codec_iter_t iter = NULL; const vpx_codec_cx_pkt_t *pkt; - - vc_init_video_splitter_cycle(call->video.second); - - while ( (pkt = vpx_codec_get_cx_data(call->video.second->encoder, &iter)) ) { - if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { - int parts = vc_update_video_splitter_cycle(call->video.second, pkt->data.frame.buf, - pkt->data.frame.sz); - - if (parts < 0) /* Should never happen though */ - continue; - - uint16_t part_size; - const uint8_t *iter; - - int i; - for (i = 0; i < parts; i++) { - iter = vc_iterate_split_video_frame(call->video.second, &part_size); - - if (rtp_send_data(call->video.first, iter, part_size, false) < 0) { - pthread_mutex_unlock(call->mutex_video); - LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); - rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; - goto END; - } - } + + while ((pkt = vpx_codec_get_cx_data(call->video.second->encoder, &iter))) { + if (pkt->kind == VPX_CODEC_CX_FRAME_PKT && + rtp_send_data(call->video.first, pkt->data.frame.buf, pkt->data.frame.sz) < 0) { + + pthread_mutex_unlock(call->mutex_video); + LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); + rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; + goto END; } } } - - if (ba_shoud_send_dummy(&call->vba)) { - if ( vc_reconfigure_encoder(call->video.second->test_encoder, call->vba.bit_rate * 1000, width, height) != 0 ) { - pthread_mutex_unlock(call->mutex_video); - rc = TOXAV_ERR_SEND_FRAME_INVALID; - goto END; - } - - /* FIXME use the same image as before */ - vpx_image_t img; - img.w = img.h = img.d_w = img.d_h = 0; - vpx_img_alloc(&img, VPX_IMG_FMT_VPXI420, width, height, 1); - - /* I420 "It comprises an NxM Y plane followed by (N/2)x(M/2) V and U planes." - * http://fourcc.org/yuv.php#IYUV - */ - memcpy(img.planes[VPX_PLANE_Y], y, width * height); - memcpy(img.planes[VPX_PLANE_U], u, (width/2) * (height/2)); - memcpy(img.planes[VPX_PLANE_V], v, (width/2) * (height/2)); - - int vrc = vpx_codec_encode(call->video.second->test_encoder, &img, - call->video.second->test_frame_counter, 1, 0, MAX_ENCODE_TIME_US); - - vpx_img_free(&img); - if ( vrc != VPX_CODEC_OK) { - pthread_mutex_unlock(call->mutex_video); - LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); - rc = TOXAV_ERR_SEND_FRAME_INVALID; - goto END; - } - - call->video.second->test_frame_counter++; - - vpx_codec_iter_t iter = NULL; - const vpx_codec_cx_pkt_t *pkt; - - /* Send the encoded data as dummy packets */ - while ( (pkt = vpx_codec_get_cx_data(call->video.second->test_encoder, &iter)) ) { - if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { - - int parts = pkt->data.frame.sz / 1300; - int i; - for (i = 0; i < parts; i++) { - if (rtp_send_data(call->video.first, pkt->data.frame.buf + i * 1300, 1300, true) < 0) { - pthread_mutex_unlock(call->mutex_video); - LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); - rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; - goto END; - } - } - - if (pkt->data.frame.sz % 1300) { - if (rtp_send_data(call->video.first, pkt->data.frame.buf + parts * 1300, pkt->data.frame.sz % 1300, true) < 0) { - pthread_mutex_unlock(call->mutex_video); - LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); - rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; - goto END; - } - } - } - } - - if (call->vba.end_time == (uint64_t) ~0) - call->vba.end_time = current_time_monotonic() + BITRATE_CHANGE_TESTING_TIME_MS; - } - + pthread_mutex_unlock(call->mutex_video); - + END: if (error) *error = rc; - + return rc == TOXAV_ERR_SEND_FRAME_OK; } -void toxav_callback_audio_receive_frame(ToxAV* av, toxav_audio_receive_frame_cb* function, void* user_data) +void toxav_callback_audio_receive_frame(ToxAV *av, toxav_audio_receive_frame_cb *function, void *user_data) { pthread_mutex_lock(av->mutex); av->acb.first = function; av->acb.second = user_data; pthread_mutex_unlock(av->mutex); } -void toxav_callback_video_receive_frame(ToxAV* av, toxav_video_receive_frame_cb* function, void* user_data) +void toxav_callback_video_receive_frame(ToxAV *av, toxav_video_receive_frame_cb *function, void *user_data) { pthread_mutex_lock(av->mutex); av->vcb.first = function; @@ -1039,108 +830,144 @@ void toxav_callback_video_receive_frame(ToxAV* av, toxav_video_receive_frame_cb* * :: Internal * ******************************************************************************/ -int callback_invite(void* toxav_inst, MSICall* call) +void callback_bwc(BWControler* bwc, uint32_t friend_number, float loss, void* user_data) { - ToxAV* toxav = toxav_inst; - pthread_mutex_lock(toxav->mutex); + /* Callback which is called when the internal measure mechanism reported packet loss. + * We report suggested lowered bitrate to an app. If app is sending both audio and video, + * we will report lowered bitrate for video only because in that case video probably + * takes more than 90% bandwidth. Otherwise, we report lowered bitrate on audio. + * The application may choose to disable video totally if the stream is too bad. + */ - ToxAVCall* av_call = call_new(toxav, call->friend_number, NULL); + ToxAVCall* call = user_data; + assert(call); + + LOGGER_DEBUG("Reported loss of %f%%", loss*100); + + if (loss < .01f) + return; + + pthread_mutex_lock(call->av->mutex); + if (!call->av->bcb.first) { + pthread_mutex_unlock(call->av->mutex); + LOGGER_WARNING("No callback to report loss on"); + return; + } + + if (call->video_bit_rate) + (*call->av->bcb.first) (call->av, friend_number, call->audio_bit_rate, + call->video_bit_rate - (call->video_bit_rate * loss), + call->av->bcb.second); + else if (call->audio_bit_rate) + (*call->av->bcb.first) (call->av, friend_number, + call->audio_bit_rate - (call->audio_bit_rate * loss), + 0, call->av->bcb.second); + + pthread_mutex_unlock(call->av->mutex); +} +int callback_invite(void *toxav_inst, MSICall *call) +{ + ToxAV *toxav = toxav_inst; + pthread_mutex_lock(toxav->mutex); + + ToxAVCall *av_call = call_new(toxav, call->friend_number, NULL); + if (av_call == NULL) { LOGGER_WARNING("Failed to initialize call..."); pthread_mutex_unlock(toxav->mutex); return -1; } - + call->av_call = av_call; av_call->msi_call = call; - + if (toxav->ccb.first) - toxav->ccb.first(toxav, call->friend_number, call->peer_capabilities & msi_CapSAudio, + toxav->ccb.first(toxav, call->friend_number, call->peer_capabilities & msi_CapSAudio, call->peer_capabilities & msi_CapSVideo, toxav->ccb.second); else { /* No handler to capture the call request, send failure */ pthread_mutex_unlock(toxav->mutex); return -1; } - + pthread_mutex_unlock(toxav->mutex); return 0; } -int callback_start(void* toxav_inst, MSICall* call) +int callback_start(void *toxav_inst, MSICall *call) { - ToxAV* toxav = toxav_inst; + ToxAV *toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - - ToxAVCall* av_call = call_get(toxav, call->friend_number); - + + ToxAVCall *av_call = call_get(toxav, call->friend_number); + if (av_call == NULL) { /* Should this ever happen? */ pthread_mutex_unlock(toxav->mutex); return -1; } - + if (!call_prepare_transmission(av_call)) { callback_error(toxav_inst, call); pthread_mutex_unlock(toxav->mutex); return -1; } - + if (!invoke_call_state_callback(toxav, call->friend_number, call->peer_capabilities)) { callback_error(toxav_inst, call); pthread_mutex_unlock(toxav->mutex); return -1; } - + pthread_mutex_unlock(toxav->mutex); return 0; } -int callback_end(void* toxav_inst, MSICall* call) +int callback_end(void *toxav_inst, MSICall *call) { - ToxAV* toxav = toxav_inst; + ToxAV *toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - + invoke_call_state_callback(toxav, call->friend_number, TOXAV_FRIEND_CALL_STATE_FINISHED); - + if (call->av_call) { call_kill_transmission(call->av_call); call_remove(call->av_call); } - + pthread_mutex_unlock(toxav->mutex); return 0; } -int callback_error(void* toxav_inst, MSICall* call) +int callback_error(void *toxav_inst, MSICall *call) { - ToxAV* toxav = toxav_inst; + ToxAV *toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - + invoke_call_state_callback(toxav, call->friend_number, TOXAV_FRIEND_CALL_STATE_ERROR); - + if (call->av_call) { call_kill_transmission(call->av_call); call_remove(call->av_call); } - + pthread_mutex_unlock(toxav->mutex); return 0; } -int callback_capabilites(void* toxav_inst, MSICall* call) +int callback_capabilites(void *toxav_inst, MSICall *call) { - ToxAV* toxav = toxav_inst; + ToxAV *toxav = toxav_inst; pthread_mutex_lock(toxav->mutex); - + if (call->peer_capabilities & msi_CapSAudio) - rtp_start_receiving(((ToxAVCall*)call->av_call)->audio.first); + rtp_allow_receiving(((ToxAVCall *)call->av_call)->audio.first); else - rtp_stop_receiving(((ToxAVCall*)call->av_call)->audio.first); - + rtp_stop_receiving(((ToxAVCall *)call->av_call)->audio.first); + if (call->peer_capabilities & msi_CapSVideo) - rtp_start_receiving(((ToxAVCall*)call->av_call)->video.first); + rtp_allow_receiving(((ToxAVCall *)call->av_call)->video.first); else - rtp_stop_receiving(((ToxAVCall*)call->av_call)->video.first); - + rtp_stop_receiving(((ToxAVCall *)call->av_call)->video.first); + invoke_call_state_callback(toxav, call->friend_number, call->peer_capabilities); - + pthread_mutex_unlock(toxav->mutex); return 0; } @@ -1157,201 +984,212 @@ bool video_bit_rate_invalid(uint32_t bit_rate) /* TODO: If anyone knows the answer to this one please fill it up */ return false; } -bool invoke_call_state_callback(ToxAV* av, uint32_t friend_number, uint32_t state) +bool invoke_call_state_callback(ToxAV *av, uint32_t friend_number, uint32_t state) { if (av->scb.first) av->scb.first(av, friend_number, state, av->scb.second); else return false; + return true; } -ToxAVCall* call_new(ToxAV* av, uint32_t friend_number, TOXAV_ERR_CALL* error) +ToxAVCall *call_new(ToxAV *av, uint32_t friend_number, TOXAV_ERR_CALL *error) { /* Assumes mutex locked */ TOXAV_ERR_CALL rc = TOXAV_ERR_CALL_OK; - ToxAVCall* call = NULL; - + ToxAVCall *call = NULL; + if (m_friend_exists(av->m, friend_number) == 0) { rc = TOXAV_ERR_CALL_FRIEND_NOT_FOUND; goto END; } - + if (m_get_friend_connectionstatus(av->m, friend_number) < 1) { rc = TOXAV_ERR_CALL_FRIEND_NOT_CONNECTED; goto END; } - + if (call_get(av, friend_number) != NULL) { rc = TOXAV_ERR_CALL_FRIEND_ALREADY_IN_CALL; goto END; } - - + + call = calloc(sizeof(ToxAVCall), 1); - + if (call == NULL) { rc = TOXAV_ERR_CALL_MALLOC; goto END; } - + call->av = av; call->friend_number = friend_number; - + if (av->calls == NULL) { /* Creating */ - av->calls = calloc (sizeof(ToxAVCall*), friend_number + 1); - + av->calls = calloc (sizeof(ToxAVCall *), friend_number + 1); + if (av->calls == NULL) { free(call); call = NULL; rc = TOXAV_ERR_CALL_MALLOC; goto END; } - + av->calls_tail = av->calls_head = friend_number; - + } else if (av->calls_tail < friend_number) { /* Appending */ - void* tmp = realloc(av->calls, sizeof(ToxAVCall*) * friend_number + 1); - + void *tmp = realloc(av->calls, sizeof(ToxAVCall *) * friend_number + 1); + if (tmp == NULL) { free(call); call = NULL; rc = TOXAV_ERR_CALL_MALLOC; goto END; } - + av->calls = tmp; - + /* Set fields in between to null */ uint32_t i = av->calls_tail + 1; + for (; i < friend_number; i ++) av->calls[i] = NULL; - + call->prev = av->calls[av->calls_tail]; av->calls[av->calls_tail]->next = call; - + av->calls_tail = friend_number; - + } else if (av->calls_head > friend_number) { /* Inserting at front */ call->next = av->calls[av->calls_head]; av->calls[av->calls_head]->prev = call; av->calls_head = friend_number; } - + av->calls[friend_number] = call; - + END: + if (error) *error = rc; - + return call; } -ToxAVCall* call_get(ToxAV* av, uint32_t friend_number) +ToxAVCall *call_get(ToxAV *av, uint32_t friend_number) { /* Assumes mutex locked */ if (av->calls == NULL || av->calls_tail < friend_number) return NULL; - + return av->calls[friend_number]; } -ToxAVCall* call_remove(ToxAVCall* call) +ToxAVCall *call_remove(ToxAVCall *call) { if (call == NULL) return NULL; - + uint32_t friend_number = call->friend_number; - ToxAV* av = call->av; - - ToxAVCall* prev = call->prev; - ToxAVCall* next = call->next; - + ToxAV *av = call->av; + + ToxAVCall *prev = call->prev; + ToxAVCall *next = call->next; + /* Set av call in msi to NULL in order to know if call if ToxAVCall is * removed from the msi call. */ call->msi_call->av_call = NULL; free(call); - + if (prev) prev->next = next; else if (next) av->calls_head = next->friend_number; else goto CLEAR; - + if (next) next->prev = prev; else if (prev) av->calls_tail = prev->friend_number; else goto CLEAR; - + av->calls[friend_number] = NULL; return next; - + CLEAR: av->calls_head = av->calls_tail = 0; free(av->calls); av->calls = NULL; - + return NULL; } -bool call_prepare_transmission(ToxAVCall* call) +bool call_prepare_transmission(ToxAVCall *call) { /* Assumes mutex locked */ - + if (call == NULL) return false; - - ToxAV* av = call->av; - + + ToxAV *av = call->av; + if (!av->acb.first && !av->vcb.first) /* It makes no sense to have CSession without callbacks */ return false; - + if (call->active) { LOGGER_WARNING("Call already active!\n"); return true; } - + if (create_recursive_mutex(call->mutex_audio) != 0) return false; - + if (create_recursive_mutex(call->mutex_video) != 0) goto FAILURE_3; - - if (create_recursive_mutex(call->mutex) != 0) + + if (create_recursive_mutex(call->mutex) != 0) goto FAILURE_2; - + + /* Prepare bwc */ + call->bwc = bwc_new(av->m, call->friend_number, callback_bwc, call); { /* Prepare audio */ call->audio.second = ac_new(av, call->friend_number, av->acb.first, av->acb.second); + if (!call->audio.second) { LOGGER_ERROR("Failed to create audio codec session"); goto FAILURE; } - - call->audio.first = rtp_new(rtp_TypeAudio, av->m, call->friend_number, call->audio.second, ac_queue_message); + + call->audio.first = rtp_new(rtp_TypeAudio, av->m, call->friend_number, call->bwc, + call->audio.second, ac_queue_message); + if (!call->audio.first) { LOGGER_ERROR("Failed to create audio rtp session");; goto FAILURE; } } - { /* Prepare video */ - call->video.second = vc_new(av, call->friend_number, av->vcb.first, av->vcb.second, call->msi_call->peer_vfpsz); + call->video.second = vc_new(av, call->friend_number, av->vcb.first, av->vcb.second); + if (!call->video.second) { LOGGER_ERROR("Failed to create video codec session"); goto FAILURE; } - - call->video.first = rtp_new(rtp_TypeVideo, av->m, call->friend_number, call->video.second, vc_queue_message); + + call->video.first = rtp_new(rtp_TypeVideo, av->m, call->friend_number, call->bwc, + call->video.second, vc_queue_message); + if (!call->video.first) { LOGGER_ERROR("Failed to create video rtp session"); goto FAILURE; } } - + call->active = 1; return true; - + FAILURE: + bwc_kill(call->bwc); rtp_kill(call->audio.first); ac_kill(call->audio.second); call->audio.first = NULL; @@ -1367,49 +1205,33 @@ FAILURE_3: pthread_mutex_destroy(call->mutex_audio); return false; } -void call_kill_transmission(ToxAVCall* call) +void call_kill_transmission(ToxAVCall *call) { if (call == NULL || call->active == 0) return; - + call->active = 0; - + pthread_mutex_lock(call->mutex_audio); pthread_mutex_unlock(call->mutex_audio); pthread_mutex_lock(call->mutex_video); pthread_mutex_unlock(call->mutex_video); pthread_mutex_lock(call->mutex); pthread_mutex_unlock(call->mutex); + + bwc_kill(call->bwc); rtp_kill(call->audio.first); ac_kill(call->audio.second); call->audio.first = NULL; call->audio.second = NULL; - + rtp_kill(call->video.first); vc_kill(call->video.second); call->video.first = NULL; call->video.second = NULL; - + pthread_mutex_destroy(call->mutex_audio); pthread_mutex_destroy(call->mutex_video); pthread_mutex_destroy(call->mutex); } -void ba_set(ToxAvBitrateAdapter* ba, uint32_t bit_rate) -{ - ba->bit_rate = bit_rate; - ba->next_send = current_time_monotonic(); - ba->end_time = ~0; - ba->next_send_interval = 1000; - ba->active = true; -} -bool ba_shoud_send_dummy(ToxAvBitrateAdapter* ba) -{ - if (!ba->active || ba->next_send > current_time_monotonic()) - return false; - - ba->next_send_interval *= 0.8; - ba->next_send = current_time_monotonic() + ba->next_send_interval; - - return true; -} \ No newline at end of file diff --git a/toxav/toxav.h b/toxav/toxav.h index 58d5503f5..e83f4edcd 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -52,12 +52,19 @@ extern "C" { /** \subsection threading Threading implications * * Unlike the Core API, this API is fully thread-safe. The library will ensure - * the proper synchronisation of parallel calls. + * the proper synchronization of parallel calls. * * A common way to run ToxAV (multiple or single instance) is to have a thread, * separate from tox instance thread, running a simple toxav_iterate loop, * sleeping for toxav_iteration_interval * milliseconds on each iteration. * + * An important thing to note is that events are triggered from both tox and + * toxav thread (see above). Audio and video receive frame events are triggered + * from toxav thread while all the other events are triggered from tox thread. + * + * Tox thread has priority with mutex mechanisms. Any api function can + * fail if mutexes are held by tox thread in which case they will set SYNC + * error code. */ /** * External Tox type. @@ -80,8 +87,10 @@ typedef struct Tox Tox; */ #ifndef TOXAV_DEFINED #define TOXAV_DEFINED -typedef struct ToxAV ToxAV; +typedef struct ToxAV_s ToxAV; #endif /* TOXAV_DEFINED */ + + /******************************************************************************* * * :: API version @@ -92,17 +101,20 @@ typedef struct ToxAV ToxAV; * incompatible way. */ #define TOXAV_VERSION_MAJOR 0u + /** * The minor version number. Incremented when functionality is added without * breaking the API or ABI. Set to 0 when the major version number is * incremented. */ #define TOXAV_VERSION_MINOR 0u + /** * The patch or revision number. Incremented when bugfixes are applied without * changing any functionality or API or ABI. */ #define TOXAV_VERSION_PATCH 0u + /** * A macro to check at preprocessing time whether the client code is compatible * with the installed version of ToxAV. @@ -112,37 +124,45 @@ typedef struct ToxAV ToxAV; (TOXAV_VERSION_MINOR > MINOR || \ (TOXAV_VERSION_MINOR == MINOR && \ TOXAV_VERSION_PATCH >= PATCH))) + /** * A macro to make compilation fail if the client code is not compatible with * the installed version of ToxAV. */ #define TOXAV_VERSION_REQUIRE(MAJOR, MINOR, PATCH) \ typedef char toxav_required_version[TOXAV_IS_COMPATIBLE(MAJOR, MINOR, PATCH) ? 1 : -1] + /** * A convenience macro to call toxav_version_is_compatible with the currently * compiling API version. */ #define TOXAV_VERSION_IS_ABI_COMPATIBLE() \ toxav_version_is_compatible(TOXAV_VERSION_MAJOR, TOXAV_VERSION_MINOR, TOXAV_VERSION_PATCH) + /** * Return the major version number of the library. Can be used to display the * ToxAV library version or to check whether the client is compatible with the * dynamically linked version of ToxAV. */ uint32_t toxav_version_major(void); + /** * Return the minor version number of the library. */ uint32_t toxav_version_minor(void); + /** * Return the patch number of the library. */ uint32_t toxav_version_patch(void); + /** * Return whether the compiled library version is compatible with the passed * version numbers. */ bool toxav_version_is_compatible(uint32_t major, uint32_t minor, uint32_t patch); + + /******************************************************************************* * * :: Creation and destruction @@ -167,10 +187,12 @@ typedef enum TOXAV_ERR_NEW { */ TOXAV_ERR_NEW_MULTIPLE, } TOXAV_ERR_NEW; + /** * Start new A/V session. There can only be only one session per Tox instance. */ ToxAV *toxav_new(Tox *tox, TOXAV_ERR_NEW *error); + /** * Releases all resources associated with the A/V session. * @@ -179,10 +201,13 @@ ToxAV *toxav_new(Tox *tox, TOXAV_ERR_NEW *error); * called and the av pointer becomes invalid. */ void toxav_kill(ToxAV *toxAV); + /** * Returns the Tox instance the A/V object was created for. */ Tox *toxav_get_tox(const ToxAV *toxAV); + + /******************************************************************************* * * :: A/V event loop @@ -193,12 +218,15 @@ Tox *toxav_get_tox(const ToxAV *toxAV); * be. If no call is active at the moment, this function returns 200. */ uint32_t toxav_iteration_interval(const ToxAV *toxAV); + /** * Main loop for the session. This function needs to be called in intervals of * toxav_iteration_interval() milliseconds. It is best called in the separate * thread from tox_iterate. */ void toxav_iterate(ToxAV *toxAV); + + /******************************************************************************* * * :: Call setup @@ -214,6 +242,10 @@ typedef enum TOXAV_ERR_CALL { * required for the call. */ TOXAV_ERR_CALL_MALLOC, + /** + * Synchronization error occurred. + */ + TOXAV_ERR_CALL_SYNC, /** * The friend number did not designate a valid friend. */ @@ -232,6 +264,7 @@ typedef enum TOXAV_ERR_CALL { */ TOXAV_ERR_CALL_INVALID_BIT_RATE, } TOXAV_ERR_CALL; + /** * Call a friend. This will start ringing the friend. * @@ -246,7 +279,9 @@ typedef enum TOXAV_ERR_CALL { * @param video_bit_rate Video bit rate in Kb/sec. Set this to 0 to disable * video sending. */ -bool toxav_call(ToxAV *toxAV, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL *error); +bool toxav_call(ToxAV *toxAV, uint32_t friend_number, uint32_t audio_bit_rate, + uint32_t video_bit_rate, TOXAV_ERR_CALL *error); + /** * The function type for the call callback. * @@ -254,17 +289,24 @@ bool toxav_call(ToxAV *toxAV, uint32_t friend_number, uint32_t audio_bit_rate, u * @param audio_enabled True if friend is sending audio. * @param video_enabled True if friend is sending video. */ -typedef void toxav_call_cb(ToxAV *toxAV, uint32_t friend_number, bool audio_enabled, bool video_enabled, void *user_data); +typedef void toxav_call_cb(ToxAV *toxAV, uint32_t friend_number, bool audio_enabled, + bool video_enabled, void *user_data); + /** * Set the callback for the `call` event. Pass NULL to unset. * */ void toxav_callback_call(ToxAV *toxAV, toxav_call_cb *callback, void *user_data); + typedef enum TOXAV_ERR_ANSWER { /** * The function returned successfully. */ TOXAV_ERR_ANSWER_OK, + /** + * Synchronization error occurred. + */ + TOXAV_ERR_ANSWER_SYNC, /** * Failed to initialize codecs for call session. Note that codec initiation * will fail if there is no receive callback registered for either audio or @@ -285,6 +327,7 @@ typedef enum TOXAV_ERR_ANSWER { */ TOXAV_ERR_ANSWER_INVALID_BIT_RATE, } TOXAV_ERR_ANSWER; + /** * Accept an incoming call. * @@ -299,6 +342,8 @@ typedef enum TOXAV_ERR_ANSWER { * video sending. */ bool toxav_answer(ToxAV *toxAV, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER *error); + + /******************************************************************************* * * :: Call state graph @@ -336,7 +381,6 @@ enum TOXAV_FRIEND_CALL_STATE { TOXAV_FRIEND_CALL_STATE_ACCEPTING_V = 32, }; - /** * The function type for the call_state callback. * @@ -347,11 +391,13 @@ enum TOXAV_FRIEND_CALL_STATE { * friend. */ typedef void toxav_call_state_cb(ToxAV *toxAV, uint32_t friend_number, uint32_t state, void *user_data); + /** * Set the callback for the `call_state` event. Pass NULL to unset. * */ void toxav_callback_call_state(ToxAV *toxAV, toxav_call_state_cb *callback, void *user_data); + /******************************************************************************* * * :: Call control @@ -393,11 +439,16 @@ typedef enum TOXAV_CALL_CONTROL { */ TOXAV_CALL_CONTROL_SHOW_VIDEO, } TOXAV_CALL_CONTROL; + typedef enum TOXAV_ERR_CALL_CONTROL { /** * The function returned successfully. */ TOXAV_ERR_CALL_CONTROL_OK, + /** + * Synchronization error occurred. + */ + TOXAV_ERR_CALL_CONTROL_SYNC, /** * The friend_number passed did not designate a valid friend. */ @@ -413,6 +464,7 @@ typedef enum TOXAV_ERR_CALL_CONTROL { */ TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION, } TOXAV_ERR_CALL_CONTROL; + /** * Sends a call control command to a friend. * @@ -423,48 +475,40 @@ typedef enum TOXAV_ERR_CALL_CONTROL { * @return true on success. */ bool toxav_call_control(ToxAV *toxAV, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL *error); + + /******************************************************************************* * * :: Controlling bit rates * ******************************************************************************/ -typedef enum TOXAV_ERR_SET_BIT_RATE { +typedef enum TOXAV_ERR_BIT_RATE_SET { /** * The function returned successfully. */ - TOXAV_ERR_SET_BIT_RATE_OK, + TOXAV_ERR_BIT_RATE_SET_OK, /** - * The bit rate passed was not one of the supported values. + * Synchronization error occurred. */ - TOXAV_ERR_SET_BIT_RATE_INVALID, + TOXAV_ERR_BIT_RATE_SET_SYNC, + /** + * The audio bit rate passed was not one of the supported values. + */ + TOXAV_ERR_BIT_RATE_SET_INVALID_AUDIO_BIT_RATE, + /** + * The video bit rate passed was not one of the supported values. + */ + TOXAV_ERR_BIT_RATE_SET_INVALID_VIDEO_BIT_RATE, /** * The friend_number passed did not designate a valid friend. */ - TOXAV_ERR_SET_BIT_RATE_FRIEND_NOT_FOUND, + TOXAV_ERR_BIT_RATE_SET_FRIEND_NOT_FOUND, /** * This client is currently not in a call with the friend. */ - TOXAV_ERR_SET_BIT_RATE_FRIEND_NOT_IN_CALL, -} TOXAV_ERR_SET_BIT_RATE; -/** - * The function type for the audio_bit_rate_status callback. - * - * @param friend_number The friend number of the friend for which to set the - * audio bit rate. - * @param stable Is the stream stable enough to keep the bit rate. - * Upon successful, non forceful, bit rate change, this is set to - * true and 'bit_rate' is set to new bit rate. - * The stable is set to false with bit_rate set to the unstable - * bit rate when either current stream is unstable with said bit rate - * or the non forceful change failed. - * @param bit_rate The bit rate in Kb/sec. - */ -typedef void toxav_audio_bit_rate_status_cb(ToxAV *toxAV, uint32_t friend_number, bool stable, uint32_t bit_rate, void *user_data); -/** - * Set the callback for the `audio_bit_rate_status` event. Pass NULL to unset. - * - */ -void toxav_callback_audio_bit_rate_status(ToxAV *toxAV, toxav_audio_bit_rate_status_cb *callback, void *user_data); + TOXAV_ERR_BIT_RATE_SET_FRIEND_NOT_IN_CALL, +} TOXAV_ERR_BIT_RATE_SET; + /** * Set the audio bit rate to be used in subsequent audio frames. If the passed * bit rate is the same as the current bit rate this function will return true @@ -476,46 +520,33 @@ void toxav_callback_audio_bit_rate_status(ToxAV *toxAV, toxav_audio_bit_rate_sta * @param friend_number The friend number of the friend for which to set the * audio bit rate. * @param audio_bit_rate The new audio bit rate in Kb/sec. Set to 0 to disable - * audio sending. - * @param force True if the bit rate change is forceful. - * - */ -bool toxav_audio_bit_rate_set(ToxAV *toxAV, uint32_t friend_number, uint32_t audio_bit_rate, bool force, TOXAV_ERR_SET_BIT_RATE *error); -/** - * The function type for the video_bit_rate_status callback. - * - * @param friend_number The friend number of the friend for which to set the - * video bit rate. - * @param stable Is the stream stable enough to keep the bit rate. - * Upon successful, non forceful, bit rate change, this is set to - * true and 'bit_rate' is set to new bit rate. - * The stable is set to false with bit_rate set to the unstable - * bit rate when either current stream is unstable with said bit rate - * or the non forceful change failed. - * @param bit_rate The bit rate in Kb/sec. - */ -typedef void toxav_video_bit_rate_status_cb(ToxAV *toxAV, uint32_t friend_number, bool stable, uint32_t bit_rate, void *user_data); -/** - * Set the callback for the `video_bit_rate_status` event. Pass NULL to unset. - * - */ -void toxav_callback_video_bit_rate_status(ToxAV *toxAV, toxav_video_bit_rate_status_cb *callback, void *user_data); -/** - * Set the video bit rate to be used in subsequent video frames. If the passed - * bit rate is the same as the current bit rate this function will return true - * without calling a callback. If there is an active non forceful setup with the - * passed video bit rate and the new set request is forceful, the bit rate is - * forcefully set and the previous non forceful request is cancelled. The active - * non forceful setup will be canceled in favour of new non forceful setup. - * - * @param friend_number The friend number of the friend for which to set the - * video bit rate. + * audio sending. Set to -1 to leave unchanged. * @param video_bit_rate The new video bit rate in Kb/sec. Set to 0 to disable - * video sending. - * @param force True if the bit rate change is forceful. + * video sending. Set to -1 to leave unchanged. * */ -bool toxav_video_bit_rate_set(ToxAV *toxAV, uint32_t friend_number, uint32_t video_bit_rate, bool force, TOXAV_ERR_SET_BIT_RATE *error); +bool toxav_bit_rate_set(ToxAV *toxAV, uint32_t friend_number, int32_t audio_bit_rate, + int32_t video_bit_rate, TOXAV_ERR_BIT_RATE_SET *error); + +/** + * The function type for the bit_rate_status callback. The event is triggered + * when the network becomes too saturated for current bit rates at which + * point core suggests new bit rates. + * + * @param friend_number The friend number of the friend for which to set the + * audio bit rate. + * @param audio_bit_rate Suggested maximum audio bit rate in Kb/sec. + * @param video_bit_rate Suggested maximum video bit rate in Kb/sec. + */ +typedef void toxav_bit_rate_status_cb(ToxAV *toxAV, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, void *user_data); + +/** + * Set the callback for the `bit_rate_status` event. Pass NULL to unset. + * + */ +void toxav_callback_bit_rate_status(ToxAV *toxAV, toxav_bit_rate_status_cb *callback, void *user_data); + + /******************************************************************************* * * :: A/V sending @@ -554,6 +585,7 @@ typedef enum TOXAV_ERR_SEND_FRAME { */ TOXAV_ERR_SEND_FRAME_RTP_FAILED, } TOXAV_ERR_SEND_FRAME; + /** * Send an audio frame to a friend. * @@ -574,7 +606,10 @@ typedef enum TOXAV_ERR_SEND_FRAME { * @param sampling_rate Audio sampling rate used in this frame. Valid sampling * rates are 8000, 12000, 16000, 24000, or 48000. */ -bool toxav_audio_send_frame(ToxAV *toxAV, uint32_t friend_number, const int16_t *pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME *error); +bool toxav_audio_send_frame(ToxAV *toxAV, uint32_t friend_number, const int16_t *pcm, + size_t sample_count, uint8_t channels, uint32_t sampling_rate, + TOXAV_ERR_SEND_FRAME *error); + /** * Send a video frame to a friend. * @@ -590,7 +625,11 @@ bool toxav_audio_send_frame(ToxAV *toxAV, uint32_t friend_number, const int16_t * @param u U (Chroma) plane data. * @param v V (Chroma) plane data. */ -bool toxav_video_send_frame(ToxAV *toxAV, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t *y, const uint8_t *u, const uint8_t *v, TOXAV_ERR_SEND_FRAME *error); +bool toxav_video_send_frame(ToxAV *toxAV, uint32_t friend_number, uint16_t width, + uint16_t height, const uint8_t *y, const uint8_t *u, const uint8_t *v, + TOXAV_ERR_SEND_FRAME *error); + + /******************************************************************************* * * :: A/V receiving @@ -600,7 +639,7 @@ bool toxav_video_send_frame(ToxAV *toxAV, uint32_t friend_number, uint16_t width * The function type for the audio_receive_frame callback. The callback can be * called multiple times per single iteration depending on the amount of queued * frames in the buffer. The received format is the same as in send function. - * + * * @param friend_number The friend number of the friend who sent an audio frame. * @param pcm An array of audio samples (sample_count * channels elements). * @param sample_count The number of audio samples per channel in the PCM array. @@ -608,12 +647,16 @@ bool toxav_video_send_frame(ToxAV *toxAV, uint32_t friend_number, uint16_t width * @param sampling_rate Sampling rate used in this frame. * */ -typedef void toxav_audio_receive_frame_cb(ToxAV *toxAV, uint32_t friend_number, const int16_t *pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, void *user_data); +typedef void toxav_audio_receive_frame_cb(ToxAV *toxAV, uint32_t friend_number, const int16_t *pcm, + size_t sample_count, uint8_t channels, uint32_t sampling_rate, + void *user_data); + /** * Set the callback for the `audio_receive_frame` event. Pass NULL to unset. * */ void toxav_callback_audio_receive_frame(ToxAV *toxAV, toxav_audio_receive_frame_cb *callback, void *user_data); + /** * The function type for the video_receive_frame callback. * @@ -635,60 +678,17 @@ void toxav_callback_audio_receive_frame(ToxAV *toxAV, toxav_audio_receive_frame_ * image is bottom-up hence why you MUST abs() it when * calculating plane buffer size. */ -typedef void toxav_video_receive_frame_cb(ToxAV *toxAV, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t *y, const uint8_t *u, const uint8_t *v, int32_t ystride, int32_t ustride, int32_t vstride, void *user_data); +typedef void toxav_video_receive_frame_cb(ToxAV *toxAV, uint32_t friend_number, uint16_t width, + uint16_t height, const uint8_t *y, const uint8_t *u, const uint8_t *v, + int32_t ystride, int32_t ustride, int32_t vstride, void *user_data); + /** * Set the callback for the `video_receive_frame` event. Pass NULL to unset. * */ void toxav_callback_video_receive_frame(ToxAV *toxAV, toxav_video_receive_frame_cb *callback, void *user_data); -/** - * NOTE Compatibility with old toxav group calls TODO remove - */ -/* Create a new toxav group. - * - * return group number on success. - * return -1 on failure. - * - * Audio data callback format: - * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) - * - * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). - */ -int toxav_add_av_groupchat(Tox *tox, void (*audio_callback)(void*, int, int, const int16_t *, unsigned int, uint8_t, - unsigned int, void *), void *userdata); - -/* Join a AV group (you need to have been invited first.) - * - * returns group number on success - * returns -1 on failure. - * - * Audio data callback format (same as the one for toxav_add_av_groupchat()): - * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) - * - * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). - */ -int toxav_join_av_groupchat(Tox *tox, int32_t friendnumber, const uint8_t *data, uint16_t length, - void (*audio_callback)(void*, int, int, const int16_t *, unsigned int, uint8_t, unsigned int, void *), void *userdata); - -/* Send audio to the group chat. - * - * return 0 on success. - * return -1 on failure. - * - * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). - * - * Valid number of samples are ((sample rate) * (audio length (Valid ones are: 2.5, 5, 10, 20, 40 or 60 ms)) / 1000) - * Valid number of channels are 1 or 2. - * Valid sample rates are 8000, 12000, 16000, 24000, or 48000. - * - * Recommended values are: samples = 960, channels = 1, sample_rate = 48000 - */ -int toxav_group_send_audio(Tox *tox, int groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels, - unsigned int sample_rate); - #ifdef __cplusplus } #endif - #endif /* TOXAV_H */ diff --git a/toxav/toxav_old.c b/toxav/toxav_old.c index 61c2f0207..7d7e5e7bd 100644 --- a/toxav/toxav_old.c +++ b/toxav/toxav_old.c @@ -1,5 +1,5 @@ /* toxav_old.h - * + * * Copyright (C) 2013-2015 Tox project All Rights Reserved. * * This file is part of Tox. @@ -16,7 +16,7 @@ * * You should have received a copy of the GNU General Public License * along with Tox. If not, see . - * + * */ /** * This file contains the group chats code for the backwards compatibility. diff --git a/toxav/video.c b/toxav/video.c index 389d2e1c2..919e3c81a 100644 --- a/toxav/video.c +++ b/toxav/video.c @@ -19,6 +19,10 @@ * */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif /* HAVE_CONFIG_H */ + #include #include @@ -29,281 +33,166 @@ #include "../toxcore/logger.h" #include "../toxcore/network.h" -/* Good quality encode. */ -#define MAX_DECODE_TIME_US 0 - -#define MAX_VIDEOFRAME_SIZE 0x40000 /* 256KiB */ -#define VIDEOFRAME_HEADER_SIZE 0x2 - +#define MAX_DECODE_TIME_US 0 /* Good quality encode. */ #define VIDEO_DECODE_BUFFER_SIZE 20 -typedef struct { uint16_t size; uint8_t data[]; } Payload; -bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bit_rate); +bool create_video_encoder (vpx_codec_ctx_t *dest, int32_t bit_rate); - -VCSession* vc_new(ToxAV* av, uint32_t friend_number, toxav_video_receive_frame_cb* cb, void* cb_data, uint32_t mvfpsz) +VCSession *vc_new(ToxAV *av, uint32_t friend_number, toxav_video_receive_frame_cb *cb, void *cb_data) { VCSession *vc = calloc(sizeof(VCSession), 1); - + if (!vc) { LOGGER_WARNING("Allocation failed! Application might misbehave!"); return NULL; } - + if (create_recursive_mutex(vc->queue_mutex) != 0) { LOGGER_WARNING("Failed to create recursive mutex!"); free(vc); return NULL; } - - if ( !(vc->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) ) + + if (!(vc->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE))) goto BASE_CLEANUP; - if ( !(vc->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) ) - goto BASE_CLEANUP; - if ( !(vc->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE)) ) - goto BASE_CLEANUP; - - int rc = vpx_codec_dec_init_ver(vc->decoder, VIDEO_CODEC_DECODER_INTERFACE, - NULL, 0, VPX_DECODER_ABI_VERSION); - if ( rc != VPX_CODEC_OK) { + + int rc = vpx_codec_dec_init(vc->decoder, VIDEO_CODEC_DECODER_INTERFACE, NULL, 0); + + if (rc != VPX_CODEC_OK) { LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); goto BASE_CLEANUP; } - + if (!create_video_encoder(vc->encoder, 500000)) { vpx_codec_destroy(vc->decoder); goto BASE_CLEANUP; } - if (!create_video_encoder(vc->test_encoder, 500000)) { - vpx_codec_destroy(vc->encoder); - vpx_codec_destroy(vc->decoder); - goto BASE_CLEANUP; - } - + vc->linfts = current_time_monotonic(); vc->lcfd = 60; vc->vcb.first = cb; vc->vcb.second = cb_data; vc->friend_number = friend_number; - vc->peer_video_frame_piece_size = mvfpsz; vc->av = av; - + return vc; - + BASE_CLEANUP: pthread_mutex_destroy(vc->queue_mutex); - rb_free(vc->vbuf_raw); - free(vc->split_video_frame); - free(vc->frame_buf); + rb_kill(vc->vbuf_raw); free(vc); return NULL; } -void vc_kill(VCSession* vc) +void vc_kill(VCSession *vc) { if (!vc) return; - + vpx_codec_destroy(vc->encoder); - vpx_codec_destroy(vc->test_encoder); vpx_codec_destroy(vc->decoder); - rb_free(vc->vbuf_raw); - free(vc->split_video_frame); - free(vc->frame_buf); - + + void *p; + + while (rb_read(vc->vbuf_raw, (void **)&p)) + free(p); + + rb_kill(vc->vbuf_raw); + pthread_mutex_destroy(vc->queue_mutex); - + LOGGER_DEBUG("Terminated video handler: %p", vc); free(vc); } -void vc_do(VCSession* vc) +void vc_iterate(VCSession *vc) { if (!vc) return; - - Payload *p; + + struct RTPMessage *p; int rc; - + pthread_mutex_lock(vc->queue_mutex); - if (rb_read(vc->vbuf_raw, (void**)&p)) { + + if (rb_read(vc->vbuf_raw, (void **)&p)) { pthread_mutex_unlock(vc->queue_mutex); - - rc = vpx_codec_decode(vc->decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); + + rc = vpx_codec_decode(vc->decoder, p->data, p->len, NULL, MAX_DECODE_TIME_US); free(p); - - if (rc != VPX_CODEC_OK) { + + if (rc != VPX_CODEC_OK) LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc)); - } else { + else { vpx_codec_iter_t iter = NULL; vpx_image_t *dest = vpx_codec_get_frame(vc->decoder, &iter); - + /* Play decoded images */ for (; dest; dest = vpx_codec_get_frame(vc->decoder, &iter)) { - if (vc->vcb.first) - vc->vcb.first(vc->av, vc->friend_number, dest->d_w, dest->d_h, - (const uint8_t*)dest->planes[0], (const uint8_t*)dest->planes[1], (const uint8_t*)dest->planes[2], + if (vc->vcb.first) + vc->vcb.first(vc->av, vc->friend_number, dest->d_w, dest->d_h, + (const uint8_t *)dest->planes[0], (const uint8_t *)dest->planes[1], (const uint8_t *)dest->planes[2], dest->stride[0], dest->stride[1], dest->stride[2], vc->vcb.second); - + vpx_img_free(dest); } } - + return; } + pthread_mutex_unlock(vc->queue_mutex); } -void vc_init_video_splitter_cycle(VCSession* vc) +int vc_queue_message(void *vcp, struct RTPMessage *msg) { - if (!vc) - return; - - vc->split_video_frame[0] = vc->frameid_out++; - vc->split_video_frame[1] = 0; -} -int vc_update_video_splitter_cycle(VCSession* vc, const uint8_t* payload, uint16_t length) -{ - if (!vc) - return 0; - - vc->processing_video_frame = payload; - vc->processing_video_frame_size = length; - - return ((length - 1) / VIDEOFRAME_PIECE_SIZE) + 1; -} -const uint8_t* vc_iterate_split_video_frame(VCSession* vc, uint16_t* size) -{ - if (!vc || !size) - return NULL; - - if (vc->processing_video_frame_size > VIDEOFRAME_PIECE_SIZE) { - memcpy(vc->split_video_frame + VIDEOFRAME_HEADER_SIZE, - vc->processing_video_frame, - VIDEOFRAME_PIECE_SIZE); - - vc->processing_video_frame += VIDEOFRAME_PIECE_SIZE; - vc->processing_video_frame_size -= VIDEOFRAME_PIECE_SIZE; - - *size = VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE; - } else { - memcpy(vc->split_video_frame + VIDEOFRAME_HEADER_SIZE, - vc->processing_video_frame, - vc->processing_video_frame_size); - - *size = vc->processing_video_frame_size + VIDEOFRAME_HEADER_SIZE; - } - - vc->split_video_frame[1]++; - - return vc->split_video_frame; -} -int vc_queue_message(void* vcp, struct RTPMessage_s *msg) -{ - /* This function does the reconstruction of video packets. + /* This function does the reconstruction of video packets. * See more info about video splitting in docs */ if (!vcp || !msg) return -1; - - if ((msg->header->marker_payloadt & 0x7f) == (rtp_TypeVideo + 2) % 128) { + + if (msg->header.pt == (rtp_TypeVideo + 2) % 128) { LOGGER_WARNING("Got dummy!"); - rtp_free_msg(msg); + free(msg); return 0; } - - if ((msg->header->marker_payloadt & 0x7f) != rtp_TypeVideo % 128) { + + if (msg->header.pt != rtp_TypeVideo % 128) { LOGGER_WARNING("Invalid payload type!"); - rtp_free_msg(msg); + free(msg); return -1; } - - VCSession* vc = vcp; - - uint8_t *packet = msg->data; - uint32_t packet_size = msg->length; - if (packet_size < VIDEOFRAME_HEADER_SIZE) - goto end; + VCSession *vc = vcp; - uint8_t diff = packet[0] - vc->frameid_in; - - if (diff != 0) { - if (diff < 225) { /* New frame */ - /* Flush last frames' data and get ready for this frame */ - Payload *p = malloc(sizeof(Payload) + vc->frame_size); - - if (p) { - pthread_mutex_lock(vc->queue_mutex); - - if (rb_full(vc->vbuf_raw)) { - LOGGER_DEBUG("Dropped video frame"); - Payload *tp; - rb_read(vc->vbuf_raw, (void**)&tp); - free(tp); - } else { - p->size = vc->frame_size; - memcpy(p->data, vc->frame_buf, vc->frame_size); - } - - /* Calculate time took for peer to send us this frame */ - uint32_t t_lcfd = current_time_monotonic() - vc->linfts; - vc->lcfd = t_lcfd > 100 ? vc->lcfd : t_lcfd; - vc->linfts = current_time_monotonic(); - - rb_write(vc->vbuf_raw, p); - pthread_mutex_unlock(vc->queue_mutex); - } else { - LOGGER_WARNING("Allocation failed! Program might misbehave!"); - goto end; - } - - vc->frameid_in = packet[0]; - memset(vc->frame_buf, 0, vc->frame_size); - vc->frame_size = 0; - - } else { /* Old frame; drop */ - LOGGER_DEBUG("Old packet: %u", packet[0]); - goto end; - } + pthread_mutex_lock(vc->queue_mutex); + free(rb_write(vc->vbuf_raw, msg)); + { + /* Calculate time took for peer to send us this frame */ + uint32_t t_lcfd = current_time_monotonic() - vc->linfts; + vc->lcfd = t_lcfd > 100 ? vc->lcfd : t_lcfd; + vc->linfts = current_time_monotonic(); } + pthread_mutex_unlock(vc->queue_mutex); - uint8_t piece_number = packet[1]; - - uint32_t length_before_piece = ((piece_number - 1) * vc->peer_video_frame_piece_size); - uint32_t framebuf_new_length = length_before_piece + (packet_size - VIDEOFRAME_HEADER_SIZE); - - if (framebuf_new_length > MAX_VIDEOFRAME_SIZE) - goto end; - - - /* Otherwise it's part of the frame so just process */ - /* LOGGER_DEBUG("Video Packet: %u %u", packet[0], packet[1]); */ - - memcpy(vc->frame_buf + length_before_piece, - packet + VIDEOFRAME_HEADER_SIZE, - packet_size - VIDEOFRAME_HEADER_SIZE); - - if (framebuf_new_length > vc->frame_size) - vc->frame_size = framebuf_new_length; - -end: - rtp_free_msg(msg); return 0; } -int vc_reconfigure_encoder(vpx_codec_ctx_t* vccdc, uint32_t bit_rate, uint16_t width, uint16_t height) +int vc_reconfigure_encoder(vpx_codec_ctx_t *vccdc, uint32_t bit_rate, uint16_t width, uint16_t height) { if (!vccdc) return -1; - + vpx_codec_enc_cfg_t cfg = *vccdc->config.enc; + if (cfg.rc_target_bitrate == bit_rate && cfg.g_w == width && cfg.g_h == height) return 0; /* Nothing changed */ - + cfg.rc_target_bitrate = bit_rate; cfg.g_w = width; cfg.g_h = height; - + int rc = vpx_codec_enc_config_set(vccdc, &cfg); - if ( rc != VPX_CODEC_OK) { + + if (rc != VPX_CODEC_OK) { LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); return -1; } @@ -312,42 +201,43 @@ int vc_reconfigure_encoder(vpx_codec_ctx_t* vccdc, uint32_t bit_rate, uint16_t w } -bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bit_rate) +bool create_video_encoder (vpx_codec_ctx_t *dest, int32_t bit_rate) { assert(dest); - + vpx_codec_enc_cfg_t cfg; int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); - + if (rc != VPX_CODEC_OK) { LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); return false; } - + cfg.rc_target_bitrate = bit_rate; - cfg.g_w = 4000; - cfg.g_h = 4000; + cfg.g_w = 800; + cfg.g_h = 600; cfg.g_pass = VPX_RC_ONE_PASS; - cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; + /* FIXME If we set error resilience the app will crash due to bug in vp8. + Perhaps vp9 has solved it?*/ +// cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; cfg.g_lag_in_frames = 0; cfg.kf_min_dist = 0; cfg.kf_max_dist = 48; cfg.kf_mode = VPX_KF_AUTO; - - rc = vpx_codec_enc_init_ver(dest, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, - VPX_ENCODER_ABI_VERSION); - - if ( rc != VPX_CODEC_OK) { + + rc = vpx_codec_enc_init(dest, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); + + if (rc != VPX_CODEC_OK) { LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); return false; } - + rc = vpx_codec_control(dest, VP8E_SET_CPUUSED, 8); - - if ( rc != VPX_CODEC_OK) { + + if (rc != VPX_CODEC_OK) { LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); vpx_codec_destroy(dest); } - + return true; -} \ No newline at end of file +} diff --git a/toxav/video.h b/toxav/video.h index ac165df67..1ad1f6f50 100644 --- a/toxav/video.h +++ b/toxav/video.h @@ -36,77 +36,32 @@ #include "../toxcore/util.h" -struct RTPMessage_s; +struct RTPMessage; -/* - * Base Video Codec session type. - */ typedef struct VCSession_s { - /* encoding */ vpx_codec_ctx_t encoder[1]; - vpx_codec_ctx_t test_encoder[1]; uint32_t frame_counter; - uint32_t test_frame_counter; /* decoding */ vpx_codec_ctx_t decoder[1]; - void *vbuf_raw; /* Un-decoded data */ + void *vbuf_raw; /* Un-decoded data */ - /* Data handling */ - uint8_t *frame_buf; /* buffer for split video payloads */ - uint32_t frame_size; /* largest address written to in frame_buf for current input frame */ - uint8_t frameid_in, frameid_out; /* id of input and output video frame */ uint64_t linfts; /* Last received frame time stamp */ uint32_t lcfd; /* Last calculated frame duration for incoming video payload */ - - /* Limits */ - uint32_t peer_video_frame_piece_size; - /* Splitting */ - uint8_t *split_video_frame; - const uint8_t *processing_video_frame; - uint16_t processing_video_frame_size; - ToxAV *av; uint32_t friend_number; - + PAIR(toxav_video_receive_frame_cb *, void *) vcb; /* Video frame receive callback */ - + pthread_mutex_t queue_mutex[1]; } VCSession; -/* - * Create new Video Codec session. - */ -VCSession* vc_new(ToxAV* av, uint32_t friend_number, toxav_video_receive_frame_cb *cb, void *cb_data, uint32_t mvfpsz); -/* - * Kill the Video Codec session. - */ -void vc_kill(VCSession* vc); -/* - * Do periodic work. Work is consisted out of decoding only. - */ -void vc_do(VCSession* vc); -/* - * Set new video splitting cycle. This is requirement in order to send video packets. - */ -void vc_init_video_splitter_cycle(VCSession* vc); -/* - * Update the video splitter cycle with new data. - */ -int vc_update_video_splitter_cycle(VCSession* vc, const uint8_t* payload, uint16_t length); -/* - * Iterate over splitted cycle. - */ -const uint8_t *vc_iterate_split_video_frame(VCSession* vc, uint16_t *size); -/* - * Queue new rtp message. - */ -int vc_queue_message(void *vcp, struct RTPMessage_s *msg); -/* - * Set new values to the encoders. - */ -int vc_reconfigure_encoder(vpx_codec_ctx_t* vccdc, uint32_t bit_rate, uint16_t width, uint16_t height); +VCSession *vc_new(ToxAV* av, uint32_t friend_number, toxav_video_receive_frame_cb* cb, void* cb_data); +void vc_kill(VCSession *vc); +void vc_iterate(VCSession *vc); +int vc_queue_message(void *vcp, struct RTPMessage *msg); +int vc_reconfigure_encoder(vpx_codec_ctx_t *vccdc, uint32_t bit_rate, uint16_t width, uint16_t height); -#endif /* VIDEO_H */ \ No newline at end of file +#endif /* VIDEO_H */ diff --git a/toxcore/Messenger.c b/toxcore/Messenger.c index 4277f16a6..4cd9e1d6f 100644 --- a/toxcore/Messenger.c +++ b/toxcore/Messenger.c @@ -2239,7 +2239,7 @@ static void connection_status_cb(Messenger *m) } -#ifdef LOGGING +#ifdef TOX_LOGGER #define DUMPING_CLIENTS_FRIENDS_EVERY_N_SECONDS 60UL static time_t lastdump = 0; static char IDString[crypto_box_PUBLICKEYBYTES * 2 + 1]; @@ -2315,7 +2315,7 @@ void do_messenger(Messenger *m) do_friends(m); connection_status_cb(m); -#ifdef LOGGING +#ifdef TOX_LOGGER if (unix_time() > lastdump + DUMPING_CLIENTS_FRIENDS_EVERY_N_SECONDS) { @@ -2414,7 +2414,7 @@ void do_messenger(Messenger *m) } } -#endif /* LOGGING */ +#endif /* TOX_LOGGER */ } /* new messenger format for load/save, more robust and forward compatible */ diff --git a/toxcore/assoc.c b/toxcore/assoc.c index 44c4cc30c..932adc76c 100644 --- a/toxcore/assoc.c +++ b/toxcore/assoc.c @@ -878,9 +878,9 @@ void Assoc_self_client_id_changed(Assoc *assoc, const uint8_t *id) } } -#ifdef LOGGING +#ifdef TOX_LOGGER static char *idpart2str(uint8_t *id, size_t len); -#endif /* LOGGING */ +#endif /* TOX_LOGGER */ /* refresh buckets */ void do_Assoc(Assoc *assoc, DHT *dht) @@ -974,7 +974,7 @@ void kill_Assoc(Assoc *assoc) } } -#ifdef LOGGING +#ifdef TOX_LOGGER static char buffer[crypto_box_PUBLICKEYBYTES * 2 + 1]; static char *idpart2str(uint8_t *id, size_t len) @@ -1028,4 +1028,4 @@ void Assoc_status(const Assoc *assoc) } } -#endif /* LOGGING */ +#endif /* TOX_LOGGER */ diff --git a/toxcore/assoc.h b/toxcore/assoc.h index 1b4e1ff9e..65a2745d8 100644 --- a/toxcore/assoc.h +++ b/toxcore/assoc.h @@ -97,8 +97,8 @@ void do_Assoc(Assoc *assoc, DHT *dht); /* destroy */ void kill_Assoc(Assoc *assoc); -#ifdef LOGGING +#ifdef TOX_LOGGER void Assoc_status(const Assoc *assoc); -#endif /* LOGGING */ +#endif /* TOX_LOGGER */ #endif /* !__ASSOC_H__ */ diff --git a/toxcore/logger.c b/toxcore/logger.c index fc6a989aa..f19f76b10 100644 --- a/toxcore/logger.c +++ b/toxcore/logger.c @@ -44,7 +44,7 @@ #endif -struct logger { +struct Logger { FILE *log_file; LOG_LEVEL level; uint64_t start_time; /* Time when lib loaded */ @@ -87,7 +87,7 @@ char *strtime(char *dest, size_t max_len) */ Logger *logger_new (const char *file_name, LOG_LEVEL level, const char *id) { -#ifndef LOGGING /* Disabled */ +#ifndef TOX_LOGGER /* Disabled */ return NULL; #endif @@ -96,7 +96,7 @@ Logger *logger_new (const char *file_name, LOG_LEVEL level, const char *id) if (!retu) return NULL; - if ( pthread_mutex_init(retu->mutex, NULL) != 0 ) { + if (pthread_mutex_init(retu->mutex, NULL) != 0) { free(retu); return NULL; } @@ -110,7 +110,7 @@ Logger *logger_new (const char *file_name, LOG_LEVEL level, const char *id) if (!(retu->tstr = calloc(16, sizeof (char))) || !(retu->posstr = calloc(300, sizeof (char))) || - !(retu->msg = calloc(4096, sizeof (char))) ) + !(retu->msg = calloc(4096, sizeof (char)))) goto FAILURE; if (id) { @@ -147,7 +147,7 @@ FAILURE: void logger_kill(Logger *log) { -#ifndef LOGGING /* Disabled */ +#ifndef TOX_LOGGER /* Disabled */ return; #endif @@ -160,7 +160,7 @@ void logger_kill(Logger *log) free(log->posstr); free(log->msg); - if (fclose(log->log_file) != 0 ) + if (fclose(log->log_file) != 0) perror("Could not close log file"); pthread_mutex_unlock(log->mutex); @@ -177,7 +177,7 @@ void logger_kill_global(void) void logger_set_global(Logger *log) { -#ifndef LOGGING /* Disabled */ +#ifndef TOX_LOGGER /* Disabled */ return; #endif @@ -186,7 +186,7 @@ void logger_set_global(Logger *log) Logger *logger_get_global(void) { -#ifndef LOGGING /* Disabled */ +#ifndef TOX_LOGGER /* Disabled */ return NULL; #endif @@ -195,7 +195,7 @@ Logger *logger_get_global(void) void logger_write (Logger *log, LOG_LEVEL level, const char *file, int line, const char *format, ...) { -#ifndef LOGGING /* Disabled */ +#ifndef TOX_LOGGER /* Disabled */ return; #endif diff --git a/toxcore/logger.h b/toxcore/logger.h index 0513b32cc..4d3e3b542 100644 --- a/toxcore/logger.h +++ b/toxcore/logger.h @@ -43,7 +43,7 @@ typedef enum { LOG_ERROR } LOG_LEVEL; -typedef struct logger Logger; +typedef struct Logger Logger; /** * Set 'level' as the lowest printable level. If id == NULL, random number is used. @@ -66,21 +66,22 @@ void logger_write (Logger *log, LOG_LEVEL level, const char *file, int line, con /* To do some checks or similar only when logging, use this */ -#ifdef LOGGING +#ifdef TOX_LOGGER # define LOGGER_SCOPE(__SCOPE_DO__) do { __SCOPE_DO__ } while(0) # define LOGGER_WRITE(log, level, format, ...) \ - logger_write(log, level, __FILE__, __LINE__, format, ##__VA_ARGS__ ) + logger_write(log, level, __FILE__, __LINE__, format, ##__VA_ARGS__) #else +/* # warning "Logging disabled" */ # define LOGGER_SCOPE(__SCOPE_DO__) do {} while(0) # define LOGGER_WRITE(log, level, format, ...) do {} while(0) -#endif /* LOGGING */ +#endif /* TOX_LOGGER */ /* To log with an logger */ -#define LOGGER_TRACE_(log, format, ...) LOGGER_WRITE(log, LOG_TRACE, format, ##__VA_ARGS__ ) -#define LOGGER_DEBUG_(log, format, ...) LOGGER_WRITE(log, LOG_DEBUG, format, ##__VA_ARGS__ ) -#define LOGGER_INFO_(log, format, ...) LOGGER_WRITE(log, LOG_INFO, format, ##__VA_ARGS__ ) -#define LOGGER_WARNING_(log, format, ...) LOGGER_WRITE(log, LOG_WARNING, format, ##__VA_ARGS__ ) -#define LOGGER_ERROR_(log, format, ...) LOGGER_WRITE(log, LOG_ERROR, format, ##__VA_ARGS__ ) +#define LOGGER_TRACE_(log, format, ...) LOGGER_WRITE(log, LOG_TRACE, format, ##__VA_ARGS__) +#define LOGGER_DEBUG_(log, format, ...) LOGGER_WRITE(log, LOG_DEBUG, format, ##__VA_ARGS__) +#define LOGGER_INFO_(log, format, ...) LOGGER_WRITE(log, LOG_INFO, format, ##__VA_ARGS__) +#define LOGGER_WARNING_(log, format, ...) LOGGER_WRITE(log, LOG_WARNING, format, ##__VA_ARGS__) +#define LOGGER_ERROR_(log, format, ...) LOGGER_WRITE(log, LOG_ERROR, format, ##__VA_ARGS__) /* To log with the global logger */ #define LOGGER_TRACE(format, ...) LOGGER_TRACE_(NULL, format, ##__VA_ARGS__) diff --git a/toxcore/network.c b/toxcore/network.c index 22ee42028..965e65f91 100644 --- a/toxcore/network.c +++ b/toxcore/network.c @@ -266,7 +266,7 @@ uint64_t current_time_monotonic(void) } /* In case no logging */ -#ifndef LOGGING +#ifndef TOX_LOGGER #define loglogdata(__message__, __buffer__, __buflen__, __ip_port__, __res__) #else #define data_0(__buflen__, __buffer__) __buflen__ > 4 ? ntohl(*(uint32_t *)&__buffer__[1]) : 0 @@ -287,7 +287,7 @@ uint64_t current_time_monotonic(void) __buffer__[0], __message__, (size_t)__res__, (!__res__ ? '!' : '>'), __buflen__, \ ip_ntoa(&((__ip_port__).ip)), ntohs((__ip_port__).port), 0, "OK", data_0(__buflen__, __buffer__), data_1(__buflen__, __buffer__)); -#endif /* LOGGING */ +#endif /* TOX_LOGGER */ /* Basic network functions: * Function to send packet(data) of length length to ip_port. @@ -615,9 +615,9 @@ Networking_Core *new_networking_ex(IP ip, uint16_t port_from, uint16_t port_to, } if (ip.family == AF_INET6) { -#ifdef LOGGING +#ifdef TOX_LOGGER int is_dualstack = -#endif /* LOGGING */ +#endif /* TOX_LOGGER */ set_socket_dualstack(temp->sock); LOGGER_DEBUG( "Dual-stack socket: %s", is_dualstack ? "enabled" : "Failed to enable, won't be able to receive from/send to IPv4 addresses" ); @@ -628,9 +628,9 @@ Networking_Core *new_networking_ex(IP ip, uint16_t port_from, uint16_t port_to, mreq.ipv6mr_multiaddr.s6_addr[ 1] = 0x02; mreq.ipv6mr_multiaddr.s6_addr[15] = 0x01; mreq.ipv6mr_interface = 0; -#ifdef LOGGING +#ifdef TOX_LOGGER int res = -#endif /* LOGGING */ +#endif /* TOX_LOGGER */ setsockopt(temp->sock, IPPROTO_IPV6, IPV6_ADD_MEMBERSHIP, (char *)&mreq, sizeof(mreq)); LOGGER_DEBUG(res < 0 ? "Failed to activate local multicast membership. (%u, %s)" : diff --git a/toxcore/util.c b/toxcore/util.c index 5865a172d..81fa84c6e 100644 --- a/toxcore/util.c +++ b/toxcore/util.c @@ -234,14 +234,6 @@ bool rb_read(RingBuffer *b, void **p) b->start = (b->start + 1) % b->size; return true; } -void rb_clear(RingBuffer *b) -{ - while (!rb_empty(b)) { - void *p; - rb_read(b, &p); - free(p); - } -} RingBuffer *rb_new(int size) { RingBuffer *buf = calloc(sizeof(RingBuffer), 1); @@ -257,11 +249,28 @@ RingBuffer *rb_new(int size) return buf; } -void rb_free(RingBuffer *b) +void rb_kill(RingBuffer *b) { if (b) { - rb_clear(b); free(b->data); free(b); } } +uint16_t rb_size(const RingBuffer* b) +{ + if (rb_empty(b)) + return 0; + + return + b->end > b->start ? + b->end - b->start : + (b->size - b->start) + b->end; +} +uint16_t rb_data(const RingBuffer* b, void** dest) +{ + uint16_t i = 0; + for (; i < rb_size(b); i++) + dest[i] = b->data[(b->start + i) % b->size]; + + return i; +} diff --git a/toxcore/util.h b/toxcore/util.h index 7670a80f2..7cf631789 100644 --- a/toxcore/util.h +++ b/toxcore/util.h @@ -64,7 +64,9 @@ bool rb_full(const RingBuffer *b); bool rb_empty(const RingBuffer *b); void* rb_write(RingBuffer* b, void* p); bool rb_read(RingBuffer* b, void** p); -void rb_clear(RingBuffer *b); RingBuffer *rb_new(int size); -void rb_free(RingBuffer *b); +void rb_kill(RingBuffer *b); +uint16_t rb_size(const RingBuffer *b); +uint16_t rb_data(const RingBuffer* b, void** dest); + #endif /* __UTIL_H__ */ From 8f130b6f256badd93e96c6a7d074d461c6fe106d Mon Sep 17 00:00:00 2001 From: Eniz Vukovic Date: Sun, 11 Oct 2015 00:01:44 +0200 Subject: [PATCH 64/97] Remove old comment --- other/apidsl/toxav.in.h | 7 +------ toxav/Makefile.inc | 1 - toxav/bwcontroler.c | 2 +- toxav/toxav.c | 2 +- toxav/toxav.h | 9 ++------- 5 files changed, 5 insertions(+), 16 deletions(-) diff --git a/other/apidsl/toxav.in.h b/other/apidsl/toxav.in.h index c272b9347..20efa1d1f 100644 --- a/other/apidsl/toxav.in.h +++ b/other/apidsl/toxav.in.h @@ -433,12 +433,7 @@ bool call_control (uint32_t friend_number, CALL_CONTROL control) { ******************************************************************************/ namespace bit_rate { /** - * Set the audio bit rate to be used in subsequent audio frames. If the passed - * bit rate is the same as the current bit rate this function will return true - * without calling a callback. If there is an active non forceful setup with the - * passed audio bit rate and the new set request is forceful, the bit rate is - * forcefully set and the previous non forceful request is cancelled. The active - * non forceful setup will be canceled in favour of new non forceful setup. + * Set the audio bit rate to be used in subsequent audio/video frames. * * @param friend_number The friend number. * @param audio_bit_rate The new audio bit rate in Kb/sec. Set to 0 to disable diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc index 232c06de5..083f862fd 100644 --- a/toxav/Makefile.inc +++ b/toxav/Makefile.inc @@ -18,7 +18,6 @@ libtoxav_la_SOURCES = ../toxav/rtp.h \ ../toxav/bwcontroler.c \ ../toxav/toxav.h \ ../toxav/toxav.c \ - ../toxav/toxav_old.h \ ../toxav/toxav_old.c libtoxav_la_CFLAGS = -I../toxcore \ diff --git a/toxav/bwcontroler.c b/toxav/bwcontroler.c index 2c468ce3f..a4328045f 100644 --- a/toxav/bwcontroler.c +++ b/toxav/bwcontroler.c @@ -200,7 +200,7 @@ int on_update (BWControler *bwc, struct BWCMessage *msg) int bwc_handle_data(Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object) { if (length != sizeof(struct BWCMessage)) - return; + return -1; /* NOTE the data is mutable */ return on_update(object, (struct BWCMessage *) data); diff --git a/toxav/toxav.c b/toxav/toxav.c index 4a413b66c..7585206c4 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -63,7 +63,7 @@ typedef struct ToxAVCall_s { struct ToxAVCall_s *next; } ToxAVCall; -struct ToxAV_s { +struct ToxAV { Messenger *m; MSISession *msi; diff --git a/toxav/toxav.h b/toxav/toxav.h index befae5dc8..d558991f8 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -87,7 +87,7 @@ typedef struct Tox Tox; */ #ifndef TOXAV_DEFINED #define TOXAV_DEFINED -typedef struct ToxAV_s ToxAV; +typedef struct ToxAV ToxAV; #endif /* TOXAV_DEFINED */ @@ -510,12 +510,7 @@ typedef enum TOXAV_ERR_BIT_RATE_SET { } TOXAV_ERR_BIT_RATE_SET; /** - * Set the audio bit rate to be used in subsequent audio frames. If the passed - * bit rate is the same as the current bit rate this function will return true - * without calling a callback. If there is an active non forceful setup with the - * passed audio bit rate and the new set request is forceful, the bit rate is - * forcefully set and the previous non forceful request is cancelled. The active - * non forceful setup will be canceled in favour of new non forceful setup. + * Set the audio bit rate to be used in subsequent audio/video frames. * * @param friend_number The friend number of the friend for which to set the * audio bit rate. From e5ddc0af03d2852fbfb723a48e6e99dfbdb63a82 Mon Sep 17 00:00:00 2001 From: Eniz Vukovic Date: Sun, 11 Oct 2015 00:43:05 +0200 Subject: [PATCH 65/97] Test should not assert when hangup fails --- auto_tests/toxav_many_test.c | 1 - 1 file changed, 1 deletion(-) diff --git a/auto_tests/toxav_many_test.c b/auto_tests/toxav_many_test.c index 7dc82c6f2..9819b6949 100644 --- a/auto_tests/toxav_many_test.c +++ b/auto_tests/toxav_many_test.c @@ -192,7 +192,6 @@ void* call_thread(void* pd) if (rc != TOXAV_ERR_CALL_CONTROL_OK) { printf("toxav_call_control failed: %d %p %p\n", rc, AliceAV, BobAV); - ck_assert(0); } } From bb67d9b273b534205df2326fca5712b2d0efc4a5 Mon Sep 17 00:00:00 2001 From: Eniz Vukovic Date: Mon, 12 Oct 2015 22:30:55 +0200 Subject: [PATCH 66/97] Return removed groupchats code --- toxav/toxav.h | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/toxav/toxav.h b/toxav/toxav.h index d558991f8..b36debb1e 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -683,6 +683,51 @@ typedef void toxav_video_receive_frame_cb(ToxAV *toxAV, uint32_t friend_number, */ void toxav_callback_video_receive_frame(ToxAV *toxAV, toxav_video_receive_frame_cb *callback, void *user_data); +/** + * NOTE Compatibility with old toxav group calls TODO remove + */ +/* Create a new toxav group. + * + * return group number on success. + * return -1 on failure. + * + * Audio data callback format: + * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + */ +int toxav_add_av_groupchat(Tox *tox, void (*audio_callback)(void*, int, int, const int16_t *, unsigned int, uint8_t, + unsigned int, void *), void *userdata); + +/* Join a AV group (you need to have been invited first.) + * + * returns group number on success + * returns -1 on failure. + * + * Audio data callback format (same as the one for toxav_add_av_groupchat()): + * audio_callback(Tox *tox, int groupnumber, int peernumber, const int16_t *pcm, unsigned int samples, uint8_t channels, unsigned int sample_rate, void *userdata) + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + */ +int toxav_join_av_groupchat(Tox *tox, int32_t friendnumber, const uint8_t *data, uint16_t length, + void (*audio_callback)(void*, int, int, const int16_t *, unsigned int, uint8_t, unsigned int, void *), void *userdata); + +/* Send audio to the group chat. + * + * return 0 on success. + * return -1 on failure. + * + * Note that total size of pcm in bytes is equal to (samples * channels * sizeof(int16_t)). + * + * Valid number of samples are ((sample rate) * (audio length (Valid ones are: 2.5, 5, 10, 20, 40 or 60 ms)) / 1000) + * Valid number of channels are 1 or 2. + * Valid sample rates are 8000, 12000, 16000, 24000, or 48000. + * + * Recommended values are: samples = 960, channels = 1, sample_rate = 48000 + */ +int toxav_group_send_audio(Tox *tox, int groupnumber, const int16_t *pcm, unsigned int samples, uint8_t channels, + unsigned int sample_rate); + #ifdef __cplusplus } #endif From 363771ea3c13e11dd2bf6f4fb6aed724652fdabb Mon Sep 17 00:00:00 2001 From: Eniz Vukovic Date: Tue, 13 Oct 2015 00:35:46 +0200 Subject: [PATCH 67/97] Fix typo and change logger config option --- configure.ac | 4 ++-- other/apidsl/toxav.in.h | 2 +- toxav/toxav.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 639fc20c5..01680ac63 100644 --- a/configure.ac +++ b/configure.ac @@ -82,8 +82,8 @@ AC_ARG_ENABLE([randombytes-stir], ] ) -AC_ARG_ENABLE([logger], - [AC_HELP_STRING([--enable-logger], [enable logging (default: auto)]) ], +AC_ARG_ENABLE([logging], + [AC_HELP_STRING([--enable-logging], [enable logging (default: auto)]) ], [ if test "x$enableval" = "xyes"; then TOX_LOGGER="yes" diff --git a/other/apidsl/toxav.in.h b/other/apidsl/toxav.in.h index 20efa1d1f..592c3def7 100644 --- a/other/apidsl/toxav.in.h +++ b/other/apidsl/toxav.in.h @@ -389,7 +389,7 @@ enum class CALL_CONTROL { /** * Request that the friend stops sending video. Regardless of the friend's * compliance, this will cause the ${event video.receive_frame} event to stop being - * triggered on receiving an video frame from the friend. + * triggered on receiving a video frame from the friend. */ HIDE_VIDEO, /** diff --git a/toxav/toxav.h b/toxav/toxav.h index b36debb1e..859fe9afd 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -431,7 +431,7 @@ typedef enum TOXAV_CALL_CONTROL { /** * Request that the friend stops sending video. Regardless of the friend's * compliance, this will cause the video_receive_frame event to stop being - * triggered on receiving an video frame from the friend. + * triggered on receiving a video frame from the friend. */ TOXAV_CALL_CONTROL_HIDE_VIDEO, /** From 118a59de3ceb818474efce21cfb56dc36a387d87 Mon Sep 17 00:00:00 2001 From: Zetok Zalbavar Date: Wed, 14 Oct 2015 08:26:40 +0100 Subject: [PATCH 68/97] Fix file modes revert changes to it from 4f76a8c89eea77ebc190a324ad128e3c7dda298c --- autogen.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 autogen.sh diff --git a/autogen.sh b/autogen.sh old mode 100644 new mode 100755 From 90b1ca872731d7911d6318c3a6e05133ea6071b8 Mon Sep 17 00:00:00 2001 From: Eniz Vukovic Date: Fri, 23 Oct 2015 22:52:32 +0200 Subject: [PATCH 69/97] Fix setting resolution for vpx v1.4 (or newer i suppose) --- toxav/toxav.c | 2 +- toxav/video.c | 142 +++++++++++++++++++++++++++++--------------------- toxav/video.h | 2 +- 3 files changed, 85 insertions(+), 61 deletions(-) diff --git a/toxav/toxav.c b/toxav/toxav.c index 7585206c4..a9e8b6f1f 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -752,7 +752,7 @@ bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, u goto END; } - if (vc_reconfigure_encoder(call->video.second->encoder, call->video_bit_rate * 1000, width, height) != 0) { + if (vc_reconfigure_encoder(call->video.second, call->video_bit_rate * 1000, width, height) != 0) { pthread_mutex_unlock(call->mutex_video); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; diff --git a/toxav/video.c b/toxav/video.c index 919e3c81a..acc1852bc 100644 --- a/toxav/video.c +++ b/toxav/video.c @@ -36,9 +36,6 @@ #define MAX_DECODE_TIME_US 0 /* Good quality encode. */ #define VIDEO_DECODE_BUFFER_SIZE 20 - -bool create_video_encoder (vpx_codec_ctx_t *dest, int32_t bit_rate); - VCSession *vc_new(ToxAV *av, uint32_t friend_number, toxav_video_receive_frame_cb *cb, void *cb_data) { VCSession *vc = calloc(sizeof(VCSession), 1); @@ -64,9 +61,41 @@ VCSession *vc_new(ToxAV *av, uint32_t friend_number, toxav_video_receive_frame_c goto BASE_CLEANUP; } - if (!create_video_encoder(vc->encoder, 500000)) { - vpx_codec_destroy(vc->decoder); - goto BASE_CLEANUP; + /* Set encoder to some initial values + */ + vpx_codec_enc_cfg_t cfg; + rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); + + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); + goto BASE_CLEANUP_1; + } + + cfg.rc_target_bitrate = 500000; + cfg.g_w = 800; + cfg.g_h = 600; + cfg.g_pass = VPX_RC_ONE_PASS; + /* FIXME If we set error resilience the app will crash due to bug in vp8. + Perhaps vp9 has solved it?*/ +// cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; + cfg.g_lag_in_frames = 0; + cfg.kf_min_dist = 0; + cfg.kf_max_dist = 48; + cfg.kf_mode = VPX_KF_AUTO; + + rc = vpx_codec_enc_init(vc->encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); + + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); + goto BASE_CLEANUP_1; + } + + rc = vpx_codec_control(vc->encoder, VP8E_SET_CPUUSED, 8); + + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); + vpx_codec_destroy(vc->encoder); + goto BASE_CLEANUP_1; } vc->linfts = current_time_monotonic(); @@ -78,6 +107,8 @@ VCSession *vc_new(ToxAV *av, uint32_t friend_number, toxav_video_receive_frame_c return vc; +BASE_CLEANUP_1: + vpx_codec_destroy(vc->decoder); BASE_CLEANUP: pthread_mutex_destroy(vc->queue_mutex); rb_kill(vc->vbuf_raw); @@ -176,68 +207,61 @@ int vc_queue_message(void *vcp, struct RTPMessage *msg) return 0; } -int vc_reconfigure_encoder(vpx_codec_ctx_t *vccdc, uint32_t bit_rate, uint16_t width, uint16_t height) +int vc_reconfigure_encoder(VCSession* vc, uint32_t bit_rate, uint16_t width, uint16_t height) { - if (!vccdc) + if (!vc) return -1; - vpx_codec_enc_cfg_t cfg = *vccdc->config.enc; - + vpx_codec_enc_cfg_t cfg = *vc->encoder->config.enc; + int rc; + if (cfg.rc_target_bitrate == bit_rate && cfg.g_w == width && cfg.g_h == height) return 0; /* Nothing changed */ - cfg.rc_target_bitrate = bit_rate; - cfg.g_w = width; - cfg.g_h = height; + if (cfg.g_w == width && cfg.g_h == height) + { + /* Only bit rate changed */ + cfg.rc_target_bitrate = bit_rate; + + rc = vpx_codec_enc_config_set(vc->encoder, &cfg); + + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); + return -1; + } + } + else + { + /* Resolution is changed, must reinitialize encoder since libvpx v1.4 doesn't support + * reconfiguring encoder to use resolutions greater than initially set. + */ + + LOGGER_DEBUG("Have to reinitialize vpx encoder on session %p", vc); + + cfg.rc_target_bitrate = bit_rate; + cfg.g_w = width; + cfg.g_h = height; - int rc = vpx_codec_enc_config_set(vccdc, &cfg); + vpx_codec_ctx_t new_c; + + rc = vpx_codec_enc_init(&new_c, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); - if (rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - return -1; + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); + return -1; + } + + rc = vpx_codec_control(&new_c, VP8E_SET_CPUUSED, 8); + + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); + vpx_codec_destroy(&new_c); + return -1; + } + + vpx_codec_destroy(vc->encoder); + memcpy(vc->encoder, &new_c, sizeof(new_c)); } return 0; } - - -bool create_video_encoder (vpx_codec_ctx_t *dest, int32_t bit_rate) -{ - assert(dest); - - vpx_codec_enc_cfg_t cfg; - int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); - - if (rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); - return false; - } - - cfg.rc_target_bitrate = bit_rate; - cfg.g_w = 800; - cfg.g_h = 600; - cfg.g_pass = VPX_RC_ONE_PASS; - /* FIXME If we set error resilience the app will crash due to bug in vp8. - Perhaps vp9 has solved it?*/ -// cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; - cfg.g_lag_in_frames = 0; - cfg.kf_min_dist = 0; - cfg.kf_max_dist = 48; - cfg.kf_mode = VPX_KF_AUTO; - - rc = vpx_codec_enc_init(dest, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); - - if (rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); - return false; - } - - rc = vpx_codec_control(dest, VP8E_SET_CPUUSED, 8); - - if (rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - vpx_codec_destroy(dest); - } - - return true; -} diff --git a/toxav/video.h b/toxav/video.h index 1ad1f6f50..51f343188 100644 --- a/toxav/video.h +++ b/toxav/video.h @@ -62,6 +62,6 @@ VCSession *vc_new(ToxAV* av, uint32_t friend_number, toxav_video_receive_frame_c void vc_kill(VCSession *vc); void vc_iterate(VCSession *vc); int vc_queue_message(void *vcp, struct RTPMessage *msg); -int vc_reconfigure_encoder(vpx_codec_ctx_t *vccdc, uint32_t bit_rate, uint16_t width, uint16_t height); +int vc_reconfigure_encoder(VCSession *vc, uint32_t bit_rate, uint16_t width, uint16_t height); #endif /* VIDEO_H */ From 283297a10d228c3bed28485106a18b6b0980cdda Mon Sep 17 00:00:00 2001 From: irungentoo Date: Sun, 9 Aug 2015 21:47:54 -0400 Subject: [PATCH 70/97] Increased timeouts. --- toxcore/friend_connection.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/toxcore/friend_connection.h b/toxcore/friend_connection.h index 32e947acb..044b8d7f2 100644 --- a/toxcore/friend_connection.h +++ b/toxcore/friend_connection.h @@ -40,10 +40,10 @@ #define PACKET_ID_FRIEND_REQUESTS 18 /* Interval between the sending of ping packets. */ -#define FRIEND_PING_INTERVAL 7 +#define FRIEND_PING_INTERVAL 8 /* If no packets are received from friend in this time interval, kill the connection. */ -#define FRIEND_CONNECTION_TIMEOUT (FRIEND_PING_INTERVAL * 3) +#define FRIEND_CONNECTION_TIMEOUT (FRIEND_PING_INTERVAL * 4) /* Time before friend is removed from the DHT after last hearing about him. */ #define FRIEND_DHT_TIMEOUT BAD_NODE_TIMEOUT From ef8b6ccf5658f9d09515a55d555fea1ed1d0a941 Mon Sep 17 00:00:00 2001 From: randoms Date: Sat, 8 Aug 2015 13:53:05 +0800 Subject: [PATCH 71/97] Install instructions: Use 1.0.3 tag for libsodium --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index a31be6450..ba9557923 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -395,7 +395,7 @@ Now we will build sodium crypto library: ```bash git clone https://github.com/jedisct1/libsodium/ cd libsodium -git checkout tags/1.0.0 +git checkout tags/1.0.3 ./autogen.sh ./configure --host="$WINDOWS_TOOLCHAIN" --prefix="$PREFIX_DIR" --disable-shared --enable-static make From 0ad1fadb97af7ead5ae5783ab6948b3f8f7e795b Mon Sep 17 00:00:00 2001 From: irungentoo Date: Tue, 11 Aug 2015 23:54:05 -0400 Subject: [PATCH 72/97] Remove useless code. --- toxcore/DHT.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/toxcore/DHT.c b/toxcore/DHT.c index ae8233027..6060a9eb6 100644 --- a/toxcore/DHT.c +++ b/toxcore/DHT.c @@ -2422,8 +2422,6 @@ static int dht_load_state_callback(void *outer, const uint8_t *data, uint32_t le int num = unpack_nodes(dht->loaded_nodes_list, MAX_SAVED_DHT_NODES, NULL, data, length, 0); - Node_format *client_list = (Node_format *)data; - if (num > 0) { dht->loaded_num_nodes = num; } else { From 7da177cfd398a7be267d4f893ddb7fbf1d0b7556 Mon Sep 17 00:00:00 2001 From: irungentoo Date: Wed, 12 Aug 2015 23:18:55 -0400 Subject: [PATCH 73/97] Increased max number of stored announce entries. --- toxcore/onion_announce.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toxcore/onion_announce.h b/toxcore/onion_announce.h index 0e041e506..a9128be89 100644 --- a/toxcore/onion_announce.h +++ b/toxcore/onion_announce.h @@ -25,7 +25,7 @@ #include "onion.h" -#define ONION_ANNOUNCE_MAX_ENTRIES 96 +#define ONION_ANNOUNCE_MAX_ENTRIES 160 #define ONION_ANNOUNCE_TIMEOUT 300 #define ONION_PING_ID_SIZE crypto_hash_sha256_BYTES From 826e18315bdcda7757cc82536e841716c94643d8 Mon Sep 17 00:00:00 2001 From: irungentoo Date: Wed, 9 Sep 2015 14:40:22 -0400 Subject: [PATCH 74/97] Increased cookie timeout. --- toxcore/net_crypto.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toxcore/net_crypto.c b/toxcore/net_crypto.c index 628666c0f..8f7c611ea 100644 --- a/toxcore/net_crypto.c +++ b/toxcore/net_crypto.c @@ -47,7 +47,7 @@ static uint8_t crypt_connection_id_not_valid(const Net_Crypto *c, int crypt_conn } /* cookie timeout in seconds */ -#define COOKIE_TIMEOUT 10 +#define COOKIE_TIMEOUT 15 #define COOKIE_DATA_LENGTH (crypto_box_PUBLICKEYBYTES * 2) #define COOKIE_CONTENTS_LENGTH (sizeof(uint64_t) + COOKIE_DATA_LENGTH) #define COOKIE_LENGTH (crypto_box_NONCEBYTES + COOKIE_CONTENTS_LENGTH + crypto_box_MACBYTES) From 9c824105a91f8ee37d61c9356d6a80f58ca00fba Mon Sep 17 00:00:00 2001 From: tux3 Date: Mon, 14 Sep 2015 19:05:44 +0200 Subject: [PATCH 75/97] Fix PACKET_ID_OFFLINE fallinthrough PACKET_ID_NICKNAME This fixes the PACKET_ID_OFFLINE handling code incorrectly falling through to the PACKET_ID_NICKNAME handling code This bug resulted in a friend B's name being wiped on friend A's client after B removes A Fixes tux3/qTox#1967 Fixes #1426 --- toxcore/Messenger.c | 1 + 1 file changed, 1 insertion(+) diff --git a/toxcore/Messenger.c b/toxcore/Messenger.c index 4cd9e1d6f..ac5d4ed2e 100644 --- a/toxcore/Messenger.c +++ b/toxcore/Messenger.c @@ -1930,6 +1930,7 @@ static int handle_packet(void *object, int i, uint8_t *temp, uint16_t len) break; set_friend_status(m, i, FRIEND_CONFIRMED); + break; } case PACKET_ID_NICKNAME: { From 916bfedff572a94a482e6e5f5d5ab0a98f91efae Mon Sep 17 00:00:00 2001 From: Jeppler Date: Tue, 1 Sep 2015 21:42:20 +0200 Subject: [PATCH 76/97] small typo change bob -> Bob Bobs -> Bob's fixed some wired sentences --- docs/updates/Crypto.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/updates/Crypto.md b/docs/updates/Crypto.md index cee03f0aa..87cdb71d0 100644 --- a/docs/updates/Crypto.md +++ b/docs/updates/Crypto.md @@ -9,27 +9,27 @@ The generated public key is set as the client_id of the peer. Adding a friend --------------- -Alice adds bob to her friends list by adding his 32 byte public key (client_id) to her friends list. +Alice adds Bob to her friends list by adding his 32 byte public key (client_id) to her friends list. 2 cases: -case 1: Alice adds Bobs public key and bob waits for Alice to attempt to connect to him. +case 1: Alice adds the public key Bob, then Bob waits for Alice to attempt to connect to him. case 2: Bob and Alice add their respective public keys to their friends list at the same time. case 1: -Alice sends a onion data (see: Prevent_tracking.txt) packet to bob with the encrypted part containing the friends request like so: +Alice sends an onion data (see: Prevent_tracking.txt) packet to Bob with the encrypted part containing the friend request like so: ``` [char with a value of 32][nospam number (4 bytes)][Message] ``` -Ex message: hello bob it's me alice -_- add me pl0x. +Ex message: hello Bob it's me Alice -_- add me pl0x. For more info on the nospam see: Spam_Prevention.txt Bob receives the request and decrypts the message using the function crypto_box_open() If the message decrypts successfully: -If Alice is already in Bobs friends list: case 2 +If Alice is already in Bob's friends list: case 2 If Alice is not in Bob's friends list and the nospam is good: Bob is prompt to add Alice and is shown the message from her. -If Bobs accepts Alice's friends request he adds her public key to his friends list. +If Bob accepts Alice friend request he adds her public key to his friends list. case 2: Bob and Alice both have the others public key in their friends list, they are ready for the next step: Connecting to an already added friend @@ -46,9 +46,9 @@ Crypto request packets -------------------------------------- ``` -[char with a value of 32][Bob's (The reciever's) Public key (client_id) (32 bytes))][Alice's (The sender's) Public key (client_id) (32 bytes)][Random nonce (24 bytes)][Encrypted message] +[char with a value of 32][Bob (The reciever's) Public key (client_id) (32 bytes))][Alice's (The sender's) Public key (client_id) (32 bytes)][Random nonce (24 bytes)][Encrypted message] ``` -The encrypted message is encrypted with crypto_box() (using Bobs public key, Alice's private key and the nonce (randomly generated 24 bytes)) and is a message from Alice in which she tells Bob who she is. +The encrypted message is encrypted with crypto_box() (using Bob's public key, Alice's private key and the nonce (randomly generated 24 bytes)) and is a message from Alice in which she tells Bob who she is. Each node can route the request to the receiver if they are connected to him. This is to bypass bad NATs. From ea06bf8e573ccd244576af30c4ea360046f4d275 Mon Sep 17 00:00:00 2001 From: Jeppler Date: Tue, 1 Sep 2015 21:46:08 +0200 Subject: [PATCH 77/97] small typo change really small change --- docs/updates/Crypto.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/updates/Crypto.md b/docs/updates/Crypto.md index 87cdb71d0..cdefec30f 100644 --- a/docs/updates/Crypto.md +++ b/docs/updates/Crypto.md @@ -11,7 +11,7 @@ Adding a friend Alice adds Bob to her friends list by adding his 32 byte public key (client_id) to her friends list. 2 cases: -case 1: Alice adds the public key Bob, then Bob waits for Alice to attempt to connect to him. +case 1: Alice adds the public key of Bob, then Bob waits for Alice to attempt to connect to him. case 2: Bob and Alice add their respective public keys to their friends list at the same time. case 1: From 4afe6a295e26daed2f817355924a6e26aada5452 Mon Sep 17 00:00:00 2001 From: Jeppler Date: Wed, 2 Sep 2015 00:03:51 +0200 Subject: [PATCH 78/97] changed friends list to friend list --- docs/updates/Crypto.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/updates/Crypto.md b/docs/updates/Crypto.md index cdefec30f..bfb010342 100644 --- a/docs/updates/Crypto.md +++ b/docs/updates/Crypto.md @@ -9,10 +9,10 @@ The generated public key is set as the client_id of the peer. Adding a friend --------------- -Alice adds Bob to her friends list by adding his 32 byte public key (client_id) to her friends list. +Alice adds Bob to her friend list by adding his 32 byte public key (client_id) to her friend list. 2 cases: case 1: Alice adds the public key of Bob, then Bob waits for Alice to attempt to connect to him. -case 2: Bob and Alice add their respective public keys to their friends list at the same time. +case 2: Bob and Alice add their respective public keys to their friend lists at the same time. case 1: Alice sends an onion data (see: Prevent_tracking.txt) packet to Bob with the encrypted part containing the friend request like so: @@ -27,12 +27,12 @@ For more info on the nospam see: Spam_Prevention.txt Bob receives the request and decrypts the message using the function crypto_box_open() If the message decrypts successfully: -If Alice is already in Bob's friends list: case 2 -If Alice is not in Bob's friends list and the nospam is good: Bob is prompt to add Alice and is shown the message from her. -If Bob accepts Alice friend request he adds her public key to his friends list. +If Alice is already in Bob's friend list: case 2 +If Alice is not in Bob's friend list and the nospam is good: Bob is prompt to add Alice and is shown the message from her. +If Bob accepts Alice friend request he adds her public key to his friend list. case 2: -Bob and Alice both have the others public key in their friends list, they are ready for the next step: Connecting to an already added friend +Bob and Alice both have the others public key in their friend list, they are ready for the next step: Connecting to an already added friend In the next step only crypto_box() is used for encryption and only crypto_box_open() for decryption (just like in the last step.) From 90c2a6964b592f2fb5c64bc92bda332a7ad57c0c Mon Sep 17 00:00:00 2001 From: anoadragon453 Date: Wed, 9 Sep 2015 12:10:56 -0700 Subject: [PATCH 79/97] Changed Binaries/Downloads link to new Wiki page --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ab18debce..4d08dcda2 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ With the rise of governmental monitoring programs, Tox, a FOSS initiative, aims to be an easy to use, all-in-one communication platform that ensures their users full privacy and secure message delivery.

-[**Website**](https://tox.chat) **|** [**Wiki**](https://wiki.tox.chat/) **|** [**Blog**](https://blog.tox.chat/) **|** [**FAQ**](https://wiki.tox.chat/doku.php?id=users:faq) **|** [**Binaries/Downloads**](https://wiki.tox.chat/doku.php?id=developers:binaries) **|** [**Clients**](https://wiki.tox.chat/doku.php?id=clients) **|** [**Compiling**](/INSTALL.md) **|** **IRC Channel:** [#tox@freenode](https://webchat.freenode.net/?channels=tox) +[**Website**](https://tox.chat) **|** [**Wiki**](https://wiki.tox.chat/) **|** [**Blog**](https://blog.tox.chat/) **|** [**FAQ**](https://wiki.tox.chat/doku.php?id=users:faq) **|** [**Binaries/Downloads**](https://wiki.tox.chat/Binaries) **|** [**Clients**](https://wiki.tox.chat/doku.php?id=clients) **|** [**Compiling**](/INSTALL.md) **|** **IRC Channel:** [#tox@freenode](https://webchat.freenode.net/?channels=tox) ## The Complex Stuff: From b69959355c8ab9a870abd984e150656295631021 Mon Sep 17 00:00:00 2001 From: Pascal Weinberger Date: Wed, 9 Sep 2015 11:53:28 +0200 Subject: [PATCH 80/97] Fix ubuntu git link for libvpx --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index ba9557923..adb0810a6 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -534,7 +534,7 @@ Install from source (example for most unix-like OS's): libvpx: ```bash -git clone http://git.chromium.org/webm/libvpx.git +git clone https://chromium.googlesource.com/webm/libvpx cd libvpx ./configure make -j3 From 4e5dd8a6f6169e7ea04ac48f078204f2cd867b88 Mon Sep 17 00:00:00 2001 From: irungentoo Date: Mon, 28 Sep 2015 19:22:02 -0400 Subject: [PATCH 81/97] Lowered interval in onion_client. --- toxcore/onion_client.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toxcore/onion_client.h b/toxcore/onion_client.h index ad28ac515..416d593a6 100644 --- a/toxcore/onion_client.h +++ b/toxcore/onion_client.h @@ -29,7 +29,7 @@ #include "ping_array.h" #define MAX_ONION_CLIENTS 8 -#define ONION_NODE_PING_INTERVAL 20 +#define ONION_NODE_PING_INTERVAL 15 #define ONION_NODE_TIMEOUT (ONION_NODE_PING_INTERVAL * 3) /* The interval in seconds at which to tell our friends where we are */ From a9060b77af7c7829b60cb1cdbfd2d954fd12f6a7 Mon Sep 17 00:00:00 2001 From: Jacob Henner Date: Fri, 2 Oct 2015 00:30:10 -0400 Subject: [PATCH 82/97] Adding dev irc channel --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 4d08dcda2..851110bfb 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ With the rise of governmental monitoring programs, Tox, a FOSS initiative, aims to be an easy to use, all-in-one communication platform that ensures their users full privacy and secure message delivery.

-[**Website**](https://tox.chat) **|** [**Wiki**](https://wiki.tox.chat/) **|** [**Blog**](https://blog.tox.chat/) **|** [**FAQ**](https://wiki.tox.chat/doku.php?id=users:faq) **|** [**Binaries/Downloads**](https://wiki.tox.chat/Binaries) **|** [**Clients**](https://wiki.tox.chat/doku.php?id=clients) **|** [**Compiling**](/INSTALL.md) **|** **IRC Channel:** [#tox@freenode](https://webchat.freenode.net/?channels=tox) +[**Website**](https://tox.chat) **|** [**Wiki**](https://wiki.tox.chat/) **|** [**Blog**](https://blog.tox.chat/) **|** [**FAQ**](https://wiki.tox.chat/doku.php?id=users:faq) **|** [**Binaries/Downloads**](https://wiki.tox.chat/Binaries) **|** [**Clients**](https://wiki.tox.chat/doku.php?id=clients) **|** [**Compiling**](/INSTALL.md) **|** **IRC Channel:** [#tox@freenode](https://webchat.freenode.net/?channels=tox), [#tox-dev@freenode](https://webchat.freenode.net/?channels=tox-dev) ## The Complex Stuff: @@ -36,4 +36,3 @@ The goal of this project is to create a configuration-free P2P Skype replacement - [Compiling](/INSTALL.md) - [DHT Protocol](/docs/updates/DHT.md)
- [Crypto](/docs/updates/Crypto.md)
- From e7804c626840c7d8478244bc64c7b491aa3c21b5 Mon Sep 17 00:00:00 2001 From: Jacob Henner Date: Fri, 2 Oct 2015 00:38:15 -0400 Subject: [PATCH 83/97] Editing some README phrasing, IRC channels on new line --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 851110bfb..ed62a9b2c 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ ![Project Tox](https://raw.github.com/irungentoo/toxcore/master/other/tox.png "Project Tox") *** -With the rise of governmental monitoring programs, Tox, a FOSS initiative, aims to be an easy to use, all-in-one communication platform that ensures their users full privacy and secure message delivery.

+With the rise of government surveillance programs, Tox, a FOSS initiative, aims to be an easy to use, all-in-one communication platform that ensures full privacy and secure message delivery.

-[**Website**](https://tox.chat) **|** [**Wiki**](https://wiki.tox.chat/) **|** [**Blog**](https://blog.tox.chat/) **|** [**FAQ**](https://wiki.tox.chat/doku.php?id=users:faq) **|** [**Binaries/Downloads**](https://wiki.tox.chat/Binaries) **|** [**Clients**](https://wiki.tox.chat/doku.php?id=clients) **|** [**Compiling**](/INSTALL.md) **|** **IRC Channel:** [#tox@freenode](https://webchat.freenode.net/?channels=tox), [#tox-dev@freenode](https://webchat.freenode.net/?channels=tox-dev) +[**Website**](https://tox.chat) **|** [**Wiki**](https://wiki.tox.chat/) **|** [**Blog**](https://blog.tox.chat/) **|** [**FAQ**](https://wiki.tox.chat/doku.php?id=users:faq) **|** [**Binaries/Downloads**](https://wiki.tox.chat/Binaries) **|** [**Clients**](https://wiki.tox.chat/doku.php?id=clients) **|** [**Compiling**](/INSTALL.md) + +**IRC Channels:** [#tox@freenode](https://webchat.freenode.net/?channels=tox), [#tox-dev@freenode](https://webchat.freenode.net/?channels=tox-dev) ## The Complex Stuff: From 21b45b8b741cb407cacb477c2f301abdc0215b7a Mon Sep 17 00:00:00 2001 From: Soumitra Date: Fri, 9 Oct 2015 11:02:07 +0530 Subject: [PATCH 84/97] Added ommited words in comment --- auto_tests/messenger_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_tests/messenger_test.c b/auto_tests/messenger_test.c index 2a813c9b3..ad5d40aeb 100644 --- a/auto_tests/messenger_test.c +++ b/auto_tests/messenger_test.c @@ -135,7 +135,7 @@ START_TEST(test_m_addfriend) if(m_addfriend(m, (uint8_t *)friend_id, (uint8_t *)good_data, really_bad_len) != FAERR_TOOLONG) ck_abort_msg("m_addfriend did NOT catch the following length: %d\n", really_bad_len); */ -/* this will error if the original m_addfriend_norequest() failed */ +/* this will give an error if the original m_addfriend_norequest() failed */ /* if(m_addfriend(m, (uint8_t *)friend_id, (uint8_t *)good_data, good_len) != FAERR_ALREADYSENT) ck_abort_msg("m_addfriend did NOT catch adding a friend we already have.\n" "(this can be caused by the error of m_addfriend_norequest in" @@ -144,7 +144,7 @@ START_TEST(test_m_addfriend) if(m_addfriend(m, (uint8_t *)good_id_b, (uint8_t *)bad_data, bad_len) != FAERR_NOMESSAGE) ck_abort_msg("m_addfriend did NOT catch the following length: %d\n", bad_len); */ -/* this should REALLY error */ +/* this should REALLY give an error */ /* * TODO: validate client_id in m_addfriend? if(m_addfriend((uint8_t *)bad_id, (uint8_t *)good_data, good_len) >= 0) From fc25ca11d10725974737e0abd9ada5e7d4d037ff Mon Sep 17 00:00:00 2001 From: Soumitra Date: Mon, 12 Oct 2015 12:04:52 +0530 Subject: [PATCH 85/97] Changed gramatically incorrect comment in messenger_test --- auto_tests/messenger_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_tests/messenger_test.c b/auto_tests/messenger_test.c index ad5d40aeb..2b0b2519f 100644 --- a/auto_tests/messenger_test.c +++ b/auto_tests/messenger_test.c @@ -135,7 +135,7 @@ START_TEST(test_m_addfriend) if(m_addfriend(m, (uint8_t *)friend_id, (uint8_t *)good_data, really_bad_len) != FAERR_TOOLONG) ck_abort_msg("m_addfriend did NOT catch the following length: %d\n", really_bad_len); */ -/* this will give an error if the original m_addfriend_norequest() failed */ +/* this will return an error if the original m_addfriend_norequest() failed */ /* if(m_addfriend(m, (uint8_t *)friend_id, (uint8_t *)good_data, good_len) != FAERR_ALREADYSENT) ck_abort_msg("m_addfriend did NOT catch adding a friend we already have.\n" "(this can be caused by the error of m_addfriend_norequest in" @@ -144,7 +144,7 @@ START_TEST(test_m_addfriend) if(m_addfriend(m, (uint8_t *)good_id_b, (uint8_t *)bad_data, bad_len) != FAERR_NOMESSAGE) ck_abort_msg("m_addfriend did NOT catch the following length: %d\n", bad_len); */ -/* this should REALLY give an error */ +/* this should REALLY return an error */ /* * TODO: validate client_id in m_addfriend? if(m_addfriend((uint8_t *)bad_id, (uint8_t *)good_data, good_len) >= 0) From 7eaf97a6c3663f71722b2eaad9f6e289261723ce Mon Sep 17 00:00:00 2001 From: Vishnu Ks Date: Wed, 14 Oct 2015 15:49:44 +0530 Subject: [PATCH 86/97] updated git repo urls updated the git urls from `git` to `https`. The old urls are no longer supported --- INSTALL.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index adb0810a6..b5a925ada 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -78,7 +78,7 @@ Note, if you install from ports select NaCl for performance, and sodium if you w You should get and install [libsodium](https://github.com/jedisct1/libsodium). If you have installed `libsodium` from repo, ommit this step, and jump directly to [compiling toxcore](#compile-toxcore): ```bash -git clone git://github.com/jedisct1/libsodium.git +git clone https://github.com/jedisct1/libsodium.git cd libsodium git checkout tags/1.0.3 ./autogen.sh @@ -93,7 +93,7 @@ Or if checkinstall is not easily available for your distribution (e.g., Fedora), this will install the libs to /usr/local/lib and the headers to /usr/local/include: ```bash -git clone git://github.com/jedisct1/libsodium.git +git clone https://github.com/jedisct1/libsodium.git cd libsodium git checkout tags/1.0.3 ./autogen.sh @@ -121,7 +121,7 @@ sudo ldconfig Then clone this repo, generate makefile, and install `toxcore` system-wide: ```bash -git clone git://github.com/irungentoo/toxcore.git +git clone https://github.com/irungentoo/toxcore.git cd toxcore autoreconf -i ./configure @@ -153,7 +153,7 @@ brew install libtool automake autoconf libsodium check ``` Then clone this repo and generate makefile: ```bash -git clone git://github.com/irungentoo/toxcore.git +git clone https://github.com/irungentoo/toxcore.git cd toxcore autoreconf -i ./configure From 01570177603a201128279f4d176f013764d95bd8 Mon Sep 17 00:00:00 2001 From: irungentoo Date: Wed, 21 Oct 2015 21:01:13 -0400 Subject: [PATCH 87/97] Prevent seek equal to size of file transfer so that toxcore respects what the docs say. --- toxcore/Messenger.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/toxcore/Messenger.c b/toxcore/Messenger.c index ac5d4ed2e..6d45077ab 100644 --- a/toxcore/Messenger.c +++ b/toxcore/Messenger.c @@ -1264,7 +1264,7 @@ int file_seek(const Messenger *m, int32_t friendnumber, uint32_t filenumber, uin if (ft->status != FILESTATUS_NOT_ACCEPTED) return -5; - if (position > ft->size) { + if (position >= ft->size) { return -6; } @@ -1569,7 +1569,7 @@ static int handle_filecontrol(Messenger *m, int32_t friendnumber, uint8_t receiv return -1; } - /* seek can only be sent by the receiver to seek before resuming broken tranfers. */ + /* seek can only be sent by the receiver to seek before resuming broken transfers. */ if (ft->status != FILESTATUS_NOT_ACCEPTED || !receive_send) { return -1; } @@ -1577,7 +1577,7 @@ static int handle_filecontrol(Messenger *m, int32_t friendnumber, uint8_t receiv memcpy(&position, data, sizeof(position)); net_to_host((uint8_t *) &position, sizeof(position)); - if (position > ft->size) { + if (position >= ft->size) { return -1; } @@ -2128,6 +2128,11 @@ static int handle_packet(void *object, int i, uint8_t *temp, uint16_t len) file_data = data + 1; } + /* Prevent more data than the filesize from being passed to clients. */ + if ((ft->transferred + file_data_length) > ft->size) { + file_data_length = ft->size - ft->transferred; + } + if (m->file_filedata) (*m->file_filedata)(m, i, real_filenumber, position, file_data, file_data_length, m->file_filedata_userdata); From 4f3be9f354dca109a41c844f81d9ffa5873f67d1 Mon Sep 17 00:00:00 2001 From: mannol Date: Mon, 19 Oct 2015 14:18:58 +0200 Subject: [PATCH 88/97] Fix docs --- toxav/toxav.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/toxav/toxav.h b/toxav/toxav.h index 859fe9afd..b82a2ace8 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -510,10 +510,10 @@ typedef enum TOXAV_ERR_BIT_RATE_SET { } TOXAV_ERR_BIT_RATE_SET; /** - * Set the audio bit rate to be used in subsequent audio/video frames. + * Set the bit rate to be used in subsequent audio/video frames. * * @param friend_number The friend number of the friend for which to set the - * audio bit rate. + * bit rate. * @param audio_bit_rate The new audio bit rate in Kb/sec. Set to 0 to disable * audio sending. Set to -1 to leave unchanged. * @param video_bit_rate The new video bit rate in Kb/sec. Set to 0 to disable @@ -529,7 +529,7 @@ bool toxav_bit_rate_set(ToxAV *toxAV, uint32_t friend_number, int32_t audio_bit_ * point core suggests new bit rates. * * @param friend_number The friend number of the friend for which to set the - * audio bit rate. + * bit rate. * @param audio_bit_rate Suggested maximum audio bit rate in Kb/sec. * @param video_bit_rate Suggested maximum video bit rate in Kb/sec. */ From b23819a4d17b5ec083e47a2e0f659bc7fb9980b3 Mon Sep 17 00:00:00 2001 From: irungentoo Date: Fri, 23 Oct 2015 09:59:21 -0400 Subject: [PATCH 89/97] Fixed some memory related bugs. --- toxav/msi.c | 2 +- toxav/toxav.c | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/toxav/msi.c b/toxav/msi.c index ef307bcbd..7ad39a54e 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -528,7 +528,7 @@ MSICall *new_call (MSISession *session, uint32_t friend_number) session->calls_tail = session->calls_head = friend_number; } else if (session->calls_tail < friend_number) { /* Appending */ - void *tmp = realloc(session->calls, sizeof(MSICall *) * friend_number + 1); + void *tmp = realloc(session->calls, sizeof(MSICall *) * (friend_number + 1)); if (tmp == NULL) { free(rc); diff --git a/toxav/toxav.c b/toxav/toxav.c index a9e8b6f1f..ab73ab4e8 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -445,6 +445,7 @@ bool toxav_call_control(ToxAV *av, uint32_t friend_number, TOXAV_CALL_CONTROL co goto END; } + call->msi_call = NULL; /* No mather the case, terminate the call */ call_kill_transmission(call); call_remove(call); @@ -1037,8 +1038,8 @@ ToxAVCall *call_new(ToxAV *av, uint32_t friend_number, TOXAV_ERR_CALL *error) av->calls_tail = av->calls_head = friend_number; - } else if (av->calls_tail < friend_number) { /* Appending */ - void *tmp = realloc(av->calls, sizeof(ToxAVCall *) * friend_number + 1); + } else if (av->calls_tail <= friend_number) { /* Appending */ + void *tmp = realloc(av->calls, sizeof(ToxAVCall *) * (friend_number + 1)); if (tmp == NULL) { free(call); @@ -1097,7 +1098,10 @@ ToxAVCall *call_remove(ToxAVCall *call) /* Set av call in msi to NULL in order to know if call if ToxAVCall is * removed from the msi call. */ - call->msi_call->av_call = NULL; + if (call->msi_call) { + call->msi_call->av_call = NULL; + } + free(call); if (prev) From d8a85d9a78149497c8e7ad39197934a288d9a7c2 Mon Sep 17 00:00:00 2001 From: irungentoo Date: Fri, 23 Oct 2015 13:01:51 -0400 Subject: [PATCH 90/97] Fixed error in last commit. --- toxav/toxav.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toxav/toxav.c b/toxav/toxav.c index ab73ab4e8..645774be6 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -1038,7 +1038,7 @@ ToxAVCall *call_new(ToxAV *av, uint32_t friend_number, TOXAV_ERR_CALL *error) av->calls_tail = av->calls_head = friend_number; - } else if (av->calls_tail <= friend_number) { /* Appending */ + } else if (av->calls_tail < friend_number) { /* Appending */ void *tmp = realloc(av->calls, sizeof(ToxAVCall *) * (friend_number + 1)); if (tmp == NULL) { From d012bb1e8e0012950c00cd72471c5c77c6a5e9ac Mon Sep 17 00:00:00 2001 From: irungentoo Date: Fri, 23 Oct 2015 14:42:48 -0400 Subject: [PATCH 91/97] Added mutex lock+unlock. Should it be also added to the other parts of this function? --- toxav/toxav.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/toxav/toxav.c b/toxav/toxav.c index 645774be6..1607a9521 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -440,12 +440,16 @@ bool toxav_call_control(ToxAV *av, uint32_t friend_number, TOXAV_CALL_CONTROL co case TOXAV_CALL_CONTROL_CANCEL: { /* Hang up */ + pthread_mutex_lock(call->mutex); if (msi_hangup(call->msi_call) != 0) { rc = TOXAV_ERR_CALL_CONTROL_SYNC; + pthread_mutex_unlock(call->mutex); goto END; } call->msi_call = NULL; + pthread_mutex_unlock(call->mutex); + /* No mather the case, terminate the call */ call_kill_transmission(call); call_remove(call); From 7972db5c41b1cd573e4ba362d5d76ad4cd691c9f Mon Sep 17 00:00:00 2001 From: Eniz Vukovic Date: Fri, 23 Oct 2015 22:56:54 +0200 Subject: [PATCH 92/97] Removed mutex operations --- toxav/toxav.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/toxav/toxav.c b/toxav/toxav.c index 1607a9521..a88a6cf4a 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -440,15 +440,12 @@ bool toxav_call_control(ToxAV *av, uint32_t friend_number, TOXAV_CALL_CONTROL co case TOXAV_CALL_CONTROL_CANCEL: { /* Hang up */ - pthread_mutex_lock(call->mutex); if (msi_hangup(call->msi_call) != 0) { rc = TOXAV_ERR_CALL_CONTROL_SYNC; - pthread_mutex_unlock(call->mutex); goto END; } call->msi_call = NULL; - pthread_mutex_unlock(call->mutex); /* No mather the case, terminate the call */ call_kill_transmission(call); From 87828a1b425fb46d3276f6166f9de49744b8e583 Mon Sep 17 00:00:00 2001 From: Eniz Vukovic Date: Sat, 24 Oct 2015 01:56:49 +0200 Subject: [PATCH 93/97] Add synchronization protection for send_frame API functions --- other/apidsl/toxav.in.h | 4 ++++ toxav/toxav.c | 12 ++++++++++-- toxav/toxav.h | 4 ++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/other/apidsl/toxav.in.h b/other/apidsl/toxav.in.h index 592c3def7..ab89b0ea1 100644 --- a/other/apidsl/toxav.in.h +++ b/other/apidsl/toxav.in.h @@ -492,6 +492,10 @@ error for send_frame { * This client is currently not in a call with the friend. */ FRIEND_NOT_IN_CALL, + /** + * Synchronization error occurred. + */ + SYNC, /** * One of the frame parameters was invalid. E.g. the resolution may be too * small or too large, or the audio sampling rate may be unsupported. diff --git a/toxav/toxav.c b/toxav/toxav.c index a88a6cf4a..fac6b0bc0 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -649,7 +649,11 @@ bool toxav_audio_send_frame(ToxAV *av, uint32_t friend_number, const int16_t *pc goto END; } - pthread_mutex_lock(av->mutex); + if (pthread_mutex_trylock(av->mutex) != 0) { + rc = TOXAV_ERR_SEND_FRAME_SYNC; + goto END; + } + call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { @@ -728,7 +732,11 @@ bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, u goto END; } - pthread_mutex_lock(av->mutex); + if (pthread_mutex_trylock(av->mutex) != 0) { + rc = TOXAV_ERR_SEND_FRAME_SYNC; + goto END; + } + call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { diff --git a/toxav/toxav.h b/toxav/toxav.h index b82a2ace8..5c5195b33 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -565,6 +565,10 @@ typedef enum TOXAV_ERR_SEND_FRAME { * This client is currently not in a call with the friend. */ TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL, + /** + * Synchronization error occurred. + */ + TOXAV_ERR_SEND_FRAME_SYNC, /** * One of the frame parameters was invalid. E.g. the resolution may be too * small or too large, or the audio sampling rate may be unsupported. From caa038287717e6fd47df420660c0854246fbc4c8 Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 24 Oct 2015 16:16:42 +0200 Subject: [PATCH 94/97] Fix misplaced log message. --- toxav/audio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toxav/audio.c b/toxav/audio.c index 3ba95c03e..4f9d35625 100644 --- a/toxav/audio.c +++ b/toxav/audio.c @@ -228,7 +228,6 @@ int ac_reconfigure_encoder(ACSession *ac, int32_t bit_rate, int32_t sampling_rat &ac->le_channel_count)) return -1; - LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bit_rate, sampling_rate, channels); return 0; } @@ -409,6 +408,7 @@ bool reconfigure_audio_encoder(OpusEncoder **e, int32_t new_br, int32_t new_sr, *old_sr = new_sr; *old_ch = new_ch; + LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", new_br, new_sr, new_ch); return true; } bool reconfigure_audio_decoder(ACSession *ac, int32_t sampling_rate, int8_t channels) From e730bd54e6abd159dc776adce42094f20f2bb8e5 Mon Sep 17 00:00:00 2001 From: Eniz Vukovic Date: Mon, 26 Oct 2015 22:29:23 +0100 Subject: [PATCH 95/97] Fix possible misalignment of RTP header --- toxav/rtp.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/toxav/rtp.h b/toxav/rtp.h index fddbce3de..0393ac273 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -37,21 +37,21 @@ enum { struct RTPHeader { /* Standard RTP header */ #ifndef WORDS_BIGENDIAN - unsigned cc: 4; /* Contributing sources count */ - unsigned xe: 1; /* Extra header */ - unsigned pe: 1; /* Padding */ - unsigned ve: 2; /* Version */ + uint16_t cc: 4; /* Contributing sources count */ + uint16_t xe: 1; /* Extra header */ + uint16_t pe: 1; /* Padding */ + uint16_t ve: 2; /* Version */ - unsigned pt: 7; /* Payload type */ - unsigned ma: 1; /* Marker */ + uint16_t pt: 7; /* Payload type */ + uint16_t ma: 1; /* Marker */ #else - unsigned ve: 2; /* Version */ - unsigned pe: 1; /* Padding */ - unsigned xe: 1; /* Extra header */ - unsigned cc: 4; /* Contributing sources count */ + uint16_t ve: 2; /* Version */ + uint16_t pe: 1; /* Padding */ + uint16_t xe: 1; /* Extra header */ + uint16_t cc: 4; /* Contributing sources count */ - unsigned ma: 1; /* Marker */ - unsigned pt: 7; /* Payload type */ + uint16_t ma: 1; /* Marker */ + uint16_t pt: 7; /* Payload type */ #endif uint16_t sequnum; From fa0c87fa55e93a7e54ac61e8d4955c3c21b5939a Mon Sep 17 00:00:00 2001 From: Eniz Vukovic Date: Sat, 31 Oct 2015 14:53:20 +0100 Subject: [PATCH 96/97] Revert "Removed mutex operations" --- toxav/toxav.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/toxav/toxav.c b/toxav/toxav.c index fac6b0bc0..6a17f55dd 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -440,12 +440,15 @@ bool toxav_call_control(ToxAV *av, uint32_t friend_number, TOXAV_CALL_CONTROL co case TOXAV_CALL_CONTROL_CANCEL: { /* Hang up */ + pthread_mutex_lock(call->mutex); if (msi_hangup(call->msi_call) != 0) { rc = TOXAV_ERR_CALL_CONTROL_SYNC; + pthread_mutex_unlock(call->mutex); goto END; } call->msi_call = NULL; + pthread_mutex_unlock(call->mutex); /* No mather the case, terminate the call */ call_kill_transmission(call); From ca4214bbcfe554c5754ebc4650cbab09f507567a Mon Sep 17 00:00:00 2001 From: Eniz Vukovic Date: Tue, 3 Nov 2015 00:08:46 +0100 Subject: [PATCH 97/97] Fix compile error on mingw --- toxav/bwcontroler.c | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/toxav/bwcontroler.c b/toxav/bwcontroler.c index a4328045f..2530e7fd2 100644 --- a/toxav/bwcontroler.c +++ b/toxav/bwcontroler.c @@ -138,14 +138,9 @@ void bwc_add_recv(BWControler *bwc, uint32_t bytes) struct BWCMessage { - uint8_t core_type; /* Aligner for payload type which is always 196 */ - uint32_t lost; uint32_t recv; -} __attribute__((packed)); - -/* Check alignment */ -typedef char __fail_if_misaligned [ sizeof(struct BWCMessage) == 9 ? 1 : -1 ]; +}; void send_update(BWControler *bwc) { @@ -159,15 +154,18 @@ void send_update(BWControler *bwc) if (bwc->cycle.lost) { - LOGGER_DEBUG ("%p Sent update", bwc); + LOGGER_DEBUG ("%p Sent update rcv: %u lost: %u", + bwc, bwc->cycle.recv, bwc->cycle.lost); - struct BWCMessage msg; - msg.core_type = BWC_PACKET_ID; - msg.lost = htonl(bwc->cycle.lost); - msg.recv = htonl(bwc->cycle.recv); + uint8_t p_msg[sizeof(struct BWCMessage) + 1]; + struct BWCMessage* b_msg = (struct BWCMessage*)(p_msg + 1); + + p_msg[0] = BWC_PACKET_ID; + b_msg->lost = htonl(bwc->cycle.lost); + b_msg->recv = htonl(bwc->cycle.recv); - if (-1 == send_custom_lossy_packet(bwc->m, bwc->friend_number, (uint8_t *)&msg, sizeof(msg))) - LOGGER_WARNING("BWC send failed (len: %d)! std error: %s", sizeof(msg), strerror(errno)); + if (-1 == send_custom_lossy_packet(bwc->m, bwc->friend_number, p_msg, sizeof(p_msg))) + LOGGER_WARNING("BWC send failed (len: %d)! std error: %s", sizeof(p_msg), strerror(errno)); } bwc->cycle.lsu = current_time_monotonic(); @@ -199,9 +197,9 @@ int on_update (BWControler *bwc, struct BWCMessage *msg) } int bwc_handle_data(Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object) { - if (length != sizeof(struct BWCMessage)) + if (length - 1 != sizeof(struct BWCMessage)) return -1; /* NOTE the data is mutable */ - return on_update(object, (struct BWCMessage *) data); + return on_update(object, (struct BWCMessage *) (data + 1)); }