Asterisk - The Open Source Telephony Project  GIT-master-4a4f1a5
bridge_softmix.c
Go to the documentation of this file.
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19 
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29 
30 /*** MODULEINFO
31  <support_level>core</support_level>
32  ***/
33 
34 #include "asterisk.h"
35 
36 #include <math.h>
37 
38 #include "asterisk/stream.h"
39 #include "asterisk/test.h"
40 #include "asterisk/vector.h"
41 #include "asterisk/message.h"
43 
44 /*! The minimum sample rate of the bridge. */
45 #define SOFTMIX_MIN_SAMPLE_RATE 8000 /* 8 kHz sample rate */
46 
47 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
48 #define DEFAULT_SOFTMIX_INTERVAL 20
49 
50 /*! \brief Size of the buffer used for sample manipulation */
51 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
52 
53 /*! \brief Number of samples we are dealing with */
54 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
55 
56 /*! \brief Number of mixing iterations to perform between gathering statistics. */
57 #define SOFTMIX_STAT_INTERVAL 100
58 
59 /*!
60  * \brief Default time in ms of silence necessary to declare talking stopped by the bridge.
61  *
62  * \details
63  * This is the time at which a channel's own audio will stop getting
64  * mixed out of its own write audio stream because it is no longer talking.
65  */
66 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
67 
68 /*! Default minimum average magnitude threshold to determine talking by the DSP. */
69 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
70 
71 #define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
72 #define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
73 #define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
74 
76  /*! The frame which will be given to each source stream */
77  struct ast_frame frame;
78  /*! The REMB to send to the source which is collecting REMB reports */
80  /*! The maximum bitrate (A single precision floating point is big enough) */
81  float bitrate;
82 };
83 
84 struct softmix_stats {
85  /*! Each index represents a sample rate used above the internal rate. */
86  unsigned int sample_rates[16];
87  /*! Each index represents the number of channels using the same index in the sample_rates array. */
88  unsigned int num_channels[16];
89  /*! The number of channels above the internal sample rate */
91  /*! The number of channels above the maximum sample rate */
92  unsigned int num_above_maximum_rate;
93  /*! The number of channels at the internal sample rate */
94  unsigned int num_at_internal_rate;
95  /*! The absolute highest sample rate preferred by any channel in the bridge */
96  unsigned int highest_supported_rate;
97  /*! Is the sample rate locked by the bridge, if so what is that rate.*/
98  unsigned int locked_rate;
99  /*! The maximum sample rate the bridge may use */
100  unsigned int maximum_rate;
101 };
102 
104  int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
105  and re-init if it was usable. */
106  struct ast_format *dst_format; /*!< The destination format for this helper */
107  struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
108  struct ast_frame *out_frame; /*!< The output frame from the last translation */
110 };
111 
113  struct ast_format *slin_src; /*!< the source format expected for all the translators */
115 };
116 
118 {
120  if (!(entry = ast_calloc(1, sizeof(*entry)))) {
121  return NULL;
122  }
123  entry->dst_format = ao2_bump(dst);
124  /* initialize this to one so that the first time through the cleanup code after
125  allocation it won't be removed from the entry list */
126  entry->num_times_requested = 1;
127  return entry;
128 }
129 
131 {
132  ao2_cleanup(entry->dst_format);
133 
134  if (entry->trans_pvt) {
135  ast_translator_free_path(entry->trans_pvt);
136  }
137  if (entry->out_frame) {
138  ast_frfree(entry->out_frame);
139  }
140  ast_free(entry);
141  return NULL;
142 }
143 
144 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
145 {
146  memset(trans_helper, 0, sizeof(*trans_helper));
147  trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
148 }
149 
151 {
153 
154  while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
156  }
157 }
158 
159 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
160 {
162 
163  trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
164  AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
165  if (entry->trans_pvt) {
166  ast_translator_free_path(entry->trans_pvt);
167  if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
170  }
171  }
172  }
174 }
175 
176 /*!
177  * \internal
178  * \brief Get the next available audio on the softmix channel's read stream
179  * and determine if it should be mixed out or not on the write stream.
180  *
181  * \retval pointer to buffer containing the exact number of samples requested on success.
182  * \retval NULL if no samples are present
183  */
184 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
185 {
186  if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
187  ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
188  sc->have_audio = 1;
189  return sc->our_buf;
190  }
191  sc->have_audio = 0;
192  return NULL;
193 }
194 
195 /*!
196  * \internal
197  * \brief Process a softmix channel's write audio
198  *
199  * \details This function will remove the channel's talking from its own audio if present and
200  * possibly even do the channel's write translation for it depending on how many other
201  * channels use the same write format.
202  */
203 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
204  struct ast_format *raw_write_fmt,
205  struct softmix_channel *sc, unsigned int default_sample_size)
206 {
208  int i;
209 
210  /* If we provided any audio then take it out while in slinear format. */
211  if (sc->have_audio && !sc->binaural) {
212  for (i = 0; i < sc->write_frame.samples; i++) {
214  }
215  /* check to see if any entries exist for the format. if not we'll want
216  to remove it during cleanup */
217  AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
218  if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
219  ++entry->num_times_requested;
220  break;
221  }
222  }
223  /* do not do any special write translate optimization if we had to make
224  * a special mix for them to remove their own audio. */
225  return;
226  } else if (sc->have_audio && sc->binaural > 0) {
227  /*
228  * Binaural audio requires special saturated substract since we have two
229  * audio signals per channel now.
230  */
231  softmix_process_write_binaural_audio(sc, default_sample_size);
232  return;
233  }
234 
235  /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
236  of translation paths and track the number of references for each type. Each one of the same
237  type should be able to use the same out_frame. Since the optimization is only necessary for
238  multiple channels (>=2) using the same codec make sure resources are allocated only when
239  needed and released when not (see also softmix_translate_helper_cleanup */
240  AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
241  if (sc->binaural != 0) {
242  continue;
243  }
244  if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
245  entry->num_times_requested++;
246  } else {
247  continue;
248  }
249  if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
250  entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
251  }
252  if (entry->trans_pvt && !entry->out_frame) {
253  entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
254  }
255  if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
256  && entry->out_frame->datalen < MAX_DATALEN) {
257  ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
258  memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
259  sc->write_frame.datalen = entry->out_frame->datalen;
260  sc->write_frame.samples = entry->out_frame->samples;
261  }
262  break;
263  }
264 
265  /* add new entry into list if this format destination was not matched. */
266  if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
267  AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
268  }
269 }
270 
272 {
274 
275  AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
276  /* if it hasn't been requested then remove it */
277  if (!entry->num_times_requested) {
280  continue;
281  }
282 
283  if (entry->out_frame) {
284  ast_frfree(entry->out_frame);
285  entry->out_frame = NULL;
286  }
287 
288  /* nothing is optimized for a single path reference, so there is
289  no reason to continue to hold onto the codec */
290  if (entry->num_times_requested == 1 && entry->trans_pvt) {
291  ast_translator_free_path(entry->trans_pvt);
292  entry->trans_pvt = NULL;
293  }
294 
295  /* for each iteration (a mixing run) in the bridge softmix thread the number
296  of references to a given entry is recalculated, so reset the number of
297  times requested */
298  entry->num_times_requested = 0;
299  }
301 }
302 
303 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
304 {
305  struct softmix_channel *sc = bridge_channel->tech_pvt;
306  struct ast_format *slin_format;
307  int setup_fail;
308 
309 #ifdef BINAURAL_RENDERING
310  if (interval != BINAURAL_MIXING_INTERVAL) {
311  interval = BINAURAL_MIXING_INTERVAL;
312  }
313 #endif
314 
315  /* The callers have already ensured that sc is never NULL. */
316  ast_assert(sc != NULL);
317 
318  slin_format = ast_format_cache_get_slin_by_rate(rate);
319 
320  ast_mutex_lock(&sc->lock);
321  if (reset) {
323  ast_dsp_free(sc->dsp);
324  }
325 
326  /* Setup write frame parameters */
328  /*
329  * NOTE: The write_frame format holds a reference because translation
330  * could be needed and the format changed to the translated format
331  * for the channel. The translated format may not be a
332  * static cached format.
333  */
334  ao2_replace(sc->write_frame.subclass.format, slin_format);
335  sc->write_frame.data.ptr = sc->final_buf;
336  sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
337  sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
338 
339  /* We will store the rate here cause we need to set the data again when a channel is unsuspended */
340  sc->rate = rate;
341 
342  /* If the channel will contain binaural data we will set a identifier in the channel
343  * if set_binaural == -1 this is just a sample rate update, will ignore it. */
344  if (set_binaural == 1) {
345  sc->binaural = 1;
346  } else if (set_binaural == 0) {
347  sc->binaural = 0;
348  }
349 
350  /* Setting the binaural position. This doesn't require a change of the overlaying channel infos
351  * and doesn't have to be done if we just updating sample rates. */
352  if (binaural_pos_id != -1) {
353  sc->binaural_pos = binaural_pos_id;
354  }
355  if (is_announcement != -1) {
356  sc->is_announcement = is_announcement;
357  }
358 
359  /*
360  * NOTE: The read_slin_format does not hold a reference because it
361  * will always be a signed linear format.
362  */
363  sc->read_slin_format = slin_format;
364 
365  /* Setup smoother */
366  setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
367 
368  /* set new read and write formats on channel. */
369  ast_channel_lock(bridge_channel->chan);
370  setup_fail |= ast_set_read_format_path(bridge_channel->chan,
371  ast_channel_rawreadformat(bridge_channel->chan), slin_format);
372  ast_channel_unlock(bridge_channel->chan);
373 
374  /* If channel contains binaural data we will set it here for the trans_pvt. */
375  if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
376  setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
377  } else if (set_binaural == 0) {
378  setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
379  }
380 
381  /* set up new DSP. This is on the read side only right before the read frame enters the smoother. */
382  sc->dsp = ast_dsp_new_with_rate(rate);
383  if (setup_fail || !sc->dsp) {
384  /* Bad news. Could not setup the channel for softmix. */
385  ast_mutex_unlock(&sc->lock);
387  return;
388  }
389 
390  /* we want to aggressively detect silence to avoid feedback */
391  if (bridge_channel->tech_args.talking_threshold) {
393  } else {
395  }
396 
397  ast_mutex_unlock(&sc->lock);
398 }
399 
400 /*!
401  * \internal
402  * \brief Poke the mixing thread in case it is waiting for an active channel.
403  * \since 12.0.0
404  *
405  * \param softmix_data Bridge mixing data.
406  */
407 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
408 {
409  ast_mutex_lock(&softmix_data->lock);
410  ast_cond_signal(&softmix_data->cond);
411  ast_mutex_unlock(&softmix_data->lock);
412 }
413 
414 /*! \brief Function called when a channel is unsuspended from the bridge */
415 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
416 {
417 #ifdef BINAURAL_RENDERING
418  struct softmix_channel *sc = bridge_channel->tech_pvt;
419  if (sc->binaural) {
420  /* Restore some usefull data if it was a binaural channel */
421  struct ast_format *slin_format;
422 
423  slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
424  ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
425  }
426 #endif
427  if (bridge->tech_pvt) {
428  softmix_poke_thread(bridge->tech_pvt);
429  }
430 }
431 
432 /*!
433  * \brief Determine if a stream is a video source stream.
434  *
435  * \param stream The stream to test
436  * \retval 1 The stream is a video source
437  * \retval 0 The stream is not a video source
438  */
439 static int is_video_source(const struct ast_stream *stream)
440 {
445  return 1;
446  }
447 
448  return 0;
449 }
450 
451 /*!
452  * \brief Determine if a stream is a video destination stream.
453  *
454  * A source channel name can be provided to narrow this to a destination stream
455  * for a particular source channel. Further, a source stream name can be provided
456  * to narrow this to a particular source stream's destination. However, empty strings
457  * can be provided to match any destination video stream, regardless of source channel
458  * or source stream.
459  *
460  * \param stream The stream to test
461  * \param source_channel_name The name of a source video channel to match
462  * \param source_channel_stream_position The position of the video on the source channel
463  * \retval 1 The stream is a video destination stream
464  * \retval 0 The stream is not a video destination stream
465  */
466 static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
467  int source_channel_stream_position)
468 {
469  char *dest_video_name;
470  size_t dest_video_name_len;
471 
474  return 0;
475  }
476 
477  dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
478  if (!ast_strlen_zero(source_channel_name)) {
479  dest_video_name_len += strlen(source_channel_name) + 1;
480  if (source_channel_stream_position != -1) {
481  dest_video_name_len += 11;
482  }
483 
484  dest_video_name = ast_alloca(dest_video_name_len);
485  if (source_channel_stream_position != -1) {
486  /* We are looking for an exact stream position */
487  snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%d",
489  source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
490  source_channel_stream_position);
491  return !strcmp(ast_stream_get_name(stream), dest_video_name);
492  }
493  snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
495  source_channel_name);
496  } else {
497  dest_video_name = SOFTBRIDGE_VIDEO_DEST_PREFIX;
498  }
499 
500  return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
501 }
502 
504  const char *channel_name, const char *sdp_label,
505  struct ast_stream *stream, int index)
506 {
507  char *stream_clone_name = NULL;
508  struct ast_stream *stream_clone;
509 
510  /* We use the stream topology index for the stream to uniquely identify and recognize it.
511  * This is guaranteed to remain the same across renegotiation of the source channel and
512  * ensures that the stream name is unique.
513  */
514  if (ast_asprintf(&stream_clone_name, "%s%c%s%c%d", SOFTBRIDGE_VIDEO_DEST_PREFIX,
516  index) < 0) {
517  return -1;
518  }
519 
520  stream_clone = ast_stream_clone(stream, stream_clone_name);
521  ast_free(stream_clone_name);
522  if (!stream_clone) {
523  return -1;
524  }
525 
526  /* Sends an "a:label" attribute in the SDP for participant event correlation */
527  if (!ast_strlen_zero(sdp_label)) {
528  ast_stream_set_metadata(stream_clone, "SDP:LABEL", sdp_label);
529  }
530 
531  /* We will be sending them a stream and not expecting anything in return */
533 
534  if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
535  ast_stream_free(stream_clone);
536  return -1;
537  }
538 
539  return 0;
540 }
541 
542 
544  const char *channel_name, const char *sdp_label,
545  const struct ast_stream_topology *source)
546 {
547  int i;
548 
549  for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
550  struct ast_stream *stream;
551 
552  stream = ast_stream_topology_get_stream(source, i);
553 
554  if (!is_video_source(stream)) {
555  continue;
556  }
557 
558  if (append_source_stream(dest, channel_name, sdp_label, stream, i)) {
559  return -1;
560  }
561  }
562 
563  return 0;
564 }
565 
566 static int append_all_streams(struct ast_stream_topology *dest,
567  const struct ast_stream_topology *source)
568 {
569  int i;
570  int dest_index = 0;
571 
572  for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
573  struct ast_stream *clone;
574  int added = 0;
575 
577  if (!clone) {
578  return -1;
579  }
580 
581  /* If we can reuse an existing removed stream then do so */
582  while (dest_index < ast_stream_topology_get_count(dest)) {
583  struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
584 
585  dest_index++;
586 
588  /* This cannot fail because dest_index - 1 is less than the
589  * current count in dest. */
590  ast_stream_topology_set_stream(dest, dest_index - 1, clone);
591  added = 1;
592  break;
593  }
594  }
595 
596  /* If no removed stream exists that we took the place of append the stream */
597  if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
598  ast_stream_free(clone);
599  return -1;
600  }
601  }
602 
603  return 0;
604 }
605 
606 /*!
607  * \brief Issue channel stream topology change requests.
608  *
609  * When in SFU mode, each participant needs to be able to
610  * send video directly to other participants in the bridge.
611  * This means that all participants need to have their topologies
612  * updated. The joiner needs to have destination streams for
613  * all current participants, and the current participants need
614  * to have destinations streams added for the joiner's sources.
615  *
616  * \param bridge
617  * \param joiner The channel that is joining the softmix bridge
618  */
619 static void sfu_topologies_on_join(struct ast_bridge *bridge,
620  struct ast_bridge_channel *joiner)
621 {
622  RAII_VAR(struct ast_stream_topology *, joiner_video, NULL, ast_stream_topology_free);
623  struct ast_bridge_channels_list *participants = &bridge->channels;
624  struct ast_bridge_channel *participant;
625  int res;
626  struct softmix_channel *sc;
627  SCOPE_ENTER(3, "%s: \n", ast_channel_name(joiner->chan));
628 
629  joiner_video = ast_stream_topology_alloc();
630  if (!joiner_video) {
631  SCOPE_EXIT_LOG_RTN(LOG_ERROR, "%s: Couldn't alloc topology\n", ast_channel_name(joiner->chan));
632  }
633 
634  sc = joiner->tech_pvt;
635 
636  ast_channel_lock(joiner->chan);
637  res = append_source_streams(joiner_video, ast_channel_name(joiner->chan),
638  bridge->softmix.send_sdp_label ? ast_channel_uniqueid(joiner->chan) : NULL,
641  ast_channel_unlock(joiner->chan);
642 
643  if (res || !sc->topology) {
644  SCOPE_EXIT_LOG_RTN(LOG_ERROR, "%s: Couldn't append source streams\n", ast_channel_name(joiner->chan));
645  }
646 
647  AST_LIST_TRAVERSE(participants, participant, entry) {
648  if (participant == joiner) {
649  continue;
650  }
651  ast_trace(-1, "%s: Appending existing participant %s\n", ast_channel_name(joiner->chan),
652  ast_channel_name(participant->chan));
653  ast_channel_lock(participant->chan);
654  res = append_source_streams(sc->topology, ast_channel_name(participant->chan),
655  bridge->softmix.send_sdp_label ? ast_channel_uniqueid(participant->chan) : NULL,
656  ast_channel_get_stream_topology(participant->chan));
657  ast_channel_unlock(participant->chan);
658  if (res) {
659  SCOPE_EXIT_LOG_RTN(LOG_ERROR, "%s/%s: Couldn't append source streams\n",
660  ast_channel_name(participant->chan), ast_channel_name(joiner->chan));
661  }
662  }
663 
664  ast_trace(-1, "%s: Requesting topology change.\n", ast_channel_name(joiner->chan));
666  if (res) {
667  /* There are conditions under which this is expected so no need to log an ERROR. */
668  SCOPE_EXIT_RTN("%s: Couldn't request topology change\n", ast_channel_name(joiner->chan));
669  }
670 
671  AST_LIST_TRAVERSE(participants, participant, entry) {
672  if (participant == joiner) {
673  continue;
674  }
675 
676  sc = participant->tech_pvt;
677  ast_trace(-1, "%s: Appending joiner %s\n", ast_channel_name(participant->chan),
678  ast_channel_name(joiner->chan));
679 
680  if (append_all_streams(sc->topology, joiner_video)) {
681  SCOPE_EXIT_LOG_RTN(LOG_ERROR, "%s/%s: Couldn't append streams\n",
682  ast_channel_name(participant->chan), ast_channel_name(joiner->chan));
683  }
684  ast_trace(-1, "%s: Requesting topology change\n", ast_channel_name(participant->chan));
686  if (res) {
687  ast_trace(-1, "%s/%s: Couldn't request topology change\n",
688  ast_channel_name(participant->chan), ast_channel_name(joiner->chan));
689  }
690  }
691 
692  SCOPE_EXIT();
693 }
694 
695 /*! \brief Function called when a channel is joined into the bridge */
696 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
697 {
698  struct softmix_channel *sc;
699  struct softmix_bridge_data *softmix_data;
700  int set_binaural = 0;
701  /*
702  * If false, the channel will be convolved, but since it is a non stereo channel, output
703  * will be mono.
704  */
705  int skip_binaural_output = 1;
706  int pos_id;
707  int is_announcement = 0;
708  int samplerate_change;
709  SCOPE_ENTER(3, "%s:\n", ast_channel_name(bridge_channel->chan));
710 
711  softmix_data = bridge->tech_pvt;
712  if (!softmix_data) {
713  SCOPE_EXIT_RTN_VALUE(-1, "No tech_pvt\n");
714  }
715 
716  /* Create a new softmix_channel structure and allocate various things on it */
717  if (!(sc = ast_calloc(1, sizeof(*sc)))) {
718  SCOPE_EXIT_RTN_VALUE(-1, "Couldn't alloc tech_pvt\n");
719  }
720 
721  samplerate_change = softmix_data->internal_rate;
722  pos_id = -1;
724  if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
725  set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
726  if (set_binaural) {
727  softmix_data->internal_rate = samplerate_change;
728  }
729  skip_binaural_output = 0;
730  } else {
731  is_announcement = 1;
732  }
733  if (set_binaural) {
734  softmix_data->convolve.binaural_active = 1;
735  }
736  if (!skip_binaural_output) {
737  pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
738  if (pos_id == -1) {
739  ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
740  "Could not allocate enough memory.\n", bridge->uniqueid,
741  ast_channel_name(bridge_channel->chan));
742  ast_free(sc);
743  SCOPE_EXIT_RTN_VALUE(-1, "Couldn't do binaural join\n");
744  }
745  }
746  }
747 
748  /* Can't forget the lock */
749  ast_mutex_init(&sc->lock);
750 
751  /* Can't forget to record our pvt structure within the bridged channel structure */
752  bridge_channel->tech_pvt = sc;
753 
755  softmix_data->internal_mixing_interval
756  ? softmix_data->internal_mixing_interval
758  bridge_channel, 0, set_binaural, pos_id, is_announcement);
759 
761  sfu_topologies_on_join(bridge, bridge_channel);
762  }
763 
764  /* Complete any active hold before entering, or transitioning to softmix. */
765  if (ast_channel_hold_state(bridge_channel->chan) == AST_CONTROL_HOLD) {
766  ast_debug(1, "Channel %s simulating UNHOLD for bridge softmix join.\n",
767  ast_channel_name(bridge_channel->chan));
768  ast_indicate(bridge_channel->chan, AST_CONTROL_UNHOLD);
769  }
770 
771  softmix_poke_thread(softmix_data);
773 }
774 
776  const char *channel_name)
777 {
778  int i;
779  int stream_removed = 0;
780 
781  for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
782  struct ast_stream *stream;
783 
784  stream = ast_stream_topology_get_stream(topology, i);
785 
786  if (is_video_dest(stream, channel_name, -1)) {
788  stream_removed = 1;
789  }
790  }
791  return stream_removed;
792 }
793 
794 static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
795 {
796  struct ast_bridge_channel *participant;
797  struct softmix_channel *sc;
798 
799  AST_LIST_TRAVERSE(participants, participant, entry) {
800  sc = participant->tech_pvt;
802  continue;
803  }
805  }
806 
807  sc = leaver->tech_pvt;
808  if (remove_destination_streams(sc->topology, "")) {
810  }
811 
812  return 0;
813 }
814 
815 /*! \brief Function called when a channel leaves the bridge */
816 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
817 {
818  struct softmix_channel *sc;
819  struct softmix_bridge_data *softmix_data;
820 
821  softmix_data = bridge->tech_pvt;
822  sc = bridge_channel->tech_pvt;
823  if (!sc) {
824  return;
825  }
826 
828  sfu_topologies_on_leave(bridge_channel, &bridge->channels);
829  }
830 
832  if (sc->binaural) {
833  set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
834  softmix_data->default_sample_size);
835  }
836  }
837 
838  bridge_channel->tech_pvt = NULL;
839 
841 
843 
844  AST_VECTOR_FREE(&sc->video_sources);
845 
846  /* Drop mutex lock */
847  ast_mutex_destroy(&sc->lock);
848 
849  /* Drop the factory */
851 
852  /* Drop any formats on the frames */
854 
855  /* Drop the DSP */
856  ast_dsp_free(sc->dsp);
857 
858  /* Eep! drop ourselves */
859  ast_free(sc);
860 }
861 
862 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
863 {
864  struct ast_bridge_channel *cur;
865 
867  if (cur->suspended) {
868  continue;
869  }
870  if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
871  ast_bridge_channel_queue_frame(cur, frame);
872  break;
873  }
874  }
875 }
876 
877 /*!
878  * \internal
879  * \brief Determine what to do with a video frame.
880  * \since 12.0.0
881  *
882  * \param bridge Which bridge is getting the frame
883  * \param bridge_channel Which channel is writing the frame.
884  * \param frame What is being written.
885  */
886 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
887 {
888  struct softmix_channel *sc;
889  int video_src_priority;
890 
891  /* Determine if the video frame should be distributed or not */
892  switch (bridge->softmix.video_mode.mode) {
894  break;
896  video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
897  if (video_src_priority == 1) {
898  /* Pass to me and everyone else. */
899  ast_bridge_queue_everyone_else(bridge, NULL, frame);
900  }
901  break;
903  sc = bridge_channel->tech_pvt;
904  ast_mutex_lock(&sc->lock);
905  ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
907  frame->subclass.frame_ending);
908  ast_mutex_unlock(&sc->lock);
909  video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
910  if (video_src_priority == 1) {
911  int num_src = ast_bridge_number_video_src(bridge);
912  int echo = num_src > 1 ? 0 : 1;
913 
914  ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
915  } else if (video_src_priority == 2) {
916  softmix_pass_video_top_priority(bridge, frame);
917  }
918  break;
920  /* Nothing special to do here, the bridge channel stream map will ensure the
921  * video goes everywhere it needs to
922  */
923  ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
924  break;
925  }
926 }
927 
928 /*!
929  * \internal
930  * \brief Determine what to do with a voice frame.
931  * \since 12.0.0
932  *
933  * \param bridge Which bridge is getting the frame
934  * \param bridge_channel Which channel is writing the frame.
935  * \param frame What is being written.
936  */
937 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
938 {
939  struct softmix_channel *sc = bridge_channel->tech_pvt;
940  struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
941  int silent = 0;
942  int totalsilence = 0;
943  int cur_energy = 0;
944  int silence_threshold = bridge_channel->tech_args.silence_threshold ?
945  bridge_channel->tech_args.silence_threshold :
947  /*
948  * If update_talking is set to 0 or 1, tell the bridge that the channel
949  * has started or stopped talking.
950  */
951  char update_talking = -1;
952 
953  /* Write the frame into the conference */
954  ast_mutex_lock(&sc->lock);
955 
957  /*
958  * The incoming frame is not the expected format. Update
959  * the channel's translation path to get us slinear from
960  * the new format for the next frame.
961  *
962  * There is the possibility that this frame is an old slinear
963  * rate frame that was in flight when the softmix bridge
964  * changed rates. If so it will self correct on subsequent
965  * frames.
966  */
967  ast_channel_lock(bridge_channel->chan);
968  ast_debug(1, "Channel %s wrote unexpected format into bridge. Got %s, expected %s.\n",
969  ast_channel_name(bridge_channel->chan),
972  ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
973  sc->read_slin_format);
974  ast_channel_unlock(bridge_channel->chan);
975  }
976 
977  /* The channel will be leaving soon if there is no dsp. */
978  if (sc->dsp) {
979  silent = ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
980  }
981 
983  int cur_slot = sc->video_talker.energy_history_cur_slot;
984 
986  sc->video_talker.energy_accum += cur_energy;
987  sc->video_talker.energy_history[cur_slot] = cur_energy;
991  sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
992  }
993  }
994 
995  if (totalsilence < silence_threshold) {
996  if (!sc->talking && !silent) {
997  /* Tell the write process we have audio to be mixed out */
998  sc->talking = 1;
999  update_talking = 1;
1000  }
1001  } else {
1002  if (sc->talking) {
1003  sc->talking = 0;
1004  update_talking = 0;
1005  }
1006  }
1007 
1008  /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
1009  * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
1010  * the audio by flushing the buffer before adding new audio in. */
1011  if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
1013  }
1014 
1015  if (sc->talking || !bridge_channel->tech_args.drop_silence) {
1016  /* Add frame to the smoother for mixing with other channels. */
1017  ast_slinfactory_feed(&sc->factory, frame);
1018  }
1019 
1020  /* Alllll done */
1021  ast_mutex_unlock(&sc->lock);
1022 
1023  if (update_talking != -1) {
1024  ast_bridge_channel_notify_talking(bridge_channel, update_talking);
1025  }
1026 }
1027 
1028 /*!
1029  * \internal
1030  * \brief Clear talking flag, stop contributing to mixing and notify handlers.
1031  * \since 13.21.0, 15.4.0
1032  *
1033  * \param bridge_channel Which channel's talking to clear
1034  */
1035 static void clear_talking(struct ast_bridge_channel *bridge_channel)
1036 {
1037  struct softmix_channel *sc = bridge_channel->tech_pvt;
1038 
1039  if (sc->talking) {
1040  ast_mutex_lock(&sc->lock);
1042  sc->talking = 0;
1043  ast_mutex_unlock(&sc->lock);
1044 
1045  /* Notify that we are no longer talking. */
1046  ast_bridge_channel_notify_talking(bridge_channel, 0);
1047  }
1048 }
1049 
1050 /*!
1051  * \internal
1052  * \brief Check for voice status updates.
1053  * \since 13.20.0
1054  *
1055  * \param bridge Which bridge we are in
1056  * \param bridge_channel Which channel we are checking
1057  */
1058 static void softmix_bridge_check_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
1059 {
1060  if (bridge_channel->features->mute) {
1061  /*
1062  * We were muted while we were talking.
1063  *
1064  * Immediately stop contributing to mixing
1065  * and report no longer talking.
1066  */
1067  clear_talking(bridge_channel);
1068  }
1069 }
1070 
1072  const struct ast_stream_topology *source,
1073  const struct ast_stream_topology *original)
1074 {
1075  int i;
1076 
1077  for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
1078  struct ast_stream *stream;
1079  int original_index;
1080 
1081  stream = ast_stream_topology_get_stream(source, i);
1082 
1083  /* Mark the existing stream as removed so we get a new one, this will get
1084  * reused on a subsequent renegotiation.
1085  */
1086  for (original_index = 0; original_index < ast_stream_topology_get_count(original); ++original_index) {
1087  struct ast_stream *original_stream = ast_stream_topology_get_stream(original, original_index);
1088 
1089  if (!strcmp(ast_stream_get_name(stream), ast_stream_get_name(original_stream))) {
1090  struct ast_stream *removed;
1091 
1092  removed = ast_stream_clone(stream, NULL);
1093  if (!removed) {
1094  return -1;
1095  }
1096 
1098 
1099  /* The destination topology can only ever contain the same, or more,
1100  * streams than the original so this is safe.
1101  */
1102  if (ast_stream_topology_set_stream(dest, original_index, removed)) {
1103  ast_stream_free(removed);
1104  return -1;
1105  }
1106 
1107  break;
1108  }
1109  }
1110  }
1111 
1112  return 0;
1113 }
1114 
1115 static void sfu_topologies_on_source_change(struct ast_bridge *bridge,
1116  struct ast_bridge_channel *source)
1117 {
1118  struct ast_stream_topology *source_video = NULL;
1119  struct ast_bridge_channels_list *participants = &bridge->channels;
1120  struct ast_bridge_channel *participant;
1121  int res;
1122 
1123  source_video = ast_stream_topology_alloc();
1124  if (!source_video) {
1125  return;
1126  }
1127 
1128  ast_channel_lock(source->chan);
1129  res = append_source_streams(source_video, ast_channel_name(source->chan),
1132  ast_channel_unlock(source->chan);
1133  if (res) {
1134  goto cleanup;
1135  }
1136 
1137  AST_LIST_TRAVERSE(participants, participant, entry) {
1138  struct ast_stream_topology *original_topology;
1139  struct softmix_channel *sc;
1140 
1141  if (participant == source) {
1142  continue;
1143  }
1144 
1145  sc = participant->tech_pvt;
1146 
1147  original_topology = ast_stream_topology_clone(sc->topology);
1148  if (!original_topology) {
1149  goto cleanup;
1150  }
1151 
1152  /* We add all the source streams back in, if any removed streams are already present they will
1153  * get used first followed by appending new ones.
1154  */
1155  if (append_all_streams(sc->topology, source_video)) {
1156  ast_stream_topology_free(original_topology);
1157  goto cleanup;
1158  }
1159 
1160  /* And the original existing streams get marked as removed. This causes the remote side to see
1161  * a new stream for the source streams.
1162  */
1163  if (remove_all_original_streams(sc->topology, source_video, original_topology)) {
1164  ast_stream_topology_free(original_topology);
1165  goto cleanup;
1166  }
1167 
1169  ast_stream_topology_free(original_topology);
1170  }
1171 
1172 cleanup:
1173  ast_stream_topology_free(source_video);
1174 }
1175 
1176 /*!
1177  * \internal
1178  * \brief Determine what to do with a text frame.
1179  * \since 13.22.0
1180  * \since 15.5.0
1181  *
1182  * \param bridge Which bridge is getting the frame
1183  * \param bridge_channel Which channel is writing the frame.
1184  * \param frame What is being written.
1185  */
1186 static void softmix_bridge_write_text(struct ast_bridge *bridge,
1187  struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1188 {
1189  if (DEBUG_ATLEAST(1)) {
1190  struct ast_msg_data *msg = frame->data.ptr;
1191  char frame_type[64];
1192 
1194 
1195  if (frame->frametype == AST_FRAME_TEXT_DATA) {
1196  ast_log(LOG_DEBUG, "Received %s frame from '%s:%s': %s\n", frame_type,
1198  ast_channel_name(bridge_channel->chan),
1200  } else {
1201  ast_log(LOG_DEBUG, "Received %s frame from '%s': %.*s\n", frame_type,
1202  ast_channel_name(bridge_channel->chan), frame->datalen,
1203  (char *)frame->data.ptr);
1204  }
1205  }
1206 
1207  ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1208 }
1209 
1210 /*!
1211  * \internal
1212  * \brief Determine what to do with a control frame.
1213  * \since 12.0.0
1214  *
1215  * \param bridge Which bridge is getting the frame
1216  * \param bridge_channel Which channel is writing the frame.
1217  * \param frame What is being written.
1218  *
1219  * \retval 0 Frame accepted into the bridge.
1220  * \retval -1 Frame needs to be deferred.
1221  */
1222 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1223 {
1224  struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1225 
1226  /*
1227  * XXX Softmix needs to use channel roles to determine what to
1228  * do with control frames.
1229  */
1230 
1231  switch (frame->subclass.integer) {
1232  case AST_CONTROL_HOLD:
1233  /*
1234  * Doing anything for holds in a conference bridge could be considered a bit
1235  * odd. That being said, in most cases one would probably want the talking
1236  * flag cleared when 'hold' is pressed by the remote endpoint, so go ahead
1237  * and do that here. However, that is all we'll do. Meaning if for some reason
1238  * the endpoint continues to send audio frames despite pressing 'hold' talking
1239  * will once again be detected for that channel.
1240  */
1241  clear_talking(bridge_channel);
1242  break;
1243  case AST_CONTROL_VIDUPDATE:
1247  softmix_data->last_video_update = ast_tvnow();
1248  }
1249  break;
1252  sfu_topologies_on_source_change(bridge, bridge_channel);
1253  }
1254  break;
1255  default:
1256  break;
1257  }
1258 
1259  return 0;
1260 }
1261 
1262 /*!
1263  * \internal
1264  * \brief Determine what to do with an RTCP frame.
1265  * \since 15.4.0
1266  *
1267  * \param bridge Which bridge is getting the frame
1268  * \param bridge_channel Which channel is writing the frame.
1269  * \param frame What is being written.
1270  */
1271 static void softmix_bridge_write_rtcp(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1272 {
1273  struct ast_rtp_rtcp_feedback *feedback = frame->data.ptr;
1274  struct softmix_channel *sc = bridge_channel->tech_pvt;
1275 
1276  /* We only care about REMB reports right now. In the future we may be able to use sender or
1277  * receiver reports to further tweak things, but not yet.
1278  */
1279  if (frame->subclass.integer != AST_RTP_RTCP_PSFB || feedback->fmt != AST_RTP_RTCP_FMT_REMB ||
1282  return;
1283  }
1284 
1285  /* REMB is the total estimated maximum bitrate across all streams within the session, so we store
1286  * only the latest report and use it everywhere.
1287  */
1288  ast_mutex_lock(&sc->lock);
1289  sc->remb = feedback->remb;
1290  ast_mutex_unlock(&sc->lock);
1291 
1292  return;
1293 }
1294 
1295 /*!
1296  * \internal
1297  * \brief Determine what to do with a frame written into the bridge.
1298  * \since 12.0.0
1299  *
1300  * \param bridge Which bridge is getting the frame
1301  * \param bridge_channel Which channel is writing the frame.
1302  * \param frame What is being written.
1303  *
1304  * \retval 0 Frame accepted into the bridge.
1305  * \retval -1 Frame needs to be deferred.
1306  *
1307  * \note On entry, bridge is already locked.
1308  */
1309 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1310 {
1311  int res = 0;
1312 
1313  if (!bridge->tech_pvt || !bridge_channel || !bridge_channel->tech_pvt) {
1314  /* "Accept" the frame and discard it. */
1315  return 0;
1316  }
1317 
1318  /*
1319  * XXX Softmix needs to use channel roles to determine who gets
1320  * what frame. Possible roles: announcer, recorder, agent,
1321  * supervisor.
1322  */
1323  switch (frame->frametype) {
1324  case AST_FRAME_NULL:
1325  /* "Accept" the frame and discard it. */
1326  softmix_bridge_check_voice(bridge, bridge_channel);
1327  break;
1328  case AST_FRAME_DTMF_BEGIN:
1329  case AST_FRAME_DTMF_END:
1330  res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1331  break;
1332  case AST_FRAME_VOICE:
1333  softmix_bridge_write_voice(bridge, bridge_channel, frame);
1334  break;
1335  case AST_FRAME_VIDEO:
1336  softmix_bridge_write_video(bridge, bridge_channel, frame);
1337  break;
1338  case AST_FRAME_TEXT:
1339  case AST_FRAME_TEXT_DATA:
1340  softmix_bridge_write_text(bridge, bridge_channel, frame);
1341  break;
1342  case AST_FRAME_CONTROL:
1343  res = softmix_bridge_write_control(bridge, bridge_channel, frame);
1344  break;
1345  case AST_FRAME_RTCP:
1346  softmix_bridge_write_rtcp(bridge, bridge_channel, frame);
1347  break;
1349  res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1350  break;
1352  ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
1353  ast_assert(0);
1354  default:
1355  ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
1356  /* "Accept" the frame and discard it. */
1357  break;
1358  }
1359 
1360  return res;
1361 }
1362 
1363 static void remb_collect_report_all(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data,
1364  float bitrate)
1365 {
1366  if (!softmix_data->bitrate) {
1367  softmix_data->bitrate = bitrate;
1368  return;
1369  }
1370 
1373  softmix_data->bitrate = (softmix_data->bitrate + bitrate) / 2;
1374  break;
1376  if (bitrate < softmix_data->bitrate) {
1377  softmix_data->bitrate = bitrate;
1378  }
1379  break;
1381  if (bitrate > softmix_data->bitrate) {
1382  softmix_data->bitrate = bitrate;
1383  }
1384  break;
1389  /* These will never actually get hit due to being handled by remb_collect_report below */
1390  break;
1391  }
1392 }
1393 
1394 static void remb_collect_report(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel,
1395  struct softmix_bridge_data *softmix_data, struct softmix_channel *sc)
1396 {
1397  int i;
1398  float bitrate;
1399 
1400  /* If there are no video sources that we are a receiver of then we have noone to
1401  * report REMB to.
1402  */
1403  if (!AST_VECTOR_SIZE(&sc->video_sources)) {
1404  return;
1405  }
1406 
1407  /* We evenly divide the available maximum bitrate across the video sources
1408  * to this receiver so each source gets an equal slice.
1409  */
1410 
1413  return;
1414  }
1415 
1416  bitrate = (sc->remb.br_mantissa << sc->remb.br_exp) / AST_VECTOR_SIZE(&sc->video_sources);
1417 
1418  /* If this receiver has no bitrate yet ignore it */
1419  if (!bitrate) {
1420  return;
1421  }
1422 
1423  /* If we are using the "all" variants then we should use the bridge bitrate to store information */
1427  remb_collect_report_all(bridge, softmix_data, bitrate);
1428  return;
1429  }
1430 
1431  for (i = 0; i < AST_VECTOR_SIZE(&sc->video_sources); ++i) {
1432  struct softmix_remb_collector *collector;
1433 
1434  /* The collector will always exist if a video source is in our list */
1435  collector = AST_VECTOR_GET(&softmix_data->remb_collectors, AST_VECTOR_GET(&sc->video_sources, i));
1436 
1437  if (!collector->bitrate) {
1438  collector->bitrate = bitrate;
1439  continue;
1440  }
1441 
1444  collector->bitrate = (collector->bitrate + bitrate) / 2;
1445  break;
1447  if (bitrate < collector->bitrate) {
1448  collector->bitrate = bitrate;
1449  }
1450  break;
1452  if (bitrate > collector->bitrate) {
1453  collector->bitrate = bitrate;
1454  }
1455  break;
1459  /* These will never actually get hit due to being handled by remb_collect_report_all above */
1460  break;
1462  /* Don't do anything, we've already forced it */
1463  break;
1464  }
1465  }
1466 
1467  /* After the report is integrated we reset this to 0 in case they stop producing
1468  * REMB reports.
1469  */
1470  sc->remb.br_mantissa = 0;
1471  sc->remb.br_exp = 0;
1472 }
1473 
1474 static void remb_send_report(struct ast_bridge_channel *bridge_channel, struct softmix_bridge_data *softmix_data,
1475  struct softmix_channel *sc)
1476 {
1477  float bitrate = softmix_data->bitrate;
1478  int i;
1479  int exp;
1480 
1481  if (!sc->remb_collector) {
1482  return;
1483  }
1484 
1485  /* If there is no bridge level bitrate fall back to collector level */
1486  if (!bitrate) {
1488  sc->remb_collector->bitrate = 0;
1489  }
1490 
1491  /* We always do this calculation as even when the bitrate is zero the browser
1492  * still prefers it to be accurate instead of lying.
1493  *
1494  * The mantissa only has 18 bits available, so make sure it fits. Adjust the
1495  * value and exponent for those values that don't.
1496  *
1497  * For example given the following:
1498  *
1499  * bitrate = 123456789.0
1500  * frexp(bitrate, &exp);
1501  *
1502  * 'exp' should now equal 27 (number of bits needed to represent the value). Since
1503  * the mantissa must fit into an 18-bit unsigned integer, and the given bitrate is
1504  * too large to fit, we must subtract 18 from the exponent in order to get the
1505  * number of times the bitrate will fit into that size integer.
1506  *
1507  * exp -= 18;
1508  *
1509  * 'exp' is now equal to 9. Now we can get the mantissa that fits into an 18-bit
1510  * unsigned integer by dividing the bitrate by 2^exp:
1511  *
1512  * mantissa = 123456789.0 / 2^9
1513  *
1514  * This makes the final mantissa equal to 241126 (implicitly cast), which is less
1515  * than 262143 (the max value that can be put into an unsigned 18-bit integer).
1516  * So now we have the following:
1517  *
1518  * exp = 9;
1519  * mantissa = 241126;
1520  *
1521  * If we multiply that back we should come up with something close to the original
1522  * bit rate:
1523  *
1524  * 241126 * 2^9 = 123456512
1525  *
1526  * Precision is lost due to the nature of floating point values. Easier to why from
1527  * the binary:
1528  *
1529  * 241126 * 2^9 = 241126 << 9 = 111010110111100110 << 9 = 111010110111100110000000000
1530  *
1531  * Precision on the "lower" end is lost due to zeros being shifted in. This loss is
1532  * both expected and acceptable.
1533  */
1534  frexp(bitrate, &exp);
1535  exp = exp > 18 ? exp - 18 : 0;
1536 
1537  sc->remb_collector->feedback.remb.br_mantissa = bitrate / (1 << exp);
1538  sc->remb_collector->feedback.remb.br_exp = exp;
1539 
1540  for (i = 0; i < AST_VECTOR_SIZE(&bridge_channel->stream_map.to_bridge); ++i) {
1541  int bridge_num = AST_VECTOR_GET(&bridge_channel->stream_map.to_bridge, i);
1542 
1543  /* If this stream is not being provided to the bridge there can be no receivers of it
1544  * so therefore no REMB reports.
1545  */
1546  if (bridge_num == -1) {
1547  continue;
1548  }
1549 
1550  /* We need to update the frame with this stream, or else it won't be
1551  * properly routed. We don't use the actual channel stream identifier as
1552  * the bridging core will do the translation from bridge stream identifier to
1553  * channel stream identifier.
1554  */
1555  sc->remb_collector->frame.stream_num = bridge_num;
1556  ast_bridge_channel_queue_frame(bridge_channel, &sc->remb_collector->frame);
1557  }
1558 }
1559 
1560 static void gather_softmix_stats(struct softmix_stats *stats,
1561  const struct softmix_bridge_data *softmix_data,
1562  struct ast_bridge_channel *bridge_channel)
1563 {
1564  int channel_native_rate;
1565 
1566  /* Gather stats about channel sample rates. */
1567  ast_channel_lock(bridge_channel->chan);
1568  channel_native_rate = MAX(SOFTMIX_MIN_SAMPLE_RATE,
1570  ast_channel_unlock(bridge_channel->chan);
1571 
1572  if (stats->highest_supported_rate < channel_native_rate) {
1573  stats->highest_supported_rate = channel_native_rate;
1574  }
1575  if (stats->maximum_rate && stats->maximum_rate < channel_native_rate) {
1576  stats->num_above_maximum_rate++;
1577  } else if (softmix_data->internal_rate < channel_native_rate) {
1578  int i;
1579 
1580  for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
1581  if (stats->sample_rates[i] == channel_native_rate) {
1582  stats->num_channels[i]++;
1583  break;
1584  } else if (!stats->sample_rates[i]) {
1585  stats->sample_rates[i] = channel_native_rate;
1586  stats->num_channels[i]++;
1587  break;
1588  }
1589  }
1590  stats->num_above_internal_rate++;
1591  } else if (softmix_data->internal_rate == channel_native_rate) {
1592  stats->num_at_internal_rate++;
1593  }
1594 }
1595 
1596 /*!
1597  * \internal
1598  * \brief Analyse mixing statistics and change bridges internal rate
1599  * if necessary.
1600  *
1601  * \retval 0 no changes to internal rate
1602  * \retval 1 internal rate was changed, update all the channels on the next mixing iteration.
1603  */
1604 static unsigned int analyse_softmix_stats(struct softmix_stats *stats,
1605  struct softmix_bridge_data *softmix_data, int binaural_active)
1606 {
1607  int i;
1608 
1609  if (binaural_active) {
1611  }
1612 
1613  /*
1614  * Re-adjust the internal bridge sample rate if
1615  * 1. The bridge's internal sample rate is locked in at a sample
1616  * rate other than the current sample rate being used.
1617  * 2. two or more channels support a higher sample rate
1618  * 3. no channels support the current sample rate or a higher rate
1619  */
1620  if (stats->locked_rate) {
1621  /* if the rate is locked by the bridge, only update it if it differs
1622  * from the current rate we are using. */
1623  if (softmix_data->internal_rate != stats->locked_rate) {
1624  ast_debug(1, "Locking at new rate. Bridge changed from %u to %u.\n",
1625  softmix_data->internal_rate, stats->locked_rate);
1626  softmix_data->internal_rate = stats->locked_rate;
1627  return 1;
1628  }
1629  } else if (stats->num_above_maximum_rate) {
1630  /* if the bridge has a maximum rate set and channels are above it only
1631  * update if it differs from the current rate we are using. */
1632  if (softmix_data->internal_rate != stats->maximum_rate) {
1633  ast_debug(1, "Locking at new maximum rate. Bridge changed from %u to %u.\n",
1634  softmix_data->internal_rate, stats->maximum_rate);
1635  softmix_data->internal_rate = stats->maximum_rate;
1636  return 1;
1637  }
1638  } else if (stats->num_above_internal_rate >= 2) {
1639  /* the highest rate is just used as a starting point */
1640  unsigned int best_rate = stats->highest_supported_rate;
1641  int best_index = -1;
1642 
1643  for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
1644  if (stats->num_channels[i]) {
1645  break;
1646  }
1647  if (2 <= stats->num_channels[i]) {
1648  /* Two or more channels support this rate. */
1649  if (best_index == -1
1650  || stats->sample_rates[best_index] < stats->sample_rates[i]) {
1651  /*
1652  * best_rate starts out being the first sample rate
1653  * greater than the internal sample rate that two or
1654  * more channels support.
1655  *
1656  * or
1657  *
1658  * There are multiple rates above the internal rate
1659  * and this rate is higher than the previous rate two
1660  * or more channels support.
1661  */
1662  best_rate = stats->sample_rates[i];
1663  best_index = i;
1664  }
1665  } else if (best_index == -1) {
1666  /*
1667  * It is possible that multiple channels exist with native sample
1668  * rates above the internal sample rate, but none of those channels
1669  * have the same rate in common. In this case, the lowest sample
1670  * rate among those channels is picked. Over time as additional
1671  * statistic runs are made the internal sample rate number will
1672  * adjust to the most optimal sample rate, but it may take multiple
1673  * iterations.
1674  */
1675  best_rate = MIN(best_rate, stats->sample_rates[i]);
1676  }
1677  }
1678 
1679  ast_debug(1, "Multiple above internal rate. Bridge changed from %u to %u.\n",
1680  softmix_data->internal_rate, best_rate);
1681  softmix_data->internal_rate = best_rate;
1682  return 1;
1683  } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
1684  /* In this case, the highest supported rate is actually lower than the internal rate */
1685  ast_debug(1, "All below internal rate. Bridge changed from %u to %u.\n",
1686  softmix_data->internal_rate, stats->highest_supported_rate);
1687  softmix_data->internal_rate = stats->highest_supported_rate;
1688  return 1;
1689  }
1690  return 0;
1691 }
1692 
1693 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array,
1694  unsigned int starting_num_entries, unsigned int binaural_active)
1695 {
1696  memset(mixing_array, 0, sizeof(*mixing_array));
1697  mixing_array->max_num_entries = starting_num_entries;
1698  if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
1699  ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1700  return -1;
1701  }
1702  if (binaural_active) {
1703  if (!(mixing_array->chan_pairs = ast_calloc(mixing_array->max_num_entries,
1704  sizeof(struct convolve_channel_pair *)))) {
1705  ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1706  return -1;
1707  }
1708  }
1709  return 0;
1710 }
1711 
1712 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array,
1713  unsigned int binaural_active)
1714 {
1715  ast_free(mixing_array->buffers);
1716  if (binaural_active) {
1717  ast_free(mixing_array->chan_pairs);
1718  }
1719 }
1720 
1721 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array,
1722  unsigned int num_entries, unsigned int binaural_active)
1723 {
1724  int16_t **tmp;
1725 
1726  /* give it some room to grow since memory is cheap but allocations can be expensive */
1727  mixing_array->max_num_entries = num_entries;
1728  if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
1729  ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1730  return -1;
1731  }
1732  mixing_array->buffers = tmp;
1733 
1734  if (binaural_active) {
1735  struct convolve_channel_pair **tmp2;
1736  if (!(tmp2 = ast_realloc(mixing_array->chan_pairs,
1737  (mixing_array->max_num_entries * sizeof(struct convolve_channel_pair *))))) {
1738  ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1739  return -1;
1740  }
1741  mixing_array->chan_pairs = tmp2;
1742  }
1743  return 0;
1744 }
1745 
1746 /*!
1747  * \brief Mixing loop.
1748  *
1749  * \retval 0 on success
1750  * \retval -1 on failure
1751  */
1752 static int softmix_mixing_loop(struct ast_bridge *bridge)
1753 {
1754  struct softmix_stats stats = { { 0 }, };
1755  struct softmix_mixing_array mixing_array;
1756  struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1757  struct ast_timer *timer;
1758  struct softmix_translate_helper trans_helper;
1760 #ifdef BINAURAL_RENDERING
1761  int16_t bin_buf[MAX_DATALEN];
1762  int16_t ann_buf[MAX_DATALEN];
1763 #endif
1764  unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
1765  int timingfd;
1766  int update_all_rates = 0; /* set this when the internal sample rate has changed */
1767  unsigned int idx;
1768  unsigned int x;
1769  int res = -1;
1770 
1771  timer = softmix_data->timer;
1772  timingfd = ast_timer_fd(timer);
1773  softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
1774  ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1775 
1776  /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
1777  if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10,
1778  bridge->softmix.binaural_active)) {
1779  goto softmix_cleanup;
1780  }
1781 
1782  /*
1783  * XXX Softmix needs to use channel roles to determine who gets
1784  * what audio mixed.
1785  */
1786  while (!softmix_data->stop && bridge->num_active) {
1787  struct ast_bridge_channel *bridge_channel;
1788  int timeout = -1;
1789  struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
1790  unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1791  unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1792  int remb_update = 0;
1793 
1794  if (softmix_datalen > MAX_DATALEN) {
1795  /* This should NEVER happen, but if it does we need to know about it. Almost
1796  * all the memcpys used during this process depend on this assumption. Rather
1797  * than checking this over and over again through out the code, this single
1798  * verification is done on each iteration. */
1800  "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
1801  bridge->uniqueid);
1802  goto softmix_cleanup;
1803  }
1804 
1805  /* Grow the mixing array buffer as participants are added. */
1806  if (mixing_array.max_num_entries < bridge->num_channels
1807  && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5,
1808  bridge->softmix.binaural_active)) {
1809  goto softmix_cleanup;
1810  }
1811 
1812  /* init the number of buffers stored in the mixing array to 0.
1813  * As buffers are added for mixing, this number is incremented. */
1814  mixing_array.used_entries = 0;
1815 
1816  /* These variables help determine if a rate change is required */
1817  if (!stat_iteration_counter) {
1818  memset(&stats, 0, sizeof(stats));
1819  stats.locked_rate = bridge->softmix.internal_sample_rate;
1820  stats.maximum_rate = bridge->softmix.maximum_sample_rate;
1821  }
1822 
1823  /* If the sample rate has changed, update the translator helper */
1824  if (update_all_rates) {
1825  softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
1826  }
1827 
1828 #ifdef BINAURAL_RENDERING
1829  check_binaural_position_change(bridge, softmix_data);
1830 #endif
1831 
1832  /* If we need to do a REMB update to all video sources then do so */
1836  remb_update = 1;
1837  softmix_data->last_remb_update = ast_tvnow();
1838  }
1839 
1840  /* Go through pulling audio from each factory that has it available */
1841  AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1842  struct softmix_channel *sc = bridge_channel->tech_pvt;
1843 
1844  if (!sc) {
1845  /* This channel failed to join successfully. */
1846  continue;
1847  }
1848 
1849  /* Update the sample rate to match the bridge's native sample rate if necessary. */
1850  if (update_all_rates) {
1851  set_softmix_bridge_data(softmix_data->internal_rate,
1852  softmix_data->internal_mixing_interval, bridge_channel, 1, -1, -1, -1);
1853  }
1854 
1855  /* If stat_iteration_counter is 0, then collect statistics during this mixing interaction */
1856  if (!stat_iteration_counter) {
1857  gather_softmix_stats(&stats, softmix_data, bridge_channel);
1858  }
1859 
1860  /* if the channel is suspended, don't check for audio, but still gather stats */
1861  if (bridge_channel->suspended) {
1862  continue;
1863  }
1864 
1865  /* Try to get audio from the factory if available */
1866  ast_mutex_lock(&sc->lock);
1867  if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
1868 #ifdef BINAURAL_RENDERING
1869  add_binaural_mixing(bridge, softmix_data, softmix_samples, &mixing_array, sc,
1870  ast_channel_name(bridge_channel->chan));
1871 #endif
1872  mixing_array.used_entries++;
1873  }
1874  if (remb_update) {
1875  remb_collect_report(bridge, bridge_channel, softmix_data, sc);
1876  }
1877  ast_mutex_unlock(&sc->lock);
1878  }
1879 
1880  /* mix it like crazy (non binaural channels)*/
1881  memset(buf, 0, softmix_datalen);
1882  for (idx = 0; idx < mixing_array.used_entries; ++idx) {
1883  for (x = 0; x < softmix_samples; ++x) {
1884  ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
1885  }
1886  }
1887 
1888 #ifdef BINAURAL_RENDERING
1889  binaural_mixing(bridge, softmix_data, &mixing_array, bin_buf, ann_buf);
1890 #endif
1891 
1892  /* Next step go through removing the channel's own audio and creating a good frame... */
1893  AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1894  struct softmix_channel *sc = bridge_channel->tech_pvt;
1895 
1896  if (!sc || bridge_channel->suspended) {
1897  /* This channel failed to join successfully or is suspended. */
1898  continue;
1899  }
1900 
1901  ast_mutex_lock(&sc->lock);
1902 
1903  /* Make SLINEAR write frame from local buffer */
1904  ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
1905  "Replace softmix channel slin format");
1906 #ifdef BINAURAL_RENDERING
1907  if (bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
1908  && sc->binaural) {
1909  create_binaural_frame(bridge_channel, sc, bin_buf, ann_buf, softmix_datalen,
1910  softmix_samples, buf);
1911  } else
1912 #endif
1913  {
1914  sc->write_frame.datalen = softmix_datalen;
1915  sc->write_frame.samples = softmix_samples;
1916  memcpy(sc->final_buf, buf, softmix_datalen);
1917  }
1918  /* process the softmix channel's new write audio */
1919  softmix_process_write_audio(&trans_helper,
1920  ast_channel_rawwriteformat(bridge_channel->chan), sc,
1921  softmix_data->default_sample_size);
1922 
1923  ast_mutex_unlock(&sc->lock);
1924 
1925  /* A frame is now ready for the channel. */
1926  ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1927 
1928  if (remb_update) {
1929  remb_send_report(bridge_channel, softmix_data, sc);
1930  }
1931  }
1932 
1933  if (remb_update) {
1934  /* In case we are doing bridge level REMB reset the bitrate so we start fresh */
1935  softmix_data->bitrate = 0;
1936  }
1937 
1938  update_all_rates = 0;
1939  if (!stat_iteration_counter) {
1940  update_all_rates = analyse_softmix_stats(&stats, softmix_data,
1941  bridge->softmix.binaural_active);
1942  stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1943  }
1944  stat_iteration_counter--;
1945 
1946  ast_bridge_unlock(bridge);
1947  /* cleanup any translation frame data from the previous mixing iteration. */
1948  softmix_translate_helper_cleanup(&trans_helper);
1949  /* Wait for the timing source to tell us to wake up and get things done */
1950  ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1951  if (ast_timer_ack(timer, 1) < 0) {
1952  ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1953  bridge->uniqueid);
1954  ast_bridge_lock(bridge);
1955  goto softmix_cleanup;
1956  }
1957  ast_bridge_lock(bridge);
1958 
1959  /* make sure to detect mixing interval changes if they occur. */
1960  if (bridge->softmix.internal_mixing_interval
1961  && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1963  ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1964  update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1965  }
1966  }
1967 
1968  res = 0;
1969 
1970 softmix_cleanup:
1971  softmix_translate_helper_destroy(&trans_helper);
1972  softmix_mixing_array_destroy(&mixing_array, bridge->softmix.binaural_active);
1973  return res;
1974 }
1975 
1976 /*!
1977  * \internal
1978  * \brief Mixing thread.
1979  * \since 12.0.0
1980  *
1981  * \note The thread does not have its own reference to the
1982  * bridge. The lifetime of the thread is tied to the lifetime
1983  * of the mixing technology association with the bridge.
1984  */
1985 static void *softmix_mixing_thread(void *data)
1986 {
1987  struct softmix_bridge_data *softmix_data = data;
1988  struct ast_bridge *bridge = softmix_data->bridge;
1989 
1990  ast_bridge_lock(bridge);
1991  if (bridge->callid) {
1993  }
1994 
1995  ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1996 
1997  while (!softmix_data->stop) {
1998  if (!bridge->num_active) {
1999  /* Wait for something to happen to the bridge. */
2000  ast_bridge_unlock(bridge);
2001  ast_mutex_lock(&softmix_data->lock);
2002  if (!softmix_data->stop) {
2003  ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
2004  }
2005  ast_mutex_unlock(&softmix_data->lock);
2006  ast_bridge_lock(bridge);
2007  continue;
2008  }
2009 
2010  if (bridge->softmix.binaural_active && !softmix_data->binaural_init) {
2011 #ifndef BINAURAL_RENDERING
2012  ast_bridge_lock(bridge);
2013  bridge->softmix.binaural_active = 0;
2014  ast_bridge_unlock(bridge);
2015  ast_log(LOG_WARNING, "Bridge: %s: Binaural rendering active by config but not "
2016  "compiled.\n", bridge->uniqueid);
2017 #else
2018  /* Set and init binaural data if binaural is activated in the configuration. */
2020  softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
2021  softmix_data->internal_mixing_interval);
2022  /* If init for binaural processing fails we will fall back to mono audio processing. */
2023  if (init_convolve_data(&softmix_data->convolve, softmix_data->default_sample_size)
2024  == -1) {
2025  ast_bridge_lock(bridge);
2026  bridge->softmix.binaural_active = 0;
2027  ast_bridge_unlock(bridge);
2028  ast_log(LOG_ERROR, "Bridge: %s: Unable to allocate memory for "
2029  "binaural processing, Will only process mono audio.\n",
2030  bridge->uniqueid);
2031  }
2032  softmix_data->binaural_init = 1;
2033 #endif
2034  }
2035 
2036  if (softmix_mixing_loop(bridge)) {
2037  /*
2038  * A mixing error occurred. Sleep and try again later so we
2039  * won't flood the logs.
2040  */
2041  ast_bridge_unlock(bridge);
2042  sleep(1);
2043  ast_bridge_lock(bridge);
2044  }
2045  }
2046 
2047  ast_bridge_unlock(bridge);
2048 
2049  ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
2050 
2051  return NULL;
2052 }
2053 
2054 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
2055 {
2056  if (softmix_data->timer) {
2057  ast_timer_close(softmix_data->timer);
2058  softmix_data->timer = NULL;
2059  }
2060  ast_mutex_destroy(&softmix_data->lock);
2061  ast_cond_destroy(&softmix_data->cond);
2062  AST_VECTOR_RESET(&softmix_data->remb_collectors, ao2_cleanup);
2063  AST_VECTOR_FREE(&softmix_data->remb_collectors);
2064  ast_free(softmix_data);
2065 }
2066 
2067 /*! \brief Function called when a bridge is created */
2068 static int softmix_bridge_create(struct ast_bridge *bridge)
2069 {
2070  struct softmix_bridge_data *softmix_data;
2071 
2072  softmix_data = ast_calloc(1, sizeof(*softmix_data));
2073  if (!softmix_data) {
2074  return -1;
2075  }
2076  softmix_data->bridge = bridge;
2077  ast_mutex_init(&softmix_data->lock);
2078  ast_cond_init(&softmix_data->cond, NULL);
2079  softmix_data->timer = ast_timer_open();
2080  if (!softmix_data->timer) {
2081  ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
2082  softmix_bridge_data_destroy(softmix_data);
2083  return -1;
2084  }
2085  /* start at minimum rate, let it grow from there */
2086  softmix_data->internal_rate = SOFTMIX_MIN_SAMPLE_RATE;
2088 
2089 #ifdef BINAURAL_RENDERING
2090  softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
2091  softmix_data->internal_mixing_interval);
2092 #endif
2093 
2094  AST_VECTOR_INIT(&softmix_data->remb_collectors, 0);
2095 
2096  bridge->tech_pvt = softmix_data;
2097 
2098  /* Start the mixing thread. */
2099  if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
2100  softmix_data)) {
2101  softmix_data->thread = AST_PTHREADT_NULL;
2102  softmix_bridge_data_destroy(softmix_data);
2103  bridge->tech_pvt = NULL;
2104  return -1;
2105  }
2106 
2107  return 0;
2108 }
2109 
2110 /*!
2111  * \internal
2112  * \brief Request the softmix mixing thread stop.
2113  * \since 12.0.0
2114  *
2115  * \param bridge Which bridge is being stopped.
2116  */
2118 {
2119  struct softmix_bridge_data *softmix_data;
2120 
2121  softmix_data = bridge->tech_pvt;
2122  if (!softmix_data) {
2123  return;
2124  }
2125 
2126  ast_mutex_lock(&softmix_data->lock);
2127  softmix_data->stop = 1;
2128  ast_mutex_unlock(&softmix_data->lock);
2129 }
2130 
2131 /*! \brief Function called when a bridge is destroyed */
2133 {
2134  struct softmix_bridge_data *softmix_data;
2135  pthread_t thread;
2136 
2137  softmix_data = bridge->tech_pvt;
2138  if (!softmix_data) {
2139  return;
2140  }
2141 
2142  /* Stop the mixing thread. */
2143  ast_mutex_lock(&softmix_data->lock);
2144  softmix_data->stop = 1;
2145  ast_cond_signal(&softmix_data->cond);
2146  thread = softmix_data->thread;
2147  softmix_data->thread = AST_PTHREADT_NULL;
2148  ast_mutex_unlock(&softmix_data->lock);
2149  if (thread != AST_PTHREADT_NULL) {
2150  ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
2151  pthread_join(thread, NULL);
2152  }
2153 #ifdef BINAURAL_RENDERING
2154  free_convolve_data(&softmix_data->convolve);
2155 #endif
2156  softmix_bridge_data_destroy(softmix_data);
2157  bridge->tech_pvt = NULL;
2158 }
2159 
2160 /*!
2161  * \brief Map a source stream to all of its destination streams.
2162  *
2163  * \param source_channel_name Name of channel where the source stream originates
2164  * \param bridge_stream_position The slot in the bridge where source video will come from
2165  * \param participants The bridge_channels in the bridge
2166  * \param source_channel_stream_position The position of the stream on the source channel
2167  */
2168 static void map_source_to_destinations(const char *source_channel_name,
2169  size_t bridge_stream_position, struct ast_bridge_channels_list *participants, int source_channel_stream_position)
2170 {
2171  struct ast_bridge_channel *participant;
2172 
2173  AST_LIST_TRAVERSE(participants, participant, entry) {
2174  int i;
2175  struct ast_stream_topology *topology;
2176 
2177  if (!strcmp(source_channel_name, ast_channel_name(participant->chan))) {
2178  continue;
2179  }
2180 
2181  ast_bridge_channel_lock(participant);
2182  ast_channel_lock(participant->chan);
2183  topology = ast_channel_get_stream_topology(participant->chan);
2184 
2185  for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
2186  struct ast_stream *stream;
2187 
2188  stream = ast_stream_topology_get_stream(topology, i);
2189  if (is_video_dest(stream, source_channel_name, source_channel_stream_position)) {
2190  struct softmix_channel *sc = participant->tech_pvt;
2191 
2192  AST_VECTOR_REPLACE(&participant->stream_map.to_channel, bridge_stream_position, i);
2193  AST_VECTOR_APPEND(&sc->video_sources, bridge_stream_position);
2194  break;
2195  }
2196  }
2197  ast_channel_unlock(participant->chan);
2198  ast_bridge_channel_unlock(participant);
2199  }
2200 }
2201 
2202 /*!
2203  * \brief Allocate a REMB collector
2204  *
2205  * \retval non-NULL success
2206  * \retval NULL failure
2207  */
2209 {
2210  struct softmix_remb_collector *collector;
2211 
2212  collector = ao2_alloc_options(sizeof(*collector), NULL, AO2_ALLOC_OPT_LOCK_NOLOCK);
2213  if (!collector) {
2214  return NULL;
2215  }
2216 
2217  collector->frame.frametype = AST_FRAME_RTCP;
2218  collector->frame.subclass.integer = AST_RTP_RTCP_PSFB;
2219  collector->feedback.fmt = AST_RTP_RTCP_FMT_REMB;
2220  collector->frame.data.ptr = &collector->feedback;
2221  collector->frame.datalen = sizeof(collector->feedback);
2222 
2223  return collector;
2224 }
2225 
2226 /*!
2227  * \brief Setup REMB collection for a particular bridge stream and channel.
2228  *
2229  * \param bridge The bridge
2230  * \param bridge_channel Channel that is collecting REMB information
2231  * \param bridge_stream_position The slot in the bridge where source video comes from
2232  */
2233 static void remb_enable_collection(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel,
2234  size_t bridge_stream_position)
2235 {
2236  struct softmix_channel *sc = bridge_channel->tech_pvt;
2237  struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
2238 
2239  if (!sc->remb_collector) {
2241  if (!sc->remb_collector) {
2242  /* This is not fatal. Things will still continue to work but we won't
2243  * produce a REMB report to the sender.
2244  */
2245  return;
2246  }
2247  }
2248 
2249  ao2_ref(sc->remb_collector, +1);
2250  if (AST_VECTOR_REPLACE(&softmix_data->remb_collectors, bridge_stream_position,
2251  sc->remb_collector)) {
2252  ao2_ref(sc->remb_collector, -1);
2253  }
2254 }
2255 
2256 static void softmix_bridge_stream_sources_update(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel,
2257  struct softmix_channel *sc)
2258 {
2259  int index;
2260  struct ast_stream_topology *old_topology = sc->topology;
2261  struct ast_stream_topology *new_topology = ast_channel_get_stream_topology(bridge_channel->chan);
2262  int removed_streams[MAX(ast_stream_topology_get_count(sc->topology), ast_stream_topology_get_count(new_topology))];
2263  size_t removed_streams_count = 0;
2264  struct ast_stream_topology *added_streams;
2265  struct ast_bridge_channels_list *participants = &bridge->channels;
2266  struct ast_bridge_channel *participant;
2267  SCOPE_ENTER(3, "%s: OT: %s NT: %s\n", ast_channel_name(bridge_channel->chan),
2268  ast_str_tmp(256, ast_stream_topology_to_str(old_topology, &STR_TMP)),
2269  ast_str_tmp(256, ast_stream_topology_to_str(new_topology, &STR_TMP)));
2270 
2271  added_streams = ast_stream_topology_alloc();
2272  if (!added_streams) {
2273  SCOPE_EXIT_LOG(LOG_ERROR, "%s: Couldn't alloc topology\n", ast_channel_name(bridge_channel->chan));
2274  }
2275 
2276  /* We go through the old topology comparing it to the new topology to determine what streams
2277  * changed state. A state transition can result in the stream being considered a new source
2278  * (for example it was removed and is now present) or being removed (a stream became inactive).
2279  * Added streams are copied into a topology and added to each other participant while for
2280  * removed streams we merely store their position and mark them as removed later.
2281  */
2282  ast_trace(-1, "%s: Checking for state changes\n", ast_channel_name(bridge_channel->chan));
2283  for (index = 0; index < ast_stream_topology_get_count(sc->topology) && index < ast_stream_topology_get_count(new_topology); ++index) {
2284  struct ast_stream *old_stream = ast_stream_topology_get_stream(sc->topology, index);
2285  struct ast_stream *new_stream = ast_stream_topology_get_stream(new_topology, index);
2286  SCOPE_ENTER(4, "%s: Slot: %d Old stream: %s New stream: %s\n", ast_channel_name(bridge_channel->chan),
2287  index, ast_str_tmp(256, ast_stream_to_str(old_stream, &STR_TMP)),
2288  ast_str_tmp(256, ast_stream_to_str(new_stream, &STR_TMP)));
2289 
2290  /* Ignore all streams that don't carry video and streams that are strictly outgoing destination streams */
2291  if ((ast_stream_get_type(old_stream) != AST_MEDIA_TYPE_VIDEO && ast_stream_get_type(new_stream) != AST_MEDIA_TYPE_VIDEO) ||
2292  !strncmp(ast_stream_get_name(new_stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
2294  SCOPE_EXIT_EXPR(continue, "%s: Stream %d ignored\n", ast_channel_name(bridge_channel->chan), index);
2295  }
2296 
2298  /* If a stream renegotiates from video to non-video then we need to remove it as a source */
2299  ast_trace(-1, "%s: Stream %d added to remove list\n", ast_channel_name(bridge_channel->chan), index);
2300  removed_streams[removed_streams_count++] = index;
2301  } else if (ast_stream_get_type(old_stream) != AST_MEDIA_TYPE_VIDEO && ast_stream_get_type(new_stream) == AST_MEDIA_TYPE_VIDEO) {
2302  if (ast_stream_get_state(new_stream) != AST_STREAM_STATE_REMOVED) {
2303  /* If a stream renegotiates from non-video to video in a non-removed state we need to add it as a source */
2304  if (append_source_stream(added_streams, ast_channel_name(bridge_channel->chan),
2305  bridge->softmix.send_sdp_label ? ast_channel_uniqueid(bridge_channel->chan) : NULL,
2306  new_stream, index)) {
2307  SCOPE_EXIT_EXPR(goto cleanup, "%s: Couldn't append source stream %d:%s\n", ast_channel_name(bridge_channel->chan),
2308  index, ast_stream_get_name(new_stream));
2309  }
2310  ast_trace(-1, "%s: Stream %d changed from non-video to video\n", ast_channel_name(bridge_channel->chan), index);
2311  }
2312  } else if (ast_stream_get_state(old_stream) != AST_STREAM_STATE_REMOVED &&
2314  ast_trace(-1, "%s: Stream %d added to remove list\n", ast_channel_name(bridge_channel->chan), index);
2315  /* If a stream renegotiates and is removed then we remove it */
2316  removed_streams[removed_streams_count++] = index;
2321  /* If a stream renegotiates and is added then we add it */
2322  if (append_source_stream(added_streams, ast_channel_name(bridge_channel->chan),
2323  bridge->softmix.send_sdp_label ? ast_channel_uniqueid(bridge_channel->chan) : NULL,
2324  new_stream, index)) {
2325  SCOPE_EXIT_EXPR(goto cleanup, "%s: Couldn't append source stream %d:%s\n", ast_channel_name(bridge_channel->chan),
2326  index, ast_stream_get_name(new_stream));
2327  }
2328  ast_trace(-1, "%s: Stream %d:%s changed state from %s to %s\n", ast_channel_name(bridge_channel->chan),
2329  index, ast_stream_get_name(old_stream), ast_stream_state2str(ast_stream_get_state(old_stream)),
2331  } else {
2332  ast_trace(-1, "%s: Stream %d:%s didn't do anything\n", ast_channel_name(bridge_channel->chan),
2333  index, ast_stream_get_name(old_stream));
2334  }
2335  SCOPE_EXIT();
2336  }
2337 
2338  /* Any newly added streams that did not take the position of a removed stream
2339  * will be present at the end of the new topology. Since streams are never
2340  * removed from the topology but merely marked as removed we can pick up where we
2341  * left off when comparing the old and new topologies.
2342  */
2343  ast_trace(-1, "%s: Checking for newly added streams\n", ast_channel_name(bridge_channel->chan));
2344 
2345  for (; index < ast_stream_topology_get_count(new_topology); ++index) {
2346  struct ast_stream *stream = ast_stream_topology_get_stream(new_topology, index);
2347  SCOPE_ENTER(4, "%s: Checking stream %d:%s\n", ast_channel_name(bridge_channel->chan), index,
2348  ast_stream_get_name(stream));
2349 
2350  if (!is_video_source(stream)) {
2351  SCOPE_EXIT_EXPR(continue, "%s: Stream %d:%s is not video source\n", ast_channel_name(bridge_channel->chan),
2352  index, ast_stream_get_name(stream));
2353  }
2354 
2355  if (append_source_stream(added_streams, ast_channel_name(bridge_channel->chan),
2356  bridge->softmix.send_sdp_label ? ast_channel_uniqueid(bridge_channel->chan) : NULL,
2357  stream, index)) {
2358  SCOPE_EXIT_EXPR(goto cleanup, "%s: Couldn't append stream %d:%s\n", ast_channel_name(bridge_channel->chan),
2359  index, ast_stream_get_name(stream));
2360  }
2361  SCOPE_EXIT("%s: Added new stream %s\n", ast_channel_name(bridge_channel->chan),
2362  ast_str_tmp(256, ast_stream_to_str(stream, &STR_TMP)));
2363  }
2364 
2365  /* We always update the stored topology if we can to reflect what is currently negotiated */
2366  sc->topology = ast_stream_topology_clone(new_topology);
2367  if (!sc->topology) {
2368  sc->topology = old_topology;
2369  } else {
2370  ast_stream_topology_free(old_topology);
2371  }
2372 
2373  /* If there are no removed sources and no added sources we don't need to renegotiate the
2374  * other participants.
2375  */
2376  if (!removed_streams_count && !ast_stream_topology_get_count(added_streams)) {
2377  ast_trace(-1, "%s: Nothing added or removed\n", ast_channel_name(bridge_channel->chan));
2378  goto cleanup;
2379  }
2380 
2381  ast_trace(-1, "%s: Processing adds and removes\n", ast_channel_name(bridge_channel->chan));
2382  /* Go through each participant adding in the new streams and removing the old ones */
2383  AST_LIST_TRAVERSE(participants, participant, entry)
2384  {
2385  struct softmix_channel *participant_sc = participant->tech_pvt;
2386  SCOPE_ENTER(4, "%s/%s: Old participant topology %s\n",
2387  ast_channel_name(bridge_channel->chan),
2388  ast_channel_name(participant->chan),
2389  ast_str_tmp(256, ast_stream_topology_to_str(participant_sc->topology, &STR_TMP)));
2390 
2391  if (participant == bridge_channel) {
2392  SCOPE_EXIT_EXPR(continue, "%s/%s: Same channel. Skipping\n",
2393  ast_channel_name(bridge_channel->chan),
2394  ast_channel_name(participant->chan));
2395  }
2396 
2397  /* We add in all the new streams first so that they do not take the place
2398  * of any of our removed streams, allowing the remote side to reset the state
2399  * for each removed stream. */
2400  if (append_all_streams(participant_sc->topology, added_streams)) {
2401  SCOPE_EXIT_EXPR(goto cleanup, "%s/%s: Couldn't append streams\n", ast_channel_name(bridge_channel->chan),
2402  ast_channel_name(participant->chan));
2403  }
2404  ast_trace(-1, "%s/%s: Adding streams %s\n", ast_channel_name(bridge_channel->chan),
2405  ast_channel_name(participant->chan),
2406  ast_str_tmp(256, ast_stream_topology_to_str(added_streams, &STR_TMP)));
2407 
2408  /* Then we go through and remove any ones that were removed */
2409  for (index = 0;
2410  removed_streams_count && index < ast_stream_topology_get_count(sc->topology); ++index) {
2411  struct ast_stream *stream = ast_stream_topology_get_stream(sc->topology, index);
2412  int removed_stream;
2413 
2414  for (removed_stream = 0; removed_stream < removed_streams_count; ++removed_stream) {
2415  if (is_video_dest(stream, ast_channel_name(bridge_channel->chan),
2416  removed_streams[removed_stream])) {
2417  ast_trace(-1, "%s/%s: Removing stream %s\n",
2418  ast_channel_name(bridge_channel->chan),
2419  ast_channel_name(participant->chan),
2420  ast_str_tmp(256, ast_stream_to_str(stream, &STR_TMP)));
2422  }
2423  }
2424  }
2425  ast_channel_request_stream_topology_change(participant->chan, participant_sc->topology, NULL);
2426  SCOPE_EXIT("%s/%s: New participant topology %s\n",
2427  ast_channel_name(bridge_channel->chan),
2428  ast_channel_name(participant->chan),
2429  ast_str_tmp(256, ast_stream_topology_to_str(participant_sc->topology, &STR_TMP)));
2430  }
2431 
2432  ast_trace(-1, "%s: New topology %s\n", ast_channel_name(bridge_channel->chan),
2433  ast_str_tmp(256, ast_stream_topology_to_str(sc->topology, &STR_TMP)));
2434 
2435 cleanup:
2436  ast_stream_topology_free(added_streams);
2437  SCOPE_EXIT();
2438 }
2439 
2440 /*!
2441  * \brief stream_topology_changed callback
2442  *
2443  * For most video modes, nothing beyond the ordinary is required.
2444  * For the SFU case, though, we need to completely remap the streams
2445  * in order to ensure video gets directed where it is expected to go.
2446  *
2447  * \param bridge The bridge
2448  * \param bridge_channel Channel whose topology has changed
2449  */
2450 static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
2451 {
2452  struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
2453  struct softmix_channel *sc = bridge_channel->tech_pvt;
2454  struct ast_bridge_channel *participant;
2455  struct ast_vector_int media_types;
2456  int nths[AST_MEDIA_TYPE_END] = {0};
2457  int idx;
2458  SCOPE_ENTER(3, "%s: \n", ast_channel_name(bridge_channel->chan));
2459 
2460  switch (bridge->softmix.video_mode.mode) {
2464  default:
2465  ast_bridge_channel_stream_map(bridge_channel);
2466  SCOPE_EXIT_RTN("%s: Not in SFU mode\n", ast_channel_name(bridge_channel->chan));
2468  break;
2469  }
2470 
2471  ast_channel_lock(bridge_channel->chan);
2472  softmix_bridge_stream_sources_update(bridge, bridge_channel, sc);
2473  ast_channel_unlock(bridge_channel->chan);
2474 
2475  AST_VECTOR_INIT(&media_types, AST_MEDIA_TYPE_END);
2476 
2477  /* The bridge stream identifiers may change, so reset the mapping for them.
2478  * When channels end up getting added back in they'll reuse their existing
2479  * collector and won't need to allocate a new one (unless they were just added).
2480  */
2481  for (idx = 0; idx < AST_VECTOR_SIZE(&softmix_data->remb_collectors); ++idx) {
2482  ao2_cleanup(AST_VECTOR_GET(&softmix_data->remb_collectors, idx));
2483  AST_VECTOR_REPLACE(&softmix_data->remb_collectors, idx, NULL);
2484  }
2485 
2486  /* First traversal: re-initialize all of the participants' stream maps */
2487  AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
2488  ast_bridge_channel_lock(participant);
2489 
2492 
2493  sc = participant->tech_pvt;
2494  AST_VECTOR_RESET(&sc->video_sources, AST_VECTOR_ELEM_CLEANUP_NOOP);
2495 
2496  ast_bridge_channel_unlock(participant);
2497  }
2498 
2499  /* Second traversal: Map specific video channels from their source to their destinations.
2500  *
2501  * This is similar to what is done in ast_stream_topology_map(),
2502  * except that video channels are handled differently. Each video
2503  * source has it's own unique index on the bridge. This way, a
2504  * particular channel's source video can be distributed to the
2505  * appropriate destination streams on the other channels.
2506  */
2507  AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
2508  int i;
2509  struct ast_stream_topology *topology;
2510 
2511  ast_bridge_channel_lock(participant);
2512  ast_channel_lock(participant->chan);
2513 
2514  topology = ao2_bump(ast_channel_get_stream_topology(participant->chan));
2515  if (!topology) {
2516  /* Oh, my, we are in trouble. */
2517  ast_channel_unlock(participant->chan);
2518  ast_bridge_channel_unlock(participant);
2519  continue;
2520  }
2521 
2522  for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
2523  struct ast_stream *stream = ast_stream_topology_get_stream(topology, i);
2524 
2525  if (is_video_source(stream)) {
2526  AST_VECTOR_APPEND(&media_types, AST_MEDIA_TYPE_VIDEO);
2527  AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, AST_VECTOR_SIZE(&media_types) - 1);
2528  /*
2529  * There are cases where we need to bidirectionally send frames, such as for REMB reports
2530  * so we also map back to the channel.
2531  */
2532  AST_VECTOR_REPLACE(&participant->stream_map.to_channel, AST_VECTOR_SIZE(&media_types) - 1, i);
2533  remb_enable_collection(bridge, participant, AST_VECTOR_SIZE(&media_types) - 1);
2534  /*
2535  * Unlock the channel and participant to prevent
2536  * potential deadlock in map_source_to_destinations().
2537  */
2538  ast_channel_unlock(participant->chan);
2539  ast_bridge_channel_unlock(participant);
2540 
2542  AST_VECTOR_SIZE(&media_types) - 1, &bridge->channels, i);
2543  ast_bridge_channel_lock(participant);
2544  ast_channel_lock(participant->chan);
2545  } else if (ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO) {
2546  /* Video stream mapping occurs directly when a video source stream
2547  * is found on a channel. Video streams should otherwise remain
2548  * unmapped.
2549  */
2550  AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, -1);
2551  } else if (ast_stream_get_state(stream) != AST_STREAM_STATE_REMOVED) {
2552  /* XXX This is copied from ast_stream_topology_map(). This likely could
2553  * be factored out in some way
2554  */
2555  enum ast_media_type type = ast_stream_get_type(stream);
2556  int index = AST_VECTOR_GET_INDEX_NTH(&media_types, ++nths[type],
2558 
2559  if (index == -1) {
2560  AST_VECTOR_APPEND(&media_types, type);
2561  index = AST_VECTOR_SIZE(&media_types) - 1;
2562  }
2563 
2564  AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, index);
2565  AST_VECTOR_REPLACE(&participant->stream_map.to_channel, index, i);
2566  }
2567  }
2568 
2569  ast_stream_topology_free(topology);
2570 
2571  ast_channel_unlock(participant->chan);
2572  ast_bridge_channel_unlock(participant);
2573  }
2574 
2575  AST_VECTOR_FREE(&media_types);
2576  SCOPE_EXIT_RTN("%s\n", ast_channel_name(bridge_channel->chan));
2577 }
2578 
2579 static struct ast_bridge_technology softmix_bridge = {
2580  .name = "softmix",
2581  .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
2583  .create = softmix_bridge_create,
2584  .stop = softmix_bridge_stop,
2585  .destroy = softmix_bridge_destroy,
2586  .join = softmix_bridge_join,
2587  .leave = softmix_bridge_leave,
2588  .unsuspend = softmix_bridge_unsuspend,
2589  .write = softmix_bridge_write,
2590  .stream_topology_changed = softmix_bridge_stream_topology_changed,
2591 };
2592 
2593 #ifdef TEST_FRAMEWORK
2594 struct stream_parameters {
2595  const char *name;
2596  const char *formats;
2597  enum ast_media_type type;
2598 };
2599 
2600 static struct ast_stream_topology *build_topology(const struct stream_parameters *params, size_t num_streams)
2601 {
2602  struct ast_stream_topology *topology;
2603  size_t i;
2604 
2605  topology = ast_stream_topology_alloc();
2606  if (!topology) {
2607  return NULL;
2608  }
2609 
2610  for (i = 0; i < num_streams; ++i) {
2611  RAII_VAR(struct ast_format_cap *, caps, NULL, ao2_cleanup);
2612  struct ast_stream *stream;
2613 
2615  if (!caps) {
2616  goto fail;
2617  }
2618  if (ast_format_cap_update_by_allow_disallow(caps, params[i].formats, 1) < 0) {
2619  goto fail;
2620  }
2621  stream = ast_stream_alloc(params[i].name, params[i].type);
2622  if (!stream) {
2623  goto fail;
2624  }
2625  ast_stream_set_formats(stream, caps);
2626  if (ast_stream_topology_append_stream(topology, stream) < 0) {
2627  ast_stream_free(stream);
2628  goto fail;
2629  }
2630  }
2631 
2632  return topology;
2633 
2634 fail:
2635  ast_stream_topology_free(topology);
2636  return NULL;
2637 }
2638 
2639 static int validate_stream(struct ast_test *test, struct ast_stream *stream,
2640  const struct stream_parameters *params)
2641 {
2642  const struct ast_format_cap *stream_caps;
2643  struct ast_format_cap *params_caps;
2644 
2645  if (ast_stream_get_type(stream) != params->type) {
2646  ast_test_status_update(test, "Expected stream type '%s' but got type '%s'\n",
2647  ast_codec_media_type2str(params->type),
2649  return -1;
2650  }
2651  if (strcmp(ast_stream_get_name(stream), params->name)) {
2652  ast_test_status_update(test, "Expected stream name '%s' but got type '%s'\n",
2653  params->name, ast_stream_get_name(stream));
2654  return -1;
2655  }
2656 
2657  stream_caps = ast_stream_get_formats(stream);
2659  if (!params_caps) {
2660  ast_test_status_update(test, "Allocation error on capabilities\n");
2661  return -1;
2662  }
2663  ast_format_cap_update_by_allow_disallow(params_caps, params->formats, 1);
2664 
2665  if (!ast_format_cap_identical(stream_caps, params_caps)) {
2666  ast_test_status_update(test, "Formats are not as expected on stream '%s'\n",
2667  ast_stream_get_name(stream));
2668  ao2_cleanup(params_caps);
2669  return -1;
2670  }
2671 
2672  ao2_cleanup(params_caps);
2673  return 0;
2674 }
2675 
2676 static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology,
2677  const struct stream_parameters *params, size_t num_streams)
2678 {
2679  int i;
2680 
2681  if (ast_stream_topology_get_count(topology) < num_streams) {
2682  ast_test_status_update(test, "Topology only has %d streams. Needs to have at least %zu\n",
2683  ast_stream_topology_get_count(topology), num_streams);
2684  return -1;
2685  }
2686 
2687  for (i = 0; i < num_streams; ++i) {
2688  if (validate_stream(test, ast_stream_topology_get_stream(topology, i), &params[i])) {
2689  return -1;
2690  }
2691  }
2692 
2693  return 0;
2694 }
2695 
2696 AST_TEST_DEFINE(sfu_append_source_streams)
2697 {
2699  static const struct stream_parameters bob_streams[] = {
2700  { "bob_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2701  { "bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2702  };
2703  static const struct stream_parameters alice_streams[] = {
2704  { "alice_audio", "ulaw,opus", AST_MEDIA_TYPE_AUDIO, },
2705  { "alice_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2706  };
2707  static const struct stream_parameters alice_dest_stream = {
2708  "softbridge_dest_PJSIP/Bob-00000001_1", "h264,vp8", AST_MEDIA_TYPE_VIDEO,
2709  };
2710  static const struct stream_parameters bob_dest_stream = {
2711  "softbridge_dest_PJSIP/Alice-00000000_1", "vp8", AST_MEDIA_TYPE_VIDEO,
2712  };
2713  struct ast_stream_topology *topology_alice = NULL;
2714  struct ast_stream_topology *topology_bob = NULL;
2715 
2716  switch (cmd) {
2717  case TEST_INIT:
2718  info->name = "sfu_append_source_streams";
2719  info->category = "/bridges/bridge_softmix/";
2720  info->summary = "Test appending of video streams";
2721  info->description =
2722  "This tests does stuff.";
2723  return AST_TEST_NOT_RUN;
2724  case TEST_EXECUTE:
2725  break;
2726  }
2727 
2728  topology_alice = build_topology(alice_streams, ARRAY_LEN(alice_streams));
2729  if (!topology_alice) {
2730  goto end;
2731  }
2732 
2733  topology_bob = build_topology(bob_streams, ARRAY_LEN(bob_streams));
2734  if (!topology_bob) {
2735  goto end;
2736  }
2737 
2738  if (append_source_streams(topology_alice, "PJSIP/Bob-00000001", NULL, topology_bob)) {
2739  ast_test_status_update(test, "Failed to append Bob's streams to Alice\n");
2740  goto end;
2741  }
2742 
2743  if (ast_stream_topology_get_count(topology_alice) != 3) {
2744  ast_test_status_update(test, "Alice's topology isn't large enough! It's %d but needs to be %d\n",
2745  ast_stream_topology_get_count(topology_alice), 3);
2746  goto end;
2747  }
2748 
2749  if (validate_original_streams(test, topology_alice, alice_streams, ARRAY_LEN(alice_streams))) {
2750  goto end;
2751  }
2752 
2753  if (validate_stream(test, ast_stream_topology_get_stream(topology_alice, 2), &alice_dest_stream)) {
2754  goto end;
2755  }
2756 
2757  if (append_source_streams(topology_bob, "PJSIP/Alice-00000000", NULL, topology_alice)) {
2758  ast_test_status_update(test, "Failed to append Alice's streams to Bob\n");
2759  goto end;
2760  }
2761 
2762  if (ast_stream_topology_get_count(topology_bob) != 3) {
2763  ast_test_status_update(test, "Bob's topology isn't large enough! It's %d but needs to be %d\n",
2764  ast_stream_topology_get_count(topology_bob), 3);
2765  goto end;
2766  }
2767 
2768  if (validate_original_streams(test, topology_bob, bob_streams, ARRAY_LEN(bob_streams))) {
2769  goto end;
2770  }
2771 
2772  if (validate_stream(test, ast_stream_topology_get_stream(topology_bob, 2), &bob_dest_stream)) {
2773  goto end;
2774  }
2775 
2776  res = AST_TEST_PASS;
2777 
2778 end:
2779  ast_stream_topology_free(topology_alice);
2780  ast_stream_topology_free(topology_bob);
2781  return res;
2782 }
2783 
2784 AST_TEST_DEFINE(sfu_remove_destination_streams)
2785 {
2787  static const struct stream_parameters params[] = {
2788  { "alice_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2789  { "alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2790  { "softbridge_dest_PJSIP/Bob-00000001_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2791  { "softbridge_dest_PJSIP/Carol-00000002_video", "h264", AST_MEDIA_TYPE_VIDEO, },
2792  };
2793  static const struct {
2794  const char *channel_name;
2795  int num_streams;
2796  int params_index[4];
2797  } removal_results[] = {
2798  { "PJSIP/Bob-00000001", 4, { 0, 1, 2, 3 }, },
2799  { "PJSIP/Edward-00000004", 4, { 0, 1, 2, 3 }, },
2800  { "", 4, { 0, 1, 2, 3 }, },
2801  };
2802  struct ast_stream_topology *orig = NULL;
2803  int i;
2804 
2805  switch (cmd) {
2806  case TEST_INIT:
2807  info->name = "sfu_remove_destination_streams";
2808  info->category = "/bridges/bridge_softmix/";
2809  info->summary = "Test removal of destination video streams";
2810  info->description =
2811  "This tests does stuff.";
2812  return AST_TEST_NOT_RUN;
2813  case TEST_EXECUTE:
2814  break;
2815  }
2816 
2817  orig = build_topology(params, ARRAY_LEN(params));
2818  if (!orig) {
2819  ast_test_status_update(test, "Unable to build initial stream topology\n");
2820  goto end;
2821  }
2822 
2823  for (i = 0; i < ARRAY_LEN(removal_results); ++i) {
2824  int j;
2825 
2826  remove_destination_streams(orig, removal_results[i].channel_name);
2827 
2828  if (ast_stream_topology_get_count(orig) != removal_results[i].num_streams) {
2829  ast_test_status_update(test, "Resulting topology has %d streams, when %d are expected\n",
2830  ast_stream_topology_get_count(orig), removal_results[i].num_streams);
2831  goto end;
2832  }
2833 
2834  for (j = 0; j < removal_results[i].num_streams; ++j) {
2835  struct ast_stream *actual;
2836  struct ast_stream *expected;
2837  int orig_index;
2838 
2839  actual = ast_stream_topology_get_stream(orig, j);
2840 
2841  orig_index = removal_results[i].params_index[j];
2842  expected = ast_stream_topology_get_stream(orig, orig_index);
2843 
2845  ast_stream_get_formats(expected))) {
2846  struct ast_str *expected_str;
2847  struct ast_str *actual_str;
2848 
2849  expected_str = ast_str_alloca(64);
2850  actual_str = ast_str_alloca(64);
2851 
2852  ast_test_status_update(test, "Mismatch between expected (%s) and actual (%s) stream formats\n",
2853  ast_format_cap_get_names(ast_stream_get_formats(expected), &expected_str),
2854  ast_format_cap_get_names(ast_stream_get_formats(actual), &actual_str));
2855  goto end;
2856  }
2857 
2858  if (is_video_dest(actual, removal_results[i].channel_name, -1) &&
2860  ast_test_status_update(test, "Removed stream %s does not have a state of removed\n", ast_stream_get_name(actual));
2861  goto end;
2862  }
2863  }
2864  }
2865 
2866  res = AST_TEST_PASS;
2867 
2868 end:
2870  return res;
2871 }
2872 
2873 #endif
2874 
2875 static int unload_module(void)
2876 {
2878  AST_TEST_UNREGISTER(sfu_append_source_streams);
2879  AST_TEST_UNREGISTER(sfu_remove_destination_streams);
2880  return 0;
2881 }
2882 
2883 static int load_module(void)
2884 {
2886  unload_module();
2887  return AST_MODULE_LOAD_DECLINE;
2888  }
2889  AST_TEST_REGISTER(sfu_append_source_streams);
2890  AST_TEST_REGISTER(sfu_remove_destination_streams);
2891  return AST_MODULE_LOAD_SUCCESS;
2892 }
2893 
2894 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");
pthread_t thread
Definition: app_meetme.c:1091
Asterisk main include file. File version handling, generic pbx functions.
#define ast_alloca(size)
call __builtin_alloca to ensure we get gcc builtin semantics
Definition: astmm.h:290
#define ast_free(a)
Definition: astmm.h:182
#define ast_realloc(p, len)
A wrapper for realloc()
Definition: astmm.h:228
#define ast_asprintf(ret, fmt,...)
A wrapper for asprintf()
Definition: astmm.h:269
#define ast_calloc(num, len)
A wrapper for calloc()
Definition: astmm.h:204
#define ast_log
Definition: astobj2.c:42
@ AO2_ALLOC_OPT_LOCK_NOLOCK
Definition: astobj2.h:369
#define ao2_t_replace(dst, src, tag)
Replace one object reference with another cleaning up the original.
Definition: astobj2.h:503
#define ao2_cleanup(obj)
Definition: astobj2.h:1958
#define ao2_replace(dst, src)
Definition: astobj2.h:517
#define ao2_ref(o, delta)
Definition: astobj2.h:464
#define ao2_alloc_options(data_size, destructor_fn, options)
Definition: astobj2.h:406
#define ao2_bump(obj)
Definition: astobj2.h:491
#define ast_bridge_unlock(bridge)
Unlock the bridge.
Definition: bridge.h:493
@ AST_BRIDGE_VIDEO_SFU_REMB_LOWEST
Definition: bridge.h:135
@ AST_BRIDGE_VIDEO_SFU_REMB_HIGHEST_ALL
Definition: bridge.h:143
@ AST_BRIDGE_VIDEO_SFU_REMB_AVERAGE
Definition: bridge.h:133
@ AST_BRIDGE_VIDEO_SFU_REMB_HIGHEST
Definition: bridge.h:137
@ AST_BRIDGE_VIDEO_SFU_REMB_AVERAGE_ALL
Definition: bridge.h:139
@ AST_BRIDGE_VIDEO_SFU_REMB_FORCE
Definition: bridge.h:145
@ AST_BRIDGE_VIDEO_SFU_REMB_LOWEST_ALL
Definition: bridge.h:141
int ast_bridge_number_video_src(struct ast_bridge *bridge)
Returns the number of video sources currently active in the bridge.
Definition: bridge.c:3931
@ AST_BRIDGE_CAPABILITY_MULTIMIX
Definition: bridge.h:94
@ AST_BRIDGE_VIDEO_MODE_SINGLE_SRC
Definition: bridge.h:102
@ AST_BRIDGE_VIDEO_MODE_TALKER_SRC
Definition: bridge.h:105
@ AST_BRIDGE_VIDEO_MODE_NONE
Definition: bridge.h:100
@ AST_BRIDGE_VIDEO_MODE_SFU
Definition: bridge.h:109
#define ast_bridge_lock(bridge)
Lock the bridge.
Definition: bridge.h:480
void ast_bridge_update_talker_src_video_mode(struct ast_bridge *bridge, struct ast_channel *chan, int talker_energy, int is_keyframe)
Update information about talker energy for talker src video mode.
Definition: bridge.c:3883
int ast_bridge_is_video_src(struct ast_bridge *bridge, struct ast_channel *chan)
Determine if a channel is a video src for the bridge.
Definition: bridge.c:3958
int ast_bridge_queue_everyone_else(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
Queue the given frame to everyone else.
#define ast_bridge_channel_unlock(bridge_channel)
Unlock the bridge_channel.
int ast_bridge_channel_notify_talking(struct ast_bridge_channel *bridge_channel, int started_talking)
Lets the bridging indicate when a bridge channel has stopped or started talking.
void ast_bridge_channel_stream_map(struct ast_bridge_channel *bridge_channel)
Maps a channel's stream topology to and from the bridge.
#define ast_bridge_channel_lock(bridge_channel)
Lock the bridge_channel.
@ BRIDGE_CHANNEL_STATE_END
int ast_bridge_channel_queue_frame(struct ast_bridge_channel *bridge_channel, struct ast_frame *fr)
Write a frame to the specified bridge_channel.
void ast_bridge_channel_leave_bridge(struct ast_bridge_channel *bridge_channel, enum bridge_channel_state new_state, int cause)
Set bridge channel state to leave bridge (if not leaving already).
static void remb_collect_report_all(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data, float bitrate)
static void gather_softmix_stats(struct softmix_stats *stats, const struct softmix_bridge_data *softmix_data, struct ast_bridge_channel *bridge_channel)
static void * softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
static void remb_collect_report(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct softmix_bridge_data *softmix_data, struct softmix_channel *sc)
#define SOFTMIX_MIN_SAMPLE_RATE
static int append_all_streams(struct ast_stream_topology *dest, const struct ast_stream_topology *source)
static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper, struct ast_format *raw_write_fmt, struct softmix_channel *sc, unsigned int default_sample_size)
static struct ast_bridge_technology softmix_bridge
static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
static int append_source_stream(struct ast_stream_topology *dest, const char *channel_name, const char *sdp_label, struct ast_stream *stream, int index)
static void remb_enable_collection(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, size_t bridge_stream_position)
Setup REMB collection for a particular bridge stream and channel.
static int is_video_source(const struct ast_stream *stream)
Determine if a stream is a video source stream.
static void remb_send_report(struct ast_bridge_channel *bridge_channel, struct softmix_bridge_data *softmix_data, struct softmix_channel *sc)
static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
static struct softmix_translate_helper_entry * softmix_translate_helper_entry_alloc(struct ast_format *dst)
static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
static unsigned int analyse_softmix_stats(struct softmix_stats *stats, struct softmix_bridge_data *softmix_data, int binaural_active)
static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array, unsigned int binaural_active)
static int remove_destination_streams(struct ast_stream_topology *topology, const char *channel_name)
#define SOFTMIX_DATALEN(rate, interval)
Size of the buffer used for sample manipulation.
#define SOFTMIX_SAMPLES(rate, interval)
Number of samples we are dealing with.
static struct softmix_remb_collector * remb_collector_alloc(void)
Allocate a REMB collector.
static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
Function called when a channel is joined into the bridge.
static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array, unsigned int starting_num_entries, unsigned int binaural_active)
static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name, int source_channel_stream_position)
Determine if a stream is a video destination stream.
static int append_source_streams(struct ast_stream_topology *dest, const char *channel_name, const char *sdp_label, const struct ast_stream_topology *source)
static int remove_all_original_streams(struct ast_stream_topology *dest, const struct ast_stream_topology *source, const struct ast_stream_topology *original)
static void softmix_bridge_check_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
Function called when a channel leaves the bridge.
static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
static void softmix_bridge_stream_sources_update(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct softmix_channel *sc)
static void softmix_bridge_stop(struct ast_bridge *bridge)
#define SOFTMIX_STAT_INTERVAL
Number of mixing iterations to perform between gathering statistics.
static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
Function called when a channel is unsuspended from the bridge.
static int16_t * softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
#define DEFAULT_SOFTMIX_TALKING_THRESHOLD
static void softmix_bridge_write_rtcp(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
static void sfu_topologies_on_source_change(struct ast_bridge *bridge, struct ast_bridge_channel *source)
#define DEFAULT_SOFTMIX_SILENCE_THRESHOLD
Default time in ms of silence necessary to declare talking stopped by the bridge.
static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
#define SOFTBRIDGE_VIDEO_DEST_LEN
static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array, unsigned int num_entries, unsigned int binaural_active)
#define SOFTBRIDGE_VIDEO_DEST_PREFIX
static void sfu_topologies_on_join(struct ast_bridge *bridge, struct ast_bridge_channel *joiner)
Issue channel stream topology change requests.
static void softmix_bridge_destroy(struct ast_bridge *bridge)
Function called when a bridge is destroyed.
static int load_module(void)
#define SOFTBRIDGE_VIDEO_DEST_SEPARATOR
static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
stream_topology_changed callback
static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
static int softmix_mixing_loop(struct ast_bridge *bridge)
Mixing loop.
static int unload_module(void)
#define DEFAULT_SOFTMIX_INTERVAL
Interval at which mixing will take place. Valid options are 10, 20, and 40.
static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
static int softmix_bridge_create(struct ast_bridge *bridge)
Function called when a bridge is created.
static void clear_talking(struct ast_bridge_channel *bridge_channel)
static void map_source_to_destinations(const char *source_channel_name, size_t bridge_stream_position, struct ast_bridge_channels_list *participants, int source_channel_stream_position)
Map a source stream to all of its destination streams.
static void * softmix_mixing_thread(void *data)
static void softmix_bridge_write_text(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
int set_binaural_data_join(struct convolve_data *data, unsigned int default_sample_size)
Joins a channel into a virtual enviroment build with the help of binaural synthesis.
void create_binaural_frame(struct ast_bridge_channel *bridge_channel, struct softmix_channel *sc, int16_t *bin_buf, int16_t *ann_buf, unsigned int softmix_datalen, unsigned int softmix_samples, int16_t *buf)
Creates a frame out of binaural audio data.
void softmix_process_write_binaural_audio(struct softmix_channel *sc, unsigned int default_sample_size)
Writes the binaural audio to a channel.
void binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data, struct softmix_mixing_array *mixing_array, int16_t *bin_buf, int16_t *ann_buf)
Mixes all binaural audio data contained in the mixing array.
void add_binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data, unsigned int softmix_samples, struct softmix_mixing_array *mixing_array, struct softmix_channel *sc, const char *channel_name)
Processes audio data with the binaural synthesis and adds the result to the mixing array.
int init_convolve_data(struct convolve_data *data, unsigned int default_sample_size)
Preinits a specific number of channels (CONVOLVE_CHANNEL_PREALLOC) at the beginning of a conference.
void check_binaural_position_change(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data)
Checks if a position change in the virtual enviroment is requested by one of the participants.
void free_convolve_data(struct convolve_data *data)
Frees all channels and data needed for binaural audio processing.
void set_binaural_data_leave(struct convolve_data *data, unsigned int pos, unsigned int default_sample_size)
Removes a channel from the binaural conference bridge. Marks the position in the virtual room as unus...
Multi-party software based channel mixing (header)
#define BINAURAL_MIXING_INTERVAL
#define SOFTMIX_BINAURAL_SAMPLE_RATE
#define DEFAULT_ENERGY_HISTORY_LEN
#define MAX_DATALEN
@ AST_BRIDGE_PREFERENCE_BASE_MULTIMIX
#define ast_bridge_technology_register(technology)
See __ast_bridge_technology_register()
int ast_bridge_technology_unregister(struct ast_bridge_technology *technology)
Unregister a bridge technology from use.
Definition: bridge.c:265
static int tmp()
Definition: bt_open.c:389
static struct ast_timer * timer
Definition: chan_iax2.c:357
static const char type[]
Definition: chan_ooh323.c:109
int ast_channel_request_stream_topology_change(struct ast_channel *chan, struct ast_stream_topology *topology, void *change_source)
Request that the stream topology of a channel change.
Definition: channel.c:11069
int ast_set_read_format_path(struct ast_channel *chan, struct ast_format *raw_format, struct ast_format *core_format)
Set specific read path on channel.
Definition: channel.c:5484
const char * ast_channel_uniqueid(const struct ast_channel *chan)
#define ast_channel_lock(chan)
Definition: channel.h:2888
int ast_waitfor_n_fd(int *fds, int n, int *ms, int *exception)
Waits for input on an fd.
Definition: channel.c:2977
int ast_set_write_format_interleaved_stereo(struct ast_channel *chan, struct ast_format *format)
Sets write format for a channel. All internal data will than be handled in an interleaved format....
Definition: channel.c:5781
const char * ast_channel_name(const struct ast_channel *chan)
struct ast_format * ast_channel_rawreadformat(struct ast_channel *chan)
int ast_set_write_format(struct ast_channel *chan, struct ast_format *format)
Sets write format on channel chan.
Definition: channel.c:5799
int ast_channel_hold_state(const struct ast_channel *chan)
struct ast_format * ast_channel_rawwriteformat(struct ast_channel *chan)
struct ast_stream_topology * ast_channel_get_stream_topology(const struct ast_channel *chan)
Retrieve the topology of streams on a channel.
int ast_indicate(struct ast_channel *chan, int condition)
Indicates condition of channel.
Definition: channel.c:4301
#define ast_channel_unlock(chan)
Definition: channel.h:2889
ast_media_type
Types of media.
Definition: codec.h:30
@ AST_MEDIA_TYPE_AUDIO
Definition: codec.h:32
@ AST_MEDIA_TYPE_VIDEO
Definition: codec.h:33
@ AST_MEDIA_TYPE_END
Definition: codec.h:36
const char * ast_codec_media_type2str(enum ast_media_type type)
Conversion function to take a media type and turn it into a string.
Definition: codec.c:347
frame_type
Definition: codec_builtin.c:44
short int16_t
Definition: db.h:59
void ast_dsp_set_threshold(struct ast_dsp *dsp, int threshold)
Set the minimum average magnitude threshold to determine talking by the DSP.
Definition: dsp.c:1775
void ast_dsp_free(struct ast_dsp *dsp)
Definition: dsp.c:1770
int ast_dsp_silence_with_energy(struct ast_dsp *dsp, struct ast_frame *f, int *totalsilence, int *frames_energy)
Process the audio frame for silence.
Definition: dsp.c:1478
struct ast_dsp * ast_dsp_new_with_rate(unsigned int sample_rate)
Allocates a new dsp with a specific internal sample rate used during processing.
Definition: dsp.c:1750
char * end
Definition: eagi_proxy.c:73
char buf[BUFSIZE]
Definition: eagi_proxy.c:66
static struct formats formats
const char * ast_format_get_name(const struct ast_format *format)
Get the name associated with a format.
Definition: format.c:334
unsigned int ast_format_get_sample_rate(const struct ast_format *format)
Get the sample rate of a media format.
Definition: format.c:379
enum ast_format_cmp_res ast_format_cmp(const struct ast_format *format1, const struct ast_format *format2)
Compare two formats.
Definition: format.c:201
@ AST_FORMAT_CMP_EQUAL
Definition: format.h:36
unsigned int ast_format_get_channel_count(const struct ast_format *format)
Get the channel count on a format.
Definition: format.c:135
struct ast_format * ast_format_cache_get_slin_by_rate(unsigned int rate)
Retrieve the best signed linear format given a sample rate.
Definition: format_cache.c:512
int ast_format_cap_update_by_allow_disallow(struct ast_format_cap *cap, const char *list, int allowing)
Parse an "allow" or "deny" list and modify a format capabilities structure accordingly.
Definition: format_cap.c:320
const char * ast_format_cap_get_names(const struct ast_format_cap *cap, struct ast_str **buf)
Get the names of codecs of a set of formats.
Definition: format_cap.c:736
@ AST_FORMAT_CAP_FLAG_DEFAULT
Definition: format_cap.h:38
int ast_format_cap_identical(const struct ast_format_cap *cap1, const struct ast_format_cap *cap2)
Determine if two capabilities structures are identical.
Definition: format_cap.c:689
#define ast_format_cap_alloc(flags)
Definition: format_cap.h:52
static const char name[]
Definition: format_mp3.c:68
unsigned int fmt
Definition: rtp_engine.h:359
struct ast_rtp_rtcp_feedback_remb remb
Definition: rtp_engine.h:361
const char * ast_msg_data_get_attribute(struct ast_msg_data *msg, enum ast_msg_data_attribute_type attribute_type)
Get attribute from ast_msg_data.
Definition: message.c:1560
@ AST_MSG_DATA_ATTR_BODY
Definition: message.h:458
@ AST_MSG_DATA_ATTR_FROM
Definition: message.h:456
#define ast_frfree(fr)
char * ast_frame_type2str(enum ast_frame_type frame_type, char *ftype, size_t len)
Copy the discription of a frame type into the provided string.
Definition: main/frame.c:671
@ AST_FRAME_VIDEO
@ AST_FRAME_NULL
@ AST_FRAME_DTMF_END
@ AST_FRAME_DTMF_BEGIN
@ AST_FRAME_BRIDGE_ACTION_SYNC
@ AST_FRAME_VOICE
@ AST_FRAME_RTCP
@ AST_FRAME_TEXT_DATA
@ AST_FRAME_CONTROL
@ AST_FRAME_BRIDGE_ACTION
@ AST_FRAME_TEXT
@ AST_CONTROL_STREAM_TOPOLOGY_SOURCE_CHANGED
@ AST_CONTROL_UNHOLD
@ AST_CONTROL_VIDUPDATE
@ AST_CONTROL_HOLD
#define AST_LIST_TRAVERSE(head, var, field)
Loops over (traverses) the entries in a list.
Definition: linkedlists.h:491
#define AST_LIST_ENTRY(type)
Declare a forward link structure inside a list entry.
Definition: linkedlists.h:410
#define AST_LIST_TRAVERSE_SAFE_END
Closes a safe loop traversal block.
Definition: linkedlists.h:615
#define AST_LIST_INSERT_HEAD(head, elm, field)
Inserts a list entry at the head of a list.
Definition: linkedlists.h:711
#define AST_LIST_TRAVERSE_SAFE_BEGIN(head, var, field)
Loops safely over (traverses) the entries in a list.
Definition: linkedlists.h:529
#define AST_LIST_REMOVE_CURRENT(field)
Removes the current entry from a list during a traversal.
Definition: linkedlists.h:557
#define AST_LIST_REMOVE_HEAD(head, field)
Removes and returns the head entry from a list.
Definition: linkedlists.h:833
#define ast_cond_destroy(cond)
Definition: lock.h:200
#define ast_cond_wait(cond, mutex)
Definition: lock.h:203
#define AST_PTHREADT_NULL
Definition: lock.h:66
#define ast_cond_init(cond, attr)
Definition: lock.h:199
#define ast_mutex_init(pmutex)
Definition: lock.h:184
#define ast_mutex_unlock(a)
Definition: lock.h:188
#define ast_mutex_destroy(a)
Definition: lock.h:186
#define ast_mutex_lock(a)
Definition: lock.h:187
#define ast_cond_signal(cond)
Definition: lock.h:201
#define AST_LOG_WARNING
Definition: logger.h:280
#define DEBUG_ATLEAST(level)
Definition: logger.h:442
#define ast_debug(level,...)
Log a DEBUG message.
Definition: logger.h:453
#define SCOPE_EXIT_LOG(__log_level,...)
Definition: logger.h:930
#define SCOPE_EXIT_RTN(...)
Definition: logger.h:915
#define SCOPE_EXIT_RTN_VALUE(__return_value,...)
Definition: logger.h:919
int ast_callid_threadassoc_add(ast_callid callid)
Adds a known callid to thread storage of the calling thread.
Definition: logger.c:2067
#define SCOPE_ENTER(level,...)
Definition: logger.h:900
#define LOG_DEBUG
Definition: logger.h:242
#define SCOPE_EXIT_EXPR(__expr,...)
Definition: logger.h:911
#define SCOPE_EXIT(...)
Definition: logger.h:908
#define SCOPE_EXIT_LOG_RTN(__log_level,...)
Definition: logger.h:936
#define ast_trace(level,...)
Definition: logger.h:891
#define LOG_ERROR
Definition: logger.h:286
#define LOG_NOTICE
Definition: logger.h:264
#define LOG_WARNING
Definition: logger.h:275
Out-of-call text message support.
#define AST_MODULE_INFO_STANDARD(keystr, desc)
Definition: module.h:567
#define ASTERISK_GPL_KEY
The text the key() function should return.
Definition: module.h:46
@ AST_MODULE_LOAD_SUCCESS
Definition: module.h:70
@ AST_MODULE_LOAD_DECLINE
Module has failed to load, may be in an inconsistent state.
Definition: module.h:78
def info(msg)
static void * cleanup(void *unused)
Definition: pbx_realtime.c:124
AST_LIST_HEAD_NOLOCK(contactliststruct, contact)
#define NULL
Definition: resample.c:96
#define AST_RTP_RTCP_PSFB
Definition: rtp_engine.h:299
#define AST_RTP_RTCP_FMT_REMB
Definition: rtp_engine.h:309
int ast_slinfactory_init_with_format(struct ast_slinfactory *sf, struct ast_format *slin_out)
Initialize a slinfactory.
Definition: slinfactory.c:46
unsigned int ast_slinfactory_available(const struct ast_slinfactory *sf)
Retrieve number of samples currently in a slinfactory.
Definition: slinfactory.c:199
int ast_slinfactory_read(struct ast_slinfactory *sf, short *buf, size_t samples)
Read samples from a slinfactory.
Definition: slinfactory.c:145
void ast_slinfactory_flush(struct ast_slinfactory *sf)
Flush the contents of a slinfactory.
Definition: slinfactory.c:204
int ast_slinfactory_feed(struct ast_slinfactory *sf, struct ast_frame *f)
Feed audio into a slinfactory.
Definition: slinfactory.c:77
void ast_slinfactory_destroy(struct ast_slinfactory *sf)
Destroy the contents of a slinfactory.
Definition: slinfactory.c:58
Media Stream API.
struct ast_stream_topology * ast_stream_topology_clone(const struct ast_stream_topology *topology)
Create a deep clone of an existing stream topology.
Definition: stream.c:667
int ast_stream_topology_set_stream(struct ast_stream_topology *topology, unsigned int position, struct ast_stream *stream)
Set a specific position in a topology.
Definition: stream.c:796
int ast_stream_set_metadata(struct ast_stream *stream, const char *m_key, const char *value)
Set a stream metadata value.
Definition: stream.c:460
@ AST_STREAM_STATE_RECVONLY
Set when the stream is receiving media only.
Definition: stream.h:90
@ AST_STREAM_STATE_INACTIVE
Set when the stream is not sending OR receiving media.
Definition: stream.h:94
@ AST_STREAM_STATE_REMOVED
Set when the stream has been removed/declined.
Definition: stream.h:78
@ AST_STREAM_STATE_SENDRECV
Set when the stream is sending and receiving media.
Definition: stream.h:82
@ AST_STREAM_STATE_SENDONLY
Set when the stream is sending media only.
Definition: stream.h:86
const char * ast_stream_topology_to_str(const struct ast_stream_topology *topology, struct ast_str **buf)
Get a string representing the topology for debugging/display purposes.
Definition: stream.c:936
void ast_stream_set_state(struct ast_stream *stream, enum ast_stream_state state)
Set the state of a stream.
Definition: stream.c:380
int ast_stream_topology_append_stream(struct ast_stream_topology *topology, struct ast_stream *stream)
Append a stream to the topology.
Definition: stream.c:748
int ast_stream_topology_get_count(const struct ast_stream_topology *topology)
Get the number of streams in a topology.
Definition: stream.c:765
enum ast_stream_state ast_stream_get_state(const struct ast_stream *stream)
Get the current state of a stream.
Definition: stream.c:373
enum ast_media_type ast_stream_get_type(const struct ast_stream *stream)
Get the media type of a stream.
Definition: stream.c:316
void ast_stream_topology_free(struct ast_stream_topology *topology)
Unreference and destroy a stream topology.
Definition: stream.c:743
void ast_stream_set_formats(struct ast_stream *stream, struct ast_format_cap *caps)
Set the current negotiated formats of a stream.
Definition: stream.c:365
struct ast_stream * ast_stream_clone(const struct ast_stream *stream, const char *name)
Create a deep clone of an existing stream.
Definition: stream.c:257
struct ast_stream * ast_stream_topology_get_stream(const struct ast_stream_topology *topology, unsigned int position)
Get a specific stream from the topology.
Definition: stream.c:788
void ast_stream_free(struct ast_stream *stream)
Destroy a media stream representation.
Definition: stream.c:292
struct ast_stream_topology * ast_stream_topology_alloc(void)
Create a stream topology.
Definition: stream.c:650
const struct ast_format_cap * ast_stream_get_formats(const struct ast_stream *stream)
Get the current negotiated formats of a stream.
Definition: stream.c:330
const char * ast_stream_state2str(enum ast_stream_state state)
Convert the state of a stream into a string.
Definition: stream.c:388
const char * ast_stream_to_str(const struct ast_stream *stream, struct ast_str **buf)
Get a string representing the stream for debugging/display purposes.
Definition: stream.c:337
const char * ast_stream_get_name(const struct ast_stream *stream)
Get the name of a stream.
Definition: stream.c:309
struct ast_stream * ast_stream_alloc(const char *name, enum ast_media_type type)
Create a new media stream representation.
Definition: stream.c:233
static force_inline int attribute_pure ast_strlen_zero(const char *s)
Definition: strings.h:65
#define ast_str_tmp(init_len, __expr)
Provides a temporary ast_str and returns a copy of its buffer.
Definition: strings.h:1154
#define ast_str_alloca(init_len)
Definition: strings.h:813
Structure that contains information regarding a channel in a bridge.
struct ast_vector_int to_bridge
void * tech_pvt
Private information unique to the bridge technology.
unsigned int suspended
struct ast_format * write_format
struct ast_bridge * bridge
Bridge this channel is participating in.
struct ast_bridge_features * features
struct ast_channel * chan
struct ast_vector_int to_channel
struct ast_bridge_channel::@223 stream_map
struct ast_bridge_tech_optimizations tech_args
unsigned int send_sdp_label
Definition: bridge.h:308
unsigned int maximum_sample_rate
The maximum sample rate softmix uses to mix channels.
Definition: bridge.h:314
unsigned int internal_sample_rate
The internal sample rate softmix uses to mix channels.
Definition: bridge.h:293
unsigned int binaural_active
Definition: bridge.h:303
struct ast_bridge_video_mode video_mode
Definition: bridge.h:287
unsigned int internal_mixing_interval
The mixing interval indicates how quickly softmix mixing should occur to mix audio.
Definition: bridge.h:301
Structure that is the essence of a bridge technology.
unsigned int video_update_discard
Definition: bridge.h:168
union ast_bridge_video_mode::@214 mode_data
enum ast_bridge_video_mode_type mode
Definition: bridge.h:160
struct ast_bridge_video_sfu_data sfu_data
Definition: bridge.h:165
unsigned int remb_send_interval
Definition: bridge.h:151
enum ast_bridge_video_sfu_remb_behavior remb_behavior
Definition: bridge.h:153
Structure that contains information about a bridge.
Definition: bridge.h:357
struct ast_bridge_softmix softmix
Definition: bridge.h:375
void * tech_pvt
Definition: bridge.h:365
unsigned int num_active
Definition: bridge.h:383
ast_callid callid
Definition: bridge.h:369
const ast_string_field uniqueid
Definition: bridge.h:409
struct ast_bridge_channels_list channels
Definition: bridge.h:371
unsigned int num_channels
Definition: bridge.h:381
Format capabilities structure, holds formats + preference order + etc.
Definition: format_cap.c:54
Definition of a media format.
Definition: format.c:43
struct ast_format * format
Data structure associated with a single frame of data.
union ast_frame::@250 data
struct ast_frame_subclass subclass
enum ast_frame_type frametype
Structure used to transport a message through the frame core.
Definition: message.c:1433
An object that represents data received in a feedback report.
Definition: rtp_engine.h:358
The descriptor of a dynamic string XXX storage will be optimized later if needed We use the ts field ...
Definition: strings.h:597
Default structure for translators, with the basic fields and buffers, all allocated as part of the sa...
Definition: translate.h:213
Definition: search.h:40
char * data
Definition: search.h:42
Definition: file.c:69
struct convolve_data convolve
struct ast_bridge * bridge
Bridge pointer passed to the softmix mixing thread.
Structure which contains per-channel mixing information.
unsigned int talking
TRUE if a channel is talking.
struct ast_stream_topology * topology
struct softmix_remb_collector * remb_collector
short final_buf[MAX_DATALEN]
short our_buf[MAX_DATALEN]
struct ast_format * read_slin_format
struct ast_slinfactory factory
struct video_follow_talker_data video_talker
struct ast_frame write_frame
struct ast_rtp_rtcp_feedback_remb remb
struct convolve_channel_pair ** chan_pairs
struct ast_rtp_rtcp_feedback feedback
struct ast_frame frame
unsigned int sample_rates[16]
unsigned int num_above_internal_rate
unsigned int num_channels[16]
unsigned int highest_supported_rate
unsigned int num_above_maximum_rate
unsigned int locked_rate
unsigned int num_at_internal_rate
unsigned int maximum_rate
int num_times_requested
struct ast_trans_pvt * trans_pvt
struct ast_format * dst_format
struct ast_frame * out_frame
struct softmix_translate_helper_entry::@89 entry
struct ast_format * slin_src
struct softmix_translate_helper::@90 entries
const char * name
Definition: test_logger.c:44
int energy_history[DEFAULT_ENERGY_HISTORY_LEN]
Test Framework API.
@ TEST_INIT
Definition: test.h:207
@ TEST_EXECUTE
Definition: test.h:208
#define AST_TEST_REGISTER(cb)
Definition: test.h:127
#define ast_test_status_update(a, b, c...)
Definition: test.h:129
#define AST_TEST_UNREGISTER(cb)
Definition: test.h:128
#define AST_TEST_DEFINE(hdr)
Definition: test.h:126
ast_test_result_state
Definition: test.h:200
@ AST_TEST_PASS
Definition: test.h:202
@ AST_TEST_FAIL
Definition: test.h:203
@ AST_TEST_NOT_RUN
Definition: test.h:201
int64_t ast_tvdiff_ms(struct timeval end, struct timeval start)
Computes the difference (in milliseconds) between two struct timeval instances.
Definition: time.h:98
struct timeval ast_tvnow(void)
Returns current timeval. Meant to replace calls to gettimeofday().
Definition: time.h:150
void ast_timer_close(struct ast_timer *handle)
Close an opened timing handle.
Definition: timing.c:154
int ast_timer_ack(const struct ast_timer *handle, unsigned int quantity)
Acknowledge a timer event.
Definition: timing.c:171
int ast_timer_set_rate(const struct ast_timer *handle, unsigned int rate)
Set the timing tick rate.
Definition: timing.c:166
int ast_timer_fd(const struct ast_timer *handle)
Get a poll()-able file descriptor for a timer.
Definition: timing.c:161
struct ast_timer * ast_timer_open(void)
Open a timer.
Definition: timing.c:122
struct ast_trans_pvt * ast_translator_build_path(struct ast_format *dest, struct ast_format *source)
Builds a translator path Build a path (possibly NULL) from source to dest.
Definition: translate.c:485
void ast_translator_free_path(struct ast_trans_pvt *tr)
Frees a translator path Frees the given translator path structure.
Definition: translate.c:475
struct ast_frame * ast_translate(struct ast_trans_pvt *tr, struct ast_frame *f, int consume)
translates one or more frames Apply an input frame into the translator and receive zero or one output...
Definition: translate.c:565
#define RAII_VAR(vartype, varname, initval, dtor)
Declare a variable that will call a destructor function when it goes out of scope.
Definition: utils.h:911
static force_inline void ast_slinear_saturated_subtract(short *input, short *value)
Definition: utils.h:438
#define ast_assert(a)
Definition: utils.h:710
#define MIN(a, b)
Definition: utils.h:226
#define ast_pthread_create(a, b, c, d)
Definition: utils.h:559
static force_inline void ast_slinear_saturated_add(short *input, short *value)
Definition: utils.h:425
#define ARRAY_LEN(a)
Definition: utils.h:639
#define MAX(a, b)
Definition: utils.h:228
Vector container support.
#define AST_VECTOR_REPLACE(vec, idx, elem)
Replace an element at a specific position in a vector, growing the vector if needed.
Definition: vector.h:284
#define AST_VECTOR_RESET(vec, cleanup)
Reset vector.
Definition: vector.h:627
#define AST_VECTOR_ELEM_CLEANUP_NOOP(elem)
Vector element cleanup that does nothing.
Definition: vector.h:573
#define AST_VECTOR_SIZE(vec)
Get the number of elements in a vector.
Definition: vector.h:611
#define AST_VECTOR_GET_INDEX_NTH(vec, nth, value, cmp)
Get the nth index from a vector that matches the given comparison.
Definition: vector.h:698
#define AST_VECTOR_FREE(vec)
Deallocates this vector.
Definition: vector.h:174
#define AST_VECTOR_ELEM_DEFAULT_CMP(elem, value)
Default comparator for AST_VECTOR_REMOVE_ELEM_UNORDERED()
Definition: vector.h:564
#define AST_VECTOR_INIT(vec, size)
Initialize a vector.
Definition: vector.h:113
#define AST_VECTOR_APPEND(vec, elem)
Append an element to a vector, growing the vector if needed.
Definition: vector.h:256
#define AST_VECTOR_GET(vec, idx)
Get an element from a vector.
Definition: vector.h:682