Asterisk - The Open Source Telephony Project  GIT-master-b91fb3c
bridge_softmix_internal.h
Go to the documentation of this file.
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19 
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing (header)
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29 
30 #ifndef _ASTERISK_BRIDGE_SOFTMIX_INTERNAL_H
31 #define _ASTERISK_BRIDGE_SOFTMIX_INTERNAL_H
32 
33 #include "asterisk.h"
34 #include <stdio.h>
35 #include <stdlib.h>
36 #include <string.h>
37 #include <sys/time.h>
38 #include <signal.h>
39 #include <errno.h>
40 #include <unistd.h>
41 
42 #include "asterisk/module.h"
43 #include "asterisk/channel.h"
44 #include "asterisk/bridge.h"
46 #include "asterisk/frame.h"
47 #include "asterisk/options.h"
48 #include "asterisk/logger.h"
49 #include "asterisk/slinfactory.h"
50 #include "asterisk/astobj2.h"
51 #include "asterisk/timing.h"
52 #include "asterisk/translate.h"
53 #include "asterisk/rtp_engine.h"
54 #include "asterisk/vector.h"
55 
56 #ifdef BINAURAL_RENDERING
57 #include <fftw3.h>
58 #endif
59 
60 #if defined(__Darwin__) || defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__DragonFly__) || defined(__NetBSD__) || defined(__CYGWIN__)
61 #include <float.h>
62 #else
63 #include <values.h>
64 #endif
65 
66 #define MAX_DATALEN 8096
67 #define DEFAULT_ENERGY_HISTORY_LEN 150
68 
69 /*! Setting the sample rate to 48000 by default if binaural is activated. */
70 #define SOFTMIX_BINAURAL_SAMPLE_RATE 48000
71 /*! We only support 20 ms interval length with binaural data at the moment. */
72 #define BINAURAL_MIXING_INTERVAL 20
73 
75  /*! The head related transfer function used for convolving */
76  double *hrtf;
77  /*! Input signals for fftw */
78  double *fftw_in;
79  /*! Output signals from the fftw */
80  double *fftw_out;
81  /*! Signals for overlap add */
82  float *overlap_add;
83  /*! The resulting data after the convolution */
85 #ifdef BINAURAL_RENDERING
86  /*! The fftw plan for binaural signaling */
87  fftw_plan fftw_plan;
88  /*! The inverse fftw plan for binaural signaling */
89  fftw_plan fftw_plan_inverse;
90 #endif
91 };
92 
94  /*! The left channel of a stereo channel pair */
95  struct convolve_channel chan_left;
96  /*! The right channel of a stereo channel pair */
97  struct convolve_channel chan_right;
98 };
99 
101  /*! A count of all channels potentialy having input data for the conference. */
103  /*! Will set to true if there is at least one binaural output.
104  * Only if set to true data will be convolved. */
106  /*! The length of the head related transfer function */
107  unsigned int hrtf_length;
108  /*! Number of channels available for convolving.
109  * We do not delete a channel when a member leaves, cause we can reuse it for the next one. */
111  /*! The positions of the single channels in the virtual room */
112  int *pos_ids;
113  /*! Each channel has a stereo pair of channels for the convolution */
115 };
116 
118  /*! audio energy history */
119  int energy_history[DEFAULT_ENERGY_HISTORY_LEN];
120  /*! The current slot being used in the history buffer, this
121  * increments and wraps around */
123  /*! The current energy sum used for averages. */
125  /*! The current energy average */
127 };
128 
130 
131 /*! \brief Structure which contains per-channel mixing information */
133  /*! Lock to protect this structure */
135  /*! Factory which contains audio read in from the channel */
136  struct ast_slinfactory factory;
137  /*! Frame that contains mixed audio to be written out to the channel */
138  struct ast_frame write_frame;
139  /*! Current expected read slinear format. */
141  /*! DSP for detecting silence */
142  struct ast_dsp *dsp;
143  /*!
144  * \brief TRUE if a channel is talking.
145  *
146  * \note This affects how the channel's audio is mixed back to
147  * it.
148  */
149  unsigned int talking:1;
150  /*! TRUE if the channel provided audio for this mixing interval */
151  unsigned int have_audio:1;
152  /*! We set binaural also as channel data, to have better tracking.
153  * It is also present in transpvt.
154  */
155  unsigned int binaural:1;
156  /*! TRUE if this is an announcement channel (data will not be convolved) */
157  unsigned int is_announcement:1;
158  /*! The position of the channel in the virtual room represented by an id
159  * This ID has to be set even if the channel has no binaural output!
160  */
161  unsigned int binaural_pos;
162  /*! The channel pair for this channel */
164  /*! Marks the channel for suspending all binaural activity on the output */
165  unsigned int binaural_suspended:1;
166  /*! Channel sample rate, stored to retrieve it after unsuspending the channel */
167  int rate;
168  /*! Buffer containing final mixed audio from all sources */
169  short final_buf[MAX_DATALEN];
170  /*! Buffer containing only the audio from the channel */
171  short our_buf[MAX_DATALEN];
172  /*! Data pertaining to talker mode for video conferencing */
173  struct video_follow_talker_data video_talker;
174  /*! The ideal stream topology for the channel */
176  /*! The latest REMB report from this participant */
178  /*! The REMB collector for this channel, collects REMB from all video receivers */
180  /*! The bridge streams which are feeding us video sources */
181  AST_VECTOR(, int) video_sources;
182 };
183 
185  struct ast_timer *timer;
186  /*!
187  * \brief Bridge pointer passed to the softmix mixing thread.
188  *
189  * \note Does not need a reference because the bridge will
190  * always exist while the mixing thread exists even if the
191  * bridge is no longer actively using the softmix technology.
192  */
194  /*! Lock for signaling the mixing thread. */
196  /*! Condition, used if we need to wake up the mixing thread. */
198  /*! Thread handling the mixing */
199  pthread_t thread;
200  unsigned int internal_rate;
202  /*! TRUE if the mixing thread should stop */
203  unsigned int stop:1;
204  /*! The default sample size (e.g. using Opus at 48khz and 20 ms mixing
205  * interval, sample size is 960) */
206  unsigned int default_sample_size;
207  /*! All data needed for binaural signaling */
208  struct convolve_data convolve;
209  /*! TRUE if the first attempt to init binaural rendering data was done
210  * (does not guarantee success)
211  */
212  unsigned int binaural_init;
213  /*! The last time a video update was sent into the bridge */
214  struct timeval last_video_update;
215  /*! The last time a REMB frame was sent to each source of video */
216  struct timeval last_remb_update;
217  /*! Per-bridge stream REMB collectors, which flow back to video source */
218  AST_VECTOR(, struct softmix_remb_collector *) remb_collectors;
219  /*! Per-bridge REMB bitrate */
220  float bitrate;
221 };
222 
224  unsigned int max_num_entries;
225  unsigned int used_entries;
227  /*! Stereo channel pairs used to store convolved binaural signals */
229 };
230 
231 /*!
232  * \brief Deletes left over signals on a channel that it can be reused.
233  *
234  * \param channel_pair The channel pair which contains the left and right audio channel.
235  * \param default_sample_size The sample size which the channel pair uses.
236  */
237 void reset_channel_pair(struct convolve_channel_pair *channel_pair,
238  unsigned int default_sample_size);
239 
240 /*!
241  * \brief Randomly changes the virtual positions of conference participants.
242  *
243  * \param softmix_data The structure containing all position informations.
244  */
245 void random_binaural_pos_change(struct softmix_bridge_data *softmix_data);
246 
247 /*!
248  * \brief Binaural convolving of audio data for a channel.
249  *
250  * \param chan The channel that will contain the binaural audio data as result.
251  * \param in_samples The audio data which will be convolved.
252  * \param in_sample_size The size of the audio data.
253  * \param hrtf_length The length of the head related transfer function used to convolve the audio.
254  *
255  * \retval 0 success
256  * \retval -1 failure
257  */
258 int do_convolve(struct convolve_channel *chan, int16_t *in_samples,
259  unsigned int in_sample_size, unsigned int hrtf_length);
260 
261 /*!
262  * \brief Binaural convolving of audio data for a channel pair (left and right channel).
263  *
264  * \param data Contains the left and right audio channel.
265  * \param pos_id The position the channel has in the virtual enviroment.
266  * \param in_samples The audio data which will be convolved for both channels.
267  * \param in_sample_size The size of the audio data.
268  * \param channel_name The name of the channel
269  *
270  * \retval The channel pair with convolved audio on success.
271  * \retval NULL failure
272  */
274  unsigned int pos_id, int16_t *in_samples, unsigned int in_sample_size,
275  const char *channel_name);
276 
277 /*!
278  * \brief Provides a head related impulse response for the given position in the virtual
279  * enviroment.
280  *
281  * \param chan_pos The position of the channel in the virtual enviroment.
282  * \param chan_side 0 for the left audio channel, 1 for the right.
283  *
284  * \retval The hrir for the given position in the virtual room for either the left or right
285  * channels.
286  * \retval NULL on failure.
287  *
288  */
289 float *get_hrir(unsigned int chan_pos, unsigned int chan_side);
290 
291 /*!
292  * \brief Initializes all data needed for binaural audio processing.
293  *
294  * \param channel The channel used for binaural audio processing.
295  * \param hrtf_len The length of the head related impulse response used for binaural processing.
296  * \param chan_pos The position of the channel in the virtual enviroment.
297  * \param chan_side 0 for the left audio channel, 1 for the right.
298  * \param default_sample_size The default size of audio samples.
299  *
300  * \retval 0 on success
301  * \retval -1 on failure
302  */
303 int init_convolve_channel(struct convolve_channel *channel, unsigned int hrtf_len,
304  unsigned int chan_pos, unsigned int chan_side, unsigned int default_sample_size);
305 
306 /*!
307  * \brief Initializies all data needed for binaural audio processing of a channel pair
308  * (left and right).
309  *
310  * \param cchan_pair The channel pair used for binaural audio processing.
311  * \param hrtf_len The length of the head related impulse response used for binaural processing.
312  * \param chan_pos The position of the channel in the virtual enviroment.
313  * \param default_sample_size The default size of audio samples.
314  *
315  * \retval 0 on success
316  * \retval -1 on failure
317  */
319  unsigned int hrtf_len, unsigned int chan_pos, unsigned int default_sample_size);
320 
321 /*!
322  * \brief Preinits a specific number of channels (CONVOVLE_CHANNEL_PREALLOC)
323  * at the beginning of a conference.
324  *
325  * \param data Contains all channels and data needed for binaural processing
326  * (e.g. head related transfer functions).
327  * \param default_sample_size The default size of audio samples.
328  *
329  * \retval 0 on success
330  * \retval -1 on failure
331  */
332 int init_convolve_data(struct convolve_data *data, unsigned int default_sample_size);
333 
334 /*!
335  * \brief Frees all data needed for binaural processing by an audio channel.
336  *
337  * \param cchan The channel to clean up.
338  */
339 void free_convolve_channel(struct convolve_channel *cchan);
340 
341 /*!
342  * \brief Frees all data needed for binaural processing by a pair of audio channels
343  * (left and right).
344  *
345  * \param cchan_pair The channel pair to clean up.
346  */
347 void free_convolve_channel_pair(struct convolve_channel_pair *cchan_pair);
348 
349 /*!
350  * \brief Frees all channels and data needed for binaural audio processing.
351  *
352  * \param data Contains all channels and data for the cleanup process.
353  */
354 void free_convolve_data(struct convolve_data *data);
355 
356 /*!
357  * \brief Joins a channel into a virtual enviroment build with the help of binaural sythesis.
358  *
359  * \param data Contains all channels and data needed for binaural processing
360  * (e.g. head related transfer functions).
361  * \param default_sample_size The default size of audio samples.
362  *
363  * \retval The position of the channel in the virtual enviroment.
364  * \retval -1 on failure
365  */
366 int set_binaural_data_join(struct convolve_data *data, unsigned int default_sample_size);
367 
368 /*!
369  * \brief Removes a channel from the binaural conference bridge. Marks the position in
370  * the virtual room as unused that it can be reused by the next channel which enters the
371  * conference.
372  *
373  * \param data Contains all channels and data needed for binaural processing
374  * (e.g. head related transfer functions).
375  * \param pos The position of the channel in the virtual enviroment.
376  * \param default_sample_size The default size of audio samples.
377  */
378 void set_binaural_data_leave(struct convolve_data *data, unsigned int pos,
379  unsigned int default_sample_size);
380 
381 /*!
382  * \brief Writes the binaural audio to a channel.
383  *
384  * \param sc The softmix channel.
385  * \param default_sample_size The default size of audio samples.
386  */
388  unsigned int default_sample_size);
389 
390 /*!
391  * \brief Checks if a position change in the virual enviroment is requested by one of
392  * the participants.
393  *
394  * \param bridge The conference bridge.
395  * \param softmix_data The data used by the softmix bridge.
396  */
397 void check_binaural_position_change(struct ast_bridge *bridge,
398  struct softmix_bridge_data *softmix_data);
399 
400 /*!
401  * \brief Processes audio data with the binaural synthesis and adds the result to the mixing array.
402  *
403  * \param bridge The conference bridge needed to check if binaural processing is active or not.
404  * \param softmix_data Contains all data for the softmix bridge and for the binaural processing.
405  * \param softmix_samples The sample size.
406  * \param mixing_array The array which holds all audio data for mixing.
407  * \param sc The channel which contains the audio data to process.
408  * \param channel_name The name of the channel
409  */
410 void add_binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data,
411  unsigned int softmix_samples, struct softmix_mixing_array *mixing_array,
412  struct softmix_channel *sc, const char *channel_name);
413 
414 /*!
415  * \brief Mixes all binaural audio data contained in the mixing array.
416  *
417  * \param bridge The conference bridge needed to check if binaural processing is active or not.
418  * \param softmix_data Contains all data for the softmix bridge and for the binaural processing.
419  * \param mixing_array The array which holds all audio data for mixing.
420  * \param bin_buf The buffer that will contain the mixing results.
421  * \param ann_buf The buffer that will contain mixed announcements in an interleaved format.
422  */
423 void binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data,
424  struct softmix_mixing_array *mixing_array, int16_t *bin_buf, int16_t *ann_buf);
425 
426 /*!
427  * \brief Creates a frame out of binaural audio data.
428  *
429  * \param bridge_channel Contains the information if binaural processing is active or not.
430  * If active binaural audio data will be copied, if not mono data will be provided in an
431  * interleaved format.
432  * \param sc The softmix channel holding all informations for the process.
433  * \param bin_buf The buffer that contains all mixing results.
434  * \param ann_buf The buffer that contains mixed announcements in an interleaved format.
435  * \param softmix_datalen The size of the audio data.
436  * \param softmix_samples The number of audio samples.
437  * \param buf The buffer that contains all mono mixing results, used if binaural processing is
438  * inactive.
439  */
440 void create_binaural_frame(struct ast_bridge_channel *bridge_channel,
441  struct softmix_channel *sc, int16_t *bin_buf, int16_t *ann_buf,
442  unsigned int softmix_datalen, unsigned int softmix_samples, int16_t *buf);
443 
444 #endif /* _ASTERISK_BRIDGE_SOFTMIX_INTERNAL_H */
struct convolve_channel_pair ** chan_pairs
Asterisk main include file. File version handling, generic pbx functions.
float * get_hrir(unsigned int chan_pos, unsigned int chan_side)
Provides a head related impulse response for the given position in the virtual enviroment.
struct ast_stream_topology * topology
short int16_t
Definition: db.h:59
Support for translation of data formats. translate.c.
char buf[BUFSIZE]
Definition: eagi_proxy.c:66
struct convolve_channel_pair * our_chan_pair
Definition of a media format.
Definition: format.c:43
unsigned int binaural_suspended
void softmix_process_write_binaural_audio(struct softmix_channel *sc, unsigned int default_sample_size)
Writes the binaural audio to a channel.
Definition: muted.c:95
A REMB feedback message (see draft-alvestrand-rmcat-remb-03 for details)
Definition: rtp_engine.h:350
pthread_cond_t ast_cond_t
Definition: lock.h:176
void binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data, struct softmix_mixing_array *mixing_array, int16_t *bin_buf, int16_t *ann_buf)
Mixes all binaural audio data contained in the mixing array.
void check_binaural_position_change(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data)
Checks if a position change in the virual enviroment is requested by one of the participants.
int do_convolve(struct convolve_channel *chan, int16_t *in_samples, unsigned int in_sample_size, unsigned int hrtf_length)
Binaural convolving of audio data for a channel.
General Asterisk PBX channel definitions.
Definition: dsp.c:405
void random_binaural_pos_change(struct softmix_bridge_data *softmix_data)
Randomly changes the virtual positions of conference participants.
struct ast_format * read_slin_format
int init_convolve_channel(struct convolve_channel *channel, unsigned int hrtf_len, unsigned int chan_pos, unsigned int chan_side, unsigned int default_sample_size)
Initializes all data needed for binaural audio processing.
Channel Bridging API.
Asterisk internal frame definitions.
#define AST_VECTOR(name, type)
Define a vector structure.
Definition: vector.h:44
Structure that contains information about a bridge.
Definition: bridge.h:357
int set_binaural_data_join(struct convolve_data *data, unsigned int default_sample_size)
Joins a channel into a virtual enviroment build with the help of binaural sythesis.
Structure which contains per-channel mixing information.
void create_binaural_frame(struct ast_bridge_channel *bridge_channel, struct softmix_channel *sc, int16_t *bin_buf, int16_t *ann_buf, unsigned int softmix_datalen, unsigned int softmix_samples, int16_t *buf)
Creates a frame out of binaural audio data.
void free_convolve_channel_pair(struct convolve_channel_pair *cchan_pair)
Frees all data needed for binaural processing by a pair of audio channels (left and right)...
A machine to gather up arbitrary frames and convert them to raw slinear on demand.
struct ast_bridge * bridge
Bridge pointer passed to the softmix mixing thread.
int init_convolve_data(struct convolve_data *data, unsigned int default_sample_size)
Preinits a specific number of channels (CONVOVLE_CHANNEL_PREALLOC) at the beginning of a conference...
int init_convolve_channel_pair(struct convolve_channel_pair *cchan_pair, unsigned int hrtf_len, unsigned int chan_pos, unsigned int default_sample_size)
Initializies all data needed for binaural audio processing of a channel pair (left and right)...
Vector container support.
unsigned int talking
TRUE if a channel is talking.
#define MAX_DATALEN
Support for logging to various files, console and syslog Configuration in file logger.conf.
struct convolve_channel_pair ** cchan_pair
Structure that contains information regarding a channel in a bridge.
void set_binaural_data_leave(struct convolve_data *data, unsigned int pos, unsigned int default_sample_size)
Removes a channel from the binaural conference bridge. Marks the position in the virtual room as unus...
Data structure associated with a single frame of data.
Options provided by main asterisk program.
void add_binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data, unsigned int softmix_samples, struct softmix_mixing_array *mixing_array, struct softmix_channel *sc, const char *channel_name)
Processes audio data with the binaural synthesis and adds the result to the mixing array...
Pluggable RTP Architecture.
Bridging API.
void free_convolve_data(struct convolve_data *data)
Frees all channels and data needed for binaural audio processing.
Asterisk module definitions.
struct convolve_channel_pair * do_convolve_pair(struct convolve_data *data, unsigned int pos_id, int16_t *in_samples, unsigned int in_sample_size, const char *channel_name)
Binaural convolving of audio data for a channel pair (left and right channel).
Timing source management.
struct softmix_remb_collector * remb_collector
Structure for mutex and tracking information.
Definition: lock.h:135
void reset_channel_pair(struct convolve_channel_pair *channel_pair, unsigned int default_sample_size)
Deletes left over signals on a channel that it can be reused.
#define DEFAULT_ENERGY_HISTORY_LEN
void free_convolve_channel(struct convolve_channel *cchan)
Frees all data needed for binaural processing by an audio channel.