Asterisk - The Open Source Telephony Project GIT-master-2de1a68
bridge_softmix_internal.h
Go to the documentation of this file.
1/*
2 * Asterisk -- An open source telephony toolkit.
3 *
4 * Copyright (C) 2011, Digium, Inc.
5 *
6 * Joshua Colp <jcolp@digium.com>
7 * David Vossel <dvossel@digium.com>
8 *
9 * See http://www.asterisk.org for more information about
10 * the Asterisk project. Please do not directly contact
11 * any of the maintainers of this project for assistance;
12 * the project provides a web site, mailing lists and IRC
13 * channels for your use.
14 *
15 * This program is free software, distributed under the terms of
16 * the GNU General Public License Version 2. See the LICENSE file
17 * at the top of the source tree.
18 */
19
20/*! \file
21 *
22 * \brief Multi-party software based channel mixing (header)
23 *
24 * \author Joshua Colp <jcolp@digium.com>
25 * \author David Vossel <dvossel@digium.com>
26 *
27 * \ingroup bridges
28 */
29
30#ifndef _ASTERISK_BRIDGE_SOFTMIX_INTERNAL_H
31#define _ASTERISK_BRIDGE_SOFTMIX_INTERNAL_H
32
33#include "asterisk.h"
34#include <stdio.h>
35#include <stdlib.h>
36#include <string.h>
37#include <sys/time.h>
38#include <signal.h>
39#include <errno.h>
40#include <unistd.h>
41
42#include "asterisk/module.h"
43#include "asterisk/channel.h"
44#include "asterisk/bridge.h"
46#include "asterisk/frame.h"
47#include "asterisk/options.h"
48#include "asterisk/logger.h"
50#include "asterisk/astobj2.h"
51#include "asterisk/timing.h"
52#include "asterisk/translate.h"
53#include "asterisk/rtp_engine.h"
54#include "asterisk/vector.h"
55
56#ifdef BINAURAL_RENDERING
57#include <fftw3.h>
58#endif
59
60#if defined(__Darwin__) || defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__DragonFly__) || defined(__NetBSD__) || defined(__CYGWIN__)
61#include <float.h>
62#else
63#include <values.h>
64#endif
65
66#define MAX_DATALEN 8096
67#define DEFAULT_ENERGY_HISTORY_LEN 150
68
69/*! Setting the sample rate to 48000 by default if binaural is activated. */
70#define SOFTMIX_BINAURAL_SAMPLE_RATE 48000
71/*! We only support 20 ms interval length with binaural data at the moment. */
72#define BINAURAL_MIXING_INTERVAL 20
73
75 /*! The head related transfer function used for convolving */
76 double *hrtf;
77 /*! Input signals for fftw */
78 double *fftw_in;
79 /*! Output signals from the fftw */
80 double *fftw_out;
81 /*! Signals for overlap add */
83 /*! The resulting data after the convolution */
85#ifdef BINAURAL_RENDERING
86 /*! The fftw plan for binaural signaling */
87 fftw_plan fftw_plan;
88 /*! The inverse fftw plan for binaural signaling */
89 fftw_plan fftw_plan_inverse;
90#endif
91};
92
94 /*! The left channel of a stereo channel pair */
96 /*! The right channel of a stereo channel pair */
98};
99
101 /*! A count of all channels potentially having input data for the conference. */
103 /*! Will set to true if there is at least one binaural output.
104 * Only if set to true data will be convolved. */
106 /*! The length of the head related transfer function */
107 unsigned int hrtf_length;
108 /*! Number of channels available for convolving.
109 * We do not delete a channel when a member leaves, cause we can reuse it for the next one. */
111 /*! The positions of the single channels in the virtual room */
113 /*! Each channel has a stereo pair of channels for the convolution */
115};
116
118 /*! audio energy history */
120 /*! The current slot being used in the history buffer, this
121 * increments and wraps around */
123 /*! The current energy sum used for averages. */
125 /*! The current energy average */
127};
128
130
131/*! \brief Structure which contains per-channel mixing information */
133 /*! Lock to protect this structure */
135 /*! Factory which contains audio read in from the channel */
137 /*! Frame that contains mixed audio to be written out to the channel */
139 /*! Current expected read slinear format. */
141 /*! DSP for detecting silence */
142 struct ast_dsp *dsp;
143 /*!
144 * \brief TRUE if a channel is talking.
145 *
146 * \note This affects how the channel's audio is mixed back to
147 * it.
148 */
149 unsigned int talking:1;
150 /*! TRUE if the channel provided audio for this mixing interval */
151 unsigned int have_audio:1;
152 /*! We set binaural also as channel data, to have better tracking.
153 * It is also present in transpvt.
154 */
155 unsigned int binaural:1;
156 /*! TRUE if this is an announcement channel (data will not be convolved) */
157 unsigned int is_announcement:1;
158 /*! The position of the channel in the virtual room represented by an id
159 * This ID has to be set even if the channel has no binaural output!
160 */
161 unsigned int binaural_pos;
162 /*! The channel pair for this channel */
164 /*! Marks the channel for suspending all binaural activity on the output */
165 unsigned int binaural_suspended:1;
166 /*! Channel sample rate, stored to retrieve it after unsuspending the channel */
167 int rate;
168 /*! Buffer containing final mixed audio from all sources */
170 /*! Buffer containing only the audio from the channel */
172 /*! Data pertaining to talker mode for video conferencing */
174 /*! The ideal stream topology for the channel */
176 /*! The latest REMB report from this participant */
178 /*! The REMB collector for this channel, collects REMB from all video receivers */
180 /*! The bridge streams which are feeding us video sources */
182};
183
186 /*!
187 * \brief Bridge pointer passed to the softmix mixing thread.
188 *
189 * \note Does not need a reference because the bridge will
190 * always exist while the mixing thread exists even if the
191 * bridge is no longer actively using the softmix technology.
192 */
194 /*! Lock for signaling the mixing thread. */
196 /*! Condition, used if we need to wake up the mixing thread. */
198 /*! Thread handling the mixing */
199 pthread_t thread;
200 unsigned int internal_rate;
202 /*! TRUE if the mixing thread should stop */
203 unsigned int stop:1;
204 /*! The default sample size (e.g. using Opus at 48khz and 20 ms mixing
205 * interval, sample size is 960) */
207 /*! All data needed for binaural signaling */
209 /*! TRUE if the first attempt to init binaural rendering data was done
210 * (does not guarantee success)
211 */
212 unsigned int binaural_init;
213 /*! The last time a video update was sent into the bridge */
214 struct timeval last_video_update;
215 /*! The last time a REMB frame was sent to each source of video */
216 struct timeval last_remb_update;
217 /*! Per-bridge stream REMB collectors, which flow back to video source */
219 /*! Per-bridge REMB bitrate */
220 float bitrate;
221};
222
224 unsigned int max_num_entries;
225 unsigned int used_entries;
227 /*! Stereo channel pairs used to store convolved binaural signals */
229};
230
231/*!
232 * \brief Deletes left over signals on a channel that it can be reused.
233 *
234 * \param channel_pair The channel pair which contains the left and right audio channel.
235 * \param default_sample_size The sample size which the channel pair uses.
236 */
237void reset_channel_pair(struct convolve_channel_pair *channel_pair,
238 unsigned int default_sample_size);
239
240/*!
241 * \brief Randomly changes the virtual positions of conference participants.
242 *
243 * \param softmix_data The structure containing all position informations.
244 */
245void random_binaural_pos_change(struct softmix_bridge_data *softmix_data);
246
247/*!
248 * \brief Binaural convolving of audio data for a channel.
249 *
250 * \param chan The channel that will contain the binaural audio data as result.
251 * \param in_samples The audio data which will be convolved.
252 * \param in_sample_size The size of the audio data.
253 * \param hrtf_length The length of the head related transfer function used to convolve the audio.
254 *
255 * \retval 0 success
256 * \retval -1 failure
257 */
258int do_convolve(struct convolve_channel *chan, int16_t *in_samples,
259 unsigned int in_sample_size, unsigned int hrtf_length);
260
261/*!
262 * \brief Binaural convolving of audio data for a channel pair (left and right channel).
263 *
264 * \param data Contains the left and right audio channel.
265 * \param pos_id The position the channel has in the virtual enviroment.
266 * \param in_samples The audio data which will be convolved for both channels.
267 * \param in_sample_size The size of the audio data.
268 * \param channel_name The name of the channel
269 *
270 * \retval The channel pair with convolved audio on success.
271 * \retval NULL failure
272 */
274 unsigned int pos_id, int16_t *in_samples, unsigned int in_sample_size,
275 const char *channel_name);
276
277/*!
278 * \brief Provides a head related impulse response for the given position in the virtual
279 * enviroment.
280 *
281 * \param chan_pos The position of the channel in the virtual enviroment.
282 * \param chan_side 0 for the left audio channel, 1 for the right.
283 *
284 * \retval The hrir for the given position in the virtual room for either the left or right
285 * channels.
286 * \retval NULL on failure.
287 *
288 */
289float *get_hrir(unsigned int chan_pos, unsigned int chan_side);
290
291/*!
292 * \brief Initializes all data needed for binaural audio processing.
293 *
294 * \param channel The channel used for binaural audio processing.
295 * \param hrtf_len The length of the head related impulse response used for binaural processing.
296 * \param chan_pos The position of the channel in the virtual enviroment.
297 * \param chan_side 0 for the left audio channel, 1 for the right.
298 * \param default_sample_size The default size of audio samples.
299 *
300 * \retval 0 on success
301 * \retval -1 on failure
302 */
303int init_convolve_channel(struct convolve_channel *channel, unsigned int hrtf_len,
304 unsigned int chan_pos, unsigned int chan_side, unsigned int default_sample_size);
305
306/*!
307 * \brief Initializes all data needed for binaural audio processing of a channel pair
308 * (left and right).
309 *
310 * \param cchan_pair The channel pair used for binaural audio processing.
311 * \param hrtf_len The length of the head related impulse response used for binaural processing.
312 * \param chan_pos The position of the channel in the virtual enviroment.
313 * \param default_sample_size The default size of audio samples.
314 *
315 * \retval 0 on success
316 * \retval -1 on failure
317 */
319 unsigned int hrtf_len, unsigned int chan_pos, unsigned int default_sample_size);
320
321/*!
322 * \brief Preinits a specific number of channels (CONVOLVE_CHANNEL_PREALLOC)
323 * at the beginning of a conference.
324 *
325 * \param data Contains all channels and data needed for binaural processing
326 * (e.g. head related transfer functions).
327 * \param default_sample_size The default size of audio samples.
328 *
329 * \retval 0 on success
330 * \retval -1 on failure
331 */
332int init_convolve_data(struct convolve_data *data, unsigned int default_sample_size);
333
334/*!
335 * \brief Frees all data needed for binaural processing by an audio channel.
336 *
337 * \param cchan The channel to clean up.
338 */
339void free_convolve_channel(struct convolve_channel *cchan);
340
341/*!
342 * \brief Frees all data needed for binaural processing by a pair of audio channels
343 * (left and right).
344 *
345 * \param cchan_pair The channel pair to clean up.
346 */
348
349/*!
350 * \brief Frees all channels and data needed for binaural audio processing.
351 *
352 * \param data Contains all channels and data for the cleanup process.
353 */
354void free_convolve_data(struct convolve_data *data);
355
356/*!
357 * \brief Joins a channel into a virtual enviroment build with the help of binaural synthesis.
358 *
359 * \param data Contains all channels and data needed for binaural processing
360 * (e.g. head related transfer functions).
361 * \param default_sample_size The default size of audio samples.
362 *
363 * \retval The position of the channel in the virtual enviroment.
364 * \retval -1 on failure
365 */
366int set_binaural_data_join(struct convolve_data *data, unsigned int default_sample_size);
367
368/*!
369 * \brief Removes a channel from the binaural conference bridge. Marks the position in
370 * the virtual room as unused that it can be reused by the next channel which enters the
371 * conference.
372 *
373 * \param data Contains all channels and data needed for binaural processing
374 * (e.g. head related transfer functions).
375 * \param pos The position of the channel in the virtual enviroment.
376 * \param default_sample_size The default size of audio samples.
377 */
378void set_binaural_data_leave(struct convolve_data *data, unsigned int pos,
379 unsigned int default_sample_size);
380
381/*!
382 * \brief Writes the binaural audio to a channel.
383 *
384 * \param sc The softmix channel.
385 * \param default_sample_size The default size of audio samples.
386 */
388 unsigned int default_sample_size);
389
390/*!
391 * \brief Checks if a position change in the virtual enviroment is requested by one of
392 * the participants.
393 *
394 * \param bridge The conference bridge.
395 * \param softmix_data The data used by the softmix bridge.
396 */
398 struct softmix_bridge_data *softmix_data);
399
400/*!
401 * \brief Processes audio data with the binaural synthesis and adds the result to the mixing array.
402 *
403 * \param bridge The conference bridge needed to check if binaural processing is active or not.
404 * \param softmix_data Contains all data for the softmix bridge and for the binaural processing.
405 * \param softmix_samples The sample size.
406 * \param mixing_array The array which holds all audio data for mixing.
407 * \param sc The channel which contains the audio data to process.
408 * \param channel_name The name of the channel
409 */
410void add_binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data,
411 unsigned int softmix_samples, struct softmix_mixing_array *mixing_array,
412 struct softmix_channel *sc, const char *channel_name);
413
414/*!
415 * \brief Mixes all binaural audio data contained in the mixing array.
416 *
417 * \param bridge The conference bridge needed to check if binaural processing is active or not.
418 * \param softmix_data Contains all data for the softmix bridge and for the binaural processing.
419 * \param mixing_array The array which holds all audio data for mixing.
420 * \param bin_buf The buffer that will contain the mixing results.
421 * \param ann_buf The buffer that will contain mixed announcements in an interleaved format.
422 */
423void binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data,
424 struct softmix_mixing_array *mixing_array, int16_t *bin_buf, int16_t *ann_buf);
425
426/*!
427 * \brief Creates a frame out of binaural audio data.
428 *
429 * \param bridge_channel Contains the information if binaural processing is active or not.
430 * If active binaural audio data will be copied, if not mono data will be provided in an
431 * interleaved format.
432 * \param sc The softmix channel holding all informations for the process.
433 * \param bin_buf The buffer that contains all mixing results.
434 * \param ann_buf The buffer that contains mixed announcements in an interleaved format.
435 * \param softmix_datalen The size of the audio data.
436 * \param softmix_samples The number of audio samples.
437 * \param buf The buffer that contains all mono mixing results, used if binaural processing is
438 * inactive.
439 */
440void create_binaural_frame(struct ast_bridge_channel *bridge_channel,
441 struct softmix_channel *sc, int16_t *bin_buf, int16_t *ann_buf,
442 unsigned int softmix_datalen, unsigned int softmix_samples, int16_t *buf);
443
444#endif /* _ASTERISK_BRIDGE_SOFTMIX_INTERNAL_H */
Asterisk main include file. File version handling, generic pbx functions.
Bridging API.
int set_binaural_data_join(struct convolve_data *data, unsigned int default_sample_size)
Joins a channel into a virtual enviroment build with the help of binaural synthesis.
int do_convolve(struct convolve_channel *chan, int16_t *in_samples, unsigned int in_sample_size, unsigned int hrtf_length)
Binaural convolving of audio data for a channel.
float * get_hrir(unsigned int chan_pos, unsigned int chan_side)
Provides a head related impulse response for the given position in the virtual enviroment.
void create_binaural_frame(struct ast_bridge_channel *bridge_channel, struct softmix_channel *sc, int16_t *bin_buf, int16_t *ann_buf, unsigned int softmix_datalen, unsigned int softmix_samples, int16_t *buf)
Creates a frame out of binaural audio data.
int init_convolve_channel(struct convolve_channel *channel, unsigned int hrtf_len, unsigned int chan_pos, unsigned int chan_side, unsigned int default_sample_size)
Initializes all data needed for binaural audio processing.
void softmix_process_write_binaural_audio(struct softmix_channel *sc, unsigned int default_sample_size)
Writes the binaural audio to a channel.
void binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data, struct softmix_mixing_array *mixing_array, int16_t *bin_buf, int16_t *ann_buf)
Mixes all binaural audio data contained in the mixing array.
void reset_channel_pair(struct convolve_channel_pair *channel_pair, unsigned int default_sample_size)
Deletes left over signals on a channel that it can be reused.
void add_binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data, unsigned int softmix_samples, struct softmix_mixing_array *mixing_array, struct softmix_channel *sc, const char *channel_name)
Processes audio data with the binaural synthesis and adds the result to the mixing array.
#define DEFAULT_ENERGY_HISTORY_LEN
void free_convolve_channel_pair(struct convolve_channel_pair *cchan_pair)
Frees all data needed for binaural processing by a pair of audio channels (left and right).
void free_convolve_channel(struct convolve_channel *cchan)
Frees all data needed for binaural processing by an audio channel.
int init_convolve_data(struct convolve_data *data, unsigned int default_sample_size)
Preinits a specific number of channels (CONVOLVE_CHANNEL_PREALLOC) at the beginning of a conference.
void random_binaural_pos_change(struct softmix_bridge_data *softmix_data)
Randomly changes the virtual positions of conference participants.
int init_convolve_channel_pair(struct convolve_channel_pair *cchan_pair, unsigned int hrtf_len, unsigned int chan_pos, unsigned int default_sample_size)
Initializes all data needed for binaural audio processing of a channel pair (left and right).
void check_binaural_position_change(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data)
Checks if a position change in the virtual enviroment is requested by one of the participants.
struct convolve_channel_pair * do_convolve_pair(struct convolve_data *data, unsigned int pos_id, int16_t *in_samples, unsigned int in_sample_size, const char *channel_name)
Binaural convolving of audio data for a channel pair (left and right channel).
void free_convolve_data(struct convolve_data *data)
Frees all channels and data needed for binaural audio processing.
#define MAX_DATALEN
void set_binaural_data_leave(struct convolve_data *data, unsigned int pos, unsigned int default_sample_size)
Removes a channel from the binaural conference bridge. Marks the position in the virtual room as unus...
Channel Bridging API.
General Asterisk PBX channel definitions.
short int16_t
Definition: db.h:59
char buf[BUFSIZE]
Definition: eagi_proxy.c:66
Asterisk internal frame definitions.
Support for logging to various files, console and syslog Configuration in file logger....
pthread_cond_t ast_cond_t
Definition: lock.h:178
Asterisk module definitions.
Options provided by main asterisk program.
Pluggable RTP Architecture.
A machine to gather up arbitrary frames and convert them to raw slinear on demand.
Structure that contains information regarding a channel in a bridge.
Structure that contains information about a bridge.
Definition: bridge.h:349
Definition: dsp.c:407
Definition of a media format.
Definition: format.c:43
Data structure associated with a single frame of data.
Structure for mutex and tracking information.
Definition: lock.h:135
A REMB feedback message (see draft-alvestrand-rmcat-remb-03 for details)
Definition: rtp_engine.h:375
struct convolve_channel chan_left
struct convolve_channel chan_right
struct convolve_channel_pair ** cchan_pair
struct convolve_data convolve
struct ast_bridge * bridge
Bridge pointer passed to the softmix mixing thread.
struct softmix_bridge_data::@102 remb_collectors
Structure which contains per-channel mixing information.
unsigned int talking
TRUE if a channel is talking.
struct ast_stream_topology * topology
struct softmix_remb_collector * remb_collector
struct softmix_channel::@101 video_sources
unsigned int binaural_suspended
struct convolve_channel_pair * our_chan_pair
short final_buf[MAX_DATALEN]
short our_buf[MAX_DATALEN]
struct ast_format * read_slin_format
struct ast_slinfactory factory
struct video_follow_talker_data video_talker
struct ast_frame write_frame
struct ast_rtp_rtcp_feedback_remb remb
struct convolve_channel_pair ** chan_pairs
int energy_history[DEFAULT_ENERGY_HISTORY_LEN]
Timing source management.
Support for translation of data formats. translate.c.
Vector container support.
#define AST_VECTOR(name, type)
Define a vector structure.
Definition: vector.h:44