GNOME Bugzilla – Bug 768836
audioresample crash when sample rate changes within a buffer gap (<= 1.8 only)
Last modified: 2016-08-31 08:10:46 UTC
Hi, In my application, I need to change the audio samplerate dynamical during runtime. My code crashes with "audioresample assertion failed: (out_len >= out_processed)", when I change the sink pad sample rate while a GAP is active. related GStreamer-devel discussion: =================================== http://gstreamer-devel.966125.n4.nabble.com/audioresample-assertion-failed-out-len-gt-out-processed-td4678533.html Comment from S. Dröge: I didn't look closely at your code (or the crash), but I can confirm that it crashes like that with 1.8 but it doesn't seem to crash at all here with 1.9.1. It is also using a completely new resampler, so that's not too surprising. Test Results: ============= with 1.4.5: rate change not audible, no crash, but GStreamer-CRITICAL **: gst_buffer_resize_range: assertion 'bufmax >= bufoffs + offset + size' failed with 1.6.4: rate change not audible, no crash, but GStreamer-CRITICAL **: gst_buffer_resize_range: assertion 'bufmax >= bufoffs + offset + size' failed with 1.8.1 & 1.8.2: ./test src=120ms sink=120ms set audioresample input rate to 47950 src=1200ms sink=1200ms set audioresample input rate to 48050 src=2279ms sink=2280ms set audioresample input rate to 47950 src=3360ms sink=3360ms set audioresample input rate to 48050 src=4439ms sink=4440ms set audioresample input rate to 47950 ERROR:gstaudioresample.c:1125:gst_audio_resample_process: assertion failed: (out_len >= out_processed) with 1.9.1: no crash but "WARN audio-resampler audio-resampler.c:362:convert_taps_gint16_c: can't find exact taps" on each rate change test.c Compiled with: ===================== gcc -o test `pkg-config --cflags --libs gstreamer-base-1.0 gstreamer-audio-1.0 gstreamer-1.0 gstreamer-app-1.0 gstreamer-plugins-good-1.0 gstreamer-audio-1.0` test.c -lm `pkg-config --libs gstreamer-base-1.0 gstreamer-audio-1.0 gstreamer-1.0 gstreamer-app-1.0 gstreamer-plugins-good-1.0 gstreamer-audio-1.0` test.c: ======= #include <stdio.h> #include <math.h> #include <gst/gst.h> #include <gst/app/gstappsrc.h> #include <gst/gstcaps.h> #define TESTCAPS "audio/x-raw,format=S16LE,channels=2,rate=48000,layout=interleaved" GstPad* eB3_sink_pad; GstClockTime eB3_sink_time = 0; GstPad* eB3_src_pad; GstClockTime eB3_src_time = 0; GstClockTime eB3_next_src_time = 0; typedef struct { GstElement* e; guint frequency; guint samplerate; guint period; GstClockTime src_timestamp; gboolean gap; } AudioSrcDataType; gboolean appsrc_feed_function (void* me); GstPadProbeReturn audioresample_pad_probe (GstPad *pad, GstPadProbeInfo *info, gpointer user_data); /* Main */ int main(void) { gst_init(NULL, NULL); GstElement* p = gst_pipeline_new("pipeline"); AudioSrcDataType live_audio_B = {NULL,300,48000,120,0,TRUE}; live_audio_B.e = gst_element_factory_make("appsrc",NULL); gst_app_src_set_caps(GST_APP_SRC(live_audio_B.e),gst_caps_from_string(TESTCAPS)); gst_base_src_set_live(GST_BASE_SRC(live_audio_B.e),TRUE); g_object_set (G_OBJECT(GST_APP_SRC(live_audio_B.e)), "format", GST_FORMAT_TIME, NULL); g_object_set (G_OBJECT(GST_APP_SRC(live_audio_B.e)), "block", FALSE, NULL); gst_app_src_set_stream_type(GST_APP_SRC(live_audio_B.e),GST_APP_STREAM_TYPE_STREAM); gst_app_src_set_latency(GST_APP_SRC(live_audio_B.e),1*live_audio_B.period*GST_MSECOND,1*live_audio_B.period*GST_MSECOND); g_timeout_add (live_audio_B.period/2,appsrc_feed_function,&live_audio_B); GstElement* eB2 = gst_element_factory_make("audiorate",NULL); g_object_set(eB2, "tolerance", (120+10)*GST_MSECOND, NULL); GstElement* eB3 = gst_element_factory_make("audioresample",NULL); g_object_set(eB3, "quality", 4, NULL); g_object_set(eB3, "sinc-filter-mode", 2, NULL); GstElement* e_mixer = gst_element_factory_make("audiomixer",NULL); g_object_set(e_mixer, "latency", 512*GST_MSECOND, NULL); GstElement* e_sink = gst_element_factory_make("autoaudiosink",NULL); /* add ... */ gst_bin_add_many(GST_BIN(p),live_audio_B.e,eB2,/*eB3,*/eB3,NULL); gst_bin_add_many(GST_BIN(p),e_mixer,e_sink,NULL); /* link ... */ //gst_element_link_filtered(eA1, eA2, gst_caps_from_string(TESTCAPS)); gst_element_link_many(live_audio_B.e, eB2, eB3, NULL); gst_pad_link (gst_element_get_static_pad (eB3,"src"), gst_element_request_pad (e_mixer,gst_element_class_get_pad_template(GST_ELEMENT_GET_CLASS(e_mixer),"sink_%u"), NULL, NULL)); gst_element_link_many(e_mixer, e_sink, NULL); #if 1 /* Enable audioresample control loop, if disabled, the error does not ocure */ { eB3_sink_pad = gst_element_get_static_pad (eB3, "sink"); gst_pad_add_probe (eB3_sink_pad,GST_PAD_PROBE_TYPE_BUFFER,(GstPadProbeCallback) audioresample_pad_probe,NULL,NULL); eB3_src_pad = gst_element_get_static_pad (eB3, "src"); gst_pad_add_probe (eB3_src_pad,GST_PAD_PROBE_TYPE_BUFFER,(GstPadProbeCallback) audioresample_pad_probe,NULL,NULL); } #endif /* run ... */ { static GMainLoop *loop; loop = g_main_loop_new(NULL, FALSE); gst_element_set_state(GST_ELEMENT(p), GST_STATE_PLAYING); g_main_loop_run(loop); } return 0; } /* timer based appsrc feed function. * 5 seconds sine + 5 seconds gap */ gboolean appsrc_feed_function (void* _me) { AudioSrcDataType* me = (AudioSrcDataType*)_me; if( GST_BASE_SRC_IS_STARTED(me->e) ) { GstClockTime t_now; while( ( t_now = gst_clock_get_time(gst_element_get_clock(GST_ELEMENT(me->e)))-gst_element_get_base_time(GST_ELEMENT(me->e))) > me->src_timestamp ) { GstMapInfo buffer_info; guint i; guint buffer_size = 2*sizeof(gint16)*me->samplerate*me->period/1000; GstBuffer* buffer = gst_buffer_new_and_alloc(buffer_size); if( gst_buffer_map(buffer,&buffer_info,GST_MAP_WRITE) ) { gst_buffer_ref(buffer); for(i=0;i<(buffer_size/sizeof(gint16));) { static glong t=0; gint16 v = (gint16)(1000.0*sin(me->frequency*(t++)*2*3.14/me->samplerate)); ((gint16*)(buffer_info.data))[i++] = v; ((gint16*)(buffer_info.data))[i++] = v; } GST_BUFFER_PTS(buffer) = t_now; GST_BUFFER_DURATION (buffer) = GST_CLOCK_TIME_NONE; //GST_BUFFER_DURATION (buffer) = me->period * GST_MSECOND; me->src_timestamp += me->period*GST_MSECOND; gst_buffer_unmap(buffer,&buffer_info); } if( (GST_TIME_AS_SECONDS(t_now)%20 ) < 10 ) { /* 5 seconds push audio to pipeline */ if(me->gap) { me->gap = FALSE; GST_BUFFER_FLAG_SET(buffer,GST_BUFFER_FLAG_RESYNC); } if( gst_app_src_push_buffer(GST_APP_SRC(me->e),buffer) != GST_FLOW_OK ) { gst_buffer_unref (buffer); } } else { /* 5 seconds GAP */ gst_buffer_unref (buffer); if(!me->gap) { me->gap = TRUE; } gst_pad_push_event(GST_BASE_SRC_PAD(me->e),gst_event_new_gap(t_now,GST_CLOCK_TIME_NONE)); } } } return TRUE; } /* audioresample PAD prob */ GstPadProbeReturn audioresample_pad_probe (GstPad *pad, GstPadProbeInfo *info, gpointer user_data) { (void)user_data; if(((info->type&GST_PAD_PROBE_TYPE_BUFFER)!=0)&&(info->data != NULL)) { if( pad == eB3_sink_pad ) { eB3_sink_time = GST_BUFFER_TIMESTAMP(GST_BUFFER(info->data)); } else if( pad == eB3_src_pad ) { eB3_src_time = GST_BUFFER_TIMESTAMP(GST_BUFFER(info->data)); if(( eB3_next_src_time < eB3_src_time )&&(GST_PAD_STREAM_TRYLOCK(eB3_sink_pad))) { gint rate; GstCaps* caps = gst_pad_get_current_caps(eB3_sink_pad); GstStructure* structure; caps = gst_caps_make_writable(caps); structure = gst_caps_get_structure(caps,0); if (gst_structure_get_int (structure, "rate", &rate) ) { if( eB3_src_time > eB3_sink_time ) { rate = 48000 + 8; } else { rate = 48000 - 8; } printf("src=%dms sink=%dms set audioresample input rate to %d\n", (gint)GST_TIME_AS_MSECONDS(eB3_src_time), (gint)GST_TIME_AS_MSECONDS(eB3_sink_time), rate); gst_caps_set_simple(caps,"rate",G_TYPE_INT,rate,NULL); gst_pad_set_caps(eB3_sink_pad,caps); gst_pad_send_event (eB3_sink_pad, gst_event_new_caps (caps)); } eB3_next_src_time = eB3_src_time + 1000 * GST_MSECOND; /* next adjustment in 5 seconds */ GST_PAD_STREAM_UNLOCK(eB3_sink_pad); } } } return GST_PAD_PROBE_OK; }
I don't know what the tap code is trying to do, but it appears to attempt to converge some value. Adding some logs, it falls off by just one (32766 or 32768 with a target of 32767). Since first sentence, I don't know whether this is "within error bars" or "really bad", but thought I'd mention it :)
(with current git, of course)
Hi, is there a chance to get this issue fixed with the next 1.8.x release?
If someone finds a solution, sure
Hi, I did some analysis by myself. For me (gstreamer beginner), it looks like, when the rate ratio is changed during a gap buffer timespan. Then the out_processed calculation results into a wrong value. Limiting the out_processed to out_len, avoids the assert. if( out_processed > out_len ) { /* take care that out_processed is not greater then out_len */ out_processed = out_len; } But I am not sure about the side effects? With that change, the above testcase works fine. I will do some tests within out application. Maik gstaudioresample.c: =================== static GstFlowReturn gst_audio_resample_process (GstAudioResample * resample, GstBuffer * inbuf, GstBuffer * outbuf) { GstMapInfo in_map, out_map; gsize outsize; guint32 in_len, in_processed; guint32 out_len, out_processed; guint filt_len = resample->funcs->get_filt_len (resample->state); gst_buffer_map (inbuf, &in_map, GST_MAP_READ); gst_buffer_map (outbuf, &out_map, GST_MAP_WRITE); in_len = in_map.size / resample->channels; out_len = out_map.size / resample->channels; in_len /= (resample->width / 8); out_len /= (resample->width / 8); in_processed = in_len; out_processed = out_len; if (GST_BUFFER_FLAG_IS_SET (inbuf, GST_BUFFER_FLAG_GAP)) { resample->num_nongap_samples = 0; if (resample->num_gap_samples < filt_len) { guint zeros_to_push; if (in_len >= filt_len - resample->num_gap_samples) zeros_to_push = filt_len - resample->num_gap_samples; else zeros_to_push = in_len; gst_audio_resample_push_drain (resample, zeros_to_push); in_len -= zeros_to_push; resample->num_gap_samples += zeros_to_push; } { guint num, den; resample->funcs->get_ratio (resample->state, &num, &den); if (resample->samples_in + in_len >= filt_len / 2) out_processed = gst_util_uint64_scale_int_ceil (resample->samples_in + in_len - filt_len / 2, den, num) - resample->samples_out; else out_processed = 0; => POSSIBLE BUGFIX if( out_processed > out_len ) { /* take care that out_processed is not greater then out_len */ out_processed = out_len; } <= POSSIBLE BUGFIX memset (out_map.data, 0, out_map.size); GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_GAP); resample->num_gap_samples += in_len; in_processed = in_len; } } else { /* not a gap */ gint err; if (resample->num_gap_samples > filt_len) { /* push in enough zeros to restore the filter to the right offset */ guint num, den; resample->funcs->get_ratio (resample->state, &num, &den); gst_audio_resample_dump_drain (resample, (resample->num_gap_samples - filt_len) % num); } resample->num_gap_samples = 0; if (resample->num_nongap_samples < filt_len) { resample->num_nongap_samples += in_len; if (resample->num_nongap_samples > filt_len) resample->num_nongap_samples = filt_len; } if (resample->funcs->width != resample->width) { /* need to convert data format for processing; ensure we have enough * workspace available */ if (!gst_audio_resample_workspace_realloc (&resample->tmp_in, &resample->tmp_in_size, in_len * resample->channels * (resample->funcs->width / 8)) || !gst_audio_resample_workspace_realloc (&resample->tmp_out, &resample->tmp_out_size, out_len * resample->channels * (resample->funcs->width / 8))) { GST_ERROR_OBJECT (resample, "failed to allocate workspace"); gst_buffer_unmap (inbuf, &in_map); gst_buffer_unmap (outbuf, &out_map); return GST_FLOW_ERROR; } /* convert input */ gst_audio_resample_convert_buffer (resample, in_map.data, resample->tmp_in, in_len, FALSE); /* process */ err = resample->funcs->process (resample->state, resample->tmp_in, &in_processed, resample->tmp_out, &out_processed); /* convert output */ gst_audio_resample_convert_buffer (resample, resample->tmp_out, out_map.data, out_processed, TRUE); } else { /* no format conversion required; process */ err = resample->funcs->process (resample->state, in_map.data, &in_processed, out_map.data, &out_processed); } if (G_UNLIKELY (err != RESAMPLER_ERR_SUCCESS)) { GST_ERROR_OBJECT (resample, "Failed to convert data: %s", resample->funcs->strerror (err)); gst_buffer_unmap (inbuf, &in_map); gst_buffer_unmap (outbuf, &out_map); return GST_FLOW_ERROR; } } /* If we wrote more than allocated something is really wrong now and we * should better abort immediately */ g_assert (out_len >= out_processed);
commit c5ddadd472521127bdb810fd4ebb5e3e35f10951 Author: Maik Scholz <scholz.maik@t-online.de> Date: Wed Aug 31 11:00:37 2016 +0300 audioresample: Don't produce more data than expected in GAP mode Due to rounding errors in combination with rate changes, we might otherwise end up producing too much and run into an assertion later. This is not a problem with the new audioresample in GIT master. https://bugzilla.gnome.org/show_bug.cgi?id=768836