GNOME Bugzilla – Bug 758060
gst_mini_object_unlock: assertion 'state >= SHARE_ONE' failed
Last modified: 2018-11-03 15:05:53 UTC
Created attachment 315406 [details] minimal pipeline exposing this bug upon setting pipeline state to NULL at shutdown the following assertion is thrown repeatedly countless times: ``` GStreamer-CRITICAL **: gst_mini_object_unlock: assertion 'state >= SHARE_ONE' failed ``` this is accompanied by wild disk thrashing and in the larger project where this was discovered it quite often also renders the entire system unresponsive where the only remedy is to pull the plug i have isolated this behavior to a bin containing a pngdec and imagefreeze and ghostpad src to another bin a .cpp file with a minimal pipeline exposing this is attached - a thread dump and valgrind profile is available in this gist along with the same .cpp file https://gist.github.com/bill-auger/b003a77c20cfbe8a704b
Comment on attachment 315406 [details] minimal pipeline exposing this bug >/* > g++ `pkg-config --cflags gstreamer-1.0` ./pngdec.cpp \ > `pkg-config --libs gstreamer-1.0` -o ./pngdec >*/ > >#include <gst/gst.h> >#include <stdio.h> >#include <unistd.h> > > >GstElement *Pipeline , *ImageBin , *CompositorBin ; > > >bool newGhostPad(GstElement* a_bin , GstElement* an_element , > const gchar* pad_template , const gchar* public_id ) >{ > GstPad *private_pad , *public_pad ; > > bool is_err = !(private_pad = gst_element_get_static_pad(an_element , pad_template)) || > !(public_pad = gst_ghost_pad_new (public_id , private_pad )) || > !gst_pad_set_active(public_pad , TRUE) ; > gst_object_unref(private_pad) ; > > if (is_err || a_bin == NULL|| !gst_element_add_pad(a_bin , public_pad)) > return false ; > > return true ; >} > >int configureImageBin() >{ >#define IMAGE_W "640" >#define IMAGE_H "480" >#define IMAGE_FILENAME "a.png" >#define SCALER_CAPS_STR "video/x-raw, " \ > "width=(int)"IMAGE_W", " \ > "height=(int)"IMAGE_H", " \ > "framerate=(fraction)0/1, " \ > "format=(string)YUY2," \ > "interlace-mode=(string)progressive, " \ > "pixel-aspect-ratio=(fraction)1/1" >#define FREEZER_CAPS_STR "video/x-raw, " \ > "width=(int)"IMAGE_W", " \ > "height=(int)"IMAGE_H", " \ > "framerate=(fraction)8/1, " \ > "format=(string)YUY2," \ > "interlace-mode=(string)progressive, " \ > "pixel-aspect-ratio=(fraction)1/1" > > > GstElement *source , *decoder , *converter , > *scaler , *scaler_filter , > *freezer , *freezer_filter , *queue ; > GstCaps *scaler_caps , *freezer_caps ; > > // instantiate elements > if (!(source = gst_element_factory_make("filesrc" , "image-real-source" )) || > !(decoder = gst_element_factory_make("pngdec" , "image-decoder" )) || > !(converter = gst_element_factory_make("videoconvert" , "image-converter" )) || > !(scaler = gst_element_factory_make("videoscale" , "image-scaler" )) || > !(scaler_filter = gst_element_factory_make("capsfilter" , "image-scaler-caps" )) || > !(freezer = gst_element_factory_make("imagefreeze" , "image-freezer" )) || > !(freezer_filter = gst_element_factory_make("capsfilter" , "image-freezer-caps")) || > !(queue = gst_element_factory_make("queue" , "image-queue" )) || > !(scaler_caps = gst_caps_from_string(SCALER_CAPS_STR ) ) || > !(freezer_caps = gst_caps_from_string(FREEZER_CAPS_STR) ) ) > { printf("IMAGE_BIN_INIT_ERROR_MSG\n") ; return false ; } > > // configure elements > g_object_set(G_OBJECT(source ) , "location" , IMAGE_FILENAME , NULL) ; > g_object_set(G_OBJECT(scaler_filter ) , "caps" , scaler_caps , NULL) ; > g_object_set(G_OBJECT(freezer_filter) , "caps" , freezer_caps , NULL) ; > g_object_set(G_OBJECT(queue ) , "max-size-bytes" , (guint)0 , NULL) ; > g_object_set(G_OBJECT(queue ) , "max-size-time" , (guint64)0 , NULL) ; > g_object_set(G_OBJECT(queue ) , "max-size-buffers" , (guint)0 , NULL) ; > gst_caps_unref(scaler_caps) ; gst_caps_unref(freezer_caps) ; > > // link elements > if (!gst_bin_add(GST_BIN(ImageBin) , source ) || > !gst_bin_add(GST_BIN(ImageBin) , decoder ) || > !gst_bin_add(GST_BIN(ImageBin) , converter ) || > !gst_bin_add(GST_BIN(ImageBin) , scaler ) || > !gst_bin_add(GST_BIN(ImageBin) , scaler_filter ) || > !gst_bin_add(GST_BIN(ImageBin) , freezer ) || > !gst_bin_add(GST_BIN(ImageBin) , freezer_filter) || > !gst_bin_add(GST_BIN(ImageBin) , queue ) || > !gst_element_link(source , decoder ) || > !gst_element_link(decoder , converter ) || > !gst_element_link(converter , scaler ) || > !gst_element_link(scaler , scaler_filter ) || > !gst_element_link(scaler_filter , freezer ) || > !gst_element_link(freezer , freezer_filter) || > !gst_element_link(freezer_filter , queue ) || > !newGhostPad(ImageBin , queue , "src" , "image-source") ) > { printf("IMAGE_BIN_LINK_ERROR_MSG\n") ; return false ; } > > return true ; >} > > >int main(int argc , char* argv[]) >{ > GstElement* sink ; > > gst_init(NULL , NULL) ; > > Pipeline = gst_pipeline_new("pipeline" ) ; > ImageBin = gst_bin_new ("image-bin") ; > CompositorBin = gst_bin_new ("compositor-bin") ; > > gst_bin_add(GST_BIN(Pipeline) , ImageBin ) ; > gst_bin_add(GST_BIN(Pipeline) , CompositorBin) ; > > if (!(sink = gst_element_factory_make("xvimagesink" , "image-sink")) || > !gst_bin_add(GST_BIN(CompositorBin) , sink) || > !newGhostPad(CompositorBin , sink , "sink" , "image-sink") ) > return 255 ; > > if (!configureImageBin()) return 255 ; > > if (!gst_element_link(ImageBin , CompositorBin)) return 255 ; > > gst_element_set_state(Pipeline , GST_STATE_PLAYING) ; > > usleep(10000000); > > gst_element_set_state(Pipeline , GST_STATE_NULL) ; // this line cause the berserk > gst_object_unref(Pipeline) ; > > return 0 ; >}
Thanks, let's keep the code and debug logs in attachments. I can reproduce the issue. It happens because you made the queue unlimited, and imagefreeze will produce buffers as fast as it can, but the video sink will only consumer buffers according to the timestamps. So more and more buffers queue up in the queue. When the first critical happens, this is what queue says it contains: current-level-buffers=33085, current-level-bytes=3 150 626 816, current-level-time=4136 500 000 000 which is a lot, but the buffers all share the same memory, so not sure yet why this would be a problem. So if you want a solution: set a sane limit on the queue (defaults should be fine). I'd like to keep the bug open still until I understand what's going on here.
Hello!! My team and I also noticed the same problem using a large pipeline and happening when freeing buffers in GstRtpRtxQueues with "max-size-packets" of 512. Hence, they do not have an unlimited size and it should be a different case than the reported by @bill-auger. There were a lot of occurrences while both pushing buffers and changing the state as you can see in the next GDB stacktrace (related to GStreamer version 1.8.1): #0 0x00007f4e625e30c1 in g_logv (log_domain=0x7f4e62beaa80 <g_log_domain_gstreamer> "GStreamer", log_level=G_LOG_LEVEL_CRITICAL, format=<optimized out>, args=args@entry=0x7f450da8b558) at gmessages.c:1059 #1 0x00007f4e625e3352 in g_log (log_domain=<optimized out>, log_level=log_level@entry=G_LOG_LEVEL_CRITICAL, format=format@entry=0x7f4e6264f230 "%s: assertion '%s' failed") at gmessages.c:1119 #2 0x00007f4e625e3379 in g_return_if_fail_warning (log_domain=<optimized out>, pretty_function=<optimized out>, expression=<optimized out>) at gmessages.c:1134 #3 0x00007f4e62b543e1 in _gst_buffer_free (buffer=0x7f4cbd2b9790) at gstbuffer.c:676 #4 0x00007f4e2c5c37cd in gst_buffer_unref (buf=<optimized out>) at /usr/include/gstreamer-1.5/gst/gstbuffer.h:366 #5 gst_rtx_queue_buffer_item_free (item=0x7f4c34e58170) at gstrtprtxqueue.c:81 #6 0x00007f4e625ebfef in g_queue_foreach (queue=0x7f4e5cd3d9f0 <list_all_lock>, func=0x7f4e2c5c34d0 <gst_rtx_queue_buffer_item_unref>, user_data=0x0) at gqueue.c:250 #7 0x00007f4e2c5c45bd in gst_rtp_rtx_queue_reset (rtx=0x7f4da40cfb70, full=<optimized out>) at gstrtprtxqueue.c:306 #8 0x00007f4e2c5c4629 in gst_rtp_rtx_queue_change_state (element=0x7f4da40cfb70, transition=GST_STATE_CHANGE_PAUSED_TO_READY) at gstrtprtxqueue.c:695 #9 0x00007f4e62b752ae in gst_element_change_state (element=element@entry=0x7f4da40cfb70, transition=transition@entry=GST_STATE_CHANGE_PAUSED_TO_READY) at gstelement.c:2656 #10 0x00007f4e62b75954 in gst_element_set_state_func (element=0x7f4da40cfb70, state=GST_STATE_READY) at gstelement.c:2610 ...
It might be fixed by https://bugzilla.gnome.org/show_bug.cgi?id=796692, at least one potential symptom of that was the same.
Created attachment 373237 [details] [review] miniobject: lock: check share refs does not go out of bounds I am proposing this patch in order to detect if the counter used for GST_LOCK_FLAG_EXCLUSIVE mode is going out of bounds. It seems that it can have up to 32767 (0x7FFF0000 / 0x10000) refs. Could this be not enough for some use cases and then provoke the warning in gst_mini_object_unlock ? I am not sure if the use cases exposed here are dealing with that, but this check is harmless and useful if the counter is aiming to go out of bounds.
(In reply to Sebastian Dröge (slomo) from comment #4) > It might be fixed by https://bugzilla.gnome.org/show_bug.cgi?id=796692, at > least one potential symptom of that was the same. Great Sebastian!! I will take a look ;)
(In reply to Miguel París Díaz from comment #5) > Created attachment 373237 [details] [review] [review] > miniobject: lock: check share refs does not go out of bounds > > I am proposing this patch in order to detect if the counter used for > GST_LOCK_FLAG_EXCLUSIVE mode is going out of bounds. > > It seems that it can have up to 32767 (0x7FFF0000 / 0x10000) refs. > Could this be not enough for some use cases and then provoke the warning in > gst_mini_object_unlock ? > > I am not sure if the use cases exposed here are dealing with that, but this > check is harmless and useful if the counter is aiming to go out of bounds. How would you get to 32767 *valid* references? :)
I think it is difficult, but if it happens the check will notify you and will avoid you to make crazy.
-- GitLab Migration Automatic Message -- This bug has been migrated to freedesktop.org's GitLab instance and has been closed from further activity. You can subscribe and participate further through the new bug through this link to our GitLab instance: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/237.