construct pipeline as 

videotestsrc !  glvideomixer name=mix ! capsfilter name=capsfilter ! glimagesink videotestsrc name=s2 ! mix. 

and set capsfilter.caps to GL:Memory video, log gives error as 
reason not-negotiated (-4)

seems that gst_gl_mixer_query_caps didn't add GL capsfeatures to caps (line 605)
call gst_gl_mixer_update_caps here fix the problem .

following demo can trigger this problem:

#include <gst/gst.h>

#include <gst/gl/gstglmemory.h>

    
int main(int argc, char *argv[])
{

    gst_init(NULL,NULL);

    GstElement* pipeline=gst_parse_launch("videotestsrc !  glvideomixer name=mix ! capsfilter name=capsfilter ! glimagesink videotestsrc  ! mix. ",NULL);

    GstElement *capsfilter=gst_bin_get_by_name(GST_BIN(pipeline),"capsfilter");


    GstCaps *caps=gst_caps_new_simple("video/x-raw",
            "width",G_TYPE_INT,700,
            "height",G_TYPE_INT,700,NULL);

    gst_caps_set_features(caps,0,
            gst_caps_features_new(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,NULL));

    g_object_set(G_OBJECT(capsfilter),"caps",caps,NULL);

    GMainLoop *loop=g_main_loop_new(NULL,FALSE);


    gst_element_set_state(pipeline,GST_STATE_PLAYING);

    g_main_loop_run(loop);

}