diff --git a/sys/msdk/gstmsdkbufferpool.c b/sys/msdk/gstmsdkbufferpool.c
index 3ee4f84..bbdc4c0 100644
--- a/sys/msdk/gstmsdkbufferpool.c
+++ b/sys/msdk/gstmsdkbufferpool.c
@@ -236,7 +236,9 @@ gst_msdk_buffer_pool_acquire_buffer (GstBufferPool * pool,
   GstBuffer *buf = NULL;
   GstFlowReturn ret;
   mfxFrameSurface1 *surface;
+  gint retry = 0, retry_max = 4;
 
+retry:
   ret =
       GST_BUFFER_POOL_CLASS (parent_class)->acquire_buffer (pool, &buf, params);
 
@@ -254,15 +256,29 @@ gst_msdk_buffer_pool_acquire_buffer (GstBufferPool * pool,
   surface = gst_msdk_get_surface_from_buffer (buf);
 
   if (surface->Data.Locked > 0) {
-    if (!gst_msdk_video_memory_replace_surface (GST_MSDK_VIDEO_MEMORY_CAST
-            (gst_buffer_peek_memory (buf, 0)))) {
-      GST_WARNING_OBJECT (pool, "failed to replace with new surface");
-      return GST_FLOW_ERROR;
+    /* We should increase outstanding, but impossible here since it's private.
+     * g_atomic_int_inc (&pool->priv->outstanding);
+     */
+
+    /* Also we should reffing pool before unreffing,
+     * this is already being done in gst_buffer_pool_acquire_buffer */
+    buf->pool = gst_object_ref (pool);
+    gst_buffer_unref (buf);
+
+    if (retry < retry_max) {
+      retry++;
+      goto retry;
     }
   }
 
+  if (retry == retry_max) {
+    /* Do allocation just like do_alloc_buffer in gstbufferpool.c
+     * But some are impossible since some of them try to access to private members.
+     */
+  }
+
   *out_buffer_ptr = buf;
-  return GST_FLOW_OK;
+  return ret;
 }
 
 static void
