diff --git a/ext/ffmpeg/gstffmpegdec.c b/ext/ffmpeg/gstffmpegdec.c
index 67dcf22..46ad517 100644
--- a/ext/ffmpeg/gstffmpegdec.c
+++ b/ext/ffmpeg/gstffmpegdec.c
@@ -2084,11 +2084,14 @@ gst_ffmpegdec_audio_frame (GstFFMpegDec * ffmpegdec,
     /*
      * Timestamps:
      *
-     *  1) Copy input timestamp if valid
+     *  1) Copy input timestamp if valid - but only if codec parser doesn't have any data buffered
+     *     If the codec parser has any data buffered input timestamps can't be used (because the actual
+     *     data might be originating from previous buffer)
      *  2) else interpolate from previous input timestamp
      */
     /* always take timestamps from the input buffer if any */
-    if (GST_CLOCK_TIME_IS_VALID (dec_info->timestamp)) {
+    if (GST_CLOCK_TIME_IS_VALID (dec_info->timestamp) && 
+        (ffmpegdec->pctx->offset == 0 || !GST_CLOCK_TIME_IS_VALID(ffmpegdec->next_out))) {
       out_timestamp = dec_info->timestamp;
     } else {
       out_timestamp = ffmpegdec->next_out;
