Index: gstffmpegcodecmap.c
===================================================================
RCS file: /cvs/gstreamer/gst-plugins-base/gst/ffmpegcolorspace/gstffmpegcodecmap.c,v
retrieving revision 1.31
diff -a -u -r1.31 gstffmpegcodecmap.c
--- gstffmpegcodecmap.c	22 Dec 2006 11:09:34 -0000	1.31
+++ gstffmpegcodecmap.c	7 Jan 2007 17:18:04 -0000
@@ -512,13 +512,15 @@
 
 /* Convert a GstCaps (video/raw) to a FFMPEG PixFmt
  * and other video properties in a AVCodecContext.
+ * In guess mode the function always return a pixel format
+ * (though it may be completely wrong)
  *
  * For usefulness, see below
  */
 
-static void
+void
 gst_ffmpeg_caps_to_pixfmt (const GstCaps * caps,
-    AVCodecContext * context, gboolean raw)
+    AVCodecContext * context, gboolean raw, gboolean guess_mode)
 {
   GstStructure *structure;
   const GValue *fps;
@@ -529,18 +531,23 @@
 
   ret = gst_structure_get_int (structure, "width", &context->width);
   ret &= gst_structure_get_int (structure, "height", &context->height);
-  g_return_if_fail (ret == TRUE);
+  g_return_if_fail (guess_mode || ret == TRUE);
 
   fps = gst_structure_get_value (structure, "framerate");
-  g_return_if_fail (GST_VALUE_HOLDS_FRACTION (fps));
-
-  /* framerate does not really matter */
-  context->frame_rate = gst_value_get_fraction_numerator (fps);
-  context->frame_rate_base = gst_value_get_fraction_denominator (fps);
+  if (GST_VALUE_HOLDS_FRACTION (fps)) {
+    /* framerate does not really matter */
+    context->frame_rate = gst_value_get_fraction_numerator (fps);
+    context->frame_rate_base = gst_value_get_fraction_denominator (fps);
+  } else {
+    g_return_if_fail (guess_mode);
+  }
 
   if (!raw)
     return;
 
+  if (guess_mode)
+    context->pix_fmt = PIX_FMT_NB;
+
   if (gst_structure_has_name (structure, "video/x-raw-yuv")) {
     guint32 fourcc;
 
@@ -578,6 +585,11 @@
           break;
       }
     }
+
+    /* Choose a some popular format YUV in guess mode */
+    if (guess_mode && context->pix_fmt == PIX_FMT_NB)
+      context->pix_fmt = PIX_FMT_YUV420P;
+
   } else if (gst_structure_has_name (structure, "video/x-raw-rgb")) {
     gint bpp = 0, rmask = 0, endianness = 0, amask = 0, depth = 0;
 
@@ -655,6 +667,15 @@
         }
       }
     }
+
+    /* Choose a some popular format RGB in guess mode */
+    if (guess_mode && context->pix_fmt == PIX_FMT_NB) {
+      if (gst_structure_get_int (structure, "alpha_mask", &amask))
+        context->pix_fmt = PIX_FMT_RGBA32;
+      else
+        context->pix_fmt = PIX_FMT_RGB32;
+    }
+
   } else if (gst_structure_has_name (structure, "video/x-raw-gray")) {
     gint bpp = 0;
 
@@ -665,7 +686,15 @@
           break;
       }
     }
+
+    /* Choose a some popular grayscale format in guess mode */
+    if (guess_mode && context->pix_fmt == PIX_FMT_NB)
+      context->pix_fmt = PIX_FMT_GRAY8;
   }
+
+  /* Choose a some popular format in guess mode */
+  if (guess_mode && context->pix_fmt == PIX_FMT_NB)
+    context->pix_fmt = PIX_FMT_YUV420P;
 }
 
 /* Convert a GstCaps and a FFMPEG codec Type to a
@@ -684,7 +713,7 @@
 
   switch (type) {
     case CODEC_TYPE_VIDEO:
-      gst_ffmpeg_caps_to_pixfmt (caps, context, TRUE);
+      gst_ffmpeg_caps_to_pixfmt (caps, context, TRUE, FALSE);
       break;
 
     case CODEC_TYPE_AUDIO:
Index: gstffmpegcodecmap.h
===================================================================
RCS file: /cvs/gstreamer/gst-plugins-base/gst/ffmpegcolorspace/gstffmpegcodecmap.h,v
retrieving revision 1.4
diff -a -u -r1.4 gstffmpegcodecmap.h
--- gstffmpegcodecmap.h	6 Dec 2005 19:41:57 -0000	1.4
+++ gstffmpegcodecmap.h	7 Jan 2007 17:18:05 -0000
@@ -43,6 +43,15 @@
                                    AVCodecContext *context);
 
 /*
+ * Convert caps to pixel format.
+ * In guess mode some pixel format is always returned even if it can be wrong.
+ */
+void
+gst_ffmpeg_caps_to_pixfmt (const GstCaps * caps,
+    AVCodecContext * context, gboolean raw, gboolean guess_mode);
+
+
+/*
  * Fill in pointers in an AVPicture, aligned by 4 (required by X).
  */
 
Index: gstffmpegcolorspace.c
===================================================================
RCS file: /cvs/gstreamer/gst-plugins-base/gst/ffmpegcolorspace/gstffmpegcolorspace.c,v
retrieving revision 1.54
diff -a -u -r1.54 gstffmpegcolorspace.c
--- gstffmpegcolorspace.c	30 Aug 2006 17:22:27 -0000	1.54
+++ gstffmpegcolorspace.c	7 Jan 2007 17:18:05 -0000
@@ -84,52 +84,79 @@
 
 /*static guint gst_ffmpegcsp_signals[LAST_SIGNAL] = { 0 }; */
 
-/* copies the given caps */
+/* helper for gst_ffmpegcsp_caps_set_format_info */
+static void
+copy_field (GstStructure * to, GstStructure * from, const gchar *name)
+{
+  const GValue *value;
+
+  value = gst_structure_get_value (from, name);
+  if (value)
+    gst_structure_set_value (to, name, value);
+  else
+    gst_structure_remove_field (to, name);
+}
+
+/* sets format info to caps copy */
 static GstCaps *
-gst_ffmpegcsp_caps_remove_format_info (GstCaps * caps)
+gst_ffmpegcsp_caps_set_format_info (GstCaps * caps, int fmt)
 {
-  int i;
   GstStructure *structure;
-  GstCaps *rgbcaps;
-  GstCaps *graycaps;
+  AVCodecContext *ctx = NULL;
+  GstCaps *fmtcaps;
+  GstStructure *fmtstruct;
+  int i;
+
+  /* Get caps for format fmt */
+  ctx = avcodec_alloc_context ();
+  ctx->pix_fmt = fmt;
+  ctx->width = -1;
+  fmtcaps = gst_ffmpegcsp_codectype_to_caps (CODEC_TYPE_VIDEO, ctx);
+  av_free (ctx);
+  if (!fmtcaps)
+    return NULL;
+  fmtstruct = gst_caps_get_structure (fmtcaps, 0);
 
   caps = gst_caps_copy (caps);
 
+  /* Copy format to each struct */
   for (i = 0; i < gst_caps_get_size (caps); i++) {
     structure = gst_caps_get_structure (caps, i);
 
-    gst_structure_set_name (structure, "video/x-raw-yuv");
-    gst_structure_remove_field (structure, "format");
-    gst_structure_remove_field (structure, "endianness");
-    gst_structure_remove_field (structure, "depth");
-    gst_structure_remove_field (structure, "bpp");
-    gst_structure_remove_field (structure, "red_mask");
-    gst_structure_remove_field (structure, "green_mask");
-    gst_structure_remove_field (structure, "blue_mask");
-    gst_structure_remove_field (structure, "alpha_mask");
-    gst_structure_remove_field (structure, "palette_data");
-  }
-
-  gst_caps_do_simplify (caps);
-  rgbcaps = gst_caps_copy (caps);
-
-  for (i = 0; i < gst_caps_get_size (rgbcaps); i++) {
-    structure = gst_caps_get_structure (rgbcaps, i);
-
-    gst_structure_set_name (structure, "video/x-raw-rgb");
+    gst_structure_set_name (structure, gst_structure_get_name (fmtstruct));
+    copy_field (structure, fmtstruct, "format");
+    copy_field (structure, fmtstruct, "endianness");
+    copy_field (structure, fmtstruct, "depth");
+    copy_field (structure, fmtstruct, "bpp");
+    copy_field (structure, fmtstruct, "red_mask");
+    copy_field (structure, fmtstruct, "green_mask");
+    copy_field (structure, fmtstruct, "blue_mask");
+    copy_field (structure, fmtstruct, "alpha_mask");
+    copy_field (structure, fmtstruct, "palette_data");
   }
-  graycaps = gst_caps_copy (caps);
-
-  for (i = 0; i < gst_caps_get_size (graycaps); i++) {
-    structure = gst_caps_get_structure (graycaps, i);
+  gst_caps_unref (fmtcaps);
+  return caps;
+}
 
-    gst_structure_set_name (structure, "video/x-raw-gray");
-  }
+/* Struct used for sorting formats according to goodness */
+struct FmtGoodness {
+  gint fmt;
+  gint goodness;
+  GstCaps *caps;
+};
 
-  gst_caps_append (caps, graycaps);
-  gst_caps_append (caps, rgbcaps);
+/* function used to sort pixel formats */
+static gint
+compare_fmts (struct FmtGoodness * f1, struct FmtGoodness * f2)
+{
+  return f1->goodness - f2->goodness;
+}
 
-  return caps;
+/* Append caps from format to caps */
+static void
+add_fmt (struct FmtGoodness * f, GstCaps * caps)
+{
+  gst_caps_append (caps, f->caps);
 }
 
 /* The caps can be transformed into any other caps with format info removed.
@@ -142,18 +169,57 @@
   GstFFMpegCsp *space;
   GstCaps *template;
   GstCaps *result;
+  struct FmtGoodness formats[PIX_FMT_NB];
+  GList *sorted_fmts = NULL;
+  AVCodecContext *ctx = NULL;
+  gint fmt;
 
   space = GST_FFMPEGCSP (btrans);
 
+  /* Prefer pass-through mode */
   template = gst_ffmpegcsp_codectype_to_caps (CODEC_TYPE_VIDEO, NULL);
   result = gst_caps_intersect (caps, template);
   gst_caps_unref (template);
 
-  gst_caps_append (result, gst_ffmpegcsp_caps_remove_format_info (caps));
+  /* Nothing more to do for ANY caps */
+  if (G_UNLIKELY (gst_caps_get_size (caps) < 1))
+    goto beach;
+
+  /* Basetransform will provide simple caps, get format */
+  g_return_val_if_fail (gst_caps_get_size (caps) == 1, result);
+  ctx = avcodec_alloc_context ();
+  ctx->pix_fmt = PIX_FMT_NB;
+  gst_ffmpeg_caps_to_pixfmt (caps, ctx, TRUE, TRUE);
+  if (ctx->pix_fmt == PIX_FMT_NB) {
+    GST_DEBUG_OBJECT (space, "could not guess caps colorspace");
+    goto beach;
+  }
+
+  /* Create caps for each format and compute goodness */
+  for (fmt = 0; fmt < PIX_FMT_NB; ++fmt) {
+    formats[fmt].fmt = fmt;
+    formats[fmt].goodness = avcodec_get_pix_fmt_loss (fmt, ctx->pix_fmt, 1);
+    formats[fmt].caps = gst_ffmpegcsp_caps_set_format_info (caps, fmt);
+    if (formats[fmt].caps)
+      sorted_fmts = g_list_append (sorted_fmts, &formats[fmt]);
+  }
+  if (!sorted_fmts)
+    goto beach;
+
+  /* Sort formats according to goodness */
+  sorted_fmts = g_list_sort (sorted_fmts, (GCompareFunc) compare_fmts);
+
+  /* Append formats */
+  g_list_foreach (sorted_fmts, (GFunc)add_fmt, result);
 
   GST_DEBUG_OBJECT (btrans, "transformed %" GST_PTR_FORMAT " into %"
       GST_PTR_FORMAT, caps, result);
 
+beach:
+  if (ctx)
+    av_free (ctx);
+  if (sorted_fmts)
+    g_list_free (sorted_fmts);
   return result;
 }
 
