final word on RGBA byte order

Originally committed as revision 2268 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/avcodec.h

@@ -115,7 +115,19 @@ enum CodecType {
 };
 
 /**
- * Pixel format.
+ * Pixel format. Notes: 
+ *
+ * PIX_FMT_RGBA32 is handled in an endian-specific manner. A RGBA
+ * color is put together as:
+ *  (A << 24) | (R << 16) | (G << 8) | B
+ * This is stored as BGRA on little endian CPU architectures and ARGB on
+ * big endian CPUs.
+ *
+ * When the pixel format is palettized RGB (PIX_FMT_PAL8), the palettized
+ * image data is stored in AVFrame.data[0]. The palette is transported in
+ * AVFrame.data[1] and, is 1024 bytes long (256 4-byte entries) and is
+ * formatted the same as in PIX_FMT_RGBA32 described above (i.e., it is
+ * also endian-specific).
  */
 enum PixelFormat {
     PIX_FMT_YUV420P,   ///< Planar YUV 4:2:0 (1 Cr & Cb sample per 2x2 Y samples)
@@ -124,7 +136,7 @@ enum PixelFormat {
     PIX_FMT_BGR24,     ///< Packed pixel, 3 bytes per pixel, BGRBGR...
     PIX_FMT_YUV422P,   ///< Planar YUV 4:2:2 (1 Cr & Cb sample per 2x1 Y samples)
     PIX_FMT_YUV444P,   ///< Planar YUV 4:4:4 (1 Cr & Cb sample per 1x1 Y samples)
-    PIX_FMT_RGBA32,    ///< Packed pixel, 4 bytes per pixel, BGRABGRA...
+    PIX_FMT_RGBA32,    ///< Packed pixel, 4 bytes per pixel, BGRABGRA..., stored in cpu endianness
     PIX_FMT_YUV410P,   ///< Planar YUV 4:1:0 (1 Cr & Cb sample per 4x4 Y samples)
     PIX_FMT_YUV411P,   ///< Planar YUV 4:1:1 (1 Cr & Cb sample per 4x1 Y samples)
     PIX_FMT_RGB565,    ///< always stored in cpu endianness 

libavcodec/xan.c

@@ -265,17 +265,6 @@ static void inline xan_wc3_build_palette(XanContext *s,
 
     switch (s->avctx->pix_fmt) {
 
-    case PIX_FMT_PAL8:
-        for (i = 0; i < PALETTE_COUNT; i++) {
-            r = *palette_data++;
-            g = *palette_data++;
-            b = *palette_data++;
-            s->palette[i * 4 + 0] = b;
-            s->palette[i * 4 + 1] = g;
-            s->palette[i * 4 + 2] = r;
-        }
-        break;
-
     case PIX_FMT_RGB555:
         palette16 = (unsigned short *)s->palette;
         for (i = 0; i < PALETTE_COUNT; i++) {
@@ -321,6 +310,7 @@ static void inline xan_wc3_build_palette(XanContext *s,
         }
         break;
 
+    case PIX_FMT_PAL8:
     case PIX_FMT_RGBA32:
         palette32 = (unsigned int *)s->palette;
         for (i = 0; i < PALETTE_COUNT; i++) {