avcodec/huffman: extend ff_huff_gen_len_table() to allow >8bit

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/huffman.c

@@ -52,13 +52,18 @@ static void heap_sift(HeapElem *h, int root, int size)
     }
 }
 
-void ff_huff_gen_len_table(uint8_t *dst, const uint64_t *stats)
+int ff_huff_gen_len_table(uint8_t *dst, const uint64_t *stats, int size)
 {
-    HeapElem h[256];
-    int up[2*256];
-    int len[2*256];
+    HeapElem *h  = av_malloc(sizeof(*h) * size);
+    int *up      = av_malloc(sizeof(*up) * 2 * size);
+    uint8_t *len = av_malloc(sizeof(*len) * 2 * size);
     int offset, i, next;
-    int size = 256;
+    int ret = 0;
+
+    if (!h || !up || !len) {
+        ret = AVERROR(ENOMEM);
+        goto end;
+    }
 
     for (offset = 1; ; offset <<= 1) {
         for (i=0; i < size; i++) {
@@ -89,6 +94,11 @@ void ff_huff_gen_len_table(uint8_t *dst, const uint64_t *stats)
         }
         if (i==size) break;
     }
+end:
+    av_free(h);
+    av_free(up);
+    av_free(len);
+    return ret;
 }
 
 static void get_tree_codes(uint32_t *bits, int16_t *lens, uint8_t *xlat,

libavcodec/huffman.h

@@ -43,6 +43,6 @@ typedef int (*HuffCmp)(const void *va, const void *vb);
 int ff_huff_build_tree(AVCodecContext *avctx, VLC *vlc, int nb_codes, int nb_bits,
                        Node *nodes, HuffCmp cmp, int flags);
 
-void ff_huff_gen_len_table(uint8_t *dst, const uint64_t *stats);
+int ff_huff_gen_len_table(uint8_t *dst, const uint64_t *stats, int n);
 
 #endif /* AVCODEC_HUFFMAN_H */

libavcodec/huffyuvenc.c

@@ -144,7 +144,7 @@ static int store_table(HYuvContext *s, const uint8_t *len, uint8_t *buf)
 
 static int store_huffman_tables(HYuvContext *s, uint8_t *buf)
 {
-    int i;
+    int i, ret;
     int size = 0;
     int count = 3;
 
@@ -152,7 +152,8 @@ static int store_huffman_tables(HYuvContext *s, uint8_t *buf)
         count = 1 + s->alpha + 2*s->chroma;
 
     for (i = 0; i < count; i++) {
-        ff_huff_gen_len_table(s->len[i], s->stats[i]);
+        if ((ret = ff_huff_gen_len_table(s->len[i], s->stats[i], 256)) < 0)
+            return ret;
 
         if (ff_huffyuv_generate_bits_table(s->bits[i], s->len[i]) < 0) {
             return -1;

libavcodec/utvideoenc.c

@@ -363,6 +363,7 @@ static int encode_plane(AVCodecContext *avctx, uint8_t *src,
     uint32_t offset = 0, slice_len = 0;
     int      i, sstart, send = 0;
     int      symbol;
+    int      ret;
 
     /* Do prediction / make planes */
     switch (c->frame_pred) {
@@ -429,7 +430,8 @@ static int encode_plane(AVCodecContext *avctx, uint8_t *src,
     }
 
     /* Calculate huffman lengths */
-    ff_huff_gen_len_table(lengths, counts);
+    if ((ret = ff_huff_gen_len_table(lengths, counts, 256)) < 0)
+        return ret;
 
     /*
      * Write the plane's header into the output packet: