Commit 1ba08c94 authored by Justin Ruggles's avatar Justin Ruggles

vorbisenc: add output buffer overwrite protection

parent fe78470a
...@@ -137,13 +137,16 @@ typedef struct { ...@@ -137,13 +137,16 @@ typedef struct {
#define RESIDUE_PART_SIZE 32 #define RESIDUE_PART_SIZE 32
#define NUM_RESIDUE_PARTITIONS (RESIDUE_SIZE/RESIDUE_PART_SIZE) #define NUM_RESIDUE_PARTITIONS (RESIDUE_SIZE/RESIDUE_PART_SIZE)
static inline void put_codeword(PutBitContext *pb, vorbis_enc_codebook *cb, static inline int put_codeword(PutBitContext *pb, vorbis_enc_codebook *cb,
int entry) int entry)
{ {
assert(entry >= 0); assert(entry >= 0);
assert(entry < cb->nentries); assert(entry < cb->nentries);
assert(cb->lens[entry]); assert(cb->lens[entry]);
if (pb->size_in_bits - put_bits_count(pb) < cb->lens[entry])
return AVERROR(EINVAL);
put_bits(pb, cb->lens[entry], cb->codewords[entry]); put_bits(pb, cb->lens[entry], cb->codewords[entry]);
return 0;
} }
static int cb_lookup_vals(int lookup, int dimentions, int entries) static int cb_lookup_vals(int lookup, int dimentions, int entries)
...@@ -751,7 +754,7 @@ static int render_point(int x0, int y0, int x1, int y1, int x) ...@@ -751,7 +754,7 @@ static int render_point(int x0, int y0, int x1, int y1, int x)
return y0 + (x - x0) * (y1 - y0) / (x1 - x0); return y0 + (x - x0) * (y1 - y0) / (x1 - x0);
} }
static void floor_encode(vorbis_enc_context *venc, vorbis_enc_floor *fc, static int floor_encode(vorbis_enc_context *venc, vorbis_enc_floor *fc,
PutBitContext *pb, uint16_t *posts, PutBitContext *pb, uint16_t *posts,
float *floor, int samples) float *floor, int samples)
{ {
...@@ -759,6 +762,8 @@ static void floor_encode(vorbis_enc_context *venc, vorbis_enc_floor *fc, ...@@ -759,6 +762,8 @@ static void floor_encode(vorbis_enc_context *venc, vorbis_enc_floor *fc,
int coded[MAX_FLOOR_VALUES]; // first 2 values are unused int coded[MAX_FLOOR_VALUES]; // first 2 values are unused
int i, counter; int i, counter;
if (pb->size_in_bits - put_bits_count(pb) < 1 + 2 * ilog(range - 1))
return AVERROR(EINVAL);
put_bits(pb, 1, 1); // non zero put_bits(pb, 1, 1); // non zero
put_bits(pb, ilog(range - 1), posts[0]); put_bits(pb, ilog(range - 1), posts[0]);
put_bits(pb, ilog(range - 1), posts[1]); put_bits(pb, ilog(range - 1), posts[1]);
...@@ -816,7 +821,8 @@ static void floor_encode(vorbis_enc_context *venc, vorbis_enc_floor *fc, ...@@ -816,7 +821,8 @@ static void floor_encode(vorbis_enc_context *venc, vorbis_enc_floor *fc,
cval |= l << cshift; cval |= l << cshift;
cshift += c->subclass; cshift += c->subclass;
} }
put_codeword(pb, book, cval); if (put_codeword(pb, book, cval))
return AVERROR(EINVAL);
} }
for (k = 0; k < c->dim; k++) { for (k = 0; k < c->dim; k++) {
int book = c->books[cval & (csub-1)]; int book = c->books[cval & (csub-1)];
...@@ -826,12 +832,15 @@ static void floor_encode(vorbis_enc_context *venc, vorbis_enc_floor *fc, ...@@ -826,12 +832,15 @@ static void floor_encode(vorbis_enc_context *venc, vorbis_enc_floor *fc,
continue; continue;
if (entry == -1) if (entry == -1)
entry = 0; entry = 0;
put_codeword(pb, &venc->codebooks[book], entry); if (put_codeword(pb, &venc->codebooks[book], entry))
return AVERROR(EINVAL);
} }
} }
ff_vorbis_floor1_render_list(fc->list, fc->values, posts, coded, ff_vorbis_floor1_render_list(fc->list, fc->values, posts, coded,
fc->multiplier, floor, samples); fc->multiplier, floor, samples);
return 0;
} }
static float *put_vector(vorbis_enc_codebook *book, PutBitContext *pb, static float *put_vector(vorbis_enc_codebook *book, PutBitContext *pb,
...@@ -852,11 +861,12 @@ static float *put_vector(vorbis_enc_codebook *book, PutBitContext *pb, ...@@ -852,11 +861,12 @@ static float *put_vector(vorbis_enc_codebook *book, PutBitContext *pb,
distance = d; distance = d;
} }
} }
put_codeword(pb, book, entry); if (put_codeword(pb, book, entry))
return NULL;
return &book->dimentions[entry * book->ndimentions]; return &book->dimentions[entry * book->ndimentions];
} }
static void residue_encode(vorbis_enc_context *venc, vorbis_enc_residue *rc, static int residue_encode(vorbis_enc_context *venc, vorbis_enc_residue *rc,
PutBitContext *pb, float *coeffs, int samples, PutBitContext *pb, float *coeffs, int samples,
int real_ch) int real_ch)
{ {
...@@ -894,7 +904,8 @@ static void residue_encode(vorbis_enc_context *venc, vorbis_enc_residue *rc, ...@@ -894,7 +904,8 @@ static void residue_encode(vorbis_enc_context *venc, vorbis_enc_residue *rc,
entry *= rc->classifications; entry *= rc->classifications;
entry += classes[j][p + i]; entry += classes[j][p + i];
} }
put_codeword(pb, book, entry); if (put_codeword(pb, book, entry))
return AVERROR(EINVAL);
} }
for (i = 0; i < classwords && p < partitions; i++, p++) { for (i = 0; i < classwords && p < partitions; i++, p++) {
for (j = 0; j < channels; j++) { for (j = 0; j < channels; j++) {
...@@ -909,8 +920,10 @@ static void residue_encode(vorbis_enc_context *venc, vorbis_enc_residue *rc, ...@@ -909,8 +920,10 @@ static void residue_encode(vorbis_enc_context *venc, vorbis_enc_residue *rc,
if (rc->type == 0) { if (rc->type == 0) {
for (k = 0; k < psize; k += book->ndimentions) { for (k = 0; k < psize; k += book->ndimentions) {
float *a = put_vector(book, pb, &buf[k]);
int l; int l;
float *a = put_vector(book, pb, &buf[k]);
if (!a)
return AVERROR(EINVAL);
for (l = 0; l < book->ndimentions; l++) for (l = 0; l < book->ndimentions; l++)
buf[k + l] -= a[l]; buf[k + l] -= a[l];
} }
...@@ -930,6 +943,8 @@ static void residue_encode(vorbis_enc_context *venc, vorbis_enc_residue *rc, ...@@ -930,6 +943,8 @@ static void residue_encode(vorbis_enc_context *venc, vorbis_enc_residue *rc,
} }
} }
pv = put_vector(book, pb, vec); pv = put_vector(book, pb, vec);
if (!pv)
return AVERROR(EINVAL);
for (dim = book->ndimentions; dim--; ) { for (dim = book->ndimentions; dim--; ) {
coeffs[a1 + b1] -= *pv++; coeffs[a1 + b1] -= *pv++;
if ((a1 += samples) == s) { if ((a1 += samples) == s) {
...@@ -943,6 +958,7 @@ static void residue_encode(vorbis_enc_context *venc, vorbis_enc_residue *rc, ...@@ -943,6 +958,7 @@ static void residue_encode(vorbis_enc_context *venc, vorbis_enc_residue *rc,
} }
} }
} }
return 0;
} }
static int apply_window_and_mdct(vorbis_enc_context *venc, const signed short *audio, static int apply_window_and_mdct(vorbis_enc_context *venc, const signed short *audio,
...@@ -1017,6 +1033,11 @@ static int vorbis_encode_frame(AVCodecContext *avccontext, ...@@ -1017,6 +1033,11 @@ static int vorbis_encode_frame(AVCodecContext *avccontext,
init_put_bits(&pb, packets, buf_size); init_put_bits(&pb, packets, buf_size);
if (pb.size_in_bits - put_bits_count(&pb) < 1 + ilog(venc->nmodes - 1)) {
av_log(avccontext, AV_LOG_ERROR, "output buffer is too small\n");
return AVERROR(EINVAL);
}
put_bits(&pb, 1, 0); // magic bit put_bits(&pb, 1, 0); // magic bit
put_bits(&pb, ilog(venc->nmodes - 1), 0); // 0 bits, the mode put_bits(&pb, ilog(venc->nmodes - 1), 0); // 0 bits, the mode
...@@ -1032,7 +1053,10 @@ static int vorbis_encode_frame(AVCodecContext *avccontext, ...@@ -1032,7 +1053,10 @@ static int vorbis_encode_frame(AVCodecContext *avccontext,
vorbis_enc_floor *fc = &venc->floors[mapping->floor[mapping->mux[i]]]; vorbis_enc_floor *fc = &venc->floors[mapping->floor[mapping->mux[i]]];
uint16_t posts[MAX_FLOOR_VALUES]; uint16_t posts[MAX_FLOOR_VALUES];
floor_fit(venc, fc, &venc->coeffs[i * samples], posts, samples); floor_fit(venc, fc, &venc->coeffs[i * samples], posts, samples);
floor_encode(venc, fc, &pb, posts, &venc->floor[i * samples], samples); if (floor_encode(venc, fc, &pb, posts, &venc->floor[i * samples], samples)) {
av_log(avccontext, AV_LOG_ERROR, "output buffer is too small\n");
return AVERROR(EINVAL);
}
} }
for (i = 0; i < venc->channels * samples; i++) for (i = 0; i < venc->channels * samples; i++)
...@@ -1052,8 +1076,11 @@ static int vorbis_encode_frame(AVCodecContext *avccontext, ...@@ -1052,8 +1076,11 @@ static int vorbis_encode_frame(AVCodecContext *avccontext,
} }
} }
residue_encode(venc, &venc->residues[mapping->residue[mapping->mux[0]]], if (residue_encode(venc, &venc->residues[mapping->residue[mapping->mux[0]]],
&pb, venc->coeffs, samples, venc->channels); &pb, venc->coeffs, samples, venc->channels)) {
av_log(avccontext, AV_LOG_ERROR, "output buffer is too small\n");
return AVERROR(EINVAL);
}
avccontext->coded_frame->pts = venc->sample_count; avccontext->coded_frame->pts = venc->sample_count;
venc->sample_count += avccontext->frame_size; venc->sample_count += avccontext->frame_size;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment