Reapply: Avoid reading out of bounds due to negative aaIccIndexMapped

Fixes: 10325/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_LIBFDK_AAC_fuzzer-5740113355603968

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
This commit is contained in:
Martin Storsjo 2017-11-20 12:35:32 +02:00
parent 651ff34d8d
commit e6bb256130
1 changed files with 6 additions and 3 deletions

View File

@ -329,7 +329,7 @@ void initSlotBasedRotation(
FIXP_SGL invL;
FIXP_DBL ScaleL, ScaleR;
FIXP_DBL Alpha, Beta;
FIXP_DBL Alpha, Beta, AlphasValue;
FIXP_DBL h11r, h12r, h21r, h22r;
const FIXP_DBL *PScaleFactors;
@ -363,12 +363,15 @@ void initSlotBasedRotation(
ScaleL = PScaleFactors[noIidSteps - h_ps_d->specificTo.mpeg.pCoef
->aaIidIndexMapped[env][bin]];
AlphasValue = 0;
if (h_ps_d->specificTo.mpeg.pCoef->aaIccIndexMapped[env][bin] >= 0)
AlphasValue = Alphas[h_ps_d->specificTo.mpeg.pCoef->aaIccIndexMapped[env][bin]];
Beta = fMult(
fMult(Alphas[h_ps_d->specificTo.mpeg.pCoef->aaIccIndexMapped[env][bin]],
fMult(AlphasValue,
(ScaleR - ScaleL)),
FIXP_SQRT05);
Alpha =
Alphas[h_ps_d->specificTo.mpeg.pCoef->aaIccIndexMapped[env][bin]] >> 1;
AlphasValue >> 1;
/* Alpha and Beta are now both scaled by 2 shifts right */