fenix
/
fdk-aac
mirror of https://github.com/mstorsjo/fdk-aac.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Don't try to shift more bits than the variable length 

Shifting by more than (or equal to) the variable length is
undefined in C.

This caused the quantized spectrum values to go out of range,
triggering asserts later.

Change-Id: If81b6c8caa7b9c75941ad9d280b686d2069c968c
This commit is contained in:
Martin Storsjo 2012-08-10 10:57:24 +03:00
parent d2854e2ac7
commit 6574569496
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
libAACenc/src/quantize.cpp
View File

@ -127,7 +127,10 @@ static void FDKaacEnc_quantizeLines(INT gain,
accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]); accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]);
totalShift = (16-4)-(3*(totalShift>>2)); totalShift = (16-4)-(3*(totalShift>>2));
FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */ FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */
if (totalShift < 32)
accu>>=totalShift; accu>>=totalShift;
else
accu = 0;
quaSpectrum[line] = (SHORT)(-((LONG)(k + accu) >> (DFRACT_BITS-1-16))); quaSpectrum[line] = (SHORT)(-((LONG)(k + accu) >> (DFRACT_BITS-1-16)));
} }
else if(accu > FL2FXCONST_DBL(0.0f)) else if(accu > FL2FXCONST_DBL(0.0f))
@ -140,7 +143,10 @@ static void FDKaacEnc_quantizeLines(INT gain,
accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]); accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]);
totalShift = (16-4)-(3*(totalShift>>2)); totalShift = (16-4)-(3*(totalShift>>2));
FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */ FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */
if (totalShift < 32)
accu>>=totalShift; accu>>=totalShift;
else
accu = 0;
quaSpectrum[line] = (SHORT)((LONG)(k + accu) >> (DFRACT_BITS-1-16)); quaSpectrum[line] = (SHORT)((LONG)(k + accu) >> (DFRACT_BITS-1-16));
} }
else else