From 09e7c40a3aa31a47edd272269325e72b40668e90 Mon Sep 17 00:00:00 2001 From: Fraunhofer IIS FDK Date: Fri, 17 Apr 2020 15:07:13 +0200 Subject: [PATCH] Fix USAC time domain limiter latency at config change. We have observed quality problems regarding config switching for USAC streams. Crossfading did not consider the USAC time domain limiter latency correctly. The limiter memory still contained the last part of the frame before the config change. With this patch we were able to improve the quality by moving the limiter processing to the end of the processing chain (crossfade -> DRC -> limiter). By that we don't have to consider the limiter latency at the crossfader anymore and can resolve the quality issue. Bug: 176246647 Test: atest android.media.cts.DecoderTestAacFormat android.media.cts.DecoderTestXheAac android.media.cts.DecoderTestAacDrc Change-Id: I0dfd3b76ff2b0daf495ad406283f56a39982ad8f Change-Id: I26f5da65ef8344602007e180e837820c6a25f173 --- libAACdec/src/aac_ram.cpp | 4 +- libAACdec/src/aac_ram.h | 4 +- libAACdec/src/aacdecoder.cpp | 23 +-- libAACdec/src/aacdecoder.h | 18 ++- libAACdec/src/aacdecoder_lib.cpp | 248 ++++++++++++++++--------------- 5 files changed, 156 insertions(+), 141 deletions(-) diff --git a/libAACdec/src/aac_ram.cpp b/libAACdec/src/aac_ram.cpp index aa8f6a6..fac1540 100644 --- a/libAACdec/src/aac_ram.cpp +++ b/libAACdec/src/aac_ram.cpp @@ -1,7 +1,7 @@ /* ----------------------------------------------------------------------------- Software License for The Fraunhofer FDK AAC Codec Library for Android -© Copyright 1995 - 2019 Fraunhofer-Gesellschaft zur Förderung der angewandten +© Copyright 1995 - 2020 Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. All rights reserved. 1. INTRODUCTION @@ -148,7 +148,7 @@ C_ALLOC_MEM(CplxPredictionData, CCplxPredictionData, 1) /*! The buffer holds time samples for the crossfade in case of an USAC DASH IPF config change Dimension: (8) */ -C_ALLOC_MEM2(TimeDataFlush, INT_PCM, TIME_DATA_FLUSH_SIZE, (8)) +C_ALLOC_MEM2(TimeDataFlush, PCM_DEC, TIME_DATA_FLUSH_SIZE, (8)) /* @} */ diff --git a/libAACdec/src/aac_ram.h b/libAACdec/src/aac_ram.h index b9b95b7..395b2b2 100644 --- a/libAACdec/src/aac_ram.h +++ b/libAACdec/src/aac_ram.h @@ -1,7 +1,7 @@ /* ----------------------------------------------------------------------------- Software License for The Fraunhofer FDK AAC Codec Library for Android -© Copyright 1995 - 2019 Fraunhofer-Gesellschaft zur Förderung der angewandten +© Copyright 1995 - 2020 Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. All rights reserved. 1. INTRODUCTION @@ -132,7 +132,7 @@ H_ALLOC_MEM(CplxPredictionData, CCplxPredictionData) H_ALLOC_MEM(SpectralCoeffs, FIXP_DBL) H_ALLOC_MEM(SpecScale, SHORT) -H_ALLOC_MEM(TimeDataFlush, INT_PCM) +H_ALLOC_MEM(TimeDataFlush, PCM_DEC) H_ALLOC_MEM_OVERLAY(WorkBufferCore1, CWorkBufferCore1) H_ALLOC_MEM_OVERLAY(WorkBufferCore2, FIXP_DBL) diff --git a/libAACdec/src/aacdecoder.cpp b/libAACdec/src/aacdecoder.cpp index c6d1832..c18e5e9 100644 --- a/libAACdec/src/aacdecoder.cpp +++ b/libAACdec/src/aacdecoder.cpp @@ -568,7 +568,7 @@ static int CProgramConfigElement_Read(HANDLE_FDK_BITSTREAM bs, \return Error code */ LINKSPEC_CPP AAC_DECODER_ERROR CAacDecoder_PrepareCrossFade( - const INT_PCM *pTimeData, INT_PCM **pTimeDataFlush, const INT numChannels, + const PCM_DEC *pTimeData, PCM_DEC **pTimeDataFlush, const INT numChannels, const INT frameSize, const INT interleaved) { int i, ch, s1, s2; AAC_DECODER_ERROR ErrorStatus; @@ -584,7 +584,7 @@ LINKSPEC_CPP AAC_DECODER_ERROR CAacDecoder_PrepareCrossFade( } for (ch = 0; ch < numChannels; ch++) { - const INT_PCM *pIn = &pTimeData[ch * s1]; + const PCM_DEC *pIn = &pTimeData[ch * s1]; for (i = 0; i < TIME_DATA_FLUSH_SIZE; i++) { pTimeDataFlush[ch][i] = *pIn; pIn += s2; @@ -606,7 +606,7 @@ LINKSPEC_CPP AAC_DECODER_ERROR CAacDecoder_PrepareCrossFade( \return Error code */ LINKSPEC_CPP AAC_DECODER_ERROR CAacDecoder_ApplyCrossFade( - INT_PCM *pTimeData, INT_PCM **pTimeDataFlush, const INT numChannels, + PCM_DEC *pTimeData, PCM_DEC **pTimeDataFlush, const INT numChannels, const INT frameSize, const INT interleaved) { int i, ch, s1, s2; AAC_DECODER_ERROR ErrorStatus; @@ -622,15 +622,15 @@ LINKSPEC_CPP AAC_DECODER_ERROR CAacDecoder_ApplyCrossFade( } for (ch = 0; ch < numChannels; ch++) { - INT_PCM *pIn = &pTimeData[ch * s1]; + PCM_DEC *pIn = &pTimeData[ch * s1]; for (i = 0; i < TIME_DATA_FLUSH_SIZE; i++) { FIXP_SGL alpha = (FIXP_SGL)i << (FRACT_BITS - 1 - TIME_DATA_FLUSH_SIZE_SF); - FIXP_DBL time = FX_PCM2FX_DBL(*pIn); - FIXP_DBL timeFlush = FX_PCM2FX_DBL(pTimeDataFlush[ch][i]); + FIXP_DBL time = PCM_DEC2FIXP_DBL(*pIn); + FIXP_DBL timeFlush = PCM_DEC2FIXP_DBL(pTimeDataFlush[ch][i]); - *pIn = (INT_PCM)(FIXP_PCM)FX_DBL2FX_PCM( - timeFlush - fMult(timeFlush, alpha) + fMult(time, alpha)); + *pIn = FIXP_DBL2PCM_DEC(timeFlush - fMult(timeFlush, alpha) + + fMult(time, alpha)); pIn += s2; } } @@ -753,7 +753,12 @@ LINKSPEC_CPP AAC_DECODER_ERROR CAacDecoder_PreRollExtensionPayloadParse( /* We are interested in preroll AUs if an explicit or an implicit config * change is signalized in other words if the build up status is set. */ if (self->buildUpStatus == AACDEC_USAC_BUILD_UP_ON) { - self->applyCrossfade |= FDKreadBit(hBs); + UCHAR applyCrossfade = FDKreadBit(hBs); + if (applyCrossfade) { + self->applyCrossfade |= AACDEC_CROSSFADE_BITMASK_PREROLL; + } else { + self->applyCrossfade &= ~AACDEC_CROSSFADE_BITMASK_PREROLL; + } FDKreadBit(hBs); /* reserved */ /* Read num preroll AU's */ *numPrerollAU = escapedValue(hBs, 2, 4, 0); diff --git a/libAACdec/src/aacdecoder.h b/libAACdec/src/aacdecoder.h index bd1f38f..002807f 100644 --- a/libAACdec/src/aacdecoder.h +++ b/libAACdec/src/aacdecoder.h @@ -1,7 +1,7 @@ /* ----------------------------------------------------------------------------- Software License for The Fraunhofer FDK AAC Codec Library for Android -© Copyright 1995 - 2019 Fraunhofer-Gesellschaft zur Förderung der angewandten +© Copyright 1995 - 2020 Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. All rights reserved. 1. INTRODUCTION @@ -172,6 +172,12 @@ enum { AACDEC_RSV60_BUILD_UP_IDLE_IN_BAND = 5 }; +#define AACDEC_CROSSFADE_BITMASK_OFF \ + ((UCHAR)0) /*!< No cross-fade between frames shall be applied at next \ + config change. */ +#define AACDEC_CROSSFADE_BITMASK_PREROLL \ + ((UCHAR)1 << 1) /*!< applyCrossfade is signaled in AudioPreRoll */ + typedef struct { /* Usac Extension Elements */ USAC_EXT_ELEMENT_TYPE usacExtElementType[(3)]; @@ -325,7 +331,7 @@ This structure is allocated once for each CPE. */ UINT loudnessInfoSetPosition[3]; SCHAR defaultTargetLoudness; - INT_PCM + PCM_DEC *pTimeDataFlush[((8) * 2)]; /*!< Pointer to the flushed time data which will be used for the crossfade in case of an USAC DASH IPF config change */ @@ -341,8 +347,8 @@ This structure is allocated once for each CPE. */ start position in the bitstream */ INT accessUnit; /*!< Number of the actual processed preroll accessUnit */ - UCHAR applyCrossfade; /*!< if set crossfade for seamless stream switching is - applied */ + UCHAR applyCrossfade; /*!< If any bit is set, cross-fade for seamless stream + switching is applied */ FDK_SignalDelay usacResidualDelay; /*!< Delay residual signal to compensate for eSBR delay of DMX signal in case of @@ -439,12 +445,12 @@ LINKSPEC_H AAC_DECODER_ERROR CAacDecoder_FreeMem(HANDLE_AACDECODER self, /* Prepare crossfade for USAC DASH IPF config change */ LINKSPEC_H AAC_DECODER_ERROR CAacDecoder_PrepareCrossFade( - const INT_PCM *pTimeData, INT_PCM **pTimeDataFlush, const INT numChannels, + const PCM_DEC *pTimeData, PCM_DEC **pTimeDataFlush, const INT numChannels, const INT frameSize, const INT interleaved); /* Apply crossfade for USAC DASH IPF config change */ LINKSPEC_H AAC_DECODER_ERROR CAacDecoder_ApplyCrossFade( - INT_PCM *pTimeData, INT_PCM **pTimeDataFlush, const INT numChannels, + PCM_DEC *pTimeData, PCM_DEC **pTimeDataFlush, const INT numChannels, const INT frameSize, const INT interleaved); /* Set flush and build up mode */ diff --git a/libAACdec/src/aacdecoder_lib.cpp b/libAACdec/src/aacdecoder_lib.cpp index 3ef7140..90563ea 100644 --- a/libAACdec/src/aacdecoder_lib.cpp +++ b/libAACdec/src/aacdecoder_lib.cpp @@ -1155,6 +1155,8 @@ LINKSPEC_CPP AAC_DECODER_ERROR aacDecoder_DecodeFrame(HANDLE_AACDECODER self, int applyCrossfade = 1; /* flag indicates if flushing was possible */ PCM_DEC *pTimeData2; PCM_AAC *pTimeData3; + INT pcmLimiterScale = 0; + INT interleaved = 0; if (self == NULL) { return AAC_DEC_INVALID_HANDLE; @@ -1800,8 +1802,7 @@ LINKSPEC_CPP AAC_DECODER_ERROR aacDecoder_DecodeFrame(HANDLE_AACDECODER self, } if (self->streamInfo.extAot != AOT_AAC_SLS) { - INT pcmLimiterScale = 0; - INT interleaved = 0; + interleaved = 0; interleaved |= (self->sbrEnabled) ? 1 : 0; interleaved |= (self->mpsEnableCurr) ? 1 : 0; PCMDMX_ERROR dmxErr = PCMDMX_OK; @@ -1832,145 +1833,38 @@ LINKSPEC_CPP AAC_DECODER_ERROR aacDecoder_DecodeFrame(HANDLE_AACDECODER self, * predictable behavior and thus maybe produce strange output. */ ErrorStatus = AAC_DEC_DECODE_FRAME_ERROR; } - - pcmLimiterScale += PCM_OUT_HEADROOM; - - if (flags & AACDEC_CLRHIST) { - if (!(self->flags[0] & AC_USAC)) { - /* Reset DRC data */ - aacDecoder_drcReset(self->hDrcInfo); - /* Delete the delayed signal. */ - pcmLimiter_Reset(self->hLimiter); - } - } - - /* Set applyExtGain if DRC processing is enabled and if - progRefLevelPresent is present for the first time. Consequences: The - headroom of the output signal can be set to AACDEC_DRC_GAIN_SCALING - only for audio formats which support legacy DRC Level Normalization. - For all other audio formats the headroom of the output - signal is set to PCM_OUT_HEADROOM. */ - if (self->hDrcInfo->enable && - (self->hDrcInfo->progRefLevelPresent == 1)) { - self->hDrcInfo->applyExtGain |= 1; - } - - /* Check whether time data buffer is large enough. */ - if (timeDataSize < - (self->streamInfo.numChannels * self->streamInfo.frameSize)) { - ErrorStatus = AAC_DEC_OUTPUT_BUFFER_TOO_SMALL; - goto bail; - } - - if (self->limiterEnableCurr) { - /* use workBufferCore2 buffer for interleaving */ - PCM_LIM *pInterleaveBuffer; - int blockLength = self->streamInfo.frameSize; - - /* Set actual signal parameters */ - pcmLimiter_SetNChannels(self->hLimiter, self->streamInfo.numChannels); - pcmLimiter_SetSampleRate(self->hLimiter, self->streamInfo.sampleRate); - - if ((self->streamInfo.numChannels == 1) || (self->sbrEnabled) || - (self->mpsEnableCurr)) { - pInterleaveBuffer = (PCM_LIM *)pTimeData2; - } else { - pInterleaveBuffer = (PCM_LIM *)self->workBufferCore2; - - /* applyLimiter requests for interleaved data */ - /* Interleave ouput buffer */ - FDK_interleave(pTimeData2, pInterleaveBuffer, - self->streamInfo.numChannels, blockLength, - self->streamInfo.frameSize); - } - - FIXP_DBL *pGainPerSample = NULL; - - if (self->hDrcInfo->enable && self->hDrcInfo->applyExtGain) { - pGainPerSample = self->workBufferCore1; - - if ((INT)GetRequiredMemWorkBufferCore1() < - (INT)(self->streamInfo.frameSize * sizeof(FIXP_DBL))) { - ErrorStatus = AAC_DEC_UNKNOWN; - goto bail; - } - - pcmLimiterScale = applyDrcLevelNormalization( - self->hDrcInfo, (PCM_DEC *)pInterleaveBuffer, self->extGain, - pGainPerSample, pcmLimiterScale, self->extGainDelay, - self->streamInfo.frameSize, self->streamInfo.numChannels, 1, 1); - } - - pcmLimiter_Apply(self->hLimiter, pInterleaveBuffer, pTimeData, - pGainPerSample, pcmLimiterScale, - self->streamInfo.frameSize); - - { - /* Announce the additional limiter output delay */ - self->streamInfo.outputDelay += pcmLimiter_GetDelay(self->hLimiter); - } - } else { - if (self->hDrcInfo->enable && self->hDrcInfo->applyExtGain) { - pcmLimiterScale = applyDrcLevelNormalization( - self->hDrcInfo, pTimeData2, self->extGain, NULL, - pcmLimiterScale, self->extGainDelay, self->streamInfo.frameSize, - self->streamInfo.numChannels, - (interleaved || (self->streamInfo.numChannels == 1)) - ? 1 - : self->streamInfo.frameSize, - 0); - } - - /* If numChannels = 1 we do not need interleaving. The same applies if - SBR or MPS are used, since their output is interleaved already - (resampled or not) */ - if ((self->streamInfo.numChannels == 1) || (self->sbrEnabled) || - (self->mpsEnableCurr)) { - scaleValuesSaturate( - pTimeData, pTimeData2, - self->streamInfo.frameSize * self->streamInfo.numChannels, - pcmLimiterScale); - - } else { - scaleValuesSaturate( - (INT_PCM *)self->workBufferCore2, pTimeData2, - self->streamInfo.frameSize * self->streamInfo.numChannels, - pcmLimiterScale); - /* Interleave ouput buffer */ - FDK_interleave((INT_PCM *)self->workBufferCore2, pTimeData, - self->streamInfo.numChannels, - self->streamInfo.frameSize, - self->streamInfo.frameSize); - } - } - } /* if (self->streamInfo.extAot != AOT_AAC_SLS)*/ + } if (self->flags[0] & AC_USAC) { if (self->flushStatus == AACDEC_USAC_DASH_IPF_FLUSH_ON && !(flags & AACDEC_CONCEAL)) { - CAacDecoder_PrepareCrossFade(pTimeData, self->pTimeDataFlush, + CAacDecoder_PrepareCrossFade(pTimeData2, self->pTimeDataFlush, self->streamInfo.numChannels, - self->streamInfo.frameSize, 1); + self->streamInfo.frameSize, interleaved); } /* prepare crossfade buffer for fade in */ - if (!applyCrossfade && self->applyCrossfade && + if (!applyCrossfade && + (self->applyCrossfade != AACDEC_CROSSFADE_BITMASK_OFF) && !(flags & AACDEC_CONCEAL)) { for (int ch = 0; ch < self->streamInfo.numChannels; ch++) { for (int i = 0; i < TIME_DATA_FLUSH_SIZE; i++) { - self->pTimeDataFlush[ch][i] = 0; + self->pTimeDataFlush[ch][i] = (PCM_DEC)0; } } applyCrossfade = 1; } - if (applyCrossfade && self->applyCrossfade && + if (applyCrossfade && + (self->applyCrossfade != AACDEC_CROSSFADE_BITMASK_OFF) && !(accessUnit < numPrerollAU) && (self->buildUpStatus == AACDEC_USAC_BUILD_UP_ON)) { - CAacDecoder_ApplyCrossFade(pTimeData, self->pTimeDataFlush, + CAacDecoder_ApplyCrossFade(pTimeData2, self->pTimeDataFlush, self->streamInfo.numChannels, - self->streamInfo.frameSize, 1); - self->applyCrossfade = 0; + self->streamInfo.frameSize, interleaved); + self->applyCrossfade = + AACDEC_CROSSFADE_BITMASK_OFF; /* disable cross-fade between frames + at nect config change */ } } @@ -2012,6 +1906,116 @@ LINKSPEC_CPP AAC_DECODER_ERROR aacDecoder_DecodeFrame(HANDLE_AACDECODER self, ((self->flushStatus == AACDEC_USAC_DASH_IPF_FLUSH_ON) && !(flags & AACDEC_CONCEAL))); + if (self->streamInfo.extAot != AOT_AAC_SLS) { + pcmLimiterScale += PCM_OUT_HEADROOM; + + if (flags & AACDEC_CLRHIST) { + if (!(self->flags[0] & AC_USAC)) { + /* Reset DRC data */ + aacDecoder_drcReset(self->hDrcInfo); + /* Delete the delayed signal. */ + pcmLimiter_Reset(self->hLimiter); + } + } + + /* Set applyExtGain if DRC processing is enabled and if progRefLevelPresent + is present for the first time. Consequences: The headroom of the output + signal can be set to AACDEC_DRC_GAIN_SCALING only for audio formats which + support legacy DRC Level Normalization. For all other audio formats the + headroom of the output signal is set to PCM_OUT_HEADROOM. */ + if (self->hDrcInfo->enable && (self->hDrcInfo->progRefLevelPresent == 1)) { + self->hDrcInfo->applyExtGain |= 1; + } + + /* Check whether time data buffer is large enough. */ + if (timeDataSize < + (self->streamInfo.numChannels * self->streamInfo.frameSize)) { + ErrorStatus = AAC_DEC_OUTPUT_BUFFER_TOO_SMALL; + goto bail; + } + + if (self->limiterEnableCurr) { + /* use workBufferCore2 buffer for interleaving */ + PCM_LIM *pInterleaveBuffer; + int blockLength = self->streamInfo.frameSize; + + /* Set actual signal parameters */ + pcmLimiter_SetNChannels(self->hLimiter, self->streamInfo.numChannels); + pcmLimiter_SetSampleRate(self->hLimiter, self->streamInfo.sampleRate); + + if ((self->streamInfo.numChannels == 1) || (self->sbrEnabled) || + (self->mpsEnableCurr)) { + pInterleaveBuffer = (PCM_LIM *)pTimeData2; + } else { + pInterleaveBuffer = (PCM_LIM *)self->workBufferCore2; + + /* applyLimiter requests for interleaved data */ + /* Interleave ouput buffer */ + FDK_interleave(pTimeData2, pInterleaveBuffer, + self->streamInfo.numChannels, blockLength, + self->streamInfo.frameSize); + } + + FIXP_DBL *pGainPerSample = NULL; + + if (self->hDrcInfo->enable && self->hDrcInfo->applyExtGain) { + pGainPerSample = self->workBufferCore1; + + if ((INT)GetRequiredMemWorkBufferCore1() < + (INT)(self->streamInfo.frameSize * sizeof(FIXP_DBL))) { + ErrorStatus = AAC_DEC_UNKNOWN; + goto bail; + } + + pcmLimiterScale = applyDrcLevelNormalization( + self->hDrcInfo, (PCM_DEC *)pInterleaveBuffer, self->extGain, + pGainPerSample, pcmLimiterScale, self->extGainDelay, + self->streamInfo.frameSize, self->streamInfo.numChannels, 1, 1); + } + + pcmLimiter_Apply(self->hLimiter, pInterleaveBuffer, pTimeData, + pGainPerSample, pcmLimiterScale, + self->streamInfo.frameSize); + + { + /* Announce the additional limiter output delay */ + self->streamInfo.outputDelay += pcmLimiter_GetDelay(self->hLimiter); + } + } else { + if (self->hDrcInfo->enable && self->hDrcInfo->applyExtGain) { + pcmLimiterScale = applyDrcLevelNormalization( + self->hDrcInfo, pTimeData2, self->extGain, NULL, pcmLimiterScale, + self->extGainDelay, self->streamInfo.frameSize, + self->streamInfo.numChannels, + (interleaved || (self->streamInfo.numChannels == 1)) + ? 1 + : self->streamInfo.frameSize, + 0); + } + + /* If numChannels = 1 we do not need interleaving. The same applies if SBR + or MPS are used, since their output is interleaved already (resampled or + not) */ + if ((self->streamInfo.numChannels == 1) || (self->sbrEnabled) || + (self->mpsEnableCurr)) { + scaleValuesSaturate( + pTimeData, pTimeData2, + self->streamInfo.frameSize * self->streamInfo.numChannels, + pcmLimiterScale); + + } else { + scaleValuesSaturate( + (INT_PCM *)self->workBufferCore2, pTimeData2, + self->streamInfo.frameSize * self->streamInfo.numChannels, + pcmLimiterScale); + /* Interleave ouput buffer */ + FDK_interleave((INT_PCM *)self->workBufferCore2, pTimeData, + self->streamInfo.numChannels, self->streamInfo.frameSize, + self->streamInfo.frameSize); + } + } + } /* if (self->streamInfo.extAot != AOT_AAC_SLS)*/ + bail: /* error in renderer part occurred, ErrorStatus was set to invalid output */