Fix USAC time domain limiter latency at config change.

We have observed quality problems regarding config switching for USAC streams. Crossfading did not consider the USAC time domain limiter latency correctly. The limiter memory still contained the last part of the frame before the config change.
With this patch we were able to improve the quality by moving the limiter processing to the end of the processing chain (crossfade -> DRC -> limiter). By that we don't have to consider the limiter latency at the crossfader anymore and can resolve the quality issue.
Bug: 176246647
Test: atest android.media.cts.DecoderTestAacFormat android.media.cts.DecoderTestXheAac android.media.cts.DecoderTestAacDrc
Change-Id: I0dfd3b76ff2b0daf495ad406283f56a39982ad8f

Change-Id: I26f5da65ef8344602007e180e837820c6a25f173

libAACdec/src/aac_ram.cpp

@@ -1,7 +1,7 @@
 /* -----------------------------------------------------------------------------
 Software License for The Fraunhofer FDK AAC Codec Library for Android
 
-© Copyright  1995 - 2019 Fraunhofer-Gesellschaft zur Förderung der angewandten
+© Copyright  1995 - 2020 Fraunhofer-Gesellschaft zur Förderung der angewandten
 Forschung e.V. All rights reserved.
 
  1.    INTRODUCTION
@@ -148,7 +148,7 @@ C_ALLOC_MEM(CplxPredictionData, CCplxPredictionData, 1)
 /*! The buffer holds time samples for the crossfade in case of an USAC DASH IPF
    config change Dimension: (8)
  */
-C_ALLOC_MEM2(TimeDataFlush, INT_PCM, TIME_DATA_FLUSH_SIZE, (8))
+C_ALLOC_MEM2(TimeDataFlush, PCM_DEC, TIME_DATA_FLUSH_SIZE, (8))
 
 /* @} */
 

libAACdec/src/aac_ram.h

@@ -1,7 +1,7 @@
 /* -----------------------------------------------------------------------------
 Software License for The Fraunhofer FDK AAC Codec Library for Android
 
-© Copyright  1995 - 2019 Fraunhofer-Gesellschaft zur Förderung der angewandten
+© Copyright  1995 - 2020 Fraunhofer-Gesellschaft zur Förderung der angewandten
 Forschung e.V. All rights reserved.
 
  1.    INTRODUCTION
@@ -132,7 +132,7 @@ H_ALLOC_MEM(CplxPredictionData, CCplxPredictionData)
 H_ALLOC_MEM(SpectralCoeffs, FIXP_DBL)
 H_ALLOC_MEM(SpecScale, SHORT)
 
-H_ALLOC_MEM(TimeDataFlush, INT_PCM)
+H_ALLOC_MEM(TimeDataFlush, PCM_DEC)
 
 H_ALLOC_MEM_OVERLAY(WorkBufferCore1, CWorkBufferCore1)
 H_ALLOC_MEM_OVERLAY(WorkBufferCore2, FIXP_DBL)

libAACdec/src/aacdecoder.cpp

@@ -568,7 +568,7 @@ static int CProgramConfigElement_Read(HANDLE_FDK_BITSTREAM bs,
   \return  Error code
 */
 LINKSPEC_CPP AAC_DECODER_ERROR CAacDecoder_PrepareCrossFade(
-    const INT_PCM *pTimeData, INT_PCM **pTimeDataFlush, const INT numChannels,
+    const PCM_DEC *pTimeData, PCM_DEC **pTimeDataFlush, const INT numChannels,
     const INT frameSize, const INT interleaved) {
   int i, ch, s1, s2;
   AAC_DECODER_ERROR ErrorStatus;
@@ -584,7 +584,7 @@ LINKSPEC_CPP AAC_DECODER_ERROR CAacDecoder_PrepareCrossFade(
   }
 
   for (ch = 0; ch < numChannels; ch++) {
-    const INT_PCM *pIn = &pTimeData[ch * s1];
+    const PCM_DEC *pIn = &pTimeData[ch * s1];
     for (i = 0; i < TIME_DATA_FLUSH_SIZE; i++) {
       pTimeDataFlush[ch][i] = *pIn;
       pIn += s2;
@@ -606,7 +606,7 @@ LINKSPEC_CPP AAC_DECODER_ERROR CAacDecoder_PrepareCrossFade(
   \return  Error code
 */
 LINKSPEC_CPP AAC_DECODER_ERROR CAacDecoder_ApplyCrossFade(
-    INT_PCM *pTimeData, INT_PCM **pTimeDataFlush, const INT numChannels,
+    PCM_DEC *pTimeData, PCM_DEC **pTimeDataFlush, const INT numChannels,
     const INT frameSize, const INT interleaved) {
   int i, ch, s1, s2;
   AAC_DECODER_ERROR ErrorStatus;
@@ -622,15 +622,15 @@ LINKSPEC_CPP AAC_DECODER_ERROR CAacDecoder_ApplyCrossFade(
   }
 
   for (ch = 0; ch < numChannels; ch++) {
-    INT_PCM *pIn = &pTimeData[ch * s1];
+    PCM_DEC *pIn = &pTimeData[ch * s1];
     for (i = 0; i < TIME_DATA_FLUSH_SIZE; i++) {
       FIXP_SGL alpha = (FIXP_SGL)i
                        << (FRACT_BITS - 1 - TIME_DATA_FLUSH_SIZE_SF);
-      FIXP_DBL time = FX_PCM2FX_DBL(*pIn);
+      FIXP_DBL time = PCM_DEC2FIXP_DBL(*pIn);
-      FIXP_DBL timeFlush = FX_PCM2FX_DBL(pTimeDataFlush[ch][i]);
+      FIXP_DBL timeFlush = PCM_DEC2FIXP_DBL(pTimeDataFlush[ch][i]);
 
-      *pIn = (INT_PCM)(FIXP_PCM)FX_DBL2FX_PCM(
+      *pIn = FIXP_DBL2PCM_DEC(timeFlush - fMult(timeFlush, alpha) +
-          timeFlush - fMult(timeFlush, alpha) + fMult(time, alpha));
+                              fMult(time, alpha));
       pIn += s2;
     }
   }
@@ -753,7 +753,12 @@ LINKSPEC_CPP AAC_DECODER_ERROR CAacDecoder_PreRollExtensionPayloadParse(
     /* We are interested in preroll AUs if an explicit or an implicit config
      * change is signalized in other words if the build up status is set. */
     if (self->buildUpStatus == AACDEC_USAC_BUILD_UP_ON) {
-      self->applyCrossfade |= FDKreadBit(hBs);
+      UCHAR applyCrossfade = FDKreadBit(hBs);
+      if (applyCrossfade) {
+        self->applyCrossfade |= AACDEC_CROSSFADE_BITMASK_PREROLL;
+      } else {
+        self->applyCrossfade &= ~AACDEC_CROSSFADE_BITMASK_PREROLL;
+      }
       FDKreadBit(hBs); /* reserved */
       /* Read num preroll AU's */
       *numPrerollAU = escapedValue(hBs, 2, 4, 0);

libAACdec/src/aacdecoder.h

@@ -1,7 +1,7 @@
 /* -----------------------------------------------------------------------------
 Software License for The Fraunhofer FDK AAC Codec Library for Android
 
-© Copyright  1995 - 2019 Fraunhofer-Gesellschaft zur Förderung der angewandten
+© Copyright  1995 - 2020 Fraunhofer-Gesellschaft zur Förderung der angewandten
 Forschung e.V. All rights reserved.
 
  1.    INTRODUCTION
@@ -172,6 +172,12 @@ enum {
   AACDEC_RSV60_BUILD_UP_IDLE_IN_BAND = 5
 };
 
+#define AACDEC_CROSSFADE_BITMASK_OFF                                    \
+  ((UCHAR)0) /*!< No cross-fade between frames shall be applied at next \
+                config change. */
+#define AACDEC_CROSSFADE_BITMASK_PREROLL \
+  ((UCHAR)1 << 1) /*!< applyCrossfade is signaled in AudioPreRoll */
+
 typedef struct {
   /* Usac Extension Elements */
   USAC_EXT_ELEMENT_TYPE usacExtElementType[(3)];
@@ -325,7 +331,7 @@ This structure is allocated once for each CPE. */
   UINT loudnessInfoSetPosition[3];
   SCHAR defaultTargetLoudness;
 
-  INT_PCM
+  PCM_DEC
   *pTimeDataFlush[((8) * 2)]; /*!< Pointer to the flushed time data which
                                  will be used for the crossfade in case of
                                  an USAC DASH IPF config change */
@@ -341,8 +347,8 @@ This structure is allocated once for each CPE. */
                                                           start position in the
                                                           bitstream */
   INT accessUnit; /*!< Number of the actual processed preroll accessUnit */
-  UCHAR applyCrossfade; /*!< if set crossfade for seamless stream switching is
+  UCHAR applyCrossfade; /*!< If any bit is set, cross-fade for seamless stream
-                           applied */
+                           switching is applied */
 
   FDK_SignalDelay usacResidualDelay; /*!< Delay residual signal to compensate
                                         for eSBR delay of DMX signal in case of
@@ -439,12 +445,12 @@ LINKSPEC_H AAC_DECODER_ERROR CAacDecoder_FreeMem(HANDLE_AACDECODER self,
 
 /* Prepare crossfade for USAC DASH IPF config change */
 LINKSPEC_H AAC_DECODER_ERROR CAacDecoder_PrepareCrossFade(
-    const INT_PCM *pTimeData, INT_PCM **pTimeDataFlush, const INT numChannels,
+    const PCM_DEC *pTimeData, PCM_DEC **pTimeDataFlush, const INT numChannels,
     const INT frameSize, const INT interleaved);
 
 /* Apply crossfade for USAC DASH IPF config change */
 LINKSPEC_H AAC_DECODER_ERROR CAacDecoder_ApplyCrossFade(
-    INT_PCM *pTimeData, INT_PCM **pTimeDataFlush, const INT numChannels,
+    PCM_DEC *pTimeData, PCM_DEC **pTimeDataFlush, const INT numChannels,
     const INT frameSize, const INT interleaved);
 
 /* Set flush and build up mode */

libAACdec/src/aacdecoder_lib.cpp

@@ -1155,6 +1155,8 @@ LINKSPEC_CPP AAC_DECODER_ERROR aacDecoder_DecodeFrame(HANDLE_AACDECODER self,
   int applyCrossfade = 1;    /* flag indicates if flushing was possible */
   PCM_DEC *pTimeData2;
   PCM_AAC *pTimeData3;
+  INT pcmLimiterScale = 0;
+  INT interleaved = 0;
 
   if (self == NULL) {
     return AAC_DEC_INVALID_HANDLE;
@@ -1800,8 +1802,7 @@ LINKSPEC_CPP AAC_DECODER_ERROR aacDecoder_DecodeFrame(HANDLE_AACDECODER self,
       }
 
       if (self->streamInfo.extAot != AOT_AAC_SLS) {
-        INT pcmLimiterScale = 0;
+        interleaved = 0;
-        INT interleaved = 0;
         interleaved |= (self->sbrEnabled) ? 1 : 0;
         interleaved |= (self->mpsEnableCurr) ? 1 : 0;
         PCMDMX_ERROR dmxErr = PCMDMX_OK;
@@ -1832,7 +1833,80 @@ LINKSPEC_CPP AAC_DECODER_ERROR aacDecoder_DecodeFrame(HANDLE_AACDECODER self,
            * predictable behavior and thus maybe produce strange output. */
           ErrorStatus = AAC_DEC_DECODE_FRAME_ERROR;
         }
+      }
 
+      if (self->flags[0] & AC_USAC) {
+        if (self->flushStatus == AACDEC_USAC_DASH_IPF_FLUSH_ON &&
+            !(flags & AACDEC_CONCEAL)) {
+          CAacDecoder_PrepareCrossFade(pTimeData2, self->pTimeDataFlush,
+                                       self->streamInfo.numChannels,
+                                       self->streamInfo.frameSize, interleaved);
+        }
+
+        /* prepare crossfade buffer for fade in */
+        if (!applyCrossfade &&
+            (self->applyCrossfade != AACDEC_CROSSFADE_BITMASK_OFF) &&
+            !(flags & AACDEC_CONCEAL)) {
+          for (int ch = 0; ch < self->streamInfo.numChannels; ch++) {
+            for (int i = 0; i < TIME_DATA_FLUSH_SIZE; i++) {
+              self->pTimeDataFlush[ch][i] = (PCM_DEC)0;
+            }
+          }
+          applyCrossfade = 1;
+        }
+
+        if (applyCrossfade &&
+            (self->applyCrossfade != AACDEC_CROSSFADE_BITMASK_OFF) &&
+            !(accessUnit < numPrerollAU) &&
+            (self->buildUpStatus == AACDEC_USAC_BUILD_UP_ON)) {
+          CAacDecoder_ApplyCrossFade(pTimeData2, self->pTimeDataFlush,
+                                     self->streamInfo.numChannels,
+                                     self->streamInfo.frameSize, interleaved);
+          self->applyCrossfade =
+              AACDEC_CROSSFADE_BITMASK_OFF; /* disable cross-fade between frames
+                                               at nect config change */
+        }
+      }
+
+      /* Signal interruption to take effect in next frame. */
+      if ((flags & AACDEC_FLUSH || self->flushStatus) &&
+          !(flags & AACDEC_CONCEAL)) {
+        aacDecoder_SignalInterruption(self);
+      }
+
+      /* Update externally visible copy of flags */
+      self->streamInfo.flags = self->flags[0];
+
+    } /* USAC DASH IPF flushing possible end */
+    if (accessUnit < numPrerollAU) {
+      FDKpushBack(hBsAu, auStartAnchor - (INT)FDKgetValidBits(hBsAu));
+    } else {
+      if ((self->buildUpStatus == AACDEC_RSV60_BUILD_UP_ON) ||
+          (self->buildUpStatus == AACDEC_RSV60_BUILD_UP_ON_IN_BAND) ||
+          (self->buildUpStatus == AACDEC_USAC_BUILD_UP_ON)) {
+        self->buildUpCnt--;
+
+        if (self->buildUpCnt < 0) {
+          self->buildUpStatus = 0;
+        }
+      }
+
+      if (self->flags[0] & AC_USAC) {
+        if (self->flushStatus == AACDEC_USAC_DASH_IPF_FLUSH_ON &&
+            !(flags & AACDEC_CONCEAL)) {
+          self->streamInfo.frameSize = 0;
+        }
+      }
+    }
+
+    if (self->flushStatus != AACDEC_USAC_DASH_IPF_FLUSH_ON) {
+      accessUnit++;
+    }
+  } while ((accessUnit < numAccessUnits) ||
+           ((self->flushStatus == AACDEC_USAC_DASH_IPF_FLUSH_ON) &&
+            !(flags & AACDEC_CONCEAL)));
+
+  if (self->streamInfo.extAot != AOT_AAC_SLS) {
     pcmLimiterScale += PCM_OUT_HEADROOM;
 
     if (flags & AACDEC_CLRHIST) {
@@ -1844,14 +1918,12 @@ LINKSPEC_CPP AAC_DECODER_ERROR aacDecoder_DecodeFrame(HANDLE_AACDECODER self,
       }
     }
 
-        /* Set applyExtGain if DRC processing is enabled and if
+    /* Set applyExtGain if DRC processing is enabled and if progRefLevelPresent
-           progRefLevelPresent is present for the first time. Consequences: The
+       is present for the first time. Consequences: The headroom of the output
-           headroom of the output signal can be set to AACDEC_DRC_GAIN_SCALING
+       signal can be set to AACDEC_DRC_GAIN_SCALING only for audio formats which
-           only for audio formats which support legacy DRC Level Normalization.
+       support legacy DRC Level Normalization. For all other audio formats the
-                         For all other audio formats the headroom of the output
+       headroom of the output signal is set to PCM_OUT_HEADROOM. */
-           signal is set to PCM_OUT_HEADROOM. */
+    if (self->hDrcInfo->enable && (self->hDrcInfo->progRefLevelPresent == 1)) {
-        if (self->hDrcInfo->enable &&
-            (self->hDrcInfo->progRefLevelPresent == 1)) {
       self->hDrcInfo->applyExtGain |= 1;
     }
 
@@ -1912,8 +1984,8 @@ LINKSPEC_CPP AAC_DECODER_ERROR aacDecoder_DecodeFrame(HANDLE_AACDECODER self,
     } else {
       if (self->hDrcInfo->enable && self->hDrcInfo->applyExtGain) {
         pcmLimiterScale = applyDrcLevelNormalization(
-                self->hDrcInfo, pTimeData2, self->extGain, NULL,
+            self->hDrcInfo, pTimeData2, self->extGain, NULL, pcmLimiterScale,
-                pcmLimiterScale, self->extGainDelay, self->streamInfo.frameSize,
+            self->extGainDelay, self->streamInfo.frameSize,
             self->streamInfo.numChannels,
             (interleaved || (self->streamInfo.numChannels == 1))
                 ? 1
@@ -1921,9 +1993,9 @@ LINKSPEC_CPP AAC_DECODER_ERROR aacDecoder_DecodeFrame(HANDLE_AACDECODER self,
             0);
       }
 
-          /* If numChannels = 1 we do not need interleaving. The same applies if
+      /* If numChannels = 1 we do not need interleaving. The same applies if SBR
-          SBR or MPS are used, since their output is interleaved already
+      or MPS are used, since their output is interleaved already (resampled or
-          (resampled or not) */
+      not) */
       if ((self->streamInfo.numChannels == 1) || (self->sbrEnabled) ||
           (self->mpsEnableCurr)) {
         scaleValuesSaturate(
@@ -1938,80 +2010,12 @@ LINKSPEC_CPP AAC_DECODER_ERROR aacDecoder_DecodeFrame(HANDLE_AACDECODER self,
             pcmLimiterScale);
         /* Interleave ouput buffer */
         FDK_interleave((INT_PCM *)self->workBufferCore2, pTimeData,
-                           self->streamInfo.numChannels,
+                       self->streamInfo.numChannels, self->streamInfo.frameSize,
-                           self->streamInfo.frameSize,
                        self->streamInfo.frameSize);
       }
     }
   } /* if (self->streamInfo.extAot != AOT_AAC_SLS)*/
 
-      if (self->flags[0] & AC_USAC) {
-        if (self->flushStatus == AACDEC_USAC_DASH_IPF_FLUSH_ON &&
-            !(flags & AACDEC_CONCEAL)) {
-          CAacDecoder_PrepareCrossFade(pTimeData, self->pTimeDataFlush,
-                                       self->streamInfo.numChannels,
-                                       self->streamInfo.frameSize, 1);
-        }
-
-        /* prepare crossfade buffer for fade in */
-        if (!applyCrossfade && self->applyCrossfade &&
-            !(flags & AACDEC_CONCEAL)) {
-          for (int ch = 0; ch < self->streamInfo.numChannels; ch++) {
-            for (int i = 0; i < TIME_DATA_FLUSH_SIZE; i++) {
-              self->pTimeDataFlush[ch][i] = 0;
-            }
-          }
-          applyCrossfade = 1;
-        }
-
-        if (applyCrossfade && self->applyCrossfade &&
-            !(accessUnit < numPrerollAU) &&
-            (self->buildUpStatus == AACDEC_USAC_BUILD_UP_ON)) {
-          CAacDecoder_ApplyCrossFade(pTimeData, self->pTimeDataFlush,
-                                     self->streamInfo.numChannels,
-                                     self->streamInfo.frameSize, 1);
-          self->applyCrossfade = 0;
-        }
-      }
-
-      /* Signal interruption to take effect in next frame. */
-      if ((flags & AACDEC_FLUSH || self->flushStatus) &&
-          !(flags & AACDEC_CONCEAL)) {
-        aacDecoder_SignalInterruption(self);
-      }
-
-      /* Update externally visible copy of flags */
-      self->streamInfo.flags = self->flags[0];
-
-    } /* USAC DASH IPF flushing possible end */
-    if (accessUnit < numPrerollAU) {
-      FDKpushBack(hBsAu, auStartAnchor - (INT)FDKgetValidBits(hBsAu));
-    } else {
-      if ((self->buildUpStatus == AACDEC_RSV60_BUILD_UP_ON) ||
-          (self->buildUpStatus == AACDEC_RSV60_BUILD_UP_ON_IN_BAND) ||
-          (self->buildUpStatus == AACDEC_USAC_BUILD_UP_ON)) {
-        self->buildUpCnt--;
-
-        if (self->buildUpCnt < 0) {
-          self->buildUpStatus = 0;
-        }
-      }
-
-      if (self->flags[0] & AC_USAC) {
-        if (self->flushStatus == AACDEC_USAC_DASH_IPF_FLUSH_ON &&
-            !(flags & AACDEC_CONCEAL)) {
-          self->streamInfo.frameSize = 0;
-        }
-      }
-    }
-
-    if (self->flushStatus != AACDEC_USAC_DASH_IPF_FLUSH_ON) {
-      accessUnit++;
-    }
-  } while ((accessUnit < numAccessUnits) ||
-           ((self->flushStatus == AACDEC_USAC_DASH_IPF_FLUSH_ON) &&
-            !(flags & AACDEC_CONCEAL)));
-
 bail:
 
   /* error in renderer part occurred, ErrorStatus was set to invalid output */