From 5def92cb0fa9284064890d9fe5cb0a54d9e03589 Mon Sep 17 00:00:00 2001
From: Fraunhofer IIS FDK <audio-fdk@iis.fraunhofer.de>
Date: Fri, 3 May 2019 10:25:48 +0200
Subject: [PATCH] Integer overflow fix in FDKaacEnc_initAvoidHoleFlag()

Bug: 121226473
Test: see bug with specific audio stream to encode
Change-Id: I282506f0856130fa00c7f7c6e94b18e1162c8593
---
 libAACenc/src/adj_thr.cpp | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/libAACenc/src/adj_thr.cpp b/libAACenc/src/adj_thr.cpp
index 6e19680..226d003 100644
--- a/libAACenc/src/adj_thr.cpp
+++ b/libAACenc/src/adj_thr.cpp
@@ -1,7 +1,7 @@
 /* -----------------------------------------------------------------------------
 Software License for The Fraunhofer FDK AAC Codec Library for Android
 
-© Copyright  1995 - 2018 Fraunhofer-Gesellschaft zur Förderung der angewandten
+© Copyright  1995 - 2019 Fraunhofer-Gesellschaft zur Förderung der angewandten
 Forschung e.V. All rights reserved.
 
  1.    INTRODUCTION
@@ -591,13 +591,11 @@ static void FDKaacEnc_initAvoidHoleFlag(
           if (sfbEn > avgEn) {
             FIXP_DBL tmpMinSnrLdData;
             if (psyOutChannel[ch]->lastWindowSequence == LONG_WINDOW)
-              tmpMinSnrLdData =
-                  fixMax(SnrLdFac + (FIXP_DBL)(avgEnLdData - sfbEnLdData),
-                         (FIXP_DBL)SnrLdMin1);
+              tmpMinSnrLdData = SnrLdFac + fixMax(avgEnLdData - sfbEnLdData,
+                                                  SnrLdMin1 - SnrLdFac);
             else
-              tmpMinSnrLdData =
-                  fixMax(SnrLdFac + (FIXP_DBL)(avgEnLdData - sfbEnLdData),
-                         (FIXP_DBL)SnrLdMin3);
+              tmpMinSnrLdData = SnrLdFac + fixMax(avgEnLdData - sfbEnLdData,
+                                                  SnrLdMin3 - SnrLdFac);
 
             qcOutChan->sfbMinSnrLdData[sfbGrp + sfb] = fixMin(
                 qcOutChan->sfbMinSnrLdData[sfbGrp + sfb], tmpMinSnrLdData);