diff --git a/app/src/main/kotlin/com/simplemobiletools/contacts/adapters/ContactsAdapter.kt b/app/src/main/kotlin/com/simplemobiletools/contacts/adapters/ContactsAdapter.kt
index d3716462..820e26e7 100644
--- a/app/src/main/kotlin/com/simplemobiletools/contacts/adapters/ContactsAdapter.kt
+++ b/app/src/main/kotlin/com/simplemobiletools/contacts/adapters/ContactsAdapter.kt
@@ -131,7 +131,7 @@ class ContactsAdapter(activity: SimpleActivity, var contactItems: MutableList<Co
         contactItems.removeAll(favoritesToRemove)
 
         val favoriteIDsToRemove = HashSet<String>()
-        favoritesToRemove.map { favoriteIDsToRemove.add(it.id.toString()) }
+        favoritesToRemove.mapTo(favoriteIDsToRemove, { it.id.toString() })
         activity.config.removeFavorites(favoriteIDsToRemove)
         removeSelectedItems()
     }
diff --git a/app/src/main/kotlin/com/simplemobiletools/contacts/dialogs/AddFavoritesDialog.kt b/app/src/main/kotlin/com/simplemobiletools/contacts/dialogs/AddFavoritesDialog.kt
index 94b83796..bdf5ed80 100644
--- a/app/src/main/kotlin/com/simplemobiletools/contacts/dialogs/AddFavoritesDialog.kt
+++ b/app/src/main/kotlin/com/simplemobiletools/contacts/dialogs/AddFavoritesDialog.kt
@@ -9,6 +9,8 @@ import com.simplemobiletools.contacts.extensions.config
 import com.simplemobiletools.contacts.helpers.ContactsHelper
 import com.simplemobiletools.contacts.models.Contact
 import kotlinx.android.synthetic.main.dialog_add_favorites.view.*
+import java.util.HashSet
+import kotlin.collections.ArrayList
 
 class AddFavoritesDialog(val activity: SimpleActivity, val callback: () -> Unit) {
     private var dialog: AlertDialog? = null
@@ -38,11 +40,16 @@ class AddFavoritesDialog(val activity: SimpleActivity, val callback: () -> Unit)
     }
 
     private fun dialogConfirmed() {
+        val allDisplayedIDs = ArrayList<String>()
+        allContacts.mapTo(allDisplayedIDs, { it.id.toString() })
         val selectedItems = (view.add_favorites_list.adapter as AddFavoritesAdapter).getSelectedItemsSet()
-        if (config.favorites != selectedItems) {
-            config.favorites = selectedItems
-            callback()
-        }
+        config.addFavorites(selectedItems)
+        allDisplayedIDs.removeAll(selectedItems)
+
+        val favoriteIDsToRemove = HashSet<String>()
+        allDisplayedIDs.mapTo(favoriteIDsToRemove, { it })
+        config.removeFavorites(favoriteIDsToRemove)
+        callback()
         dialog?.dismiss()
     }
 }
diff --git a/app/src/main/kotlin/com/simplemobiletools/contacts/helpers/Config.kt b/app/src/main/kotlin/com/simplemobiletools/contacts/helpers/Config.kt
index cbfdf516..e97c84d6 100644
--- a/app/src/main/kotlin/com/simplemobiletools/contacts/helpers/Config.kt
+++ b/app/src/main/kotlin/com/simplemobiletools/contacts/helpers/Config.kt
@@ -49,8 +49,8 @@ class Config(context: Context) : BaseConfig(context) {
         addFavorites(HashSet<String>(Arrays.asList(id)))
     }
 
-    private fun addFavorites(favs: Set<String>) {
-        val currFavorites = HashSet<String>(favs)
+    fun addFavorites(favs: Set<String>) {
+        val currFavorites = HashSet<String>(favorites)
         currFavorites.addAll(favs)
         favorites = currFavorites
     }
diff --git a/app/src/main/res/layout/item_add_favorite_with_number.xml b/app/src/main/res/layout/item_add_favorite_with_number.xml
index b146c91b..be6790eb 100644
--- a/app/src/main/res/layout/item_add_favorite_with_number.xml
+++ b/app/src/main/res/layout/item_add_favorite_with_number.xml
@@ -57,6 +57,7 @@
             android:layout_alignBottom="@+id/contact_tmb"
             android:layout_alignParentRight="true"
             android:layout_alignTop="@+id/contact_tmb"
+            android:clickable="false"
             android:gravity="center_vertical"/>
 
     </RelativeLayout>