diff --git a/.github/workflows/pr-auto-manager.yml b/.github/workflows/pr-auto-manager.yml
index d2cf29ee4..46ea54e90 100644
--- a/.github/workflows/pr-auto-manager.yml
+++ b/.github/workflows/pr-auto-manager.yml
@@ -46,6 +46,7 @@ jobs:
         uses: actions/labeler@v5
         with:
           configuration-path: .github/pr-auto-labels-by-branch.yml
+          sync-labels: true # Remove labels when they are not relevant anymore
           repo-token: ${{ secrets.BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
 
   label-by-files:
@@ -59,6 +60,7 @@ jobs:
         uses: actions/labeler@v5
         with:
           configuration-path: .github/pr-auto-labels-by-files.yml
+          sync-labels: true # Remove labels when they are not relevant anymore
           repo-token: ${{ secrets.BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
 
   remove-stale-label: