diff --git a/app/src/main/java/eu/faircode/email/FragmentMessages.java b/app/src/main/java/eu/faircode/email/FragmentMessages.java
index 4a985b3a91..2ba3a49a72 100644
--- a/app/src/main/java/eu/faircode/email/FragmentMessages.java
+++ b/app/src/main/java/eu/faircode/email/FragmentMessages.java
@@ -430,8 +430,8 @@ public class FragmentMessages extends FragmentEx {
             selectionTracker.addObserver(new SelectionTracker.SelectionObserver() {
                 @Override
                 public void onSelectionChanged() {
-                    swipeRefresh.setEnabled(false);
                     if (selectionTracker.hasSelection()) {
+                        swipeRefresh.setEnabled(false);
                         if (messages != null)
                             messages.removeObservers(getViewLifecycleOwner());
                         fabMore.show();