diff --git a/src/Application.vala b/src/Application.vala
index 1f195cd..be4aa3b 100644
--- a/src/Application.vala
+++ b/src/Application.vala
@@ -21,6 +21,7 @@
 public class Badger.Application : Gtk.Application {
     public bool headless = false;
     public bool ask_autostart = false;
+    public Reminder[] reminders;
 
     private Badger.MainWindow window;
 
@@ -40,7 +41,7 @@ public class Badger.Application : Gtk.Application {
         var quit_action = new SimpleAction ("quit", null);
         add_action (quit_action);
         set_accels_for_action ("app.quit", {"<Control>q"});
-        quit_action.activate.connect (quit);
+        quit_action.activate.connect (tidy_up);
     }
 
     protected override void activate () {
@@ -76,7 +77,7 @@ public class Badger.Application : Gtk.Application {
         }
 
         if (window == null) {
-            var reminders = set_up_reminders ();
+            reminders = set_up_reminders ();
             var main = new MainGrid (reminders);
             window = new MainWindow (this, main);
 
@@ -214,4 +215,11 @@ public class Badger.Application : Gtk.Application {
         );
         return reminders;
     }
+
+    public void tidy_up () {
+        foreach (var reminder in reminders) {
+            withdraw_notification (reminder.name);
+        };
+        quit ();
+    }
 }
diff --git a/src/MainWindow.vala b/src/MainWindow.vala
index a7577e1..5c9f2e8 100644
--- a/src/MainWindow.vala
+++ b/src/MainWindow.vala
@@ -89,8 +89,8 @@ public class Badger.MainWindow : Gtk.Window {
 
         if (!settings.get_boolean ("all")) {
             debug ("All reminders are disabled, Badger will now go to sleep");
-            application.quit ();
-        };
+            ((Badger.Application)this.application).tidy_up ();
+        }
 
         return false;
     }