diff --git a/libs/ardour/session.cc b/libs/ardour/session.cc
index d8dd4c76a1..7a9f9fb993 100644
--- a/libs/ardour/session.cc
+++ b/libs/ardour/session.cc
@@ -4657,7 +4657,6 @@ Session::remove_source (std::weak_ptr<Source> src, bool drop_references)
 		 */
 		_history.clear();
 	}
-	assert (!source->used ());
 
 	if (source->empty ()) {
 		/* No need to save when empty sources are removed.
@@ -4667,6 +4666,8 @@ Session::remove_source (std::weak_ptr<Source> src, bool drop_references)
 		return;
 	}
 
+	assert (!source->used ());
+
 	if (!in_cleanup () && !loading ()) {
 		/* save state so we don't end up with a session file
 		 * referring to non-existent sources.