diff --git a/include/base.h b/include/base.h
index c2c7397d8..f655eb4b6 100644
--- a/include/base.h
+++ b/include/base.h
@@ -116,6 +116,7 @@ public:
 		return *this;
 	}
 
+	inline T* ptr() const { return value; }
 	inline operator bool() const { return (value != nullptr); }
 	inline operator T*() const { return value; }
 	inline T* operator->() const { return value; }
diff --git a/src/coremods/core_channel/extban.cpp b/src/coremods/core_channel/extban.cpp
index 13ecfcb7f..c9c2611fd 100644
--- a/src/coremods/core_channel/extban.cpp
+++ b/src/coremods/core_channel/extban.cpp
@@ -153,8 +153,13 @@ ModResult ExtBanManager::GetStatus(ExtBan::ActingBase* extban, User* user, Chann
 
 void ExtBanManager::DelExtBan(ExtBan::Base* extban)
 {
-	byletter.erase(extban->GetLetter());
-	byname.erase(extban->GetName());
+	auto lit = byletter.find(extban->GetLetter());
+	if (lit != byletter.end() && lit->second->creator.ptr() == extban->creator.ptr())
+		byletter.erase(lit);
+
+	auto nit = byname.find(extban->GetName());
+	if (nit != byname.end() && nit->second->creator.ptr() == extban->creator.ptr())
+		byname.erase(nit);
 }
 
 ExtBan::Base* ExtBanManager::FindName(const std::string& xbname) const