diff --git a/bot/access.go b/bot/access.go
index 18208c3..297cf1c 100644
--- a/bot/access.go
+++ b/bot/access.go
@@ -13,8 +13,10 @@ func (b *Bot) allowAnyone(actorID id.UserID, targetRoomID id.RoomID) bool {
 }
 
 func (b *Bot) allowOwner(actorID id.UserID, targetRoomID id.RoomID) bool {
-	if !utils.Match(actorID.String(), b.allowedUsers) {
-		return false
+	if len(b.allowedUsers) != 0 {
+		if !utils.Match(actorID.String(), b.allowedUsers) {
+			return false
+		}
 	}
 
 	if b.noowner {
diff --git a/utils/user.go b/utils/user.go
index 130d91a..3b93c49 100644
--- a/utils/user.go
+++ b/utils/user.go
@@ -23,11 +23,6 @@ func WildcardMXIDsToRegexes(wildCardPatterns []string) ([]*regexp.Regexp, error)
 
 // Match tells if the given user id is allowed to use the bot, according to the given whitelist
 func Match(userID string, allowed []*regexp.Regexp) bool {
-	// No whitelisted users means everyone is whitelisted
-	if len(allowed) == 0 {
-		return true
-	}
-
 	for _, regex := range allowed {
 		if regex.MatchString(userID) {
 			return true
diff --git a/utils/user_test.go b/utils/user_test.go
index f59cbdb..28e87c9 100644
--- a/utils/user_test.go
+++ b/utils/user_test.go
@@ -127,10 +127,10 @@ func TestMatch(t *testing.T) {
 
 	tests := []testDataDefinition{
 		{
-			name:           "Empty allowed users allows anyone",
+			name:           "Empty allowed users allows no one",
 			checkedValue:   "@someone:example.com",
 			allowedUsers:   []string{},
-			expectedResult: true,
+			expectedResult: false,
 		},
 		{
 			name:           "Direct full mxid match is allowed",