diff --git a/autoload/tagbar.vim b/autoload/tagbar.vim
index 163bb4e..eb0cd91 100644
--- a/autoload/tagbar.vim
+++ b/autoload/tagbar.vim
@@ -3889,7 +3889,7 @@ function! s:QuitIfOnlyWindow() abort
     endif
 
     let curwinnr = winnr()
-    let prevwinnr = winnr('#')
+    let prevwinnr = winnr('#') == 0 ? curwinnr : winnr('#')
     call s:goto_win(tagbarwinnr, 1)
 
     " Check if there is more than one window