diff --git a/src/SelectionManager.ts b/src/SelectionManager.ts
index 11fb65b4..55e95735 100644
--- a/src/SelectionManager.ts
+++ b/src/SelectionManager.ts
@@ -317,15 +317,10 @@ export class SelectionManager extends EventEmitter {
    * @param event The mousedown event.
    */
   private _onMouseDown(event: MouseEvent) {
-    // Ignore this event when selection is disabled
-    if (!this._enabled) {
-      if (!(Browser.isLinux ? event.shiftKey : event.altKey)) {
-        return;
-      } else {
-
-        // Don't send the mouse down event to the current process
-        event.stopPropagation();
-      }
+    // If we have selection, we want the context menu on right click
+    if (event.button === 2 && this.hasSelection) {
+      event.stopPropagation();
+      return;
     }
 
     // Only action the primary button
@@ -333,6 +328,18 @@ export class SelectionManager extends EventEmitter {
       return;
     }
 
+    // Allow selection when using a specific modifier key, even when disabled
+    if (!this._enabled) {
+      const shouldForceSelection = Browser.isMac ? event.altKey : event.shiftKey;
+
+      if (!shouldForceSelection) {
+        return;
+      }
+
+      // Don't send the mouse down event to the current process, we want to select
+      event.stopPropagation();
+    }
+
     // Tell the browser not to start a regular selection
     event.preventDefault();