diff --git a/src/browser/services/SelectionService.ts b/src/browser/services/SelectionService.ts
index 90c85ecd..2efb095d 100644
--- a/src/browser/services/SelectionService.ts
+++ b/src/browser/services/SelectionService.ts
@@ -664,7 +664,7 @@ export class SelectionService implements ISelectionService {
     this._removeMouseDownListeners();
 
     if (this.selectionText.length <= 1 && timeElapsed < ALT_CLICK_MOVE_CURSOR_TIME) {
-      if (event.altKey) {
+      if (event.altKey && this._bufferService.buffer.ybase === this._bufferService.buffer.ydisp) {
         const coordinates = this._mouseService.getCoords(
           event,
           this._element,