diff --git a/src/main_gui.cpp b/src/main_gui.cpp
index df9417f523..469c15c488 100644
--- a/src/main_gui.cpp
+++ b/src/main_gui.cpp
@@ -87,11 +87,10 @@ void CcPlaySound_EXPLOSION(Commands, const CommandCost &result, TileIndex tile)
  * Zooms a viewport in a window in or out.
  * @param how Zooming direction.
  * @param w   Window owning the viewport.
- * @param stop_following Should we stop following the vehicle in the viewport?
  * @return Returns \c true if zooming step could be done, \c false if further zooming is not possible.
  * @note No button handling or what so ever is done.
  */
-bool DoZoomInOutWindow(ZoomStateChange how, Window *w, bool stop_following)
+bool DoZoomInOutWindow(ZoomStateChange how, Window *w)
 {
 	Viewport *vp;
 
@@ -113,7 +112,6 @@ bool DoZoomInOutWindow(ZoomStateChange how, Window *w, bool stop_following)
 			w->viewport->scrollpos_y += vp->virtual_height >> 1;
 			w->viewport->dest_scrollpos_x = w->viewport->scrollpos_x;
 			w->viewport->dest_scrollpos_y = w->viewport->scrollpos_y;
-			if (stop_following) w->viewport->follow_vehicle = INVALID_VEHICLE;
 			break;
 		case ZOOM_OUT:
 			if (vp->zoom >= _settings_client.gui.zoom_max) return false;
@@ -126,7 +124,6 @@ bool DoZoomInOutWindow(ZoomStateChange how, Window *w, bool stop_following)
 
 			vp->virtual_width <<= 1;
 			vp->virtual_height <<= 1;
-			if (stop_following) w->viewport->follow_vehicle = INVALID_VEHICLE;
 			break;
 	}
 	if (vp != nullptr) { // the vp can be null when how == ZOOM_NONE
@@ -436,7 +433,14 @@ struct MainWindow : Window
 	void OnMouseWheel(int wheel) override
 	{
 		if (_settings_client.gui.scrollwheel_scrolling != SWS_OFF) {
-			ZoomInOrOutToCursorWindow(wheel < 0, this);
+			bool in = wheel < 0;
+
+			/* When following, only change zoom - otherwise zoom to the cursor. */
+			if (this->viewport->follow_vehicle != INVALID_VEHICLE) {
+				DoZoomInOutWindow(in ? ZOOM_IN : ZOOM_OUT, this);
+			} else {
+				ZoomInOrOutToCursorWindow(in, this);
+			}
 		}
 	}
 
diff --git a/src/vehicle_gui.cpp b/src/vehicle_gui.cpp
index e6123606d4..2a43b351a3 100644
--- a/src/vehicle_gui.cpp
+++ b/src/vehicle_gui.cpp
@@ -3234,7 +3234,7 @@ public:
 					ShowExtraViewportWindow(TileVirtXY(v->x_pos, v->y_pos));
 				} else {
 					const Window *mainwindow = GetMainWindow();
-					if (click_count > 1 && mainwindow->viewport->zoom <= ZOOM_LVL_NORMAL) {
+					if (click_count > 1) {
 						/* main window 'follows' vehicle */
 						mainwindow->viewport->follow_vehicle = v->index;
 					} else {
@@ -3320,7 +3320,7 @@ public:
 	void OnMouseWheel(int wheel) override
 	{
 		if (_settings_client.gui.scrollwheel_scrolling != SWS_OFF) {
-			DoZoomInOutWindow(wheel < 0 ? ZOOM_IN : ZOOM_OUT, this, false);
+			DoZoomInOutWindow(wheel < 0 ? ZOOM_IN : ZOOM_OUT, this);
 		}
 	}
 
diff --git a/src/viewport_func.h b/src/viewport_func.h
index 30fec9d287..5b1537478c 100644
--- a/src/viewport_func.h
+++ b/src/viewport_func.h
@@ -30,7 +30,7 @@ void UpdateViewportPosition(Window *w, uint32_t delta_ms);
 
 bool MarkAllViewportsDirty(int left, int top, int right, int bottom);
 
-bool DoZoomInOutWindow(ZoomStateChange how, Window *w, bool stop_following = true);
+bool DoZoomInOutWindow(ZoomStateChange how, Window *w);
 void ZoomInOrOutToCursorWindow(bool in, Window * w);
 void ConstrainAllViewportsZoom();
 Point GetTileZoomCenterWindow(bool in, Window * w);
diff --git a/src/window.cpp b/src/window.cpp
index 75d02ee713..9f215d4593 100644
--- a/src/window.cpp
+++ b/src/window.cpp
@@ -2683,15 +2683,20 @@ static void HandleAutoscroll()
 	y -= vp->top;
 
 	/* here allows scrolling in both x and y axis */
+	/* If we succeed at scrolling in any direction, stop following a vehicle. */
 	static const int SCROLLSPEED = 3;
 	if (x - 15 < 0) {
+		w->viewport->follow_vehicle = INVALID_VEHICLE;
 		w->viewport->dest_scrollpos_x += ScaleByZoom((x - 15) * SCROLLSPEED, vp->zoom);
 	} else if (15 - (vp->width - x) > 0) {
+		w->viewport->follow_vehicle = INVALID_VEHICLE;
 		w->viewport->dest_scrollpos_x += ScaleByZoom((15 - (vp->width - x)) * SCROLLSPEED, vp->zoom);
 	}
 	if (y - 15 < 0) {
+		w->viewport->follow_vehicle = INVALID_VEHICLE;
 		w->viewport->dest_scrollpos_y += ScaleByZoom((y - 15) * SCROLLSPEED, vp->zoom);
 	} else if (15 - (vp->height - y) > 0) {
+		w->viewport->follow_vehicle = INVALID_VEHICLE;
 		w->viewport->dest_scrollpos_y += ScaleByZoom((15 - (vp->height - y)) * SCROLLSPEED, vp->zoom);
 	}
 }
@@ -2755,6 +2760,10 @@ static void HandleKeyScrolling()
 	 */
 	if (_dirkeys && !EditBoxInGlobalFocus()) {
 		int factor = _shift_pressed ? 50 : 10;
+
+		/* Key scrolling stops following a vehicle. */
+		GetMainWindow()->viewport->follow_vehicle = INVALID_VEHICLE;
+
 		ScrollMainViewport(scrollamt[_dirkeys][0] * factor, scrollamt[_dirkeys][1] * factor);
 	}
 }