diff --git a/labelme/app.py b/labelme/app.py
index 7933fba3a..ad419bc00 100644
--- a/labelme/app.py
+++ b/labelme/app.py
@@ -1611,6 +1611,9 @@ def _set_zoom(self, value: int, pos: QtCore.QPointF | None = None) -> None:
 
         self.actions.fitWidth.setChecked(self._zoom_mode == _ZoomMode.FIT_WIDTH)
         self.actions.fitWindow.setChecked(self._zoom_mode == _ZoomMode.FIT_WINDOW)
+        self.canvas.enableDragging(
+            enabled=value > int(self.scalers[_ZoomMode.FIT_WINDOW]() * 100)
+        )
         self.zoomWidget.setValue(value)  # triggers self._paint_canvas
         self._zoom_values[self.filename] = (self._zoom_mode, value)
 
diff --git a/labelme/widgets/canvas.py b/labelme/widgets/canvas.py
index 85517bef0..cebd3eac6 100644
--- a/labelme/widgets/canvas.py
+++ b/labelme/widgets/canvas.py
@@ -61,6 +61,10 @@ class Canvas(QtWidgets.QWidget):
     prevMovePoint: QPointF
     offsets: tuple[QPointF, QPointF]
 
+    _dragging_start_pos: QPointF
+    _is_dragging: bool
+    _is_dragging_enabled: bool
+
     hVertex: int | None
     hShape: Shape | None
 
@@ -122,6 +126,9 @@ def __init__(self, *args, **kwargs):
         self.hShapeIsSelected = False
         self._painter = QtGui.QPainter()
         self._cursor = CURSOR_DEFAULT
+        self._dragging_start_pos = QPointF()
+        self._is_dragging = False
+        self._is_dragging_enabled = False
         # Menus:
         # 0: right-click without selection and dragging of shapes
         # 1: right-click with selection and dragging of shapes
@@ -313,6 +320,13 @@ def mouseMoveEvent(self, ev):
 
         is_shift_pressed = ev.modifiers() & Qt.ShiftModifier
 
+        if self._is_dragging:
+            self.overrideCursor(CURSOR_GRAB)
+            delta: QPointF = pos - self._dragging_start_pos
+            self.scrollRequest.emit(int(delta.x()), Qt.Horizontal)
+            self.scrollRequest.emit(int(delta.y()), Qt.Vertical)
+            return
+
         # Polygon drawing.
         if self.drawing():
             if self.createMode in ["ai_polygon", "ai_mask"]:
@@ -570,6 +584,10 @@ def mousePressEvent(self, ev):
                 self.selectShapePoint(pos, multiple_selection_mode=group_mode)
                 self.repaint()
             self.prevPoint = pos
+        elif ev.button() == Qt.MiddleButton and self._is_dragging_enabled:
+            self.overrideCursor(CURSOR_GRAB)
+            self._dragging_start_pos = pos
+            self._is_dragging = True
         self._update_status()
 
     def mouseReleaseEvent(self, ev):
@@ -590,6 +608,9 @@ def mouseReleaseEvent(self, ev):
                     self.selectionChanged.emit(
                         [x for x in self.selectedShapes if x != self.hShape]
                     )
+        elif ev.button() == Qt.MiddleButton:
+            self._is_dragging = False
+            self.restoreCursor()
 
         if self.movingShape and self.hShape:
             index = self.shapes.index(self.hShape)
@@ -848,6 +869,9 @@ def transformPos(self, point: QPointF) -> QPointF:
         """Convert from widget-logical coordinates to painter-logical ones."""
         return point / self.scale - self.offsetToCenter()
 
+    def enableDragging(self, enabled: bool):
+        self._is_dragging_enabled = enabled
+
     def offsetToCenter(self) -> QPointF:
         s = self.scale
         area = super().size()
@@ -947,9 +971,15 @@ def sizeHint(self):
         return self.minimumSizeHint()
 
     def minimumSizeHint(self):
-        if self.pixmap:
-            return self.scale * self.pixmap.size()
-        return super().minimumSizeHint()
+        if not self.pixmap:
+            return super().minimumSizeHint()
+
+        min_size = self.scale * self.pixmap.size()
+        if self._is_dragging_enabled:
+            # When drag buffer should be enabled, add a bit of buffer around the image
+            # This lets dragging the image around have a bit of give on the edges
+            min_size = 1.167 * min_size
+        return min_size
 
     def wheelEvent(self, ev: QtGui.QWheelEvent) -> None:
         mods: Qt.KeyboardModifiers = ev.modifiers()