diff --git a/packages/block-editor/src/components/block-tools/back-compat.js b/packages/block-editor/src/components/block-tools/back-compat.js
index 3597bc07b07f79..029419926e9ed5 100644
--- a/packages/block-editor/src/components/block-tools/back-compat.js
+++ b/packages/block-editor/src/components/block-tools/back-compat.js
@@ -9,7 +9,7 @@ import deprecated from '@wordpress/deprecated';
  * Internal dependencies
  */
 import InsertionPoint, { InsertionPointOpenRef } from './insertion-point';
-import BlockPopover from './selected-block-popover';
+import BlockPopover from './selected-block-tools';
 
 export default function BlockToolsBackCompat( { children } ) {
 	const openRef = useContext( InsertionPointOpenRef );
diff --git a/packages/block-editor/src/components/block-tools/empty-block-inserter.js b/packages/block-editor/src/components/block-tools/empty-block-inserter.js
new file mode 100644
index 00000000000000..1d520ed72b1c69
--- /dev/null
+++ b/packages/block-editor/src/components/block-tools/empty-block-inserter.js
@@ -0,0 +1,56 @@
+/**
+ * External dependencies
+ */
+import classnames from 'classnames';
+
+/**
+ * Internal dependencies
+ */
+import BlockPopover from '../block-popover';
+import useBlockToolbarPopoverProps from './use-block-toolbar-popover-props';
+import Inserter from '../inserter';
+import useSelectedBlockToolProps from './use-selected-block-tool-props';
+
+export default function EmptyBlockInserter( {
+	clientId,
+	__unstableContentRef,
+} ) {
+	const {
+		capturingClientId,
+		isInsertionPointVisible,
+		lastClientId,
+		rootClientId,
+	} = useSelectedBlockToolProps( clientId );
+
+	const popoverProps = useBlockToolbarPopoverProps( {
+		contentElement: __unstableContentRef?.current,
+		clientId,
+	} );
+
+	return (
+		<BlockPopover
+			clientId={ capturingClientId || clientId }
+			__unstableCoverTarget
+			bottomClientId={ lastClientId }
+			className={ classnames(
+				'block-editor-block-list__block-side-inserter-popover',
+				{
+					'is-insertion-point-visible': isInsertionPointVisible,
+				}
+			) }
+			__unstableContentRef={ __unstableContentRef }
+			resize={ false }
+			shift={ false }
+			{ ...popoverProps }
+		>
+			<div className="block-editor-block-list__empty-block-inserter">
+				<Inserter
+					position="bottom right"
+					rootClientId={ rootClientId }
+					clientId={ clientId }
+					__experimentalIsQuick
+				/>
+			</div>
+		</BlockPopover>
+	);
+}
diff --git a/packages/block-editor/src/components/block-tools/index.js b/packages/block-editor/src/components/block-tools/index.js
index 8e3b240838fd04..3a570712778182 100644
--- a/packages/block-editor/src/components/block-tools/index.js
+++ b/packages/block-editor/src/components/block-tools/index.js
@@ -6,28 +6,48 @@ import { useViewportMatch } from '@wordpress/compose';
 import { Popover } from '@wordpress/components';
 import { __unstableUseShortcutEventMatch as useShortcutEventMatch } from '@wordpress/keyboard-shortcuts';
 import { useRef } from '@wordpress/element';
+import { isUnmodifiedDefaultBlock } from '@wordpress/blocks';
 
 /**
  * Internal dependencies
  */
+import EmptyBlockInserter from './empty-block-inserter';
 import {
 	InsertionPointOpenRef,
 	default as InsertionPoint,
 } from './insertion-point';
-import SelectedBlockPopover from './selected-block-popover';
+import SelectedBlockTools from './selected-block-tools';
 import { store as blockEditorStore } from '../../store';
 import BlockContextualToolbar from './block-contextual-toolbar';
 import usePopoverScroll from '../block-popover/use-popover-scroll';
 import ZoomOutModeInserters from './zoom-out-mode-inserters';
 
 function selector( select ) {
-	const { __unstableGetEditorMode, getSettings, isTyping } =
-		select( blockEditorStore );
+	const {
+		getSelectedBlockClientId,
+		getFirstMultiSelectedBlockClientId,
+		getBlock,
+		getSettings,
+		__unstableGetEditorMode,
+		isTyping,
+	} = select( blockEditorStore );
+
+	const clientId =
+		getSelectedBlockClientId() || getFirstMultiSelectedBlockClientId();
+
+	const { name = '', attributes = {} } = getBlock( clientId ) || {};
 
 	return {
-		isZoomOutMode: __unstableGetEditorMode() === 'zoom-out',
+		clientId,
 		hasFixedToolbar: getSettings().hasFixedToolbar,
+		hasSelectedBlock: clientId && name,
 		isTyping: isTyping(),
+		isZoomOutMode: __unstableGetEditorMode() === 'zoom-out',
+		showEmptyBlockSideInserter:
+			clientId &&
+			! isTyping() &&
+			__unstableGetEditorMode() === 'edit' &&
+			isUnmodifiedDefaultBlock( { name, attributes } ),
 	};
 }
 
@@ -46,10 +66,14 @@ export default function BlockTools( {
 	...props
 } ) {
 	const isLargeViewport = useViewportMatch( 'medium' );
-	const { hasFixedToolbar, isZoomOutMode, isTyping } = useSelect(
-		selector,
-		[]
-	);
+	const {
+		clientId,
+		hasFixedToolbar,
+		hasSelectedBlock,
+		isTyping,
+		isZoomOutMode,
+		showEmptyBlockSideInserter,
+	} = useSelect( selector, [] );
 	const isMatch = useShortcutEventMatch();
 	const { getSelectedBlockClientIds, getBlockRootClientId } =
 		useSelect( blockEditorStore );
@@ -142,11 +166,22 @@ export default function BlockTools( {
 					( hasFixedToolbar || ! isLargeViewport ) && (
 						<BlockContextualToolbar isFixed />
 					) }
+
+				{ showEmptyBlockSideInserter && (
+					<EmptyBlockInserter
+						__unstableContentRef={ __unstableContentRef }
+						clientId={ clientId }
+					/>
+				) }
 				{ /* Even if the toolbar is fixed, the block popover is still
 					needed for navigation and zoom-out mode. */ }
-				<SelectedBlockPopover
-					__unstableContentRef={ __unstableContentRef }
-				/>
+				{ ! showEmptyBlockSideInserter && hasSelectedBlock && (
+					<SelectedBlockTools
+						__unstableContentRef={ __unstableContentRef }
+						clientId={ clientId }
+					/>
+				) }
+
 				{ /* Used for the inline rich text toolbar. */ }
 				<Popover.Slot name="block-toolbar" ref={ blockToolbarRef } />
 				{ children }
diff --git a/packages/block-editor/src/components/block-tools/selected-block-popover.js b/packages/block-editor/src/components/block-tools/selected-block-popover.js
deleted file mode 100644
index 8c87d8ea3a4739..00000000000000
--- a/packages/block-editor/src/components/block-tools/selected-block-popover.js
+++ /dev/null
@@ -1,265 +0,0 @@
-/**
- * External dependencies
- */
-import classnames from 'classnames';
-
-/**
- * WordPress dependencies
- */
-import { useRef, useEffect } from '@wordpress/element';
-import { isUnmodifiedDefaultBlock } from '@wordpress/blocks';
-import { useDispatch, useSelect } from '@wordpress/data';
-import { useShortcut } from '@wordpress/keyboard-shortcuts';
-
-/**
- * Internal dependencies
- */
-import BlockSelectionButton from './block-selection-button';
-import BlockContextualToolbar from './block-contextual-toolbar';
-import { store as blockEditorStore } from '../../store';
-import BlockPopover from '../block-popover';
-import useBlockToolbarPopoverProps from './use-block-toolbar-popover-props';
-import Inserter from '../inserter';
-import { useShouldContextualToolbarShow } from '../../utils/use-should-contextual-toolbar-show';
-
-function selector( select ) {
-	const {
-		__unstableGetEditorMode,
-		hasMultiSelection,
-		isTyping,
-		getLastMultiSelectedBlockClientId,
-	} = select( blockEditorStore );
-
-	return {
-		editorMode: __unstableGetEditorMode(),
-		hasMultiSelection: hasMultiSelection(),
-		isTyping: isTyping(),
-		lastClientId: hasMultiSelection()
-			? getLastMultiSelectedBlockClientId()
-			: null,
-	};
-}
-
-function SelectedBlockPopover( {
-	clientId,
-	rootClientId,
-	isEmptyDefaultBlock,
-	capturingClientId,
-	__unstablePopoverSlot,
-	__unstableContentRef,
-} ) {
-	const { editorMode, hasMultiSelection, isTyping, lastClientId } = useSelect(
-		selector,
-		[]
-	);
-
-	const isInsertionPointVisible = useSelect(
-		( select ) => {
-			const {
-				isBlockInsertionPointVisible,
-				getBlockInsertionPoint,
-				getBlockOrder,
-			} = select( blockEditorStore );
-
-			if ( ! isBlockInsertionPointVisible() ) {
-				return false;
-			}
-
-			const insertionPoint = getBlockInsertionPoint();
-			const order = getBlockOrder( insertionPoint.rootClientId );
-			return order[ insertionPoint.index ] === clientId;
-		},
-		[ clientId ]
-	);
-	const isToolbarForced = useRef( false );
-	const { shouldShowContextualToolbar, canFocusHiddenToolbar } =
-		useShouldContextualToolbarShow();
-
-	const { stopTyping } = useDispatch( blockEditorStore );
-
-	const showEmptyBlockSideInserter =
-		! isTyping && editorMode === 'edit' && isEmptyDefaultBlock;
-	const shouldShowBreadcrumb =
-		! hasMultiSelection &&
-		( editorMode === 'navigation' || editorMode === 'zoom-out' );
-
-	useShortcut(
-		'core/block-editor/focus-toolbar',
-		() => {
-			isToolbarForced.current = true;
-			stopTyping( true );
-		},
-		{
-			isDisabled: ! canFocusHiddenToolbar,
-		}
-	);
-
-	useEffect( () => {
-		isToolbarForced.current = false;
-	} );
-
-	// Stores the active toolbar item index so the block toolbar can return focus
-	// to it when re-mounting.
-	const initialToolbarItemIndexRef = useRef();
-
-	useEffect( () => {
-		// Resets the index whenever the active block changes so this is not
-		// persisted. See https://github.com/WordPress/gutenberg/pull/25760#issuecomment-717906169
-		initialToolbarItemIndexRef.current = undefined;
-	}, [ clientId ] );
-
-	const popoverProps = useBlockToolbarPopoverProps( {
-		contentElement: __unstableContentRef?.current,
-		clientId,
-	} );
-
-	if ( showEmptyBlockSideInserter ) {
-		return (
-			<BlockPopover
-				clientId={ capturingClientId || clientId }
-				__unstableCoverTarget
-				bottomClientId={ lastClientId }
-				className={ classnames(
-					'block-editor-block-list__block-side-inserter-popover',
-					{
-						'is-insertion-point-visible': isInsertionPointVisible,
-					}
-				) }
-				__unstablePopoverSlot={ __unstablePopoverSlot }
-				__unstableContentRef={ __unstableContentRef }
-				resize={ false }
-				shift={ false }
-				{ ...popoverProps }
-			>
-				<div className="block-editor-block-list__empty-block-inserter">
-					<Inserter
-						position="bottom right"
-						rootClientId={ rootClientId }
-						clientId={ clientId }
-						__experimentalIsQuick
-					/>
-				</div>
-			</BlockPopover>
-		);
-	}
-
-	if ( shouldShowBreadcrumb || shouldShowContextualToolbar ) {
-		return (
-			<BlockPopover
-				clientId={ capturingClientId || clientId }
-				bottomClientId={ lastClientId }
-				className={ classnames(
-					'block-editor-block-list__block-popover',
-					{
-						'is-insertion-point-visible': isInsertionPointVisible,
-					}
-				) }
-				__unstablePopoverSlot={ __unstablePopoverSlot }
-				__unstableContentRef={ __unstableContentRef }
-				resize={ false }
-				{ ...popoverProps }
-			>
-				{ shouldShowContextualToolbar && (
-					<BlockContextualToolbar
-						// If the toolbar is being shown because of being forced
-						// it should focus the toolbar right after the mount.
-						focusOnMount={ isToolbarForced.current }
-						__experimentalInitialIndex={
-							initialToolbarItemIndexRef.current
-						}
-						__experimentalOnIndexChange={ ( index ) => {
-							initialToolbarItemIndexRef.current = index;
-						} }
-						// Resets the index whenever the active block changes so
-						// this is not persisted. See https://github.com/WordPress/gutenberg/pull/25760#issuecomment-717906169
-						key={ clientId }
-					/>
-				) }
-				{ shouldShowBreadcrumb && (
-					<BlockSelectionButton
-						clientId={ clientId }
-						rootClientId={ rootClientId }
-					/>
-				) }
-			</BlockPopover>
-		);
-	}
-
-	return null;
-}
-
-function wrapperSelector( select ) {
-	const {
-		getSelectedBlockClientId,
-		getFirstMultiSelectedBlockClientId,
-		getBlockRootClientId,
-		getBlock,
-		getBlockParents,
-		__experimentalGetBlockListSettingsForBlocks,
-	} = select( blockEditorStore );
-
-	const clientId =
-		getSelectedBlockClientId() || getFirstMultiSelectedBlockClientId();
-
-	if ( ! clientId ) {
-		return;
-	}
-
-	const { name, attributes = {} } = getBlock( clientId ) || {};
-	const blockParentsClientIds = getBlockParents( clientId );
-
-	// Get Block List Settings for all ancestors of the current Block clientId.
-	const parentBlockListSettings = __experimentalGetBlockListSettingsForBlocks(
-		blockParentsClientIds
-	);
-
-	// Get the clientId of the topmost parent with the capture toolbars setting.
-	const capturingClientId = blockParentsClientIds.find(
-		( parentClientId ) =>
-			parentBlockListSettings[ parentClientId ]
-				?.__experimentalCaptureToolbars
-	);
-
-	return {
-		clientId,
-		rootClientId: getBlockRootClientId( clientId ),
-		name,
-		isEmptyDefaultBlock:
-			name && isUnmodifiedDefaultBlock( { name, attributes } ),
-		capturingClientId,
-	};
-}
-
-export default function WrappedBlockPopover( {
-	__unstablePopoverSlot,
-	__unstableContentRef,
-} ) {
-	const selected = useSelect( wrapperSelector, [] );
-
-	if ( ! selected ) {
-		return null;
-	}
-
-	const {
-		clientId,
-		rootClientId,
-		name,
-		isEmptyDefaultBlock,
-		capturingClientId,
-	} = selected;
-
-	if ( ! name ) {
-		return null;
-	}
-
-	return (
-		<SelectedBlockPopover
-			clientId={ clientId }
-			rootClientId={ rootClientId }
-			isEmptyDefaultBlock={ isEmptyDefaultBlock }
-			capturingClientId={ capturingClientId }
-			__unstablePopoverSlot={ __unstablePopoverSlot }
-			__unstableContentRef={ __unstableContentRef }
-		/>
-	);
-}
diff --git a/packages/block-editor/src/components/block-tools/selected-block-tools.js b/packages/block-editor/src/components/block-tools/selected-block-tools.js
new file mode 100644
index 00000000000000..8fa0ac97d5c941
--- /dev/null
+++ b/packages/block-editor/src/components/block-tools/selected-block-tools.js
@@ -0,0 +1,130 @@
+/**
+ * External dependencies
+ */
+import classnames from 'classnames';
+
+/**
+ * WordPress dependencies
+ */
+import { useRef, useEffect } from '@wordpress/element';
+import { useDispatch, useSelect } from '@wordpress/data';
+import { useShortcut } from '@wordpress/keyboard-shortcuts';
+
+/**
+ * Internal dependencies
+ */
+import BlockSelectionButton from './block-selection-button';
+import BlockContextualToolbar from './block-contextual-toolbar';
+import { store as blockEditorStore } from '../../store';
+import BlockPopover from '../block-popover';
+import useBlockToolbarPopoverProps from './use-block-toolbar-popover-props';
+import useSelectedBlockToolProps from './use-selected-block-tool-props';
+import { useShouldContextualToolbarShow } from '../../utils/use-should-contextual-toolbar-show';
+
+export default function SelectedBlockTools( {
+	clientId,
+	showEmptyBlockSideInserter,
+	__unstableContentRef,
+} ) {
+	const {
+		capturingClientId,
+		isInsertionPointVisible,
+		lastClientId,
+		rootClientId,
+	} = useSelectedBlockToolProps( clientId );
+
+	const { shouldShowBreadcrumb } = useSelect( ( select ) => {
+		const { hasMultiSelection, __unstableGetEditorMode } =
+			select( blockEditorStore );
+
+		const editorMode = __unstableGetEditorMode();
+
+		return {
+			shouldShowBreadcrumb:
+				! hasMultiSelection() &&
+				( editorMode === 'navigation' || editorMode === 'zoom-out' ),
+		};
+	}, [] );
+
+	const isToolbarForced = useRef( false );
+	const { shouldShowContextualToolbar, canFocusHiddenToolbar } =
+		useShouldContextualToolbarShow();
+
+	const { stopTyping } = useDispatch( blockEditorStore );
+
+	useShortcut(
+		'core/block-editor/focus-toolbar',
+		() => {
+			isToolbarForced.current = true;
+			stopTyping( true );
+		},
+		{
+			isDisabled: ! canFocusHiddenToolbar,
+		}
+	);
+
+	useEffect( () => {
+		isToolbarForced.current = false;
+	} );
+
+	// Stores the active toolbar item index so the block toolbar can return focus
+	// to it when re-mounting.
+	const initialToolbarItemIndexRef = useRef();
+
+	useEffect( () => {
+		// Resets the index whenever the active block changes so this is not
+		// persisted. See https://github.com/WordPress/gutenberg/pull/25760#issuecomment-717906169
+		initialToolbarItemIndexRef.current = undefined;
+	}, [ clientId ] );
+
+	const popoverProps = useBlockToolbarPopoverProps( {
+		contentElement: __unstableContentRef?.current,
+		clientId,
+	} );
+
+	if ( showEmptyBlockSideInserter ) {
+		return null;
+	}
+
+	if ( shouldShowBreadcrumb || shouldShowContextualToolbar ) {
+		return (
+			<BlockPopover
+				clientId={ capturingClientId || clientId }
+				bottomClientId={ lastClientId }
+				className={ classnames(
+					'block-editor-block-list__block-popover',
+					{
+						'is-insertion-point-visible': isInsertionPointVisible,
+					}
+				) }
+				resize={ false }
+				{ ...popoverProps }
+			>
+				{ shouldShowContextualToolbar && (
+					<BlockContextualToolbar
+						// If the toolbar is being shown because of being forced
+						// it should focus the toolbar right after the mount.
+						focusOnMount={ isToolbarForced.current }
+						__experimentalInitialIndex={
+							initialToolbarItemIndexRef.current
+						}
+						__experimentalOnIndexChange={ ( index ) => {
+							initialToolbarItemIndexRef.current = index;
+						} }
+						// Resets the index whenever the active block changes so
+						// this is not persisted. See https://github.com/WordPress/gutenberg/pull/25760#issuecomment-717906169
+						key={ clientId }
+					/>
+				) }
+				{ shouldShowBreadcrumb && (
+					<BlockSelectionButton
+						clientId={ clientId }
+						rootClientId={ rootClientId }
+					/>
+				) }
+			</BlockPopover>
+		);
+	}
+
+	return null;
+}
diff --git a/packages/block-editor/src/components/block-tools/use-selected-block-tool-props.js b/packages/block-editor/src/components/block-tools/use-selected-block-tool-props.js
new file mode 100644
index 00000000000000..a2783f49bb5389
--- /dev/null
+++ b/packages/block-editor/src/components/block-tools/use-selected-block-tool-props.js
@@ -0,0 +1,66 @@
+/**
+ * WordPress dependencies
+ */
+import { useSelect } from '@wordpress/data';
+
+/**
+ * Internal dependencies
+ */
+import { store as blockEditorStore } from '../../store';
+
+/**
+ * Returns props for the selected block tools and empty block inserter.
+ *
+ * @param {string} clientId Selected block client ID.
+ */
+export default function useSelectedBlockToolProps( clientId ) {
+	const selectedBlockProps = useSelect(
+		( select ) => {
+			const {
+				getBlockRootClientId,
+				getBlockParents,
+				__experimentalGetBlockListSettingsForBlocks,
+				isBlockInsertionPointVisible,
+				getBlockInsertionPoint,
+				getBlockOrder,
+				hasMultiSelection,
+				getLastMultiSelectedBlockClientId,
+			} = select( blockEditorStore );
+
+			const blockParentsClientIds = getBlockParents( clientId );
+
+			// Get Block List Settings for all ancestors of the current Block clientId.
+			const parentBlockListSettings =
+				__experimentalGetBlockListSettingsForBlocks(
+					blockParentsClientIds
+				);
+
+			// Get the clientId of the topmost parent with the capture toolbars setting.
+			const capturingClientId = blockParentsClientIds.find(
+				( parentClientId ) =>
+					parentBlockListSettings[ parentClientId ]
+						?.__experimentalCaptureToolbars
+			);
+
+			let isInsertionPointVisible = false;
+			if ( isBlockInsertionPointVisible() ) {
+				const insertionPoint = getBlockInsertionPoint();
+				const order = getBlockOrder( insertionPoint.rootClientId );
+				isInsertionPointVisible =
+					order[ insertionPoint.index ] === clientId;
+			}
+
+			return {
+				capturingClientId,
+				isInsertionPointVisible,
+				lastClientId: hasMultiSelection()
+					? getLastMultiSelectedBlockClientId()
+					: null,
+				rootClientId: getBlockRootClientId( clientId ),
+			};
+		},
+		[ clientId ]
+	);
+
+	return selectedBlockProps;
+}