diff --git a/packages/block-library/src/audio/edit.js b/packages/block-library/src/audio/edit.js
index 5b2737873fce44..2c36f2272571b6 100644
--- a/packages/block-library/src/audio/edit.js
+++ b/packages/block-library/src/audio/edit.js
@@ -22,6 +22,7 @@ import {
 	MediaPlaceholder,
 	MediaReplaceFlow,
 	useBlockProps,
+	useBlockEditingMode,
 } from '@wordpress/block-editor';
 import { __, _x } from '@wordpress/i18n';
 import { useDispatch } from '@wordpress/data';
@@ -51,6 +52,8 @@ function AudioEdit( {
 } ) {
 	const { id, autoplay, loop, preload, src } = attributes;
 	const [ temporaryURL, setTemporaryURL ] = useState( attributes.blob );
+	const blockEditingMode = useBlockEditingMode();
+	const hasNonContentControls = blockEditingMode === 'default';
 
 	useUploadMediaFromBlobURL( {
 		url: temporaryURL,
@@ -264,7 +267,9 @@ function AudioEdit( {
 					isSelected={ isSingleSelected }
 					insertBlocksAfter={ insertBlocksAfter }
 					label={ __( 'Audio caption text' ) }
-					showToolbarButton={ isSingleSelected }
+					showToolbarButton={
+						isSingleSelected && hasNonContentControls
+					}
 				/>
 			</figure>
 		</>
diff --git a/packages/block-library/src/video/edit.js b/packages/block-library/src/video/edit.js
index fbd724b2160a09..125b2cbc78741e 100644
--- a/packages/block-library/src/video/edit.js
+++ b/packages/block-library/src/video/edit.js
@@ -20,6 +20,7 @@ import {
 	MediaPlaceholder,
 	MediaReplaceFlow,
 	useBlockProps,
+	useBlockEditingMode,
 } from '@wordpress/block-editor';
 import { useRef, useEffect, useState } from '@wordpress/element';
 import { __ } from '@wordpress/i18n';
@@ -55,6 +56,8 @@ function VideoEdit( {
 	const { id, controls, poster, src, tracks } = attributes;
 	const [ temporaryURL, setTemporaryURL ] = useState( attributes.blob );
 	const dropdownMenuProps = useToolsPanelDropdownMenuProps();
+	const blockEditingMode = useBlockEditingMode();
+	const hasNonContentControls = blockEditingMode === 'default';
 
 	useUploadMediaFromBlobURL( {
 		url: temporaryURL,
@@ -251,7 +254,9 @@ function VideoEdit( {
 					isSelected={ isSingleSelected }
 					insertBlocksAfter={ insertBlocksAfter }
 					label={ __( 'Video caption text' ) }
-					showToolbarButton={ isSingleSelected }
+					showToolbarButton={
+						isSingleSelected && hasNonContentControls
+					}
 				/>
 			</figure>
 		</>