diff --git a/packages/block-editor/src/hooks/border-color.js b/packages/block-editor/src/hooks/border-color.js
index 036ff345a8ccdd..9060f21236d69c 100644
--- a/packages/block-editor/src/hooks/border-color.js
+++ b/packages/block-editor/src/hooks/border-color.js
@@ -9,6 +9,7 @@ import classnames from 'classnames';
 import { addFilter } from '@wordpress/hooks';
 import { __ } from '@wordpress/i18n';
 import { createHigherOrderComponent } from '@wordpress/compose';
+import { useState } from '@wordpress/element';
 
 /**
  * Internal dependencies
@@ -48,8 +49,18 @@ export function BorderColorEdit( props ) {
 	const colors = useSetting( 'color.palette' ) || EMPTY_ARRAY;
 	const disableCustomColors = ! useSetting( 'color.custom' );
 	const disableCustomGradients = ! useSetting( 'color.customGradient' );
+	const [ colorValue, setColorValue ] = useState(
+		() =>
+			getColorObjectByAttributeValues(
+				colors,
+				borderColor,
+				style?.border?.color
+			)?.color
+	);
 
 	const onChangeColor = ( value ) => {
+		setColorValue( value );
+
 		const colorObject = getColorObjectByColorValue( colors, value );
 		const newStyle = {
 			...style,
@@ -71,7 +82,7 @@ export function BorderColorEdit( props ) {
 	return (
 		<ColorGradientControl
 			label={ __( 'Color' ) }
-			value={ borderColor || style?.border?.color }
+			colorValue={ colorValue }
 			colors={ colors }
 			gradients={ undefined }
 			disableCustomColors={ disableCustomColors }
diff --git a/packages/edit-site/src/components/sidebar/border-panel.js b/packages/edit-site/src/components/sidebar/border-panel.js
index e7d5c801ff9355..a9af374e9b1f32 100644
--- a/packages/edit-site/src/components/sidebar/border-panel.js
+++ b/packages/edit-site/src/components/sidebar/border-panel.js
@@ -123,7 +123,7 @@ export default function BorderPanel( {
 			{ hasBorderColor && (
 				<ColorGradientControl
 					label={ __( 'Color' ) }
-					value={ borderColor }
+					colorValue={ borderColor }
 					colors={ colors }
 					gradients={ undefined }
 					disableCustomColors={ disableCustomColors }