diff --git a/packages/block-editor/src/components/colors/with-colors.js b/packages/block-editor/src/components/colors/with-colors.js
index 2f40462857d33f..66fdaabfcda2ed 100644
--- a/packages/block-editor/src/components/colors/with-colors.js
+++ b/packages/block-editor/src/components/colors/with-colors.js
@@ -36,6 +36,8 @@ const withCustomColorPalette = ( colorsArray ) =>
 		'withCustomColorPalette'
 	);
 
+const EMPTY_OBJECT = {};
+
 /**
  * Higher order component factory for injecting the editor colors as the
  * `colors` prop in the `withColors` HOC.
@@ -45,7 +47,11 @@ const withCustomColorPalette = ( colorsArray ) =>
 const withEditorColorPalette = () =>
 	createHigherOrderComponent(
 		( WrappedComponent ) => ( props ) => {
-			const { palette: colorPerOrigin } = useSetting( 'color' ) || {};
+			// Some color settings have a special handling for deprecated flags in `useSetting`,
+			// so we can't unwrap them by doing const { ... } = useSetting('color')
+			// until https://github.com/WordPress/gutenberg/issues/37094 is fixed.
+			const colorPerOrigin =
+				useSetting( 'color.palette' ) || EMPTY_OBJECT;
 			const allColors = useMemo(
 				() => [
 					...( colorPerOrigin?.custom || [] ),
diff --git a/packages/block-editor/src/components/gradients/use-gradient.js b/packages/block-editor/src/components/gradients/use-gradient.js
index 52a4fee408b702..07130f37add24a 100644
--- a/packages/block-editor/src/components/gradients/use-gradient.js
+++ b/packages/block-editor/src/components/gradients/use-gradient.js
@@ -59,13 +59,15 @@ export function getGradientSlugByValue( gradients, value ) {
 	return gradient && gradient.slug;
 }
 
+const EMPTY_OBJECT = {};
+
 export function __experimentalUseGradient( {
 	gradientAttribute = 'gradient',
 	customGradientAttribute = 'customGradient',
 } = {} ) {
 	const { clientId } = useBlockEditContext();
 
-	const { gradients: gradientsPerOrigin } = useSetting( 'color' ) || {};
+	const gradientsPerOrigin = useSetting( 'color.gradients' ) || EMPTY_OBJECT;
 	const allGradients = useMemo(
 		() => [
 			...( gradientsPerOrigin?.custom || [] ),
diff --git a/packages/block-editor/src/hooks/color.js b/packages/block-editor/src/hooks/color.js
index 6dc0fe991a2c63..16fca0f4035879 100644
--- a/packages/block-editor/src/hooks/color.js
+++ b/packages/block-editor/src/hooks/color.js
@@ -32,6 +32,8 @@ import useSetting from '../components/use-setting';
 
 export const COLOR_SUPPORT_KEY = 'color';
 
+const EMPTY_OBJECT = {};
+
 const hasColorSupport = ( blockType ) => {
 	const colorSupport = getBlockSupport( blockType, COLOR_SUPPORT_KEY );
 	return (
@@ -216,15 +218,16 @@ function immutableSet( object, path, value ) {
  */
 export function ColorEdit( props ) {
 	const { name: blockName, attributes } = props;
-	const {
-		palette: solidsPerOrigin,
-		gradients: gradientsPerOrigin,
-		customGradient: areCustomGradientsEnabled,
-		custom: areCustomSolidsEnabled,
-		text: isTextEnabled,
-		background: isBackgroundEnabled,
-		link: isLinkEnabled,
-	} = useSetting( 'color' ) || {};
+	// Some color settings have a special handling for deprecated flags in `useSetting`,
+	// so we can't unwrap them by doing const { ... } = useSetting('color')
+	// until https://github.com/WordPress/gutenberg/issues/37094 is fixed.
+	const solidsPerOrigin = useSetting( 'color.palette' ) || EMPTY_OBJECT;
+	const gradientsPerOrigin = useSetting( 'color.gradients' ) || EMPTY_OBJECT;
+	const areCustomSolidsEnabled = useSetting( 'color.custom' );
+	const areCustomGradientsEnabled = useSetting( 'color.customGradient' );
+	const isBackgroundEnabled = useSetting( 'color.background' );
+	const isLinkEnabled = useSetting( 'color.link' );
+	const isTextEnabled = useSetting( 'color.text' );
 
 	const solidsEnabled =
 		areCustomSolidsEnabled ||
@@ -441,7 +444,7 @@ export const withColorPaletteStyles = createHigherOrderComponent(
 	( BlockListBlock ) => ( props ) => {
 		const { name, attributes } = props;
 		const { backgroundColor, textColor } = attributes;
-		const { palette: solidsPerOrigin } = useSetting( 'color' ) || {};
+		const solidsPerOrigin = useSetting( 'color.palette' ) || EMPTY_OBJECT;
 		const colors = useMemo(
 			() => [
 				...( solidsPerOrigin?.custom || [] ),
diff --git a/packages/block-editor/src/hooks/use-color-props.js b/packages/block-editor/src/hooks/use-color-props.js
index a3aa77901f0281..f46f7b11fdec8f 100644
--- a/packages/block-editor/src/hooks/use-color-props.js
+++ b/packages/block-editor/src/hooks/use-color-props.js
@@ -73,6 +73,8 @@ export function getColorClassesAndStyles( attributes ) {
 	};
 }
 
+const EMPTY_OBJECT = {};
+
 /**
  * Determines the color related props for a block derived from its color block
  * support attributes.
@@ -87,8 +89,12 @@ export function getColorClassesAndStyles( attributes ) {
 export function useColorProps( attributes ) {
 	const { backgroundColor, textColor, gradient } = attributes;
 
-	const { palette: solidsPerOrigin, gradients: gradientsPerOrigin } =
-		useSetting( 'color' ) || {};
+	// Some color settings have a special handling for deprecated flags in `useSetting`,
+	// so we can't unwrap them by doing const { ... } = useSetting('color')
+	// until https://github.com/WordPress/gutenberg/issues/37094 is fixed.
+	const solidsPerOrigin = useSetting( 'color.palette' ) || EMPTY_OBJECT;
+	const gradientsPerOrigin = useSetting( 'color.gradients' ) || EMPTY_OBJECT;
+
 	const colors = useMemo(
 		() => [
 			...( solidsPerOrigin?.custom || [] ),
diff --git a/packages/components/CHANGELOG.md b/packages/components/CHANGELOG.md
index a15f95fb47414c..d50af888476547 100644
--- a/packages/components/CHANGELOG.md
+++ b/packages/components/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 ### Bug Fix
 
+-   Fixed `GradientPicker` not displaying `CustomGradientPicker` when no gradients are provided ([#36900](https://github.com/WordPress/gutenberg/pull/36900)).
 -   Fixed error thrown in `ColorPicker` when used in controlled state in color gradients ([#36941](https://github.com/WordPress/gutenberg/pull/36941)).
 -   Updated readme to include default value introduced in fix for unexpected movements in the `ColorPicker` ([#35670](https://github.com/WordPress/gutenberg/pull/35670)).
 -   Replaced hardcoded blue in `ColorPicker` with UI theme color ([#36153](https://github.com/WordPress/gutenberg/pull/36153)).
diff --git a/packages/components/src/gradient-picker/index.js b/packages/components/src/gradient-picker/index.js
index df5bb07704b6f4..237c10627d2c06 100644
--- a/packages/components/src/gradient-picker/index.js
+++ b/packages/components/src/gradient-picker/index.js
@@ -110,9 +110,11 @@ export default function GradientPicker( {
 	const clearGradient = useCallback( () => onChange( undefined ), [
 		onChange,
 	] );
-	const Component = __experimentalHasMultipleOrigins
-		? MultipleOrigin
-		: SingleOrigin;
+	const Component =
+		__experimentalHasMultipleOrigins && gradients?.length
+			? MultipleOrigin
+			: SingleOrigin;
+
 	return (
 		<Component
 			className={ className }
@@ -122,7 +124,8 @@ export default function GradientPicker( {
 			onChange={ onChange }
 			value={ value }
 			actions={
-				clearable && (
+				clearable &&
+				( gradients?.length || ! disableCustomGradients ) && (
 					<CircularOptionPicker.ButtonAction
 						onClick={ clearGradient }
 					>
diff --git a/packages/components/src/gradient-picker/stories/index.js b/packages/components/src/gradient-picker/stories/index.js
index 0b15c04226a972..0b54f6a4b06d6a 100644
--- a/packages/components/src/gradient-picker/stories/index.js
+++ b/packages/components/src/gradient-picker/stories/index.js
@@ -84,3 +84,26 @@ export const _default = () => {
 		/>
 	);
 };
+
+export const WithNoExistingGradients = () => {
+	const disableCustomGradients = boolean( 'Disable Custom Gradients', false );
+	const __experimentalHasMultipleOrigins = boolean(
+		'Experimental Has Multiple Origins',
+		true
+	);
+	const clearable = boolean( 'Clearable', true );
+	const className = text( 'Class Name', '' );
+	const gradients = object( 'Gradients', [] );
+
+	return (
+		<GradientPickerWithState
+			__experimentalHasMultipleOrigins={
+				__experimentalHasMultipleOrigins
+			}
+			disableCustomGradients={ disableCustomGradients }
+			gradients={ gradients }
+			clearable={ clearable }
+			className={ className }
+		/>
+	);
+};