add backwards compatibility options

deploy/crds/Grafana.yaml

@@ -118,6 +118,9 @@ spec:
             client:
               type: object
               description: Grafana client settings
+            compat:
+              type: object
+              description: Backwards compatibility switches
             dashboardLabelSelectors:
               type: array
               items:

pkg/apis/integreatly/v1alpha1/grafana_types.go

@@ -28,8 +28,15 @@ type GrafanaSpec struct {
 	Resources              *v1.ResourceRequirements `json:"resources,omitempty"`
 	ServiceAccount         *GrafanaServiceAccount   `json:"serviceAccount,omitempty"`
 	Client                 *GrafanaClient           `json:"client,omitempty"`
+	Compat                 *GrafanaCompat           `json:"compat"`
 }
 
+// Backwards compatibility switches
+type GrafanaCompat struct {
+	FixAnnotations bool `json:"fixAnnotations"`
+}
+
+// Grafana API client settings
 type GrafanaClient struct {
 	TimeoutSeconds *int `json:"timeout"`
 	PreferService  bool `json:"preferService"`

pkg/controller/common/controllerState.go

@@ -11,4 +11,5 @@ type ControllerState struct {
 	AdminUrl           string
 	GrafanaReady       bool
 	ClientTimeout      int
+	FixAnnotations     bool
 }

pkg/controller/config/controller_config.go

@@ -14,25 +14,17 @@ const (
 	ConfigGrafanaImageTag           = "grafana.image.tag"
 	ConfigPluginsInitContainerImage = "grafana.plugins.init.container.image.url"
 	ConfigPluginsInitContainerTag   = "grafana.plugins.init.container.image.tag"
-	ConfigPodLabelValue             = "grafana.pod.label"
 	ConfigOperatorNamespace         = "grafana.operator.namespace"
 	ConfigDashboardLabelSelector    = "grafana.dashboard.selector"
 	ConfigOpenshift                 = "mode.openshift"
-	GrafanaConfigMapName            = "grafana-config"
-	GrafanaConfigFileName           = "grafana.ini"
-	GrafanaDashboardsConfigMapName  = "grafana-dashboards"
-	GrafanaDeploymentName           = "grafana-deployment"
 	GrafanaDataPath                 = "/var/lib/grafana"
 	GrafanaLogsPath                 = "/var/log/grafana"
 	GrafanaPluginsPath              = "/var/lib/grafana/plugins"
 	GrafanaProvisioningPath         = "/etc/grafana/provisioning/"
 	PluginsInitContainerImage       = "quay.io/integreatly/grafana_plugins_init"
 	PluginsInitContainerTag         = "0.0.2"
-	PluginsEnvVar                   = "GRAFANA_PLUGINS"
 	PluginsUrl                      = "https://grafana.com/api/plugins/%s/versions/%s"
-	InitContainerName               = "grafana-plugins-init"
-	RequeueDelay                    = time.Second * 5
-	PodLabelDefaultValue            = "grafana"
+	RequeueDelay                    = time.Second * 10
 	SecretsMountDir                 = "/etc/grafana-secrets/"
 	ConfigMapsMountDir              = "/etc/grafana-configmaps/"
 	ConfigRouteWatch                = "watch.routes"

pkg/controller/grafana/grafana_controller.go

@@ -251,6 +251,10 @@ func (r *ReconcileGrafana) manageSuccess(cr *i8ly.Grafana, state *common.Cluster
 		}
 	}
 
+	if state.AdminSecret == nil || state.AdminSecret.Data == nil {
+		return r.manageError(cr, stdErr.New("admin secret not found or invalud"))
+	}
+
 	err := r.client.Status().Update(r.context, cr)
 	if err != nil {
 		return r.manageError(cr, err)
@@ -262,6 +266,12 @@ func (r *ReconcileGrafana) manageSuccess(cr *i8ly.Grafana, state *common.Cluster
 		return r.manageError(cr, err)
 	}
 
+	// Try to fix annotations on older dashboards?
+	fixAnnotations := false
+	if cr.Spec.Compat != nil && cr.Spec.Compat.FixAnnotations {
+		fixAnnotations = true
+	}
+
 	// Publish controller state
 	controllerState := common.ControllerState{
 		DashboardSelectors: cr.Spec.DashboardLabelSelector,
@@ -270,6 +280,7 @@ func (r *ReconcileGrafana) manageSuccess(cr *i8ly.Grafana, state *common.Cluster
 		AdminUrl:           url,
 		GrafanaReady:       true,
 		ClientTimeout:      DefaultClientTimeoutSeconds,
+		FixAnnotations:     fixAnnotations,
 	}
 
 	if cr.Spec.Client != nil && cr.Spec.Client.TimeoutSeconds != nil {

pkg/controller/grafanadashboard/dashboard_controller.go

@@ -146,6 +146,8 @@ func (r *ReconcileGrafanaDashboard) Reconcile(request reconcile.Request) (reconc
 	// If the dashboard does not match the label selectors then we ignore it
 	cr := instance.DeepCopy()
 	if !r.isMatch(cr) {
+		log.Info(fmt.Sprintf("dashboard %v/%v found but selectors do not match",
+			cr.Namespace, cr.Name))
 		return reconcile.Result{}, nil
 	}
 
@@ -198,28 +200,31 @@ func (r *ReconcileGrafanaDashboard) reconcileDashboards(request reconcile.Reques
 	for _, dashboard := range namespaceDashboards.Items {
 		// Is this a dashboard we care about (matches the label selectors)?
 		if !r.isMatch(&dashboard) {
+			log.Info(fmt.Sprintf("dashboard %v/%v found but selectors do not match",
+				dashboard.Namespace, dashboard.Name))
 			continue
 		}
 
 		// Process the dashboard. Use the known hash of an existing dashboard
 		// to determine if an update is required
 		knownHash := findHash(&dashboard)
-		pipeline := NewDashboardPipeline(&dashboard)
+		pipeline := NewDashboardPipeline(&dashboard, r.state.FixAnnotations)
 		processed, err := pipeline.ProcessDashboard(knownHash)
 
 		if err != nil {
+			log.Info(fmt.Sprintf("cannot process dashboard %v/%v", dashboard.Namespace, dashboard.Name))
 			r.manageError(&dashboard, err)
 			continue
 		}
 
 		if processed == nil {
-			log.Info(fmt.Sprintf("dashboard %v unchanged", dashboard.Name))
 			r.config.SetPluginsFor(&dashboard)
 			continue
 		}
 
 		status, err := grafanaClient.CreateOrUpdateDashboard(*processed)
 		if err != nil {
+			log.Info(fmt.Sprintf("cannot submit dashboard %v/%v", dashboard.Namespace, dashboard.Name))
 			r.manageError(&dashboard, err)
 			continue
 		}

pkg/controller/grafanadashboard/dashboard_pipeline.go

@@ -20,18 +20,20 @@ type DashboardPipeline interface {
 }
 
 type DashboardPipelineImpl struct {
-	Dashboard *v1alpha1.GrafanaDashboard
-	JSON      string
-	Board     sdk.Board
-	Logger    logr.Logger
-	Hash      string
+	Dashboard      *v1alpha1.GrafanaDashboard
+	JSON           string
+	Board          sdk.Board
+	Logger         logr.Logger
+	Hash           string
+	FixAnnotations bool
 }
 
-func NewDashboardPipeline(dashboard *v1alpha1.GrafanaDashboard) DashboardPipeline {
+func NewDashboardPipeline(dashboard *v1alpha1.GrafanaDashboard, fixAnnotations bool) DashboardPipeline {
 	return &DashboardPipelineImpl{
-		Dashboard: dashboard,
-		JSON:      "",
-		Logger:    logf.Log.WithName(fmt.Sprintf("dashboard-%v", dashboard.Name)),
+		Dashboard:      dashboard,
+		JSON:           "",
+		Logger:         logf.Log.WithName(fmt.Sprintf("dashboard-%v", dashboard.Name)),
+		FixAnnotations: fixAnnotations,
 	}
 }
 
@@ -55,6 +57,10 @@ func (r *DashboardPipelineImpl) ProcessDashboard(knownHash string) (*sdk.Board,
 		return nil, err
 	}
 
+	// Dashboards are never expected to come with an ID, it is
+	// always assigned by Grafana. If there is one, we ignore it
+	r.Board.ID = 0
+
 	// This dashboard has previously been imported
 	// To make sure its updated we have to set the metadata
 	if r.Dashboard.Status.Phase == v1alpha1.PhaseReconciling {
@@ -68,9 +74,14 @@ func (r *DashboardPipelineImpl) ProcessDashboard(knownHash string) (*sdk.Board,
 
 // Make sure the dashboard contains valid JSON
 func (r *DashboardPipelineImpl) validateJson() error {
-	log.Info("validating dashboard contents")
+	dashboardBytes := []byte(r.JSON)
+	dashboardBytes, err := r.fixAnnotations(dashboardBytes)
+	if err != nil {
+		return err
+	}
+
 	var dashboard sdk.Board
-	err := json.Unmarshal([]byte(r.JSON), &dashboard)
+	err = json.Unmarshal(dashboardBytes, &dashboard)
 	if err != nil {
 		return err
 	}
@@ -82,8 +93,6 @@ func (r *DashboardPipelineImpl) validateJson() error {
 // Try to get the dashboard json definition either from a provided URL or from the
 // raw json in the dashboard resource
 func (r *DashboardPipelineImpl) obtainJson() error {
-	r.Logger.Info("obtaining dashboard json")
-
 	if r.Dashboard.Spec.Url != "" {
 		err := r.loadDashboardFromURL()
 		if err != nil {
@@ -105,13 +114,11 @@ func (r *DashboardPipelineImpl) obtainJson() error {
 // If there are no changes we should avoid sending update requests as this will create
 // a new dashboard version in Grafana
 func (r *DashboardPipelineImpl) generateHash() string {
-	r.Logger.Info("generating dashboard hash")
 	return fmt.Sprintf("%x", md5.Sum([]byte(r.Dashboard.Spec.Json+r.Dashboard.Spec.Url)))
 }
 
 // Try to obtain the dashboard json from a provided url
 func (r *DashboardPipelineImpl) loadDashboardFromURL() error {
-	r.Logger.Info("loading dashboard from url")
 	_, err := url.ParseRequestURI(r.Dashboard.Spec.Url)
 	if err != nil {
 		return errors.New(fmt.Sprintf("invalid url %v", r.Dashboard.Spec.Url))
@@ -136,3 +143,39 @@ func (r *DashboardPipelineImpl) loadDashboardFromURL() error {
 func (r *DashboardPipelineImpl) NewHash() string {
 	return r.Hash
 }
+
+// Some older dashboards provide the tags list of an annotation as an array
+// instead of a string
+func (r *DashboardPipelineImpl) fixAnnotations(dashboardBytes []byte) ([]byte, error) {
+	if !r.FixAnnotations {
+		return dashboardBytes, nil
+	}
+
+	raw := map[string]interface{}{}
+	err := json.Unmarshal(dashboardBytes, &raw)
+	if err != nil {
+		return nil, err
+	}
+
+	if raw != nil && raw["annotations"] != nil {
+		annotations := raw["annotations"].(map[string]interface{})
+		if annotations != nil && annotations["list"] != nil {
+			annotationsList := annotations["list"].([]interface{})
+			for _, annotation := range annotationsList {
+				rawAnnotation := annotation.(map[string]interface{})
+				if rawAnnotation["tags"] != nil {
+					// Don't attempty to convert the tags, just replace them
+					// with something that is compatible
+					rawAnnotation["tags"] = ""
+				}
+			}
+		}
+	}
+
+	dashboardBytes, err = json.Marshal(raw)
+	if err != nil {
+		return nil, err
+	}
+
+	return dashboardBytes, nil
+}

pkg/controller/model/constants.go

@@ -20,7 +20,7 @@ const (
 	LastConfigAnnotation                = "last-config"
 	LastConfigEnvVar                    = "LAST_CONFIG"
 	LastDatasourcesConfigEnvVar         = "LAST_DATASOURCES"
-	GrafanaAdminSecretName              = "admin-credentials"
+	GrafanaAdminSecretName              = "grafana-admin-credentials"
 	DefaultAdminUser                    = "admin"
 	GrafanaAdminUserEnvVar              = "GF_SECURITY_ADMIN_USER"
 	GrafanaAdminPasswordEnvVar          = "GF_SECURITY_ADMIN_PASSWORD"