Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 99 additions & 0 deletions pkg/cli/admin/upgrade/status/alerts.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package status

import (
"time"

configv1 "github.com/openshift/api/config/v1"
)

// Alerts that will be included in the health upgrade evaluation, even if they were triggered before the upgrade began.
type AllowedAlerts map[string]struct{}

var allowedAlerts AllowedAlerts = map[string]struct{}{
"PodDisruptionBudgetLimit": {},
"PodDisruptionBudgetAtLimit": {},
}

func (al AllowedAlerts) Contains(alert string) bool {
_, exists := al[alert]
return exists
}

type AlertLabels struct {
AlertName string `json:"alertname,omitempty"`
Name string `json:"name,omitempty"`
Namespace string `json:"namespace,omitempty"`
Reason string `json:"reason,omitempty"`
Severity string `json:"severity,omitempty"`
}

type AlertAnnotations struct {
Description string `json:"description,omitempty"`
Summary string `json:"summary,omitempty"`
Runbook string `json:"runbook_url,omitempty"`
}

type Alert struct {
Labels AlertLabels `json:"labels,omitempty"`
Annotations AlertAnnotations `json:"annotations,omitempty"`
State string `json:"state,omitempty"`
Value string `json:"value,omitempty"`
ActiveAt time.Time `json:"activeAt,omitempty"`
PartialResponseStrategy string `json:"partialResponseStrategy,omitempty"`
}

// Stores alert data returned by thanos
type AlertData struct {
Status string `json:"status"`
Data Data `json:"data"`
}

type Data struct {
Alerts []Alert `json:"alerts"`
}

func parseAlertDataToInsights(alertData AlertData, startedAt time.Time) []updateInsight {
var alerts []Alert = alertData.Data.Alerts
var updateInsights []updateInsight = []updateInsight{}

for _, alert := range alerts {
if startedAt.After(alert.ActiveAt) && !allowedAlerts.Contains(alert.Labels.AlertName) {
continue
}
if alert.State == "pending" {
continue
}
updateInsights = append(updateInsights, updateInsight{
startedAt: alert.ActiveAt,
impact: updateInsightImpact{
level: alertImpactLevel(alert.Labels.Severity),
impactType: unknownImpactType,
summary: "Alert: " + alert.Annotations.Summary,
description: alert.Annotations.Description,
},
remediation: updateInsightRemediation{reference: alert.Annotations.Runbook},
scope: updateInsightScope{
scopeType: scopeTypeCluster,
resources: []scopeResource{{
kind: scopeGroupKind{group: configv1.GroupName, kind: "Alert"},
namespace: alert.Labels.Namespace,
name: alert.Labels.AlertName,
}},
},
})
}
return updateInsights
}

func alertImpactLevel(ail string) impactLevel {
switch ail {
case "warning":
return warningImpactLevel
case "critical":
return criticalInfoLevel
case "info":
return infoImpactLevel
default:
return infoImpactLevel
}
}
156 changes: 156 additions & 0 deletions pkg/cli/admin/upgrade/status/alerts_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
package status

import (
"reflect"
"testing"
"time"

configv1 "github.com/openshift/api/config/v1"
)

func TestParseAlertDataToInsights(t *testing.T) {
now := time.Now()

// Define test cases
tests := []struct {
name string
alertData AlertData
startedAt time.Time
expectedCount int
}{
{
name: "Empty Alerts",
alertData: AlertData{
Data: Data{Alerts: []Alert{}},
},
startedAt: now,
expectedCount: 0,
},
{
name: "Alert Active After Start Time",
alertData: AlertData{
Data: Data{
Alerts: []Alert{
{ActiveAt: now.Add(10 * time.Minute), Labels: AlertLabels{Severity: "critical", Namespace: "default", AlertName: "NodeDown"}, Annotations: AlertAnnotations{Summary: "Node is down"}},
{ActiveAt: now.Add(-10 * time.Minute), Labels: AlertLabels{Severity: "warning", Namespace: "default", AlertName: "DiskSpaceLow"}, Annotations: AlertAnnotations{Summary: "Disk space low"}},
},
},
},
startedAt: now,
expectedCount: 1,
},
{
name: "Alert Active Before Start Time",
alertData: AlertData{
Data: Data{
Alerts: []Alert{
{ActiveAt: now.Add(-10 * time.Minute), Labels: AlertLabels{Severity: "warning", Namespace: "default", AlertName: "DiskSpaceLow"}, Annotations: AlertAnnotations{Summary: "Disk space low"}},
},
},
},
startedAt: now,
expectedCount: 0,
},
{
name: "Alert Active Before Start Time, Allowed",
alertData: AlertData{
Data: Data{
Alerts: []Alert{
{ActiveAt: now.Add(-20 * time.Minute), Labels: AlertLabels{Severity: "info", Namespace: "default", AlertName: "PodDisruptionBudgetAtLimit"}, Annotations: AlertAnnotations{Summary: "PodDisruptionBudgetAtLimit is at limit"}},
{ActiveAt: now.Add(-20 * time.Minute), Labels: AlertLabels{Severity: "info", Namespace: "default", AlertName: "AlertmanagerReceiversNotConfigured"}, Annotations: AlertAnnotations{Summary: "Receivers (notification integrations) are not configured on Alertmanager"}},
},
},
},
startedAt: now,
expectedCount: 1,
},
{
name: "Alert Active Before Start Time, Not Allowed",
alertData: AlertData{
Data: Data{
Alerts: []Alert{
{ActiveAt: now.Add(-20 * time.Minute), Labels: AlertLabels{Severity: "info", Namespace: "default", AlertName: "AlertmanagerReceiversNotConfigured"}, Annotations: AlertAnnotations{Summary: "Receivers (notification integrations) are not configured on Alertmanager"}},
},
},
},
startedAt: now,
expectedCount: 0,
},
}

// Execute test cases
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
insights := parseAlertDataToInsights(tt.alertData, tt.startedAt)
if got := len(insights); got != tt.expectedCount {
t.Errorf("parseAlertDataToInsights() = %v, want %v", got, tt.expectedCount)
}
})
}
}

func TestParseAlertDataToInsightsWithData(t *testing.T) {
now := time.Now()

tests := []struct {
name string
alertData AlertData
startedAt time.Time
expectedInsights []updateInsight
}{
{
name: "Alert Active After Start Time",
alertData: AlertData{
Data: Data{
Alerts: []Alert{
{ActiveAt: now.Add(10 * time.Minute), Labels: AlertLabels{Severity: "critical", Namespace: "default", AlertName: "NodeDown"}, Annotations: AlertAnnotations{Summary: "Node is down"}},
},
},
},
startedAt: now,
expectedInsights: []updateInsight{
{
startedAt: now.Add(10 * time.Minute),
impact: updateInsightImpact{
level: alertImpactLevel("critical"),
impactType: unknownImpactType,
summary: "Alert: Node is down",
},
scope: updateInsightScope{
scopeType: scopeTypeCluster,
resources: []scopeResource{
{
kind: scopeGroupKind{group: configv1.GroupName, kind: "Alert"},
namespace: "default",
name: "NodeDown",
},
},
},
},
},
},
{
name: "Alert Active Before Start Time",
alertData: AlertData{
Data: Data{
Alerts: []Alert{
{ActiveAt: now.Add(-10 * time.Minute), Labels: AlertLabels{Severity: "warning", Namespace: "default", AlertName: "DiskSpaceLow"}, Annotations: AlertAnnotations{Summary: "Disk space low"}},
},
},
},
startedAt: now,
expectedInsights: []updateInsight{},
},
}

// Execute test cases
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
insights := parseAlertDataToInsights(tt.alertData, tt.startedAt)
if !reflect.DeepEqual(insights, tt.expectedInsights) {
t.Errorf("parseAlertDataToInsights() got %#v, want %#v", insights, tt.expectedInsights)
}
})
}

}
Loading